LCOV - code coverage report
Current view: top level - gcc - tree-vect-data-refs.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 89.8 % 2812 2526
Test Date: 2026-11-14 16:39:50 Functions: 97.4 % 76 74
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Data References Analysis and Manipulation Utilities for Vectorization.
       2              :    Copyright (C) 2003-2026 Free Software Foundation, Inc.
       3              :    Contributed by Dorit Naishlos <dorit@il.ibm.com>
       4              :    and Ira Rosen <irar@il.ibm.com>
       5              : 
       6              : This file is part of GCC.
       7              : 
       8              : GCC is free software; you can redistribute it and/or modify it under
       9              : the terms of the GNU General Public License as published by the Free
      10              : Software Foundation; either version 3, or (at your option) any later
      11              : version.
      12              : 
      13              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      14              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      15              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      16              : for more details.
      17              : 
      18              : You should have received a copy of the GNU General Public License
      19              : along with GCC; see the file COPYING3.  If not see
      20              : <http://www.gnu.org/licenses/>.  */
      21              : 
      22              : #define INCLUDE_ALGORITHM
      23              : #include "config.h"
      24              : #include "system.h"
      25              : #include "coretypes.h"
      26              : #include "backend.h"
      27              : #include "target.h"
      28              : #include "rtl.h"
      29              : #include "tree.h"
      30              : #include "gimple.h"
      31              : #include "predict.h"
      32              : #include "memmodel.h"
      33              : #include "tm_p.h"
      34              : #include "ssa.h"
      35              : #include "optabs-tree.h"
      36              : #include "cgraph.h"
      37              : #include "dumpfile.h"
      38              : #include "pretty-print.h"
      39              : #include "alias.h"
      40              : #include "fold-const.h"
      41              : #include "stor-layout.h"
      42              : #include "tree-eh.h"
      43              : #include "gimplify.h"
      44              : #include "gimple-iterator.h"
      45              : #include "gimplify-me.h"
      46              : #include "tree-ssa-loop-ivopts.h"
      47              : #include "tree-ssa-loop-manip.h"
      48              : #include "tree-ssa-loop.h"
      49              : #include "cfgloop.h"
      50              : #include "tree-scalar-evolution.h"
      51              : #include "tree-vectorizer.h"
      52              : #include "expr.h"
      53              : #include "builtins.h"
      54              : #include "tree-cfg.h"
      55              : #include "tree-hash-traits.h"
      56              : #include "vec-perm-indices.h"
      57              : #include "internal-fn.h"
      58              : #include "gimple-fold.h"
      59              : #include "optabs-query.h"
      60              : 
      61              : /* Return true if load- or store-lanes optab OPTAB is implemented for
      62              :    COUNT vectors of type VECTYPE.  NAME is the name of OPTAB.
      63              : 
      64              :    If it is implemented and ELSVALS is nonzero store the possible else
      65              :    values in the vector it points to.  */
      66              : 
      67              : static bool
      68       387392 : vect_lanes_optab_supported_p (const char *name, convert_optab optab,
      69              :                               tree vectype, unsigned HOST_WIDE_INT count,
      70              :                               vec<int> *elsvals = nullptr)
      71              : {
      72       387392 :   machine_mode mode, array_mode;
      73       387392 :   bool limit_p;
      74              : 
      75       387392 :   mode = TYPE_MODE (vectype);
      76       387392 :   if (!targetm.array_mode (mode, count).exists (&array_mode))
      77              :     {
      78       774784 :       poly_uint64 bits = count * GET_MODE_BITSIZE (mode);
      79       387392 :       limit_p = !targetm.array_mode_supported_p (mode, count);
      80       387392 :       if (!int_mode_for_size (bits, limit_p).exists (&array_mode))
      81              :         {
      82       326472 :           if (dump_enabled_p ())
      83        12910 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
      84              :                              "no array mode for %s[%wu]\n",
      85        12910 :                              GET_MODE_NAME (mode), count);
      86       326472 :           return false;
      87              :         }
      88              :     }
      89              : 
      90        60920 :   enum insn_code icode;
      91        60920 :   if ((icode = convert_optab_handler (optab, array_mode, mode))
      92              :       == CODE_FOR_nothing)
      93              :     {
      94        60920 :       if (dump_enabled_p ())
      95         4266 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
      96              :                          "cannot use %s<%s><%s>\n", name,
      97         4266 :                          GET_MODE_NAME (array_mode), GET_MODE_NAME (mode));
      98              :       return false;
      99              :     }
     100              : 
     101            0 :   if (dump_enabled_p ())
     102            0 :     dump_printf_loc (MSG_NOTE, vect_location,
     103            0 :                      "can use %s<%s><%s>\n", name, GET_MODE_NAME (array_mode),
     104            0 :                      GET_MODE_NAME (mode));
     105              : 
     106            0 :   if (elsvals)
     107            0 :     get_supported_else_vals (icode,
     108            0 :                              internal_fn_else_index (IFN_MASK_LEN_LOAD_LANES),
     109              :                              *elsvals);
     110              : 
     111              :   return true;
     112              : }
     113              : 
     114              : 
     115              : /* Insert DDR into LOOP_VINFO list of ddrs that may alias and need to be
     116              :    tested at run-time.  Return TRUE if DDR was successfully inserted.
     117              :    Return false if versioning is not supported.  */
     118              : 
     119              : static opt_result
     120       184430 : vect_mark_for_runtime_alias_test (ddr_p ddr, loop_vec_info loop_vinfo)
     121              : {
     122       184430 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
     123              : 
     124       184430 :   if ((unsigned) param_vect_max_version_for_alias_checks == 0)
     125           54 :     return opt_result::failure_at (vect_location,
     126              :                                    "will not create alias checks, as"
     127              :                                    " --param vect-max-version-for-alias-checks"
     128              :                                    " == 0\n");
     129              : 
     130       184376 :   opt_result res
     131       184376 :     = runtime_alias_check_p (ddr, loop,
     132              :                              optimize_loop_nest_for_speed_p (loop));
     133       184376 :   if (!res)
     134          149 :     return res;
     135              : 
     136       184227 :   LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo).safe_push (ddr);
     137       184227 :   return opt_result::success ();
     138              : }
     139              : 
     140              : /* Record that loop LOOP_VINFO needs to check that VALUE is nonzero.  */
     141              : 
     142              : static void
     143         1512 : vect_check_nonzero_value (loop_vec_info loop_vinfo, tree value)
     144              : {
     145         1512 :   const vec<tree> &checks = LOOP_VINFO_CHECK_NONZERO (loop_vinfo);
     146         2267 :   for (unsigned int i = 0; i < checks.length(); ++i)
     147          755 :     if (checks[i] == value)
     148              :       return;
     149              : 
     150         1512 :   if (dump_enabled_p ())
     151          418 :     dump_printf_loc (MSG_NOTE, vect_location,
     152              :                      "need run-time check that %T is nonzero\n",
     153              :                      value);
     154         1512 :   LOOP_VINFO_CHECK_NONZERO (loop_vinfo).safe_push (value);
     155              : }
     156              : 
     157              : /* Return true if we know that the order of vectorized DR_INFO_A and
     158              :    vectorized DR_INFO_B will be the same as the order of DR_INFO_A and
     159              :    DR_INFO_B.  At least one of the accesses is a write.  */
     160              : 
     161              : static bool
     162       152465 : vect_preserves_scalar_order_p (dr_vec_info *dr_info_a, dr_vec_info *dr_info_b)
     163              : {
     164       152465 :   stmt_vec_info stmtinfo_a = dr_info_a->stmt;
     165       152465 :   stmt_vec_info stmtinfo_b = dr_info_b->stmt;
     166              : 
     167              :   /* Single statements are always kept in their original order.  */
     168       152465 :   if (!STMT_VINFO_GROUPED_ACCESS (stmtinfo_a)
     169       253163 :       && !STMT_VINFO_GROUPED_ACCESS (stmtinfo_b))
     170              :     return true;
     171              : 
     172              :   /* If there is a loop invariant read involved we might vectorize it in
     173              :      the prologue, breaking scalar order with respect to the in-loop store.  */
     174        28383 :   if ((DR_IS_READ (dr_info_a->dr) && integer_zerop (DR_STEP (dr_info_a->dr)))
     175        85816 :       || (DR_IS_READ (dr_info_b->dr) && integer_zerop (DR_STEP (dr_info_b->dr))))
     176              :     return false;
     177              : 
     178              :   /* STMT_A and STMT_B belong to overlapping groups.  All loads are
     179              :      emitted at the position of the first scalar load.
     180              :      Stores in a group are emitted at the position of the last scalar store.
     181              :      Compute that position and check whether the resulting order matches
     182              :      the current one.  */
     183        56722 :   stmt_vec_info il_a = DR_GROUP_FIRST_ELEMENT (stmtinfo_a);
     184        56722 :   if (il_a)
     185              :     {
     186        51056 :       if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_a)))
     187       214117 :         for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_a); s;
     188       188486 :              s = DR_GROUP_NEXT_ELEMENT (s))
     189       188486 :           il_a = get_later_stmt (il_a, s);
     190              :       else /* DR_IS_READ */
     191        96583 :         for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_a); s;
     192        71158 :              s = DR_GROUP_NEXT_ELEMENT (s))
     193        71158 :           if (get_later_stmt (il_a, s) == il_a)
     194         2456 :             il_a = s;
     195              :     }
     196              :   else
     197              :     il_a = stmtinfo_a;
     198        56722 :   stmt_vec_info il_b = DR_GROUP_FIRST_ELEMENT (stmtinfo_b);
     199        56722 :   if (il_b)
     200              :     {
     201        48322 :       if (DR_IS_WRITE (STMT_VINFO_DATA_REF (stmtinfo_b)))
     202       275908 :         for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_b); s;
     203       237066 :              s = DR_GROUP_NEXT_ELEMENT (s))
     204       237066 :           il_b = get_later_stmt (il_b, s);
     205              :       else /* DR_IS_READ */
     206        43266 :         for (stmt_vec_info s = DR_GROUP_NEXT_ELEMENT (il_b); s;
     207        33786 :              s = DR_GROUP_NEXT_ELEMENT (s))
     208        33786 :           if (get_later_stmt (il_b, s) == il_b)
     209          365 :             il_b = s;
     210              :     }
     211              :   else
     212              :     il_b = stmtinfo_b;
     213        56722 :   bool a_after_b = (get_later_stmt (stmtinfo_a, stmtinfo_b) == stmtinfo_a);
     214        56722 :   return (get_later_stmt (il_a, il_b) == il_a) == a_after_b;
     215              : }
     216              : 
     217              : /* A subroutine of vect_analyze_data_ref_dependence.  Handle
     218              :    DDR_COULD_BE_INDEPENDENT_P ddr DDR that has a known set of dependence
     219              :    distances.  These distances are conservatively correct but they don't
     220              :    reflect a guaranteed dependence.
     221              : 
     222              :    Return true if this function does all the work necessary to avoid
     223              :    an alias or false if the caller should use the dependence distances
     224              :    to limit the vectorization factor in the usual way.  LOOP_DEPTH is
     225              :    the depth of the loop described by LOOP_VINFO and the other arguments
     226              :    are as for vect_analyze_data_ref_dependence.  */
     227              : 
     228              : static bool
     229         9283 : vect_analyze_possibly_independent_ddr (data_dependence_relation *ddr,
     230              :                                        loop_vec_info loop_vinfo,
     231              :                                        int loop_depth, unsigned int *max_vf)
     232              : {
     233         9283 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
     234        37150 :   for (lambda_vector &dist_v : DDR_DIST_VECTS (ddr))
     235              :     {
     236        18295 :       int dist = dist_v[loop_depth];
     237        18295 :       if (dist != 0 && !(dist > 0 && DDR_REVERSED_P (ddr)))
     238              :         {
     239              :           /* If the user asserted safelen >= DIST consecutive iterations
     240              :              can be executed concurrently, assume independence.
     241              : 
     242              :              ??? An alternative would be to add the alias check even
     243              :              in this case, and vectorize the fallback loop with the
     244              :              maximum VF set to safelen.  However, if the user has
     245              :              explicitly given a length, it's less likely that that
     246              :              would be a win.  */
     247         9026 :           if (loop->safelen >= 2 && abs_hwi (dist) <= loop->safelen)
     248              :             {
     249           32 :               if ((unsigned int) loop->safelen < *max_vf)
     250            2 :                 *max_vf = loop->safelen;
     251           32 :               LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = false;
     252           32 :               continue;
     253              :             }
     254              : 
     255              :           /* For dependence distances of 2 or more, we have the option
     256              :              of limiting VF or checking for an alias at runtime.
     257              :              Prefer to check at runtime if we can, to avoid limiting
     258              :              the VF unnecessarily when the bases are in fact independent.
     259              : 
     260              :              Note that the alias checks will be removed if the VF ends up
     261              :              being small enough.  */
     262         8994 :           dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
     263         8994 :           dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
     264         8994 :           return (!STMT_VINFO_GATHER_SCATTER_P (dr_info_a->stmt)
     265         8994 :                   && !STMT_VINFO_GATHER_SCATTER_P (dr_info_b->stmt)
     266        17996 :                   && vect_mark_for_runtime_alias_test (ddr, loop_vinfo));
     267              :         }
     268              :     }
     269              :   return true;
     270              : }
     271              : 
     272              : 
     273              : /* Function vect_analyze_data_ref_dependence.
     274              : 
     275              :    FIXME: I needed to change the sense of the returned flag.
     276              : 
     277              :    Return FALSE if there (might) exist a dependence between a memory-reference
     278              :    DRA and a memory-reference DRB.  When versioning for alias may check a
     279              :    dependence at run-time, return TRUE.  Adjust *MAX_VF according to
     280              :    the data dependence.  */
     281              : 
     282              : static opt_result
     283      1527512 : vect_analyze_data_ref_dependence (struct data_dependence_relation *ddr,
     284              :                                   loop_vec_info loop_vinfo,
     285              :                                   unsigned int *max_vf)
     286              : {
     287      1527512 :   unsigned int i;
     288      1527512 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
     289      1527512 :   struct data_reference *dra = DDR_A (ddr);
     290      1527512 :   struct data_reference *drb = DDR_B (ddr);
     291      1527512 :   dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (dra);
     292      1527512 :   dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (drb);
     293      1527512 :   stmt_vec_info stmtinfo_a = dr_info_a->stmt;
     294      1527512 :   stmt_vec_info stmtinfo_b = dr_info_b->stmt;
     295      1527512 :   lambda_vector dist_v;
     296      1527512 :   unsigned int loop_depth;
     297              : 
     298              :   /* If user asserted safelen consecutive iterations can be
     299              :      executed concurrently, assume independence.  */
     300      1719642 :   auto apply_safelen = [&]()
     301              :     {
     302       192130 :       if (loop->safelen >= 2)
     303              :         {
     304         7653 :           if ((unsigned int) loop->safelen < *max_vf)
     305         1914 :             *max_vf = loop->safelen;
     306         7653 :           LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = false;
     307         7653 :           return true;
     308              :         }
     309              :       return false;
     310      1527512 :     };
     311              : 
     312              :   /* In loop analysis all data references should be vectorizable.  */
     313      1527512 :   if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a)
     314      1527512 :       || !STMT_VINFO_VECTORIZABLE (stmtinfo_b))
     315            0 :     gcc_unreachable ();
     316              : 
     317              :   /* Independent data accesses.  */
     318      1527512 :   if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
     319      1246247 :     return opt_result::success ();
     320              : 
     321       281265 :   if (dra == drb
     322       281265 :       || (DR_IS_READ (dra) && DR_IS_READ (drb)))
     323            0 :     return opt_result::success ();
     324              : 
     325              :   /* We do not have to consider dependences between accesses that belong
     326              :      to the same group, unless the stride could be smaller than the
     327              :      group size.  */
     328       281265 :   if (DR_GROUP_FIRST_ELEMENT (stmtinfo_a)
     329       124212 :       && (DR_GROUP_FIRST_ELEMENT (stmtinfo_a)
     330       124212 :           == DR_GROUP_FIRST_ELEMENT (stmtinfo_b))
     331       300907 :       && !STMT_VINFO_STRIDED_P (stmtinfo_a))
     332         2511 :     return opt_result::success ();
     333              : 
     334              :   /* Even if we have an anti-dependence then, as the vectorized loop covers at
     335              :      least two scalar iterations, there is always also a true dependence.
     336              :      As the vectorizer does not re-order loads and stores we can ignore
     337              :      the anti-dependence if TBAA can disambiguate both DRs similar to the
     338              :      case with known negative distance anti-dependences (positive
     339              :      distance anti-dependences would violate TBAA constraints).  */
     340       135554 :   if (((DR_IS_READ (dra) && DR_IS_WRITE (drb))
     341       143200 :        || (DR_IS_WRITE (dra) && DR_IS_READ (drb)))
     342       433121 :       && !alias_sets_conflict_p (get_alias_set (DR_REF (dra)),
     343              :                                  get_alias_set (DR_REF (drb))))
     344         6112 :     return opt_result::success ();
     345              : 
     346       272642 :   if (STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a)
     347       264843 :       || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
     348              :     {
     349        10434 :       if (apply_safelen ())
     350         1393 :         return opt_result::success ();
     351              : 
     352         9041 :       return opt_result::failure_at
     353         9041 :         (stmtinfo_a->stmt,
     354              :          "possible alias involving gather/scatter between %T and %T\n",
     355              :          DR_REF (dra), DR_REF (drb));
     356              :     }
     357              : 
     358              :   /* Unknown data dependence.  */
     359       262208 :   if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
     360              :     {
     361       181135 :       if (apply_safelen ())
     362         6260 :         return opt_result::success ();
     363              : 
     364       174875 :       if (dump_enabled_p ())
     365         7698 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, stmtinfo_a->stmt,
     366              :                          "versioning for alias required: "
     367              :                          "can't determine dependence between %T and %T\n",
     368              :                          DR_REF (dra), DR_REF (drb));
     369              : 
     370              :       /* Add to list of ddrs that need to be tested at run-time.  */
     371       174875 :       return vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
     372              :     }
     373              : 
     374              :   /* Known data dependence.  */
     375        81073 :   if (DDR_NUM_DIST_VECTS (ddr) == 0)
     376              :     {
     377          561 :       if (apply_safelen ())
     378            0 :         return opt_result::success ();
     379              : 
     380          561 :       if (dump_enabled_p ())
     381          156 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, stmtinfo_a->stmt,
     382              :                          "versioning for alias required: "
     383              :                          "bad dist vector for %T and %T\n",
     384              :                          DR_REF (dra), DR_REF (drb));
     385              :       /* Add to list of ddrs that need to be tested at run-time.  */
     386          561 :       return vect_mark_for_runtime_alias_test (ddr, loop_vinfo);
     387              :     }
     388              : 
     389        80512 :   loop_depth = index_in_loop_nest (loop->num, DDR_LOOP_NEST (ddr));
     390              : 
     391        80512 :   if (DDR_COULD_BE_INDEPENDENT_P (ddr)
     392        80512 :       && vect_analyze_possibly_independent_ddr (ddr, loop_vinfo,
     393              :                                                 loop_depth, max_vf))
     394         9275 :     return opt_result::success ();
     395              : 
     396       135253 :   FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
     397              :     {
     398        71265 :       int dist = dist_v[loop_depth];
     399              : 
     400        71265 :       if (dump_enabled_p ())
     401         4781 :         dump_printf_loc (MSG_NOTE, vect_location,
     402              :                          "dependence distance  = %d.\n", dist);
     403              : 
     404        71265 :       if (dist == 0)
     405              :         {
     406        59142 :           if (dump_enabled_p ())
     407         3678 :             dump_printf_loc (MSG_NOTE, vect_location,
     408              :                              "dependence distance == 0 between %T and %T\n",
     409              :                              DR_REF (dra), DR_REF (drb));
     410              : 
     411              :           /* When we perform grouped accesses and perform implicit CSE
     412              :              by detecting equal accesses and doing disambiguation with
     413              :              runtime alias tests like for
     414              :                 .. = a[i];
     415              :                 .. = a[i+1];
     416              :                 a[i] = ..;
     417              :                 a[i+1] = ..;
     418              :                 *p = ..;
     419              :                 .. = a[i];
     420              :                 .. = a[i+1];
     421              :              where we will end up loading { a[i], a[i+1] } once, make
     422              :              sure that inserting group loads before the first load and
     423              :              stores after the last store will do the right thing.
     424              :              Similar for groups like
     425              :                 a[i] = ...;
     426              :                 ... = a[i];
     427              :                 a[i+1] = ...;
     428              :              where loads from the group interleave with the store.  */
     429        59142 :           if (!vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
     430            0 :             return opt_result::failure_at (stmtinfo_a->stmt,
     431              :                                            "READ_WRITE dependence"
     432              :                                            " in interleaving.\n");
     433              : 
     434        59142 :           if (loop->safelen < 2)
     435              :             {
     436        55404 :               tree indicator = dr_zero_step_indicator (dra);
     437        55404 :               if (!indicator || integer_zerop (indicator))
     438            0 :                 return opt_result::failure_at (stmtinfo_a->stmt,
     439              :                                                "access also has a zero step\n");
     440        55404 :               else if (TREE_CODE (indicator) != INTEGER_CST)
     441         1512 :                 vect_check_nonzero_value (loop_vinfo, indicator);
     442              :             }
     443        59142 :           continue;
     444        59142 :         }
     445              : 
     446        12123 :       if (dist > 0 && DDR_REVERSED_P (ddr))
     447              :         {
     448              :           /* If DDR_REVERSED_P the order of the data-refs in DDR was
     449              :              reversed (to make distance vector positive), and the actual
     450              :              distance is negative.  */
     451         3627 :           if (dump_enabled_p ())
     452          114 :             dump_printf_loc (MSG_NOTE, vect_location,
     453              :                              "dependence distance negative.\n");
     454              :           /* When doing outer loop vectorization, we need to check if there is
     455              :              a backward dependence at the inner loop level if the dependence
     456              :              at the outer loop is reversed.  See PR81740.  */
     457         3627 :           if (nested_in_vect_loop_p (loop, stmtinfo_a)
     458         3615 :               || nested_in_vect_loop_p (loop, stmtinfo_b))
     459              :             {
     460           12 :               unsigned inner_depth = index_in_loop_nest (loop->inner->num,
     461           12 :                                                          DDR_LOOP_NEST (ddr));
     462           12 :               if (dist_v[inner_depth] < 0)
     463            9 :                 return opt_result::failure_at (stmtinfo_a->stmt,
     464              :                                                "not vectorized, dependence "
     465              :                                                "between data-refs %T and %T\n",
     466              :                                                DR_REF (dra), DR_REF (drb));
     467              :             }
     468         3618 :           if (!vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
     469           18 :             return opt_result::failure_at (stmtinfo_a->stmt,
     470              :                                            "dependence in interleaving.\n");
     471              :           /* Record a negative dependence distance to later limit the
     472              :              amount of stmt copying / unrolling we can perform.
     473              :              Only need to handle read-after-write dependence.  */
     474         3600 :           if (DR_IS_READ (drb)
     475          147 :               && (STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) == 0
     476           36 :                   || STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) > (unsigned)dist))
     477          147 :             STMT_VINFO_MIN_NEG_DIST (stmtinfo_b) = dist;
     478         3600 :           continue;
     479         3600 :         }
     480              : 
     481         8496 :       unsigned int abs_dist = abs (dist);
     482         8496 :       if (abs_dist >= 2 && abs_dist < *max_vf)
     483              :         {
     484              :           /* The dependence distance requires reduction of the maximal
     485              :              vectorization factor.  */
     486          431 :           *max_vf = abs_dist;
     487          431 :           if (dump_enabled_p ())
     488           32 :             dump_printf_loc (MSG_NOTE, vect_location,
     489              :                              "adjusting maximal vectorization factor to %i\n",
     490              :                              *max_vf);
     491              :         }
     492              : 
     493         8496 :       if (abs_dist >= *max_vf)
     494              :         {
     495              :           /* Dependence distance does not create dependence, as far as
     496              :              vectorization is concerned, in this case.  */
     497         1274 :           if (dump_enabled_p ())
     498          737 :             dump_printf_loc (MSG_NOTE, vect_location,
     499              :                              "dependence distance >= VF.\n");
     500         1274 :           continue;
     501              :         }
     502              : 
     503         7222 :       return opt_result::failure_at (stmtinfo_a->stmt,
     504              :                                      "not vectorized, possible dependence "
     505              :                                      "between data-refs %T and %T\n",
     506              :                                      DR_REF (dra), DR_REF (drb));
     507              :     }
     508              : 
     509        63988 :   return opt_result::success ();
     510              : }
     511              : 
     512              : /* Function vect_analyze_early_break_dependences.
     513              : 
     514              :    Examine all the data references in the loop and make sure that if we have
     515              :    multiple exits that we are able to safely move stores such that they become
     516              :    safe for vectorization.  The function also calculates the place where to move
     517              :    the instructions to and computes what the new vUSE chain should be.
     518              : 
     519              :    This works in tandem with the CFG that will be produced by
     520              :    slpeel_tree_duplicate_loop_to_edge_cfg later on.
     521              : 
     522              :    This function tries to validate whether an early break vectorization
     523              :    is possible for the current instruction sequence. Returns True i
     524              :    possible, otherwise False.
     525              : 
     526              :    Requirements:
     527              :      - Any memory access must be to a fixed size buffer.
     528              :      - There must not be any loads and stores to the same object.
     529              :      - Multiple loads are allowed as long as they don't alias.
     530              : 
     531              :    NOTE:
     532              :      This implementation is very conservative. Any overlapping loads/stores
     533              :      that take place before the early break statement gets rejected aside from
     534              :      WAR dependencies.
     535              : 
     536              :      i.e.:
     537              : 
     538              :         a[i] = 8
     539              :         c = a[i]
     540              :         if (b[i])
     541              :           ...
     542              : 
     543              :         is not allowed, but
     544              : 
     545              :         c = a[i]
     546              :         a[i] = 8
     547              :         if (b[i])
     548              :           ...
     549              : 
     550              :         is which is the common case.  */
     551              : 
     552              : static opt_result
     553       156724 : vect_analyze_early_break_dependences (loop_vec_info loop_vinfo)
     554              : {
     555       156724 :   DUMP_VECT_SCOPE ("vect_analyze_early_break_dependences");
     556              : 
     557              :   /* List of all load data references found during traversal.  */
     558       156724 :   auto_vec<data_reference *> bases;
     559       156724 :   basic_block dest_bb = NULL;
     560              : 
     561       156724 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
     562       156724 :   class loop *loop_nest = loop_outer (loop);
     563              : 
     564       156724 :   if (dump_enabled_p ())
     565         1860 :     dump_printf_loc (MSG_NOTE, vect_location,
     566              :                      "loop contains multiple exits, analyzing"
     567              :                      " statement dependencies.\n");
     568              : 
     569       156724 :   if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
     570        31471 :     if (dump_enabled_p ())
     571          342 :       dump_printf_loc (MSG_NOTE, vect_location,
     572              :                        "alternate exit has been chosen as main exit.\n");
     573              : 
     574              :   /* Since we don't support general control flow, the location we'll move the
     575              :      side-effects to is always the latch connected exit.  When we support
     576              :      general control flow we can do better but for now this is fine.  Move
     577              :      side-effects to the in-loop destination of the last early exit.  For the
     578              :      PEELED case we move the side-effects to the latch block as this is
     579              :      guaranteed to be the last block to be executed when a vector iteration
     580              :      finished.  */
     581       156724 :   if (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo))
     582        31471 :     dest_bb = loop->latch;
     583              :   else
     584       125253 :     dest_bb = single_pred (loop->latch);
     585              : 
     586              :   /* We start looking from dest_bb, for the non-PEELED case we don't want to
     587              :      move any stores already present, but we do want to read and validate the
     588              :      loads.  */
     589       156724 :   basic_block bb = dest_bb;
     590              : 
     591              :   /* We move stores across all loads to the beginning of dest_bb, so
     592              :      the first block processed below doesn't need dependence checking.  */
     593       156724 :   bool check_deps = false;
     594              : 
     595       566040 :   do
     596              :     {
     597       361382 :       gimple_stmt_iterator gsi = gsi_last_bb (bb);
     598              : 
     599              :       /* Now analyze all the remaining statements and try to determine which
     600              :          instructions are allowed/needed to be moved.  */
     601      2744955 :       while (!gsi_end_p (gsi))
     602              :         {
     603      2389580 :           gimple *stmt = gsi_stmt (gsi);
     604      2389580 :           gsi_prev (&gsi);
     605      2389580 :           if (is_gimple_debug (stmt) || is_a <glabel *> (stmt))
     606      2116745 :             continue;
     607              : 
     608      1236846 :           stmt_vec_info orig_stmt_vinfo = loop_vinfo->lookup_stmt (stmt);
     609      1236846 :           stmt_vec_info stmt_vinfo
     610      1236846 :             = vect_stmt_to_vectorize (orig_stmt_vinfo);
     611      1236846 :           auto dr_ref = STMT_VINFO_DATA_REF (stmt_vinfo);
     612      1236846 :           if (!dr_ref)
     613              :             {
     614              :               /* Trapping statements after the last early exit are fine.  */
     615       957770 :               if (check_deps)
     616              :                 {
     617       590950 :                   bool could_trap_p = false;
     618       590950 :                   gimple *cur_stmt = STMT_VINFO_STMT (stmt_vinfo);
     619       590950 :                   could_trap_p = gimple_could_trap_p (cur_stmt);
     620       590950 :                   if (STMT_VINFO_IN_PATTERN_P (orig_stmt_vinfo))
     621              :                     {
     622       216015 :                       gimple_stmt_iterator gsi2;
     623       216015 :                       auto stmt_seq
     624       216015 :                         = STMT_VINFO_PATTERN_DEF_SEQ (orig_stmt_vinfo);
     625       216015 :                       for (gsi2 = gsi_start (stmt_seq);
     626       436100 :                            !could_trap_p && !gsi_end_p (gsi2); gsi_next (&gsi2))
     627              :                         {
     628       220085 :                           cur_stmt = gsi_stmt (gsi2);
     629       220085 :                           could_trap_p = gimple_could_trap_p (cur_stmt);
     630              :                         }
     631              :                     }
     632              : 
     633       590950 :                   if (could_trap_p)
     634              :                     {
     635         5617 :                       if (dump_enabled_p ())
     636          153 :                         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     637              :                              "cannot vectorize as operation may trap.\n");
     638         5617 :                       return opt_result::failure_at (cur_stmt,
     639              :                              "can't safely apply code motion to dependencies"
     640              :                              " to vectorize the early exit. %G may trap.\n",
     641              :                              cur_stmt);
     642              :                     }
     643              :                 }
     644              : 
     645       952153 :               continue;
     646       952153 :             }
     647              : 
     648              :           /* We know everything below dest_bb is safe since we know we
     649              :              had a full vector iteration when reaching it.  Either by
     650              :              the loop entry / IV exit test being last or because this
     651              :              is the loop latch itself.  */
     652       279076 :           if (!check_deps)
     653        11858 :             continue;
     654              : 
     655              :           /* Check if vector accesses to the object will be within bounds.
     656              :              must be a constant or assume loop will be versioned or niters
     657              :              bounded by VF so accesses are within range.  We only need to check
     658              :              the reads since writes are moved to a safe place where if we get
     659              :              there we know they are safe to perform.  */
     660       267218 :           if (DR_IS_READ (dr_ref))
     661              :             {
     662       246856 :               dr_set_safe_speculative_read_required (stmt_vinfo, true);
     663       246856 :               bool inbounds = ref_within_array_bound (stmt, DR_REF (dr_ref));
     664       246856 :               DR_SCALAR_KNOWN_BOUNDS (STMT_VINFO_DR_INFO (stmt_vinfo)) = inbounds;
     665              : 
     666       246856 :               if (dump_enabled_p ())
     667         2846 :                 dump_printf_loc (MSG_NOTE, vect_location,
     668              :                                  "marking DR (read) as possibly needing peeling "
     669              :                                  "for alignment at %G", stmt);
     670              :             }
     671              : 
     672       267218 :           if (DR_IS_READ (dr_ref))
     673       246856 :             bases.safe_push (dr_ref);
     674        20362 :           else if (DR_IS_WRITE (dr_ref))
     675              :             {
     676              :               /* We are moving writes down in the CFG.  To be sure that this
     677              :                  is valid after vectorization we have to check all the loads
     678              :                  we are sinking the stores past to see if any of them may
     679              :                  alias or are the same object.
     680              : 
     681              :                  Same objects will not be an issue because unless the store
     682              :                  is marked volatile the value can be forwarded.  If the
     683              :                  store is marked volatile we don't vectorize the loop
     684              :                  anyway.
     685              : 
     686              :                  That leaves the check for aliasing.  We don't really need
     687              :                  to care about the stores aliasing with each other since the
     688              :                  stores are moved in order so the effects are still observed
     689              :                  correctly.  This leaves the check for WAR dependencies
     690              :                  which we would be introducing here if the DR can alias.
     691              :                  The check is quadratic in loads/stores but I have not found
     692              :                  a better API to do this.  I believe all loads and stores
     693              :                  must be checked.  We also must check them when we
     694              :                  encountered the store, since we don't care about loads past
     695              :                  the store.  */
     696              : 
     697        55656 :               for (auto dr_read : bases)
     698        16114 :                 if (dr_may_alias_p (dr_ref, dr_read, loop_nest))
     699              :                   {
     700          390 :                     if (dump_enabled_p ())
     701            4 :                       dump_printf_loc (MSG_MISSED_OPTIMIZATION,
     702              :                                        vect_location,
     703              :                                        "early breaks not supported: "
     704              :                                        "overlapping loads and stores "
     705              :                                        "found before the break "
     706              :                                        "statement.\n");
     707              : 
     708          390 :                     return opt_result::failure_at (stmt,
     709              :                              "can't safely apply code motion to dependencies"
     710              :                              " to vectorize the early exit. %G may alias with"
     711              :                              " %G\n", stmt, dr_read->stmt);
     712              :                   }
     713              :             }
     714              : 
     715       533656 :           if (gimple_vdef (stmt))
     716              :             {
     717        19972 :               if (dump_enabled_p ())
     718          299 :                 dump_printf_loc (MSG_NOTE, vect_location,
     719              :                                  "==> recording stmt %G", stmt);
     720              : 
     721        19972 :               LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).safe_push (stmt);
     722              :             }
     723       760540 :           else if (gimple_vuse (stmt))
     724              :             {
     725       246856 :               LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo).safe_insert (0, stmt);
     726       246856 :               if (dump_enabled_p ())
     727         2846 :                 dump_printf_loc (MSG_NOTE, vect_location,
     728              :                                  "marked statement for vUSE update: %G", stmt);
     729              :             }
     730              :         }
     731              : 
     732       355375 :       if (!single_pred_p (bb))
     733              :         {
     734       150717 :           gcc_assert (bb == loop->header);
     735       150717 :           break;
     736              :         }
     737              : 
     738              :       /* If we possibly sink through a virtual PHI make sure to elide that.  */
     739       204658 :       if (gphi *vphi = get_virtual_phi (bb))
     740          107 :         LOOP_VINFO_EARLY_BRK_STORES (loop_vinfo).safe_push (vphi);
     741              : 
     742              :       /* All earlier blocks need dependence checking.  */
     743       204658 :       check_deps = true;
     744       204658 :       bb = single_pred (bb);
     745       204658 :     }
     746              :   while (1);
     747              : 
     748              :   /* We don't allow outer -> inner loop transitions which should have been
     749              :      trapped already during loop form analysis.  */
     750       150717 :   gcc_assert (dest_bb->loop_father == loop);
     751              : 
     752              :   /* Check that the destination block we picked has only one pred.  To relax this we
     753              :      have to take special care when moving the statements.  We don't currently support
     754              :      such control flow however this check is there to simplify how we handle
     755              :      labels that may be present anywhere in the IL.  This check is to ensure that the
     756              :      labels aren't significant for the CFG.  */
     757       150717 :   if (!single_pred (dest_bb))
     758            0 :     return opt_result::failure_at (vect_location,
     759              :                              "chosen loop exit block (BB %d) does not have a "
     760              :                              "single predecessor which is currently not "
     761              :                              "supported for early break vectorization.\n",
     762              :                              dest_bb->index);
     763              : 
     764       150717 :   LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo) = dest_bb;
     765              :   /* Check if loop has a side-effect (stores), force scalar epilogue.  */
     766       677162 :   for (auto dr : LOOP_VINFO_DATAREFS (loop_vinfo))
     767       254707 :     if (DR_IS_WRITE (dr))
     768              :       {
     769        15748 :         LOOP_VINFO_EARLY_BRK_NEEDS_EPILOG (loop_vinfo) = true;
     770        15748 :         break;
     771              :       }
     772              : 
     773       150717 :   if (!LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo).is_empty ())
     774              :     {
     775              :       /* All uses shall be updated to that of the first load.  Entries are
     776              :          stored in reverse order.  */
     777       138181 :       tree vuse = gimple_vuse (LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo).last ());
     778       384111 :       for (auto g : LOOP_VINFO_EARLY_BRK_VUSES (loop_vinfo))
     779              :         {
     780       245930 :           if (dump_enabled_p ())
     781         2783 :           dump_printf_loc (MSG_NOTE, vect_location,
     782              :                            "will update use: %T, mem_ref: %G", vuse, g);
     783              :         }
     784              :     }
     785              : 
     786       150717 :   if (dump_enabled_p ())
     787         1703 :     dump_printf_loc (MSG_NOTE, vect_location,
     788              :                      "recorded statements to be moved to BB %d\n",
     789         1703 :                      LOOP_VINFO_EARLY_BRK_DEST_BB (loop_vinfo)->index);
     790              : 
     791       150717 :   return opt_result::success ();
     792       156724 : }
     793              : 
     794              : /* Function vect_analyze_data_ref_dependences.
     795              : 
     796              :    Examine all the data references in the loop, and make sure there do not
     797              :    exist any data dependences between them.  Set *MAX_VF according to
     798              :    the maximum vectorization factor the data dependences allow.  */
     799              : 
     800              : opt_result
     801       409134 : vect_analyze_data_ref_dependences (loop_vec_info loop_vinfo,
     802              :                                    unsigned int *max_vf)
     803              : {
     804       409134 :   unsigned int i;
     805       409134 :   struct data_dependence_relation *ddr;
     806              : 
     807       409134 :   DUMP_VECT_SCOPE ("vect_analyze_data_ref_dependences");
     808              : 
     809       409134 :   if (!LOOP_VINFO_DDRS (loop_vinfo).exists ())
     810              :     {
     811       169473 :       LOOP_VINFO_DDRS (loop_vinfo)
     812       169473 :         .create (LOOP_VINFO_DATAREFS (loop_vinfo).length ()
     813       169473 :                  * LOOP_VINFO_DATAREFS (loop_vinfo).length ());
     814              :       /* We do not need read-read dependences.  */
     815       338946 :       bool res = compute_all_dependences (LOOP_VINFO_DATAREFS (loop_vinfo),
     816              :                                           &LOOP_VINFO_DDRS (loop_vinfo),
     817       169473 :                                           LOOP_VINFO_LOOP_NEST (loop_vinfo),
     818              :                                           false);
     819       169473 :       gcc_assert (res);
     820              :     }
     821              : 
     822       409134 :   LOOP_VINFO_NO_DATA_DEPENDENCIES (loop_vinfo) = true;
     823              : 
     824              :   /* For epilogues we either have no aliases or alias versioning
     825              :      was applied to original loop.  Therefore we may just get max_vf
     826              :      using VF of original loop.  */
     827       409134 :   if (LOOP_VINFO_EPILOGUE_P (loop_vinfo))
     828        13147 :     *max_vf = LOOP_VINFO_ORIG_MAX_VECT_FACTOR (loop_vinfo);
     829              :   else
     830      1907014 :     FOR_EACH_VEC_ELT (LOOP_VINFO_DDRS (loop_vinfo), i, ddr)
     831              :       {
     832      1527512 :         opt_result res
     833      1527512 :           = vect_analyze_data_ref_dependence (ddr, loop_vinfo, max_vf);
     834      1527512 :         if (!res)
     835        16485 :           return res;
     836              :       }
     837              : 
     838              :   /* If we have early break statements in the loop, check to see if they
     839              :      are of a form we can vectorizer.  */
     840       392649 :   if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
     841       156724 :     return vect_analyze_early_break_dependences (loop_vinfo);
     842              : 
     843       235925 :   return opt_result::success ();
     844              : }
     845              : 
     846              : 
     847              : /* Function vect_slp_analyze_data_ref_dependence.
     848              : 
     849              :    Classify the dependence between the memory-references DRA and DRB of DDR
     850              :    for VINFO using the classical (affine) data-dependence test.  Return
     851              :    chrec_known if they are provably independent, chrec_dont_know if the test
     852              :    cannot analyze them (in which case the caller can still try to disambiguate
     853              :    them with the alias oracle), and the dependence (NULL_TREE) otherwise.  */
     854              : 
     855              : static tree
     856      6949186 : vect_slp_analyze_data_ref_dependence (vec_info *vinfo,
     857              :                                       struct data_dependence_relation *ddr)
     858              : {
     859      6949186 :   struct data_reference *dra = DDR_A (ddr);
     860      6949186 :   struct data_reference *drb = DDR_B (ddr);
     861      6949186 :   dr_vec_info *dr_info_a = vinfo->lookup_dr (dra);
     862      6949186 :   dr_vec_info *dr_info_b = vinfo->lookup_dr (drb);
     863              : 
     864              :   /* We need to check dependences of statements marked as unvectorizable
     865              :      as well, they still can prohibit vectorization.  */
     866              : 
     867              :   /* Independent data accesses.  */
     868      6949186 :   if (DDR_ARE_DEPENDENT (ddr) == chrec_known)
     869              :     return chrec_known;
     870              : 
     871      1129576 :   if (dra == drb)
     872              :     return chrec_known;
     873              : 
     874              :   /* Read-read is OK.  */
     875         9150 :   if (DR_IS_READ (dra) && DR_IS_READ (drb))
     876              :     return chrec_known;
     877              : 
     878              :   /* If dra and drb are part of the same interleaving chain consider
     879              :      them independent.  */
     880         9150 :   if (STMT_VINFO_GROUPED_ACCESS (dr_info_a->stmt)
     881         9150 :       && (DR_GROUP_FIRST_ELEMENT (dr_info_a->stmt)
     882         9150 :           == DR_GROUP_FIRST_ELEMENT (dr_info_b->stmt)))
     883              :     return chrec_known;
     884              : 
     885              :   /* Unknown data dependence.  */
     886         9150 :   if (DDR_ARE_DEPENDENT (ddr) == chrec_dont_know)
     887              :     {
     888         9150 :       if  (dump_enabled_p ())
     889           60 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
     890              :                          "can't determine dependence between %T and %T\n",
     891              :                          DR_REF (dra), DR_REF (drb));
     892              :     }
     893            0 :   else if (dump_enabled_p ())
     894            0 :     dump_printf_loc (MSG_NOTE, vect_location,
     895              :                      "determined dependence between %T and %T\n",
     896              :                      DR_REF (dra), DR_REF (drb));
     897              : 
     898         9150 :   return DDR_ARE_DEPENDENT (ddr);
     899              : }
     900              : 
     901              : 
     902              : /* Analyze dependences involved in the transform of a store SLP NODE.  */
     903              : 
     904              : static bool
     905       676694 : vect_slp_analyze_store_dependences (vec_info *vinfo, slp_tree node)
     906              : {
     907              :   /* This walks over all stmts involved in the SLP store done
     908              :      in NODE verifying we can sink them up to the last stmt in the
     909              :      group.  */
     910       676694 :   stmt_vec_info last_access_info = vect_find_last_scalar_stmt_in_slp (node);
     911       676694 :   gcc_assert (DR_IS_WRITE (STMT_VINFO_DATA_REF (last_access_info)));
     912              : 
     913      3810630 :   for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
     914              :     {
     915      1789311 :       stmt_vec_info access_info
     916      1789311 :         = vect_orig_stmt (stmt_vinfo);
     917      1789311 :       if (access_info == last_access_info)
     918       668819 :         continue;
     919      1120492 :       data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
     920      1120492 :       ao_ref ref;
     921      1120492 :       bool ref_initialized_p = false;
     922      1120492 :       for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt);
     923     10738807 :            gsi_stmt (gsi) != last_access_info->stmt; gsi_next (&gsi))
     924              :         {
     925      9627078 :           gimple *stmt = gsi_stmt (gsi);
     926     17046589 :           if (! gimple_vuse (stmt))
     927      2677544 :             continue;
     928              : 
     929              :           /* If we couldn't record a (single) data reference for this stmt,
     930              :              or the classical dependence test cannot analyze it, we have to
     931              :              resort to the alias oracle.  */
     932      6949534 :           stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt);
     933      6949534 :           data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info);
     934      6949534 :           if (dr_b)
     935              :             {
     936      6948906 :               gcc_assert (!gimple_visited_p (stmt));
     937              : 
     938      6948906 :               ddr_p ddr = initialize_data_dependence_relation (dr_a,
     939      6948906 :                                                                dr_b, vNULL);
     940      6948906 :               tree dep = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
     941      6948906 :               free_dependence_relation (ddr);
     942      6948906 :               if (dep == chrec_known)
     943      6939792 :                 continue;
     944         9114 :               if (dep != chrec_dont_know)
     945         8763 :                 return false;
     946              :               /* Unknown dependence - fall through to the alias oracle.  */
     947              :             }
     948              : 
     949              :           /* We are moving a store - this means we cannot use TBAA for
     950              :              disambiguation.  */
     951         9742 :           if (!ref_initialized_p)
     952              :             {
     953         9458 :               ao_ref_init (&ref, DR_REF (dr_a));
     954         9458 :               ref_initialized_p = true;
     955              :             }
     956         9742 :           if (stmt_may_clobber_ref_p_1 (stmt, &ref, false)
     957         9742 :               || ref_maybe_used_by_stmt_p (stmt, &ref, false))
     958              :             return false;
     959              :         }
     960              :     }
     961              :   return true;
     962              : }
     963              : 
     964              : /* Analyze dependences involved in the transform of a load SLP NODE.  STORES
     965              :    contain the vector of scalar stores of this instance if we are
     966              :    disambiguating the loads.  */
     967              : 
     968              : static bool
     969       178194 : vect_slp_analyze_load_dependences (vec_info *vinfo, slp_tree node,
     970              :                                    vec<stmt_vec_info> stores,
     971              :                                    stmt_vec_info last_store_info)
     972              : {
     973              :   /* This walks over all stmts involved in the SLP load done
     974              :      in NODE verifying we can hoist them up to the first stmt in the
     975              :      group.  */
     976       178194 :   stmt_vec_info first_access_info = vect_find_first_scalar_stmt_in_slp (node);
     977       178194 :   gcc_assert (DR_IS_READ (STMT_VINFO_DATA_REF (first_access_info)));
     978              : 
     979       980634 :   for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (node))
     980              :     {
     981       446088 :       if (! stmt_vinfo)
     982       187885 :         continue;
     983       446088 :       stmt_vec_info access_info
     984       446088 :         = vect_orig_stmt (stmt_vinfo);
     985       446088 :       if (access_info == first_access_info)
     986       187885 :         continue;
     987       258203 :       data_reference *dr_a = STMT_VINFO_DATA_REF (access_info);
     988       258203 :       ao_ref ref;
     989       258203 :       bool ref_initialized_p = false;
     990       258203 :       hash_set<stmt_vec_info> grp_visited;
     991       258203 :       for (gimple_stmt_iterator gsi = gsi_for_stmt (access_info->stmt);
     992      3350468 :            gsi_stmt (gsi) != first_access_info->stmt; gsi_prev (&gsi))
     993              :         {
     994      3092301 :           gimple *stmt = gsi_stmt (gsi);
     995      5159867 :           if (! gimple_vdef (stmt))
     996      3034468 :             continue;
     997              : 
     998       282853 :           stmt_vec_info stmt_info = vinfo->lookup_stmt (stmt);
     999              : 
    1000              :           /* If we run into a store of this same instance (we've just
    1001              :              marked those) then delay dependence checking until we run
    1002              :              into the last store because this is where it will have
    1003              :              been sunk to (and we verified that we can do that already).  */
    1004       282853 :           if (gimple_visited_p (stmt))
    1005              :             {
    1006       225020 :               if (stmt_info != last_store_info)
    1007       225018 :                 continue;
    1008              : 
    1009           10 :               for (stmt_vec_info &store_info : stores)
    1010              :                 {
    1011            4 :                   data_reference *store_dr = STMT_VINFO_DATA_REF (store_info);
    1012            4 :                   ddr_p ddr = initialize_data_dependence_relation
    1013            4 :                                 (dr_a, store_dr, vNULL);
    1014            4 :                   tree dep
    1015            4 :                     = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
    1016            4 :                   free_dependence_relation (ddr);
    1017            4 :                   if (dep == chrec_known)
    1018            4 :                     continue;
    1019            0 :                   if (dep != chrec_dont_know)
    1020           36 :                     return false;
    1021              :                   /* The classical dependence test cannot analyze this;
    1022              :                      resort to the alias oracle.  We are hoisting a load
    1023              :                      so TBAA may be used for disambiguation.  */
    1024            0 :                   if (!ref_initialized_p)
    1025              :                     {
    1026            0 :                       ao_ref_init (&ref, DR_REF (dr_a));
    1027            0 :                       ref_initialized_p = true;
    1028              :                     }
    1029            0 :                   if (stmt_may_clobber_ref_p_1 (store_info->stmt, &ref, true))
    1030              :                     return false;
    1031              :                 }
    1032            2 :               continue;
    1033            2 :             }
    1034              : 
    1035       119703 :           auto check_hoist = [&] (stmt_vec_info stmt_info) -> bool
    1036              :             {
    1037              :               /* We are hoisting a load - this means we can use TBAA for
    1038              :                  disambiguation.  */
    1039        61870 :               if (!ref_initialized_p)
    1040              :                 {
    1041        11658 :                   ao_ref_init (&ref, DR_REF (dr_a));
    1042        11658 :                   ref_initialized_p = true;
    1043              :                 }
    1044        61870 :               if (stmt_may_clobber_ref_p_1 (stmt_info->stmt, &ref, true))
    1045              :                 {
    1046              :                   /* If we couldn't record a (single) data reference for this
    1047              :                      stmt we have to give up now.  */
    1048          276 :                   data_reference *dr_b = STMT_VINFO_DATA_REF (stmt_info);
    1049          276 :                   if (!dr_b)
    1050              :                     return false;
    1051          276 :                   ddr_p ddr = initialize_data_dependence_relation (dr_a,
    1052          276 :                                                                    dr_b, vNULL);
    1053          276 :                   tree dep
    1054          276 :                     = vect_slp_analyze_data_ref_dependence (vinfo, ddr);
    1055          276 :                   free_dependence_relation (ddr);
    1056              :                   /* The alias oracle above could not rule out a conflict;
    1057              :                      only a proven-independent (chrec_known) result lets us
    1058              :                      hoist the load past this store.  */
    1059          276 :                   if (dep != chrec_known)
    1060           36 :                     return false;
    1061              :                 }
    1062              :               /* No dependence.  */
    1063              :               return true;
    1064        57833 :             };
    1065        57833 :           if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    1066              :             {
    1067              :               /* When we run into a store group we have to honor
    1068              :                  that earlier stores might be moved here.  We don't
    1069              :                  know exactly which and where to since we lack a
    1070              :                  back-mapping from DR to SLP node, so assume all
    1071              :                  earlier stores are sunk here.  It's enough to
    1072              :                  consider the last stmt of a group for this.
    1073              :                  ???  Both this and the fact that we disregard that
    1074              :                  the conflicting instance might be removed later
    1075              :                  is overly conservative.  */
    1076        57198 :               if (!grp_visited.add (DR_GROUP_FIRST_ELEMENT (stmt_info)))
    1077        11580 :                 for (auto store_info = DR_GROUP_FIRST_ELEMENT (stmt_info);
    1078       137193 :                      store_info != NULL;
    1079       125613 :                      store_info = DR_GROUP_NEXT_ELEMENT (store_info))
    1080       125649 :                   if ((store_info == stmt_info
    1081       114078 :                        || get_later_stmt (store_info, stmt_info) == stmt_info)
    1082       175313 :                       && !check_hoist (store_info))
    1083              :                     return false;
    1084              :             }
    1085              :           else
    1086              :             {
    1087          635 :               if (!check_hoist (stmt_info))
    1088              :                 return false;
    1089              :             }
    1090              :         }
    1091       258203 :     }
    1092              :   return true;
    1093              : }
    1094              : 
    1095              : 
    1096              : /* Function vect_analyze_data_ref_dependences.
    1097              : 
    1098              :    Examine all the data references in the basic-block, and make sure there
    1099              :    do not exist any data dependences between them.  Set *MAX_VF according to
    1100              :    the maximum vectorization factor the data dependences allow.  */
    1101              : 
    1102              : bool
    1103       834481 : vect_slp_analyze_instance_dependence (vec_info *vinfo, slp_instance instance)
    1104              : {
    1105       834481 :   DUMP_VECT_SCOPE ("vect_slp_analyze_instance_dependence");
    1106              : 
    1107              :   /* The stores of this instance are at the root of the SLP tree.  */
    1108       834481 :   slp_tree store = NULL;
    1109       834481 :   if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store)
    1110       676694 :     store = SLP_INSTANCE_TREE (instance);
    1111              : 
    1112              :   /* Verify we can sink stores to the vectorized stmt insert location.  */
    1113       676694 :   stmt_vec_info last_store_info = NULL;
    1114       676694 :   if (store)
    1115              :     {
    1116       676694 :       if (! vect_slp_analyze_store_dependences (vinfo, store))
    1117              :         return false;
    1118              : 
    1119              :       /* Mark stores in this instance and remember the last one.  */
    1120       667931 :       last_store_info = vect_find_last_scalar_stmt_in_slp (store);
    1121      3783423 :       for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (store))
    1122      1779630 :         gimple_set_visited (STMT_VINFO_STMT (stmt_vinfo), true);
    1123              :     }
    1124              : 
    1125       825718 :   bool res = true;
    1126              : 
    1127              :   /* Verify we can sink loads to the vectorized stmt insert location,
    1128              :      special-casing stores of this instance.  */
    1129      1280764 :   for (slp_tree &load : SLP_INSTANCE_LOADS (instance))
    1130       178194 :     if (! vect_slp_analyze_load_dependences (vinfo, load,
    1131              :                                              store
    1132              :                                              ? SLP_TREE_SCALAR_STMTS (store)
    1133              :                                              : vNULL, last_store_info))
    1134              :       {
    1135              :         res = false;
    1136              :         break;
    1137              :       }
    1138              : 
    1139              :   /* Unset the visited flag.  */
    1140       825718 :   if (store)
    1141      3783423 :     for (auto stmt_vinfo : SLP_TREE_SCALAR_STMTS (store))
    1142      1779630 :       gimple_set_visited (STMT_VINFO_STMT (stmt_vinfo), false);
    1143              : 
    1144              :   /* If this is a SLP instance with a store check if there's a dependent
    1145              :      load that cannot be forwarded from a previous iteration of a loop
    1146              :      both are in.  This is to avoid situations like that in PR115777.  */
    1147       825718 :   if (res && store)
    1148              :     {
    1149       667907 :       stmt_vec_info store_info
    1150       667907 :         = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (store)[0]);
    1151       667907 :       class loop *store_loop = gimple_bb (store_info->stmt)->loop_father;
    1152       667907 :       if (! loop_outer (store_loop))
    1153       568809 :         return res;
    1154        99098 :       vec<loop_p> loop_nest;
    1155        99098 :       loop_nest.create (1);
    1156        99098 :       loop_nest.quick_push (store_loop);
    1157        99098 :       data_reference *drs = nullptr;
    1158       183305 :       for (slp_tree &load : SLP_INSTANCE_LOADS (instance))
    1159              :         {
    1160        37114 :           if (! STMT_VINFO_GROUPED_ACCESS (SLP_TREE_SCALAR_STMTS (load)[0]))
    1161            0 :             continue;
    1162        37114 :           stmt_vec_info load_info
    1163        37114 :             = DR_GROUP_FIRST_ELEMENT (SLP_TREE_SCALAR_STMTS (load)[0]);
    1164        37114 :           if (gimple_bb (load_info->stmt)->loop_father != store_loop)
    1165         5039 :             continue;
    1166              : 
    1167              :           /* For now concern ourselves with write-after-read as we also
    1168              :              only look for re-use of the store within the same SLP instance.
    1169              :              We can still get a RAW here when the instance contains a PHI
    1170              :              with a backedge though, thus this test.  */
    1171        32075 :           if (! vect_stmt_dominates_stmt_p (STMT_VINFO_STMT (load_info),
    1172              :                                             STMT_VINFO_STMT (store_info)))
    1173        11721 :             continue;
    1174              : 
    1175        20354 :           if (! drs)
    1176              :             {
    1177        19490 :               drs = create_data_ref (loop_preheader_edge (store_loop),
    1178              :                                      store_loop,
    1179        19490 :                                      DR_REF (STMT_VINFO_DATA_REF (store_info)),
    1180              :                                      store_info->stmt, false, false);
    1181        19490 :               if (! DR_BASE_ADDRESS (drs)
    1182        16452 :                   || TREE_CODE (DR_STEP (drs)) != INTEGER_CST)
    1183              :                 break;
    1184              :             }
    1185        17037 :           data_reference *drl
    1186        17037 :             = create_data_ref (loop_preheader_edge (store_loop),
    1187              :                                store_loop,
    1188        17037 :                                DR_REF (STMT_VINFO_DATA_REF (load_info)),
    1189              :                                load_info->stmt, true, false);
    1190              : 
    1191              :           /* See whether the DRs have a known constant distance throughout
    1192              :              the containing loop iteration.  */
    1193        32370 :           if (! DR_BASE_ADDRESS (drl)
    1194        14966 :               || ! operand_equal_p (DR_STEP (drs), DR_STEP (drl))
    1195         8952 :               || ! operand_equal_p (DR_BASE_ADDRESS (drs),
    1196         8952 :                                     DR_BASE_ADDRESS (drl))
    1197        18751 :               || ! operand_equal_p (DR_OFFSET (drs), DR_OFFSET (drl)))
    1198              :             {
    1199        15333 :               free_data_ref (drl);
    1200        15333 :               continue;
    1201              :             }
    1202              : 
    1203              :           /* If the next iteration load overlaps with a non-power-of-two offset
    1204              :              we are surely failing any STLF attempt.  */
    1205         1704 :           HOST_WIDE_INT step = TREE_INT_CST_LOW (DR_STEP (drl));
    1206         1704 :           unsigned HOST_WIDE_INT sizes
    1207         1704 :             = (TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drs))))
    1208         1704 :                * DR_GROUP_SIZE (store_info));
    1209         1704 :           unsigned HOST_WIDE_INT sizel
    1210         1704 :             = (TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drl))))
    1211         1704 :                * DR_GROUP_SIZE (load_info));
    1212         1704 :           if (ranges_overlap_p (TREE_INT_CST_LOW (DR_INIT (drl)) + step, sizel,
    1213         1704 :                                 TREE_INT_CST_LOW (DR_INIT (drs)), sizes))
    1214              :             {
    1215          854 :               unsigned HOST_WIDE_INT dist
    1216          854 :                 = absu_hwi (TREE_INT_CST_LOW (DR_INIT (drl)) + step
    1217          854 :                             - TREE_INT_CST_LOW (DR_INIT (drs)));
    1218          854 :               poly_uint64 loadsz = tree_to_poly_uint64
    1219          854 :                                      (TYPE_SIZE_UNIT (SLP_TREE_VECTYPE (load)));
    1220          854 :               poly_uint64 storesz = tree_to_poly_uint64
    1221          854 :                                     (TYPE_SIZE_UNIT (SLP_TREE_VECTYPE (store)));
    1222              :               /* When the overlap aligns with vector sizes used for the loads
    1223              :                  and the vector stores are larger or equal to the loads
    1224              :                  forwarding should work.  */
    1225         1708 :               if (maybe_gt (loadsz, storesz) || ! multiple_p (dist, loadsz))
    1226           66 :                 load->avoid_stlf_fail = true;
    1227              :             }
    1228         1704 :           free_data_ref (drl);
    1229              :         }
    1230        99098 :       if (drs)
    1231        19490 :         free_data_ref (drs);
    1232        99098 :       loop_nest.release ();
    1233              :     }
    1234              : 
    1235              :   return res;
    1236              : }
    1237              : 
    1238              : /* Return the misalignment of DR_INFO accessed in VECTYPE with OFFSET
    1239              :    applied.  */
    1240              : 
    1241              : int
    1242      7025148 : dr_misalignment (dr_vec_info *dr_info, tree vectype, poly_int64 offset)
    1243              : {
    1244      7025148 :   HOST_WIDE_INT diff = 0;
    1245              :   /* Alignment is only analyzed for the first element of a DR group,
    1246              :      use that but adjust misalignment by the offset of the access.  */
    1247      7025148 :   if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt))
    1248              :     {
    1249      2354427 :       dr_vec_info *first_dr
    1250      2354427 :         = STMT_VINFO_DR_INFO (DR_GROUP_FIRST_ELEMENT (dr_info->stmt));
    1251              :       /* vect_analyze_data_ref_accesses guarantees that DR_INIT are
    1252              :          INTEGER_CSTs and the first element in the group has the lowest
    1253              :          address.  */
    1254      2354427 :       diff = (TREE_INT_CST_LOW (DR_INIT (dr_info->dr))
    1255      2354427 :               - TREE_INT_CST_LOW (DR_INIT (first_dr->dr)));
    1256      2354427 :       gcc_assert (diff >= 0);
    1257              :       dr_info = first_dr;
    1258              :     }
    1259              : 
    1260      7025148 :   int misalign = dr_info->misalignment;
    1261      7025148 :   gcc_assert (misalign != DR_MISALIGNMENT_UNINITIALIZED);
    1262      7025148 :   if (misalign == DR_MISALIGNMENT_UNKNOWN)
    1263              :     return misalign;
    1264              : 
    1265              :   /* If the access is only aligned for a vector type with smaller alignment
    1266              :      requirement the access has unknown misalignment.  */
    1267      4252810 :   if (maybe_lt (dr_info->target_alignment * BITS_PER_UNIT,
    1268      4252810 :                 targetm.vectorize.preferred_vector_alignment (vectype)))
    1269              :     return DR_MISALIGNMENT_UNKNOWN;
    1270              : 
    1271              :   /* Apply the offset from the DR group start and the externally supplied
    1272              :      offset which can for example result from a negative stride access.  */
    1273      4252801 :   poly_int64 misalignment = misalign + diff + offset;
    1274              : 
    1275              :   /* Below we reject compile-time non-constant target alignments, but if
    1276              :      our misalignment is zero, then we are known to already be aligned
    1277              :      w.r.t. any such possible target alignment.  */
    1278      4252801 :   if (known_eq (misalignment, 0))
    1279              :     return 0;
    1280              : 
    1281       659821 :   unsigned HOST_WIDE_INT target_alignment_c;
    1282       659821 :   if (!dr_info->target_alignment.is_constant (&target_alignment_c)
    1283       659821 :       || !known_misalignment (misalignment, target_alignment_c, &misalign))
    1284              :     return DR_MISALIGNMENT_UNKNOWN;
    1285       659821 :   return misalign;
    1286              : }
    1287              : 
    1288              : /* Record the base alignment guarantee given by DRB, which occurs
    1289              :    in STMT_INFO.  */
    1290              : 
    1291              : static void
    1292      4948933 : vect_record_base_alignment (vec_info *vinfo, stmt_vec_info stmt_info,
    1293              :                             innermost_loop_behavior *drb)
    1294              : {
    1295      4948933 :   bool existed;
    1296      4948933 :   std::pair<stmt_vec_info, innermost_loop_behavior *> &entry
    1297      4948933 :     = vinfo->base_alignments.get_or_insert (drb->base_address, &existed);
    1298      4948933 :   if (!existed || entry.second->base_alignment < drb->base_alignment)
    1299              :     {
    1300      1503585 :       entry = std::make_pair (stmt_info, drb);
    1301      1503585 :       if (dump_enabled_p ())
    1302        35155 :         dump_printf_loc (MSG_NOTE, vect_location,
    1303              :                          "recording new base alignment for %T\n"
    1304              :                          "  alignment:    %d\n"
    1305              :                          "  misalignment: %d\n"
    1306              :                          "  based on:     %G",
    1307              :                          drb->base_address,
    1308              :                          drb->base_alignment,
    1309              :                          drb->base_misalignment,
    1310              :                          stmt_info->stmt);
    1311              :     }
    1312      4948933 : }
    1313              : 
    1314              : /* If the region we're going to vectorize is reached, all unconditional
    1315              :    data references occur at least once.  We can therefore pool the base
    1316              :    alignment guarantees from each unconditional reference.  Do this by
    1317              :    going through all the data references in VINFO and checking whether
    1318              :    the containing statement makes the reference unconditionally.  If so,
    1319              :    record the alignment of the base address in VINFO so that it can be
    1320              :    used for all other references with the same base.  */
    1321              : 
    1322              : void
    1323      1126979 : vect_record_base_alignments (vec_info *vinfo)
    1324              : {
    1325      1126979 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    1326       449612 :   class loop *loop = loop_vinfo ? LOOP_VINFO_LOOP (loop_vinfo) : NULL;
    1327     16382576 :   for (data_reference *dr : vinfo->shared->datarefs)
    1328              :     {
    1329     13108851 :       dr_vec_info *dr_info = vinfo->lookup_dr (dr);
    1330     13108851 :       stmt_vec_info stmt_info = dr_info->stmt;
    1331     13108851 :       if (!DR_IS_CONDITIONAL_IN_STMT (dr)
    1332     13099729 :           && STMT_VINFO_VECTORIZABLE (stmt_info)
    1333      4970640 :           && !STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    1334              :         {
    1335      4947375 :           vect_record_base_alignment (vinfo, stmt_info, &DR_INNERMOST (dr));
    1336              : 
    1337              :           /* If DR is nested in the loop that is being vectorized, we can also
    1338              :              record the alignment of the base wrt the outer loop.  */
    1339     14104488 :           if (loop && nested_in_vect_loop_p (loop, stmt_info))
    1340         1558 :             vect_record_base_alignment
    1341         1558 :               (vinfo, stmt_info, &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info));
    1342              :         }
    1343              :     }
    1344      1126979 : }
    1345              : 
    1346              : /* Function vect_compute_data_ref_alignment
    1347              : 
    1348              :    Compute the misalignment of the data reference DR_INFO when vectorizing
    1349              :    with VECTYPE.
    1350              : 
    1351              :    Output:
    1352              :    1. initialized misalignment info for DR_INFO
    1353              : 
    1354              :    FOR NOW: No analysis is actually performed. Misalignment is calculated
    1355              :    only for trivial cases. TODO.  */
    1356              : 
    1357              : static void
    1358      1695121 : vect_compute_data_ref_alignment (vec_info *vinfo, dr_vec_info *dr_info,
    1359              :                                  tree vectype)
    1360              : {
    1361      1695121 :   stmt_vec_info stmt_info = dr_info->stmt;
    1362      1695121 :   vec_base_alignments *base_alignments = &vinfo->base_alignments;
    1363      1695121 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    1364      1695121 :   class loop *loop = NULL;
    1365      1695121 :   tree ref = DR_REF (dr_info->dr);
    1366              : 
    1367      1695121 :   if (dump_enabled_p ())
    1368        55502 :     dump_printf_loc (MSG_NOTE, vect_location,
    1369              :                      "vect_compute_data_ref_alignment:\n");
    1370              : 
    1371      1695121 :   if (loop_vinfo)
    1372       889961 :     loop = LOOP_VINFO_LOOP (loop_vinfo);
    1373              : 
    1374              :   /* Initialize misalignment to unknown.  */
    1375      1695121 :   SET_DR_MISALIGNMENT (dr_info, DR_MISALIGNMENT_UNKNOWN);
    1376              : 
    1377      1695121 :   if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    1378              :     return;
    1379              : 
    1380      1670764 :   innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
    1381      1670764 :   bool step_preserves_misalignment_p;
    1382              : 
    1383      1670764 :   poly_uint64 vector_alignment
    1384      1670764 :     = exact_div (targetm.vectorize.preferred_vector_alignment (vectype),
    1385              :                  BITS_PER_UNIT);
    1386              : 
    1387      1670764 :   if (loop_vinfo
    1388      1670764 :       && dr_safe_speculative_read_required (stmt_info))
    1389              :     {
    1390              :       /* The required target alignment must be a power-of-2 value and is
    1391              :          computed as the product of vector element size, VF and group size.
    1392              :          We compute the constant part first as VF may be a variable.  For
    1393              :          variable VF, the power-of-2 check of VF is deferred to runtime.  */
    1394       337525 :       auto align_factor_c
    1395       337525 :         = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
    1396       337525 :       if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    1397        90930 :         align_factor_c *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
    1398              : 
    1399       337525 :       poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    1400       337525 :       poly_uint64 new_alignment = vf * align_factor_c;
    1401              : 
    1402       675050 :       if ((vf.is_constant () && pow2p_hwi (new_alignment.to_constant ()))
    1403              :           || (!vf.is_constant () && pow2p_hwi (align_factor_c)))
    1404              :         {
    1405       274008 :           if (dump_enabled_p ())
    1406              :             {
    1407         3617 :               dump_printf_loc (MSG_NOTE, vect_location,
    1408              :                                "alignment increased due to early break to ");
    1409         3617 :               dump_dec (MSG_NOTE, new_alignment);
    1410         3617 :               dump_printf (MSG_NOTE, " bytes.\n");
    1411              :             }
    1412       274008 :           vector_alignment = new_alignment;
    1413              :         }
    1414              :     }
    1415              : 
    1416      1670764 :   SET_DR_TARGET_ALIGNMENT (dr_info, vector_alignment);
    1417              : 
    1418              :   /* If the main loop has peeled for alignment we have no way of knowing
    1419              :      whether the data accesses in the epilogues are aligned.  We can't at
    1420              :      compile time answer the question whether we have entered the main loop or
    1421              :      not.  Fixes PR 92351.  */
    1422      1670764 :   if (loop_vinfo)
    1423              :     {
    1424       865604 :       loop_vec_info orig_loop_vinfo = LOOP_VINFO_ORIG_LOOP_INFO (loop_vinfo);
    1425       865604 :       if (orig_loop_vinfo
    1426        34131 :           && LOOP_VINFO_PEELING_FOR_ALIGNMENT (orig_loop_vinfo) != 0)
    1427              :         return;
    1428              :     }
    1429              : 
    1430      1670545 :   unsigned HOST_WIDE_INT vect_align_c;
    1431      1670545 :   if (!vector_alignment.is_constant (&vect_align_c))
    1432              :     return;
    1433              : 
    1434              :   /* No step for BB vectorization.  */
    1435      1670545 :   if (!loop)
    1436              :     {
    1437       805160 :       gcc_assert (integer_zerop (drb->step));
    1438              :       step_preserves_misalignment_p = true;
    1439              :     }
    1440              : 
    1441              :   else
    1442              :     {
    1443              :       /* We can only use base and misalignment information relative to
    1444              :          an innermost loop if the misalignment stays the same throughout the
    1445              :          execution of the loop.  As above, this is the case if the stride of
    1446              :          the dataref evenly divides by the alignment.  Make sure to check
    1447              :          previous epilogues and the main loop.  */
    1448              :       step_preserves_misalignment_p = true;
    1449              :       auto lvinfo = loop_vinfo;
    1450      1765536 :       while (lvinfo)
    1451              :         {
    1452       900151 :           poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (lvinfo);
    1453       900151 :           step_preserves_misalignment_p
    1454       900151 :             &= multiple_p (drb->step_alignment * vf, vect_align_c);
    1455       900151 :           lvinfo = LOOP_VINFO_ORIG_LOOP_INFO (lvinfo);
    1456              :         }
    1457              : 
    1458       865385 :       if (!step_preserves_misalignment_p && dump_enabled_p ())
    1459          334 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1460              :                          "step doesn't divide the vector alignment.\n");
    1461              : 
    1462              :       /* In case the dataref is in an inner-loop of the loop that is being
    1463              :          vectorized (LOOP), we use the base and misalignment information
    1464              :          relative to the outer-loop (LOOP).  This is ok only if the
    1465              :          misalignment stays the same throughout the execution of the
    1466              :          inner-loop, which is why we have to check that the stride of the
    1467              :          dataref in the inner-loop evenly divides by the vector alignment.  */
    1468       865385 :       if (step_preserves_misalignment_p
    1469       865385 :           && nested_in_vect_loop_p (loop, stmt_info))
    1470              :         {
    1471         1557 :           step_preserves_misalignment_p
    1472         1557 :             = (DR_STEP_ALIGNMENT (dr_info->dr) % vect_align_c) == 0;
    1473              : 
    1474         1557 :           if (dump_enabled_p ())
    1475              :             {
    1476          503 :               if (step_preserves_misalignment_p)
    1477          358 :                 dump_printf_loc (MSG_NOTE, vect_location,
    1478              :                                  "inner step divides the vector alignment.\n");
    1479              :               else
    1480          145 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1481              :                                  "inner step doesn't divide the vector"
    1482              :                                  " alignment.\n");
    1483              :             }
    1484              :         }
    1485              :     }
    1486              : 
    1487      1670545 :   unsigned int base_alignment = drb->base_alignment;
    1488      1670545 :   unsigned int base_misalignment = drb->base_misalignment;
    1489              : 
    1490              :   /* Calculate the maximum of the pooled base address alignment and the
    1491              :      alignment that we can compute for DR itself.  */
    1492      1670545 :   std::pair<stmt_vec_info, innermost_loop_behavior *> *entry
    1493      1670545 :     = base_alignments->get (drb->base_address);
    1494      1670545 :   if (entry
    1495      1665898 :       && base_alignment < (*entry).second->base_alignment
    1496      1674288 :       && (loop_vinfo
    1497         2817 :           || (dominated_by_p (CDI_DOMINATORS, gimple_bb (stmt_info->stmt),
    1498         2817 :                               gimple_bb (entry->first->stmt))
    1499         2706 :               && (gimple_bb (stmt_info->stmt) != gimple_bb (entry->first->stmt)
    1500         2421 :                   || (entry->first->dr_aux.group <= dr_info->group)))))
    1501              :     {
    1502         3615 :       base_alignment = entry->second->base_alignment;
    1503         3615 :       base_misalignment = entry->second->base_misalignment;
    1504              :     }
    1505              : 
    1506      1670545 :   if (drb->offset_alignment < vect_align_c
    1507      1602298 :       || !step_preserves_misalignment_p
    1508              :       /* We need to know whether the step wrt the vectorized loop is
    1509              :          negative when computing the starting misalignment below.  */
    1510      1593424 :       || TREE_CODE (drb->step) != INTEGER_CST)
    1511              :     {
    1512       105631 :       if (dump_enabled_p ())
    1513         3646 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1514              :                          "Unknown alignment for access: %T\n", ref);
    1515              :       return;
    1516              :     }
    1517              : 
    1518      1564914 :   if (base_alignment < vect_align_c)
    1519              :     {
    1520       791220 :       unsigned int max_alignment;
    1521       791220 :       tree base = get_base_for_alignment (drb->base_address, &max_alignment);
    1522       791220 :       if (max_alignment < vect_align_c
    1523       788608 :           || (loop_vinfo && LOOP_VINFO_EPILOGUE_P (loop_vinfo))
    1524      1579828 :           || !vect_can_force_dr_alignment_p (base,
    1525       766790 :                                              vect_align_c * BITS_PER_UNIT))
    1526              :         {
    1527       579140 :           if (dump_enabled_p ())
    1528        16344 :             dump_printf_loc (MSG_NOTE, vect_location,
    1529              :                              "can't force alignment of ref: %T\n", ref);
    1530       579140 :           return;
    1531              :         }
    1532              : 
    1533              :       /* Force the alignment of the decl.
    1534              :          NOTE: This is the only change to the code we make during
    1535              :          the analysis phase, before deciding to vectorize the loop.  */
    1536       212080 :       if (dump_enabled_p ())
    1537         8358 :         dump_printf_loc (MSG_NOTE, vect_location,
    1538              :                          "force alignment of %T\n", ref);
    1539              : 
    1540       212080 :       dr_info->base_decl = base;
    1541       212080 :       dr_info->base_misaligned = true;
    1542       212080 :       base_misalignment = 0;
    1543              :     }
    1544       985774 :   poly_int64 misalignment
    1545       985774 :     = base_misalignment + wi::to_poly_offset (drb->init).force_shwi ();
    1546              : 
    1547       985774 :   unsigned int const_misalignment;
    1548       985774 :   if (!known_misalignment (misalignment, vect_align_c, &const_misalignment))
    1549              :     {
    1550              :       if (dump_enabled_p ())
    1551              :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1552              :                          "Non-constant misalignment for access: %T\n", ref);
    1553              :       return;
    1554              :     }
    1555              : 
    1556       985774 :   SET_DR_MISALIGNMENT (dr_info, const_misalignment);
    1557              : 
    1558       985774 :   if (dump_enabled_p ())
    1559        34179 :     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1560              :                      "misalign = %d bytes of ref %T\n",
    1561              :                      const_misalignment, ref);
    1562              : 
    1563              :   return;
    1564              : }
    1565              : 
    1566              : /* Return whether DR_INFO, which is related to DR_PEEL_INFO in
    1567              :    that it only differs in DR_INIT, is aligned if DR_PEEL_INFO
    1568              :    is made aligned via peeling.  */
    1569              : 
    1570              : static bool
    1571      2058286 : vect_dr_aligned_if_related_peeled_dr_is (dr_vec_info *dr_info,
    1572              :                                          dr_vec_info *dr_peel_info)
    1573              : {
    1574      2058286 :   if (multiple_p (DR_TARGET_ALIGNMENT (dr_peel_info),
    1575      2058908 :                   DR_TARGET_ALIGNMENT (dr_info)))
    1576              :     {
    1577      2057664 :       poly_offset_int diff
    1578      2057664 :         = (wi::to_poly_offset (DR_INIT (dr_peel_info->dr))
    1579      2057664 :            - wi::to_poly_offset (DR_INIT (dr_info->dr)));
    1580      2057664 :       if (known_eq (diff, 0)
    1581      2057664 :           || multiple_p (diff, DR_TARGET_ALIGNMENT (dr_info)))
    1582       780409 :         return true;
    1583              :     }
    1584              :   return false;
    1585              : }
    1586              : 
    1587              : /* Return whether DR_INFO is aligned if DR_PEEL_INFO is made
    1588              :    aligned via peeling.  */
    1589              : 
    1590              : static bool
    1591       212577 : vect_dr_aligned_if_peeled_dr_is (dr_vec_info *dr_info,
    1592              :                                  dr_vec_info *dr_peel_info)
    1593              : {
    1594       212577 :   if (!operand_equal_p (DR_BASE_ADDRESS (dr_info->dr),
    1595       212577 :                         DR_BASE_ADDRESS (dr_peel_info->dr), 0)
    1596        53394 :       || !operand_equal_p (DR_OFFSET (dr_info->dr),
    1597        53394 :                            DR_OFFSET (dr_peel_info->dr), 0)
    1598       265059 :       || !operand_equal_p (DR_STEP (dr_info->dr),
    1599        52482 :                            DR_STEP (dr_peel_info->dr), 0))
    1600              :     return false;
    1601              : 
    1602        52084 :   return vect_dr_aligned_if_related_peeled_dr_is (dr_info, dr_peel_info);
    1603              : }
    1604              : 
    1605              : /* Compute the value for dr_info->misalign so that the access appears
    1606              :    aligned.  This is used by peeling to compensate for dr_misalignment
    1607              :    applying the offset for negative step.  */
    1608              : 
    1609              : int
    1610        22766 : vect_dr_misalign_for_aligned_access (dr_vec_info *dr_info)
    1611              : {
    1612        22766 :   if (tree_int_cst_sgn (DR_STEP (dr_info->dr)) >= 0)
    1613              :     return 0;
    1614              : 
    1615          243 :   tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
    1616          243 :   poly_int64 misalignment
    1617          243 :     = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
    1618          243 :        * TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
    1619              : 
    1620          243 :   unsigned HOST_WIDE_INT target_alignment_c;
    1621          243 :   int misalign;
    1622          243 :   if (!dr_info->target_alignment.is_constant (&target_alignment_c)
    1623          243 :       || !known_misalignment (misalignment, target_alignment_c, &misalign))
    1624              :     return DR_MISALIGNMENT_UNKNOWN;
    1625          243 :   return misalign;
    1626              : }
    1627              : 
    1628              : /* Function vect_update_misalignment_for_peel.
    1629              :    Sets DR_INFO's misalignment
    1630              :    - to 0 if it has the same alignment as DR_PEEL_INFO,
    1631              :    - to the misalignment computed using NPEEL if DR_INFO's salignment is known,
    1632              :    - to -1 (unknown) otherwise.
    1633              : 
    1634              :    DR_INFO - the data reference whose misalignment is to be adjusted.
    1635              :    DR_PEEL_INFO - the data reference whose misalignment is being made
    1636              :                   zero in the vector loop by the peel.
    1637              :    NPEEL - the number of iterations in the peel loop if the misalignment
    1638              :            of DR_PEEL_INFO is known at compile time.  */
    1639              : 
    1640              : static void
    1641         2907 : vect_update_misalignment_for_peel (dr_vec_info *dr_info,
    1642              :                                    dr_vec_info *dr_peel_info, int npeel)
    1643              : {
    1644              :   /* If dr_info is aligned of dr_peel_info is, then mark it so.  */
    1645         2907 :   if (vect_dr_aligned_if_peeled_dr_is (dr_info, dr_peel_info))
    1646              :     {
    1647          448 :       SET_DR_MISALIGNMENT (dr_info,
    1648              :                            vect_dr_misalign_for_aligned_access (dr_peel_info));
    1649          448 :       return;
    1650              :     }
    1651              : 
    1652         2459 :   unsigned HOST_WIDE_INT alignment;
    1653         2459 :   if (DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment)
    1654         2459 :       && known_alignment_for_access_p (dr_info,
    1655         2459 :                                        STMT_VINFO_VECTYPE (dr_info->stmt))
    1656          222 :       && known_alignment_for_access_p (dr_peel_info,
    1657          222 :                                        STMT_VINFO_VECTYPE (dr_peel_info->stmt)))
    1658              :     {
    1659          202 :       int misal = dr_info->misalignment;
    1660          202 :       misal += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
    1661          202 :       misal &= alignment - 1;
    1662          202 :       set_dr_misalignment (dr_info, misal);
    1663          202 :       return;
    1664              :     }
    1665              : 
    1666         2257 :   if (dump_enabled_p ())
    1667           40 :     dump_printf_loc (MSG_NOTE, vect_location, "Setting misalignment " \
    1668              :                      "to unknown (-1).\n");
    1669         2257 :   SET_DR_MISALIGNMENT (dr_info, DR_MISALIGNMENT_UNKNOWN);
    1670              : }
    1671              : 
    1672              : /* Return true if alignment is relevant for DR_INFO.  */
    1673              : 
    1674              : static bool
    1675      2331244 : vect_relevant_for_alignment_p (dr_vec_info *dr_info)
    1676              : {
    1677      2331244 :   stmt_vec_info stmt_info = dr_info->stmt;
    1678              : 
    1679      2331244 :   if (!STMT_VINFO_RELEVANT_P (stmt_info))
    1680              :     return false;
    1681              : 
    1682              :   /* For interleaving, only the alignment of the first access matters.  */
    1683      2329792 :   if (STMT_VINFO_GROUPED_ACCESS (stmt_info)
    1684      2691910 :       && DR_GROUP_FIRST_ELEMENT (stmt_info) != stmt_info)
    1685              :     return false;
    1686              : 
    1687              :   /* Scatter-gather and invariant accesses continue to address individual
    1688              :      scalars, so vector-level alignment is irrelevant.  */
    1689      2204283 :   if (STMT_VINFO_GATHER_SCATTER_P (stmt_info)
    1690      2204283 :       || integer_zerop (DR_STEP (dr_info->dr)))
    1691              :     return false;
    1692              : 
    1693              :   /* Strided accesses perform only component accesses, alignment is
    1694              :      irrelevant for them.  */
    1695      2121759 :   if (STMT_VINFO_STRIDED_P (stmt_info)
    1696      2121759 :       && !STMT_VINFO_GROUPED_ACCESS (stmt_info))
    1697        85486 :     return false;
    1698              : 
    1699              :   return true;
    1700              : }
    1701              : 
    1702              : /* Given an memory reference EXP return whether its alignment is less
    1703              :    than its size.  */
    1704              : 
    1705              : static bool
    1706      1683389 : not_size_aligned (tree exp)
    1707              : {
    1708      1683389 :   if (!tree_fits_uhwi_p (TYPE_SIZE (TREE_TYPE (exp))))
    1709              :     return true;
    1710              : 
    1711      1683389 :   return (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (exp)))
    1712      1683389 :           > get_object_alignment (exp));
    1713              : }
    1714              : 
    1715              : /* Function vector_alignment_reachable_p
    1716              : 
    1717              :    Return true if vector alignment for DR_INFO is reachable by peeling
    1718              :    a few loop iterations.  Return false otherwise.  */
    1719              : 
    1720              : static bool
    1721       661996 : vector_alignment_reachable_p (dr_vec_info *dr_info, poly_uint64 vf)
    1722              : {
    1723       661996 :   stmt_vec_info stmt_info = dr_info->stmt;
    1724       661996 :   tree vectype = STMT_VINFO_VECTYPE (stmt_info);
    1725       661996 :   poly_uint64 nelements = TYPE_VECTOR_SUBPARTS (vectype);
    1726      1323992 :   poly_uint64 vector_size = GET_MODE_SIZE (TYPE_MODE (vectype));
    1727       661996 :   unsigned elem_size = vector_element_size (vector_size, nelements);
    1728       661996 :   unsigned group_size = 1;
    1729              : 
    1730       661996 :   if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    1731              :     {
    1732              :       /* For interleaved access we peel only if number of iterations in
    1733              :          the prolog loop ({VF - misalignment}), is a multiple of the
    1734              :          number of the interleaved accesses.  */
    1735              : 
    1736              :       /* FORNOW: handle only known alignment.  */
    1737        92430 :       if (!known_alignment_for_access_p (dr_info, vectype))
    1738       661996 :         return false;
    1739              : 
    1740        52738 :       unsigned mis_in_elements = dr_misalignment (dr_info, vectype) / elem_size;
    1741        65444 :       if (!multiple_p (nelements - mis_in_elements, DR_GROUP_SIZE (stmt_info)))
    1742              :         return false;
    1743              : 
    1744        12706 :       group_size = DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
    1745              :     }
    1746              : 
    1747              :   /* If the vectorization factor does not guarantee DR advancement of
    1748              :      a multiple of the target alignment no peeling will help.  */
    1749       582272 :   if (!multiple_p (elem_size * group_size * vf, dr_target_alignment (dr_info)))
    1750          167 :     return false;
    1751              : 
    1752              :   /* If misalignment is known at the compile time then allow peeling
    1753              :      only if natural alignment is reachable through peeling.  */
    1754       582105 :   if (known_alignment_for_access_p (dr_info, vectype)
    1755       904425 :       && !aligned_access_p (dr_info, vectype))
    1756              :     {
    1757        16329 :       HOST_WIDE_INT elmsize =
    1758        16329 :                 int_cst_value (TYPE_SIZE_UNIT (TREE_TYPE (vectype)));
    1759        16329 :       if (dump_enabled_p ())
    1760              :         {
    1761          784 :           dump_printf_loc (MSG_NOTE, vect_location,
    1762              :                            "data size = %wd. misalignment = %d.\n", elmsize,
    1763              :                            dr_misalignment (dr_info, vectype));
    1764              :         }
    1765        16329 :       if (dr_misalignment (dr_info, vectype) % elmsize)
    1766              :         {
    1767           72 :           if (dump_enabled_p ())
    1768            7 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1769              :                              "data size does not divide the misalignment.\n");
    1770              :           return false;
    1771              :         }
    1772              :     }
    1773              : 
    1774       582033 :   if (!known_alignment_for_access_p (dr_info, vectype))
    1775              :     {
    1776       259785 :       tree type = TREE_TYPE (DR_REF (dr_info->dr));
    1777       259785 :       bool is_packed = not_size_aligned (DR_REF (dr_info->dr));
    1778       259785 :       if (dump_enabled_p ())
    1779        18643 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    1780              :                          "Unknown misalignment, %snaturally aligned\n",
    1781              :                          is_packed ? "not " : "");
    1782       259785 :       return targetm.vectorize.vector_alignment_reachable (type, is_packed);
    1783              :     }
    1784              : 
    1785              :   return true;
    1786              : }
    1787              : 
    1788              : 
    1789              : /* Calculate the cost of the memory access represented by DR_INFO.  */
    1790              : 
    1791              : static void
    1792       794778 : vect_get_data_access_cost (vec_info *vinfo, dr_vec_info *dr_info,
    1793              :                            dr_alignment_support alignment_support_scheme,
    1794              :                            int misalignment,
    1795              :                            unsigned int *inside_cost,
    1796              :                            unsigned int *outside_cost,
    1797              :                            stmt_vector_for_cost *body_cost_vec,
    1798              :                            stmt_vector_for_cost *prologue_cost_vec)
    1799              : {
    1800       794778 :   stmt_vec_info stmt_info = dr_info->stmt;
    1801              : 
    1802       794778 :   if (DR_IS_READ (dr_info->dr))
    1803       557038 :     vect_get_load_cost (vinfo, stmt_info, NULL, 1,
    1804              :                         alignment_support_scheme, misalignment, true,
    1805              :                         inside_cost, outside_cost, prologue_cost_vec,
    1806              :                         body_cost_vec, false);
    1807              :   else
    1808       237740 :     vect_get_store_cost (vinfo,stmt_info, NULL, 1,
    1809              :                          alignment_support_scheme, misalignment, inside_cost,
    1810              :                          body_cost_vec);
    1811              : 
    1812       794778 :   if (dump_enabled_p ())
    1813        33316 :     dump_printf_loc (MSG_NOTE, vect_location,
    1814              :                      "vect_get_data_access_cost: inside_cost = %d, "
    1815              :                      "outside_cost = %d.\n", *inside_cost, *outside_cost);
    1816       794778 : }
    1817              : 
    1818              : 
    1819              : typedef struct _vect_peel_info
    1820              : {
    1821              :   dr_vec_info *dr_info;
    1822              :   int npeel;
    1823              :   unsigned int count;
    1824              : } *vect_peel_info;
    1825              : 
    1826              : typedef struct _vect_peel_extended_info
    1827              : {
    1828              :   vec_info *vinfo;
    1829              :   struct _vect_peel_info peel_info;
    1830              :   unsigned int inside_cost;
    1831              :   unsigned int outside_cost;
    1832              : } *vect_peel_extended_info;
    1833              : 
    1834              : 
    1835              : /* Peeling hashtable helpers.  */
    1836              : 
    1837              : struct peel_info_hasher : free_ptr_hash <_vect_peel_info>
    1838              : {
    1839              :   static inline hashval_t hash (const _vect_peel_info *);
    1840              :   static inline bool equal (const _vect_peel_info *, const _vect_peel_info *);
    1841              : };
    1842              : 
    1843              : inline hashval_t
    1844       797232 : peel_info_hasher::hash (const _vect_peel_info *peel_info)
    1845              : {
    1846       797232 :   return (hashval_t) peel_info->npeel;
    1847              : }
    1848              : 
    1849              : inline bool
    1850       413511 : peel_info_hasher::equal (const _vect_peel_info *a, const _vect_peel_info *b)
    1851              : {
    1852       413511 :   return (a->npeel == b->npeel);
    1853              : }
    1854              : 
    1855              : 
    1856              : /* Insert DR_INFO into peeling hash table with NPEEL as key.  */
    1857              : 
    1858              : static void
    1859       384675 : vect_peeling_hash_insert (hash_table<peel_info_hasher> *peeling_htab,
    1860              :                           loop_vec_info loop_vinfo, dr_vec_info *dr_info,
    1861              :                           int npeel, bool supportable_if_not_aligned)
    1862              : {
    1863       384675 :   struct _vect_peel_info elem, *slot;
    1864       384675 :   _vect_peel_info **new_slot;
    1865              : 
    1866       384675 :   elem.npeel = npeel;
    1867       384675 :   slot = peeling_htab->find (&elem);
    1868       384675 :   if (slot)
    1869       165865 :     slot->count++;
    1870              :   else
    1871              :     {
    1872       218810 :       slot = XNEW (struct _vect_peel_info);
    1873       218810 :       slot->npeel = npeel;
    1874       218810 :       slot->dr_info = dr_info;
    1875       218810 :       slot->count = 1;
    1876       218810 :       new_slot = peeling_htab->find_slot (slot, INSERT);
    1877       218810 :       *new_slot = slot;
    1878              :     }
    1879              : 
    1880              :   /* If this DR is not supported with unknown misalignment then bias
    1881              :      this slot when the cost model is disabled.  */
    1882       384675 :   if (!supportable_if_not_aligned
    1883       384675 :       && unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
    1884         5649 :     slot->count += VECT_MAX_COST;
    1885       384675 : }
    1886              : 
    1887              : 
    1888              : /* Traverse peeling hash table to find peeling option that aligns maximum
    1889              :    number of data accesses.  */
    1890              : 
    1891              : int
    1892        38987 : vect_peeling_hash_get_most_frequent (_vect_peel_info **slot,
    1893              :                                      _vect_peel_extended_info *max)
    1894              : {
    1895        38987 :   vect_peel_info elem = *slot;
    1896              : 
    1897        38987 :   if (elem->count > max->peel_info.count
    1898        23796 :       || (elem->count == max->peel_info.count
    1899        18872 :           && max->peel_info.npeel > elem->npeel))
    1900              :     {
    1901        15207 :       max->peel_info.npeel = elem->npeel;
    1902        15207 :       max->peel_info.count = elem->count;
    1903        15207 :       max->peel_info.dr_info = elem->dr_info;
    1904              :     }
    1905              : 
    1906        38987 :   return 1;
    1907              : }
    1908              : 
    1909              : /* Get the costs of peeling NPEEL iterations for LOOP_VINFO, checking
    1910              :    data access costs for all data refs.  If UNKNOWN_MISALIGNMENT is true,
    1911              :    npeel is computed at runtime but DR0_INFO's misalignment will be zero
    1912              :    after peeling.  */
    1913              : 
    1914              : static void
    1915       437511 : vect_get_peeling_costs_all_drs (loop_vec_info loop_vinfo,
    1916              :                                 dr_vec_info *dr0_info,
    1917              :                                 unsigned int *inside_cost,
    1918              :                                 unsigned int *outside_cost,
    1919              :                                 stmt_vector_for_cost *body_cost_vec,
    1920              :                                 stmt_vector_for_cost *prologue_cost_vec,
    1921              :                                 unsigned int npeel)
    1922              : {
    1923       437511 :   vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
    1924              : 
    1925       437511 :   bool dr0_alignment_known_p
    1926              :     = (dr0_info
    1927       801289 :        && known_alignment_for_access_p (dr0_info,
    1928       363778 :                                         STMT_VINFO_VECTYPE (dr0_info->stmt)));
    1929              : 
    1930      2145458 :   for (data_reference *dr : datarefs)
    1931              :     {
    1932       832925 :       dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
    1933       832925 :       if (!vect_relevant_for_alignment_p (dr_info))
    1934        38147 :         continue;
    1935              : 
    1936       794778 :       tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
    1937       794778 :       dr_alignment_support alignment_support_scheme;
    1938       794778 :       int misalignment;
    1939       794778 :       unsigned HOST_WIDE_INT alignment;
    1940              : 
    1941       794778 :       bool negative = tree_int_cst_compare (DR_STEP (dr_info->dr),
    1942       794778 :                                             size_zero_node) < 0;
    1943       794778 :       poly_int64 off = 0;
    1944       794778 :       if (negative)
    1945        24943 :         off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
    1946        24943 :                * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
    1947              : 
    1948       794778 :       if (npeel == 0)
    1949       403544 :         misalignment = dr_misalignment (dr_info, vectype, off);
    1950       391234 :       else if (dr_info == dr0_info
    1951       391234 :                || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
    1952              :         misalignment = 0;
    1953       133526 :       else if (!dr0_alignment_known_p
    1954         9768 :                || !known_alignment_for_access_p (dr_info, vectype)
    1955       143294 :                || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment))
    1956              :         misalignment = DR_MISALIGNMENT_UNKNOWN;
    1957              :       else
    1958              :         {
    1959         8676 :           misalignment = dr_misalignment (dr_info, vectype, off);
    1960         8676 :           misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
    1961         8676 :           misalignment &= alignment - 1;
    1962              :         }
    1963       794778 :       alignment_support_scheme
    1964       794778 :         = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
    1965              :                                          misalignment);
    1966              : 
    1967       794778 :       vect_get_data_access_cost (loop_vinfo, dr_info,
    1968              :                                  alignment_support_scheme, misalignment,
    1969              :                                  inside_cost, outside_cost,
    1970              :                                  body_cost_vec, prologue_cost_vec);
    1971              :     }
    1972       437511 : }
    1973              : 
    1974              : /* Traverse peeling hash table and calculate cost for each peeling option.
    1975              :    Find the one with the lowest cost.  */
    1976              : 
    1977              : int
    1978       159228 : vect_peeling_hash_get_lowest_cost (_vect_peel_info **slot,
    1979              :                                    _vect_peel_extended_info *min)
    1980              : {
    1981       159228 :   vect_peel_info elem = *slot;
    1982       159228 :   unsigned int inside_cost = 0, outside_cost = 0;
    1983       159228 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (min->vinfo);
    1984       159228 :   stmt_vector_for_cost prologue_cost_vec, body_cost_vec;
    1985              : 
    1986       159228 :   prologue_cost_vec.create (2);
    1987       159228 :   body_cost_vec.create (2);
    1988              : 
    1989       159228 :   vect_get_peeling_costs_all_drs (loop_vinfo, elem->dr_info, &inside_cost,
    1990              :                                   &outside_cost, &body_cost_vec,
    1991       159228 :                                   &prologue_cost_vec, elem->npeel);
    1992              : 
    1993       159228 :   body_cost_vec.release ();
    1994       159228 :   prologue_cost_vec.release ();
    1995              : 
    1996       159228 :   outside_cost += vect_get_known_peeling_cost (loop_vinfo, elem->npeel);
    1997              : 
    1998       159228 :   if (inside_cost < min->inside_cost
    1999         2631 :       || (inside_cost == min->inside_cost
    2000         1283 :           && outside_cost < min->outside_cost))
    2001              :     {
    2002       156603 :       min->inside_cost = inside_cost;
    2003       156603 :       min->outside_cost = outside_cost;
    2004       156603 :       min->peel_info.dr_info = elem->dr_info;
    2005       156603 :       min->peel_info.npeel = elem->npeel;
    2006       156603 :       min->peel_info.count = elem->count;
    2007              :     }
    2008              : 
    2009       159228 :   return 1;
    2010              : }
    2011              : 
    2012              : 
    2013              : /* Choose best peeling option by traversing peeling hash table and either
    2014              :    choosing an option with the lowest cost (if cost model is enabled) or the
    2015              :    option that aligns as many accesses as possible.  */
    2016              : 
    2017              : static struct _vect_peel_extended_info
    2018       169785 : vect_peeling_hash_choose_best_peeling (hash_table<peel_info_hasher> *peeling_htab,
    2019              :                                        loop_vec_info loop_vinfo)
    2020              : {
    2021       169785 :    struct _vect_peel_extended_info res;
    2022              : 
    2023       169785 :    res.peel_info.dr_info = NULL;
    2024       169785 :    res.vinfo = loop_vinfo;
    2025              : 
    2026       169785 :    if (!unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
    2027              :      {
    2028       154647 :        res.inside_cost = INT_MAX;
    2029       154647 :        res.outside_cost = INT_MAX;
    2030       154647 :        peeling_htab->traverse <_vect_peel_extended_info *,
    2031       313875 :                                vect_peeling_hash_get_lowest_cost> (&res);
    2032              :      }
    2033              :    else
    2034              :      {
    2035        15138 :        res.peel_info.count = 0;
    2036        15138 :        peeling_htab->traverse <_vect_peel_extended_info *,
    2037        54125 :                                vect_peeling_hash_get_most_frequent> (&res);
    2038        15138 :        res.inside_cost = 0;
    2039        15138 :        res.outside_cost = 0;
    2040              :      }
    2041              : 
    2042       169785 :    return res;
    2043              : }
    2044              : 
    2045              : /* Return if vectorization is definitely, possibly, or unlikely to be
    2046              :    supportable after loop peeling.  */
    2047              : 
    2048              : static enum peeling_support
    2049        85481 : vect_peeling_supportable (loop_vec_info loop_vinfo, dr_vec_info *dr0_info,
    2050              :                           unsigned npeel)
    2051              : {
    2052        85481 :   vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
    2053        85481 :   enum dr_alignment_support supportable_dr_alignment;
    2054              : 
    2055        85481 :   bool dr0_alignment_known_p
    2056       170962 :     = known_alignment_for_access_p (dr0_info,
    2057        85481 :                                     STMT_VINFO_VECTYPE (dr0_info->stmt));
    2058        85481 :   bool has_unsupported_dr_p = false;
    2059        85481 :   unsigned int dr0_step = tree_to_shwi (DR_STEP (dr0_info->dr));
    2060        85481 :   int known_unsupported_misalignment = DR_MISALIGNMENT_UNKNOWN;
    2061              : 
    2062              :   /* Check if each data ref can be vectorized after peeling.  */
    2063       363910 :   for (data_reference *dr : datarefs)
    2064              :     {
    2065       124503 :       if (dr == dr0_info->dr)
    2066        84475 :         continue;
    2067              : 
    2068        40028 :       dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
    2069        40028 :       if (!vect_relevant_for_alignment_p (dr_info)
    2070        40028 :           || vect_dr_aligned_if_peeled_dr_is (dr_info, dr0_info))
    2071         6761 :         continue;
    2072              : 
    2073        33267 :       tree vectype = STMT_VINFO_VECTYPE (dr_info->stmt);
    2074        33267 :       int misalignment;
    2075        33267 :       unsigned HOST_WIDE_INT alignment;
    2076        33267 :       if (!dr0_alignment_known_p
    2077         2534 :           || !known_alignment_for_access_p (dr_info, vectype)
    2078        35801 :           || !DR_TARGET_ALIGNMENT (dr_info).is_constant (&alignment))
    2079              :         misalignment = DR_MISALIGNMENT_UNKNOWN;
    2080              :       else
    2081              :         {
    2082         2520 :           misalignment = dr_misalignment (dr_info, vectype);
    2083         2520 :           misalignment += npeel * TREE_INT_CST_LOW (DR_STEP (dr_info->dr));
    2084         2520 :           misalignment &= alignment - 1;
    2085              :         }
    2086        33267 :       supportable_dr_alignment
    2087        33267 :         = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
    2088              :                                          misalignment);
    2089        33267 :       if (supportable_dr_alignment == dr_unaligned_unsupported)
    2090              :         {
    2091        31892 :           has_unsupported_dr_p = true;
    2092              : 
    2093              :           /* If unaligned unsupported DRs exist, we do following checks to see
    2094              :              if they can be mutually aligned to support vectorization.  If yes,
    2095              :              we can try peeling and create a runtime (mutual alignment) check
    2096              :              to guard the peeled loop.  If no, return PEELING_UNSUPPORTED.  */
    2097              : 
    2098              :           /* 1) If unaligned unsupported DRs have different alignment steps, the
    2099              :                 probability of DRs being mutually aligned is very low, and it's
    2100              :                 quite complex to check mutual alignment at runtime.  We return
    2101              :                 PEELING_UNSUPPORTED in this case.  */
    2102        31892 :           if (tree_to_shwi (DR_STEP (dr)) != dr0_step)
    2103        85481 :             return peeling_unsupported;
    2104              : 
    2105              :           /* 2) Based on above same alignment step condition, if one known
    2106              :                 misaligned DR has zero misalignment, or different misalignment
    2107              :                 amount from another known misaligned DR, peeling is unable to
    2108              :                 help make all these DRs aligned together.  We won't try peeling
    2109              :                 with versioning anymore.  */
    2110        27572 :           int curr_dr_misalignment = dr_misalignment (dr_info, vectype);
    2111        27572 :           if (curr_dr_misalignment == 0)
    2112              :             return peeling_unsupported;
    2113        14856 :           if (known_unsupported_misalignment != DR_MISALIGNMENT_UNKNOWN)
    2114              :             {
    2115            8 :               if (curr_dr_misalignment != DR_MISALIGNMENT_UNKNOWN
    2116            8 :                   && curr_dr_misalignment != known_unsupported_misalignment)
    2117              :                 return peeling_unsupported;
    2118              :             }
    2119              :           else
    2120              :             known_unsupported_misalignment = curr_dr_misalignment;
    2121              :         }
    2122              :     }
    2123              : 
    2124              :   /* Vectorization is known to be supportable with peeling alone when there is
    2125              :      no unsupported DR.  */
    2126        68445 :   return has_unsupported_dr_p ? peeling_maybe_supported
    2127              :                               : peeling_known_supported;
    2128              : }
    2129              : 
    2130              : /* Compare two data-references DRA and DRB to group them into chunks
    2131              :    with related alignment.  */
    2132              : 
    2133              : static int
    2134      4776375 : dr_align_group_sort_cmp (const void *dra_, const void *drb_)
    2135              : {
    2136      4776375 :   data_reference_p dra = *(data_reference_p *)const_cast<void *>(dra_);
    2137      4776375 :   data_reference_p drb = *(data_reference_p *)const_cast<void *>(drb_);
    2138      4776375 :   int cmp;
    2139              : 
    2140              :   /* Stabilize sort.  */
    2141      4776375 :   if (dra == drb)
    2142              :     return 0;
    2143              : 
    2144              :   /* Ordering of DRs according to base.  */
    2145      4776375 :   cmp = data_ref_compare_tree (DR_BASE_ADDRESS (dra),
    2146              :                                DR_BASE_ADDRESS (drb));
    2147      4776375 :   if (cmp != 0)
    2148              :     return cmp;
    2149              : 
    2150              :   /* And according to DR_OFFSET.  */
    2151      2102388 :   cmp = data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb));
    2152      2102388 :   if (cmp != 0)
    2153              :     return cmp;
    2154              : 
    2155              :   /* And after step.  */
    2156      2087867 :   cmp = data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb));
    2157      2087867 :   if (cmp != 0)
    2158              :     return cmp;
    2159              : 
    2160              :   /* Then sort after DR_INIT.  In case of identical DRs sort after stmt UID.  */
    2161      2082664 :   cmp = data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb));
    2162      2082664 :   if (cmp == 0)
    2163       256413 :     return gimple_uid (DR_STMT (dra)) < gimple_uid (DR_STMT (drb)) ? -1 : 1;
    2164              :   return cmp;
    2165              : }
    2166              : 
    2167              : /* Function vect_enhance_data_refs_alignment
    2168              : 
    2169              :    This pass will use loop versioning and loop peeling in order to enhance
    2170              :    the alignment of data references in the loop.
    2171              : 
    2172              :    FOR NOW: we assume that whatever versioning/peeling takes place, only the
    2173              :    original loop is to be vectorized.  Any other loops that are created by
    2174              :    the transformations performed in this pass - are not supposed to be
    2175              :    vectorized.  This restriction will be relaxed.
    2176              : 
    2177              :    This pass will require a cost model to guide it whether to apply peeling
    2178              :    or versioning or a combination of the two.  For example, the scheme that
    2179              :    intel uses when given a loop with several memory accesses, is as follows:
    2180              :    choose one memory access ('p') which alignment you want to force by doing
    2181              :    peeling.  Then, either (1) generate a loop in which 'p' is aligned and all
    2182              :    other accesses are not necessarily aligned, or (2) use loop versioning to
    2183              :    generate one loop in which all accesses are aligned, and another loop in
    2184              :    which only 'p' is necessarily aligned.
    2185              : 
    2186              :    ("Automatic Intra-Register Vectorization for the Intel Architecture",
    2187              :    Aart J.C. Bik, Milind Girkar, Paul M. Grey and Ximmin Tian, International
    2188              :    Journal of Parallel Programming, Vol. 30, No. 2, April 2002.)
    2189              : 
    2190              :    Devising a cost model is the most critical aspect of this work.  It will
    2191              :    guide us on which access to peel for, whether to use loop versioning, how
    2192              :    many versions to create, etc.  The cost model will probably consist of
    2193              :    generic considerations as well as target specific considerations (on
    2194              :    powerpc for example, misaligned stores are more painful than misaligned
    2195              :    loads).
    2196              : 
    2197              :    Here are the general steps involved in alignment enhancements:
    2198              : 
    2199              :      -- original loop, before alignment analysis:
    2200              :         for (i=0; i<N; i++){
    2201              :           x = q[i];                     # DR_MISALIGNMENT(q) = unknown
    2202              :           p[i] = y;                     # DR_MISALIGNMENT(p) = unknown
    2203              :         }
    2204              : 
    2205              :      -- After vect_compute_data_refs_alignment:
    2206              :         for (i=0; i<N; i++){
    2207              :           x = q[i];                     # DR_MISALIGNMENT(q) = 3
    2208              :           p[i] = y;                     # DR_MISALIGNMENT(p) = unknown
    2209              :         }
    2210              : 
    2211              :      -- Possibility 1: we do loop versioning:
    2212              :      if (p is aligned) {
    2213              :         for (i=0; i<N; i++){ # loop 1A
    2214              :           x = q[i];                     # DR_MISALIGNMENT(q) = 3
    2215              :           p[i] = y;                     # DR_MISALIGNMENT(p) = 0
    2216              :         }
    2217              :      }
    2218              :      else {
    2219              :         for (i=0; i<N; i++){ # loop 1B
    2220              :           x = q[i];                     # DR_MISALIGNMENT(q) = 3
    2221              :           p[i] = y;                     # DR_MISALIGNMENT(p) = unaligned
    2222              :         }
    2223              :      }
    2224              : 
    2225              :      -- Possibility 2: we do loop peeling:
    2226              :      for (i = 0; i < 3; i++){        # (scalar loop, not to be vectorized).
    2227              :         x = q[i];
    2228              :         p[i] = y;
    2229              :      }
    2230              :      for (i = 3; i < N; i++){        # loop 2A
    2231              :         x = q[i];                       # DR_MISALIGNMENT(q) = 0
    2232              :         p[i] = y;                       # DR_MISALIGNMENT(p) = unknown
    2233              :      }
    2234              : 
    2235              :      -- Possibility 3: combination of loop peeling and versioning:
    2236              :      if (p & q are mutually aligned) {
    2237              :         for (i=0; i<3; i++){ # (peeled loop iterations).
    2238              :           x = q[i];
    2239              :           p[i] = y;
    2240              :         }
    2241              :         for (i=3; i<N; i++){ # loop 3A
    2242              :           x = q[i];                     # DR_MISALIGNMENT(q) = 0
    2243              :           p[i] = y;                     # DR_MISALIGNMENT(p) = 0
    2244              :         }
    2245              :      }
    2246              :      else {
    2247              :         for (i=0; i<N; i++){ # (scalar loop, not to be vectorized).
    2248              :           x = q[i];                     # DR_MISALIGNMENT(q) = 3
    2249              :           p[i] = y;                     # DR_MISALIGNMENT(p) = unknown
    2250              :         }
    2251              :      }
    2252              : 
    2253              :      These loops are later passed to loop_transform to be vectorized.  The
    2254              :      vectorizer will use the alignment information to guide the transformation
    2255              :      (whether to generate regular loads/stores, or with special handling for
    2256              :      misalignment).  */
    2257              : 
    2258              : opt_result
    2259       416353 : vect_enhance_data_refs_alignment (loop_vec_info loop_vinfo)
    2260              : {
    2261       416353 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    2262       416353 :   dr_vec_info *first_store = NULL;
    2263       416353 :   dr_vec_info *dr0_info = NULL;
    2264       416353 :   struct data_reference *dr;
    2265       416353 :   unsigned int i;
    2266       416353 :   bool do_peeling = false;
    2267       416353 :   bool do_versioning = false;
    2268       416353 :   bool try_peeling_with_versioning = false;
    2269       416353 :   unsigned int npeel = 0;
    2270       416353 :   bool one_misalignment_known = false;
    2271       416353 :   bool one_misalignment_unknown = false;
    2272       416353 :   bool one_dr_unsupportable = false;
    2273       416353 :   dr_vec_info *unsupportable_dr_info = NULL;
    2274       416353 :   unsigned int dr0_same_align_drs = 0, first_store_same_align_drs = 0;
    2275       416353 :   hash_table<peel_info_hasher> peeling_htab (1);
    2276              : 
    2277       416353 :   DUMP_VECT_SCOPE ("vect_enhance_data_refs_alignment");
    2278              : 
    2279              :   /* Reset data so we can safely be called multiple times.  */
    2280       416353 :   LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).truncate (0);
    2281       416353 :   LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = 0;
    2282              : 
    2283       416353 :   if (LOOP_VINFO_DATAREFS (loop_vinfo).is_empty ())
    2284        15063 :     return opt_result::success ();
    2285              : 
    2286              :   /* Sort the vector of datarefs so DRs that have the same or dependent
    2287              :      alignment are next to each other.  */
    2288       401290 :   auto_vec<data_reference_p> datarefs
    2289       401290 :     = LOOP_VINFO_DATAREFS (loop_vinfo).copy ();
    2290       401290 :   datarefs.qsort (dr_align_group_sort_cmp);
    2291              : 
    2292              :   /* Compute the number of DRs that become aligned when we peel
    2293              :      a dataref so it becomes aligned.  */
    2294       802580 :   auto_vec<unsigned> n_same_align_refs (datarefs.length ());
    2295       401290 :   n_same_align_refs.quick_grow_cleared (datarefs.length ());
    2296       401290 :   unsigned i0;
    2297       826390 :   for (i0 = 0; i0 < datarefs.length (); ++i0)
    2298       418605 :     if (DR_BASE_ADDRESS (datarefs[i0]))
    2299              :       break;
    2300      2572386 :   for (i = i0 + 1; i <= datarefs.length (); ++i)
    2301              :     {
    2302       884903 :       if (i == datarefs.length ()
    2303       490108 :           || !operand_equal_p (DR_BASE_ADDRESS (datarefs[i0]),
    2304       490108 :                                DR_BASE_ADDRESS (datarefs[i]), 0)
    2305       227227 :           || !operand_equal_p (DR_OFFSET (datarefs[i0]),
    2306       227227 :                                DR_OFFSET (datarefs[i]), 0)
    2307      1110779 :           || !operand_equal_p (DR_STEP (datarefs[i0]),
    2308       225876 :                                DR_STEP (datarefs[i]), 0))
    2309              :         {
    2310              :           /* The subgroup [i0, i-1] now only differs in DR_INIT and
    2311              :              possibly DR_TARGET_ALIGNMENT.  Still the whole subgroup
    2312              :              will get known misalignment if we align one of the refs
    2313              :              with the largest DR_TARGET_ALIGNMENT.  */
    2314      1544468 :           for (unsigned j = i0; j < i; ++j)
    2315              :             {
    2316       884903 :               dr_vec_info *dr_infoj = loop_vinfo->lookup_dr (datarefs[j]);
    2317      4660911 :               for (unsigned k = i0; k < i; ++k)
    2318              :                 {
    2319      2891105 :                   if (k == j)
    2320       884903 :                     continue;
    2321      2006202 :                   dr_vec_info *dr_infok = loop_vinfo->lookup_dr (datarefs[k]);
    2322      2006202 :                   if (vect_dr_aligned_if_related_peeled_dr_is (dr_infok,
    2323              :                                                                dr_infoj))
    2324       737084 :                     n_same_align_refs[j]++;
    2325              :                 }
    2326              :             }
    2327              :           i0 = i;
    2328              :         }
    2329              :     }
    2330              : 
    2331              :   /* See if we can relax the flags on speculative reads for early break.  Do
    2332              :      this outside of the other loops below because they can exit early leading
    2333              :      to the flag not being cleared for known in bounds cases.  */
    2334       401290 :   poly_uint64 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    2335       401290 :   if (LOOP_VINFO_EARLY_BREAKS (loop_vinfo))
    2336      1006311 :     for (auto dr : datarefs)
    2337              :       {
    2338       391896 :         dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
    2339       391896 :         if (!vect_relevant_for_alignment_p (dr_info))
    2340              :           continue;
    2341              : 
    2342       391896 :         stmt_vec_info stmt_info = dr_info->stmt;
    2343              : 
    2344              :         /* With variable VF, unsafe speculative read can be avoided for known
    2345              :            inbounds DRs as long as partial vectors are used.  */
    2346       391896 :         if (!vf.is_constant ()
    2347              :             && dr_safe_speculative_read_required (stmt_info)
    2348              :             && DR_SCALAR_KNOWN_BOUNDS (dr_info))
    2349              :           {
    2350              :             dr_set_safe_speculative_read_required (stmt_info, false);
    2351              :             LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P (loop_vinfo) = true;
    2352              :           }
    2353              :       }
    2354              : 
    2355              :   /* While cost model enhancements are expected in the future, the high level
    2356              :      view of the code at this time is as follows:
    2357              : 
    2358              :      A) If there is a misaligned access then see if doing peeling alone can
    2359              :         make all data references satisfy vect_supportable_dr_alignment.  If so,
    2360              :         update data structures and return.
    2361              : 
    2362              :      B) If peeling alone wasn't possible and there is a data reference with an
    2363              :         unknown misalignment that does not satisfy vect_supportable_dr_alignment
    2364              :         then we may use either of the following two approaches.
    2365              : 
    2366              :         B1) Try peeling with versioning: Add a runtime loop versioning check to
    2367              :             see if all unsupportable data references are mutually aligned, which
    2368              :             means they will be uniformly aligned after a certain amount of loop
    2369              :             peeling.  If peeling and versioning can be used together, set
    2370              :             LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT_P to TRUE and return.
    2371              : 
    2372              :         B2) Try versioning alone: Add a runtime loop versioning check to see if
    2373              :             all unsupportable data references are already uniformly aligned
    2374              :             without loop peeling.  If versioning can be applied alone, set
    2375              :             LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT_P to FALSE and return.
    2376              : 
    2377              :         Above B1 is more powerful and more likely to be adopted than B2.  But B2
    2378              :         is still available and useful in some cases, for example, the cost model
    2379              :         does not allow much peeling.
    2380              : 
    2381              :      C) If none of above was successful then the alignment was not enhanced,
    2382              :         just return.  */
    2383              : 
    2384              :   /* (1) Peeling to force alignment.  */
    2385              : 
    2386              :   /* (1.1) Decide whether to perform peeling, how many iterations to peel, and
    2387              :      if vectorization may be supported by peeling with versioning.
    2388              :      Considerations:
    2389              :      - How many accesses will become aligned due to the peeling
    2390              :      - How many accesses will become unaligned due to the peeling,
    2391              :        and the cost of misaligned accesses.
    2392              :      - The cost of peeling (the extra runtime checks, the increase
    2393              :        in code size).  */
    2394              : 
    2395      1131595 :   FOR_EACH_VEC_ELT (datarefs, i, dr)
    2396              :     {
    2397       780223 :       dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
    2398       780223 :       if (!vect_relevant_for_alignment_p (dr_info))
    2399       118227 :         continue;
    2400              : 
    2401       661996 :       stmt_vec_info stmt_info = dr_info->stmt;
    2402       661996 :       tree vectype = STMT_VINFO_VECTYPE (stmt_info);
    2403              : 
    2404       661996 :       do_peeling = vector_alignment_reachable_p (dr_info, vf);
    2405       661996 :       if (do_peeling)
    2406              :         {
    2407       579836 :           if (known_alignment_for_access_p (dr_info, vectype))
    2408              :             {
    2409       322248 :               unsigned int npeel_tmp = 0;
    2410       322248 :               bool negative = tree_int_cst_compare (DR_STEP (dr),
    2411       322248 :                                                     size_zero_node) < 0;
    2412              : 
    2413              :               /* If known_alignment_for_access_p then we have set
    2414              :                  DR_MISALIGNMENT which is only done if we know it at compiler
    2415              :                  time, so it is safe to assume target alignment is constant.
    2416              :                */
    2417       322248 :               unsigned int target_align =
    2418       322248 :                 DR_TARGET_ALIGNMENT (dr_info).to_constant ();
    2419       322248 :               unsigned HOST_WIDE_INT dr_size = vect_get_scalar_dr_size (dr_info);
    2420       322248 :               poly_int64 off = 0;
    2421       322248 :               if (negative)
    2422         2627 :                 off = (TYPE_VECTOR_SUBPARTS (vectype) - 1) * -dr_size;
    2423       322248 :               unsigned int mis = dr_misalignment (dr_info, vectype, off);
    2424       322248 :               mis = negative ? mis : -mis;
    2425       322248 :               if (mis != 0)
    2426        15320 :                 npeel_tmp = (mis & (target_align - 1)) / dr_size;
    2427              : 
    2428              :               /* For multiple types, it is possible that the bigger type access
    2429              :                  will have more than one peeling option.  E.g., a loop with two
    2430              :                  types: one of size (vector size / 4), and the other one of
    2431              :                  size (vector size / 8).  Vectorization factor will 8.  If both
    2432              :                  accesses are misaligned by 3, the first one needs one scalar
    2433              :                  iteration to be aligned, and the second one needs 5.  But the
    2434              :                  first one will be aligned also by peeling 5 scalar
    2435              :                  iterations, and in that case both accesses will be aligned.
    2436              :                  Hence, except for the immediate peeling amount, we also want
    2437              :                  to try to add full vector size, while we don't exceed
    2438              :                  vectorization factor.
    2439              :                  We do this automatically for cost model, since we calculate
    2440              :                  cost for every peeling option.  */
    2441       322248 :               poly_uint64 nscalars = npeel_tmp;
    2442       322248 :               if (unlimited_cost_model (LOOP_VINFO_LOOP (loop_vinfo)))
    2443              :                 {
    2444        42013 :                   unsigned group_size = 1;
    2445        42013 :                   if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    2446         1917 :                     group_size = DR_GROUP_SIZE (stmt_info);
    2447        42013 :                   nscalars = vf * group_size;
    2448              :                 }
    2449              : 
    2450              :               /* Save info about DR in the hash table.  Also include peeling
    2451              :                  amounts according to the explanation above.  Indicate
    2452              :                  the alignment status when the ref is not aligned.
    2453              :                  ???  Rather than using unknown alignment here we should
    2454              :                  prune all entries from the peeling hashtable which cause
    2455              :                  DRs to be not supported.  */
    2456       322248 :               bool supportable_if_not_aligned
    2457              :                 = vect_supportable_dr_alignment
    2458       322248 :                     (loop_vinfo, dr_info, vectype, DR_MISALIGNMENT_UNKNOWN);
    2459       706923 :               while (known_le (npeel_tmp, nscalars))
    2460              :                 {
    2461       384675 :                   vect_peeling_hash_insert (&peeling_htab, loop_vinfo,
    2462              :                                             dr_info, npeel_tmp,
    2463              :                                             supportable_if_not_aligned);
    2464       384675 :                   npeel_tmp += MAX (1, target_align / dr_size);
    2465              :                 }
    2466              : 
    2467       322248 :               one_misalignment_known = true;
    2468              :             }
    2469              :           else
    2470              :             {
    2471              :               /* If we don't know any misalignment values, we prefer
    2472              :                  peeling for data-ref that has the maximum number of data-refs
    2473              :                  with the same alignment, unless the target prefers to align
    2474              :                  stores over load.  */
    2475       257588 :               unsigned same_align_drs = n_same_align_refs[i];
    2476       257588 :               if (!dr0_info
    2477       257588 :                   || dr0_same_align_drs < same_align_drs)
    2478              :                 {
    2479              :                   dr0_same_align_drs = same_align_drs;
    2480              :                   dr0_info = dr_info;
    2481              :                 }
    2482              :               /* For data-refs with the same number of related
    2483              :                  accesses prefer the one where the misalign
    2484              :                  computation will be invariant in the outermost loop.  */
    2485        81896 :               else if (dr0_same_align_drs == same_align_drs)
    2486              :                 {
    2487        80376 :                   class loop *ivloop0, *ivloop;
    2488        80376 :                   ivloop0 = outermost_invariant_loop_for_expr
    2489        80376 :                     (loop, DR_BASE_ADDRESS (dr0_info->dr));
    2490        80376 :                   ivloop = outermost_invariant_loop_for_expr
    2491        80376 :                     (loop, DR_BASE_ADDRESS (dr));
    2492        80376 :                   if ((ivloop && !ivloop0)
    2493        80376 :                       || (ivloop && ivloop0
    2494        80368 :                           && flow_loop_nested_p (ivloop, ivloop0)))
    2495              :                     dr0_info = dr_info;
    2496              :                 }
    2497              : 
    2498       257588 :               one_misalignment_unknown = true;
    2499              : 
    2500              :               /* Check for data refs with unsupportable alignment that
    2501              :                  can be peeled.  */
    2502       257588 :               enum dr_alignment_support supportable_dr_alignment
    2503       257588 :                 = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
    2504              :                                                  DR_MISALIGNMENT_UNKNOWN);
    2505       257588 :               if (supportable_dr_alignment == dr_unaligned_unsupported)
    2506              :                 {
    2507       107795 :                   one_dr_unsupportable = true;
    2508       107795 :                   unsupportable_dr_info = dr_info;
    2509              :                 }
    2510              : 
    2511       257588 :               if (!first_store && DR_IS_WRITE (dr))
    2512              :                 {
    2513        56162 :                   first_store = dr_info;
    2514        56162 :                   first_store_same_align_drs = same_align_drs;
    2515              :                 }
    2516              :             }
    2517              :         }
    2518              :       else
    2519              :         {
    2520        82160 :           if (!aligned_access_p (dr_info, vectype))
    2521              :             {
    2522        49918 :               if (dump_enabled_p ())
    2523         2112 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    2524              :                                  "vector alignment may not be reachable\n");
    2525              :               break;
    2526              :             }
    2527              :         }
    2528              :     }
    2529              : 
    2530              :   /* Check if we can possibly peel the loop.  */
    2531       401290 :   if (!vect_can_advance_ivs_p (loop_vinfo)
    2532       392978 :       || !slpeel_can_duplicate_loop_p (loop, LOOP_VINFO_MAIN_EXIT (loop_vinfo),
    2533       392978 :                                        loop_preheader_edge (loop))
    2534       392978 :       || loop->inner
    2535              :       /* We don't currently maintain the LCSSA for prologue peeled inversed
    2536              :          loops.  */
    2537       792654 :       || (LOOP_VINFO_EARLY_BREAKS_VECT_PEELED (loop_vinfo)
    2538        34934 :           && !LOOP_VINFO_NITERS_UNCOUNTED_P (loop_vinfo)))
    2539              :     do_peeling = false;
    2540              : 
    2541       401290 :   struct _vect_peel_extended_info peel_for_known_alignment;
    2542       401290 :   struct _vect_peel_extended_info peel_for_unknown_alignment;
    2543       401290 :   struct _vect_peel_extended_info best_peel;
    2544              : 
    2545       401290 :   peel_for_unknown_alignment.inside_cost = INT_MAX;
    2546       401290 :   peel_for_unknown_alignment.outside_cost = INT_MAX;
    2547       401290 :   peel_for_unknown_alignment.peel_info.count = 0;
    2548              : 
    2549       401290 :   if (do_peeling
    2550       401290 :       && one_misalignment_unknown)
    2551              :     {
    2552              :       /* Check if the target requires to prefer stores over loads, i.e., if
    2553              :          misaligned stores are more expensive than misaligned loads (taking
    2554              :          drs with same alignment into account).  */
    2555       155612 :       unsigned int load_inside_cost = 0;
    2556       155612 :       unsigned int load_outside_cost = 0;
    2557       155612 :       unsigned int store_inside_cost = 0;
    2558       155612 :       unsigned int store_outside_cost = 0;
    2559       155612 :       unsigned int estimated_npeels = vect_vf_for_cost (loop_vinfo) / 2;
    2560              : 
    2561       155612 :       stmt_vector_for_cost dummy;
    2562       155612 :       dummy.create (2);
    2563       155612 :       vect_get_peeling_costs_all_drs (loop_vinfo, dr0_info,
    2564              :                                       &load_inside_cost,
    2565              :                                       &load_outside_cost,
    2566              :                                       &dummy, &dummy, estimated_npeels);
    2567       155612 :       dummy.release ();
    2568              : 
    2569       155612 :       if (first_store)
    2570              :         {
    2571        48938 :           dummy.create (2);
    2572        48938 :           vect_get_peeling_costs_all_drs (loop_vinfo, first_store,
    2573              :                                           &store_inside_cost,
    2574              :                                           &store_outside_cost,
    2575              :                                           &dummy, &dummy,
    2576              :                                           estimated_npeels);
    2577        48938 :           dummy.release ();
    2578              :         }
    2579              :       else
    2580              :         {
    2581       106674 :           store_inside_cost = INT_MAX;
    2582       106674 :           store_outside_cost = INT_MAX;
    2583              :         }
    2584              : 
    2585       155612 :       if (load_inside_cost > store_inside_cost
    2586       155612 :           || (load_inside_cost == store_inside_cost
    2587        48481 :               && load_outside_cost > store_outside_cost))
    2588              :         {
    2589       155612 :           dr0_info = first_store;
    2590       155612 :           dr0_same_align_drs = first_store_same_align_drs;
    2591       155612 :           peel_for_unknown_alignment.inside_cost = store_inside_cost;
    2592       155612 :           peel_for_unknown_alignment.outside_cost = store_outside_cost;
    2593              :         }
    2594              :       else
    2595              :         {
    2596       155612 :           peel_for_unknown_alignment.inside_cost = load_inside_cost;
    2597       155612 :           peel_for_unknown_alignment.outside_cost = load_outside_cost;
    2598              :         }
    2599              : 
    2600       155612 :       peel_for_unknown_alignment.outside_cost
    2601       155612 :         += vect_get_known_peeling_cost (loop_vinfo, estimated_npeels);
    2602              : 
    2603       155612 :       peel_for_unknown_alignment.peel_info.count = dr0_same_align_drs + 1;
    2604              :     }
    2605              : 
    2606       401290 :   peel_for_unknown_alignment.peel_info.npeel = 0;
    2607       401290 :   peel_for_unknown_alignment.peel_info.dr_info = dr0_info;
    2608              : 
    2609       401290 :   best_peel = peel_for_unknown_alignment;
    2610              : 
    2611       401290 :   peel_for_known_alignment.inside_cost = INT_MAX;
    2612       401290 :   peel_for_known_alignment.outside_cost = INT_MAX;
    2613       401290 :   peel_for_known_alignment.peel_info.count = 0;
    2614       401290 :   peel_for_known_alignment.peel_info.dr_info = NULL;
    2615              : 
    2616       401290 :   if (do_peeling && one_misalignment_known)
    2617              :     {
    2618              :       /* Peeling is possible, but there is no data access that is not supported
    2619              :          unless aligned.  So we try to choose the best possible peeling from
    2620              :          the hash table.  */
    2621       169785 :       peel_for_known_alignment = vect_peeling_hash_choose_best_peeling
    2622       169785 :         (&peeling_htab, loop_vinfo);
    2623              :     }
    2624              : 
    2625              :   /* Compare costs of peeling for known and unknown alignment. */
    2626       401290 :   if (peel_for_known_alignment.peel_info.dr_info != NULL
    2627       169785 :       && peel_for_unknown_alignment.inside_cost
    2628              :       >= peel_for_known_alignment.inside_cost)
    2629              :     {
    2630       155245 :       best_peel = peel_for_known_alignment;
    2631              : 
    2632              :       /* If the best peeling for known alignment has NPEEL == 0, perform no
    2633              :          peeling at all except if there is an unsupportable dr that we can
    2634              :          align.  */
    2635       155245 :       if (best_peel.peel_info.npeel == 0 && !one_dr_unsupportable)
    2636              :         do_peeling = false;
    2637              :     }
    2638              : 
    2639              :   /* If there is an unsupportable data ref, prefer this over all choices so far
    2640              :      since we'd have to discard a chosen peeling except when it accidentally
    2641              :      aligned the unsupportable data ref.  */
    2642       254795 :   if (one_dr_unsupportable)
    2643              :     dr0_info = unsupportable_dr_info;
    2644       311877 :   else if (do_peeling)
    2645              :     {
    2646              :       /* Calculate the penalty for no peeling, i.e. leaving everything as-is.
    2647              :          TODO: Use nopeel_outside_cost or get rid of it?  */
    2648        73733 :       unsigned nopeel_inside_cost = 0;
    2649        73733 :       unsigned nopeel_outside_cost = 0;
    2650              : 
    2651        73733 :       stmt_vector_for_cost dummy;
    2652        73733 :       dummy.create (2);
    2653        73733 :       vect_get_peeling_costs_all_drs (loop_vinfo, NULL, &nopeel_inside_cost,
    2654              :                                       &nopeel_outside_cost, &dummy, &dummy, 0);
    2655        73733 :       dummy.release ();
    2656              : 
    2657              :       /* Add epilogue costs.  As we do not peel for alignment here, no prologue
    2658              :          costs will be recorded.  */
    2659        73733 :       nopeel_outside_cost += vect_get_known_peeling_cost (loop_vinfo, 0);
    2660              : 
    2661        73733 :       npeel = best_peel.peel_info.npeel;
    2662        73733 :       dr0_info = best_peel.peel_info.dr_info;
    2663              : 
    2664              :       /* If no peeling is not more expensive than the best peeling we
    2665              :          have so far, don't perform any peeling.  */
    2666        73733 :       if (nopeel_inside_cost <= best_peel.inside_cost)
    2667        66236 :         do_peeling = false;
    2668              :     }
    2669              : 
    2670       163146 :   if (do_peeling)
    2671              :     {
    2672        85481 :       stmt_vec_info stmt_info = dr0_info->stmt;
    2673        85481 :       if (known_alignment_for_access_p (dr0_info,
    2674              :                                         STMT_VINFO_VECTYPE (stmt_info)))
    2675              :         {
    2676         7474 :           bool negative = tree_int_cst_compare (DR_STEP (dr0_info->dr),
    2677         7474 :                                                 size_zero_node) < 0;
    2678         7474 :           if (!npeel)
    2679              :             {
    2680              :               /* Since it's known at compile time, compute the number of
    2681              :                  iterations in the peeled loop (the peeling factor) for use in
    2682              :                  updating DR_MISALIGNMENT values.  The peeling factor is the
    2683              :                  vectorization factor minus the misalignment as an element
    2684              :                  count.  */
    2685            0 :               tree vectype = STMT_VINFO_VECTYPE (stmt_info);
    2686            0 :               poly_int64 off = 0;
    2687            0 :               if (negative)
    2688            0 :                 off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
    2689            0 :                        * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
    2690            0 :               unsigned int mis
    2691            0 :                 = dr_misalignment (dr0_info, vectype, off);
    2692            0 :               mis = negative ? mis : -mis;
    2693              :               /* If known_alignment_for_access_p then we have set
    2694              :                  DR_MISALIGNMENT which is only done if we know it at compiler
    2695              :                  time, so it is safe to assume target alignment is constant.
    2696              :                */
    2697            0 :               unsigned int target_align =
    2698            0 :                 DR_TARGET_ALIGNMENT (dr0_info).to_constant ();
    2699            0 :               npeel = ((mis & (target_align - 1))
    2700            0 :                        / vect_get_scalar_dr_size (dr0_info));
    2701              :             }
    2702              : 
    2703              :           /* For interleaved data access every iteration accesses all the
    2704              :              members of the group, therefore we divide the number of iterations
    2705              :              by the group size.  */
    2706         7474 :           if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
    2707          305 :             npeel /= DR_GROUP_SIZE (stmt_info);
    2708              : 
    2709         7474 :           if (dump_enabled_p ())
    2710          296 :             dump_printf_loc (MSG_NOTE, vect_location,
    2711              :                              "Try peeling by %d\n", npeel);
    2712              :         }
    2713              : 
    2714              :       /* Check how peeling for alignment can support vectorization.  Function
    2715              :          vect_peeling_supportable returns one of the three possible values:
    2716              :          - PEELING_KNOWN_SUPPORTED: indicates that we know all unsupported
    2717              :            datarefs can be aligned after peeling.  We can use peeling alone.
    2718              :          - PEELING_MAYBE_SUPPORTED: indicates that peeling may be able to make
    2719              :            these datarefs aligned but we are not sure about it at compile time.
    2720              :            We will try peeling with versioning to add a runtime check to guard
    2721              :            the peeled loop.
    2722              :          - PEELING_UNSUPPORTED: indicates that peeling is almost impossible to
    2723              :            support vectorization.  We will stop trying peeling.  */
    2724        85481 :       switch (vect_peeling_supportable (loop_vinfo, dr0_info, npeel))
    2725              :         {
    2726              :         case peeling_known_supported:
    2727              :           break;
    2728        13674 :         case peeling_maybe_supported:
    2729        13674 :           try_peeling_with_versioning = true;
    2730        13674 :           break;
    2731        17036 :         case peeling_unsupported:
    2732        17036 :           do_peeling = false;
    2733        17036 :           break;
    2734              :         }
    2735              : 
    2736              :       /* Check if all datarefs are supportable and log.  */
    2737        85481 :       if (do_peeling
    2738        85481 :           && npeel == 0
    2739        85481 :           && known_alignment_for_access_p (dr0_info,
    2740              :                                            STMT_VINFO_VECTYPE (stmt_info)))
    2741            3 :         return opt_result::success ();
    2742              : 
    2743              :       /* Cost model #1 - honor --param vect-max-peeling-for-alignment.  */
    2744        85478 :       if (do_peeling)
    2745              :         {
    2746        68442 :           unsigned max_allowed_peel
    2747        68442 :             = param_vect_max_peeling_for_alignment;
    2748        68442 :           if (loop_cost_model (loop) <= VECT_COST_MODEL_CHEAP)
    2749              :             max_allowed_peel = 0;
    2750        14730 :           if (max_allowed_peel != (unsigned)-1)
    2751              :             {
    2752        53733 :               unsigned max_peel = npeel;
    2753        53733 :               if (max_peel == 0)
    2754              :                 {
    2755        50710 :                   poly_uint64 target_align = DR_TARGET_ALIGNMENT (dr0_info);
    2756        50710 :                   unsigned HOST_WIDE_INT target_align_c;
    2757        50710 :                   if (target_align.is_constant (&target_align_c))
    2758       101420 :                     max_peel =
    2759        50710 :                       target_align_c / vect_get_scalar_dr_size (dr0_info) - 1;
    2760              :                   else
    2761              :                     {
    2762              :                       do_peeling = false;
    2763              :                       if (dump_enabled_p ())
    2764              :                         dump_printf_loc (MSG_NOTE, vect_location,
    2765              :                           "Disable peeling, max peels set and vector"
    2766              :                           " alignment unknown\n");
    2767              :                     }
    2768              :                 }
    2769        53733 :               if (max_peel > max_allowed_peel)
    2770              :                 {
    2771        53725 :                   do_peeling = false;
    2772        53725 :                   if (dump_enabled_p ())
    2773           53 :                     dump_printf_loc (MSG_NOTE, vect_location,
    2774              :                         "Disable peeling, max peels reached: %d\n", max_peel);
    2775              :                 }
    2776              :             }
    2777              :         }
    2778              : 
    2779              :       /* Cost model #2 - if peeling may result in a remaining loop not
    2780              :          iterating enough to be vectorized then do not peel.  Since this
    2781              :          is a cost heuristic rather than a correctness decision, use the
    2782              :          most likely runtime value for variable vectorization factors.  */
    2783           53 :       if (do_peeling
    2784        14717 :           && LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo))
    2785              :         {
    2786         3252 :           unsigned int assumed_vf = vect_vf_for_cost (loop_vinfo);
    2787         3252 :           unsigned int max_peel = npeel == 0 ? assumed_vf - 1 : npeel;
    2788         3252 :           if ((unsigned HOST_WIDE_INT) LOOP_VINFO_INT_NITERS (loop_vinfo)
    2789         3252 :               < assumed_vf + max_peel)
    2790              :             do_peeling = false;
    2791              :         }
    2792              : 
    2793              :       if (do_peeling)
    2794              :         {
    2795              :           /* (1.2) Update the DR_MISALIGNMENT of each data reference DR_i.
    2796              :              If the misalignment of DR_i is identical to that of dr0 then set
    2797              :              DR_MISALIGNMENT (DR_i) to zero.  If the misalignment of DR_i and
    2798              :              dr0 are known at compile time then increment DR_MISALIGNMENT (DR_i)
    2799              :              by the peeling factor times the element size of DR_i (MOD the
    2800              :              vectorization factor times the size).  Otherwise, the
    2801              :              misalignment of DR_i must be set to unknown.  */
    2802        30969 :           FOR_EACH_VEC_ELT (datarefs, i, dr)
    2803        17085 :             if (dr != dr0_info->dr)
    2804              :               {
    2805         3201 :                 dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
    2806         3201 :                 if (!vect_relevant_for_alignment_p (dr_info))
    2807          294 :                   continue;
    2808              : 
    2809         2907 :                 vect_update_misalignment_for_peel (dr_info, dr0_info, npeel);
    2810              :               }
    2811              :         }
    2812              : 
    2813        85478 :       if (do_peeling && !try_peeling_with_versioning)
    2814              :         {
    2815              :           /* Update data structures if peeling will be applied alone.  */
    2816        12752 :           LOOP_VINFO_UNALIGNED_DR (loop_vinfo) = dr0_info;
    2817        12752 :           if (npeel)
    2818        12752 :             LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = npeel;
    2819              :           else
    2820        10621 :             LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = -1;
    2821        12752 :           SET_DR_MISALIGNMENT (dr0_info,
    2822              :                                vect_dr_misalign_for_aligned_access (dr0_info));
    2823        12752 :           if (dump_enabled_p ())
    2824              :             {
    2825          384 :               dump_printf_loc (MSG_NOTE, vect_location,
    2826              :                                "Alignment of access forced using peeling.\n");
    2827          384 :               dump_printf_loc (MSG_NOTE, vect_location,
    2828              :                                "Peeling for alignment will be applied.\n");
    2829              :             }
    2830              : 
    2831              :           /* The inside-loop cost will be accounted for in vectorizable_load
    2832              :              and vectorizable_store correctly with adjusted alignments.
    2833              :              Drop the body_cst_vec on the floor here.  */
    2834        12752 :           return opt_result::success ();
    2835              :         }
    2836              :     }
    2837              : 
    2838              :   /* (2) Versioning to force alignment.  */
    2839              : 
    2840              :   /* Try versioning if:
    2841              :      1) optimize loop for speed and the cost-model is not cheap
    2842              :      2) there is at least one unsupported misaligned data ref with an unknown
    2843              :         misalignment, and
    2844              :      3) all misaligned data refs with a known misalignment are supported, and
    2845              :      4) the number of runtime alignment checks is within reason.  */
    2846              : 
    2847       388535 :   do_versioning
    2848       388535 :     = (optimize_loop_nest_for_speed_p (loop)
    2849       388085 :        && !loop->inner /* FORNOW */
    2850       775006 :        && loop_cost_model (loop) > VECT_COST_MODEL_CHEAP);
    2851              : 
    2852              :   if (do_versioning)
    2853              :     {
    2854       377014 :       FOR_EACH_VEC_ELT (datarefs, i, dr)
    2855              :         {
    2856       282971 :           dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
    2857       282971 :           if (!vect_relevant_for_alignment_p (dr_info))
    2858       198728 :             continue;
    2859              : 
    2860       196226 :           stmt_vec_info stmt_info = dr_info->stmt;
    2861       196226 :           if (STMT_VINFO_STRIDED_P (stmt_info))
    2862              :             {
    2863              :               do_versioning = false;
    2864         5185 :               break;
    2865              :             }
    2866              : 
    2867       195114 :           tree vectype = STMT_VINFO_VECTYPE (stmt_info);
    2868       195114 :           bool negative = tree_int_cst_compare (DR_STEP (dr),
    2869       195114 :                                                 size_zero_node) < 0;
    2870       195114 :           poly_int64 off = 0;
    2871       195114 :           if (negative)
    2872         3553 :             off = ((TYPE_VECTOR_SUBPARTS (vectype) - 1)
    2873         3553 :                    * -TREE_INT_CST_LOW (TYPE_SIZE_UNIT (TREE_TYPE (vectype))));
    2874       195114 :           int misalignment;
    2875       195114 :           if ((misalignment = dr_misalignment (dr_info, vectype, off)) == 0)
    2876       111983 :             continue;
    2877              : 
    2878        83131 :           enum dr_alignment_support supportable_dr_alignment
    2879        83131 :             = vect_supportable_dr_alignment (loop_vinfo, dr_info, vectype,
    2880              :                                              misalignment);
    2881        83131 :           if (supportable_dr_alignment == dr_unaligned_unsupported)
    2882              :             {
    2883        16279 :               if (misalignment != DR_MISALIGNMENT_UNKNOWN
    2884        16279 :                   || (LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).length ()
    2885        12730 :                       >= (unsigned) param_vect_max_version_for_alignment_checks))
    2886              :                 {
    2887              :                   do_versioning = false;
    2888         5185 :                   break;
    2889              :                 }
    2890              : 
    2891              :               /* Forcing alignment in the first iteration is no good if
    2892              :                  we don't keep it across iterations.  For now, just disable
    2893              :                  versioning in this case.
    2894              :                  ?? We could actually unroll the loop to achieve the required
    2895              :                  overall step alignment, and forcing the alignment could be
    2896              :                  done by doing some iterations of the non-vectorized loop.  */
    2897        12322 :               if (!multiple_p (vf * DR_STEP_ALIGNMENT (dr),
    2898        12322 :                                DR_TARGET_ALIGNMENT (dr_info)))
    2899              :                 {
    2900              :                   do_versioning = false;
    2901              :                   break;
    2902              :                 }
    2903              : 
    2904              :               /* Use "mask = DR_TARGET_ALIGNMENT - 1" to test rightmost address
    2905              :                  bits for runtime alignment check.  For example, for 16 bytes
    2906              :                  target alignment the mask is 15 = 0xf.  */
    2907        12322 :               poly_uint64 mask = DR_TARGET_ALIGNMENT (dr_info) - 1;
    2908              : 
    2909              :               /* FORNOW: use the same mask to test all potentially unaligned
    2910              :                  references in the loop.  */
    2911        12322 :               if (maybe_ne (LOOP_VINFO_PTR_MASK (loop_vinfo), 0U)
    2912        12322 :                   && maybe_ne (LOOP_VINFO_PTR_MASK (loop_vinfo), mask))
    2913              :                 {
    2914              :                   do_versioning = false;
    2915              :                   break;
    2916              :                 }
    2917              : 
    2918        12206 :               LOOP_VINFO_PTR_MASK (loop_vinfo) = mask;
    2919        12206 :               LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).safe_push (stmt_info);
    2920              :             }
    2921              :         }
    2922              : 
    2923              :       /* Versioning requires at least one misaligned data reference.  */
    2924        99228 :       if (!LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (loop_vinfo))
    2925              :         do_versioning = false;
    2926         6294 :       else if (!do_versioning)
    2927          540 :         LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo).truncate (0);
    2928              :     }
    2929              : 
    2930              :   /* If we are trying peeling with versioning but versioning is disabled for
    2931              :      some reason, peeling should be turned off together.  */
    2932       388535 :   if (try_peeling_with_versioning && !do_versioning)
    2933              :     do_peeling = false;
    2934              : 
    2935       376093 :   if (do_versioning)
    2936              :     {
    2937              :       const vec<stmt_vec_info> &may_misalign_stmts
    2938              :         = LOOP_VINFO_MAY_MISALIGN_STMTS (loop_vinfo);
    2939              :       stmt_vec_info stmt_info;
    2940              : 
    2941              :       /* It can now be assumed that the data references in the statements
    2942              :          in LOOP_VINFO_MAY_MISALIGN_STMTS will be aligned in the version
    2943              :          of the loop being vectorized.  */
    2944        15320 :       FOR_EACH_VEC_ELT (may_misalign_stmts, i, stmt_info)
    2945              :         {
    2946         9566 :           dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
    2947         9566 :           SET_DR_MISALIGNMENT (dr_info,
    2948              :                                vect_dr_misalign_for_aligned_access (dr_info));
    2949         9566 :           if (dump_enabled_p ())
    2950          272 :             dump_printf_loc (MSG_NOTE, vect_location,
    2951              :                              "Alignment of access forced using versioning.\n");
    2952              :         }
    2953              : 
    2954         5754 :       if (do_peeling)
    2955              :         {
    2956              :           /* This point is reached if peeling and versioning are used together
    2957              :              to ensure alignment.  Update data structures to make sure the loop
    2958              :              is correctly peeled and a right runtime check is added for loop
    2959              :              versioning.  */
    2960         1132 :           gcc_assert (try_peeling_with_versioning);
    2961         1132 :           LOOP_VINFO_UNALIGNED_DR (loop_vinfo) = dr0_info;
    2962         1132 :           LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo) = -1;
    2963         1132 :           LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (loop_vinfo) = true;
    2964         1132 :           if (dump_enabled_p ())
    2965           11 :             dump_printf_loc (MSG_NOTE, vect_location,
    2966              :                              "Both peeling and versioning will be applied.\n");
    2967              :         }
    2968              :       else
    2969              :         {
    2970              :           /* This point is reached if versioning is used alone.  */
    2971         4622 :           LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (loop_vinfo) = false;
    2972         4622 :           if (dump_enabled_p ())
    2973          142 :             dump_printf_loc (MSG_NOTE, vect_location,
    2974              :                              "Versioning for alignment will be applied.\n");
    2975              :         }
    2976              : 
    2977         5754 :       return opt_result::success ();
    2978              :     }
    2979              : 
    2980              :   /* This point is reached if neither peeling nor versioning is being done.  */
    2981       382781 :   gcc_assert (! (do_peeling || do_versioning));
    2982              : 
    2983       382781 :   return opt_result::success ();
    2984       817643 : }
    2985              : 
    2986              : 
    2987              : /* Function vect_analyze_data_refs_alignment
    2988              : 
    2989              :    Analyze the alignment of the data-references in the loop.  */
    2990              : 
    2991              : void
    2992       449612 : vect_analyze_data_refs_alignment (loop_vec_info loop_vinfo)
    2993              : {
    2994       449612 :   DUMP_VECT_SCOPE ("vect_analyze_data_refs_alignment");
    2995              : 
    2996       449612 :   vec<data_reference_p> datarefs = LOOP_VINFO_DATAREFS (loop_vinfo);
    2997       449612 :   struct data_reference *dr;
    2998       449612 :   unsigned int i;
    2999              : 
    3000       449612 :   vect_record_base_alignments (loop_vinfo);
    3001      2358837 :   FOR_EACH_VEC_ELT (datarefs, i, dr)
    3002              :     {
    3003      1025366 :       dr_vec_info *dr_info = loop_vinfo->lookup_dr (dr);
    3004      1025366 :       if (STMT_VINFO_VECTORIZABLE (dr_info->stmt))
    3005              :         {
    3006      1025366 :           if (STMT_VINFO_GROUPED_ACCESS (dr_info->stmt)
    3007      1327496 :               && DR_GROUP_FIRST_ELEMENT (dr_info->stmt) != dr_info->stmt)
    3008       135405 :             continue;
    3009              : 
    3010       889961 :           vect_compute_data_ref_alignment (loop_vinfo, dr_info,
    3011              :                                            STMT_VINFO_VECTYPE (dr_info->stmt));
    3012              :         }
    3013              :     }
    3014       449612 : }
    3015              : 
    3016              : 
    3017              : /* Analyze alignment of DRs of stmts in NODE.  */
    3018              : 
    3019              : static bool
    3020       858190 : vect_slp_analyze_node_alignment (vec_info *vinfo, slp_tree node)
    3021              : {
    3022              :   /* Alignment is maintained in the first element of the group.  */
    3023       858190 :   stmt_vec_info first_stmt_info = SLP_TREE_SCALAR_STMTS (node)[0];
    3024       858190 :   first_stmt_info = DR_GROUP_FIRST_ELEMENT (first_stmt_info);
    3025       858190 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (first_stmt_info);
    3026       858190 :   tree vectype = SLP_TREE_VECTYPE (node);
    3027       858190 :   poly_uint64 vector_alignment
    3028       858190 :     = exact_div (targetm.vectorize.preferred_vector_alignment (vectype),
    3029              :                  BITS_PER_UNIT);
    3030       858190 :   if (dr_info->misalignment == DR_MISALIGNMENT_UNINITIALIZED)
    3031       805067 :     vect_compute_data_ref_alignment (vinfo, dr_info, SLP_TREE_VECTYPE (node));
    3032              :   /* Re-analyze alignment when we're facing a vectorization with a bigger
    3033              :      alignment requirement.  */
    3034        53123 :   else if (known_lt (dr_info->target_alignment, vector_alignment))
    3035              :     {
    3036           93 :       poly_uint64 old_target_alignment = dr_info->target_alignment;
    3037           93 :       int old_misalignment = dr_info->misalignment;
    3038           93 :       vect_compute_data_ref_alignment (vinfo, dr_info, SLP_TREE_VECTYPE (node));
    3039              :       /* But keep knowledge about a smaller alignment.  */
    3040           93 :       if (old_misalignment != DR_MISALIGNMENT_UNKNOWN
    3041           39 :           && dr_info->misalignment == DR_MISALIGNMENT_UNKNOWN)
    3042              :         {
    3043            4 :           dr_info->target_alignment = old_target_alignment;
    3044            4 :           dr_info->misalignment = old_misalignment;
    3045              :         }
    3046              :     }
    3047              :   /* When we ever face unordered target alignments the first one wins in terms
    3048              :      of analyzing and the other will become unknown in dr_misalignment.  */
    3049       858190 :   return true;
    3050              : }
    3051              : 
    3052              : /* Function vect_slp_analyze_instance_alignment
    3053              : 
    3054              :    Analyze the alignment of the data-references in the SLP instance.
    3055              :    Return FALSE if a data reference is found that cannot be vectorized.  */
    3056              : 
    3057              : bool
    3058       834481 : vect_slp_analyze_instance_alignment (vec_info *vinfo,
    3059              :                                                 slp_instance instance)
    3060              : {
    3061       834481 :   DUMP_VECT_SCOPE ("vect_slp_analyze_instance_alignment");
    3062              : 
    3063       834481 :   slp_tree node;
    3064       834481 :   unsigned i;
    3065      1015977 :   FOR_EACH_VEC_ELT (SLP_INSTANCE_LOADS (instance), i, node)
    3066       181496 :     if (! vect_slp_analyze_node_alignment (vinfo, node))
    3067              :       return false;
    3068              : 
    3069       834481 :   if (SLP_INSTANCE_KIND (instance) == slp_inst_kind_store
    3070       834481 :       && ! vect_slp_analyze_node_alignment
    3071       676694 :              (vinfo, SLP_INSTANCE_TREE (instance)))
    3072              :     return false;
    3073              : 
    3074              :   return true;
    3075              : }
    3076              : 
    3077              : 
    3078              : /* Analyze groups of accesses: check that DR_INFO belongs to a group of
    3079              :    accesses of legal size, step, etc.  Detect gaps, single element
    3080              :    interleaving, and other special cases. Set grouped access info.
    3081              :    Collect groups of strided stores for further use in SLP analysis.
    3082              :    Worker for vect_analyze_group_access.  */
    3083              : 
    3084              : static bool
    3085     13213086 : vect_analyze_group_access_1 (vec_info *vinfo, dr_vec_info *dr_info)
    3086              : {
    3087     13213086 :   data_reference *dr = dr_info->dr;
    3088     13213086 :   tree step = DR_STEP (dr);
    3089     13213086 :   tree scalar_type = TREE_TYPE (DR_REF (dr));
    3090     13213086 :   HOST_WIDE_INT type_size = TREE_INT_CST_LOW (TYPE_SIZE_UNIT (scalar_type));
    3091     13213086 :   stmt_vec_info stmt_info = dr_info->stmt;
    3092     13213086 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    3093     13213086 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    3094     13213086 :   HOST_WIDE_INT dr_step = -1;
    3095     13213086 :   HOST_WIDE_INT groupsize, last_accessed_element = 1;
    3096     13213086 :   bool slp_impossible = false;
    3097              : 
    3098              :   /* For interleaving, GROUPSIZE is STEP counted in elements, i.e., the
    3099              :      size of the interleaving group (including gaps).  */
    3100     13213086 :   if (tree_fits_shwi_p (step))
    3101              :     {
    3102     13203434 :       dr_step = tree_to_shwi (step);
    3103              :       /* Check that STEP is a multiple of type size.  Otherwise there is
    3104              :          a non-element-sized gap at the end of the group which we
    3105              :          cannot represent in DR_GROUP_GAP or DR_GROUP_SIZE.
    3106              :          ???  As we can handle non-constant step fine here we should
    3107              :          simply remove uses of DR_GROUP_GAP between the last and first
    3108              :          element and instead rely on DR_STEP.  DR_GROUP_SIZE then would
    3109              :          simply not include that gap.  */
    3110     13203434 :       if ((dr_step % type_size) != 0)
    3111              :         {
    3112          505 :           if (dump_enabled_p ())
    3113           27 :             dump_printf_loc (MSG_NOTE, vect_location,
    3114              :                              "Step %T is not a multiple of the element size"
    3115              :                              " for %T\n",
    3116              :                              step, DR_REF (dr));
    3117              :           return false;
    3118              :         }
    3119     13202929 :       groupsize = absu_hwi (dr_step) / type_size;
    3120              :     }
    3121              :   else
    3122              :     groupsize = 0;
    3123              : 
    3124              :   /* Not consecutive access is possible only if it is a part of interleaving.  */
    3125     13212581 :   if (!DR_GROUP_FIRST_ELEMENT (stmt_info))
    3126              :     {
    3127              :       /* Check if it this DR is a part of interleaving, and is a single
    3128              :          element of the group that is accessed in the loop.  */
    3129              : 
    3130              :       /* Gaps are supported only for loads. STEP must be a multiple of the type
    3131              :          size.  */
    3132      8844616 :       if (DR_IS_READ (dr)
    3133      5285498 :           && (dr_step % type_size) == 0
    3134              :           && groupsize > 0
    3135              :           /* This could be UINT_MAX but as we are generating code in a very
    3136              :              inefficient way we have to cap earlier.
    3137              :              See PR91403 for example.  */
    3138      5285498 :           && groupsize <= 4096)
    3139              :         {
    3140        74460 :           DR_GROUP_FIRST_ELEMENT (stmt_info) = stmt_info;
    3141        74460 :           DR_GROUP_SIZE (stmt_info) = groupsize;
    3142        74460 :           DR_GROUP_GAP (stmt_info) = groupsize - 1;
    3143        74460 :           if (dump_enabled_p ())
    3144         1501 :             dump_printf_loc (MSG_NOTE, vect_location,
    3145              :                              "Detected single element interleaving %T"
    3146              :                              " step %T\n",
    3147              :                              DR_REF (dr), step);
    3148              : 
    3149              :           return true;
    3150              :         }
    3151              : 
    3152      8770156 :       if (dump_enabled_p ())
    3153         3441 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3154              :                          "not consecutive access %G", stmt_info->stmt);
    3155              : 
    3156      8770156 :       if (bb_vinfo)
    3157              :         {
    3158              :           /* Mark the statement as unvectorizable.  */
    3159      8748970 :           STMT_VINFO_VECTORIZABLE (stmt_info) = false;
    3160      8748970 :           return true;
    3161              :         }
    3162              : 
    3163        21186 :       if (dump_enabled_p ())
    3164          317 :         dump_printf_loc (MSG_NOTE, vect_location, "using strided accesses\n");
    3165        21186 :       STMT_VINFO_STRIDED_P (stmt_info) = true;
    3166        21186 :       return true;
    3167              :     }
    3168              : 
    3169      4367965 :   if (DR_GROUP_FIRST_ELEMENT (stmt_info) == stmt_info)
    3170              :     {
    3171              :       /* First stmt in the interleaving chain. Check the chain.  */
    3172      1583220 :       stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
    3173      1583220 :       struct data_reference *data_ref = dr;
    3174      1583220 :       unsigned int count = 1;
    3175      1583220 :       tree prev_init = DR_INIT (data_ref);
    3176      1583220 :       HOST_WIDE_INT diff, gaps = 0;
    3177              : 
    3178              :       /* By construction, all group members have INTEGER_CST DR_INITs.  */
    3179      4367974 :       while (next)
    3180              :         {
    3181              :           /* We never have the same DR multiple times.  */
    3182      2784816 :           gcc_assert (tree_int_cst_compare (DR_INIT (data_ref),
    3183              :                                 DR_INIT (STMT_VINFO_DATA_REF (next))) != 0);
    3184              : 
    3185      2784816 :           data_ref = STMT_VINFO_DATA_REF (next);
    3186              : 
    3187              :           /* All group members have the same STEP by construction.  */
    3188      2784816 :           gcc_checking_assert (operand_equal_p (DR_STEP (data_ref), step, 0));
    3189              : 
    3190              :           /* Check that the distance between two accesses is equal to the type
    3191              :              size. Otherwise, we have gaps.  */
    3192      2784816 :           diff = (TREE_INT_CST_LOW (DR_INIT (data_ref))
    3193      2784816 :                   - TREE_INT_CST_LOW (prev_init)) / type_size;
    3194      2784816 :           if (diff < 1 || diff > UINT_MAX)
    3195              :             {
    3196              :               /* For artificial testcases with array accesses with large
    3197              :                  constant indices we can run into overflow issues which
    3198              :                  can end up fooling the groupsize constraint below so
    3199              :                  check the individual gaps (which are represented as
    3200              :                  unsigned int) as well.  */
    3201            0 :               if (dump_enabled_p ())
    3202            0 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3203              :                                  "interleaved access with gap larger "
    3204              :                                  "than representable\n");
    3205              :               return false;
    3206              :             }
    3207      2784816 :           if (diff != 1)
    3208              :             {
    3209              :               /* FORNOW: SLP of accesses with gaps is not supported.  */
    3210       126246 :               slp_impossible = true;
    3211       126246 :               if (DR_IS_WRITE (data_ref))
    3212              :                 {
    3213           62 :                   if (dump_enabled_p ())
    3214            0 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3215              :                                      "interleaved store with gaps\n");
    3216              :                   return false;
    3217              :                 }
    3218              : 
    3219       126184 :               gaps += diff - 1;
    3220              :             }
    3221              : 
    3222      2784754 :           last_accessed_element += diff;
    3223              : 
    3224              :           /* Store the gap from the previous member of the group. If there is no
    3225              :              gap in the access, DR_GROUP_GAP is always 1.  */
    3226      2784754 :           DR_GROUP_GAP (next) = diff;
    3227              : 
    3228      2784754 :           prev_init = DR_INIT (data_ref);
    3229      2784754 :           next = DR_GROUP_NEXT_ELEMENT (next);
    3230              :           /* Count the number of data-refs in the chain.  */
    3231      2784754 :           count++;
    3232              :         }
    3233              : 
    3234      1583158 :       if (groupsize == 0)
    3235      1509337 :         groupsize = count + gaps;
    3236              : 
    3237              :       /* This could be UINT_MAX but as we are generating code in a very
    3238              :          inefficient way we have to cap earlier.  See PR78699 for example.  */
    3239      1583158 :       if (groupsize > 4096)
    3240              :         {
    3241            1 :           if (dump_enabled_p ())
    3242            1 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3243              :                              "group is too large\n");
    3244              :           return false;
    3245              :         }
    3246              : 
    3247              :       /* Check that the size of the interleaving is equal to count for stores,
    3248              :          i.e., that there are no gaps.  */
    3249      1583157 :       if (groupsize != count
    3250       129609 :           && !DR_IS_READ (dr))
    3251              :         {
    3252        12204 :           groupsize = count;
    3253        12204 :           STMT_VINFO_STRIDED_P (stmt_info) = true;
    3254              :         }
    3255              : 
    3256              :       /* If there is a gap after the last load in the group it is the
    3257              :          difference between the groupsize and the last accessed
    3258              :          element.
    3259              :          When there is no gap, this difference should be 0.  */
    3260      1583157 :       DR_GROUP_GAP (stmt_info) = groupsize - last_accessed_element;
    3261              : 
    3262      1583157 :       DR_GROUP_SIZE (stmt_info) = groupsize;
    3263      1583157 :       if (dump_enabled_p ())
    3264              :         {
    3265         8238 :           dump_printf_loc (MSG_NOTE, vect_location,
    3266              :                            "Detected interleaving ");
    3267         8238 :           if (DR_IS_READ (dr))
    3268         4438 :             dump_printf (MSG_NOTE, "load ");
    3269         3800 :           else if (STMT_VINFO_STRIDED_P (stmt_info))
    3270          466 :             dump_printf (MSG_NOTE, "strided store ");
    3271              :           else
    3272         3334 :             dump_printf (MSG_NOTE, "store ");
    3273         8238 :           dump_printf (MSG_NOTE, "of size %u\n",
    3274              :                        (unsigned)groupsize);
    3275         8238 :           dump_printf_loc (MSG_NOTE, vect_location, "\t%G", stmt_info->stmt);
    3276         8238 :           next = DR_GROUP_NEXT_ELEMENT (stmt_info);
    3277        40333 :           while (next)
    3278              :             {
    3279        32095 :               if (DR_GROUP_GAP (next) != 1)
    3280          329 :                 dump_printf_loc (MSG_NOTE, vect_location,
    3281              :                                  "\t<gap of %d elements>\n",
    3282          329 :                                  DR_GROUP_GAP (next) - 1);
    3283        32095 :               dump_printf_loc (MSG_NOTE, vect_location, "\t%G", next->stmt);
    3284        32095 :               next = DR_GROUP_NEXT_ELEMENT (next);
    3285              :             }
    3286         8238 :           if (DR_GROUP_GAP (stmt_info) != 0)
    3287          404 :             dump_printf_loc (MSG_NOTE, vect_location,
    3288              :                              "\t<gap of %d elements>\n",
    3289          404 :                              DR_GROUP_GAP (stmt_info));
    3290              :         }
    3291              : 
    3292              :       /* SLP: create an SLP data structure for every interleaving group of
    3293              :          stores for further analysis in vect_analyse_slp.  */
    3294      1583157 :       if (DR_IS_WRITE (dr) && !slp_impossible)
    3295              :         {
    3296       948145 :           if (loop_vinfo)
    3297        32493 :             LOOP_VINFO_GROUPED_STORES (loop_vinfo).safe_push (stmt_info);
    3298       948145 :           if (bb_vinfo)
    3299       915652 :             BB_VINFO_GROUPED_STORES (bb_vinfo).safe_push (stmt_info);
    3300              :         }
    3301              :     }
    3302              : 
    3303              :   return true;
    3304              : }
    3305              : 
    3306              : /* Analyze groups of accesses: check that DR_INFO belongs to a group of
    3307              :    accesses of legal size, step, etc.  Detect gaps, single element
    3308              :    interleaving, and other special cases. Set grouped access info.
    3309              :    Collect groups of strided stores for further use in SLP analysis.  */
    3310              : 
    3311              : static bool
    3312     13213086 : vect_analyze_group_access (vec_info *vinfo, dr_vec_info *dr_info)
    3313              : {
    3314     13213086 :   if (!vect_analyze_group_access_1 (vinfo, dr_info))
    3315              :     {
    3316              :       /* Dissolve the group if present.  */
    3317          568 :       stmt_vec_info stmt_info = DR_GROUP_FIRST_ELEMENT (dr_info->stmt);
    3318          799 :       while (stmt_info)
    3319              :         {
    3320          231 :           stmt_vec_info next = DR_GROUP_NEXT_ELEMENT (stmt_info);
    3321          231 :           DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
    3322          231 :           DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
    3323          231 :           stmt_info = next;
    3324              :         }
    3325              :       return false;
    3326              :     }
    3327              :   return true;
    3328              : }
    3329              : 
    3330              : /* Analyze the access pattern of the data-reference DR_INFO.
    3331              :    In case of non-consecutive accesses call vect_analyze_group_access() to
    3332              :    analyze groups of accesses.  */
    3333              : 
    3334              : static bool
    3335     14000411 : vect_analyze_data_ref_access (vec_info *vinfo, dr_vec_info *dr_info)
    3336              : {
    3337     14000411 :   data_reference *dr = dr_info->dr;
    3338     14000411 :   tree step = DR_STEP (dr);
    3339     14000411 :   tree scalar_type = TREE_TYPE (DR_REF (dr));
    3340     14000411 :   stmt_vec_info stmt_info = dr_info->stmt;
    3341     14000411 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    3342     14000411 :   class loop *loop = NULL;
    3343              : 
    3344     14000411 :   if (STMT_VINFO_GATHER_SCATTER_P (stmt_info))
    3345              :     return true;
    3346              : 
    3347     13904900 :   if (loop_vinfo)
    3348      1018538 :     loop = LOOP_VINFO_LOOP (loop_vinfo);
    3349              : 
    3350     13904900 :   if (loop_vinfo && !step)
    3351              :     {
    3352            0 :       if (dump_enabled_p ())
    3353            0 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3354              :                          "bad data-ref access in loop\n");
    3355              :       return false;
    3356              :     }
    3357              : 
    3358              :   /* Allow loads with zero step in inner-loop vectorization.  */
    3359     13904900 :   if (loop_vinfo && integer_zerop (step))
    3360              :     {
    3361        16515 :       DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
    3362        16515 :       DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
    3363        16515 :       if (!nested_in_vect_loop_p (loop, stmt_info))
    3364        16249 :         return DR_IS_READ (dr);
    3365              :       /* Allow references with zero step for outer loops marked
    3366              :          with pragma omp simd only - it guarantees absence of
    3367              :          loop-carried dependencies between inner loop iterations.  */
    3368          266 :       if (loop->safelen < 2)
    3369              :         {
    3370          230 :           if (dump_enabled_p ())
    3371            6 :             dump_printf_loc (MSG_NOTE, vect_location,
    3372              :                              "zero step in inner loop of nest\n");
    3373              :           return false;
    3374              :         }
    3375              :     }
    3376              : 
    3377     13888385 :   if (loop && nested_in_vect_loop_p (loop, stmt_info))
    3378              :     {
    3379              :       /* Interleaved accesses are not yet supported within outer-loop
    3380              :         vectorization for references in the inner-loop.  */
    3381         6024 :       DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
    3382         6024 :       DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
    3383              : 
    3384              :       /* For the rest of the analysis we use the outer-loop step.  */
    3385         6024 :       step = STMT_VINFO_DR_STEP (stmt_info);
    3386         6024 :       if (integer_zerop (step))
    3387              :         {
    3388         1357 :           if (dump_enabled_p ())
    3389          244 :             dump_printf_loc (MSG_NOTE, vect_location,
    3390              :                              "zero step in outer loop.\n");
    3391         1357 :           return DR_IS_READ (dr);
    3392              :         }
    3393              :     }
    3394              : 
    3395              :   /* Consecutive?  */
    3396     13887064 :   if (TREE_CODE (step) == INTEGER_CST)
    3397              :     {
    3398     13846963 :       HOST_WIDE_INT dr_step = TREE_INT_CST_LOW (step);
    3399     13846963 :       if (!tree_int_cst_compare (step, TYPE_SIZE_UNIT (scalar_type))
    3400     13846963 :           || (dr_step < 0
    3401        34527 :               && !compare_tree_int (TYPE_SIZE_UNIT (scalar_type), -dr_step)))
    3402              :         {
    3403              :           /* Mark that it is not interleaving.  */
    3404       640563 :           DR_GROUP_FIRST_ELEMENT (stmt_info) = NULL;
    3405       640563 :           DR_GROUP_NEXT_ELEMENT (stmt_info) = NULL;
    3406       640563 :           return true;
    3407              :         }
    3408              :     }
    3409              : 
    3410     13246501 :   if (loop && nested_in_vect_loop_p (loop, stmt_info))
    3411              :     {
    3412         3442 :       if (dump_enabled_p ())
    3413          163 :         dump_printf_loc (MSG_NOTE, vect_location,
    3414              :                          "grouped access in outer loop.\n");
    3415              :       return false;
    3416              :     }
    3417              : 
    3418              : 
    3419              :   /* Assume this is a DR handled by non-constant strided load case.  */
    3420     13243059 :   if (TREE_CODE (step) != INTEGER_CST)
    3421        39625 :     return (STMT_VINFO_STRIDED_P (stmt_info)
    3422        39625 :             && (!STMT_VINFO_GROUPED_ACCESS (stmt_info)
    3423         9652 :                 || vect_analyze_group_access (vinfo, dr_info)));
    3424              : 
    3425              :   /* Not consecutive access - check if it's a part of interleaving group.  */
    3426     13203434 :   return vect_analyze_group_access (vinfo, dr_info);
    3427              : }
    3428              : 
    3429              : /* Compare two data-references DRA and DRB to group them into chunks
    3430              :    suitable for grouping.  */
    3431              : 
    3432              : static int
    3433    371125950 : dr_group_sort_cmp (const void *dra_, const void *drb_)
    3434              : {
    3435    371125950 :   dr_vec_info *dra_info = *(dr_vec_info **)const_cast<void *>(dra_);
    3436    371125950 :   dr_vec_info *drb_info = *(dr_vec_info **)const_cast<void *>(drb_);
    3437    371125950 :   data_reference_p dra = dra_info->dr;
    3438    371125950 :   data_reference_p drb = drb_info->dr;
    3439    371125950 :   int cmp;
    3440              : 
    3441              :   /* Stabilize sort.  */
    3442    371125950 :   if (dra == drb)
    3443              :     return 0;
    3444              : 
    3445              :   /* Different group IDs lead never belong to the same group.  */
    3446    371125950 :   if (dra_info->group != drb_info->group)
    3447    265258159 :     return dra_info->group < drb_info->group ? -1 : 1;
    3448              : 
    3449              :   /* Ordering of DRs according to base.  */
    3450    105867791 :   cmp = data_ref_compare_tree (DR_BASE_ADDRESS (dra),
    3451              :                                DR_BASE_ADDRESS (drb));
    3452    105867791 :   if (cmp != 0)
    3453              :     return cmp;
    3454              : 
    3455              :   /* And according to DR_OFFSET.  */
    3456     56400104 :   cmp = data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb));
    3457     56400104 :   if (cmp != 0)
    3458              :     return cmp;
    3459              : 
    3460              :   /* Put reads before writes.  */
    3461     56033110 :   if (DR_IS_READ (dra) != DR_IS_READ (drb))
    3462      3045868 :     return DR_IS_READ (dra) ? -1 : 1;
    3463              : 
    3464              :   /* Then sort after access size.  */
    3465     52987242 :   cmp = data_ref_compare_tree (TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dra))),
    3466     52987242 :                                TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drb))));
    3467     52987242 :   if (cmp != 0)
    3468              :     return cmp;
    3469              : 
    3470              :   /* And after step.  */
    3471     45955034 :   cmp = data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb));
    3472     45955034 :   if (cmp != 0)
    3473              :     return cmp;
    3474              : 
    3475              :   /* Then sort after DR_INIT.  In case of identical DRs sort after stmt UID.  */
    3476     45947456 :   cmp = data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb));
    3477     45947456 :   if (cmp == 0)
    3478       487747 :     return gimple_uid (DR_STMT (dra)) < gimple_uid (DR_STMT (drb)) ? -1 : 1;
    3479              :   return cmp;
    3480              : }
    3481              : 
    3482              : /* If OP is the result of a conversion, return the unconverted value,
    3483              :    otherwise return null.  */
    3484              : 
    3485              : static tree
    3486          342 : strip_conversion (tree op)
    3487              : {
    3488          342 :   if (TREE_CODE (op) != SSA_NAME)
    3489              :     return NULL_TREE;
    3490          342 :   gimple *stmt = SSA_NAME_DEF_STMT (op);
    3491          342 :   if (!is_gimple_assign (stmt)
    3492          342 :       || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (stmt)))
    3493              :     return NULL_TREE;
    3494          186 :   return gimple_assign_rhs1 (stmt);
    3495              : }
    3496              : 
    3497              : /* Return true if vectorizable_* routines can handle statements STMT1_INFO
    3498              :    and STMT2_INFO being in a single group.  When ALLOW_SLP_P, masked loads can
    3499              :    be grouped in SLP mode.  */
    3500              : 
    3501              : static bool
    3502      7400797 : can_group_stmts_p (stmt_vec_info stmt1_info, stmt_vec_info stmt2_info,
    3503              :                    bool allow_slp_p)
    3504              : {
    3505      7400797 :   if (gimple_assign_single_p (stmt1_info->stmt))
    3506      7399405 :     return gimple_assign_single_p (stmt2_info->stmt);
    3507              : 
    3508         1392 :   gcall *call1 = dyn_cast <gcall *> (stmt1_info->stmt);
    3509         1392 :   if (call1 && gimple_call_internal_p (call1))
    3510              :     {
    3511              :       /* Check for two masked loads or two masked stores.  */
    3512         1392 :       gcall *call2 = dyn_cast <gcall *> (stmt2_info->stmt);
    3513         1376 :       if (!call2 || !gimple_call_internal_p (call2))
    3514              :         return false;
    3515         1376 :       internal_fn ifn = gimple_call_internal_fn (call1);
    3516         1376 :       if (ifn != IFN_MASK_LOAD && ifn != IFN_MASK_STORE)
    3517              :         return false;
    3518         1376 :       if (ifn != gimple_call_internal_fn (call2))
    3519              :         return false;
    3520              : 
    3521              :       /* Check that the masks are the same.  Cope with casts of masks,
    3522              :          like those created by build_mask_conversion.  */
    3523         1376 :       tree mask1 = gimple_call_arg (call1, 2);
    3524         1376 :       tree mask2 = gimple_call_arg (call2, 2);
    3525         1376 :       if (!operand_equal_p (mask1, mask2, 0) && !allow_slp_p)
    3526              :         {
    3527          249 :           mask1 = strip_conversion (mask1);
    3528          249 :           if (!mask1)
    3529              :             return false;
    3530           93 :           mask2 = strip_conversion (mask2);
    3531           93 :           if (!mask2)
    3532              :             return false;
    3533           93 :           if (!operand_equal_p (mask1, mask2, 0))
    3534              :             return false;
    3535              :         }
    3536         1183 :       return true;
    3537              :     }
    3538              : 
    3539              :   return false;
    3540              : }
    3541              : 
    3542              : /* Function vect_analyze_data_ref_accesses.
    3543              : 
    3544              :    Analyze the access pattern of all the data references in the loop.
    3545              : 
    3546              :    FORNOW: the only access pattern that is considered vectorizable is a
    3547              :            simple step 1 (consecutive) access.
    3548              : 
    3549              :    FORNOW: handle only arrays and pointer accesses.  */
    3550              : 
    3551              : opt_result
    3552      2777795 : vect_analyze_data_ref_accesses (vec_info *vinfo,
    3553              :                                 vec<int> *dataref_groups)
    3554              : {
    3555      2777795 :   unsigned int i;
    3556      2777795 :   vec<data_reference_p> datarefs = vinfo->shared->datarefs;
    3557              : 
    3558      2777795 :   DUMP_VECT_SCOPE ("vect_analyze_data_ref_accesses");
    3559              : 
    3560      2777795 :   if (datarefs.is_empty ())
    3561      1107933 :     return opt_result::success ();
    3562              : 
    3563              :   /* Sort the array of datarefs to make building the interleaving chains
    3564              :      linear.  Don't modify the original vector's order, it is needed for
    3565              :      determining what dependencies are reversed.  */
    3566      1669862 :   vec<dr_vec_info *> datarefs_copy;
    3567      1669862 :   datarefs_copy.create (datarefs.length ());
    3568     19419642 :   for (unsigned i = 0; i < datarefs.length (); i++)
    3569              :     {
    3570     16079918 :       dr_vec_info *dr_info = vinfo->lookup_dr (datarefs[i]);
    3571              :       /* If the caller computed DR grouping use that, otherwise group by
    3572              :          basic blocks.  */
    3573     16079918 :       if (dataref_groups)
    3574     14951489 :         dr_info->group = (*dataref_groups)[i];
    3575              :       else
    3576      1128429 :         dr_info->group = gimple_bb (DR_STMT (datarefs[i]))->index;
    3577     16079918 :       datarefs_copy.quick_push (dr_info);
    3578              :     }
    3579      1669862 :   datarefs_copy.qsort (dr_group_sort_cmp);
    3580      1669862 :   hash_set<stmt_vec_info> to_fixup;
    3581              : 
    3582              :   /* Build the interleaving chains.  */
    3583     15178277 :   for (i = 0; i < datarefs_copy.length () - 1;)
    3584              :     {
    3585     11838553 :       dr_vec_info *dr_info_a = datarefs_copy[i];
    3586     11838553 :       data_reference_p dra = dr_info_a->dr;
    3587     11838553 :       int dra_group_id = dr_info_a->group;
    3588     11838553 :       stmt_vec_info stmtinfo_a = dr_info_a->stmt;
    3589     11838553 :       stmt_vec_info lastinfo = NULL;
    3590     11838553 :       if (!STMT_VINFO_VECTORIZABLE (stmtinfo_a)
    3591      9933477 :           || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_a))
    3592              :         {
    3593      1966982 :           ++i;
    3594      1966982 :           continue;
    3595              :         }
    3596     26200208 :       for (i = i + 1; i < datarefs_copy.length (); ++i)
    3597              :         {
    3598     12443074 :           dr_vec_info *dr_info_b = datarefs_copy[i];
    3599     12443074 :           data_reference_p drb = dr_info_b->dr;
    3600     12443074 :           int drb_group_id = dr_info_b->group;
    3601     12443074 :           stmt_vec_info stmtinfo_b = dr_info_b->stmt;
    3602     12443074 :           if (!STMT_VINFO_VECTORIZABLE (stmtinfo_b)
    3603     12092576 :               || STMT_VINFO_GATHER_SCATTER_P (stmtinfo_b))
    3604              :             break;
    3605              : 
    3606              :           /* ???  Imperfect sorting (non-compatible types, non-modulo
    3607              :              accesses, same accesses) can lead to a group to be artificially
    3608              :              split here as we don't just skip over those.  If it really
    3609              :              matters we can push those to a worklist and re-iterate
    3610              :              over them.  The we can just skip ahead to the next DR here.  */
    3611              : 
    3612              :           /* DRs in a different DR group should not be put into the same
    3613              :              interleaving group.  */
    3614     12086047 :           if (dra_group_id != drb_group_id)
    3615              :             break;
    3616              : 
    3617              :           /* Check that the data-refs have same first location (except init)
    3618              :              and they are both either store or load (not load and store,
    3619              :              not masked loads or stores).  */
    3620      7674391 :           if (DR_IS_READ (dra) != DR_IS_READ (drb)
    3621      6314332 :               || data_ref_compare_tree (DR_BASE_ADDRESS (dra),
    3622              :                                         DR_BASE_ADDRESS (drb)) != 0
    3623      4601043 :               || data_ref_compare_tree (DR_OFFSET (dra), DR_OFFSET (drb)) != 0
    3624     12255236 :               || !can_group_stmts_p (stmtinfo_a, stmtinfo_b, true))
    3625              :             break;
    3626              : 
    3627              :           /* Check that the data-refs have the same constant size.  */
    3628      4580820 :           tree sza = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (dra)));
    3629      4580820 :           tree szb = TYPE_SIZE_UNIT (TREE_TYPE (DR_REF (drb)));
    3630      4580820 :           if (!tree_fits_uhwi_p (sza)
    3631      4580820 :               || !tree_fits_uhwi_p (szb)
    3632      9161640 :               || !tree_int_cst_equal (sza, szb))
    3633              :             break;
    3634              : 
    3635              :           /* Check that the data-refs have the same step.  */
    3636      4220636 :           if (data_ref_compare_tree (DR_STEP (dra), DR_STEP (drb)) != 0)
    3637              :             break;
    3638              : 
    3639              :           /* Check the types are compatible.
    3640              :              ???  We don't distinguish this during sorting.  */
    3641      4219868 :           if (!types_compatible_p (TREE_TYPE (DR_REF (dra)),
    3642      4219868 :                                    TREE_TYPE (DR_REF (drb))))
    3643              :             break;
    3644              : 
    3645              :           /* Check that the DR_INITs are compile-time constants.  */
    3646      3040619 :           if (!tree_fits_shwi_p (DR_INIT (dra))
    3647      3040619 :               || !tree_fits_shwi_p (DR_INIT (drb)))
    3648              :             break;
    3649              : 
    3650              :           /* Different .GOMP_SIMD_LANE calls still give the same lane,
    3651              :              just hold extra information.  */
    3652      3040619 :           if (STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_a)
    3653         1240 :               && STMT_VINFO_SIMD_LANE_ACCESS_P (stmtinfo_b)
    3654      3041859 :               && data_ref_compare_tree (DR_INIT (dra), DR_INIT (drb)) == 0)
    3655              :             break;
    3656              : 
    3657              :           /* Sorting has ensured that DR_INIT (dra) <= DR_INIT (drb).  */
    3658      3039379 :           HOST_WIDE_INT init_a = TREE_INT_CST_LOW (DR_INIT (dra));
    3659      3039379 :           HOST_WIDE_INT init_b = TREE_INT_CST_LOW (DR_INIT (drb));
    3660      3039379 :           HOST_WIDE_INT init_prev
    3661      3039379 :             = TREE_INT_CST_LOW (DR_INIT (datarefs_copy[i-1]->dr));
    3662      3039379 :           gcc_assert (init_a <= init_b
    3663              :                       && init_a <= init_prev
    3664              :                       && init_prev <= init_b);
    3665              : 
    3666              :           /* Do not place the same access in the interleaving chain twice.  */
    3667      3039379 :           if (init_b == init_prev)
    3668              :             {
    3669        31619 :               gcc_assert (gimple_uid (DR_STMT (datarefs_copy[i-1]->dr))
    3670              :                           < gimple_uid (DR_STMT (drb)));
    3671              :               /* Simply link in duplicates and fix up the chain below.  */
    3672              :             }
    3673              :           else
    3674              :             {
    3675              :               /* If init_b == init_a + the size of the type * k, we have an
    3676              :                  interleaving, and DRA is accessed before DRB.  */
    3677      3007760 :               unsigned HOST_WIDE_INT type_size_a = tree_to_uhwi (sza);
    3678      3007760 :               if (type_size_a == 0
    3679      3007760 :                   || (((unsigned HOST_WIDE_INT)init_b - init_a)
    3680      3007760 :                       % type_size_a != 0))
    3681              :                 break;
    3682              : 
    3683              :               /* If we have a store, the accesses are adjacent.  This splits
    3684              :                  groups into chunks we support (we don't support vectorization
    3685              :                  of stores with gaps).  */
    3686      3005980 :               if (!DR_IS_READ (dra)
    3687      1924057 :                   && (((unsigned HOST_WIDE_INT)init_b - init_prev)
    3688              :                       != type_size_a))
    3689              :                 break;
    3690              : 
    3691              :               /* For datarefs with big gap, it's better to split them into different
    3692              :                  groups.
    3693              :                  .i.e a[0], a[1], a[2], .. a[7], a[100], a[101],..., a[107]  */
    3694      2818027 :               if ((unsigned HOST_WIDE_INT)(init_b - init_prev)
    3695              :                   > MAX_BITSIZE_MODE_ANY_MODE / BITS_PER_UNIT)
    3696              :                 break;
    3697              : 
    3698              :               /* If the step (if not zero or non-constant) is smaller than the
    3699              :                  difference between data-refs' inits this splits groups into
    3700              :                  suitable sizes.  */
    3701      2808159 :               if (tree_fits_shwi_p (DR_STEP (dra)))
    3702              :                 {
    3703      2801796 :                   unsigned HOST_WIDE_INT step
    3704      2801796 :                     = absu_hwi (tree_to_shwi (DR_STEP (dra)));
    3705      2801796 :                   if (step != 0
    3706       169912 :                       && step <= ((unsigned HOST_WIDE_INT)init_b - init_a))
    3707              :                     break;
    3708              :                 }
    3709              :             }
    3710              : 
    3711      2820222 :           if (dump_enabled_p ())
    3712        33013 :             dump_printf_loc (MSG_NOTE, vect_location,
    3713        33013 :                              DR_IS_READ (dra)
    3714              :                              ? "Detected interleaving load %T and %T\n"
    3715              :                              : "Detected interleaving store %T and %T\n",
    3716              :                              DR_REF (dra), DR_REF (drb));
    3717              : 
    3718              :           /* Link the found element into the group list.  */
    3719      2820222 :           if (!DR_GROUP_FIRST_ELEMENT (stmtinfo_a))
    3720              :             {
    3721      1560126 :               DR_GROUP_FIRST_ELEMENT (stmtinfo_a) = stmtinfo_a;
    3722      1560126 :               lastinfo = stmtinfo_a;
    3723              :             }
    3724      2820222 :           DR_GROUP_FIRST_ELEMENT (stmtinfo_b) = stmtinfo_a;
    3725      2820222 :           DR_GROUP_NEXT_ELEMENT (lastinfo) = stmtinfo_b;
    3726      2820222 :           lastinfo = stmtinfo_b;
    3727              : 
    3728      2820222 :           if (! STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a))
    3729              :             {
    3730      2819952 :               STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a)
    3731      2819952 :                 = !can_group_stmts_p (stmtinfo_a, stmtinfo_b, false);
    3732              : 
    3733      2819952 :               if (dump_enabled_p () && STMT_VINFO_SLP_VECT_ONLY (stmtinfo_a))
    3734          114 :                 dump_printf_loc (MSG_NOTE, vect_location,
    3735              :                                  "Load suitable for SLP vectorization only.\n");
    3736              :             }
    3737              : 
    3738      2820222 :           if (init_b == init_prev
    3739        31619 :               && !to_fixup.add (DR_GROUP_FIRST_ELEMENT (stmtinfo_a))
    3740      2838080 :               && dump_enabled_p ())
    3741          214 :             dump_printf_loc (MSG_NOTE, vect_location,
    3742              :                              "Queuing group with duplicate access for fixup\n");
    3743              :         }
    3744              :     }
    3745              : 
    3746              :   /* Fixup groups with duplicate entries by splitting it.  */
    3747      1715881 :   while (1)
    3748              :     {
    3749      1715881 :       hash_set<stmt_vec_info>::iterator it = to_fixup.begin ();
    3750      1715881 :       if (!(it != to_fixup.end ()))
    3751              :         break;
    3752        46019 :       stmt_vec_info grp = *it;
    3753        46019 :       to_fixup.remove (grp);
    3754              : 
    3755              :       /* Find the earliest duplicate group member.  */
    3756        46019 :       unsigned first_duplicate = -1u;
    3757        46019 :       stmt_vec_info next, g = grp;
    3758       285289 :       while ((next = DR_GROUP_NEXT_ELEMENT (g)))
    3759              :         {
    3760       193251 :           if (tree_int_cst_equal (DR_INIT (STMT_VINFO_DR_INFO (next)->dr),
    3761       193251 :                                   DR_INIT (STMT_VINFO_DR_INFO (g)->dr))
    3762       193251 :               && gimple_uid (STMT_VINFO_STMT (next)) < first_duplicate)
    3763              :             first_duplicate = gimple_uid (STMT_VINFO_STMT (next));
    3764       193251 :           g = next;
    3765              :         }
    3766        46019 :       if (first_duplicate == -1U)
    3767        17858 :         continue;
    3768              : 
    3769              :       /* Then move all stmts after the first duplicate to a new group.
    3770              :          Note this is a heuristic but one with the property that *it
    3771              :          is fixed up completely.  */
    3772        28161 :       g = grp;
    3773        28161 :       stmt_vec_info newgroup = NULL, ng = grp;
    3774       246995 :       while ((next = DR_GROUP_NEXT_ELEMENT (g)))
    3775              :         {
    3776       190673 :           if (gimple_uid (STMT_VINFO_STMT (next)) >= first_duplicate)
    3777              :             {
    3778       184209 :               DR_GROUP_NEXT_ELEMENT (g) = DR_GROUP_NEXT_ELEMENT (next);
    3779       184209 :               if (!newgroup)
    3780              :                 {
    3781        28161 :                   newgroup = next;
    3782        28161 :                   STMT_VINFO_SLP_VECT_ONLY (newgroup)
    3783        28161 :                     = STMT_VINFO_SLP_VECT_ONLY (grp);
    3784              :                 }
    3785              :               else
    3786       156048 :                 DR_GROUP_NEXT_ELEMENT (ng) = next;
    3787       184209 :               ng = next;
    3788       184209 :               DR_GROUP_FIRST_ELEMENT (ng) = newgroup;
    3789              :             }
    3790              :           else
    3791              :             g = DR_GROUP_NEXT_ELEMENT (g);
    3792              :         }
    3793        28161 :       DR_GROUP_NEXT_ELEMENT (ng) = NULL;
    3794              : 
    3795              :       /* Fixup the new group which still may contain duplicates.  */
    3796        28161 :       to_fixup.add (newgroup);
    3797              :     }
    3798              : 
    3799      1669862 :   dr_vec_info *dr_info;
    3800     17727228 :   FOR_EACH_VEC_ELT (datarefs_copy, i, dr_info)
    3801              :     {
    3802     16065538 :       if (STMT_VINFO_VECTORIZABLE (dr_info->stmt)
    3803     16065538 :           && !vect_analyze_data_ref_access (vinfo, dr_info))
    3804              :         {
    3805         8226 :           if (dump_enabled_p ())
    3806          292 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    3807              :                              "not vectorized: complicated access pattern.\n");
    3808              : 
    3809         8226 :           if (is_a <bb_vec_info> (vinfo))
    3810              :             {
    3811              :               /* Mark the statement as not vectorizable.  */
    3812           54 :               STMT_VINFO_VECTORIZABLE (dr_info->stmt) = false;
    3813           54 :               continue;
    3814              :             }
    3815              :           else
    3816              :             {
    3817         8172 :               datarefs_copy.release ();
    3818         8172 :               return opt_result::failure_at (dr_info->stmt->stmt,
    3819              :                                              "not vectorized:"
    3820              :                                              " complicated access pattern.\n");
    3821              :             }
    3822              :         }
    3823              :     }
    3824              : 
    3825      1661690 :   datarefs_copy.release ();
    3826      1661690 :   return opt_result::success ();
    3827      1669862 : }
    3828              : 
    3829              : /* Function vect_vfa_segment_size.
    3830              : 
    3831              :    Input:
    3832              :      DR_INFO: The data reference.
    3833              :      LENGTH_FACTOR: segment length to consider.
    3834              : 
    3835              :    Return a value suitable for the dr_with_seg_len::seg_len field.
    3836              :    This is the "distance travelled" by the pointer from the first
    3837              :    iteration in the segment to the last.  Note that it does not include
    3838              :    the size of the access; in effect it only describes the first byte.  */
    3839              : 
    3840              : static tree
    3841       153174 : vect_vfa_segment_size (dr_vec_info *dr_info, tree length_factor)
    3842              : {
    3843       153174 :   length_factor = size_binop (MINUS_EXPR,
    3844              :                               fold_convert (sizetype, length_factor),
    3845              :                               size_one_node);
    3846       153174 :   return size_binop (MULT_EXPR, fold_convert (sizetype, DR_STEP (dr_info->dr)),
    3847              :                      length_factor);
    3848              : }
    3849              : 
    3850              : /* Return a value that, when added to abs (vect_vfa_segment_size (DR_INFO)),
    3851              :    gives the worst-case number of bytes covered by the segment.  */
    3852              : 
    3853              : static unsigned HOST_WIDE_INT
    3854       153728 : vect_vfa_access_size (vec_info *vinfo, dr_vec_info *dr_info)
    3855              : {
    3856       153728 :   stmt_vec_info stmt_vinfo = dr_info->stmt;
    3857       153728 :   tree ref_type = TREE_TYPE (DR_REF (dr_info->dr));
    3858       153728 :   unsigned HOST_WIDE_INT ref_size = tree_to_uhwi (TYPE_SIZE_UNIT (ref_type));
    3859       153728 :   unsigned HOST_WIDE_INT access_size = ref_size;
    3860       153728 :   if (DR_GROUP_FIRST_ELEMENT (stmt_vinfo))
    3861              :     {
    3862        44226 :       gcc_assert (DR_GROUP_FIRST_ELEMENT (stmt_vinfo) == stmt_vinfo);
    3863        44226 :       access_size *= DR_GROUP_SIZE (stmt_vinfo) - DR_GROUP_GAP (stmt_vinfo);
    3864              :     }
    3865       153728 :   tree vectype = STMT_VINFO_VECTYPE (stmt_vinfo);
    3866       153728 :   int misalignment;
    3867       307456 :   if (((misalignment = dr_misalignment (dr_info, vectype)), true)
    3868       153728 :       && (vect_supportable_dr_alignment (vinfo, dr_info, vectype, misalignment)
    3869              :           == dr_explicit_realign_optimized))
    3870              :     {
    3871              :       /* We might access a full vector's worth.  */
    3872            0 :       access_size += tree_to_uhwi (TYPE_SIZE_UNIT (vectype)) - ref_size;
    3873              :     }
    3874       153728 :   return access_size;
    3875              : }
    3876              : 
    3877              : /* Get the minimum alignment for all the scalar accesses that DR_INFO
    3878              :    describes.  */
    3879              : 
    3880              : static unsigned int
    3881       153728 : vect_vfa_align (dr_vec_info *dr_info)
    3882              : {
    3883            0 :   return dr_alignment (dr_info->dr);
    3884              : }
    3885              : 
    3886              : /* Function vect_no_alias_p.
    3887              : 
    3888              :    Given data references A and B with equal base and offset, see whether
    3889              :    the alias relation can be decided at compilation time.  Return 1 if
    3890              :    it can and the references alias, 0 if it can and the references do
    3891              :    not alias, and -1 if we cannot decide at compile time.  SEGMENT_LENGTH_A,
    3892              :    SEGMENT_LENGTH_B, ACCESS_SIZE_A and ACCESS_SIZE_B are the equivalent
    3893              :    of dr_with_seg_len::{seg_len,access_size} for A and B.  */
    3894              : 
    3895              : static int
    3896         4592 : vect_compile_time_alias (dr_vec_info *a, dr_vec_info *b,
    3897              :                          tree segment_length_a, tree segment_length_b,
    3898              :                          unsigned HOST_WIDE_INT access_size_a,
    3899              :                          unsigned HOST_WIDE_INT access_size_b)
    3900              : {
    3901         4592 :   poly_offset_int offset_a = wi::to_poly_offset (DR_INIT (a->dr));
    3902         4592 :   poly_offset_int offset_b = wi::to_poly_offset (DR_INIT (b->dr));
    3903         4592 :   poly_uint64 const_length_a;
    3904         4592 :   poly_uint64 const_length_b;
    3905              : 
    3906              :   /* For negative step, we need to adjust address range by TYPE_SIZE_UNIT
    3907              :      bytes, e.g., int a[3] -> a[1] range is [a+4, a+16) instead of
    3908              :      [a, a+12) */
    3909         4592 :   if (tree_int_cst_compare (DR_STEP (a->dr), size_zero_node) < 0)
    3910              :     {
    3911          228 :       const_length_a = (-wi::to_poly_wide (segment_length_a)).force_uhwi ();
    3912          228 :       offset_a -= const_length_a;
    3913              :     }
    3914              :   else
    3915         4364 :     const_length_a = tree_to_poly_uint64 (segment_length_a);
    3916         4592 :   if (tree_int_cst_compare (DR_STEP (b->dr), size_zero_node) < 0)
    3917              :     {
    3918          386 :       const_length_b = (-wi::to_poly_wide (segment_length_b)).force_uhwi ();
    3919          386 :       offset_b -= const_length_b;
    3920              :     }
    3921              :   else
    3922         4206 :     const_length_b = tree_to_poly_uint64 (segment_length_b);
    3923              : 
    3924         4592 :   const_length_a += access_size_a;
    3925         4592 :   const_length_b += access_size_b;
    3926              : 
    3927         4592 :   if (ranges_known_overlap_p (offset_a, const_length_a,
    3928              :                               offset_b, const_length_b))
    3929              :     return 1;
    3930              : 
    3931          536 :   if (!ranges_maybe_overlap_p (offset_a, const_length_a,
    3932              :                                offset_b, const_length_b))
    3933          536 :     return 0;
    3934              : 
    3935              :   return -1;
    3936              : }
    3937              : 
    3938              : /* Return true if the minimum nonzero dependence distance for loop LOOP_DEPTH
    3939              :    in DDR is >= VF.  */
    3940              : 
    3941              : static bool
    3942        89842 : dependence_distance_ge_vf (data_dependence_relation *ddr,
    3943              :                            unsigned int loop_depth, poly_uint64 vf)
    3944              : {
    3945        89842 :   if (DDR_ARE_DEPENDENT (ddr) != NULL_TREE
    3946        89842 :       || DDR_NUM_DIST_VECTS (ddr) == 0)
    3947              :     return false;
    3948              : 
    3949              :   /* If the dependence is exact, we should have limited the VF instead.  */
    3950         6030 :   gcc_checking_assert (DDR_COULD_BE_INDEPENDENT_P (ddr));
    3951              : 
    3952              :   unsigned int i;
    3953              :   lambda_vector dist_v;
    3954        12091 :   FOR_EACH_VEC_ELT (DDR_DIST_VECTS (ddr), i, dist_v)
    3955              :     {
    3956        12056 :       HOST_WIDE_INT dist = dist_v[loop_depth];
    3957        12056 :       if (dist != 0
    3958         6030 :           && !(dist > 0 && DDR_REVERSED_P (ddr))
    3959        18086 :           && maybe_lt ((unsigned HOST_WIDE_INT) abs_hwi (dist), vf))
    3960              :         return false;
    3961              :     }
    3962              : 
    3963           35 :   if (dump_enabled_p ())
    3964            2 :     dump_printf_loc (MSG_NOTE, vect_location,
    3965              :                      "dependence distance between %T and %T is >= VF\n",
    3966            2 :                      DR_REF (DDR_A (ddr)), DR_REF (DDR_B (ddr)));
    3967              : 
    3968              :   return true;
    3969              : }
    3970              : 
    3971              : /* Dump LOWER_BOUND using flags DUMP_KIND.  Dumps are known to be enabled.  */
    3972              : 
    3973              : static void
    3974          432 : dump_lower_bound (dump_flags_t dump_kind, const vec_lower_bound &lower_bound)
    3975              : {
    3976          432 :   dump_printf (dump_kind, "%s (%T) >= ",
    3977          432 :                lower_bound.unsigned_p ? "unsigned" : "abs",
    3978          432 :                lower_bound.expr);
    3979          432 :   dump_dec (dump_kind, lower_bound.min_value);
    3980          432 : }
    3981              : 
    3982              : /* Record that the vectorized loop requires the vec_lower_bound described
    3983              :    by EXPR, UNSIGNED_P and MIN_VALUE.  */
    3984              : 
    3985              : static void
    3986         6750 : vect_check_lower_bound (loop_vec_info loop_vinfo, tree expr, bool unsigned_p,
    3987              :                         poly_uint64 min_value)
    3988              : {
    3989         6750 :   vec<vec_lower_bound> &lower_bounds
    3990              :     = LOOP_VINFO_LOWER_BOUNDS (loop_vinfo);
    3991         7722 :   for (unsigned int i = 0; i < lower_bounds.length (); ++i)
    3992         5953 :     if (operand_equal_p (lower_bounds[i].expr, expr, 0))
    3993              :       {
    3994         4981 :         unsigned_p &= lower_bounds[i].unsigned_p;
    3995         4981 :         min_value = upper_bound (lower_bounds[i].min_value, min_value);
    3996         4981 :         if (lower_bounds[i].unsigned_p != unsigned_p
    3997         4981 :             || maybe_lt (lower_bounds[i].min_value, min_value))
    3998              :           {
    3999          830 :             lower_bounds[i].unsigned_p = unsigned_p;
    4000          830 :             lower_bounds[i].min_value = min_value;
    4001          830 :             if (dump_enabled_p ())
    4002              :               {
    4003          252 :                 dump_printf_loc (MSG_NOTE, vect_location,
    4004              :                                  "updating run-time check to ");
    4005          252 :                 dump_lower_bound (MSG_NOTE, lower_bounds[i]);
    4006          252 :                 dump_printf (MSG_NOTE, "\n");
    4007              :               }
    4008              :           }
    4009         4981 :         return;
    4010              :       }
    4011              : 
    4012         1769 :   vec_lower_bound lower_bound (expr, unsigned_p, min_value);
    4013         1769 :   if (dump_enabled_p ())
    4014              :     {
    4015          180 :       dump_printf_loc (MSG_NOTE, vect_location, "need a run-time check that ");
    4016          180 :       dump_lower_bound (MSG_NOTE, lower_bound);
    4017          180 :       dump_printf (MSG_NOTE, "\n");
    4018              :     }
    4019         1769 :   LOOP_VINFO_LOWER_BOUNDS (loop_vinfo).safe_push (lower_bound);
    4020              : }
    4021              : 
    4022              : /* Return true if it's unlikely that the step of the vectorized form of DR_INFO
    4023              :    will span fewer than GAP bytes.  */
    4024              : 
    4025              : static bool
    4026         5440 : vect_small_gap_p (loop_vec_info loop_vinfo, dr_vec_info *dr_info,
    4027              :                   poly_int64 gap)
    4028              : {
    4029         5440 :   stmt_vec_info stmt_info = dr_info->stmt;
    4030         5440 :   HOST_WIDE_INT count
    4031         5440 :     = estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
    4032         5440 :   if (DR_GROUP_FIRST_ELEMENT (stmt_info))
    4033         4680 :     count *= DR_GROUP_SIZE (DR_GROUP_FIRST_ELEMENT (stmt_info));
    4034         5440 :   return (estimated_poly_value (gap)
    4035         5440 :           <= count * vect_get_scalar_dr_size (dr_info));
    4036              : }
    4037              : 
    4038              : /* Return true if we know that there is no alias between DR_INFO_A and
    4039              :    DR_INFO_B when abs (DR_STEP (DR_INFO_A->dr)) >= N for some N.
    4040              :    When returning true, set *LOWER_BOUND_OUT to this N.  */
    4041              : 
    4042              : static bool
    4043        19793 : vectorizable_with_step_bound_p (dr_vec_info *dr_info_a, dr_vec_info *dr_info_b,
    4044              :                                 poly_uint64 *lower_bound_out)
    4045              : {
    4046              :   /* Check that there is a constant gap of known sign between DR_A
    4047              :      and DR_B.  */
    4048        19793 :   data_reference *dr_a = dr_info_a->dr;
    4049        19793 :   data_reference *dr_b = dr_info_b->dr;
    4050        19793 :   poly_int64 init_a, init_b;
    4051        19793 :   if (!operand_equal_p (DR_BASE_ADDRESS (dr_a), DR_BASE_ADDRESS (dr_b), 0)
    4052         8808 :       || !operand_equal_p (DR_OFFSET (dr_a), DR_OFFSET (dr_b), 0)
    4053         8122 :       || !operand_equal_p (DR_STEP (dr_a), DR_STEP (dr_b), 0)
    4054         8112 :       || !poly_int_tree_p (DR_INIT (dr_a), &init_a)
    4055         8112 :       || !poly_int_tree_p (DR_INIT (dr_b), &init_b)
    4056        19793 :       || !ordered_p (init_a, init_b))
    4057              :     return false;
    4058              : 
    4059              :   /* Sort DR_A and DR_B by the address they access.  */
    4060         8112 :   if (maybe_lt (init_b, init_a))
    4061              :     {
    4062          119 :       std::swap (init_a, init_b);
    4063          119 :       std::swap (dr_info_a, dr_info_b);
    4064          119 :       std::swap (dr_a, dr_b);
    4065              :     }
    4066              : 
    4067              :   /* If the two accesses could be dependent within a scalar iteration,
    4068              :      make sure that we'd retain their order.  */
    4069         8112 :   if (maybe_gt (init_a + vect_get_scalar_dr_size (dr_info_a), init_b)
    4070         8112 :       && !vect_preserves_scalar_order_p (dr_info_a, dr_info_b))
    4071              :     return false;
    4072              : 
    4073              :   /* There is no alias if abs (DR_STEP) is greater than or equal to
    4074              :      the bytes spanned by the combination of the two accesses.  */
    4075         8112 :   *lower_bound_out = init_b + vect_get_scalar_dr_size (dr_info_b) - init_a;
    4076         8112 :   return true;
    4077              : }
    4078              : 
    4079              : /* Function vect_prune_runtime_alias_test_list.
    4080              : 
    4081              :    Prune a list of ddrs to be tested at run-time by versioning for alias.
    4082              :    Merge several alias checks into one if possible.
    4083              :    Return FALSE if resulting list of ddrs is longer then allowed by
    4084              :    PARAM_VECT_MAX_VERSION_FOR_ALIAS_CHECKS, otherwise return TRUE.  */
    4085              : 
    4086              : opt_result
    4087       449612 : vect_prune_runtime_alias_test_list (loop_vec_info loop_vinfo)
    4088              : {
    4089       449612 :   typedef pair_hash <tree_operand_hash, tree_operand_hash> tree_pair_hash;
    4090       449612 :   hash_set <tree_pair_hash> compared_objects;
    4091              : 
    4092       449612 :   const vec<ddr_p> &may_alias_ddrs = LOOP_VINFO_MAY_ALIAS_DDRS (loop_vinfo);
    4093       449612 :   vec<dr_with_seg_len_pair_t> &comp_alias_ddrs
    4094              :     = LOOP_VINFO_COMP_ALIAS_DDRS (loop_vinfo);
    4095       449612 :   const vec<vec_object_pair> &check_unequal_addrs
    4096              :     = LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo);
    4097       449612 :   poly_uint64 vect_factor = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
    4098       449612 :   tree scalar_loop_iters = LOOP_VINFO_NITERS (loop_vinfo);
    4099              : 
    4100       449612 :   ddr_p ddr;
    4101       449612 :   unsigned int i;
    4102       449612 :   tree length_factor;
    4103              : 
    4104       449612 :   DUMP_VECT_SCOPE ("vect_prune_runtime_alias_test_list");
    4105              : 
    4106              :   /* Step values are irrelevant for aliasing if the number of vector
    4107              :      iterations is equal to the number of scalar iterations (which can
    4108              :      happen for fully-SLP loops).  */
    4109       449612 :   bool vf_one_p = known_eq (LOOP_VINFO_VECT_FACTOR (loop_vinfo), 1U);
    4110              : 
    4111       449612 :   if (!vf_one_p)
    4112              :     {
    4113              :       /* Convert the checks for nonzero steps into bound tests.  */
    4114              :       tree value;
    4115       446170 :       FOR_EACH_VEC_ELT (LOOP_VINFO_CHECK_NONZERO (loop_vinfo), i, value)
    4116         1667 :         vect_check_lower_bound (loop_vinfo, value, true, 1);
    4117              :     }
    4118              : 
    4119       449612 :   if (may_alias_ddrs.is_empty ())
    4120       421774 :     return opt_result::success ();
    4121              : 
    4122        27838 :   comp_alias_ddrs.create (may_alias_ddrs.length ());
    4123              : 
    4124        27838 :   unsigned int loop_depth
    4125        27838 :     = index_in_loop_nest (LOOP_VINFO_LOOP (loop_vinfo)->num,
    4126        27838 :                           LOOP_VINFO_LOOP_NEST (loop_vinfo));
    4127              : 
    4128              :   /* First, we collect all data ref pairs for aliasing checks.  */
    4129       113612 :   FOR_EACH_VEC_ELT (may_alias_ddrs, i, ddr)
    4130              :     {
    4131        89842 :       poly_uint64 lower_bound;
    4132        89842 :       tree segment_length_a, segment_length_b;
    4133        89842 :       unsigned HOST_WIDE_INT access_size_a, access_size_b;
    4134        89842 :       unsigned HOST_WIDE_INT align_a, align_b;
    4135              : 
    4136              :       /* Ignore the alias if the VF we chose ended up being no greater
    4137              :          than the dependence distance.  */
    4138        89842 :       if (dependence_distance_ge_vf (ddr, loop_depth, vect_factor))
    4139        13502 :         continue;
    4140              : 
    4141        89807 :       if (DDR_OBJECT_A (ddr))
    4142              :         {
    4143          106 :           vec_object_pair new_pair (DDR_OBJECT_A (ddr), DDR_OBJECT_B (ddr));
    4144          106 :           if (!compared_objects.add (new_pair))
    4145              :             {
    4146           22 :               if (dump_enabled_p ())
    4147           16 :                 dump_printf_loc (MSG_NOTE, vect_location,
    4148              :                                  "checking that %T and %T"
    4149              :                                  " have different addresses\n",
    4150              :                                  new_pair.first, new_pair.second);
    4151           22 :               LOOP_VINFO_CHECK_UNEQUAL_ADDRS (loop_vinfo).safe_push (new_pair);
    4152              :             }
    4153          106 :           continue;
    4154          106 :         }
    4155              : 
    4156        89701 :       dr_vec_info *dr_info_a = loop_vinfo->lookup_dr (DDR_A (ddr));
    4157        89701 :       stmt_vec_info stmt_info_a = dr_info_a->stmt;
    4158              : 
    4159        89701 :       dr_vec_info *dr_info_b = loop_vinfo->lookup_dr (DDR_B (ddr));
    4160        89701 :       stmt_vec_info stmt_info_b = dr_info_b->stmt;
    4161              : 
    4162        89701 :       bool preserves_scalar_order_p
    4163        89701 :         = vect_preserves_scalar_order_p (dr_info_a, dr_info_b);
    4164        89701 :       bool ignore_step_p
    4165              :           = (vf_one_p
    4166        89701 :              && (preserves_scalar_order_p
    4167         4193 :                  || operand_equal_p (DR_STEP (dr_info_a->dr),
    4168         4193 :                                      DR_STEP (dr_info_b->dr))));
    4169              : 
    4170              :       /* Skip the pair if inter-iteration dependencies are irrelevant
    4171              :          and intra-iteration dependencies are guaranteed to be honored.  */
    4172         7742 :       if (ignore_step_p
    4173         8019 :           && (preserves_scalar_order_p
    4174         3242 :               || vectorizable_with_step_bound_p (dr_info_a, dr_info_b,
    4175              :                                                  &lower_bound)))
    4176              :         {
    4177         7742 :           if (dump_enabled_p ())
    4178         2416 :             dump_printf_loc (MSG_NOTE, vect_location,
    4179              :                              "no need for alias check between "
    4180              :                              "%T and %T when VF is 1\n",
    4181         2416 :                              DR_REF (dr_info_a->dr), DR_REF (dr_info_b->dr));
    4182         7742 :           continue;
    4183              :         }
    4184              : 
    4185              :       /* See whether we can handle the alias using a bounds check on
    4186              :          the step, and whether that's likely to be the best approach.
    4187              :          (It might not be, for example, if the minimum step is much larger
    4188              :          than the number of bytes handled by one vector iteration.)  */
    4189        81959 :       if (!ignore_step_p
    4190        81682 :           && TREE_CODE (DR_STEP (dr_info_a->dr)) != INTEGER_CST
    4191        16551 :           && vectorizable_with_step_bound_p (dr_info_a, dr_info_b,
    4192              :                                              &lower_bound)
    4193        87106 :           && (vect_small_gap_p (loop_vinfo, dr_info_a, lower_bound)
    4194         5147 :               || vect_small_gap_p (loop_vinfo, dr_info_b, lower_bound)))
    4195              :         {
    4196         5083 :           bool unsigned_p = dr_known_forward_stride_p (dr_info_a->dr);
    4197         5083 :           if (dump_enabled_p ())
    4198              :             {
    4199         3394 :               dump_printf_loc (MSG_NOTE, vect_location, "no alias between "
    4200              :                                "%T and %T when the step %T is outside ",
    4201              :                                DR_REF (dr_info_a->dr),
    4202         1697 :                                DR_REF (dr_info_b->dr),
    4203         1697 :                                DR_STEP (dr_info_a->dr));
    4204         1697 :               if (unsigned_p)
    4205          540 :                 dump_printf (MSG_NOTE, "[0");
    4206              :               else
    4207              :                 {
    4208         1157 :                   dump_printf (MSG_NOTE, "(");
    4209         1157 :                   dump_dec (MSG_NOTE, poly_int64 (-lower_bound));
    4210              :                 }
    4211         1697 :               dump_printf (MSG_NOTE, ", ");
    4212         1697 :               dump_dec (MSG_NOTE, lower_bound);
    4213         1697 :               dump_printf (MSG_NOTE, ")\n");
    4214              :             }
    4215         5083 :           vect_check_lower_bound (loop_vinfo, DR_STEP (dr_info_a->dr),
    4216              :                                   unsigned_p, lower_bound);
    4217         5083 :           continue;
    4218         5083 :         }
    4219              : 
    4220        76876 :       stmt_vec_info dr_group_first_a = DR_GROUP_FIRST_ELEMENT (stmt_info_a);
    4221        76876 :       if (dr_group_first_a)
    4222              :         {
    4223        21781 :           stmt_info_a = dr_group_first_a;
    4224        21781 :           dr_info_a = STMT_VINFO_DR_INFO (stmt_info_a);
    4225              :         }
    4226              : 
    4227        76876 :       stmt_vec_info dr_group_first_b = DR_GROUP_FIRST_ELEMENT (stmt_info_b);
    4228        76876 :       if (dr_group_first_b)
    4229              :         {
    4230        22445 :           stmt_info_b = dr_group_first_b;
    4231        22445 :           dr_info_b = STMT_VINFO_DR_INFO (stmt_info_b);
    4232              :         }
    4233              : 
    4234        76876 :       if (ignore_step_p)
    4235              :         {
    4236          277 :           segment_length_a = size_zero_node;
    4237          277 :           segment_length_b = size_zero_node;
    4238              :         }
    4239              :       else
    4240              :         {
    4241        76599 :           if (!operand_equal_p (DR_STEP (dr_info_a->dr),
    4242        76599 :                                 DR_STEP (dr_info_b->dr), 0))
    4243              :             {
    4244        17279 :               length_factor = scalar_loop_iters;
    4245        17279 :               if (TREE_CODE (length_factor) == SCEV_NOT_KNOWN)
    4246           12 :                 return opt_result::failure_at (vect_location,
    4247              :                                                "Unsupported alias check on"
    4248              :                                                " uncounted loop\n");
    4249              :             }
    4250              :           else
    4251        59320 :             length_factor = size_int (vect_factor);
    4252        76587 :           segment_length_a = vect_vfa_segment_size (dr_info_a, length_factor);
    4253        76587 :           segment_length_b = vect_vfa_segment_size (dr_info_b, length_factor);
    4254              :         }
    4255        76864 :       access_size_a = vect_vfa_access_size (loop_vinfo, dr_info_a);
    4256        76864 :       access_size_b = vect_vfa_access_size (loop_vinfo, dr_info_b);
    4257        76864 :       align_a = vect_vfa_align (dr_info_a);
    4258        76864 :       align_b = vect_vfa_align (dr_info_b);
    4259              : 
    4260              :       /* See whether the alias is known at compilation time.  */
    4261        76864 :       if (operand_equal_p (DR_BASE_ADDRESS (dr_info_a->dr),
    4262        76864 :                            DR_BASE_ADDRESS (dr_info_b->dr), 0)
    4263         6456 :           && operand_equal_p (DR_OFFSET (dr_info_a->dr),
    4264         6456 :                               DR_OFFSET (dr_info_b->dr), 0)
    4265         4748 :           && TREE_CODE (DR_STEP (dr_info_a->dr)) == INTEGER_CST
    4266         4674 :           && TREE_CODE (DR_STEP (dr_info_b->dr)) == INTEGER_CST
    4267         4674 :           && poly_int_tree_p (segment_length_a)
    4268        81478 :           && poly_int_tree_p (segment_length_b))
    4269              :         {
    4270         4592 :           int res = vect_compile_time_alias (dr_info_a, dr_info_b,
    4271              :                                              segment_length_a,
    4272              :                                              segment_length_b,
    4273              :                                              access_size_a,
    4274              :                                              access_size_b);
    4275         4592 :           if (res >= 0 && dump_enabled_p ())
    4276              :             {
    4277          208 :               dump_printf_loc (MSG_NOTE, vect_location,
    4278              :                                "can tell at compile time that %T and %T",
    4279          104 :                                DR_REF (dr_info_a->dr), DR_REF (dr_info_b->dr));
    4280          104 :               if (res == 0)
    4281           57 :                 dump_printf (MSG_NOTE, " do not alias\n");
    4282              :               else
    4283           47 :                 dump_printf (MSG_NOTE, " alias\n");
    4284              :             }
    4285              : 
    4286         4592 :           if (res == 0)
    4287          536 :             continue;
    4288              : 
    4289         4056 :           if (res == 1)
    4290         4056 :             return opt_result::failure_at (stmt_info_b->stmt,
    4291              :                                            "not vectorized:"
    4292              :                                            " compilation time alias: %G%G",
    4293              :                                            stmt_info_a->stmt,
    4294              :                                            stmt_info_b->stmt);
    4295              :         }
    4296              : 
    4297              :       /* dr_with_seg_len requires the alignment to apply to the segment length
    4298              :          and access size, not just the start address.  The access size can be
    4299              :          smaller than the pointer alignment for grouped accesses and bitfield
    4300              :          references; see PR115192 and PR116125 respectively.  */
    4301        72272 :       align_a = std::min (align_a, least_bit_hwi (access_size_a));
    4302        72272 :       align_b = std::min (align_b, least_bit_hwi (access_size_b));
    4303              : 
    4304        72272 :       dr_with_seg_len dr_a (dr_info_a->dr, segment_length_a,
    4305        72272 :                             access_size_a, align_a);
    4306        72272 :       dr_with_seg_len dr_b (dr_info_b->dr, segment_length_b,
    4307        72272 :                             access_size_b, align_b);
    4308              :       /* Canonicalize the order to be the one that's needed for accurate
    4309              :          RAW, WAR and WAW flags, in cases where the data references are
    4310              :          well-ordered.  The order doesn't really matter otherwise,
    4311              :          but we might as well be consistent.  */
    4312        72272 :       if (get_later_stmt (stmt_info_a, stmt_info_b) == stmt_info_a)
    4313         5531 :         std::swap (dr_a, dr_b);
    4314              : 
    4315        72272 :       dr_with_seg_len_pair_t dr_with_seg_len_pair
    4316              :         (dr_a, dr_b, (preserves_scalar_order_p
    4317              :                       ? dr_with_seg_len_pair_t::WELL_ORDERED
    4318        79809 :                       : dr_with_seg_len_pair_t::REORDERED));
    4319              : 
    4320        72272 :       comp_alias_ddrs.safe_push (dr_with_seg_len_pair);
    4321              :     }
    4322              : 
    4323        23770 :   prune_runtime_alias_test_list (&comp_alias_ddrs, vect_factor);
    4324              : 
    4325        47540 :   unsigned int count = (comp_alias_ddrs.length ()
    4326        23770 :                         + check_unequal_addrs.length ());
    4327              : 
    4328        23770 :   if (count
    4329        23770 :       && (loop_cost_model (LOOP_VINFO_LOOP (loop_vinfo))
    4330              :           == VECT_COST_MODEL_VERY_CHEAP))
    4331        13556 :     return opt_result::failure_at
    4332        13556 :       (vect_location, "would need a runtime alias check\n");
    4333              : 
    4334        10214 :   if (dump_enabled_p ())
    4335         2026 :     dump_printf_loc (MSG_NOTE, vect_location,
    4336              :                      "improved number of alias checks from %d to %d\n",
    4337              :                      may_alias_ddrs.length (), count);
    4338        10214 :   unsigned limit = param_vect_max_version_for_alias_checks;
    4339        10214 :   if (loop_cost_model (LOOP_VINFO_LOOP (loop_vinfo)) == VECT_COST_MODEL_CHEAP)
    4340          938 :     limit = param_vect_max_version_for_alias_checks * 6 / 10;
    4341        10214 :   if (count > limit)
    4342          162 :     return opt_result::failure_at
    4343          162 :       (vect_location,
    4344              :        "number of versioning for alias run-time tests exceeds %d "
    4345              :        "(--param vect-max-version-for-alias-checks)\n", limit);
    4346              : 
    4347        10052 :   return opt_result::success ();
    4348       449612 : }
    4349              : 
    4350              : /* Structure to hold information about a supported gather/scatter
    4351              :    configuration.  */
    4352              : struct gather_scatter_config
    4353              : {
    4354              :   internal_fn ifn;
    4355              :   tree offset_vectype;
    4356              :   int scale;
    4357              :   vec<int> elsvals;
    4358              : };
    4359              : 
    4360              : /* Determine which gather/scatter IFN is supported for the given parameters.
    4361              :    IFN_MASK_GATHER_LOAD, IFN_GATHER_LOAD, and IFN_MASK_LEN_GATHER_LOAD
    4362              :    are mutually exclusive, so we only need to find one.  Return the
    4363              :    supported IFN or IFN_LAST if none are supported.  */
    4364              : 
    4365              : static internal_fn
    4366      1193704 : vect_gather_scatter_which_ifn (bool read_p, bool masked_p,
    4367              :                                tree vectype, tree memory_type,
    4368              :                                tree offset_vectype, int scale,
    4369              :                                vec<int> *elsvals)
    4370              : {
    4371              :   /* Work out which functions to try.  */
    4372      1193704 :   internal_fn ifn, alt_ifn, alt_ifn2;
    4373      1193704 :   if (read_p)
    4374              :     {
    4375       894470 :       ifn = masked_p ? IFN_MASK_GATHER_LOAD : IFN_GATHER_LOAD;
    4376              :       alt_ifn = IFN_MASK_GATHER_LOAD;
    4377              :       alt_ifn2 = IFN_MASK_LEN_GATHER_LOAD;
    4378              :     }
    4379              :   else
    4380              :     {
    4381       299234 :       ifn = masked_p ? IFN_MASK_SCATTER_STORE : IFN_SCATTER_STORE;
    4382              :       alt_ifn = IFN_MASK_SCATTER_STORE;
    4383              :       alt_ifn2 = IFN_MASK_LEN_SCATTER_STORE;
    4384              :     }
    4385              : 
    4386      1193704 :   if (!offset_vectype)
    4387              :     return IFN_LAST;
    4388              : 
    4389      1193704 :   if (internal_gather_scatter_fn_supported_p (ifn, vectype, memory_type,
    4390              :                                               offset_vectype, scale, elsvals))
    4391              :     return ifn;
    4392      1193704 :   if (internal_gather_scatter_fn_supported_p (alt_ifn, vectype, memory_type,
    4393              :                                               offset_vectype, scale, elsvals))
    4394              :     return alt_ifn;
    4395      1193704 :   if (internal_gather_scatter_fn_supported_p (alt_ifn2, vectype, memory_type,
    4396              :                                               offset_vectype, scale, elsvals))
    4397            0 :     return alt_ifn2;
    4398              : 
    4399              :   return IFN_LAST;
    4400              : }
    4401              : 
    4402              : /* Collect all supported offset vector types for a gather load or scatter
    4403              :    store.  READ_P is true for loads and false for stores.  MASKED_P is true
    4404              :    if the load or store is conditional.  VECTYPE is the data vector type.
    4405              :    MEMORY_TYPE is the type of the memory elements being loaded or stored,
    4406              :    and OFFSET_TYPE is the type of the offset.
    4407              :    SCALE is the amount by which the offset should be multiplied.
    4408              : 
    4409              :    Return a vector of all configurations the target supports (which can
    4410              :    be none).  */
    4411              : 
    4412              : static auto_vec<gather_scatter_config>
    4413        85171 : vect_gather_scatter_get_configs (vec_info *vinfo, bool read_p, bool masked_p,
    4414              :                                  tree vectype, tree memory_type,
    4415              :                                  tree offset_type, int scale)
    4416              : {
    4417        85171 :   auto_vec<gather_scatter_config> configs;
    4418              : 
    4419        85171 :   auto_vec<tree, 8> offset_types_to_try;
    4420              : 
    4421              :   /* Try all sizes from the offset type's precision up to POINTER_SIZE.  */
    4422        85171 :   for (unsigned int bits = TYPE_PRECISION (offset_type);
    4423       401384 :        bits <= POINTER_SIZE;
    4424       302741 :        bits *= 2)
    4425              :     {
    4426              :       /* Signed variant.  */
    4427       302741 :       offset_types_to_try.safe_push
    4428       302741 :         (build_nonstandard_integer_type (bits, 0));
    4429              :       /* Unsigned variant.  */
    4430       302741 :       offset_types_to_try.safe_push
    4431       302741 :         (build_nonstandard_integer_type (bits, 1));
    4432              :     }
    4433              : 
    4434              :   /* Once we find which IFN works for one offset type, we know that it
    4435              :      will work for other offset types as well.  Then we can perform
    4436              :      the checks for the remaining offset types with only that IFN.
    4437              :      However, we might need to try different offset types to find which
    4438              :      IFN is supported, since the check is offset-type-specific.  */
    4439              :   internal_fn ifn = IFN_LAST;
    4440              : 
    4441              :   /* Try each offset type.  */
    4442       690653 :   for (unsigned int i = 0; i < offset_types_to_try.length (); i++)
    4443              :     {
    4444       605482 :       tree offset_type = offset_types_to_try[i];
    4445       605482 :       tree offset_vectype = get_vectype_for_scalar_type (vinfo, offset_type);
    4446       605482 :       if (!offset_vectype)
    4447        10070 :         continue;
    4448              : 
    4449              :       /* Try multiple scale values.  Start with exact match, then try
    4450              :          smaller common scales that a target might support .  */
    4451       595412 :       int scales_to_try[] = {scale, 1, 2, 4, 8};
    4452              : 
    4453      3572472 :       for (unsigned int j = 0;
    4454      3572472 :            j < sizeof (scales_to_try) / sizeof (*scales_to_try);
    4455              :            j++)
    4456              :         {
    4457      2977060 :           int try_scale = scales_to_try[j];
    4458              : 
    4459              :           /* Skip scales >= requested scale (except for exact match).  */
    4460      2977060 :           if (j > 0 && try_scale >= scale)
    4461      1783356 :             continue;
    4462              : 
    4463              :           /* Skip if requested scale is not a multiple of this scale.  */
    4464      1193848 :           if (j > 0 && scale % try_scale != 0)
    4465          144 :             continue;
    4466              : 
    4467      1193704 :           vec<int> elsvals = vNULL;
    4468              : 
    4469              :           /* If we haven't determined which IFN is supported yet, try all three
    4470              :              to find which one the target supports.  */
    4471      1193704 :           if (ifn == IFN_LAST)
    4472              :             {
    4473      1193704 :               ifn = vect_gather_scatter_which_ifn (read_p, masked_p,
    4474              :                                                    vectype, memory_type,
    4475              :                                                    offset_vectype, try_scale,
    4476              :                                                    &elsvals);
    4477      1193704 :               if (ifn != IFN_LAST)
    4478              :                 {
    4479              :                   /* Found which IFN is supported.  Save this configuration.  */
    4480            0 :                   gather_scatter_config config;
    4481            0 :                   config.ifn = ifn;
    4482            0 :                   config.offset_vectype = offset_vectype;
    4483            0 :                   config.scale = try_scale;
    4484            0 :                   config.elsvals = elsvals;
    4485            0 :                   configs.safe_push (config);
    4486              :                 }
    4487              :             }
    4488              :           else
    4489              :             {
    4490              :               /* We already know which IFN is supported, just check if this
    4491              :                  offset type and scale work with it.  */
    4492            0 :               if (internal_gather_scatter_fn_supported_p (ifn, vectype,
    4493              :                                                           memory_type,
    4494              :                                                           offset_vectype,
    4495              :                                                           try_scale,
    4496              :                                                           &elsvals))
    4497              :                 {
    4498            0 :                   gather_scatter_config config;
    4499            0 :                   config.ifn = ifn;
    4500            0 :                   config.offset_vectype = offset_vectype;
    4501            0 :                   config.scale = try_scale;
    4502            0 :                   config.elsvals = elsvals;
    4503            0 :                   configs.safe_push (config);
    4504              :                 }
    4505              :             }
    4506              :         }
    4507              :     }
    4508              : 
    4509        85171 :   return configs;
    4510        85171 : }
    4511              : 
    4512              : /* Check whether we can use an internal function for a gather load
    4513              :    or scatter store.  READ_P is true for loads and false for stores.
    4514              :    MASKED_P is true if the load or store is conditional.  MEMORY_TYPE is
    4515              :    the type of the memory elements being loaded or stored.  OFFSET_TYPE
    4516              :    is the type of the offset that is being applied to the invariant
    4517              :    base address.  If OFFSET_TYPE is scalar the function chooses an
    4518              :    appropriate vector type for it.  SCALE is the amount by which the
    4519              :    offset should be multiplied *after* it has been converted to address width.
    4520              :    If the target does not support the requested SCALE, SUPPORTED_SCALE
    4521              :    will contain the scale that is actually supported
    4522              :    (which may be smaller, requiring additional multiplication).
    4523              :    Otherwise SUPPORTED_SCALE is 0.
    4524              : 
    4525              :    Return true if the function is supported, storing the function id in
    4526              :    *IFN_OUT and the vector type for the offset in *OFFSET_VECTYPE_OUT.
    4527              :    If we support an offset vector type with different signedness than
    4528              :    OFFSET_TYPE store it in SUPPORTED_OFFSET_VECTYPE.
    4529              : 
    4530              :    If we can use gather/scatter and ELSVALS is nonzero, store the possible
    4531              :    else values in ELSVALS.  */
    4532              : 
    4533              : bool
    4534        85171 : vect_gather_scatter_fn_p (vec_info *vinfo, bool read_p, bool masked_p,
    4535              :                           tree vectype, tree memory_type, tree offset_type,
    4536              :                           int scale, int *supported_scale,
    4537              :                           internal_fn *ifn_out,
    4538              :                           tree *offset_vectype_out,
    4539              :                           tree *supported_offset_vectype,
    4540              :                           vec<int> *elsvals)
    4541              : {
    4542        85171 :   *supported_offset_vectype = NULL_TREE;
    4543        85171 :   *supported_scale = 0;
    4544        85171 :   unsigned int memory_bits = tree_to_uhwi (TYPE_SIZE (memory_type));
    4545        85171 :   unsigned int element_bits = vector_element_bits (vectype);
    4546        85171 :   if (element_bits != memory_bits)
    4547              :     /* For now the vector elements must be the same width as the
    4548              :        memory elements.  */
    4549              :     return false;
    4550              : 
    4551              :   /* Get the original offset vector type for comparison.  */
    4552        85171 :   tree offset_vectype = VECTOR_TYPE_P (offset_type)
    4553        85171 :     ? offset_type : get_vectype_for_scalar_type (vinfo, offset_type);
    4554              : 
    4555              :   /* If there is no offset vectype, bail.  */
    4556        72054 :   if (!offset_vectype)
    4557              :     return false;
    4558              : 
    4559        85171 :   offset_type = TREE_TYPE (offset_vectype);
    4560              : 
    4561              :   /* Get all supported configurations for this data vector type.  */
    4562        85171 :   auto_vec<gather_scatter_config> configs
    4563              :     = vect_gather_scatter_get_configs (vinfo, read_p, masked_p, vectype,
    4564        85171 :                                        memory_type, offset_type, scale);
    4565              : 
    4566        85171 :   if (configs.is_empty ())
    4567              :     return false;
    4568              : 
    4569              :   /* Selection priority:
    4570              :      1 - Exact scale match + offset type match
    4571              :      2 - Exact scale match + sign-swapped offset
    4572              :      3 - Smaller scale + offset type match
    4573              :      4 - Smaller scale + sign-swapped offset
    4574              :      Within each category, prefer smaller offset types.  */
    4575              : 
    4576              :   /* First pass: exact scale match with no conversion.  */
    4577            0 :   for (unsigned int i = 0; i < configs.length (); i++)
    4578              :     {
    4579            0 :       if (configs[i].scale == scale
    4580            0 :           && TYPE_SIGN (configs[i].offset_vectype)
    4581            0 :              == TYPE_SIGN (offset_vectype))
    4582              :         {
    4583            0 :           *ifn_out = configs[i].ifn;
    4584            0 :           *offset_vectype_out = configs[i].offset_vectype;
    4585            0 :           if (elsvals)
    4586            0 :             *elsvals = configs[i].elsvals;
    4587              :           return true;
    4588              :         }
    4589              :     }
    4590              : 
    4591              :   /* No direct match.  This means we try to find either
    4592              :       - a sign-swapped offset vectype or
    4593              :       - a different scale and 2x larger offset type
    4594              :       - a different scale and larger sign-swapped offset vectype.  */
    4595            0 :   unsigned int offset_precision = TYPE_PRECISION (TREE_TYPE (offset_vectype));
    4596            0 :   unsigned int needed_precision
    4597            0 :     = TYPE_UNSIGNED (offset_vectype) ? offset_precision * 2 : POINTER_SIZE;
    4598            0 :   needed_precision = std::min (needed_precision, (unsigned) POINTER_SIZE);
    4599              : 
    4600              :   /* Second pass: No direct match.  This means we try to find a sign-swapped
    4601              :      offset vectype.  */
    4602            0 :   for (unsigned int i = 0; i < configs.length (); i++)
    4603              :     {
    4604            0 :       unsigned int precision
    4605            0 :         = TYPE_PRECISION (TREE_TYPE (configs[i].offset_vectype));
    4606            0 :       if (configs[i].scale == scale
    4607            0 :           && precision >= needed_precision
    4608            0 :           && (supportable_convert_operation (CONVERT_EXPR,
    4609            0 :                                              configs[i].offset_vectype,
    4610              :                                              offset_vectype)
    4611            0 :               || (needed_precision == offset_precision
    4612            0 :                   && tree_nop_conversion_p (configs[i].offset_vectype,
    4613              :                                             offset_vectype))))
    4614              :         {
    4615            0 :           *ifn_out = configs[i].ifn;
    4616            0 :           *offset_vectype_out = offset_vectype;
    4617            0 :           *supported_offset_vectype = configs[i].offset_vectype;
    4618            0 :           if (elsvals)
    4619            0 :             *elsvals = configs[i].elsvals;
    4620              :           return true;
    4621              :         }
    4622              :     }
    4623              : 
    4624              :   /* Third pass: Try a smaller scale with the same signedness.  */
    4625            0 :   needed_precision = offset_precision * 2;
    4626            0 :   needed_precision = std::min (needed_precision, (unsigned) POINTER_SIZE);
    4627              : 
    4628            0 :   for (unsigned int i = 0; i < configs.length (); i++)
    4629              :     {
    4630            0 :       unsigned int precision
    4631            0 :         = TYPE_PRECISION (TREE_TYPE (configs[i].offset_vectype));
    4632            0 :       if (configs[i].scale < scale
    4633            0 :           && TYPE_SIGN (configs[i].offset_vectype)
    4634            0 :              == TYPE_SIGN (offset_vectype)
    4635            0 :           && precision >= needed_precision)
    4636              :         {
    4637            0 :           *ifn_out = configs[i].ifn;
    4638            0 :           *offset_vectype_out = configs[i].offset_vectype;
    4639            0 :           *supported_scale = configs[i].scale;
    4640              :           /* Only set SUPPORTED_OFFSET_VECTYPE if this is a real
    4641              :              conversion.  */
    4642            0 :           if (!useless_type_conversion_p (offset_vectype,
    4643            0 :                                           configs[i].offset_vectype))
    4644            0 :             *supported_offset_vectype = configs[i].offset_vectype;
    4645            0 :           if (elsvals)
    4646            0 :             *elsvals = configs[i].elsvals;
    4647              :           return true;
    4648              :         }
    4649              :     }
    4650              : 
    4651              :   /* Fourth pass: Try a smaller scale and sign-swapped offset vectype.  */
    4652            0 :   needed_precision
    4653            0 :     = TYPE_UNSIGNED (offset_vectype) ? offset_precision * 2 : POINTER_SIZE;
    4654            0 :   needed_precision = std::min (needed_precision, (unsigned) POINTER_SIZE);
    4655              : 
    4656        85171 :   for (unsigned int i = 0; i < configs.length (); i++)
    4657              :     {
    4658            0 :       unsigned int precision
    4659            0 :         = TYPE_PRECISION (TREE_TYPE (configs[i].offset_vectype));
    4660            0 :       if (configs[i].scale < scale
    4661            0 :           && precision >= needed_precision
    4662            0 :           && (supportable_convert_operation (CONVERT_EXPR,
    4663            0 :                                              configs[i].offset_vectype,
    4664              :                                              offset_vectype)
    4665            0 :               || (needed_precision == offset_precision
    4666            0 :                   && tree_nop_conversion_p (configs[i].offset_vectype,
    4667              :                                             offset_vectype))))
    4668              :         {
    4669            0 :           *ifn_out = configs[i].ifn;
    4670            0 :           *offset_vectype_out = offset_vectype;
    4671            0 :           *supported_offset_vectype = configs[i].offset_vectype;
    4672            0 :           *supported_scale = configs[i].scale;
    4673            0 :           if (elsvals)
    4674            0 :             *elsvals = configs[i].elsvals;
    4675              :           return true;
    4676              :         }
    4677              :     }
    4678              : 
    4679              :   return false;
    4680        85171 : }
    4681              : 
    4682              : /* STMT_INFO is a call to an internal gather load or scatter store function.
    4683              :    Describe the operation in INFO.  */
    4684              : 
    4685              : void
    4686            0 : vect_describe_gather_scatter_call (stmt_vec_info stmt_info,
    4687              :                                    gather_scatter_info *info)
    4688              : {
    4689            0 :   gcall *call = as_a <gcall *> (stmt_info->stmt);
    4690            0 :   tree vectype = STMT_VINFO_VECTYPE (stmt_info);
    4691            0 :   data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
    4692              : 
    4693            0 :   info->ifn = gimple_call_internal_fn (call);
    4694            0 :   info->decl = NULL_TREE;
    4695            0 :   info->base = gimple_call_arg (call, 0);
    4696            0 :   info->alias_ptr = gimple_call_arg
    4697            0 :                      (call, internal_fn_alias_ptr_index (info->ifn));
    4698            0 :   info->offset = gimple_call_arg
    4699            0 :                   (call, internal_fn_offset_index (info->ifn));
    4700            0 :   info->offset_vectype = NULL_TREE;
    4701            0 :   info->scale = TREE_INT_CST_LOW (gimple_call_arg
    4702              :                                   (call, internal_fn_scale_index (info->ifn)));
    4703            0 :   info->element_type = TREE_TYPE (vectype);
    4704            0 :   info->memory_type = TREE_TYPE (DR_REF (dr));
    4705            0 : }
    4706              : 
    4707              : /* Return true if a non-affine read or write in STMT_INFO is suitable for a
    4708              :    gather load or scatter store with VECTYPE.  Describe the operation in *INFO
    4709              :    if so.  If it is suitable and ELSVALS is nonzero store the supported else
    4710              :    values in the vector it points to.  */
    4711              : 
    4712              : bool
    4713       346422 : vect_check_gather_scatter (stmt_vec_info stmt_info, tree vectype,
    4714              :                            loop_vec_info loop_vinfo,
    4715              :                            gather_scatter_info *info, vec<int> *elsvals)
    4716              : {
    4717       346422 :   HOST_WIDE_INT scale = 1;
    4718       346422 :   poly_int64 pbitpos, pbitsize;
    4719       346422 :   class loop *loop = LOOP_VINFO_LOOP (loop_vinfo);
    4720       346422 :   struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
    4721       346422 :   tree offtype = NULL_TREE;
    4722       346422 :   tree decl = NULL_TREE, base, off;
    4723       346422 :   tree memory_type = TREE_TYPE (DR_REF (dr));
    4724       346422 :   machine_mode pmode;
    4725       346422 :   int punsignedp, reversep, pvolatilep = 0;
    4726       346422 :   internal_fn ifn;
    4727       346422 :   tree offset_vectype;
    4728       346422 :   bool masked_p = false;
    4729              : 
    4730              :   /* See whether this is already a call to a gather/scatter internal function.
    4731              :      If not, see whether it's a masked load or store.  */
    4732       346422 :   gcall *call = dyn_cast <gcall *> (stmt_info->stmt);
    4733         4261 :   if (call && gimple_call_internal_p (call))
    4734              :     {
    4735         4261 :       ifn = gimple_call_internal_fn (call);
    4736         4261 :       if (internal_gather_scatter_fn_p (ifn))
    4737              :         {
    4738            0 :           vect_describe_gather_scatter_call (stmt_info, info);
    4739              : 
    4740              :           /* In pattern recog we simply used a ZERO else value that
    4741              :              we need to correct here.  To that end just re-use the
    4742              :              (already successful) check if we support a gather IFN
    4743              :              and have it populate the else values.  */
    4744            0 :           if (DR_IS_READ (dr) && internal_fn_mask_index (ifn) >= 0 && elsvals)
    4745            0 :             supports_vec_gather_load_p (TYPE_MODE (vectype), elsvals);
    4746              :           return true;
    4747              :         }
    4748         4261 :       masked_p = (ifn == IFN_MASK_LOAD || ifn == IFN_MASK_STORE);
    4749              :     }
    4750              : 
    4751              :   /* True if we should aim to use internal functions rather than
    4752              :      built-in functions.  */
    4753       346422 :   bool use_ifn_p = (DR_IS_READ (dr)
    4754       346422 :                     ? supports_vec_gather_load_p (TYPE_MODE (vectype),
    4755              :                                                   elsvals)
    4756       346422 :                     : supports_vec_scatter_store_p (TYPE_MODE (vectype)));
    4757              : 
    4758       346422 :   base = DR_REF (dr);
    4759              :   /* For masked loads/stores, DR_REF (dr) is an artificial MEM_REF,
    4760              :      see if we can use the def stmt of the address.  */
    4761       346422 :   if (masked_p
    4762         4261 :       && TREE_CODE (base) == MEM_REF
    4763         4261 :       && TREE_CODE (TREE_OPERAND (base, 0)) == SSA_NAME
    4764         4261 :       && integer_zerop (TREE_OPERAND (base, 1))
    4765       350683 :       && !expr_invariant_in_loop_p (loop, TREE_OPERAND (base, 0)))
    4766              :     {
    4767         4261 :       gimple *def_stmt = SSA_NAME_DEF_STMT (TREE_OPERAND (base, 0));
    4768         4261 :       if (is_gimple_assign (def_stmt)
    4769         4261 :           && gimple_assign_rhs_code (def_stmt) == ADDR_EXPR)
    4770          589 :         base = TREE_OPERAND (gimple_assign_rhs1 (def_stmt), 0);
    4771              :     }
    4772              : 
    4773              :   /* The gather and scatter builtins need address of the form
    4774              :      loop_invariant + vector * {1, 2, 4, 8}
    4775              :      or
    4776              :      loop_invariant + sign_extend (vector) * { 1, 2, 4, 8 }.
    4777              :      Unfortunately DR_BASE_ADDRESS/DR_OFFSET can be a mixture
    4778              :      of loop invariants/SSA_NAMEs defined in the loop, with casts,
    4779              :      multiplications and additions in it.  To get a vector, we need
    4780              :      a single SSA_NAME that will be defined in the loop and will
    4781              :      contain everything that is not loop invariant and that can be
    4782              :      vectorized.  The following code attempts to find such a preexistng
    4783              :      SSA_NAME OFF and put the loop invariants into a tree BASE
    4784              :      that can be gimplified before the loop.  */
    4785       346422 :   base = get_inner_reference (base, &pbitsize, &pbitpos, &off, &pmode,
    4786              :                               &punsignedp, &reversep, &pvolatilep);
    4787       346422 :   if (reversep)
    4788              :     return false;
    4789              : 
    4790              :   /* PR 107346.  Packed structs can have fields at offsets that are not
    4791              :      multiples of BITS_PER_UNIT.  Do not use gather/scatters in such cases.  */
    4792       346407 :   if (!multiple_p (pbitpos, BITS_PER_UNIT))
    4793              :     return false;
    4794              : 
    4795              :   /* We need to be able to form an address to the base which for example
    4796              :      isn't possible for hard registers.  */
    4797       346407 :   if (may_be_nonaddressable_p (base))
    4798              :     return false;
    4799              : 
    4800       346399 :   poly_int64 pbytepos = exact_div (pbitpos, BITS_PER_UNIT);
    4801              : 
    4802       346399 :   if (TREE_CODE (base) == MEM_REF)
    4803              :     {
    4804       272346 :       if (!integer_zerop (TREE_OPERAND (base, 1)))
    4805              :         {
    4806        34963 :           if (off == NULL_TREE)
    4807        34670 :             off = wide_int_to_tree (sizetype, mem_ref_offset (base));
    4808              :           else
    4809          293 :             off = size_binop (PLUS_EXPR, off,
    4810              :                               fold_convert (sizetype, TREE_OPERAND (base, 1)));
    4811              :         }
    4812       272346 :       base = TREE_OPERAND (base, 0);
    4813              :     }
    4814              :   else
    4815        74053 :     base = build_fold_addr_expr (base);
    4816              : 
    4817       346399 :   if (off == NULL_TREE)
    4818       211948 :     off = size_zero_node;
    4819              : 
    4820              :   /* BASE must be loop invariant.  If it is not invariant, but OFF is, then we
    4821              :    * can fix that by swapping BASE and OFF.  */
    4822       346399 :   if (!expr_invariant_in_loop_p (loop, base))
    4823              :     {
    4824       247129 :       if (!expr_invariant_in_loop_p (loop, off))
    4825              :         return false;
    4826              : 
    4827       246884 :       std::swap (base, off);
    4828              :     }
    4829              : 
    4830       346154 :   base = fold_convert (sizetype, base);
    4831       346154 :   base = size_binop (PLUS_EXPR, base, size_int (pbytepos));
    4832       346154 :   int tmp_scale;
    4833       346154 :   tree tmp_offset_vectype;
    4834              : 
    4835              :   /* OFF at this point may be either a SSA_NAME or some tree expression
    4836              :      from get_inner_reference.  Try to peel off loop invariants from it
    4837              :      into BASE as long as possible.  */
    4838       346154 :   STRIP_NOPS (off);
    4839       871675 :   while (offtype == NULL_TREE)
    4840              :     {
    4841       768643 :       enum tree_code code;
    4842       768643 :       tree op0, op1, add = NULL_TREE;
    4843              : 
    4844       768643 :       if (TREE_CODE (off) == SSA_NAME)
    4845              :         {
    4846       573331 :           gimple *def_stmt = SSA_NAME_DEF_STMT (off);
    4847              : 
    4848       573331 :           if (expr_invariant_in_loop_p (loop, off))
    4849            0 :             return false;
    4850              : 
    4851       573331 :           if (gimple_code (def_stmt) != GIMPLE_ASSIGN)
    4852              :             break;
    4853              : 
    4854       434734 :           op0 = gimple_assign_rhs1 (def_stmt);
    4855       434734 :           code = gimple_assign_rhs_code (def_stmt);
    4856       434734 :           op1 = gimple_assign_rhs2 (def_stmt);
    4857              :         }
    4858              :       else
    4859              :         {
    4860       195312 :           if (get_gimple_rhs_class (TREE_CODE (off)) == GIMPLE_TERNARY_RHS)
    4861              :             return false;
    4862       195312 :           code = TREE_CODE (off);
    4863       195312 :           extract_ops_from_tree (off, &code, &op0, &op1);
    4864              :         }
    4865       630046 :       switch (code)
    4866              :         {
    4867       185847 :         case POINTER_PLUS_EXPR:
    4868       185847 :         case PLUS_EXPR:
    4869       185847 :           if (expr_invariant_in_loop_p (loop, op0))
    4870              :             {
    4871       114465 :               add = op0;
    4872       114465 :               off = op1;
    4873       169619 :             do_add:
    4874       169619 :               add = fold_convert (sizetype, add);
    4875       169619 :               if (scale != 1)
    4876        50219 :                 add = size_binop (MULT_EXPR, add, size_int (scale));
    4877       169619 :               base = size_binop (PLUS_EXPR, base, add);
    4878       525521 :               continue;
    4879              :             }
    4880        71382 :           if (expr_invariant_in_loop_p (loop, op1))
    4881              :             {
    4882        54860 :               add = op1;
    4883        54860 :               off = op0;
    4884        54860 :               goto do_add;
    4885              :             }
    4886              :           break;
    4887          464 :         case MINUS_EXPR:
    4888          464 :           if (expr_invariant_in_loop_p (loop, op1))
    4889              :             {
    4890          294 :               add = fold_convert (sizetype, op1);
    4891          294 :               add = size_binop (MINUS_EXPR, size_zero_node, add);
    4892          294 :               off = op0;
    4893          294 :               goto do_add;
    4894              :             }
    4895              :           break;
    4896       199776 :         case MULT_EXPR:
    4897       199776 :           if (scale == 1 && tree_fits_shwi_p (op1))
    4898              :             {
    4899       165217 :               int new_scale = tree_to_shwi (op1);
    4900              :               /* Only treat this as a scaling operation if the target
    4901              :                  supports it for at least some offset type.  */
    4902       165217 :               if (use_ifn_p
    4903            0 :                   && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
    4904              :                                                 masked_p, vectype, memory_type,
    4905              :                                                 signed_char_type_node,
    4906              :                                                 new_scale, &tmp_scale,
    4907              :                                                 &ifn,
    4908              :                                                 &offset_vectype,
    4909              :                                                 &tmp_offset_vectype,
    4910              :                                                 elsvals)
    4911       165217 :                   && !vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
    4912              :                                                 masked_p, vectype, memory_type,
    4913              :                                                 unsigned_char_type_node,
    4914              :                                                 new_scale, &tmp_scale,
    4915              :                                                 &ifn,
    4916              :                                                 &offset_vectype,
    4917              :                                                 &tmp_offset_vectype,
    4918              :                                                 elsvals))
    4919              :                 break;
    4920       165217 :               scale = new_scale;
    4921       165217 :               off = op0;
    4922       165217 :               continue;
    4923       165217 :             }
    4924              :           break;
    4925            0 :         case SSA_NAME:
    4926            0 :           off = op0;
    4927            0 :           continue;
    4928       197282 :         CASE_CONVERT:
    4929       394548 :           if (!POINTER_TYPE_P (TREE_TYPE (op0))
    4930       394548 :               && !INTEGRAL_TYPE_P (TREE_TYPE (op0)))
    4931              :             break;
    4932              : 
    4933              :           /* Don't include the conversion if the target is happy with
    4934              :              the current offset type.  */
    4935       197282 :           if (use_ifn_p
    4936            0 :               && TREE_CODE (off) == SSA_NAME
    4937            0 :               && !POINTER_TYPE_P (TREE_TYPE (off))
    4938       197282 :               && vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr),
    4939              :                                            masked_p, vectype, memory_type,
    4940            0 :                                            TREE_TYPE (off),
    4941              :                                            scale, &tmp_scale,
    4942              :                                            &ifn,
    4943              :                                            &offset_vectype,
    4944              :                                            &tmp_offset_vectype,
    4945              :                                            elsvals))
    4946              :             break;
    4947              : 
    4948       197282 :           if (TYPE_PRECISION (TREE_TYPE (op0))
    4949       197282 :               == TYPE_PRECISION (TREE_TYPE (off)))
    4950              :             {
    4951        87653 :               off = op0;
    4952        87653 :               continue;
    4953              :             }
    4954              : 
    4955              :           /* Include the conversion if it is widening and we're using
    4956              :              the IFN path or the target can handle the converted from
    4957              :              offset or the current size is not already the same as the
    4958              :              data vector element size.  */
    4959       109629 :           if ((TYPE_PRECISION (TREE_TYPE (op0))
    4960       109629 :                < TYPE_PRECISION (TREE_TYPE (off)))
    4961       109629 :               && (use_ifn_p
    4962       108761 :                   || (DR_IS_READ (dr)
    4963        79672 :                       ? (targetm.vectorize.builtin_gather
    4964        79672 :                          && targetm.vectorize.builtin_gather (vectype,
    4965        79672 :                                                               TREE_TYPE (op0),
    4966              :                                                               scale))
    4967        29089 :                       : (targetm.vectorize.builtin_scatter
    4968        29089 :                          && targetm.vectorize.builtin_scatter (vectype,
    4969        29089 :                                                                TREE_TYPE (op0),
    4970              :                                                                scale)))
    4971       107685 :                   || !operand_equal_p (TYPE_SIZE (TREE_TYPE (off)),
    4972       107685 :                                        TYPE_SIZE (TREE_TYPE (vectype)), 0)))
    4973              :             {
    4974       103032 :               off = op0;
    4975       103032 :               offtype = TREE_TYPE (off);
    4976       103032 :               STRIP_NOPS (off);
    4977       103032 :               continue;
    4978              :             }
    4979              :           break;
    4980              :         default:
    4981              :           break;
    4982            0 :         }
    4983              :       break;
    4984              :     }
    4985              : 
    4986              :   /* If at the end OFF still isn't a SSA_NAME or isn't
    4987              :      defined in the loop, punt.  */
    4988       346154 :   if (TREE_CODE (off) != SSA_NAME
    4989       346154 :       || expr_invariant_in_loop_p (loop, off))
    4990              :     return false;
    4991              : 
    4992       339414 :   if (offtype == NULL_TREE)
    4993       236736 :     offtype = TREE_TYPE (off);
    4994              : 
    4995       339414 :   if (use_ifn_p)
    4996              :     {
    4997            0 :       if (!vect_gather_scatter_fn_p (loop_vinfo, DR_IS_READ (dr), masked_p,
    4998              :                                      vectype, memory_type, offtype,
    4999              :                                      scale, &tmp_scale,
    5000              :                                      &ifn, &offset_vectype,
    5001              :                                      &tmp_offset_vectype,
    5002              :                                      elsvals))
    5003            0 :         ifn = IFN_LAST;
    5004              :       decl = NULL_TREE;
    5005              :     }
    5006              :   else
    5007              :     {
    5008       339414 :       if (DR_IS_READ (dr))
    5009              :         {
    5010       264539 :           if (targetm.vectorize.builtin_gather)
    5011       264539 :             decl = targetm.vectorize.builtin_gather (vectype, offtype, scale);
    5012              :         }
    5013              :       else
    5014              :         {
    5015        74875 :           if (targetm.vectorize.builtin_scatter)
    5016        74875 :             decl = targetm.vectorize.builtin_scatter (vectype, offtype, scale);
    5017              :         }
    5018       339414 :       ifn = IFN_LAST;
    5019              :       /* The offset vector type will be read from DECL when needed.  */
    5020       339414 :       offset_vectype = NULL_TREE;
    5021              :     }
    5022              : 
    5023       339414 :   gcc_checking_assert (expr_invariant_in_loop_p (loop, base));
    5024       339414 :   gcc_checking_assert (!expr_invariant_in_loop_p (loop, off));
    5025              : 
    5026       339414 :   info->ifn = ifn;
    5027       339414 :   info->decl = decl;
    5028       339414 :   info->base = base;
    5029              : 
    5030       678828 :   info->alias_ptr = build_int_cst
    5031       339414 :     (reference_alias_ptr_type (DR_REF (dr)),
    5032       339414 :      get_object_alignment (DR_REF (dr)));
    5033              : 
    5034       339414 :   info->offset = off;
    5035       339414 :   info->offset_vectype = offset_vectype;
    5036       339414 :   info->scale = scale;
    5037       339414 :   info->element_type = TREE_TYPE (vectype);
    5038       339414 :   info->memory_type = memory_type;
    5039       339414 :   return true;
    5040              : }
    5041              : 
    5042              : /* Find the data references in STMT, analyze them with respect to LOOP and
    5043              :    append them to DATAREFS.  Return false if datarefs in this stmt cannot
    5044              :    be handled.  */
    5045              : 
    5046              : opt_result
    5047     33423485 : vect_find_stmt_data_reference (loop_p loop, gimple *stmt,
    5048              :                                vec<data_reference_p> *datarefs,
    5049              :                                vec<int> *dataref_groups, int group_id)
    5050              : {
    5051              :   /* We can ignore clobbers for dataref analysis - they are removed during
    5052              :      loop vectorization and BB vectorization checks dependences with a
    5053              :      stmt walk.  */
    5054     33423485 :   if (gimple_clobber_p (stmt))
    5055      1198543 :     return opt_result::success ();
    5056              : 
    5057     59986956 :   if (gimple_has_volatile_ops (stmt))
    5058       366662 :     return opt_result::failure_at (stmt, "not vectorized: volatile type: %G",
    5059              :                                    stmt);
    5060              : 
    5061     31858280 :   if (stmt_can_throw_internal (cfun, stmt))
    5062       741460 :     return opt_result::failure_at (stmt,
    5063              :                                    "not vectorized:"
    5064              :                                    " statement can throw an exception: %G",
    5065              :                                    stmt);
    5066              : 
    5067     31116820 :   auto_vec<data_reference_p, 2> refs;
    5068     31116820 :   opt_result res = find_data_references_in_stmt (loop, stmt, &refs, true);
    5069     31116820 :   if (!res)
    5070      3770658 :     return res;
    5071              : 
    5072     27346162 :   if (refs.is_empty ())
    5073     15725339 :     return opt_result::success ();
    5074              : 
    5075     11620823 :   if (refs.length () > 1)
    5076              :     {
    5077      1258999 :       while (!refs.is_empty ())
    5078       839635 :         free_data_ref (refs.pop ());
    5079       419364 :       return opt_result::failure_at (stmt,
    5080              :                                      "not vectorized: more than one "
    5081              :                                      "data ref in stmt: %G", stmt);
    5082              :     }
    5083              : 
    5084     11201459 :   data_reference_p dr = refs.pop ();
    5085     11201459 :   if (gcall *call = dyn_cast <gcall *> (stmt))
    5086        25472 :     if (!gimple_call_internal_p (call)
    5087        25472 :         || (gimple_call_internal_fn (call) != IFN_MASK_LOAD
    5088        22804 :             && gimple_call_internal_fn (call) != IFN_MASK_STORE))
    5089              :       {
    5090        22161 :         free_data_ref (dr);
    5091        22161 :         return opt_result::failure_at (stmt,
    5092              :                                        "not vectorized: dr in a call %G", stmt);
    5093              :       }
    5094              : 
    5095     11179298 :   if (TREE_CODE (DR_REF (dr)) == COMPONENT_REF
    5096     11179298 :       && DECL_BIT_FIELD (TREE_OPERAND (DR_REF (dr), 1)))
    5097              :     {
    5098        57777 :       free_data_ref (dr);
    5099        57777 :       return opt_result::failure_at (stmt,
    5100              :                                      "not vectorized:"
    5101              :                                      " statement is an unsupported"
    5102              :                                      " bitfield access %G", stmt);
    5103              :     }
    5104              : 
    5105     11121521 :   if (DR_BASE_ADDRESS (dr)
    5106     11035280 :       && TREE_CODE (DR_BASE_ADDRESS (dr)) == INTEGER_CST)
    5107              :     {
    5108          998 :       free_data_ref (dr);
    5109          998 :       return opt_result::failure_at (stmt,
    5110              :                                      "not vectorized:"
    5111              :                                      " base addr of dr is a constant\n");
    5112              :     }
    5113              : 
    5114              :   /* Check whether this may be a SIMD lane access and adjust the
    5115              :      DR to make it easier for us to handle it.  */
    5116     11120523 :   if (loop
    5117       620258 :       && loop->simduid
    5118        10707 :       && (!DR_BASE_ADDRESS (dr)
    5119         2963 :           || !DR_OFFSET (dr)
    5120         2963 :           || !DR_INIT (dr)
    5121         2963 :           || !DR_STEP (dr)))
    5122              :     {
    5123         7744 :       struct data_reference *newdr
    5124        15488 :         = create_data_ref (NULL, loop_containing_stmt (stmt), DR_REF (dr), stmt,
    5125              :                            DR_IS_READ (dr), DR_IS_CONDITIONAL_IN_STMT (dr),
    5126         7744 :                            true);
    5127         7744 :       if (DR_BASE_ADDRESS (newdr)
    5128         7744 :           && DR_OFFSET (newdr)
    5129         7744 :           && DR_INIT (newdr)
    5130         7744 :           && DR_STEP (newdr)
    5131         7744 :           && TREE_CODE (DR_INIT (newdr)) == INTEGER_CST
    5132        15488 :           && integer_zerop (DR_STEP (newdr)))
    5133              :         {
    5134         7744 :           tree base_address = DR_BASE_ADDRESS (newdr);
    5135         7744 :           tree off = DR_OFFSET (newdr);
    5136         7744 :           tree step = ssize_int (1);
    5137         7744 :           if (integer_zerop (off)
    5138         7744 :               && TREE_CODE (base_address) == POINTER_PLUS_EXPR)
    5139              :             {
    5140           82 :               off = TREE_OPERAND (base_address, 1);
    5141           82 :               base_address = TREE_OPERAND (base_address, 0);
    5142              :             }
    5143         7744 :           STRIP_NOPS (off);
    5144         7744 :           if (TREE_CODE (off) == MULT_EXPR
    5145         7744 :               && tree_fits_uhwi_p (TREE_OPERAND (off, 1)))
    5146              :             {
    5147         7493 :               step = TREE_OPERAND (off, 1);
    5148         7493 :               off = TREE_OPERAND (off, 0);
    5149         7493 :               STRIP_NOPS (off);
    5150              :             }
    5151          539 :           if (CONVERT_EXPR_P (off)
    5152         7744 :               && (TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (off, 0)))
    5153         7205 :                   < TYPE_PRECISION (TREE_TYPE (off))))
    5154         7205 :             off = TREE_OPERAND (off, 0);
    5155         7744 :           if (TREE_CODE (off) == SSA_NAME)
    5156              :             {
    5157         7221 :               gimple *def = SSA_NAME_DEF_STMT (off);
    5158              :               /* Look through widening conversion.  */
    5159         7221 :               if (is_gimple_assign (def)
    5160         7221 :                   && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (def)))
    5161              :                 {
    5162            0 :                   tree rhs1 = gimple_assign_rhs1 (def);
    5163            0 :                   if (TREE_CODE (rhs1) == SSA_NAME
    5164            0 :                       && INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
    5165            0 :                       && (TYPE_PRECISION (TREE_TYPE (off))
    5166            0 :                           > TYPE_PRECISION (TREE_TYPE (rhs1))))
    5167            0 :                     def = SSA_NAME_DEF_STMT (rhs1);
    5168              :                 }
    5169         7221 :               if (is_gimple_call (def)
    5170         7085 :                   && gimple_call_internal_p (def)
    5171        14306 :                   && (gimple_call_internal_fn (def) == IFN_GOMP_SIMD_LANE))
    5172              :                 {
    5173         7085 :                   tree arg = gimple_call_arg (def, 0);
    5174         7085 :                   tree reft = TREE_TYPE (DR_REF (newdr));
    5175         7085 :                   gcc_assert (TREE_CODE (arg) == SSA_NAME);
    5176         7085 :                   arg = SSA_NAME_VAR (arg);
    5177         7085 :                   if (arg == loop->simduid
    5178              :                       /* For now.  */
    5179         7085 :                       && tree_int_cst_equal (TYPE_SIZE_UNIT (reft), step))
    5180              :                     {
    5181         7060 :                       DR_BASE_ADDRESS (newdr) = base_address;
    5182         7060 :                       DR_OFFSET (newdr) = ssize_int (0);
    5183         7060 :                       DR_STEP (newdr) = step;
    5184         7060 :                       DR_OFFSET_ALIGNMENT (newdr) = BIGGEST_ALIGNMENT;
    5185         7060 :                       DR_STEP_ALIGNMENT (newdr) = highest_pow2_factor (step);
    5186              :                       /* Mark as simd-lane access.  */
    5187         7060 :                       tree arg2 = gimple_call_arg (def, 1);
    5188         7060 :                       newdr->aux = (void *) (-1 - tree_to_uhwi (arg2));
    5189         7060 :                       free_data_ref (dr);
    5190         7060 :                       datarefs->safe_push (newdr);
    5191         7060 :                       if (dataref_groups)
    5192            0 :                         dataref_groups->safe_push (group_id);
    5193         7060 :                       return opt_result::success ();
    5194              :                     }
    5195              :                 }
    5196              :             }
    5197              :         }
    5198          684 :       free_data_ref (newdr);
    5199              :     }
    5200              : 
    5201     11113463 :   datarefs->safe_push (dr);
    5202     11113463 :   if (dataref_groups)
    5203     10500265 :     dataref_groups->safe_push (group_id);
    5204     11113463 :   return opt_result::success ();
    5205     31116820 : }
    5206              : 
    5207              : /* Function vect_analyze_data_refs.
    5208              : 
    5209              :   Find all the data references in the loop or basic block.
    5210              : 
    5211              :    The general structure of the analysis of data refs in the vectorizer is as
    5212              :    follows:
    5213              :    1- vect_analyze_data_refs(loop/bb): call
    5214              :       compute_data_dependences_for_loop/bb to find and analyze all data-refs
    5215              :       in the loop/bb and their dependences.
    5216              :    2- vect_analyze_dependences(): apply dependence testing using ddrs.
    5217              :    3- vect_analyze_drs_alignment(): check that ref_stmt.alignment is ok.
    5218              :    4- vect_analyze_drs_access(): check that ref_stmt.step is ok.
    5219              : 
    5220              : */
    5221              : 
    5222              : opt_result
    5223      2853644 : vect_analyze_data_refs (vec_info *vinfo, bool *fatal)
    5224              : {
    5225      2853644 :   class loop *loop = NULL;
    5226      2853644 :   unsigned int i;
    5227      2853644 :   struct data_reference *dr;
    5228      2853644 :   tree scalar_type;
    5229              : 
    5230      2853644 :   DUMP_VECT_SCOPE ("vect_analyze_data_refs");
    5231              : 
    5232      2853644 :   if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
    5233       540719 :     loop = LOOP_VINFO_LOOP (loop_vinfo);
    5234              : 
    5235              :   /* Go through the data-refs, check that the analysis succeeded.  Update
    5236              :      pointer from stmt_vec_info struct to DR and vectype.  */
    5237              : 
    5238      2853644 :   vec<data_reference_p> datarefs = vinfo->shared->datarefs;
    5239     18945988 :   FOR_EACH_VEC_ELT (datarefs, i, dr)
    5240              :     {
    5241     16168193 :       enum { SG_NONE, GATHER, SCATTER } gatherscatter = SG_NONE;
    5242              : 
    5243     16168193 :       gcc_assert (DR_REF (dr));
    5244     16168193 :       stmt_vec_info stmt_info = vinfo->lookup_stmt (DR_STMT (dr));
    5245     16168193 :       gcc_assert (!stmt_info->dr_aux.dr);
    5246     16168193 :       stmt_info->dr_aux.dr = dr;
    5247     16168193 :       stmt_info->dr_aux.stmt = stmt_info;
    5248              : 
    5249              :       /* Check that analysis of the data-ref succeeded.  */
    5250     16168193 :       if (!DR_BASE_ADDRESS (dr) || !DR_OFFSET (dr) || !DR_INIT (dr)
    5251     16054887 :           || !DR_STEP (dr))
    5252              :         {
    5253       226612 :           bool maybe_gather
    5254       113306 :             = DR_IS_READ (dr)
    5255       113306 :               && !TREE_THIS_VOLATILE (DR_REF (dr));
    5256       226612 :           bool maybe_scatter
    5257              :             = DR_IS_WRITE (dr)
    5258       113306 :               && !TREE_THIS_VOLATILE (DR_REF (dr));
    5259              : 
    5260              :           /* If target supports vector gather loads or scatter stores,
    5261              :              see if they can't be used.  */
    5262       113306 :           if (is_a <loop_vec_info> (vinfo)
    5263       113306 :               && !nested_in_vect_loop_p (loop, stmt_info))
    5264              :             {
    5265       109809 :               if (maybe_gather || maybe_scatter)
    5266              :                 {
    5267       109809 :                   if (maybe_gather)
    5268              :                     gatherscatter = GATHER;
    5269              :                   else
    5270        18801 :                     gatherscatter = SCATTER;
    5271              :                 }
    5272              :             }
    5273              : 
    5274        18801 :           if (gatherscatter == SG_NONE)
    5275              :             {
    5276         3497 :               if (dump_enabled_p ())
    5277            5 :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5278              :                                  "not vectorized: data ref analysis "
    5279              :                                  "failed %G", stmt_info->stmt);
    5280         3497 :               if (is_a <bb_vec_info> (vinfo))
    5281              :                 {
    5282              :                   /* In BB vectorization the ref can still participate
    5283              :                      in dependence analysis, we just can't vectorize it.  */
    5284         3153 :                   STMT_VINFO_VECTORIZABLE (stmt_info) = false;
    5285         3153 :                   continue;
    5286              :                 }
    5287          344 :               return opt_result::failure_at (stmt_info->stmt,
    5288              :                                              "not vectorized:"
    5289              :                                              " data ref analysis failed: %G",
    5290              :                                              stmt_info->stmt);
    5291              :             }
    5292              :         }
    5293              : 
    5294              :       /* See if this was detected as SIMD lane access.  */
    5295     16164696 :       if (dr->aux == (void *)-1
    5296     16164696 :           || dr->aux == (void *)-2
    5297     16155801 :           || dr->aux == (void *)-3
    5298     16154961 :           || dr->aux == (void *)-4)
    5299              :         {
    5300        10535 :           if (nested_in_vect_loop_p (loop, stmt_info))
    5301            0 :             return opt_result::failure_at (stmt_info->stmt,
    5302              :                                            "not vectorized:"
    5303              :                                            " data ref analysis failed: %G",
    5304              :                                            stmt_info->stmt);
    5305        10535 :           STMT_VINFO_SIMD_LANE_ACCESS_P (stmt_info)
    5306        10535 :             = -(uintptr_t) dr->aux;
    5307              :         }
    5308              : 
    5309     16164696 :       tree base = get_base_address (DR_REF (dr));
    5310     16164696 :       if (base && VAR_P (base) && DECL_NONALIASED (base))
    5311              :         {
    5312         9136 :           if (dump_enabled_p ())
    5313          186 :             dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5314              :                              "not vectorized: base object not addressable "
    5315              :                              "for stmt: %G", stmt_info->stmt);
    5316         9136 :           if (is_a <bb_vec_info> (vinfo))
    5317              :             {
    5318              :               /* In BB vectorization the ref can still participate
    5319              :                  in dependence analysis, we just can't vectorize it.  */
    5320         9135 :               STMT_VINFO_VECTORIZABLE (stmt_info) = false;
    5321         9135 :               continue;
    5322              :             }
    5323            1 :           return opt_result::failure_at (stmt_info->stmt,
    5324              :                                          "not vectorized: base object not"
    5325              :                                          " addressable for stmt: %G",
    5326              :                                          stmt_info->stmt);
    5327              :         }
    5328              : 
    5329     16155560 :       if (is_a <loop_vec_info> (vinfo)
    5330      1216359 :           && DR_STEP (dr)
    5331     17262110 :           && TREE_CODE (DR_STEP (dr)) != INTEGER_CST)
    5332              :         {
    5333        45815 :           if (nested_in_vect_loop_p (loop, stmt_info))
    5334          408 :             return opt_result::failure_at (stmt_info->stmt,
    5335              :                                            "not vectorized: "
    5336              :                                            "not suitable for strided load %G",
    5337              :                                            stmt_info->stmt);
    5338        45407 :           STMT_VINFO_STRIDED_P (stmt_info) = true;
    5339              :         }
    5340              : 
    5341              :       /* Update DR field in stmt_vec_info struct.  */
    5342              : 
    5343              :       /* If the dataref is in an inner-loop of the loop that is considered for
    5344              :          for vectorization, we also want to analyze the access relative to
    5345              :          the outer-loop (DR contains information only relative to the
    5346              :          inner-most enclosing loop).  We do that by building a reference to the
    5347              :          first location accessed by the inner-loop, and analyze it relative to
    5348              :          the outer-loop.  */
    5349     16155152 :       if (loop && nested_in_vect_loop_p (loop, stmt_info))
    5350              :         {
    5351              :           /* Build a reference to the first location accessed by the
    5352              :              inner loop: *(BASE + INIT + OFFSET).  By construction,
    5353              :              this address must be invariant in the inner loop, so we
    5354              :              can consider it as being used in the outer loop.  */
    5355        12337 :           tree base = unshare_expr (DR_BASE_ADDRESS (dr));
    5356        12337 :           tree offset = unshare_expr (DR_OFFSET (dr));
    5357        12337 :           tree init = unshare_expr (DR_INIT (dr));
    5358        12337 :           tree init_offset = fold_build2 (PLUS_EXPR, TREE_TYPE (offset),
    5359              :                                           init, offset);
    5360        12337 :           tree init_addr = fold_build_pointer_plus (base, init_offset);
    5361        12337 :           tree init_ref = build2 (MEM_REF, TREE_TYPE (DR_REF (dr)),
    5362              :                                   init_addr, build_zero_cst (ptr_type_node));
    5363              : 
    5364        12337 :           if (dump_enabled_p ())
    5365         1231 :             dump_printf_loc (MSG_NOTE, vect_location,
    5366              :                              "analyze in outer loop: %T\n", init_ref);
    5367              : 
    5368        12337 :           opt_result res
    5369        12337 :             = dr_analyze_innermost (&STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info),
    5370        12337 :                                     init_ref, loop, stmt_info->stmt);
    5371        12337 :           if (!res)
    5372              :             /* dr_analyze_innermost already explained the failure.  */
    5373          165 :             return res;
    5374              : 
    5375        12172 :           if (dump_enabled_p ())
    5376         1227 :             dump_printf_loc (MSG_NOTE, vect_location,
    5377              :                              "\touter base_address: %T\n"
    5378              :                              "\touter offset from base address: %T\n"
    5379              :                              "\touter constant offset from base address: %T\n"
    5380              :                              "\touter step: %T\n"
    5381              :                              "\touter base alignment: %d\n\n"
    5382              :                              "\touter base misalignment: %d\n"
    5383              :                              "\touter offset alignment: %d\n"
    5384              :                              "\touter step alignment: %d\n",
    5385              :                              STMT_VINFO_DR_BASE_ADDRESS (stmt_info),
    5386              :                              STMT_VINFO_DR_OFFSET (stmt_info),
    5387              :                              STMT_VINFO_DR_INIT (stmt_info),
    5388              :                              STMT_VINFO_DR_STEP (stmt_info),
    5389              :                              STMT_VINFO_DR_BASE_ALIGNMENT (stmt_info),
    5390              :                              STMT_VINFO_DR_BASE_MISALIGNMENT (stmt_info),
    5391              :                              STMT_VINFO_DR_OFFSET_ALIGNMENT (stmt_info),
    5392              :                              STMT_VINFO_DR_STEP_ALIGNMENT (stmt_info));
    5393              :         }
    5394              : 
    5395              :       /* Set vectype for STMT.  */
    5396     16154987 :       scalar_type = TREE_TYPE (DR_REF (dr));
    5397     16154987 :       tree vectype = get_vectype_for_scalar_type (vinfo, scalar_type);
    5398     16154987 :       if (!vectype)
    5399              :         {
    5400      2119888 :           if (dump_enabled_p ())
    5401              :             {
    5402         2337 :               dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    5403              :                                "not vectorized: no vectype for stmt: %G",
    5404              :                                stmt_info->stmt);
    5405         2337 :               dump_printf (MSG_MISSED_OPTIMIZATION, " scalar_type: ");
    5406         2337 :               dump_generic_expr (MSG_MISSED_OPTIMIZATION, TDF_DETAILS,
    5407              :                                  scalar_type);
    5408         2337 :               dump_printf (MSG_MISSED_OPTIMIZATION, "\n");
    5409              :             }
    5410              : 
    5411      2119888 :           if (is_a <bb_vec_info> (vinfo))
    5412              :             {
    5413              :               /* No vector type is fine, the ref can still participate
    5414              :                  in dependence analysis, we just can't vectorize it.  */
    5415      2052839 :               STMT_VINFO_VECTORIZABLE (stmt_info) = false;
    5416      2052839 :               continue;
    5417              :             }
    5418        67049 :           if (fatal)
    5419        67049 :             *fatal = false;
    5420        67049 :           return opt_result::failure_at (stmt_info->stmt,
    5421              :                                          "not vectorized:"
    5422              :                                          " no vectype for stmt: %G"
    5423              :                                          " scalar_type: %T\n",
    5424              :                                          stmt_info->stmt, scalar_type);
    5425              :         }
    5426              :       else
    5427              :         {
    5428     14035099 :           if (dump_enabled_p ())
    5429        86682 :             dump_printf_loc (MSG_NOTE, vect_location,
    5430              :                              "got vectype for stmt: %G%T\n",
    5431              :                              stmt_info->stmt, vectype);
    5432              :         }
    5433              : 
    5434              :       /* Leave the BB vectorizer to pick the vector type later, based on
    5435              :          the final dataref group size and SLP node size.  */
    5436     14035099 :       if (is_a <loop_vec_info> (vinfo))
    5437      1148737 :         STMT_VINFO_VECTYPE (stmt_info) = vectype;
    5438              : 
    5439     14035099 :       if (gatherscatter != SG_NONE)
    5440              :         {
    5441       103916 :           gather_scatter_info gs_info;
    5442       103916 :           if (!vect_check_gather_scatter (stmt_info, vectype,
    5443              :                                           as_a <loop_vec_info> (vinfo),
    5444              :                                           &gs_info)
    5445       203746 :               || !get_vectype_for_scalar_type (vinfo,
    5446        99830 :                                                TREE_TYPE (gs_info.offset)))
    5447              :             {
    5448         7882 :               if (fatal)
    5449         7882 :                 *fatal = false;
    5450         7882 :               return opt_result::failure_at
    5451         8569 :                         (stmt_info->stmt,
    5452              :                          (gatherscatter == GATHER)
    5453              :                          ? "not vectorized: not suitable for gather load %G"
    5454              :                          : "not vectorized: not suitable for scatter store %G",
    5455              :                          stmt_info->stmt);
    5456              :             }
    5457        96034 :           STMT_VINFO_GATHER_SCATTER_P (stmt_info) = gatherscatter;
    5458              :         }
    5459              :     }
    5460              : 
    5461              :   /* We used to stop processing and prune the list here.  Verify we no
    5462              :      longer need to.  */
    5463      4447657 :   gcc_assert (i == datarefs.length ());
    5464              : 
    5465      2777795 :   return opt_result::success ();
    5466              : }
    5467              : 
    5468              : 
    5469              : /* Function vect_get_new_vect_var.
    5470              : 
    5471              :    Returns a name for a new variable.  The current naming scheme appends the
    5472              :    prefix "vect_" or "vect_p" (depending on the value of VAR_KIND) to
    5473              :    the name of vectorizer generated variables, and appends that to NAME if
    5474              :    provided.  */
    5475              : 
    5476              : tree
    5477      1949125 : vect_get_new_vect_var (tree type, enum vect_var_kind var_kind, const char *name)
    5478              : {
    5479      1949125 :   const char *prefix;
    5480      1949125 :   tree new_vect_var;
    5481              : 
    5482      1949125 :   switch (var_kind)
    5483              :   {
    5484              :   case vect_simple_var:
    5485              :     prefix = "vect";
    5486              :     break;
    5487        23071 :   case vect_scalar_var:
    5488        23071 :     prefix = "stmp";
    5489        23071 :     break;
    5490        20633 :   case vect_mask_var:
    5491        20633 :     prefix = "mask";
    5492        20633 :     break;
    5493      1403688 :   case vect_pointer_var:
    5494      1403688 :     prefix = "vectp";
    5495      1403688 :     break;
    5496            0 :   default:
    5497            0 :     gcc_unreachable ();
    5498              :   }
    5499              : 
    5500      1949125 :   if (name)
    5501              :     {
    5502      1099945 :       char* tmp = concat (prefix, "_", name, NULL);
    5503      1099945 :       new_vect_var = create_tmp_reg (type, tmp);
    5504      1099945 :       free (tmp);
    5505              :     }
    5506              :   else
    5507       849180 :     new_vect_var = create_tmp_reg (type, prefix);
    5508              : 
    5509      1949125 :   return new_vect_var;
    5510              : }
    5511              : 
    5512              : /* Like vect_get_new_vect_var but return an SSA name.  */
    5513              : 
    5514              : tree
    5515         7223 : vect_get_new_ssa_name (tree type, enum vect_var_kind var_kind, const char *name)
    5516              : {
    5517         7223 :   const char *prefix;
    5518         7223 :   tree new_vect_var;
    5519              : 
    5520         7223 :   switch (var_kind)
    5521              :   {
    5522              :   case vect_simple_var:
    5523              :     prefix = "vect";
    5524              :     break;
    5525          307 :   case vect_scalar_var:
    5526          307 :     prefix = "stmp";
    5527          307 :     break;
    5528            0 :   case vect_pointer_var:
    5529            0 :     prefix = "vectp";
    5530            0 :     break;
    5531            0 :   default:
    5532            0 :     gcc_unreachable ();
    5533              :   }
    5534              : 
    5535         7223 :   if (name)
    5536              :     {
    5537         6749 :       char* tmp = concat (prefix, "_", name, NULL);
    5538         6749 :       new_vect_var = make_temp_ssa_name (type, NULL, tmp);
    5539         6749 :       free (tmp);
    5540              :     }
    5541              :   else
    5542          474 :     new_vect_var = make_temp_ssa_name (type, NULL, prefix);
    5543              : 
    5544         7223 :   return new_vect_var;
    5545              : }
    5546              : 
    5547              : /* Duplicate points-to info on NAME from DR_INFO.  */
    5548              : 
    5549              : static void
    5550       444641 : vect_duplicate_ssa_name_pt_solution (tree name, dr_vec_info *dr_info)
    5551              : {
    5552       444641 :   if (DR_PTR_INFO (dr_info->dr))
    5553       297765 :     duplicate_ssa_name_pt_solution (name, DR_PTR_INFO (dr_info->dr));
    5554       146876 :   else if (!ssa_name_pt_solution (name))
    5555              :     {
    5556       146876 :       tree base = get_base_address (dr_info->dr->ref);
    5557       146876 :       if (VAR_P (base)
    5558              :           || TREE_CODE (base) == PARM_DECL
    5559              :           || TREE_CODE (base) == RESULT_DECL)
    5560              :         {
    5561       134055 :           pt_solution pt;
    5562       134055 :           pt_solution_set_var (&pt, base);
    5563       134055 :           set_ssa_name_pt_solution (name, pt);
    5564              :         }
    5565              :     }
    5566       444641 : }
    5567              : 
    5568              : /* Function vect_create_addr_base_for_vector_ref.
    5569              : 
    5570              :    Create an expression that computes the address of the first memory location
    5571              :    that will be accessed for a data reference.
    5572              : 
    5573              :    Input:
    5574              :    STMT_INFO: The statement containing the data reference.
    5575              :    NEW_STMT_LIST: Must be initialized to NULL_TREE or a statement list.
    5576              :    OFFSET: Optional. If supplied, it is be added to the initial address.
    5577              :    LOOP:    Specify relative to which loop-nest should the address be computed.
    5578              :             For example, when the dataref is in an inner-loop nested in an
    5579              :             outer-loop that is now being vectorized, LOOP can be either the
    5580              :             outer-loop, or the inner-loop.  The first memory location accessed
    5581              :             by the following dataref ('in' points to short):
    5582              : 
    5583              :                 for (i=0; i<N; i++)
    5584              :                    for (j=0; j<M; j++)
    5585              :                      s += in[i+j]
    5586              : 
    5587              :             is as follows:
    5588              :             if LOOP=i_loop:     &in         (relative to i_loop)
    5589              :             if LOOP=j_loop:     &in+i*2B    (relative to j_loop)
    5590              : 
    5591              :    Output:
    5592              :    1. Return an SSA_NAME whose value is the address of the memory location of
    5593              :       the first vector of the data reference.
    5594              :    2. If new_stmt_list is not NULL_TREE after return then the caller must insert
    5595              :       these statement(s) which define the returned SSA_NAME.
    5596              : 
    5597              :    FORNOW: We are only handling array accesses with step 1.  */
    5598              : 
    5599              : tree
    5600       692927 : vect_create_addr_base_for_vector_ref (vec_info *vinfo, stmt_vec_info stmt_info,
    5601              :                                       gimple_seq *new_stmt_list,
    5602              :                                       tree offset)
    5603              : {
    5604       692927 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
    5605       692927 :   struct data_reference *dr = dr_info->dr;
    5606       692927 :   const char *base_name;
    5607       692927 :   tree addr_base;
    5608       692927 :   tree dest;
    5609       692927 :   gimple_seq seq = NULL;
    5610       692927 :   tree vect_ptr_type;
    5611       692927 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    5612       692927 :   innermost_loop_behavior *drb = vect_dr_behavior (vinfo, dr_info);
    5613              : 
    5614       692927 :   tree data_ref_base = unshare_expr (drb->base_address);
    5615       692927 :   tree vector_offset = NULL_TREE;
    5616       692927 :   if (loop_vinfo && dr_info->offset)
    5617        18184 :     vector_offset = unshare_expr (dr_info->offset);
    5618       692927 :   tree base_offset = unshare_expr (vector_offset
    5619              :                                    ? drb->offset
    5620       674743 :                                    : get_dr_vinfo_offset (vinfo, dr_info,
    5621              :                                                           true));
    5622       692927 :   tree init = unshare_expr (drb->init);
    5623              : 
    5624       692927 :   if (loop_vinfo)
    5625       133507 :     base_name = get_name (data_ref_base);
    5626              :   else
    5627              :     {
    5628       559420 :       base_offset = ssize_int (0);
    5629       559420 :       init = ssize_int (0);
    5630       559420 :       base_name = get_name (DR_REF (dr));
    5631              :     }
    5632              : 
    5633              :   /* Create base_offset */
    5634       692927 :   base_offset = size_binop (PLUS_EXPR,
    5635              :                             fold_convert (sizetype, base_offset),
    5636              :                             fold_convert (sizetype, init));
    5637              : 
    5638       692927 :   if (offset)
    5639              :     {
    5640         3249 :       offset = fold_convert (sizetype, offset);
    5641         3249 :       base_offset = fold_build2 (PLUS_EXPR, sizetype,
    5642              :                                  base_offset, offset);
    5643              :     }
    5644              : 
    5645              :   /* base + base_offset */
    5646       692927 :   if (loop_vinfo)
    5647       133507 :     addr_base = fold_build_pointer_plus (data_ref_base, base_offset);
    5648              :   else
    5649      1118840 :     addr_base = build1 (ADDR_EXPR,
    5650       559420 :                         build_pointer_type (TREE_TYPE (DR_REF (dr))),
    5651              :                         /* Strip zero offset components since we don't need
    5652              :                            them and they can confuse late diagnostics if
    5653              :                            we CSE them wrongly.  See PR106904 for example.  */
    5654              :                         unshare_expr (strip_zero_offset_components
    5655              :                                                                 (DR_REF (dr))));
    5656              : 
    5657       692927 :   vect_ptr_type = build_pointer_type (TREE_TYPE (DR_REF (dr)));
    5658       692927 :   dest = vect_get_new_vect_var (vect_ptr_type, vect_pointer_var, base_name);
    5659              : 
    5660              :   /* Keep vectorizer-added offsets separate from the original scalar access
    5661              :      address.  Forming "base + scalar offset" first gives the target a better
    5662              :      chance of sharing it with other address calculations, such as the
    5663              :      misalignment check used for masked alignment peeling.  */
    5664       692927 :   if (vector_offset)
    5665              :     {
    5666        18184 :       tree scalar_dest = vect_get_new_vect_var (vect_ptr_type,
    5667              :                                                 vect_pointer_var, base_name);
    5668        18184 :       gimple_seq addr_seq = NULL;
    5669        18184 :       addr_base = force_gimple_operand (addr_base, &addr_seq, true,
    5670              :                                         scalar_dest);
    5671        18184 :       gimple_seq_add_seq (&seq, addr_seq);
    5672        18184 :       addr_base = fold_build_pointer_plus (addr_base,
    5673              :                                            fold_convert (sizetype,
    5674              :                                                          vector_offset));
    5675              :     }
    5676              : 
    5677       692927 :   gimple_seq addr_seq = NULL;
    5678       692927 :   addr_base = force_gimple_operand (addr_base, &addr_seq, true, dest);
    5679       692927 :   gimple_seq_add_seq (&seq, addr_seq);
    5680       692927 :   gimple_seq_add_seq (new_stmt_list, seq);
    5681              : 
    5682       692927 :   if (TREE_CODE (addr_base) == SSA_NAME
    5683              :       /* We should only duplicate pointer info to newly created SSA names.  */
    5684       700135 :       && SSA_NAME_VAR (addr_base) == dest)
    5685              :     {
    5686       177663 :       gcc_assert (!ssa_name_pt_solution (addr_base));
    5687       177663 :       vect_duplicate_ssa_name_pt_solution (addr_base, dr_info);
    5688              :     }
    5689              : 
    5690       692927 :   if (dump_enabled_p ())
    5691        26465 :     dump_printf_loc (MSG_NOTE, vect_location, "created %T\n", addr_base);
    5692              : 
    5693       692927 :   return addr_base;
    5694              : }
    5695              : 
    5696              : 
    5697              : /* Function vect_create_data_ref_ptr.
    5698              : 
    5699              :    Create a new pointer-to-AGGR_TYPE variable (ap), that points to the first
    5700              :    location accessed in the loop by STMT_INFO, along with the def-use update
    5701              :    chain to appropriately advance the pointer through the loop iterations.
    5702              :    Also set aliasing information for the pointer.  This pointer is used by
    5703              :    the callers to this function to create a memory reference expression for
    5704              :    vector load/store access.
    5705              : 
    5706              :    Input:
    5707              :    1. STMT_INFO: a stmt that references memory. Expected to be of the form
    5708              :          GIMPLE_ASSIGN <name, data-ref> or
    5709              :          GIMPLE_ASSIGN <data-ref, name>.
    5710              :    2. AGGR_TYPE: the type of the reference, which should be either a vector
    5711              :         or an array.
    5712              :    3. AT_LOOP: the loop where the vector memref is to be created.
    5713              :    4. OFFSET (optional): a byte offset to be added to the initial address
    5714              :         accessed by the data-ref in STMT_INFO.
    5715              :    5. BSI: location where the new stmts are to be placed if there is no loop
    5716              :    6. ONLY_INIT: indicate if ap is to be updated in the loop, or remain
    5717              :         pointing to the initial address.
    5718              :    8. IV_STEP (optional, defaults to NULL): the amount that should be added
    5719              :         to the IV during each iteration of the loop.  NULL says to move
    5720              :         by one copy of AGGR_TYPE up or down, depending on the step of the
    5721              :         data reference.
    5722              : 
    5723              :    Output:
    5724              :    1. Declare a new ptr to vector_type, and have it point to the base of the
    5725              :       data reference (initial addressed accessed by the data reference).
    5726              :       For example, for vector of type V8HI, the following code is generated:
    5727              : 
    5728              :       v8hi *ap;
    5729              :       ap = (v8hi *)initial_address;
    5730              : 
    5731              :       if OFFSET is not supplied:
    5732              :          initial_address = &a[init];
    5733              :       if OFFSET is supplied:
    5734              :          initial_address = &a[init] + OFFSET;
    5735              :       if BYTE_OFFSET is supplied:
    5736              :          initial_address = &a[init] + BYTE_OFFSET;
    5737              : 
    5738              :       Return the initial_address in INITIAL_ADDRESS.
    5739              : 
    5740              :    2. If ONLY_INIT is true, just return the initial pointer.  Otherwise, also
    5741              :       update the pointer in each iteration of the loop.
    5742              : 
    5743              :       Return the increment stmt that updates the pointer in PTR_INCR.
    5744              : 
    5745              :    3. Return the pointer.  */
    5746              : 
    5747              : tree
    5748       692577 : vect_create_data_ref_ptr (vec_info *vinfo, stmt_vec_info stmt_info,
    5749              :                           tree aggr_type, class loop *at_loop, tree offset,
    5750              :                           tree *initial_address, gimple_stmt_iterator *gsi,
    5751              :                           gimple **ptr_incr, bool only_init,
    5752              :                           tree iv_step)
    5753              : {
    5754       692577 :   const char *base_name;
    5755       692577 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    5756       692577 :   class loop *loop = NULL;
    5757       692577 :   bool nested_in_vect_loop = false;
    5758       692577 :   class loop *containing_loop = NULL;
    5759       692577 :   tree aggr_ptr_type;
    5760       692577 :   tree aggr_ptr;
    5761       692577 :   tree new_temp;
    5762       692577 :   gimple_seq new_stmt_list = NULL;
    5763       692577 :   edge pe = NULL;
    5764       692577 :   basic_block new_bb;
    5765       692577 :   tree aggr_ptr_init;
    5766       692577 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
    5767       692577 :   struct data_reference *dr = dr_info->dr;
    5768       692577 :   tree aptr;
    5769       692577 :   gimple_stmt_iterator incr_gsi;
    5770       692577 :   bool insert_after;
    5771       692577 :   tree indx_before_incr, indx_after_incr;
    5772       692577 :   gimple *incr;
    5773       692577 :   bb_vec_info bb_vinfo = dyn_cast <bb_vec_info> (vinfo);
    5774              : 
    5775       692577 :   gcc_assert (iv_step != NULL_TREE
    5776              :               || TREE_CODE (aggr_type) == ARRAY_TYPE
    5777              :               || TREE_CODE (aggr_type) == VECTOR_TYPE);
    5778              : 
    5779       692577 :   if (loop_vinfo)
    5780              :     {
    5781       133157 :       loop = LOOP_VINFO_LOOP (loop_vinfo);
    5782       133157 :       nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
    5783       133157 :       containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
    5784       133157 :       pe = loop_preheader_edge (loop);
    5785              :     }
    5786              :   else
    5787              :     {
    5788       559420 :       gcc_assert (bb_vinfo);
    5789       559420 :       only_init = true;
    5790       559420 :       if (ptr_incr)
    5791            0 :         *ptr_incr = NULL;
    5792              :     }
    5793              : 
    5794              :   /* Create an expression for the first address accessed by this load
    5795              :      in LOOP.  */
    5796       692577 :   base_name = get_name (DR_BASE_ADDRESS (dr));
    5797              : 
    5798       692577 :   if (dump_enabled_p ())
    5799              :     {
    5800        26280 :       tree dr_base_type = TREE_TYPE (DR_BASE_OBJECT (dr));
    5801        26280 :       dump_printf_loc (MSG_NOTE, vect_location,
    5802              :                        "create %s-pointer variable to type: %T",
    5803        26280 :                        get_tree_code_name (TREE_CODE (aggr_type)),
    5804              :                        aggr_type);
    5805        26280 :       if (TREE_CODE (dr_base_type) == ARRAY_TYPE)
    5806        13792 :         dump_printf (MSG_NOTE, "  vectorizing an array ref: ");
    5807        12488 :       else if (TREE_CODE (dr_base_type) == VECTOR_TYPE)
    5808            0 :         dump_printf (MSG_NOTE, "  vectorizing a vector ref: ");
    5809        12488 :       else if (TREE_CODE (dr_base_type) == RECORD_TYPE)
    5810         1672 :         dump_printf (MSG_NOTE, "  vectorizing a record based array ref: ");
    5811              :       else
    5812        10816 :         dump_printf (MSG_NOTE, "  vectorizing a pointer ref: ");
    5813        26280 :       dump_printf (MSG_NOTE, "%T\n", DR_BASE_OBJECT (dr));
    5814              :     }
    5815              : 
    5816              :   /* (1) Create the new aggregate-pointer variable.
    5817              :      Vector and array types inherit the alias set of their component
    5818              :      type by default so we need to use a ref-all pointer if the data
    5819              :      reference does not conflict with the created aggregated data
    5820              :      reference because it is not addressable.  */
    5821       692577 :   bool need_ref_all = false;
    5822       692577 :   if (!alias_sets_conflict_p (get_alias_set (aggr_type),
    5823              :                               get_alias_set (DR_REF (dr))))
    5824              :     need_ref_all = true;
    5825              :   /* Likewise for any of the data references in the stmt group.  */
    5826       590657 :   else if (DR_GROUP_SIZE (stmt_info) > 1)
    5827              :     {
    5828       473497 :       stmt_vec_info sinfo = DR_GROUP_FIRST_ELEMENT (stmt_info);
    5829      1324594 :       do
    5830              :         {
    5831      1324594 :           struct data_reference *sdr = STMT_VINFO_DATA_REF (sinfo);
    5832      1324594 :           if (!alias_sets_conflict_p (get_alias_set (aggr_type),
    5833              :                                       get_alias_set (DR_REF (sdr))))
    5834              :             {
    5835              :               need_ref_all = true;
    5836              :               break;
    5837              :             }
    5838      1298819 :           sinfo = DR_GROUP_NEXT_ELEMENT (sinfo);
    5839              :         }
    5840      1298819 :       while (sinfo);
    5841              :     }
    5842       692577 :   aggr_ptr_type = build_pointer_type_for_mode (aggr_type, VOIDmode,
    5843              :                                                need_ref_all);
    5844       692577 :   aggr_ptr = vect_get_new_vect_var (aggr_ptr_type, vect_pointer_var, base_name);
    5845              : 
    5846              : 
    5847              :   /* Note: If the dataref is in an inner-loop nested in LOOP, and we are
    5848              :      vectorizing LOOP (i.e., outer-loop vectorization), we need to create two
    5849              :      def-use update cycles for the pointer: one relative to the outer-loop
    5850              :      (LOOP), which is what steps (3) and (4) below do.  The other is relative
    5851              :      to the inner-loop (which is the inner-most loop containing the dataref),
    5852              :      and this is done be step (5) below.
    5853              : 
    5854              :      When vectorizing inner-most loops, the vectorized loop (LOOP) is also the
    5855              :      inner-most loop, and so steps (3),(4) work the same, and step (5) is
    5856              :      redundant.  Steps (3),(4) create the following:
    5857              : 
    5858              :         vp0 = &base_addr;
    5859              :         LOOP:   vp1 = phi(vp0,vp2)
    5860              :                 ...
    5861              :                 ...
    5862              :                 vp2 = vp1 + step
    5863              :                 goto LOOP
    5864              : 
    5865              :      If there is an inner-loop nested in loop, then step (5) will also be
    5866              :      applied, and an additional update in the inner-loop will be created:
    5867              : 
    5868              :         vp0 = &base_addr;
    5869              :         LOOP:   vp1 = phi(vp0,vp2)
    5870              :                 ...
    5871              :         inner:     vp3 = phi(vp1,vp4)
    5872              :                    vp4 = vp3 + inner_step
    5873              :                    if () goto inner
    5874              :                 ...
    5875              :                 vp2 = vp1 + step
    5876              :                 if () goto LOOP   */
    5877              : 
    5878              :   /* (2) Calculate the initial address of the aggregate-pointer, and set
    5879              :      the aggregate-pointer to point to it before the loop.  */
    5880              : 
    5881              :   /* Create: (&(base[init_val]+offset) in the loop preheader.  */
    5882              : 
    5883       692577 :   new_temp = vect_create_addr_base_for_vector_ref (vinfo,
    5884              :                                                    stmt_info, &new_stmt_list,
    5885              :                                                    offset);
    5886       692577 :   if (new_stmt_list)
    5887              :     {
    5888       177491 :       if (pe)
    5889              :         {
    5890        56283 :           new_bb = gsi_insert_seq_on_edge_immediate (pe, new_stmt_list);
    5891        56283 :           gcc_assert (!new_bb);
    5892              :         }
    5893              :       else
    5894       121208 :         gsi_insert_seq_before (gsi, new_stmt_list, GSI_SAME_STMT);
    5895              :     }
    5896              : 
    5897       692577 :   *initial_address = new_temp;
    5898       692577 :   aggr_ptr_init = new_temp;
    5899              : 
    5900              :   /* (3) Handle the updating of the aggregate-pointer inside the loop.
    5901              :      This is needed when ONLY_INIT is false, and also when AT_LOOP is the
    5902              :      inner-loop nested in LOOP (during outer-loop vectorization).  */
    5903              : 
    5904              :   /* No update in loop is required.  */
    5905       692577 :   if (only_init && (!loop_vinfo || at_loop == loop))
    5906              :     aptr = aggr_ptr_init;
    5907              :   else
    5908              :     {
    5909              :       /* Accesses to invariant addresses should be handled specially
    5910              :          by the caller.  */
    5911       133149 :       tree step = vect_dr_behavior (vinfo, dr_info)->step;
    5912       133149 :       gcc_assert (!integer_zerop (step));
    5913              : 
    5914       133149 :       if (iv_step == NULL_TREE)
    5915              :         {
    5916              :           /* The step of the aggregate pointer is the type size,
    5917              :              negated for downward accesses.  */
    5918            0 :           iv_step = TYPE_SIZE_UNIT (aggr_type);
    5919            0 :           if (tree_int_cst_sgn (step) == -1)
    5920            0 :             iv_step = fold_build1 (NEGATE_EXPR, TREE_TYPE (iv_step), iv_step);
    5921              :         }
    5922              : 
    5923       133149 :       standard_iv_increment_position (loop, &incr_gsi, &insert_after);
    5924              : 
    5925       266298 :       create_iv (aggr_ptr_init, PLUS_EXPR,
    5926              :                  iv_step, aggr_ptr, loop, &incr_gsi, insert_after,
    5927              :                  &indx_before_incr, &indx_after_incr,
    5928              :                  !loop_vinfo
    5929       133149 :                    || LOOP_VINFO_IV_INCREMENT_INVARIANT_P (loop_vinfo));
    5930       133149 :       incr = gsi_stmt (incr_gsi);
    5931              : 
    5932              :       /* Copy the points-to information if it exists. */
    5933       133149 :       vect_duplicate_ssa_name_pt_solution (indx_before_incr, dr_info);
    5934       133149 :       vect_duplicate_ssa_name_pt_solution (indx_after_incr, dr_info);
    5935       133149 :       if (ptr_incr)
    5936            0 :         *ptr_incr = incr;
    5937              : 
    5938       133149 :       aptr = indx_before_incr;
    5939              :     }
    5940              : 
    5941       692577 :   if (!nested_in_vect_loop || only_init)
    5942              :     return aptr;
    5943              : 
    5944              : 
    5945              :   /* (4) Handle the updating of the aggregate-pointer inside the inner-loop
    5946              :      nested in LOOP, if exists.  */
    5947              : 
    5948          340 :   gcc_assert (nested_in_vect_loop);
    5949          340 :   if (!only_init)
    5950              :     {
    5951          340 :       standard_iv_increment_position (containing_loop, &incr_gsi,
    5952              :                                       &insert_after);
    5953          340 :       create_iv (aptr, PLUS_EXPR, DR_STEP (dr),
    5954              :                  aggr_ptr, containing_loop, &incr_gsi, insert_after,
    5955              :                  &indx_before_incr, &indx_after_incr);
    5956          340 :       incr = gsi_stmt (incr_gsi);
    5957              : 
    5958              :       /* Copy the points-to information if it exists. */
    5959          340 :       vect_duplicate_ssa_name_pt_solution (indx_before_incr, dr_info);
    5960          340 :       vect_duplicate_ssa_name_pt_solution (indx_after_incr, dr_info);
    5961          340 :       if (ptr_incr)
    5962            0 :         *ptr_incr = incr;
    5963              : 
    5964          340 :       return indx_before_incr;
    5965              :     }
    5966              :   else
    5967              :     gcc_unreachable ();
    5968              : }
    5969              : 
    5970              : 
    5971              : /* Function bump_vector_ptr
    5972              : 
    5973              :    Increment DATAREF_PTR by UPDATE.
    5974              : 
    5975              :    Input:
    5976              :    DATAREF_PTR - ssa_name of a pointer (to vector type) that is being updated
    5977              :                  in the loop.
    5978              :    GSI - location where the new update stmt is to be placed.
    5979              :    STMT_INFO - the original scalar memory-access stmt that is being vectorized.
    5980              :    UPDATE - The offset by which to bump the pointer.
    5981              : 
    5982              :    Output: Return NEW_DATAREF_PTR as illustrated above.
    5983              : 
    5984              : */
    5985              : 
    5986              : tree
    5987       236104 : bump_vector_ptr (vec_info *vinfo,
    5988              :                  tree dataref_ptr, gimple_stmt_iterator *gsi,
    5989              :                  stmt_vec_info stmt_info, tree update)
    5990              : {
    5991       236104 :   struct data_reference *dr = STMT_VINFO_DATA_REF (stmt_info);
    5992       236104 :   gimple *incr_stmt;
    5993       236104 :   tree new_dataref_ptr;
    5994              : 
    5995       236104 :   if (TREE_CODE (dataref_ptr) == SSA_NAME)
    5996       114055 :     new_dataref_ptr = copy_ssa_name (dataref_ptr);
    5997       122049 :   else if (is_gimple_min_invariant (dataref_ptr))
    5998              :     /* When possible avoid emitting a separate increment stmt that will
    5999              :        force the addressed object addressable.  */
    6000       244098 :     return build1 (ADDR_EXPR, TREE_TYPE (dataref_ptr),
    6001       122049 :                    fold_build2 (MEM_REF,
    6002              :                                 TREE_TYPE (TREE_TYPE (dataref_ptr)),
    6003              :                                 dataref_ptr,
    6004       122049 :                                 fold_convert (ptr_type_node, update)));
    6005              :   else
    6006            0 :     new_dataref_ptr = make_ssa_name (TREE_TYPE (dataref_ptr));
    6007       114055 :   incr_stmt = gimple_build_assign (new_dataref_ptr, POINTER_PLUS_EXPR,
    6008              :                                    dataref_ptr, update);
    6009       114055 :   vect_finish_stmt_generation (vinfo, stmt_info, incr_stmt, gsi);
    6010              :   /* Fold the increment, avoiding excessive chains use-def chains of
    6011              :      those, leading to compile-time issues for passes until the next
    6012              :      forwprop pass which would do this as well.  */
    6013       114055 :   gimple_stmt_iterator fold_gsi = gsi_for_stmt (incr_stmt);
    6014       114055 :   if (fold_stmt (&fold_gsi, follow_all_ssa_edges))
    6015              :     {
    6016        73881 :       incr_stmt = gsi_stmt (fold_gsi);
    6017        73881 :       update_stmt (incr_stmt);
    6018              :     }
    6019              : 
    6020              :   /* Copy the points-to information if it exists. */
    6021       114055 :   duplicate_ssa_name_pt_solution (new_dataref_ptr, DR_PTR_INFO (dr));
    6022              : 
    6023       114055 :   return new_dataref_ptr;
    6024              : }
    6025              : 
    6026              : 
    6027              : /* Copy memory reference info such as base/clique from the SRC reference
    6028              :    to the DEST MEM_REF.  */
    6029              : 
    6030              : void
    6031       944096 : vect_copy_ref_info (tree dest, tree src)
    6032              : {
    6033       944096 :   if (TREE_CODE (dest) != MEM_REF)
    6034              :     return;
    6035              : 
    6036              :   tree src_base = src;
    6037      1912176 :   while (handled_component_p (src_base))
    6038       972734 :     src_base = TREE_OPERAND (src_base, 0);
    6039       939442 :   if (TREE_CODE (src_base) != MEM_REF
    6040       939442 :       && TREE_CODE (src_base) != TARGET_MEM_REF)
    6041              :     return;
    6042              : 
    6043       508999 :   MR_DEPENDENCE_CLIQUE (dest) = MR_DEPENDENCE_CLIQUE (src_base);
    6044       508999 :   MR_DEPENDENCE_BASE (dest) = MR_DEPENDENCE_BASE (src_base);
    6045              : }
    6046              : 
    6047              : 
    6048              : /* Function vect_create_destination_var.
    6049              : 
    6050              :    Create a new temporary of type VECTYPE.  */
    6051              : 
    6052              : tree
    6053       527795 : vect_create_destination_var (tree scalar_dest, tree vectype)
    6054              : {
    6055       527795 :   tree vec_dest;
    6056       527795 :   const char *name;
    6057       527795 :   char *new_name;
    6058       527795 :   tree type;
    6059       527795 :   enum vect_var_kind kind;
    6060              : 
    6061       527795 :   kind = vectype
    6062      1032519 :     ? VECTOR_BOOLEAN_TYPE_P (vectype)
    6063       504724 :     ? vect_mask_var
    6064              :     : vect_simple_var
    6065              :     : vect_scalar_var;
    6066        23071 :   type = vectype ? vectype : TREE_TYPE (scalar_dest);
    6067              : 
    6068       527795 :   gcc_assert (TREE_CODE (scalar_dest) == SSA_NAME);
    6069              : 
    6070       527795 :   name = get_name (scalar_dest);
    6071       527795 :   if (name)
    6072       177116 :     new_name = xasprintf ("%s_%u", name, SSA_NAME_VERSION (scalar_dest));
    6073              :   else
    6074       350679 :     new_name = xasprintf ("_%u", SSA_NAME_VERSION (scalar_dest));
    6075       527795 :   vec_dest = vect_get_new_vect_var (type, kind, new_name);
    6076       527795 :   free (new_name);
    6077              : 
    6078       527795 :   return vec_dest;
    6079              : }
    6080              : 
    6081              : /* Function vect_grouped_store_supported.
    6082              : 
    6083              :    Returns TRUE if interleave high and interleave low permutations
    6084              :    are supported, and FALSE otherwise.  */
    6085              : 
    6086              : bool
    6087         4589 : vect_grouped_store_supported (tree vectype, unsigned HOST_WIDE_INT count)
    6088              : {
    6089         4589 :   machine_mode mode = TYPE_MODE (vectype);
    6090              : 
    6091              :   /* vect_permute_store_chain requires the group size to be equal to 3 or
    6092              :      be a power of two.  */
    6093         4589 :   if (count != 3 && exact_log2 (count) == -1)
    6094              :     {
    6095          551 :       if (dump_enabled_p ())
    6096           15 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6097              :                          "the size of the group of accesses"
    6098              :                          " is not a power of 2 or not equal to 3\n");
    6099              :       return false;
    6100              :     }
    6101              : 
    6102              :   /* Check that the permutation is supported.  */
    6103         4038 :   if (VECTOR_MODE_P (mode))
    6104              :     {
    6105         4038 :       unsigned int i;
    6106         4038 :       if (count == 3)
    6107              :         {
    6108          950 :           unsigned int j0 = 0, j1 = 0, j2 = 0;
    6109          950 :           unsigned int i, j;
    6110              : 
    6111          950 :           unsigned int nelt;
    6112         1900 :           if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
    6113              :             {
    6114              :               if (dump_enabled_p ())
    6115              :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6116              :                                  "cannot handle groups of 3 stores for"
    6117              :                                  " variable-length vectors\n");
    6118              :               return false;
    6119              :             }
    6120              : 
    6121          950 :           vec_perm_builder sel (nelt, nelt, 1);
    6122          950 :           sel.quick_grow (nelt);
    6123          950 :           vec_perm_indices indices;
    6124         3575 :           for (j = 0; j < 3; j++)
    6125              :             {
    6126         2700 :               int nelt0 = ((3 - j) * nelt) % 3;
    6127         2700 :               int nelt1 = ((3 - j) * nelt + 1) % 3;
    6128         2700 :               int nelt2 = ((3 - j) * nelt + 2) % 3;
    6129         9606 :               for (i = 0; i < nelt; i++)
    6130              :                 {
    6131         6906 :                   if (3 * i + nelt0 < nelt)
    6132         2340 :                     sel[3 * i + nelt0] = j0++;
    6133         6906 :                   if (3 * i + nelt1 < nelt)
    6134         2301 :                     sel[3 * i + nelt1] = nelt + j1++;
    6135         6906 :                   if (3 * i + nelt2 < nelt)
    6136         2265 :                     sel[3 * i + nelt2] = 0;
    6137              :                 }
    6138         2700 :               indices.new_vector (sel, 2, nelt);
    6139         2700 :               if (!can_vec_perm_const_p (mode, mode, indices))
    6140              :                 {
    6141           66 :                   if (dump_enabled_p ())
    6142           37 :                     dump_printf (MSG_MISSED_OPTIMIZATION,
    6143              :                                  "permutation op not supported by target.\n");
    6144              :                   return false;
    6145              :                 }
    6146              : 
    6147         8964 :               for (i = 0; i < nelt; i++)
    6148              :                 {
    6149         6330 :                   if (3 * i + nelt0 < nelt)
    6150         2116 :                     sel[3 * i + nelt0] = 3 * i + nelt0;
    6151         6330 :                   if (3 * i + nelt1 < nelt)
    6152         2107 :                     sel[3 * i + nelt1] = 3 * i + nelt1;
    6153         6330 :                   if (3 * i + nelt2 < nelt)
    6154         2107 :                     sel[3 * i + nelt2] = nelt + j2++;
    6155              :                 }
    6156         2634 :               indices.new_vector (sel, 2, nelt);
    6157         2634 :               if (!can_vec_perm_const_p (mode, mode, indices))
    6158              :                 {
    6159            9 :                   if (dump_enabled_p ())
    6160            9 :                     dump_printf (MSG_MISSED_OPTIMIZATION,
    6161              :                                  "permutation op not supported by target.\n");
    6162              :                   return false;
    6163              :                 }
    6164              :             }
    6165              :           return true;
    6166          950 :         }
    6167              :       else
    6168              :         {
    6169              :           /* If length is not equal to 3 then only power of 2 is supported.  */
    6170         3088 :           gcc_assert (pow2p_hwi (count));
    6171         6176 :           poly_uint64 nelt = GET_MODE_NUNITS (mode);
    6172              : 
    6173              :           /* The encoding has 2 interleaved stepped patterns.  */
    6174         6176 :           if(!multiple_p (nelt, 2))
    6175         3042 :             return false;
    6176         3088 :           vec_perm_builder sel (nelt, 2, 3);
    6177         3088 :           sel.quick_grow (6);
    6178        15440 :           for (i = 0; i < 3; i++)
    6179              :             {
    6180         9264 :               sel[i * 2] = i;
    6181         9264 :               sel[i * 2 + 1] = i + nelt;
    6182              :             }
    6183         3088 :           vec_perm_indices indices (sel, 2, nelt);
    6184         3088 :           if (can_vec_perm_const_p (mode, mode, indices))
    6185              :             {
    6186        21294 :               for (i = 0; i < 6; i++)
    6187        18252 :                 sel[i] += exact_div (nelt, 2);
    6188         3042 :               indices.new_vector (sel, 2, nelt);
    6189         3042 :               if (can_vec_perm_const_p (mode, mode, indices))
    6190         3042 :                 return true;
    6191              :             }
    6192         3088 :         }
    6193              :     }
    6194              : 
    6195           46 :   if (dump_enabled_p ())
    6196            3 :     dump_printf (MSG_MISSED_OPTIMIZATION,
    6197              :                  "permutation op not supported by target.\n");
    6198              :   return false;
    6199              : }
    6200              : 
    6201              : /* Return FN if vec_{mask_,mask_len_}store_lanes is available for COUNT vectors
    6202              :    of type VECTYPE.  MASKED_P says whether the masked form is needed.  */
    6203              : 
    6204              : internal_fn
    6205        47301 : vect_store_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
    6206              :                             bool masked_p)
    6207              : {
    6208        47301 :   if (vect_lanes_optab_supported_p ("vec_mask_len_store_lanes",
    6209              :                                     vec_mask_len_store_lanes_optab, vectype,
    6210              :                                     count))
    6211              :     return IFN_MASK_LEN_STORE_LANES;
    6212        47301 :   else if (masked_p)
    6213              :     {
    6214          111 :       if (vect_lanes_optab_supported_p ("vec_mask_store_lanes",
    6215              :                                         vec_mask_store_lanes_optab, vectype,
    6216              :                                         count))
    6217            0 :         return IFN_MASK_STORE_LANES;
    6218              :     }
    6219              :   else
    6220              :     {
    6221        47190 :       if (vect_lanes_optab_supported_p ("vec_store_lanes",
    6222              :                                         vec_store_lanes_optab, vectype, count))
    6223            0 :         return IFN_STORE_LANES;
    6224              :     }
    6225              :   return IFN_LAST;
    6226              : }
    6227              : 
    6228              : 
    6229              : /* Function vect_setup_realignment
    6230              : 
    6231              :    This function is called when vectorizing an unaligned load using
    6232              :    the dr_explicit_realign[_optimized] scheme.
    6233              :    This function generates the following code at the loop prolog:
    6234              : 
    6235              :       p = initial_addr;
    6236              :    x  msq_init = *(floor(p));   # prolog load
    6237              :       realignment_token = call target_builtin;
    6238              :     loop:
    6239              :    x  msq = phi (msq_init, ---)
    6240              : 
    6241              :    The stmts marked with x are generated only for the case of
    6242              :    dr_explicit_realign_optimized.
    6243              : 
    6244              :    The code above sets up a new (vector) pointer, pointing to the first
    6245              :    location accessed by STMT_INFO, and a "floor-aligned" load using that
    6246              :    pointer.  It also generates code to compute the "realignment-token"
    6247              :    (if the relevant target hook was defined), and creates a phi-node at the
    6248              :    loop-header bb whose arguments are the result of the prolog-load (created
    6249              :    by this function) and the result of a load that takes place in the loop
    6250              :    (to be created by the caller to this function).
    6251              : 
    6252              :    For the case of dr_explicit_realign_optimized:
    6253              :    The caller to this function uses the phi-result (msq) to create the
    6254              :    realignment code inside the loop, and sets up the missing phi argument,
    6255              :    as follows:
    6256              :     loop:
    6257              :       msq = phi (msq_init, lsq)
    6258              :       lsq = *(floor(p'));        # load in loop
    6259              :       result = realign_load (msq, lsq, realignment_token);
    6260              : 
    6261              :    For the case of dr_explicit_realign:
    6262              :     loop:
    6263              :       msq = *(floor(p));        # load in loop
    6264              :       p' = p + (VS-1);
    6265              :       lsq = *(floor(p'));       # load in loop
    6266              :       result = realign_load (msq, lsq, realignment_token);
    6267              : 
    6268              :    Input:
    6269              :    STMT_INFO - (scalar) load stmt to be vectorized. This load accesses
    6270              :                a memory location that may be unaligned.
    6271              :    BSI - place where new code is to be inserted.
    6272              :    ALIGNMENT_SUPPORT_SCHEME - which of the two misalignment handling schemes
    6273              :                               is used.
    6274              : 
    6275              :    Output:
    6276              :    REALIGNMENT_TOKEN - the result of a call to the builtin_mask_for_load
    6277              :                        target hook, if defined.
    6278              :    Return value - the result of the loop-header phi node.  */
    6279              : 
    6280              : tree
    6281            0 : vect_setup_realignment (vec_info *vinfo, stmt_vec_info stmt_info, tree vectype,
    6282              :                         gimple_stmt_iterator *gsi, tree *realignment_token,
    6283              :                         enum dr_alignment_support alignment_support_scheme,
    6284              :                         tree init_addr,
    6285              :                         class loop **at_loop)
    6286              : {
    6287            0 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    6288            0 :   dr_vec_info *dr_info = STMT_VINFO_DR_INFO (stmt_info);
    6289            0 :   struct data_reference *dr = dr_info->dr;
    6290            0 :   class loop *loop = NULL;
    6291            0 :   edge pe = NULL;
    6292            0 :   tree scalar_dest = gimple_assign_lhs (stmt_info->stmt);
    6293            0 :   tree vec_dest;
    6294            0 :   gimple *inc;
    6295            0 :   tree ptr;
    6296            0 :   tree data_ref;
    6297            0 :   basic_block new_bb;
    6298            0 :   tree msq_init = NULL_TREE;
    6299            0 :   tree new_temp;
    6300            0 :   gphi *phi_stmt;
    6301            0 :   tree msq = NULL_TREE;
    6302            0 :   gimple_seq stmts = NULL;
    6303            0 :   bool compute_in_loop = false;
    6304            0 :   bool nested_in_vect_loop = false;
    6305            0 :   class loop *containing_loop = (gimple_bb (stmt_info->stmt))->loop_father;
    6306            0 :   class loop *loop_for_initial_load = NULL;
    6307              : 
    6308            0 :   if (loop_vinfo)
    6309              :     {
    6310            0 :       loop = LOOP_VINFO_LOOP (loop_vinfo);
    6311            0 :       nested_in_vect_loop = nested_in_vect_loop_p (loop, stmt_info);
    6312              :     }
    6313              : 
    6314            0 :   gcc_assert (alignment_support_scheme == dr_explicit_realign
    6315              :               || alignment_support_scheme == dr_explicit_realign_optimized);
    6316              : 
    6317              :   /* We need to generate three things:
    6318              :      1. the misalignment computation
    6319              :      2. the extra vector load (for the optimized realignment scheme).
    6320              :      3. the phi node for the two vectors from which the realignment is
    6321              :       done (for the optimized realignment scheme).  */
    6322              : 
    6323              :   /* 1. Determine where to generate the misalignment computation.
    6324              : 
    6325              :      If INIT_ADDR is NULL_TREE, this indicates that the misalignment
    6326              :      calculation will be generated by this function, outside the loop (in the
    6327              :      preheader).  Otherwise, INIT_ADDR had already been computed for us by the
    6328              :      caller, inside the loop.
    6329              : 
    6330              :      Background: If the misalignment remains fixed throughout the iterations of
    6331              :      the loop, then both realignment schemes are applicable, and also the
    6332              :      misalignment computation can be done outside LOOP.  This is because we are
    6333              :      vectorizing LOOP, and so the memory accesses in LOOP advance in steps that
    6334              :      are a multiple of VS (the Vector Size), and therefore the misalignment in
    6335              :      different vectorized LOOP iterations is always the same.
    6336              :      The problem arises only if the memory access is in an inner-loop nested
    6337              :      inside LOOP, which is now being vectorized using outer-loop vectorization.
    6338              :      This is the only case when the misalignment of the memory access may not
    6339              :      remain fixed throughout the iterations of the inner-loop (as explained in
    6340              :      detail in vect_supportable_dr_alignment).  In this case, not only is the
    6341              :      optimized realignment scheme not applicable, but also the misalignment
    6342              :      computation (and generation of the realignment token that is passed to
    6343              :      REALIGN_LOAD) have to be done inside the loop.
    6344              : 
    6345              :      In short, INIT_ADDR indicates whether we are in a COMPUTE_IN_LOOP mode
    6346              :      or not, which in turn determines if the misalignment is computed inside
    6347              :      the inner-loop, or outside LOOP.  */
    6348              : 
    6349            0 :   if (init_addr != NULL_TREE || !loop_vinfo)
    6350              :     {
    6351            0 :       compute_in_loop = true;
    6352            0 :       gcc_assert (alignment_support_scheme == dr_explicit_realign);
    6353              :     }
    6354              : 
    6355              : 
    6356              :   /* 2. Determine where to generate the extra vector load.
    6357              : 
    6358              :      For the optimized realignment scheme, instead of generating two vector
    6359              :      loads in each iteration, we generate a single extra vector load in the
    6360              :      preheader of the loop, and in each iteration reuse the result of the
    6361              :      vector load from the previous iteration.  In case the memory access is in
    6362              :      an inner-loop nested inside LOOP, which is now being vectorized using
    6363              :      outer-loop vectorization, we need to determine whether this initial vector
    6364              :      load should be generated at the preheader of the inner-loop, or can be
    6365              :      generated at the preheader of LOOP.  If the memory access has no evolution
    6366              :      in LOOP, it can be generated in the preheader of LOOP. Otherwise, it has
    6367              :      to be generated inside LOOP (in the preheader of the inner-loop).  */
    6368              : 
    6369            0 :   if (nested_in_vect_loop)
    6370              :     {
    6371            0 :       tree outerloop_step = STMT_VINFO_DR_STEP (stmt_info);
    6372            0 :       bool invariant_in_outerloop =
    6373            0 :             (tree_int_cst_compare (outerloop_step, size_zero_node) == 0);
    6374            0 :       loop_for_initial_load = (invariant_in_outerloop ? loop : loop->inner);
    6375              :     }
    6376              :   else
    6377              :     loop_for_initial_load = loop;
    6378            0 :   if (at_loop)
    6379            0 :     *at_loop = loop_for_initial_load;
    6380              : 
    6381            0 :   tree vuse = NULL_TREE;
    6382            0 :   if (loop_for_initial_load)
    6383              :     {
    6384            0 :       pe = loop_preheader_edge (loop_for_initial_load);
    6385            0 :       if (gphi *vphi = get_virtual_phi (loop_for_initial_load->header))
    6386            0 :         vuse = PHI_ARG_DEF_FROM_EDGE (vphi, pe);
    6387              :     }
    6388            0 :   if (!vuse)
    6389            0 :     vuse = gimple_vuse (gsi_stmt (*gsi));
    6390              : 
    6391              :   /* 3. For the case of the optimized realignment, create the first vector
    6392              :       load at the loop preheader.  */
    6393              : 
    6394            0 :   if (alignment_support_scheme == dr_explicit_realign_optimized)
    6395              :     {
    6396              :       /* Create msq_init = *(floor(p1)) in the loop preheader  */
    6397            0 :       gassign *new_stmt;
    6398              : 
    6399            0 :       gcc_assert (!compute_in_loop);
    6400            0 :       vec_dest = vect_create_destination_var (scalar_dest, vectype);
    6401            0 :       ptr = vect_create_data_ref_ptr (vinfo, stmt_info, vectype,
    6402              :                                       loop_for_initial_load, NULL_TREE,
    6403              :                                       &init_addr, NULL, &inc, true);
    6404            0 :       if (TREE_CODE (ptr) == SSA_NAME)
    6405            0 :         new_temp = copy_ssa_name (ptr);
    6406              :       else
    6407            0 :         new_temp = make_ssa_name (TREE_TYPE (ptr));
    6408            0 :       poly_uint64 align = DR_TARGET_ALIGNMENT (dr_info);
    6409            0 :       tree type = TREE_TYPE (ptr);
    6410            0 :       new_stmt = gimple_build_assign
    6411            0 :                    (new_temp, BIT_AND_EXPR, ptr,
    6412            0 :                     fold_build2 (MINUS_EXPR, type,
    6413              :                                  build_int_cst (type, 0),
    6414              :                                  build_int_cst (type, align)));
    6415            0 :       new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
    6416            0 :       gcc_assert (!new_bb);
    6417            0 :       data_ref
    6418            0 :         = build2 (MEM_REF, TREE_TYPE (vec_dest), new_temp,
    6419              :                   build_int_cst (reference_alias_ptr_type (DR_REF (dr)), 0));
    6420            0 :       vect_copy_ref_info (data_ref, DR_REF (dr));
    6421            0 :       new_stmt = gimple_build_assign (vec_dest, data_ref);
    6422            0 :       new_temp = make_ssa_name (vec_dest, new_stmt);
    6423            0 :       gimple_assign_set_lhs (new_stmt, new_temp);
    6424            0 :       gimple_set_vuse (new_stmt, vuse);
    6425            0 :       if (pe)
    6426              :         {
    6427            0 :           new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
    6428            0 :           gcc_assert (!new_bb);
    6429              :         }
    6430              :       else
    6431            0 :          gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    6432              : 
    6433            0 :       msq_init = gimple_assign_lhs (new_stmt);
    6434              :     }
    6435              : 
    6436              :   /* 4. Create realignment token using a target builtin, if available.
    6437              :       It is done either inside the containing loop, or before LOOP (as
    6438              :       determined above).  */
    6439              : 
    6440            0 :   if (targetm.vectorize.builtin_mask_for_load)
    6441              :     {
    6442            0 :       gcall *new_stmt;
    6443            0 :       tree builtin_decl;
    6444              : 
    6445              :       /* Compute INIT_ADDR - the initial addressed accessed by this memref.  */
    6446            0 :       if (!init_addr)
    6447              :         {
    6448              :           /* Generate the INIT_ADDR computation outside LOOP.  */
    6449            0 :           init_addr = vect_create_addr_base_for_vector_ref (vinfo,
    6450              :                                                             stmt_info, &stmts,
    6451              :                                                             NULL_TREE);
    6452            0 :           if (loop)
    6453              :             {
    6454            0 :               pe = loop_preheader_edge (loop);
    6455            0 :               new_bb = gsi_insert_seq_on_edge_immediate (pe, stmts);
    6456            0 :               gcc_assert (!new_bb);
    6457              :             }
    6458              :           else
    6459            0 :              gsi_insert_seq_before (gsi, stmts, GSI_SAME_STMT);
    6460              :         }
    6461              : 
    6462            0 :       builtin_decl = targetm.vectorize.builtin_mask_for_load ();
    6463            0 :       new_stmt = gimple_build_call (builtin_decl, 1, init_addr);
    6464            0 :       vec_dest =
    6465            0 :         vect_create_destination_var (scalar_dest,
    6466              :                                      gimple_call_return_type (new_stmt));
    6467            0 :       new_temp = make_ssa_name (vec_dest, new_stmt);
    6468            0 :       gimple_call_set_lhs (new_stmt, new_temp);
    6469              : 
    6470            0 :       if (compute_in_loop)
    6471            0 :         gsi_insert_before (gsi, new_stmt, GSI_SAME_STMT);
    6472              :       else
    6473              :         {
    6474              :           /* Generate the misalignment computation outside LOOP.  */
    6475            0 :           pe = loop_preheader_edge (loop);
    6476            0 :           new_bb = gsi_insert_on_edge_immediate (pe, new_stmt);
    6477            0 :           gcc_assert (!new_bb);
    6478              :         }
    6479              : 
    6480            0 :       *realignment_token = gimple_call_lhs (new_stmt);
    6481              : 
    6482              :       /* The result of the CALL_EXPR to this builtin is determined from
    6483              :          the value of the parameter and no global variables are touched
    6484              :          which makes the builtin a "const" function.  Requiring the
    6485              :          builtin to have the "const" attribute makes it unnecessary
    6486              :          to call mark_call_clobbered.  */
    6487            0 :       gcc_assert (TREE_READONLY (builtin_decl));
    6488              :     }
    6489              : 
    6490            0 :   if (alignment_support_scheme == dr_explicit_realign)
    6491              :     return msq;
    6492              : 
    6493            0 :   gcc_assert (!compute_in_loop);
    6494            0 :   gcc_assert (alignment_support_scheme == dr_explicit_realign_optimized);
    6495              : 
    6496              : 
    6497              :   /* 5. Create msq = phi <msq_init, lsq> in loop  */
    6498              : 
    6499            0 :   pe = loop_preheader_edge (containing_loop);
    6500            0 :   vec_dest = vect_create_destination_var (scalar_dest, vectype);
    6501            0 :   msq = make_ssa_name (vec_dest);
    6502            0 :   phi_stmt = create_phi_node (msq, containing_loop->header);
    6503            0 :   add_phi_arg (phi_stmt, msq_init, pe, UNKNOWN_LOCATION);
    6504              : 
    6505            0 :   return msq;
    6506              : }
    6507              : 
    6508              : 
    6509              : /* Function vect_grouped_load_supported.
    6510              : 
    6511              :    COUNT is the size of the load group (the number of statements plus the
    6512              :    number of gaps).  SINGLE_ELEMENT_P is true if there is actually
    6513              :    only one statement, with a gap of COUNT - 1.
    6514              : 
    6515              :    Returns true if a suitable permute exists.  */
    6516              : 
    6517              : bool
    6518         2026 : vect_grouped_load_supported (tree vectype, bool single_element_p,
    6519              :                              unsigned HOST_WIDE_INT count)
    6520              : {
    6521         2026 :   machine_mode mode = TYPE_MODE (vectype);
    6522              : 
    6523              :   /* If this is single-element interleaving with an element distance
    6524              :      that leaves unused vector loads around punt - we at least create
    6525              :      very sub-optimal code in that case (and blow up memory,
    6526              :      see PR65518).  */
    6527         2026 :   if (single_element_p && maybe_gt (count, TYPE_VECTOR_SUBPARTS (vectype)))
    6528              :     {
    6529           46 :       if (dump_enabled_p ())
    6530            3 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6531              :                          "single-element interleaving not supported "
    6532              :                          "for not adjacent vector loads\n");
    6533              :       return false;
    6534              :     }
    6535              : 
    6536              :   /* vect_permute_load_chain requires the group size to be equal to 3 or
    6537              :      be a power of two.  */
    6538         1980 :   if (count != 3 && exact_log2 (count) == -1)
    6539              :     {
    6540          222 :       if (dump_enabled_p ())
    6541            8 :         dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6542              :                          "the size of the group of accesses"
    6543              :                          " is not a power of 2 or not equal to 3\n");
    6544              :       return false;
    6545              :     }
    6546              : 
    6547              :   /* Check that the permutation is supported.  */
    6548         1758 :   if (VECTOR_MODE_P (mode))
    6549              :     {
    6550         1758 :       unsigned int i, j;
    6551         1758 :       if (count == 3)
    6552              :         {
    6553          838 :           unsigned int nelt;
    6554         1676 :           if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
    6555              :             {
    6556              :               if (dump_enabled_p ())
    6557              :                 dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6558              :                                  "cannot handle groups of 3 loads for"
    6559              :                                  " variable-length vectors\n");
    6560              :               return false;
    6561              :             }
    6562              : 
    6563          838 :           vec_perm_builder sel (nelt, nelt, 1);
    6564          838 :           sel.quick_grow (nelt);
    6565          838 :           vec_perm_indices indices;
    6566          838 :           unsigned int k;
    6567         3295 :           for (k = 0; k < 3; k++)
    6568              :             {
    6569         8848 :               for (i = 0; i < nelt; i++)
    6570         6372 :                 if (3 * i + k < 2 * nelt)
    6571         4256 :                   sel[i] = 3 * i + k;
    6572              :                 else
    6573         2116 :                   sel[i] = 0;
    6574         2476 :               indices.new_vector (sel, 2, nelt);
    6575         2476 :               if (!can_vec_perm_const_p (mode, mode, indices))
    6576              :                 {
    6577           19 :                   if (dump_enabled_p ())
    6578            6 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6579              :                                      "shuffle of 3 loads is not supported by"
    6580              :                                      " target\n");
    6581              :                   return false;
    6582              :                 }
    6583         8649 :               for (i = 0, j = 0; i < nelt; i++)
    6584         6192 :                 if (3 * i + k < 2 * nelt)
    6585         4128 :                   sel[i] = i;
    6586              :                 else
    6587         2064 :                   sel[i] = nelt + ((nelt + k) % 3) + 3 * (j++);
    6588         2457 :               indices.new_vector (sel, 2, nelt);
    6589         2457 :               if (!can_vec_perm_const_p (mode, mode, indices))
    6590              :                 {
    6591            0 :                   if (dump_enabled_p ())
    6592            0 :                     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6593              :                                      "shuffle of 3 loads is not supported by"
    6594              :                                      " target\n");
    6595              :                   return false;
    6596              :                 }
    6597              :             }
    6598              :           return true;
    6599          838 :         }
    6600              :       else
    6601              :         {
    6602              :           /* If length is not equal to 3 then only power of 2 is supported.  */
    6603          920 :           gcc_assert (pow2p_hwi (count));
    6604         1840 :           poly_uint64 nelt = GET_MODE_NUNITS (mode);
    6605              : 
    6606              :           /* The encoding has a single stepped pattern.  */
    6607          920 :           vec_perm_builder sel (nelt, 1, 3);
    6608          920 :           sel.quick_grow (3);
    6609         4600 :           for (i = 0; i < 3; i++)
    6610         2760 :             sel[i] = i * 2;
    6611          920 :           vec_perm_indices indices (sel, 2, nelt);
    6612          920 :           if (can_vec_perm_const_p (mode, mode, indices))
    6613              :             {
    6614         3628 :               for (i = 0; i < 3; i++)
    6615         2721 :                 sel[i] = i * 2 + 1;
    6616          907 :               indices.new_vector (sel, 2, nelt);
    6617          907 :               if (can_vec_perm_const_p (mode, mode, indices))
    6618          907 :                 return true;
    6619              :             }
    6620          920 :         }
    6621              :     }
    6622              : 
    6623           13 :   if (dump_enabled_p ())
    6624            2 :     dump_printf_loc (MSG_MISSED_OPTIMIZATION, vect_location,
    6625              :                      "extract even/odd not supported by target\n");
    6626              :   return false;
    6627              : }
    6628              : 
    6629              : /* Return FN if vec_{masked_,mask_len_}load_lanes is available for COUNT vectors
    6630              :    of type VECTYPE.  MASKED_P says whether the masked form is needed.
    6631              :    If it is available and ELSVALS is nonzero store the possible else values
    6632              :    in the vector it points to.  */
    6633              : 
    6634              : internal_fn
    6635       146395 : vect_load_lanes_supported (tree vectype, unsigned HOST_WIDE_INT count,
    6636              :                            bool masked_p, vec<int> *elsvals)
    6637              : {
    6638       146395 :   if (vect_lanes_optab_supported_p ("vec_mask_len_load_lanes",
    6639              :                                     vec_mask_len_load_lanes_optab, vectype,
    6640              :                                     count, elsvals))
    6641              :     return IFN_MASK_LEN_LOAD_LANES;
    6642       146395 :   else if (masked_p)
    6643              :     {
    6644           30 :       if (vect_lanes_optab_supported_p ("vec_mask_load_lanes",
    6645              :                                         vec_mask_load_lanes_optab, vectype,
    6646              :                                         count, elsvals))
    6647            0 :         return IFN_MASK_LOAD_LANES;
    6648              :     }
    6649              :   else
    6650              :     {
    6651       146365 :       if (vect_lanes_optab_supported_p ("vec_load_lanes", vec_load_lanes_optab,
    6652              :                                         vectype, count, elsvals))
    6653            0 :         return IFN_LOAD_LANES;
    6654              :     }
    6655              :   return IFN_LAST;
    6656              : }
    6657              : 
    6658              : /* Function vect_force_dr_alignment_p.
    6659              : 
    6660              :    Returns whether the alignment of a DECL can be forced to be aligned
    6661              :    on ALIGNMENT bit boundary.  */
    6662              : 
    6663              : bool
    6664       766790 : vect_can_force_dr_alignment_p (const_tree decl, poly_uint64 alignment)
    6665              : {
    6666       766790 :   if (!VAR_P (decl))
    6667              :     return false;
    6668              : 
    6669       228351 :   if (decl_in_symtab_p (decl)
    6670       228351 :       && (!symtab_node::get (decl)
    6671        27083 :           || !symtab_node::get (decl)->can_increase_alignment_p ()))
    6672              :     return false;
    6673              : 
    6674       212080 :   if (TREE_STATIC (decl))
    6675        10812 :     return (known_le (alignment,
    6676        10812 :                       (unsigned HOST_WIDE_INT) MAX_OFILE_ALIGNMENT));
    6677              :   else
    6678       201268 :     return (known_le (alignment, (unsigned HOST_WIDE_INT) MAX_STACK_ALIGNMENT));
    6679              : }
    6680              : 
    6681              : /* Return whether the data reference DR_INFO is supported with respect to its
    6682              :    alignment.
    6683              :    If CHECK_ALIGNED_ACCESSES is TRUE, check if the access is supported even
    6684              :    it is aligned, i.e., check if it is possible to vectorize it with different
    6685              :    alignment.  If IS_GATHER_SCATTER is true we are dealing with a
    6686              :    gather/scatter.  */
    6687              : 
    6688              : enum dr_alignment_support
    6689      3023099 : vect_supportable_dr_alignment (vec_info *vinfo, dr_vec_info *dr_info,
    6690              :                                tree vectype, int misalignment,
    6691              :                                bool is_gather_scatter)
    6692              : {
    6693      3023099 :   data_reference *dr = dr_info->dr;
    6694      3023099 :   stmt_vec_info stmt_info = dr_info->stmt;
    6695      3023099 :   machine_mode mode = TYPE_MODE (vectype);
    6696      3023099 :   loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo);
    6697      3023099 :   class loop *vect_loop = NULL;
    6698      3023099 :   bool nested_in_vect_loop = false;
    6699              : 
    6700      3023099 :   if (misalignment == 0)
    6701              :     return dr_aligned;
    6702      1855457 :   else if (dr_safe_speculative_read_required (stmt_info))
    6703              :     return dr_unaligned_unsupported;
    6704              : 
    6705      1423604 :   if (loop_vinfo)
    6706              :     {
    6707      1002170 :       vect_loop = LOOP_VINFO_LOOP (loop_vinfo);
    6708      1002170 :       nested_in_vect_loop = nested_in_vect_loop_p (vect_loop, stmt_info);
    6709              :     }
    6710              : 
    6711              :   /* Possibly unaligned access.  */
    6712              : 
    6713              :   /* We can choose between using the implicit realignment scheme (generating
    6714              :      a misaligned_move stmt) and the explicit realignment scheme (generating
    6715              :      aligned loads with a REALIGN_LOAD).  There are two variants to the
    6716              :      explicit realignment scheme: optimized, and unoptimized.
    6717              :      We can optimize the realignment only if the step between consecutive
    6718              :      vector loads is equal to the vector size.  Since the vector memory
    6719              :      accesses advance in steps of VS (Vector Size) in the vectorized loop, it
    6720              :      is guaranteed that the misalignment amount remains the same throughout the
    6721              :      execution of the vectorized loop.  Therefore, we can create the
    6722              :      "realignment token" (the permutation mask that is passed to REALIGN_LOAD)
    6723              :      at the loop preheader.
    6724              : 
    6725              :      However, in the case of outer-loop vectorization, when vectorizing a
    6726              :      memory access in the inner-loop nested within the LOOP that is now being
    6727              :      vectorized, while it is guaranteed that the misalignment of the
    6728              :      vectorized memory access will remain the same in different outer-loop
    6729              :      iterations, it is *not* guaranteed that is will remain the same throughout
    6730              :      the execution of the inner-loop.  This is because the inner-loop advances
    6731              :      with the original scalar step (and not in steps of VS).  If the inner-loop
    6732              :      step happens to be a multiple of VS, then the misalignment remains fixed
    6733              :      and we can use the optimized realignment scheme.  For example:
    6734              : 
    6735              :       for (i=0; i<N; i++)
    6736              :         for (j=0; j<M; j++)
    6737              :           s += a[i+j];
    6738              : 
    6739              :      When vectorizing the i-loop in the above example, the step between
    6740              :      consecutive vector loads is 1, and so the misalignment does not remain
    6741              :      fixed across the execution of the inner-loop, and the realignment cannot
    6742              :      be optimized (as illustrated in the following pseudo vectorized loop):
    6743              : 
    6744              :       for (i=0; i<N; i+=4)
    6745              :         for (j=0; j<M; j++){
    6746              :           vs += vp[i+j]; // misalignment of &vp[i+j] is {0,1,2,3,0,1,2,3,...}
    6747              :                          // when j is {0,1,2,3,4,5,6,7,...} respectively.
    6748              :                          // (assuming that we start from an aligned address).
    6749              :           }
    6750              : 
    6751              :      We therefore have to use the unoptimized realignment scheme:
    6752              : 
    6753              :       for (i=0; i<N; i+=4)
    6754              :           for (j=k; j<M; j+=4)
    6755              :           vs += vp[i+j]; // misalignment of &vp[i+j] is always k (assuming
    6756              :                            // that the misalignment of the initial address is
    6757              :                            // 0).
    6758              : 
    6759              :      The loop can then be vectorized as follows:
    6760              : 
    6761              :       for (k=0; k<4; k++){
    6762              :         rt = get_realignment_token (&vp[k]);
    6763              :         for (i=0; i<N; i+=4){
    6764              :           v1 = vp[i+k];
    6765              :           for (j=k; j<M; j+=4){
    6766              :             v2 = vp[i+j+VS-1];
    6767              :             va = REALIGN_LOAD <v1,v2,rt>;
    6768              :             vs += va;
    6769              :             v1 = v2;
    6770              :           }
    6771              :         }
    6772              :     } */
    6773              : 
    6774      1423604 :   if (DR_IS_READ (dr) && !is_gather_scatter)
    6775              :     {
    6776       645718 :       if (can_implement_p (vec_realign_load_optab, mode)
    6777       645718 :           && (!targetm.vectorize.builtin_mask_for_load
    6778            0 :               || targetm.vectorize.builtin_mask_for_load ()))
    6779              :         {
    6780              :           /* If we are doing SLP then the accesses need not have the
    6781              :              same alignment, instead it depends on the SLP group size.  */
    6782            0 :           if (loop_vinfo
    6783            0 :               && STMT_VINFO_GROUPED_ACCESS (stmt_info)
    6784            0 :               && !multiple_p (LOOP_VINFO_VECT_FACTOR (loop_vinfo)
    6785            0 :                               * (DR_GROUP_SIZE
    6786            0 :                                    (DR_GROUP_FIRST_ELEMENT (stmt_info))),
    6787            0 :                               TYPE_VECTOR_SUBPARTS (vectype)))
    6788              :             ;
    6789            0 :           else if (!loop_vinfo
    6790            0 :                    || (nested_in_vect_loop
    6791            0 :                        && maybe_ne (TREE_INT_CST_LOW (DR_STEP (dr)),
    6792            0 :                                     GET_MODE_SIZE (TYPE_MODE (vectype)))))
    6793            0 :             return dr_explicit_realign;
    6794              :           else
    6795              :             return dr_explicit_realign_optimized;
    6796              :         }
    6797              :     }
    6798              : 
    6799      1423604 :   bool is_packed = not_size_aligned (DR_REF (dr));
    6800      1423604 :   if (misalignment == DR_MISALIGNMENT_UNKNOWN
    6801      1423604 :       && is_gather_scatter)
    6802         2836 :     misalignment = (get_object_alignment (DR_REF (dr))
    6803         2836 :                     % (GET_MODE_BITSIZE (GET_MODE_INNER (mode))))
    6804         2836 :       / BITS_PER_UNIT;
    6805      1423604 :   if (targetm.vectorize.support_vector_misalignment (mode, misalignment,
    6806              :                                                      is_packed,
    6807              :                                                      is_gather_scatter))
    6808      1423409 :     return dr_unaligned_supported;
    6809              : 
    6810              :   /* Unsupported.  */
    6811              :   return dr_unaligned_unsupported;
    6812              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.