21#ifndef GCC_TREE_VECTORIZER_H 
   22#define GCC_TREE_VECTORIZER_H 
  114#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def)           \ 
  115                                   || ((D) == vect_double_reduction_def) \ 
  116                                   || ((D) == vect_nested_cycle)) 
 
  135                 std::pair<stmt_vec_info, innermost_loop_behavior *> >
 
  142  tree op (
unsigned int i) 
const;
 
 
  376  static void *
operator new (size_t);
 
  379  static void operator delete (
void *, size_t);
 
 
  434#define SLP_INSTANCE_TREE(S)                     (S)->root 
  435#define SLP_INSTANCE_LOADS(S)                    (S)->loads 
  436#define SLP_INSTANCE_ROOT_STMTS(S)               (S)->root_stmts 
  437#define SLP_INSTANCE_REMAIN_DEFS(S)              (S)->remain_defs 
  438#define SLP_INSTANCE_KIND(S)                     (S)->kind 
  440#define SLP_TREE_CHILDREN(S)                     (S)->children 
  441#define SLP_TREE_SCALAR_STMTS(S)                 (S)->stmts 
  442#define SLP_TREE_SCALAR_OPS(S)                   (S)->ops 
  443#define SLP_TREE_REF_COUNT(S)                    (S)->refcnt 
  444#define SLP_TREE_VEC_DEFS(S)                     (S)->vec_defs 
  445#define SLP_TREE_LOAD_PERMUTATION(S)             (S)->load_permutation 
  446#define SLP_TREE_LANE_PERMUTATION(S)             (S)->lane_permutation 
  447#define SLP_TREE_DEF_TYPE(S)                     (S)->def_type 
  448#define SLP_TREE_VECTYPE(S)                      (S)->vectype 
  449#define SLP_TREE_REPRESENTATIVE(S)               (S)->representative 
  450#define SLP_TREE_LANES(S)                        (S)->lanes 
  451#define SLP_TREE_CODE(S)                         (S)->code 
  452#define SLP_TREE_TYPE(S)                         (S)->type 
  453#define SLP_TREE_GS_SCALE(S)                     (S)->gs_scale 
  454#define SLP_TREE_GS_BASE(S)                      (S)->gs_base 
  455#define SLP_TREE_REDUC_IDX(S)                    (S)->cycle_info.reduc_idx 
  456#define SLP_TREE_PERMUTE_P(S)                    ((S)->code == VEC_PERM_EXPR) 
  501  static inline hashval_t
 
 
  885#define VECT_REDUC_INFO_DEF_TYPE(I) ((I)->def_type) 
  886#define VECT_REDUC_INFO_TYPE(I) ((I)->reduc_type) 
  887#define VECT_REDUC_INFO_CODE(I) ((I)->reduc_code) 
  888#define VECT_REDUC_INFO_FN(I) ((I)->reduc_fn) 
  889#define VECT_REDUC_INFO_SCALAR_RESULTS(I) ((I)->reduc_scalar_results) 
  890#define VECT_REDUC_INFO_INITIAL_VALUES(I) ((I)->reduc_initial_values) 
  891#define VECT_REDUC_INFO_REUSED_ACCUMULATOR(I) ((I)->reused_accumulator) 
  892#define VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL(I) ((I)->induc_cond_initial_val) 
  893#define VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT(I) ((I)->reduc_epilogue_adjustment) 
  894#define VECT_REDUC_INFO_VECTYPE(I) ((I)->reduc_vectype) 
  895#define VECT_REDUC_INFO_VECTYPE_FOR_MASK(I) ((I)->reduc_vectype_for_mask) 
  896#define VECT_REDUC_INFO_FORCE_SINGLE_CYCLE(I) ((I)->force_single_cycle) 
  897#define VECT_REDUC_INFO_RESULT_POS(I) ((I)->reduc_result_pos) 
 1242#define LOOP_VINFO_LOOP(L)                 (L)->loop 
 1243#define LOOP_VINFO_IV_EXIT(L)              (L)->vec_loop_iv_exit 
 1244#define LOOP_VINFO_EPILOGUE_IV_EXIT(L)     (L)->vec_epilogue_loop_iv_exit 
 1245#define LOOP_VINFO_SCALAR_IV_EXIT(L)       (L)->scalar_loop_iv_exit 
 1246#define LOOP_VINFO_BBS(L)                  (L)->bbs 
 1247#define LOOP_VINFO_NBBS(L)                 (L)->nbbs 
 1248#define LOOP_VINFO_NITERSM1(L)             (L)->num_itersm1 
 1249#define LOOP_VINFO_NITERS(L)               (L)->num_iters 
 1253#define LOOP_VINFO_NITERS_UNCHANGED(L)     (L)->num_iters_unchanged 
 1254#define LOOP_VINFO_NITERS_ASSUMPTIONS(L)   (L)->num_iters_assumptions 
 1255#define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th 
 1256#define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold 
 1257#define LOOP_VINFO_VECTORIZABLE_P(L)       (L)->vectorizable 
 1258#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p 
 1259#define LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P(L) (L)->must_use_partial_vectors_p 
 1260#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p 
 1261#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p 
 1262#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p 
 1263#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L)                             \ 
 1264  (L)->epil_using_partial_vectors_p 
 
 1265#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment 
 1266#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias 
 1267#define LOOP_VINFO_VECT_FACTOR(L)          (L)->vectorization_factor 
 1268#define LOOP_VINFO_MAX_VECT_FACTOR(L)      (L)->max_vectorization_factor 
 1269#define LOOP_VINFO_MASKS(L)                (L)->masks 
 1270#define LOOP_VINFO_LENS(L)                 (L)->lens 
 1271#define LOOP_VINFO_MASK_SKIP_NITERS(L)     (L)->mask_skip_niters 
 1272#define LOOP_VINFO_MASK_NITERS_PFA_OFFSET(L) (L)->mask_skip_niters_pfa_offset 
 1273#define LOOP_VINFO_RGROUP_COMPARE_TYPE(L)  (L)->rgroup_compare_type 
 1274#define LOOP_VINFO_RGROUP_IV_TYPE(L)       (L)->rgroup_iv_type 
 1275#define LOOP_VINFO_PARTIAL_VECTORS_STYLE(L) (L)->partial_vector_style 
 1276#define LOOP_VINFO_PTR_MASK(L)             (L)->ptr_mask 
 1277#define LOOP_VINFO_MAX_SPEC_READ_AMOUNT(L) (L)->max_spec_read_amount 
 1278#define LOOP_VINFO_LOOP_NEST(L)            (L)->shared->loop_nest 
 1279#define LOOP_VINFO_DATAREFS(L)             (L)->shared->datarefs 
 1280#define LOOP_VINFO_DDRS(L)                 (L)->shared->ddrs 
 1281#define LOOP_VINFO_INT_NITERS(L)           (TREE_INT_CST_LOW ((L)->num_iters)) 
 1282#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment 
 1283#define LOOP_VINFO_NON_LINEAR_IV(L)        (L)->nonlinear_iv 
 1284#define LOOP_VINFO_UNALIGNED_DR(L)         (L)->unaligned_dr 
 1285#define LOOP_VINFO_MAY_MISALIGN_STMTS(L)   (L)->may_misalign_stmts 
 1286#define LOOP_VINFO_MAY_ALIAS_DDRS(L)       (L)->may_alias_ddrs 
 1287#define LOOP_VINFO_COMP_ALIAS_DDRS(L)      (L)->comp_alias_ddrs 
 1288#define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L)  (L)->check_unequal_addrs 
 1289#define LOOP_VINFO_CHECK_NONZERO(L)        (L)->check_nonzero 
 1290#define LOOP_VINFO_LOWER_BOUNDS(L)         (L)->lower_bounds 
 1291#define LOOP_VINFO_USER_UNROLL(L)          (L)->user_unroll 
 1292#define LOOP_VINFO_GROUPED_STORES(L)       (L)->grouped_stores 
 1293#define LOOP_VINFO_SLP_INSTANCES(L)        (L)->slp_instances 
 1294#define LOOP_VINFO_REDUCTIONS(L)           (L)->reductions 
 1295#define LOOP_VINFO_PEELING_FOR_GAPS(L)     (L)->peeling_for_gaps 
 1296#define LOOP_VINFO_PEELING_FOR_NITER(L)    (L)->peeling_for_niter 
 1297#define LOOP_VINFO_EARLY_BREAKS(L)         (L)->early_breaks 
 1298#define LOOP_VINFO_EARLY_BRK_STORES(L)     (L)->early_break_stores 
 1299#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L)  \ 
 1300  (single_pred ((L)->loop->latch) != (L)->vec_loop_iv_exit->src) 
 
 1301#define LOOP_VINFO_EARLY_BREAKS_LIVE_IVS(L)  \ 
 1302  (L)->early_break_live_ivs 
 
 1303#define LOOP_VINFO_EARLY_BRK_DEST_BB(L)    (L)->early_break_dest_bb 
 1304#define LOOP_VINFO_EARLY_BRK_VUSES(L)      (L)->early_break_vuses 
 1305#define LOOP_VINFO_LOOP_CONDS(L)           (L)->conds 
 1306#define LOOP_VINFO_LOOP_IV_COND(L)         (L)->loop_iv_cond 
 1307#define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies 
 1308#define LOOP_VINFO_SCALAR_LOOP(L)          (L)->scalar_loop 
 1309#define LOOP_VINFO_SCALAR_LOOP_SCALING(L)  (L)->scalar_loop_scaling 
 1310#define LOOP_VINFO_HAS_MASK_STORE(L)       (L)->has_mask_store 
 1311#define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec 
 1312#define LOOP_VINFO_MAIN_LOOP_INFO(L)       (L)->main_loop_info 
 1313#define LOOP_VINFO_ORIG_LOOP_INFO(L)       (L)->orig_loop_info 
 1314#define LOOP_VINFO_SIMD_IF_COND(L)         (L)->simd_if_cond 
 1315#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor 
 1316#define LOOP_VINFO_INV_PATTERN_DEF_SEQ(L)  (L)->inv_pattern_def_seq 
 1317#define LOOP_VINFO_DRS_ADVANCED_BY(L)      (L)->drs_advanced_by 
 1318#define LOOP_VINFO_ALTERNATE_DEFS(L)       (L)->alternate_defs 
 1320#define LOOP_VINFO_FULLY_MASKED_P(L)            \ 
 1321  (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L)       \ 
 1322   && !LOOP_VINFO_MASKS (L).is_empty ()) 
 
 1324#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L)       \ 
 1325  (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L)       \ 
 1326   && !LOOP_VINFO_LENS (L).is_empty ()) 
 
 1328#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L)       \ 
 1329  ((L)->may_misalign_stmts.length () > 0) 
 
 1330#define LOOP_REQUIRES_VERSIONING_FOR_SPEC_READ(L)       \ 
 1331  (maybe_gt ((L)->max_spec_read_amount, 0U)) 
 
 1332#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L)           \ 
 1333  ((L)->comp_alias_ddrs.length () > 0 \ 
 1334   || (L)->check_unequal_addrs.length () > 0 \ 
 1335   || (L)->lower_bounds.length () > 0) 
 
 1336#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L)          \ 
 1337  (LOOP_VINFO_NITERS_ASSUMPTIONS (L)) 
 
 1338#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L)    \ 
 1339  (LOOP_VINFO_SIMD_IF_COND (L)) 
 
 1340#define LOOP_REQUIRES_VERSIONING(L)                     \ 
 1341  (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L)           \ 
 1342   || LOOP_REQUIRES_VERSIONING_FOR_SPEC_READ (L)        \ 
 1343   || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L)            \ 
 1344   || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L)           \ 
 1345   || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L)) 
 
 1347#define LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING(L)    \ 
 1348  ((L)->may_misalign_stmts.length () > 0                \ 
 1349   && !LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (L)) 
 
 1351#define LOOP_VINFO_NITERS_KNOWN_P(L)          \ 
 1352  (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0) 
 
 1354#define LOOP_VINFO_EPILOGUE_P(L) \ 
 1355  (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL) 
 
 1357#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \ 
 1358  (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L))) 
 
 1392#define BB_VINFO_BBS(B)              (B)->bbs 
 1393#define BB_VINFO_NBBS(B)             (B)->nbbs 
 1394#define BB_VINFO_GROUPED_STORES(B)   (B)->grouped_stores 
 1395#define BB_VINFO_SLP_INSTANCES(B)    (B)->slp_instances 
 1396#define BB_VINFO_DATAREFS(B)         (B)->shared->datarefs 
 1397#define BB_VINFO_DDRS(B)             (B)->shared->ddrs 
 1662#define STMT_VINFO_STMT(S)                 (S)->stmt 
 1663#define STMT_VINFO_RELEVANT(S)             (S)->relevant 
 1664#define STMT_VINFO_LIVE_P(S)               (S)->live 
 1665#define STMT_VINFO_VECTYPE(S)              (S)->vectype 
 1666#define STMT_VINFO_VECTORIZABLE(S)         (S)->vectorizable 
 1667#define STMT_VINFO_DATA_REF(S)             ((S)->dr_aux.dr + 0) 
 1668#define STMT_VINFO_GATHER_SCATTER_P(S)     (S)->gather_scatter_p 
 1669#define STMT_VINFO_STRIDED_P(S)            (S)->strided_p 
 1670#define STMT_VINFO_SIMD_LANE_ACCESS_P(S)   (S)->simd_lane_access_p 
 1671#define STMT_VINFO_REDUC_IDX(S)            (S)->reduc_idx 
 1673#define STMT_VINFO_DR_WRT_VEC_LOOP(S)      (S)->dr_wrt_vec_loop 
 1674#define STMT_VINFO_DR_BASE_ADDRESS(S)      (S)->dr_wrt_vec_loop.base_address 
 1675#define STMT_VINFO_DR_INIT(S)              (S)->dr_wrt_vec_loop.init 
 1676#define STMT_VINFO_DR_OFFSET(S)            (S)->dr_wrt_vec_loop.offset 
 1677#define STMT_VINFO_DR_STEP(S)              (S)->dr_wrt_vec_loop.step 
 1678#define STMT_VINFO_DR_BASE_ALIGNMENT(S)    (S)->dr_wrt_vec_loop.base_alignment 
 1679#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \ 
 1680  (S)->dr_wrt_vec_loop.base_misalignment 
 
 1681#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \ 
 1682  (S)->dr_wrt_vec_loop.offset_alignment 
 
 1683#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \ 
 1684  (S)->dr_wrt_vec_loop.step_alignment 
 
 1686#define STMT_VINFO_DR_INFO(S) \ 
 1687  (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux) 
 
 1689#define STMT_VINFO_IN_PATTERN_P(S)         (S)->in_pattern_p 
 1690#define STMT_VINFO_RELATED_STMT(S)         (S)->related_stmt 
 1691#define STMT_VINFO_PATTERN_DEF_SEQ(S)      (S)->pattern_def_seq 
 1692#define STMT_VINFO_DEF_TYPE(S)             (S)->def_type 
 1693#define STMT_VINFO_GROUPED_ACCESS(S) \ 
 1694  ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S)) 
 
 1695#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged 
 1696#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part 
 1697#define STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE(S) (S)->loop_phi_evolution_type 
 1698#define STMT_VINFO_MIN_NEG_DIST(S)      (S)->min_neg_dist 
 1699#define STMT_VINFO_REDUC_TYPE(S)        (S)->reduc_type 
 1700#define STMT_VINFO_REDUC_CODE(S)        (S)->reduc_code 
 1701#define STMT_VINFO_REDUC_DEF(S)         (S)->reduc_def 
 1702#define STMT_VINFO_SLP_VECT_ONLY(S)     (S)->slp_vect_only_p 
 1703#define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p 
 1704#define STMT_VINFO_REDUC_VECTYPE_IN(S)  (S)->reduc_vectype_in 
 1706#define DR_GROUP_FIRST_ELEMENT(S) \ 
 1707  (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element) 
 
 1708#define DR_GROUP_NEXT_ELEMENT(S) \ 
 1709  (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element) 
 
 1710#define DR_GROUP_SIZE(S) \ 
 1711  (gcc_checking_assert ((S)->dr_aux.dr), (S)->size) 
 
 1712#define DR_GROUP_GAP(S) \ 
 1713  (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap) 
 
 1715#define STMT_VINFO_RELEVANT_P(S)          ((S)->relevant != vect_unused_in_scope) 
 1717#define PURE_SLP_STMT(S)                  ((S)->slp_type == pure_slp) 
 1718#define STMT_SLP_TYPE(S)                   (S)->slp_type 
 1749                                      tree vectype, 
int misalign,
 
 
 1895#define VECT_MAX_COST 1000 
 1899#define MAX_INTERM_CVT_STEPS         3 
 1901#define MAX_VECTORIZATION_FACTOR INT_MAX 
 1908#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \ 
 1909  (TREE_CODE (TYPE) == BOOLEAN_TYPE             \ 
 1910   || ((TREE_CODE (TYPE) == INTEGER_TYPE        \ 
 1911        || TREE_CODE (TYPE) == ENUMERAL_TYPE)   \ 
 1912       && TYPE_PRECISION (TYPE) == 1            \ 
 1913       && TYPE_UNSIGNED (TYPE))) 
 
 1941  return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
 
 
 1950  return stmt_info->pattern_stmt_p;
 
 
 2019  for (
i = 0; 
i < x; 
i++)
 
 
 2029                            tree vectype, 
int misalign)
 
 2031  return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
 
 
 2048  return targetm.vectorize.create_costs (vinfo, costing_for_scalar);
 
 
 2061               tree vectype, 
int misalign,
 
 2066  if (
costs->costing_for_scalar ())
 
 2071  unsigned cost = 
costs->add_stmt_cost (
count, kind, stmt_info, 
node, vectype,
 
 
 2092                        i->vectype, 
i->misalign, 
i->where);
 
 
 2108#define DR_MISALIGNMENT_UNKNOWN (-1) 
 2109#define DR_MISALIGNMENT_UNINITIALIZED (-2) 
 2120#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL) 
 2128  return dr_info->target_alignment;
 
 
 2130#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR) 
 2131#define DR_SCALAR_KNOWN_BOUNDS(DR) (DR)->scalar_access_known_in_bounds 
 2143  return dr_info->safe_speculative_read_required;
 
 
 2150                                       bool requires_alignment)
 
 2158  dr_info->safe_speculative_read_required = requires_alignment;
 
 
 2164  dr_info->target_alignment = val;
 
 
 2166#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL) 
 2197  else if (misalignment == 0)
 
 2199  return misalignment & -misalignment;
 
 
 2211  if (loop_vinfo == 
NULL 
 
 2250    return flag_simd_cost_model;
 
 2251  return flag_vect_cost_model;
 
 
 2316  *max_nunits = force_common_multiple (*max_nunits, nunits);
 
 
 2352inline unsigned HOST_WIDE_INT
 
 2355  unsigned HOST_WIDE_INT vf;
 
 
 2390  return code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR;
 
 
 2418#define DUMP_VECT_SCOPE(MSG) \ 
 2419  AUTO_DUMP_SCOPE (MSG, vect_location) 
 
 2493                                                    vec<std::pair<tree, tree_code> > &,
 
 2556                                unsigned int *, 
unsigned int *,
 
 2567                            const char * = 
nullptr);
 
 2571                                                  tree *, 
unsigned int = 0);
 
 2623                                   const char * = 
NULL);
 
 2650                                unsigned int, 
tree, 
unsigned int);
 
 2652                                  tree, 
unsigned int);
 
 2655                               unsigned int, 
unsigned int);
 
 2658                                    unsigned int, 
tree, 
tree, 
unsigned int,
 
 2724                                          unsigned * = 
nullptr, 
bool = 
false);
 
 2744                                            unsigned int * = 
NULL,
 
 2793typedef hash_map <slp_tree, complex_perm_kinds_t>
 
 2797typedef pair_hash <nofree_ptr_hash <_slp_tree>,
 
 2825      this->m_node = 
node;
 
 2826      this->m_ops.create (0);
 
 2828        this->m_ops.safe_splice (*
m_ops);
 
 
 2843        this->m_ops.release ();
 
 
 
 2948  return (def_stmt_info
 
 
edge loop_preheader_edge(const class loop *loop)
Definition cfgloop.cc:1760
class loop * loop_p
Definition cfgloop.h:98
Definition tree-vectorizer.h:1384
vec< slp_root > roots
Definition tree-vectorizer.h:1389
_bb_vec_info(vec< basic_block > bbs, vec_info_shared *)
Definition tree-vect-slp.cc:8234
~_bb_vec_info()
Definition tree-vect-slp.cc:8271
Definition tree-vectorizer.h:914
tree mask_skip_niters_pfa_offset
Definition tree-vectorizer.h:1010
bool peeling_for_gaps
Definition tree-vectorizer.h:1145
bool can_use_partial_vectors_p
Definition tree-vectorizer.h:1105
poly_uint64 ptr_mask
Definition tree-vectorizer.h:1046
signed char partial_load_store_bias
Definition tree-vectorizer.h:1140
class vector_costs * vector_costs
Definition tree-vectorizer.h:932
class dr_vec_info * unaligned_dr
Definition tree-vectorizer.h:1033
_loop_vec_info(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:711
edge skip_main_loop_edge
Definition tree-vectorizer.h:959
unsigned int th
Definition tree-vectorizer.h:940
_loop_vec_info * epilogue_vinfo
Definition tree-vectorizer.h:1205
_loop_vec_info * main_loop_info
Definition tree-vectorizer.h:1196
auto_vec< gimple * > early_break_vuses
Definition tree-vectorizer.h:1234
bool using_partial_vectors_p
Definition tree-vectorizer.h:1112
bool no_data_dependencies
Definition tree-vectorizer.h:1182
class loop * scalar_loop
Definition tree-vectorizer.h:1192
tree drs_advanced_by
Definition tree-vectorizer.h:1208
scalar_cond_masked_set_type scalar_cond_masked_set
Definition tree-vectorizer.h:996
auto_vec< ddr_p > may_alias_ddrs
Definition tree-vectorizer.h:1056
tree num_itersm1
Definition tree-vectorizer.h:923
unsigned int inner_loop_cost_factor
Definition tree-vectorizer.h:1096
bool vectorizable
Definition tree-vectorizer.h:1099
vect_partial_vector_style partial_vector_style
Definition tree-vectorizer.h:1030
poly_uint64 max_spec_read_amount
Definition tree-vectorizer.h:1049
auto_vec< vect_reduc_info > reduc_infos
Definition tree-vectorizer.h:967
edge vec_epilogue_loop_iv_exit
Definition tree-vectorizer.h:1216
tree num_iters
Definition tree-vectorizer.h:925
hash_map< tree, tree > * scan_map
Definition tree-vectorizer.h:1092
int peeling_for_alignment
Definition tree-vectorizer.h:1043
vec_cond_masked_set_type vec_cond_masked_set
Definition tree-vectorizer.h:999
bool epil_using_partial_vectors_p
Definition tree-vectorizer.h:1128
bool early_breaks
Definition tree-vectorizer.h:1153
poly_uint64 versioning_threshold
Definition tree-vectorizer.h:946
auto_vec< vec_lower_bound > lower_bounds
Definition tree-vectorizer.h:1071
profile_probability scalar_loop_scaling
Definition tree-vectorizer.h:1188
auto_vec< gimple * > early_break_stores
Definition tree-vectorizer.h:1226
edge main_loop_edge
Definition tree-vectorizer.h:958
edge skip_this_loop_edge
Definition tree-vectorizer.h:963
bool must_use_partial_vectors_p
Definition tree-vectorizer.h:1108
auto_vec< stmt_info_for_cost > scalar_cost_vec
Definition tree-vectorizer.h:1085
gcond * loop_iv_cond
Definition tree-vectorizer.h:1159
basic_block early_break_dest_bb
Definition tree-vectorizer.h:1230
bool using_select_vl_p
Definition tree-vectorizer.h:1124
tree rgroup_compare_type
Definition tree-vectorizer.h:1015
poly_uint64 vectorization_factor
Definition tree-vectorizer.h:950
bool peeling_for_niter
Definition tree-vectorizer.h:1149
auto_vec< dr_with_seg_len_pair_t > comp_alias_ddrs
Definition tree-vectorizer.h:1060
edge vec_loop_iv_exit
Definition tree-vectorizer.h:1212
tree num_iters_unchanged
Definition tree-vectorizer.h:927
vec_loop_lens lens
Definition tree-vectorizer.h:993
unsigned HOST_WIDE_INT max_vectorization_factor
Definition tree-vectorizer.h:985
bool allow_mutual_alignment
Definition tree-vectorizer.h:1135
unsigned int suggested_unroll_factor
Definition tree-vectorizer.h:981
hash_map< tree_operand_hash, tree > * ivexpr_map
Definition tree-vectorizer.h:1088
auto_vec< stmt_vec_info > may_misalign_stmts
Definition tree-vectorizer.h:1075
tree rgroup_iv_type
Definition tree-vectorizer.h:1026
vec_loop_masks masks
Definition tree-vectorizer.h:989
tree num_iters_assumptions
Definition tree-vectorizer.h:929
tree simd_if_cond
Definition tree-vectorizer.h:1022
auto_vec< stmt_vec_info > alternate_defs
Definition tree-vectorizer.h:1082
auto_vec< tree > check_nonzero
Definition tree-vectorizer.h:1068
bool user_unroll
Definition tree-vectorizer.h:1163
auto_vec< stmt_vec_info > early_break_live_ivs
Definition tree-vectorizer.h:1238
tree mask_skip_niters
Definition tree-vectorizer.h:1005
bool has_mask_store
Definition tree-vectorizer.h:1185
auto_vec< stmt_vec_info > reductions
Definition tree-vectorizer.h:1078
edge scalar_loop_iv_exit
Definition tree-vectorizer.h:1220
class loop * loop
Definition tree-vectorizer.h:920
bool nonlinear_iv
Definition tree-vectorizer.h:1052
~_loop_vec_info()
Definition tree-vect-loop.cc:835
bool using_decrementing_iv_p
Definition tree-vectorizer.h:1119
auto_vec< gcond * > conds
Definition tree-vectorizer.h:1156
_loop_vec_info * orig_loop_info
Definition tree-vectorizer.h:1201
hash_map< tree, vect_reusable_accumulator > reusable_accumulators
Definition tree-vectorizer.h:974
class vector_costs * scalar_costs
Definition tree-vectorizer.h:935
auto_vec< vec_object_pair > check_unequal_addrs
Definition tree-vectorizer.h:1063
Definition tree-vectorizer.h:400
slp_instance_kind kind
Definition tree-vectorizer.h:427
vec< stmt_vec_info > root_stmts
Definition tree-vectorizer.h:407
slp_tree root
Definition tree-vectorizer.h:403
dump_user_location_t location() const
Definition tree-vect-slp.cc:206
vec< slp_tree > loads
Definition tree-vectorizer.h:414
vec< _slp_instance * > subgraph_entries
Definition tree-vectorizer.h:424
vec< tree > remain_defs
Definition tree-vectorizer.h:411
slp_tree reduc_phis
Definition tree-vectorizer.h:417
stmt_vector_for_cost cost_vec
Definition tree-vectorizer.h:420
Definition tree-vectorizer.h:1474
unsigned int min_input_precision
Definition tree-vectorizer.h:1594
code_helper reduc_code
Definition tree-vectorizer.h:1577
tree vectype
Definition tree-vectorizer.h:1498
bool in_pattern_p
Definition tree-vectorizer.h:1482
stmt_vec_info first_element
Definition tree-vectorizer.h:1544
unsigned int mask_precision
Definition tree-vectorizer.h:1617
int reduc_idx
Definition tree-vectorizer.h:1581
gimple * stmt
Definition tree-vectorizer.h:1495
tree loop_phi_evolution_base_unchanged
Definition tree-vectorizer.h:1516
enum vect_induction_op_type loop_phi_evolution_type
Definition tree-vectorizer.h:1518
innermost_loop_behavior dr_wrt_vec_loop
Definition tree-vectorizer.h:1508
bool slp_vect_only_p
Definition tree-vectorizer.h:1620
bool live
Definition tree-vectorizer.h:1479
unsigned int gap
Definition tree-vectorizer.h:1551
unsigned int min_output_precision
Definition tree-vectorizer.h:1589
unsigned int min_neg_dist
Definition tree-vectorizer.h:1555
bool vectorizable
Definition tree-vectorizer.h:1492
bool pattern_stmt_p
Definition tree-vectorizer.h:1488
bool strided_p
Definition tree-vectorizer.h:1566
enum vect_relevant relevant
Definition tree-vectorizer.h:1560
unsigned int size
Definition tree-vectorizer.h:1548
signop operation_sign
Definition tree-vectorizer.h:1600
gimple_seq pattern_def_seq
Definition tree-vectorizer.h:1534
stmt_vec_info related_stmt
Definition tree-vectorizer.h:1529
unsigned simd_lane_access_p
Definition tree-vectorizer.h:1569
bool slp_vect_pattern_only_p
Definition tree-vectorizer.h:1624
stmt_vec_info reduc_def
Definition tree-vectorizer.h:1585
bool gather_scatter_p
Definition tree-vectorizer.h:1563
enum vect_def_type def_type
Definition tree-vectorizer.h:1537
stmt_vec_info next_element
Definition tree-vectorizer.h:1546
unsigned int operation_precision
Definition tree-vectorizer.h:1599
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:1573
enum slp_vect_type slp_type
Definition tree-vectorizer.h:1540
dr_vec_info dr_aux
Definition tree-vectorizer.h:1504
tree loop_phi_evolution_part
Definition tree-vectorizer.h:1517
Definition tree-vectorizer.h:2434
~auto_purge_vect_location()
Definition tree-vectorizer.cc:94
Definition tree-vectorizer.h:1440
bool scalar_access_known_in_bounds
Definition tree-vectorizer.h:1463
unsigned group
Definition tree-vectorizer.h:1448
tree base_decl
Definition tree-vectorizer.h:1465
bool base_misaligned
Definition tree-vectorizer.h:1455
tree offset
Definition tree-vectorizer.h:1469
int misalignment
Definition tree-vectorizer.h:1450
poly_uint64 target_alignment
Definition tree-vectorizer.h:1453
data_reference * dr
Definition tree-vectorizer.h:1443
bool safe_speculative_read_required
Definition tree-vectorizer.h:1459
stmt_vec_info stmt
Definition tree-vectorizer.h:1445
Definition dumpfile.h:340
void add_int(unsigned v)
Definition inchash.h:55
hashval_t end() const
Definition inchash.h:49
void add_flag(bool flag)
Definition inchash.h:124
class loop * inner
Definition cfgloop.h:148
unsigned force_vectorize
Definition cfgloop.h:227
void * aux
Definition cfgloop.h:154
Definition opt-problem.h:179
Definition profile-count.h:149
Definition tree-vectorizer.h:575
vec< data_reference_p > datarefs
Definition tree-vectorizer.h:584
vec< data_reference > datarefs_copy
Definition tree-vectorizer.h:585
auto_vec< loop_p > loop_nest
Definition tree-vectorizer.h:588
void check_datarefs()
Definition tree-vectorizer.cc:507
void save_datarefs()
Definition tree-vectorizer.cc:497
~vec_info_shared()
Definition tree-vectorizer.cc:489
vec< ddr_p > ddrs
Definition tree-vectorizer.h:592
vec_info_shared()
Definition tree-vectorizer.cc:482
Definition tree-vectorizer.h:596
stmt_vec_info lookup_stmt(gimple *)
Definition tree-vectorizer.cc:571
basic_block * bbs
Definition tree-vectorizer.h:656
auto_vec< stmt_vec_info > grouped_stores
Definition tree-vectorizer.h:641
~vec_info()
Definition tree-vectorizer.cc:474
auto_vec< slp_instance > slp_instances
Definition tree-vectorizer.h:633
void replace_stmt(gimple_stmt_iterator *, stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:660
void remove_stmt(stmt_vec_info)
Definition tree-vectorizer.cc:644
void insert_on_entry(stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:702
void set_vinfo_for_stmt(gimple *, stmt_vec_info, bool=true)
Definition tree-vectorizer.cc:745
vec_base_alignments base_alignments
Definition tree-vectorizer.h:637
machine_mode vector_mode
Definition tree-vectorizer.h:649
stmt_vec_info resync_stmt_addr(gimple *)
Definition tree-vectorizer.cc:551
stmt_vec_info new_stmt_vec_info(gimple *stmt)
Definition tree-vectorizer.cc:713
class dr_vec_info * lookup_dr(data_reference *)
Definition tree-vectorizer.cc:613
vec< stmt_vec_info > stmt_vec_infos
Definition tree-vectorizer.h:624
void move_dr(stmt_vec_info, stmt_vec_info)
Definition tree-vectorizer.cc:625
bool any_known_not_updated_vssa
Definition tree-vectorizer.h:630
vec_kind
Definition tree-vectorizer.h:599
@ loop
Definition tree-vectorizer.h:599
@ bb
Definition tree-vectorizer.h:599
void free_stmt_vec_infos()
Definition tree-vectorizer.cc:766
vec_info_shared * shared
Definition tree-vectorizer.h:621
gimple_seq inv_pattern_def_seq
Definition tree-vectorizer.h:665
unsigned int nbbs
Definition tree-vectorizer.h:659
stmt_vec_info add_stmt(gimple *)
Definition tree-vectorizer.cc:522
vec_kind kind
Definition tree-vectorizer.h:618
mode_set used_vector_modes
Definition tree-vectorizer.h:644
hash_set< int_hash< machine_mode, E_VOIDmode, E_BLKmode > > mode_set
Definition tree-vectorizer.h:598
void insert_seq_on_entry(stmt_vec_info, gimple_seq)
Definition tree-vectorizer.cc:676
stmt_vec_info lookup_def(tree)
Definition tree-vectorizer.cc:588
void free_stmt_vec_info(stmt_vec_info)
Definition tree-vectorizer.cc:777
stmt_vec_info lookup_single_use(tree)
Definition tree-vectorizer.cc:601
vec_info(vec_kind, vec_info_shared *)
Definition tree-vectorizer.cc:463
stmt_vec_info add_pattern_stmt(gimple *, stmt_vec_info)
Definition tree-vectorizer.cc:534
bool stmt_vec_info_ro
Definition tree-vectorizer.h:626
vec_lower_bound()
Definition tree-vectorizer.h:564
poly_uint64 min_value
Definition tree-vectorizer.h:570
bool unsigned_p
Definition tree-vectorizer.h:569
vec_lower_bound(tree e, bool u, poly_uint64 m)
Definition tree-vectorizer.h:565
tree expr
Definition tree-vectorizer.h:568
Definition vec-perm-indices.h:51
Definition tree-vectorizer.h:2806
static vect_pattern * recognize(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
internal_fn m_ifn
Definition tree-vectorizer.h:2812
vec< slp_tree > m_ops
Definition tree-vectorizer.h:2819
virtual ~vect_pattern()
Definition tree-vectorizer.h:2841
virtual void build(vec_info *)=0
vect_pattern(slp_tree *node, vec< slp_tree > *m_ops, internal_fn ifn)
Definition tree-vectorizer.h:2822
slp_tree * m_node
Definition tree-vectorizer.h:2815
unsigned m_num_args
Definition tree-vectorizer.h:2809
Definition tree-vectorizer.h:827
tree reduc_epilogue_adjustment
Definition tree-vectorizer.h:866
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:835
struct vect_reusable_accumulator * reused_accumulator
Definition tree-vectorizer.h:870
internal_fn reduc_fn
Definition tree-vectorizer.h:841
tree reduc_vectype_for_mask
Definition tree-vectorizer.h:860
bool is_reduc_chain
Definition tree-vectorizer.h:850
code_helper reduc_code
Definition tree-vectorizer.h:838
auto_vec< tree > reduc_initial_values
Definition tree-vectorizer.h:874
tree induc_cond_initial_val
Definition tree-vectorizer.h:863
tree reduc_vectype
Definition tree-vectorizer.h:856
bool force_single_cycle
Definition tree-vectorizer.h:853
enum vect_def_type def_type
Definition tree-vectorizer.h:831
unsigned int reduc_result_pos
Definition tree-vectorizer.h:847
auto_vec< tree > reduc_scalar_results
Definition tree-vectorizer.h:880
Definition tree-vectorizer.h:1723
unsigned int adjust_cost_for_freq(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1874
virtual ~vector_costs()
Definition tree-vectorizer.h:1726
bool m_costing_for_scalar
Definition tree-vectorizer.h:1796
vector_costs(vec_info *, bool)
Definition tree-vectorizer.h:1821
bool costing_for_scalar() const
Definition tree-vectorizer.h:1781
vec_info * m_vinfo
Definition tree-vectorizer.h:1792
virtual unsigned int add_stmt_cost(int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, vect_cost_model_location where)
Definition tree-vectorizer.cc:1836
machine_mode m_suggested_epilogue_mode
Definition tree-vectorizer.h:1810
virtual bool better_main_loop_than_p(const vector_costs *other) const
Definition tree-vectorizer.cc:1894
unsigned int m_costs[3]
Definition tree-vectorizer.h:1799
unsigned int outside_cost() const
Definition tree-vectorizer.h:1862
unsigned int record_stmt_cost(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1860
virtual bool better_epilogue_loop_than_p(const vector_costs *other, loop_vec_info main_loop) const
Definition tree-vectorizer.cc:1913
unsigned int epilogue_cost() const
Definition tree-vectorizer.h:1853
int m_masked_epilogue
Definition tree-vectorizer.h:1811
machine_mode suggested_epilogue_mode(int &masked) const
Definition tree-vectorizer.h:1888
unsigned int m_suggested_unroll_factor
Definition tree-vectorizer.h:1802
unsigned int suggested_unroll_factor() const
Definition tree-vectorizer.h:1879
bool m_finished
Definition tree-vectorizer.h:1814
int compare_outside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:2089
unsigned int body_cost() const
Definition tree-vectorizer.h:1844
virtual void finish_cost(const vector_costs *scalar_costs)
Definition tree-vectorizer.cc:1849
unsigned int prologue_cost() const
Definition tree-vectorizer.h:1835
unsigned int total_cost() const
Definition tree-vectorizer.h:1871
int compare_inside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:1996
struct basic_block_def * basic_block
Definition coretypes.h:370
class edge_def * edge
Definition coretypes.h:367
gimple * gimple_seq
Definition coretypes.h:100
const class edge_def * const_edge
Definition coretypes.h:368
const union tree_node * const_tree
Definition coretypes.h:98
union tree_node * tree
Definition coretypes.h:97
static unsigned int count[debug_counter_number_of_counters]
Definition dbgcnt.cc:50
bool dominated_by_p(enum cdi_direction dir, const_basic_block bb1, const_basic_block bb2)
Definition dominance.cc:1125
@ CDI_DOMINATORS
Definition dominance.h:25
dump_flags_t dump_flags
Definition dumpfile.cc:65
FILE * dump_file
Definition dumpfile.cc:63
@ TDF_DETAILS
Definition dumpfile.h:92
vect_cost_model
Definition flag-types.h:279
@ VECT_COST_MODEL_UNLIMITED
Definition flag-types.h:283
@ VECT_COST_MODEL_DEFAULT
Definition flag-types.h:284
bool operand_equal_p(const_tree arg0, const_tree arg1, unsigned int flags)
Definition fold-const.cc:4355
#define fold_convert(T1, T2)
Definition fold-const.h:96
#define fold_build2(c, t1, t2, t3)
Definition fold-const.h:76
static struct token T
Definition gengtype-parse.cc:45
internal_fn
Definition genmatch.cc:1015
tree_code
Definition genmatch.cc:1002
unsigned gimple_uid(const gimple *g)
Definition gimple.h:2091
enum tree_code gimple_assign_rhs_code(const gassign *gs)
Definition gimple.h:2857
tree gimple_assign_rhs1(const gassign *gs)
Definition gimple.h:2667
basic_block gimple_bb(const gimple *g)
Definition gimple.h:1867
tree gimple_assign_lhs(const gassign *gs)
Definition gimple.h:2616
#define init_cost
Definition ira-costs.cc:63
T dyn_cast(U *p)
Definition is-a.h:280
Definition custom-sarif-properties/state-graphs.h:33
void add_expr(const_tree t, inchash::hash &hstate, unsigned int flags)
Definition fold-const.cc:4369
opt_pointer_wrapper< tree > opt_tree
Definition opt-problem.h:287
poly_int< NUM_POLY_INT_COEFFS, unsigned HOST_WIDE_INT > poly_uint64
Definition poly-int-types.h:25
poly_int< NUM_POLY_INT_COEFFS, HOST_WIDE_INT > poly_int64
Definition poly-int-types.h:24
i
Definition poly-int.h:776
signop
Definition signop.h:28
Definition tree-vectorizer.h:300
stmt_vec_info representative
Definition tree-vectorizer.h:316
tree gs_base
Definition tree-vectorizer.h:352
unsigned int lanes
Definition tree-vectorizer.h:345
bool avoid_stlf_fail
Definition tree-vectorizer.h:358
load_permutation_t load_permutation
Definition tree-vectorizer.h:327
slp_tree prev_node
Definition tree-vectorizer.h:383
enum tree_code code
Definition tree-vectorizer.h:347
bool ldst_lanes
Definition tree-vectorizer.h:355
vec< tree > vec_defs
Definition tree-vectorizer.h:335
~_slp_tree()
Definition tree-vect-slp.cc:141
T & get_data(T &else_)
Definition tree-vectorizer.h:368
int gs_scale
Definition tree-vectorizer.h:350
unsigned int refcnt
Definition tree-vectorizer.h:338
enum stmt_vec_info_type type
Definition tree-vectorizer.h:364
slp_tree next_node
Definition tree-vectorizer.h:382
int vertex
Definition tree-vectorizer.h:360
enum vect_def_type def_type
Definition tree-vectorizer.h:343
bool * failed
Definition tree-vectorizer.h:373
vec< tree > ops
Definition tree-vectorizer.h:313
void push_vec_def(gimple *def)
Definition tree-vect-slp.cc:164
int reduc_idx
Definition tree-vectorizer.h:322
poly_uint64 max_nunits
Definition tree-vectorizer.h:341
lane_permutation_t lane_permutation
Definition tree-vectorizer.h:331
_slp_tree()
Definition tree-vect-slp.cc:108
int id
Definition tree-vectorizer.h:320
struct _slp_tree::@120146070312105203102335350023136130264031166255 cycle_info
tree vectype
Definition tree-vectorizer.h:333
void push_vec_def(tree def)
Definition tree-vectorizer.h:305
vec< slp_tree > children
Definition tree-vectorizer.h:308
vect_data * data
Definition tree-vectorizer.h:365
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:311
class loop * loop_father
Definition basic-block.h:126
Definition ira-costs.cc:54
Definition tree-data-ref.h:151
struct innermost_loop_behavior innermost
Definition tree-data-ref.h:173
Definition loop-invariant.cc:88
static const bool empty_zero_p
Definition tree-vectorizer.h:523
static void mark_empty(value_type &v)
Definition tree-vectorizer.h:526
scalar_cond_masked_key value_type
Definition tree-vectorizer.h:499
static bool equal(value_type existing, value_type candidate)
Definition tree-vectorizer.h:514
static void remove(value_type &)
Definition tree-vectorizer.h:545
static hashval_t hash(value_type v)
Definition tree-vectorizer.h:502
static void mark_deleted(value_type &)
Definition tree-vectorizer.h:538
static bool is_deleted(const value_type &)
Definition tree-vectorizer.h:540
static bool is_empty(value_type v)
Definition tree-vectorizer.h:533
scalar_cond_masked_key compare_type
Definition tree-vectorizer.h:498
Definition hash-traits.h:466
Definition graphite.h:338
data_reference_p dr
Definition graphite.h:343
Definition function.h:249
Definition tree-vectorizer.h:1628
tree decl
Definition tree-vectorizer.h:1635
tree element_type
Definition tree-vectorizer.h:1655
tree base
Definition tree-vectorizer.h:1638
tree offset_vectype
Definition tree-vectorizer.h:1652
tree alias_ptr
Definition tree-vectorizer.h:1642
int scale
Definition tree-vectorizer.h:1649
tree offset
Definition tree-vectorizer.h:1645
internal_fn ifn
Definition tree-vectorizer.h:1631
tree memory_type
Definition tree-vectorizer.h:1658
Definition gimple-iterator.h:26
Definition tree-data-ref.h:50
tree offset
Definition tree-data-ref.h:52
Definition hash-traits.h:122
Definition hash-traits.h:345
Definition tree-vectorizer.h:778
tree compare_type
Definition tree-vectorizer.h:798
unsigned int max_nscalars_per_iter
Definition tree-vectorizer.h:782
unsigned int factor
Definition tree-vectorizer.h:788
tree type
Definition tree-vectorizer.h:794
tree bias_adjusted_ctrl
Definition tree-vectorizer.h:805
vec< tree > controls
Definition tree-vectorizer.h:801
unsigned ncopies
Definition tree-vectorizer.h:488
tree_code code
Definition tree-vectorizer.h:490
scalar_cond_masked_key(tree t, unsigned ncopies_)
Definition tree-vectorizer.h:480
void get_cond_ops_from_tree(tree)
Definition tree-vectorizer.cc:1786
tree op0
Definition tree-vectorizer.h:491
tree op1
Definition tree-vectorizer.h:492
bool inverted_p
Definition tree-vectorizer.h:489
vec< tree > remain
Definition tree-vectorizer.h:1380
slp_instance_kind kind
Definition tree-vectorizer.h:1377
vec< stmt_vec_info > roots
Definition tree-vectorizer.h:1379
slp_root(slp_instance_kind kind_, vec< stmt_vec_info > stmts_, vec< stmt_vec_info > roots_, vec< tree > remain_=vNULL)
Definition tree-vectorizer.h:1374
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:1378
Definition tree-vectorizer.h:120
tree vectype
Definition tree-vectorizer.h:126
int misalign
Definition tree-vectorizer.h:127
slp_tree node
Definition tree-vectorizer.h:125
enum vect_cost_model_location where
Definition tree-vectorizer.h:123
enum vect_cost_for_stmt kind
Definition tree-vectorizer.h:122
stmt_vec_info stmt_info
Definition tree-vectorizer.h:124
int count
Definition tree-vectorizer.h:121
Definition tree-hash-traits.h:25
Definition hash-traits.h:75
Definition tree-vectorizer.h:809
hash_set< pair_hash< nofree_ptr_hash< tree_node >, int_hash< unsigned, 0 > > > mask_set
Definition tree-vectorizer.h:814
auto_vec< rgroup_controls > rgc_vec
Definition tree-vectorizer.h:817
bool is_empty() const
Definition tree-vectorizer.h:810
Definition tree-vectorizer.h:257
virtual ~vect_data()=default
Definition tree-vectorizer.h:276
int misalignment
Definition tree-vectorizer.h:283
tree decl
Definition tree-vectorizer.h:288
virtual ~vect_load_store_data()=default
vect_load_store_data()=default
internal_fn ifn
Definition tree-vectorizer.h:287
unsigned n_perms
Definition tree-vectorizer.h:295
poly_int64 poffset
Definition tree-vectorizer.h:285
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:281
vect_load_store_data(vect_load_store_data &&other)=default
tree strided_offset_vectype
Definition tree-vectorizer.h:290
tree ls_type
Definition tree-vectorizer.h:291
union vect_load_store_data::@336230030006311336100276173266177172342371306160 gs
auto_vec< int > elsvals
Definition tree-vectorizer.h:292
dr_alignment_support alignment_support_scheme
Definition tree-vectorizer.h:282
internal_fn lanes_ifn
Definition tree-vectorizer.h:284
bool slp_perm
Definition tree-vectorizer.h:294
Definition tree-vectorizer.h:901
tree reduc_input
Definition tree-vectorizer.h:904
vect_reduc_info reduc_info
Definition tree-vectorizer.h:908
Definition tree-vectorizer.h:159
static void mark_deleted(value_type &s)
Definition tree-vectorizer.h:165
static bool is_empty(const value_type &s)
Definition tree-vectorizer.h:168
static bool is_deleted(const value_type &s)
Definition tree-vectorizer.h:167
static const bool empty_zero_p
Definition tree-vectorizer.h:163
static void mark_empty(value_type &s)
Definition tree-vectorizer.h:166
static hashval_t hash(const value_type &)
Definition tree-vect-slp.cc:8452
vect_scalar_ops_slice value_type
Definition tree-vectorizer.h:160
static bool equal(const value_type &, const compare_type &)
Definition tree-vect-slp.cc:8461
vect_scalar_ops_slice compare_type
Definition tree-vectorizer.h:161
Definition tree-vectorizer.h:141
unsigned int length
Definition tree-vectorizer.h:147
unsigned int start
Definition tree-vectorizer.h:146
bool all_same_p() const
Definition tree-vect-slp.cc:8443
tree op(unsigned int i) const
Definition tree-vectorizer.h:152
vec< tree > * ops
Definition tree-vectorizer.h:145
virtual ~vect_simd_clone_data()=default
vect_simd_clone_data(vect_simd_clone_data &&other)=default
vect_simd_clone_data()=default
auto_vec< tree > simd_clone_info
Definition tree-vectorizer.h:271
#define NULL
Definition system.h:50
#define gcc_assert(EXPR)
Definition system.h:814
#define gcc_unreachable()
Definition system.h:841
#define false
Definition system.h:888
#define gcc_checking_assert(EXPR)
Definition system.h:821
HOST_WIDE_INT estimated_poly_value(poly_int64 x, poly_value_estimate_kind kind=POLY_VALUE_LIKELY)
Definition target.h:337
struct gcc_target targetm
vect_cost_model_location
Definition target.h:211
@ vect_body
Definition target.h:213
@ vect_prologue
Definition target.h:212
@ vect_epilogue
Definition target.h:214
vect_cost_for_stmt
Definition target.h:189
@ scalar_stmt
Definition target.h:190
@ cond_branch_taken
Definition target.h:203
@ vec_to_scalar
Definition target.h:200
@ cond_branch_not_taken
Definition target.h:202
@ tcc_comparison
Definition tree-core.h:240
@ vec_kind
Definition tree-core.h:1042
#define DR_IS_WRITE(DR)
Definition tree-data-ref.h:192
#define DR_IS_READ(DR)
Definition tree-data-ref.h:191
#define DR_REF(DR)
Definition tree-data-ref.h:185
#define DR_INNERMOST(DR)
Definition tree-data-ref.h:203
static tree candidate(unsigned uid)
Definition tree-sra.cc:326
#define PHI_ARG_DEF_FROM_EDGE(PHI, E)
Definition tree-ssa-operands.h:84
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset)
Definition tree-vect-data-refs.cc:1303
vect_pattern_decl_t slp_patterns[]
Definition tree-vect-slp-patterns.cc:1703
_complex_operation
Definition tree-vect-slp-patterns.cc:119
size_t num__slp_patterns
Definition tree-vect-slp-patterns.cc:1714
dump_user_location_t vect_location
Definition tree-vectorizer.cc:88
stmt_vec_info vect_orig_stmt(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1957
#define LOOP_VINFO_COST_MODEL_THRESHOLD(L)
Definition tree-vectorizer.h:1255
bool vect_can_vectorize_without_simd_p(tree_code)
Definition tree-vect-loop.cc:8809
tree cse_and_gimplify_to_preheader(loop_vec_info, tree)
Definition tree-vect-loop.cc:859
int builtin_vectorization_cost(enum vect_cost_for_stmt type_of_cost, tree vectype, int misalign)
Definition tree-vectorizer.h:2028
loop_vec_info loop_vec_info_for_loop(class loop *loop)
Definition tree-vectorizer.h:1367
bool vect_is_extending_load(class vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2931
bool vect_transform_stmt(vec_info *, stmt_vec_info, gimple_stmt_iterator *, slp_tree, slp_instance)
Definition tree-vect-stmts.cc:12926
bool vect_is_simple_use(tree, vec_info *, enum vect_def_type *, stmt_vec_info *=NULL, gimple **=NULL)
Definition tree-vect-stmts.cc:13393
bool vect_rgroup_iv_might_wrap_p(loop_vec_info, rgroup_controls *)
Definition tree-vect-loop.cc:11644
tree vect_halve_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:10499
void dump_stmt_cost(FILE *, int, enum vect_cost_for_stmt, stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location)
Definition tree-vectorizer.cc:102
vec< unsigned > load_permutation_t
Definition tree-vectorizer.h:254
bool vect_slp_analyze_instance_alignment(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:3142
internal_fn vect_store_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:6045
tree vect_get_new_vect_var(tree, enum vect_var_kind, const char *)
Definition tree-vect-data-refs.cc:5311
void vect_mark_pattern_stmts(vec_info *, stmt_vec_info, gimple *, tree)
Definition tree-vect-patterns.cc:7435
class loop * vect_loop_versioning(loop_vec_info, gimple *)
Definition tree-vect-loop-manip.cc:4121
tree vect_get_slp_vect_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10807
opt_tree vect_get_mask_type_for_stmt(stmt_vec_info, unsigned int=0)
#define LOOP_VINFO_VECT_FACTOR(L)
Definition tree-vectorizer.h:1267
void vect_free_slp_instance(slp_instance)
Definition tree-vect-slp.cc:218
#define SLP_TREE_LANES(S)
Definition tree-vectorizer.h:450
bool vectorizable_early_exit(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:12502
bool vect_slp_function(function *)
Definition tree-vect-slp.cc:10340
std::pair< tree, tree > vec_object_pair
Definition tree-vectorizer.h:558
tree vect_gen_perm_mask_any(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9360
tree vect_setup_realignment(vec_info *, stmt_vec_info, tree, gimple_stmt_iterator *, tree *, enum dr_alignment_support, tree, class loop **)
Definition tree-vect-data-refs.cc:6121
_bb_vec_info * bb_vec_info
#define LOOP_VINFO_FULLY_MASKED_P(L)
Definition tree-vectorizer.h:1320
int vect_get_known_peeling_cost(loop_vec_info, int, int *, stmt_vector_for_cost *, stmt_vector_for_cost *, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:3908
opt_result vect_analyze_data_refs(vec_info *, bool *)
Definition tree-vect-data-refs.cc:5058
bool compatible_calls_p(gcall *, gcall *, bool)
Definition tree-vect-slp.cc:1000
bool vect_transform_slp_perm_load(vec_info *, slp_tree, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, unsigned *, unsigned *=nullptr, bool=false)
tree vect_gen_while_not(gimple_seq *, tree, tree, tree)
Definition tree-vect-stmts.cc:14247
#define VECT_REDUC_INFO_TYPE(I)
Definition tree-vectorizer.h:886
tree vect_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2921
bool is_simple_and_all_uses_invariant(stmt_vec_info, loop_vec_info)
Definition tree-vect-stmts.cc:322
#define STMT_VINFO_GROUPED_ACCESS(S)
Definition tree-vectorizer.h:1693
void vect_schedule_slp(vec_info *, const vec< slp_instance > &)
Definition tree-vect-slp.cc:12230
tree vect_peel_nonlinear_iv_init(gimple_seq *, tree, tree, tree, enum vect_induction_op_type)
Definition tree-vect-loop.cc:8935
bool check_reduction_path(dump_user_location_t, loop_p, gphi *, tree, enum tree_code)
Definition tree-vect-loop.cc:3626
bool can_duplicate_and_interleave_p(vec_info *, unsigned int, tree, unsigned int *=NULL, tree *=NULL, tree *=NULL)
Definition tree-vect-slp.cc:430
auto_vec< unsigned, 16 > auto_load_permutation_t
Definition tree-vectorizer.h:255
#define SLP_TREE_VECTYPE(S)
Definition tree-vectorizer.h:448
tree get_vectype_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13216
#define LOOP_VINFO_LOOP(L)
Definition tree-vectorizer.h:1242
_complex_perm_kinds
Definition tree-vectorizer.h:2782
@ PERM_EVENEVEN
Definition tree-vectorizer.h:2787
@ PERM_TOP
Definition tree-vectorizer.h:2789
@ PERM_ODDEVEN
Definition tree-vectorizer.h:2785
@ PERM_EVENODD
Definition tree-vectorizer.h:2784
@ PERM_UNKNOWN
Definition tree-vectorizer.h:2783
@ PERM_ODDODD
Definition tree-vectorizer.h:2786
#define SLP_TREE_TYPE(S)
Definition tree-vectorizer.h:452
void vect_record_base_alignments(vec_info *)
Definition tree-vect-data-refs.cc:1384
vec_load_store_type
Definition tree-vectorizer.h:1434
@ VLS_LOAD
Definition tree-vectorizer.h:1435
@ VLS_STORE
Definition tree-vectorizer.h:1436
@ VLS_STORE_INVARIANT
Definition tree-vectorizer.h:1437
tree get_mask_type_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13291
pair_hash< tree_operand_hash, tree_operand_hash > tree_cond_mask_hash
Definition tree-vectorizer.h:553
_loop_vec_info * loop_vec_info
vect_reduc_info info_for_reduction(loop_vec_info, slp_tree)
Definition tree-vect-loop.cc:5016
tree vect_double_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:10509
vect_var_kind
Definition tree-vectorizer.h:36
@ vect_pointer_var
Definition tree-vectorizer.h:38
@ vect_scalar_var
Definition tree-vectorizer.h:39
@ vect_mask_var
Definition tree-vectorizer.h:40
@ vect_simple_var
Definition tree-vectorizer.h:37
int vect_get_place_in_interleaving_chain(stmt_vec_info, stmt_vec_info)
Definition tree-vect-slp.cc:401
class loop * vect_do_peeling(loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, tree *)
Definition tree-vect-loop-manip.cc:3135
void optimize_mask_stores(class loop *)
Definition tree-vect-loop.cc:11408
class _stmt_vec_info * stmt_vec_info
Definition tree-vectorizer.h:24
#define STMT_VINFO_DATA_REF(S)
Definition tree-vectorizer.h:1667
tree vect_get_loop_len(loop_vec_info, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:10696
bool vect_transform_cycle_phi(loop_vec_info, stmt_vec_info, slp_tree, slp_instance)
Definition tree-vect-loop.cc:8205
unsigned int vect_get_num_copies(vec_info *vinfo, slp_tree node)
Definition tree-vectorizer.h:2292
bool vect_is_store_elt_extraction(vect_cost_for_stmt kind, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2868
opt_result vect_get_vector_types_for_stmt(vec_info *, stmt_vec_info, tree *, tree *, unsigned int=0)
Definition tree-vect-stmts.cc:14271
const poly_uint64 dr_target_alignment(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2124
void vect_update_inits_of_drs(loop_vec_info, tree, tree_code)
Definition tree-vect-loop-manip.cc:2615
struct data_reference * dr_p
Definition tree-vectorizer.h:1472
bool vect_can_advance_ivs_p(loop_vec_info)
Definition tree-vect-loop-manip.cc:2211
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vect-data-refs.cc:1303
tree vect_init_vector(vec_info *, stmt_vec_info, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1206
void vect_copy_ref_info(tree, tree)
Definition tree-vect-data-refs.cc:5871
bool vect_transform_reduction(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree)
Definition tree-vect-loop.cc:7861
bool vectorizable_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:8420
bool vectorizable_lane_reducing(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:6747
tree vect_get_new_ssa_name(tree, enum vect_var_kind, const char *=NULL)
Definition tree-vect-data-refs.cc:5349
#define STMT_VINFO_VECTYPE(S)
Definition tree-vectorizer.h:1665
opt_result vect_prune_runtime_alias_test_list(loop_vec_info)
Definition tree-vect-data-refs.cc:4171
void vect_describe_gather_scatter_call(stmt_vec_info, gather_scatter_info *)
Definition tree-vect-data-refs.cc:4535
bool needs_fold_left_reduction_p(tree, code_helper)
Definition tree-vect-loop.cc:3387
bool vect_nop_conversion_p(stmt_vec_info)
Definition tree-vect-stmts.cc:5790
class vect_reduc_info_s * vect_reduc_info
Definition tree-vectorizer.h:883
void vect_finish_stmt_generation(vec_info *, stmt_vec_info, gimple *, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1331
#define LOOP_VINFO_NON_LINEAR_IV(L)
Definition tree-vectorizer.h:1283
unsigned HOST_WIDE_INT vect_max_vf(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2353
internal_fn vect_load_lanes_supported(tree, unsigned HOST_WIDE_INT, bool, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:6475
loop_vec_info vect_create_loop_vinfo(class loop *, vec_info_shared *, const vect_loop_form_info *, loop_vec_info=nullptr)
Definition tree-vect-loop.cc:1639
bool vect_stmt_dominates_stmt_p(gimple *, gimple *)
Definition tree-vectorizer.cc:793
tree vect_get_slp_scalar_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10789
bool vect_supportable_shift(vec_info *, enum tree_code, tree)
Definition tree-vect-stmts.cc:5969
void vect_optimize_slp(vec_info *)
Definition tree-vect-slp.cc:8105
void vect_iv_increment_position(edge, gimple_stmt_iterator *, bool *)
Definition tree-vect-loop-manip.cc:457
hash_set< scalar_cond_masked_key > scalar_cond_masked_set_type
Definition tree-vectorizer.h:548
tree vect_phi_initial_value(gphi *phi)
Definition tree-vectorizer.h:1927
opt_pointer_wrapper< loop_vec_info > opt_loop_vec_info
Definition tree-vectorizer.h:1364
bool vectorizable_reduction(loop_vec_info, stmt_vec_info, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:6907
unsigned record_stmt_cost(stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location)
Definition tree-vect-stmts.cc:113
bool vect_emulated_vector_p(tree)
Definition tree-vect-loop.cc:8798
tree get_same_sized_vectype(tree, tree)
Definition tree-vect-stmts.cc:13327
peeling_support
Definition tree-vectorizer.h:61
@ peeling_known_supported
Definition tree-vectorizer.h:62
@ peeling_maybe_supported
Definition tree-vectorizer.h:63
@ peeling_unsupported
Definition tree-vectorizer.h:64
gimple * vect_gimple_build(tree, code_helper, tree, tree=NULL_TREE)
Definition tree-vect-patterns.cc:7760
void add_stmt_costs(vector_costs *costs, stmt_vector_for_cost *cost_vec)
Definition tree-vectorizer.h:2096
void vect_finish_replace_stmt(vec_info *, stmt_vec_info, gimple *)
Definition tree-vect-stmts.cc:1315
tree prepare_vec_mask(loop_vec_info, tree, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1608
void vect_slp_fini(void)
Definition tree-vect-slp.cc:83
#define LOOP_VINFO_NITERS_KNOWN_P(L)
Definition tree-vectorizer.h:1351
stmt_vec_info get_later_stmt(stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
Definition tree-vectorizer.h:1967
vec< std::pair< unsigned, unsigned > > lane_permutation_t
Definition tree-vectorizer.h:252
tree vect_gen_loop_len_mask(loop_vec_info, gimple_stmt_iterator *, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:10763
widest_int vect_iv_limit_for_partial_vectors(loop_vec_info loop_vinfo)
Definition tree-vect-loop.cc:11603
void vect_get_vec_defs(vec_info *, slp_tree, tree, vec< tree > *, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL)
Definition tree-vect-stmts.cc:1268
void vect_gather_slp_loads(vec_info *)
Definition tree-vect-slp.cc:8124
vect_partial_vector_style
Definition tree-vectorizer.h:467
@ vect_partial_vectors_none
Definition tree-vectorizer.h:468
@ vect_partial_vectors_len
Definition tree-vectorizer.h:471
@ vect_partial_vectors_avx512
Definition tree-vectorizer.h:470
@ vect_partial_vectors_while_ult
Definition tree-vectorizer.h:469
bool vect_gather_scatter_fn_p(vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4443
class _slp_instance * slp_instance
bool vect_maybe_update_slp_op_vectype(slp_tree, tree)
Definition tree-vect-stmts.cc:13544
#define STMT_VINFO_REDUC_IDX(S)
Definition tree-vectorizer.h:1671
class loop * slpeel_tree_duplicate_loop_to_edge_cfg(class loop *, edge, class loop *, edge, edge, edge *, bool=true, vec< basic_block > *=NULL)
Definition tree-vect-loop-manip.cc:1478
unsigned int vect_vf_for_cost(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2335
unsigned int vect_nunits_for_cost(tree vec_type)
Definition tree-vectorizer.h:2345
bool vect_get_range_info(tree, wide_int *, wide_int *)
Definition tree-vect-patterns.cc:69
bool vect_grouped_load_supported(tree, bool, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:6358
#define STMT_VINFO_IN_PATTERN_P(S)
Definition tree-vectorizer.h:1689
tree vect_create_addr_base_for_vector_ref(vec_info *, stmt_vec_info, gimple_seq *, tree)
Definition tree-vect-data-refs.cc:5424
stmt_vec_info vect_find_last_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3532
tree vect_get_main_loop_result(loop_vec_info, tree, tree)
Definition tree-vect-loop-manip.cc:3026
#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L)
Definition tree-vectorizer.h:1282
#define STMT_VINFO_DR_INFO(S)
Definition tree-vectorizer.h:1686
hash_map< slp_node_hash, bool > slp_compat_nodes_map_t
Definition tree-vectorizer.h:2799
tree get_related_vectype_for_scalar_type(machine_mode, tree, poly_uint64=0)
Definition tree-vect-stmts.cc:13098
bool lane_reducing_op_p(code_helper code)
Definition tree-vectorizer.h:2388
bool nested_in_vect_loop_p(class loop *loop, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1916
bool vect_slp_analyze_instance_dependence(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:1164
bool vect_detect_hybrid_slp(loop_vec_info)
auto_vec< std::pair< data_reference *, tree > > drs_init_vec
Definition tree-vectorizer.h:822
edge vec_init_loop_exit_info(class loop *)
Definition tree-vect-loop.cc:655
opt_result vect_analyze_data_ref_accesses(vec_info *, vec< int > *)
Definition tree-vect-data-refs.cc:3636
int vect_get_stmt_cost(enum vect_cost_for_stmt type_of_cost)
Definition tree-vectorizer.h:2038
opt_result vect_analyze_stmt(vec_info *, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:12776
dump_user_location_t find_loop_location(class loop *)
Definition tree-vect-loop-manip.cc:2070
bool vectorizable_recurr(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:8629
bool vect_slp_analyze_operations(vec_info *)
Definition tree-vect-slp.cc:8988
tree neutral_op_for_reduction(tree, code_helper, tree, bool=true)
Definition tree-vect-loop.cc:3329
void vect_free_slp_tree(slp_tree)
Definition tree-vect-slp.cc:178
unsigned vectorize_loops(void)
bool vect_can_force_dr_alignment_p(const_tree, poly_uint64)
Definition tree-vect-data-refs.cc:6504
tree bump_vector_ptr(vec_info *, tree, gimple *, gimple_stmt_iterator *, stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:5807
bool vect_apply_runtime_profitability_check_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2378
unsigned int vect_get_num_vectors(poly_uint64 nunits, tree vectype)
Definition tree-vectorizer.h:2283
#define DR_TARGET_ALIGNMENT(DR)
Definition tree-vectorizer.h:2130
void vect_free_loop_info_assumptions(class loop *)
Definition tree-vectorizer.cc:855
void set_dr_target_alignment(dr_vec_info *dr_info, poly_uint64 val)
Definition tree-vectorizer.h:2162
stmt_vec_info vect_stmt_to_vectorize(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1994
bool supportable_indirect_convert_operation(code_helper, tree, tree, vec< std::pair< tree, tree_code > > &, tree=NULL_TREE, slp_tree=NULL)
Definition tree-vect-stmts.cc:14088
opt_loop_vec_info vect_analyze_loop(class loop *, gimple *, vec_info_shared *)
Definition tree-vect-loop.cc:2894
void dr_set_safe_speculative_read_required(stmt_vec_info stmt_info, bool requires_alignment)
Definition tree-vectorizer.h:2149
#define STMT_VINFO_RELATED_STMT(S)
Definition tree-vectorizer.h:1690
bool is_pattern_stmt_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1948
#define MAX_VECTORIZATION_FACTOR
Definition tree-vectorizer.h:1901
int vect_pow2(int x)
Definition tree-vectorizer.h:2015
bool reduction_fn_for_scalar_code(code_helper, internal_fn *)
Definition tree-vect-loop.cc:3245
void vect_pattern_recog(vec_info *)
Definition tree-vect-patterns.cc:7724
int vect_slp_child_index_for_operand(const gimple *, int op, bool)
Definition tree-vect-slp.cc:620
vect_induction_op_type
Definition tree-vectorizer.h:83
@ vect_step_op_add
Definition tree-vectorizer.h:84
@ vect_step_op_shl
Definition tree-vectorizer.h:87
@ vect_step_op_mul
Definition tree-vectorizer.h:86
@ vect_step_op_neg
Definition tree-vectorizer.h:85
@ vect_step_op_shr
Definition tree-vectorizer.h:88
void vect_set_loop_condition(class loop *, edge, loop_vec_info, tree, tree, tree, bool)
Definition tree-vect-loop-manip.cc:1393
bool lane_reducing_stmt_p(gimple *stmt)
Definition tree-vectorizer.h:2396
void vect_prepare_for_masked_peels(loop_vec_info)
Definition tree-vect-loop-manip.cc:2645
void vect_get_store_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:1007
stmt_vec_info vect_find_first_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3550
enum _complex_perm_kinds complex_perm_kinds_t
bool vect_is_integer_truncation(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2955
auto_vec< std::pair< unsigned, unsigned >, 16 > auto_lane_permutation_t
Definition tree-vectorizer.h:253
bool vect_use_loop_mask_for_alignment_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2266
tree vect_build_loop_niters(loop_vec_info, bool *=NULL)
Definition tree-vect-loop-manip.cc:2691
bool vect_get_loop_mask_type(loop_vec_info)
void vect_update_max_nunits(poly_uint64 *max_nunits, poly_uint64 nunits)
Definition tree-vectorizer.h:2311
slp_instance_kind
Definition tree-vectorizer.h:389
@ slp_inst_kind_ctor
Definition tree-vectorizer.h:394
@ slp_inst_kind_reduc_group
Definition tree-vectorizer.h:391
@ slp_inst_kind_store
Definition tree-vectorizer.h:390
@ slp_inst_kind_reduc_chain
Definition tree-vectorizer.h:392
@ slp_inst_kind_gcond
Definition tree-vectorizer.h:395
@ slp_inst_kind_bb_reduc
Definition tree-vectorizer.h:393
enum vect_cost_model loop_cost_model(loop_p loop)
Definition tree-vectorizer.h:2245
unsigned int vect_known_alignment_in_bytes(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vectorizer.h:2191
#define DR_MISALIGNMENT_UNKNOWN
Definition tree-vectorizer.h:2108
bool mat_gather_scatter_p(vect_memory_access_type mat)
Definition tree-vectorizer.h:215
hash_map< slp_tree, complex_perm_kinds_t > slp_tree_to_load_perm_map_t
Definition tree-vectorizer.h:2794
opt_result vect_enhance_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2332
bool vect_is_reduction(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2877
#define LOOP_VINFO_EARLY_BREAKS(L)
Definition tree-vectorizer.h:1297
enum dr_alignment_support vect_supportable_dr_alignment(vec_info *, dr_vec_info *, tree, int, bool=false)
Definition tree-vect-data-refs.cc:6529
bool vectorizable_induction(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9411
#define SLP_TREE_REDUC_IDX(S)
Definition tree-vectorizer.h:455
innermost_loop_behavior * vect_dr_behavior(vec_info *vinfo, dr_vec_info *dr_info)
Definition tree-vectorizer.h:2207
tree perm_mask_for_reverse(tree)
Definition tree-vect-stmts.cc:1883
vect_memory_access_type SLP_TREE_MEMORY_ACCESS_TYPE(slp_tree node)
Definition tree-vectorizer.h:459
stmt_vec_info_type
Definition tree-vectorizer.h:225
@ comparison_vec_info_type
Definition tree-vectorizer.h:235
@ condition_vec_info_type
Definition tree-vectorizer.h:234
@ type_conversion_vec_info_type
Definition tree-vectorizer.h:240
@ load_vec_info_type
Definition tree-vectorizer.h:227
@ loop_exit_ctrl_vec_info_type
Definition tree-vectorizer.h:245
@ permute_info_type
Definition tree-vectorizer.h:246
@ recurr_info_type
Definition tree-vectorizer.h:244
@ op_vec_info_type
Definition tree-vectorizer.h:230
@ reduc_vec_info_type
Definition tree-vectorizer.h:236
@ assignment_vec_info_type
Definition tree-vectorizer.h:233
@ undef_vec_info_type
Definition tree-vectorizer.h:226
@ shift_vec_info_type
Definition tree-vectorizer.h:229
@ type_demotion_vec_info_type
Definition tree-vectorizer.h:239
@ store_vec_info_type
Definition tree-vectorizer.h:228
@ cycle_phi_info_type
Definition tree-vectorizer.h:241
@ phi_info_type
Definition tree-vectorizer.h:243
@ lc_phi_info_type
Definition tree-vectorizer.h:242
@ type_promotion_vec_info_type
Definition tree-vectorizer.h:238
@ call_simd_clone_vec_info_type
Definition tree-vectorizer.h:232
@ call_vec_info_type
Definition tree-vectorizer.h:231
@ induc_vec_info_type
Definition tree-vectorizer.h:237
void vect_get_slp_defs(slp_tree, vec< tree > *)
Definition tree-vect-slp.cc:10815
void vect_record_loop_len(loop_vec_info, vec_loop_lens *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:10661
struct _slp_tree * slp_tree
Definition tree-vectorizer.h:25
gimple_seq vect_gen_len(tree, tree, tree, tree)
Definition tree-vect-stmts.cc:14435
bool dr_safe_speculative_read_required(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2135
bool vect_check_gather_scatter(stmt_vec_info, tree, loop_vec_info, gather_scatter_info *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4562
tree vect_embedded_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2906
gimple * vect_loop_vectorized_call(class loop *, gcond **cond=NULL)
Definition tree-vectorizer.cc:870
void duplicate_and_interleave(vec_info *, gimple_seq *, tree, const vec< tree > &, unsigned int, vec< tree > &)
Definition tree-vect-slp.cc:10472
bool vect_grouped_store_supported(tree, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:5927
void vect_get_load_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool)
Definition tree-vect-stmts.cc:1059
vect_def_type
Definition tree-vectorizer.h:68
@ vect_induction_def
Definition tree-vectorizer.h:73
@ vect_uninitialized_def
Definition tree-vectorizer.h:69
@ vect_nested_cycle
Definition tree-vectorizer.h:76
@ vect_constant_def
Definition tree-vectorizer.h:70
@ vect_external_def
Definition tree-vectorizer.h:71
@ vect_condition_def
Definition tree-vectorizer.h:78
@ vect_internal_def
Definition tree-vectorizer.h:72
@ vect_reduction_def
Definition tree-vectorizer.h:74
@ vect_double_reduction_def
Definition tree-vectorizer.h:75
@ vect_first_order_recurrence
Definition tree-vectorizer.h:77
@ vect_unknown_def_type
Definition tree-vectorizer.h:79
opt_result vect_analyze_data_ref_dependences(loop_vec_info, unsigned int *)
Definition tree-vect-data-refs.cc:887
vect_relevant
Definition tree-vectorizer.h:1401
@ vect_used_in_outer
Definition tree-vectorizer.h:1411
@ vect_used_in_scope
Definition tree-vectorizer.h:1421
@ vect_unused_in_scope
Definition tree-vectorizer.h:1402
@ vect_used_by_reduction
Definition tree-vectorizer.h:1419
@ vect_used_in_outer_by_reduction
Definition tree-vectorizer.h:1408
@ vect_used_only_live
Definition tree-vectorizer.h:1405
tree vect_get_mask_load_else(int, tree)
Definition tree-vect-stmts.cc:2883
bool is_loop_header_bb_p(basic_block bb)
Definition tree-vectorizer.h:2004
opt_result vect_analyze_loop_form(class loop *, gimple *, vect_loop_form_info *)
Definition tree-vect-loop.cc:1432
#define STMT_VINFO_DR_WRT_VEC_LOOP(S)
Definition tree-vectorizer.h:1673
bool vectorizable_live_operation(vec_info *, stmt_vec_info, slp_tree, slp_instance, int, bool, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:10095
slp_tree vect_create_new_slp_node(unsigned, tree_code)
Definition tree-vect-slp.cc:233
opt_result vect_analyze_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:3074
bool unlimited_cost_model(loop_p loop)
Definition tree-vectorizer.h:2256
opt_result vect_mark_stmts_to_be_vectorized(loop_vec_info, bool *)
Definition tree-vect-stmts.cc:701
tree vect_gen_while(gimple_seq *, tree, tree, tree, const char *=nullptr)
Definition tree-vect-stmts.cc:14223
tree vect_get_store_rhs(stmt_vec_info)
Definition tree-vect-stmts.cc:1964
class loop * vect_transform_loop(loop_vec_info, gimple *)
Definition tree-vect-loop.cc:11015
void vect_slp_init(void)
Definition tree-vect-slp.cc:77
unsigned int vect_get_scalar_dr_size(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2369
opt_result vect_determine_partial_vectors_and_peeling(loop_vec_info)
Definition tree-vect-loop.cc:2022
bool ref_within_array_bound(gimple *, tree)
Definition tree-if-conv.cc:852
dr_alignment_support
Definition tree-vectorizer.h:51
@ dr_unaligned_supported
Definition tree-vectorizer.h:53
@ dr_explicit_realign_optimized
Definition tree-vectorizer.h:55
@ dr_aligned
Definition tree-vectorizer.h:56
@ dr_explicit_realign
Definition tree-vectorizer.h:54
@ dr_unaligned_unsupported
Definition tree-vectorizer.h:52
int compare_step_with_zero(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:1872
bool supportable_narrowing_operation(code_helper, tree, tree, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:13915
operation_type
Definition tree-vectorizer.h:44
@ ternary_op
Definition tree-vectorizer.h:47
@ unary_op
Definition tree-vectorizer.h:45
@ binary_op
Definition tree-vectorizer.h:46
void vect_record_loop_mask(loop_vec_info, vec_loop_masks *, unsigned int, tree, tree)
Definition tree-vect-loop.cc:10521
vect_memory_access_type
Definition tree-vectorizer.h:175
@ VMAT_STRIDED_SLP
Definition tree-vectorizer.h:204
@ VMAT_GATHER_SCATTER_LEGACY
Definition tree-vectorizer.h:207
@ VMAT_GATHER_SCATTER_EMULATED
Definition tree-vectorizer.h:209
@ VMAT_INVARIANT
Definition tree-vectorizer.h:179
@ VMAT_UNINITIALIZED
Definition tree-vectorizer.h:176
@ VMAT_ELEMENTWISE
Definition tree-vectorizer.h:198
@ VMAT_GATHER_SCATTER_IFN
Definition tree-vectorizer.h:208
@ VMAT_CONTIGUOUS_DOWN
Definition tree-vectorizer.h:187
@ VMAT_CONTIGUOUS
Definition tree-vectorizer.h:182
@ VMAT_LOAD_STORE_LANES
Definition tree-vectorizer.h:194
@ VMAT_CONTIGUOUS_REVERSE
Definition tree-vectorizer.h:191
bool vect_chooses_same_modes_p(vec_info *, machine_mode)
Definition tree-vect-stmts.cc:13345
bool slpeel_can_duplicate_loop_p(const class loop *, const_edge, const_edge)
Definition tree-vect-loop-manip.cc:2038
tree vect_get_loop_mask(loop_vec_info, gimple_stmt_iterator *, vec_loop_masks *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:10543
slp_vect_type
Definition tree-vectorizer.h:1427
@ not_vect
Definition tree-vectorizer.h:1428
@ pure_slp
Definition tree-vectorizer.h:1429
vect_reduction_type
Definition tree-vectorizer.h:92
@ TREE_CODE_REDUCTION
Definition tree-vectorizer.h:93
@ FOLD_LEFT_REDUCTION
Definition tree-vectorizer.h:111
@ EXTRACT_LAST_REDUCTION
Definition tree-vectorizer.h:103
@ INTEGER_INDUC_COND_REDUCTION
Definition tree-vectorizer.h:95
@ CONST_COND_REDUCTION
Definition tree-vectorizer.h:96
@ COND_REDUCTION
Definition tree-vectorizer.h:94
bool supportable_widening_operation(vec_info *, code_helper, stmt_vec_info, tree, tree, code_helper *, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:13584
auto_vec< rgroup_controls > vec_loop_lens
Definition tree-vectorizer.h:820
void set_dr_misalignment(dr_vec_info *dr_info, int val)
Definition tree-vectorizer.h:2112
bool vect_use_mask_type_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1939
pair_hash< nofree_ptr_hash< _slp_tree >, nofree_ptr_hash< _slp_tree > > slp_node_hash
Definition tree-vectorizer.h:2798
hash_map< tree_operand_hash, std::pair< stmt_vec_info, innermost_loop_behavior * > > vec_base_alignments
Definition tree-vectorizer.h:136
unsigned add_stmt_cost(vector_costs *costs, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where)
Definition tree-vectorizer.h:2058
tree vect_get_smallest_scalar_type(stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:161
vec< stmt_info_for_cost > stmt_vector_for_cost
Definition tree-vectorizer.h:130
bool known_alignment_for_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2182
void vect_remove_stores(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:13068
#define DR_GROUP_FIRST_ELEMENT(S)
Definition tree-vectorizer.h:1706
hash_set< tree_cond_mask_hash > vec_cond_masked_set_type
Definition tree-vectorizer.h:554
opt_result vect_find_stmt_data_reference(loop_p, gimple *, vec< data_reference_p > *, vec< int > *, int)
Definition tree-vect-data-refs.cc:4883
tree get_dr_vinfo_offset(vec_info *vinfo, dr_vec_info *dr_info, bool check_outer=false)
Definition tree-vectorizer.h:2223
bool vect_slp_if_converted_bb(basic_block bb, loop_p orig_loop)
Definition tree-vect-slp.cc:10329
bool vectorizable_slp_permutation(vec_info *, gimple_stmt_iterator *, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-slp.cc:11615
tree vect_gen_perm_mask_checked(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9375
opt_result vect_analyze_slp(vec_info *, unsigned, bool)
Definition tree-vect-slp.cc:5489
vect_pattern *(* vect_pattern_decl_t)(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
Definition tree-vectorizer.h:2848
tree vect_create_destination_var(tree, tree)
Definition tree-vect-data-refs.cc:5893
bool vect_transform_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:8460
tree vect_create_data_ref_ptr(vec_info *, stmt_vec_info, tree, class loop *, tree, tree *, gimple_stmt_iterator *, gimple **, bool, tree=NULL_TREE)
Definition tree-vect-data-refs.cc:5547
int vect_reduc_type(vec_info *vinfo, slp_tree node)
Definition tree-vectorizer.h:2892
void vect_gen_vector_loop_niters(loop_vec_info, tree, tree *, tree *, bool)
Definition tree-vect-loop-manip.cc:2782
bool vectorizable_phi(bb_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:8487
bool vect_make_slp_decision(loop_vec_info)
Definition tree-vect-slp.cc:8182
bool aligned_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2172
unsigned HOST_WIDE_INT tree_to_uhwi(const_tree t)
Definition tree.cc:6672
#define TYPE_ALIGN_UNIT(NODE)
Definition tree.h:2454
poly_uint64 TYPE_VECTOR_SUBPARTS(const_tree node)
Definition tree.h:4365
#define TYPE_PRECISION(NODE)
Definition tree.h:2360
#define TREE_OPERAND(NODE, I)
Definition tree.h:1319
#define sizetype
Definition tree.h:5242
#define TREE_CODE_CLASS(CODE)
Definition tree.h:203
#define TYPE_SIZE_UNIT(NODE)
Definition tree.h:2357
#define CONVERT_EXPR_CODE_P(CODE)
Definition tree.h:554
#define TREE_TYPE(NODE)
Definition tree.h:513
#define INTEGRAL_TYPE_P(TYPE)
Definition tree.h:614
#define COMPARISON_CLASS_P(NODE)
Definition tree.h:257
#define NULL_TREE
Definition tree.h:318
#define FOR_EACH_VEC_ELT(V, I, P)
Definition vec.h:1895
constexpr vnull vNULL
Definition vec.h:569
generic_wide_int< widest_int_storage< WIDEST_INT_MAX_PRECISION > > widest_int
Definition wide-int.h:345
generic_wide_int< wide_int_storage > wide_int
Definition wide-int.h:343