21#ifndef GCC_TREE_VECTORIZER_H
22#define GCC_TREE_VECTORIZER_H
114#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
115 || ((D) == vect_double_reduction_def) \
116 || ((D) == vect_nested_cycle))
135 std::pair<stmt_vec_info, innermost_loop_behavior *> >
142 tree op (
unsigned int i)
const;
373 static void *
operator new (size_t);
376 static void operator delete (
void *, size_t);
431#define SLP_INSTANCE_TREE(S) (S)->root
432#define SLP_INSTANCE_LOADS(S) (S)->loads
433#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
434#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
435#define SLP_INSTANCE_KIND(S) (S)->kind
437#define SLP_TREE_CHILDREN(S) (S)->children
438#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
439#define SLP_TREE_SCALAR_OPS(S) (S)->ops
440#define SLP_TREE_REF_COUNT(S) (S)->refcnt
441#define SLP_TREE_VEC_DEFS(S) (S)->vec_defs
442#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
443#define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
444#define SLP_TREE_DEF_TYPE(S) (S)->def_type
445#define SLP_TREE_VECTYPE(S) (S)->vectype
446#define SLP_TREE_REPRESENTATIVE(S) (S)->representative
447#define SLP_TREE_LANES(S) (S)->lanes
448#define SLP_TREE_CODE(S) (S)->code
449#define SLP_TREE_TYPE(S) (S)->type
450#define SLP_TREE_GS_SCALE(S) (S)->gs_scale
451#define SLP_TREE_GS_BASE(S) (S)->gs_base
452#define SLP_TREE_REDUC_IDX(S) (S)->cycle_info.reduc_idx
453#define SLP_TREE_PERMUTE_P(S) ((S)->code == VEC_PERM_EXPR)
498 static inline hashval_t
875#define VECT_REDUC_INFO_DEF_TYPE(I) ((I)->def_type)
876#define VECT_REDUC_INFO_TYPE(I) ((I)->reduc_type)
877#define VECT_REDUC_INFO_CODE(I) ((I)->reduc_code)
878#define VECT_REDUC_INFO_FN(I) ((I)->reduc_fn)
879#define VECT_REDUC_INFO_SCALAR_RESULTS(I) ((I)->reduc_scalar_results)
880#define VECT_REDUC_INFO_INITIAL_VALUES(I) ((I)->reduc_initial_values)
881#define VECT_REDUC_INFO_REUSED_ACCUMULATOR(I) ((I)->reused_accumulator)
882#define VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL(I) ((I)->induc_cond_initial_val)
883#define VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT(I) ((I)->reduc_epilogue_adjustment)
884#define VECT_REDUC_INFO_VECTYPE(I) ((I)->reduc_vectype)
885#define VECT_REDUC_INFO_FORCE_SINGLE_CYCLE(I) ((I)->force_single_cycle)
886#define VECT_REDUC_INFO_RESULT_POS(I) ((I)->reduc_result_pos)
1238#define LOOP_VINFO_LOOP(L) (L)->loop
1239#define LOOP_VINFO_IV_EXIT(L) (L)->vec_loop_iv_exit
1240#define LOOP_VINFO_EPILOGUE_IV_EXIT(L) (L)->vec_epilogue_loop_iv_exit
1241#define LOOP_VINFO_SCALAR_IV_EXIT(L) (L)->scalar_loop_iv_exit
1242#define LOOP_VINFO_BBS(L) (L)->bbs
1243#define LOOP_VINFO_NBBS(L) (L)->nbbs
1244#define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1
1245#define LOOP_VINFO_NITERS(L) (L)->num_iters
1249#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
1250#define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions
1251#define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th
1252#define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold
1253#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
1254#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
1255#define LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P(L) (L)->must_use_partial_vectors_p
1256#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
1257#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
1258#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
1259#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \
1260 (L)->epil_using_partial_vectors_p
1261#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment
1262#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
1263#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
1264#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
1265#define LOOP_VINFO_MASKS(L) (L)->masks
1266#define LOOP_VINFO_LENS(L) (L)->lens
1267#define LOOP_VINFO_MASK_SKIP_NITERS(L) (L)->mask_skip_niters
1268#define LOOP_VINFO_MASK_NITERS_PFA_OFFSET(L) (L)->mask_skip_niters_pfa_offset
1269#define LOOP_VINFO_RGROUP_COMPARE_TYPE(L) (L)->rgroup_compare_type
1270#define LOOP_VINFO_RGROUP_IV_TYPE(L) (L)->rgroup_iv_type
1271#define LOOP_VINFO_PARTIAL_VECTORS_STYLE(L) (L)->partial_vector_style
1272#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
1273#define LOOP_VINFO_MAX_SPEC_READ_AMOUNT(L) (L)->max_spec_read_amount
1274#define LOOP_VINFO_LOOP_NEST(L) (L)->shared->loop_nest
1275#define LOOP_VINFO_DATAREFS(L) (L)->shared->datarefs
1276#define LOOP_VINFO_DDRS(L) (L)->shared->ddrs
1277#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
1278#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
1279#define LOOP_VINFO_NON_LINEAR_IV(L) (L)->nonlinear_iv
1280#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
1281#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
1282#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
1283#define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs
1284#define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs
1285#define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero
1286#define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds
1287#define LOOP_VINFO_USER_UNROLL(L) (L)->user_unroll
1288#define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
1289#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
1290#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
1291#define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
1292#define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
1293#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
1294#define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter
1295#define LOOP_VINFO_EARLY_BREAKS(L) (L)->early_breaks
1296#define LOOP_VINFO_EARLY_BRK_STORES(L) (L)->early_break_stores
1297#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L) \
1298 (single_pred ((L)->loop->latch) != (L)->vec_loop_iv_exit->src)
1299#define LOOP_VINFO_EARLY_BREAKS_LIVE_IVS(L) \
1300 (L)->early_break_live_ivs
1301#define LOOP_VINFO_EARLY_BRK_DEST_BB(L) (L)->early_break_dest_bb
1302#define LOOP_VINFO_EARLY_BRK_VUSES(L) (L)->early_break_vuses
1303#define LOOP_VINFO_LOOP_CONDS(L) (L)->conds
1304#define LOOP_VINFO_LOOP_IV_COND(L) (L)->loop_iv_cond
1305#define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
1306#define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop
1307#define LOOP_VINFO_SCALAR_LOOP_SCALING(L) (L)->scalar_loop_scaling
1308#define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store
1309#define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
1310#define LOOP_VINFO_MAIN_LOOP_INFO(L) (L)->main_loop_info
1311#define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info
1312#define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
1313#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
1314#define LOOP_VINFO_INV_PATTERN_DEF_SEQ(L) (L)->inv_pattern_def_seq
1315#define LOOP_VINFO_DRS_ADVANCED_BY(L) (L)->drs_advanced_by
1316#define LOOP_VINFO_ALTERNATE_DEFS(L) (L)->alternate_defs
1318#define LOOP_VINFO_FULLY_MASKED_P(L) \
1319 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1320 && !LOOP_VINFO_MASKS (L).is_empty ())
1322#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \
1323 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1324 && !LOOP_VINFO_LENS (L).is_empty ())
1326#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
1327 ((L)->may_misalign_stmts.length () > 0)
1328#define LOOP_REQUIRES_VERSIONING_FOR_SPEC_READ(L) \
1329 (maybe_gt ((L)->max_spec_read_amount, 0U))
1330#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
1331 ((L)->comp_alias_ddrs.length () > 0 \
1332 || (L)->check_unequal_addrs.length () > 0 \
1333 || (L)->lower_bounds.length () > 0)
1334#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \
1335 (LOOP_VINFO_NITERS_ASSUMPTIONS (L))
1336#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \
1337 (LOOP_VINFO_SIMD_IF_COND (L))
1338#define LOOP_REQUIRES_VERSIONING(L) \
1339 (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \
1340 || LOOP_REQUIRES_VERSIONING_FOR_SPEC_READ (L) \
1341 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \
1342 || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
1343 || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
1345#define LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING(L) \
1346 ((L)->may_misalign_stmts.length () > 0 \
1347 && !LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (L))
1349#define LOOP_VINFO_NITERS_KNOWN_P(L) \
1350 (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
1352#define LOOP_VINFO_EPILOGUE_P(L) \
1353 (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL)
1355#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \
1356 (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L)))
1390#define BB_VINFO_BBS(B) (B)->bbs
1391#define BB_VINFO_NBBS(B) (B)->nbbs
1392#define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
1393#define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
1394#define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
1395#define BB_VINFO_DDRS(B) (B)->shared->ddrs
1660#define STMT_VINFO_STMT(S) (S)->stmt
1661#define STMT_VINFO_RELEVANT(S) (S)->relevant
1662#define STMT_VINFO_LIVE_P(S) (S)->live
1663#define STMT_VINFO_VECTYPE(S) (S)->vectype
1664#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
1665#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
1666#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
1667#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
1668#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
1669#define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
1671#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop
1672#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address
1673#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init
1674#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset
1675#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step
1676#define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment
1677#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \
1678 (S)->dr_wrt_vec_loop.base_misalignment
1679#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \
1680 (S)->dr_wrt_vec_loop.offset_alignment
1681#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
1682 (S)->dr_wrt_vec_loop.step_alignment
1684#define STMT_VINFO_DR_INFO(S) \
1685 (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
1687#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
1688#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
1689#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
1690#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
1691#define STMT_VINFO_GROUPED_ACCESS(S) \
1692 ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
1693#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
1694#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
1695#define STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE(S) (S)->loop_phi_evolution_type
1696#define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
1697#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
1698#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
1699#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
1700#define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
1701#define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p
1702#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
1704#define DR_GROUP_FIRST_ELEMENT(S) \
1705 (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
1706#define DR_GROUP_NEXT_ELEMENT(S) \
1707 (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
1708#define DR_GROUP_SIZE(S) \
1709 (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
1710#define DR_GROUP_GAP(S) \
1711 (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
1713#define REDUC_GROUP_FIRST_ELEMENT(S) \
1714 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
1715#define REDUC_GROUP_NEXT_ELEMENT(S) \
1716 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
1717#define REDUC_GROUP_SIZE(S) \
1718 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
1720#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
1722#define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
1723#define STMT_SLP_TYPE(S) (S)->slp_type
1754 tree vectype,
int misalign,
1900#define VECT_MAX_COST 1000
1904#define MAX_INTERM_CVT_STEPS 3
1906#define MAX_VECTORIZATION_FACTOR INT_MAX
1913#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \
1914 (TREE_CODE (TYPE) == BOOLEAN_TYPE \
1915 || ((TREE_CODE (TYPE) == INTEGER_TYPE \
1916 || TREE_CODE (TYPE) == ENUMERAL_TYPE) \
1917 && TYPE_PRECISION (TYPE) == 1 \
1918 && TYPE_UNSIGNED (TYPE)))
1946 return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
1955 return stmt_info->pattern_stmt_p;
2024 for (
i = 0;
i < x;
i++)
2034 tree vectype,
int misalign)
2036 return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
2053 return targetm.vectorize.create_costs (vinfo, costing_for_scalar);
2066 tree vectype,
int misalign,
2071 if (
costs->costing_for_scalar ())
2076 unsigned cost =
costs->add_stmt_cost (
count, kind, stmt_info, node, vectype,
2097 i->vectype,
i->misalign,
i->where);
2113#define DR_MISALIGNMENT_UNKNOWN (-1)
2114#define DR_MISALIGNMENT_UNINITIALIZED (-2)
2125#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
2133 return dr_info->target_alignment;
2135#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR)
2136#define DR_SCALAR_KNOWN_BOUNDS(DR) (DR)->scalar_access_known_in_bounds
2148 return dr_info->safe_speculative_read_required;
2155 bool requires_alignment)
2163 dr_info->safe_speculative_read_required = requires_alignment;
2169 dr_info->target_alignment = val;
2171#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL)
2202 else if (misalignment == 0)
2204 return misalignment & -misalignment;
2216 if (loop_vinfo ==
NULL
2255 return flag_simd_cost_model;
2256 return flag_vect_cost_model;
2321 *max_nunits = force_common_multiple (*max_nunits, nunits);
2357inline unsigned HOST_WIDE_INT
2360 unsigned HOST_WIDE_INT vf;
2395 return code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR;
2423#define DUMP_VECT_SCOPE(MSG) \
2424 AUTO_DUMP_SCOPE (MSG, vect_location)
2498 vec<std::pair<tree, tree_code> > &,
2561 unsigned int *,
unsigned int *,
2572 const char * =
nullptr);
2576 tree *,
unsigned int = 0);
2628 const char * =
NULL);
2655 unsigned int,
tree,
unsigned int);
2657 tree,
unsigned int);
2660 unsigned int,
unsigned int);
2663 unsigned int,
tree,
tree,
unsigned int,
2729 unsigned * =
nullptr,
bool =
false);
2749 unsigned int * =
NULL,
2798typedef hash_map <slp_tree, complex_perm_kinds_t>
2802typedef pair_hash <nofree_ptr_hash <_slp_tree>,
2830 this->m_node = node;
2831 this->m_ops.create (0);
2833 this->m_ops.safe_splice (*
m_ops);
2848 this->m_ops.release ();
2953 return (def_stmt_info
edge loop_preheader_edge(const class loop *loop)
Definition cfgloop.cc:1760
class loop * loop_p
Definition cfgloop.h:98
Definition tree-vectorizer.h:1382
vec< slp_root > roots
Definition tree-vectorizer.h:1387
_bb_vec_info(vec< basic_block > bbs, vec_info_shared *)
Definition tree-vect-slp.cc:7969
~_bb_vec_info()
Definition tree-vect-slp.cc:8006
Definition tree-vectorizer.h:903
tree mask_skip_niters_pfa_offset
Definition tree-vectorizer.h:998
bool peeling_for_gaps
Definition tree-vectorizer.h:1141
bool can_use_partial_vectors_p
Definition tree-vectorizer.h:1101
poly_uint64 ptr_mask
Definition tree-vectorizer.h:1034
signed char partial_load_store_bias
Definition tree-vectorizer.h:1136
class vector_costs * vector_costs
Definition tree-vectorizer.h:921
class dr_vec_info * unaligned_dr
Definition tree-vectorizer.h:1021
_loop_vec_info(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:801
edge skip_main_loop_edge
Definition tree-vectorizer.h:947
auto_vec< stmt_vec_info > reduction_chains
Definition tree-vectorizer.h:1070
unsigned int th
Definition tree-vectorizer.h:929
_loop_vec_info * epilogue_vinfo
Definition tree-vectorizer.h:1201
_loop_vec_info * main_loop_info
Definition tree-vectorizer.h:1192
auto_vec< gimple * > early_break_vuses
Definition tree-vectorizer.h:1230
bool using_partial_vectors_p
Definition tree-vectorizer.h:1108
bool no_data_dependencies
Definition tree-vectorizer.h:1178
class loop * scalar_loop
Definition tree-vectorizer.h:1188
tree drs_advanced_by
Definition tree-vectorizer.h:1204
scalar_cond_masked_set_type scalar_cond_masked_set
Definition tree-vectorizer.h:984
auto_vec< ddr_p > may_alias_ddrs
Definition tree-vectorizer.h:1044
tree num_itersm1
Definition tree-vectorizer.h:912
unsigned int inner_loop_cost_factor
Definition tree-vectorizer.h:1092
bool vectorizable
Definition tree-vectorizer.h:1095
vect_partial_vector_style partial_vector_style
Definition tree-vectorizer.h:1018
poly_uint64 slp_unrolling_factor
Definition tree-vectorizer.h:1088
poly_uint64 max_spec_read_amount
Definition tree-vectorizer.h:1037
auto_vec< vect_reduc_info > reduc_infos
Definition tree-vectorizer.h:955
edge vec_epilogue_loop_iv_exit
Definition tree-vectorizer.h:1212
tree num_iters
Definition tree-vectorizer.h:914
hash_map< tree, tree > * scan_map
Definition tree-vectorizer.h:1084
int peeling_for_alignment
Definition tree-vectorizer.h:1031
vec_cond_masked_set_type vec_cond_masked_set
Definition tree-vectorizer.h:987
bool epil_using_partial_vectors_p
Definition tree-vectorizer.h:1124
bool early_breaks
Definition tree-vectorizer.h:1149
poly_uint64 versioning_threshold
Definition tree-vectorizer.h:935
auto_vec< vec_lower_bound > lower_bounds
Definition tree-vectorizer.h:1059
profile_probability scalar_loop_scaling
Definition tree-vectorizer.h:1184
auto_vec< gimple * > early_break_stores
Definition tree-vectorizer.h:1222
edge main_loop_edge
Definition tree-vectorizer.h:946
edge skip_this_loop_edge
Definition tree-vectorizer.h:951
bool must_use_partial_vectors_p
Definition tree-vectorizer.h:1104
auto_vec< stmt_info_for_cost > scalar_cost_vec
Definition tree-vectorizer.h:1077
gcond * loop_iv_cond
Definition tree-vectorizer.h:1155
basic_block early_break_dest_bb
Definition tree-vectorizer.h:1226
bool using_select_vl_p
Definition tree-vectorizer.h:1120
tree rgroup_compare_type
Definition tree-vectorizer.h:1003
poly_uint64 vectorization_factor
Definition tree-vectorizer.h:938
bool peeling_for_niter
Definition tree-vectorizer.h:1145
auto_vec< dr_with_seg_len_pair_t > comp_alias_ddrs
Definition tree-vectorizer.h:1048
edge vec_loop_iv_exit
Definition tree-vectorizer.h:1208
tree num_iters_unchanged
Definition tree-vectorizer.h:916
vec_loop_lens lens
Definition tree-vectorizer.h:981
unsigned HOST_WIDE_INT max_vectorization_factor
Definition tree-vectorizer.h:973
bool allow_mutual_alignment
Definition tree-vectorizer.h:1131
unsigned int suggested_unroll_factor
Definition tree-vectorizer.h:969
hash_map< tree_operand_hash, tree > * ivexpr_map
Definition tree-vectorizer.h:1080
auto_vec< stmt_vec_info > may_misalign_stmts
Definition tree-vectorizer.h:1063
tree rgroup_iv_type
Definition tree-vectorizer.h:1014
vec_loop_masks masks
Definition tree-vectorizer.h:977
tree num_iters_assumptions
Definition tree-vectorizer.h:918
tree simd_if_cond
Definition tree-vectorizer.h:1010
auto_vec< stmt_vec_info > alternate_defs
Definition tree-vectorizer.h:1074
auto_vec< tree > check_nonzero
Definition tree-vectorizer.h:1056
bool user_unroll
Definition tree-vectorizer.h:1159
auto_vec< stmt_vec_info > early_break_live_ivs
Definition tree-vectorizer.h:1234
tree mask_skip_niters
Definition tree-vectorizer.h:993
bool has_mask_store
Definition tree-vectorizer.h:1181
auto_vec< stmt_vec_info > reductions
Definition tree-vectorizer.h:1066
edge scalar_loop_iv_exit
Definition tree-vectorizer.h:1216
class loop * loop
Definition tree-vectorizer.h:909
bool nonlinear_iv
Definition tree-vectorizer.h:1040
~_loop_vec_info()
Definition tree-vect-loop.cc:926
bool using_decrementing_iv_p
Definition tree-vectorizer.h:1115
auto_vec< gcond * > conds
Definition tree-vectorizer.h:1152
_loop_vec_info * orig_loop_info
Definition tree-vectorizer.h:1197
hash_map< tree, vect_reusable_accumulator > reusable_accumulators
Definition tree-vectorizer.h:962
class vector_costs * scalar_costs
Definition tree-vectorizer.h:924
auto_vec< vec_object_pair > check_unequal_addrs
Definition tree-vectorizer.h:1051
Definition tree-vectorizer.h:397
slp_instance_kind kind
Definition tree-vectorizer.h:424
vec< stmt_vec_info > root_stmts
Definition tree-vectorizer.h:404
slp_tree root
Definition tree-vectorizer.h:400
dump_user_location_t location() const
Definition tree-vect-slp.cc:203
vec< slp_tree > loads
Definition tree-vectorizer.h:411
vec< _slp_instance * > subgraph_entries
Definition tree-vectorizer.h:421
vec< tree > remain_defs
Definition tree-vectorizer.h:408
slp_tree reduc_phis
Definition tree-vectorizer.h:414
stmt_vector_for_cost cost_vec
Definition tree-vectorizer.h:417
Definition tree-vectorizer.h:1472
unsigned int min_input_precision
Definition tree-vectorizer.h:1592
code_helper reduc_code
Definition tree-vectorizer.h:1575
tree vectype
Definition tree-vectorizer.h:1496
bool in_pattern_p
Definition tree-vectorizer.h:1480
stmt_vec_info first_element
Definition tree-vectorizer.h:1542
unsigned int mask_precision
Definition tree-vectorizer.h:1615
int reduc_idx
Definition tree-vectorizer.h:1579
gimple * stmt
Definition tree-vectorizer.h:1493
tree loop_phi_evolution_base_unchanged
Definition tree-vectorizer.h:1514
enum vect_induction_op_type loop_phi_evolution_type
Definition tree-vectorizer.h:1516
innermost_loop_behavior dr_wrt_vec_loop
Definition tree-vectorizer.h:1506
bool slp_vect_only_p
Definition tree-vectorizer.h:1618
bool live
Definition tree-vectorizer.h:1477
unsigned int gap
Definition tree-vectorizer.h:1549
unsigned int min_output_precision
Definition tree-vectorizer.h:1587
unsigned int min_neg_dist
Definition tree-vectorizer.h:1553
bool vectorizable
Definition tree-vectorizer.h:1490
bool pattern_stmt_p
Definition tree-vectorizer.h:1486
bool strided_p
Definition tree-vectorizer.h:1564
enum vect_relevant relevant
Definition tree-vectorizer.h:1558
unsigned int size
Definition tree-vectorizer.h:1546
signop operation_sign
Definition tree-vectorizer.h:1598
gimple_seq pattern_def_seq
Definition tree-vectorizer.h:1532
stmt_vec_info related_stmt
Definition tree-vectorizer.h:1527
unsigned simd_lane_access_p
Definition tree-vectorizer.h:1567
bool slp_vect_pattern_only_p
Definition tree-vectorizer.h:1622
stmt_vec_info reduc_def
Definition tree-vectorizer.h:1583
bool gather_scatter_p
Definition tree-vectorizer.h:1561
enum vect_def_type def_type
Definition tree-vectorizer.h:1535
stmt_vec_info next_element
Definition tree-vectorizer.h:1544
unsigned int operation_precision
Definition tree-vectorizer.h:1597
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:1571
enum slp_vect_type slp_type
Definition tree-vectorizer.h:1538
dr_vec_info dr_aux
Definition tree-vectorizer.h:1502
tree loop_phi_evolution_part
Definition tree-vectorizer.h:1515
Definition tree-vectorizer.h:2439
~auto_purge_vect_location()
Definition tree-vectorizer.cc:94
Definition tree-vectorizer.h:1438
bool scalar_access_known_in_bounds
Definition tree-vectorizer.h:1461
unsigned group
Definition tree-vectorizer.h:1446
tree base_decl
Definition tree-vectorizer.h:1463
bool base_misaligned
Definition tree-vectorizer.h:1453
tree offset
Definition tree-vectorizer.h:1467
int misalignment
Definition tree-vectorizer.h:1448
poly_uint64 target_alignment
Definition tree-vectorizer.h:1451
data_reference * dr
Definition tree-vectorizer.h:1441
bool safe_speculative_read_required
Definition tree-vectorizer.h:1457
stmt_vec_info stmt
Definition tree-vectorizer.h:1443
Definition dumpfile.h:340
void add_int(unsigned v)
Definition inchash.h:55
hashval_t end() const
Definition inchash.h:49
void add_flag(bool flag)
Definition inchash.h:124
class loop * inner
Definition cfgloop.h:148
unsigned force_vectorize
Definition cfgloop.h:227
void * aux
Definition cfgloop.h:154
Definition opt-problem.h:179
Definition profile-count.h:149
Definition tree-vectorizer.h:572
vec< data_reference_p > datarefs
Definition tree-vectorizer.h:581
vec< data_reference > datarefs_copy
Definition tree-vectorizer.h:582
auto_vec< loop_p > loop_nest
Definition tree-vectorizer.h:585
void check_datarefs()
Definition tree-vectorizer.cc:507
void save_datarefs()
Definition tree-vectorizer.cc:497
~vec_info_shared()
Definition tree-vectorizer.cc:489
vec< ddr_p > ddrs
Definition tree-vectorizer.h:589
vec_info_shared()
Definition tree-vectorizer.cc:482
Definition tree-vectorizer.h:593
stmt_vec_info lookup_stmt(gimple *)
Definition tree-vectorizer.cc:571
basic_block * bbs
Definition tree-vectorizer.h:653
auto_vec< stmt_vec_info > grouped_stores
Definition tree-vectorizer.h:638
~vec_info()
Definition tree-vectorizer.cc:474
auto_vec< slp_instance > slp_instances
Definition tree-vectorizer.h:630
void replace_stmt(gimple_stmt_iterator *, stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:660
void remove_stmt(stmt_vec_info)
Definition tree-vectorizer.cc:644
void insert_on_entry(stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:702
void set_vinfo_for_stmt(gimple *, stmt_vec_info, bool=true)
Definition tree-vectorizer.cc:745
vec_base_alignments base_alignments
Definition tree-vectorizer.h:634
machine_mode vector_mode
Definition tree-vectorizer.h:646
stmt_vec_info resync_stmt_addr(gimple *)
Definition tree-vectorizer.cc:551
stmt_vec_info new_stmt_vec_info(gimple *stmt)
Definition tree-vectorizer.cc:713
class dr_vec_info * lookup_dr(data_reference *)
Definition tree-vectorizer.cc:613
vec< stmt_vec_info > stmt_vec_infos
Definition tree-vectorizer.h:621
void move_dr(stmt_vec_info, stmt_vec_info)
Definition tree-vectorizer.cc:625
bool any_known_not_updated_vssa
Definition tree-vectorizer.h:627
vec_kind
Definition tree-vectorizer.h:596
@ loop
Definition tree-vectorizer.h:596
@ bb
Definition tree-vectorizer.h:596
void free_stmt_vec_infos()
Definition tree-vectorizer.cc:766
vec_info_shared * shared
Definition tree-vectorizer.h:618
gimple_seq inv_pattern_def_seq
Definition tree-vectorizer.h:662
unsigned int nbbs
Definition tree-vectorizer.h:656
stmt_vec_info add_stmt(gimple *)
Definition tree-vectorizer.cc:522
vec_kind kind
Definition tree-vectorizer.h:615
mode_set used_vector_modes
Definition tree-vectorizer.h:641
hash_set< int_hash< machine_mode, E_VOIDmode, E_BLKmode > > mode_set
Definition tree-vectorizer.h:595
void insert_seq_on_entry(stmt_vec_info, gimple_seq)
Definition tree-vectorizer.cc:676
stmt_vec_info lookup_def(tree)
Definition tree-vectorizer.cc:588
void free_stmt_vec_info(stmt_vec_info)
Definition tree-vectorizer.cc:777
stmt_vec_info lookup_single_use(tree)
Definition tree-vectorizer.cc:601
vec_info(vec_kind, vec_info_shared *)
Definition tree-vectorizer.cc:463
stmt_vec_info add_pattern_stmt(gimple *, stmt_vec_info)
Definition tree-vectorizer.cc:534
bool stmt_vec_info_ro
Definition tree-vectorizer.h:623
vec_lower_bound()
Definition tree-vectorizer.h:561
poly_uint64 min_value
Definition tree-vectorizer.h:567
bool unsigned_p
Definition tree-vectorizer.h:566
vec_lower_bound(tree e, bool u, poly_uint64 m)
Definition tree-vectorizer.h:562
tree expr
Definition tree-vectorizer.h:565
Definition vec-perm-indices.h:51
Definition tree-vectorizer.h:2811
static vect_pattern * recognize(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
internal_fn m_ifn
Definition tree-vectorizer.h:2817
vec< slp_tree > m_ops
Definition tree-vectorizer.h:2824
virtual ~vect_pattern()
Definition tree-vectorizer.h:2846
virtual void build(vec_info *)=0
vect_pattern(slp_tree *node, vec< slp_tree > *m_ops, internal_fn ifn)
Definition tree-vectorizer.h:2827
slp_tree * m_node
Definition tree-vectorizer.h:2820
unsigned m_num_args
Definition tree-vectorizer.h:2814
Definition tree-vectorizer.h:824
tree reduc_epilogue_adjustment
Definition tree-vectorizer.h:856
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:832
struct vect_reusable_accumulator * reused_accumulator
Definition tree-vectorizer.h:860
internal_fn reduc_fn
Definition tree-vectorizer.h:838
code_helper reduc_code
Definition tree-vectorizer.h:835
auto_vec< tree > reduc_initial_values
Definition tree-vectorizer.h:864
tree induc_cond_initial_val
Definition tree-vectorizer.h:853
tree reduc_vectype
Definition tree-vectorizer.h:850
bool force_single_cycle
Definition tree-vectorizer.h:847
enum vect_def_type def_type
Definition tree-vectorizer.h:828
unsigned int reduc_result_pos
Definition tree-vectorizer.h:844
auto_vec< tree > reduc_scalar_results
Definition tree-vectorizer.h:870
Definition tree-vectorizer.h:1728
unsigned int adjust_cost_for_freq(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1874
virtual ~vector_costs()
Definition tree-vectorizer.h:1731
bool m_costing_for_scalar
Definition tree-vectorizer.h:1801
vector_costs(vec_info *, bool)
Definition tree-vectorizer.h:1826
bool costing_for_scalar() const
Definition tree-vectorizer.h:1786
vec_info * m_vinfo
Definition tree-vectorizer.h:1797
virtual unsigned int add_stmt_cost(int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, vect_cost_model_location where)
Definition tree-vectorizer.cc:1836
machine_mode m_suggested_epilogue_mode
Definition tree-vectorizer.h:1815
virtual bool better_main_loop_than_p(const vector_costs *other) const
Definition tree-vectorizer.cc:1894
unsigned int m_costs[3]
Definition tree-vectorizer.h:1804
unsigned int outside_cost() const
Definition tree-vectorizer.h:1867
unsigned int record_stmt_cost(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1860
virtual bool better_epilogue_loop_than_p(const vector_costs *other, loop_vec_info main_loop) const
Definition tree-vectorizer.cc:1913
unsigned int epilogue_cost() const
Definition tree-vectorizer.h:1858
int m_masked_epilogue
Definition tree-vectorizer.h:1816
machine_mode suggested_epilogue_mode(int &masked) const
Definition tree-vectorizer.h:1893
unsigned int m_suggested_unroll_factor
Definition tree-vectorizer.h:1807
unsigned int suggested_unroll_factor() const
Definition tree-vectorizer.h:1884
bool m_finished
Definition tree-vectorizer.h:1819
int compare_outside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:2089
unsigned int body_cost() const
Definition tree-vectorizer.h:1849
virtual void finish_cost(const vector_costs *scalar_costs)
Definition tree-vectorizer.cc:1849
unsigned int prologue_cost() const
Definition tree-vectorizer.h:1840
unsigned int total_cost() const
Definition tree-vectorizer.h:1876
int compare_inside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:1996
struct basic_block_def * basic_block
Definition coretypes.h:357
class edge_def * edge
Definition coretypes.h:354
gimple * gimple_seq
Definition coretypes.h:100
const class edge_def * const_edge
Definition coretypes.h:355
const union tree_node * const_tree
Definition coretypes.h:98
union tree_node * tree
Definition coretypes.h:97
static unsigned int count[debug_counter_number_of_counters]
Definition dbgcnt.cc:50
bool dominated_by_p(enum cdi_direction dir, const_basic_block bb1, const_basic_block bb2)
Definition dominance.cc:1125
@ CDI_DOMINATORS
Definition dominance.h:25
dump_flags_t dump_flags
Definition dumpfile.cc:65
FILE * dump_file
Definition dumpfile.cc:63
@ TDF_DETAILS
Definition dumpfile.h:92
vect_cost_model
Definition flag-types.h:279
@ VECT_COST_MODEL_UNLIMITED
Definition flag-types.h:283
@ VECT_COST_MODEL_DEFAULT
Definition flag-types.h:284
bool operand_equal_p(const_tree arg0, const_tree arg1, unsigned int flags)
Definition fold-const.cc:4340
#define fold_convert(T1, T2)
Definition fold-const.h:96
#define fold_build2(c, t1, t2, t3)
Definition fold-const.h:76
static struct token T
Definition gengtype-parse.cc:45
internal_fn
Definition genmatch.cc:1015
tree_code
Definition genmatch.cc:1002
unsigned gimple_uid(const gimple *g)
Definition gimple.h:2091
enum tree_code gimple_assign_rhs_code(const gassign *gs)
Definition gimple.h:2857
tree gimple_assign_rhs1(const gassign *gs)
Definition gimple.h:2667
basic_block gimple_bb(const gimple *g)
Definition gimple.h:1867
tree gimple_assign_lhs(const gassign *gs)
Definition gimple.h:2616
#define init_cost
Definition ira-costs.cc:63
T dyn_cast(U *p)
Definition is-a.h:280
void add_expr(const_tree t, inchash::hash &hstate, unsigned int flags)
Definition fold-const.cc:4354
opt_pointer_wrapper< tree > opt_tree
Definition opt-problem.h:287
poly_int< NUM_POLY_INT_COEFFS, unsigned HOST_WIDE_INT > poly_uint64
Definition poly-int-types.h:25
poly_int< NUM_POLY_INT_COEFFS, HOST_WIDE_INT > poly_int64
Definition poly-int-types.h:24
i
Definition poly-int.h:776
signop
Definition signop.h:28
Definition tree-vectorizer.h:297
stmt_vec_info representative
Definition tree-vectorizer.h:313
tree gs_base
Definition tree-vectorizer.h:349
unsigned int lanes
Definition tree-vectorizer.h:342
bool avoid_stlf_fail
Definition tree-vectorizer.h:355
load_permutation_t load_permutation
Definition tree-vectorizer.h:324
slp_tree prev_node
Definition tree-vectorizer.h:380
enum tree_code code
Definition tree-vectorizer.h:344
bool ldst_lanes
Definition tree-vectorizer.h:352
vec< tree > vec_defs
Definition tree-vectorizer.h:332
~_slp_tree()
Definition tree-vect-slp.cc:138
T & get_data(T &else_)
Definition tree-vectorizer.h:365
int gs_scale
Definition tree-vectorizer.h:347
unsigned int refcnt
Definition tree-vectorizer.h:335
enum stmt_vec_info_type type
Definition tree-vectorizer.h:361
slp_tree next_node
Definition tree-vectorizer.h:379
int vertex
Definition tree-vectorizer.h:357
enum vect_def_type def_type
Definition tree-vectorizer.h:340
bool * failed
Definition tree-vectorizer.h:370
vec< tree > ops
Definition tree-vectorizer.h:310
void push_vec_def(gimple *def)
Definition tree-vect-slp.cc:161
int reduc_idx
Definition tree-vectorizer.h:319
poly_uint64 max_nunits
Definition tree-vectorizer.h:338
lane_permutation_t lane_permutation
Definition tree-vectorizer.h:328
_slp_tree()
Definition tree-vect-slp.cc:105
int id
Definition tree-vectorizer.h:317
struct _slp_tree::@120146070312105203102335350023136130264031166255 cycle_info
tree vectype
Definition tree-vectorizer.h:330
void push_vec_def(tree def)
Definition tree-vectorizer.h:302
vec< slp_tree > children
Definition tree-vectorizer.h:305
vect_data * data
Definition tree-vectorizer.h:362
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:308
class loop * loop_father
Definition basic-block.h:126
Definition ira-costs.cc:54
Definition tree-data-ref.h:151
struct innermost_loop_behavior innermost
Definition tree-data-ref.h:173
Definition loop-invariant.cc:88
static const bool empty_zero_p
Definition tree-vectorizer.h:520
static void mark_empty(value_type &v)
Definition tree-vectorizer.h:523
scalar_cond_masked_key value_type
Definition tree-vectorizer.h:496
static bool equal(value_type existing, value_type candidate)
Definition tree-vectorizer.h:511
static void remove(value_type &)
Definition tree-vectorizer.h:542
static hashval_t hash(value_type v)
Definition tree-vectorizer.h:499
static void mark_deleted(value_type &)
Definition tree-vectorizer.h:535
static bool is_deleted(const value_type &)
Definition tree-vectorizer.h:537
static bool is_empty(value_type v)
Definition tree-vectorizer.h:530
scalar_cond_masked_key compare_type
Definition tree-vectorizer.h:495
Definition hash-traits.h:466
Definition graphite.h:338
data_reference_p dr
Definition graphite.h:343
Definition function.h:249
Definition tree-vectorizer.h:1626
tree decl
Definition tree-vectorizer.h:1633
tree element_type
Definition tree-vectorizer.h:1653
tree base
Definition tree-vectorizer.h:1636
tree offset_vectype
Definition tree-vectorizer.h:1650
tree alias_ptr
Definition tree-vectorizer.h:1640
int scale
Definition tree-vectorizer.h:1647
tree offset
Definition tree-vectorizer.h:1643
internal_fn ifn
Definition tree-vectorizer.h:1629
tree memory_type
Definition tree-vectorizer.h:1656
Definition gimple-iterator.h:26
Definition tree-data-ref.h:50
tree offset
Definition tree-data-ref.h:52
Definition hash-traits.h:122
Definition hash-traits.h:345
Definition tree-vectorizer.h:775
tree compare_type
Definition tree-vectorizer.h:795
unsigned int max_nscalars_per_iter
Definition tree-vectorizer.h:779
unsigned int factor
Definition tree-vectorizer.h:785
tree type
Definition tree-vectorizer.h:791
tree bias_adjusted_ctrl
Definition tree-vectorizer.h:802
vec< tree > controls
Definition tree-vectorizer.h:798
unsigned ncopies
Definition tree-vectorizer.h:485
tree_code code
Definition tree-vectorizer.h:487
scalar_cond_masked_key(tree t, unsigned ncopies_)
Definition tree-vectorizer.h:477
void get_cond_ops_from_tree(tree)
Definition tree-vectorizer.cc:1786
tree op0
Definition tree-vectorizer.h:488
tree op1
Definition tree-vectorizer.h:489
bool inverted_p
Definition tree-vectorizer.h:486
vec< tree > remain
Definition tree-vectorizer.h:1378
slp_instance_kind kind
Definition tree-vectorizer.h:1375
vec< stmt_vec_info > roots
Definition tree-vectorizer.h:1377
slp_root(slp_instance_kind kind_, vec< stmt_vec_info > stmts_, vec< stmt_vec_info > roots_, vec< tree > remain_=vNULL)
Definition tree-vectorizer.h:1372
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:1376
Definition tree-vectorizer.h:120
tree vectype
Definition tree-vectorizer.h:126
int misalign
Definition tree-vectorizer.h:127
slp_tree node
Definition tree-vectorizer.h:125
enum vect_cost_model_location where
Definition tree-vectorizer.h:123
enum vect_cost_for_stmt kind
Definition tree-vectorizer.h:122
stmt_vec_info stmt_info
Definition tree-vectorizer.h:124
int count
Definition tree-vectorizer.h:121
Definition tree-hash-traits.h:25
Definition hash-traits.h:75
Definition tree-vectorizer.h:806
hash_set< pair_hash< nofree_ptr_hash< tree_node >, int_hash< unsigned, 0 > > > mask_set
Definition tree-vectorizer.h:811
auto_vec< rgroup_controls > rgc_vec
Definition tree-vectorizer.h:814
bool is_empty() const
Definition tree-vectorizer.h:807
Definition tree-vectorizer.h:257
virtual ~vect_data()=default
Definition tree-vectorizer.h:276
int misalignment
Definition tree-vectorizer.h:283
tree decl
Definition tree-vectorizer.h:288
virtual ~vect_load_store_data()=default
vect_load_store_data()=default
internal_fn ifn
Definition tree-vectorizer.h:287
unsigned n_perms
Definition tree-vectorizer.h:292
poly_int64 poffset
Definition tree-vectorizer.h:285
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:281
vect_load_store_data(vect_load_store_data &&other)=default
tree strided_offset_vectype
Definition tree-vectorizer.h:290
union vect_load_store_data::@336230030006311336100276173266177172342371306160 gs
auto_vec< int > elsvals
Definition tree-vectorizer.h:291
dr_alignment_support alignment_support_scheme
Definition tree-vectorizer.h:282
internal_fn lanes_ifn
Definition tree-vectorizer.h:284
Definition tree-vectorizer.h:890
tree reduc_input
Definition tree-vectorizer.h:893
vect_reduc_info reduc_info
Definition tree-vectorizer.h:897
Definition tree-vectorizer.h:159
static void mark_deleted(value_type &s)
Definition tree-vectorizer.h:165
static bool is_empty(const value_type &s)
Definition tree-vectorizer.h:168
static bool is_deleted(const value_type &s)
Definition tree-vectorizer.h:167
static const bool empty_zero_p
Definition tree-vectorizer.h:163
static void mark_empty(value_type &s)
Definition tree-vectorizer.h:166
static hashval_t hash(const value_type &)
Definition tree-vect-slp.cc:8187
vect_scalar_ops_slice value_type
Definition tree-vectorizer.h:160
static bool equal(const value_type &, const compare_type &)
Definition tree-vect-slp.cc:8196
vect_scalar_ops_slice compare_type
Definition tree-vectorizer.h:161
Definition tree-vectorizer.h:141
unsigned int length
Definition tree-vectorizer.h:147
unsigned int start
Definition tree-vectorizer.h:146
bool all_same_p() const
Definition tree-vect-slp.cc:8178
tree op(unsigned int i) const
Definition tree-vectorizer.h:152
vec< tree > * ops
Definition tree-vectorizer.h:145
virtual ~vect_simd_clone_data()=default
vect_simd_clone_data(vect_simd_clone_data &&other)=default
vect_simd_clone_data()=default
auto_vec< tree > simd_clone_info
Definition tree-vectorizer.h:271
#define NULL
Definition system.h:50
#define gcc_assert(EXPR)
Definition system.h:814
#define gcc_unreachable()
Definition system.h:841
#define false
Definition system.h:888
#define gcc_checking_assert(EXPR)
Definition system.h:821
HOST_WIDE_INT estimated_poly_value(poly_int64 x, poly_value_estimate_kind kind=POLY_VALUE_LIKELY)
Definition target.h:337
struct gcc_target targetm
vect_cost_model_location
Definition target.h:211
@ vect_body
Definition target.h:213
@ vect_prologue
Definition target.h:212
@ vect_epilogue
Definition target.h:214
vect_cost_for_stmt
Definition target.h:189
@ scalar_stmt
Definition target.h:190
@ cond_branch_taken
Definition target.h:203
@ vec_to_scalar
Definition target.h:200
@ cond_branch_not_taken
Definition target.h:202
@ tcc_comparison
Definition tree-core.h:233
@ vec_kind
Definition tree-core.h:1035
#define DR_IS_WRITE(DR)
Definition tree-data-ref.h:192
#define DR_IS_READ(DR)
Definition tree-data-ref.h:191
#define DR_REF(DR)
Definition tree-data-ref.h:185
#define DR_INNERMOST(DR)
Definition tree-data-ref.h:203
static tree candidate(unsigned uid)
Definition tree-sra.cc:326
#define PHI_ARG_DEF_FROM_EDGE(PHI, E)
Definition tree-ssa-operands.h:84
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset)
Definition tree-vect-data-refs.cc:1303
vect_pattern_decl_t slp_patterns[]
Definition tree-vect-slp-patterns.cc:1703
_complex_operation
Definition tree-vect-slp-patterns.cc:119
size_t num__slp_patterns
Definition tree-vect-slp-patterns.cc:1714
dump_user_location_t vect_location
Definition tree-vectorizer.cc:88
stmt_vec_info vect_orig_stmt(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1962
#define LOOP_VINFO_COST_MODEL_THRESHOLD(L)
Definition tree-vectorizer.h:1251
bool vect_can_vectorize_without_simd_p(tree_code)
Definition tree-vect-loop.cc:8838
tree cse_and_gimplify_to_preheader(loop_vec_info, tree)
Definition tree-vect-loop.cc:950
int builtin_vectorization_cost(enum vect_cost_for_stmt type_of_cost, tree vectype, int misalign)
Definition tree-vectorizer.h:2033
loop_vec_info loop_vec_info_for_loop(class loop *loop)
Definition tree-vectorizer.h:1365
bool vect_is_extending_load(class vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2936
bool vect_transform_stmt(vec_info *, stmt_vec_info, gimple_stmt_iterator *, slp_tree, slp_instance)
Definition tree-vect-stmts.cc:12720
bool vect_is_simple_use(tree, vec_info *, enum vect_def_type *, stmt_vec_info *=NULL, gimple **=NULL)
Definition tree-vect-stmts.cc:13187
bool vect_rgroup_iv_might_wrap_p(loop_vec_info, rgroup_controls *)
Definition tree-vect-loop.cc:11673
tree vect_halve_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:10528
void dump_stmt_cost(FILE *, int, enum vect_cost_for_stmt, stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location)
Definition tree-vectorizer.cc:102
vec< unsigned > load_permutation_t
Definition tree-vectorizer.h:254
bool vect_slp_analyze_instance_alignment(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:3142
internal_fn vect_store_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:6045
tree vect_get_new_vect_var(tree, enum vect_var_kind, const char *)
Definition tree-vect-data-refs.cc:5311
void vect_mark_pattern_stmts(vec_info *, stmt_vec_info, gimple *, tree)
Definition tree-vect-patterns.cc:7010
class loop * vect_loop_versioning(loop_vec_info, gimple *)
Definition tree-vect-loop-manip.cc:4121
tree vect_get_slp_vect_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10542
opt_tree vect_get_mask_type_for_stmt(stmt_vec_info, unsigned int=0)
#define LOOP_VINFO_VECT_FACTOR(L)
Definition tree-vectorizer.h:1263
void vect_free_slp_instance(slp_instance)
Definition tree-vect-slp.cc:215
#define SLP_TREE_LANES(S)
Definition tree-vectorizer.h:447
bool vectorizable_early_exit(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:12296
bool vect_slp_function(function *)
Definition tree-vect-slp.cc:10075
std::pair< tree, tree > vec_object_pair
Definition tree-vectorizer.h:555
tree vect_gen_perm_mask_any(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9143
tree vect_setup_realignment(vec_info *, stmt_vec_info, tree, gimple_stmt_iterator *, tree *, enum dr_alignment_support, tree, class loop **)
Definition tree-vect-data-refs.cc:6121
_bb_vec_info * bb_vec_info
#define LOOP_VINFO_FULLY_MASKED_P(L)
Definition tree-vectorizer.h:1318
int vect_get_known_peeling_cost(loop_vec_info, int, int *, stmt_vector_for_cost *, stmt_vector_for_cost *, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:4020
opt_result vect_analyze_data_refs(vec_info *, bool *)
Definition tree-vect-data-refs.cc:5058
bool compatible_calls_p(gcall *, gcall *, bool)
Definition tree-vect-slp.cc:997
bool vect_transform_slp_perm_load(vec_info *, slp_tree, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, unsigned *, unsigned *=nullptr, bool=false)
tree vect_gen_while_not(gimple_seq *, tree, tree, tree)
Definition tree-vect-stmts.cc:14041
#define VECT_REDUC_INFO_TYPE(I)
Definition tree-vectorizer.h:876
tree vect_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2926
bool is_simple_and_all_uses_invariant(stmt_vec_info, loop_vec_info)
Definition tree-vect-stmts.cc:319
#define STMT_VINFO_GROUPED_ACCESS(S)
Definition tree-vectorizer.h:1691
void vect_schedule_slp(vec_info *, const vec< slp_instance > &)
Definition tree-vect-slp.cc:11965
tree vect_peel_nonlinear_iv_init(gimple_seq *, tree, tree, tree, enum vect_induction_op_type)
Definition tree-vect-loop.cc:8964
bool check_reduction_path(dump_user_location_t, loop_p, gphi *, tree, enum tree_code)
Definition tree-vect-loop.cc:3702
bool can_duplicate_and_interleave_p(vec_info *, unsigned int, tree, unsigned int *=NULL, tree *=NULL, tree *=NULL)
Definition tree-vect-slp.cc:427
auto_vec< unsigned, 16 > auto_load_permutation_t
Definition tree-vectorizer.h:255
#define SLP_TREE_VECTYPE(S)
Definition tree-vectorizer.h:445
tree get_vectype_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13010
#define LOOP_VINFO_LOOP(L)
Definition tree-vectorizer.h:1238
_complex_perm_kinds
Definition tree-vectorizer.h:2787
@ PERM_EVENEVEN
Definition tree-vectorizer.h:2792
@ PERM_TOP
Definition tree-vectorizer.h:2794
@ PERM_ODDEVEN
Definition tree-vectorizer.h:2790
@ PERM_EVENODD
Definition tree-vectorizer.h:2789
@ PERM_UNKNOWN
Definition tree-vectorizer.h:2788
@ PERM_ODDODD
Definition tree-vectorizer.h:2791
#define SLP_TREE_TYPE(S)
Definition tree-vectorizer.h:449
void vect_record_base_alignments(vec_info *)
Definition tree-vect-data-refs.cc:1384
vec_load_store_type
Definition tree-vectorizer.h:1432
@ VLS_LOAD
Definition tree-vectorizer.h:1433
@ VLS_STORE
Definition tree-vectorizer.h:1434
@ VLS_STORE_INVARIANT
Definition tree-vectorizer.h:1435
tree get_mask_type_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13085
pair_hash< tree_operand_hash, tree_operand_hash > tree_cond_mask_hash
Definition tree-vectorizer.h:550
_loop_vec_info * loop_vec_info
vect_reduc_info info_for_reduction(loop_vec_info, slp_tree)
Definition tree-vect-loop.cc:5118
tree vect_double_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:10538
vect_var_kind
Definition tree-vectorizer.h:36
@ vect_pointer_var
Definition tree-vectorizer.h:38
@ vect_scalar_var
Definition tree-vectorizer.h:39
@ vect_mask_var
Definition tree-vectorizer.h:40
@ vect_simple_var
Definition tree-vectorizer.h:37
int vect_get_place_in_interleaving_chain(stmt_vec_info, stmt_vec_info)
Definition tree-vect-slp.cc:398
class loop * vect_do_peeling(loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, tree *)
Definition tree-vect-loop-manip.cc:3135
void optimize_mask_stores(class loop *)
Definition tree-vect-loop.cc:11437
class _stmt_vec_info * stmt_vec_info
Definition tree-vectorizer.h:24
#define STMT_VINFO_DATA_REF(S)
Definition tree-vectorizer.h:1665
tree vect_get_loop_len(loop_vec_info, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:10725
bool vect_transform_cycle_phi(loop_vec_info, stmt_vec_info, slp_tree, slp_instance)
Definition tree-vect-loop.cc:8236
unsigned int vect_get_num_copies(vec_info *vinfo, slp_tree node)
Definition tree-vectorizer.h:2297
bool vect_is_store_elt_extraction(vect_cost_for_stmt kind, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2873
opt_result vect_get_vector_types_for_stmt(vec_info *, stmt_vec_info, tree *, tree *, unsigned int=0)
Definition tree-vect-stmts.cc:14065
const poly_uint64 dr_target_alignment(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2129
void vect_update_inits_of_drs(loop_vec_info, tree, tree_code)
Definition tree-vect-loop-manip.cc:2615
struct data_reference * dr_p
Definition tree-vectorizer.h:1470
bool vect_can_advance_ivs_p(loop_vec_info)
Definition tree-vect-loop-manip.cc:2211
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vect-data-refs.cc:1303
tree vect_init_vector(vec_info *, stmt_vec_info, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1203
void vect_copy_ref_info(tree, tree)
Definition tree-vect-data-refs.cc:5871
bool vect_transform_reduction(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree)
Definition tree-vect-loop.cc:7894
bool vectorizable_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:8449
bool vectorizable_lane_reducing(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:6790
tree vect_get_new_ssa_name(tree, enum vect_var_kind, const char *=NULL)
Definition tree-vect-data-refs.cc:5349
#define STMT_VINFO_VECTYPE(S)
Definition tree-vectorizer.h:1663
opt_result vect_prune_runtime_alias_test_list(loop_vec_info)
Definition tree-vect-data-refs.cc:4171
void vect_describe_gather_scatter_call(stmt_vec_info, gather_scatter_info *)
Definition tree-vect-data-refs.cc:4535
bool needs_fold_left_reduction_p(tree, code_helper)
Definition tree-vect-loop.cc:3463
bool vect_nop_conversion_p(stmt_vec_info)
Definition tree-vect-stmts.cc:5595
class vect_reduc_info_s * vect_reduc_info
Definition tree-vectorizer.h:873
void vect_finish_stmt_generation(vec_info *, stmt_vec_info, gimple *, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1328
#define LOOP_VINFO_NON_LINEAR_IV(L)
Definition tree-vectorizer.h:1279
unsigned HOST_WIDE_INT vect_max_vf(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2358
internal_fn vect_load_lanes_supported(tree, unsigned HOST_WIDE_INT, bool, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:6475
loop_vec_info vect_create_loop_vinfo(class loop *, vec_info_shared *, const vect_loop_form_info *, loop_vec_info=nullptr)
Definition tree-vect-loop.cc:1730
bool vect_stmt_dominates_stmt_p(gimple *, gimple *)
Definition tree-vectorizer.cc:793
tree vect_get_slp_scalar_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10524
bool vect_supportable_shift(vec_info *, enum tree_code, tree)
Definition tree-vect-stmts.cc:5773
void vect_optimize_slp(vec_info *)
Definition tree-vect-slp.cc:7840
void vect_iv_increment_position(edge, gimple_stmt_iterator *, bool *)
Definition tree-vect-loop-manip.cc:457
hash_set< scalar_cond_masked_key > scalar_cond_masked_set_type
Definition tree-vectorizer.h:545
tree vect_phi_initial_value(gphi *phi)
Definition tree-vectorizer.h:1932
opt_pointer_wrapper< loop_vec_info > opt_loop_vec_info
Definition tree-vectorizer.h:1362
bool vectorizable_reduction(loop_vec_info, stmt_vec_info, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:6950
unsigned record_stmt_cost(stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location)
Definition tree-vect-stmts.cc:110
bool vect_emulated_vector_p(tree)
Definition tree-vect-loop.cc:8827
tree get_same_sized_vectype(tree, tree)
Definition tree-vect-stmts.cc:13121
peeling_support
Definition tree-vectorizer.h:61
@ peeling_known_supported
Definition tree-vectorizer.h:62
@ peeling_maybe_supported
Definition tree-vectorizer.h:63
@ peeling_unsupported
Definition tree-vectorizer.h:64
gimple * vect_gimple_build(tree, code_helper, tree, tree=NULL_TREE)
Definition tree-vect-patterns.cc:7332
void add_stmt_costs(vector_costs *costs, stmt_vector_for_cost *cost_vec)
Definition tree-vectorizer.h:2101
void vect_finish_replace_stmt(vec_info *, stmt_vec_info, gimple *)
Definition tree-vect-stmts.cc:1312
tree prepare_vec_mask(loop_vec_info, tree, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1605
void vect_slp_fini(void)
Definition tree-vect-slp.cc:80
#define LOOP_VINFO_NITERS_KNOWN_P(L)
Definition tree-vectorizer.h:1349
stmt_vec_info get_later_stmt(stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
Definition tree-vectorizer.h:1972
vec< std::pair< unsigned, unsigned > > lane_permutation_t
Definition tree-vectorizer.h:252
tree vect_gen_loop_len_mask(loop_vec_info, gimple_stmt_iterator *, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:10792
widest_int vect_iv_limit_for_partial_vectors(loop_vec_info loop_vinfo)
Definition tree-vect-loop.cc:11632
void vect_get_vec_defs(vec_info *, slp_tree, tree, vec< tree > *, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL)
Definition tree-vect-stmts.cc:1265
void vect_gather_slp_loads(vec_info *)
Definition tree-vect-slp.cc:7859
vect_partial_vector_style
Definition tree-vectorizer.h:464
@ vect_partial_vectors_none
Definition tree-vectorizer.h:465
@ vect_partial_vectors_len
Definition tree-vectorizer.h:468
@ vect_partial_vectors_avx512
Definition tree-vectorizer.h:467
@ vect_partial_vectors_while_ult
Definition tree-vectorizer.h:466
bool vect_gather_scatter_fn_p(vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4443
class _slp_instance * slp_instance
bool vect_maybe_update_slp_op_vectype(slp_tree, tree)
Definition tree-vect-stmts.cc:13338
#define STMT_VINFO_REDUC_IDX(S)
Definition tree-vectorizer.h:1669
class loop * slpeel_tree_duplicate_loop_to_edge_cfg(class loop *, edge, class loop *, edge, edge, edge *, bool=true, vec< basic_block > *=NULL)
Definition tree-vect-loop-manip.cc:1478
unsigned int vect_vf_for_cost(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2340
unsigned int vect_nunits_for_cost(tree vec_type)
Definition tree-vectorizer.h:2350
bool vect_get_range_info(tree, wide_int *, wide_int *)
Definition tree-vect-patterns.cc:69
bool vect_grouped_load_supported(tree, bool, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:6358
#define STMT_VINFO_IN_PATTERN_P(S)
Definition tree-vectorizer.h:1687
tree vect_create_addr_base_for_vector_ref(vec_info *, stmt_vec_info, gimple_seq *, tree)
Definition tree-vect-data-refs.cc:5424
stmt_vec_info vect_find_last_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3529
tree vect_get_main_loop_result(loop_vec_info, tree, tree)
Definition tree-vect-loop-manip.cc:3026
#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L)
Definition tree-vectorizer.h:1278
#define STMT_VINFO_DR_INFO(S)
Definition tree-vectorizer.h:1684
hash_map< slp_node_hash, bool > slp_compat_nodes_map_t
Definition tree-vectorizer.h:2804
tree get_related_vectype_for_scalar_type(machine_mode, tree, poly_uint64=0)
Definition tree-vect-stmts.cc:12892
bool lane_reducing_op_p(code_helper code)
Definition tree-vectorizer.h:2393
bool nested_in_vect_loop_p(class loop *loop, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1921
bool vect_slp_analyze_instance_dependence(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:1164
bool vect_detect_hybrid_slp(loop_vec_info)
auto_vec< std::pair< data_reference *, tree > > drs_init_vec
Definition tree-vectorizer.h:819
edge vec_init_loop_exit_info(class loop *)
Definition tree-vect-loop.cc:745
opt_result vect_analyze_data_ref_accesses(vec_info *, vec< int > *)
Definition tree-vect-data-refs.cc:3636
int vect_get_stmt_cost(enum vect_cost_for_stmt type_of_cost)
Definition tree-vectorizer.h:2043
opt_result vect_analyze_stmt(vec_info *, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:12570
dump_user_location_t find_loop_location(class loop *)
Definition tree-vect-loop-manip.cc:2070
bool vectorizable_recurr(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:8658
bool vect_slp_analyze_operations(vec_info *)
Definition tree-vect-slp.cc:8723
tree neutral_op_for_reduction(tree, code_helper, tree, bool=true)
Definition tree-vect-loop.cc:3405
void vect_free_slp_tree(slp_tree)
Definition tree-vect-slp.cc:175
unsigned vectorize_loops(void)
bool vect_can_force_dr_alignment_p(const_tree, poly_uint64)
Definition tree-vect-data-refs.cc:6504
tree bump_vector_ptr(vec_info *, tree, gimple *, gimple_stmt_iterator *, stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:5807
bool vect_apply_runtime_profitability_check_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2383
unsigned int vect_get_num_vectors(poly_uint64 nunits, tree vectype)
Definition tree-vectorizer.h:2288
#define DR_TARGET_ALIGNMENT(DR)
Definition tree-vectorizer.h:2135
void vect_free_loop_info_assumptions(class loop *)
Definition tree-vectorizer.cc:855
void set_dr_target_alignment(dr_vec_info *dr_info, poly_uint64 val)
Definition tree-vectorizer.h:2167
stmt_vec_info vect_stmt_to_vectorize(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1999
bool supportable_indirect_convert_operation(code_helper, tree, tree, vec< std::pair< tree, tree_code > > &, tree=NULL_TREE, slp_tree=NULL)
Definition tree-vect-stmts.cc:13882
opt_loop_vec_info vect_analyze_loop(class loop *, gimple *, vec_info_shared *)
Definition tree-vect-loop.cc:2992
void dr_set_safe_speculative_read_required(stmt_vec_info stmt_info, bool requires_alignment)
Definition tree-vectorizer.h:2154
#define STMT_VINFO_RELATED_STMT(S)
Definition tree-vectorizer.h:1688
bool is_pattern_stmt_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1953
#define MAX_VECTORIZATION_FACTOR
Definition tree-vectorizer.h:1906
int vect_pow2(int x)
Definition tree-vectorizer.h:2020
bool reduction_fn_for_scalar_code(code_helper, internal_fn *)
Definition tree-vect-loop.cc:3343
void vect_pattern_recog(vec_info *)
Definition tree-vect-patterns.cc:7296
int vect_slp_child_index_for_operand(const gimple *, int op, bool)
Definition tree-vect-slp.cc:617
vect_induction_op_type
Definition tree-vectorizer.h:83
@ vect_step_op_add
Definition tree-vectorizer.h:84
@ vect_step_op_shl
Definition tree-vectorizer.h:87
@ vect_step_op_mul
Definition tree-vectorizer.h:86
@ vect_step_op_neg
Definition tree-vectorizer.h:85
@ vect_step_op_shr
Definition tree-vectorizer.h:88
void vect_set_loop_condition(class loop *, edge, loop_vec_info, tree, tree, tree, bool)
Definition tree-vect-loop-manip.cc:1393
bool lane_reducing_stmt_p(gimple *stmt)
Definition tree-vectorizer.h:2401
void vect_prepare_for_masked_peels(loop_vec_info)
Definition tree-vect-loop-manip.cc:2645
void vect_get_store_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:1004
stmt_vec_info vect_find_first_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3547
enum _complex_perm_kinds complex_perm_kinds_t
bool vect_is_integer_truncation(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2960
auto_vec< std::pair< unsigned, unsigned >, 16 > auto_lane_permutation_t
Definition tree-vectorizer.h:253
bool vect_use_loop_mask_for_alignment_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2271
tree vect_build_loop_niters(loop_vec_info, bool *=NULL)
Definition tree-vect-loop-manip.cc:2691
bool vect_get_loop_mask_type(loop_vec_info)
void vect_update_max_nunits(poly_uint64 *max_nunits, poly_uint64 nunits)
Definition tree-vectorizer.h:2316
slp_instance_kind
Definition tree-vectorizer.h:386
@ slp_inst_kind_ctor
Definition tree-vectorizer.h:391
@ slp_inst_kind_reduc_group
Definition tree-vectorizer.h:388
@ slp_inst_kind_store
Definition tree-vectorizer.h:387
@ slp_inst_kind_reduc_chain
Definition tree-vectorizer.h:389
@ slp_inst_kind_gcond
Definition tree-vectorizer.h:392
@ slp_inst_kind_bb_reduc
Definition tree-vectorizer.h:390
enum vect_cost_model loop_cost_model(loop_p loop)
Definition tree-vectorizer.h:2250
unsigned int vect_known_alignment_in_bytes(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vectorizer.h:2196
#define DR_MISALIGNMENT_UNKNOWN
Definition tree-vectorizer.h:2113
bool mat_gather_scatter_p(vect_memory_access_type mat)
Definition tree-vectorizer.h:215
hash_map< slp_tree, complex_perm_kinds_t > slp_tree_to_load_perm_map_t
Definition tree-vectorizer.h:2799
opt_result vect_enhance_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2332
bool vect_is_reduction(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2882
#define LOOP_VINFO_EARLY_BREAKS(L)
Definition tree-vectorizer.h:1295
enum dr_alignment_support vect_supportable_dr_alignment(vec_info *, dr_vec_info *, tree, int, bool=false)
Definition tree-vect-data-refs.cc:6528
bool vectorizable_induction(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9440
#define SLP_TREE_REDUC_IDX(S)
Definition tree-vectorizer.h:452
innermost_loop_behavior * vect_dr_behavior(vec_info *vinfo, dr_vec_info *dr_info)
Definition tree-vectorizer.h:2212
tree perm_mask_for_reverse(tree)
Definition tree-vect-stmts.cc:1791
vect_memory_access_type SLP_TREE_MEMORY_ACCESS_TYPE(slp_tree node)
Definition tree-vectorizer.h:456
stmt_vec_info_type
Definition tree-vectorizer.h:225
@ comparison_vec_info_type
Definition tree-vectorizer.h:235
@ condition_vec_info_type
Definition tree-vectorizer.h:234
@ type_conversion_vec_info_type
Definition tree-vectorizer.h:240
@ load_vec_info_type
Definition tree-vectorizer.h:227
@ loop_exit_ctrl_vec_info_type
Definition tree-vectorizer.h:245
@ permute_info_type
Definition tree-vectorizer.h:246
@ recurr_info_type
Definition tree-vectorizer.h:244
@ op_vec_info_type
Definition tree-vectorizer.h:230
@ reduc_vec_info_type
Definition tree-vectorizer.h:236
@ assignment_vec_info_type
Definition tree-vectorizer.h:233
@ undef_vec_info_type
Definition tree-vectorizer.h:226
@ shift_vec_info_type
Definition tree-vectorizer.h:229
@ type_demotion_vec_info_type
Definition tree-vectorizer.h:239
@ store_vec_info_type
Definition tree-vectorizer.h:228
@ cycle_phi_info_type
Definition tree-vectorizer.h:241
@ phi_info_type
Definition tree-vectorizer.h:243
@ lc_phi_info_type
Definition tree-vectorizer.h:242
@ type_promotion_vec_info_type
Definition tree-vectorizer.h:238
@ call_simd_clone_vec_info_type
Definition tree-vectorizer.h:232
@ call_vec_info_type
Definition tree-vectorizer.h:231
@ induc_vec_info_type
Definition tree-vectorizer.h:237
void vect_get_slp_defs(slp_tree, vec< tree > *)
Definition tree-vect-slp.cc:10550
void vect_record_loop_len(loop_vec_info, vec_loop_lens *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:10690
struct _slp_tree * slp_tree
Definition tree-vectorizer.h:25
gimple_seq vect_gen_len(tree, tree, tree, tree)
Definition tree-vect-stmts.cc:14229
bool dr_safe_speculative_read_required(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2140
bool vect_check_gather_scatter(stmt_vec_info, tree, loop_vec_info, gather_scatter_info *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4562
tree vect_embedded_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2911
gimple * vect_loop_vectorized_call(class loop *, gcond **cond=NULL)
Definition tree-vectorizer.cc:870
void duplicate_and_interleave(vec_info *, gimple_seq *, tree, const vec< tree > &, unsigned int, vec< tree > &)
Definition tree-vect-slp.cc:10207
bool vect_grouped_store_supported(tree, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:5927
void vect_get_load_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool)
Definition tree-vect-stmts.cc:1056
vect_def_type
Definition tree-vectorizer.h:68
@ vect_induction_def
Definition tree-vectorizer.h:73
@ vect_uninitialized_def
Definition tree-vectorizer.h:69
@ vect_nested_cycle
Definition tree-vectorizer.h:76
@ vect_constant_def
Definition tree-vectorizer.h:70
@ vect_external_def
Definition tree-vectorizer.h:71
@ vect_condition_def
Definition tree-vectorizer.h:78
@ vect_internal_def
Definition tree-vectorizer.h:72
@ vect_reduction_def
Definition tree-vectorizer.h:74
@ vect_double_reduction_def
Definition tree-vectorizer.h:75
@ vect_first_order_recurrence
Definition tree-vectorizer.h:77
@ vect_unknown_def_type
Definition tree-vectorizer.h:79
opt_result vect_analyze_data_ref_dependences(loop_vec_info, unsigned int *)
Definition tree-vect-data-refs.cc:887
vect_relevant
Definition tree-vectorizer.h:1399
@ vect_used_in_outer
Definition tree-vectorizer.h:1409
@ vect_used_in_scope
Definition tree-vectorizer.h:1419
@ vect_unused_in_scope
Definition tree-vectorizer.h:1400
@ vect_used_by_reduction
Definition tree-vectorizer.h:1417
@ vect_used_in_outer_by_reduction
Definition tree-vectorizer.h:1406
@ vect_used_only_live
Definition tree-vectorizer.h:1403
tree vect_get_mask_load_else(int, tree)
Definition tree-vect-stmts.cc:2691
bool is_loop_header_bb_p(basic_block bb)
Definition tree-vectorizer.h:2009
opt_result vect_analyze_loop_form(class loop *, gimple *, vect_loop_form_info *)
Definition tree-vect-loop.cc:1523
#define STMT_VINFO_DR_WRT_VEC_LOOP(S)
Definition tree-vectorizer.h:1671
bool vectorizable_live_operation(vec_info *, stmt_vec_info, slp_tree, slp_instance, int, bool, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:10124
slp_tree vect_create_new_slp_node(unsigned, tree_code)
Definition tree-vect-slp.cc:230
opt_result vect_analyze_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:3074
bool unlimited_cost_model(loop_p loop)
Definition tree-vectorizer.h:2261
opt_result vect_mark_stmts_to_be_vectorized(loop_vec_info, bool *)
Definition tree-vect-stmts.cc:698
tree vect_gen_while(gimple_seq *, tree, tree, tree, const char *=nullptr)
Definition tree-vect-stmts.cc:14017
tree vect_get_store_rhs(stmt_vec_info)
Definition tree-vect-stmts.cc:1872
class loop * vect_transform_loop(loop_vec_info, gimple *)
Definition tree-vect-loop.cc:11044
void vect_slp_init(void)
Definition tree-vect-slp.cc:74
unsigned int vect_get_scalar_dr_size(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2374
opt_result vect_determine_partial_vectors_and_peeling(loop_vec_info)
Definition tree-vect-loop.cc:2113
bool ref_within_array_bound(gimple *, tree)
Definition tree-if-conv.cc:852
dr_alignment_support
Definition tree-vectorizer.h:51
@ dr_unaligned_supported
Definition tree-vectorizer.h:53
@ dr_explicit_realign_optimized
Definition tree-vectorizer.h:55
@ dr_aligned
Definition tree-vectorizer.h:56
@ dr_explicit_realign
Definition tree-vectorizer.h:54
@ dr_unaligned_unsupported
Definition tree-vectorizer.h:52
int compare_step_with_zero(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:1780
bool supportable_narrowing_operation(code_helper, tree, tree, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:13709
operation_type
Definition tree-vectorizer.h:44
@ ternary_op
Definition tree-vectorizer.h:47
@ unary_op
Definition tree-vectorizer.h:45
@ binary_op
Definition tree-vectorizer.h:46
void vect_record_loop_mask(loop_vec_info, vec_loop_masks *, unsigned int, tree, tree)
Definition tree-vect-loop.cc:10550
vect_memory_access_type
Definition tree-vectorizer.h:175
@ VMAT_STRIDED_SLP
Definition tree-vectorizer.h:204
@ VMAT_GATHER_SCATTER_LEGACY
Definition tree-vectorizer.h:207
@ VMAT_GATHER_SCATTER_EMULATED
Definition tree-vectorizer.h:209
@ VMAT_INVARIANT
Definition tree-vectorizer.h:179
@ VMAT_UNINITIALIZED
Definition tree-vectorizer.h:176
@ VMAT_ELEMENTWISE
Definition tree-vectorizer.h:198
@ VMAT_GATHER_SCATTER_IFN
Definition tree-vectorizer.h:208
@ VMAT_CONTIGUOUS_DOWN
Definition tree-vectorizer.h:187
@ VMAT_CONTIGUOUS
Definition tree-vectorizer.h:182
@ VMAT_LOAD_STORE_LANES
Definition tree-vectorizer.h:194
@ VMAT_CONTIGUOUS_REVERSE
Definition tree-vectorizer.h:191
bool vect_chooses_same_modes_p(vec_info *, machine_mode)
Definition tree-vect-stmts.cc:13139
bool slpeel_can_duplicate_loop_p(const class loop *, const_edge, const_edge)
Definition tree-vect-loop-manip.cc:2038
tree vect_get_loop_mask(loop_vec_info, gimple_stmt_iterator *, vec_loop_masks *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:10572
slp_vect_type
Definition tree-vectorizer.h:1425
@ not_vect
Definition tree-vectorizer.h:1426
@ pure_slp
Definition tree-vectorizer.h:1427
vect_reduction_type
Definition tree-vectorizer.h:92
@ TREE_CODE_REDUCTION
Definition tree-vectorizer.h:93
@ FOLD_LEFT_REDUCTION
Definition tree-vectorizer.h:111
@ EXTRACT_LAST_REDUCTION
Definition tree-vectorizer.h:103
@ INTEGER_INDUC_COND_REDUCTION
Definition tree-vectorizer.h:95
@ CONST_COND_REDUCTION
Definition tree-vectorizer.h:96
@ COND_REDUCTION
Definition tree-vectorizer.h:94
bool supportable_widening_operation(vec_info *, code_helper, stmt_vec_info, tree, tree, code_helper *, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:13378
auto_vec< rgroup_controls > vec_loop_lens
Definition tree-vectorizer.h:817
void set_dr_misalignment(dr_vec_info *dr_info, int val)
Definition tree-vectorizer.h:2117
bool vect_use_mask_type_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1944
pair_hash< nofree_ptr_hash< _slp_tree >, nofree_ptr_hash< _slp_tree > > slp_node_hash
Definition tree-vectorizer.h:2803
hash_map< tree_operand_hash, std::pair< stmt_vec_info, innermost_loop_behavior * > > vec_base_alignments
Definition tree-vectorizer.h:136
unsigned add_stmt_cost(vector_costs *costs, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where)
Definition tree-vectorizer.h:2063
tree vect_get_smallest_scalar_type(stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:161
vec< stmt_info_for_cost > stmt_vector_for_cost
Definition tree-vectorizer.h:130
bool known_alignment_for_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2187
void vect_remove_stores(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:12862
#define DR_GROUP_FIRST_ELEMENT(S)
Definition tree-vectorizer.h:1704
hash_set< tree_cond_mask_hash > vec_cond_masked_set_type
Definition tree-vectorizer.h:551
opt_result vect_find_stmt_data_reference(loop_p, gimple *, vec< data_reference_p > *, vec< int > *, int)
Definition tree-vect-data-refs.cc:4883
tree get_dr_vinfo_offset(vec_info *vinfo, dr_vec_info *dr_info, bool check_outer=false)
Definition tree-vectorizer.h:2228
bool vect_slp_if_converted_bb(basic_block bb, loop_p orig_loop)
Definition tree-vect-slp.cc:10064
bool vectorizable_slp_permutation(vec_info *, gimple_stmt_iterator *, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-slp.cc:11350
tree vect_gen_perm_mask_checked(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9158
opt_result vect_analyze_slp(vec_info *, unsigned, bool)
Definition tree-vect-slp.cc:5173
vect_pattern *(* vect_pattern_decl_t)(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
Definition tree-vectorizer.h:2853
tree vect_create_destination_var(tree, tree)
Definition tree-vect-data-refs.cc:5893
bool vect_transform_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:8489
tree vect_create_data_ref_ptr(vec_info *, stmt_vec_info, tree, class loop *, tree, tree *, gimple_stmt_iterator *, gimple **, bool, tree=NULL_TREE)
Definition tree-vect-data-refs.cc:5547
int vect_reduc_type(vec_info *vinfo, slp_tree node)
Definition tree-vectorizer.h:2897
void vect_gen_vector_loop_niters(loop_vec_info, tree, tree *, tree *, bool)
Definition tree-vect-loop-manip.cc:2782
bool vectorizable_phi(bb_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:8516
bool vect_make_slp_decision(loop_vec_info)
Definition tree-vect-slp.cc:7917
bool aligned_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2177
unsigned HOST_WIDE_INT tree_to_uhwi(const_tree t)
Definition tree.cc:6671
#define TYPE_ALIGN_UNIT(NODE)
Definition tree.h:2454
poly_uint64 TYPE_VECTOR_SUBPARTS(const_tree node)
Definition tree.h:4365
#define TYPE_PRECISION(NODE)
Definition tree.h:2360
#define TREE_OPERAND(NODE, I)
Definition tree.h:1319
#define sizetype
Definition tree.h:5242
#define TREE_CODE_CLASS(CODE)
Definition tree.h:203
#define TYPE_SIZE_UNIT(NODE)
Definition tree.h:2357
#define CONVERT_EXPR_CODE_P(CODE)
Definition tree.h:554
#define TREE_TYPE(NODE)
Definition tree.h:513
#define INTEGRAL_TYPE_P(TYPE)
Definition tree.h:614
#define COMPARISON_CLASS_P(NODE)
Definition tree.h:257
#define NULL_TREE
Definition tree.h:318
#define FOR_EACH_VEC_ELT(V, I, P)
Definition vec.h:1895
constexpr vnull vNULL
Definition vec.h:569
generic_wide_int< widest_int_storage< WIDEST_INT_MAX_PRECISION > > widest_int
Definition wide-int.h:345
generic_wide_int< wide_int_storage > wide_int
Definition wide-int.h:343