21#ifndef GCC_TREE_VECTORIZER_H
22#define GCC_TREE_VECTORIZER_H
106#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
107 || ((D) == vect_double_reduction_def) \
108 || ((D) == vect_nested_cycle))
106#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \ …
127 std::pair<stmt_vec_info, innermost_loop_behavior *> >
134 tree op (
unsigned int i)
const;
285 static void *
operator new (size_t);
288 static void operator delete (
void *, size_t);
343#define SLP_INSTANCE_TREE(S) (S)->root
344#define SLP_INSTANCE_LOADS(S) (S)->loads
345#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
346#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
347#define SLP_INSTANCE_KIND(S) (S)->kind
349#define SLP_TREE_CHILDREN(S) (S)->children
350#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
351#define SLP_TREE_SCALAR_OPS(S) (S)->ops
352#define SLP_TREE_REF_COUNT(S) (S)->refcnt
353#define SLP_TREE_VEC_DEFS(S) (S)->vec_defs
354#define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
355#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
356#define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
357#define SLP_TREE_SIMD_CLONE_INFO(S) (S)->simd_clone_info
358#define SLP_TREE_DEF_TYPE(S) (S)->def_type
359#define SLP_TREE_VECTYPE(S) (S)->vectype
360#define SLP_TREE_REPRESENTATIVE(S) (S)->representative
361#define SLP_TREE_LANES(S) (S)->lanes
362#define SLP_TREE_CODE(S) (S)->code
363#define SLP_TREE_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
399 static inline hashval_t
1050#define LOOP_VINFO_LOOP(L) (L)->loop
1051#define LOOP_VINFO_IV_EXIT(L) (L)->vec_loop_iv_exit
1052#define LOOP_VINFO_EPILOGUE_IV_EXIT(L) (L)->vec_epilogue_loop_iv_exit
1053#define LOOP_VINFO_SCALAR_IV_EXIT(L) (L)->scalar_loop_iv_exit
1054#define LOOP_VINFO_BBS(L) (L)->bbs
1055#define LOOP_VINFO_NBBS(L) (L)->nbbs
1056#define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1
1057#define LOOP_VINFO_NITERS(L) (L)->num_iters
1061#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
1062#define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions
1063#define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th
1064#define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold
1065#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
1066#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
1067#define LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P(L) (L)->must_use_partial_vectors_p
1068#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
1069#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
1070#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
1071#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \
1072 (L)->epil_using_partial_vectors_p
1071#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \ …
1073#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
1074#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
1075#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
1076#define LOOP_VINFO_MASKS(L) (L)->masks
1077#define LOOP_VINFO_LENS(L) (L)->lens
1078#define LOOP_VINFO_MASK_SKIP_NITERS(L) (L)->mask_skip_niters
1079#define LOOP_VINFO_MASK_NITERS_PFA_OFFSET(L) (L)->mask_skip_niters_pfa_offset
1080#define LOOP_VINFO_RGROUP_COMPARE_TYPE(L) (L)->rgroup_compare_type
1081#define LOOP_VINFO_RGROUP_IV_TYPE(L) (L)->rgroup_iv_type
1082#define LOOP_VINFO_PARTIAL_VECTORS_STYLE(L) (L)->partial_vector_style
1083#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
1084#define LOOP_VINFO_LOOP_NEST(L) (L)->shared->loop_nest
1085#define LOOP_VINFO_DATAREFS(L) (L)->shared->datarefs
1086#define LOOP_VINFO_DDRS(L) (L)->shared->ddrs
1087#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
1088#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
1089#define LOOP_VINFO_NON_LINEAR_IV(L) (L)->nonlinear_iv
1090#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
1091#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
1092#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
1093#define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs
1094#define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs
1095#define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero
1096#define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds
1097#define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
1098#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
1099#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
1100#define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
1101#define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
1102#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
1103#define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter
1104#define LOOP_VINFO_EARLY_BREAKS(L) (L)->early_breaks
1105#define LOOP_VINFO_EARLY_BRK_STORES(L) (L)->early_break_stores
1106#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L) \
1107 (single_pred ((L)->loop->latch) != (L)->vec_loop_iv_exit->src)
1106#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L) \ …
1108#define LOOP_VINFO_EARLY_BREAKS_LIVE_IVS(L) \
1109 (L)->early_break_live_ivs
1108#define LOOP_VINFO_EARLY_BREAKS_LIVE_IVS(L) \ …
1110#define LOOP_VINFO_EARLY_BRK_DEST_BB(L) (L)->early_break_dest_bb
1111#define LOOP_VINFO_EARLY_BRK_VUSES(L) (L)->early_break_vuses
1112#define LOOP_VINFO_LOOP_CONDS(L) (L)->conds
1113#define LOOP_VINFO_LOOP_IV_COND(L) (L)->loop_iv_cond
1114#define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
1115#define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop
1116#define LOOP_VINFO_SCALAR_LOOP_SCALING(L) (L)->scalar_loop_scaling
1117#define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store
1118#define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
1119#define LOOP_VINFO_MAIN_LOOP_INFO(L) (L)->main_loop_info
1120#define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info
1121#define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
1122#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
1123#define LOOP_VINFO_INV_PATTERN_DEF_SEQ(L) (L)->inv_pattern_def_seq
1124#define LOOP_VINFO_DRS_ADVANCED_BY(L) (L)->drs_advanced_by
1126#define LOOP_VINFO_FULLY_MASKED_P(L) \
1127 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1128 && !LOOP_VINFO_MASKS (L).is_empty ())
1126#define LOOP_VINFO_FULLY_MASKED_P(L) \ …
1130#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \
1131 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1132 && !LOOP_VINFO_LENS (L).is_empty ())
1130#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \ …
1134#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
1135 ((L)->may_misalign_stmts.length () > 0)
1134#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \ …
1136#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
1137 ((L)->comp_alias_ddrs.length () > 0 \
1138 || (L)->check_unequal_addrs.length () > 0 \
1139 || (L)->lower_bounds.length () > 0)
1136#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \ …
1140#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \
1141 (LOOP_VINFO_NITERS_ASSUMPTIONS (L))
1140#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \ …
1142#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \
1143 (LOOP_VINFO_SIMD_IF_COND (L))
1142#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \ …
1144#define LOOP_REQUIRES_VERSIONING(L) \
1145 (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \
1146 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \
1147 || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
1148 || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
1144#define LOOP_REQUIRES_VERSIONING(L) \ …
1150#define LOOP_VINFO_NITERS_KNOWN_P(L) \
1151 (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
1150#define LOOP_VINFO_NITERS_KNOWN_P(L) \ …
1153#define LOOP_VINFO_EPILOGUE_P(L) \
1154 (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL)
1153#define LOOP_VINFO_EPILOGUE_P(L) \ …
1156#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \
1157 (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L)))
1156#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \ …
1191#define BB_VINFO_BBS(B) (B)->bbs
1192#define BB_VINFO_NBBS(B) (B)->nbbs
1193#define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
1194#define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
1195#define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
1196#define BB_VINFO_DDRS(B) (B)->shared->ddrs
1560#define STMT_VINFO_TYPE(S) (S)->type
1561#define STMT_VINFO_STMT(S) (S)->stmt
1562#define STMT_VINFO_RELEVANT(S) (S)->relevant
1563#define STMT_VINFO_LIVE_P(S) (S)->live
1564#define STMT_VINFO_VECTYPE(S) (S)->vectype
1565#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
1566#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
1567#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
1568#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
1569#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
1570#define STMT_VINFO_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
1571#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
1572#define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
1573#define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
1574#define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
1575#define STMT_VINFO_FORCE_SINGLE_CYCLE(S) (S)->force_single_cycle
1577#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop
1578#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address
1579#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init
1580#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset
1581#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step
1582#define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment
1583#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \
1584 (S)->dr_wrt_vec_loop.base_misalignment
1583#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \ …
1585#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \
1586 (S)->dr_wrt_vec_loop.offset_alignment
1585#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \ …
1587#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
1588 (S)->dr_wrt_vec_loop.step_alignment
1587#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \ …
1590#define STMT_VINFO_DR_INFO(S) \
1591 (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
1590#define STMT_VINFO_DR_INFO(S) \ …
1593#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
1594#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
1595#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
1596#define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info
1597#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
1598#define STMT_VINFO_GROUPED_ACCESS(S) \
1599 ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
1598#define STMT_VINFO_GROUPED_ACCESS(S) \ …
1600#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
1601#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
1602#define STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE(S) (S)->loop_phi_evolution_type
1603#define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
1604#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
1605#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
1606#define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn
1607#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
1608#define STMT_VINFO_REDUC_VECTYPE(S) (S)->reduc_vectype
1609#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
1610#define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
1611#define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p
1613#define DR_GROUP_FIRST_ELEMENT(S) \
1614 (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
1613#define DR_GROUP_FIRST_ELEMENT(S) \ …
1615#define DR_GROUP_NEXT_ELEMENT(S) \
1616 (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
1615#define DR_GROUP_NEXT_ELEMENT(S) \ …
1617#define DR_GROUP_SIZE(S) \
1618 (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
1617#define DR_GROUP_SIZE(S) \ …
1619#define DR_GROUP_STORE_COUNT(S) \
1620 (gcc_checking_assert ((S)->dr_aux.dr), (S)->store_count)
1619#define DR_GROUP_STORE_COUNT(S) \ …
1621#define DR_GROUP_GAP(S) \
1622 (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
1621#define DR_GROUP_GAP(S) \ …
1624#define REDUC_GROUP_FIRST_ELEMENT(S) \
1625 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
1624#define REDUC_GROUP_FIRST_ELEMENT(S) \ …
1626#define REDUC_GROUP_NEXT_ELEMENT(S) \
1627 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
1626#define REDUC_GROUP_NEXT_ELEMENT(S) \ …
1628#define REDUC_GROUP_SIZE(S) \
1629 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
1628#define REDUC_GROUP_SIZE(S) \ …
1631#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
1633#define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
1634#define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
1635#define STMT_SLP_TYPE(S) (S)->slp_type
1665 tree vectype,
int misalign,
1803#define VECT_MAX_COST 1000
1807#define MAX_INTERM_CVT_STEPS 3
1809#define MAX_VECTORIZATION_FACTOR INT_MAX
1816#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \
1817 (TREE_CODE (TYPE) == BOOLEAN_TYPE \
1818 || ((TREE_CODE (TYPE) == INTEGER_TYPE \
1819 || TREE_CODE (TYPE) == ENUMERAL_TYPE) \
1820 && TYPE_PRECISION (TYPE) == 1 \
1821 && TYPE_UNSIGNED (TYPE)))
1816#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \ …
1849 return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
1858 return stmt_info->pattern_stmt_p;
1927 for (
i = 0;
i < x;
i++)
1937 tree vectype,
int misalign)
1939 return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
1956 return targetm.vectorize.create_costs (vinfo, costing_for_scalar);
1969 tree vectype,
int misalign,
1972 unsigned cost =
costs->add_stmt_cost (
count, kind, stmt_info, node, vectype,
1993 i->vectype,
i->misalign,
i->where);
2009#define DR_MISALIGNMENT_UNKNOWN (-1)
2010#define DR_MISALIGNMENT_UNINITIALIZED (-2)
2021#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
2029 return dr_info->target_alignment;
2031#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR)
2032#define DR_SCALAR_KNOWN_BOUNDS(DR) (DR)->scalar_access_known_in_bounds
2044 return dr_info->safe_speculative_read_required;
2051 bool requires_alignment)
2059 dr_info->safe_speculative_read_required = requires_alignment;
2065 dr_info->target_alignment = val;
2067#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL)
2098 else if (misalignment == 0)
2100 return misalignment & -misalignment;
2112 if (loop_vinfo ==
NULL
2151 return flag_simd_cost_model;
2152 return flag_vect_cost_model;
2235 *max_nunits = force_common_multiple (*max_nunits, nunits);
2271inline unsigned HOST_WIDE_INT
2274 unsigned HOST_WIDE_INT vf;
2309 return code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR;
2337#define DUMP_VECT_SCOPE(MSG) \
2338 AUTO_DUMP_SCOPE (MSG, vect_location)
2337#define DUMP_VECT_SCOPE(MSG) \ …
2414 vec<std::pair<tree, tree_code> > &,
2487 unsigned int *,
unsigned int *,
2498 const char * =
nullptr);
2502 tree *,
unsigned int = 0);
2558 const char * =
NULL);
2585 unsigned int,
tree,
unsigned int);
2587 tree,
unsigned int);
2590 unsigned int,
unsigned int);
2593 unsigned int,
tree,
tree,
unsigned int,
2660 unsigned * =
nullptr,
bool =
false);
2678 unsigned int * =
NULL,
2727typedef hash_map <slp_tree, complex_perm_kinds_t>
2731typedef pair_hash <nofree_ptr_hash <_slp_tree>,
2759 this->m_node = node;
2760 this->m_ops.create (0);
2762 this->m_ops.safe_splice (*
m_ops);
2777 this->m_ops.release ();
2887 return (def_stmt_info
edge loop_preheader_edge(const class loop *loop)
Definition cfgloop.cc:1760
class loop * loop_p
Definition cfgloop.h:98
Definition tree-vectorizer.h:1183
vec< slp_root > roots
Definition tree-vectorizer.h:1188
_bb_vec_info(vec< basic_block > bbs, vec_info_shared *)
Definition tree-vect-slp.cc:7737
~_bb_vec_info()
Definition tree-vect-slp.cc:7774
Definition tree-vectorizer.h:737
tree mask_skip_niters_pfa_offset
Definition tree-vectorizer.h:828
bool peeling_for_gaps
Definition tree-vectorizer.h:957
bool can_use_partial_vectors_p
Definition tree-vectorizer.h:924
signed char partial_load_store_bias
Definition tree-vectorizer.h:952
class vector_costs * vector_costs
Definition tree-vectorizer.h:755
class dr_vec_info * unaligned_dr
Definition tree-vectorizer.h:851
_loop_vec_info(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:1034
edge skip_main_loop_edge
Definition tree-vectorizer.h:781
auto_vec< stmt_vec_info > reduction_chains
Definition tree-vectorizer.h:897
unsigned int th
Definition tree-vectorizer.h:763
_loop_vec_info * epilogue_vinfo
Definition tree-vectorizer.h:1013
_loop_vec_info * main_loop_info
Definition tree-vectorizer.h:1004
auto_vec< gimple * > early_break_vuses
Definition tree-vectorizer.h:1042
bool using_partial_vectors_p
Definition tree-vectorizer.h:931
bool no_data_dependencies
Definition tree-vectorizer.h:990
class loop * scalar_loop
Definition tree-vectorizer.h:1000
tree drs_advanced_by
Definition tree-vectorizer.h:1016
scalar_cond_masked_set_type scalar_cond_masked_set
Definition tree-vectorizer.h:814
auto_vec< ddr_p > may_alias_ddrs
Definition tree-vectorizer.h:871
tree num_itersm1
Definition tree-vectorizer.h:746
unsigned int inner_loop_cost_factor
Definition tree-vectorizer.h:915
bool vectorizable
Definition tree-vectorizer.h:918
vect_partial_vector_style partial_vector_style
Definition tree-vectorizer.h:848
poly_uint64 slp_unrolling_factor
Definition tree-vectorizer.h:911
edge vec_epilogue_loop_iv_exit
Definition tree-vectorizer.h:1024
tree num_iters
Definition tree-vectorizer.h:748
hash_map< tree, tree > * scan_map
Definition tree-vectorizer.h:907
int peeling_for_alignment
Definition tree-vectorizer.h:861
vec_cond_masked_set_type vec_cond_masked_set
Definition tree-vectorizer.h:817
bool epil_using_partial_vectors_p
Definition tree-vectorizer.h:947
bool early_breaks
Definition tree-vectorizer.h:965
poly_uint64 versioning_threshold
Definition tree-vectorizer.h:769
auto_vec< vec_lower_bound > lower_bounds
Definition tree-vectorizer.h:886
profile_probability scalar_loop_scaling
Definition tree-vectorizer.h:996
auto_vec< gimple * > early_break_stores
Definition tree-vectorizer.h:1034
edge main_loop_edge
Definition tree-vectorizer.h:780
edge skip_this_loop_edge
Definition tree-vectorizer.h:785
bool must_use_partial_vectors_p
Definition tree-vectorizer.h:927
auto_vec< stmt_info_for_cost > scalar_cost_vec
Definition tree-vectorizer.h:900
gcond * loop_iv_cond
Definition tree-vectorizer.h:971
basic_block early_break_dest_bb
Definition tree-vectorizer.h:1038
bool using_select_vl_p
Definition tree-vectorizer.h:943
tree rgroup_compare_type
Definition tree-vectorizer.h:833
poly_uint64 vectorization_factor
Definition tree-vectorizer.h:772
bool peeling_for_niter
Definition tree-vectorizer.h:961
auto_vec< dr_with_seg_len_pair_t > comp_alias_ddrs
Definition tree-vectorizer.h:875
edge vec_loop_iv_exit
Definition tree-vectorizer.h:1020
tree num_iters_unchanged
Definition tree-vectorizer.h:750
vec_loop_lens lens
Definition tree-vectorizer.h:811
unsigned HOST_WIDE_INT max_vectorization_factor
Definition tree-vectorizer.h:803
unsigned int suggested_unroll_factor
Definition tree-vectorizer.h:799
hash_map< tree_operand_hash, tree > * ivexpr_map
Definition tree-vectorizer.h:903
auto_vec< stmt_vec_info > may_misalign_stmts
Definition tree-vectorizer.h:890
tree rgroup_iv_type
Definition tree-vectorizer.h:844
vec_loop_masks masks
Definition tree-vectorizer.h:807
tree num_iters_assumptions
Definition tree-vectorizer.h:752
tree simd_if_cond
Definition tree-vectorizer.h:840
auto_vec< tree > check_nonzero
Definition tree-vectorizer.h:883
auto_vec< stmt_vec_info > early_break_live_ivs
Definition tree-vectorizer.h:1046
tree mask_skip_niters
Definition tree-vectorizer.h:823
bool has_mask_store
Definition tree-vectorizer.h:993
auto_vec< stmt_vec_info > reductions
Definition tree-vectorizer.h:893
edge scalar_loop_iv_exit
Definition tree-vectorizer.h:1028
class loop * loop
Definition tree-vectorizer.h:743
bool nonlinear_iv
Definition tree-vectorizer.h:867
~_loop_vec_info()
Definition tree-vect-loop.cc:1156
bool using_decrementing_iv_p
Definition tree-vectorizer.h:938
auto_vec< gcond * > conds
Definition tree-vectorizer.h:968
_loop_vec_info * orig_loop_info
Definition tree-vectorizer.h:1009
hash_map< tree, vect_reusable_accumulator > reusable_accumulators
Definition tree-vectorizer.h:792
int ptr_mask
Definition tree-vectorizer.h:864
class vector_costs * scalar_costs
Definition tree-vectorizer.h:758
auto_vec< vec_object_pair > check_unequal_addrs
Definition tree-vectorizer.h:878
Definition tree-vectorizer.h:309
slp_instance_kind kind
Definition tree-vectorizer.h:336
vec< stmt_vec_info > root_stmts
Definition tree-vectorizer.h:316
slp_tree root
Definition tree-vectorizer.h:312
dump_user_location_t location() const
Definition tree-vect-slp.cc:201
vec< slp_tree > loads
Definition tree-vectorizer.h:323
vec< _slp_instance * > subgraph_entries
Definition tree-vectorizer.h:333
vec< tree > remain_defs
Definition tree-vectorizer.h:320
slp_tree reduc_phis
Definition tree-vectorizer.h:326
stmt_vector_for_cost cost_vec
Definition tree-vectorizer.h:329
Definition tree-vectorizer.h:1313
unsigned int min_input_precision
Definition tree-vectorizer.h:1493
code_helper reduc_code
Definition tree-vectorizer.h:1438
unsigned int store_count
Definition tree-vectorizer.h:1400
tree vectype
Definition tree-vectorizer.h:1339
vec< tree > simd_clone_info
Definition tree-vectorizer.h:1383
tree induc_cond_initial_val
Definition tree-vectorizer.h:1428
bool in_pattern_p
Definition tree-vectorizer.h:1323
stmt_vec_info first_element
Definition tree-vectorizer.h:1393
unsigned int mask_precision
Definition tree-vectorizer.h:1516
vec< tree > reduc_initial_values
Definition tree-vectorizer.h:1466
int reduc_idx
Definition tree-vectorizer.h:1444
gimple * stmt
Definition tree-vectorizer.h:1336
tree loop_phi_evolution_base_unchanged
Definition tree-vectorizer.h:1360
enum vect_induction_op_type loop_phi_evolution_type
Definition tree-vectorizer.h:1362
innermost_loop_behavior dr_wrt_vec_loop
Definition tree-vectorizer.h:1352
vec< gimple * > vec_stmts
Definition tree-vectorizer.h:1342
bool slp_vect_only_p
Definition tree-vectorizer.h:1519
bool live
Definition tree-vectorizer.h:1320
unsigned int gap
Definition tree-vectorizer.h:1403
unsigned int min_output_precision
Definition tree-vectorizer.h:1488
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:1425
internal_fn reduc_fn
Definition tree-vectorizer.h:1440
enum stmt_vec_info_type type
Definition tree-vectorizer.h:1316
unsigned int min_neg_dist
Definition tree-vectorizer.h:1407
vec< tree > reduc_scalar_results
Definition tree-vectorizer.h:1473
tree reduc_vectype_in
Definition tree-vectorizer.h:1452
bool vectorizable
Definition tree-vectorizer.h:1333
bool pattern_stmt_p
Definition tree-vectorizer.h:1329
bool strided_p
Definition tree-vectorizer.h:1418
enum vect_relevant relevant
Definition tree-vectorizer.h:1412
unsigned int size
Definition tree-vectorizer.h:1397
signop operation_sign
Definition tree-vectorizer.h:1499
bool force_single_cycle
Definition tree-vectorizer.h:1481
bool is_reduc_info
Definition tree-vectorizer.h:1484
gimple_seq pattern_def_seq
Definition tree-vectorizer.h:1378
stmt_vec_info related_stmt
Definition tree-vectorizer.h:1373
unsigned simd_lane_access_p
Definition tree-vectorizer.h:1421
bool slp_vect_pattern_only_p
Definition tree-vectorizer.h:1523
tree reduc_vectype
Definition tree-vectorizer.h:1455
stmt_vec_info reduc_def
Definition tree-vectorizer.h:1449
bool gather_scatter_p
Definition tree-vectorizer.h:1415
enum vect_def_type def_type
Definition tree-vectorizer.h:1386
stmt_vec_info next_element
Definition tree-vectorizer.h:1395
unsigned int operation_precision
Definition tree-vectorizer.h:1498
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:1435
tree reduc_epilogue_adjustment
Definition tree-vectorizer.h:1431
unsigned int reduc_result_pos
Definition tree-vectorizer.h:1461
enum slp_vect_type slp_type
Definition tree-vectorizer.h:1389
dr_vec_info dr_aux
Definition tree-vectorizer.h:1348
tree loop_phi_evolution_part
Definition tree-vectorizer.h:1361
vect_reusable_accumulator * reused_accumulator
Definition tree-vectorizer.h:1478
Definition tree-vectorizer.h:2353
~auto_purge_vect_location()
Definition tree-vectorizer.cc:94
Definition tree-vectorizer.h:1279
bool scalar_access_known_in_bounds
Definition tree-vectorizer.h:1302
unsigned group
Definition tree-vectorizer.h:1287
tree base_decl
Definition tree-vectorizer.h:1304
bool base_misaligned
Definition tree-vectorizer.h:1294
tree offset
Definition tree-vectorizer.h:1308
int misalignment
Definition tree-vectorizer.h:1289
poly_uint64 target_alignment
Definition tree-vectorizer.h:1292
data_reference * dr
Definition tree-vectorizer.h:1282
bool safe_speculative_read_required
Definition tree-vectorizer.h:1298
stmt_vec_info stmt
Definition tree-vectorizer.h:1284
Definition dumpfile.h:340
void add_int(unsigned v)
Definition inchash.h:55
hashval_t end() const
Definition inchash.h:49
void add_flag(bool flag)
Definition inchash.h:124
class loop * inner
Definition cfgloop.h:148
unsigned force_vectorize
Definition cfgloop.h:227
void * aux
Definition cfgloop.h:154
Definition opt-problem.h:179
Definition profile-count.h:147
Definition tree-vectorizer.h:473
vec< data_reference_p > datarefs
Definition tree-vectorizer.h:482
vec< data_reference > datarefs_copy
Definition tree-vectorizer.h:483
auto_vec< loop_p > loop_nest
Definition tree-vectorizer.h:486
void check_datarefs()
Definition tree-vectorizer.cc:507
void save_datarefs()
Definition tree-vectorizer.cc:497
~vec_info_shared()
Definition tree-vectorizer.cc:489
vec< ddr_p > ddrs
Definition tree-vectorizer.h:490
vec_info_shared()
Definition tree-vectorizer.cc:482
Definition tree-vectorizer.h:494
stmt_vec_info lookup_stmt(gimple *)
Definition tree-vectorizer.cc:571
basic_block * bbs
Definition tree-vectorizer.h:554
auto_vec< stmt_vec_info > grouped_stores
Definition tree-vectorizer.h:539
~vec_info()
Definition tree-vectorizer.cc:474
auto_vec< slp_instance > slp_instances
Definition tree-vectorizer.h:531
void replace_stmt(gimple_stmt_iterator *, stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:660
void remove_stmt(stmt_vec_info)
Definition tree-vectorizer.cc:644
void insert_on_entry(stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:702
void set_vinfo_for_stmt(gimple *, stmt_vec_info, bool=true)
Definition tree-vectorizer.cc:749
vec_base_alignments base_alignments
Definition tree-vectorizer.h:535
machine_mode vector_mode
Definition tree-vectorizer.h:547
stmt_vec_info resync_stmt_addr(gimple *)
Definition tree-vectorizer.cc:551
stmt_vec_info new_stmt_vec_info(gimple *stmt)
Definition tree-vectorizer.cc:713
class dr_vec_info * lookup_dr(data_reference *)
Definition tree-vectorizer.cc:613
vec< stmt_vec_info > stmt_vec_infos
Definition tree-vectorizer.h:522
void move_dr(stmt_vec_info, stmt_vec_info)
Definition tree-vectorizer.cc:625
bool any_known_not_updated_vssa
Definition tree-vectorizer.h:528
vec_kind
Definition tree-vectorizer.h:497
@ loop
Definition tree-vectorizer.h:497
@ bb
Definition tree-vectorizer.h:497
void free_stmt_vec_infos()
Definition tree-vectorizer.cc:770
vec_info_shared * shared
Definition tree-vectorizer.h:519
gimple_seq inv_pattern_def_seq
Definition tree-vectorizer.h:563
unsigned int nbbs
Definition tree-vectorizer.h:557
stmt_vec_info add_stmt(gimple *)
Definition tree-vectorizer.cc:522
vec_kind kind
Definition tree-vectorizer.h:516
mode_set used_vector_modes
Definition tree-vectorizer.h:542
hash_set< int_hash< machine_mode, E_VOIDmode, E_BLKmode > > mode_set
Definition tree-vectorizer.h:496
void insert_seq_on_entry(stmt_vec_info, gimple_seq)
Definition tree-vectorizer.cc:676
stmt_vec_info lookup_def(tree)
Definition tree-vectorizer.cc:588
void free_stmt_vec_info(stmt_vec_info)
Definition tree-vectorizer.cc:781
stmt_vec_info lookup_single_use(tree)
Definition tree-vectorizer.cc:601
vec_info(vec_kind, vec_info_shared *)
Definition tree-vectorizer.cc:463
stmt_vec_info add_pattern_stmt(gimple *, stmt_vec_info)
Definition tree-vectorizer.cc:534
bool stmt_vec_info_ro
Definition tree-vectorizer.h:524
vec_lower_bound()
Definition tree-vectorizer.h:462
poly_uint64 min_value
Definition tree-vectorizer.h:468
bool unsigned_p
Definition tree-vectorizer.h:467
vec_lower_bound(tree e, bool u, poly_uint64 m)
Definition tree-vectorizer.h:463
tree expr
Definition tree-vectorizer.h:466
Definition vec-perm-indices.h:51
Definition tree-vectorizer.h:2740
static vect_pattern * recognize(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
internal_fn m_ifn
Definition tree-vectorizer.h:2746
vec< slp_tree > m_ops
Definition tree-vectorizer.h:2753
virtual ~vect_pattern()
Definition tree-vectorizer.h:2775
virtual void build(vec_info *)=0
vect_pattern(slp_tree *node, vec< slp_tree > *m_ops, internal_fn ifn)
Definition tree-vectorizer.h:2756
slp_tree * m_node
Definition tree-vectorizer.h:2749
unsigned m_num_args
Definition tree-vectorizer.h:2743
Definition tree-vectorizer.h:1639
unsigned int adjust_cost_for_freq(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1873
virtual ~vector_costs()
Definition tree-vectorizer.h:1642
bool m_costing_for_scalar
Definition tree-vectorizer.h:1711
vector_costs(vec_info *, bool)
Definition tree-vectorizer.h:1731
vec_info * m_vinfo
Definition tree-vectorizer.h:1707
virtual unsigned int add_stmt_cost(int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, vect_cost_model_location where)
Definition tree-vectorizer.cc:1835
machine_mode suggested_epilogue_mode() const
Definition tree-vectorizer.h:1797
machine_mode m_suggested_epilogue_mode
Definition tree-vectorizer.h:1721
virtual bool better_main_loop_than_p(const vector_costs *other) const
Definition tree-vectorizer.cc:1893
unsigned int m_costs[3]
Definition tree-vectorizer.h:1714
unsigned int outside_cost() const
Definition tree-vectorizer.h:1771
unsigned int record_stmt_cost(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1859
virtual bool better_epilogue_loop_than_p(const vector_costs *other, loop_vec_info main_loop) const
Definition tree-vectorizer.cc:1912
unsigned int epilogue_cost() const
Definition tree-vectorizer.h:1762
unsigned int m_suggested_unroll_factor
Definition tree-vectorizer.h:1717
unsigned int suggested_unroll_factor() const
Definition tree-vectorizer.h:1788
bool m_finished
Definition tree-vectorizer.h:1724
int compare_outside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:2088
unsigned int body_cost() const
Definition tree-vectorizer.h:1753
virtual void finish_cost(const vector_costs *scalar_costs)
Definition tree-vectorizer.cc:1848
unsigned int prologue_cost() const
Definition tree-vectorizer.h:1744
unsigned int total_cost() const
Definition tree-vectorizer.h:1780
int compare_inside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:1995
struct basic_block_def * basic_block
Definition coretypes.h:355
class edge_def * edge
Definition coretypes.h:352
gimple * gimple_seq
Definition coretypes.h:100
const class edge_def * const_edge
Definition coretypes.h:353
const union tree_node * const_tree
Definition coretypes.h:98
union tree_node * tree
Definition coretypes.h:97
static unsigned int count[debug_counter_number_of_counters]
Definition dbgcnt.cc:50
bool dominated_by_p(enum cdi_direction dir, const_basic_block bb1, const_basic_block bb2)
Definition dominance.cc:1125
@ CDI_DOMINATORS
Definition dominance.h:25
dump_flags_t dump_flags
Definition dumpfile.cc:65
FILE * dump_file
Definition dumpfile.cc:63
@ TDF_DETAILS
Definition dumpfile.h:92
vect_cost_model
Definition flag-types.h:279
@ VECT_COST_MODEL_UNLIMITED
Definition flag-types.h:283
@ VECT_COST_MODEL_DEFAULT
Definition flag-types.h:284
bool operand_equal_p(const_tree arg0, const_tree arg1, unsigned int flags)
Definition fold-const.cc:4332
#define fold_convert(T1, T2)
Definition fold-const.h:96
#define fold_build2(c, t1, t2, t3)
Definition fold-const.h:76
internal_fn
Definition genmatch.cc:1015
tree_code
Definition genmatch.cc:1002
unsigned gimple_uid(const gimple *g)
Definition gimple.h:2087
enum tree_code gimple_assign_rhs_code(const gassign *gs)
Definition gimple.h:2853
tree gimple_assign_rhs1(const gassign *gs)
Definition gimple.h:2663
basic_block gimple_bb(const gimple *g)
Definition gimple.h:1863
tree gimple_assign_lhs(const gassign *gs)
Definition gimple.h:2612
#define init_cost
Definition ira-costs.cc:63
T dyn_cast(U *p)
Definition is-a.h:280
void add_expr(const_tree t, inchash::hash &hstate, unsigned int flags)
Definition fold-const.cc:4346
opt_pointer_wrapper< tree > opt_tree
Definition opt-problem.h:287
poly_int< NUM_POLY_INT_COEFFS, unsigned HOST_WIDE_INT > poly_uint64
Definition poly-int-types.h:25
poly_int< NUM_POLY_INT_COEFFS, HOST_WIDE_INT > poly_int64
Definition poly-int-types.h:24
i
Definition poly-int.h:776
signop
Definition signop.h:28
Definition tree-vectorizer.h:215
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:277
stmt_vec_info representative
Definition tree-vectorizer.h:231
unsigned int lanes
Definition tree-vectorizer.h:263
bool avoid_stlf_fail
Definition tree-vectorizer.h:271
load_permutation_t load_permutation
Definition tree-vectorizer.h:235
slp_tree prev_node
Definition tree-vectorizer.h:292
enum tree_code code
Definition tree-vectorizer.h:265
bool ldst_lanes
Definition tree-vectorizer.h:268
unsigned int vec_stmts_size
Definition tree-vectorizer.h:253
vec< tree > vec_defs
Definition tree-vectorizer.h:248
~_slp_tree()
Definition tree-vect-slp.cc:137
unsigned int refcnt
Definition tree-vectorizer.h:256
slp_tree next_node
Definition tree-vectorizer.h:291
int vertex
Definition tree-vectorizer.h:273
enum vect_def_type def_type
Definition tree-vectorizer.h:261
bool * failed
Definition tree-vectorizer.h:282
vec< tree > simd_clone_info
Definition tree-vectorizer.h:244
vec< tree > ops
Definition tree-vectorizer.h:228
void push_vec_def(gimple *def)
Definition tree-vect-slp.cc:159
poly_uint64 max_nunits
Definition tree-vectorizer.h:259
lane_permutation_t lane_permutation
Definition tree-vectorizer.h:239
_slp_tree()
Definition tree-vect-slp.cc:107
tree vectype
Definition tree-vectorizer.h:246
void push_vec_def(tree def)
Definition tree-vectorizer.h:220
vec< slp_tree > children
Definition tree-vectorizer.h:223
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:226
class loop * loop_father
Definition basic-block.h:126
Definition ira-costs.cc:54
Definition tree-data-ref.h:151
struct innermost_loop_behavior innermost
Definition tree-data-ref.h:173
Definition loop-invariant.cc:88
static const bool empty_zero_p
Definition tree-vectorizer.h:421
static void mark_empty(value_type &v)
Definition tree-vectorizer.h:424
scalar_cond_masked_key value_type
Definition tree-vectorizer.h:397
static bool equal(value_type existing, value_type candidate)
Definition tree-vectorizer.h:412
static void remove(value_type &)
Definition tree-vectorizer.h:443
static hashval_t hash(value_type v)
Definition tree-vectorizer.h:400
static void mark_deleted(value_type &)
Definition tree-vectorizer.h:436
static bool is_deleted(const value_type &)
Definition tree-vectorizer.h:438
static bool is_empty(value_type v)
Definition tree-vectorizer.h:431
scalar_cond_masked_key compare_type
Definition tree-vectorizer.h:396
Definition hash-traits.h:466
Definition graphite.h:338
data_reference_p dr
Definition graphite.h:343
Definition function.h:249
Definition tree-vectorizer.h:1527
tree decl
Definition tree-vectorizer.h:1534
tree element_type
Definition tree-vectorizer.h:1553
tree base
Definition tree-vectorizer.h:1537
tree offset_vectype
Definition tree-vectorizer.h:1550
int scale
Definition tree-vectorizer.h:1544
enum vect_def_type offset_dt
Definition tree-vectorizer.h:1547
tree offset
Definition tree-vectorizer.h:1540
internal_fn ifn
Definition tree-vectorizer.h:1530
tree memory_type
Definition tree-vectorizer.h:1556
Definition gimple-iterator.h:26
Definition tree-data-ref.h:50
tree offset
Definition tree-data-ref.h:52
Definition hash-traits.h:122
Definition hash-traits.h:345
Definition tree-vectorizer.h:676
tree compare_type
Definition tree-vectorizer.h:696
unsigned int max_nscalars_per_iter
Definition tree-vectorizer.h:680
unsigned int factor
Definition tree-vectorizer.h:686
tree type
Definition tree-vectorizer.h:692
tree bias_adjusted_ctrl
Definition tree-vectorizer.h:703
vec< tree > controls
Definition tree-vectorizer.h:699
unsigned ncopies
Definition tree-vectorizer.h:386
tree_code code
Definition tree-vectorizer.h:388
scalar_cond_masked_key(tree t, unsigned ncopies_)
Definition tree-vectorizer.h:378
void get_cond_ops_from_tree(tree)
Definition tree-vectorizer.cc:1785
tree op0
Definition tree-vectorizer.h:389
tree op1
Definition tree-vectorizer.h:390
bool inverted_p
Definition tree-vectorizer.h:387
vec< tree > remain
Definition tree-vectorizer.h:1179
slp_instance_kind kind
Definition tree-vectorizer.h:1176
vec< stmt_vec_info > roots
Definition tree-vectorizer.h:1178
slp_root(slp_instance_kind kind_, vec< stmt_vec_info > stmts_, vec< stmt_vec_info > roots_, vec< tree > remain_=vNULL)
Definition tree-vectorizer.h:1173
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:1177
Definition tree-vectorizer.h:112
tree vectype
Definition tree-vectorizer.h:118
int misalign
Definition tree-vectorizer.h:119
slp_tree node
Definition tree-vectorizer.h:117
enum vect_cost_model_location where
Definition tree-vectorizer.h:115
enum vect_cost_for_stmt kind
Definition tree-vectorizer.h:114
stmt_vec_info stmt_info
Definition tree-vectorizer.h:116
int count
Definition tree-vectorizer.h:113
Definition tree-hash-traits.h:25
Definition hash-traits.h:75
Definition tree-vectorizer.h:707
hash_set< pair_hash< nofree_ptr_hash< tree_node >, int_hash< unsigned, 0 > > > mask_set
Definition tree-vectorizer.h:712
auto_vec< rgroup_controls > rgc_vec
Definition tree-vectorizer.h:715
bool is_empty() const
Definition tree-vectorizer.h:708
Definition tree-vectorizer.h:724
stmt_vec_info reduc_info
Definition tree-vectorizer.h:731
tree reduc_input
Definition tree-vectorizer.h:727
Definition tree-vectorizer.h:151
static void mark_deleted(value_type &s)
Definition tree-vectorizer.h:157
static bool is_empty(const value_type &s)
Definition tree-vectorizer.h:160
static bool is_deleted(const value_type &s)
Definition tree-vectorizer.h:159
static const bool empty_zero_p
Definition tree-vectorizer.h:155
static void mark_empty(value_type &s)
Definition tree-vectorizer.h:158
static hashval_t hash(const value_type &)
Definition tree-vect-slp.cc:7966
vect_scalar_ops_slice value_type
Definition tree-vectorizer.h:152
static bool equal(const value_type &, const compare_type &)
Definition tree-vect-slp.cc:7975
vect_scalar_ops_slice compare_type
Definition tree-vectorizer.h:153
Definition tree-vectorizer.h:133
unsigned int length
Definition tree-vectorizer.h:139
unsigned int start
Definition tree-vectorizer.h:138
bool all_same_p() const
Definition tree-vect-slp.cc:7957
tree op(unsigned int i) const
Definition tree-vectorizer.h:144
vec< tree > * ops
Definition tree-vectorizer.h:137
#define NULL
Definition system.h:50
#define gcc_assert(EXPR)
Definition system.h:814
#define gcc_unreachable()
Definition system.h:841
#define false
Definition system.h:888
#define gcc_checking_assert(EXPR)
Definition system.h:821
HOST_WIDE_INT estimated_poly_value(poly_int64 x, poly_value_estimate_kind kind=POLY_VALUE_LIKELY)
Definition target.h:338
struct gcc_target targetm
vect_cost_model_location
Definition target.h:212
@ vect_body
Definition target.h:214
@ vect_prologue
Definition target.h:213
@ vect_epilogue
Definition target.h:215
vect_cost_for_stmt
Definition target.h:190
@ scalar_stmt
Definition target.h:191
@ cond_branch_taken
Definition target.h:204
@ vec_to_scalar
Definition target.h:201
@ cond_branch_not_taken
Definition target.h:203
@ tcc_comparison
Definition tree-core.h:233
@ vec_kind
Definition tree-core.h:1031
#define DR_IS_WRITE(DR)
Definition tree-data-ref.h:192
#define DR_IS_READ(DR)
Definition tree-data-ref.h:191
#define DR_REF(DR)
Definition tree-data-ref.h:185
#define DR_INNERMOST(DR)
Definition tree-data-ref.h:203
static tree candidate(unsigned uid)
Definition tree-sra.cc:326
#define PHI_ARG_DEF_FROM_EDGE(PHI, E)
Definition tree-ssa-operands.h:84
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset)
Definition tree-vect-data-refs.cc:1303
vect_pattern_decl_t slp_patterns[]
Definition tree-vect-slp-patterns.cc:1658
_complex_operation
Definition tree-vect-slp-patterns.cc:119
size_t num__slp_patterns
Definition tree-vect-slp-patterns.cc:1669
dump_user_location_t vect_location
Definition tree-vectorizer.cc:88
stmt_vec_info vect_orig_stmt(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1865
#define LOOP_VINFO_COST_MODEL_THRESHOLD(L)
Definition tree-vectorizer.h:1063
bool vect_can_vectorize_without_simd_p(tree_code)
Definition tree-vect-loop.cc:9385
tree cse_and_gimplify_to_preheader(loop_vec_info, tree)
Definition tree-vect-loop.cc:1178
int builtin_vectorization_cost(enum vect_cost_for_stmt type_of_cost, tree vectype, int misalign)
Definition tree-vectorizer.h:1936
loop_vec_info loop_vec_info_for_loop(class loop *loop)
Definition tree-vectorizer.h:1166
bool vect_is_extending_load(class vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2870
bool vect_transform_stmt(vec_info *, stmt_vec_info, gimple_stmt_iterator *, slp_tree, slp_instance)
Definition tree-vect-stmts.cc:14053
bool vect_is_simple_use(tree, vec_info *, enum vect_def_type *, stmt_vec_info *=NULL, gimple **=NULL)
Definition tree-vect-stmts.cc:14533
bool vect_rgroup_iv_might_wrap_p(loop_vec_info, rgroup_controls *)
Definition tree-vect-loop.cc:12663
tree vect_halve_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11437
void dump_stmt_cost(FILE *, int, enum vect_cost_for_stmt, stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location)
Definition tree-vectorizer.cc:102
vec< unsigned > load_permutation_t
Definition tree-vectorizer.h:210
bool vect_slp_analyze_instance_alignment(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:3052
internal_fn vect_store_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:5954
tree vect_get_new_vect_var(tree, enum vect_var_kind, const char *)
Definition tree-vect-data-refs.cc:5213
void vect_mark_pattern_stmts(vec_info *, stmt_vec_info, gimple *, tree)
Definition tree-vect-patterns.cc:6910
class loop * vect_loop_versioning(loop_vec_info, gimple *)
Definition tree-vect-loop-manip.cc:4003
tree vect_get_slp_vect_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10324
opt_tree vect_get_mask_type_for_stmt(stmt_vec_info, unsigned int=0)
#define LOOP_VINFO_VECT_FACTOR(L)
Definition tree-vectorizer.h:1074
void vect_free_slp_instance(slp_instance)
Definition tree-vect-slp.cc:213
#define SLP_TREE_LANES(S)
Definition tree-vectorizer.h:361
bool vect_slp_function(function *)
Definition tree-vect-slp.cc:9857
std::pair< tree, tree > vec_object_pair
Definition tree-vectorizer.h:456
tree vect_gen_perm_mask_any(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:10052
_bb_vec_info * bb_vec_info
#define LOOP_VINFO_FULLY_MASKED_P(L)
Definition tree-vectorizer.h:1126
int vect_get_known_peeling_cost(loop_vec_info, int, int *, stmt_vector_for_cost *, stmt_vector_for_cost *, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:4499
bool vect_transform_slp_perm_load(vec_info *, slp_tree, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, unsigned *, unsigned *=nullptr, bool=false)
tree vect_gen_while_not(gimple_seq *, tree, tree, tree)
Definition tree-vect-stmts.cc:15460
bool vectorizable_early_exit(vec_info *, stmt_vec_info, gimple_stmt_iterator *, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:13476
tree vect_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2860
bool is_simple_and_all_uses_invariant(stmt_vec_info, loop_vec_info)
Definition tree-vect-stmts.cc:326
bool vectorizable_phi(vec_info *, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9076
#define STMT_VINFO_GROUPED_ACCESS(S)
Definition tree-vectorizer.h:1598
#define STMT_VINFO_MEMORY_ACCESS_TYPE(S)
Definition tree-vectorizer.h:1570
void vect_schedule_slp(vec_info *, const vec< slp_instance > &)
Definition tree-vect-slp.cc:11765
tree vect_peel_nonlinear_iv_init(gimple_seq *, tree, tree, tree, enum vect_induction_op_type)
Definition tree-vect-loop.cc:9511
bool check_reduction_path(dump_user_location_t, loop_p, gphi *, tree, enum tree_code)
Definition tree-vect-loop.cc:4178
bool can_duplicate_and_interleave_p(vec_info *, unsigned int, tree, unsigned int *=NULL, tree *=NULL, tree *=NULL)
Definition tree-vect-slp.cc:425
auto_vec< unsigned, 16 > auto_load_permutation_t
Definition tree-vectorizer.h:211
#define SLP_TREE_VECTYPE(S)
Definition tree-vectorizer.h:359
tree get_vectype_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:14373
#define LOOP_VINFO_LOOP(L)
Definition tree-vectorizer.h:1050
_complex_perm_kinds
Definition tree-vectorizer.h:2716
@ PERM_EVENEVEN
Definition tree-vectorizer.h:2721
@ PERM_TOP
Definition tree-vectorizer.h:2723
@ PERM_ODDEVEN
Definition tree-vectorizer.h:2719
@ PERM_EVENODD
Definition tree-vectorizer.h:2718
@ PERM_UNKNOWN
Definition tree-vectorizer.h:2717
@ PERM_ODDODD
Definition tree-vectorizer.h:2720
void vect_record_base_alignments(vec_info *)
Definition tree-vect-data-refs.cc:1384
void vect_get_vec_defs(vec_info *, stmt_vec_info, slp_tree, unsigned, tree, vec< tree > *, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL)
Definition tree-vect-stmts.cc:1375
vec_load_store_type
Definition tree-vectorizer.h:1273
@ VLS_LOAD
Definition tree-vectorizer.h:1274
@ VLS_STORE
Definition tree-vectorizer.h:1275
@ VLS_STORE_INVARIANT
Definition tree-vectorizer.h:1276
tree get_mask_type_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:14448
pair_hash< tree_operand_hash, tree_operand_hash > tree_cond_mask_hash
Definition tree-vectorizer.h:451
_loop_vec_info * loop_vec_info
tree vect_double_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11447
vect_var_kind
Definition tree-vectorizer.h:36
@ vect_pointer_var
Definition tree-vectorizer.h:38
@ vect_scalar_var
Definition tree-vectorizer.h:39
@ vect_mask_var
Definition tree-vectorizer.h:40
@ vect_simple_var
Definition tree-vectorizer.h:37
int vect_get_place_in_interleaving_chain(stmt_vec_info, stmt_vec_info)
Definition tree-vect-slp.cc:396
class loop * vect_do_peeling(loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, tree *)
Definition tree-vect-loop-manip.cc:3128
void optimize_mask_stores(class loop *)
Definition tree-vect-loop.cc:12427
class _stmt_vec_info * stmt_vec_info
Definition tree-vectorizer.h:24
#define STMT_VINFO_DATA_REF(S)
Definition tree-vectorizer.h:1567
tree vect_get_loop_len(loop_vec_info, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:11634
bool vect_transform_cycle_phi(loop_vec_info, stmt_vec_info, slp_tree, slp_instance)
Definition tree-vect-loop.cc:8809
bool vect_is_store_elt_extraction(vect_cost_for_stmt kind, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2802
void vect_detect_hybrid_slp(loop_vec_info)
Definition tree-vect-slp.cc:7652
opt_result vect_get_vector_types_for_stmt(vec_info *, stmt_vec_info, tree *, tree *, unsigned int=0)
Definition tree-vect-stmts.cc:15484
const poly_uint64 dr_target_alignment(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2025
void vect_update_inits_of_drs(loop_vec_info, tree, tree_code)
Definition tree-vect-loop-manip.cc:2622
struct data_reference * dr_p
Definition tree-vectorizer.h:1311
bool vect_can_advance_ivs_p(loop_vec_info)
Definition tree-vect-loop-manip.cc:2211
opt_result vect_analyze_data_refs(vec_info *, poly_uint64 *, bool *)
Definition tree-vect-data-refs.cc:4954
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vect-data-refs.cc:1303
tree vect_init_vector(vec_info *, stmt_vec_info, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1204
void vect_copy_ref_info(tree, tree)
Definition tree-vect-data-refs.cc:5780
void vect_record_grouped_load_vectors(vec_info *, stmt_vec_info, vec< tree >)
Definition tree-vect-data-refs.cc:7188
bool vect_transform_reduction(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree)
Definition tree-vect-loop.cc:8462
bool vectorizable_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:9021
bool vectorizable_lane_reducing(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:7339
tree vect_get_new_ssa_name(tree, enum vect_var_kind, const char *=NULL)
Definition tree-vect-data-refs.cc:5251
#define STMT_VINFO_VECTYPE(S)
Definition tree-vectorizer.h:1564
opt_result vect_prune_runtime_alias_test_list(loop_vec_info)
Definition tree-vect-data-refs.cc:4075
bool needs_fold_left_reduction_p(tree, code_helper)
Definition tree-vect-loop.cc:3943
bool vect_nop_conversion_p(stmt_vec_info)
Definition tree-vect-stmts.cc:6137
void vect_finish_stmt_generation(vec_info *, stmt_vec_info, gimple *, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1436
#define LOOP_VINFO_NON_LINEAR_IV(L)
Definition tree-vectorizer.h:1089
unsigned HOST_WIDE_INT vect_max_vf(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2272
internal_fn vect_load_lanes_supported(tree, unsigned HOST_WIDE_INT, bool, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:6583
loop_vec_info vect_create_loop_vinfo(class loop *, vec_info_shared *, const vect_loop_form_info *, loop_vec_info=nullptr)
Definition tree-vect-loop.cc:1947
bool vect_stmt_dominates_stmt_p(gimple *, gimple *)
Definition tree-vectorizer.cc:801
tree vect_get_slp_scalar_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10306
bool vect_supportable_shift(vec_info *, enum tree_code, tree)
Definition tree-vect-stmts.cc:6335
void vect_optimize_slp(vec_info *)
Definition tree-vect-slp.cc:7433
void vect_iv_increment_position(edge, gimple_stmt_iterator *, bool *)
Definition tree-vect-loop-manip.cc:457
hash_set< scalar_cond_masked_key > scalar_cond_masked_set_type
Definition tree-vectorizer.h:446
tree vect_phi_initial_value(gphi *phi)
Definition tree-vectorizer.h:1835
opt_pointer_wrapper< loop_vec_info > opt_loop_vec_info
Definition tree-vectorizer.h:1163
bool vectorizable_reduction(loop_vec_info, stmt_vec_info, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:7501
unsigned record_stmt_cost(stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location)
Definition tree-vect-stmts.cc:118
bool vect_emulated_vector_p(tree)
Definition tree-vect-loop.cc:9374
tree get_same_sized_vectype(tree, tree)
Definition tree-vect-stmts.cc:14484
gimple * vect_gimple_build(tree, code_helper, tree, tree=NULL_TREE)
Definition tree-vect-patterns.cc:7214
void add_stmt_costs(vector_costs *costs, stmt_vector_for_cost *cost_vec)
Definition tree-vectorizer.h:1997
void vect_finish_replace_stmt(vec_info *, stmt_vec_info, gimple *)
Definition tree-vect-stmts.cc:1420
tree prepare_vec_mask(loop_vec_info, tree, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1693
void vect_slp_fini(void)
Definition tree-vect-slp.cc:82
#define LOOP_VINFO_NITERS_KNOWN_P(L)
Definition tree-vectorizer.h:1150
stmt_vec_info get_later_stmt(stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
Definition tree-vectorizer.h:1875
vec< std::pair< unsigned, unsigned > > lane_permutation_t
Definition tree-vectorizer.h:208
tree vect_gen_loop_len_mask(loop_vec_info, gimple_stmt_iterator *, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:11701
widest_int vect_iv_limit_for_partial_vectors(loop_vec_info loop_vinfo)
Definition tree-vect-loop.cc:12622
void vect_get_vec_defs_for_operand(vec_info *vinfo, stmt_vec_info, unsigned, tree op, vec< tree > *, tree=NULL)
Definition tree-vect-stmts.cc:1276
void vect_gather_slp_loads(vec_info *)
Definition tree-vect-slp.cc:7452
vect_partial_vector_style
Definition tree-vectorizer.h:365
@ vect_partial_vectors_none
Definition tree-vectorizer.h:366
@ vect_partial_vectors_len
Definition tree-vectorizer.h:369
@ vect_partial_vectors_avx512
Definition tree-vectorizer.h:368
@ vect_partial_vectors_while_ult
Definition tree-vectorizer.h:367
bool vect_gather_scatter_fn_p(vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4346
class _slp_instance * slp_instance
bool vect_maybe_update_slp_op_vectype(slp_tree, tree)
Definition tree-vect-stmts.cc:14758
bool compatible_calls_p(gcall *, gcall *)
Definition tree-vect-slp.cc:989
#define STMT_VINFO_REDUC_IDX(S)
Definition tree-vectorizer.h:1574
class loop * slpeel_tree_duplicate_loop_to_edge_cfg(class loop *, edge, class loop *, edge, edge, edge *, bool=true, vec< basic_block > *=NULL)
Definition tree-vect-loop-manip.cc:1478
unsigned int vect_vf_for_cost(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2254
unsigned int vect_nunits_for_cost(tree vec_type)
Definition tree-vectorizer.h:2264
bool vect_get_range_info(tree, wide_int *, wide_int *)
Definition tree-vect-patterns.cc:69
bool vect_grouped_load_supported(tree, bool, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:6466
#define STMT_VINFO_IN_PATTERN_P(S)
Definition tree-vectorizer.h:1593
tree vect_create_addr_base_for_vector_ref(vec_info *, stmt_vec_info, gimple_seq *, tree)
Definition tree-vect-data-refs.cc:5326
stmt_vec_info vect_find_last_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3391
tree vect_get_main_loop_result(loop_vec_info, tree, tree)
Definition tree-vect-loop-manip.cc:3019
#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L)
Definition tree-vectorizer.h:1088
#define STMT_VINFO_DR_INFO(S)
Definition tree-vectorizer.h:1590
hash_map< slp_node_hash, bool > slp_compat_nodes_map_t
Definition tree-vectorizer.h:2733
tree get_related_vectype_for_scalar_type(machine_mode, tree, poly_uint64=0)
Definition tree-vect-stmts.cc:14255
bool lane_reducing_op_p(code_helper code)
Definition tree-vectorizer.h:2307
bool nested_in_vect_loop_p(class loop *loop, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1824
bool vect_slp_analyze_instance_dependence(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:1164
auto_vec< std::pair< data_reference *, tree > > drs_init_vec
Definition tree-vectorizer.h:720
edge vec_init_loop_exit_info(class loop *)
Definition tree-vect-loop.cc:978
opt_result vect_analyze_data_ref_accesses(vec_info *, vec< int > *)
Definition tree-vect-data-refs.cc:3546
int vect_get_stmt_cost(enum vect_cost_for_stmt type_of_cost)
Definition tree-vectorizer.h:1946
void vect_transform_grouped_load(vec_info *, stmt_vec_info, vec< tree >, int, gimple_stmt_iterator *)
Definition tree-vect-data-refs.cc:7157
dump_user_location_t find_loop_location(class loop *)
Definition tree-vect-loop-manip.cc:2070
bool vect_slp_analyze_operations(vec_info *)
Definition tree-vect-slp.cc:8504
tree neutral_op_for_reduction(tree, code_helper, tree, bool=true)
Definition tree-vect-loop.cc:3885
void vect_free_slp_tree(slp_tree)
Definition tree-vect-slp.cc:173
unsigned vectorize_loops(void)
bool vect_can_force_dr_alignment_p(const_tree, poly_uint64)
Definition tree-vect-data-refs.cc:7239
tree bump_vector_ptr(vec_info *, tree, gimple *, gimple_stmt_iterator *, stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:5711
bool vect_apply_runtime_profitability_check_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2297
unsigned int vect_get_num_vectors(poly_uint64 nunits, tree vectype)
Definition tree-vectorizer.h:2184
#define DR_TARGET_ALIGNMENT(DR)
Definition tree-vectorizer.h:2031
void vect_free_loop_info_assumptions(class loop *)
Definition tree-vectorizer.cc:863
void set_dr_target_alignment(dr_vec_info *dr_info, poly_uint64 val)
Definition tree-vectorizer.h:2063
stmt_vec_info vect_stmt_to_vectorize(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1902
bool supportable_indirect_convert_operation(code_helper, tree, tree, vec< std::pair< tree, tree_code > > &, tree=NULL_TREE, slp_tree=NULL)
Definition tree-vect-stmts.cc:15300
opt_loop_vec_info vect_analyze_loop(class loop *, gimple *, vec_info_shared *)
Definition tree-vect-loop.cc:3507
void dr_set_safe_speculative_read_required(stmt_vec_info stmt_info, bool requires_alignment)
Definition tree-vectorizer.h:2050
int vect_reduc_type(vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2831
#define STMT_VINFO_RELATED_STMT(S)
Definition tree-vectorizer.h:1594
bool is_pattern_stmt_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1856
#define SLP_TREE_MEMORY_ACCESS_TYPE(S)
Definition tree-vectorizer.h:363
#define MAX_VECTORIZATION_FACTOR
Definition tree-vectorizer.h:1809
int vect_pow2(int x)
Definition tree-vectorizer.h:1923
bool reduction_fn_for_scalar_code(code_helper, internal_fn *)
Definition tree-vect-loop.cc:3823
void vect_pattern_recog(vec_info *)
Definition tree-vect-patterns.cc:7178
int vect_slp_child_index_for_operand(const gimple *, int op, bool)
Definition tree-vect-slp.cc:618
vect_induction_op_type
Definition tree-vectorizer.h:75
@ vect_step_op_add
Definition tree-vectorizer.h:76
@ vect_step_op_shl
Definition tree-vectorizer.h:79
@ vect_step_op_mul
Definition tree-vectorizer.h:78
@ vect_step_op_neg
Definition tree-vectorizer.h:77
@ vect_step_op_shr
Definition tree-vectorizer.h:80
void vect_set_loop_condition(class loop *, edge, loop_vec_info, tree, tree, tree, bool)
Definition tree-vect-loop-manip.cc:1393
bool lane_reducing_stmt_p(gimple *stmt)
Definition tree-vectorizer.h:2315
void vect_prepare_for_masked_peels(loop_vec_info)
Definition tree-vect-loop-manip.cc:2652
void vect_get_store_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:1015
stmt_vec_info vect_find_first_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3409
enum _complex_perm_kinds complex_perm_kinds_t
bool vect_is_integer_truncation(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2894
auto_vec< std::pair< unsigned, unsigned >, 16 > auto_lane_permutation_t
Definition tree-vectorizer.h:209
bool vect_use_loop_mask_for_alignment_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2167
tree vect_build_loop_niters(loop_vec_info, bool *=NULL)
Definition tree-vect-loop-manip.cc:2698
bool vect_get_loop_mask_type(loop_vec_info)
void vect_update_max_nunits(poly_uint64 *max_nunits, poly_uint64 nunits)
Definition tree-vectorizer.h:2230
slp_instance_kind
Definition tree-vectorizer.h:298
@ slp_inst_kind_ctor
Definition tree-vectorizer.h:303
@ slp_inst_kind_reduc_group
Definition tree-vectorizer.h:300
@ slp_inst_kind_store
Definition tree-vectorizer.h:299
@ slp_inst_kind_reduc_chain
Definition tree-vectorizer.h:301
@ slp_inst_kind_gcond
Definition tree-vectorizer.h:304
@ slp_inst_kind_bb_reduc
Definition tree-vectorizer.h:302
enum vect_cost_model loop_cost_model(loop_p loop)
Definition tree-vectorizer.h:2146
unsigned int vect_known_alignment_in_bytes(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vectorizer.h:2092
#define DR_MISALIGNMENT_UNKNOWN
Definition tree-vectorizer.h:2009
hash_map< slp_tree, complex_perm_kinds_t > slp_tree_to_load_perm_map_t
Definition tree-vectorizer.h:2728
opt_result vect_enhance_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2290
bool vect_is_reduction(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2811
#define LOOP_VINFO_EARLY_BREAKS(L)
Definition tree-vectorizer.h:1104
innermost_loop_behavior * vect_dr_behavior(vec_info *vinfo, dr_vec_info *dr_info)
Definition tree-vectorizer.h:2108
void vect_permute_store_chain(vec_info *, vec< tree > &, unsigned int, stmt_vec_info, gimple_stmt_iterator *, vec< tree > *)
Definition tree-vect-data-refs.cc:6040
tree perm_mask_for_reverse(tree)
Definition tree-vect-stmts.cc:1864
stmt_vec_info_type
Definition tree-vectorizer.h:1201
@ comparison_vec_info_type
Definition tree-vectorizer.h:1211
@ condition_vec_info_type
Definition tree-vectorizer.h:1210
@ type_conversion_vec_info_type
Definition tree-vectorizer.h:1216
@ load_vec_info_type
Definition tree-vectorizer.h:1203
@ loop_exit_ctrl_vec_info_type
Definition tree-vectorizer.h:1221
@ recurr_info_type
Definition tree-vectorizer.h:1220
@ op_vec_info_type
Definition tree-vectorizer.h:1206
@ reduc_vec_info_type
Definition tree-vectorizer.h:1212
@ assignment_vec_info_type
Definition tree-vectorizer.h:1209
@ undef_vec_info_type
Definition tree-vectorizer.h:1202
@ shift_vec_info_type
Definition tree-vectorizer.h:1205
@ type_demotion_vec_info_type
Definition tree-vectorizer.h:1215
@ store_vec_info_type
Definition tree-vectorizer.h:1204
@ cycle_phi_info_type
Definition tree-vectorizer.h:1217
@ phi_info_type
Definition tree-vectorizer.h:1219
@ lc_phi_info_type
Definition tree-vectorizer.h:1218
@ type_promotion_vec_info_type
Definition tree-vectorizer.h:1214
@ call_simd_clone_vec_info_type
Definition tree-vectorizer.h:1208
@ call_vec_info_type
Definition tree-vectorizer.h:1207
@ induc_vec_info_type
Definition tree-vectorizer.h:1213
void vect_get_slp_defs(slp_tree, vec< tree > *)
Definition tree-vect-slp.cc:10332
void vect_record_loop_len(loop_vec_info, vec_loop_lens *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11599
struct _slp_tree * slp_tree
Definition tree-vectorizer.h:25
gimple_seq vect_gen_len(tree, tree, tree, tree)
Definition tree-vect-stmts.cc:15648
bool dr_safe_speculative_read_required(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2036
tree vect_embedded_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2845
gimple * vect_loop_vectorized_call(class loop *, gcond **cond=NULL)
Definition tree-vectorizer.cc:878
void duplicate_and_interleave(vec_info *, gimple_seq *, tree, const vec< tree > &, unsigned int, vec< tree > &)
Definition tree-vect-slp.cc:9989
bool vect_grouped_store_supported(tree, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:5836
tree vect_setup_realignment(vec_info *, stmt_vec_info, gimple_stmt_iterator *, tree *, enum dr_alignment_support, tree, class loop **)
Definition tree-vect-data-refs.cc:6228
void vect_get_load_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool)
Definition tree-vect-stmts.cc:1065
vect_def_type
Definition tree-vectorizer.h:60
@ vect_induction_def
Definition tree-vectorizer.h:65
@ vect_uninitialized_def
Definition tree-vectorizer.h:61
@ vect_nested_cycle
Definition tree-vectorizer.h:68
@ vect_constant_def
Definition tree-vectorizer.h:62
@ vect_external_def
Definition tree-vectorizer.h:63
@ vect_condition_def
Definition tree-vectorizer.h:70
@ vect_internal_def
Definition tree-vectorizer.h:64
@ vect_reduction_def
Definition tree-vectorizer.h:66
@ vect_double_reduction_def
Definition tree-vectorizer.h:67
@ vect_first_order_recurrence
Definition tree-vectorizer.h:69
@ vect_unknown_def_type
Definition tree-vectorizer.h:71
opt_result vect_analyze_data_ref_dependences(loop_vec_info, unsigned int *)
Definition tree-vect-data-refs.cc:887
vect_relevant
Definition tree-vectorizer.h:1226
@ vect_used_in_outer
Definition tree-vectorizer.h:1236
@ vect_used_in_scope
Definition tree-vectorizer.h:1246
@ vect_unused_in_scope
Definition tree-vectorizer.h:1227
@ vect_used_by_reduction
Definition tree-vectorizer.h:1244
@ vect_used_in_outer_by_reduction
Definition tree-vectorizer.h:1233
@ vect_used_only_live
Definition tree-vectorizer.h:1230
tree vect_get_mask_load_else(int, tree)
Definition tree-vect-stmts.cc:2964
bool is_loop_header_bb_p(basic_block bb)
Definition tree-vectorizer.h:1912
opt_result vect_analyze_loop_form(class loop *, gimple *, vect_loop_form_info *)
Definition tree-vect-loop.cc:1751
#define STMT_VINFO_DR_WRT_VEC_LOOP(S)
Definition tree-vectorizer.h:1577
bool vectorizable_live_operation(vec_info *, stmt_vec_info, slp_tree, slp_instance, int, bool, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:11001
slp_tree vect_create_new_slp_node(unsigned, tree_code)
Definition tree-vect-slp.cc:228
opt_result vect_analyze_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2984
bool unlimited_cost_model(loop_p loop)
Definition tree-vectorizer.h:2157
bool vectorizable_recurr(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9218
unsigned int vect_get_num_copies(vec_info *vinfo, slp_tree node, tree vectype=NULL)
Definition tree-vectorizer.h:2196
opt_result vect_mark_stmts_to_be_vectorized(loop_vec_info, bool *)
Definition tree-vect-stmts.cc:699
tree vect_gen_while(gimple_seq *, tree, tree, tree, const char *=nullptr)
Definition tree-vect-stmts.cc:15436
tree vect_get_store_rhs(stmt_vec_info)
Definition tree-vect-stmts.cc:1945
class loop * vect_transform_loop(loop_vec_info, gimple *)
Definition tree-vect-loop.cc:12042
void vect_slp_init(void)
Definition tree-vect-slp.cc:76
bool vect_check_gather_scatter(stmt_vec_info, loop_vec_info, gather_scatter_info *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4451
unsigned int vect_get_scalar_dr_size(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2288
enum dr_alignment_support vect_supportable_dr_alignment(vec_info *, dr_vec_info *, tree, int)
Definition tree-vect-data-refs.cc:7262
opt_result vect_determine_partial_vectors_and_peeling(loop_vec_info)
Definition tree-vect-loop.cc:2634
bool ref_within_array_bound(gimple *, tree)
Definition tree-if-conv.cc:848
dr_alignment_support
Definition tree-vectorizer.h:51
@ dr_unaligned_supported
Definition tree-vectorizer.h:53
@ dr_explicit_realign_optimized
Definition tree-vectorizer.h:55
@ dr_aligned
Definition tree-vectorizer.h:56
@ dr_explicit_realign
Definition tree-vectorizer.h:54
@ dr_unaligned_unsupported
Definition tree-vectorizer.h:52
int compare_step_with_zero(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:1853
bool supportable_narrowing_operation(code_helper, tree, tree, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:15127
#define STMT_VINFO_REDUC_TYPE(S)
Definition tree-vectorizer.h:1604
operation_type
Definition tree-vectorizer.h:44
@ ternary_op
Definition tree-vectorizer.h:47
@ unary_op
Definition tree-vectorizer.h:45
@ binary_op
Definition tree-vectorizer.h:46
void vect_record_loop_mask(loop_vec_info, vec_loop_masks *, unsigned int, tree, tree)
Definition tree-vect-loop.cc:11459
vect_memory_access_type
Definition tree-vectorizer.h:167
@ VMAT_STRIDED_SLP
Definition tree-vectorizer.h:199
@ VMAT_INVARIANT
Definition tree-vectorizer.h:169
@ VMAT_GATHER_SCATTER
Definition tree-vectorizer.h:202
@ VMAT_ELEMENTWISE
Definition tree-vectorizer.h:193
@ VMAT_CONTIGUOUS_DOWN
Definition tree-vectorizer.h:177
@ VMAT_CONTIGUOUS
Definition tree-vectorizer.h:172
@ VMAT_CONTIGUOUS_PERMUTE
Definition tree-vectorizer.h:182
@ VMAT_LOAD_STORE_LANES
Definition tree-vectorizer.h:189
@ VMAT_CONTIGUOUS_REVERSE
Definition tree-vectorizer.h:186
bool vect_chooses_same_modes_p(vec_info *, machine_mode)
Definition tree-vect-stmts.cc:14502
bool slpeel_can_duplicate_loop_p(const class loop *, const_edge, const_edge)
Definition tree-vect-loop-manip.cc:2038
opt_result vect_analyze_stmt(vec_info *, stmt_vec_info, bool *, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:13787
tree vect_get_loop_mask(loop_vec_info, gimple_stmt_iterator *, vec_loop_masks *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11481
slp_vect_type
Definition tree-vectorizer.h:1265
@ loop_vect
Definition tree-vectorizer.h:1266
@ hybrid
Definition tree-vectorizer.h:1268
@ pure_slp
Definition tree-vectorizer.h:1267
vect_reduction_type
Definition tree-vectorizer.h:84
@ TREE_CODE_REDUCTION
Definition tree-vectorizer.h:85
@ FOLD_LEFT_REDUCTION
Definition tree-vectorizer.h:103
@ EXTRACT_LAST_REDUCTION
Definition tree-vectorizer.h:95
@ INTEGER_INDUC_COND_REDUCTION
Definition tree-vectorizer.h:87
@ CONST_COND_REDUCTION
Definition tree-vectorizer.h:88
@ COND_REDUCTION
Definition tree-vectorizer.h:86
bool supportable_widening_operation(vec_info *, code_helper, stmt_vec_info, tree, tree, code_helper *, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:14798
auto_vec< rgroup_controls > vec_loop_lens
Definition tree-vectorizer.h:718
void set_dr_misalignment(dr_vec_info *dr_info, int val)
Definition tree-vectorizer.h:2013
bool vect_use_mask_type_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1847
pair_hash< nofree_ptr_hash< _slp_tree >, nofree_ptr_hash< _slp_tree > > slp_node_hash
Definition tree-vectorizer.h:2732
hash_map< tree_operand_hash, std::pair< stmt_vec_info, innermost_loop_behavior * > > vec_base_alignments
Definition tree-vectorizer.h:128
vect_memory_access_type vect_mem_access_type(stmt_vec_info stmt_info, slp_tree node)
Definition tree-vectorizer.h:2820
unsigned add_stmt_cost(vector_costs *costs, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where)
Definition tree-vectorizer.h:1966
tree vect_get_smallest_scalar_type(stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:161
vec< stmt_info_for_cost > stmt_vector_for_cost
Definition tree-vectorizer.h:122
stmt_vec_info info_for_reduction(vec_info *, stmt_vec_info)
Definition tree-vect-loop.cc:5597
bool known_alignment_for_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2083
void vect_remove_stores(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:14225
#define DR_GROUP_FIRST_ELEMENT(S)
Definition tree-vectorizer.h:1613
hash_set< tree_cond_mask_hash > vec_cond_masked_set_type
Definition tree-vectorizer.h:452
#define STMT_VINFO_REDUC_DEF(S)
Definition tree-vectorizer.h:1607
opt_result vect_find_stmt_data_reference(loop_p, gimple *, vec< data_reference_p > *, vec< int > *, int)
Definition tree-vect-data-refs.cc:4779
tree get_dr_vinfo_offset(vec_info *vinfo, dr_vec_info *dr_info, bool check_outer=false)
Definition tree-vectorizer.h:2124
bool vect_slp_if_converted_bb(basic_block bb, loop_p orig_loop)
Definition tree-vect-slp.cc:9846
tree vect_gen_perm_mask_checked(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:10067
bool vectorizable_induction(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9996
opt_result vect_analyze_slp(vec_info *, unsigned, bool)
Definition tree-vect-slp.cc:4812
vect_pattern *(* vect_pattern_decl_t)(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
Definition tree-vectorizer.h:2782
tree vect_create_destination_var(tree, tree)
Definition tree-vect-data-refs.cc:5802
bool vect_transform_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:9049
tree vect_create_data_ref_ptr(vec_info *, stmt_vec_info, tree, class loop *, tree, tree *, gimple_stmt_iterator *, gimple **, bool, tree=NULL_TREE)
Definition tree-vect-data-refs.cc:5449
void vect_gen_vector_loop_niters(loop_vec_info, tree, tree *, tree *, bool)
Definition tree-vect-loop-manip.cc:2789
bool vect_make_slp_decision(loop_vec_info)
Definition tree-vect-slp.cc:7511
bool aligned_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2073
unsigned HOST_WIDE_INT tree_to_uhwi(const_tree t)
Definition tree.cc:6594
#define TYPE_ALIGN_UNIT(NODE)
Definition tree.h:2400
poly_uint64 TYPE_VECTOR_SUBPARTS(const_tree node)
Definition tree.h:4311
#define TYPE_PRECISION(NODE)
Definition tree.h:2306
#define TREE_OPERAND(NODE, I)
Definition tree.h:1300
#define sizetype
Definition tree.h:5182
#define TREE_CODE_CLASS(CODE)
Definition tree.h:202
#define TYPE_SIZE_UNIT(NODE)
Definition tree.h:2303
#define CONVERT_EXPR_CODE_P(CODE)
Definition tree.h:553
#define TREE_TYPE(NODE)
Definition tree.h:512
#define INTEGRAL_TYPE_P(TYPE)
Definition tree.h:613
#define COMPARISON_CLASS_P(NODE)
Definition tree.h:256
#define NULL_TREE
Definition tree.h:317
#define FOR_EACH_VEC_ELT(V, I, P)
Definition vec.h:1895
constexpr vnull vNULL
Definition vec.h:569
generic_wide_int< widest_int_storage< WIDEST_INT_MAX_PRECISION > > widest_int
Definition wide-int.h:345
generic_wide_int< wide_int_storage > wide_int
Definition wide-int.h:343