21#ifndef GCC_TREE_VECTORIZER_H
22#define GCC_TREE_VECTORIZER_H
105#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
106 || ((D) == vect_double_reduction_def) \
107 || ((D) == vect_nested_cycle))
126 std::pair<stmt_vec_info, innermost_loop_behavior *> >
133 tree op (
unsigned int i)
const;
234 static void *
operator new (size_t);
237 static void operator delete (
void *, size_t);
294#define SLP_INSTANCE_TREE(S) (S)->root
295#define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
296#define SLP_INSTANCE_LOADS(S) (S)->loads
297#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
298#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
299#define SLP_INSTANCE_KIND(S) (S)->kind
301#define SLP_TREE_CHILDREN(S) (S)->children
302#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
303#define SLP_TREE_SCALAR_OPS(S) (S)->ops
304#define SLP_TREE_REF_COUNT(S) (S)->refcnt
305#define SLP_TREE_VEC_DEFS(S) (S)->vec_defs
306#define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
307#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
308#define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
309#define SLP_TREE_SIMD_CLONE_INFO(S) (S)->simd_clone_info
310#define SLP_TREE_DEF_TYPE(S) (S)->def_type
311#define SLP_TREE_VECTYPE(S) (S)->vectype
312#define SLP_TREE_REPRESENTATIVE(S) (S)->representative
313#define SLP_TREE_LANES(S) (S)->lanes
314#define SLP_TREE_CODE(S) (S)->code
350 static inline hashval_t
372 static const bool empty_zero_p =
true;
974#define LOOP_VINFO_LOOP(L) (L)->loop
975#define LOOP_VINFO_IV_EXIT(L) (L)->vec_loop_iv_exit
976#define LOOP_VINFO_EPILOGUE_IV_EXIT(L) (L)->vec_epilogue_loop_iv_exit
977#define LOOP_VINFO_SCALAR_IV_EXIT(L) (L)->scalar_loop_iv_exit
978#define LOOP_VINFO_BBS(L) (L)->bbs
979#define LOOP_VINFO_NBBS(L) (L)->nbbs
980#define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1
981#define LOOP_VINFO_NITERS(L) (L)->num_iters
985#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
986#define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions
987#define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th
988#define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold
989#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
990#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
991#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
992#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
993#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
994#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \
995 (L)->epil_using_partial_vectors_p
996#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
997#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
998#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
999#define LOOP_VINFO_MASKS(L) (L)->masks
1000#define LOOP_VINFO_LENS(L) (L)->lens
1001#define LOOP_VINFO_MASK_SKIP_NITERS(L) (L)->mask_skip_niters
1002#define LOOP_VINFO_RGROUP_COMPARE_TYPE(L) (L)->rgroup_compare_type
1003#define LOOP_VINFO_RGROUP_IV_TYPE(L) (L)->rgroup_iv_type
1004#define LOOP_VINFO_PARTIAL_VECTORS_STYLE(L) (L)->partial_vector_style
1005#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
1006#define LOOP_VINFO_N_STMTS(L) (L)->shared->n_stmts
1007#define LOOP_VINFO_LOOP_NEST(L) (L)->shared->loop_nest
1008#define LOOP_VINFO_DATAREFS(L) (L)->shared->datarefs
1009#define LOOP_VINFO_DDRS(L) (L)->shared->ddrs
1010#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
1011#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
1012#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
1013#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
1014#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
1015#define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs
1016#define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs
1017#define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero
1018#define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds
1019#define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
1020#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
1021#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
1022#define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
1023#define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
1024#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
1025#define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter
1026#define LOOP_VINFO_EARLY_BREAKS(L) (L)->early_breaks
1027#define LOOP_VINFO_EARLY_BRK_STORES(L) (L)->early_break_stores
1028#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L) \
1029 (single_pred ((L)->loop->latch) != (L)->vec_loop_iv_exit->src)
1030#define LOOP_VINFO_EARLY_BRK_DEST_BB(L) (L)->early_break_dest_bb
1031#define LOOP_VINFO_EARLY_BRK_VUSES(L) (L)->early_break_vuses
1032#define LOOP_VINFO_LOOP_CONDS(L) (L)->conds
1033#define LOOP_VINFO_LOOP_IV_COND(L) (L)->loop_iv_cond
1034#define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
1035#define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop
1036#define LOOP_VINFO_SCALAR_LOOP_SCALING(L) (L)->scalar_loop_scaling
1037#define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store
1038#define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
1039#define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info
1040#define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
1041#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
1043#define LOOP_VINFO_FULLY_MASKED_P(L) \
1044 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1045 && !LOOP_VINFO_MASKS (L).is_empty ())
1047#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \
1048 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1049 && !LOOP_VINFO_LENS (L).is_empty ())
1051#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
1052 ((L)->may_misalign_stmts.length () > 0)
1053#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
1054 ((L)->comp_alias_ddrs.length () > 0 \
1055 || (L)->check_unequal_addrs.length () > 0 \
1056 || (L)->lower_bounds.length () > 0)
1057#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \
1058 (LOOP_VINFO_NITERS_ASSUMPTIONS (L))
1059#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \
1060 (LOOP_VINFO_SIMD_IF_COND (L))
1061#define LOOP_REQUIRES_VERSIONING(L) \
1062 (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \
1063 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \
1064 || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
1065 || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
1067#define LOOP_VINFO_NITERS_KNOWN_P(L) \
1068 (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
1070#define LOOP_VINFO_EPILOGUE_P(L) \
1071 (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL)
1073#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \
1074 (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L)))
1108#define BB_VINFO_BBS(B) (B)->bbs
1109#define BB_VINFO_NBBS(B) (B)->nbbs
1110#define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
1111#define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
1112#define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
1113#define BB_VINFO_DDRS(B) (B)->shared->ddrs
1508#define STMT_VINFO_TYPE(S) (S)->type
1509#define STMT_VINFO_STMT(S) (S)->stmt
1510#define STMT_VINFO_RELEVANT(S) (S)->relevant
1511#define STMT_VINFO_LIVE_P(S) (S)->live
1512#define STMT_VINFO_VECTYPE(S) (S)->vectype
1513#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
1514#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
1515#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
1516#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
1517#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
1518#define STMT_VINFO_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
1519#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
1520#define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
1521#define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
1522#define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
1523#define STMT_VINFO_FORCE_SINGLE_CYCLE(S) (S)->force_single_cycle
1525#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop
1526#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address
1527#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init
1528#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset
1529#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step
1530#define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment
1531#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \
1532 (S)->dr_wrt_vec_loop.base_misalignment
1533#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \
1534 (S)->dr_wrt_vec_loop.offset_alignment
1535#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
1536 (S)->dr_wrt_vec_loop.step_alignment
1538#define STMT_VINFO_DR_INFO(S) \
1539 (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
1541#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
1542#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
1543#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
1544#define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info
1545#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
1546#define STMT_VINFO_GROUPED_ACCESS(S) \
1547 ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
1548#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
1549#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
1550#define STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE(S) (S)->loop_phi_evolution_type
1551#define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
1552#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
1553#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
1554#define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn
1555#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
1556#define STMT_VINFO_REDUC_VECTYPE(S) (S)->reduc_vectype
1557#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
1558#define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
1559#define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p
1561#define DR_GROUP_FIRST_ELEMENT(S) \
1562 (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
1563#define DR_GROUP_NEXT_ELEMENT(S) \
1564 (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
1565#define DR_GROUP_SIZE(S) \
1566 (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
1567#define DR_GROUP_STORE_COUNT(S) \
1568 (gcc_checking_assert ((S)->dr_aux.dr), (S)->store_count)
1569#define DR_GROUP_GAP(S) \
1570 (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
1572#define REDUC_GROUP_FIRST_ELEMENT(S) \
1573 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
1574#define REDUC_GROUP_NEXT_ELEMENT(S) \
1575 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
1576#define REDUC_GROUP_SIZE(S) \
1577 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
1579#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
1581#define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
1582#define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
1583#define STMT_SLP_TYPE(S) (S)->slp_type
1613 tree vectype,
int misalign,
1676 m_costing_for_scalar (costing_for_scalar),
1678 m_suggested_unroll_factor(1),
1736#define VECT_MAX_COST 1000
1740#define MAX_INTERM_CVT_STEPS 3
1742#define MAX_VECTORIZATION_FACTOR INT_MAX
1749#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \
1750 (TREE_CODE (TYPE) == BOOLEAN_TYPE \
1751 || ((TREE_CODE (TYPE) == INTEGER_TYPE \
1752 || TREE_CODE (TYPE) == ENUMERAL_TYPE) \
1753 && TYPE_PRECISION (TYPE) == 1 \
1754 && TYPE_UNSIGNED (TYPE)))
1782 return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
1791 return stmt_info->pattern_stmt_p;
1846 for (
i = 0;
i < x;
i++)
1856 tree vectype,
int misalign)
1858 return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
1875 return targetm.vectorize.create_costs (vinfo, costing_for_scalar);
1888 tree vectype,
int misalign,
1891 unsigned cost =
costs->add_stmt_cost (
count, kind, stmt_info, node, vectype,
1914 i->vectype,
i->misalign,
i->where);
1921 unsigned *prologue_cost,
unsigned *body_cost,
1922 unsigned *epilogue_cost,
unsigned *suggested_unroll_factor =
NULL)
1924 costs->finish_cost (scalar_costs);
1925 *prologue_cost =
costs->prologue_cost ();
1926 *body_cost =
costs->body_cost ();
1927 *epilogue_cost =
costs->epilogue_cost ();
1928 if (suggested_unroll_factor)
1929 *suggested_unroll_factor =
costs->suggested_unroll_factor ();
1945#define DR_MISALIGNMENT_UNKNOWN (-1)
1946#define DR_MISALIGNMENT_UNINITIALIZED (-2)
1957#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
1965 return dr_info->target_alignment;
1967#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR)
1972 dr_info->target_alignment = val;
1974#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL)
2004 else if (misalignment == 0)
2006 return misalignment & -misalignment;
2018 if (loop_vinfo ==
NULL
2057 return flag_simd_cost_model;
2058 return flag_vect_cost_model;
2135 *max_nunits = force_common_multiple (*max_nunits, nunits);
2171inline unsigned HOST_WIDE_INT
2174 unsigned HOST_WIDE_INT vf;
2209 return code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR;
2237#define DUMP_VECT_SCOPE(MSG) \
2238 AUTO_DUMP_SCOPE (MSG, vect_location)
2314 vec<std::pair<tree, tree_code> > *,
2368 unsigned int *,
unsigned int *,
2379 const char * =
nullptr);
2383 tree *,
unsigned int = 0);
2436 const char * =
NULL);
2462 unsigned int,
tree,
unsigned int);
2464 tree,
unsigned int);
2467 unsigned int,
unsigned int);
2470 unsigned int,
tree,
tree,
unsigned int,
2534 unsigned * =
nullptr,
bool =
false);
2551 unsigned int * =
NULL,
2599typedef hash_map <slp_tree, complex_perm_kinds_t>
2603typedef pair_hash <nofree_ptr_hash <_slp_tree>,
2631 this->m_node = node;
2632 this->m_ops.create (0);
2634 this->m_ops.safe_splice (*
m_ops);
2649 this->m_ops.release ();
2747 return (def_stmt_info
edge loop_preheader_edge(const class loop *loop)
Definition cfgloop.cc:1760
class loop * loop_p
Definition cfgloop.h:98
Definition tree-vectorizer.h:1100
vec< slp_root > roots
Definition tree-vectorizer.h:1105
_bb_vec_info(vec< basic_block > bbs, vec_info_shared *)
Definition tree-vect-slp.cc:6480
~_bb_vec_info()
Definition tree-vect-slp.cc:6517
Definition tree-vectorizer.h:684
bool peeling_for_gaps
Definition tree-vectorizer.h:893
bool can_use_partial_vectors_p
Definition tree-vectorizer.h:863
signed char partial_load_store_bias
Definition tree-vectorizer.h:888
class vector_costs * vector_costs
Definition tree-vectorizer.h:702
class dr_vec_info * unaligned_dr
Definition tree-vectorizer.h:793
_loop_vec_info(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:1030
edge skip_main_loop_edge
Definition tree-vectorizer.h:728
auto_vec< stmt_vec_info > reduction_chains
Definition tree-vectorizer.h:836
unsigned int th
Definition tree-vectorizer.h:710
auto_vec< gimple * > early_break_vuses
Definition tree-vectorizer.h:970
bool using_partial_vectors_p
Definition tree-vectorizer.h:867
bool no_data_dependencies
Definition tree-vectorizer.h:926
class loop * scalar_loop
Definition tree-vectorizer.h:936
vec< _loop_vec_info * > epilogue_vinfos
Definition tree-vectorizer.h:944
scalar_cond_masked_set_type scalar_cond_masked_set
Definition tree-vectorizer.h:761
auto_vec< ddr_p > may_alias_ddrs
Definition tree-vectorizer.h:810
tree num_itersm1
Definition tree-vectorizer.h:693
unsigned int inner_loop_cost_factor
Definition tree-vectorizer.h:854
bool vectorizable
Definition tree-vectorizer.h:857
vect_partial_vector_style partial_vector_style
Definition tree-vectorizer.h:790
poly_uint64 slp_unrolling_factor
Definition tree-vectorizer.h:850
edge vec_epilogue_loop_iv_exit
Definition tree-vectorizer.h:952
tree num_iters
Definition tree-vectorizer.h:695
hash_map< tree, tree > * scan_map
Definition tree-vectorizer.h:846
int peeling_for_alignment
Definition tree-vectorizer.h:803
vec_cond_masked_set_type vec_cond_masked_set
Definition tree-vectorizer.h:764
bool epil_using_partial_vectors_p
Definition tree-vectorizer.h:883
bool early_breaks
Definition tree-vectorizer.h:901
poly_uint64 versioning_threshold
Definition tree-vectorizer.h:716
auto_vec< vec_lower_bound > lower_bounds
Definition tree-vectorizer.h:825
profile_probability scalar_loop_scaling
Definition tree-vectorizer.h:932
auto_vec< gimple * > early_break_stores
Definition tree-vectorizer.h:962
edge main_loop_edge
Definition tree-vectorizer.h:727
edge skip_this_loop_edge
Definition tree-vectorizer.h:732
auto_vec< stmt_info_for_cost > scalar_cost_vec
Definition tree-vectorizer.h:839
gcond * loop_iv_cond
Definition tree-vectorizer.h:907
basic_block early_break_dest_bb
Definition tree-vectorizer.h:966
bool using_select_vl_p
Definition tree-vectorizer.h:879
tree rgroup_compare_type
Definition tree-vectorizer.h:775
poly_uint64 vectorization_factor
Definition tree-vectorizer.h:719
bool peeling_for_niter
Definition tree-vectorizer.h:897
auto_vec< dr_with_seg_len_pair_t > comp_alias_ddrs
Definition tree-vectorizer.h:814
edge vec_loop_iv_exit
Definition tree-vectorizer.h:948
tree num_iters_unchanged
Definition tree-vectorizer.h:697
vec_loop_lens lens
Definition tree-vectorizer.h:758
unsigned HOST_WIDE_INT max_vectorization_factor
Definition tree-vectorizer.h:750
unsigned int suggested_unroll_factor
Definition tree-vectorizer.h:746
hash_map< tree_operand_hash, tree > * ivexpr_map
Definition tree-vectorizer.h:842
auto_vec< stmt_vec_info > may_misalign_stmts
Definition tree-vectorizer.h:829
tree rgroup_iv_type
Definition tree-vectorizer.h:786
vec_loop_masks masks
Definition tree-vectorizer.h:754
tree num_iters_assumptions
Definition tree-vectorizer.h:699
tree simd_if_cond
Definition tree-vectorizer.h:782
auto_vec< tree > check_nonzero
Definition tree-vectorizer.h:822
tree mask_skip_niters
Definition tree-vectorizer.h:770
bool has_mask_store
Definition tree-vectorizer.h:929
auto_vec< stmt_vec_info > reductions
Definition tree-vectorizer.h:832
edge scalar_loop_iv_exit
Definition tree-vectorizer.h:956
class loop * loop
Definition tree-vectorizer.h:690
~_loop_vec_info()
Definition tree-vect-loop.cc:1148
bool using_decrementing_iv_p
Definition tree-vectorizer.h:874
auto_vec< gcond * > conds
Definition tree-vectorizer.h:904
_loop_vec_info * orig_loop_info
Definition tree-vectorizer.h:940
hash_map< tree, vect_reusable_accumulator > reusable_accumulators
Definition tree-vectorizer.h:739
int ptr_mask
Definition tree-vectorizer.h:806
class vector_costs * scalar_costs
Definition tree-vectorizer.h:705
auto_vec< vec_object_pair > check_unequal_addrs
Definition tree-vectorizer.h:817
Definition tree-vectorizer.h:257
poly_uint64 unrolling_factor
Definition tree-vectorizer.h:271
slp_instance_kind kind
Definition tree-vectorizer.h:287
vec< stmt_vec_info > root_stmts
Definition tree-vectorizer.h:264
slp_tree root
Definition tree-vectorizer.h:260
dump_user_location_t location() const
Definition tree-vect-slp.cc:197
vec< slp_tree > loads
Definition tree-vectorizer.h:274
vec< _slp_instance * > subgraph_entries
Definition tree-vectorizer.h:284
vec< tree > remain_defs
Definition tree-vectorizer.h:268
slp_tree reduc_phis
Definition tree-vectorizer.h:277
stmt_vector_for_cost cost_vec
Definition tree-vectorizer.h:280
Definition tree-vectorizer.h:1261
unsigned int min_input_precision
Definition tree-vectorizer.h:1441
code_helper reduc_code
Definition tree-vectorizer.h:1386
unsigned int store_count
Definition tree-vectorizer.h:1348
tree vectype
Definition tree-vectorizer.h:1287
vec< tree > simd_clone_info
Definition tree-vectorizer.h:1331
tree induc_cond_initial_val
Definition tree-vectorizer.h:1376
bool in_pattern_p
Definition tree-vectorizer.h:1271
stmt_vec_info first_element
Definition tree-vectorizer.h:1341
unsigned int mask_precision
Definition tree-vectorizer.h:1464
vec< tree > reduc_initial_values
Definition tree-vectorizer.h:1414
int reduc_idx
Definition tree-vectorizer.h:1392
gimple * stmt
Definition tree-vectorizer.h:1284
tree loop_phi_evolution_base_unchanged
Definition tree-vectorizer.h:1308
enum vect_induction_op_type loop_phi_evolution_type
Definition tree-vectorizer.h:1310
innermost_loop_behavior dr_wrt_vec_loop
Definition tree-vectorizer.h:1300
vec< gimple * > vec_stmts
Definition tree-vectorizer.h:1290
bool slp_vect_only_p
Definition tree-vectorizer.h:1467
bool live
Definition tree-vectorizer.h:1268
unsigned int gap
Definition tree-vectorizer.h:1351
unsigned int min_output_precision
Definition tree-vectorizer.h:1436
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:1373
internal_fn reduc_fn
Definition tree-vectorizer.h:1388
enum stmt_vec_info_type type
Definition tree-vectorizer.h:1264
unsigned int min_neg_dist
Definition tree-vectorizer.h:1355
vec< tree > reduc_scalar_results
Definition tree-vectorizer.h:1421
tree reduc_vectype_in
Definition tree-vectorizer.h:1400
bool vectorizable
Definition tree-vectorizer.h:1281
bool pattern_stmt_p
Definition tree-vectorizer.h:1277
bool strided_p
Definition tree-vectorizer.h:1366
enum vect_relevant relevant
Definition tree-vectorizer.h:1360
unsigned int size
Definition tree-vectorizer.h:1345
signop operation_sign
Definition tree-vectorizer.h:1447
bool force_single_cycle
Definition tree-vectorizer.h:1429
bool is_reduc_info
Definition tree-vectorizer.h:1432
gimple_seq pattern_def_seq
Definition tree-vectorizer.h:1326
stmt_vec_info related_stmt
Definition tree-vectorizer.h:1321
unsigned simd_lane_access_p
Definition tree-vectorizer.h:1369
bool slp_vect_pattern_only_p
Definition tree-vectorizer.h:1471
tree reduc_vectype
Definition tree-vectorizer.h:1403
stmt_vec_info reduc_def
Definition tree-vectorizer.h:1397
bool gather_scatter_p
Definition tree-vectorizer.h:1363
enum vect_def_type def_type
Definition tree-vectorizer.h:1334
stmt_vec_info next_element
Definition tree-vectorizer.h:1343
unsigned int operation_precision
Definition tree-vectorizer.h:1446
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:1383
tree reduc_epilogue_adjustment
Definition tree-vectorizer.h:1379
unsigned int reduc_result_pos
Definition tree-vectorizer.h:1409
enum slp_vect_type slp_type
Definition tree-vectorizer.h:1337
dr_vec_info dr_aux
Definition tree-vectorizer.h:1296
tree loop_phi_evolution_part
Definition tree-vectorizer.h:1309
vect_reusable_accumulator * reused_accumulator
Definition tree-vectorizer.h:1426
Definition tree-vectorizer.h:2253
~auto_purge_vect_location()
Definition tree-vectorizer.cc:94
Definition tree-vectorizer.h:1236
unsigned group
Definition tree-vectorizer.h:1244
tree base_decl
Definition tree-vectorizer.h:1252
bool base_misaligned
Definition tree-vectorizer.h:1251
tree offset
Definition tree-vectorizer.h:1256
int misalignment
Definition tree-vectorizer.h:1246
poly_uint64 target_alignment
Definition tree-vectorizer.h:1249
data_reference * dr
Definition tree-vectorizer.h:1239
stmt_vec_info stmt
Definition tree-vectorizer.h:1241
Definition dumpfile.h:340
Definition genmatch.cc:845
bool is_empty() const
Definition hash-set.h:107
void add_int(unsigned v)
Definition inchash.h:55
void add_flag(bool flag)
Definition inchash.h:124
hashval_t end()
Definition inchash.h:49
class loop * inner
Definition cfgloop.h:148
unsigned force_vectorize
Definition cfgloop.h:227
void * aux
Definition cfgloop.h:154
Definition opt-problem.h:230
Definition opt-problem.h:179
Definition profile-count.h:147
Definition tree-vectorizer.h:424
vec< data_reference_p > datarefs
Definition tree-vectorizer.h:436
vec< data_reference > datarefs_copy
Definition tree-vectorizer.h:437
auto_vec< loop_p > loop_nest
Definition tree-vectorizer.h:440
void check_datarefs()
Definition tree-vectorizer.cc:507
unsigned n_stmts
Definition tree-vectorizer.h:433
void save_datarefs()
Definition tree-vectorizer.cc:497
~vec_info_shared()
Definition tree-vectorizer.cc:489
vec< ddr_p > ddrs
Definition tree-vectorizer.h:444
vec_info_shared()
Definition tree-vectorizer.cc:481
Definition tree-vectorizer.h:448
stmt_vec_info lookup_stmt(gimple *)
Definition tree-vectorizer.cc:547
basic_block * bbs
Definition tree-vectorizer.h:507
auto_vec< stmt_vec_info > grouped_stores
Definition tree-vectorizer.h:492
~vec_info()
Definition tree-vectorizer.cc:473
auto_vec< slp_instance > slp_instances
Definition tree-vectorizer.h:484
void replace_stmt(gimple_stmt_iterator *, stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:632
void remove_stmt(stmt_vec_info)
Definition tree-vectorizer.cc:616
void insert_on_entry(stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:674
void set_vinfo_for_stmt(gimple *, stmt_vec_info, bool=true)
Definition tree-vectorizer.cc:721
vec_base_alignments base_alignments
Definition tree-vectorizer.h:488
machine_mode vector_mode
Definition tree-vectorizer.h:500
stmt_vec_info new_stmt_vec_info(gimple *stmt)
Definition tree-vectorizer.cc:685
class dr_vec_info * lookup_dr(data_reference *)
Definition tree-vectorizer.cc:589
vec< stmt_vec_info > stmt_vec_infos
Definition tree-vectorizer.h:475
void move_dr(stmt_vec_info, stmt_vec_info)
Definition tree-vectorizer.cc:601
bool any_known_not_updated_vssa
Definition tree-vectorizer.h:481
vec_kind
Definition tree-vectorizer.h:451
@ loop
Definition tree-vectorizer.h:451
@ bb
Definition tree-vectorizer.h:451
void free_stmt_vec_infos()
Definition tree-vectorizer.cc:742
vec_info_shared * shared
Definition tree-vectorizer.h:472
unsigned int nbbs
Definition tree-vectorizer.h:510
stmt_vec_info add_stmt(gimple *)
Definition tree-vectorizer.cc:522
vec_kind kind
Definition tree-vectorizer.h:469
mode_set used_vector_modes
Definition tree-vectorizer.h:495
hash_set< int_hash< machine_mode, E_VOIDmode, E_BLKmode > > mode_set
Definition tree-vectorizer.h:450
void insert_seq_on_entry(stmt_vec_info, gimple_seq)
Definition tree-vectorizer.cc:648
stmt_vec_info lookup_def(tree)
Definition tree-vectorizer.cc:564
void free_stmt_vec_info(stmt_vec_info)
Definition tree-vectorizer.cc:753
stmt_vec_info lookup_single_use(tree)
Definition tree-vectorizer.cc:577
vec_info(vec_kind, vec_info_shared *)
Definition tree-vectorizer.cc:463
stmt_vec_info add_pattern_stmt(gimple *, stmt_vec_info)
Definition tree-vectorizer.cc:534
bool stmt_vec_info_ro
Definition tree-vectorizer.h:477
Definition tree-vectorizer.h:411
vec_lower_bound()
Definition tree-vectorizer.h:413
poly_uint64 min_value
Definition tree-vectorizer.h:419
bool unsigned_p
Definition tree-vectorizer.h:418
vec_lower_bound(tree e, bool u, poly_uint64 m)
Definition tree-vectorizer.h:414
tree expr
Definition tree-vectorizer.h:417
Definition vec-perm-indices.h:51
Definition tree-vectorizer.h:2612
static vect_pattern * recognize(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
internal_fn m_ifn
Definition tree-vectorizer.h:2618
vec< slp_tree > m_ops
Definition tree-vectorizer.h:2625
virtual ~vect_pattern()
Definition tree-vectorizer.h:2647
virtual void build(vec_info *)=0
vect_pattern(slp_tree *node, vec< slp_tree > *m_ops, internal_fn ifn)
Definition tree-vectorizer.h:2628
slp_tree * m_node
Definition tree-vectorizer.h:2621
unsigned m_num_args
Definition tree-vectorizer.h:2615
Definition tree-vectorizer.h:1587
unsigned int adjust_cost_for_freq(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1840
virtual ~vector_costs()
Definition tree-vectorizer.h:1590
bool m_costing_for_scalar
Definition tree-vectorizer.h:1658
vector_costs(vec_info *, bool)
Definition tree-vectorizer.h:1674
vec_info * m_vinfo
Definition tree-vectorizer.h:1654
virtual unsigned int add_stmt_cost(int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, vect_cost_model_location where)
Definition tree-vectorizer.cc:1802
virtual bool better_main_loop_than_p(const vector_costs *other) const
Definition tree-vectorizer.cc:1860
unsigned int m_costs[3]
Definition tree-vectorizer.h:1661
unsigned int outside_cost() const
Definition tree-vectorizer.h:1713
unsigned int record_stmt_cost(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1826
virtual bool better_epilogue_loop_than_p(const vector_costs *other, loop_vec_info main_loop) const
Definition tree-vectorizer.cc:1879
unsigned int epilogue_cost() const
Definition tree-vectorizer.h:1704
unsigned int m_suggested_unroll_factor
Definition tree-vectorizer.h:1664
unsigned int suggested_unroll_factor() const
Definition tree-vectorizer.h:1730
bool m_finished
Definition tree-vectorizer.h:1667
int compare_outside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:2055
unsigned int body_cost() const
Definition tree-vectorizer.h:1695
virtual void finish_cost(const vector_costs *scalar_costs)
Definition tree-vectorizer.cc:1815
unsigned int prologue_cost() const
Definition tree-vectorizer.h:1686
unsigned int total_cost() const
Definition tree-vectorizer.h:1722
int compare_inside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:1962
class edge_def * edge
Definition coretypes.h:345
const class edge_def * const_edge
Definition coretypes.h:346
const union tree_node * const_tree
Definition coretypes.h:98
union tree_node * tree
Definition coretypes.h:97
static unsigned int count[debug_counter_number_of_counters]
Definition dbgcnt.cc:50
dump_flags_t dump_flags
Definition dumpfile.cc:64
FILE * dump_file
Definition dumpfile.cc:62
@ TDF_DETAILS
Definition dumpfile.h:92
vect_cost_model
Definition flag-types.h:279
@ VECT_COST_MODEL_UNLIMITED
Definition flag-types.h:283
@ VECT_COST_MODEL_DEFAULT
Definition flag-types.h:284
bool operand_equal_p(const_tree arg0, const_tree arg1, unsigned int flags)
Definition fold-const.cc:4234
#define fold_convert(T1, T2)
Definition fold-const.h:92
#define fold_build2(c, t1, t2, t3)
Definition fold-const.h:72
internal_fn
Definition genmatch.cc:360
tree_code
Definition genmatch.cc:347
unsigned gimple_uid(const gimple *g)
Definition gimple.h:2084
enum tree_code gimple_assign_rhs_code(const gassign *gs)
Definition gimple.h:2850
tree gimple_assign_rhs1(const gassign *gs)
Definition gimple.h:2660
basic_block gimple_bb(const gimple *g)
Definition gimple.h:1860
tree gimple_assign_lhs(const gassign *gs)
Definition gimple.h:2609
#define init_cost
Definition ira-costs.cc:63
T dyn_cast(U *p)
Definition is-a.h:280
void add_expr(const_tree t, inchash::hash &hstate, unsigned int flags)
Definition fold-const.cc:4248
i
Definition poly-int.h:772
rtx offset
Definition postreload.cc:691
signop
Definition signop.h:28
Definition tree-vectorizer.h:174
stmt_vec_info representative
Definition tree-vectorizer.h:190
unsigned int lanes
Definition tree-vectorizer.h:222
load_permutation_t load_permutation
Definition tree-vectorizer.h:194
slp_tree prev_node
Definition tree-vectorizer.h:241
enum tree_code code
Definition tree-vectorizer.h:224
unsigned int vec_stmts_size
Definition tree-vectorizer.h:212
vec< tree > vec_defs
Definition tree-vectorizer.h:207
~_slp_tree()
Definition tree-vect-slp.cc:133
unsigned int refcnt
Definition tree-vectorizer.h:215
slp_tree next_node
Definition tree-vectorizer.h:240
int vertex
Definition tree-vectorizer.h:226
enum vect_def_type def_type
Definition tree-vectorizer.h:220
bool * failed
Definition tree-vectorizer.h:231
vec< tree > simd_clone_info
Definition tree-vectorizer.h:203
vec< tree > ops
Definition tree-vectorizer.h:187
void push_vec_def(gimple *def)
Definition tree-vect-slp.cc:155
poly_uint64 max_nunits
Definition tree-vectorizer.h:218
lane_permutation_t lane_permutation
Definition tree-vectorizer.h:198
_slp_tree()
Definition tree-vect-slp.cc:106
tree vectype
Definition tree-vectorizer.h:205
void push_vec_def(tree def)
Definition tree-vectorizer.h:179
vec< slp_tree > children
Definition tree-vectorizer.h:182
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:185
Definition basic-block.h:117
class loop * loop_father
Definition basic-block.h:126
Definition ira-costs.cc:54
Definition tree-data-ref.h:151
struct innermost_loop_behavior innermost
Definition tree-data-ref.h:173
Definition loop-invariant.cc:88
static void mark_empty(value_type &v)
Definition tree-vectorizer.h:375
scalar_cond_masked_key value_type
Definition tree-vectorizer.h:348
static bool equal(value_type existing, value_type candidate)
Definition tree-vectorizer.h:363
static void remove(value_type &)
Definition tree-vectorizer.h:394
static hashval_t hash(value_type v)
Definition tree-vectorizer.h:351
static void mark_deleted(value_type &)
Definition tree-vectorizer.h:387
static bool is_deleted(const value_type &)
Definition tree-vectorizer.h:389
static bool is_empty(value_type v)
Definition tree-vectorizer.h:382
scalar_cond_masked_key compare_type
Definition tree-vectorizer.h:347
Definition hash-traits.h:466
Definition graphite.h:338
data_reference_p dr
Definition graphite.h:343
Definition function.h:249
Definition tree-vectorizer.h:1475
tree decl
Definition tree-vectorizer.h:1482
tree element_type
Definition tree-vectorizer.h:1501
tree base
Definition tree-vectorizer.h:1485
tree offset_vectype
Definition tree-vectorizer.h:1498
int scale
Definition tree-vectorizer.h:1492
enum vect_def_type offset_dt
Definition tree-vectorizer.h:1495
tree offset
Definition tree-vectorizer.h:1488
internal_fn ifn
Definition tree-vectorizer.h:1478
tree memory_type
Definition tree-vectorizer.h:1504
Definition gimple-iterator.h:26
Definition tree-data-ref.h:50
tree offset
Definition tree-data-ref.h:52
Definition hash-traits.h:122
Definition hash-traits.h:345
Definition tree-vectorizer.h:623
tree compare_type
Definition tree-vectorizer.h:643
unsigned int max_nscalars_per_iter
Definition tree-vectorizer.h:627
unsigned int factor
Definition tree-vectorizer.h:633
tree type
Definition tree-vectorizer.h:639
tree bias_adjusted_ctrl
Definition tree-vectorizer.h:650
vec< tree > controls
Definition tree-vectorizer.h:646
Definition tree-vectorizer.h:328
unsigned ncopies
Definition tree-vectorizer.h:337
tree_code code
Definition tree-vectorizer.h:339
scalar_cond_masked_key(tree t, unsigned ncopies_)
Definition tree-vectorizer.h:329
void get_cond_ops_from_tree(tree)
Definition tree-vectorizer.cc:1752
tree op0
Definition tree-vectorizer.h:340
tree op1
Definition tree-vectorizer.h:341
bool inverted_p
Definition tree-vectorizer.h:338
Definition tree-vectorizer.h:1089
vec< tree > remain
Definition tree-vectorizer.h:1096
slp_instance_kind kind
Definition tree-vectorizer.h:1093
vec< stmt_vec_info > roots
Definition tree-vectorizer.h:1095
slp_root(slp_instance_kind kind_, vec< stmt_vec_info > stmts_, vec< stmt_vec_info > roots_, vec< tree > remain_=vNULL)
Definition tree-vectorizer.h:1090
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:1094
Definition tree-vectorizer.h:111
tree vectype
Definition tree-vectorizer.h:117
int misalign
Definition tree-vectorizer.h:118
slp_tree node
Definition tree-vectorizer.h:116
enum vect_cost_model_location where
Definition tree-vectorizer.h:114
enum vect_cost_for_stmt kind
Definition tree-vectorizer.h:113
stmt_vec_info stmt_info
Definition tree-vectorizer.h:115
int count
Definition tree-vectorizer.h:112
Definition tree-hash-traits.h:25
Definition hash-traits.h:75
Definition tree-vectorizer.h:654
hash_set< pair_hash< nofree_ptr_hash< tree_node >, int_hash< unsigned, 0 > > > mask_set
Definition tree-vectorizer.h:659
auto_vec< rgroup_controls > rgc_vec
Definition tree-vectorizer.h:662
bool is_empty() const
Definition tree-vectorizer.h:655
Definition tree-vectorizer.h:671
stmt_vec_info reduc_info
Definition tree-vectorizer.h:678
tree reduc_input
Definition tree-vectorizer.h:674
Definition tree-vectorizer.h:150
static void mark_deleted(value_type &s)
Definition tree-vectorizer.h:156
static bool is_empty(const value_type &s)
Definition tree-vectorizer.h:159
static bool is_deleted(const value_type &s)
Definition tree-vectorizer.h:158
static const bool empty_zero_p
Definition tree-vectorizer.h:154
static void mark_empty(value_type &s)
Definition tree-vectorizer.h:157
static hashval_t hash(const value_type &)
Definition tree-vect-slp.cc:6645
vect_scalar_ops_slice value_type
Definition tree-vectorizer.h:151
static bool equal(const value_type &, const compare_type &)
Definition tree-vect-slp.cc:6654
vect_scalar_ops_slice compare_type
Definition tree-vectorizer.h:152
Definition tree-vectorizer.h:132
unsigned int length
Definition tree-vectorizer.h:138
unsigned int start
Definition tree-vectorizer.h:137
bool all_same_p() const
Definition tree-vect-slp.cc:6636
tree op(unsigned int i) const
Definition tree-vectorizer.h:143
vec< tree > * ops
Definition tree-vectorizer.h:136
#define NULL
Definition system.h:50
#define gcc_assert(EXPR)
Definition system.h:821
#define false
Definition system.h:895
#define gcc_checking_assert(EXPR)
Definition system.h:828
HOST_WIDE_INT estimated_poly_value(poly_int64 x, poly_value_estimate_kind kind=POLY_VALUE_LIKELY)
Definition target.h:306
struct gcc_target targetm
vect_cost_model_location
Definition target.h:209
@ vect_body
Definition target.h:211
@ vect_prologue
Definition target.h:210
@ vect_epilogue
Definition target.h:212
vect_cost_for_stmt
Definition target.h:187
@ scalar_stmt
Definition target.h:188
@ cond_branch_taken
Definition target.h:201
@ vec_to_scalar
Definition target.h:198
@ cond_branch_not_taken
Definition target.h:200
@ tcc_comparison
Definition tree-core.h:233
#define DR_IS_WRITE(DR)
Definition tree-data-ref.h:192
#define DR_IS_READ(DR)
Definition tree-data-ref.h:191
#define DR_REF(DR)
Definition tree-data-ref.h:185
#define DR_INNERMOST(DR)
Definition tree-data-ref.h:203
static tree candidate(unsigned uid)
Definition tree-sra.cc:325
#define PHI_ARG_DEF_FROM_EDGE(PHI, E)
Definition tree-ssa-operands.h:88
_complex_operation
Definition tree-vect-slp-patterns.cc:119
stmt_vec_info vect_orig_stmt(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1798
#define LOOP_VINFO_COST_MODEL_THRESHOLD(L)
Definition tree-vectorizer.h:987
bool vect_can_vectorize_without_simd_p(tree_code)
Definition tree-vect-loop.cc:9705
tree cse_and_gimplify_to_preheader(loop_vec_info, tree)
Definition tree-vect-loop.cc:1171
void finish_cost(vector_costs *costs, const vector_costs *scalar_costs, unsigned *prologue_cost, unsigned *body_cost, unsigned *epilogue_cost, unsigned *suggested_unroll_factor=NULL)
Definition tree-vectorizer.h:1920
int builtin_vectorization_cost(enum vect_cost_for_stmt type_of_cost, tree vectype, int misalign)
Definition tree-vectorizer.h:1855
loop_vec_info loop_vec_info_for_loop(class loop *loop)
Definition tree-vectorizer.h:1083
bool vect_is_extending_load(class vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2730
bool vect_transform_stmt(vec_info *, stmt_vec_info, gimple_stmt_iterator *, slp_tree, slp_instance)
Definition tree-vect-stmts.cc:13375
bool vect_is_simple_use(tree, vec_info *, enum vect_def_type *, stmt_vec_info *=NULL, gimple **=NULL)
Definition tree-vect-stmts.cc:13848
bool vect_rgroup_iv_might_wrap_p(loop_vec_info, rgroup_controls *)
Definition tree-vect-loop.cc:12967
tree vect_halve_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11539
void dump_stmt_cost(FILE *, int, enum vect_cost_for_stmt, stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location)
Definition tree-vectorizer.cc:102
vec< unsigned > load_permutation_t
Definition tree-vectorizer.h:169
bool vect_slp_analyze_instance_alignment(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:2901
internal_fn vect_store_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:5763
tree vect_get_new_vect_var(tree, enum vect_var_kind, const char *)
Definition tree-vect-data-refs.cc:5022
void vect_mark_pattern_stmts(vec_info *, stmt_vec_info, gimple *, tree)
Definition tree-vect-patterns.cc:7206
class loop * vect_loop_versioning(loop_vec_info, gimple *)
Definition tree-vect-loop-manip.cc:3979
bool vect_check_gather_scatter(stmt_vec_info, loop_vec_info, gather_scatter_info *)
Definition tree-vect-data-refs.cc:4277
void vect_get_store_cost(vec_info *, stmt_vec_info, int, dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:987
tree vect_get_slp_vect_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:8923
internal_fn vect_load_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:6390
opt_tree vect_get_mask_type_for_stmt(stmt_vec_info, unsigned int=0)
#define LOOP_VINFO_VECT_FACTOR(L)
Definition tree-vectorizer.h:997
void vect_free_slp_instance(slp_instance)
Definition tree-vect-slp.cc:209
#define SLP_TREE_LANES(S)
Definition tree-vectorizer.h:313
bool vect_slp_function(function *)
Definition tree-vect-slp.cc:8475
std::pair< tree, tree > vec_object_pair
Definition tree-vectorizer.h:407
tree vect_gen_perm_mask_any(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9675
bool vect_transform_reduction(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, gimple **, slp_tree)
Definition tree-vect-loop.cc:8657
_bb_vec_info * bb_vec_info
#define LOOP_VINFO_FULLY_MASKED_P(L)
Definition tree-vectorizer.h:1043
int vect_get_known_peeling_cost(loop_vec_info, int, int *, stmt_vector_for_cost *, stmt_vector_for_cost *, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:4525
bool vect_transform_slp_perm_load(vec_info *, slp_tree, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, unsigned *, unsigned *=nullptr, bool=false)
tree vect_gen_while_not(gimple_seq *, tree, tree, tree)
Definition tree-vect-stmts.cc:14766
tree vect_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2720
bool is_simple_and_all_uses_invariant(stmt_vec_info, loop_vec_info)
Definition tree-vect-stmts.cc:308
bool vectorizable_phi(vec_info *, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9398
#define STMT_VINFO_GROUPED_ACCESS(S)
Definition tree-vectorizer.h:1546
void vect_schedule_slp(vec_info *, const vec< slp_instance > &)
Definition tree-vect-slp.cc:10216
tree vect_peel_nonlinear_iv_init(gimple_seq *, tree, tree, tree, enum vect_induction_op_type)
Definition tree-vect-loop.cc:9831
bool check_reduction_path(dump_user_location_t, loop_p, gphi *, tree, enum tree_code)
Definition tree-vect-loop.cc:4205
bool can_duplicate_and_interleave_p(vec_info *, unsigned int, tree, unsigned int *=NULL, tree *=NULL, tree *=NULL)
Definition tree-vect-slp.cc:421
auto_vec< unsigned, 16 > auto_load_permutation_t
Definition tree-vectorizer.h:170
#define SLP_TREE_VECTYPE(S)
Definition tree-vectorizer.h:311
tree get_vectype_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13688
#define LOOP_VINFO_LOOP(L)
Definition tree-vectorizer.h:974
_complex_perm_kinds
Definition tree-vectorizer.h:2588
@ PERM_EVENEVEN
Definition tree-vectorizer.h:2593
@ PERM_TOP
Definition tree-vectorizer.h:2595
@ PERM_ODDEVEN
Definition tree-vectorizer.h:2591
@ PERM_EVENODD
Definition tree-vectorizer.h:2590
@ PERM_UNKNOWN
Definition tree-vectorizer.h:2589
@ PERM_ODDODD
Definition tree-vectorizer.h:2592
void vect_record_base_alignments(vec_info *)
Definition tree-vect-data-refs.cc:1273
void vect_get_vec_defs(vec_info *, stmt_vec_info, slp_tree, unsigned, tree, vec< tree > *, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL)
Definition tree-vect-stmts.cc:1335
vec_load_store_type
Definition tree-vectorizer.h:1190
@ VLS_LOAD
Definition tree-vectorizer.h:1191
@ VLS_STORE
Definition tree-vectorizer.h:1192
@ VLS_STORE_INVARIANT
Definition tree-vectorizer.h:1193
tree get_mask_type_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13763
pair_hash< tree_operand_hash, tree_operand_hash > tree_cond_mask_hash
Definition tree-vectorizer.h:402
vect_pattern_decl_t slp_patterns[]
Definition tree-vect-slp-patterns.cc:1636
_loop_vec_info * loop_vec_info
tree vect_double_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11549
vect_var_kind
Definition tree-vectorizer.h:35
@ vect_pointer_var
Definition tree-vectorizer.h:37
@ vect_scalar_var
Definition tree-vectorizer.h:38
@ vect_mask_var
Definition tree-vectorizer.h:39
@ vect_simple_var
Definition tree-vectorizer.h:36
int vect_get_place_in_interleaving_chain(stmt_vec_info, stmt_vec_info)
Definition tree-vect-slp.cc:392
class loop * vect_do_peeling(loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, tree *)
Definition tree-vect-loop-manip.cc:3118
void optimize_mask_stores(class loop *)
Definition tree-vect-loop.cc:12732
class _stmt_vec_info * stmt_vec_info
Definition tree-vectorizer.h:24
#define STMT_VINFO_DATA_REF(S)
Definition tree-vectorizer.h:1515
tree vect_get_loop_len(loop_vec_info, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:11736
bool vect_is_store_elt_extraction(vect_cost_for_stmt kind, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2674
void vect_detect_hybrid_slp(loop_vec_info)
Definition tree-vect-slp.cc:6395
opt_result vect_get_vector_types_for_stmt(vec_info *, stmt_vec_info, tree *, tree *, unsigned int=0)
Definition tree-vect-stmts.cc:14790
const poly_uint64 dr_target_alignment(dr_vec_info *dr_info)
Definition tree-vectorizer.h:1961
opt_result vect_analyze_loop_form(class loop *, vect_loop_form_info *)
Definition tree-vect-loop.cc:1739
void vect_update_inits_of_drs(loop_vec_info, tree, tree_code)
Definition tree-vect-loop-manip.cc:2612
struct data_reference * dr_p
Definition tree-vectorizer.h:1259
bool vect_can_advance_ivs_p(loop_vec_info)
Definition tree-vect-loop-manip.cc:2201
opt_result vect_analyze_data_refs(vec_info *, poly_uint64 *, bool *)
Definition tree-vect-data-refs.cc:4763
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vect-data-refs.cc:1196
tree vect_init_vector(vec_info *, stmt_vec_info, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1174
void vect_copy_ref_info(tree, tree)
Definition tree-vect-data-refs.cc:5589
void vect_record_grouped_load_vectors(vec_info *, stmt_vec_info, vec< tree >)
Definition tree-vect-data-refs.cc:6995
bool vectorizable_lc_phi(loop_vec_info, stmt_vec_info, gimple **, slp_tree)
Definition tree-vect-loop.cc:9340
bool vectorizable_lane_reducing(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:7504
tree vect_get_new_ssa_name(tree, enum vect_var_kind, const char *=NULL)
Definition tree-vect-data-refs.cc:5060
#define STMT_VINFO_VECTYPE(S)
Definition tree-vectorizer.h:1512
opt_result vect_prune_runtime_alias_test_list(loop_vec_info)
Definition tree-vect-data-refs.cc:3914
bool needs_fold_left_reduction_p(tree, code_helper)
Definition tree-vect-loop.cc:3972
bool vect_nop_conversion_p(stmt_vec_info)
Definition tree-vect-stmts.cc:5831
void vect_finish_stmt_generation(vec_info *, stmt_vec_info, gimple *, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1396
unsigned HOST_WIDE_INT vect_max_vf(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2172
loop_vec_info vect_create_loop_vinfo(class loop *, vec_info_shared *, const vect_loop_form_info *, loop_vec_info=nullptr)
Definition tree-vect-loop.cc:1916
bool vect_stmt_dominates_stmt_p(gimple *, gimple *)
Definition tree-vectorizer.cc:773
bool vect_supportable_shift(vec_info *, enum tree_code, tree)
Definition tree-vect-stmts.cc:6029
void vect_optimize_slp(vec_info *)
Definition tree-vect-slp.cc:6218
void vect_iv_increment_position(edge, gimple_stmt_iterator *, bool *)
Definition tree-vect-loop-manip.cc:457
hash_set< scalar_cond_masked_key > scalar_cond_masked_set_type
Definition tree-vectorizer.h:397
tree vect_phi_initial_value(gphi *phi)
Definition tree-vectorizer.h:1768
opt_pointer_wrapper< loop_vec_info > opt_loop_vec_info
Definition tree-vectorizer.h:1080
bool vectorizable_reduction(loop_vec_info, stmt_vec_info, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:7666
unsigned record_stmt_cost(stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location)
Definition tree-vect-stmts.cc:117
bool vect_emulated_vector_p(tree)
Definition tree-vect-loop.cc:9694
tree get_same_sized_vectype(tree, tree)
Definition tree-vect-stmts.cc:13799
gimple * vect_gimple_build(tree, code_helper, tree, tree=NULL_TREE)
Definition tree-vect-patterns.cc:7510
void add_stmt_costs(vector_costs *costs, stmt_vector_for_cost *cost_vec)
Definition tree-vectorizer.h:1933
void vect_finish_replace_stmt(vec_info *, stmt_vec_info, gimple *)
Definition tree-vect-stmts.cc:1380
tree prepare_vec_mask(loop_vec_info, tree, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1647
void vect_slp_fini(void)
Definition tree-vect-slp.cc:81
#define LOOP_VINFO_NITERS_KNOWN_P(L)
Definition tree-vectorizer.h:1067
stmt_vec_info get_later_stmt(stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
Definition tree-vectorizer.h:1808
vec< std::pair< unsigned, unsigned > > lane_permutation_t
Definition tree-vectorizer.h:167
tree vect_gen_loop_len_mask(loop_vec_info, gimple_stmt_iterator *, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:11803
widest_int vect_iv_limit_for_partial_vectors(loop_vec_info loop_vinfo)
Definition tree-vect-loop.cc:12926
void vect_get_vec_defs_for_operand(vec_info *vinfo, stmt_vec_info, unsigned, tree op, vec< tree > *, tree=NULL)
Definition tree-vect-stmts.cc:1246
void vect_gather_slp_loads(vec_info *)
Definition tree-vect-slp.cc:6237
vect_partial_vector_style
Definition tree-vectorizer.h:316
@ vect_partial_vectors_none
Definition tree-vectorizer.h:317
@ vect_partial_vectors_len
Definition tree-vectorizer.h:320
@ vect_partial_vectors_avx512
Definition tree-vectorizer.h:319
@ vect_partial_vectors_while_ult
Definition tree-vectorizer.h:318
class _slp_instance * slp_instance
bool vect_maybe_update_slp_op_vectype(slp_tree, tree)
Definition tree-vect-stmts.cc:14078
bool compatible_calls_p(gcall *, gcall *)
Definition tree-vect-slp.cc:958
#define STMT_VINFO_REDUC_IDX(S)
Definition tree-vectorizer.h:1522
class loop * slpeel_tree_duplicate_loop_to_edge_cfg(class loop *, edge, class loop *, edge, edge, edge *, bool=true, vec< basic_block > *=NULL)
Definition tree-vect-loop-manip.cc:1478
unsigned int vect_vf_for_cost(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2154
unsigned int vect_nunits_for_cost(tree vec_type)
Definition tree-vectorizer.h:2164
bool vect_get_range_info(tree, wide_int *, wide_int *)
Definition tree-vect-patterns.cc:67
bool vect_grouped_load_supported(tree, bool, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:6275
#define STMT_VINFO_IN_PATTERN_P(S)
Definition tree-vectorizer.h:1541
dump_user_location_t vect_location
Definition tree-vectorizer.cc:88
tree vect_create_addr_base_for_vector_ref(vec_info *, stmt_vec_info, gimple_seq *, tree)
Definition tree-vect-data-refs.cc:5135
stmt_vec_info vect_find_last_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3030
tree vect_get_main_loop_result(loop_vec_info, tree, tree)
Definition tree-vect-loop-manip.cc:3009
#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L)
Definition tree-vectorizer.h:1011
#define STMT_VINFO_DR_INFO(S)
Definition tree-vectorizer.h:1538
hash_map< slp_node_hash, bool > slp_compat_nodes_map_t
Definition tree-vectorizer.h:2605
bool vect_gather_scatter_fn_p(vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *)
Definition tree-vect-data-refs.cc:4176
tree get_related_vectype_for_scalar_type(machine_mode, tree, poly_uint64=0)
Definition tree-vect-stmts.cc:13570
bool lane_reducing_op_p(code_helper code)
Definition tree-vectorizer.h:2207
bool nested_in_vect_loop_p(class loop *loop, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1757
bool vect_slp_analyze_instance_dependence(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:1148
auto_vec< std::pair< data_reference *, tree > > drs_init_vec
Definition tree-vectorizer.h:667
edge vec_init_loop_exit_info(class loop *)
Definition tree-vect-loop.cc:974
opt_result vect_analyze_data_ref_accesses(vec_info *, vec< int > *)
Definition tree-vect-data-refs.cc:3392
int vect_get_stmt_cost(enum vect_cost_for_stmt type_of_cost)
Definition tree-vectorizer.h:1865
void vect_transform_grouped_load(vec_info *, stmt_vec_info, vec< tree >, int, gimple_stmt_iterator *)
Definition tree-vect-data-refs.cc:6964
dump_user_location_t find_loop_location(class loop *)
Definition tree-vect-loop-manip.cc:2060
opt_result vect_analyze_slp(vec_info *, unsigned)
Definition tree-vect-slp.cc:3951
bool vect_slp_analyze_operations(vec_info *)
Definition tree-vect-slp.cc:7169
tree neutral_op_for_reduction(tree, code_helper, tree, bool=true)
Definition tree-vect-loop.cc:3914
void vect_free_slp_tree(slp_tree)
Definition tree-vect-slp.cc:169
unsigned vectorize_loops(void)
bool vect_can_force_dr_alignment_p(const_tree, poly_uint64)
Definition tree-vect-data-refs.cc:7046
tree bump_vector_ptr(vec_info *, tree, gimple *, gimple_stmt_iterator *, stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:5520
bool vect_apply_runtime_profitability_check_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2197
unsigned int vect_get_num_vectors(poly_uint64 nunits, tree vectype)
Definition tree-vectorizer.h:2084
#define DR_TARGET_ALIGNMENT(DR)
Definition tree-vectorizer.h:1967
void vect_free_loop_info_assumptions(class loop *)
Definition tree-vectorizer.cc:835
void set_dr_target_alignment(dr_vec_info *dr_info, poly_uint64 val)
Definition tree-vectorizer.h:1970
stmt_vec_info vect_stmt_to_vectorize(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1821
int vect_reduc_type(vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2691
#define STMT_VINFO_RELATED_STMT(S)
Definition tree-vectorizer.h:1542
bool is_pattern_stmt_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1789
#define MAX_VECTORIZATION_FACTOR
Definition tree-vectorizer.h:1742
int vect_pow2(int x)
Definition tree-vectorizer.h:1842
bool reduction_fn_for_scalar_code(code_helper, internal_fn *)
Definition tree-vect-loop.cc:3852
void vect_pattern_recog(vec_info *)
Definition tree-vect-patterns.cc:7474
int vect_slp_child_index_for_operand(const gimple *, int op, bool)
Definition tree-vect-slp.cc:598
vect_induction_op_type
Definition tree-vectorizer.h:74
@ vect_step_op_add
Definition tree-vectorizer.h:75
@ vect_step_op_shl
Definition tree-vectorizer.h:78
@ vect_step_op_mul
Definition tree-vectorizer.h:77
@ vect_step_op_neg
Definition tree-vectorizer.h:76
@ vect_step_op_shr
Definition tree-vectorizer.h:79
void vect_set_loop_condition(class loop *, edge, loop_vec_info, tree, tree, tree, bool)
Definition tree-vect-loop-manip.cc:1393
bool lane_reducing_stmt_p(gimple *stmt)
Definition tree-vectorizer.h:2215
void vect_prepare_for_masked_peels(loop_vec_info)
Definition tree-vect-loop-manip.cc:2642
stmt_vec_info vect_find_first_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3048
enum _complex_perm_kinds complex_perm_kinds_t
bool vect_is_integer_truncation(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2754
auto_vec< std::pair< unsigned, unsigned >, 16 > auto_lane_permutation_t
Definition tree-vectorizer.h:168
bool vect_use_loop_mask_for_alignment_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2073
tree vect_build_loop_niters(loop_vec_info, bool *=NULL)
Definition tree-vect-loop-manip.cc:2688
bool vect_get_loop_mask_type(loop_vec_info)
void vect_update_max_nunits(poly_uint64 *max_nunits, poly_uint64 nunits)
Definition tree-vectorizer.h:2130
slp_instance_kind
Definition tree-vectorizer.h:247
@ slp_inst_kind_ctor
Definition tree-vectorizer.h:252
@ slp_inst_kind_reduc_group
Definition tree-vectorizer.h:249
@ slp_inst_kind_store
Definition tree-vectorizer.h:248
@ slp_inst_kind_reduc_chain
Definition tree-vectorizer.h:250
@ slp_inst_kind_bb_reduc
Definition tree-vectorizer.h:251
enum vect_cost_model loop_cost_model(loop_p loop)
Definition tree-vectorizer.h:2052
#define DR_MISALIGNMENT_UNKNOWN
Definition tree-vectorizer.h:1945
hash_map< slp_tree, complex_perm_kinds_t > slp_tree_to_load_perm_map_t
Definition tree-vectorizer.h:2600
opt_result vect_enhance_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2144
bool vect_is_reduction(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2683
innermost_loop_behavior * vect_dr_behavior(vec_info *vinfo, dr_vec_info *dr_info)
Definition tree-vectorizer.h:2014
void vect_permute_store_chain(vec_info *, vec< tree > &, unsigned int, stmt_vec_info, gimple_stmt_iterator *, vec< tree > *)
Definition tree-vect-data-refs.cc:5849
tree perm_mask_for_reverse(tree)
Definition tree-vect-stmts.cc:1809
stmt_vec_info_type
Definition tree-vectorizer.h:1118
@ comparison_vec_info_type
Definition tree-vectorizer.h:1128
@ condition_vec_info_type
Definition tree-vectorizer.h:1127
@ type_conversion_vec_info_type
Definition tree-vectorizer.h:1133
@ load_vec_info_type
Definition tree-vectorizer.h:1120
@ loop_exit_ctrl_vec_info_type
Definition tree-vectorizer.h:1138
@ recurr_info_type
Definition tree-vectorizer.h:1137
@ op_vec_info_type
Definition tree-vectorizer.h:1123
@ reduc_vec_info_type
Definition tree-vectorizer.h:1129
@ assignment_vec_info_type
Definition tree-vectorizer.h:1126
@ undef_vec_info_type
Definition tree-vectorizer.h:1119
@ shift_vec_info_type
Definition tree-vectorizer.h:1122
@ type_demotion_vec_info_type
Definition tree-vectorizer.h:1132
@ store_vec_info_type
Definition tree-vectorizer.h:1121
@ cycle_phi_info_type
Definition tree-vectorizer.h:1134
@ phi_info_type
Definition tree-vectorizer.h:1136
@ lc_phi_info_type
Definition tree-vectorizer.h:1135
@ type_promotion_vec_info_type
Definition tree-vectorizer.h:1131
@ call_simd_clone_vec_info_type
Definition tree-vectorizer.h:1125
@ call_vec_info_type
Definition tree-vectorizer.h:1124
@ induc_vec_info_type
Definition tree-vectorizer.h:1130
void vect_get_slp_defs(slp_tree, vec< tree > *)
Definition tree-vect-slp.cc:8931
void vect_record_loop_len(loop_vec_info, vec_loop_lens *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11701
struct _slp_tree * slp_tree
Definition tree-vectorizer.h:25
gimple_seq vect_gen_len(tree, tree, tree, tree)
Definition tree-vect-stmts.cc:14954
bool vect_transform_cycle_phi(loop_vec_info, stmt_vec_info, gimple **, slp_tree, slp_instance)
Definition tree-vect-loop.cc:9041
tree vect_embedded_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2705
gimple * vect_loop_vectorized_call(class loop *, gcond **cond=NULL)
Definition tree-vectorizer.cc:850
void duplicate_and_interleave(vec_info *, gimple_seq *, tree, const vec< tree > &, unsigned int, vec< tree > &)
Definition tree-vect-slp.cc:8607
bool vect_grouped_store_supported(tree, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:5645
tree vect_setup_realignment(vec_info *, stmt_vec_info, gimple_stmt_iterator *, tree *, enum dr_alignment_support, tree, class loop **)
Definition tree-vect-data-refs.cc:6037
vect_def_type
Definition tree-vectorizer.h:59
@ vect_induction_def
Definition tree-vectorizer.h:64
@ vect_uninitialized_def
Definition tree-vectorizer.h:60
@ vect_nested_cycle
Definition tree-vectorizer.h:67
@ vect_constant_def
Definition tree-vectorizer.h:61
@ vect_external_def
Definition tree-vectorizer.h:62
@ vect_condition_def
Definition tree-vectorizer.h:69
@ vect_internal_def
Definition tree-vectorizer.h:63
@ vect_reduction_def
Definition tree-vectorizer.h:65
@ vect_double_reduction_def
Definition tree-vectorizer.h:66
@ vect_first_order_recurrence
Definition tree-vectorizer.h:68
@ vect_unknown_def_type
Definition tree-vectorizer.h:70
opt_result vect_analyze_data_ref_dependences(loop_vec_info, unsigned int *)
Definition tree-vect-data-refs.cc:871
vect_relevant
Definition tree-vectorizer.h:1143
@ vect_used_in_outer
Definition tree-vectorizer.h:1153
@ vect_used_in_scope
Definition tree-vectorizer.h:1163
@ vect_unused_in_scope
Definition tree-vectorizer.h:1144
@ vect_used_by_reduction
Definition tree-vectorizer.h:1161
@ vect_used_in_outer_by_reduction
Definition tree-vectorizer.h:1150
@ vect_used_only_live
Definition tree-vectorizer.h:1147
bool is_loop_header_bb_p(basic_block bb)
Definition tree-vectorizer.h:1831
#define STMT_VINFO_DR_WRT_VEC_LOOP(S)
Definition tree-vectorizer.h:1525
bool vectorizable_live_operation(vec_info *, stmt_vec_info, slp_tree, slp_instance, int, bool, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:11109
slp_tree vect_create_new_slp_node(unsigned, tree_code)
Definition tree-vect-slp.cc:224
opt_result vect_analyze_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2834
unsigned int vect_known_alignment_in_bytes(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:1999
bool unlimited_cost_model(loop_p loop)
Definition tree-vectorizer.h:2063
bool vectorizable_recurr(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9541
unsigned int vect_get_num_copies(vec_info *vinfo, slp_tree node, tree vectype=NULL)
Definition tree-vectorizer.h:2096
opt_result vect_mark_stmts_to_be_vectorized(loop_vec_info, bool *)
Definition tree-vect-stmts.cc:673
tree vect_gen_while(gimple_seq *, tree, tree, tree, const char *=nullptr)
Definition tree-vect-stmts.cc:14742
tree vect_get_store_rhs(stmt_vec_info)
Definition tree-vect-stmts.cc:1890
class loop * vect_transform_loop(loop_vec_info, gimple *)
Definition tree-vect-loop.cc:12259
void vect_slp_init(void)
Definition tree-vect-slp.cc:75
unsigned int vect_get_scalar_dr_size(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2188
enum dr_alignment_support vect_supportable_dr_alignment(vec_info *, dr_vec_info *, tree, int)
Definition tree-vect-data-refs.cc:7069
opt_result vect_determine_partial_vectors_and_peeling(loop_vec_info)
Definition tree-vect-loop.cc:2646
bool ref_within_array_bound(gimple *, tree)
Definition tree-if-conv.cc:848
dr_alignment_support
Definition tree-vectorizer.h:50
@ dr_unaligned_supported
Definition tree-vectorizer.h:52
@ dr_explicit_realign_optimized
Definition tree-vectorizer.h:54
@ dr_aligned
Definition tree-vectorizer.h:55
@ dr_explicit_realign
Definition tree-vectorizer.h:53
@ dr_unaligned_unsupported
Definition tree-vectorizer.h:51
bool supportable_narrowing_operation(code_helper, tree, tree, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:14446
#define STMT_VINFO_REDUC_TYPE(S)
Definition tree-vectorizer.h:1552
operation_type
Definition tree-vectorizer.h:43
@ ternary_op
Definition tree-vectorizer.h:46
@ unary_op
Definition tree-vectorizer.h:44
@ binary_op
Definition tree-vectorizer.h:45
void vect_record_loop_mask(loop_vec_info, vec_loop_masks *, unsigned int, tree, tree)
Definition tree-vect-loop.cc:11561
vect_memory_access_type
Definition tree-vectorizer.h:1198
@ VMAT_STRIDED_SLP
Definition tree-vectorizer.h:1230
@ VMAT_INVARIANT
Definition tree-vectorizer.h:1200
@ VMAT_GATHER_SCATTER
Definition tree-vectorizer.h:1233
@ VMAT_ELEMENTWISE
Definition tree-vectorizer.h:1224
@ VMAT_CONTIGUOUS_DOWN
Definition tree-vectorizer.h:1208
@ VMAT_CONTIGUOUS
Definition tree-vectorizer.h:1203
@ VMAT_CONTIGUOUS_PERMUTE
Definition tree-vectorizer.h:1213
@ VMAT_LOAD_STORE_LANES
Definition tree-vectorizer.h:1220
@ VMAT_CONTIGUOUS_REVERSE
Definition tree-vectorizer.h:1217
bool vect_chooses_same_modes_p(vec_info *, machine_mode)
Definition tree-vect-stmts.cc:13817
bool slpeel_can_duplicate_loop_p(const class loop *, const_edge, const_edge)
Definition tree-vect-loop-manip.cc:2028
opt_result vect_analyze_stmt(vec_info *, stmt_vec_info, bool *, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:13122
tree vect_get_loop_mask(loop_vec_info, gimple_stmt_iterator *, vec_loop_masks *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11583
slp_vect_type
Definition tree-vectorizer.h:1182
@ loop_vect
Definition tree-vectorizer.h:1183
@ hybrid
Definition tree-vectorizer.h:1185
@ pure_slp
Definition tree-vectorizer.h:1184
vect_reduction_type
Definition tree-vectorizer.h:83
@ TREE_CODE_REDUCTION
Definition tree-vectorizer.h:84
@ FOLD_LEFT_REDUCTION
Definition tree-vectorizer.h:102
@ EXTRACT_LAST_REDUCTION
Definition tree-vectorizer.h:94
@ INTEGER_INDUC_COND_REDUCTION
Definition tree-vectorizer.h:86
@ CONST_COND_REDUCTION
Definition tree-vectorizer.h:87
@ COND_REDUCTION
Definition tree-vectorizer.h:85
bool supportable_widening_operation(vec_info *, code_helper, stmt_vec_info, tree, tree, code_helper *, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:14115
auto_vec< rgroup_controls > vec_loop_lens
Definition tree-vectorizer.h:665
void set_dr_misalignment(dr_vec_info *dr_info, int val)
Definition tree-vectorizer.h:1949
bool vect_use_mask_type_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1780
pair_hash< nofree_ptr_hash< _slp_tree >, nofree_ptr_hash< _slp_tree > > slp_node_hash
Definition tree-vectorizer.h:2604
hash_map< tree_operand_hash, std::pair< stmt_vec_info, innermost_loop_behavior * > > vec_base_alignments
Definition tree-vectorizer.h:127
unsigned add_stmt_cost(vector_costs *costs, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where)
Definition tree-vectorizer.h:1885
tree vect_get_smallest_scalar_type(stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:147
vec< stmt_info_for_cost > stmt_vector_for_cost
Definition tree-vectorizer.h:121
stmt_vec_info info_for_reduction(vec_info *, stmt_vec_info)
Definition tree-vect-loop.cc:5688
bool known_alignment_for_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:1990
void vect_remove_stores(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:13540
#define DR_GROUP_FIRST_ELEMENT(S)
Definition tree-vectorizer.h:1561
void vect_get_load_cost(vec_info *, stmt_vec_info, int, dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool)
Definition tree-vect-stmts.cc:1037
hash_set< tree_cond_mask_hash > vec_cond_masked_set_type
Definition tree-vectorizer.h:403
#define STMT_VINFO_REDUC_DEF(S)
Definition tree-vectorizer.h:1555
opt_result vect_find_stmt_data_reference(loop_p, gimple *, vec< data_reference_p > *, vec< int > *, int)
Definition tree-vect-data-refs.cc:4588
opt_loop_vec_info vect_analyze_loop(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:3569
tree get_dr_vinfo_offset(vec_info *vinfo, dr_vec_info *dr_info, bool check_outer=false)
Definition tree-vectorizer.h:2030
bool vect_slp_if_converted_bb(basic_block bb, loop_p orig_loop)
Definition tree-vect-slp.cc:8464
size_t num__slp_patterns
Definition tree-vect-slp-patterns.cc:1647
bool supportable_indirect_convert_operation(code_helper, tree, tree, vec< std::pair< tree, tree_code > > *, tree=NULL_TREE)
Definition tree-vect-stmts.cc:14619
tree vect_gen_perm_mask_checked(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9690
bool vectorizable_induction(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:10310
vect_pattern *(* vect_pattern_decl_t)(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
Definition tree-vectorizer.h:2654
tree vect_create_destination_var(tree, tree)
Definition tree-vect-data-refs.cc:5611
tree vect_create_data_ref_ptr(vec_info *, stmt_vec_info, tree, class loop *, tree, tree *, gimple_stmt_iterator *, gimple **, bool, tree=NULL_TREE)
Definition tree-vect-data-refs.cc:5258
void vect_gen_vector_loop_niters(loop_vec_info, tree, tree *, tree *, bool)
Definition tree-vect-loop-manip.cc:2779
bool vect_make_slp_decision(loop_vec_info)
Definition tree-vect-slp.cc:6255
bool aligned_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:1980
unsigned HOST_WIDE_INT tree_to_uhwi(const_tree t)
Definition tree.cc:6483
#define TYPE_ALIGN_UNIT(NODE)
Definition tree.h:2347
poly_uint64 TYPE_VECTOR_SUBPARTS(const_tree node)
Definition tree.h:4251
#define TYPE_PRECISION(NODE)
Definition tree.h:2253
#define TREE_OPERAND(NODE, I)
Definition tree.h:1285
#define sizetype
Definition tree.h:5110
#define TREE_CODE_CLASS(CODE)
Definition tree.h:202
#define TYPE_SIZE_UNIT(NODE)
Definition tree.h:2250
#define CONVERT_EXPR_CODE_P(CODE)
Definition tree.h:553
#define TREE_TYPE(NODE)
Definition tree.h:512
#define INTEGRAL_TYPE_P(TYPE)
Definition tree.h:613
#define COMPARISON_CLASS_P(NODE)
Definition tree.h:256
#define NULL_TREE
Definition tree.h:317
#define FOR_EACH_VEC_ELT(V, I, P)
Definition vec.h:1884
constexpr vnull vNULL
Definition vec.h:569