21#ifndef GCC_TREE_VECTORIZER_H
22#define GCC_TREE_VECTORIZER_H
105#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
106 || ((D) == vect_double_reduction_def) \
107 || ((D) == vect_nested_cycle))
126 std::pair<stmt_vec_info, innermost_loop_behavior *> >
133 tree op (
unsigned int i)
const;
234 static void *
operator new (
size_t);
237 static void operator delete (
void *,
size_t);
294#define SLP_INSTANCE_TREE(S) (S)->root
295#define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
296#define SLP_INSTANCE_LOADS(S) (S)->loads
297#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
298#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
299#define SLP_INSTANCE_KIND(S) (S)->kind
301#define SLP_TREE_CHILDREN(S) (S)->children
302#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
303#define SLP_TREE_SCALAR_OPS(S) (S)->ops
304#define SLP_TREE_REF_COUNT(S) (S)->refcnt
305#define SLP_TREE_VEC_DEFS(S) (S)->vec_defs
306#define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
307#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
308#define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
309#define SLP_TREE_SIMD_CLONE_INFO(S) (S)->simd_clone_info
310#define SLP_TREE_DEF_TYPE(S) (S)->def_type
311#define SLP_TREE_VECTYPE(S) (S)->vectype
312#define SLP_TREE_REPRESENTATIVE(S) (S)->representative
313#define SLP_TREE_LANES(S) (S)->lanes
314#define SLP_TREE_CODE(S) (S)->code
372 static const bool empty_zero_p =
true;
967#define LOOP_VINFO_LOOP(L) (L)->loop
968#define LOOP_VINFO_IV_EXIT(L) (L)->vec_loop_iv_exit
969#define LOOP_VINFO_EPILOGUE_IV_EXIT(L) (L)->vec_epilogue_loop_iv_exit
970#define LOOP_VINFO_SCALAR_IV_EXIT(L) (L)->scalar_loop_iv_exit
971#define LOOP_VINFO_BBS(L) (L)->bbs
972#define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1
973#define LOOP_VINFO_NITERS(L) (L)->num_iters
977#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
978#define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions
979#define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th
980#define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold
981#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
982#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
983#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
984#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
985#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
986#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \
987 (L)->epil_using_partial_vectors_p
988#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
989#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
990#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
991#define LOOP_VINFO_MASKS(L) (L)->masks
992#define LOOP_VINFO_LENS(L) (L)->lens
993#define LOOP_VINFO_MASK_SKIP_NITERS(L) (L)->mask_skip_niters
994#define LOOP_VINFO_RGROUP_COMPARE_TYPE(L) (L)->rgroup_compare_type
995#define LOOP_VINFO_RGROUP_IV_TYPE(L) (L)->rgroup_iv_type
996#define LOOP_VINFO_PARTIAL_VECTORS_STYLE(L) (L)->partial_vector_style
997#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
998#define LOOP_VINFO_N_STMTS(L) (L)->shared->n_stmts
999#define LOOP_VINFO_LOOP_NEST(L) (L)->shared->loop_nest
1000#define LOOP_VINFO_DATAREFS(L) (L)->shared->datarefs
1001#define LOOP_VINFO_DDRS(L) (L)->shared->ddrs
1002#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
1003#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
1004#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
1005#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
1006#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
1007#define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs
1008#define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs
1009#define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero
1010#define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds
1011#define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
1012#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
1013#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
1014#define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
1015#define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
1016#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
1017#define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter
1018#define LOOP_VINFO_EARLY_BREAKS(L) (L)->early_breaks
1019#define LOOP_VINFO_EARLY_BRK_STORES(L) (L)->early_break_stores
1020#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L) \
1021 (single_pred ((L)->loop->latch) != (L)->vec_loop_iv_exit->src)
1022#define LOOP_VINFO_EARLY_BRK_DEST_BB(L) (L)->early_break_dest_bb
1023#define LOOP_VINFO_EARLY_BRK_VUSES(L) (L)->early_break_vuses
1024#define LOOP_VINFO_LOOP_CONDS(L) (L)->conds
1025#define LOOP_VINFO_LOOP_IV_COND(L) (L)->loop_iv_cond
1026#define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
1027#define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop
1028#define LOOP_VINFO_SCALAR_LOOP_SCALING(L) (L)->scalar_loop_scaling
1029#define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store
1030#define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
1031#define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info
1032#define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
1033#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
1035#define LOOP_VINFO_FULLY_MASKED_P(L) \
1036 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1037 && !LOOP_VINFO_MASKS (L).is_empty ())
1039#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \
1040 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1041 && !LOOP_VINFO_LENS (L).is_empty ())
1043#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
1044 ((L)->may_misalign_stmts.length () > 0)
1045#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
1046 ((L)->comp_alias_ddrs.length () > 0 \
1047 || (L)->check_unequal_addrs.length () > 0 \
1048 || (L)->lower_bounds.length () > 0)
1049#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \
1050 (LOOP_VINFO_NITERS_ASSUMPTIONS (L))
1051#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \
1052 (LOOP_VINFO_SIMD_IF_COND (L))
1053#define LOOP_REQUIRES_VERSIONING(L) \
1054 (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \
1055 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \
1056 || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
1057 || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
1059#define LOOP_VINFO_NITERS_KNOWN_P(L) \
1060 (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
1062#define LOOP_VINFO_EPILOGUE_P(L) \
1063 (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL)
1065#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \
1066 (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L)))
1106#define BB_VINFO_BB(B) (B)->bb
1107#define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
1108#define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
1109#define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
1110#define BB_VINFO_DDRS(B) (B)->shared->ddrs
1499#define STMT_VINFO_TYPE(S) (S)->type
1500#define STMT_VINFO_STMT(S) (S)->stmt
1501#define STMT_VINFO_RELEVANT(S) (S)->relevant
1502#define STMT_VINFO_LIVE_P(S) (S)->live
1503#define STMT_VINFO_VECTYPE(S) (S)->vectype
1504#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
1505#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
1506#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
1507#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
1508#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
1509#define STMT_VINFO_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
1510#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
1511#define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
1512#define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
1513#define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
1514#define STMT_VINFO_FORCE_SINGLE_CYCLE(S) (S)->force_single_cycle
1516#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop
1517#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address
1518#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init
1519#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset
1520#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step
1521#define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment
1522#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \
1523 (S)->dr_wrt_vec_loop.base_misalignment
1524#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \
1525 (S)->dr_wrt_vec_loop.offset_alignment
1526#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
1527 (S)->dr_wrt_vec_loop.step_alignment
1529#define STMT_VINFO_DR_INFO(S) \
1530 (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
1532#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
1533#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
1534#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
1535#define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info
1536#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
1537#define STMT_VINFO_GROUPED_ACCESS(S) \
1538 ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
1539#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
1540#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
1541#define STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE(S) (S)->loop_phi_evolution_type
1542#define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
1543#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
1544#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
1545#define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn
1546#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
1547#define STMT_VINFO_REDUC_VECTYPE(S) (S)->reduc_vectype
1548#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
1549#define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
1550#define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p
1552#define DR_GROUP_FIRST_ELEMENT(S) \
1553 (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
1554#define DR_GROUP_NEXT_ELEMENT(S) \
1555 (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
1556#define DR_GROUP_SIZE(S) \
1557 (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
1558#define DR_GROUP_STORE_COUNT(S) \
1559 (gcc_checking_assert ((S)->dr_aux.dr), (S)->store_count)
1560#define DR_GROUP_GAP(S) \
1561 (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
1563#define REDUC_GROUP_FIRST_ELEMENT(S) \
1564 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
1565#define REDUC_GROUP_NEXT_ELEMENT(S) \
1566 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
1567#define REDUC_GROUP_SIZE(S) \
1568 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
1570#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
1572#define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
1573#define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
1574#define STMT_SLP_TYPE(S) (S)->slp_type
1604 tree vectype,
int misalign,
1669 m_suggested_unroll_factor(1),
1727#define VECT_MAX_COST 1000
1731#define MAX_INTERM_CVT_STEPS 3
1733#define MAX_VECTORIZATION_FACTOR INT_MAX
1740#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \
1741 (TREE_CODE (TYPE) == BOOLEAN_TYPE \
1742 || ((TREE_CODE (TYPE) == INTEGER_TYPE \
1743 || TREE_CODE (TYPE) == ENUMERAL_TYPE) \
1744 && TYPE_PRECISION (TYPE) == 1 \
1745 && TYPE_UNSIGNED (TYPE)))
1773 return stmt_info->mask_precision && stmt_info->mask_precision != ~0
U;
1782 return stmt_info->pattern_stmt_p;
1837 for (
i = 0;
i < x;
i++)
1847 tree vectype,
int misalign)
1879 tree vectype,
int misalign,
1882 unsigned cost =
costs->add_stmt_cost (
count, kind, stmt_info, node, vectype,
1905 i->vectype,
i->misalign,
i->where);
1912 unsigned *prologue_cost,
unsigned *body_cost,
1913 unsigned *epilogue_cost,
unsigned *suggested_unroll_factor =
NULL)
1915 costs->finish_cost (scalar_costs);
1916 *prologue_cost =
costs->prologue_cost ();
1917 *body_cost =
costs->body_cost ();
1918 *epilogue_cost =
costs->epilogue_cost ();
1919 if (suggested_unroll_factor)
1920 *suggested_unroll_factor =
costs->suggested_unroll_factor ();
1936#define DR_MISALIGNMENT_UNKNOWN (-1)
1937#define DR_MISALIGNMENT_UNINITIALIZED (-2)
1948#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
1956 return dr_info->target_alignment;
1958#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR)
1963 dr_info->target_alignment = val;
1965#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL)
1995 else if (misalignment == 0)
1997 return misalignment & -misalignment;
2009 if (loop_vinfo ==
NULL
2184#define DUMP_VECT_SCOPE(MSG) \
2185 AUTO_DUMP_SCOPE (MSG, vect_location)
2311 unsigned int *,
unsigned int *,
2322 const char * =
nullptr);
2326 tree *,
unsigned int = 0);
2379 const char * =
NULL);
2405 unsigned int,
tree,
unsigned int);
2407 tree,
unsigned int);
2410 unsigned int,
unsigned int);
2413 unsigned int,
tree,
tree,
unsigned int,
2475 unsigned * =
nullptr,
bool =
false);
2492 unsigned int * =
NULL,
2569 this->m_node = node;
2570 this->m_ops.create (0);
2572 this->m_ops.safe_splice (*
m_ops);
2587 this->m_ops.release ();
edge loop_preheader_edge(const class loop *loop)
Definition cfgloop.cc:1760
class loop * loop_p
Definition cfgloop.h:98
Definition tree-vectorizer.h:1092
vec< slp_root > roots
Definition tree-vectorizer.h:1103
_bb_vec_info(vec< basic_block > bbs, vec_info_shared *)
Definition tree-vect-slp.cc:6091
vec< basic_block > bbs
Definition tree-vectorizer.h:1101
~_bb_vec_info()
Definition tree-vect-slp.cc:6122
Definition tree-vectorizer.h:674
bool peeling_for_gaps
Definition tree-vectorizer.h:886
bool can_use_partial_vectors_p
Definition tree-vectorizer.h:856
signed char partial_load_store_bias
Definition tree-vectorizer.h:881
class vector_costs * vector_costs
Definition tree-vectorizer.h:695
class dr_vec_info * unaligned_dr
Definition tree-vectorizer.h:786
_loop_vec_info(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:1028
edge skip_main_loop_edge
Definition tree-vectorizer.h:721
auto_vec< stmt_vec_info > reduction_chains
Definition tree-vectorizer.h:829
unsigned int th
Definition tree-vectorizer.h:703
auto_vec< gimple * > early_break_vuses
Definition tree-vectorizer.h:963
bool using_partial_vectors_p
Definition tree-vectorizer.h:860
bool no_data_dependencies
Definition tree-vectorizer.h:919
class loop * scalar_loop
Definition tree-vectorizer.h:929
vec< _loop_vec_info * > epilogue_vinfos
Definition tree-vectorizer.h:937
scalar_cond_masked_set_type scalar_cond_masked_set
Definition tree-vectorizer.h:754
auto_vec< ddr_p > may_alias_ddrs
Definition tree-vectorizer.h:803
tree num_itersm1
Definition tree-vectorizer.h:686
unsigned int inner_loop_cost_factor
Definition tree-vectorizer.h:847
bool vectorizable
Definition tree-vectorizer.h:850
vect_partial_vector_style partial_vector_style
Definition tree-vectorizer.h:783
poly_uint64 slp_unrolling_factor
Definition tree-vectorizer.h:843
edge vec_epilogue_loop_iv_exit
Definition tree-vectorizer.h:945
basic_block * bbs
Definition tree-vectorizer.h:683
tree num_iters
Definition tree-vectorizer.h:688
hash_map< tree, tree > * scan_map
Definition tree-vectorizer.h:839
int peeling_for_alignment
Definition tree-vectorizer.h:796
vec_cond_masked_set_type vec_cond_masked_set
Definition tree-vectorizer.h:757
bool epil_using_partial_vectors_p
Definition tree-vectorizer.h:876
bool early_breaks
Definition tree-vectorizer.h:894
poly_uint64 versioning_threshold
Definition tree-vectorizer.h:709
auto_vec< vec_lower_bound > lower_bounds
Definition tree-vectorizer.h:818
profile_probability scalar_loop_scaling
Definition tree-vectorizer.h:925
auto_vec< gimple * > early_break_stores
Definition tree-vectorizer.h:955
edge main_loop_edge
Definition tree-vectorizer.h:720
edge skip_this_loop_edge
Definition tree-vectorizer.h:725
auto_vec< stmt_info_for_cost > scalar_cost_vec
Definition tree-vectorizer.h:832
gcond * loop_iv_cond
Definition tree-vectorizer.h:900
basic_block early_break_dest_bb
Definition tree-vectorizer.h:959
bool using_select_vl_p
Definition tree-vectorizer.h:872
tree rgroup_compare_type
Definition tree-vectorizer.h:768
poly_uint64 vectorization_factor
Definition tree-vectorizer.h:712
bool peeling_for_niter
Definition tree-vectorizer.h:890
auto_vec< dr_with_seg_len_pair_t > comp_alias_ddrs
Definition tree-vectorizer.h:807
edge vec_loop_iv_exit
Definition tree-vectorizer.h:941
tree num_iters_unchanged
Definition tree-vectorizer.h:690
vec_loop_lens lens
Definition tree-vectorizer.h:751
unsigned HOST_WIDE_INT max_vectorization_factor
Definition tree-vectorizer.h:743
unsigned int suggested_unroll_factor
Definition tree-vectorizer.h:739
hash_map< tree_operand_hash, tree > * ivexpr_map
Definition tree-vectorizer.h:835
auto_vec< stmt_vec_info > may_misalign_stmts
Definition tree-vectorizer.h:822
tree rgroup_iv_type
Definition tree-vectorizer.h:779
vec_loop_masks masks
Definition tree-vectorizer.h:747
tree num_iters_assumptions
Definition tree-vectorizer.h:692
tree simd_if_cond
Definition tree-vectorizer.h:775
auto_vec< tree > check_nonzero
Definition tree-vectorizer.h:815
tree mask_skip_niters
Definition tree-vectorizer.h:763
bool has_mask_store
Definition tree-vectorizer.h:922
auto_vec< stmt_vec_info > reductions
Definition tree-vectorizer.h:825
edge scalar_loop_iv_exit
Definition tree-vectorizer.h:949
class loop * loop
Definition tree-vectorizer.h:680
~_loop_vec_info()
Definition tree-vect-loop.cc:1146
bool using_decrementing_iv_p
Definition tree-vectorizer.h:867
auto_vec< gcond * > conds
Definition tree-vectorizer.h:897
_loop_vec_info * orig_loop_info
Definition tree-vectorizer.h:933
hash_map< tree, vect_reusable_accumulator > reusable_accumulators
Definition tree-vectorizer.h:732
int ptr_mask
Definition tree-vectorizer.h:799
class vector_costs * scalar_costs
Definition tree-vectorizer.h:698
auto_vec< vec_object_pair > check_unequal_addrs
Definition tree-vectorizer.h:810
Definition tree-vectorizer.h:257
poly_uint64 unrolling_factor
Definition tree-vectorizer.h:271
slp_instance_kind kind
Definition tree-vectorizer.h:287
vec< stmt_vec_info > root_stmts
Definition tree-vectorizer.h:264
slp_tree root
Definition tree-vectorizer.h:260
dump_user_location_t location() const
Definition tree-vect-slp.cc:197
vec< slp_tree > loads
Definition tree-vectorizer.h:274
vec< _slp_instance * > subgraph_entries
Definition tree-vectorizer.h:284
vec< tree > remain_defs
Definition tree-vectorizer.h:268
slp_tree reduc_phis
Definition tree-vectorizer.h:277
stmt_vector_for_cost cost_vec
Definition tree-vectorizer.h:280
Definition tree-vectorizer.h:1258
unsigned int min_input_precision
Definition tree-vectorizer.h:1432
code_helper reduc_code
Definition tree-vectorizer.h:1383
unsigned int store_count
Definition tree-vectorizer.h:1345
tree vectype
Definition tree-vectorizer.h:1284
vec< tree > simd_clone_info
Definition tree-vectorizer.h:1328
tree induc_cond_initial_val
Definition tree-vectorizer.h:1373
bool in_pattern_p
Definition tree-vectorizer.h:1268
stmt_vec_info first_element
Definition tree-vectorizer.h:1338
unsigned int mask_precision
Definition tree-vectorizer.h:1455
vec< tree > reduc_initial_values
Definition tree-vectorizer.h:1405
int reduc_idx
Definition tree-vectorizer.h:1389
gimple * stmt
Definition tree-vectorizer.h:1281
tree loop_phi_evolution_base_unchanged
Definition tree-vectorizer.h:1305
enum vect_induction_op_type loop_phi_evolution_type
Definition tree-vectorizer.h:1307
innermost_loop_behavior dr_wrt_vec_loop
Definition tree-vectorizer.h:1297
vec< gimple * > vec_stmts
Definition tree-vectorizer.h:1287
bool slp_vect_only_p
Definition tree-vectorizer.h:1458
bool live
Definition tree-vectorizer.h:1265
unsigned int gap
Definition tree-vectorizer.h:1348
unsigned int min_output_precision
Definition tree-vectorizer.h:1427
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:1370
internal_fn reduc_fn
Definition tree-vectorizer.h:1385
enum stmt_vec_info_type type
Definition tree-vectorizer.h:1261
unsigned int min_neg_dist
Definition tree-vectorizer.h:1352
vec< tree > reduc_scalar_results
Definition tree-vectorizer.h:1412
tree reduc_vectype_in
Definition tree-vectorizer.h:1397
bool vectorizable
Definition tree-vectorizer.h:1278
bool pattern_stmt_p
Definition tree-vectorizer.h:1274
bool strided_p
Definition tree-vectorizer.h:1363
enum vect_relevant relevant
Definition tree-vectorizer.h:1357
unsigned int size
Definition tree-vectorizer.h:1342
signop operation_sign
Definition tree-vectorizer.h:1438
bool force_single_cycle
Definition tree-vectorizer.h:1420
bool is_reduc_info
Definition tree-vectorizer.h:1423
gimple_seq pattern_def_seq
Definition tree-vectorizer.h:1323
stmt_vec_info related_stmt
Definition tree-vectorizer.h:1318
unsigned simd_lane_access_p
Definition tree-vectorizer.h:1366
bool slp_vect_pattern_only_p
Definition tree-vectorizer.h:1462
tree reduc_vectype
Definition tree-vectorizer.h:1400
stmt_vec_info reduc_def
Definition tree-vectorizer.h:1394
bool gather_scatter_p
Definition tree-vectorizer.h:1360
enum vect_def_type def_type
Definition tree-vectorizer.h:1331
stmt_vec_info next_element
Definition tree-vectorizer.h:1340
unsigned int operation_precision
Definition tree-vectorizer.h:1437
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:1380
tree reduc_epilogue_adjustment
Definition tree-vectorizer.h:1376
enum slp_vect_type slp_type
Definition tree-vectorizer.h:1334
dr_vec_info dr_aux
Definition tree-vectorizer.h:1293
tree loop_phi_evolution_part
Definition tree-vectorizer.h:1306
vect_reusable_accumulator * reused_accumulator
Definition tree-vectorizer.h:1417
Definition tree-vectorizer.h:2200
~auto_purge_vect_location()
Definition tree-vectorizer.cc:94
Definition tree-vectorizer.h:1233
unsigned group
Definition tree-vectorizer.h:1241
tree base_decl
Definition tree-vectorizer.h:1249
bool base_misaligned
Definition tree-vectorizer.h:1248
tree offset
Definition tree-vectorizer.h:1253
int misalignment
Definition tree-vectorizer.h:1243
poly_uint64 target_alignment
Definition tree-vectorizer.h:1246
data_reference * dr
Definition tree-vectorizer.h:1236
stmt_vec_info stmt
Definition tree-vectorizer.h:1238
Definition dumpfile.h:340
Definition genmatch.cc:845
bool is_empty() const
Definition hash-set.h:107
class loop * inner
Definition cfgloop.h:148
unsigned force_vectorize
Definition cfgloop.h:227
void * aux
Definition cfgloop.h:154
Definition opt-problem.h:230
Definition opt-problem.h:179
Definition profile-count.h:147
Definition tree-vectorizer.h:424
vec< data_reference_p > datarefs
Definition tree-vectorizer.h:436
vec< data_reference > datarefs_copy
Definition tree-vectorizer.h:437
auto_vec< loop_p > loop_nest
Definition tree-vectorizer.h:440
void check_datarefs()
Definition tree-vectorizer.cc:505
unsigned n_stmts
Definition tree-vectorizer.h:433
void save_datarefs()
Definition tree-vectorizer.cc:495
~vec_info_shared()
Definition tree-vectorizer.cc:487
vec< ddr_p > ddrs
Definition tree-vectorizer.h:444
vec_info_shared()
Definition tree-vectorizer.cc:479
Definition tree-vectorizer.h:448
stmt_vec_info lookup_stmt(gimple *)
Definition tree-vectorizer.cc:545
auto_vec< stmt_vec_info > grouped_stores
Definition tree-vectorizer.h:492
~vec_info()
Definition tree-vectorizer.cc:471
auto_vec< slp_instance > slp_instances
Definition tree-vectorizer.h:484
void replace_stmt(gimple_stmt_iterator *, stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:630
void remove_stmt(stmt_vec_info)
Definition tree-vectorizer.cc:614
void insert_on_entry(stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:673
void set_vinfo_for_stmt(gimple *, stmt_vec_info, bool=true)
Definition tree-vectorizer.cc:720
vec_base_alignments base_alignments
Definition tree-vectorizer.h:488
machine_mode vector_mode
Definition tree-vectorizer.h:500
stmt_vec_info new_stmt_vec_info(gimple *stmt)
Definition tree-vectorizer.cc:684
class dr_vec_info * lookup_dr(data_reference *)
Definition tree-vectorizer.cc:587
vec< stmt_vec_info > stmt_vec_infos
Definition tree-vectorizer.h:475
void move_dr(stmt_vec_info, stmt_vec_info)
Definition tree-vectorizer.cc:599
bool any_known_not_updated_vssa
Definition tree-vectorizer.h:481
vec_kind
Definition tree-vectorizer.h:451
@ loop
Definition tree-vectorizer.h:451
@ bb
Definition tree-vectorizer.h:451
void free_stmt_vec_infos()
Definition tree-vectorizer.cc:741
vec_info_shared * shared
Definition tree-vectorizer.h:472
stmt_vec_info add_stmt(gimple *)
Definition tree-vectorizer.cc:520
vec_kind kind
Definition tree-vectorizer.h:469
mode_set used_vector_modes
Definition tree-vectorizer.h:495
hash_set< int_hash< machine_mode, E_VOIDmode, E_BLKmode > > mode_set
Definition tree-vectorizer.h:450
void insert_seq_on_entry(stmt_vec_info, gimple_seq)
Definition tree-vectorizer.cc:646
stmt_vec_info lookup_def(tree)
Definition tree-vectorizer.cc:562
void free_stmt_vec_info(stmt_vec_info)
Definition tree-vectorizer.cc:752
stmt_vec_info lookup_single_use(tree)
Definition tree-vectorizer.cc:575
vec_info(vec_kind, vec_info_shared *)
Definition tree-vectorizer.cc:463
stmt_vec_info add_pattern_stmt(gimple *, stmt_vec_info)
Definition tree-vectorizer.cc:532
bool stmt_vec_info_ro
Definition tree-vectorizer.h:477
Definition tree-vectorizer.h:411
vec_lower_bound()
Definition tree-vectorizer.h:413
poly_uint64 min_value
Definition tree-vectorizer.h:419
bool unsigned_p
Definition tree-vectorizer.h:418
vec_lower_bound(tree e, bool u, poly_uint64 m)
Definition tree-vectorizer.h:414
tree expr
Definition tree-vectorizer.h:417
Definition vec-perm-indices.h:51
Definition tree-vectorizer.h:2550
static vect_pattern * recognize(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
internal_fn m_ifn
Definition tree-vectorizer.h:2556
vec< slp_tree > m_ops
Definition tree-vectorizer.h:2563
virtual ~vect_pattern()
Definition tree-vectorizer.h:2585
virtual void build(vec_info *)=0
vect_pattern(slp_tree *node, vec< slp_tree > *m_ops, internal_fn ifn)
Definition tree-vectorizer.h:2566
slp_tree * m_node
Definition tree-vectorizer.h:2559
unsigned m_num_args
Definition tree-vectorizer.h:2553
Definition tree-vectorizer.h:1578
unsigned int adjust_cost_for_freq(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1839
virtual ~vector_costs()
Definition tree-vectorizer.h:1581
bool m_costing_for_scalar
Definition tree-vectorizer.h:1649
vector_costs(vec_info *, bool)
Definition tree-vectorizer.h:1665
vec_info * m_vinfo
Definition tree-vectorizer.h:1645
virtual unsigned int add_stmt_cost(int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, vect_cost_model_location where)
Definition tree-vectorizer.cc:1801
virtual bool better_main_loop_than_p(const vector_costs *other) const
Definition tree-vectorizer.cc:1859
unsigned int m_costs[3]
Definition tree-vectorizer.h:1652
unsigned int outside_cost() const
Definition tree-vectorizer.h:1704
unsigned int record_stmt_cost(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1825
virtual bool better_epilogue_loop_than_p(const vector_costs *other, loop_vec_info main_loop) const
Definition tree-vectorizer.cc:1878
unsigned int epilogue_cost() const
Definition tree-vectorizer.h:1695
unsigned int m_suggested_unroll_factor
Definition tree-vectorizer.h:1655
unsigned int suggested_unroll_factor() const
Definition tree-vectorizer.h:1721
bool m_finished
Definition tree-vectorizer.h:1658
int compare_outside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:2054
unsigned int body_cost() const
Definition tree-vectorizer.h:1686
virtual void finish_cost(const vector_costs *scalar_costs)
Definition tree-vectorizer.cc:1814
unsigned int prologue_cost() const
Definition tree-vectorizer.h:1677
unsigned int total_cost() const
Definition tree-vectorizer.h:1713
int compare_inside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:1961
class edge_def * edge
Definition coretypes.h:342
const class edge_def * const_edge
Definition coretypes.h:343
const union tree_node * const_tree
Definition coretypes.h:98
union tree_node * tree
Definition coretypes.h:97
static unsigned int count[debug_counter_number_of_counters]
Definition dbgcnt.cc:50
dump_flags_t dump_flags
Definition dumpfile.cc:64
FILE * dump_file
Definition dumpfile.cc:62
@ TDF_DETAILS
Definition dumpfile.h:92
vect_cost_model
Definition flag-types.h:279
@ VECT_COST_MODEL_UNLIMITED
Definition flag-types.h:283
@ VECT_COST_MODEL_DEFAULT
Definition flag-types.h:284
bool operand_equal_p(const_tree arg0, const_tree arg1, unsigned int flags)
Definition fold-const.cc:4223
#define fold_convert(T1, T2)
Definition fold-const.h:92
#define fold_build2(c, t1, t2, t3)
Definition fold-const.h:72
internal_fn
Definition genmatch.cc:360
tree_code
Definition genmatch.cc:347
T * ggc_alloc(ALONE_CXX_MEM_STAT_INFO)
Definition ggc.h:184
unsigned gimple_uid(const gimple *g)
Definition gimple.h:2084
enum tree_code gimple_assign_rhs_code(const gassign *gs)
Definition gimple.h:2850
tree gimple_assign_rhs1(const gassign *gs)
Definition gimple.h:2660
basic_block gimple_bb(const gimple *g)
Definition gimple.h:1860
tree gimple_assign_lhs(const gassign *gs)
Definition gimple.h:2609
#define init_cost
Definition ira-costs.cc:63
void add_expr(const_tree t, inchash::hash &hstate, unsigned int flags)
Definition fold-const.cc:4237
i
Definition poly-int.h:772
rtx offset
Definition postreload.cc:691
signop
Definition signop.h:28
Definition tree-vectorizer.h:174
stmt_vec_info representative
Definition tree-vectorizer.h:190
unsigned int lanes
Definition tree-vectorizer.h:222
load_permutation_t load_permutation
Definition tree-vectorizer.h:194
slp_tree prev_node
Definition tree-vectorizer.h:241
enum tree_code code
Definition tree-vectorizer.h:224
unsigned int vec_stmts_size
Definition tree-vectorizer.h:212
vec< tree > vec_defs
Definition tree-vectorizer.h:207
~_slp_tree()
Definition tree-vect-slp.cc:133
unsigned int refcnt
Definition tree-vectorizer.h:215
slp_tree next_node
Definition tree-vectorizer.h:240
int vertex
Definition tree-vectorizer.h:226
enum vect_def_type def_type
Definition tree-vectorizer.h:220
bool * failed
Definition tree-vectorizer.h:231
vec< tree > simd_clone_info
Definition tree-vectorizer.h:203
vec< tree > ops
Definition tree-vectorizer.h:187
void push_vec_def(gimple *def)
Definition tree-vect-slp.cc:155
poly_uint64 max_nunits
Definition tree-vectorizer.h:218
lane_permutation_t lane_permutation
Definition tree-vectorizer.h:198
_slp_tree()
Definition tree-vect-slp.cc:106
tree vectype
Definition tree-vectorizer.h:205
void push_vec_def(tree def)
Definition tree-vectorizer.h:179
vec< slp_tree > children
Definition tree-vectorizer.h:182
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:185
Definition basic-block.h:117
class loop * loop_father
Definition basic-block.h:126
Definition ira-costs.cc:54
Definition tree-data-ref.h:151
struct innermost_loop_behavior innermost
Definition tree-data-ref.h:173
Definition loop-invariant.cc:88
static void mark_empty(value_type &v)
Definition tree-vectorizer.h:375
scalar_cond_masked_key value_type
Definition tree-vectorizer.h:348
static bool equal(value_type existing, value_type candidate)
Definition tree-vectorizer.h:363
static void remove(value_type &)
Definition tree-vectorizer.h:394
static hashval_t hash(value_type v)
Definition tree-vectorizer.h:351
static void mark_deleted(value_type &)
Definition tree-vectorizer.h:387
static bool is_deleted(const value_type &)
Definition tree-vectorizer.h:389
static bool is_empty(value_type v)
Definition tree-vectorizer.h:382
scalar_cond_masked_key compare_type
Definition tree-vectorizer.h:347
Definition hash-traits.h:466
Definition graphite.h:338
data_reference_p dr
Definition graphite.h:343
Definition function.h:249
Definition tree-vectorizer.h:1466
tree decl
Definition tree-vectorizer.h:1473
tree element_type
Definition tree-vectorizer.h:1492
tree base
Definition tree-vectorizer.h:1476
tree offset_vectype
Definition tree-vectorizer.h:1489
int scale
Definition tree-vectorizer.h:1483
enum vect_def_type offset_dt
Definition tree-vectorizer.h:1486
tree offset
Definition tree-vectorizer.h:1479
internal_fn ifn
Definition tree-vectorizer.h:1469
tree memory_type
Definition tree-vectorizer.h:1495
Definition gimple-iterator.h:26
Definition tree-data-ref.h:50
tree offset
Definition tree-data-ref.h:52
Definition hash-traits.h:122
Definition hash-traits.h:345
Definition tree-vectorizer.h:613
tree compare_type
Definition tree-vectorizer.h:633
unsigned int max_nscalars_per_iter
Definition tree-vectorizer.h:617
unsigned int factor
Definition tree-vectorizer.h:623
tree type
Definition tree-vectorizer.h:629
tree bias_adjusted_ctrl
Definition tree-vectorizer.h:640
vec< tree > controls
Definition tree-vectorizer.h:636
Definition tree-vectorizer.h:328
unsigned ncopies
Definition tree-vectorizer.h:337
tree_code code
Definition tree-vectorizer.h:339
scalar_cond_masked_key(tree t, unsigned ncopies_)
Definition tree-vectorizer.h:329
void get_cond_ops_from_tree(tree)
Definition tree-vectorizer.cc:1751
tree op0
Definition tree-vectorizer.h:340
tree op1
Definition tree-vectorizer.h:341
bool inverted_p
Definition tree-vectorizer.h:338
Definition tree-vectorizer.h:1081
vec< tree > remain
Definition tree-vectorizer.h:1088
slp_instance_kind kind
Definition tree-vectorizer.h:1085
vec< stmt_vec_info > roots
Definition tree-vectorizer.h:1087
slp_root(slp_instance_kind kind_, vec< stmt_vec_info > stmts_, vec< stmt_vec_info > roots_, vec< tree > remain_=vNULL)
Definition tree-vectorizer.h:1082
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:1086
Definition tree-vectorizer.h:111
tree vectype
Definition tree-vectorizer.h:117
int misalign
Definition tree-vectorizer.h:118
slp_tree node
Definition tree-vectorizer.h:116
enum vect_cost_model_location where
Definition tree-vectorizer.h:114
enum vect_cost_for_stmt kind
Definition tree-vectorizer.h:113
stmt_vec_info stmt_info
Definition tree-vectorizer.h:115
int count
Definition tree-vectorizer.h:112
Definition tree-hash-traits.h:25
Definition hash-traits.h:75
Definition tree-vectorizer.h:644
hash_set< pair_hash< nofree_ptr_hash< tree_node >, int_hash< unsigned, 0 > > > mask_set
Definition tree-vectorizer.h:649
auto_vec< rgroup_controls > rgc_vec
Definition tree-vectorizer.h:652
bool is_empty() const
Definition tree-vectorizer.h:645
Definition tree-vectorizer.h:661
stmt_vec_info reduc_info
Definition tree-vectorizer.h:668
tree reduc_input
Definition tree-vectorizer.h:664
Definition tree-vectorizer.h:150
static void mark_deleted(value_type &s)
Definition tree-vectorizer.h:156
static bool is_empty(const value_type &s)
Definition tree-vectorizer.h:159
static bool is_deleted(const value_type &s)
Definition tree-vectorizer.h:158
static const bool empty_zero_p
Definition tree-vectorizer.h:154
static void mark_empty(value_type &s)
Definition tree-vectorizer.h:157
static hashval_t hash(const value_type &)
Definition tree-vect-slp.cc:6268
vect_scalar_ops_slice value_type
Definition tree-vectorizer.h:151
static bool equal(const value_type &, const compare_type &)
Definition tree-vect-slp.cc:6277
vect_scalar_ops_slice compare_type
Definition tree-vectorizer.h:152
Definition tree-vectorizer.h:132
unsigned int length
Definition tree-vectorizer.h:138
unsigned int start
Definition tree-vectorizer.h:137
bool all_same_p() const
Definition tree-vect-slp.cc:6259
tree op(unsigned int i) const
Definition tree-vectorizer.h:143
vec< tree > * ops
Definition tree-vectorizer.h:136
#define NULL
Definition system.h:50
#define gcc_assert(EXPR)
Definition system.h:821
#define false
Definition system.h:895
#define gcc_checking_assert(EXPR)
Definition system.h:828
HOST_WIDE_INT estimated_poly_value(poly_int64 x, poly_value_estimate_kind kind=POLY_VALUE_LIKELY)
Definition target.h:306
struct gcc_target targetm
vect_cost_model_location
Definition target.h:209
@ vect_body
Definition target.h:211
@ vect_prologue
Definition target.h:210
@ vect_epilogue
Definition target.h:212
vect_cost_for_stmt
Definition target.h:187
@ scalar_stmt
Definition target.h:188
@ cond_branch_taken
Definition target.h:201
@ vec_to_scalar
Definition target.h:198
@ cond_branch_not_taken
Definition target.h:200
@ tcc_comparison
Definition tree-core.h:233
#define DR_IS_WRITE(DR)
Definition tree-data-ref.h:192
#define DR_IS_READ(DR)
Definition tree-data-ref.h:191
#define DR_REF(DR)
Definition tree-data-ref.h:185
#define DR_INNERMOST(DR)
Definition tree-data-ref.h:203
static tree candidate(unsigned uid)
Definition tree-sra.cc:325
#define PHI_ARG_DEF_FROM_EDGE(PHI, E)
Definition tree-ssa-operands.h:88
_complex_operation
Definition tree-vect-slp-patterns.cc:119
stmt_vec_info vect_orig_stmt(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1789
#define LOOP_VINFO_COST_MODEL_THRESHOLD(L)
Definition tree-vectorizer.h:979
bool vect_can_vectorize_without_simd_p(tree_code)
Definition tree-vect-loop.cc:9327
tree cse_and_gimplify_to_preheader(loop_vec_info, tree)
Definition tree-vect-loop.cc:1169
void finish_cost(vector_costs *costs, const vector_costs *scalar_costs, unsigned *prologue_cost, unsigned *body_cost, unsigned *epilogue_cost, unsigned *suggested_unroll_factor=NULL)
Definition tree-vectorizer.h:1911
int builtin_vectorization_cost(enum vect_cost_for_stmt type_of_cost, tree vectype, int misalign)
Definition tree-vectorizer.h:1846
loop_vec_info loop_vec_info_for_loop(class loop *loop)
Definition tree-vectorizer.h:1075
bool vect_is_extending_load(class vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2668
bool vect_transform_stmt(vec_info *, stmt_vec_info, gimple_stmt_iterator *, slp_tree, slp_instance)
Definition tree-vect-stmts.cc:13365
bool vect_is_simple_use(tree, vec_info *, enum vect_def_type *, stmt_vec_info *=NULL, gimple **=NULL)
Definition tree-vect-stmts.cc:13838
bool vect_rgroup_iv_might_wrap_p(loop_vec_info, rgroup_controls *)
Definition tree-vect-loop.cc:12586
tree vect_halve_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11161
void dump_stmt_cost(FILE *, int, enum vect_cost_for_stmt, stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location)
Definition tree-vectorizer.cc:102
vec< unsigned > load_permutation_t
Definition tree-vectorizer.h:169
bool vect_slp_analyze_instance_alignment(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:2898
internal_fn vect_store_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:5760
tree vect_get_new_vect_var(tree, enum vect_var_kind, const char *)
Definition tree-vect-data-refs.cc:5019
void vect_mark_pattern_stmts(vec_info *, stmt_vec_info, gimple *, tree)
Definition tree-vect-patterns.cc:7061
class loop * vect_loop_versioning(loop_vec_info, gimple *)
Definition tree-vect-loop-manip.cc:3975
bool vect_check_gather_scatter(stmt_vec_info, loop_vec_info, gather_scatter_info *)
Definition tree-vect-data-refs.cc:4274
void vect_get_store_cost(vec_info *, stmt_vec_info, int, dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:987
tree vect_get_slp_vect_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:8546
internal_fn vect_load_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:6387
opt_tree vect_get_mask_type_for_stmt(stmt_vec_info, unsigned int=0)
#define LOOP_VINFO_VECT_FACTOR(L)
Definition tree-vectorizer.h:989
void vect_free_slp_instance(slp_instance)
Definition tree-vect-slp.cc:209
bool vect_slp_function(function *)
Definition tree-vect-slp.cc:8098
std::pair< tree, tree > vec_object_pair
Definition tree-vectorizer.h:407
tree vect_gen_perm_mask_any(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9703
bool vect_transform_reduction(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, gimple **, slp_tree)
Definition tree-vect-loop.cc:8458
_bb_vec_info * bb_vec_info
#define LOOP_VINFO_FULLY_MASKED_P(L)
Definition tree-vectorizer.h:1035
int vect_get_known_peeling_cost(loop_vec_info, int, int *, stmt_vector_for_cost *, stmt_vector_for_cost *, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:4519
bool vect_transform_slp_perm_load(vec_info *, slp_tree, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, unsigned *, unsigned *=nullptr, bool=false)
tree vect_gen_while_not(gimple_seq *, tree, tree, tree)
Definition tree-vect-stmts.cc:14603
tree vect_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2658
bool is_simple_and_all_uses_invariant(stmt_vec_info, loop_vec_info)
Definition tree-vect-stmts.cc:308
bool vectorizable_phi(vec_info *, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9020
#define STMT_VINFO_GROUPED_ACCESS(S)
Definition tree-vectorizer.h:1537
void vect_schedule_slp(vec_info *, const vec< slp_instance > &)
Definition tree-vect-slp.cc:9832
tree vect_peel_nonlinear_iv_init(gimple_seq *, tree, tree, tree, enum vect_induction_op_type)
Definition tree-vect-loop.cc:9453
bool check_reduction_path(dump_user_location_t, loop_p, gphi *, tree, enum tree_code)
Definition tree-vect-loop.cc:4199
bool can_duplicate_and_interleave_p(vec_info *, unsigned int, tree, unsigned int *=NULL, tree *=NULL, tree *=NULL)
Definition tree-vect-slp.cc:421
auto_vec< unsigned, 16 > auto_load_permutation_t
Definition tree-vectorizer.h:170
tree get_vectype_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13678
#define LOOP_VINFO_LOOP(L)
Definition tree-vectorizer.h:967
_complex_perm_kinds
Definition tree-vectorizer.h:2526
@ PERM_EVENEVEN
Definition tree-vectorizer.h:2531
@ PERM_TOP
Definition tree-vectorizer.h:2533
@ PERM_ODDEVEN
Definition tree-vectorizer.h:2529
@ PERM_EVENODD
Definition tree-vectorizer.h:2528
@ PERM_UNKNOWN
Definition tree-vectorizer.h:2527
@ PERM_ODDODD
Definition tree-vectorizer.h:2530
void vect_record_base_alignments(vec_info *)
Definition tree-vect-data-refs.cc:1271
void vect_get_vec_defs(vec_info *, stmt_vec_info, slp_tree, unsigned, tree, vec< tree > *, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL)
Definition tree-vect-stmts.cc:1335
vec_load_store_type
Definition tree-vectorizer.h:1187
@ VLS_LOAD
Definition tree-vectorizer.h:1188
@ VLS_STORE
Definition tree-vectorizer.h:1189
@ VLS_STORE_INVARIANT
Definition tree-vectorizer.h:1190
tree get_mask_type_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13753
pair_hash< tree_operand_hash, tree_operand_hash > tree_cond_mask_hash
Definition tree-vectorizer.h:402
vect_pattern_decl_t slp_patterns[]
Definition tree-vect-slp-patterns.cc:1636
_loop_vec_info * loop_vec_info
tree vect_double_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11171
vect_var_kind
Definition tree-vectorizer.h:35
@ vect_pointer_var
Definition tree-vectorizer.h:37
@ vect_scalar_var
Definition tree-vectorizer.h:38
@ vect_mask_var
Definition tree-vectorizer.h:39
@ vect_simple_var
Definition tree-vectorizer.h:36
int vect_get_place_in_interleaving_chain(stmt_vec_info, stmt_vec_info)
Definition tree-vect-slp.cc:392
class loop * vect_do_peeling(loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, tree *)
Definition tree-vect-loop-manip.cc:3114
void optimize_mask_stores(class loop *)
Definition tree-vect-loop.cc:12351
class _stmt_vec_info * stmt_vec_info
Definition tree-vectorizer.h:24
#define STMT_VINFO_DATA_REF(S)
Definition tree-vectorizer.h:1506
tree vect_get_loop_len(loop_vec_info, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:11358
bool vect_is_store_elt_extraction(vect_cost_for_stmt kind, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2612
void vect_detect_hybrid_slp(loop_vec_info)
Definition tree-vect-slp.cc:6006
opt_result vect_get_vector_types_for_stmt(vec_info *, stmt_vec_info, tree *, tree *, unsigned int=0)
Definition tree-vect-stmts.cc:14627
const poly_uint64 dr_target_alignment(dr_vec_info *dr_info)
Definition tree-vectorizer.h:1952
opt_result vect_analyze_loop_form(class loop *, vect_loop_form_info *)
Definition tree-vect-loop.cc:1737
void vect_update_inits_of_drs(loop_vec_info, tree, tree_code)
Definition tree-vect-loop-manip.cc:2608
struct data_reference * dr_p
Definition tree-vectorizer.h:1256
bool vect_can_advance_ivs_p(loop_vec_info)
Definition tree-vect-loop-manip.cc:2197
opt_result vect_analyze_data_refs(vec_info *, poly_uint64 *, bool *)
Definition tree-vect-data-refs.cc:4760
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vect-data-refs.cc:1194
tree vect_init_vector(vec_info *, stmt_vec_info, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1174
void vect_copy_ref_info(tree, tree)
Definition tree-vect-data-refs.cc:5586
void vect_record_grouped_load_vectors(vec_info *, stmt_vec_info, vec< tree >)
Definition tree-vect-data-refs.cc:6992
bool vectorizable_lc_phi(loop_vec_info, stmt_vec_info, gimple **, slp_tree)
Definition tree-vect-loop.cc:8962
tree vect_get_new_ssa_name(tree, enum vect_var_kind, const char *=NULL)
Definition tree-vect-data-refs.cc:5057
#define STMT_VINFO_VECTYPE(S)
Definition tree-vectorizer.h:1503
opt_result vect_prune_runtime_alias_test_list(loop_vec_info)
Definition tree-vect-data-refs.cc:3911
bool needs_fold_left_reduction_p(tree, code_helper)
Definition tree-vect-loop.cc:3970
bool vect_nop_conversion_p(stmt_vec_info)
Definition tree-vect-stmts.cc:5861
void vect_finish_stmt_generation(vec_info *, stmt_vec_info, gimple *, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1396
unsigned HOST_WIDE_INT vect_max_vf(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2137
loop_vec_info vect_create_loop_vinfo(class loop *, vec_info_shared *, const vect_loop_form_info *, loop_vec_info=nullptr)
Definition tree-vect-loop.cc:1914
bool vect_stmt_dominates_stmt_p(gimple *, gimple *)
Definition tree-vectorizer.cc:772
bool vect_supportable_shift(vec_info *, enum tree_code, tree)
Definition tree-vect-stmts.cc:6059
void vect_optimize_slp(vec_info *)
Definition tree-vect-slp.cc:5838
void vect_iv_increment_position(edge, gimple_stmt_iterator *, bool *)
Definition tree-vect-loop-manip.cc:457
hash_set< scalar_cond_masked_key > scalar_cond_masked_set_type
Definition tree-vectorizer.h:397
tree vect_phi_initial_value(gphi *phi)
Definition tree-vectorizer.h:1759
opt_pointer_wrapper< loop_vec_info > opt_loop_vec_info
Definition tree-vectorizer.h:1072
bool vectorizable_reduction(loop_vec_info, stmt_vec_info, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:7438
unsigned record_stmt_cost(stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location)
Definition tree-vect-stmts.cc:117
bool vect_emulated_vector_p(tree)
Definition tree-vect-loop.cc:9316
tree get_same_sized_vectype(tree, tree)
Definition tree-vect-stmts.cc:13789
gimple * vect_gimple_build(tree, code_helper, tree, tree=NULL_TREE)
Definition tree-vect-patterns.cc:7389
void add_stmt_costs(vector_costs *costs, stmt_vector_for_cost *cost_vec)
Definition tree-vectorizer.h:1924
void vect_finish_replace_stmt(vec_info *, stmt_vec_info, gimple *)
Definition tree-vect-stmts.cc:1380
void vect_slp_fini(void)
Definition tree-vect-slp.cc:81
#define LOOP_VINFO_NITERS_KNOWN_P(L)
Definition tree-vectorizer.h:1059
stmt_vec_info get_later_stmt(stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
Definition tree-vectorizer.h:1799
vec< std::pair< unsigned, unsigned > > lane_permutation_t
Definition tree-vectorizer.h:167
tree vect_gen_loop_len_mask(loop_vec_info, gimple_stmt_iterator *, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:11425
widest_int vect_iv_limit_for_partial_vectors(loop_vec_info loop_vinfo)
Definition tree-vect-loop.cc:12545
void vect_get_vec_defs_for_operand(vec_info *vinfo, stmt_vec_info, unsigned, tree op, vec< tree > *, tree=NULL)
Definition tree-vect-stmts.cc:1246
void vect_gather_slp_loads(vec_info *)
Definition tree-vect-slp.cc:5848
vect_partial_vector_style
Definition tree-vectorizer.h:316
@ vect_partial_vectors_none
Definition tree-vectorizer.h:317
@ vect_partial_vectors_len
Definition tree-vectorizer.h:320
@ vect_partial_vectors_avx512
Definition tree-vectorizer.h:319
@ vect_partial_vectors_while_ult
Definition tree-vectorizer.h:318
class _slp_instance * slp_instance
bool vect_maybe_update_slp_op_vectype(slp_tree, tree)
Definition tree-vect-stmts.cc:14050
bool compatible_calls_p(gcall *, gcall *)
Definition tree-vect-slp.cc:958
#define STMT_VINFO_REDUC_IDX(S)
Definition tree-vectorizer.h:1513
class loop * slpeel_tree_duplicate_loop_to_edge_cfg(class loop *, edge, class loop *, edge, edge, edge *, bool=true, vec< basic_block > *=NULL)
Definition tree-vect-loop-manip.cc:1474
unsigned int vect_vf_for_cost(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2119
unsigned int vect_nunits_for_cost(tree vec_type)
Definition tree-vectorizer.h:2129
bool vect_get_range_info(tree, wide_int *, wide_int *)
Definition tree-vect-patterns.cc:67
bool vect_grouped_load_supported(tree, bool, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:6272
#define STMT_VINFO_IN_PATTERN_P(S)
Definition tree-vectorizer.h:1532
dump_user_location_t vect_location
Definition tree-vectorizer.cc:88
tree vect_create_addr_base_for_vector_ref(vec_info *, stmt_vec_info, gimple_seq *, tree)
Definition tree-vect-data-refs.cc:5132
stmt_vec_info vect_find_last_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:2984
tree vect_get_main_loop_result(loop_vec_info, tree, tree)
Definition tree-vect-loop-manip.cc:3005
#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L)
Definition tree-vectorizer.h:1003
#define STMT_VINFO_DR_INFO(S)
Definition tree-vectorizer.h:1529
hash_map< slp_node_hash, bool > slp_compat_nodes_map_t
Definition tree-vectorizer.h:2543
bool vect_gather_scatter_fn_p(vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *)
Definition tree-vect-data-refs.cc:4173
tree get_related_vectype_for_scalar_type(machine_mode, tree, poly_uint64=0)
Definition tree-vect-stmts.cc:13560
bool nested_in_vect_loop_p(class loop *loop, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1748
bool vect_slp_analyze_instance_dependence(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:1146
auto_vec< std::pair< data_reference *, tree > > drs_init_vec
Definition tree-vectorizer.h:657
edge vec_init_loop_exit_info(class loop *)
Definition tree-vect-loop.cc:972
opt_result vect_analyze_data_ref_accesses(vec_info *, vec< int > *)
Definition tree-vect-data-refs.cc:3389
int vect_get_stmt_cost(enum vect_cost_for_stmt type_of_cost)
Definition tree-vectorizer.h:1856
void vect_transform_grouped_load(vec_info *, stmt_vec_info, vec< tree >, int, gimple_stmt_iterator *)
Definition tree-vect-data-refs.cc:6961
dump_user_location_t find_loop_location(class loop *)
Definition tree-vect-loop-manip.cc:2056
opt_result vect_analyze_slp(vec_info *, unsigned)
Definition tree-vect-slp.cc:3644
bool vect_slp_analyze_operations(vec_info *)
Definition tree-vect-slp.cc:6794
tree neutral_op_for_reduction(tree, code_helper, tree, bool=true)
Definition tree-vect-loop.cc:3912
void vect_free_slp_tree(slp_tree)
Definition tree-vect-slp.cc:169
unsigned vectorize_loops(void)
bool vect_can_force_dr_alignment_p(const_tree, poly_uint64)
Definition tree-vect-data-refs.cc:7043
tree bump_vector_ptr(vec_info *, tree, gimple *, gimple_stmt_iterator *, stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:5517
bool vect_apply_runtime_profitability_check_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2162
unsigned int vect_get_num_vectors(poly_uint64 nunits, tree vectype)
Definition tree-vectorizer.h:2075
#define DR_TARGET_ALIGNMENT(DR)
Definition tree-vectorizer.h:1958
void vect_free_loop_info_assumptions(class loop *)
Definition tree-vectorizer.cc:834
void set_dr_target_alignment(dr_vec_info *dr_info, poly_uint64 val)
Definition tree-vectorizer.h:1961
stmt_vec_info vect_stmt_to_vectorize(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1812
int vect_reduc_type(vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2629
#define STMT_VINFO_RELATED_STMT(S)
Definition tree-vectorizer.h:1533
bool is_pattern_stmt_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1780
#define MAX_VECTORIZATION_FACTOR
Definition tree-vectorizer.h:1733
int vect_pow2(int x)
Definition tree-vectorizer.h:1833
bool reduction_fn_for_scalar_code(code_helper, internal_fn *)
Definition tree-vect-loop.cc:3850
void vect_pattern_recog(vec_info *)
Definition tree-vect-patterns.cc:7329
int vect_slp_child_index_for_operand(const gimple *, int op, bool)
Definition tree-vect-slp.cc:598
vect_induction_op_type
Definition tree-vectorizer.h:74
@ vect_step_op_add
Definition tree-vectorizer.h:75
@ vect_step_op_shl
Definition tree-vectorizer.h:78
@ vect_step_op_mul
Definition tree-vectorizer.h:77
@ vect_step_op_neg
Definition tree-vectorizer.h:76
@ vect_step_op_shr
Definition tree-vectorizer.h:79
void vect_set_loop_condition(class loop *, edge, loop_vec_info, tree, tree, tree, bool)
Definition tree-vect-loop-manip.cc:1389
void vect_prepare_for_masked_peels(loop_vec_info)
Definition tree-vect-loop-manip.cc:2638
stmt_vec_info vect_find_first_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3001
enum _complex_perm_kinds complex_perm_kinds_t
bool vect_is_integer_truncation(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2692
auto_vec< std::pair< unsigned, unsigned >, 16 > auto_lane_permutation_t
Definition tree-vectorizer.h:168
bool vect_use_loop_mask_for_alignment_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2064
tree vect_build_loop_niters(loop_vec_info, bool *=NULL)
Definition tree-vect-loop-manip.cc:2684
bool vect_get_loop_mask_type(loop_vec_info)
void vect_update_max_nunits(poly_uint64 *max_nunits, poly_uint64 nunits)
Definition tree-vectorizer.h:2095
slp_instance_kind
Definition tree-vectorizer.h:247
@ slp_inst_kind_ctor
Definition tree-vectorizer.h:252
@ slp_inst_kind_reduc_group
Definition tree-vectorizer.h:249
@ slp_inst_kind_store
Definition tree-vectorizer.h:248
@ slp_inst_kind_reduc_chain
Definition tree-vectorizer.h:250
@ slp_inst_kind_bb_reduc
Definition tree-vectorizer.h:251
enum vect_cost_model loop_cost_model(loop_p loop)
Definition tree-vectorizer.h:2043
#define DR_MISALIGNMENT_UNKNOWN
Definition tree-vectorizer.h:1936
hash_map< slp_tree, complex_perm_kinds_t > slp_tree_to_load_perm_map_t
Definition tree-vectorizer.h:2538
opt_result vect_enhance_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2141
bool vect_is_reduction(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2621
innermost_loop_behavior * vect_dr_behavior(vec_info *vinfo, dr_vec_info *dr_info)
Definition tree-vectorizer.h:2005
void vect_permute_store_chain(vec_info *, vec< tree > &, unsigned int, stmt_vec_info, gimple_stmt_iterator *, vec< tree > *)
Definition tree-vect-data-refs.cc:5846
tree perm_mask_for_reverse(tree)
Definition tree-vect-stmts.cc:1809
stmt_vec_info_type
Definition tree-vectorizer.h:1115
@ comparison_vec_info_type
Definition tree-vectorizer.h:1125
@ condition_vec_info_type
Definition tree-vectorizer.h:1124
@ type_conversion_vec_info_type
Definition tree-vectorizer.h:1130
@ load_vec_info_type
Definition tree-vectorizer.h:1117
@ loop_exit_ctrl_vec_info_type
Definition tree-vectorizer.h:1135
@ recurr_info_type
Definition tree-vectorizer.h:1134
@ op_vec_info_type
Definition tree-vectorizer.h:1120
@ reduc_vec_info_type
Definition tree-vectorizer.h:1126
@ assignment_vec_info_type
Definition tree-vectorizer.h:1123
@ undef_vec_info_type
Definition tree-vectorizer.h:1116
@ shift_vec_info_type
Definition tree-vectorizer.h:1119
@ type_demotion_vec_info_type
Definition tree-vectorizer.h:1129
@ store_vec_info_type
Definition tree-vectorizer.h:1118
@ cycle_phi_info_type
Definition tree-vectorizer.h:1131
@ phi_info_type
Definition tree-vectorizer.h:1133
@ lc_phi_info_type
Definition tree-vectorizer.h:1132
@ type_promotion_vec_info_type
Definition tree-vectorizer.h:1128
@ call_simd_clone_vec_info_type
Definition tree-vectorizer.h:1122
@ call_vec_info_type
Definition tree-vectorizer.h:1121
@ induc_vec_info_type
Definition tree-vectorizer.h:1127
void vect_get_slp_defs(slp_tree, vec< tree > *)
Definition tree-vect-slp.cc:8554
void vect_record_loop_len(loop_vec_info, vec_loop_lens *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11323
struct _slp_tree * slp_tree
Definition tree-vectorizer.h:25
gimple_seq vect_gen_len(tree, tree, tree, tree)
Definition tree-vect-stmts.cc:14791
bool vect_transform_cycle_phi(loop_vec_info, stmt_vec_info, gimple **, slp_tree, slp_instance)
Definition tree-vect-loop.cc:8684
tree vect_embedded_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2643
gimple * vect_loop_vectorized_call(class loop *, gcond **cond=NULL)
Definition tree-vectorizer.cc:849
void duplicate_and_interleave(vec_info *, gimple_seq *, tree, const vec< tree > &, unsigned int, vec< tree > &)
Definition tree-vect-slp.cc:8230
bool vect_grouped_store_supported(tree, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:5642
tree vect_setup_realignment(vec_info *, stmt_vec_info, gimple_stmt_iterator *, tree *, enum dr_alignment_support, tree, class loop **)
Definition tree-vect-data-refs.cc:6034
vect_def_type
Definition tree-vectorizer.h:59
@ vect_induction_def
Definition tree-vectorizer.h:64
@ vect_uninitialized_def
Definition tree-vectorizer.h:60
@ vect_nested_cycle
Definition tree-vectorizer.h:67
@ vect_constant_def
Definition tree-vectorizer.h:61
@ vect_external_def
Definition tree-vectorizer.h:62
@ vect_condition_def
Definition tree-vectorizer.h:69
@ vect_internal_def
Definition tree-vectorizer.h:63
@ vect_reduction_def
Definition tree-vectorizer.h:65
@ vect_double_reduction_def
Definition tree-vectorizer.h:66
@ vect_first_order_recurrence
Definition tree-vectorizer.h:68
@ vect_unknown_def_type
Definition tree-vectorizer.h:70
opt_result vect_analyze_data_ref_dependences(loop_vec_info, unsigned int *)
Definition tree-vect-data-refs.cc:871
vect_relevant
Definition tree-vectorizer.h:1140
@ vect_used_in_outer
Definition tree-vectorizer.h:1150
@ vect_used_in_scope
Definition tree-vectorizer.h:1160
@ vect_unused_in_scope
Definition tree-vectorizer.h:1141
@ vect_used_by_reduction
Definition tree-vectorizer.h:1158
@ vect_used_in_outer_by_reduction
Definition tree-vectorizer.h:1147
@ vect_used_only_live
Definition tree-vectorizer.h:1144
bool is_loop_header_bb_p(basic_block bb)
Definition tree-vectorizer.h:1822
#define STMT_VINFO_DR_WRT_VEC_LOOP(S)
Definition tree-vectorizer.h:1516
bool vectorizable_live_operation(vec_info *, stmt_vec_info, slp_tree, slp_instance, int, bool, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:10731
slp_tree vect_create_new_slp_node(unsigned, tree_code)
Definition tree-vect-slp.cc:224
opt_result vect_analyze_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2831
unsigned int vect_known_alignment_in_bytes(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:1990
bool unlimited_cost_model(loop_p loop)
Definition tree-vectorizer.h:2054
bool vectorizable_recurr(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9163
opt_result vect_mark_stmts_to_be_vectorized(loop_vec_info, bool *)
Definition tree-vect-stmts.cc:673
tree vect_gen_while(gimple_seq *, tree, tree, tree, const char *=nullptr)
Definition tree-vect-stmts.cc:14579
tree vect_get_store_rhs(stmt_vec_info)
Definition tree-vect-stmts.cc:1890
class loop * vect_transform_loop(loop_vec_info, gimple *)
Definition tree-vect-loop.cc:11880
void vect_slp_init(void)
Definition tree-vect-slp.cc:75
unsigned int vect_get_num_copies(loop_vec_info loop_vinfo, tree vectype)
Definition tree-vectorizer.h:2086
unsigned int vect_get_scalar_dr_size(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2153
enum dr_alignment_support vect_supportable_dr_alignment(vec_info *, dr_vec_info *, tree, int)
Definition tree-vect-data-refs.cc:7066
opt_result vect_determine_partial_vectors_and_peeling(loop_vec_info)
Definition tree-vect-loop.cc:2644
bool ref_within_array_bound(gimple *, tree)
Definition tree-if-conv.cc:848
dr_alignment_support
Definition tree-vectorizer.h:50
@ dr_unaligned_supported
Definition tree-vectorizer.h:52
@ dr_explicit_realign_optimized
Definition tree-vectorizer.h:54
@ dr_aligned
Definition tree-vectorizer.h:55
@ dr_explicit_realign
Definition tree-vectorizer.h:53
@ dr_unaligned_unsupported
Definition tree-vectorizer.h:51
bool supportable_narrowing_operation(code_helper, tree, tree, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:14418
#define STMT_VINFO_REDUC_TYPE(S)
Definition tree-vectorizer.h:1543
operation_type
Definition tree-vectorizer.h:43
@ ternary_op
Definition tree-vectorizer.h:46
@ unary_op
Definition tree-vectorizer.h:44
@ binary_op
Definition tree-vectorizer.h:45
void vect_record_loop_mask(loop_vec_info, vec_loop_masks *, unsigned int, tree, tree)
Definition tree-vect-loop.cc:11183
vect_memory_access_type
Definition tree-vectorizer.h:1195
@ VMAT_STRIDED_SLP
Definition tree-vectorizer.h:1227
@ VMAT_INVARIANT
Definition tree-vectorizer.h:1197
@ VMAT_GATHER_SCATTER
Definition tree-vectorizer.h:1230
@ VMAT_ELEMENTWISE
Definition tree-vectorizer.h:1221
@ VMAT_CONTIGUOUS_DOWN
Definition tree-vectorizer.h:1205
@ VMAT_CONTIGUOUS
Definition tree-vectorizer.h:1200
@ VMAT_CONTIGUOUS_PERMUTE
Definition tree-vectorizer.h:1210
@ VMAT_LOAD_STORE_LANES
Definition tree-vectorizer.h:1217
@ VMAT_CONTIGUOUS_REVERSE
Definition tree-vectorizer.h:1214
bool vect_chooses_same_modes_p(vec_info *, machine_mode)
Definition tree-vect-stmts.cc:13807
bool slpeel_can_duplicate_loop_p(const class loop *, const_edge, const_edge)
Definition tree-vect-loop-manip.cc:2024
opt_result vect_analyze_stmt(vec_info *, stmt_vec_info, bool *, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:13118
tree vect_get_loop_mask(loop_vec_info, gimple_stmt_iterator *, vec_loop_masks *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11205
slp_vect_type
Definition tree-vectorizer.h:1179
@ loop_vect
Definition tree-vectorizer.h:1180
@ hybrid
Definition tree-vectorizer.h:1182
@ pure_slp
Definition tree-vectorizer.h:1181
vect_reduction_type
Definition tree-vectorizer.h:83
@ TREE_CODE_REDUCTION
Definition tree-vectorizer.h:84
@ FOLD_LEFT_REDUCTION
Definition tree-vectorizer.h:102
@ EXTRACT_LAST_REDUCTION
Definition tree-vectorizer.h:94
@ INTEGER_INDUC_COND_REDUCTION
Definition tree-vectorizer.h:86
@ CONST_COND_REDUCTION
Definition tree-vectorizer.h:87
@ COND_REDUCTION
Definition tree-vectorizer.h:85
bool supportable_widening_operation(vec_info *, code_helper, stmt_vec_info, tree, tree, code_helper *, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:14087
auto_vec< rgroup_controls > vec_loop_lens
Definition tree-vectorizer.h:655
void set_dr_misalignment(dr_vec_info *dr_info, int val)
Definition tree-vectorizer.h:1940
bool vect_use_mask_type_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1771
pair_hash< nofree_ptr_hash< _slp_tree >, nofree_ptr_hash< _slp_tree > > slp_node_hash
Definition tree-vectorizer.h:2542
hash_map< tree_operand_hash, std::pair< stmt_vec_info, innermost_loop_behavior * > > vec_base_alignments
Definition tree-vectorizer.h:127
unsigned add_stmt_cost(vector_costs *costs, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where)
Definition tree-vectorizer.h:1876
tree vect_get_smallest_scalar_type(stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:147
vec< stmt_info_for_cost > stmt_vector_for_cost
Definition tree-vectorizer.h:121
stmt_vec_info info_for_reduction(vec_info *, stmt_vec_info)
Definition tree-vect-loop.cc:5684
bool known_alignment_for_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:1981
void vect_remove_stores(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:13530
#define DR_GROUP_FIRST_ELEMENT(S)
Definition tree-vectorizer.h:1552
void vect_get_load_cost(vec_info *, stmt_vec_info, int, dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool)
Definition tree-vect-stmts.cc:1037
hash_set< tree_cond_mask_hash > vec_cond_masked_set_type
Definition tree-vectorizer.h:403
#define STMT_VINFO_REDUC_DEF(S)
Definition tree-vectorizer.h:1546
opt_result vect_find_stmt_data_reference(loop_p, gimple *, vec< data_reference_p > *, vec< int > *, int)
Definition tree-vect-data-refs.cc:4585
opt_loop_vec_info vect_analyze_loop(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:3567
tree get_dr_vinfo_offset(vec_info *vinfo, dr_vec_info *dr_info, bool check_outer=false)
Definition tree-vectorizer.h:2021
bool vect_slp_if_converted_bb(basic_block bb, loop_p orig_loop)
Definition tree-vect-slp.cc:8087
size_t num__slp_patterns
Definition tree-vect-slp-patterns.cc:1647
tree vect_gen_perm_mask_checked(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9718
bool vectorizable_induction(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9932
vect_pattern *(* vect_pattern_decl_t)(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
Definition tree-vectorizer.h:2592
tree vect_create_destination_var(tree, tree)
Definition tree-vect-data-refs.cc:5608
tree vect_create_data_ref_ptr(vec_info *, stmt_vec_info, tree, class loop *, tree, tree *, gimple_stmt_iterator *, gimple **, bool, tree=NULL_TREE)
Definition tree-vect-data-refs.cc:5255
void vect_gen_vector_loop_niters(loop_vec_info, tree, tree *, tree *, bool)
Definition tree-vect-loop-manip.cc:2775
bool vect_make_slp_decision(loop_vec_info)
Definition tree-vect-slp.cc:5866
bool aligned_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:1971
unsigned HOST_WIDE_INT tree_to_uhwi(const_tree t)
Definition tree.cc:6477
#define TYPE_ALIGN_UNIT(NODE)
Definition tree.h:2339
poly_uint64 TYPE_VECTOR_SUBPARTS(const_tree node)
Definition tree.h:4243
#define TYPE_PRECISION(NODE)
Definition tree.h:2245
#define TREE_OPERAND(NODE, I)
Definition tree.h:1285
#define sizetype
Definition tree.h:5102
#define TREE_CODE_CLASS(CODE)
Definition tree.h:202
#define TYPE_SIZE_UNIT(NODE)
Definition tree.h:2242
#define CONVERT_EXPR_CODE_P(CODE)
Definition tree.h:553
#define TREE_TYPE(NODE)
Definition tree.h:512
#define INTEGRAL_TYPE_P(TYPE)
Definition tree.h:613
#define COMPARISON_CLASS_P(NODE)
Definition tree.h:256
#define NULL_TREE
Definition tree.h:317
#define FOR_EACH_VEC_ELT(V, I, P)
Definition vec.h:1884
constexpr vnull vNULL
Definition vec.h:569