#include "config.h"#include "system.h"#include "coretypes.h"#include "backend.h"#include "target.h"#include "rtl.h"#include "tree.h"#include "gimple.h"#include "tree-pass.h"#include "ssa.h"#include "optabs-tree.h"#include "insn-config.h"#include "recog.h"#include "fold-const.h"#include "stor-layout.h"#include "gimple-iterator.h"#include "cfgloop.h"#include "tree-vectorizer.h"#include "langhooks.h"#include "gimple-walk.h"#include "dbgcnt.h"#include "tree-vector-builder.h"#include "vec-perm-indices.h"#include "gimple-fold.h"#include "internal-fn.h"#include "dump-context.h"#include "cfganal.h"#include "tree-eh.h"#include "tree-cfg.h"#include "alloc-pool.h"#include "sreal.h"#include "predict.h"
Data Structures | |
| struct | _slp_oprnd_info |
| struct | slp_oprnds |
| struct | bst_traits |
| struct | chain_op_t |
| struct | slpg_layout_cost |
| struct | slpg_vertex |
| struct | slpg_partition_info |
| struct | slpg_partition_layout_costs |
| class | vect_optimize_slp_pass |
| struct | slp_scc_info |
Macros | |
| #define | INCLUDE_ALGORITHM |
| #define | REDUC_GROUP_FIRST_ELEMENT(S) |
| #define | GATHER_SCATTER_OFFSET (-3) |
| #define | TEMPLATE "{depth: %f, total: %f}" |
Typedefs | |
| typedef struct _slp_oprnd_info * | slp_oprnd_info |
| typedef hash_map< vec< stmt_vec_info >, slp_tree, simple_hashmap_traits< bst_traits, slp_tree > > | scalar_stmts_to_slp_tree_map_t |
Variables | |
| static object_allocator< _slp_tree > * | slp_tree_pool |
| static slp_tree | slp_first_node |
| static unsigned | least_upthread_swappable_op_distance = -1U |
Macro Definition Documentation
◆ GATHER_SCATTER_OFFSET
| #define GATHER_SCATTER_OFFSET (-3) |
Referenced by slp_oprnds::get_op_for_slp_child(), vect_get_and_check_slp_defs(), and vect_get_operand_map().
◆ INCLUDE_ALGORITHM
| #define INCLUDE_ALGORITHM |
SLP - Basic Block Vectorization Copyright (C) 2007-2026 Free Software Foundation, Inc. Contributed by Dorit Naishlos <dorit@il.ibm.com> and Ira Rosen <irar@il.ibm.com> This file is part of GCC. GCC is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. GCC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GCC; see the file COPYING3. If not see <http://www.gnu.org/licenses/>.
◆ REDUC_GROUP_FIRST_ELEMENT
| #define REDUC_GROUP_FIRST_ELEMENT | ( | S | ) |
Referenced by vect_analyze_slp_reduc_chain(), vect_build_slp_tree_3(), and vect_get_and_check_slp_defs().
◆ TEMPLATE
| #define TEMPLATE "{depth: %f, total: %f}" |
Referenced by vect_optimize_slp_pass::dump().
Typedef Documentation
◆ scalar_stmts_to_slp_tree_map_t
| typedef hash_map<vec <stmt_vec_info>, slp_tree, simple_hashmap_traits <bst_traits, slp_tree> > scalar_stmts_to_slp_tree_map_t |
◆ slp_oprnd_info
| typedef struct _slp_oprnd_info * slp_oprnd_info |
This structure is used in creation of an SLP tree. Each instance corresponds to the same operand in a group of scalar stmts in an SLP node.
Function Documentation
◆ add_slp_costs()
|
static |
Hand over COST_VEC to the target COSTS grouped by SLP node.
References add_slp_costs(), and end().
Referenced by add_slp_costs(), vect_bb_vectorization_profitable_p(), and vect_slp_analyze_operations().
◆ calculate_unrolling_factor()
|
static |
Calculate the unrolling factor for an SLP instance with GROUP_SIZE statements and a vector of NUNITS elements.
References calculate_unrolling_factor().
Referenced by calculate_unrolling_factor(), and vect_update_slp_vf_for_node().
◆ can_duplicate_and_interleave_p()
| bool can_duplicate_and_interleave_p | ( | vec_info * | vinfo, |
| unsigned int | count, | ||
| tree | elt_type, | ||
| unsigned int * | nvectors_out, | ||
| tree * | vector_type_out, | ||
| tree * | permutes ) |
Check whether it is possible to load COUNT elements of type ELT_TYPE using the method implemented by duplicate_and_interleave. Return true if so, returning the number of intermediate vectors in *NVECTORS_OUT (if nonnull) and the type of each intermediate vector in *VECTOR_TYPE_OUT (if nonnull).
References build_nonstandard_integer_type(), can_duplicate_and_interleave_p(), can_vec_perm_const_p(), GET_MODE_BITSIZE(), GET_MODE_NUNITS(), GET_MODE_SIZE(), GET_MODE_UNIT_SIZE, get_vectype_for_scalar_type(), i, int_mode_for_size(), known_eq, TYPE_MODE, vect_gen_perm_mask_checked(), and VECTOR_MODE_P.
Referenced by can_duplicate_and_interleave_p(), duplicate_and_interleave(), vect_build_slp_tree_2(), vect_transform_slp_perm_load(), and vectorizable_reduction().
◆ compatible_calls_p()
Return true if call statements CALL1 and CALL2 are similar enough to be combined into the same SLP group.
References compatible_calls_p(), gimple_call_arg(), gimple_call_combined_fn(), gimple_call_fn(), gimple_call_fntype(), gimple_call_internal_p(), gimple_call_lhs(), gimple_call_num_args(), i, operand_equal_p(), TREE_TYPE, types_compatible_p(), and vect_get_operand_map().
Referenced by compatible_calls_p(), compatible_complex_nodes_p(), vect_build_slp_tree_3(), and vect_transform_slp_perm_load().
◆ debug() [1/2]
| DEBUG_FUNCTION void debug | ( | slp_instance | instance | ) |
References debug, dump_location_t::from_location_t(), MSG_NOTE, SLP_INSTANCE_TREE, UNKNOWN_LOCATION, and vect_print_slp_graph().
◆ debug() [2/2]
| DEBUG_FUNCTION void debug | ( | slp_tree | node | ) |
References debug, dump_location_t::from_location_t(), MSG_NOTE, UNKNOWN_LOCATION, and vect_print_slp_tree().
◆ dot_slp_tree() [1/3]
| DEBUG_FUNCTION void dot_slp_tree | ( | const char * | fname, |
| const vec< slp_instance > & | slp_instances ) |
References dot_slp_tree(), fopen, and SLP_INSTANCE_TREE.
◆ dot_slp_tree() [2/3]
| DEBUG_FUNCTION void dot_slp_tree | ( | const char * | fname, |
| slp_tree | node ) |
References dot_slp_tree(), and fopen.
◆ dot_slp_tree() [3/3]
Recursive helper for the dot producer below.
References hash_set< KeyId, Lazy, Traits >::add(), dot_slp_tree(), dump_location_t::from_location_t(), MSG_NOTE, SLP_TREE_CHILDREN, UNKNOWN_LOCATION, and vect_print_slp_tree().
Referenced by dot_slp_tree(), dot_slp_tree(), and dot_slp_tree().
◆ dt_sort_cmp()
|
static |
Comparator for sorting associatable chains.
References dt_sort_cmp().
Referenced by dt_sort_cmp(), vect_build_slp_tree_2(), and vect_slp_check_for_roots().
◆ duplicate_and_interleave()
| void duplicate_and_interleave | ( | vec_info * | vinfo, |
| gimple_seq * | seq, | ||
| tree | vector_type, | ||
| const vec< tree > & | elts, | ||
| unsigned int | nresults, | ||
| vec< tree > & | results ) |
Build a variable-length vector in which the elements in ELTS are repeated
to a fill NRESULTS vectors of type VECTOR_TYPE. Store the vectors in
RESULTS and add any new instructions to SEQ.
The approach we use is:
(1) Find a vector mode VM with integer elements of mode IM.
(2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
ELTS' has mode IM. This involves creating NELTS' VIEW_CONVERT_EXPRs
from small vectors to IM.
(3) Duplicate each ELTS'[I] into a vector of mode VM.
(4) Use a tree of interleaving VEC_PERM_EXPRs to create VMs with the
correct byte contents.
(5) Use VIEW_CONVERT_EXPR to cast the final VMs to the required type.
We try to find the largest IM for which this sequence works, in order
to cut down on the number of interleaves.
References build_vector_type(), can_duplicate_and_interleave_p(), duplicate_and_interleave(), gcc_unreachable, gimple_build(), gimple_build_assign(), gimple_build_vector(), gimple_build_vector_from_val(), gimple_seq_add_stmt(), i, make_ssa_name(), tree_vector_builder::new_vector(), TREE_TYPE, TYPE_VECTOR_SUBPARTS(), and vector_costs::vinfo().
Referenced by duplicate_and_interleave(), get_initial_defs_for_reduction(), vect_create_constant_vectors(), and vect_transform_slp_perm_load().
◆ force_split_matches()
|
static |
For isomorphic matches[], indicate a splitting point according to CONST_NUNITS.
References dump_enabled_p(), dump_printf_loc(), force_split_matches(), i, MSG_MISSED_OPTIMIZATION, and vect_location.
Referenced by force_split_matches(), and vect_build_slp_tree_2().
◆ get_ultimate_leader()
|
static |
Get the SLP instance leader from INSTANCE_LEADER thereby transitively closing the eventual chain.
References hash_map< KeyId, Value, Traits >::get(), and get_ultimate_leader().
Referenced by get_ultimate_leader(), and vect_bb_partition_graph().
◆ li_cost_vec_cmp()
|
static |
Comparator for the loop-index sorted cost vectors.
References a, b, and li_cost_vec_cmp().
Referenced by li_cost_vec_cmp(), and vect_bb_vectorization_profitable_p().
◆ optimize_load_redistribution()
|
static |
Temporary workaround for loads not being CSEd during SLP build. This function will traverse the SLP tree rooted in ROOT for INSTANCE and find VEC_PERM nodes that blend vectors from multiple nodes that all read from the same DR such that the final operation is equal to a permuted load. Such NODES are then directly converted into LOADS themselves. The nodes are CSEd using BST_MAP.
References FOR_EACH_VEC_ELT, i, optimize_load_redistribution(), optimize_load_redistribution_1(), hash_map< KeyId, Value, Traits >::remove(), SLP_TREE_CHILDREN, SLP_TREE_REF_COUNT, and vect_free_slp_tree().
Referenced by optimize_load_redistribution(), and vect_analyze_slp().
◆ optimize_load_redistribution_1()
|
static |
Helper function of optimize_load_redistribution that performs the operation recursively.
References dump_enabled_p(), dump_printf_loc(), FOR_EACH_VEC_ELT, hash_map< KeyId, Value, Traits >::get(), i, MSG_NOTE, loop::next, NULL, optimize_load_redistribution_1(), hash_map< KeyId, Value, Traits >::put(), hash_map< KeyId, Value, Traits >::remove(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_PERMUTE_P, SLP_TREE_REF_COUNT, SLP_TREE_SCALAR_STMTS, tree_size(), vect_build_slp_tree(), vect_free_slp_tree(), vect_internal_def, vect_is_slp_load_node(), and vect_location.
Referenced by optimize_load_redistribution(), and optimize_load_redistribution_1().
◆ release_scalar_stmts_to_slp_tree_map()
|
static |
Release BST_MAP.
References hash_map< KeyId, Value, Traits >::begin(), hash_map< KeyId, Value, Traits >::end(), release_scalar_stmts_to_slp_tree_map(), and vect_free_slp_tree().
Referenced by release_scalar_stmts_to_slp_tree_map(), vect_analyze_slp(), and vect_optimize_slp().
◆ skip_cfg_latch_edges()
|
static |
Return true if E corresponds to a loop latch edge in the cfg.
References skip_cfg_latch_edges().
Referenced by vect_optimize_slp_pass::create_partitions(), and skip_cfg_latch_edges().
◆ sort_ints()
|
static |
References a, b, and sort_ints().
Referenced by sort_ints(), and vect_slp_can_convert_to_external().
◆ vect_add_slp_permutation()
|
static |
Produce the next vector result for SLP permutation NODE by adding a vector
statement at GSI. If MASK_VEC is nonnull, add:
<new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF, MASK_VEC>
otherwise add:
<new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF,
{ N, N+1, N+2, ... }>
where N == IDENTITY_OFFSET which is either zero or equal to the
number of elements of the result.
References bitsize_int, build1(), build3(), build_constructor_va(), gcc_assert, gcc_unreachable, gimple_assign_lhs(), gimple_build_assign(), known_eq, known_le, make_ssa_name(), NULL, NULL_TREE, operand_equal_p(), SLP_TREE_VECTYPE, tree_to_uhwi(), TREE_TYPE, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), vect_add_slp_permutation(), vect_finish_stmt_generation(), and vector_costs::vinfo().
Referenced by vect_add_slp_permutation(), and vectorizable_slp_permutation_1().
◆ vect_analyze_slp()
| opt_result vect_analyze_slp | ( | vec_info * | vinfo, |
| unsigned | max_tree_size, | ||
| bool | force_single_lane ) |
Check if there are stmts in the loop can be vectorized using SLP. Build SLP trees of packed scalar stmts if SLP is possible.
References boolean_type_node, compare_step_with_zero(), vec_info_shared::datarefs, DR_GROUP_FIRST_ELEMENT, DR_GROUP_SIZE, DR_IS_WRITE, dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, dyn_cast(), opt_result::failure_at(), FOR_EACH_VEC_ELT, gcc_assert, get_loop_exit_edges(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_cond_code(), gimple_cond_lhs(), gimple_cond_rhs(), gimple_phi_arg_def_from_edge(), vec_info::grouped_stores, gsi_end_p(), gsi_next(), gsi_start_phis(), i, integer_zerop(), internal_fn_mask_index(), is_a(), is_gimple_call(), vec_info::lookup_def(), vec_info::lookup_dr(), vec_info::lookup_stmt(), LOOP_VINFO_ALTERNATE_DEFS, LOOP_VINFO_LOOP, LOOP_VINFO_LOOP_CONDS, LOOP_VINFO_SLP_INSTANCES, MSG_NOTE, optimize_load_redistribution(), release_scalar_stmts_to_slp_tree_map(), vec_info::shared, slp_inst_kind_gcond, slp_inst_kind_reduc_group, slp_inst_kind_store, SLP_INSTANCE_KIND, SLP_INSTANCE_TREE, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_REPRESENTATIVE, SLP_TREE_VECTYPE, dr_vec_info::stmt, STMT_VINFO_DEF_TYPE, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_LIVE_P, STMT_VINFO_REDUC_IDX, STMT_VINFO_RELEVANT, STMT_VINFO_STMT, STMT_VINFO_STRIDED_P, opt_result::success(), TREE_CODE, TREE_TYPE, vect_analyze_slp(), vect_analyze_slp_instance(), vect_analyze_slp_reductions(), vect_build_slp_instance(), vect_free_slp_instance(), vect_gather_slp_loads(), vect_load_lanes_supported(), vect_location, vect_lower_load_permutations(), vect_match_slp_patterns(), vect_print_slp_graph(), vect_slp_prefer_store_lanes_p(), vect_stmt_to_vectorize(), vect_store_lanes_supported(), vect_used_only_live, VECTORIZABLE_CYCLE_DEF, virtual_operand_p(), and vNULL.
Referenced by vect_analyze_loop_2(), vect_analyze_slp(), vect_slp_analyze_bb_1(), and vect_transform_slp_perm_load().
◆ vect_analyze_slp_instance()
|
static |
Analyze an SLP instance starting from a group of grouped stores. Call vect_build_slp_tree to build a tree of packed stmts if possible. Return FALSE if it's impossible to SLP any stmt in the loop.
Analyze an SLP instance starting from a group of grouped stores. Call vect_build_slp_tree to build a tree of packed stmts if possible. Return FALSE if it's impossible to SLP any stmt in the group.
References compare_step_with_zero(), DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, DR_REF, dump_enabled_p(), dump_printf_loc(), dyn_cast(), end(), floor_log2(), gcc_assert, get_vectype_for_scalar_type(), gimple_call_internal_fn(), gimple_call_internal_p(), i, internal_fn_mask_index(), is_a(), MSG_NOTE, NULL, NULL_TREE, slp_inst_kind_store, SLP_INSTANCE_KIND, SLP_INSTANCE_LOADS, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_CHILDREN, SLP_TREE_LANES, SLP_TREE_VECTYPE, STMT_VINFO_DATA_REF, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_SLP_VECT_ONLY, STMT_VINFO_STRIDED_P, tree_size(), TREE_TYPE, TYPE_VECTOR_SUBPARTS(), vect_analyze_slp_instance(), vect_build_slp_store_interleaving(), vect_build_slp_tree(), vect_create_new_slp_node(), vect_free_slp_tree(), vect_location, vect_print_slp_graph(), vect_slp_prefer_store_lanes_p(), vect_split_slp_store_group(), vect_stmt_to_vectorize(), vect_store_lanes_supported(), and vNULL.
Referenced by vect_analyze_slp(), and vect_analyze_slp_instance().
◆ vect_analyze_slp_reduc_chain()
|
static |
Analyze an SLP instance starting from a the start of a reduction chain. Call vect_build_slp_tree to build a tree of packed stmts if possible. Return FALSE if SLP build fails.
References associative_tree_code(), gimple_match_op::code, CONVERT_EXPR_CODE_P, _slp_tree::cycle_info, dump_enabled_p(), dump_printf_loc(), FOR_EACH_IMM_USE_FAST, gcc_assert, gcc_unreachable, hash_map< KeyId, Value, Traits >::get(), get_vectype_for_scalar_type(), gimple_arg(), gimple_assign_lhs(), gimple_assign_rhs1(), gimple_assign_rhs_code(), gimple_bb(), gimple_extract_op(), gimple_get_lhs(), loop::header, i, info_for_reduction(), is_a(), is_gimple_assign(), is_gimple_debug(), vec_info::lookup_def(), vec_info::lookup_stmt(), loop_latch_edge(), LOOP_VINFO_LOOP, MSG_NOTE, needs_fold_left_reduction_p(), NULL, gimple_match_op::ops, REDUC_GROUP_FIRST_ELEMENT, _loop_vec_info::reduc_infos, scalar_stmt, signed_or_unsigned_type_for(), slp_inst_kind_reduc_chain, SLP_INSTANCE_KIND, SLP_INSTANCE_LOADS, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_CHILDREN, SLP_TREE_LANES, SLP_TREE_REDUC_IDX, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, SSA_NAME_DEF_STMT, STMT_VINFO_DEF_TYPE, STMT_VINFO_REDUC_CODE, STMT_VINFO_REDUC_DEF, STMT_VINFO_REDUC_IDX, STMT_VINFO_REDUC_TYPE, STMT_VINFO_STMT, TREE_CODE, tree_nop_conversion_p(), tree_size(), TREE_TYPE, gimple_match_op::type, TYPE_UNSIGNED, USE_STMT, vect_analyze_slp_reduc_chain(), vect_build_slp_tree(), vect_constant_def, vect_create_new_slp_node(), vect_external_def, vect_location, vect_orig_stmt(), vect_print_slp_graph(), VECT_REDUC_INFO_CODE, VECT_REDUC_INFO_DEF_TYPE, VECT_REDUC_INFO_FN, VECT_REDUC_INFO_TYPE, vect_slp_linearize_chain(), vect_stmt_to_vectorize(), and vNULL.
Referenced by vect_analyze_slp_reduc_chain(), and vect_analyze_slp_reduction().
◆ vect_analyze_slp_reduction()
|
static |
Analyze an SLP instance starting from SCALAR_STMTS which are a group of KIND. Return true if successful.
References dump_enabled_p(), dump_printf_loc(), gcc_assert, i, MSG_NOTE, NULL, scalar_stmt, slp_inst_kind_reduc_group, SLP_INSTANCE_KIND, SLP_INSTANCE_LOADS, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_LANES, STMT_VINFO_DEF_TYPE, tree_size(), vect_analyze_slp_reduc_chain(), vect_analyze_slp_reduction(), vect_build_slp_tree(), vect_location, vect_print_slp_graph(), vect_reduction_def, and vNULL.
Referenced by vect_analyze_slp_reduction(), and vect_analyze_slp_reductions().
◆ vect_analyze_slp_reduction_group()
|
static |
Analyze a single SLP reduction group. If successful add a SLP instance for it and return true, otherwise return false and have *MATCHES populated.
References dump_enabled_p(), dump_printf_loc(), gcc_assert, MSG_NOTE, NULL, slp_inst_kind_reduc_group, SLP_INSTANCE_KIND, SLP_INSTANCE_LOADS, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_LANES, tree_size(), vect_analyze_slp_reduction_group(), vect_build_slp_tree(), vect_location, vect_print_slp_graph(), and vNULL.
Referenced by vect_analyze_slp_reduction_group(), and vect_analyze_slp_reductions().
◆ vect_analyze_slp_reductions()
|
static |
Analyze reductions in LOOP_VINFO and populate SLP instances accordingly. Returns false if something fails.
References i, lane_reducing_stmt_p(), NULL, _loop_vec_info::reductions, STMT_VINFO_DEF_TYPE, STMT_VINFO_LIVE_P, STMT_VINFO_RELEVANT_P, STMT_VINFO_STMT, vect_analyze_slp_reduction(), vect_analyze_slp_reduction_group(), vect_analyze_slp_reductions(), vect_double_reduction_def, vect_reduction_def, vect_stmt_to_vectorize(), and vNULL.
Referenced by vect_analyze_slp(), and vect_analyze_slp_reductions().
◆ vect_bb_partition_graph()
|
static |
Partition the SLP graph into pieces that can be costed independently.
References dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, get_ultimate_leader(), i, MSG_NOTE, hash_map< KeyId, Value, Traits >::put(), SLP_INSTANCE_TREE, vec_info::slp_instances, vect_bb_partition_graph(), vect_bb_partition_graph_r(), and vect_location.
Referenced by vect_bb_partition_graph(), and vect_slp_analyze_bb_1().
◆ vect_bb_partition_graph_r()
|
static |
Worker of vect_bb_partition_graph, recurse on NODE.
References SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, vect_bb_partition_graph_r(), and vect_internal_def.
Referenced by vect_bb_partition_graph(), and vect_bb_partition_graph_r().
◆ vect_bb_slp_mark_live_stmts() [1/2]
|
static |
Traverse all slp instances of BB_VINFO, and mark lanes of every node that are live outside of the basic-block vectorized region and that can be vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P.
References SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vec_info::slp_instances, STMT_VINFO_LIVE_P, vect_bb_slp_mark_live_stmts(), and vect_location.
◆ vect_bb_slp_mark_live_stmts() [2/2]
|
static |
Mark lanes of NODE that are live outside of the basic-block vectorized region and that can be vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P. Not handled live operations will cause the scalar code computing it to be retained.
References hash_set< KeyId, Lazy, Traits >::add(), CDI_DOMINATORS, hash_set< KeyId, Lazy, Traits >::contains(), DEF_FROM_PTR, dominated_by_p(), dump_enabled_p(), dump_printf_loc(), FOR_EACH_IMM_USE_FAST, FOR_EACH_PHI_OR_STMT_DEF, FOR_EACH_VEC_ELT, gimple_bb(), i, is_a(), is_gimple_debug(), vec_info::lookup_stmt(), MSG_MISSED_OPTIMIZATION, NULL, phi_arg_edge_from_use(), PURE_SLP_STMT, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, SSA_OP_DEF, STMT_VINFO_IN_PATTERN_P, STMT_VINFO_LIVE_P, STMT_VINFO_RELATED_STMT, USE_STMT, vect_bb_slp_mark_live_stmts(), vect_find_last_scalar_stmt_in_slp(), vect_internal_def, vect_location, vect_orig_stmt(), vect_stmt_dominates_stmt_p(), and vectorizable_live_operation().
Referenced by vect_bb_slp_mark_live_stmts(), vect_bb_slp_mark_live_stmts(), and vect_slp_analyze_bb_1().
◆ vect_bb_slp_mark_stmts_vectorized()
|
static |
Mark the original scalar stmt coverage of the vector SLP graph of VINFO with STMT_SLP_TYPE == pure_slp.
References BB_VINFO_SLP_INSTANCES, hash_set< KeyId, Lazy, Traits >::contains(), slp_oprnds::get_op_for_slp_child(), i, vec_info::lookup_def(), slp_oprnds::num_slp_children, pure_slp, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, SLP_TREE_SCALAR_STMTS, STMT_SLP_TYPE, vect_bb_slp_mark_stmts_vectorized(), vect_orig_stmt(), and vect_slp_gather_extern_scalar_stmts().
Referenced by vect_bb_slp_mark_stmts_vectorized(), and vect_slp_analyze_bb_1().
◆ vect_bb_slp_scalar_cost()
|
static |
Compute the scalar cost of the SLP node NODE and its children and return it. Do not account defs that are marked in LIFE and update LIFE according to uses of NODE.
References hash_set< KeyId, Lazy, Traits >::add(), as_a(), DECL_MODE, DECL_P, DEF_FROM_PTR, DR_IS_READ, DR_REF, dump_enabled_p(), dump_printf_loc(), FOR_EACH_IMM_USE_STMT, FOR_EACH_PHI_OR_STMT_DEF, get_base_address(), slp_oprnds::get_op_for_slp_child(), gimple_phi_num_args(), gimple_set_visited(), i, is_a(), is_gimple_debug(), is_global_var(), vec_info::lookup_def(), vec_info::lookup_stmt(), MSG_NOTE, NULL_TREE, slp_oprnds::num_slp_children, PURE_SLP_STMT, record_stmt_cost(), scalar_load, scalar_stmt, scalar_store, SSA_OP_DEF, STMT_VINFO_DATA_REF, STMT_VINFO_LIVE_P, TREE_ADDRESSABLE, vect_bb_slp_scalar_cost(), vect_body, vect_location, vect_nop_conversion_p(), and vect_stmt_to_vectorize().
Referenced by vect_bb_slp_scalar_cost(), and vect_bb_vectorization_profitable_p().
◆ vect_bb_vectorization_profitable_p()
|
static |
Check if vectorization of the basic block is profitable for the subgraph denoted by SLP_INSTANCES.
References hash_set< KeyId, Lazy, Traits >::add(), add_slp_costs(), add_stmt_cost(), vec_info::bbs, vector_costs::body_cost(), dump_enabled_p(), dump_printf(), dump_printf_loc(), vector_costs::epilogue_cost(), vector_costs::finish_cost(), FOR_EACH_VEC_ELT, gimple_bb(), gimple_set_visited(), i, init_cost, li_cost_vec_cmp(), basic_block_def::loop_father, MSG_NOTE, stmt_info_for_cost::node, NULL, loop::num, vector_costs::prologue_cost(), record_stmt_cost(), scalar_stmt, _slp_tree::si, si, sl, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, SLP_TREE_SCALAR_STMTS, stmt_info_for_cost::stmt_info, vect_bb_slp_scalar_cost(), vect_bb_vectorization_profitable_p(), vect_body, vect_location, vect_orig_stmt(), vect_print_slp_graph(), and vNULL.
Referenced by vect_bb_vectorization_profitable_p(), and vect_slp_region().
◆ vect_build_slp_instance()
|
static |
Analyze an SLP instance starting from SCALAR_STMTS which are a group of KIND. Return true if successful. SCALAR_STMTS is owned by this function, REMAIN and ROOT_STMT_INFOS ownership is transferred back to the caller upon failure.
References dump_enabled_p(), dump_printf_loc(), gcc_assert, gimple_get_lhs(), i, MSG_NOTE, NULL, slp_inst_kind_bb_reduc, slp_inst_kind_ctor, slp_inst_kind_gcond, slp_inst_kind_reduc_group, slp_inst_kind_store, SLP_INSTANCE_KIND, SLP_INSTANCE_LOADS, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_LANES, tree_size(), match_elt_t::v, vect_build_slp_instance(), vect_build_slp_tree(), vect_location, vect_orig_stmt(), vect_print_slp_graph(), and vNULL.
Referenced by vect_analyze_slp(), and vect_build_slp_instance().
◆ vect_build_slp_store_interleaving()
|
static |
Build an interleaving scheme for the store sources RHS_NODES from SCALAR_STMTS.
References a, b, NULL, SLP_TREE_CHILDREN, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_REPRESENTATIVE, SLP_TREE_VECTYPE, vect_build_slp_store_interleaving(), and vect_create_new_slp_node().
Referenced by vect_analyze_slp_instance(), and vect_build_slp_store_interleaving().
◆ vect_build_slp_tree()
|
static |
References dump_enabled_p(), dump_printf_loc(), gcc_assert, hash_map< KeyId, Value, Traits >::get(), i, is_a(), MSG_NOTE, NULL, hash_map< KeyId, Value, Traits >::put(), SLP_TREE_DEF_TYPE, SLP_TREE_REF_COUNT, SLP_TREE_SCALAR_STMTS, tree_size(), TYPE_VECTOR_SUBPARTS(), vect_build_slp_tree(), vect_build_slp_tree_2(), vect_internal_def, vect_location, vect_uninitialized_def, and vNULL.
Referenced by optimize_load_redistribution_1(), vect_analyze_slp_instance(), vect_analyze_slp_reduc_chain(), vect_analyze_slp_reduction(), vect_analyze_slp_reduction_group(), vect_build_slp_instance(), vect_build_slp_tree(), vect_build_slp_tree_2(), and vect_lower_load_permutations().
◆ vect_build_slp_tree_1()
|
static |
Verify if the scalar stmts STMTS are isomorphic, require data permutation or are of unsupported types of operation. Return true if they are, otherwise return false and indicate in *MATCHES the groups of isomorphic stmts. See vect_build_slp_tree_3 for details. *TWO_OPERATORS is for the first isomorphic group, knowledge whether following isomorphic groups have one or two operators is not retained.
References dump_enabled_p(), dump_printf_loc(), gcc_assert, gimple_get_lhs(), i, is_a(), known_le, MSG_MISSED_OPTIMIZATION, TYPE_VECTOR_SUBPARTS(), vect_build_slp_tree_1(), vect_build_slp_tree_3(), vect_get_vector_types_for_stmt(), and vect_location.
Referenced by vect_build_slp_tree_1(), and vect_build_slp_tree_2().
◆ vect_build_slp_tree_2()
|
static |
Recursively build an SLP tree starting from NODE. Fail (and return a value not equal to zero) if def-stmts are not isomorphic, require data permutation or are of unsupported types of operation. Otherwise, return 0. The value returned is the depth in the SLP tree where a mismatch was found.
References _slp_oprnd_info::any_pattern, as_a(), associative_tree_code(), gather_scatter_info::base, bit_field_offset(), bit_field_size(), build_zero_cst(), can_duplicate_and_interleave_p(), commutative_binary_fn_p(), commutative_ternary_fn_p(), commutative_ternary_tree_code(), commutative_tree_code(), _slp_oprnd_info::def_stmts, DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, DR_IS_READ, dt_sort_cmp(), dump_enabled_p(), dump_printf(), dump_printf_loc(), dyn_cast(), first_commutative_argument(), _slp_oprnd_info::first_dt, _slp_oprnd_info::first_gs_info, _slp_oprnd_info::first_gs_p, FLOAT_TYPE_P, FOR_EACH_VEC_ELT, force_split_matches(), gcc_assert, GET_MODE_SIZE(), hash_map< KeyId, Value, Traits >::get_or_insert(), get_tree_code_name(), get_vectype_for_scalar_type(), gimple_assign_rhs1(), gimple_assign_rhs_code(), gimple_assign_single_p(), gimple_bb(), gimple_call_combined_fn(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_num_args(), gimple_phi_num_args(), gimple_phi_result(), gimple_vuse(), loop::header, i, loop::inner, INTEGRAL_TYPE_P, is_a(), poly_int< N, C >::is_constant(), is_gimple_assign(), is_gimple_call(), is_pattern_stmt_p(), known_eq, known_ge, least_upthread_swappable_op_distance, chain::length, vec_info::lookup_def(), vec_info::lookup_stmt(), loop_latch_edge(), loop_preheader_edge(), LOOP_VINFO_LOOP, MSG_MISSED_OPTIMIZATION, MSG_NOTE, nested_in_vect_loop_p(), NULL, NULL_TREE, operand_equal_p(), _slp_oprnd_info::ops, PHI_RESULT, _loop_vec_info::reduc_infos, gather_scatter_info::scale, si, single_imm_use(), SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_GS_BASE, SLP_TREE_GS_SCALE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_REDUC_IDX, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, STMT_VINFO_DATA_REF, STMT_VINFO_DEF_TYPE, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_REDUC_CODE, STMT_VINFO_REDUC_DEF, STMT_VINFO_REDUC_IDX, STMT_VINFO_REDUC_TYPE, STMT_VINFO_SLP_VECT_ONLY, STMT_VINFO_STMT, STMT_VINFO_STRIDED_P, STMT_VINFO_VECTORIZABLE, TREE_CODE, TREE_OPERAND, tree_size(), tree_to_poly_uint64(), TREE_TYPE, TYPE_OVERFLOW_WRAPS, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), vect_build_slp_tree(), vect_build_slp_tree_1(), vect_build_slp_tree_2(), vect_constant_def, vect_create_new_slp_node(), vect_create_oprnd_info(), vect_double_reduction_def, vect_external_def, vect_first_order_recurrence, vect_free_oprnd_info(), vect_free_slp_tree(), vect_get_and_check_slp_defs(), vect_get_operand_map(), vect_get_place_in_interleaving_chain(), vect_induction_def, vect_internal_def, vect_location, vect_nested_cycle, vect_orig_stmt(), vect_print_slp_tree(), VECT_REDUC_INFO_CODE, VECT_REDUC_INFO_DEF_TYPE, VECT_REDUC_INFO_FN, VECT_REDUC_INFO_TYPE, vect_reduction_def, vect_slp_build_two_operator_nodes(), vect_slp_can_convert_to_external(), vect_slp_linearize_chain(), vect_slp_tree_uniform_p(), vect_uninitialized_def, vec_info::vector_mode, VECTOR_TYPE_P, and vNULL.
Referenced by vect_build_slp_tree(), and vect_build_slp_tree_2().
◆ vect_build_slp_tree_3()
|
static |
Verify if the scalar stmts STMTS are isomorphic, require data permutation or are of unsupported types of operation. Return false if at least one (or all in case of BB vectorization) stmt is unvectorizable or the comparison could not be carried out. Return true if they all (or at least one in case of BB vectorization) are and indicate in MATCHES[] which stmts are not isomorphic to the stmt at *START_I. This function is designed to be invoked repeatedly on the same set of STMTS with increasing *START_I. When *START_I is zero and the function returns true MATCHES[] will be initialized with, in case of BB vectorization, unvectorizable stmts marked with -2, stmts isomorphic to *START_I as *START_I and other vectorizable stmts with -1 (not isomorphic to *START_I). *START_I, when initially zero, is updated to the first vectorizable statement, so MATCHES[] can have a prefix with entries valued -2. When *START_I is not zero MATCHES[] is expected to be pre-initialized by a former call with *START_I zero and MATCHES[*START_I] should be -1. The function will return true and have the stmts isomorphic to MATCHES[*START_I] marked with *START_I. *TWO_OPERATORS indicates whether the group of isomorphic statements uses two related operations like PLUS_EXPR and MINUS_EXPR. If TWO_OPERATORS is NULL such case is not considered isomorphic. SWAP[] indicates whether for a stmt to be isomorphic to its group leader, has to have its operands swapped (1) or its predicate inverted (2). COND_EXPR is possibly isomorphic to another one after swapping its operands. Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to the first stmt by swapping the two operands of comparison; set SWAP[i] to 2 if stmt I is isormorphic to the first stmt by inverting the code of comparison. Take A1 >= B1 ? X1 : Y1 as an example, it can be swapped to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1.
References as_a(), as_internal_fn(), cfun, commutative_tree_code(), compatible_calls_p(), DECL_P, directly_supported_p(), DR_GROUP_FIRST_ELEMENT, DR_IS_READ, DR_IS_WRITE, DR_STEP, dump_enabled_p(), dump_printf_loc(), dyn_cast(), first_commutative_argument(), gcc_assert, gimple_arg(), gimple_assign_rhs1(), gimple_assign_rhs2(), gimple_assign_rhs_code(), gimple_bb(), gimple_call_arg(), gimple_call_chain(), gimple_call_combined_fn(), gimple_call_noreturn_p(), gimple_call_tail_p(), gimple_could_trap_p(), gimple_get_lhs(), gimple_has_volatile_ops(), HONOR_NANS(), i, integer_zerop(), internal_fn_p(), invert_tree_comparison(), is_a(), code_helper::is_tree_code(), MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, NULL_TREE, operand_equal_p(), optab_vector, REDUC_GROUP_FIRST_ELEMENT, REFERENCE_CLASS_P, stmt_can_throw_internal(), STMT_VINFO_DATA_REF, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_REDUC_IDX, STMT_VINFO_VECTORIZABLE, swap_tree_comparison(), tcc_binary, tcc_comparison, tcc_expression, tcc_unary, TREE_CODE, TREE_CODE_CLASS, TREE_OPERAND, TREE_TYPE, TYPE_SIZE, types_compatible_p(), vect_build_slp_tree_3(), vect_location, VECTOR_TYPE_P, and vectorizable_internal_fn_p().
Referenced by vect_build_slp_tree_1(), and vect_build_slp_tree_3().
◆ vect_contains_pattern_stmt_p()
|
static |
Return true if STMTS contains a pattern statement.
References FOR_EACH_VEC_ELT, i, is_pattern_stmt_p(), and vect_contains_pattern_stmt_p().
Referenced by vect_contains_pattern_stmt_p(), and vect_slp_convert_to_external().
◆ vect_create_constant_vectors()
For constant and loop invariant defs in OP_NODE this function creates vector defs that will be used in the vectorized stmts and stores them to SLP_TREE_VEC_DEFS of OP_NODE.
References build1(), build_all_ones_cst(), build_int_cst(), build_zero_cst(), CONSTANT_CLASS_P, duplicate_and_interleave(), find_fallthru_edge(), fold_unary, gcc_assert, gcc_checking_assert, gcc_unreachable, get_later_stmt(), gimple_bb(), gimple_build_assign(), gimple_build_vector(), gimple_build_vector_from_val(), gimple_seq_add_stmt(), gimple_seq_empty_p(), gsi_after_labels(), GSI_CONTINUE_LINKING, gsi_for_stmt(), gsi_insert_seq_after(), gsi_insert_seq_before(), gsi_insert_seq_on_edge_immediate(), i, vec_info::insert_seq_on_entry(), integer_onep(), integer_zerop(), INTEGRAL_TYPE_P, is_a(), known_eq, vec_info::lookup_def(), make_ssa_name(), tree_vector_builder::new_vector(), NULL, NULL_TREE, operand_equal_p(), _slp_tree::ops, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, SSA_NAME_IS_DEFAULT_DEF, stmt_ends_bb_p(), basic_block_def::succs, TREE_CODE, TREE_TYPE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), vect_create_constant_vectors(), vect_get_num_copies_for_invariant(), VECTOR_BOOLEAN_TYPE_P, and vector_costs::vinfo().
Referenced by vect_create_constant_vectors(), and vect_schedule_slp_node().
◆ vect_create_new_slp_node() [1/5]
|
static |
Create a SLP node inplace at NODE for SCALAR_STMTS and NOPS children.
References SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, vect_create_new_slp_node(), and vect_internal_def.
◆ vect_create_new_slp_node() [2/5]
Create a vect_external_def SLP node inplace at NODE for scalar operands OPS.
References SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_SCALAR_OPS, vect_create_new_slp_node(), and vect_external_def.
◆ vect_create_new_slp_node() [3/5]
Create a SLP node with NOPS children with CODE, either VEC_PERM_EXPR for a permute node or else ERROR_MARK.
References gcc_assert, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, vect_create_new_slp_node(), vect_internal_def, and vNULL.
Referenced by vect_optimize_slp_pass::get_result_with_layout(), vect_analyze_slp_instance(), vect_analyze_slp_reduc_chain(), vect_build_combine_node(), vect_build_slp_store_interleaving(), vect_build_slp_tree_2(), vect_build_swap_evenodd_node(), vect_create_new_slp_node(), vect_create_new_slp_node(), vect_create_new_slp_node(), vect_create_new_slp_node(), vect_create_new_slp_node(), vect_lower_load_permutations(), and vect_transform_slp_perm_load().
◆ vect_create_new_slp_node() [4/5]
|
static |
Create an SLP node for SCALAR_STMTS and NOPS children.
References vect_create_new_slp_node().
◆ vect_create_new_slp_node() [5/5]
Create a vect_external_def SLP node for scalar operands OPS.
References vect_create_new_slp_node().
◆ vect_create_oprnd_info()
|
static |
Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each operand.
References _slp_oprnd_info::any_pattern, _slp_oprnd_info::def_stmts, _slp_oprnd_info::first_dt, _slp_oprnd_info::first_gs_p, _slp_oprnd_info::first_op_type, i, NULL_TREE, _slp_oprnd_info::ops, vect_create_oprnd_info(), and vect_uninitialized_def.
Referenced by vect_build_slp_tree_2(), and vect_create_oprnd_info().
◆ vect_cse_gather_part_starts()
|
static |
Associate stmts with possible starts of a subset of lanes of NODE in PART_STARTS.
References hash_set< KeyId, Lazy, Traits >::add(), gimple_uid(), i, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_SCALAR_STMTS, vect_cse_gather_part_starts(), and vect_internal_def.
Referenced by vect_cse_gather_part_starts(), and vect_optimize_slp().
◆ vect_cse_slp_node_parts()
|
static |
Apply CSE to NODE and its children using lowparts of nodes in BST_MAP.
References hash_set< KeyId, Lazy, Traits >::add(), dump_enabled_p(), dump_printf(), gimple_uid(), i, poly_int< N, C >::is_constant(), MSG_NOTE, NULL, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_PERMUTE_P, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, STMT_VINFO_DATA_REF, TYPE_VECTOR_SUBPARTS(), vect_cse_slp_node_parts(), and vect_internal_def.
Referenced by vect_cse_slp_node_parts(), and vect_optimize_slp().
◆ vect_cse_slp_nodes()
|
static |
Apply CSE to NODE and its children using BST_MAP.
References dump_enabled_p(), dump_printf_loc(), hash_map< KeyId, Value, Traits >::get(), MSG_NOTE, hash_map< KeyId, Value, Traits >::put(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, vect_cse_slp_nodes(), vect_free_slp_tree(), vect_internal_def, and vect_location.
Referenced by vect_cse_slp_nodes(), and vect_optimize_slp().
◆ vect_def_types_match()
|
static |
Return true if DTA and DTB match.
References vect_constant_def, vect_def_types_match(), and vect_external_def.
Referenced by vect_def_types_match(), and vect_get_and_check_slp_defs().
◆ vect_find_first_scalar_stmt_in_slp()
| stmt_vec_info vect_find_first_scalar_stmt_in_slp | ( | slp_tree | node | ) |
Find the first stmt in NODE.
References get_later_stmt(), NULL, SLP_TREE_SCALAR_STMTS, vect_find_first_scalar_stmt_in_slp(), and vect_orig_stmt().
Referenced by vect_find_first_scalar_stmt_in_slp(), vect_schedule_slp_node(), vect_slp_analyze_load_dependences(), vect_transform_slp_perm_load(), and vectorizable_load().
◆ vect_find_last_scalar_stmt_in_slp()
| stmt_vec_info vect_find_last_scalar_stmt_in_slp | ( | slp_tree | node | ) |
Find the last store in SLP INSTANCE.
References get_later_stmt(), NULL, SLP_TREE_SCALAR_STMTS, vect_find_last_scalar_stmt_in_slp(), and vect_orig_stmt().
Referenced by vect_bb_slp_mark_live_stmts(), vect_find_last_scalar_stmt_in_slp(), vect_schedule_slp_node(), vect_slp_analyze_instance_dependence(), vect_slp_analyze_store_dependences(), and vect_transform_slp_perm_load().
◆ vect_fixup_store_groups_with_patterns()
|
static |
Walk the grouped store chains and replace entries with their pattern variant if any.
References DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, FOR_EACH_VEC_ELT, vec_info::grouped_stores, i, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_IN_PATTERN_P, STMT_VINFO_RELATED_STMT, vect_fixup_store_groups_with_patterns(), and vector_costs::vinfo().
Referenced by vect_fixup_store_groups_with_patterns(), and vect_slp_analyze_bb_1().
◆ vect_free_oprnd_info()
|
static |
Free operands info.
References _slp_oprnd_info::def_stmts, FOR_EACH_VEC_ELT, i, _slp_oprnd_info::ops, and vect_free_oprnd_info().
Referenced by vect_build_slp_tree_2(), and vect_free_oprnd_info().
◆ vect_free_slp_instance()
| void vect_free_slp_instance | ( | slp_instance | instance | ) |
Free the memory allocated for the SLP instance.
References free(), SLP_INSTANCE_LOADS, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vect_free_slp_instance(), and vect_free_slp_tree().
Referenced by vect_analyze_loop_2(), vect_analyze_slp(), vect_free_slp_instance(), vect_schedule_slp(), vect_slp_analyze_bb_1(), vect_slp_analyze_operations(), vect_transform_loop(), and vec_info::~vec_info().
◆ vect_free_slp_tree()
| void vect_free_slp_tree | ( | slp_tree | node | ) |
Recursively free the memory allocated for the SLP tree rooted at NODE.
References FOR_EACH_VEC_ELT, i, SLP_TREE_CHILDREN, SLP_TREE_REF_COUNT, and vect_free_slp_tree().
Referenced by addsub_pattern::build(), complex_add_pattern::build(), complex_fms_pattern::build(), complex_mul_pattern::build(), vect_optimize_slp_pass::decide_masked_load_lanes(), vect_optimize_slp_pass::materialize(), optimize_load_redistribution(), optimize_load_redistribution_1(), release_scalar_stmts_to_slp_tree_map(), vect_analyze_slp_instance(), vect_build_slp_tree_2(), vect_cse_slp_nodes(), vect_free_slp_instance(), vect_free_slp_tree(), vect_optimize_slp(), and vect_transform_slp_perm_load().
◆ vect_gather_slp_loads() [1/2]
|
static |
Gather loads in the SLP graph NODE and populate the INST loads array.
References hash_set< KeyId, Lazy, Traits >::add(), DR_IS_READ, FOR_EACH_VEC_ELT, i, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_PERMUTE_P, SLP_TREE_REPRESENTATIVE, STMT_VINFO_DATA_REF, vect_gather_slp_loads(), and vect_internal_def.
◆ vect_gather_slp_loads() [2/2]
| void vect_gather_slp_loads | ( | vec_info * | vinfo | ) |
Gather loads reachable from the individual SLP graph entries.
References FOR_EACH_VEC_ELT, i, SLP_INSTANCE_LOADS, SLP_INSTANCE_TREE, vec_info::slp_instances, and vect_gather_slp_loads().
Referenced by vect_analyze_loop_2(), vect_analyze_slp(), vect_gather_slp_loads(), vect_gather_slp_loads(), vect_lower_load_permutations(), vect_slp_analyze_bb_1(), and vect_transform_slp_perm_load().
◆ vect_get_and_check_slp_defs()
|
static |
Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that they are of a valid type and that they match the defs of the first stmt of the SLP group (stored in OPRNDS_INFO). This function tries to match stmts by swapping operands of STMTS[STMT_NUM] when possible. Non-zero SWAP indicates swap is required for cond_expr stmts. Specifically, SWAP is 1 if STMT is cond and operands of comparison need to be swapped; SWAP is 2 if STMT is cond and code of comparison needs to be inverted. If there was a fatal error return -1; if the error could be corrected by swapping operands of father node of this one, return 1; if everything is ok return 0.
References _slp_oprnd_info::any_pattern, as_a(), gather_scatter_info::base, vec_info::bbs, CDI_DOMINATORS, commutative_tree_code(), _slp_oprnd_info::def_stmts, dominated_by_p(), dump_enabled_p(), dump_printf_loc(), dyn_cast(), first_commutative_argument(), _slp_oprnd_info::first_dt, _slp_oprnd_info::first_gs_info, _slp_oprnd_info::first_gs_p, _slp_oprnd_info::first_op_type, GATHER_SCATTER_OFFSET, gcc_assert, gimple_arg(), gimple_assign_rhs_code(), gimple_bb(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_get_lhs(), gimple_num_args(), gimple_phi_arg_edge(), i, internal_gather_scatter_fn_p(), is_a(), is_pattern_stmt_p(), MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, NULL_TREE, gather_scatter_info::offset, operand_equal_p(), _slp_oprnd_info::ops, REDUC_GROUP_FIRST_ELEMENT, gather_scatter_info::scale, SSA_NAME_DEF_STMT, SSA_NAME_IS_DEFAULT_DEF, STMT_VINFO_DATA_REF, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_REDUC_IDX, STMT_VINFO_RELATED_STMT, TREE_CODE, TREE_OPERAND, TREE_TYPE, types_compatible_p(), vect_check_gather_scatter(), vect_constant_def, vect_def_types_match(), vect_describe_gather_scatter_call(), vect_double_reduction_def, vect_external_def, vect_first_order_recurrence, vect_get_and_check_slp_defs(), vect_get_operand_map(), vect_induction_def, vect_internal_def, vect_is_simple_use(), vect_location, vect_nested_cycle, vect_orig_stmt(), vect_reduction_def, and vect_uninitialized_def.
Referenced by vect_build_slp_tree_2(), and vect_get_and_check_slp_defs().
◆ vect_get_num_copies_for_invariant()
| bool vect_get_num_copies_for_invariant | ( | vec_info * | vinfo, |
| slp_tree | node, | ||
| unsigned * | nvectors, | ||
| unsigned * | excess_elts ) |
Like vect_get_num_copies but N copies of the vector might have excess elements in the last vector. Stores the number of excess elements in the last vector in *EXCESS_ELTS. Returns false if *NVECTORS or *EXCESS_ELTS cannot be computed.
References dyn_cast(), LOOP_VINFO_VECT_FACTOR, SLP_TREE_LANES, SLP_TREE_VECTYPE, TYPE_VECTOR_SUBPARTS(), and vect_get_num_copies_for_invariant().
Referenced by vect_create_constant_vectors(), vect_get_num_copies_for_invariant(), vect_slp_analyze_node_operations(), vect_transform_slp_perm_load(), and vectorizable_lane_reducing().
◆ vect_get_operand_map() [1/2]
|
static |
For most SLP statements, there is a one-to-one mapping between gimple arguments and child nodes. If that is not true for STMT, return an array that contains: - the number of child nodes, followed by - for each child node, the index of the argument associated with that node. The special index -1 is the first operand of an embedded comparison and the special index -2 is the second operand of an embedded comparison. The special index -3 is the offset of a gather as analyzed by vect_check_gather_scatter. SWAP is as for vect_get_and_check_slp_defs.
References commutative_tree_code(), COMPARISON_CLASS_P, dyn_cast(), GATHER_SCATTER_OFFSET, gcc_checking_assert, gcc_unreachable, gimple_assign_lhs(), gimple_assign_rhs1(), gimple_assign_rhs_code(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_num_args(), is_gimple_assign(), tcc_comparison, TREE_CODE, TREE_CODE_CLASS, and vect_get_operand_map().
Referenced by compatible_calls_p(), slp_oprnds::slp_oprnds(), vect_build_slp_tree_2(), vect_get_and_check_slp_defs(), vect_get_operand_map(), vect_get_operand_map(), and vect_slp_child_index_for_operand().
◆ vect_get_operand_map() [2/2]
|
static |
References STMT_VINFO_GATHER_SCATTER_P, and vect_get_operand_map().
◆ vect_get_place_in_interleaving_chain()
| int vect_get_place_in_interleaving_chain | ( | stmt_vec_info | stmt_info, |
| stmt_vec_info | first_stmt_info ) |
Find the place of the data-ref in STMT_INFO in the interleaving chain that starts from FIRST_STMT_INFO. Return -1 if the data-ref is not a part of the chain.
References DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, and vect_get_place_in_interleaving_chain().
Referenced by vect_build_slp_tree_2(), vect_get_place_in_interleaving_chain(), and vect_transform_slp_perm_load().
◆ vect_get_slp_defs() [1/2]
Get the vectorized definitions of SLP_NODE in *VEC_DEFS.
References SLP_TREE_VEC_DEFS, and vect_get_slp_defs().
Referenced by vect_get_gather_scatter_ops(), vect_get_slp_defs(), vect_get_slp_defs(), vect_get_vec_defs(), vect_transform_cycle_phi(), vect_transform_slp_perm_load(), vectorizable_call(), vectorizable_load(), vectorizable_phi(), vectorizable_simd_clone_call(), vectorizable_store(), vectorize_fold_left_reduction(), and vectorize_slp_instance_root_stmt().
◆ vect_get_slp_defs() [2/2]
| void vect_get_slp_defs | ( | vec_info * | , |
| slp_tree | slp_node, | ||
| vec< vec< tree > > * | vec_oprnds, | ||
| unsigned | n ) |
Get N vectorized definitions for SLP_NODE.
References i, SLP_TREE_CHILDREN, vect_get_slp_defs(), and vNULL.
◆ vect_get_slp_scalar_def()
Get the scalar definition of the Nth lane from SLP_NODE or NULL_TREE if there is no definition for it in the scalar IL or it is not known.
References gimple_get_lhs(), NULL_TREE, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, STMT_VINFO_STMT, vect_get_slp_scalar_def(), and vect_internal_def.
Referenced by check_load_store_for_partial_vectors(), supportable_indirect_convert_operation(), vect_get_slp_scalar_def(), vect_transform_slp_perm_load(), and vectorizable_conversion().
◆ vect_get_slp_vect_def()
Get the Ith vectorized definition from SLP_NODE.
References i, SLP_TREE_VEC_DEFS, and vect_get_slp_vect_def().
Referenced by vect_create_epilog_for_reduction(), vect_get_slp_vect_def(), vect_schedule_scc(), vectorizable_induction(), and vectorizable_slp_permutation_1().
◆ vect_is_slp_load_node()
Helper that checks to see if a node is a load node.
References DR_IS_READ, SLP_TREE_DEF_TYPE, SLP_TREE_PERMUTE_P, SLP_TREE_REPRESENTATIVE, STMT_VINFO_DATA_REF, STMT_VINFO_GROUPED_ACCESS, vect_internal_def, and vect_is_slp_load_node().
Referenced by optimize_load_redistribution_1(), and vect_is_slp_load_node().
◆ vect_load_perm_consecutive_p()
Return whether if the load permutation of NODE is consecutive starting with value START_VAL in the first element. If START_VAL is not given the first element's value is used.
References i, SLP_TREE_LOAD_PERMUTATION, UINT_MAX, and vect_load_perm_consecutive_p().
Referenced by has_consecutive_load_permutation(), vect_optimize_slp_pass::remove_redundant_permutations(), vect_load_perm_consecutive_p(), vect_lower_load_permutations(), and vect_transform_slp_perm_load().
◆ vect_lower_load_permutations() [1/2]
|
static |
Transform SLP loads in the SLP graph created by SLP discovery to group loads from the same group and lower load permutations that are unlikely to be supported into a series of permutes. In the degenerate case of having only single-lane SLP instances this should result in a series of permute nodes emulating an interleaving scheme.
References DR_GROUP_FIRST_ELEMENT, i, make_array_slice(), loop::next, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_SCALAR_STMTS, STMT_VINFO_GROUPED_ACCESS, vect_gather_slp_loads(), vect_lower_load_permutations(), and vllp_cmp().
◆ vect_lower_load_permutations() [2/2]
|
static |
Process the set of LOADS that are all from the same dataref group.
References bitmap_set_bit, ceil_log2(), compare_step_with_zero(), ctz_hwi(), DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, exact_log2(), EXECUTE_IF_SET_IN_BITMAP, gcc_assert, i, _slp_tree::ldst_lanes, maybe_gt, NULL, pow2p_hwi(), array_slice< T >::size(), SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, STMT_VINFO_STRIDED_P, tree_size(), TYPE_VECTOR_SUBPARTS(), vect_build_slp_tree(), vect_create_new_slp_node(), vect_load_lanes_supported(), vect_load_perm_consecutive_p(), and vect_lower_load_permutations().
Referenced by vect_analyze_slp(), vect_lower_load_permutations(), and vect_lower_load_permutations().
◆ vect_make_slp_decision()
| bool vect_make_slp_decision | ( | loop_vec_info | loop_vinfo | ) |
For each possible SLP instance decide whether to SLP it and calculate overall unrolling factor needed to SLP the loop. Return TRUE if decided to SLP at least one instance.
References dump_dec(), dump_enabled_p(), dump_printf(), dump_printf_loc(), DUMP_VECT_SCOPE, FOR_EACH_VEC_ELT, i, known_gt, LOOP_VINFO_SLP_INSTANCES, LOOP_VINFO_VECT_FACTOR, MSG_NOTE, SLP_INSTANCE_TREE, SLP_TREE_VECTYPE, TYPE_VECTOR_SUBPARTS(), vect_location, vect_make_slp_decision(), and vect_update_slp_vf_for_node().
Referenced by vect_analyze_loop_2(), vect_make_slp_decision(), and vect_transform_slp_perm_load().
◆ vect_mark_slp_stmts_relevant() [1/2]
|
static |
References vect_mark_slp_stmts_relevant().
◆ vect_mark_slp_stmts_relevant() [2/2]
Mark the statements of the tree rooted at NODE as relevant (vect_used).
References hash_set< KeyId, Lazy, Traits >::add(), gcc_assert, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, STMT_VINFO_RELEVANT, vect_internal_def, vect_mark_slp_stmts_relevant(), and vect_used_in_scope.
Referenced by vect_mark_slp_stmts_relevant(), vect_mark_slp_stmts_relevant(), and vect_slp_analyze_bb_1().
◆ vect_match_slp_patterns()
|
static |
Applies pattern matching to the given SLP tree rooted in REF_NODE using vec_info VINFO. The modified tree is returned. Patterns are tried in order and multiple patterns may match.
References dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, MSG_NOTE, SLP_INSTANCE_TREE, vect_location, vect_match_slp_patterns(), and vect_match_slp_patterns_2().
Referenced by vect_analyze_slp(), and vect_match_slp_patterns().
◆ vect_match_slp_patterns_2()
|
static |
Helper function of vect_match_slp_patterns. Attempts to match patterns against the slp tree rooted in REF_NODE using VINFO. Patterns are matched in post-order traversal. If matching is successful the value in REF_NODE is updated and returned, if not then it is returned unchanged.
References hash_set< KeyId, Lazy, Traits >::add(), vect_pattern::build(), FOR_EACH_VEC_ELT, i, num__slp_patterns, slp_patterns, SLP_TREE_CHILDREN, and vect_match_slp_patterns_2().
Referenced by vect_match_slp_patterns(), and vect_match_slp_patterns_2().
◆ vect_optimize_slp()
| void vect_optimize_slp | ( | vec_info * | vinfo | ) |
Optimize the SLP graph of VINFO.
References hash_set< KeyId, Lazy, Traits >::empty(), is_a(), release_scalar_stmts_to_slp_tree_map(), SLP_INSTANCE_TREE, vec_info::slp_instances, vec_info::stmt_vec_infos, vect_cse_gather_part_starts(), vect_cse_slp_node_parts(), vect_cse_slp_nodes(), vect_free_slp_tree(), vect_optimize_slp(), and vect_optimize_slp_pass::vect_optimize_slp_pass().
Referenced by vect_analyze_loop_2(), vect_optimize_slp(), vect_slp_analyze_bb_1(), and vect_transform_slp_perm_load().
◆ vect_print_slp_graph() [1/2]
|
static |
References vect_print_slp_graph().
◆ vect_print_slp_graph() [2/2]
|
static |
Dump a slp tree NODE using flags specified in DUMP_KIND.
References hash_set< KeyId, Lazy, Traits >::add(), FOR_EACH_VEC_ELT, i, SLP_TREE_CHILDREN, vect_print_slp_graph(), and vect_print_slp_tree().
Referenced by debug(), vect_analyze_slp(), vect_analyze_slp_instance(), vect_analyze_slp_reduc_chain(), vect_analyze_slp_reduction(), vect_analyze_slp_reduction_group(), vect_bb_vectorization_profitable_p(), vect_build_slp_instance(), vect_print_slp_graph(), vect_print_slp_graph(), and vect_schedule_slp().
◆ vect_print_slp_tree()
|
static |
Dump a single SLP tree NODE.
References dump_printf(), dump_printf_loc(), FOR_EACH_VEC_ELT, dump_location_t::get_impl_location(), dump_location_t::get_user_location(), i, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LIVE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_PERMUTE_P, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, STMT_VINFO_LIVE_P, vect_constant_def, vect_external_def, vect_internal_def, and vect_print_slp_tree().
Referenced by debug(), dot_slp_tree(), vect_build_slp_tree_2(), vect_print_slp_graph(), vect_print_slp_tree(), vect_schedule_slp_node(), and vect_transform_slp_perm_load_1().
◆ vect_prologue_cost_for_slp()
|
static |
Compute the prologue cost for invariant or constant operands represented by NODE.
References hash_set< KeyId, Lazy, Traits >::add(), vect_scalar_ops_slice::all_same_p(), i, record_stmt_cost(), scalar_to_vec, SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_SCALAR_OPS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, TYPE_VECTOR_SUBPARTS(), vec_construct, vect_constant_def, vect_prologue, vect_prologue_cost_for_slp(), and vector_load.
Referenced by vect_prologue_cost_for_slp(), and vect_slp_analyze_node_operations().
◆ vect_remove_slp_scalar_calls() [1/2]
References vect_remove_slp_scalar_calls(), and vector_costs::vinfo().
◆ vect_remove_slp_scalar_calls() [2/2]
|
static |
Replace scalar calls from SLP node NODE with setting of their lhs to zero. For loop vectorization this is done in vectorizable_call, but for SLP it needs to be deferred until end of vect_schedule_slp, because multiple SLP instances may refer to the same scalar stmt.
References hash_set< KeyId, Lazy, Traits >::add(), build_zero_cst(), dyn_cast(), gimple_bb(), gimple_build_assign(), gimple_build_nop(), gimple_call_lhs(), gsi_for_stmt(), NULL, vec_info::replace_stmt(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, SSA_NAME_DEF_STMT, TREE_TYPE, unlink_stmt_vdef(), vect_internal_def, vect_orig_stmt(), vect_remove_slp_scalar_calls(), and vector_costs::vinfo().
Referenced by vect_remove_slp_scalar_calls(), vect_remove_slp_scalar_calls(), and vect_schedule_slp().
◆ vect_schedule_scc()
|
static |
Schedule the SLP INSTANCE doing a DFS walk and collecting SCCs. When PLACE_ONLY, return false if there is no possible schedule.
References add_phi_arg(), as_a(), slp_scc_info::dfs, FOR_EACH_EDGE, FOR_EACH_VEC_ELT, gcc_assert, gimple_assign_rhs1(), gimple_assign_set_rhs1(), gimple_assign_set_rhs2(), gimple_bb(), gimple_phi_arg_location(), i, is_a(), slp_scc_info::lowlink, MIN, NULL, slp_scc_info::on_stack, slp_scc_info::res, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_PERMUTE_P, SLP_TREE_REPRESENTATIVE, SLP_TREE_VEC_DEFS, SSA_NAME_DEF_STMT, STMT_VINFO_DEF_TYPE, update_stmt(), vect_first_order_recurrence, vect_get_slp_vect_def(), vect_internal_def, vect_schedule_scc(), and vect_schedule_slp_node().
Referenced by vect_schedule_scc(), and vect_schedule_slp().
◆ vect_schedule_slp()
| bool vect_schedule_slp | ( | vec_info * | vinfo, |
| vec< slp_instance > & | slp_instances, | ||
| bool | place_only ) |
Generate vector code for SLP_INSTANCES in the loop/basic block. Perform vector stmt placement only when PLACE_ONLY is true, removing SLP graph entries that cannot be scheduled. If placing, return false if a schedule cannot be computed for any entry.
References DR_IS_WRITE, dump_enabled_p(), dump_printf_loc(), FOR_EACH_VEC_ELT, gcc_assert, i, is_a(), MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, vec_info::remove_stmt(), SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, STMT_VINFO_DATA_REF, vect_free_slp_instance(), vect_location, vect_orig_stmt(), vect_print_slp_graph(), vect_remove_slp_scalar_calls(), vect_schedule_scc(), vect_schedule_slp(), vect_stmt_dominates_stmt_p(), and vectorize_slp_instance_root_stmt().
Referenced by vect_schedule_slp(), vect_slp_analyze_bb_1(), vect_slp_region(), vect_transform_loop(), and vect_transform_slp_perm_load().
◆ vect_schedule_slp_node()
|
static |
Vectorize SLP NODE. Only compute the vector insertion places when PLACE_ONLY is true. When placing, return false if there is no possible schedule.
References as_a(), vec_info::bbs, CDI_DOMINATORS, cycle_phi_info_type, dominated_by_p(), DR_IS_READ, dump_enabled_p(), dump_printf_loc(), dyn_cast(), FOR_EACH_VEC_ELT, gcc_assert, gcc_checking_assert, gcc_unreachable, gimple_bb(), gimple_could_trap_p(), gimple_phi_arg_edge(), gimple_uid(), gsi_after_labels(), gsi_bb(), gsi_end_bb(), gsi_end_p(), gsi_for_stmt(), gsi_next(), gsi_none(), gsi_prev(), gsi_stmt(), i, induc_vec_info_type, is_a(), is_ctrl_altering_stmt(), vec_info::lookup_def(), loop_preheader_edge(), LOOP_VINFO_FULLY_MASKED_P, LOOP_VINFO_FULLY_WITH_LENGTH_P, LOOP_VINFO_LOOP, MSG_NOTE, NULL, phi_info_type, _slp_tree::si, si, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_PERMUTE_P, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, SLP_TREE_TYPE, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, SSA_NAME_DEF_STMT, SSA_NAME_IS_DEFAULT_DEF, STMT_VINFO_DATA_REF, TREE_CODE, vect_constant_def, vect_create_constant_vectors(), vect_external_def, vect_find_first_scalar_stmt_in_slp(), vect_find_last_scalar_stmt_in_slp(), vect_get_num_copies(), vect_internal_def, vect_location, vect_print_slp_tree(), vect_schedule_slp_node(), vect_stmt_dominates_stmt_p(), vect_transform_stmt(), and vector_costs::vinfo().
Referenced by vect_schedule_scc(), and vect_schedule_slp_node().
◆ vect_slp_analyze_bb_1()
|
static |
Check if the region described by BB_VINFO can be vectorized, returning true if so. When returning false, set FATAL to true if the same failure would prevent vectorization at other vector sizes, false if it is still worth trying other sizes. N_STMTS is the number of statements in the region.
References BB_VINFO_SLP_INSTANCES, dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, fatal(), vec_info::grouped_stores, i, MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, _bb_vec_info::roots, SLP_INSTANCE_TREE, SLP_TREE_SCALAR_STMTS, vect_analyze_data_ref_accesses(), vect_analyze_data_refs(), vect_analyze_slp(), vect_bb_partition_graph(), vect_bb_slp_mark_live_stmts(), vect_bb_slp_mark_stmts_vectorized(), vect_fixup_store_groups_with_patterns(), vect_free_slp_instance(), vect_gather_slp_loads(), vect_location, vect_mark_slp_stmts_relevant(), vect_optimize_slp(), vect_pattern_recog(), vect_record_base_alignments(), vect_schedule_slp(), vect_slp_analyze_bb_1(), vect_slp_analyze_instance_alignment(), vect_slp_analyze_instance_dependence(), vect_slp_analyze_operations(), and vect_slp_check_for_roots().
Referenced by vect_slp_analyze_bb_1(), and vect_slp_region().
◆ vect_slp_analyze_node_operations()
|
static |
Analyze statements contained in SLP tree NODE after recursively analyzing the subtree. NODE_INSTANCE contains NODE and VINFO contains INSTANCE. Return true if the operations are supported.
References hash_set< KeyId, Lazy, Traits >::add(), dump_enabled_p(), dump_printf_loc(), FOR_EACH_VEC_ELT, gcc_assert, i, load_vec_info_type, MSG_NOTE, hash_set< KeyId, Lazy, Traits >::remove(), shift_vec_info_type, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, SLP_TREE_TYPE, SLP_TREE_VECTYPE, SSA_NAME_IS_DEFAULT_DEF, SSA_NAME_VAR, TREE_CODE, VAR_P, vect_constant_def, vect_external_def, vect_get_num_copies_for_invariant(), vect_internal_def, vect_location, vect_prologue_cost_for_slp(), vect_slp_analyze_node_operations(), vect_slp_analyze_node_operations_1(), vect_slp_convert_to_external(), and vect_uninitialized_def.
Referenced by vect_slp_analyze_node_operations(), and vect_slp_analyze_operations().
◆ vect_slp_analyze_node_operations_1()
|
static |
Subroutine of vect_slp_analyze_node_operations. Handle the root of NODE, given then that child nodes have already been processed, and that their def types currently match their SLP node's def type.
References FOR_EACH_VEC_ELT, gcc_checking_assert, i, NULL, permute_info_type, SLP_TREE_PERMUTE_P, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_TYPE, STMT_VINFO_LIVE_P, vect_analyze_stmt(), vect_slp_analyze_node_operations_1(), vectorizable_live_operation(), and vectorizable_slp_permutation().
Referenced by vect_slp_analyze_node_operations(), and vect_slp_analyze_node_operations_1().
◆ vect_slp_analyze_operations()
Analyze statements in SLP instances of VINFO. Return true if the operations are supported.
References hash_set< KeyId, Lazy, Traits >::add(), add_slp_costs(), as_a(), hash_set< KeyId, Lazy, Traits >::contains(), dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, dyn_cast(), hash_set< KeyId, Lazy, Traits >::empty(), gimple_assign_rhs1(), i, is_a(), hash_set< KeyId, Lazy, Traits >::is_empty(), MSG_NOTE, NULL, hash_set< KeyId, Lazy, Traits >::remove(), slp_inst_kind_bb_reduc, slp_inst_kind_ctor, slp_inst_kind_gcond, SLP_INSTANCE_KIND, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_DEF_TYPE, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_TYPE, SLP_TREE_VECTYPE, TREE_TYPE, undef_vec_info_type, useless_type_conversion_p(), vect_free_slp_instance(), vect_internal_def, vect_location, vect_slp_analyze_node_operations(), vect_slp_analyze_operations(), vect_slp_prune_covered_roots(), vectorizable_bb_reduc_epilogue(), and vectorizable_early_exit().
Referenced by vect_analyze_loop_2(), vect_slp_analyze_bb_1(), vect_slp_analyze_operations(), and vect_transform_slp_perm_load().
◆ vect_slp_bbs()
|
static |
Main entry for the BB vectorizer. Analyze and transform BBS, returns true if anything in the basic-block was vectorized.
References gimple_location(), gsi_after_labels(), gsi_end_p(), gsi_next(), gsi_stmt(), i, is_gimple_debug(), NULL, UNKNOWN_LOCATION, vect_find_stmt_data_reference(), vect_location, vect_slp_bbs(), vect_slp_region(), and vNULL.
Referenced by vect_slp_bbs(), vect_slp_function(), and vect_slp_if_converted_bb().
◆ vect_slp_build_two_operator_nodes()
|
static |
Helper for building an associated SLP node chain.
References NULL, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, SLP_TREE_VECTYPE, vect_internal_def, and vect_slp_build_two_operator_nodes().
Referenced by vect_build_slp_tree_2(), and vect_slp_build_two_operator_nodes().
◆ vect_slp_can_convert_to_external()
|
static |
Verify if we can externalize a set of internal defs.
References BASIC_BLOCK_FOR_FN, CDI_DOMINATORS, cfun, dominated_by_p(), gcc_assert, gimple_bb(), i, basic_block_def::index, NULL, sort_ints(), and vect_slp_can_convert_to_external().
Referenced by vect_build_slp_tree_2(), vect_slp_can_convert_to_external(), vect_slp_convert_to_external(), and vect_transform_slp_perm_load_1().
◆ vect_slp_check_for_roots()
|
static |
Find any vectorizable constructors and add them to the grouped_store array.
References as_a(), associative_tree_code(), vec_info::bbs, bitmap_bit_p, bitmap_clear(), bitmap_set_bit, CONSTRUCTOR_ELT, CONSTRUCTOR_ELTS, CONSTRUCTOR_NELTS, dt_sort_cmp(), dyn_cast(), FOR_EACH_CONSTRUCTOR_VALUE, gimple_assign_lhs(), gimple_assign_rhs1(), gimple_assign_rhs2(), gimple_assign_rhs3(), gimple_assign_rhs_code(), gimple_get_lhs(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_stmt(), has_single_use(), i, integer_zerop(), is_gimple_assign(), chain::length, vec_info::lookup_def(), vec_info::lookup_stmt(), vec_info::nbbs, NULL, NULL_TREE, chain::op, reduction_fn_for_scalar_code(), _bb_vec_info::roots, single_imm_use(), slp_inst_kind_bb_reduc, slp_inst_kind_ctor, SSA_NAME_DEF_STMT, poly_int< N, C >::to_constant(), TREE_CODE, TREE_TYPE, TYPE_VECTOR_SUBPARTS(), uniform_vector_p(), useless_type_conversion_p(), vect_internal_def, vect_slp_check_for_roots(), vect_slp_is_lane_insert(), vect_slp_linearize_chain(), vect_stmt_to_vectorize(), VECTOR_TYPE_P, vld_cmp(), and vNULL.
Referenced by vect_slp_analyze_bb_1(), and vect_slp_check_for_roots().
◆ vect_slp_child_index_for_operand()
| int vect_slp_child_index_for_operand | ( | const stmt_vec_info | stmt, |
| int | op ) |
Return the SLP node child index for operand OP of STMT.
References gcc_unreachable, i, vect_get_operand_map(), and vect_slp_child_index_for_operand().
Referenced by vect_check_store_rhs(), vect_slp_child_index_for_operand(), vect_transform_slp_perm_load(), vectorizable_load(), vectorizable_simd_clone_call(), and vectorizable_store().
◆ vect_slp_convert_to_external()
|
static |
Try to build NODE from scalars, returning true on success. NODE_INSTANCE is the SLP instance that contains NODE.
References dump_enabled_p(), dump_printf_loc(), FOR_EACH_VEC_ELT, gimple_get_lhs(), i, is_a(), MSG_NOTE, NULL_TREE, SLP_INSTANCE_TREE, SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, vect_contains_pattern_stmt_p(), vect_external_def, vect_location, vect_orig_stmt(), vect_slp_can_convert_to_external(), vect_slp_convert_to_external(), and VECTOR_BOOLEAN_TYPE_P.
Referenced by vect_slp_analyze_node_operations(), and vect_slp_convert_to_external().
◆ vect_slp_fini()
| void vect_slp_fini | ( | void | ) |
References NULL, slp_first_node, slp_tree_pool, and vect_slp_fini().
Referenced by vect_slp_fini().
◆ vect_slp_function()
Main entry for the BB vectorizer. Analyze and transform BB, returns true if anything in the basic-block was vectorized.
References BASIC_BLOCK_FOR_FN, bitmap_set_bit, CDI_DOMINATORS, dominated_by_p(), loop::dont_vectorize, dump_enabled_p(), dump_printf_loc(), ECF_RETURNS_TWICE, ENTRY_BLOCK_PTR_FOR_FN, EXIT_BLOCK, first_stmt(), flow_loop_nested_p(), free(), gimple_call_flags(), gimple_get_lhs(), gsi_last_bb(), loop::header, i, basic_block_def::index, is_ctrl_altering_stmt(), basic_block_def::loop_father, MSG_MISSED_OPTIMIZATION, n_basic_blocks_for_fn, NULL, loop::num, r, rev_post_order_and_mark_dfs_back_seme(), safe_dyn_cast(), single_succ_edge(), vect_location, vect_slp_bbs(), and vect_slp_function().
Referenced by vect_slp_function(), and vect_transform_slp_perm_load().
◆ vect_slp_gather_extern_scalar_stmts()
|
static |
Compute the set of scalar stmts participating in external nodes.
References hash_set< KeyId, Lazy, Traits >::add(), FOR_EACH_VEC_ELT, i, vec_info::lookup_def(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_OPS, vect_internal_def, and vect_slp_gather_extern_scalar_stmts().
Referenced by vect_bb_slp_mark_stmts_vectorized(), and vect_slp_gather_extern_scalar_stmts().
◆ vect_slp_if_converted_bb()
| bool vect_slp_if_converted_bb | ( | basic_block | bb, |
| loop_p | orig_loop ) |
Special entry for the BB vectorizer. Analyze and transform a single if-converted BB with ORIG_LOOPs body being the not if-converted representation. Returns true if anything in the basic-block was vectorized.
References vect_slp_bbs(), and vect_slp_if_converted_bb().
Referenced by try_vectorize_loop_1(), vect_slp_if_converted_bb(), and vect_transform_slp_perm_load().
◆ vect_slp_init()
| void vect_slp_init | ( | void | ) |
In tree-vect-slp.cc.
References object_allocator< T >::object_allocator(), slp_tree_pool, and vect_slp_init().
Referenced by vect_slp_init().
◆ vect_slp_is_lane_insert()
Return true if USE_STMT is a vector lane insert into VEC and set *THIS_LANE to the lane number that is set.
References dyn_cast(), gimple_assign_rhs1(), gimple_assign_rhs2(), gimple_assign_rhs3(), gimple_assign_rhs_code(), tree_to_poly_uint64(), TREE_TYPE, TYPE_SIZE, useless_type_conversion_p(), and vect_slp_is_lane_insert().
Referenced by vect_slp_check_for_roots(), and vect_slp_is_lane_insert().
◆ vect_slp_linearize_chain()
|
static |
Linearize the associatable expression chain at START with the associatable operation CODE (where PLUS_EXPR also allows MINUS_EXPR), filling CHAIN with the result and using WORKLIST as intermediate storage. CODE_STMT and ALT_CODE_STMT are filled with the first stmt using CODE or MINUS_EXPR. *CHAIN_STMTS if not NULL is filled with all computation stmts, starting with START. When ALLOW_ALT_CODE is false, do not follow into MINUS_EXPR when building a PLUS chain (treat MINUS as leaf).
References as_a(), gcc_assert, gimple_assign_rhs_code(), gimple_get_lhs(), gimple_op(), is_gimple_assign(), is_pattern_stmt_p(), single_imm_use(), vect_internal_def, vect_is_simple_use(), and vect_slp_linearize_chain().
Referenced by vect_analyze_slp_reduc_chain(), vect_build_slp_tree_2(), vect_slp_check_for_roots(), and vect_slp_linearize_chain().
◆ vect_slp_node_weight()
Return the execution frequency of NODE (so that a higher value indicates a "more important" node when optimizing for speed).
References vec_info::bbs, cfun, basic_block_def::count, ENTRY_BLOCK_PTR_FOR_FN, gimple_bb(), SLP_TREE_REPRESENTATIVE, profile_count::to_sreal_scale(), vect_orig_stmt(), and vect_slp_node_weight().
Referenced by vect_optimize_slp_pass::start_choosing_layouts(), and vect_slp_node_weight().
◆ vect_slp_permute()
|
static |
Apply (reverse) bijectite PERM to VEC.
References gcc_assert, i, and vect_slp_permute().
Referenced by vect_optimize_slp_pass::change_layout_cost(), vect_optimize_slp_pass::change_vec_perm_layout(), vect_optimize_slp_pass::get_result_with_layout(), vect_optimize_slp_pass::internal_node_cost(), vect_optimize_slp_pass::materialize(), and vect_slp_permute().
◆ vect_slp_prefer_store_lanes_p()
|
static |
STMT_INFO is a store group of size GROUP_SIZE that we are considering vectorizing with VECTYPE that might be NULL. MASKED_P indicates whether the stores are masked. Return true if we could use IFN_STORE_LANES instead and if that appears to be the better approach.
References DR_REF, get_vectype_for_scalar_type(), STMT_VINFO_DATA_REF, TREE_TYPE, TYPE_VECTOR_SUBPARTS(), vect_slp_prefer_store_lanes_p(), and vect_store_lanes_supported().
Referenced by vect_analyze_slp(), vect_analyze_slp_instance(), and vect_slp_prefer_store_lanes_p().
◆ vect_slp_prune_covered_roots()
|
static |
Prune from ROOTS all stmts that are computed as part of lanes of NODE and recurse to children.
References hash_set< KeyId, Lazy, Traits >::add(), hash_set< KeyId, Lazy, Traits >::remove(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, vect_internal_def, vect_orig_stmt(), and vect_slp_prune_covered_roots().
Referenced by vect_slp_analyze_operations(), and vect_slp_prune_covered_roots().
◆ vect_slp_region()
|
static |
Subroutine of vect_slp_bb. Try to vectorize the statements for all basic blocks in BBS, returning true on success. The region has N_STMTS statements and has the datarefs given by DATAREFS.
References _bb_vec_info::_bb_vec_info(), BB_VINFO_DATAREFS, BB_VINFO_SLP_INSTANCES, vec_info::bbs, vec_info_shared::check_datarefs(), vec_info_shared::datarefs, dbg_cnt(), dump_enabled_p(), dump_printf_loc(), dyn_cast(), fatal(), gcc_assert, GET_MODE_INNER, GET_MODE_NAME, GET_MODE_SIZE(), gimple_assign_rhs_code(), gimple_seq_empty_p(), gimple_set_visited(), gimple_visited_p(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_stmt(), vec_info::insert_seq_on_entry(), vec_info::inv_pattern_def_seq, MSG_MISSED_OPTIMIZATION, MSG_NOTE, MSG_OPTIMIZED_LOCATIONS, vec_info::nbbs, NULL, related_vector_mode(), vec_info_shared::save_datarefs(), vec_info::shared, SLP_INSTANCE_TREE, SLP_TREE_VECTYPE, targetm, TYPE_MODE, unlimited_cost_model(), vect_bb_vectorization_profitable_p(), vect_chooses_same_modes_p(), VECT_COST_MODEL_UNLIMITED, vect_location, vect_schedule_slp(), vect_slp_analyze_bb_1(), vect_slp_region(), vec_info::vector_mode, and VECTOR_MODE_P.
Referenced by vect_slp_bbs(), and vect_slp_region().
◆ vect_slp_tree_uniform_p()
Return true when all lanes in the external or constant NODE have the same value.
References FOR_EACH_VEC_ELT, gcc_assert, i, NULL_TREE, operand_equal_p(), SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_OPS, vect_constant_def, vect_external_def, and vect_slp_tree_uniform_p().
Referenced by vect_optimize_slp_pass::get_result_with_layout(), vect_build_slp_tree_2(), and vect_slp_tree_uniform_p().
◆ vect_split_slp_store_group()
|
static |
Splits a group of stores, currently beginning at FIRST_VINFO, into two groups: one (still beginning at FIRST_VINFO) of size GROUP1_SIZE (also containing the first GROUP1_SIZE stmts, since stores are consecutive), the second containing the remainder. Return the first stmt in the second group.
References DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, dump_enabled_p(), dump_printf_loc(), gcc_assert, i, MSG_NOTE, vect_location, and vect_split_slp_store_group().
Referenced by vect_analyze_slp_instance(), and vect_split_slp_store_group().
◆ vect_transform_slp_perm_load()
| bool vect_transform_slp_perm_load | ( | vec_info * | vinfo, |
| slp_tree | node, | ||
| const vec< tree > & | dr_chain, | ||
| gimple_stmt_iterator * | gsi, | ||
| poly_uint64 | vf, | ||
| bool | analyze_only, | ||
| unsigned * | n_perms, | ||
| unsigned int * | n_loads, | ||
| bool | dce_chain ) |
Generate vector permute statements from a list of loads in DR_CHAIN. If ANALYZE_ONLY is TRUE, only check that it is possible to create valid permute statements for the SLP node NODE. Store the number of vector permute instructions in *N_PERMS and the number of vector load instructions in *N_LOADS. If DCE_CHAIN is true, remove all definitions that were not needed.
References dump_enabled_p(), SLP_TREE_LOAD_PERMUTATION, vect_transform_slp_perm_load(), vect_transform_slp_perm_load_1(), and vector_costs::vinfo().
Referenced by get_load_store_type(), vect_transform_slp_perm_load(), vect_transform_slp_perm_load(), and vectorizable_load().
◆ vect_transform_slp_perm_load_1() [1/2]
|
static |
References vect_print_slp_tree(), vect_slp_can_convert_to_external(), vect_transform_slp_perm_load_1(), and vectorizable_slp_permutation_1().
Referenced by vect_optimize_slp_pass::internal_node_cost(), vect_transform_slp_perm_load(), vect_transform_slp_perm_load_1(), and vect_transform_slp_perm_load_1().
◆ vect_transform_slp_perm_load_1() [2/2]
|
static |
A subroutine of vect_transform_slp_perm_load with two extra arguments: - PERM gives the permutation that the caller wants to use for NODE, which might be different from SLP_LOAD_PERMUTATION. - DUMP_P controls whether the function dumps information.
References as_a(), bitmap_bit_p, bitmap_clear(), bitmap_set_bit, can_vec_perm_const_p(), DR_GROUP_FIRST_ELEMENT, DR_GROUP_SIZE, dump_dec(), dump_printf(), dump_printf_loc(), gcc_assert, gimple_assign_lhs(), gimple_assign_rhs_code(), gimple_build_assign(), gsi_for_stmt(), gsi_remove(), i, poly_int< N, C >::is_constant(), is_gimple_assign(), make_ssa_name(), MSG_MISSED_OPTIMIZATION, NULL, NULL_TREE, release_defs(), single_ssa_tree_operand(), SLP_TREE_LANES, SLP_TREE_SCALAR_STMTS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, SSA_NAME_DEF_STMT, SSA_OP_USE, STMT_VINFO_GROUPED_ACCESS, poly_int< N, C >::to_constant(), TYPE_MODE, TYPE_VECTOR_SUBPARTS(), vect_create_destination_var(), vect_finish_stmt_generation(), vect_gen_perm_mask_checked(), vect_get_num_copies(), vect_location, vect_transform_slp_perm_load_1(), and vector_costs::vinfo().
◆ vect_update_slp_vf_for_node()
|
static |
For NODE update VF based on the number of lanes and the vector types used.
References hash_set< KeyId, Lazy, Traits >::add(), calculate_unrolling_factor(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_PERMUTE_P, SLP_TREE_VECTYPE, TYPE_VECTOR_SUBPARTS(), vect_internal_def, and vect_update_slp_vf_for_node().
Referenced by vect_make_slp_decision(), and vect_update_slp_vf_for_node().
◆ vectorizable_bb_reduc_epilogue()
|
static |
Determine whether we can vectorize the reduction epilogue for INSTANCE.
References as_a(), direct_internal_fn_supported_p(), dump_enabled_p(), dump_printf_loc(), floor_log2(), gimple_assign_lhs(), gimple_assign_rhs_code(), MSG_MISSED_OPTIMIZATION, needs_fold_left_reduction_p(), OPTIMIZE_FOR_BOTH, record_stmt_cost(), reduction_fn_for_scalar_code(), scalar_stmt, SLP_INSTANCE_TREE, SLP_TREE_LANES, SLP_TREE_VECTYPE, TREE_TYPE, TYPE_VECTOR_SUBPARTS(), useless_type_conversion_p(), vec_perm, vec_to_scalar, vect_body, vect_location, vect_nunits_for_cost(), vector_stmt, and vectorizable_bb_reduc_epilogue().
Referenced by vect_slp_analyze_operations(), and vectorizable_bb_reduc_epilogue().
◆ vectorizable_slp_permutation()
| bool vectorizable_slp_permutation | ( | vec_info * | vinfo, |
| gimple_stmt_iterator * | gsi, | ||
| slp_tree | node, | ||
| stmt_vector_for_cost * | cost_vec ) |
Vectorize the SLP permutations in NODE as specified
in SLP_TREE_LANE_PERMUTATION which is a vector of pairs of SLP
child number and lane number.
Interleaving of two two-lane two-child SLP subtrees (not supported):
[ { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } ]
A blend of two four-lane two-child SLP subtrees:
[ { 0, 0 }, { 1, 1 }, { 0, 2 }, { 1, 3 } ]
Highpart of a four-lane one-child SLP subtree (not supported):
[ { 0, 2 }, { 0, 3 } ]
Where currently only a subset is supported by code generating below.
References dump_enabled_p(), record_stmt_cost(), SLP_TREE_CHILDREN, SLP_TREE_LANE_PERMUTATION, SLP_TREE_VECTYPE, vec_perm, vect_body, vectorizable_slp_permutation(), vectorizable_slp_permutation_1(), and vector_costs::vinfo().
Referenced by vect_slp_analyze_node_operations_1(), vect_transform_slp_perm_load(), vect_transform_stmt(), and vectorizable_slp_permutation().
◆ vectorizable_slp_permutation_1()
|
static |
Subroutine of vectorizable_slp_permutation. Check whether the target can perform permutation PERM on the (1 or 2) input nodes in CHILDREN. If GSI is nonnull, emit the permutation there. When GSI is null, the only purpose of NODE is to give properties of the result, such as the vector type and number of SLP lanes. The node does not need to be a VEC_PERM_EXPR. If the target supports the operation, return the number of individual VEC_PERM_EXPRs needed, otherwise return -1. Print information to the dump file if DUMP_P is true.
References can_vec_perm_const_p(), CEIL, dump_dec(), dump_printf(), dump_printf_loc(), dyn_cast(), FOR_EACH_VEC_ELT, gcc_assert, i, poly_int< N, C >::is_constant(), known_eq, known_le, LOOP_VINFO_VECT_FACTOR, MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL_TREE, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_SCALAR_OPS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, poly_int< N, C >::to_constant(), TREE_TYPE, TYPE_MODE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), ui, vect_add_slp_permutation(), vect_external_def, vect_gen_perm_mask_checked(), vect_get_num_copies(), vect_get_slp_vect_def(), vect_internal_def, vect_location, vect_maybe_update_slp_op_vectype(), vectorizable_slp_permutation_1(), and vector_costs::vinfo().
Referenced by vect_optimize_slp_pass::change_layout_cost(), vect_optimize_slp_pass::get_result_with_layout(), vect_optimize_slp_pass::internal_node_cost(), vect_optimize_slp_pass::materialize(), vect_transform_slp_perm_load_1(), vectorizable_slp_permutation(), and vectorizable_slp_permutation_1().
◆ vectorize_slp_instance_root_stmt()
| void vectorize_slp_instance_root_stmt | ( | vec_info * | vinfo, |
| slp_tree | node, | ||
| slp_instance | instance ) |
Vectorize the instance root.
References ANY_INTEGRAL_TYPE_P, as_a(), as_combined_fn(), bitsize_zero_node, build1(), build_constructor(), CONSTRUCTOR_APPEND_ELT, direct_internal_fn_supported_p(), FOR_EACH_VEC_ELT, gcc_assert, gcc_unreachable, gimple_assign_rhs1(), gimple_assign_rhs_code(), gimple_assign_set_rhs_from_tree(), gimple_build(), gimple_convert(), gimple_get_lhs(), gsi_for_stmt(), gsi_insert_seq_before(), GSI_SAME_STMT, gsi_stmt(), i, known_eq, NULL, NULL_TREE, operation_can_overflow(), OPTIMIZE_FOR_BOTH, reduction_fn_for_scalar_code(), slp_inst_kind_bb_reduc, slp_inst_kind_ctor, slp_inst_kind_gcond, SLP_INSTANCE_REMAIN_DEFS, SLP_TREE_VEC_DEFS, STMT_VINFO_STMT, TREE_TYPE, TYPE_OVERFLOW_UNDEFINED, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), unsigned_type_for(), update_stmt(), useless_type_conversion_p(), vec_alloc(), vect_get_slp_defs(), vect_orig_stmt(), vectorizable_early_exit(), vectorize_slp_instance_root_stmt(), vector_costs::vinfo(), and vNULL.
Referenced by vect_schedule_slp(), and vectorize_slp_instance_root_stmt().
◆ vld_cmp()
|
static |
qsort comparator for lane defs.
References a, b, and vld_cmp().
Referenced by vect_slp_check_for_roots(), and vld_cmp().
◆ vllp_cmp()
|
static |
qsort comparator ordering SLP load nodes.
References a, b, DR_GROUP_FIRST_ELEMENT, gcc_assert, gimple_uid(), i, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_SCALAR_STMTS, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_STMT, and vllp_cmp().
Referenced by vect_lower_load_permutations(), and vllp_cmp().
Variable Documentation
◆ least_upthread_swappable_op_distance
|
static |
Distance from the node currently being discovered to the closest upthread commutative operation whose operand-zero discovery may still be fixed by retrying with swapped operands, or -1U if there is none.
Referenced by vect_build_slp_tree_2().
◆ slp_first_node
|
static |
Referenced by _slp_tree::_slp_tree(), vect_slp_fini(), and _slp_tree::~_slp_tree().
◆ slp_tree_pool
|
static |
Referenced by _slp_tree::operator delete(), _slp_tree::operator new(), vect_slp_fini(), and vect_slp_init().
