#include "config.h"#include "system.h"#include "coretypes.h"#include "backend.h"#include "target.h"#include "rtl.h"#include "tree.h"#include "gimple.h"#include "tree-pass.h"#include "ssa.h"#include "optabs-tree.h"#include "insn-config.h"#include "recog.h"#include "fold-const.h"#include "stor-layout.h"#include "gimple-iterator.h"#include "cfgloop.h"#include "tree-vectorizer.h"#include "langhooks.h"#include "gimple-walk.h"#include "dbgcnt.h"#include "tree-vector-builder.h"#include "vec-perm-indices.h"#include "gimple-fold.h"#include "internal-fn.h"#include "dump-context.h"#include "cfganal.h"#include "tree-eh.h"#include "tree-cfg.h"#include "alloc-pool.h"#include "sreal.h"#include "predict.h"
Data Structures | |
| struct | _slp_oprnd_info |
| struct | bst_traits |
| struct | chain_op_t |
| struct | slpg_layout_cost |
| struct | slpg_vertex |
| struct | slpg_partition_info |
| struct | slpg_partition_layout_costs |
| class | vect_optimize_slp_pass |
| struct | vdhs_data |
| struct | slp_scc_info |
Macros | |
| #define | INCLUDE_ALGORITHM |
| #define | TEMPLATE "{depth: %f, total: %f}" |
Typedefs | |
| typedef struct _slp_oprnd_info * | slp_oprnd_info |
| typedef hash_map< vec< stmt_vec_info >, slp_tree, simple_hashmap_traits< bst_traits, slp_tree > > | scalar_stmts_to_slp_tree_map_t |
Variables | |
| static object_allocator< _slp_tree > * | slp_tree_pool |
| static slp_tree | slp_first_node |
| static const int | cond_expr_maps [3][5] |
| static const int | no_arg_map [] = { 0 } |
| static const int | arg0_map [] = { 1, 0 } |
| static const int | arg1_map [] = { 1, 1 } |
| static const int | arg2_arg3_map [] = { 2, 2, 3 } |
| static const int | arg1_arg3_map [] = { 2, 1, 3 } |
| static const int | arg1_arg4_arg5_map [] = { 3, 1, 4, 5 } |
| static const int | arg1_arg3_arg4_map [] = { 3, 1, 3, 4 } |
| static const int | arg3_arg2_map [] = { 2, 3, 2 } |
| static const int | op1_op0_map [] = { 2, 1, 0 } |
| static const int | off_map [] = { 1, -3 } |
| static const int | off_op0_map [] = { 2, -3, 0 } |
| static const int | off_arg2_arg3_map [] = { 3, -3, 2, 3 } |
| static const int | off_arg3_arg2_map [] = { 3, -3, 3, 2 } |
| static const int | mask_call_maps [6][7] |
Macro Definition Documentation
◆ INCLUDE_ALGORITHM
| #define INCLUDE_ALGORITHM |
SLP - Basic Block Vectorization Copyright (C) 2007-2024 Free Software Foundation, Inc. Contributed by Dorit Naishlos <dorit@il.ibm.com> and Ira Rosen <irar@il.ibm.com> This file is part of GCC. GCC is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. GCC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GCC; see the file COPYING3. If not see <http://www.gnu.org/licenses/>.
◆ TEMPLATE
| #define TEMPLATE "{depth: %f, total: %f}" |
Referenced by vect_optimize_slp_pass::dump().
Typedef Documentation
◆ scalar_stmts_to_slp_tree_map_t
| typedef hash_map<vec <stmt_vec_info>, slp_tree, simple_hashmap_traits <bst_traits, slp_tree> > scalar_stmts_to_slp_tree_map_t |
◆ slp_oprnd_info
| typedef struct _slp_oprnd_info * slp_oprnd_info |
This structure is used in creation of an SLP tree. Each instance corresponds to the same operand in a group of scalar stmts in an SLP node.
Function Documentation
◆ calculate_unrolling_factor()
|
static |
Calculate the unrolling factor for an SLP instance with GROUP_SIZE statements and a vector of NUNITS elements.
Referenced by vect_build_slp_instance(), and vect_update_slp_vf_for_node().
◆ can_duplicate_and_interleave_p()
| bool can_duplicate_and_interleave_p | ( | vec_info * | vinfo, |
| unsigned int | count, | ||
| tree | elt_type, | ||
| unsigned int * | nvectors_out, | ||
| tree * | vector_type_out, | ||
| tree * | permutes ) |
Check whether it is possible to load COUNT elements of type ELT_TYPE using the method implemented by duplicate_and_interleave. Return true if so, returning the number of intermediate vectors in *NVECTORS_OUT (if nonnull) and the type of each intermediate vector in *VECTOR_TYPE_OUT (if nonnull).
References build_nonstandard_integer_type(), can_vec_perm_const_p(), count, GET_MODE_BITSIZE(), GET_MODE_NUNITS(), GET_MODE_SIZE(), GET_MODE_UNIT_SIZE, get_vectype_for_scalar_type(), i, int_mode_for_size(), known_eq, TYPE_MODE, vect_gen_perm_mask_checked(), and VECTOR_MODE_P.
Referenced by duplicate_and_interleave(), vect_build_slp_tree_2(), and vectorizable_reduction().
◆ compatible_calls_p()
Return true if call statements CALL1 and CALL2 are similar enough to be combined into the same SLP group.
References gimple_call_arg(), gimple_call_combined_fn(), gimple_call_fn(), gimple_call_fntype(), gimple_call_internal_p(), gimple_call_lhs(), gimple_call_num_args(), i, map, operand_equal_p(), TREE_TYPE, types_compatible_p(), and vect_get_operand_map().
Referenced by compatible_complex_nodes_p(), and vect_build_slp_tree_1().
◆ debug() [1/2]
| DEBUG_FUNCTION void debug | ( | slp_instance | instance | ) |
◆ debug() [2/2]
| DEBUG_FUNCTION void debug | ( | slp_tree | node | ) |
References dump_location_t::from_location_t(), MSG_NOTE, UNKNOWN_LOCATION, and vect_print_slp_tree().
◆ dot_slp_tree() [1/3]
| DEBUG_FUNCTION void dot_slp_tree | ( | const char * | fname, |
| const vec< slp_instance > & | slp_instances ) |
References dot_slp_tree(), fopen, SLP_INSTANCE_TREE, and visited.
◆ dot_slp_tree() [2/3]
| DEBUG_FUNCTION void dot_slp_tree | ( | const char * | fname, |
| slp_tree | node ) |
References dot_slp_tree(), fopen, and visited.
◆ dot_slp_tree() [3/3]
Recursive helper for the dot producer below.
References dot_slp_tree(), dump_location_t::from_location_t(), MSG_NOTE, SLP_TREE_CHILDREN, UNKNOWN_LOCATION, vect_print_slp_tree(), and visited.
Referenced by dot_slp_tree(), dot_slp_tree(), and dot_slp_tree().
◆ dt_sort_cmp()
|
static |
Comparator for sorting associatable chains.
Referenced by vect_build_slp_tree_2(), and vect_slp_check_for_roots().
◆ duplicate_and_interleave()
| void duplicate_and_interleave | ( | vec_info * | vinfo, |
| gimple_seq * | seq, | ||
| tree | vector_type, | ||
| const vec< tree > & | elts, | ||
| unsigned int | nresults, | ||
| vec< tree > & | results ) |
Build a variable-length vector in which the elements in ELTS are repeated
to a fill NRESULTS vectors of type VECTOR_TYPE. Store the vectors in
RESULTS and add any new instructions to SEQ.
The approach we use is:
(1) Find a vector mode VM with integer elements of mode IM.
(2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
ELTS' has mode IM. This involves creating NELTS' VIEW_CONVERT_EXPRs
from small vectors to IM.
(3) Duplicate each ELTS'[I] into a vector of mode VM.
(4) Use a tree of interleaving VEC_PERM_EXPRs to create VMs with the
correct byte contents.
(5) Use VIEW_CONVERT_EXPR to cast the final VMs to the required type.
We try to find the largest IM for which this sequence works, in order
to cut down on the number of interleaves.
References build_vector_type(), can_duplicate_and_interleave_p(), gcc_unreachable, gimple_build(), gimple_build_assign(), gimple_build_vector(), gimple_build_vector_from_val(), gimple_seq_add_stmt(), i, make_ssa_name(), tree_vector_builder::new_vector(), TREE_TYPE, and TYPE_VECTOR_SUBPARTS().
Referenced by get_initial_defs_for_reduction(), and vect_create_constant_vectors().
◆ get_ultimate_leader()
|
static |
Get the SLP instance leader from INSTANCE_LEADER thereby transitively closing the eventual chain.
References hash_map< KeyId, Value, Traits >::get().
Referenced by vect_bb_partition_graph().
◆ li_cost_vec_cmp()
|
static |
Comparator for the loop-index sorted cost vectors.
Referenced by vect_bb_vectorization_profitable_p().
◆ maybe_push_to_hybrid_worklist()
|
static |
Look if STMT_INFO is consumed by SLP indirectly and mark it pure_slp if so, otherwise pushing it to WORKLIST.
References DEF_FROM_PTR, dump_enabled_p(), dump_printf_loc(), FOR_EACH_IMM_USE_FAST, FOR_EACH_PHI_OR_STMT_DEF, is_gimple_debug(), vec_info::lookup_stmt(), MSG_NOTE, pure_slp, SSA_OP_DEF, STMT_SLP_TYPE, STMT_VINFO_DEF_TYPE, USE_STMT, vect_condition_def, vect_location, vect_orig_stmt(), vect_stmt_to_vectorize(), and worklist.
Referenced by vect_detect_hybrid_slp().
◆ optimize_load_redistribution()
|
static |
Temporary workaround for loads not being CSEd during SLP build. This function will traverse the SLP tree rooted in ROOT for INSTANCE and find VEC_PERM nodes that blend vectors from multiple nodes that all read from the same DR such that the final operation is equal to a permuted load. Such NODES are then directly converted into LOADS themselves. The nodes are CSEd using BST_MAP.
References FOR_EACH_VEC_ELT, i, optimize_load_redistribution_1(), hash_map< KeyId, Value, Traits >::remove(), SLP_TREE_CHILDREN, SLP_TREE_REF_COUNT, and vect_free_slp_tree().
Referenced by vect_analyze_slp().
◆ optimize_load_redistribution_1()
|
static |
Helper function of optimize_load_redistribution that performs the operation recursively.
References dump_enabled_p(), dump_printf_loc(), FOR_EACH_VEC_ELT, hash_map< KeyId, Value, Traits >::get(), i, MSG_NOTE, loop::next, NULL, optimize_load_redistribution_1(), hash_map< KeyId, Value, Traits >::put(), hash_map< KeyId, Value, Traits >::remove(), SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_REF_COUNT, SLP_TREE_SCALAR_STMTS, tree_size(), vect_build_slp_tree(), vect_free_slp_tree(), vect_internal_def, vect_is_slp_load_node(), and vect_location.
Referenced by optimize_load_redistribution(), and optimize_load_redistribution_1().
◆ release_scalar_stmts_to_slp_tree_map()
|
static |
Release BST_MAP.
References hash_map< KeyId, Value, Traits >::begin(), hash_map< KeyId, Value, Traits >::end(), and vect_free_slp_tree().
Referenced by vect_analyze_slp(), and vect_optimize_slp().
◆ skip_cfg_latch_edges()
|
static |
Return true if E corresponds to a loop latch edge in the cfg.
References graph_edge::data.
Referenced by vect_optimize_slp_pass::create_partitions().
◆ vec_slp_has_scalar_use()
|
static |
Given a definition DEF, analyze if it will have any live scalar use after performing SLP vectorization whose information is represented by BB_VINFO, and record result into hash map SCALAR_USE_MAP as cache for later fast check. If recursion DEPTH exceeds a limit, stop analysis and make a conservative assumption. Return 0 if no scalar use, 1 if there is, -1 means recursion is limited.
References end_imm_use_stmt_p(), FOR_EACH_IMM_USE_STMT, gcc_assert, hash_map< KeyId, Value, Traits >::get(), gimple_get_lhs(), is_gimple_debug(), vec_info::lookup_stmt(), PURE_SLP_STMT, hash_map< KeyId, Value, Traits >::put(), TREE_CODE, vec_slp_has_scalar_use(), and vect_stmt_to_vectorize().
Referenced by vec_slp_has_scalar_use(), and vect_bb_slp_mark_live_stmts().
◆ vect_add_slp_permutation()
|
static |
Produce the next vector result for SLP permutation NODE by adding a vector statement at GSI. If MASK_VEC is nonnull, add: <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF, MASK_VEC> otherwise add: <new SSA name> = FIRST_DEF.
References bitsize_int, build1(), build3(), build_constructor_va(), gcc_unreachable, gimple_assign_lhs(), gimple_build_assign(), known_le, make_ssa_name(), NULL, NULL_TREE, operand_equal_p(), _slp_tree::push_vec_def(), SLP_TREE_VECTYPE, tree_to_uhwi(), TREE_TYPE, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), and vect_finish_stmt_generation().
Referenced by vectorizable_slp_permutation_1().
◆ vect_analyze_slp()
| opt_result vect_analyze_slp | ( | vec_info * | vinfo, |
| unsigned | max_tree_size, | ||
| bool | force_single_lane ) |
Check if there are stmts in the loop can be vectorized using SLP. Build SLP trees of packed scalar stmts if SLP is possible.
References as_a(), compare_step_with_zero(), vec_info_shared::datarefs, DR_GROUP_FIRST_ELEMENT, DR_GROUP_SIZE, DR_IS_WRITE, dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, dyn_cast(), FOR_EACH_VEC_ELT, gcc_assert, get_loop_exit_edges(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_cond_code(), gimple_cond_lhs(), gimple_cond_rhs(), gimple_phi_arg_def_from_edge(), vec_info::grouped_stores, gsi_end_p(), gsi_next(), gsi_start_phis(), i, internal_fn_mask_index(), is_a(), is_gimple_call(), lane_reducing_stmt_p(), last, vec_info::lookup_def(), vec_info::lookup_dr(), vec_info::lookup_stmt(), loop_latch_edge(), LOOP_VINFO_EARLY_BREAKS_LIVE_IVS, LOOP_VINFO_LOOP, LOOP_VINFO_LOOP_CONDS, LOOP_VINFO_SLP_INSTANCES, MSG_NOTE, loop::next, NULL, optimize_load_redistribution(), REDUC_GROUP_FIRST_ELEMENT, REDUC_GROUP_NEXT_ELEMENT, _loop_vec_info::reduction_chains, _loop_vec_info::reductions, release_scalar_stmts_to_slp_tree_map(), vec_info::shared, slp_inst_kind_gcond, slp_inst_kind_reduc_chain, slp_inst_kind_reduc_group, slp_inst_kind_store, SLP_INSTANCE_KIND, SLP_INSTANCE_TREE, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, dr_vec_info::stmt, STMT_VINFO_DEF_TYPE, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_LIVE_P, STMT_VINFO_REDUC_IDX, STMT_VINFO_RELEVANT, STMT_VINFO_RELEVANT_P, STMT_VINFO_STMT, STMT_VINFO_STRIDED_P, STMT_VINFO_VECTYPE, opt_result::success(), TREE_CODE, vect_analyze_slp_instance(), vect_build_slp_instance(), vect_double_reduction_def, vect_free_slp_instance(), vect_gather_slp_loads(), vect_induction_def, vect_internal_def, vect_load_lanes_supported(), vect_location, vect_lower_load_permutations(), vect_match_slp_patterns(), vect_print_slp_graph(), vect_reduction_def, vect_stmt_to_vectorize(), vect_store_lanes_supported(), vect_used_only_live, virtual_operand_p(), visited, vNULL, and zerop().
Referenced by vect_analyze_loop_2(), and vect_slp_analyze_bb_1().
◆ vect_analyze_slp_instance()
|
static |
Analyze an SLP instance starting from a group of grouped stores. Call vect_build_slp_tree to build a tree of packed stmts if possible. Return FALSE if it's impossible to SLP any stmt in the loop.
References DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, gcc_unreachable, is_a(), NULL, REDUC_GROUP_NEXT_ELEMENT, REDUC_GROUP_SIZE, slp_inst_kind_reduc_chain, slp_inst_kind_store, STMT_VINFO_DEF_TYPE, STMT_VINFO_REDUC_DEF, vect_build_slp_instance(), vect_location, vect_orig_stmt(), vect_stmt_to_vectorize(), and vNULL.
Referenced by vect_analyze_slp(), and vect_build_slp_instance().
◆ vect_bb_partition_graph()
|
static |
Partition the SLP graph into pieces that can be costed independently.
References dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, get_ultimate_leader(), i, MSG_NOTE, hash_map< KeyId, Value, Traits >::put(), SLP_INSTANCE_TREE, vec_info::slp_instances, vect_bb_partition_graph_r(), and vect_location.
Referenced by vect_slp_analyze_bb_1().
◆ vect_bb_partition_graph_r()
|
static |
Worker of vect_bb_partition_graph, recurse on NODE.
References FOR_EACH_VEC_ELT, i, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, vect_bb_partition_graph_r(), and vect_internal_def.
Referenced by vect_bb_partition_graph(), and vect_bb_partition_graph_r().
◆ vect_bb_slp_mark_live_stmts() [1/2]
|
static |
Traverse all slp instances of BB_VINFO, and mark lanes of every node that are live outside of the basic-block vectorized region and that can be vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P.
References hash_map< KeyId, Value, Traits >::put(), slp_inst_kind_bb_reduc, SLP_INSTANCE_KIND, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_OPS, TREE_CODE, vect_bb_slp_mark_live_stmts(), vect_external_def, vect_location, visited, and worklist.
◆ vect_bb_slp_mark_live_stmts() [2/2]
|
static |
Mark lanes of NODE that are live outside of the basic-block vectorized region and that can be vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P. Not handled live operations will cause the scalar code computing it to be retained.
References hash_set< KeyId, Lazy, Traits >::add(), hash_set< KeyId, Lazy, Traits >::contains(), DEF_FROM_PTR, dump_enabled_p(), dump_printf_loc(), FOR_EACH_IMM_USE_STMT, FOR_EACH_PHI_OR_STMT_DEF, FOR_EACH_VEC_ELT, i, is_gimple_debug(), vec_info::lookup_stmt(), MSG_MISSED_OPTIMIZATION, PURE_SLP_STMT, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, SSA_OP_DEF, STMT_VINFO_IN_PATTERN_P, STMT_VINFO_LIVE_P, STMT_VINFO_RELATED_STMT, vec_slp_has_scalar_use(), vect_bb_slp_mark_live_stmts(), vect_find_last_scalar_stmt_in_slp(), vect_internal_def, vect_location, vect_orig_stmt(), vect_stmt_dominates_stmt_p(), vect_stmt_to_vectorize(), vectorizable_live_operation(), and visited.
Referenced by vect_bb_slp_mark_live_stmts(), vect_bb_slp_mark_live_stmts(), and vect_slp_analyze_operations().
◆ vect_bb_slp_scalar_cost()
|
static |
Compute the scalar cost of the SLP node NODE and its children and return it. Do not account defs that are marked in LIFE and update LIFE according to uses of NODE.
References hash_set< KeyId, Lazy, Traits >::add(), as_a(), hash_set< KeyId, Lazy, Traits >::contains(), DECL_MODE, DECL_P, DEF_FROM_PTR, DR_IS_READ, DR_REF, FOR_EACH_IMM_USE_STMT, FOR_EACH_PHI_OR_STMT_DEF, FOR_EACH_VEC_ELT, gcc_assert, get_base_address(), gimple_phi_num_args(), gimple_set_visited(), gimple_visited_p(), i, is_a(), is_gimple_debug(), is_global_var(), vec_info::lookup_stmt(), NULL, record_stmt_cost(), scalar_load, scalar_stmt, scalar_store, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, SSA_OP_DEF, STMT_VINFO_DATA_REF, STMT_VINFO_IN_PATTERN_P, STMT_VINFO_LIVE_P, TREE_ADDRESSABLE, vect_bb_slp_scalar_cost(), vect_body, vect_internal_def, vect_nop_conversion_p(), vect_orig_stmt(), visited, and worklist.
Referenced by vect_bb_slp_scalar_cost(), and vect_bb_vectorization_profitable_p().
◆ vect_bb_vectorization_profitable_p()
|
static |
Check if vectorization of the basic block is profitable for the subgraph denoted by SLP_INSTANCES.
References hash_set< KeyId, Lazy, Traits >::add(), add_stmt_cost(), vector_costs::body_cost(), dump_enabled_p(), dump_printf(), dump_printf_loc(), vector_costs::epilogue_cost(), vector_costs::finish_cost(), FOR_EACH_VEC_ELT, gimple_bb(), gimple_set_visited(), i, init_cost, is_empty(), li_cost_vec_cmp(), basic_block_def::loop_father, MSG_NOTE, loop::num, vector_costs::prologue_cost(), record_stmt_cost(), hash_set< KeyId, Lazy, Traits >::remove(), scalar_stmt, si, sl, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, SLP_TREE_LANES, stmt_info_for_cost::stmt_info, vect_bb_slp_scalar_cost(), vect_body, vect_location, vect_print_slp_graph(), vect_slp_gather_vectorized_scalar_stmts(), visited, and vNULL.
Referenced by vect_slp_region().
◆ vect_build_slp_instance()
|
static |
Analyze an SLP instance starting from SCALAR_STMTS which are a group of KIND. Return true if successful.
References as_a(), calculate_unrolling_factor(), compare_step_with_zero(), DR_REF, dump_enabled_p(), dump_printf_loc(), dyn_cast(), end(), floor_log2(), FOR_EACH_IMM_USE_FAST, gcc_assert, hash_map< KeyId, Value, Traits >::get(), get_vectype_for_scalar_type(), gimple_assign_lhs(), gimple_bb(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_get_lhs(), loop::header, i, internal_fn_mask_index(), is_a(), poly_int< N, C >::is_constant(), is_gimple_debug(), _slp_tree::ldst_lanes, vec_info::lookup_stmt(), loop_latch_edge(), LOOP_VINFO_LOOP, _slp_tree::max_nunits, MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, NULL_TREE, phis, r, REDUC_GROUP_FIRST_ELEMENT, REDUC_GROUP_NEXT_ELEMENT, single_imm_use(), slp_inst_kind_ctor, slp_inst_kind_gcond, slp_inst_kind_reduc_chain, slp_inst_kind_store, SLP_INSTANCE_KIND, SLP_INSTANCE_LOADS, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_CHILDREN, SLP_TREE_LANES, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, STMT_VINFO_DATA_REF, STMT_VINFO_DEF_TYPE, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_SLP_VECT_ONLY, STMT_VINFO_STRIDED_P, STMT_VINFO_VECTYPE, tree_size(), TREE_TYPE, TYPE_VECTOR_SUBPARTS(), USE_STMT, vect_analyze_slp_instance(), vect_build_slp_store_interleaving(), vect_build_slp_tree(), vect_create_new_slp_node(), vect_free_slp_tree(), vect_location, vect_orig_stmt(), vect_print_slp_graph(), vect_reduction_def, vect_slp_prefer_store_lanes_p(), vect_split_slp_store_group(), vect_stmt_to_vectorize(), vect_store_lanes_supported(), vect_update_max_nunits(), and vNULL.
Referenced by vect_analyze_slp(), and vect_analyze_slp_instance().
◆ vect_build_slp_store_interleaving()
|
static |
Build an interleaving scheme for the store sources RHS_NODES from SCALAR_STMTS.
References a, b, _slp_tree::max_nunits, SLP_TREE_CHILDREN, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, and vect_create_new_slp_node().
Referenced by vect_build_slp_instance().
◆ vect_build_slp_tree()
|
static |
References dump_enabled_p(), dump_printf_loc(), gcc_assert, hash_map< KeyId, Value, Traits >::get(), i, MSG_NOTE, NULL, hash_map< KeyId, Value, Traits >::put(), SLP_TREE_DEF_TYPE, SLP_TREE_REF_COUNT, SLP_TREE_SCALAR_STMTS, tree_size(), vect_build_slp_tree_2(), vect_internal_def, vect_location, vect_uninitialized_def, vect_update_max_nunits(), and vNULL.
Referenced by optimize_load_redistribution_1(), vect_build_slp_instance(), vect_build_slp_tree_2(), and vect_lower_load_permutations().
◆ vect_build_slp_tree_1()
|
static |
Verify if the scalar stmts STMTS are isomorphic, require data permutation or are of unsupported types of operation. Return true if they are, otherwise return false and indicate in *MATCHES which stmts are not isomorphic to the first one. If MATCHES[0] is false then this indicates the comparison could not be carried out or the stmts will never be vectorized by SLP. Note COND_EXPR is possibly isomorphic to another one after swapping its operands. Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to the first stmt by swapping the two operands of comparison; set SWAP[i] to 2 if stmt I is isormorphic to the first stmt by inverting the code of comparison. Take A1 >= B1 ? X1 : Y1 as an exmple, it can be swapped to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1.
References as_a(), as_internal_fn(), cfun, compatible_calls_p(), DECL_P, directly_supported_p(), DR_GROUP_FIRST_ELEMENT, DR_IS_READ, DR_IS_WRITE, dump_enabled_p(), dump_printf_loc(), dyn_cast(), FOR_EACH_VEC_ELT, gcc_assert, gimple_arg(), gimple_assign_rhs1(), gimple_assign_rhs2(), gimple_assign_rhs_code(), gimple_bb(), gimple_call_arg(), gimple_call_chain(), gimple_call_combined_fn(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_noreturn_p(), gimple_call_tail_p(), gimple_could_trap_p(), gimple_get_lhs(), gimple_has_volatile_ops(), HONOR_NANS(), i, internal_fn_p(), internal_store_fn_p(), invert_tree_comparison(), is_a(), code_helper::is_tree_code(), MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, NULL_TREE, operand_equal_p(), optab_scalar, optab_vector, REDUC_GROUP_FIRST_ELEMENT, REFERENCE_CLASS_P, stmt_can_throw_internal(), STMT_VINFO_DATA_REF, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_REDUC_IDX, STMT_VINFO_VECTORIZABLE, swap_tree_comparison(), tcc_binary, tcc_comparison, tcc_expression, tcc_reference, tcc_unary, TREE_CODE, TREE_CODE_CLASS, TREE_OPERAND, TREE_TYPE, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), vect_get_vector_types_for_stmt(), vect_location, vect_record_max_nunits(), VECTOR_TYPE_P, and vectorizable_internal_fn_p().
Referenced by vect_build_slp_tree_2().
◆ vect_build_slp_tree_2()
|
static |
Recursively build an SLP tree starting from NODE. Fail (and return a value not equal to zero) if def-stmts are not isomorphic, require data permutation or are of unsupported types of operation. Otherwise, return 0. The value returned is the depth in the SLP tree where a mismatch was found.
References _slp_oprnd_info::any_pattern, as_a(), associative_tree_code(), bit_field_offset(), bit_field_size(), build_zero_cst(), can_duplicate_and_interleave_p(), commutative_tree_code(), _slp_oprnd_info::def_stmts, DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, DR_IS_READ, dt_sort_cmp(), dump_enabled_p(), dump_printf(), dump_printf_loc(), dyn_cast(), _slp_oprnd_info::first_dt, FLOAT_TYPE_P, FOR_EACH_VEC_ELT, gcc_assert, GET_MODE_SIZE(), hash_map< KeyId, Value, Traits >::get_or_insert(), get_tree_code_name(), get_vectype_for_scalar_type(), gimple_assign_rhs1(), gimple_assign_rhs_code(), gimple_assign_single_p(), gimple_bb(), gimple_call_internal_p(), gimple_num_args(), gimple_vuse(), loop::header, i, loop::inner, INTEGRAL_TYPE_P, is_a(), poly_int< N, C >::is_constant(), is_gimple_assign(), is_pattern_stmt_p(), known_eq, known_ge, chain::length, vec_info::lookup_def(), vec_info::lookup_stmt(), loop_latch_edge(), loop_preheader_edge(), LOOP_VINFO_LOOP, map, _slp_tree::max_nunits, MSG_MISSED_OPTIMIZATION, MSG_NOTE, nested_in_vect_loop_p(), NULL, NULL_TREE, operand_equal_p(), _slp_oprnd_info::ops, PHI_RESULT, si, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, STMT_VINFO_DATA_REF, STMT_VINFO_DEF_TYPE, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_REDUC_IDX, STMT_VINFO_SLP_VECT_ONLY, STMT_VINFO_STMT, STMT_VINFO_STRIDED_P, STMT_VINFO_VECTORIZABLE, TREE_CODE, TREE_OPERAND, tree_size(), tree_to_poly_uint64(), TREE_TYPE, TYPE_OVERFLOW_WRAPS, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), vect_build_slp_tree(), vect_build_slp_tree_1(), vect_constant_def, vect_create_new_slp_node(), vect_create_oprnd_info(), vect_double_reduction_def, vect_external_def, vect_first_order_recurrence, vect_free_oprnd_info(), vect_free_slp_tree(), vect_get_and_check_slp_defs(), vect_get_operand_map(), vect_get_place_in_interleaving_chain(), vect_induction_def, vect_internal_def, vect_location, vect_nested_cycle, vect_print_slp_tree(), vect_record_max_nunits(), vect_reduction_def, vect_slp_build_two_operator_nodes(), vect_slp_can_convert_to_external(), vect_slp_linearize_chain(), vect_slp_tree_uniform_p(), vect_uninitialized_def, vec_info::vector_mode, VECTOR_TYPE_P, vNULL, and worklist.
Referenced by vect_build_slp_tree().
◆ vect_contains_pattern_stmt_p()
|
static |
Return true if STMTS contains a pattern statement.
References FOR_EACH_VEC_ELT, i, and is_pattern_stmt_p().
Referenced by vect_slp_convert_to_external().
◆ vect_create_constant_vectors()
For constant and loop invariant defs in OP_NODE this function creates vector defs that will be used in the vectorized stmts and stores them to SLP_TREE_VEC_DEFS of OP_NODE.
References build1(), build_all_ones_cst(), build_int_cst(), build_zero_cst(), CONSTANT_CLASS_P, duplicate_and_interleave(), find_fallthru_edge(), fold_unary, gcc_assert, gcc_unreachable, get_later_stmt(), gimple_bb(), gimple_build_assign(), gimple_build_vector(), gimple_build_vector_from_val(), gimple_seq_add_stmt(), gimple_seq_empty_p(), gsi_after_labels(), GSI_CONTINUE_LINKING, gsi_for_stmt(), gsi_insert_seq_after(), gsi_insert_seq_before(), gsi_insert_seq_on_edge_immediate(), i, vec_info::insert_seq_on_entry(), integer_onep(), integer_zerop(), INTEGRAL_TYPE_P, is_a(), known_eq, vec_info::lookup_def(), make_ssa_name(), tree_vector_builder::new_vector(), NULL, NULL_TREE, operand_equal_p(), _slp_tree::ops, SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, SSA_NAME_IS_DEFAULT_DEF, stmt_ends_bb_p(), basic_block_def::succs, TREE_CODE, TREE_TYPE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), and VECTOR_BOOLEAN_TYPE_P.
Referenced by vect_schedule_slp_node().
◆ vect_create_new_slp_node() [1/5]
|
static |
Create an SLP node for SCALAR_STMTS.
References SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, and vect_internal_def.
◆ vect_create_new_slp_node() [2/5]
Create an SLP node for OPS.
References SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_SCALAR_OPS, and vect_external_def.
◆ vect_create_new_slp_node() [3/5]
Create an SLP node for SCALAR_STMTS.
References SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, vect_internal_def, and vNULL.
Referenced by vect_optimize_slp_pass::get_result_with_layout(), vect_build_combine_node(), vect_build_slp_instance(), vect_build_slp_store_interleaving(), vect_build_slp_tree_2(), vect_build_swap_evenodd_node(), vect_create_new_slp_node(), vect_create_new_slp_node(), and vect_lower_load_permutations().
◆ vect_create_new_slp_node() [4/5]
|
static |
Create an SLP node for SCALAR_STMTS.
References vect_create_new_slp_node().
◆ vect_create_new_slp_node() [5/5]
Create an SLP node for OPS.
References vect_create_new_slp_node().
◆ vect_create_oprnd_info()
|
static |
Allocate operands info for NOPS operands, and GROUP_SIZE def-stmts for each operand.
References _slp_oprnd_info::any_pattern, _slp_oprnd_info::def_stmts, _slp_oprnd_info::first_dt, _slp_oprnd_info::first_gs_p, _slp_oprnd_info::first_op_type, i, NULL_TREE, _slp_oprnd_info::ops, and vect_uninitialized_def.
Referenced by vect_build_slp_tree_2().
◆ vect_cse_slp_nodes()
|
static |
Apply CSE to NODE and its children using BST_MAP.
References dump_enabled_p(), dump_printf_loc(), hash_map< KeyId, Value, Traits >::get(), is_empty(), MSG_NOTE, hash_map< KeyId, Value, Traits >::put(), _slp_tree::refcnt, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, vect_cse_slp_nodes(), vect_free_slp_tree(), vect_internal_def, and vect_location.
Referenced by vect_cse_slp_nodes(), and vect_optimize_slp().
◆ vect_def_types_match()
|
static |
Return true if DTA and DTB match.
References vect_constant_def, and vect_external_def.
Referenced by vect_get_and_check_slp_defs().
◆ vect_detect_hybrid_slp() [1/2]
| void vect_detect_hybrid_slp | ( | loop_vec_info | loop_vinfo | ) |
Find stmts that must be both vectorized and SLPed.
References DUMP_VECT_SCOPE, gsi_end_p(), gsi_last_bb(), gsi_next(), gsi_prev(), gsi_start(), gsi_start_phis(), gsi_stmt(), i, is_gimple_debug(), vec_info::lookup_stmt(), vdhs_data::loop_vinfo, LOOP_VINFO_BBS, LOOP_VINFO_LOOP, maybe_push_to_hybrid_worklist(), gather_scatter_info::offset, STMT_SLP_TYPE, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_IN_PATTERN_P, STMT_VINFO_PATTERN_DEF_SEQ, STMT_VINFO_RELATED_STMT, STMT_VINFO_RELEVANT, vect_check_gather_scatter(), vect_detect_hybrid_slp(), walk_gimple_op(), vdhs_data::worklist, and worklist.
◆ vect_detect_hybrid_slp() [2/2]
Walker for walk_gimple_op.
References dump_enabled_p(), dump_printf_loc(), hybrid, vec_info::lookup_def(), vdhs_data::loop_vinfo, MSG_NOTE, NULL_TREE, PURE_SLP_STMT, STMT_SLP_TYPE, vect_location, vect_stmt_to_vectorize(), and vdhs_data::worklist.
Referenced by vect_analyze_loop_2(), and vect_detect_hybrid_slp().
◆ vect_find_first_scalar_stmt_in_slp()
| stmt_vec_info vect_find_first_scalar_stmt_in_slp | ( | slp_tree | node | ) |
Find the first stmt in NODE.
References get_later_stmt(), i, NULL, SLP_TREE_SCALAR_STMTS, and vect_orig_stmt().
Referenced by vect_schedule_slp_node(), vect_slp_analyze_load_dependences(), and vectorizable_load().
◆ vect_find_last_scalar_stmt_in_slp()
| stmt_vec_info vect_find_last_scalar_stmt_in_slp | ( | slp_tree | node | ) |
Find the last store in SLP INSTANCE.
References get_later_stmt(), i, last, NULL, SLP_TREE_SCALAR_STMTS, and vect_orig_stmt().
Referenced by vect_bb_slp_mark_live_stmts(), vect_schedule_slp_node(), vect_slp_analyze_instance_dependence(), and vect_slp_analyze_store_dependences().
◆ vect_fixup_store_groups_with_patterns()
|
static |
Walk the grouped store chains and replace entries with their pattern variant if any.
References DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, FOR_EACH_VEC_ELT, vec_info::grouped_stores, i, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_IN_PATTERN_P, and STMT_VINFO_RELATED_STMT.
Referenced by vect_slp_analyze_bb_1().
◆ vect_free_oprnd_info()
|
static |
Free operands info.
References _slp_oprnd_info::def_stmts, FOR_EACH_VEC_ELT, i, and _slp_oprnd_info::ops.
Referenced by vect_build_slp_tree_2().
◆ vect_free_slp_instance()
| void vect_free_slp_instance | ( | slp_instance | instance | ) |
Free the memory allocated for the SLP instance.
References free(), SLP_INSTANCE_LOADS, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, and vect_free_slp_tree().
Referenced by vect_analyze_loop_2(), vect_analyze_slp(), vect_slp_analyze_bb_1(), vect_slp_analyze_operations(), vect_transform_loop(), and vec_info::~vec_info().
◆ vect_free_slp_tree()
| void vect_free_slp_tree | ( | slp_tree | node | ) |
Recursively free the memory allocated for the SLP tree rooted at NODE.
References FOR_EACH_VEC_ELT, i, SLP_TREE_CHILDREN, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, STMT_SLP_TYPE, STMT_VINFO_IN_PATTERN_P, STMT_VINFO_SLP_VECT_ONLY_PATTERN, vect_free_slp_tree(), and vect_orig_stmt().
Referenced by addsub_pattern::build(), complex_add_pattern::build(), complex_fms_pattern::build(), complex_mul_pattern::build(), vect_optimize_slp_pass::decide_masked_load_lanes(), vect_optimize_slp_pass::materialize(), optimize_load_redistribution(), optimize_load_redistribution_1(), release_scalar_stmts_to_slp_tree_map(), vect_build_slp_instance(), vect_build_slp_tree_2(), vect_cse_slp_nodes(), vect_free_slp_instance(), and vect_free_slp_tree().
◆ vect_gather_slp_loads() [1/2]
|
static |
Gather loads in the SLP graph NODE and populate the INST loads array.
References DR_IS_READ, FOR_EACH_VEC_ELT, i, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_REPRESENTATIVE, STMT_VINFO_DATA_REF, vect_gather_slp_loads(), vect_internal_def, and visited.
Referenced by vect_analyze_loop_2(), vect_analyze_slp(), vect_gather_slp_loads(), vect_gather_slp_loads(), vect_lower_load_permutations(), and vect_slp_analyze_bb_1().
◆ vect_gather_slp_loads() [2/2]
| void vect_gather_slp_loads | ( | vec_info * | vinfo | ) |
Gather loads reachable from the individual SLP graph entries.
References FOR_EACH_VEC_ELT, i, SLP_INSTANCE_LOADS, SLP_INSTANCE_TREE, vec_info::slp_instances, vect_gather_slp_loads(), and visited.
◆ vect_get_and_check_slp_defs()
|
static |
Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that they are of a valid type and that they match the defs of the first stmt of the SLP group (stored in OPRNDS_INFO). This function tries to match stmts by swapping operands of STMTS[STMT_NUM] when possible. Non-zero SWAP indicates swap is required for cond_expr stmts. Specifically, SWAP is 1 if STMT is cond and operands of comparison need to be swapped; SWAP is 2 if STMT is cond and code of comparison needs to be inverted. If there was a fatal error return -1; if the error could be corrected by swapping operands of father node of this one, return 1; if everything is ok return 0.
References _slp_oprnd_info::any_pattern, as_a(), gather_scatter_info::base, vec_info::bbs, CDI_DOMINATORS, commutative_op(), commutative_tree_code(), _slp_oprnd_info::def_stmts, dominated_by_p(), dump_enabled_p(), dump_printf_loc(), dyn_cast(), first_commutative_argument(), _slp_oprnd_info::first_dt, _slp_oprnd_info::first_gs_info, _slp_oprnd_info::first_gs_p, _slp_oprnd_info::first_op_type, gcc_assert, gimple_arg(), gimple_assign_rhs_code(), gimple_bb(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_get_lhs(), gimple_num_args(), gimple_phi_arg_edge(), i, is_a(), is_pattern_stmt_p(), map, MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, NULL_TREE, gather_scatter_info::offset, operand_equal_p(), _slp_oprnd_info::ops, REDUC_GROUP_FIRST_ELEMENT, gather_scatter_info::scale, SSA_NAME_DEF_STMT, SSA_NAME_IS_DEFAULT_DEF, STMT_VINFO_DATA_REF, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_REDUC_IDX, STMT_VINFO_RELATED_STMT, TREE_CODE, TREE_OPERAND, TREE_TYPE, type(), types_compatible_p(), vect_check_gather_scatter(), vect_constant_def, vect_def_types_match(), vect_double_reduction_def, vect_external_def, vect_first_order_recurrence, vect_get_operand_map(), vect_induction_def, vect_internal_def, vect_is_simple_use(), vect_location, vect_nested_cycle, vect_orig_stmt(), vect_reduction_def, and vect_uninitialized_def.
Referenced by vect_build_slp_tree_2().
◆ vect_get_operand_map()
|
static |
For most SLP statements, there is a one-to-one mapping between gimple arguments and child nodes. If that is not true for STMT, return an array that contains: - the number of child nodes, followed by - for each child node, the index of the argument associated with that node. The special index -1 is the first operand of an embedded comparison and the special index -2 is the second operand of an embedded comparison. The special indes -3 is the offset of a gather as analyzed by vect_check_gather_scatter. SWAP is as for vect_get_and_check_slp_defs.
References arg0_map, arg1_arg3_arg4_map, arg1_arg3_map, arg1_arg4_arg5_map, arg1_map, arg2_arg3_map, arg3_arg2_map, COMPARISON_CLASS_P, dyn_cast(), gcc_assert, gcc_unreachable, gimple_assign_lhs(), gimple_assign_rhs1(), gimple_assign_rhs_code(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_num_args(), mask_call_maps, no_arg_map, off_arg2_arg3_map, off_arg3_arg2_map, off_map, off_op0_map, op1_op0_map, tcc_comparison, TREE_CODE, and TREE_CODE_CLASS.
Referenced by compatible_calls_p(), vect_build_slp_tree_2(), vect_get_and_check_slp_defs(), and vect_slp_child_index_for_operand().
◆ vect_get_place_in_interleaving_chain()
| int vect_get_place_in_interleaving_chain | ( | stmt_vec_info | stmt_info, |
| stmt_vec_info | first_stmt_info ) |
Find the place of the data-ref in STMT_INFO in the interleaving chain that starts from FIRST_STMT_INFO. Return -1 if the data-ref is not a part of the chain.
References DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, and DR_GROUP_NEXT_ELEMENT.
Referenced by vect_build_slp_tree_2(), and vectorizable_load().
◆ vect_get_slp_defs() [1/2]
Get the vectorized definitions of SLP_NODE in *VEC_DEFS.
References SLP_TREE_NUMBER_OF_VEC_STMTS, and SLP_TREE_VEC_DEFS.
Referenced by vect_get_gather_scatter_ops(), vect_get_slp_defs(), vect_get_vec_defs(), vect_transform_cycle_phi(), vectorizable_call(), vectorizable_load(), vectorizable_phi(), vectorizable_simd_clone_call(), vectorizable_store(), vectorize_fold_left_reduction(), and vectorize_slp_instance_root_stmt().
◆ vect_get_slp_defs() [2/2]
| void vect_get_slp_defs | ( | vec_info * | , |
| slp_tree | slp_node, | ||
| vec< vec< tree > > * | vec_oprnds, | ||
| unsigned | n ) |
Get N vectorized definitions for SLP_NODE.
References i, SLP_TREE_CHILDREN, vect_get_slp_defs(), and vNULL.
◆ vect_get_slp_vect_def()
Get the Ith vectorized definition from SLP_NODE.
References i, and SLP_TREE_VEC_DEFS.
Referenced by vect_create_epilog_for_reduction(), vect_schedule_scc(), vectorizable_induction(), and vectorizable_slp_permutation_1().
◆ vect_is_slp_load_node()
Helper that checks to see if a node is a load node.
References DR_IS_READ, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_REPRESENTATIVE, STMT_VINFO_DATA_REF, STMT_VINFO_GROUPED_ACCESS, and vect_internal_def.
Referenced by optimize_load_redistribution_1().
◆ vect_lower_load_permutations() [1/2]
|
static |
Transform SLP loads in the SLP graph created by SLP discovery to group loads from the same group and lower load permutations that are unlikely to be supported into a series of permutes. In the degenerate case of having only single-lane SLP instances this should result in a series of permute nodes emulating an interleaving scheme.
References DR_GROUP_FIRST_ELEMENT, i, make_array_slice(), loop::next, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_SCALAR_STMTS, STMT_VINFO_GROUPED_ACCESS, vect_gather_slp_loads(), vect_lower_load_permutations(), visited, and vllp_cmp().
◆ vect_lower_load_permutations() [2/2]
|
static |
Process the set of LOADS that are all from the same dataref group.
References bitmap_set_bit, ceil_log2(), compare_step_with_zero(), ctz_hwi(), DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, exact_log2(), EXECUTE_IF_SET_IN_BITMAP, gcc_assert, i, is_empty(), _slp_tree::ldst_lanes, NULL, pow2p_hwi(), array_slice< T >::size(), SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, STMT_VINFO_STRIDED_P, tree_size(), vect_build_slp_tree(), vect_create_new_slp_node(), and vect_load_lanes_supported().
Referenced by vect_analyze_slp(), and vect_lower_load_permutations().
◆ vect_make_slp_decision()
| bool vect_make_slp_decision | ( | loop_vec_info | loop_vinfo | ) |
For each possible SLP instance decide whether to SLP it and calculate overall unrolling factor needed to SLP the loop. Return TRUE if decided to SLP at least one instance.
References dump_dec(), dump_enabled_p(), dump_printf(), dump_printf_loc(), DUMP_VECT_SCOPE, FOR_EACH_VEC_ELT, i, LOOP_VINFO_SLP_INSTANCES, LOOP_VINFO_SLP_UNROLLING_FACTOR, MSG_NOTE, SLP_INSTANCE_TREE, vect_location, vect_mark_slp_stmts(), vect_update_slp_vf_for_node(), and visited.
Referenced by vect_analyze_loop_2().
◆ vect_mark_slp_stmts() [1/2]
References vect_mark_slp_stmts(), and visited.
◆ vect_mark_slp_stmts() [2/2]
|
static |
Mark the tree rooted at NODE with PURE_SLP.
References dyn_cast(), FOR_EACH_VEC_ELT, gimple_call_arg(), gimple_call_internal_fn(), gimple_call_internal_p(), i, internal_fn_mask_index(), _slp_tree::ldst_lanes, vec_info::lookup_def(), pure_slp, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, STMT_SLP_TYPE, TREE_CODE, vect_internal_def, vect_mark_slp_stmts(), vect_stmt_to_vectorize(), and visited.
Referenced by vect_make_slp_decision(), vect_mark_slp_stmts(), vect_mark_slp_stmts(), and vect_slp_analyze_bb_1().
◆ vect_mark_slp_stmts_relevant() [1/2]
|
static |
References vect_mark_slp_stmts_relevant(), and visited.
◆ vect_mark_slp_stmts_relevant() [2/2]
Mark the statements of the tree rooted at NODE as relevant (vect_used).
References FOR_EACH_VEC_ELT, gcc_assert, i, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, STMT_VINFO_RELEVANT, vect_internal_def, vect_mark_slp_stmts_relevant(), vect_used_in_scope, and visited.
Referenced by vect_mark_slp_stmts_relevant(), vect_mark_slp_stmts_relevant(), and vect_slp_analyze_bb_1().
◆ vect_match_slp_patterns()
|
static |
Applies pattern matching to the given SLP tree rooted in REF_NODE using vec_info VINFO. The modified tree is returned. Patterns are tried in order and multiple patterns may match.
References dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, MSG_NOTE, SLP_INSTANCE_TREE, vect_location, vect_match_slp_patterns_2(), and visited.
Referenced by vect_analyze_slp().
◆ vect_match_slp_patterns_2()
|
static |
Helper function of vect_match_slp_patterns. Attempts to match patterns against the slp tree rooted in REF_NODE using VINFO. Patterns are matched in post-order traversal. If matching is successful the value in REF_NODE is updated and returned, if not then it is returned unchanged.
References vect_pattern::build(), FOR_EACH_VEC_ELT, i, num__slp_patterns, slp_patterns, SLP_TREE_CHILDREN, vect_match_slp_patterns_2(), and visited.
Referenced by vect_match_slp_patterns(), and vect_match_slp_patterns_2().
◆ vect_optimize_slp()
| void vect_optimize_slp | ( | vec_info * | vinfo | ) |
Optimize the SLP graph of VINFO.
References release_scalar_stmts_to_slp_tree_map(), vect_optimize_slp_pass::run(), SLP_INSTANCE_TREE, vec_info::slp_instances, and vect_cse_slp_nodes().
Referenced by vect_analyze_loop_2(), and vect_slp_analyze_bb_1().
◆ vect_print_slp_graph() [1/2]
|
static |
References vect_print_slp_graph(), and visited.
◆ vect_print_slp_graph() [2/2]
|
static |
Dump a slp tree NODE using flags specified in DUMP_KIND.
References FOR_EACH_VEC_ELT, i, SLP_TREE_CHILDREN, vect_print_slp_graph(), vect_print_slp_tree(), and visited.
Referenced by debug(), vect_analyze_slp(), vect_bb_vectorization_profitable_p(), vect_build_slp_instance(), vect_print_slp_graph(), vect_print_slp_graph(), and vect_schedule_slp().
◆ vect_print_slp_tree()
|
static |
Dump a single SLP tree NODE.
References dump_printf(), dump_printf_loc(), estimated_poly_value(), FOR_EACH_VEC_ELT, dump_location_t::get_impl_location(), dump_location_t::get_user_location(), HOST_WIDE_INT_PRINT_UNSIGNED, i, is_empty(), _slp_tree::ldst_lanes, _slp_tree::max_nunits, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, STMT_VINFO_LIVE_P, vect_constant_def, vect_external_def, and vect_internal_def.
Referenced by debug(), dot_slp_tree(), vect_build_slp_tree_2(), and vect_print_slp_graph().
◆ vect_prologue_cost_for_slp()
|
static |
Compute the prologue cost for invariant or constant operands represented by NODE.
References hash_set< KeyId, Lazy, Traits >::add(), vect_scalar_ops_slice::all_same_p(), i, is_empty(), record_stmt_cost(), scalar_to_vec, SLP_TREE_DEF_TYPE, SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_SCALAR_OPS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, TYPE_VECTOR_SUBPARTS(), vec_construct, vect_constant_def, vect_prologue, and vector_load.
Referenced by vect_slp_analyze_node_operations().
◆ vect_record_max_nunits()
|
static |
A subroutine of vect_build_slp_tree for checking VECTYPE, which is the caller's attempt to find the vector type in STMT_INFO with the narrowest element type. Return true if VECTYPE is nonnull and if it is valid for STMT_INFO. When returning true, update MAX_NUNITS to reflect the number of units in VECTYPE. GROUP_SIZE and MAX_NUNITS are as for vect_build_slp_tree.
References dump_enabled_p(), dump_printf_loc(), is_a(), MSG_MISSED_OPTIMIZATION, TYPE_VECTOR_SUBPARTS(), vect_location, and vect_update_max_nunits().
Referenced by vect_build_slp_tree_1(), and vect_build_slp_tree_2().
◆ vect_remove_slp_scalar_calls() [1/2]
References vect_remove_slp_scalar_calls(), and visited.
◆ vect_remove_slp_scalar_calls() [2/2]
|
static |
Replace scalar calls from SLP node NODE with setting of their lhs to zero. For loop vectorization this is done in vectorizable_call, but for SLP it needs to be deferred until end of vect_schedule_slp, because multiple SLP instances may refer to the same scalar stmt.
References build_zero_cst(), dyn_cast(), FOR_EACH_VEC_ELT, gimple_bb(), gimple_build_assign(), gimple_build_nop(), gimple_call_lhs(), gsi_for_stmt(), i, is_pattern_stmt_p(), NULL, PURE_SLP_STMT, vec_info::replace_stmt(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, SSA_NAME_DEF_STMT, TREE_TYPE, unlink_stmt_vdef(), vect_internal_def, vect_remove_slp_scalar_calls(), and visited.
Referenced by vect_remove_slp_scalar_calls(), vect_remove_slp_scalar_calls(), and vect_schedule_slp().
◆ vect_schedule_scc()
|
static |
Schedule the SLP INSTANCE doing a DFS walk and collecting SCCs.
References add_phi_arg(), as_a(), slp_scc_info::dfs, FOR_EACH_EDGE, FOR_EACH_VEC_ELT, gcc_assert, gimple_assign_rhs1(), gimple_assign_set_rhs1(), gimple_assign_set_rhs2(), gimple_bb(), gimple_phi_arg_location(), i, is_a(), slp_scc_info::lowlink, MIN, NULL, slp_scc_info::on_stack, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_REPRESENTATIVE, SLP_TREE_VEC_DEFS, SSA_NAME_DEF_STMT, STMT_VINFO_DEF_TYPE, todo, update_stmt(), vect_first_order_recurrence, vect_get_slp_vect_def(), vect_internal_def, vect_schedule_scc(), and vect_schedule_slp_node().
Referenced by vect_schedule_scc(), and vect_schedule_slp().
◆ vect_schedule_slp()
| void vect_schedule_slp | ( | vec_info * | vinfo, |
| const vec< slp_instance > & | slp_instances ) |
Generate vector code for SLP_INSTANCES in the loop/basic block.
References DR_IS_WRITE, dump_enabled_p(), dump_printf_loc(), FOR_EACH_VEC_ELT, i, is_a(), is_empty(), MSG_NOTE, NULL, vec_info::remove_stmt(), SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, STMT_VINFO_DATA_REF, vect_location, vect_orig_stmt(), vect_print_slp_graph(), vect_remove_slp_scalar_calls(), vect_schedule_scc(), and vectorize_slp_instance_root_stmt().
Referenced by vect_slp_region(), and vect_transform_loop().
◆ vect_schedule_slp_node()
|
static |
Vectorize SLP NODE.
References as_a(), vec_info::bbs, CDI_DOMINATORS, cycle_phi_info_type, dominated_by_p(), DR_IS_READ, dump_enabled_p(), dump_printf_loc(), dyn_cast(), FOR_EACH_VEC_ELT, gcc_assert, gimple_bb(), gimple_could_trap_p(), gsi_after_labels(), gsi_end_p(), gsi_for_stmt(), gsi_next(), gsi_none(), header, i, induc_vec_info_type, is_a(), is_ctrl_altering_stmt(), is_empty(), vec_info::lookup_def(), LOOP_VINFO_FULLY_MASKED_P, LOOP_VINFO_FULLY_WITH_LENGTH_P, LOOP_VINFO_LOOP, MSG_NOTE, NULL, phi_info_type, si, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, SSA_NAME_DEF_STMT, SSA_NAME_IS_DEFAULT_DEF, STMT_VINFO_DATA_REF, STMT_VINFO_LIVE_P, STMT_VINFO_TYPE, TREE_CODE, vect_constant_def, vect_create_constant_vectors(), vect_external_def, vect_find_first_scalar_stmt_in_slp(), vect_find_last_scalar_stmt_in_slp(), vect_internal_def, vect_location, vect_stmt_dominates_stmt_p(), vect_transform_stmt(), vectorizable_live_operation(), and vectorizable_slp_permutation().
Referenced by vect_schedule_scc().
◆ vect_slp_analyze_bb_1()
|
static |
Check if the region described by BB_VINFO can be vectorized, returning true if so. When returning false, set FATAL to true if the same failure would prevent vectorization at other vector sizes, false if it is still worth trying other sizes. N_STMTS is the number of statements in the region.
References BB_VINFO_SLP_INSTANCES, dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, fatal(), FOR_EACH_VEC_ELT, vec_info::grouped_stores, i, MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, pure_slp, _bb_vec_info::roots, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, SLP_TREE_SCALAR_STMTS, STMT_SLP_TYPE, vect_analyze_data_ref_accesses(), vect_analyze_data_refs(), vect_analyze_slp(), vect_bb_partition_graph(), vect_fixup_store_groups_with_patterns(), vect_free_slp_instance(), vect_gather_slp_loads(), vect_location, vect_mark_slp_stmts(), vect_mark_slp_stmts_relevant(), vect_optimize_slp(), vect_pattern_recog(), vect_record_base_alignments(), vect_slp_analyze_instance_alignment(), vect_slp_analyze_instance_dependence(), vect_slp_analyze_operations(), and vect_slp_check_for_roots().
Referenced by vect_slp_region().
◆ vect_slp_analyze_node_operations()
|
static |
Analyze statements contained in SLP tree NODE after recursively analyzing the subtree. NODE_INSTANCE contains NODE and VINFO contains INSTANCE. Return true if the operations are supported.
References hash_set< KeyId, Lazy, Traits >::add(), dump_enabled_p(), dump_printf_loc(), FOR_EACH_VEC_ELT, gcc_assert, i, is_empty(), load_vec_info_type, MSG_NOTE, hash_set< KeyId, Lazy, Traits >::remove(), shift_vec_info_type, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, SSA_NAME_IS_DEFAULT_DEF, SSA_NAME_VAR, STMT_VINFO_TYPE, TREE_CODE, VAR_P, vect_constant_def, vect_external_def, vect_get_num_copies(), vect_internal_def, vect_location, vect_prologue_cost_for_slp(), vect_slp_analyze_node_operations(), vect_slp_analyze_node_operations_1(), vect_slp_convert_to_external(), and vect_uninitialized_def.
Referenced by vect_slp_analyze_node_operations(), and vect_slp_analyze_operations().
◆ vect_slp_analyze_node_operations_1()
|
static |
Subroutine of vect_slp_analyze_node_operations. Handle the root of NODE, given then that child nodes have already been processed, and that their def types currently match their SLP node's def type.
References FOR_EACH_VEC_ELT, i, NULL, SLP_TREE_CODE, SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, STMT_VINFO_LIVE_P, vect_analyze_stmt(), vect_get_num_copies(), vectorizable_live_operation(), and vectorizable_slp_permutation().
Referenced by vect_slp_analyze_node_operations().
◆ vect_slp_analyze_operations()
Analyze statements in SLP instances of VINFO. Return true if the operations are supported.
References hash_set< KeyId, Lazy, Traits >::add(), add_stmt_costs(), hash_set< KeyId, Lazy, Traits >::contains(), dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, dyn_cast(), gimple_assign_rhs1(), i, is_a(), hash_set< KeyId, Lazy, Traits >::is_empty(), is_empty(), MSG_NOTE, NULL, slp_inst_kind_bb_reduc, slp_inst_kind_ctor, slp_inst_kind_gcond, SLP_INSTANCE_KIND, SLP_INSTANCE_ROOT_STMTS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, TREE_TYPE, useless_type_conversion_p(), vect_bb_slp_mark_live_stmts(), vect_free_slp_instance(), vect_internal_def, vect_location, vect_slp_analyze_node_operations(), vect_slp_prune_covered_roots(), vectorizable_bb_reduc_epilogue(), vectorizable_early_exit(), and visited.
Referenced by vect_analyze_loop_2(), and vect_slp_analyze_bb_1().
◆ vect_slp_bbs()
|
static |
Main entry for the BB vectorizer. Analyze and transform BBS, returns true if anything in the basic-block was vectorized.
References gimple_location(), gsi_after_labels(), gsi_end_p(), gsi_next(), gsi_stmt(), i, insns, is_gimple_debug(), NULL, UNKNOWN_LOCATION, vect_find_stmt_data_reference(), vect_location, vect_slp_region(), and vNULL.
Referenced by vect_slp_function(), and vect_slp_if_converted_bb().
◆ vect_slp_build_two_operator_nodes()
|
static |
Helper for building an associated SLP node chain.
References SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, SLP_TREE_VECTYPE, and vect_internal_def.
Referenced by vect_build_slp_tree_2().
◆ vect_slp_can_convert_to_external()
|
static |
Verify if we can externalize a set of internal defs.
References gimple_bb(), and NULL.
Referenced by vect_build_slp_tree_2(), and vect_slp_convert_to_external().
◆ vect_slp_check_for_roots()
|
static |
Find any vectorizable constructors and add them to the grouped_store array.
References as_a(), associative_tree_code(), vec_info::bbs, bitmap_bit_p, bitmap_clear(), bitmap_set_bit, CONSTRUCTOR_ELT, CONSTRUCTOR_ELTS, CONSTRUCTOR_NELTS, dt_sort_cmp(), dyn_cast(), FOR_EACH_CONSTRUCTOR_VALUE, gimple_assign_lhs(), gimple_assign_rhs1(), gimple_assign_rhs2(), gimple_assign_rhs3(), gimple_assign_rhs_code(), gimple_get_lhs(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_stmt(), has_single_use(), i, integer_zerop(), is_gimple_assign(), last, chain::length, vec_info::lookup_def(), vec_info::lookup_stmt(), vec_info::nbbs, needs_fold_left_reduction_p(), NULL, NULL_TREE, chain::op, reduction_fn_for_scalar_code(), _bb_vec_info::roots, single_imm_use(), slp_inst_kind_bb_reduc, slp_inst_kind_ctor, SSA_NAME_DEF_STMT, poly_int< N, C >::to_constant(), TREE_CODE, TREE_TYPE, TYPE_VECTOR_SUBPARTS(), uniform_vector_p(), useless_type_conversion_p(), vect_internal_def, vect_slp_is_lane_insert(), vect_slp_linearize_chain(), vect_stmt_to_vectorize(), VECTOR_TYPE_P, vld_cmp(), vNULL, and worklist.
Referenced by vect_slp_analyze_bb_1().
◆ vect_slp_child_index_for_operand()
Return the SLP node child index for operand OP of STMT.
References gcc_unreachable, i, and vect_get_operand_map().
Referenced by vect_check_store_rhs(), vectorizable_load(), vectorizable_simd_clone_call(), and vectorizable_store().
◆ vect_slp_convert_to_external()
|
static |
Try to build NODE from scalars, returning true on success. NODE_INSTANCE is the SLP instance that contains NODE.
References dump_enabled_p(), dump_printf_loc(), FOR_EACH_VEC_ELT, gimple_get_lhs(), i, is_a(), MSG_NOTE, NULL_TREE, SLP_INSTANCE_TREE, SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, SLP_TREE_VECTYPE, vect_contains_pattern_stmt_p(), vect_external_def, vect_location, vect_orig_stmt(), vect_slp_can_convert_to_external(), and VECTOR_BOOLEAN_TYPE_P.
Referenced by vect_slp_analyze_node_operations().
◆ vect_slp_fini()
| void vect_slp_fini | ( | void | ) |
References NULL, slp_first_node, and slp_tree_pool.
◆ vect_slp_function()
Main entry for the BB vectorizer. Analyze and transform BB, returns true if anything in the basic-block was vectorized.
References BASIC_BLOCK_FOR_FN, bitmap_set_bit, CDI_DOMINATORS, dominated_by_p(), loop::dont_vectorize, dump_enabled_p(), dump_printf_loc(), ECF_RETURNS_TWICE, ENTRY_BLOCK_PTR_FOR_FN, EXIT_BLOCK, first_stmt(), flow_loop_nested_p(), free(), gimple_call_flags(), gimple_get_lhs(), gsi_last_bb(), loop::header, i, basic_block_def::index, is_ctrl_altering_stmt(), last, basic_block_def::loop_father, MSG_MISSED_OPTIMIZATION, n_basic_blocks_for_fn, NULL, loop::num, r, rev_post_order_and_mark_dfs_back_seme(), safe_dyn_cast(), single_succ_edge(), vect_location, and vect_slp_bbs().
◆ vect_slp_gather_vectorized_scalar_stmts()
|
static |
Compute the set of scalar stmts participating in internal and external nodes.
References hash_set< KeyId, Lazy, Traits >::add(), FOR_EACH_VEC_ELT, i, vec_info::lookup_def(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_OPS, SLP_TREE_SCALAR_STMTS, vect_internal_def, vect_slp_gather_vectorized_scalar_stmts(), and visited.
Referenced by vect_bb_vectorization_profitable_p(), and vect_slp_gather_vectorized_scalar_stmts().
◆ vect_slp_if_converted_bb()
| bool vect_slp_if_converted_bb | ( | basic_block | bb, |
| loop_p | orig_loop ) |
Special entry for the BB vectorizer. Analyze and transform a single if-converted BB with ORIG_LOOPs body being the not if-converted representation. Returns true if anything in the basic-block was vectorized.
References vect_slp_bbs().
Referenced by try_vectorize_loop_1().
◆ vect_slp_init()
| void vect_slp_init | ( | void | ) |
In tree-vect-slp.cc.
References slp_tree_pool.
◆ vect_slp_is_lane_insert()
Return true if USE_STMT is a vector lane insert into VEC and set *THIS_LANE to the lane number that is set.
References dyn_cast(), gimple_assign_rhs1(), gimple_assign_rhs2(), gimple_assign_rhs3(), gimple_assign_rhs_code(), tree_to_poly_uint64(), TREE_TYPE, TYPE_SIZE, and useless_type_conversion_p().
Referenced by vect_slp_check_for_roots().
◆ vect_slp_linearize_chain()
|
static |
Linearize the associatable expression chain at START with the associatable operation CODE (where PLUS_EXPR also allows MINUS_EXPR), filling CHAIN with the result and using WORKLIST as intermediate storage. CODE_STMT and ALT_CODE_STMT are filled with the first stmt using CODE or MINUS_EXPR. *CHAIN_STMTS if not NULL is filled with all computation stmts, starting with START.
References as_a(), gcc_assert, gimple_assign_rhs_code(), gimple_get_lhs(), gimple_op(), is_gimple_assign(), is_pattern_stmt_p(), single_imm_use(), vect_internal_def, vect_is_simple_use(), and worklist.
Referenced by vect_build_slp_tree_2(), and vect_slp_check_for_roots().
◆ vect_slp_node_weight()
Return the execution frequency of NODE (so that a higher value indicates a "more important" node when optimizing for speed).
References cfun, basic_block_def::count, count, ENTRY_BLOCK_PTR_FOR_FN, gimple_bb(), SLP_TREE_REPRESENTATIVE, profile_count::to_sreal_scale(), and vect_orig_stmt().
Referenced by vect_optimize_slp_pass::start_choosing_layouts().
◆ vect_slp_permute()
|
static |
Apply (reverse) bijectite PERM to VEC.
References gcc_assert, and i.
Referenced by vect_optimize_slp_pass::change_layout_cost(), vect_optimize_slp_pass::change_vec_perm_layout(), vect_optimize_slp_pass::get_result_with_layout(), vect_optimize_slp_pass::internal_node_cost(), and vect_optimize_slp_pass::materialize().
◆ vect_slp_prefer_store_lanes_p()
|
static |
STMT_INFO is a store group of size GROUP_SIZE that we are considering vectorizing with VECTYPE that might be NULL. MASKED_P indicates whether the stores are masked. Return true if we could use IFN_STORE_LANES instead and if that appears to be the better approach.
References DR_REF, get_vectype_for_scalar_type(), STMT_VINFO_DATA_REF, TREE_TYPE, TYPE_VECTOR_SUBPARTS(), and vect_store_lanes_supported().
Referenced by vect_build_slp_instance().
◆ vect_slp_prune_covered_roots()
|
static |
Prune from ROOTS all stmts that are computed as part of lanes of NODE and recurse to children.
References FOR_EACH_VEC_ELT, i, hash_set< KeyId, Lazy, Traits >::remove(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, vect_internal_def, vect_orig_stmt(), vect_slp_prune_covered_roots(), and visited.
Referenced by vect_slp_analyze_operations(), and vect_slp_prune_covered_roots().
◆ vect_slp_region()
|
static |
Subroutine of vect_slp_bb. Try to vectorize the statements for all basic blocks in BBS, returning true on success. The region has N_STMTS statements and has the datarefs given by DATAREFS.
References BB_VINFO_DATAREFS, BB_VINFO_SLP_INSTANCES, vec_info::bbs, vec_info_shared::check_datarefs(), vec_info_shared::datarefs, dbg_cnt(), dump_enabled_p(), dump_printf_loc(), dyn_cast(), fatal(), gcc_assert, GET_MODE_INNER, GET_MODE_NAME, GET_MODE_SIZE(), gimple_assign_rhs_code(), gimple_seq_empty_p(), gimple_set_visited(), gimple_visited_p(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_stmt(), vec_info::insert_seq_on_entry(), vec_info::inv_pattern_def_seq, MSG_MISSED_OPTIMIZATION, MSG_NOTE, MSG_OPTIMIZED_LOCATIONS, vec_info::nbbs, NULL, related_vector_mode(), vec_info_shared::save_datarefs(), vec_info::shared, SLP_INSTANCE_TREE, SLP_TREE_VECTYPE, targetm, TYPE_MODE, unlimited_cost_model(), vect_bb_vectorization_profitable_p(), vect_chooses_same_modes_p(), VECT_COST_MODEL_UNLIMITED, vect_location, vect_schedule_slp(), vect_slp_analyze_bb_1(), vec_info::vector_mode, and VECTOR_MODE_P.
Referenced by vect_slp_bbs().
◆ vect_slp_tree_uniform_p()
Return true when all lanes in the external or constant NODE have the same value.
References FOR_EACH_VEC_ELT, gcc_assert, i, is_empty(), NULL_TREE, operand_equal_p(), SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_OPS, vect_constant_def, and vect_external_def.
Referenced by vect_optimize_slp_pass::get_result_with_layout(), and vect_build_slp_tree_2().
◆ vect_split_slp_store_group()
|
static |
Splits a group of stores, currently beginning at FIRST_VINFO, into two groups: one (still beginning at FIRST_VINFO) of size GROUP1_SIZE (also containing the first GROUP1_SIZE stmts, since stores are consecutive), the second containing the remainder. Return the first stmt in the second group.
References DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, dump_enabled_p(), dump_printf_loc(), gcc_assert, i, MSG_NOTE, and vect_location.
Referenced by vect_build_slp_instance().
◆ vect_transform_slp_perm_load()
| bool vect_transform_slp_perm_load | ( | vec_info * | vinfo, |
| slp_tree | node, | ||
| const vec< tree > & | dr_chain, | ||
| gimple_stmt_iterator * | gsi, | ||
| poly_uint64 | vf, | ||
| bool | analyze_only, | ||
| unsigned * | n_perms, | ||
| unsigned int * | n_loads, | ||
| bool | dce_chain ) |
Generate vector permute statements from a list of loads in DR_CHAIN. If ANALYZE_ONLY is TRUE, only check that it is possible to create valid permute statements for the SLP node NODE. Store the number of vector permute instructions in *N_PERMS and the number of vector load instructions in *N_LOADS. If DCE_CHAIN is true, remove all definitions that were not needed.
References dump_enabled_p(), SLP_TREE_LOAD_PERMUTATION, and vect_transform_slp_perm_load_1().
Referenced by get_group_load_store_type(), and vectorizable_load().
◆ vect_transform_slp_perm_load_1() [1/2]
|
static |
Referenced by vect_optimize_slp_pass::internal_node_cost(), and vect_transform_slp_perm_load().
◆ vect_transform_slp_perm_load_1() [2/2]
|
static |
A subroutine of vect_transform_slp_perm_load with two extra arguments: - PERM gives the permutation that the caller wants to use for NODE, which might be different from SLP_LOAD_PERMUTATION. - DUMP_P controls whether the function dumps information.
References as_a(), bitmap_bit_p, bitmap_clear(), bitmap_set_bit, can_vec_perm_const_p(), count, DR_GROUP_FIRST_ELEMENT, DR_GROUP_SIZE, dump_dec(), dump_printf(), dump_printf_loc(), gcc_assert, gimple_assign_lhs(), gimple_assign_rhs_code(), gimple_build_assign(), gsi_for_stmt(), gsi_remove(), i, poly_int< N, C >::is_constant(), is_gimple_assign(), make_ssa_name(), MSG_MISSED_OPTIMIZATION, NULL, NULL_TREE, release_defs(), single_ssa_tree_operand(), SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_SCALAR_STMTS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, SSA_NAME_DEF_STMT, SSA_OP_USE, STMT_VINFO_GROUPED_ACCESS, poly_int< N, C >::to_constant(), TYPE_MODE, TYPE_VECTOR_SUBPARTS(), vect_create_destination_var(), vect_finish_stmt_generation(), vect_gen_perm_mask_checked(), and vect_location.
◆ vect_update_slp_vf_for_node()
|
static |
For NODE update VF based on the number of lanes and the vector types used.
References calculate_unrolling_factor(), _slp_tree::ldst_lanes, _slp_tree::max_nunits, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_LANES, vect_internal_def, vect_update_slp_vf_for_node(), and visited.
Referenced by vect_make_slp_decision(), and vect_update_slp_vf_for_node().
◆ vectorizable_bb_reduc_epilogue()
|
static |
Determine whether we can vectorize the reduction epilogue for INSTANCE.
References as_a(), direct_internal_fn_supported_p(), dump_enabled_p(), dump_printf_loc(), floor_log2(), gimple_assign_lhs(), gimple_assign_rhs_code(), MSG_MISSED_OPTIMIZATION, OPTIMIZE_FOR_BOTH, record_stmt_cost(), reduction_fn_for_scalar_code(), scalar_stmt, SLP_INSTANCE_TREE, SLP_TREE_VECTYPE, TREE_TYPE, useless_type_conversion_p(), vec_perm, vec_to_scalar, vect_body, vect_location, vect_nunits_for_cost(), and vector_stmt.
Referenced by vect_slp_analyze_operations().
◆ vectorizable_slp_permutation()
|
static |
Vectorize the SLP permutations in NODE as specified
in SLP_TREE_LANE_PERMUTATION which is a vector of pairs of SLP
child number and lane number.
Interleaving of two two-lane two-child SLP subtrees (not supported):
[ { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } ]
A blend of two four-lane two-child SLP subtrees:
[ { 0, 0 }, { 1, 1 }, { 0, 2 }, { 1, 3 } ]
Highpart of a four-lane one-child SLP subtree (not supported):
[ { 0, 2 }, { 0, 3 } ]
Where currently only a subset is supported by code generating below.
References dump_enabled_p(), record_stmt_cost(), SLP_TREE_CHILDREN, SLP_TREE_LANE_PERMUTATION, SLP_TREE_VECTYPE, vec_perm, vect_body, and vectorizable_slp_permutation_1().
Referenced by vect_schedule_slp_node(), and vect_slp_analyze_node_operations_1().
◆ vectorizable_slp_permutation_1()
|
static |
Subroutine of vectorizable_slp_permutation. Check whether the target can perform permutation PERM on the (1 or 2) input nodes in CHILDREN. If GSI is nonnull, emit the permutation there. When GSI is null, the only purpose of NODE is to give properties of the result, such as the vector type and number of SLP lanes. The node does not need to be a VEC_PERM_EXPR. If the target supports the operation, return the number of individual VEC_PERM_EXPRs needed, otherwise return -1. Print information to the dump file if DUMP_P is true.
References can_vec_perm_const_p(), count, dump_dec(), dump_printf(), dump_printf_loc(), dyn_cast(), FOR_EACH_VEC_ELT, gcc_assert, i, poly_int< N, C >::is_constant(), is_empty(), known_eq, known_le, _slp_tree::ldst_lanes, LOOP_VINFO_VECT_FACTOR, MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL_TREE, _slp_tree::push_vec_def(), SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_SCALAR_OPS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, poly_int< N, C >::to_constant(), TREE_TYPE, TYPE_MODE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), ui, vect_add_slp_permutation(), vect_external_def, vect_gen_perm_mask_checked(), vect_get_slp_vect_def(), vect_internal_def, vect_location, and vect_maybe_update_slp_op_vectype().
Referenced by vect_optimize_slp_pass::change_layout_cost(), vect_optimize_slp_pass::get_result_with_layout(), vect_optimize_slp_pass::internal_node_cost(), vect_optimize_slp_pass::materialize(), and vectorizable_slp_permutation().
◆ vectorize_slp_instance_root_stmt()
| void vectorize_slp_instance_root_stmt | ( | vec_info * | vinfo, |
| slp_tree | node, | ||
| slp_instance | instance ) |
Vectorize the instance root.
References ANY_INTEGRAL_TYPE_P, as_combined_fn(), build1(), build_constructor(), CONSTRUCTOR_APPEND_ELT, FOR_EACH_VEC_ELT, gcc_assert, gcc_unreachable, gimple_assign_rhs1(), gimple_assign_rhs_code(), gimple_assign_set_rhs_from_tree(), gimple_build(), gimple_build_assign(), gimple_convert(), gimple_get_lhs(), gsi_for_stmt(), gsi_insert_seq_before(), gsi_replace(), GSI_SAME_STMT, gsi_stmt(), i, is_empty(), NULL, NULL_TREE, operation_can_overflow(), reduction_fn_for_scalar_code(), slp_inst_kind_bb_reduc, slp_inst_kind_ctor, slp_inst_kind_gcond, SLP_INSTANCE_REMAIN_DEFS, SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_VEC_DEFS, STMT_VINFO_STMT, TREE_TYPE, TYPE_OVERFLOW_UNDEFINED, unsigned_type_for(), update_stmt(), useless_type_conversion_p(), vec_alloc(), vect_get_slp_defs(), vect_orig_stmt(), vectorizable_early_exit(), and vNULL.
Referenced by vect_schedule_slp().
◆ vld_cmp()
|
static |
◆ vllp_cmp()
|
static |
qsort comparator ordering SLP load nodes.
References a, b, DR_GROUP_FIRST_ELEMENT, gcc_assert, gimple_uid(), i, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_SCALAR_STMTS, STMT_VINFO_GROUPED_ACCESS, and STMT_VINFO_STMT.
Referenced by vect_lower_load_permutations().
Variable Documentation
◆ arg0_map
|
static |
Referenced by vect_get_operand_map().
◆ arg1_arg3_arg4_map
|
static |
Referenced by vect_get_operand_map().
◆ arg1_arg3_map
|
static |
Referenced by vect_get_operand_map().
◆ arg1_arg4_arg5_map
|
static |
Referenced by vect_get_operand_map().
◆ arg1_map
|
static |
Referenced by vect_get_operand_map().
◆ arg2_arg3_map
|
static |
Referenced by vect_get_operand_map().
◆ arg3_arg2_map
|
static |
Referenced by vect_get_operand_map().
◆ cond_expr_maps
|
static |
◆ mask_call_maps
|
static |
Referenced by vect_get_operand_map().
◆ no_arg_map
|
static |
Referenced by vect_get_operand_map().
◆ off_arg2_arg3_map
|
static |
Referenced by vect_get_operand_map().
◆ off_arg3_arg2_map
|
static |
Referenced by vect_get_operand_map().
◆ off_map
|
static |
Referenced by vect_get_operand_map().
◆ off_op0_map
|
static |
Referenced by vect_get_operand_map().
◆ op1_op0_map
|
static |
Referenced by vect_get_operand_map().
◆ slp_first_node
|
static |
Referenced by _slp_tree::_slp_tree(), and vect_slp_fini().
◆ slp_tree_pool
|
static |
Referenced by _slp_tree::operator delete(), _slp_tree::operator new(), vect_slp_fini(), and vect_slp_init().
