GCC Middle and Back End API Reference
tree-vect-slp.cc File Reference
#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "target.h"
#include "rtl.h"
#include "tree.h"
#include "gimple.h"
#include "tree-pass.h"
#include "ssa.h"
#include "optabs-tree.h"
#include "insn-config.h"
#include "recog.h"
#include "fold-const.h"
#include "stor-layout.h"
#include "gimple-iterator.h"
#include "cfgloop.h"
#include "tree-vectorizer.h"
#include "langhooks.h"
#include "gimple-walk.h"
#include "dbgcnt.h"
#include "tree-vector-builder.h"
#include "vec-perm-indices.h"
#include "gimple-fold.h"
#include "internal-fn.h"
#include "dump-context.h"
#include "cfganal.h"
#include "tree-eh.h"
#include "tree-cfg.h"
#include "alloc-pool.h"
#include "sreal.h"
#include "predict.h"
Include dependency graph for tree-vect-slp.cc:

Data Structures

struct  _slp_oprnd_info
 
struct  bst_traits
 
struct  chain_op_t
 
struct  slpg_layout_cost
 
struct  slpg_vertex
 
struct  slpg_partition_info
 
struct  slpg_partition_layout_costs
 
class  vect_optimize_slp_pass
 
struct  vdhs_data
 
struct  slp_scc_info
 

Macros

#define INCLUDE_ALGORITHM
 
#define TEMPLATE   "{depth: %f, total: %f}"
 

Typedefs

typedef struct _slp_oprnd_infoslp_oprnd_info
 
typedef hash_map< vec< stmt_vec_info >, slp_tree, simple_hashmap_traits< bst_traits, slp_tree > > scalar_stmts_to_slp_tree_map_t
 

Functions

static bool vect_transform_slp_perm_load_1 (vec_info *, slp_tree, load_permutation_t &, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, bool, unsigned *, unsigned *=nullptr, bool=false)
 
static int vectorizable_slp_permutation_1 (vec_info *, gimple_stmt_iterator *, slp_tree, lane_permutation_t &, vec< slp_tree > &, bool)
 
static bool vectorizable_slp_permutation (vec_info *, gimple_stmt_iterator *, slp_tree, stmt_vector_for_cost *)
 
static void vect_print_slp_tree (dump_flags_t, dump_location_t, slp_tree)
 
void vect_slp_init (void)
 
void vect_slp_fini (void)
 
void vect_free_slp_tree (slp_tree node)
 
void vect_free_slp_instance (slp_instance instance)
 
slp_tree vect_create_new_slp_node (unsigned nops, tree_code code)
 
static slp_tree vect_create_new_slp_node (slp_tree node, vec< stmt_vec_info > scalar_stmts, unsigned nops)
 
static slp_tree vect_create_new_slp_node (vec< stmt_vec_info > scalar_stmts, unsigned nops)
 
static slp_tree vect_create_new_slp_node (slp_tree node, vec< tree > ops)
 
static slp_tree vect_create_new_slp_node (vec< tree > ops)
 
static vec< slp_oprnd_infovect_create_oprnd_info (int nops, int group_size)
 
static void vect_free_oprnd_info (vec< slp_oprnd_info > &oprnds_info)
 
static sreal vect_slp_node_weight (slp_tree node)
 
static bool vect_contains_pattern_stmt_p (vec< stmt_vec_info > stmts)
 
static bool vect_slp_tree_uniform_p (slp_tree node)
 
int vect_get_place_in_interleaving_chain (stmt_vec_info stmt_info, stmt_vec_info first_stmt_info)
 
bool can_duplicate_and_interleave_p (vec_info *vinfo, unsigned int count, tree elt_type, unsigned int *nvectors_out, tree *vector_type_out, tree *permutes)
 
static bool vect_def_types_match (enum vect_def_type dta, enum vect_def_type dtb)
 
static const int * vect_get_operand_map (const gimple *stmt, bool gather_scatter_p=false, unsigned char swap=0)
 
int vect_slp_child_index_for_operand (const gimple *stmt, int op, bool gather_scatter_p)
 
static int vect_get_and_check_slp_defs (vec_info *vinfo, unsigned char swap, bool *skip_args, vec< stmt_vec_info > stmts, unsigned stmt_num, vec< slp_oprnd_info > *oprnds_info)
 
bool compatible_calls_p (gcall *call1, gcall *call2)
 
static bool vect_record_max_nunits (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int group_size, tree vectype, poly_uint64 *max_nunits)
 
static bool vect_build_slp_tree_1 (vec_info *vinfo, unsigned char *swap, vec< stmt_vec_info > stmts, unsigned int group_size, poly_uint64 *max_nunits, bool *matches, bool *two_operators, tree *node_vectype)
 
static void release_scalar_stmts_to_slp_tree_map (scalar_stmts_to_slp_tree_map_t *bst_map)
 
static int dt_sort_cmp (const void *op1_, const void *op2_, void *)
 
static void vect_slp_linearize_chain (vec_info *vinfo, vec< std::pair< tree_code, gimple * > > &worklist, vec< chain_op_t > &chain, enum tree_code code, gimple *start, gimple *&code_stmt, gimple *&alt_code_stmt, vec< gimple * > *chain_stmts)
 
static slp_tree vect_build_slp_tree_2 (vec_info *vinfo, slp_tree node, vec< stmt_vec_info > stmts, unsigned int group_size, poly_uint64 *max_nunits, bool *matches, unsigned *limit, unsigned *tree_size, scalar_stmts_to_slp_tree_map_t *bst_map)
 
static slp_tree vect_build_slp_tree (vec_info *vinfo, vec< stmt_vec_info > stmts, unsigned int group_size, poly_uint64 *max_nunits, bool *matches, unsigned *limit, unsigned *tree_size, scalar_stmts_to_slp_tree_map_t *bst_map)
 
static void vect_slp_build_two_operator_nodes (slp_tree perm, tree vectype, slp_tree op0, slp_tree op1, stmt_vec_info oper1, stmt_vec_info oper2, vec< std::pair< unsigned, unsigned > > lperm)
 
DEBUG_FUNCTION void debug (slp_tree node)
 
static void dot_slp_tree (FILE *f, slp_tree node, hash_set< slp_tree > &visited)
 
DEBUG_FUNCTION void dot_slp_tree (const char *fname, slp_tree node)
 
DEBUG_FUNCTION void dot_slp_tree (const char *fname, const vec< slp_instance > &slp_instances)
 
static void vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc, slp_tree node, hash_set< slp_tree > &visited)
 
static void vect_print_slp_graph (dump_flags_t dump_kind, dump_location_t loc, slp_tree entry)
 
static void vect_mark_slp_stmts (slp_tree node, hash_set< slp_tree > &visited)
 
static void vect_mark_slp_stmts (slp_tree node)
 
static void vect_mark_slp_stmts_relevant (slp_tree node, hash_set< slp_tree > &visited)
 
static void vect_mark_slp_stmts_relevant (slp_tree node)
 
static void vect_gather_slp_loads (vec< slp_tree > &loads, slp_tree node, hash_set< slp_tree > &visited)
 
stmt_vec_info vect_find_last_scalar_stmt_in_slp (slp_tree node)
 
stmt_vec_info vect_find_first_scalar_stmt_in_slp (slp_tree node)
 
static stmt_vec_info vect_split_slp_store_group (stmt_vec_info first_vinfo, unsigned group1_size)
 
static poly_uint64 calculate_unrolling_factor (poly_uint64 nunits, unsigned int group_size)
 
static bool vect_is_slp_load_node (slp_tree root)
 
static slp_tree optimize_load_redistribution_1 (scalar_stmts_to_slp_tree_map_t *bst_map, vec_info *vinfo, unsigned int group_size, hash_map< slp_tree, slp_tree > *load_map, slp_tree root)
 
static void optimize_load_redistribution (scalar_stmts_to_slp_tree_map_t *bst_map, vec_info *vinfo, unsigned int group_size, hash_map< slp_tree, slp_tree > *load_map, slp_tree root)
 
static bool vect_match_slp_patterns_2 (slp_tree *ref_node, vec_info *vinfo, slp_tree_to_load_perm_map_t *perm_cache, slp_compat_nodes_map_t *compat_cache, hash_set< slp_tree > *visited)
 
static bool vect_match_slp_patterns (slp_instance instance, vec_info *vinfo, hash_set< slp_tree > *visited, slp_tree_to_load_perm_map_t *perm_cache, slp_compat_nodes_map_t *compat_cache)
 
static bool vect_slp_prefer_store_lanes_p (vec_info *vinfo, stmt_vec_info stmt_info, unsigned int group_size, unsigned int new_group_size)
 
static bool vect_analyze_slp_instance (vec_info *vinfo, scalar_stmts_to_slp_tree_map_t *bst_map, stmt_vec_info stmt_info, slp_instance_kind kind, unsigned max_tree_size, unsigned *limit)
 
static bool vect_build_slp_instance (vec_info *vinfo, slp_instance_kind kind, vec< stmt_vec_info > &scalar_stmts, vec< stmt_vec_info > &root_stmt_infos, vec< tree > &remain, unsigned max_tree_size, unsigned *limit, scalar_stmts_to_slp_tree_map_t *bst_map, stmt_vec_info stmt_info_)
 
opt_result vect_analyze_slp (vec_info *vinfo, unsigned max_tree_size)
 
template<class T >
static void vect_slp_permute (vec< unsigned > perm, vec< T > &vec, bool reverse)
 
static bool skip_cfg_latch_edges (graph_edge *e)
 
static void vect_cse_slp_nodes (scalar_stmts_to_slp_tree_map_t *bst_map, slp_tree &node)
 
void vect_optimize_slp (vec_info *vinfo)
 
void vect_gather_slp_loads (vec_info *vinfo)
 
bool vect_make_slp_decision (loop_vec_info loop_vinfo)
 
static tree vect_detect_hybrid_slp (tree *tp, int *, void *data)
 
static void maybe_push_to_hybrid_worklist (vec_info *vinfo, vec< stmt_vec_info > &worklist, stmt_vec_info stmt_info)
 
void vect_detect_hybrid_slp (loop_vec_info loop_vinfo)
 
static bool vect_slp_analyze_node_operations_1 (vec_info *vinfo, slp_tree node, slp_instance node_instance, stmt_vector_for_cost *cost_vec)
 
static bool vect_slp_convert_to_external (vec_info *vinfo, slp_tree node, slp_instance node_instance)
 
static void vect_prologue_cost_for_slp (slp_tree node, stmt_vector_for_cost *cost_vec)
 
static bool vect_slp_analyze_node_operations (vec_info *vinfo, slp_tree node, slp_instance node_instance, hash_set< slp_tree > &visited_set, vec< slp_tree > &visited_vec, stmt_vector_for_cost *cost_vec)
 
static int vec_slp_has_scalar_use (bb_vec_info bb_vinfo, tree def, hash_map< tree, int > &scalar_use_map, int depth=0)
 
static void vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo, slp_tree node, slp_instance instance, stmt_vector_for_cost *cost_vec, hash_map< tree, int > &scalar_use_map, hash_set< stmt_vec_info > &svisited, hash_set< slp_tree > &visited)
 
static void vect_bb_slp_mark_live_stmts (bb_vec_info bb_vinfo)
 
static bool vectorizable_bb_reduc_epilogue (slp_instance instance, stmt_vector_for_cost *cost_vec)
 
static void vect_slp_prune_covered_roots (slp_tree node, hash_set< stmt_vec_info > &roots, hash_set< slp_tree > &visited)
 
bool vect_slp_analyze_operations (vec_info *vinfo)
 
static slp_instance get_ultimate_leader (slp_instance instance, hash_map< slp_instance, slp_instance > &instance_leader)
 
static void vect_bb_partition_graph_r (bb_vec_info bb_vinfo, slp_instance instance, slp_tree node, hash_map< stmt_vec_info, slp_instance > &stmt_to_instance, hash_map< slp_tree, slp_instance > &node_to_instance, hash_map< slp_instance, slp_instance > &instance_leader)
 
static void vect_bb_partition_graph (bb_vec_info bb_vinfo)
 
static void vect_slp_gather_vectorized_scalar_stmts (vec_info *vinfo, slp_tree node, hash_set< slp_tree > &visited, hash_set< stmt_vec_info > &vstmts, hash_set< stmt_vec_info > &estmts)
 
static void vect_bb_slp_scalar_cost (vec_info *vinfo, slp_tree node, vec< bool, va_heap > *life, stmt_vector_for_cost *cost_vec, hash_set< stmt_vec_info > &vectorized_scalar_stmts, hash_set< slp_tree > &visited)
 
static int li_cost_vec_cmp (const void *a_, const void *b_)
 
static bool vect_bb_vectorization_profitable_p (bb_vec_info bb_vinfo, vec< slp_instance > slp_instances, loop_p orig_loop)
 
static int vld_cmp (const void *a_, const void *b_)
 
static bool vect_slp_is_lane_insert (gimple *use_stmt, tree vec, unsigned *this_lane)
 
static void vect_slp_check_for_roots (bb_vec_info bb_vinfo)
 
static void vect_fixup_store_groups_with_patterns (vec_info *vinfo)
 
static bool vect_slp_analyze_bb_1 (bb_vec_info bb_vinfo, int n_stmts, bool &fatal, vec< int > *dataref_groups)
 
static bool vect_slp_region (vec< basic_block > bbs, vec< data_reference_p > datarefs, vec< int > *dataref_groups, unsigned int n_stmts, loop_p orig_loop)
 
static bool vect_slp_bbs (const vec< basic_block > &bbs, loop_p orig_loop)
 
bool vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop)
 
bool vect_slp_function (function *fun)
 
void duplicate_and_interleave (vec_info *vinfo, gimple_seq *seq, tree vector_type, const vec< tree > &elts, unsigned int nresults, vec< tree > &results)
 
static void vect_create_constant_vectors (vec_info *vinfo, slp_tree op_node)
 
tree vect_get_slp_vect_def (slp_tree slp_node, unsigned i)
 
void vect_get_slp_defs (slp_tree slp_node, vec< tree > *vec_defs)
 
void vect_get_slp_defs (vec_info *, slp_tree slp_node, vec< vec< tree > > *vec_oprnds, unsigned n)
 
static bool vect_transform_slp_perm_load_1 (vec_info *vinfo, slp_tree node, load_permutation_t &perm, const vec< tree > &dr_chain, gimple_stmt_iterator *gsi, poly_uint64 vf, bool analyze_only, bool dump_p, unsigned *n_perms, unsigned int *n_loads, bool dce_chain)
 
bool vect_transform_slp_perm_load (vec_info *vinfo, slp_tree node, const vec< tree > &dr_chain, gimple_stmt_iterator *gsi, poly_uint64 vf, bool analyze_only, unsigned *n_perms, unsigned int *n_loads, bool dce_chain)
 
static void vect_add_slp_permutation (vec_info *vinfo, gimple_stmt_iterator *gsi, slp_tree node, tree first_def, tree second_def, tree mask_vec, poly_uint64 identity_offset)
 
static void vect_schedule_slp_node (vec_info *vinfo, slp_tree node, slp_instance instance)
 
static void vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node, hash_set< slp_tree > &visited)
 
static void vect_remove_slp_scalar_calls (vec_info *vinfo, slp_tree node)
 
void vectorize_slp_instance_root_stmt (slp_tree node, slp_instance instance)
 
static void vect_schedule_scc (vec_info *vinfo, slp_tree node, slp_instance instance, hash_map< slp_tree, slp_scc_info > &scc_info, int &maxdfs, vec< slp_tree > &stack)
 
void vect_schedule_slp (vec_info *vinfo, const vec< slp_instance > &slp_instances)
 

Variables

static object_allocator< _slp_tree > * slp_tree_pool
 
static slp_tree slp_first_node
 
static const int cond_expr_maps [3][5]
 
static const int arg0_map [] = { 1, 0 }
 
static const int arg1_map [] = { 1, 1 }
 
static const int arg2_map [] = { 1, 2 }
 
static const int arg1_arg4_map [] = { 2, 1, 4 }
 
static const int arg3_arg2_map [] = { 2, 3, 2 }
 
static const int op1_op0_map [] = { 2, 1, 0 }
 
static const int off_map [] = { 1, -3 }
 
static const int off_op0_map [] = { 2, -3, 0 }
 
static const int off_arg2_map [] = { 2, -3, 2 }
 
static const int off_arg3_arg2_map [] = { 3, -3, 3, 2 }
 
static const int mask_call_maps [6][7]
 

Macro Definition Documentation

◆ INCLUDE_ALGORITHM

#define INCLUDE_ALGORITHM
SLP - Basic Block Vectorization
   Copyright (C) 2007-2024 Free Software Foundation, Inc.
   Contributed by Dorit Naishlos <dorit@il.ibm.com>
   and Ira Rosen <irar@il.ibm.com>

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.   

◆ TEMPLATE

#define TEMPLATE   "{depth: %f, total: %f}"

Typedef Documentation

◆ scalar_stmts_to_slp_tree_map_t

◆ slp_oprnd_info

typedef struct _slp_oprnd_info * slp_oprnd_info
This structure is used in creation of an SLP tree.  Each instance
corresponds to the same operand in a group of scalar stmts in an SLP
node.   

Function Documentation

◆ calculate_unrolling_factor()

static poly_uint64 calculate_unrolling_factor ( poly_uint64 nunits,
unsigned int group_size )
static
Calculate the unrolling factor for an SLP instance with GROUP_SIZE
statements and a vector of NUNITS elements.   

Referenced by vect_build_slp_instance().

◆ can_duplicate_and_interleave_p()

bool can_duplicate_and_interleave_p ( vec_info * vinfo,
unsigned int count,
tree elt_type,
unsigned int * nvectors_out,
tree * vector_type_out,
tree * permutes )
Check whether it is possible to load COUNT elements of type ELT_TYPE
using the method implemented by duplicate_and_interleave.  Return true
if so, returning the number of intermediate vectors in *NVECTORS_OUT
(if nonnull) and the type of each intermediate vector in *VECTOR_TYPE_OUT
(if nonnull).   

References build_nonstandard_integer_type(), can_vec_perm_const_p(), count, GET_MODE_BITSIZE(), GET_MODE_NUNITS(), GET_MODE_SIZE(), GET_MODE_UNIT_SIZE, get_vectype_for_scalar_type(), i, int_mode_for_size(), known_eq, TYPE_MODE, vect_gen_perm_mask_checked(), and VECTOR_MODE_P.

Referenced by duplicate_and_interleave(), vect_build_slp_tree_2(), and vectorizable_reduction().

◆ compatible_calls_p()

bool compatible_calls_p ( gcall * call1,
gcall * call2 )

◆ debug()

◆ dot_slp_tree() [1/3]

DEBUG_FUNCTION void dot_slp_tree ( const char * fname,
const vec< slp_instance > & slp_instances )

◆ dot_slp_tree() [2/3]

DEBUG_FUNCTION void dot_slp_tree ( const char * fname,
slp_tree node )

References dot_slp_tree(), fopen, and visited.

◆ dot_slp_tree() [3/3]

static void dot_slp_tree ( FILE * f,
slp_tree node,
hash_set< slp_tree > & visited )
static

◆ dt_sort_cmp()

static int dt_sort_cmp ( const void * op1_,
const void * op2_,
void *  )
static
Comparator for sorting associatable chains.   

Referenced by vect_build_slp_tree_2(), and vect_slp_check_for_roots().

◆ duplicate_and_interleave()

void duplicate_and_interleave ( vec_info * vinfo,
gimple_seq * seq,
tree vector_type,
const vec< tree > & elts,
unsigned int nresults,
vec< tree > & results )
Build a variable-length vector in which the elements in ELTS are repeated
to a fill NRESULTS vectors of type VECTOR_TYPE.  Store the vectors in
RESULTS and add any new instructions to SEQ.

The approach we use is:

(1) Find a vector mode VM with integer elements of mode IM.

(2) Replace ELTS[0:NELTS] with ELTS'[0:NELTS'], where each element of
    ELTS' has mode IM.  This involves creating NELTS' VIEW_CONVERT_EXPRs
    from small vectors to IM.

(3) Duplicate each ELTS'[I] into a vector of mode VM.

(4) Use a tree of interleaving VEC_PERM_EXPRs to create VMs with the
    correct byte contents.

(5) Use VIEW_CONVERT_EXPR to cast the final VMs to the required type.

We try to find the largest IM for which this sequence works, in order
to cut down on the number of interleaves.   

References build_vector_type(), can_duplicate_and_interleave_p(), gcc_unreachable, gimple_build(), gimple_build_assign(), gimple_build_vector(), gimple_build_vector_from_val(), gimple_seq_add_stmt(), i, make_ssa_name(), tree_vector_builder::new_vector(), TREE_TYPE, and TYPE_VECTOR_SUBPARTS().

Referenced by get_initial_defs_for_reduction(), and vect_create_constant_vectors().

◆ get_ultimate_leader()

static slp_instance get_ultimate_leader ( slp_instance instance,
hash_map< slp_instance, slp_instance > & instance_leader )
static
Get the SLP instance leader from INSTANCE_LEADER thereby transitively
closing the eventual chain.   

References hash_map< KeyId, Value, Traits >::get().

Referenced by vect_bb_partition_graph().

◆ li_cost_vec_cmp()

static int li_cost_vec_cmp ( const void * a_,
const void * b_ )
static
Comparator for the loop-index sorted cost vectors.   

References a, and b.

Referenced by vect_bb_vectorization_profitable_p().

◆ maybe_push_to_hybrid_worklist()

static void maybe_push_to_hybrid_worklist ( vec_info * vinfo,
vec< stmt_vec_info > & worklist,
stmt_vec_info stmt_info )
static

◆ optimize_load_redistribution()

static void optimize_load_redistribution ( scalar_stmts_to_slp_tree_map_t * bst_map,
vec_info * vinfo,
unsigned int group_size,
hash_map< slp_tree, slp_tree > * load_map,
slp_tree root )
static
Temporary workaround for loads not being CSEd during SLP build.  This
function will traverse the SLP tree rooted in ROOT for INSTANCE and find
VEC_PERM nodes that blend vectors from multiple nodes that all read from the
same DR such that the final operation is equal to a permuted load.  Such
NODES are then directly converted into LOADS themselves.  The nodes are
CSEd using BST_MAP.   

References FOR_EACH_VEC_ELT, i, optimize_load_redistribution_1(), hash_map< KeyId, Value, Traits >::remove(), SLP_TREE_CHILDREN, SLP_TREE_REF_COUNT, and vect_free_slp_tree().

Referenced by vect_analyze_slp().

◆ optimize_load_redistribution_1()

◆ release_scalar_stmts_to_slp_tree_map()

static void release_scalar_stmts_to_slp_tree_map ( scalar_stmts_to_slp_tree_map_t * bst_map)
static

◆ skip_cfg_latch_edges()

static bool skip_cfg_latch_edges ( graph_edge * e)
static
Return true if E corresponds to a loop latch edge in the cfg.   

References graph_edge::data.

Referenced by vect_optimize_slp_pass::create_partitions().

◆ vec_slp_has_scalar_use()

static int vec_slp_has_scalar_use ( bb_vec_info bb_vinfo,
tree def,
hash_map< tree, int > & scalar_use_map,
int depth = 0 )
static
Given a definition DEF, analyze if it will have any live scalar use after
performing SLP vectorization whose information is represented by BB_VINFO,
and record result into hash map SCALAR_USE_MAP as cache for later fast
check.  If recursion DEPTH exceeds a limit, stop analysis and make a
conservative assumption.  Return 0 if no scalar use, 1 if there is, -1
means recursion is limited.   

References end_imm_use_stmt_p(), FOR_EACH_IMM_USE_STMT, gcc_assert, hash_map< KeyId, Value, Traits >::get(), gimple_get_lhs(), is_gimple_debug(), vec_info::lookup_stmt(), PURE_SLP_STMT, hash_map< KeyId, Value, Traits >::put(), TREE_CODE, vec_slp_has_scalar_use(), and vect_stmt_to_vectorize().

Referenced by vec_slp_has_scalar_use(), and vect_bb_slp_mark_live_stmts().

◆ vect_add_slp_permutation()

static void vect_add_slp_permutation ( vec_info * vinfo,
gimple_stmt_iterator * gsi,
slp_tree node,
tree first_def,
tree second_def,
tree mask_vec,
poly_uint64 identity_offset )
static
Produce the next vector result for SLP permutation NODE by adding a vector
statement at GSI.  If MASK_VEC is nonnull, add:

   <new SSA name> = VEC_PERM_EXPR <FIRST_DEF, SECOND_DEF, MASK_VEC>

otherwise add:

   <new SSA name> = FIRST_DEF.   

References bitsize_int, build1(), build3(), build_constructor_va(), gcc_unreachable, gimple_assign_lhs(), gimple_build_assign(), known_le, make_ssa_name(), NULL, NULL_TREE, operand_equal_p(), _slp_tree::push_vec_def(), SLP_TREE_VECTYPE, tree_to_uhwi(), TREE_TYPE, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), and vect_finish_stmt_generation().

Referenced by vectorizable_slp_permutation_1().

◆ vect_analyze_slp()

◆ vect_analyze_slp_instance()

static bool vect_analyze_slp_instance ( vec_info * vinfo,
scalar_stmts_to_slp_tree_map_t * bst_map,
stmt_vec_info stmt_info,
slp_instance_kind kind,
unsigned max_tree_size,
unsigned * limit )
static
Analyze an SLP instance starting from a group of grouped stores.  Call
vect_build_slp_tree to build a tree of packed stmts if possible.
Return FALSE if it's impossible to SLP any stmt in the loop.   

References DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, gcc_unreachable, is_a(), NULL, REDUC_GROUP_NEXT_ELEMENT, REDUC_GROUP_SIZE, slp_inst_kind_reduc_chain, slp_inst_kind_store, STMT_VINFO_DEF_TYPE, STMT_VINFO_REDUC_DEF, vect_build_slp_instance(), vect_location, vect_orig_stmt(), vect_stmt_to_vectorize(), and vNULL.

Referenced by vect_analyze_slp(), and vect_build_slp_instance().

◆ vect_bb_partition_graph()

static void vect_bb_partition_graph ( bb_vec_info bb_vinfo)
static

◆ vect_bb_partition_graph_r()

static void vect_bb_partition_graph_r ( bb_vec_info bb_vinfo,
slp_instance instance,
slp_tree node,
hash_map< stmt_vec_info, slp_instance > & stmt_to_instance,
hash_map< slp_tree, slp_instance > & node_to_instance,
hash_map< slp_instance, slp_instance > & instance_leader )
static

◆ vect_bb_slp_mark_live_stmts() [1/2]

static void vect_bb_slp_mark_live_stmts ( bb_vec_info bb_vinfo)
static
Traverse all slp instances of BB_VINFO, and mark lanes of every node that
are live outside of the basic-block vectorized region and that can be
vectorized using vectorizable_live_operation with STMT_VINFO_LIVE_P.   

References hash_map< KeyId, Value, Traits >::put(), slp_inst_kind_bb_reduc, SLP_INSTANCE_KIND, SLP_INSTANCE_REMAIN_DEFS, SLP_INSTANCE_TREE, vec_info::slp_instances, SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_OPS, TREE_CODE, vect_bb_slp_mark_live_stmts(), vect_external_def, vect_location, visited, and worklist.

◆ vect_bb_slp_mark_live_stmts() [2/2]

◆ vect_bb_slp_scalar_cost()

◆ vect_bb_vectorization_profitable_p()

◆ vect_build_slp_instance()

static bool vect_build_slp_instance ( vec_info * vinfo,
slp_instance_kind kind,
vec< stmt_vec_info > & scalar_stmts,
vec< stmt_vec_info > & root_stmt_infos,
vec< tree > & remain,
unsigned max_tree_size,
unsigned * limit,
scalar_stmts_to_slp_tree_map_t * bst_map,
stmt_vec_info stmt_info_ )
static

◆ vect_build_slp_tree()

◆ vect_build_slp_tree_1()

static bool vect_build_slp_tree_1 ( vec_info * vinfo,
unsigned char * swap,
vec< stmt_vec_info > stmts,
unsigned int group_size,
poly_uint64 * max_nunits,
bool * matches,
bool * two_operators,
tree * node_vectype )
static
Verify if the scalar stmts STMTS are isomorphic, require data
permutation or are of unsupported types of operation.  Return
true if they are, otherwise return false and indicate in *MATCHES
which stmts are not isomorphic to the first one.  If MATCHES[0]
is false then this indicates the comparison could not be
carried out or the stmts will never be vectorized by SLP.

Note COND_EXPR is possibly isomorphic to another one after swapping its
operands.  Set SWAP[i] to 1 if stmt I is COND_EXPR and isomorphic to
the first stmt by swapping the two operands of comparison; set SWAP[i]
to 2 if stmt I is isormorphic to the first stmt by inverting the code
of comparison.  Take A1 >= B1 ? X1 : Y1 as an exmple, it can be swapped
to (B1 <= A1 ? X1 : Y1); or be inverted to (A1 < B1) ? Y1 : X1.   

References as_a(), as_internal_fn(), CFN_LAST, cfun, compatible_calls_p(), DECL_P, directly_supported_p(), DR_GROUP_FIRST_ELEMENT, DR_IS_READ, DR_IS_WRITE, dump_enabled_p(), dump_printf_loc(), dyn_cast(), FOR_EACH_VEC_ELT, gcc_assert, gimple_arg(), gimple_assign_rhs1(), gimple_assign_rhs2(), gimple_assign_rhs_code(), gimple_bb(), gimple_call_arg(), gimple_call_chain(), gimple_call_combined_fn(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_noreturn_p(), gimple_call_tail_p(), gimple_could_trap_p(), gimple_get_lhs(), gimple_has_volatile_ops(), HONOR_NANS(), i, internal_fn_p(), internal_store_fn_p(), invert_tree_comparison(), is_a(), code_helper::is_tree_code(), MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, NULL_TREE, operand_equal_p(), optab_scalar, optab_vector, REDUC_GROUP_FIRST_ELEMENT, REFERENCE_CLASS_P, stmt_can_throw_internal(), STMT_VINFO_DATA_REF, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_REDUC_IDX, STMT_VINFO_VECTORIZABLE, swap_tree_comparison(), tcc_binary, tcc_comparison, tcc_expression, tcc_reference, tcc_unary, TREE_CODE, TREE_CODE_CLASS, TREE_OPERAND, TREE_TYPE, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), vect_get_vector_types_for_stmt(), vect_location, vect_record_max_nunits(), VECTOR_TYPE_P, and vectorizable_internal_fn_p().

Referenced by vect_build_slp_tree_2().

◆ vect_build_slp_tree_2()

static slp_tree vect_build_slp_tree_2 ( vec_info * vinfo,
slp_tree node,
vec< stmt_vec_info > stmts,
unsigned int group_size,
poly_uint64 * max_nunits,
bool * matches,
unsigned * limit,
unsigned * tree_size,
scalar_stmts_to_slp_tree_map_t * bst_map )
static
Recursively build an SLP tree starting from NODE.
Fail (and return a value not equal to zero) if def-stmts are not
isomorphic, require data permutation or are of unsupported types of
operation.  Otherwise, return 0.
The value returned is the depth in the SLP tree where a mismatch
was found.   

References _slp_oprnd_info::any_pattern, as_a(), associative_tree_code(), bit_field_offset(), bit_field_size(), build_zero_cst(), can_duplicate_and_interleave_p(), commutative_tree_code(), _slp_oprnd_info::def_stmts, DR_GROUP_FIRST_ELEMENT, DR_GROUP_SIZE, DR_IS_READ, dt_sort_cmp(), dump_enabled_p(), dump_printf(), dump_printf_loc(), dyn_cast(), _slp_oprnd_info::first_dt, FLOAT_TYPE_P, FOR_EACH_VEC_ELT, gcc_assert, GET_MODE_SIZE(), get_tree_code_name(), get_vectype_for_scalar_type(), gimple_assign_rhs1(), gimple_assign_rhs_code(), gimple_assign_single_p(), gimple_bb(), gimple_call_internal_p(), gimple_num_args(), gimple_vuse(), loop::header, i, loop::inner, INTEGRAL_TYPE_P, is_a(), poly_int< N, C >::is_constant(), is_gimple_assign(), is_pattern_stmt_p(), known_eq, known_ge, chain::length, vec_info::lookup_def(), vec_info::lookup_stmt(), loop_latch_edge(), loop_preheader_edge(), LOOP_VINFO_LOOP, map, MSG_MISSED_OPTIMIZATION, MSG_NOTE, nested_in_vect_loop_p(), NULL, NULL_TREE, operand_equal_p(), _slp_oprnd_info::ops, PHI_RESULT, SLP_TREE_CHILDREN, SLP_TREE_CODE, SLP_TREE_DEF_TYPE, SLP_TREE_LANE_PERMUTATION, SLP_TREE_LANES, SLP_TREE_LOAD_PERMUTATION, SLP_TREE_REF_COUNT, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, SLP_TREE_VEC_DEFS, SLP_TREE_VECTYPE, STMT_VINFO_DATA_REF, STMT_VINFO_DEF_TYPE, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_REDUC_IDX, STMT_VINFO_STRIDED_P, STMT_VINFO_VECTORIZABLE, TREE_CODE, TREE_OPERAND, tree_size(), tree_to_poly_uint64(), TREE_TYPE, TYPE_OVERFLOW_WRAPS, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), vect_build_slp_tree(), vect_build_slp_tree_1(), vect_constant_def, vect_create_new_slp_node(), vect_create_oprnd_info(), vect_double_reduction_def, vect_external_def, vect_first_order_recurrence, vect_free_oprnd_info(), vect_free_slp_tree(), vect_get_and_check_slp_defs(), vect_get_operand_map(), vect_get_place_in_interleaving_chain(), vect_induction_def, vect_internal_def, vect_location, vect_nested_cycle, vect_print_slp_tree(), vect_record_max_nunits(), vect_reduction_def, vect_slp_build_two_operator_nodes(), vect_slp_linearize_chain(), vect_slp_tree_uniform_p(), vect_uninitialized_def, vec_info::vector_mode, VECTOR_TYPE_P, vNULL, and worklist.

Referenced by vect_build_slp_tree().

◆ vect_contains_pattern_stmt_p()

static bool vect_contains_pattern_stmt_p ( vec< stmt_vec_info > stmts)
static
Return true if STMTS contains a pattern statement.   

References FOR_EACH_VEC_ELT, i, and is_pattern_stmt_p().

Referenced by vect_slp_convert_to_external().

◆ vect_create_constant_vectors()

◆ vect_create_new_slp_node() [1/5]

static slp_tree vect_create_new_slp_node ( slp_tree node,
vec< stmt_vec_info > scalar_stmts,
unsigned nops )
static

◆ vect_create_new_slp_node() [2/5]

static slp_tree vect_create_new_slp_node ( slp_tree node,
vec< tree > ops )
static
Create an SLP node for OPS.   

References SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_SCALAR_OPS, and vect_external_def.

◆ vect_create_new_slp_node() [3/5]

◆ vect_create_new_slp_node() [4/5]

static slp_tree vect_create_new_slp_node ( vec< stmt_vec_info > scalar_stmts,
unsigned nops )
static
Create an SLP node for SCALAR_STMTS.   

References vect_create_new_slp_node().

◆ vect_create_new_slp_node() [5/5]

static slp_tree vect_create_new_slp_node ( vec< tree > ops)
static
Create an SLP node for OPS.   

References vect_create_new_slp_node().

◆ vect_create_oprnd_info()

static vec< slp_oprnd_info > vect_create_oprnd_info ( int nops,
int group_size )
static

◆ vect_cse_slp_nodes()

◆ vect_def_types_match()

static bool vect_def_types_match ( enum vect_def_type dta,
enum vect_def_type dtb )
static
Return true if DTA and DTB match.   

References vect_constant_def, and vect_external_def.

Referenced by vect_get_and_check_slp_defs().

◆ vect_detect_hybrid_slp() [1/2]

◆ vect_detect_hybrid_slp() [2/2]

static tree vect_detect_hybrid_slp ( tree * tp,
int * ,
void * data )
static

◆ vect_find_first_scalar_stmt_in_slp()

stmt_vec_info vect_find_first_scalar_stmt_in_slp ( slp_tree node)

◆ vect_find_last_scalar_stmt_in_slp()

◆ vect_fixup_store_groups_with_patterns()

static void vect_fixup_store_groups_with_patterns ( vec_info * vinfo)
static
Walk the grouped store chains and replace entries with their
pattern variant if any.   

References DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, FOR_EACH_VEC_ELT, vec_info::grouped_stores, i, STMT_VINFO_GROUPED_ACCESS, STMT_VINFO_IN_PATTERN_P, and STMT_VINFO_RELATED_STMT.

Referenced by vect_slp_analyze_bb_1().

◆ vect_free_oprnd_info()

static void vect_free_oprnd_info ( vec< slp_oprnd_info > & oprnds_info)
static
Free operands info.   

References _slp_oprnd_info::def_stmts, FOR_EACH_VEC_ELT, i, and _slp_oprnd_info::ops.

Referenced by vect_build_slp_tree_2().

◆ vect_free_slp_instance()

◆ vect_free_slp_tree()

◆ vect_gather_slp_loads() [1/2]

static void vect_gather_slp_loads ( vec< slp_tree > & loads,
slp_tree node,
hash_set< slp_tree > & visited )
static

◆ vect_gather_slp_loads() [2/2]

void vect_gather_slp_loads ( vec_info * vinfo)
Gather loads reachable from the individual SLP graph entries.   

References FOR_EACH_VEC_ELT, i, SLP_INSTANCE_LOADS, SLP_INSTANCE_TREE, vec_info::slp_instances, vect_gather_slp_loads(), and visited.

◆ vect_get_and_check_slp_defs()

static int vect_get_and_check_slp_defs ( vec_info * vinfo,
unsigned char swap,
bool * skip_args,
vec< stmt_vec_info > stmts,
unsigned stmt_num,
vec< slp_oprnd_info > * oprnds_info )
static
Get the defs for the rhs of STMT (collect them in OPRNDS_INFO), check that
they are of a valid type and that they match the defs of the first stmt of
the SLP group (stored in OPRNDS_INFO).  This function tries to match stmts
by swapping operands of STMTS[STMT_NUM] when possible.  Non-zero SWAP
indicates swap is required for cond_expr stmts.  Specifically, SWAP
is 1 if STMT is cond and operands of comparison need to be swapped;
SWAP is 2 if STMT is cond and code of comparison needs to be inverted.

If there was a fatal error return -1; if the error could be corrected by
swapping operands of father node of this one, return 1; if everything is
ok return 0.   

References _slp_oprnd_info::any_pattern, as_a(), gather_scatter_info::base, vec_info::bbs, CDI_DOMINATORS, commutative_op(), commutative_tree_code(), _slp_oprnd_info::def_stmts, dominated_by_p(), dump_enabled_p(), dump_printf_loc(), dyn_cast(), first_commutative_argument(), _slp_oprnd_info::first_dt, _slp_oprnd_info::first_gs_info, _slp_oprnd_info::first_gs_p, _slp_oprnd_info::first_op_type, gcc_assert, gimple_arg(), gimple_assign_rhs_code(), gimple_bb(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_get_lhs(), gimple_num_args(), gimple_phi_arg_edge(), i, is_a(), is_pattern_stmt_p(), map, MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL, NULL_TREE, gather_scatter_info::offset, operand_equal_p(), _slp_oprnd_info::ops, REDUC_GROUP_FIRST_ELEMENT, gather_scatter_info::scale, SSA_NAME_DEF_STMT, SSA_NAME_IS_DEFAULT_DEF, STMT_VINFO_DATA_REF, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_REDUC_IDX, STMT_VINFO_RELATED_STMT, TREE_CODE, TREE_OPERAND, TREE_TYPE, type(), types_compatible_p(), vect_check_gather_scatter(), vect_constant_def, vect_def_types_match(), vect_double_reduction_def, vect_external_def, vect_first_order_recurrence, vect_get_operand_map(), vect_induction_def, vect_internal_def, vect_is_simple_use(), vect_location, vect_nested_cycle, vect_orig_stmt(), vect_reduction_def, and vect_uninitialized_def.

Referenced by vect_build_slp_tree_2().

◆ vect_get_operand_map()

static const int * vect_get_operand_map ( const gimple * stmt,
bool gather_scatter_p = false,
unsigned char swap = 0 )
static
For most SLP statements, there is a one-to-one mapping between
gimple arguments and child nodes.  If that is not true for STMT,
return an array that contains:

- the number of child nodes, followed by
- for each child node, the index of the argument associated with that node.
  The special index -1 is the first operand of an embedded comparison and
  the special index -2 is the second operand of an embedded comparison.
  The special indes -3 is the offset of a gather as analyzed by
  vect_check_gather_scatter.

SWAP is as for vect_get_and_check_slp_defs.   

References arg0_map, arg1_arg4_map, arg1_map, arg2_map, arg3_arg2_map, COMPARISON_CLASS_P, cond_expr_maps, dyn_cast(), gcc_assert, gimple_assign_lhs(), gimple_assign_rhs1(), gimple_assign_rhs_code(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_num_args(), mask_call_maps, off_arg2_map, off_arg3_arg2_map, off_map, off_op0_map, op1_op0_map, tcc_comparison, TREE_CODE, and TREE_CODE_CLASS.

Referenced by compatible_calls_p(), vect_build_slp_tree_2(), vect_get_and_check_slp_defs(), and vect_slp_child_index_for_operand().

◆ vect_get_place_in_interleaving_chain()

int vect_get_place_in_interleaving_chain ( stmt_vec_info stmt_info,
stmt_vec_info first_stmt_info )
Find the place of the data-ref in STMT_INFO in the interleaving chain
that starts from FIRST_STMT_INFO.  Return -1 if the data-ref is not a part
of the chain.   

References DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, and DR_GROUP_NEXT_ELEMENT.

Referenced by vect_build_slp_tree_2(), and vectorizable_load().

◆ vect_get_slp_defs() [1/2]

◆ vect_get_slp_defs() [2/2]

void vect_get_slp_defs ( vec_info * ,
slp_tree slp_node,
vec< vec< tree > > * vec_oprnds,
unsigned n )
Get N vectorized definitions for SLP_NODE.   

References i, SLP_TREE_CHILDREN, vect_get_slp_defs(), and vNULL.

◆ vect_get_slp_vect_def()

tree vect_get_slp_vect_def ( slp_tree slp_node,
unsigned i )
Get the Ith vectorized definition from SLP_NODE.   

References i, and SLP_TREE_VEC_DEFS.

Referenced by vect_create_epilog_for_reduction(), vect_schedule_scc(), vectorizable_induction(), and vectorizable_slp_permutation_1().

◆ vect_is_slp_load_node()

static bool vect_is_slp_load_node ( slp_tree root)
inlinestatic
Helper that checks to see if a node is a load node.   

References DR_IS_READ, SLP_TREE_DEF_TYPE, SLP_TREE_REPRESENTATIVE, STMT_VINFO_DATA_REF, STMT_VINFO_GROUPED_ACCESS, and vect_internal_def.

Referenced by optimize_load_redistribution_1().

◆ vect_make_slp_decision()

bool vect_make_slp_decision ( loop_vec_info loop_vinfo)
For each possible SLP instance decide whether to SLP it and calculate overall
unrolling factor needed to SLP the loop.  Return TRUE if decided to SLP at
least one instance.   

References dump_dec(), dump_enabled_p(), dump_printf(), dump_printf_loc(), DUMP_VECT_SCOPE, FOR_EACH_VEC_ELT, i, LOOP_VINFO_SLP_INSTANCES, LOOP_VINFO_SLP_UNROLLING_FACTOR, MSG_NOTE, SLP_INSTANCE_TREE, SLP_INSTANCE_UNROLLING_FACTOR, vect_location, and vect_mark_slp_stmts().

Referenced by vect_analyze_loop_2().

◆ vect_mark_slp_stmts() [1/2]

static void vect_mark_slp_stmts ( slp_tree node)
static

References vect_mark_slp_stmts(), and visited.

◆ vect_mark_slp_stmts() [2/2]

static void vect_mark_slp_stmts ( slp_tree node,
hash_set< slp_tree > & visited )
static

◆ vect_mark_slp_stmts_relevant() [1/2]

static void vect_mark_slp_stmts_relevant ( slp_tree node)
static

◆ vect_mark_slp_stmts_relevant() [2/2]

static void vect_mark_slp_stmts_relevant ( slp_tree node,
hash_set< slp_tree > & visited )
static

◆ vect_match_slp_patterns()

static bool vect_match_slp_patterns ( slp_instance instance,
vec_info * vinfo,
hash_set< slp_tree > * visited,
slp_tree_to_load_perm_map_t * perm_cache,
slp_compat_nodes_map_t * compat_cache )
static
Applies pattern matching to the given SLP tree rooted in REF_NODE using
vec_info VINFO.

The modified tree is returned.  Patterns are tried in order and multiple
patterns may match.   

References dump_enabled_p(), dump_printf_loc(), DUMP_VECT_SCOPE, MSG_NOTE, SLP_INSTANCE_TREE, vect_location, vect_match_slp_patterns_2(), and visited.

Referenced by vect_analyze_slp().

◆ vect_match_slp_patterns_2()

static bool vect_match_slp_patterns_2 ( slp_tree * ref_node,
vec_info * vinfo,
slp_tree_to_load_perm_map_t * perm_cache,
slp_compat_nodes_map_t * compat_cache,
hash_set< slp_tree > * visited )
static
Helper function of vect_match_slp_patterns.

Attempts to match patterns against the slp tree rooted in REF_NODE using
VINFO.  Patterns are matched in post-order traversal.

If matching is successful the value in REF_NODE is updated and returned, if
not then it is returned unchanged.   

References vect_pattern::build(), FOR_EACH_VEC_ELT, i, num__slp_patterns, slp_patterns, SLP_TREE_CHILDREN, vect_match_slp_patterns_2(), and visited.

Referenced by vect_match_slp_patterns(), and vect_match_slp_patterns_2().

◆ vect_optimize_slp()

◆ vect_print_slp_graph() [1/2]

static void vect_print_slp_graph ( dump_flags_t dump_kind,
dump_location_t loc,
slp_tree entry )
static

References vect_print_slp_graph(), and visited.

◆ vect_print_slp_graph() [2/2]

static void vect_print_slp_graph ( dump_flags_t dump_kind,
dump_location_t loc,
slp_tree node,
hash_set< slp_tree > & visited )
static

◆ vect_print_slp_tree()

◆ vect_prologue_cost_for_slp()

static void vect_prologue_cost_for_slp ( slp_tree node,
stmt_vector_for_cost * cost_vec )
static

◆ vect_record_max_nunits()

static bool vect_record_max_nunits ( vec_info * vinfo,
stmt_vec_info stmt_info,
unsigned int group_size,
tree vectype,
poly_uint64 * max_nunits )
static
A subroutine of vect_build_slp_tree for checking VECTYPE, which is the
caller's attempt to find the vector type in STMT_INFO with the narrowest
element type.  Return true if VECTYPE is nonnull and if it is valid
for STMT_INFO.  When returning true, update MAX_NUNITS to reflect the
number of units in VECTYPE.  GROUP_SIZE and MAX_NUNITS are as for
vect_build_slp_tree.   

References dump_enabled_p(), dump_printf_loc(), is_a(), MSG_MISSED_OPTIMIZATION, TYPE_VECTOR_SUBPARTS(), vect_location, and vect_update_max_nunits().

Referenced by vect_build_slp_tree_1(), and vect_build_slp_tree_2().

◆ vect_remove_slp_scalar_calls() [1/2]

static void vect_remove_slp_scalar_calls ( vec_info * vinfo,
slp_tree node )
static

◆ vect_remove_slp_scalar_calls() [2/2]

static void vect_remove_slp_scalar_calls ( vec_info * vinfo,
slp_tree node,
hash_set< slp_tree > & visited )
static
Replace scalar calls from SLP node NODE with setting of their lhs to zero.
For loop vectorization this is done in vectorizable_call, but for SLP
it needs to be deferred until end of vect_schedule_slp, because multiple
SLP instances may refer to the same scalar stmt.   

References build_zero_cst(), dyn_cast(), FOR_EACH_VEC_ELT, gimple_bb(), gimple_build_assign(), gimple_build_nop(), gimple_call_lhs(), gsi_for_stmt(), i, is_pattern_stmt_p(), NULL, PURE_SLP_STMT, vec_info::replace_stmt(), SLP_TREE_CHILDREN, SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_STMTS, SSA_NAME_DEF_STMT, TREE_TYPE, unlink_stmt_vdef(), vect_internal_def, vect_remove_slp_scalar_calls(), and visited.

Referenced by vect_remove_slp_scalar_calls(), vect_remove_slp_scalar_calls(), and vect_schedule_slp().

◆ vect_schedule_scc()

◆ vect_schedule_slp()

◆ vect_schedule_slp_node()

◆ vect_slp_analyze_bb_1()

◆ vect_slp_analyze_node_operations()

◆ vect_slp_analyze_node_operations_1()

static bool vect_slp_analyze_node_operations_1 ( vec_info * vinfo,
slp_tree node,
slp_instance node_instance,
stmt_vector_for_cost * cost_vec )
static
Subroutine of vect_slp_analyze_node_operations.  Handle the root of NODE,
given then that child nodes have already been processed, and that
their def types currently match their SLP node's def type.   

References FOR_EACH_VEC_ELT, i, NULL, SLP_TREE_CODE, SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_REPRESENTATIVE, SLP_TREE_SCALAR_STMTS, STMT_VINFO_LIVE_P, vect_analyze_stmt(), vect_get_num_copies(), vectorizable_live_operation(), and vectorizable_slp_permutation().

Referenced by vect_slp_analyze_node_operations().

◆ vect_slp_analyze_operations()

◆ vect_slp_bbs()

static bool vect_slp_bbs ( const vec< basic_block > & bbs,
loop_p orig_loop )
static
Main entry for the BB vectorizer.  Analyze and transform BBS, returns
true if anything in the basic-block was vectorized.   

References gimple_location(), gsi_after_labels(), gsi_end_p(), gsi_next(), gsi_stmt(), i, insns, is_gimple_debug(), NULL, UNKNOWN_LOCATION, vect_find_stmt_data_reference(), vect_location, vect_slp_region(), and vNULL.

Referenced by vect_slp_function(), and vect_slp_if_converted_bb().

◆ vect_slp_build_two_operator_nodes()

static void vect_slp_build_two_operator_nodes ( slp_tree perm,
tree vectype,
slp_tree op0,
slp_tree op1,
stmt_vec_info oper1,
stmt_vec_info oper2,
vec< std::pair< unsigned, unsigned > > lperm )
static

◆ vect_slp_check_for_roots()

◆ vect_slp_child_index_for_operand()

int vect_slp_child_index_for_operand ( const gimple * stmt,
int op,
bool gather_scatter_p )
Return the SLP node child index for operand OP of STMT.   

References gcc_unreachable, i, and vect_get_operand_map().

Referenced by vect_check_store_rhs(), vectorizable_load(), vectorizable_simd_clone_call(), and vectorizable_store().

◆ vect_slp_convert_to_external()

static bool vect_slp_convert_to_external ( vec_info * vinfo,
slp_tree node,
slp_instance node_instance )
static

◆ vect_slp_fini()

void vect_slp_fini ( void )

References NULL, slp_first_node, and slp_tree_pool.

◆ vect_slp_function()

◆ vect_slp_gather_vectorized_scalar_stmts()

static void vect_slp_gather_vectorized_scalar_stmts ( vec_info * vinfo,
slp_tree node,
hash_set< slp_tree > & visited,
hash_set< stmt_vec_info > & vstmts,
hash_set< stmt_vec_info > & estmts )
static

◆ vect_slp_if_converted_bb()

bool vect_slp_if_converted_bb ( basic_block bb,
loop_p orig_loop )
Special entry for the BB vectorizer.  Analyze and transform a single
if-converted BB with ORIG_LOOPs body being the not if-converted
representation.  Returns true if anything in the basic-block was
vectorized.   

References vect_slp_bbs().

Referenced by try_vectorize_loop_1().

◆ vect_slp_init()

void vect_slp_init ( void )
In tree-vect-slp.cc.   

References slp_tree_pool.

◆ vect_slp_is_lane_insert()

static bool vect_slp_is_lane_insert ( gimple * use_stmt,
tree vec,
unsigned * this_lane )
static
Return true if USE_STMT is a vector lane insert into VEC and set
*THIS_LANE to the lane number that is set.   

References dyn_cast(), gimple_assign_rhs1(), gimple_assign_rhs2(), gimple_assign_rhs3(), gimple_assign_rhs_code(), tree_to_poly_uint64(), TREE_TYPE, TYPE_SIZE, and useless_type_conversion_p().

Referenced by vect_slp_check_for_roots().

◆ vect_slp_linearize_chain()

static void vect_slp_linearize_chain ( vec_info * vinfo,
vec< std::pair< tree_code, gimple * > > & worklist,
vec< chain_op_t > & chain,
enum tree_code code,
gimple * start,
gimple *& code_stmt,
gimple *& alt_code_stmt,
vec< gimple * > * chain_stmts )
static
Linearize the associatable expression chain at START with the
associatable operation CODE (where PLUS_EXPR also allows MINUS_EXPR),
filling CHAIN with the result and using WORKLIST as intermediate storage.
CODE_STMT and ALT_CODE_STMT are filled with the first stmt using CODE
or MINUS_EXPR.  *CHAIN_STMTS if not NULL is filled with all computation
stmts, starting with START.   

References as_a(), gcc_assert, gimple_assign_rhs_code(), gimple_get_lhs(), gimple_op(), is_gimple_assign(), is_pattern_stmt_p(), single_imm_use(), vect_internal_def, vect_is_simple_use(), and worklist.

Referenced by vect_build_slp_tree_2(), and vect_slp_check_for_roots().

◆ vect_slp_node_weight()

static sreal vect_slp_node_weight ( slp_tree node)
static
Return the execution frequency of NODE (so that a higher value indicates
a "more important" node when optimizing for speed).   

References cfun, basic_block_def::count, count, ENTRY_BLOCK_PTR_FOR_FN, gimple_bb(), SLP_TREE_REPRESENTATIVE, profile_count::to_sreal_scale(), and vect_orig_stmt().

Referenced by vect_optimize_slp_pass::start_choosing_layouts().

◆ vect_slp_permute()

template<class T >
static void vect_slp_permute ( vec< unsigned > perm,
vec< T > & vec,
bool reverse )
static

◆ vect_slp_prefer_store_lanes_p()

static bool vect_slp_prefer_store_lanes_p ( vec_info * vinfo,
stmt_vec_info stmt_info,
unsigned int group_size,
unsigned int new_group_size )
static
STMT_INFO is a store group of size GROUP_SIZE that we are considering
splitting into two, with the first split group having size NEW_GROUP_SIZE.
Return true if we could use IFN_STORE_LANES instead and if that appears
to be the better approach.   

References DR_REF, get_vectype_for_scalar_type(), IFN_LAST, STMT_VINFO_DATA_REF, TREE_TYPE, TYPE_VECTOR_SUBPARTS(), and vect_store_lanes_supported().

Referenced by vect_build_slp_instance().

◆ vect_slp_prune_covered_roots()

static void vect_slp_prune_covered_roots ( slp_tree node,
hash_set< stmt_vec_info > & roots,
hash_set< slp_tree > & visited )
static

◆ vect_slp_region()

◆ vect_slp_tree_uniform_p()

static bool vect_slp_tree_uniform_p ( slp_tree node)
static
Return true when all lanes in the external or constant NODE have
the same value.   

References FOR_EACH_VEC_ELT, gcc_assert, i, is_empty(), NULL_TREE, operand_equal_p(), SLP_TREE_DEF_TYPE, SLP_TREE_SCALAR_OPS, vect_constant_def, and vect_external_def.

Referenced by vect_optimize_slp_pass::get_result_with_layout(), and vect_build_slp_tree_2().

◆ vect_split_slp_store_group()

static stmt_vec_info vect_split_slp_store_group ( stmt_vec_info first_vinfo,
unsigned group1_size )
static
Splits a group of stores, currently beginning at FIRST_VINFO, into
two groups: one (still beginning at FIRST_VINFO) of size GROUP1_SIZE
(also containing the first GROUP1_SIZE stmts, since stores are
consecutive), the second containing the remainder.
Return the first stmt in the second group.   

References DR_GROUP_FIRST_ELEMENT, DR_GROUP_GAP, DR_GROUP_NEXT_ELEMENT, DR_GROUP_SIZE, dump_enabled_p(), dump_printf_loc(), gcc_assert, i, MSG_NOTE, and vect_location.

Referenced by vect_build_slp_instance().

◆ vect_transform_slp_perm_load()

bool vect_transform_slp_perm_load ( vec_info * vinfo,
slp_tree node,
const vec< tree > & dr_chain,
gimple_stmt_iterator * gsi,
poly_uint64 vf,
bool analyze_only,
unsigned * n_perms,
unsigned int * n_loads,
bool dce_chain )
Generate vector permute statements from a list of loads in DR_CHAIN.
If ANALYZE_ONLY is TRUE, only check that it is possible to create valid
permute statements for the SLP node NODE.  Store the number of vector
permute instructions in *N_PERMS and the number of vector load
instructions in *N_LOADS.  If DCE_CHAIN is true, remove all definitions
that were not needed.   

References dump_enabled_p(), SLP_TREE_LOAD_PERMUTATION, and vect_transform_slp_perm_load_1().

Referenced by vectorizable_load().

◆ vect_transform_slp_perm_load_1() [1/2]

static bool vect_transform_slp_perm_load_1 ( vec_info * ,
slp_tree ,
load_permutation_t & ,
const vec< tree > & ,
gimple_stmt_iterator * ,
poly_uint64 ,
bool ,
bool ,
unsigned * ,
unsigned * = nullptr,
bool = false )
static

◆ vect_transform_slp_perm_load_1() [2/2]

static bool vect_transform_slp_perm_load_1 ( vec_info * vinfo,
slp_tree node,
load_permutation_t & perm,
const vec< tree > & dr_chain,
gimple_stmt_iterator * gsi,
poly_uint64 vf,
bool analyze_only,
bool dump_p,
unsigned * n_perms,
unsigned int * n_loads,
bool dce_chain )
static

◆ vectorizable_bb_reduc_epilogue()

◆ vectorizable_slp_permutation()

static bool vectorizable_slp_permutation ( vec_info * vinfo,
gimple_stmt_iterator * gsi,
slp_tree node,
stmt_vector_for_cost * cost_vec )
static
Vectorize the SLP permutations in NODE as specified
in SLP_TREE_LANE_PERMUTATION which is a vector of pairs of SLP
child number and lane number.
Interleaving of two two-lane two-child SLP subtrees (not supported):
  [ { 0, 0 }, { 1, 0 }, { 0, 1 }, { 1, 1 } ]
A blend of two four-lane two-child SLP subtrees:
  [ { 0, 0 }, { 1, 1 }, { 0, 2 }, { 1, 3 } ]
Highpart of a four-lane one-child SLP subtree (not supported):
  [ { 0, 2 }, { 0, 3 } ]
Where currently only a subset is supported by code generating below.   

References dump_enabled_p(), record_stmt_cost(), SLP_TREE_CHILDREN, SLP_TREE_LANE_PERMUTATION, SLP_TREE_VECTYPE, vec_perm, vect_body, and vectorizable_slp_permutation_1().

Referenced by vect_schedule_slp_node(), and vect_slp_analyze_node_operations_1().

◆ vectorizable_slp_permutation_1()

static int vectorizable_slp_permutation_1 ( vec_info * vinfo,
gimple_stmt_iterator * gsi,
slp_tree node,
lane_permutation_t & perm,
vec< slp_tree > & children,
bool dump_p )
static
Subroutine of vectorizable_slp_permutation.  Check whether the target
can perform permutation PERM on the (1 or 2) input nodes in CHILDREN.
If GSI is nonnull, emit the permutation there.

When GSI is null, the only purpose of NODE is to give properties
of the result, such as the vector type and number of SLP lanes.
The node does not need to be a VEC_PERM_EXPR.

If the target supports the operation, return the number of individual
VEC_PERM_EXPRs needed, otherwise return -1.  Print information to the
dump file if DUMP_P is true.   

References can_vec_perm_const_p(), count, dump_dec(), dump_printf(), dump_printf_loc(), dyn_cast(), FOR_EACH_VEC_ELT, gcc_assert, i, poly_int< N, C >::is_constant(), known_le, LOOP_VINFO_VECT_FACTOR, MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL_TREE, SLP_TREE_DEF_TYPE, SLP_TREE_LANES, SLP_TREE_NUMBER_OF_VEC_STMTS, SLP_TREE_VECTYPE, poly_int< N, C >::to_constant(), TREE_TYPE, TYPE_MODE, TYPE_VECTOR_SUBPARTS(), types_compatible_p(), vect_add_slp_permutation(), vect_gen_perm_mask_checked(), vect_get_slp_vect_def(), vect_internal_def, vect_location, and vect_maybe_update_slp_op_vectype().

Referenced by vect_optimize_slp_pass::get_result_with_layout(), vect_optimize_slp_pass::internal_node_cost(), vect_optimize_slp_pass::materialize(), and vectorizable_slp_permutation().

◆ vectorize_slp_instance_root_stmt()

◆ vld_cmp()

static int vld_cmp ( const void * a_,
const void * b_ )
static
qsort comparator for lane defs.   

References a, and b.

Referenced by vect_slp_check_for_roots().

Variable Documentation

◆ arg0_map

const int arg0_map[] = { 1, 0 }
static

Referenced by vect_get_operand_map().

◆ arg1_arg4_map

const int arg1_arg4_map[] = { 2, 1, 4 }
static

Referenced by vect_get_operand_map().

◆ arg1_map

const int arg1_map[] = { 1, 1 }
static

Referenced by vect_get_operand_map().

◆ arg2_map

const int arg2_map[] = { 1, 2 }
static

Referenced by vect_get_operand_map().

◆ arg3_arg2_map

const int arg3_arg2_map[] = { 2, 3, 2 }
static

Referenced by vect_get_operand_map().

◆ cond_expr_maps

const int cond_expr_maps[3][5]
static
Initial value:
= {
{ 4, -1, -2, 1, 2 },
{ 4, -2, -1, 1, 2 },
{ 4, -1, -2, 2, 1 }
}

Referenced by vect_get_operand_map().

◆ mask_call_maps

const int mask_call_maps[6][7]
static
Initial value:
= {
{ 1, 1, },
{ 2, 1, 2, },
{ 3, 1, 2, 3, },
{ 4, 1, 2, 3, 4, },
{ 5, 1, 2, 3, 4, 5, },
{ 6, 1, 2, 3, 4, 5, 6 },
}

Referenced by vect_get_operand_map().

◆ off_arg2_map

const int off_arg2_map[] = { 2, -3, 2 }
static

Referenced by vect_get_operand_map().

◆ off_arg3_arg2_map

const int off_arg3_arg2_map[] = { 3, -3, 3, 2 }
static

Referenced by vect_get_operand_map().

◆ off_map

const int off_map[] = { 1, -3 }
static

Referenced by vect_get_operand_map().

◆ off_op0_map

const int off_op0_map[] = { 2, -3, 0 }
static

Referenced by vect_get_operand_map().

◆ op1_op0_map

const int op1_op0_map[] = { 2, 1, 0 }
static

Referenced by vect_get_operand_map().

◆ slp_first_node

slp_tree slp_first_node
static

◆ slp_tree_pool