GCC Middle and Back End API Reference
tree-vectorizer.cc File Reference
#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "tree.h"
#include "gimple.h"
#include "predict.h"
#include "tree-pass.h"
#include "ssa.h"
#include "cgraph.h"
#include "fold-const.h"
#include "stor-layout.h"
#include "gimple-iterator.h"
#include "gimple-walk.h"
#include "tree-ssa-loop-manip.h"
#include "tree-ssa-loop-niter.h"
#include "tree-cfg.h"
#include "cfgloop.h"
#include "tree-vectorizer.h"
#include "tree-ssa-propagate.h"
#include "dbgcnt.h"
#include "tree-scalar-evolution.h"
#include "stringpool.h"
#include "attribs.h"
#include "gimple-pretty-print.h"
#include "opt-problem.h"
#include "internal-fn.h"
#include "tree-ssa-sccvn.h"
#include "tree-into-ssa.h"
Include dependency graph for tree-vectorizer.cc:

Data Structures

class  simduid_to_vf
 
struct  simd_array_to_simduid
 
struct  note_simd_array_uses_struct
 

Macros

#define INCLUDE_MEMORY
 

Functions

void dump_stmt_cost (FILE *f, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree, int misalign, unsigned cost, enum vect_cost_model_location where)
 
static void adjust_simduid_builtins (hash_table< simduid_to_vf > *htab, function *fun)
 
static tree note_simd_array_uses_cb (tree *tp, int *walk_subtrees, void *data)
 
static void note_simd_array_uses (hash_table< simd_array_to_simduid > **htab, function *fun)
 
static void shrink_simd_arrays (hash_table< simd_array_to_simduid > *simd_array_to_simduid_htab, hash_table< simduid_to_vf > *simduid_to_vf_htab)
 
bool vect_stmt_dominates_stmt_p (gimple *s1, gimple *s2)
 
void vect_free_loop_info_assumptions (class loop *loop)
 
gimplevect_loop_vectorized_call (class loop *loop, gcond **cond)
 
static gimplevect_loop_dist_alias_call (class loop *loop, function *fun)
 
static void set_uid_loop_bbs (loop_vec_info loop_vinfo, gimple *loop_vectorized_call, function *fun)
 
static unsigned vect_transform_loops (hash_table< simduid_to_vf > *&simduid_to_vf_htab, loop_p loop, gimple *loop_vectorized_call, function *fun)
 
static unsigned try_vectorize_loop_1 (hash_table< simduid_to_vf > *&simduid_to_vf_htab, unsigned *num_vectorized_loops, loop_p loop, gimple *loop_vectorized_call, gimple *loop_dist_alias_call, function *fun)
 
static unsigned try_vectorize_loop (hash_table< simduid_to_vf > *&simduid_to_vf_htab, unsigned *num_vectorized_loops, loop_p loop, function *fun)
 
gimple_opt_passmake_pass_vectorize (gcc::context *ctxt)
 
gimple_opt_passmake_pass_simduid_cleanup (gcc::context *ctxt)
 
gimple_opt_passmake_pass_slp_vectorize (gcc::context *ctxt)
 
static unsigned get_vec_alignment_for_type (tree)
 
static unsigned get_vec_alignment_for_array_type (tree type)
 
static unsigned get_vec_alignment_for_record_type (tree type)
 
static unsigned int increase_alignment (void)
 
simple_ipa_opt_passmake_pass_ipa_increase_alignment (gcc::context *ctxt)
 

Variables

dump_user_location_t vect_location
 
static hash_map< tree, unsigned > * type_align_map
 

Macro Definition Documentation

◆ INCLUDE_MEMORY

#define INCLUDE_MEMORY
Vectorizer
   Copyright (C) 2003-2024 Free Software Foundation, Inc.
   Contributed by Dorit Naishlos <dorit@il.ibm.com>

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.   
Loop and basic block vectorizer.

 This file contains drivers for the three vectorizers:
 (1) loop vectorizer (inter-iteration parallelism),
 (2) loop-aware SLP (intra-iteration parallelism) (invoked by the loop
     vectorizer)
 (3) BB vectorizer (out-of-loops), aka SLP

 The rest of the vectorizer's code is organized as follows:
 - tree-vect-loop.cc - loop specific parts such as reductions, etc. These are
   used by drivers (1) and (2).
 - tree-vect-loop-manip.cc - vectorizer's loop control-flow utilities, used by
   drivers (1) and (2).
 - tree-vect-slp.cc - BB vectorization specific analysis and transformation,
   used by drivers (2) and (3).
 - tree-vect-stmts.cc - statements analysis and transformation (used by all).
 - tree-vect-data-refs.cc - vectorizer specific data-refs analysis and
   manipulations (used by all).
 - tree-vect-patterns.cc - vectorizable code patterns detector (used by all)

 Here's a poor attempt at illustrating that:

    tree-vectorizer.cc:
    loop_vect()  loop_aware_slp()  slp_vect()
         |        /           \          /
         |       /             \        /
         tree-vect-loop.cc  tree-vect-slp.cc
               | \      \  /      /   |
               |  \      \/      /    |
               |   \     /\     /     |
               |    \   /  \   /      |
        tree-vect-stmts.cc  tree-vect-data-refs.cc
                      \      /
                   tree-vect-patterns.cc

Function Documentation

◆ adjust_simduid_builtins()

◆ dump_stmt_cost()

◆ get_vec_alignment_for_array_type()

static unsigned get_vec_alignment_for_array_type ( tree type)
static
Return alignment of array's vector type corresponding to scalar type.
0 if no vector type exists.   

References array_size, gcc_assert, get_related_vectype_for_scalar_type(), poly_int_tree_p(), strip_array_types(), TREE_CODE, TYPE_ALIGN, and TYPE_SIZE.

Referenced by get_vec_alignment_for_type().

◆ get_vec_alignment_for_record_type()

static unsigned get_vec_alignment_for_record_type ( tree type)
static
Return alignment of field having maximum alignment of vector type
corresponding to it's scalar type. For now, we only consider fields whose
offset is a multiple of it's vector alignment.
0 if no suitable field is found.   

References bit_position(), DECL_ARTIFICIAL, DECL_CHAIN, DECL_FIELD_BIT_OFFSET, DECL_FIELD_OFFSET, DECL_USER_ALIGN, first_field(), gcc_assert, hash_map< KeyId, Value, Traits >::get(), get_vec_alignment_for_type(), NULL_TREE, offset, hash_map< KeyId, Value, Traits >::put(), TREE_CODE, tree_fits_uhwi_p(), tree_to_uhwi(), TREE_TYPE, type_align_map, and TYPE_PACKED.

Referenced by get_vec_alignment_for_type().

◆ get_vec_alignment_for_type()

static unsigned get_vec_alignment_for_type ( tree type)
static
Increase alignment of global arrays to improve vectorization potential.
TODO:
- Consider also structs that have an array field.
- Use ipa analysis to prune arrays that can't be vectorized?
  This should involve global alignment analysis and in the future also
  array padding.   
Return alignment of vector type corresponding to decl's scalar type
or 0 if it doesn't exist or the vector alignment is lesser than
decl's alignment.   

References gcc_assert, get_vec_alignment_for_array_type(), get_vec_alignment_for_record_type(), NULL_TREE, TREE_CODE, TYPE_ALIGN, and TYPE_P.

Referenced by get_vec_alignment_for_record_type(), and increase_alignment().

◆ increase_alignment()

◆ make_pass_ipa_increase_alignment()

simple_ipa_opt_pass * make_pass_ipa_increase_alignment ( gcc::context * ctxt)

◆ make_pass_simduid_cleanup()

gimple_opt_pass * make_pass_simduid_cleanup ( gcc::context * ctxt)

◆ make_pass_slp_vectorize()

gimple_opt_pass * make_pass_slp_vectorize ( gcc::context * ctxt)

◆ make_pass_vectorize()

gimple_opt_pass * make_pass_vectorize ( gcc::context * ctxt)

◆ note_simd_array_uses()

◆ note_simd_array_uses_cb()

static tree note_simd_array_uses_cb ( tree * tp,
int * walk_subtrees,
void * data )
static

◆ set_uid_loop_bbs()

static void set_uid_loop_bbs ( loop_vec_info loop_vinfo,
gimple * loop_vectorized_call,
function * fun )
static
Set the uids of all the statements in basic blocks inside loop
represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal
call guarding the loop which has been if converted.   

References boolean_false_node, loop::dont_vectorize, fold_loop_internal_call(), free(), g, gcc_checking_assert, get_loop(), get_loop_body(), gimple_call_arg(), gimple_set_uid(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_start_phis(), gsi_stmt(), i, loop::inner, LOOP_VINFO_SCALAR_IV_EXIT, LOOP_VINFO_SCALAR_LOOP, loop::num_nodes, tree_to_shwi(), vec_init_loop_exit_info(), and vect_loop_vectorized_call().

Referenced by vect_transform_loops().

◆ shrink_simd_arrays()

static void shrink_simd_arrays ( hash_table< simd_array_to_simduid > * simd_array_to_simduid_htab,
hash_table< simduid_to_vf > * simduid_to_vf_htab )
static

◆ try_vectorize_loop()

static unsigned try_vectorize_loop ( hash_table< simduid_to_vf > *& simduid_to_vf_htab,
unsigned * num_vectorized_loops,
loop_p loop,
function * fun )
static

◆ try_vectorize_loop_1()

◆ vect_free_loop_info_assumptions()

void vect_free_loop_info_assumptions ( class loop * loop)

◆ vect_loop_dist_alias_call()

◆ vect_loop_vectorized_call()

gimple * vect_loop_vectorized_call ( class loop * loop,
gcond ** cond )

◆ vect_stmt_dominates_stmt_p()

◆ vect_transform_loops()

Variable Documentation

◆ type_align_map

hash_map<tree, unsigned>* type_align_map
static

◆ vect_location

dump_user_location_t vect_location
Loop or bb location, with hotness information.   

Referenced by check_load_store_for_partial_vectors(), check_scan_store(), dependence_distance_ge_vf(), vect_optimize_slp_pass::dump(), get_group_alias_ptr_type(), get_group_load_store_type(), get_load_store_type(), get_negative_load_store_type(), vect_optimize_slp_pass::get_result_with_layout(), increase_alignment(), is_simple_and_all_uses_invariant(), vect_optimize_slp_pass::materialize(), maybe_push_to_hybrid_worklist(), move_early_exit_stmts(), optimize_load_redistribution_1(), optimize_mask_stores(), parloops_is_simple_reduction(), parloops_is_slp_reduction(), process_use(), report_ploop_op(), report_vect_op(), try_vectorize_loop_1(), vect_analyze_data_ref_access(), vect_analyze_data_ref_accesses(), vect_analyze_data_ref_dependence(), vect_analyze_data_refs(), vect_analyze_early_break_dependences(), vect_analyze_group_access_1(), vect_analyze_loop(), vect_analyze_loop_1(), vect_analyze_loop_2(), vect_analyze_loop_costing(), vect_analyze_loop_form(), vect_analyze_loop_operations(), vect_analyze_scalar_cycles_1(), vect_analyze_slp(), vect_analyze_slp_instance(), vect_analyze_stmt(), vect_bb_partition_graph(), vect_bb_slp_mark_live_stmts(), vect_bb_slp_mark_live_stmts(), vect_bb_vectorization_profitable_p(), vect_build_slp_instance(), vect_build_slp_tree(), vect_build_slp_tree_1(), vect_build_slp_tree_2(), vect_can_advance_ivs_p(), vect_can_peel_nonlinear_iv_p(), vect_check_lower_bound(), vect_check_nonzero_value(), vect_check_scalar_mask(), vect_check_store_rhs(), vect_compute_data_ref_alignment(), vect_create_addr_base_for_vector_ref(), vect_create_cond_for_alias_checks(), vect_create_data_ref_ptr(), vect_create_epilog_for_reduction(), vect_cse_slp_nodes(), vect_detect_hybrid_slp(), vect_determine_mask_precision(), vect_determine_min_output_precision_1(), vect_determine_partial_vectors_and_peeling(), vect_determine_precisions_from_range(), vect_determine_precisions_from_users(), vect_determine_vectorization_factor(), vect_determine_vf_for_stmt(), vect_determine_vf_for_stmt_1(), vect_enhance_data_refs_alignment(), vect_estimate_min_profitable_iters(), vect_finish_stmt_generation_1(), vect_gen_prolog_loop_niters(), vect_get_and_check_slp_defs(), vect_get_data_access_cost(), vect_get_load_cost(), vect_get_loop_niters(), vect_get_peel_iters_epilogue(), vect_get_range_info(), vect_get_store_cost(), vect_get_vec_defs_for_operand(), vect_get_vector_types_for_stmt(), vect_grouped_load_supported(), vect_grouped_store_supported(), vect_init_vector_1(), vect_is_simple_iv_evolution(), vect_is_simple_reduction(), vect_is_simple_use(), vect_is_simple_use(), vect_joust_loop_vinfos(), vect_lanes_optab_supported_p(), vect_loop_kill_debug_uses(), vect_loop_versioning(), vect_make_slp_decision(), vect_mark_for_runtime_alias_test(), vect_mark_pattern_stmts(), vect_mark_relevant(), vect_mark_stmts_to_be_vectorized(), vect_match_slp_patterns(), vect_model_promotion_demotion_cost(), vect_model_simple_cost(), vect_pattern_detected(), vect_pattern_recog_1(), vect_pattern_validate_optab(), vect_prepare_for_masked_peels(), vect_prune_runtime_alias_test_list(), vect_recog_average_pattern(), vect_recog_cond_expr_convert_pattern(), vect_recog_ctz_ffs_pattern(), vect_recog_mulhs_pattern(), vect_recog_over_widening_pattern(), vect_recog_popcount_clz_ctz_ffs_pattern(), vect_record_base_alignment(), vect_record_max_nunits(), vect_reduction_update_partial_vector_usage(), vect_schedule_slp(), vect_schedule_slp_node(), vect_set_loop_condition(), vect_shift_permute_load_chain(), vect_slp_analyze_bb_1(), vect_slp_analyze_data_ref_dependence(), vect_slp_analyze_node_operations(), vect_slp_analyze_operations(), vect_slp_bbs(), vect_slp_convert_to_external(), vect_slp_function(), vect_slp_region(), vect_split_slp_store_group(), vect_split_statement(), vect_stmt_relevant_p(), vect_transform_loop(), vect_transform_loop_stmt(), vect_transform_loops(), vect_transform_reduction(), vect_transform_slp_perm_load_1(), vect_transform_stmt(), vect_truncate_gather_scatter_offset(), vect_update_ivs_after_vectorizer(), vect_update_misalignment_for_peel(), vect_update_vf_for_slp(), vect_use_strided_gather_scatters_p(), vect_verify_loop_lens(), vector_alignment_reachable_p(), vectorizable_assignment(), vectorizable_bb_reduc_epilogue(), vectorizable_bswap(), vectorizable_call(), vectorizable_comparison_1(), vectorizable_condition(), vectorizable_conversion(), vectorizable_early_exit(), vectorizable_induction(), vectorizable_lane_reducing(), vectorizable_lc_phi(), vectorizable_live_operation(), vectorizable_load(), vectorizable_nonlinear_induction(), vectorizable_operation(), vectorizable_phi(), vectorizable_recurr(), vectorizable_reduction(), vectorizable_scan_store(), vectorizable_shift(), vectorizable_simd_clone_call(), vectorizable_slp_permutation_1(), vectorizable_store(), and auto_purge_vect_location::~auto_purge_vect_location().