#include "config.h"#include "system.h"#include "coretypes.h"#include "backend.h"#include "tree.h"#include "gimple.h"#include "predict.h"#include "tree-pass.h"#include "ssa.h"#include "cgraph.h"#include "fold-const.h"#include "stor-layout.h"#include "gimple-iterator.h"#include "gimple-walk.h"#include "tree-ssa-loop-manip.h"#include "tree-ssa-loop-niter.h"#include "tree-cfg.h"#include "cfgloop.h"#include "tree-vectorizer.h"#include "tree-ssa-propagate.h"#include "dbgcnt.h"#include "tree-scalar-evolution.h"#include "stringpool.h"#include "attribs.h"#include "gimple-pretty-print.h"#include "opt-problem.h"#include "internal-fn.h"#include "tree-ssa-sccvn.h"#include "tree-into-ssa.h"#include "gimple-range.h"
Data Structures | |
| class | simduid_to_vf |
| struct | simd_array_to_simduid |
| struct | note_simd_array_uses_struct |
Variables | |
| dump_user_location_t | vect_location |
| static hash_map< tree, unsigned > * | type_align_map |
Function Documentation
◆ adjust_simduid_builtins()
|
static |
Fold IFN_GOMP_SIMD_LANE, IFN_GOMP_SIMD_VF, IFN_GOMP_SIMD_LAST_LANE,
into their corresponding constants and remove
IFN_GOMP_SIMD_ORDERED_{START,END}.
References adjust_simduid_builtins(), build_int_cst(), builtin_decl_explicit(), DECL_UID, hash_table< Descriptor, Lazy, Allocator >::find(), FOR_EACH_BB_FN, g, gcc_assert, gcc_unreachable, gimple_build_call(), gimple_call_arg(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_lhs(), gimple_move_vops(), gsi_end_p(), gsi_next(), gsi_remove(), gsi_replace(), gsi_start_bb(), gsi_stmt(), i, integer_onep(), is_gimple_call(), basic_block_def::loop_father, NULL, NULL_TREE, release_defs(), replace_uses_by(), loop::safelen, SSA_NAME_VAR, TREE_CODE, unlink_stmt_vdef(), unsigned_type_node, and simduid_to_vf::vf.
Referenced by adjust_simduid_builtins().
◆ dump_stmt_cost()
| void dump_stmt_cost | ( | FILE * | f, |
| int | count, | ||
| enum vect_cost_for_stmt | kind, | ||
| stmt_vec_info | stmt_info, | ||
| slp_tree | node, | ||
| tree | , | ||
| int | misalign, | ||
| unsigned | cost, | ||
| enum vect_cost_model_location | where ) |
Dump a cost entry according to args to F.
References cond_branch_not_taken, cond_branch_taken, dump_stmt_cost(), print_gimple_expr(), scalar_load, scalar_stmt, scalar_store, scalar_to_vec, STMT_VINFO_STMT, TDF_SLIM, unaligned_load, unaligned_store, vec_construct, vec_deconstruct, vec_perm, vec_promote_demote, vec_to_scalar, vect_body, vect_epilogue, vect_prologue, vector_gather_load, vector_load, vector_scatter_store, vector_stmt, and vector_store.
Referenced by add_stmt_cost(), and dump_stmt_cost().
◆ get_vec_alignment_for_array_type()
|
static |
Return alignment of array's vector type corresponding to scalar type. 0 if no vector type exists.
References gcc_assert, get_related_vectype_for_scalar_type(), get_vec_alignment_for_array_type(), poly_int_tree_p(), strip_array_types(), TREE_CODE, TYPE_ALIGN, and TYPE_SIZE.
Referenced by get_vec_alignment_for_array_type(), and get_vec_alignment_for_type().
◆ get_vec_alignment_for_record_type()
|
static |
Return alignment of field having maximum alignment of vector type corresponding to it's scalar type. For now, we only consider fields whose offset is a multiple of it's vector alignment. 0 if no suitable field is found.
References bit_position(), DECL_ARTIFICIAL, DECL_CHAIN, DECL_FIELD_BIT_OFFSET, DECL_FIELD_OFFSET, DECL_USER_ALIGN, first_field(), gcc_assert, hash_map< KeyId, Value, Traits >::get(), get_vec_alignment_for_record_type(), get_vec_alignment_for_type(), NULL_TREE, hash_map< KeyId, Value, Traits >::put(), TREE_CODE, tree_fits_uhwi_p(), tree_to_uhwi(), TREE_TYPE, type_align_map, and TYPE_PACKED.
Referenced by get_vec_alignment_for_record_type(), and get_vec_alignment_for_type().
◆ get_vec_alignment_for_type()
|
static |
Increase alignment of global arrays to improve vectorization potential. TODO: - Consider also structs that have an array field. - Use ipa analysis to prune arrays that can't be vectorized? This should involve global alignment analysis and in the future also array padding.
Return alignment of vector type corresponding to decl's scalar type or 0 if it doesn't exist or the vector alignment is lesser than decl's alignment.
References gcc_assert, get_vec_alignment_for_array_type(), get_vec_alignment_for_record_type(), get_vec_alignment_for_type(), NULL_TREE, TREE_CODE, TYPE_ALIGN, and TYPE_P.
Referenced by get_vec_alignment_for_record_type(), get_vec_alignment_for_type(), and increase_alignment().
◆ increase_alignment()
|
static |
Entry point to increase_alignment pass.
References symtab_node::decl, DECL_ARTIFICIAL, decl_in_symtab_p(), DECL_USER_ALIGN, dump_enabled_p(), dump_printf(), dump_user_location_t::dump_user_location_t(), FOR_EACH_DEFINED_VARIABLE, symtab_node::get(), get_vec_alignment_for_type(), increase_alignment(), symtab_node::increase_alignment(), MSG_NOTE, TREE_TYPE, type_align_map, vect_can_force_dr_alignment_p(), and vect_location.
Referenced by increase_alignment().
◆ make_pass_ipa_increase_alignment()
| simple_ipa_opt_pass * make_pass_ipa_increase_alignment | ( | gcc::context * | ctxt | ) |
References make_pass_ipa_increase_alignment().
Referenced by make_pass_ipa_increase_alignment(), and make_pass_sprintf_length().
◆ make_pass_simduid_cleanup()
| gimple_opt_pass * make_pass_simduid_cleanup | ( | gcc::context * | ctxt | ) |
References make_pass_simduid_cleanup().
Referenced by make_pass_empty_loop(), and make_pass_simduid_cleanup().
◆ make_pass_slp_vectorize()
| gimple_opt_pass * make_pass_slp_vectorize | ( | gcc::context * | ctxt | ) |
References make_pass_slp_vectorize().
Referenced by make_pass_empty_loop(), and make_pass_slp_vectorize().
◆ make_pass_vectorize()
| gimple_opt_pass * make_pass_vectorize | ( | gcc::context * | ctxt | ) |
References make_pass_vectorize().
Referenced by make_pass_empty_loop(), and make_pass_vectorize().
◆ note_simd_array_uses()
|
static |
Find "omp simd array" temporaries and map them to corresponding simduid.
References DECL_UID, FOR_EACH_BB_FN, FOR_EACH_IMM_USE_STMT, gimple_call_arg(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_lhs(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_stmt(), note_simd_array_uses_struct::htab, is_gimple_call(), is_gimple_debug(), note_simd_array_uses(), note_simd_array_uses_cb(), NULL_TREE, note_simd_array_uses_struct::simduid, SSA_NAME_VAR, and walk_gimple_op().
Referenced by note_simd_array_uses().
◆ note_simd_array_uses_cb()
Callback for note_simd_array_uses, called through walk_gimple_op.
References current_function_decl, DECL_ATTRIBUTES, DECL_CONTEXT, note_simd_array_uses_struct::htab, lookup_attribute(), note_simd_array_uses_cb(), NULL, NULL_TREE, note_simd_array_uses_struct::simduid, TYPE_P, and VAR_P.
Referenced by note_simd_array_uses(), and note_simd_array_uses_cb().
◆ set_uid_loop_bbs()
|
static |
Set the uids of all the statements in basic blocks inside loop represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal call guarding the loop which has been if converted.
References boolean_false_node, loop::dont_vectorize, fold_loop_internal_call(), free(), g, gcc_checking_assert, get_loop(), get_loop_body(), gimple_call_arg(), gimple_set_uid(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_start_phis(), gsi_stmt(), i, loop::inner, LOOP_VINFO_SCALAR_LOOP, LOOP_VINFO_SCALAR_MAIN_EXIT, loop::num_nodes, set_uid_loop_bbs(), tree_to_shwi(), vec_init_loop_exit_info(), and vect_loop_vectorized_call().
Referenced by set_uid_loop_bbs(), and vect_transform_loops().
◆ shrink_simd_arrays()
|
static |
Shrink arrays with "omp simd array" attribute to the corresponding vectorization factor.
References hash_table< Descriptor, Lazy, Allocator >::begin(), build_array_type_nelts(), decl::decl, hash_table< Descriptor, Lazy, Allocator >::end(), hash_table< Descriptor, Lazy, Allocator >::find(), NULL, relayout_decl(), shrink_simd_arrays(), TREE_TYPE, and simduid_to_vf::vf.
Referenced by shrink_simd_arrays().
◆ try_vectorize_loop()
|
static |
Try to vectorize LOOP.
References loop::force_vectorize, optimize_loop_nest_for_speed_p(), try_vectorize_loop(), try_vectorize_loop_1(), vect_loop_dist_alias_call(), and vect_loop_vectorized_call().
Referenced by try_vectorize_loop().
◆ try_vectorize_loop_1()
|
static |
Try to vectorize LOOP.
References loop::aux, boolean_true_node, dbg_cnt(), direct_internal_fn_p(), direct_internal_fn_supported_p(), loop::dont_vectorize, dump_enabled_p(), dump_printf(), dump_printf_loc(), dyn_cast(), find_loop_location(), fold_loop_internal_call(), dump_user_location_t::get_location_t(), get_loop(), opt_wrapper< PtrType_t >::get_problem(), gimple_call_arg(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_set_uid(), gimple_set_visited(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_stmt(), loop::header, loop::inner, LOCATION_FILE, LOCATION_LINE, LOCATION_LOCUS, LOOP_C_FINITE, loop_constraint_set_p(), LOOP_VINFO_VECTORIZABLE_P, MSG_MISSED_OPTIMIZATION, MSG_NOTE, MSG_PRIORITY_INTERNALS, NULL, OPTIMIZE_FOR_SPEED, TODO_cleanup_cfg, TODO_update_ssa_only_virtuals, tree_to_shwi(), try_vectorize_loop_1(), UNKNOWN_LOCATION, vect_analyze_loop(), vect_free_loop_info_assumptions(), vect_location, vect_slp_if_converted_bb(), and vect_transform_loops().
Referenced by try_vectorize_loop(), and try_vectorize_loop_1().
◆ vect_free_loop_info_assumptions()
| void vect_free_loop_info_assumptions | ( | class loop * | loop | ) |
A helper function to free scev and LOOP niter information, as well as clear loop constraint LOOP_C_FINITE.
References loop::any_likely_upper_bound, loop::any_upper_bound, free_numbers_of_iterations_estimates(), LOOP_C_FINITE, loop_constraint_clear(), scev_reset_htab(), and vect_free_loop_info_assumptions().
Referenced by fwd_jt_path_registry::mark_threaded_blocks(), back_threader::maybe_register_path(), try_vectorize_loop_1(), vect_free_loop_info_assumptions(), vect_loop_versioning(), and vectorize_loops().
◆ vect_loop_dist_alias_call()
If LOOP has been versioned during loop distribution, return the gurading internal call.
References CDI_DOMINATORS, ENTRY_BLOCK_PTR_FOR_FN, flow_bb_inside_loop_p(), g, get_immediate_dominator(), get_loop(), gimple_call_arg(), gimple_call_internal_p(), gsi_end_p(), gsi_last_bb(), gsi_prev(), gsi_stmt(), loop::header, basic_block_def::loop_father, loop_preheader_edge(), nearest_common_dominator(), NULL, loop::orig_loop_num, safe_is_a(), tree_to_shwi(), and vect_loop_dist_alias_call().
Referenced by try_vectorize_loop(), and vect_loop_dist_alias_call().
◆ vect_loop_vectorized_call()
If LOOP has been versioned during ifcvt, return the internal call guarding it.
References as_a(), g, gimple_call_arg(), gimple_call_internal_p(), gsi_end_p(), gsi_for_stmt(), gsi_last_bb(), gsi_prev(), gsi_stmt(), loop_preheader_edge(), NULL, loop::num, single_pred(), single_pred_p(), single_succ_p(), tree_to_shwi(), and vect_loop_vectorized_call().
Referenced by set_uid_loop_bbs(), try_vectorize_loop(), vect_loop_vectorized_call(), vect_loop_versioning(), and vectorize_loops().
◆ vect_stmt_dominates_stmt_p()
Returns true if S1 dominates S2.
References CDI_DOMINATORS, dominated_by_p(), gimple_bb(), gimple_uid(), gsi_end_p(), gsi_for_stmt(), gsi_next(), gsi_prev(), gsi_stmt(), and vect_stmt_dominates_stmt_p().
Referenced by vect_bb_slp_mark_live_stmts(), vect_phi_first_order_recurrence_p(), vect_schedule_slp(), vect_schedule_slp_node(), vect_slp_analyze_instance_dependence(), vect_stmt_dominates_stmt_p(), vectorizable_live_operation(), and vectorize_loops().
◆ vect_transform_loops()
|
static |
Generate vectorized code for LOOP and its epilogues.
References vec_info::any_known_not_updated_vssa, cfun, DECL_UID, dump_enabled_p(), dump_printf_loc(), hash_table< Descriptor, Lazy, Allocator >::find_slot(), loop::force_vectorize, gcc_assert, GET_MODE_SIZE(), function::gimple_df, hash_table< Descriptor, Lazy, Allocator >::hash_table(), poly_int< N, C >::is_constant(), loop_vec_info_for_loop(), LOOP_VINFO_EPILOGUE_P, LOOP_VINFO_USING_PARTIAL_VECTORS_P, LOOP_VINFO_VECT_FACTOR, MSG_OPTIMIZED_LOCATIONS, need_ssa_update_p(), NULL, set_uid_loop_bbs(), loop::simduid, simduid_to_vf::simduid, gimple_df::ssa_renaming_needed, TODO_update_ssa_only_virtuals, vect_location, vect_transform_loop(), vect_transform_loops(), vec_info::vector_mode, _loop_vec_info::vectorization_factor, and simduid_to_vf::vf.
Referenced by try_vectorize_loop_1(), and vect_transform_loops().
Variable Documentation
◆ type_align_map
Referenced by get_vec_alignment_for_record_type(), and increase_alignment().
◆ vect_location
| dump_user_location_t vect_location |
Vectorizer Copyright (C) 2003-2026 Free Software Foundation, Inc. Contributed by Dorit Naishlos <dorit@il.ibm.com> This file is part of GCC. GCC is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. GCC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GCC; see the file COPYING3. If not see <http://www.gnu.org/licenses/>.
Loop and basic block vectorizer.
This file contains drivers for the three vectorizers:
(1) loop vectorizer (inter-iteration parallelism),
(2) loop-aware SLP (intra-iteration parallelism) (invoked by the loop
vectorizer)
(3) BB vectorizer (out-of-loops), aka SLP
The rest of the vectorizer's code is organized as follows:
- tree-vect-loop.cc - loop specific parts such as reductions, etc. These are
used by drivers (1) and (2).
- tree-vect-loop-manip.cc - vectorizer's loop control-flow utilities, used by
drivers (1) and (2).
- tree-vect-slp.cc - BB vectorization specific analysis and transformation,
used by drivers (2) and (3).
- tree-vect-stmts.cc - statements analysis and transformation (used by all).
- tree-vect-data-refs.cc - vectorizer specific data-refs analysis and
manipulations (used by all).
- tree-vect-patterns.cc - vectorizable code patterns detector (used by all)
Here's a poor attempt at illustrating that:
tree-vectorizer.cc:
loop_vect() loop_aware_slp() slp_vect()
| / \ /
| / \ /
tree-vect-loop.cc tree-vect-slp.cc
| \ \ / / |
| \ \/ / |
| \ /\ / |
| \ / \ / |
tree-vect-stmts.cc tree-vect-data-refs.cc
\ /
tree-vect-patterns.cc
Loop or bb location, with hotness information.
Source location + hotness information.
Referenced by append_pattern_def_seq(), check_load_store_for_partial_vectors(), check_scan_store(), dependence_distance_ge_vf(), vect_optimize_slp_pass::dump(), force_split_matches(), get_group_alias_ptr_type(), get_load_store_type(), get_negative_load_store_type(), vect_optimize_slp_pass::get_result_with_layout(), increase_alignment(), is_simple_and_all_uses_invariant(), vect_optimize_slp_pass::materialize(), move_early_exit_stmts(), optimize_load_redistribution_1(), optimize_mask_stores(), parloops_is_simple_reduction(), process_use(), report_ploop_op(), report_vect_op(), try_vectorize_loop_1(), vec_init_loop_exit_info(), vect_analyze_data_ref_access(), vect_analyze_data_ref_accesses(), vect_analyze_data_ref_dependence(), vect_analyze_data_refs(), vect_analyze_early_break_dependences(), vect_analyze_group_access_1(), vect_analyze_loop(), vect_analyze_loop_1(), vect_analyze_loop_2(), vect_analyze_loop_costing(), vect_analyze_loop_form(), vect_analyze_scalar_cycles_1(), vect_analyze_slp(), vect_analyze_slp_instance(), vect_analyze_slp_reduc_chain(), vect_analyze_slp_reduction(), vect_analyze_slp_reduction_group(), vect_analyze_stmt(), vect_bb_partition_graph(), vect_bb_slp_mark_live_stmts(), vect_bb_slp_mark_live_stmts(), vect_bb_slp_scalar_cost(), vect_bb_vectorization_profitable_p(), vect_build_slp_instance(), vect_build_slp_tree(), vect_build_slp_tree_1(), vect_build_slp_tree_2(), vect_build_slp_tree_3(), vect_can_advance_ivs_p(), vect_can_peel_nonlinear_iv_p(), vect_check_lower_bound(), vect_check_nonzero_value(), vect_check_scalar_mask(), vect_check_store_rhs(), vect_compute_data_ref_alignment(), vect_compute_type_for_early_break_scalar_iv(), vect_create_addr_base_for_vector_ref(), vect_create_cond_for_alias_checks(), vect_create_data_ref_ptr(), vect_create_epilog_for_reduction(), vect_cse_slp_nodes(), vect_determine_mask_precision(), vect_determine_min_output_precision_1(), vect_determine_partial_vectors_and_peeling(), vect_determine_precisions_from_range(), vect_determine_precisions_from_users(), vect_do_peeling(), vect_enhance_data_refs_alignment(), vect_estimate_min_profitable_iters(), vect_finish_stmt_generation_1(), vect_gen_prolog_loop_niters(), vect_get_and_check_slp_defs(), vect_get_data_access_cost(), vect_get_load_cost(), vect_get_loop_niters(), vect_get_peel_iters_epilogue(), vect_get_range_info(), vect_get_store_cost(), vect_get_vector_types_for_stmt(), vect_grouped_load_supported(), vect_grouped_store_supported(), vect_init_pattern_stmt(), vect_init_vector_1(), vect_is_simple_iv_evolution(), vect_is_simple_reduction(), vect_is_simple_use(), vect_joust_loop_vinfos(), vect_lanes_optab_supported_p(), vect_loop_versioning(), vect_make_slp_decision(), vect_mark_for_runtime_alias_test(), vect_mark_pattern_stmts(), vect_mark_relevant(), vect_mark_stmts_to_be_vectorized(), vect_match_slp_patterns(), vect_model_promotion_demotion_cost(), vect_model_simple_cost(), vect_pattern_detected(), vect_pattern_recog_1(), vect_pattern_validate_optab(), vect_prepare_for_masked_peels(), vect_prune_runtime_alias_test_list(), vect_recog_average_pattern(), vect_recog_cond_expr_convert_pattern(), vect_recog_ctz_ffs_pattern(), vect_recog_mulhs_pattern(), vect_recog_over_widening_pattern(), vect_recog_popcount_clz_ctz_ffs_pattern(), vect_record_base_alignment(), vect_reduction_update_partial_vector_usage(), vect_schedule_slp(), vect_schedule_slp_node(), vect_set_loop_condition(), vect_slp_analyze_bb_1(), vect_slp_analyze_data_ref_dependence(), vect_slp_analyze_node_operations(), vect_slp_analyze_operations(), vect_slp_bbs(), vect_slp_convert_to_external(), vect_slp_function(), vect_slp_region(), vect_split_slp_store_group(), vect_split_statement(), vect_stmt_relevant_p(), vect_transform_cycle_phi(), vect_transform_loop(), vect_transform_loops(), vect_transform_reduction(), vect_transform_slp_perm_load_1(), vect_transform_stmt(), vect_truncate_gather_scatter_offset(), vect_update_ivs_after_vectorizer(), vect_update_misalignment_for_peel(), vect_use_strided_gather_scatters_p(), vect_verify_loop_lens(), vector_alignment_reachable_p(), vectorizable_assignment(), vectorizable_bb_reduc_epilogue(), vectorizable_bswap(), vectorizable_call(), vectorizable_comparison_1(), vectorizable_condition(), vectorizable_conversion(), vectorizable_early_exit(), vectorizable_induction(), vectorizable_lane_reducing(), vectorizable_lc_phi(), vectorizable_live_operation(), vectorizable_load(), vectorizable_nonlinear_induction(), vectorizable_operation(), vectorizable_phi(), vectorizable_recurr(), vectorizable_reduction(), vectorizable_scan_store(), vectorizable_shift(), vectorizable_simd_clone_call(), vectorizable_slp_permutation_1(), vectorizable_store(), and auto_purge_vect_location::~auto_purge_vect_location().
