#include "config.h"#include "system.h"#include "coretypes.h"#include "backend.h"#include "tree.h"#include "gimple.h"#include "predict.h"#include "tree-pass.h"#include "ssa.h"#include "cgraph.h"#include "fold-const.h"#include "stor-layout.h"#include "gimple-iterator.h"#include "gimple-walk.h"#include "tree-ssa-loop-manip.h"#include "tree-ssa-loop-niter.h"#include "tree-cfg.h"#include "cfgloop.h"#include "tree-vectorizer.h"#include "tree-ssa-propagate.h"#include "dbgcnt.h"#include "tree-scalar-evolution.h"#include "stringpool.h"#include "attribs.h"#include "gimple-pretty-print.h"#include "opt-problem.h"#include "internal-fn.h"#include "tree-ssa-sccvn.h"#include "tree-into-ssa.h"
Data Structures | |
| class | simduid_to_vf |
| struct | simd_array_to_simduid |
| struct | note_simd_array_uses_struct |
Variables | |
| dump_user_location_t | vect_location |
| static hash_map< tree, unsigned > * | type_align_map |
Function Documentation
◆ adjust_simduid_builtins()
|
static |
Fold IFN_GOMP_SIMD_LANE, IFN_GOMP_SIMD_VF, IFN_GOMP_SIMD_LAST_LANE,
into their corresponding constants and remove
IFN_GOMP_SIMD_ORDERED_{START,END}.
References build_int_cst(), builtin_decl_explicit(), DECL_UID, hash_table< Descriptor, Lazy, Allocator >::find(), FOR_EACH_BB_FN, g, gcc_assert, gcc_unreachable, ggc_alloc(), gimple_build_call(), gimple_call_arg(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_lhs(), gimple_move_vops(), gsi_end_p(), gsi_next(), gsi_remove(), gsi_replace(), gsi_start_bb(), gsi_stmt(), i, integer_onep(), is_gimple_call(), basic_block_def::loop_father, NULL, NULL_TREE, release_defs(), replace_uses_by(), loop::safelen, SSA_NAME_VAR, TREE_CODE, unlink_stmt_vdef(), unsigned_type_node, and simduid_to_vf::vf.
◆ dump_stmt_cost()
| void dump_stmt_cost | ( | FILE * | f, |
| int | count, | ||
| enum vect_cost_for_stmt | kind, | ||
| stmt_vec_info | stmt_info, | ||
| slp_tree | node, | ||
| tree | , | ||
| int | misalign, | ||
| unsigned | cost, | ||
| enum vect_cost_model_location | where ) |
Dump a cost entry according to args to F.
References cond_branch_not_taken, cond_branch_taken, count, ggc_alloc(), print_gimple_expr(), scalar_load, scalar_stmt, scalar_store, scalar_to_vec, STMT_VINFO_STMT, TDF_SLIM, unaligned_load, unaligned_store, vec_construct, vec_perm, vec_promote_demote, vec_to_scalar, vect_body, vect_epilogue, vect_prologue, vector_gather_load, vector_load, vector_scatter_store, vector_stmt, and vector_store.
Referenced by add_stmt_cost().
◆ get_vec_alignment_for_array_type()
Return alignment of array's vector type corresponding to scalar type. 0 if no vector type exists.
References array_size, gcc_assert, get_related_vectype_for_scalar_type(), ggc_alloc(), poly_int_tree_p(), strip_array_types(), TREE_CODE, TYPE_ALIGN, and TYPE_SIZE.
Referenced by get_vec_alignment_for_type().
◆ get_vec_alignment_for_record_type()
Return alignment of field having maximum alignment of vector type corresponding to it's scalar type. For now, we only consider fields whose offset is a multiple of it's vector alignment. 0 if no suitable field is found.
References bit_position(), DECL_ARTIFICIAL, DECL_CHAIN, DECL_FIELD_BIT_OFFSET, DECL_FIELD_OFFSET, DECL_USER_ALIGN, first_field(), gcc_assert, hash_map< KeyId, Value, Traits >::get(), get_vec_alignment_for_type(), ggc_alloc(), NULL_TREE, offset, hash_map< KeyId, Value, Traits >::put(), TREE_CODE, tree_fits_uhwi_p(), tree_to_uhwi(), TREE_TYPE, type_align_map, and TYPE_PACKED.
Referenced by get_vec_alignment_for_type().
◆ get_vec_alignment_for_type()
Increase alignment of global arrays to improve vectorization potential. TODO: - Consider also structs that have an array field. - Use ipa analysis to prune arrays that can't be vectorized? This should involve global alignment analysis and in the future also array padding.
Return alignment of vector type corresponding to decl's scalar type or 0 if it doesn't exist or the vector alignment is lesser than decl's alignment.
References gcc_assert, get_vec_alignment_for_array_type(), get_vec_alignment_for_record_type(), ggc_alloc(), NULL_TREE, TREE_CODE, TYPE_ALIGN, and TYPE_P.
Referenced by get_vec_alignment_for_record_type(), and increase_alignment().
◆ increase_alignment()
Entry point to increase_alignment pass.
References DECL_ARTIFICIAL, decl_in_symtab_p(), DECL_USER_ALIGN, dump_enabled_p(), dump_printf(), FOR_EACH_DEFINED_VARIABLE, symtab_node::get(), get_vec_alignment_for_type(), ggc_alloc(), MSG_NOTE, TREE_TYPE, type_align_map, vect_can_force_dr_alignment_p(), and vect_location.
◆ make_pass_ipa_increase_alignment()
| simple_ipa_opt_pass * make_pass_ipa_increase_alignment | ( | gcc::context * | ctxt | ) |
References ggc_alloc().
◆ make_pass_simduid_cleanup()
| gimple_opt_pass * make_pass_simduid_cleanup | ( | gcc::context * | ctxt | ) |
References ggc_alloc().
◆ make_pass_slp_vectorize()
| gimple_opt_pass * make_pass_slp_vectorize | ( | gcc::context * | ctxt | ) |
References ggc_alloc().
◆ make_pass_vectorize()
| gimple_opt_pass * make_pass_vectorize | ( | gcc::context * | ctxt | ) |
References ggc_alloc().
◆ note_simd_array_uses()
|
static |
Find "omp simd array" temporaries and map them to corresponding simduid.
References DECL_UID, FOR_EACH_BB_FN, FOR_EACH_IMM_USE_STMT, ggc_alloc(), gimple_call_arg(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_lhs(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_stmt(), note_simd_array_uses_struct::htab, is_gimple_call(), is_gimple_debug(), note_simd_array_uses_cb(), NULL_TREE, note_simd_array_uses_struct::simduid, SSA_NAME_VAR, and walk_gimple_op().
◆ note_simd_array_uses_cb()
Callback for note_simd_array_uses, called through walk_gimple_op.
References current_function_decl, DECL_ATTRIBUTES, DECL_CONTEXT, ggc_alloc(), note_simd_array_uses_struct::htab, lookup_attribute(), NULL, NULL_TREE, simd_array_to_simduid::simduid, note_simd_array_uses_struct::simduid, TYPE_P, and VAR_P.
Referenced by note_simd_array_uses().
◆ set_uid_loop_bbs()
|
static |
Set the uids of all the statements in basic blocks inside loop represented by LOOP_VINFO. LOOP_VECTORIZED_CALL is the internal call guarding the loop which has been if converted.
References boolean_false_node, loop::dont_vectorize, fold_loop_internal_call(), free(), g, gcc_checking_assert, get_loop(), get_loop_body(), ggc_alloc(), gimple_call_arg(), gimple_set_uid(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_start_phis(), gsi_stmt(), i, loop::inner, LOOP_VINFO_SCALAR_IV_EXIT, LOOP_VINFO_SCALAR_LOOP, loop::num_nodes, tree_to_shwi(), vec_init_loop_exit_info(), and vect_loop_vectorized_call().
Referenced by vect_transform_loops().
◆ shrink_simd_arrays()
|
static |
Shrink arrays with "omp simd array" attribute to the corresponding vectorization factor.
References build_array_type_nelts(), decl::decl, ggc_alloc(), NULL, relayout_decl(), TREE_TYPE, and simduid_to_vf::vf.
◆ try_vectorize_loop()
|
static |
Try to vectorize LOOP.
References loop::force_vectorize, ggc_alloc(), optimize_loop_nest_for_speed_p(), try_vectorize_loop_1(), vect_loop_dist_alias_call(), and vect_loop_vectorized_call().
◆ try_vectorize_loop_1()
|
static |
Try to vectorize LOOP.
References loop::aux, boolean_true_node, dbg_cnt(), direct_internal_fn_p(), direct_internal_fn_supported_p(), loop::dont_vectorize, dump_enabled_p(), dump_printf(), dump_printf_loc(), find_loop_location(), fold_loop_internal_call(), dump_user_location_t::get_location_t(), get_loop(), opt_wrapper< T >::get_problem(), ggc_alloc(), gimple_call_arg(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_set_uid(), gimple_set_visited(), gsi_end_p(), gsi_next(), gsi_start_bb(), gsi_stmt(), loop::header, loop::inner, LOCATION_FILE, LOCATION_LINE, LOCATION_LOCUS, LOOP_C_FINITE, loop_constraint_set_p(), LOOP_VINFO_VECTORIZABLE_P, MSG_MISSED_OPTIMIZATION, MSG_NOTE, MSG_PRIORITY_INTERNALS, NULL, OPTIMIZE_FOR_SPEED, TODO_cleanup_cfg, TODO_update_ssa_only_virtuals, tree_to_shwi(), UNKNOWN_LOCATION, vect_analyze_loop(), vect_free_loop_info_assumptions(), vect_location, vect_slp_if_converted_bb(), and vect_transform_loops().
Referenced by try_vectorize_loop().
◆ vect_free_loop_info_assumptions()
A helper function to free scev and LOOP niter information, as well as clear loop constraint LOOP_C_FINITE.
References loop::any_likely_upper_bound, loop::any_upper_bound, free_numbers_of_iterations_estimates(), LOOP_C_FINITE, loop_constraint_clear(), and scev_reset_htab().
Referenced by fwd_jt_path_registry::mark_threaded_blocks(), back_threader::maybe_register_path(), try_vectorize_loop_1(), and vect_loop_versioning().
◆ vect_loop_dist_alias_call()
If LOOP has been versioned during loop distribution, return the gurading internal call.
References CDI_DOMINATORS, ENTRY_BLOCK_PTR_FOR_FN, flow_bb_inside_loop_p(), g, get_immediate_dominator(), get_loop(), ggc_alloc(), gimple_call_arg(), gimple_call_internal_p(), gsi_end_p(), gsi_last_bb(), gsi_prev(), gsi_stmt(), loop::header, basic_block_def::loop_father, loop_preheader_edge(), nearest_common_dominator(), NULL, loop::orig_loop_num, and tree_to_shwi().
Referenced by try_vectorize_loop().
◆ vect_loop_vectorized_call()
If LOOP has been versioned during ifcvt, return the internal call guarding it.
References g, ggc_alloc(), gimple_call_arg(), gimple_call_internal_p(), gsi_end_p(), gsi_for_stmt(), gsi_last_bb(), gsi_prev(), gsi_stmt(), loop_preheader_edge(), NULL, loop::num, single_pred(), single_pred_p(), single_succ_p(), and tree_to_shwi().
Referenced by set_uid_loop_bbs(), try_vectorize_loop(), and vect_loop_versioning().
◆ vect_stmt_dominates_stmt_p()
Returns true if S1 dominates S2.
References CDI_DOMINATORS, dominated_by_p(), ggc_alloc(), gimple_bb(), gimple_uid(), gsi_end_p(), gsi_for_stmt(), gsi_next(), gsi_prev(), and gsi_stmt().
Referenced by vect_bb_slp_mark_live_stmts(), vect_phi_first_order_recurrence_p(), vect_schedule_slp_node(), and vectorizable_live_operation().
◆ vect_transform_loops()
|
static |
Generate vectorized code for LOOP and its epilogues.
References vec_info::any_known_not_updated_vssa, cfun, DECL_UID, dump_enabled_p(), dump_printf_loc(), loop::force_vectorize, gcc_assert, GET_MODE_SIZE(), ggc_alloc(), function::gimple_df, poly_int< N, C >::is_constant(), loop_vec_info_for_loop(), MSG_OPTIMIZED_LOCATIONS, need_ssa_update_p(), NULL, set_uid_loop_bbs(), loop::simduid, gimple_df::ssa_renaming_needed, todo, TODO_update_ssa_only_virtuals, vect_location, vect_transform_loop(), vect_transform_loops(), vec_info::vector_mode, and _loop_vec_info::vectorization_factor.
Referenced by try_vectorize_loop_1(), and vect_transform_loops().
Variable Documentation
◆ type_align_map
Referenced by get_vec_alignment_for_record_type(), and increase_alignment().
◆ vect_location
| dump_user_location_t vect_location |
Vectorizer Copyright (C) 2003-2024 Free Software Foundation, Inc. Contributed by Dorit Naishlos <dorit@il.ibm.com> This file is part of GCC. GCC is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. GCC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GCC; see the file COPYING3. If not see <http://www.gnu.org/licenses/>.
Loop and basic block vectorizer.
This file contains drivers for the three vectorizers:
(1) loop vectorizer (inter-iteration parallelism),
(2) loop-aware SLP (intra-iteration parallelism) (invoked by the loop
vectorizer)
(3) BB vectorizer (out-of-loops), aka SLP
The rest of the vectorizer's code is organized as follows:
- tree-vect-loop.cc - loop specific parts such as reductions, etc. These are
used by drivers (1) and (2).
- tree-vect-loop-manip.cc - vectorizer's loop control-flow utilities, used by
drivers (1) and (2).
- tree-vect-slp.cc - BB vectorization specific analysis and transformation,
used by drivers (2) and (3).
- tree-vect-stmts.cc - statements analysis and transformation (used by all).
- tree-vect-data-refs.cc - vectorizer specific data-refs analysis and
manipulations (used by all).
- tree-vect-patterns.cc - vectorizable code patterns detector (used by all)
Here's a poor attempt at illustrating that:
tree-vectorizer.cc:
loop_vect() loop_aware_slp() slp_vect()
| / \ /
| / \ /
tree-vect-loop.cc tree-vect-slp.cc
| \ \ / / |
| \ \/ / |
| \ /\ / |
| \ / \ / |
tree-vect-stmts.cc tree-vect-data-refs.cc
\ /
tree-vect-patterns.cc
Loop or bb location, with hotness information.
Referenced by check_load_store_for_partial_vectors(), check_scan_store(), dependence_distance_ge_vf(), vect_optimize_slp_pass::dump(), get_group_alias_ptr_type(), get_group_load_store_type(), get_load_store_type(), get_negative_load_store_type(), vect_optimize_slp_pass::get_result_with_layout(), increase_alignment(), is_simple_and_all_uses_invariant(), vect_optimize_slp_pass::materialize(), maybe_push_to_hybrid_worklist(), move_early_exit_stmts(), optimize_load_redistribution_1(), optimize_mask_stores(), parloops_is_simple_reduction(), parloops_is_slp_reduction(), process_use(), report_ploop_op(), report_vect_op(), try_vectorize_loop_1(), vect_analyze_data_ref_access(), vect_analyze_data_ref_accesses(), vect_analyze_data_ref_dependence(), vect_analyze_data_refs(), vect_analyze_early_break_dependences(), vect_analyze_group_access_1(), vect_analyze_loop(), vect_analyze_loop_1(), vect_analyze_loop_2(), vect_analyze_loop_costing(), vect_analyze_loop_form(), vect_analyze_loop_operations(), vect_analyze_scalar_cycles_1(), vect_analyze_slp(), vect_analyze_slp_instance(), vect_analyze_stmt(), vect_bb_partition_graph(), vect_bb_slp_mark_live_stmts(), vect_bb_slp_mark_live_stmts(), vect_bb_vectorization_profitable_p(), vect_build_slp_instance(), vect_build_slp_tree(), vect_build_slp_tree_1(), vect_build_slp_tree_2(), vect_can_advance_ivs_p(), vect_can_peel_nonlinear_iv_p(), vect_check_lower_bound(), vect_check_nonzero_value(), vect_check_scalar_mask(), vect_check_store_rhs(), vect_compute_data_ref_alignment(), vect_create_addr_base_for_vector_ref(), vect_create_cond_for_alias_checks(), vect_create_data_ref_ptr(), vect_create_epilog_for_reduction(), vect_detect_hybrid_slp(), vect_determine_mask_precision(), vect_determine_min_output_precision_1(), vect_determine_partial_vectors_and_peeling(), vect_determine_precisions_from_range(), vect_determine_precisions_from_users(), vect_determine_vectorization_factor(), vect_determine_vf_for_stmt(), vect_determine_vf_for_stmt_1(), vect_enhance_data_refs_alignment(), vect_estimate_min_profitable_iters(), vect_finish_stmt_generation_1(), vect_gen_prolog_loop_niters(), vect_get_and_check_slp_defs(), vect_get_data_access_cost(), vect_get_load_cost(), vect_get_loop_niters(), vect_get_peel_iters_epilogue(), vect_get_range_info(), vect_get_store_cost(), vect_get_vec_defs_for_operand(), vect_get_vector_types_for_stmt(), vect_grouped_load_supported(), vect_grouped_store_supported(), vect_init_vector_1(), vect_is_simple_iv_evolution(), vect_is_simple_reduction(), vect_is_simple_use(), vect_is_simple_use(), vect_joust_loop_vinfos(), vect_lanes_optab_supported_p(), vect_loop_kill_debug_uses(), vect_loop_versioning(), vect_make_slp_decision(), vect_mark_for_runtime_alias_test(), vect_mark_pattern_stmts(), vect_mark_relevant(), vect_mark_stmts_to_be_vectorized(), vect_match_slp_patterns(), vect_model_promotion_demotion_cost(), vect_model_simple_cost(), vect_pattern_detected(), vect_pattern_recog_1(), vect_pattern_validate_optab(), vect_prepare_for_masked_peels(), vect_prune_runtime_alias_test_list(), vect_recog_average_pattern(), vect_recog_cond_expr_convert_pattern(), vect_recog_ctz_ffs_pattern(), vect_recog_mulhs_pattern(), vect_recog_over_widening_pattern(), vect_recog_popcount_clz_ctz_ffs_pattern(), vect_record_base_alignment(), vect_record_max_nunits(), vect_schedule_slp(), vect_schedule_slp_node(), vect_set_loop_condition(), vect_shift_permute_load_chain(), vect_slp_analyze_bb_1(), vect_slp_analyze_data_ref_dependence(), vect_slp_analyze_node_operations(), vect_slp_analyze_operations(), vect_slp_bbs(), vect_slp_convert_to_external(), vect_slp_function(), vect_slp_region(), vect_split_slp_store_group(), vect_split_statement(), vect_stmt_relevant_p(), vect_transform_loop(), vect_transform_loop_stmt(), vect_transform_loops(), vect_transform_reduction(), vect_transform_slp_perm_load_1(), vect_transform_stmt(), vect_truncate_gather_scatter_offset(), vect_update_ivs_after_vectorizer(), vect_update_misalignment_for_peel(), vect_update_vf_for_slp(), vect_use_strided_gather_scatters_p(), vect_verify_loop_lens(), vector_alignment_reachable_p(), vectorizable_assignment(), vectorizable_bb_reduc_epilogue(), vectorizable_bswap(), vectorizable_call(), vectorizable_comparison_1(), vectorizable_condition(), vectorizable_conversion(), vectorizable_early_exit(), vectorizable_induction(), vectorizable_lc_phi(), vectorizable_live_operation(), vectorizable_load(), vectorizable_nonlinear_induction(), vectorizable_operation(), vectorizable_phi(), vectorizable_recurr(), vectorizable_reduction(), vectorizable_scan_store(), vectorizable_shift(), vectorizable_simd_clone_call(), vectorizable_slp_permutation_1(), vectorizable_store(), vectorize_fold_left_reduction(), and auto_purge_vect_location::~auto_purge_vect_location().
