GCC Middle and Back End API Reference
tree-vect-loop-manip.cc File Reference
#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "tree.h"
#include "gimple.h"
#include "cfghooks.h"
#include "tree-pass.h"
#include "ssa.h"
#include "fold-const.h"
#include "cfganal.h"
#include "gimplify.h"
#include "gimple-iterator.h"
#include "gimplify-me.h"
#include "tree-cfg.h"
#include "tree-ssa-loop-manip.h"
#include "tree-into-ssa.h"
#include "tree-ssa.h"
#include "cfgloop.h"
#include "tree-scalar-evolution.h"
#include "tree-vectorizer.h"
#include "tree-ssa-loop-ivopts.h"
#include "gimple-fold.h"
#include "tree-ssa-loop-niter.h"
#include "internal-fn.h"
#include "stor-layout.h"
#include "optabs-query.h"
#include "vec-perm-indices.h"
#include "insn-config.h"
#include "rtl.h"
#include "recog.h"
#include "langhooks.h"
#include "tree-vector-builder.h"
#include "optabs-tree.h"
Include dependency graph for tree-vect-loop-manip.cc:

Data Structures

struct  adjust_info
 

Macros

#define INCLUDE_MEMORY
 

Functions

static void rename_use_op (use_operand_p op_p)
 
static void rename_variables_in_bb (basic_block bb, bool rename_from_outer_loop)
 
static void adjust_debug_stmts_now (adjust_info *ai)
 
static void adjust_vec_debug_stmts (void)
 
static void adjust_debug_stmts (tree from, tree to, basic_block bb)
 
static void adjust_phi_and_debug_stmts (gimple *update_phi, edge e, tree new_def)
 
static void vect_set_loop_control (class loop *loop, tree ctrl, tree init_ctrl, tree next_ctrl)
 
static void add_preheader_seq (class loop *loop, gimple_seq seq)
 
static void add_header_seq (class loop *loop, gimple_seq seq)
 
static bool interleave_supported_p (vec_perm_indices *indices, tree vectype, unsigned int offset)
 
static bool vect_maybe_permute_loop_masks (gimple_seq *seq, rgroup_controls *dest_rgm, rgroup_controls *src_rgm)
 
static void vect_adjust_loop_lens_control (tree iv_type, gimple_seq *seq, rgroup_controls *dest_rgm, tree step)
 
void vect_iv_increment_position (edge loop_exit, gimple_stmt_iterator *bsi, bool *insert_after)
 
static tree vect_set_loop_controls_directly (class loop *loop, loop_vec_info loop_vinfo, gimple_seq *preheader_seq, gimple_seq *header_seq, gimple_stmt_iterator loop_cond_gsi, rgroup_controls *rgc, tree niters, tree niters_skip, bool might_wrap_p, tree *iv_step, tree *compare_step)
 
static gcondvect_set_loop_condition_partial_vectors (class loop *loop, edge exit_edge, loop_vec_info loop_vinfo, tree niters, tree final_iv, bool niters_maybe_zero, gimple_stmt_iterator loop_cond_gsi)
 
static gcondvect_set_loop_condition_partial_vectors_avx512 (class loop *loop, edge exit_edge, loop_vec_info loop_vinfo, tree niters, tree final_iv, bool niters_maybe_zero, gimple_stmt_iterator loop_cond_gsi)
 
static gcondvect_set_loop_condition_normal (loop_vec_info, edge exit_edge, class loop *loop, tree niters, tree step, tree final_iv, bool niters_maybe_zero, gimple_stmt_iterator loop_cond_gsi)
 
void vect_set_loop_condition (class loop *loop, edge loop_e, loop_vec_info loop_vinfo, tree niters, tree step, tree final_iv, bool niters_maybe_zero)
 
static tree get_live_virtual_operand_on_edge (edge e)
 
class loopslpeel_tree_duplicate_loop_to_edge_cfg (class loop *loop, edge loop_exit, class loop *scalar_loop, edge scalar_exit, edge e, edge *new_e, bool flow_loops, vec< basic_block > *updated_doms)
 
static edge slpeel_add_loop_guard (basic_block guard_bb, tree cond, basic_block guard_to, basic_block dom_bb, profile_probability probability, bool irreducible_p)
 
bool slpeel_can_duplicate_loop_p (const class loop *loop, const_edge exit_e, const_edge e)
 
dump_user_location_t find_loop_location (class loop *loop)
 
static bool iv_phi_p (stmt_vec_info stmt_info)
 
static bool vect_can_peel_nonlinear_iv_p (loop_vec_info loop_vinfo, stmt_vec_info stmt_info)
 
bool vect_can_advance_ivs_p (loop_vec_info loop_vinfo)
 
static void vect_update_ivs_after_vectorizer (loop_vec_info loop_vinfo, tree niters, edge update_e)
 
static tree get_misalign_in_elems (gimple **seq, loop_vec_info loop_vinfo)
 
static tree vect_gen_prolog_loop_niters (loop_vec_info loop_vinfo, basic_block bb, int *bound)
 
static void vect_update_init_of_dr (dr_vec_info *dr_info, tree niters, tree_code code)
 
void vect_update_inits_of_drs (loop_vec_info loop_vinfo, tree niters, tree_code code)
 
void vect_prepare_for_masked_peels (loop_vec_info loop_vinfo)
 
tree vect_build_loop_niters (loop_vec_info loop_vinfo, bool *new_var_p)
 
static tree vect_gen_scalar_loop_niters (tree niters_prolog, int int_niters_prolog, int bound_prolog, poly_int64 bound_epilog, int th, poly_uint64 *bound_scalar, bool check_profitability)
 
void vect_gen_vector_loop_niters (loop_vec_info loop_vinfo, tree niters, tree *niters_vector_ptr, tree *step_vector_ptr, bool niters_no_overflow)
 
static void vect_gen_vector_loop_niters_mult_vf (loop_vec_info loop_vinfo, tree niters_vector, tree *niters_vector_mult_vf_ptr)
 
static void slpeel_update_phi_nodes_for_guard1 (class loop *skip_loop, class loop *update_loop, edge guard_edge, edge merge_edge)
 
tree vect_get_main_loop_result (loop_vec_info loop_vinfo, tree main_loop_value, tree skip_value)
 
class loopvect_do_peeling (loop_vec_info loop_vinfo, tree niters, tree nitersm1, tree *niters_vector, tree *step_vector, tree *niters_vector_mult_vf_var, int th, bool check_profitability, bool niters_no_overflow, tree *advance)
 
static void vect_create_cond_for_niters_checks (loop_vec_info loop_vinfo, tree *cond_expr)
 
static void chain_cond_expr (tree *cond_expr, tree part_cond_expr)
 
static void vect_create_cond_for_align_checks (loop_vec_info loop_vinfo, tree *cond_expr, gimple_seq *cond_expr_stmt_list)
 
static void vect_create_cond_for_unequal_addrs (loop_vec_info loop_vinfo, tree *cond_expr)
 
static void vect_create_cond_for_lower_bounds (loop_vec_info loop_vinfo, tree *cond_expr)
 
void vect_create_cond_for_alias_checks (loop_vec_info loop_vinfo, tree *cond_expr)
 
class loopvect_loop_versioning (loop_vec_info loop_vinfo, gimple *loop_vectorized_call)
 

Variables

static vec< adjust_info, va_heapadjust_vec
 

Macro Definition Documentation

◆ INCLUDE_MEMORY

#define INCLUDE_MEMORY
Vectorizer Specific Loop Manipulations
   Copyright (C) 2003-2024 Free Software Foundation, Inc.
   Contributed by Dorit Naishlos <dorit@il.ibm.com>
   and Ira Rosen <irar@il.ibm.com>

This file is part of GCC.

GCC is free software; you can redistribute it and/or modify it under
the terms of the GNU General Public License as published by the Free
Software Foundation; either version 3, or (at your option) any later
version.

GCC is distributed in the hope that it will be useful, but WITHOUT ANY
WARRANTY; without even the implied warranty of MERCHANTABILITY or
FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
for more details.

You should have received a copy of the GNU General Public License
along with GCC; see the file COPYING3.  If not see
<http://www.gnu.org/licenses/>.   

Function Documentation

◆ add_header_seq()

static void add_header_seq ( class loop * loop,
gimple_seq seq )
static
Add SEQ to the beginning of LOOP's header block.   

References gsi_after_labels(), gsi_insert_seq_before(), GSI_SAME_STMT, and loop::header.

Referenced by vect_set_loop_condition_partial_vectors().

◆ add_preheader_seq()

static void add_preheader_seq ( class loop * loop,
gimple_seq seq )
static

◆ adjust_debug_stmts()

static void adjust_debug_stmts ( tree from,
tree to,
basic_block bb )
static
Adjust any debug stmts that referenced FROM values to use the
loop-closed TO, if the references are dominated by BB and not by
the definition of FROM.  If adjust_vec is non-NULL, adjustments
will be postponed until adjust_vec_debug_stmts is called.   

References adjust_debug_stmts_now(), adjust_vec, adjust_info::bb, adjust_info::from, MAY_HAVE_DEBUG_BIND_STMTS, SSA_NAME_IS_DEFAULT_DEF, adjust_info::to, TREE_CODE, and virtual_operand_p().

Referenced by adjust_phi_and_debug_stmts(), and slpeel_tree_duplicate_loop_to_edge_cfg().

◆ adjust_debug_stmts_now()

static void adjust_debug_stmts_now ( adjust_info * ai)
static
Adjust any debug stmts that referenced AI->from values to use the
loop-closed AI->to, if the references are dominated by AI->bb and
not by the definition of AI->from.   

References adjust_info::bb, CDI_DOMINATORS, dom_info_available_p(), dominated_by_p(), FOR_EACH_IMM_USE_ON_STMT, FOR_EACH_IMM_USE_STMT, adjust_info::from, gcc_assert, gimple_bb(), gimple_debug_bind_p(), gimple_debug_bind_reset_value(), is_gimple_debug(), SET_USE, SSA_NAME_DEF_STMT, adjust_info::to, and update_stmt().

Referenced by adjust_debug_stmts(), and adjust_vec_debug_stmts().

◆ adjust_phi_and_debug_stmts()

static void adjust_phi_and_debug_stmts ( gimple * update_phi,
edge e,
tree new_def )
static
Change E's phi arg in UPDATE_PHI to NEW_DEF, and record information
to adjust any debug stmts that referenced the old phi arg,
presumably non-loop-closed references left over from other
transformations.   

References adjust_debug_stmts(), gcc_assert, gimple_bb(), MAY_HAVE_DEBUG_BIND_STMTS, PHI_ARG_DEF_FROM_EDGE, PHI_RESULT, SET_PHI_ARG_DEF, and TREE_CODE.

Referenced by slpeel_tree_duplicate_loop_to_edge_cfg(), slpeel_update_phi_nodes_for_guard1(), vect_loop_versioning(), and vect_update_ivs_after_vectorizer().

◆ adjust_vec_debug_stmts()

static void adjust_vec_debug_stmts ( void )
static
Adjust debug stmts as scheduled before.   

References adjust_debug_stmts_now(), adjust_vec, gcc_assert, and MAY_HAVE_DEBUG_BIND_STMTS.

Referenced by vect_do_peeling().

◆ chain_cond_expr()

static void chain_cond_expr ( tree * cond_expr,
tree part_cond_expr )
static
Set *COND_EXPR to a tree that is true when both the original *COND_EXPR
and PART_COND_EXPR are true.  Treat a null *COND_EXPR as "true".   

References boolean_type_node, and fold_build2.

Referenced by vect_create_cond_for_align_checks(), vect_create_cond_for_lower_bounds(), and vect_create_cond_for_unequal_addrs().

◆ find_loop_location()

◆ get_live_virtual_operand_on_edge()

static tree get_live_virtual_operand_on_edge ( edge e)
static
Get the virtual operand live on E.  The precondition on this is valid
immediate dominators and an actual virtual definition dominating E.   
???  Costly band-aid.  For the use in question we can populate a
live-on-exit/end-of-BB virtual operand when copying stmts.   

References CDI_DOMINATORS, get_immediate_dominator(), get_virtual_phi(), gimple_phi_result(), gimple_vdef(), gimple_vuse(), gsi_end_p(), gsi_last_bb(), gsi_prev(), and gsi_stmt().

Referenced by slpeel_tree_duplicate_loop_to_edge_cfg().

◆ get_misalign_in_elems()

static tree get_misalign_in_elems ( gimple ** seq,
loop_vec_info loop_vinfo )
static
Return a gimple value containing the misalignment (measured in vector
elements) for the loop described by LOOP_VINFO, i.e. how many elements
it is away from a perfectly aligned address.  Add any new statements
to SEQ.   

References build_int_cst(), dr_info::dr, DR_STEP, DR_TARGET_ALIGNMENT, exact_log2(), fold_build2, fold_convert, int_cst_value(), LOOP_VINFO_UNALIGNED_DR, offset, size_int, size_zero_node, STMT_VINFO_VECTYPE, target_align(), tree_int_cst_compare(), TREE_INT_CST_LOW, TREE_TYPE, TYPE_SIZE_UNIT, TYPE_VECTOR_SUBPARTS(), unsigned_type_for(), and vect_create_addr_base_for_vector_ref().

Referenced by vect_gen_prolog_loop_niters(), and vect_prepare_for_masked_peels().

◆ interleave_supported_p()

static bool interleave_supported_p ( vec_perm_indices * indices,
tree vectype,
unsigned int offset )
static
Return true if the target can interleave elements of two vectors.
OFFSET is 0 if the first half of the vectors should be interleaved
or 1 if the second half should.  When returning true, store the
associated permutation in INDICES.   

References can_vec_perm_const_p(), i, offset, TYPE_MODE, and TYPE_VECTOR_SUBPARTS().

Referenced by vect_maybe_permute_loop_masks().

◆ iv_phi_p()

static bool iv_phi_p ( stmt_vec_info stmt_info)
static
Return true if the phi described by STMT_INFO defines an IV of the
loop to be vectorized.   

References as_a(), PHI_RESULT, STMT_VINFO_DEF_TYPE, vect_double_reduction_def, vect_reduction_def, and virtual_operand_p().

Referenced by vect_can_advance_ivs_p(), and vect_update_ivs_after_vectorizer().

◆ rename_use_op()

static void rename_use_op ( use_operand_p op_p)
static

◆ rename_variables_in_bb()

static void rename_variables_in_bb ( basic_block bb,
bool rename_from_outer_loop )
static

◆ slpeel_add_loop_guard()

static edge slpeel_add_loop_guard ( basic_block guard_bb,
tree cond,
basic_block guard_to,
basic_block dom_bb,
profile_probability probability,
bool irreducible_p )
static
Given the condition expression COND, put it as the last statement of
GUARD_BB; set both edges' probability; set dominator of GUARD_TO to
DOM_BB; return the skip edge.  GUARD_TO is the target basic block to
skip the loop.  PROBABILITY is the skip edge's probability.  Mark the
new edge as irreducible if IRREDUCIBLE_P is true.   

References CDI_DOMINATORS, EDGE_SUCC, force_gimple_operand_1(), gimple_build_cond_from_tree(), gsi_insert_after(), gsi_insert_seq_after(), gsi_last_bb(), GSI_NEW_STMT, profile_probability::invert(), is_gimple_condexpr_for_cond(), make_edge(), NULL, NULL_TREE, set_immediate_dominator(), and split_edge().

Referenced by vect_do_peeling().

◆ slpeel_can_duplicate_loop_p()

bool slpeel_can_duplicate_loop_p ( const class loop * loop,
const_edge exit_e,
const_edge e )
This function verifies that the following restrictions apply to LOOP:
 (1) it consists of exactly 2 basic blocks - header, and an empty latch
     for innermost loop and 5 basic blocks for outer-loop.
 (2) it is single entry, single exit
 (3) its exit condition is the last stmt in the header
 (4) E is the entry/exit edge of LOOP.

References can_copy_bbs_p(), empty_block_p(), free(), get_loop_body_with_size(), get_loop_exit_condition(), gsi_last_bb(), gsi_stmt(), loop::latch, loop_outer(), loop_preheader_edge(), and loop::num_nodes.

Referenced by vect_analyze_loop_2(), vect_do_peeling(), and vect_enhance_data_refs_alignment().

◆ slpeel_tree_duplicate_loop_to_edge_cfg()

class loop * slpeel_tree_duplicate_loop_to_edge_cfg ( class loop * loop,
edge loop_exit,
class loop * scalar_loop,
edge scalar_exit,
edge e,
edge * new_e,
bool flow_loops,
vec< basic_block > * updated_doms )
Given LOOP this function generates a new copy of it and puts it
on E which is either the entry or exit of LOOP.  If SCALAR_LOOP is
non-NULL, assume LOOP and SCALAR_LOOP are equivalent and copy the
basic blocks from SCALAR_LOOP instead of LOOP, but to either the
entry or exit of LOOP.  If FLOW_LOOPS then connect LOOP to SCALAR_LOOP as a
continuation.  This is correct for cases where one loop continues from the
other like in the vectorizer, but not true for uses in e.g. loop distribution
where the contents of the loop body are split but the iteration space of both
copies remains the same.

If UPDATED_DOMS is not NULL it is update with the list of basic blocks whoms
dominators were updated during the peeling.  When doing early break vectorization
then LOOP_VINFO needs to be provided and is used to keep track of any newly created
memory references that need to be updated should we decide to vectorize.   

References add_phi_arg(), add_phi_args_after_copy(), adjust_debug_stmts(), adjust_phi_and_debug_stmts(), CDI_DOMINATORS, checking_verify_dominators(), copy_bbs(), copy_ssa_name(), create_phi_node(), delete_basic_block(), duplicate_loop(), duplicate_subloops(), EDGE_COUNT, EDGE_PRED, first_dom_son(), flow_bb_inside_loop_p(), flush_pending_stmts(), FOR_EACH_EDGE, free(), gcc_assert, hash_map< KeyId, Value, Traits >::get(), get_all_dominated_blocks(), get_bb_copy(), get_immediate_dominator(), get_live_virtual_operand_on_edge(), get_loop_body_with_size(), get_loop_copy(), get_loop_exit_edges(), get_virtual_phi(), gimple_phi_arg_def_from_edge(), gimple_phi_num_args(), gimple_phi_result(), gsi_end_p(), gsi_for_stmt(), gsi_next(), gsi_start_phis(), gsi_stmt(), loop::header, i, loop::inner, iterate_fix_dominators(), loop::latch, loop_latch_edge(), loop_outer(), loop_preheader_edge(), MAY_HAVE_DEBUG_BIND_STMTS, next_dom_son(), NULL, NULL_TREE, loop::num_nodes, PHI_ARG_DEF_FROM_EDGE, PHI_ARG_DEF_PTR_FROM_EDGE, PHI_RESULT, basic_block_def::preds, hash_map< KeyId, Value, Traits >::put(), queue, redirect_edge_and_branch(), redirect_edge_and_branch_force(), redirect_edge_pred(), redirect_edge_var_map_clear(), remove_phi_node(), rename_use_op(), rename_variables_in_bb(), set_immediate_dominator(), SET_PHI_ARG_DEF, SET_PHI_ARG_DEF_ON_EDGE, single_pred(), single_pred_edge(), single_succ_edge(), single_succ_p(), split_edge(), TREE_CODE, true, UNKNOWN_LOCATION, and virtual_operand_p().

Referenced by copy_loop_before(), and vect_do_peeling().

◆ slpeel_update_phi_nodes_for_guard1()

static void slpeel_update_phi_nodes_for_guard1 ( class loop * skip_loop,
class loop * update_loop,
edge guard_edge,
edge merge_edge )
static
Function slpeel_add_loop_guard adds guard skipping from the beginning
of SKIP_LOOP to the beginning of UPDATE_LOOP.  GUARD_EDGE and MERGE_EDGE
are two pred edges of the merge point before UPDATE_LOOP.  The two loops
appear like below:

    guard_bb:
      if (cond)
        goto merge_bb;
      else
        goto skip_loop;

  skip_loop:
    header_a:
      i_1 = PHI<i_0, i_2>;
      ...
      i_2 = i_1 + 1;
      if (cond_a)
        goto latch_a;
      else
        goto exit_a;
    latch_a:
      goto header_a;

    exit_a:
      i_5 = PHI<i_2>;

    merge_bb:
      ;; PHI (i_x = PHI<i_0, i_5>) to be created at merge point.

  update_loop:
    header_b:
      i_3 = PHI<i_5, i_4>;  ;; Use of i_5 to be replaced with i_x.
      ...
      i_4 = i_3 + 1;
      if (cond_b)
        goto latch_b;
      else
        goto exit_bb;
    latch_b:
      goto header_b;

    exit_bb:

This function creates PHI nodes at merge_bb and replaces the use of i_5
in the update_loop's PHI node with the result of new PHI result.   

References add_phi_arg(), adjust_phi_and_debug_stmts(), copy_ssa_name(), create_phi_node(), gimple_phi_arg_location_from_edge(), gsi_end_p(), gsi_next(), gsi_start_phis(), loop::header, loop_preheader_edge(), gphi_iterator::phi(), PHI_ARG_DEF_FROM_EDGE, and PHI_RESULT.

Referenced by vect_do_peeling().

◆ vect_adjust_loop_lens_control()

static void vect_adjust_loop_lens_control ( tree iv_type,
gimple_seq * seq,
rgroup_controls * dest_rgm,
tree step )
static
Populate DEST_RGM->controls, given that they should add up to STEP.

    STEP = MIN_EXPR <ivtmp_34, VF>;

    First length (MIN (X, VF/N)):
      loop_len_15 = MIN_EXPR <STEP, VF/N>;

    Second length:
      tmp = STEP - loop_len_15;
      loop_len_16 = MIN (tmp, VF/N);

    Third length:
      tmp2 = tmp - loop_len_16;
      loop_len_17 = MIN (tmp2, VF/N);

    Last length:
      loop_len_18 = tmp2 - loop_len_17;

References build_int_cst(), rgroup_controls::controls, rgroup_controls::factor, gimple_build(), gimple_build_assign(), gimple_seq_add_stmt(), i, rgroup_controls::type, and TYPE_VECTOR_SUBPARTS().

Referenced by vect_set_loop_condition_partial_vectors().

◆ vect_build_loop_niters()

tree vect_build_loop_niters ( loop_vec_info loop_vinfo,
bool * new_var_p )
This function builds ni_name = number of iterations.  Statements
are emitted on the loop preheader edge.  If NEW_VAR_P is not NULL, set
it to TRUE if new ssa_var is generated.   

References create_tmp_var, force_gimple_operand(), gsi_insert_seq_on_edge_immediate(), loop_preheader_edge(), LOOP_VINFO_LOOP, LOOP_VINFO_NITERS, NULL, TREE_CODE, TREE_TYPE, and unshare_expr().

Referenced by vect_do_peeling(), and vect_transform_loop().

◆ vect_can_advance_ivs_p()

bool vect_can_advance_ivs_p ( loop_vec_info loop_vinfo)
Function vect_can_advance_ivs_p

In case the number of iterations that LOOP iterates is unknown at compile
time, an epilog loop will be generated, and the loop induction variables
(IVs) will be "advanced" to the value they are supposed to take just before
the epilog loop.  Here we check that the access function of the loop IVs
and the expression that represents the loop bound are simple enough.
These restrictions will be relaxed in the future.   

References dump_enabled_p(), dump_printf(), dump_printf_loc(), expr_invariant_in_loop_p(), gsi_end_p(), gsi_next(), gsi_start_phis(), loop::header, iv_phi_p(), vec_info::lookup_stmt(), LOOP_VINFO_LOOP, MSG_MISSED_OPTIMIZATION, MSG_NOTE, NULL_TREE, gphi_iterator::phi(), STMT_VINFO_LOOP_PHI_EVOLUTION_PART, STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE, tree_is_chrec(), vect_can_peel_nonlinear_iv_p(), vect_location, and vect_step_op_add.

Referenced by vect_analyze_loop_2(), vect_do_peeling(), and vect_enhance_data_refs_alignment().

◆ vect_can_peel_nonlinear_iv_p()

◆ vect_create_cond_for_alias_checks()

void vect_create_cond_for_alias_checks ( loop_vec_info loop_vinfo,
tree * cond_expr )
Function vect_create_cond_for_alias_checks.

  Create a conditional expression that represents the run-time checks for
  overlapping of address ranges represented by a list of data references
  relations passed as input.

  Input:
  COND_EXPR  - input conditional expression.  New conditions will be chained
               with logical AND operation.  If it is NULL, then the function
               is used to return the number of alias checks.
  LOOP_VINFO - field LOOP_VINFO_MAY_ALIAS_STMTS contains the list of ddrs
               to be checked.

  Output:
  COND_EXPR - conditional expression.

  The returned COND_EXPR is the conditional expression to be used in the if
  statement that controls which version of the loop gets executed at runtime.

References create_runtime_alias_checks(), dump_enabled_p(), dump_printf_loc(), LOOP_VINFO_COMP_ALIAS_DDRS, LOOP_VINFO_LOOP, MSG_NOTE, and vect_location.

Referenced by vect_loop_versioning().

◆ vect_create_cond_for_align_checks()

static void vect_create_cond_for_align_checks ( loop_vec_info loop_vinfo,
tree * cond_expr,
gimple_seq * cond_expr_stmt_list )
static
Function vect_create_cond_for_align_checks.

Create a conditional expression that represents the alignment checks for
all of data references (array element references) whose alignment must be
checked at runtime.

Input:
COND_EXPR  - input conditional expression.  New conditions will be chained
             with logical AND operation.
LOOP_VINFO - two fields of the loop information are used.
             LOOP_VINFO_PTR_MASK is the mask used to check the alignment.
             LOOP_VINFO_MAY_MISALIGN_STMTS contains the refs to be checked.

Output:
COND_EXPR_STMT_LIST - statements needed to construct the conditional
                      expression.
The returned value is the conditional expression to be used in the if
statement that controls which version of the loop gets executed at runtime.

The algorithm makes two assumptions:
  1) The number of bytes "n" in a vector is a power of 2.
  2) An address "a" is aligned if a%n is zero and that this
     test can be done as a&(n-1) == 0.  For example, for 16
     byte vectors the test is a&0xf == 0.   

References boolean_type_node, build_int_cst(), chain_cond_expr(), DR_STEP, fold_build2, FOR_EACH_VEC_ELT, gcc_assert, gimple_build_assign(), gimple_seq_add_seq(), gimple_seq_add_stmt(), i, LOOP_VINFO_MAY_MISALIGN_STMTS, LOOP_VINFO_PTR_MASK, make_temp_ssa_name(), NULL, NULL_TREE, offset, ptr_type_node, signed_type_for(), size_int, size_zero_node, STMT_VINFO_DATA_REF, STMT_VINFO_VECTYPE, tree_int_cst_compare(), TREE_INT_CST_LOW, TREE_TYPE, TYPE_SIZE_UNIT, TYPE_VECTOR_SUBPARTS(), and vect_create_addr_base_for_vector_ref().

Referenced by vect_loop_versioning().

◆ vect_create_cond_for_lower_bounds()

static void vect_create_cond_for_lower_bounds ( loop_vec_info loop_vinfo,
tree * cond_expr )
static
Create an expression that is true when all lower-bound conditions for
the vectorized loop are met.  Chain this condition with *COND_EXPR.   

References boolean_type_node, build_int_cstu(), chain_cond_expr(), fold_build2, fold_convert, i, LOOP_VINFO_LOWER_BOUNDS, TREE_TYPE, and unsigned_type_for().

Referenced by vect_loop_versioning().

◆ vect_create_cond_for_niters_checks()

static void vect_create_cond_for_niters_checks ( loop_vec_info loop_vinfo,
tree * cond_expr )
static
Function vect_create_cond_for_niters_checks.

Create a conditional expression that represents the run-time checks for
loop's niter.  The loop is guaranteed to terminate if the run-time
checks hold.

Input:
COND_EXPR  - input conditional expression.  New conditions will be chained
        with logical AND operation.  If it is NULL, then the function
        is used to return the number of alias checks.
LOOP_VINFO - field LOOP_VINFO_MAY_ALIAS_STMTS contains the list of ddrs
        to be checked.

Output:
COND_EXPR - conditional expression.

The returned COND_EXPR is the conditional expression to be used in the
if statement that controls which version of the loop gets executed at
runtime.   

References boolean_type_node, fold_build2, and LOOP_VINFO_NITERS_ASSUMPTIONS.

Referenced by vect_loop_versioning().

◆ vect_create_cond_for_unequal_addrs()

static void vect_create_cond_for_unequal_addrs ( loop_vec_info loop_vinfo,
tree * cond_expr )
static
If LOOP_VINFO_CHECK_UNEQUAL_ADDRS contains <A1, B1>, ..., <An, Bn>,
create a tree representation of: (&A1 != &B1) && ... && (&An != &Bn).
Set *COND_EXPR to a tree that is true when both the original *COND_EXPR
and this new condition are true.  Treat a null *COND_EXPR as "true".   

References boolean_type_node, build_fold_addr_expr, chain_cond_expr(), fold_build2, FOR_EACH_VEC_ELT, i, and LOOP_VINFO_CHECK_UNEQUAL_ADDRS.

Referenced by vect_loop_versioning().

◆ vect_do_peeling()

class loop * vect_do_peeling ( loop_vec_info loop_vinfo,
tree niters,
tree nitersm1,
tree * niters_vector,
tree * step_vector,
tree * niters_vector_mult_vf_var,
int th,
bool check_profitability,
bool niters_no_overflow,
tree * advance )
Function vect_do_peeling.

Input:
- LOOP_VINFO: Represent a loop to be vectorized, which looks like:

    preheader:
  LOOP:
    header_bb:
      loop_body
      if (exit_loop_cond) goto exit_bb
      else                goto header_bb
    exit_bb:

- NITERS: The number of iterations of the loop.
- NITERSM1: The number of iterations of the loop's latch.
- NITERS_NO_OVERFLOW: No overflow in computing NITERS.
- TH, CHECK_PROFITABILITY: Threshold of niters to vectorize loop if
                      CHECK_PROFITABILITY is true.
Output:
- *NITERS_VECTOR and *STEP_VECTOR describe how the main loop should
  iterate after vectorization; see vect_set_loop_condition for details.
- *NITERS_VECTOR_MULT_VF_VAR is either null or an SSA name that
  should be set to the number of scalar iterations handled by the
  vector loop.  The SSA name is only used on exit from the loop.

This function peels prolog and epilog from the loop, adds guards skipping
PROLOG and EPILOG for various conditions.  As a result, the changed CFG
would look like:

    guard_bb_1:
      if (prefer_scalar_loop) goto merge_bb_1
      else                    goto guard_bb_2

    guard_bb_2:
      if (skip_prolog) goto merge_bb_2
      else             goto prolog_preheader

    prolog_preheader:
  PROLOG:
    prolog_header_bb:
      prolog_body
      if (exit_prolog_cond) goto prolog_exit_bb
      else                  goto prolog_header_bb
    prolog_exit_bb:

    merge_bb_2:

    vector_preheader:
  VECTOR LOOP:
    vector_header_bb:
      vector_body
      if (exit_vector_cond) goto vector_exit_bb
      else                  goto vector_header_bb
    vector_exit_bb:

    guard_bb_3:
      if (skip_epilog) goto merge_bb_3
      else             goto epilog_preheader

    merge_bb_1:

    epilog_preheader:
  EPILOG:
    epilog_header_bb:
      epilog_body
      if (exit_epilog_cond) goto merge_bb_3
      else                  goto epilog_header_bb

    merge_bb_3:

Note this function peels prolog and epilog only if it's necessary,
as well as guards.
This function returns the epilogue loop if a decision was made to vectorize
it, otherwise NULL.

The analysis resulting in this epilogue loop's loop_vec_info was performed
in the same vect_analyze_loop call as the main loop's.  At that time
vect_analyze_loop constructs a list of accepted loop_vec_info's for lower
vectorization factors than the main loop.  This list is stored in the main
loop's loop_vec_info in the 'epilogue_vinfos' member.  Everytime we decide to
vectorize the epilogue loop for a lower vectorization factor,  the
loop_vec_info sitting at the top of the epilogue_vinfos list is removed,
updated and linked to the epilogue loop.  This is later used to vectorize
the epilogue.  The reason the loop_vec_info needs updating is that it was
constructed based on the original main loop, and the epilogue loop is a
copy of this loop, so all links pointing to statements in the original loop
need updating.  Furthermore, these loop_vec_infos share the
data_reference's records, which will also need to be updated.

TODO: Guard for prefer_scalar_loop should be emitted along with
versioning conditions if loop versioning is needed.   

References add_phi_arg(), adjust_vec, adjust_vec_debug_stmts(), advance(), profile_probability::always(), loop::any_estimate, loop::any_likely_upper_bound, loop::any_upper_bound, profile_probability::apply_scale(), loop::aux, boolean_type_node, build_int_cst(), build_one_cst(), build_zero_cst(), CDI_DOMINATORS, cfun, basic_block_def::count, create_phi_node(), DEF_FROM_PTR, delete_update_ssa(), DR_TARGET_ALIGNMENT, EDGE_PRED, _loop_vec_info::epilogue_vinfos, first_dom_son(), flow_bb_inside_loop_p(), flow_loop_nested_p(), fold_build2, FOR_EACH_IMM_USE_STMT, FOR_EACH_SSA_DEF_OPERAND, loop::force_vectorize, free(), free_original_copy_tables(), gcc_assert, gcc_checking_assert, get_bb_original(), get_immediate_dominator(), get_loop_body(), get_loop_copy(), get_loop_exit_edges(), gimple_bb(), gimple_build_assign(), gimple_build_nop(), gimple_debug_bind_p(), gimple_debug_bind_reset_value(), gimple_phi_arg_def_from_edge(), gimple_phi_result(), gsi_end_p(), gsi_for_stmt(), gsi_insert_after(), gsi_insert_before(), gsi_last_bb(), GSI_NEW_STMT, gsi_next(), gsi_start_bb(), gsi_start_phis(), gsi_stmt(), profile_probability::guessed_always(), i, initialize_original_copy_tables(), profile_probability::initialized_p(), integer_onep(), profile_probability::invert(), poly_int< N, C >::is_constant(), iterate_fix_dominators(), LOOP_C_INFINITE, loop_constraint_clear(), loop_preheader_edge(), LOOP_REQUIRES_VERSIONING, LOOP_VINFO_BBS, LOOP_VINFO_EARLY_BREAKS, LOOP_VINFO_EARLY_BREAKS_VECT_PEELED, LOOP_VINFO_EPILOGUE_IV_EXIT, LOOP_VINFO_INT_NITERS, LOOP_VINFO_IV_EXIT, LOOP_VINFO_LOOP, LOOP_VINFO_NITERS, LOOP_VINFO_NITERS_KNOWN_P, LOOP_VINFO_NITERSM1, LOOP_VINFO_PEELING_FOR_ALIGNMENT, LOOP_VINFO_PEELING_FOR_GAPS, LOOP_VINFO_PEELING_FOR_NITER, LOOP_VINFO_SCALAR_IV_EXIT, LOOP_VINFO_SCALAR_LOOP, LOOP_VINFO_UNALIGNED_DR, LOOP_VINFO_USING_PARTIAL_VECTORS_P, LOOP_VINFO_VECT_FACTOR, _loop_vec_info::main_loop_edge, make_ssa_name(), MAY_HAVE_DEBUG_BIND_STMTS, loop::nb_iterations_estimate, loop::nb_iterations_likely_upper_bound, loop::nb_iterations_upper_bound, need_ssa_update_p(), next_dom_son(), profile_count::nonzero_p(), NULL, NULL_TREE, loop::num_nodes, PHI_RESULT, profile_count::probability_in(), queue, record_niter_bound(), reset_original_copy_tables(), scale_bbs_frequencies(), scale_loop_profile(), scev_reset(), set_immediate_dominator(), set_range_info(), single_pred_edge(), single_pred_p(), single_succ_edge(), _loop_vec_info::skip_main_loop_edge, _loop_vec_info::skip_this_loop_edge, slpeel_add_loop_guard(), slpeel_can_duplicate_loop_p(), slpeel_tree_duplicate_loop_to_edge_cfg(), slpeel_update_phi_nodes_for_guard1(), split_edge(), SSA_NAME_DEF_STMT, SSA_OP_DEF, poly_int< N, C >::to_constant(), wi::to_wide(), TREE_CODE, TREE_TYPE, TYPE_MAX_VALUE, ui, UNKNOWN_LOCATION, update_stmt(), vect_build_loop_niters(), vect_can_advance_ivs_p(), vect_determine_partial_vectors_and_peeling(), vect_gen_prolog_loop_niters(), vect_gen_scalar_loop_niters(), vect_gen_vector_loop_niters(), vect_gen_vector_loop_niters_mult_vf(), vect_set_loop_condition(), vect_update_inits_of_drs(), vect_update_ivs_after_vectorizer(), vect_use_loop_mask_for_alignment_p(), vect_vf_for_cost(), and virtual_operand_p().

Referenced by vect_transform_loop().

◆ vect_gen_prolog_loop_niters()

static tree vect_gen_prolog_loop_niters ( loop_vec_info loop_vinfo,
basic_block bb,
int * bound )
static
Function vect_gen_prolog_loop_niters

Generate the number of iterations which should be peeled as prolog for the
loop represented by LOOP_VINFO.  It is calculated as the misalignment of
DR - the data reference recorded in LOOP_VINFO_UNALIGNED_DR (LOOP_VINFO).
As a result, after the execution of this loop, the data reference DR will
refer to an aligned location.  The following computation is generated:

If the misalignment of DR is known at compile time:
  addr_mis = int mis = DR_MISALIGNMENT (dr);
Else, compute address misalignment in bytes:
  addr_mis = addr & (target_align - 1)

prolog_niters = ((VF - addr_mis/elem_size)&(VF-1))/step

(elem_size = element type size; an element is the scalar element whose type
is the inner type of the vectype)

The computations will be emitted at the end of BB.  We also compute and
store upper bound (included) of the result in BOUND.

When the step of the data-ref in the loop is not 1 (as in interleaved data
and SLP), the number of iterations of the prolog must be divided by the step
(which is equal to the size of interleaved group).

The above formulas assume that VF == number of elements in the vector. This
may not hold when there are multiple-types in the loop.
In this case, for some data-references in the loop the VF does not represent
the number of elements that fit in the vector.  Therefore, instead of VF we
use TYPE_VECTOR_SUBPARTS.   

References build_int_cst(), create_tmp_var, dr_info::dr, DR_STEP, DR_TARGET_ALIGNMENT, dump_enabled_p(), dump_printf_loc(), fold_build2, fold_convert, force_gimple_operand(), gcc_assert, get_misalign_in_elems(), gimple_seq_add_seq(), gsi_end_p(), gsi_insert_seq_after(), gsi_insert_seq_before(), gsi_last_bb(), GSI_SAME_STMT, int_cst_value(), poly_int< N, C >::is_constant(), LOOP_VINFO_NITERS, LOOP_VINFO_PEELING_FOR_ALIGNMENT, LOOP_VINFO_UNALIGNED_DR, MSG_NOTE, NULL, single_succ_p(), size_zero_node, STMT_VINFO_VECTYPE, target_align(), tree_int_cst_compare(), TREE_TYPE, TYPE_SIZE_UNIT, and vect_location.

Referenced by vect_do_peeling().

◆ vect_gen_scalar_loop_niters()

static tree vect_gen_scalar_loop_niters ( tree niters_prolog,
int int_niters_prolog,
int bound_prolog,
poly_int64 bound_epilog,
int th,
poly_uint64 * bound_scalar,
bool check_profitability )
static
Calculate the number of iterations above which vectorized loop will be
preferred than scalar loop.  NITERS_PROLOG is the number of iterations
of prolog loop.  If it's integer const, the integer number is also passed
in INT_NITERS_PROLOG.  BOUND_PROLOG is the upper bound (inclusive) of the
number of iterations of the prolog loop.  BOUND_EPILOG is the corresponding
value for the epilog loop.  If CHECK_PROFITABILITY is true, TH is the
threshold below which the scalar (rather than vectorized) loop will be
executed.  This function stores the upper bound (inclusive) of the result
in BOUND_SCALAR.   

References build_int_cst(), fold_build2, known_ge, maybe_gt, and TREE_TYPE.

Referenced by vect_do_peeling().

◆ vect_gen_vector_loop_niters()

void vect_gen_vector_loop_niters ( loop_vec_info loop_vinfo,
tree niters,
tree * niters_vector_ptr,
tree * step_vector_ptr,
bool niters_no_overflow )
NITERS is the number of times that the original scalar loop executes
after peeling.  Work out the maximum number of iterations N that can
be handled by the vectorized form of the loop and then either:

a) set *STEP_VECTOR_PTR to the vectorization factor and generate:

     niters_vector = N

b) set *STEP_VECTOR_PTR to one and generate:

     niters_vector = N / vf

In both cases, store niters_vector in *NITERS_VECTOR_PTR and add
any new statements on the loop preheader edge.  NITERS_NO_OVERFLOW
is true if NITERS doesn't overflow (i.e. if NITERS is always nonzero).   

References build_int_cst(), build_one_cst(), create_tmp_var, exact_log2(), fold_build2, force_gimple_operand(), gsi_insert_seq_on_edge_immediate(), poly_int< N, C >::is_constant(), is_gimple_val(), loop_preheader_edge(), LOOP_VINFO_LOOP, LOOP_VINFO_PEELING_FOR_GAPS, LOOP_VINFO_USING_PARTIAL_VECTORS_P, LOOP_VINFO_VECT_FACTOR, wi::max_value(), NULL, NULL_TREE, wi::one(), wi::rshift(), set_range_info(), TREE_TYPE, TYPE_PRECISION, and TYPE_SIGN.

Referenced by vect_do_peeling(), and vect_transform_loop().

◆ vect_gen_vector_loop_niters_mult_vf()

static void vect_gen_vector_loop_niters_mult_vf ( loop_vec_info loop_vinfo,
tree niters_vector,
tree * niters_vector_mult_vf_ptr )
static
Given NITERS_VECTOR which is the number of iterations for vectorized
loop specified by LOOP_VINFO after vectorization, compute the number
of iterations before vectorization (niters_vector * vf) and store it
to NITERS_VECTOR_MULT_VF_PTR.   

References build_int_cst(), create_tmp_var, exact_log2(), fold_build2, force_gimple_operand(), gcc_assert, gsi_insert_seq_before(), GSI_SAME_STMT, gsi_start_bb(), is_gimple_val(), LOOP_VINFO_EARLY_BREAKS_VECT_PEELED, LOOP_VINFO_IV_EXIT, LOOP_VINFO_VECT_FACTOR, NULL, and TREE_TYPE.

Referenced by vect_do_peeling().

◆ vect_get_main_loop_result()

tree vect_get_main_loop_result ( loop_vec_info loop_vinfo,
tree main_loop_value,
tree skip_value )
LOOP_VINFO is an epilogue loop whose corresponding main loop can be skipped.
Return a value that equals:

- MAIN_LOOP_VALUE when LOOP_VINFO is entered from the main loop and
- SKIP_VALUE when the main loop is skipped.   

References add_phi_arg(), create_phi_node(), gcc_assert, _loop_vec_info::main_loop_edge, make_ssa_name(), _loop_vec_info::skip_main_loop_edge, TREE_TYPE, and UNKNOWN_LOCATION.

Referenced by vect_transform_cycle_phi().

◆ vect_iv_increment_position()

void vect_iv_increment_position ( edge loop_exit,
gimple_stmt_iterator * bsi,
bool * insert_after )
Stores the standard position for induction variable increment in belonging to
LOOP_EXIT (just before the exit condition of the given exit to BSI.
INSERT_AFTER is set to true if the increment should be inserted after
*BSI.   

References gsi_last_bb().

Referenced by vect_create_epilog_for_reduction(), vect_set_loop_condition_normal(), vect_set_loop_condition_partial_vectors_avx512(), and vect_set_loop_controls_directly().

◆ vect_loop_versioning()

class loop * vect_loop_versioning ( loop_vec_info loop_vinfo,
gimple * loop_vectorized_call )
Function vect_loop_versioning.

If the loop has data references that may or may not be aligned or/and
has data reference relations whose independence was not proven then
two versions of the loop need to be generated, one which is vectorized
and one which isn't.  A test is then generated to control which of the
loops is executed.  The test checks for the alignment of all of the
data references that may or may not be aligned.  An additional
sequence of runtime tests is generated for each pairs of DDRs whose
independence was not proven.  The vectorized version of loop is
executed only if both alias and alignment tests are passed.

The test generated to check which version of loop is executed
is modified to also check for profitability as indicated by the
cost model threshold TH.

The versioning precondition(s) are placed in *COND_EXPR and
*COND_EXPR_STMT_LIST.   

References add_phi_arg(), adjust_phi_and_debug_stmts(), profile_probability::always(), profile_count::apply_probability(), as_a(), bb_loop_depth(), boolean_false_node, boolean_true_node, boolean_type_node, build_int_cst(), build_zero_cst(), CDI_DOMINATORS, copy_ssa_name(), basic_block_def::count, create_phi_node(), dom_info_available_p(), dominated_by_p(), dump_enabled_p(), dump_printf_loc(), EDGE_COMPLEX, EDGE_COUNT, EDGE_PRED, EDGE_SUCC, expr, EXPR_P, extract_true_false_edges_from_block(), first_dom_son(), flow_bb_inside_loop_p(), flow_loop_nested_p(), flush_pending_stmts(), fold_build2, fold_loop_internal_call(), FOR_EACH_SSA_USE_OPERAND, force_gimple_operand_1(), free_original_copy_tables(), gcc_assert, get_current_def(), dump_user_location_t::get_location_t(), get_loop_copy(), gimple_bb(), gimple_build_cond(), gimple_call_lhs(), gimple_cond_set_condition_from_tree(), gimple_phi_arg_location_from_edge(), gimple_seq_add_seq(), gsi_end_p(), gsi_for_stmt(), gsi_insert_after(), gsi_insert_seq_before(), gsi_last_bb(), GSI_NEW_STMT, gsi_next(), GSI_SAME_STMT, gsi_start(), gsi_start_phis(), gsi_stmt(), loop::header, initialize_original_copy_tables(), loop::inner, profile_probability::invert(), is_gimple_condexpr_for_cond(), is_gimple_val(), profile_probability::likely(), LOCATION_LOCUS, LOOP_C_FINITE, loop_constraint_set_p(), loop_outer(), loop_preheader_edge(), LOOP_REQUIRES_VERSIONING_FOR_ALIAS, LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT, LOOP_REQUIRES_VERSIONING_FOR_NITERS, LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND, loop_version(), LOOP_VINFO_COST_MODEL_THRESHOLD, LOOP_VINFO_IV_EXIT, LOOP_VINFO_LOOP, LOOP_VINFO_NITERSM1, LOOP_VINFO_SCALAR_LOOP, LOOP_VINFO_SCALAR_LOOP_SCALING, LOOP_VINFO_VERSIONING_THRESHOLD, make_edge(), MSG_NOTE, MSG_OPTIMIZED_LOCATIONS, MSG_PRIORITY_USER_FACING, loop::next, next_dom_son(), NULL, NULL_TREE, loop::num, outermost_invariant_loop_for_expr(), gphi_iterator::phi(), PHI_ARG_DEF_FROM_EDGE, PHI_RESULT, basic_block_def::preds, replace_uses_by(), scale_loop_frequencies(), set_immediate_dominator(), single_exit(), split_block(), split_edge(), profile_probability::sqrt(), SSA_NAME_DEF_STMT, SSA_OP_USE, ssa_redirect_edge(), superloop_at_depth(), TODO_update_ssa_no_phi, TREE_CODE, TREE_TYPE, UNKNOWN_LOCATION, unshare_expr(), update_ssa(), update_stmt(), USE_FROM_PTR, vect_apply_runtime_profitability_check_p(), vect_create_cond_for_alias_checks(), vect_create_cond_for_align_checks(), vect_create_cond_for_lower_bounds(), vect_create_cond_for_niters_checks(), vect_create_cond_for_unequal_addrs(), vect_free_loop_info_assumptions(), vect_location, and vect_loop_vectorized_call().

Referenced by vect_transform_loop().

◆ vect_maybe_permute_loop_masks()

static bool vect_maybe_permute_loop_masks ( gimple_seq * seq,
rgroup_controls * dest_rgm,
rgroup_controls * src_rgm )
static
Try to use permutes to define the masks in DEST_RGM using the masks
in SRC_RGM, given that the former has twice as many masks as the
latter.  Return true on success, adding any new statements to SEQ.   

References build1(), rgroup_controls::controls, gcc_assert, gimple_build_assign(), gimple_seq_add_stmt(), i, insn_data, interleave_supported_p(), make_ssa_name(), rgroup_controls::max_nscalars_per_iter, insn_operand_data::mode, insn_data_d::operand, optab_handler(), rgroup_controls::type, TYPE_MODE, vect_gen_perm_mask_checked(), and vect_halve_mask_nunits().

Referenced by vect_set_loop_condition_partial_vectors().

◆ vect_prepare_for_masked_peels()

void vect_prepare_for_masked_peels ( loop_vec_info loop_vinfo)

◆ vect_set_loop_condition()

void vect_set_loop_condition ( class loop * loop,
edge loop_e,
loop_vec_info loop_vinfo,
tree niters,
tree step,
tree final_iv,
bool niters_maybe_zero )
If we're using fully-masked loops, make LOOP iterate:

   N == (NITERS - 1) / STEP + 1

times.  When NITERS is zero, this is equivalent to making the loop
execute (1 << M) / STEP times, where M is the precision of NITERS.
NITERS_MAYBE_ZERO is true if this last case might occur.

If we're not using fully-masked loops, make LOOP iterate:

   N == (NITERS - STEP) / STEP + 1

times, where NITERS is known to be outside the range [1, STEP - 1].
This is equivalent to making the loop execute NITERS / STEP times
when NITERS is nonzero and (1 << M) / STEP times otherwise.
NITERS_MAYBE_ZERO again indicates whether this last case might occur.

If FINAL_IV is nonnull, it is an SSA name that should be set to
N * STEP on exit from the loop.

Assumption: the exit-condition of LOOP is the last stmt in the loop.   

References dump_enabled_p(), dump_printf_loc(), get_loop_exit_condition(), gsi_for_stmt(), gsi_remove(), vec_info::lookup_stmt(), LOOP_VINFO_PARTIAL_VECTORS_STYLE, LOOP_VINFO_USING_PARTIAL_VECTORS_P, MSG_NOTE, vec_info::remove_stmt(), vect_location, vect_partial_vectors_avx512, vect_set_loop_condition_normal(), vect_set_loop_condition_partial_vectors(), and vect_set_loop_condition_partial_vectors_avx512().

Referenced by vect_do_peeling(), and vect_transform_loop().

◆ vect_set_loop_condition_normal()

◆ vect_set_loop_condition_partial_vectors()

static gcond * vect_set_loop_condition_partial_vectors ( class loop * loop,
edge exit_edge,
loop_vec_info loop_vinfo,
tree niters,
tree final_iv,
bool niters_maybe_zero,
gimple_stmt_iterator loop_cond_gsi )
static
Set up the iteration condition and rgroup controls for LOOP, given
that LOOP_VINFO_USING_PARTIAL_VECTORS_P is true for the vectorized
loop.  LOOP_VINFO describes the vectorization of LOOP.  NITERS is
the number of iterations of the original scalar loop that should be
handled by the vector loop.  NITERS_MAYBE_ZERO and FINAL_IV are as
for vect_set_loop_condition.

Insert the branch-back condition before LOOP_COND_GSI and return the
final gcond.   

References add_header_seq(), add_preheader_seq(), build_int_cst(), build_minus_one_cst(), build_one_cst(), build_zero_cst(), rgroup_controls::controls, rgroup_controls::factor, fold_build2, FOR_EACH_VEC_ELT, gcc_assert, gimple_build(), gimple_build_assign(), gimple_build_cond(), gimple_convert(), gsi_insert_before(), gsi_insert_on_edge_immediate(), GSI_SAME_STMT, i, LOOP_VINFO_EARLY_BREAKS_VECT_PEELED, LOOP_VINFO_FULLY_MASKED_P, LOOP_VINFO_LENS, LOOP_VINFO_MASK_SKIP_NITERS, LOOP_VINFO_MASKS, LOOP_VINFO_RGROUP_COMPARE_TYPE, LOOP_VINFO_RGROUP_IV_TYPE, LOOP_VINFO_USING_DECREMENTING_IV_P, LOOP_VINFO_USING_SELECT_VL_P, LOOP_VINFO_VECT_FACTOR, rgroup_controls::max_nscalars_per_iter, loop::nb_iterations, NULL, NULL_TREE, TREE_TYPE, TYPE_PRECISION, TYPE_UNSIGNED, vect_adjust_loop_lens_control(), vect_maybe_permute_loop_masks(), vect_rgroup_iv_might_wrap_p(), and vect_set_loop_controls_directly().

Referenced by vect_set_loop_condition().

◆ vect_set_loop_condition_partial_vectors_avx512()

static gcond * vect_set_loop_condition_partial_vectors_avx512 ( class loop * loop,
edge exit_edge,
loop_vec_info loop_vinfo,
tree niters,
tree final_iv,
bool niters_maybe_zero,
gimple_stmt_iterator loop_cond_gsi )
static
Set up the iteration condition and rgroup controls for LOOP in AVX512
style, given that LOOP_VINFO_USING_PARTIAL_VECTORS_P is true for the
vectorized loop.  LOOP_VINFO describes the vectorization of LOOP.  NITERS is
the number of iterations of the original scalar loop that should be
handled by the vector loop.  NITERS_MAYBE_ZERO and FINAL_IV are as
for vect_set_loop_condition.

Insert the branch-back condition before LOOP_COND_GSI and return the
final gcond.   

References add_preheader_seq(), tree_vector_builder::build(), build_int_cst(), build_minus_one_cst(), build_one_cst(), create_iv(), fold_build2, fold_convert, FOR_EACH_VEC_ELT_REVERSE, gcc_assert, GET_MODE_CLASS, gimple_build(), gimple_build_assign(), gimple_build_cond(), gimple_build_vector_from_val(), gimple_convert(), GSI_CONTINUE_LINKING, gsi_insert_before(), gsi_insert_on_edge_immediate(), GSI_SAME_STMT, i, known_ge, known_le, LOOP_VINFO_EARLY_BREAKS_VECT_PEELED, LOOP_VINFO_MASK_SKIP_NITERS, LOOP_VINFO_MASKS, LOOP_VINFO_RGROUP_IV_TYPE, LOOP_VINFO_VECT_FACTOR, loop::nb_iterations, NULL, NULL_TREE, poly_int_tree_p(), shift, poly_int< N, C >::to_constant(), TREE_TYPE, lang_hooks_for_types::type_for_mode, TYPE_MODE, TYPE_PRECISION, TYPE_UNSIGNED, TYPE_VECTOR_SUBPARTS(), lang_hooks::types, UNKNOWN_LOCATION, vect_iv_increment_position(), and vect_set_loop_control().

Referenced by vect_set_loop_condition().

◆ vect_set_loop_control()

static void vect_set_loop_control ( class loop * loop,
tree ctrl,
tree init_ctrl,
tree next_ctrl )
static
Define one loop rgroup control CTRL from loop LOOP.  INIT_CTRL is the value
that the control should have during the first iteration and NEXT_CTRL is the
value that it should have on subsequent iterations.   

References add_phi_arg(), create_phi_node(), loop::header, loop_latch_edge(), loop_preheader_edge(), and UNKNOWN_LOCATION.

Referenced by vect_set_loop_condition_partial_vectors_avx512(), and vect_set_loop_controls_directly().

◆ vect_set_loop_controls_directly()

static tree vect_set_loop_controls_directly ( class loop * loop,
loop_vec_info loop_vinfo,
gimple_seq * preheader_seq,
gimple_seq * header_seq,
gimple_stmt_iterator loop_cond_gsi,
rgroup_controls * rgc,
tree niters,
tree niters_skip,
bool might_wrap_p,
tree * iv_step,
tree * compare_step )
static
Helper for vect_set_loop_condition_partial_vectors.  Generate definitions
for all the rgroup controls in RGC and return a control that is nonzero
when the loop needs to iterate.  Add any new preheader statements to
PREHEADER_SEQ.  Use LOOP_COND_GSI to insert code before the exit gcond.

RGC belongs to loop LOOP.  The loop originally iterated NITERS
times and has been vectorized according to LOOP_VINFO.

If NITERS_SKIP is nonnull, the first iteration of the vectorized loop
starts with NITERS_SKIP dummy iterations of the scalar loop before
the real work starts.  The mask elements for these dummy iterations
must be 0, to ensure that the extra iterations do not have an effect.

It is known that:

  NITERS * RGC->max_nscalars_per_iter * RGC->factor

does not overflow.  However, MIGHT_WRAP_P says whether an induction
variable that starts at 0 and has step:

  VF * RGC->max_nscalars_per_iter * RGC->factor

might overflow before hitting a value above:

  (NITERS + NITERS_SKIP) * RGC->max_nscalars_per_iter * RGC->factor

This means that we cannot guarantee that such an induction variable
would ever hit a value that produces a set of all-false masks or zero
lengths for RGC.

Note: the cost of the code generated by this function is modeled
by vect_estimate_min_profitable_iters, so changes here may need
corresponding changes there.   

References rgroup_controls::bias_adjusted_ctrl, build_int_cst(), build_minus_one_cst(), rgroup_controls::controls, create_iv(), end(), rgroup_controls::factor, FOR_EACH_VEC_ELT_REVERSE, gcc_assert, gimple_build(), gimple_build_assign(), gimple_convert(), gimple_seq_add_seq(), gimple_seq_add_stmt(), gsi_insert_seq_before(), GSI_SAME_STMT, i, known_ge, known_le, LOOP_VINFO_FULLY_MASKED_P, LOOP_VINFO_IV_EXIT, LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS, LOOP_VINFO_RGROUP_COMPARE_TYPE, LOOP_VINFO_RGROUP_IV_TYPE, LOOP_VINFO_USING_DECREMENTING_IV_P, LOOP_VINFO_USING_SELECT_VL_P, LOOP_VINFO_VECT_FACTOR, make_ssa_name(), make_temp_ssa_name(), rgroup_controls::max_nscalars_per_iter, NULL, NULL_TREE, poly_int_tree_p(), TREE_TYPE, rgroup_controls::type, TYPE_VECTOR_SUBPARTS(), vect_gen_len(), vect_gen_while(), vect_gen_while_not(), vect_iv_increment_position(), and vect_set_loop_control().

Referenced by vect_set_loop_condition_partial_vectors().

◆ vect_update_init_of_dr()

static void vect_update_init_of_dr ( dr_vec_info * dr_info,
tree niters,
tree_code code )
static
Function vect_update_init_of_dr

If CODE is PLUS, the vector loop starts NITERS iterations after the
scalar one, otherwise CODE is MINUS and the vector loop starts NITERS
iterations before the scalar one (using masking to skip inactive
elements).  This function updates the information recorded in DR to
account for the difference.  Specifically, it updates the OFFSET
field of DR_INFO.   

References build_zero_cst(), dr_info::dr, DR_STEP, fold_build2, fold_convert, offset, and sizetype.

Referenced by vect_update_inits_of_drs().

◆ vect_update_inits_of_drs()

void vect_update_inits_of_drs ( loop_vec_info loop_vinfo,
tree niters,
tree_code code )
Function vect_update_inits_of_drs

Apply vect_update_inits_of_dr to all accesses in LOOP_VINFO.
CODE and NITERS are as for vect_update_inits_of_dr.   

References DUMP_VECT_SCOPE, fold_convert, FOR_EACH_VEC_ELT, i, vec_info::lookup_dr(), LOOP_VINFO_DATAREFS, sizetype, STMT_VINFO_GATHER_SCATTER_P, STMT_VINFO_SIMD_LANE_ACCESS_P, TREE_TYPE, types_compatible_p(), and vect_update_init_of_dr().

Referenced by update_epilogue_loop_vinfo(), vect_do_peeling(), and vect_prepare_for_masked_peels().

◆ vect_update_ivs_after_vectorizer()

static void vect_update_ivs_after_vectorizer ( loop_vec_info loop_vinfo,
tree niters,
edge update_e )
static
Function vect_update_ivs_after_vectorizer.

   "Advance" the induction variables of LOOP to the value they should take
   after the execution of LOOP.  This is currently necessary because the
   vectorizer does not handle induction variables that are used after the
   loop.  Such a situation occurs when the last iterations of LOOP are
   peeled, because:
   1. We introduced new uses after LOOP for IVs that were not originally used
      after LOOP: the IVs of LOOP are now used by an epilog loop.
   2. LOOP is going to be vectorized; this means that it will iterate N/VF
      times, whereas the loop IVs should be bumped N times.

   Input:
   - LOOP - a loop that is going to be vectorized. The last few iterations
            of LOOP were peeled.
   - NITERS - the number of iterations that LOOP executes (before it is
              vectorized). i.e, the number of times the ivs should be bumped.
   - UPDATE_E - a successor edge of LOOP->exit that is on the (only) path
                coming out from LOOP on which there are uses of the LOOP ivs
          (this is the path from LOOP->exit to epilog_loop->preheader).

                The new definitions of the ivs are placed in LOOP->exit.
                The phi args associated with the edge UPDATE_E in the bb
                UPDATE_E->dest are updated accordingly.

   Assumption 1: Like the rest of the vectorizer, this function assumes
   a single loop exit that has a single predecessor.

   Assumption 2: The phi nodes in the LOOP header and in update_bb are
   organized in the same order.

   Assumption 3: The access function of the ivs is simple enough (see
   vect_can_advance_ivs_p).  This assumption will be relaxed in the future.

   Assumption 4: Exactly one of the successors of LOOP exit-bb is on a path
   coming out of LOOP on which the ivs of LOOP are used (this is the path
   that leads to the epilog loop; other paths skip the epilog loop).  This
   path starts with the edge UPDATE_E, and its destination (denoted update_bb)
   needs to have its phis updated.

References adjust_phi_and_debug_stmts(), create_tmp_var, dump_enabled_p(), dump_printf_loc(), fold_build2, fold_build_pointer_plus, fold_convert, force_gimple_operand(), gcc_assert, gimple_phi_result(), gsi_end_p(), gsi_insert_seq_after(), gsi_insert_seq_before(), gsi_last_bb(), gsi_next(), GSI_SAME_STMT, gsi_start_phis(), loop::header, init_expr(), iv_phi_p(), vec_info::lookup_stmt(), loop_preheader_edge(), LOOP_VINFO_IV_EXIT, LOOP_VINFO_LOOP, MSG_NOTE, NULL, gphi_iterator::phi(), PHI_ARG_DEF_FROM_EDGE, POINTER_TYPE_P, STMT_VINFO_LOOP_PHI_EVOLUTION_PART, STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE, tree_is_chrec(), TREE_TYPE, type(), unshare_expr(), vect_location, vect_peel_nonlinear_iv_init(), vect_step_op_add, and vect_step_op_neg.

Referenced by vect_do_peeling().

Variable Documentation

◆ adjust_vec

vec<adjust_info, va_heap> adjust_vec
static
A stack of values to be adjusted in debug stmts.  We have to
process them LIFO, so that the closest substitution applies.  If we
processed them FIFO, without the stack, we might substitute uses
with a PHI DEF that would soon become non-dominant, and when we got
to the suitable one, it wouldn't have anything to substitute any
more.   

Referenced by adjust_debug_stmts(), adjust_vec_debug_stmts(), and vect_do_peeling().