GCC Middle and Back End API Reference
|
#include "config.h"
#include "system.h"
#include "coretypes.h"
#include "backend.h"
#include "target.h"
#include "rtl.h"
#include "tree.h"
#include "gimple.h"
#include "predict.h"
#include "stringpool.h"
#include "tree-vrp.h"
#include "tree-ssanames.h"
#include "expmed.h"
#include "memmodel.h"
#include "optabs.h"
#include "emit-rtl.h"
#include "diagnostic-core.h"
#include "fold-const.h"
#include "internal-fn.h"
#include "stor-layout.h"
#include "dojump.h"
#include "expr.h"
#include "attribs.h"
#include "asan.h"
#include "ubsan.h"
#include "recog.h"
#include "builtins.h"
#include "optabs-tree.h"
#include "gimple-ssa.h"
#include "tree-phinodes.h"
#include "ssa-iterators.h"
#include "explow.h"
#include "rtl-iter.h"
#include "gimple-range.h"
#include "fold-const-call.h"
#include "tree-ssa-live.h"
#include "tree-outof-ssa.h"
#include "langhooks.h"
#include "internal-fn.def"
Variables | |
const char *const | internal_fn_name_array [] |
const int | internal_fn_flags_array [] |
const_tree | internal_fn_fnspec_array [IFN_LAST+1] |
const direct_internal_fn_info | direct_internal_fn_array [IFN_LAST+1] |
static void(*const | internal_fn_expanders [])(internal_fn, gcall *) |
#define binary_direct { 0, 0, true } |
#define CASE | ( | CODE, | |
IFN ) |
#define CASE | ( | NAME | ) |
#define check_ptrs_direct { 0, 0, false } |
#define cond_binary_direct { 1, 1, true } |
#define cond_len_binary_direct { 1, 1, true } |
#define cond_len_ternary_direct { 1, 1, true } |
#define cond_len_unary_direct { 1, 1, true } |
#define cond_ternary_direct { 1, 1, true } |
#define cond_unary_direct { 1, 1, true } |
#define DEF_INTERNAL_COND_FN | ( | NAME, | |
... ) |
#define DEF_INTERNAL_FN | ( | CODE, | |
FLAGS, | |||
FNSPEC ) not_direct, |
#define DEF_INTERNAL_SIGNED_COND_FN | ( | NAME, | |
... ) |
#define DEF_INTERNAL_SIGNED_OPTAB_FN | ( | CODE, | |
FLAGS, | |||
SELECTOR, | |||
SIGNED_OPTAB, | |||
UNSIGNED_OPTAB, | |||
TYPE ) TYPE##_direct, |
#define DEF_INTERNAL_SIGNED_OPTAB_FN | ( | CODE, | |
FLAGS, | |||
SELECTOR, | |||
SIGNED_OPTAB, | |||
UNSIGNED_OPTAB, | |||
TYPE ) |
#define DEF_INTERNAL_SIGNED_OPTAB_FN | ( | CODE, | |
FLAGS, | |||
SELECTOR, | |||
SIGNED_OPTAB, | |||
UNSIGNED_OPTAB, | |||
TYPE ) |
#define DEF_INTERNAL_SIGNED_OPTAB_FN | ( | CODE, | |
FLAGS, | |||
SELECTOR, | |||
SIGNED_OPTAB, | |||
UNSIGNED_OPTAB, | |||
TYPE ) |
#define direct_binary_optab_supported_p direct_optab_supported_p |
#define direct_check_ptrs_optab_supported_p direct_optab_supported_p |
#define direct_cond_binary_optab_supported_p direct_optab_supported_p |
#define direct_cond_len_binary_optab_supported_p direct_optab_supported_p |
#define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p |
#define direct_cond_len_unary_optab_supported_p direct_optab_supported_p |
#define direct_cond_ternary_optab_supported_p direct_optab_supported_p |
#define direct_cond_unary_optab_supported_p direct_optab_supported_p |
#define direct_fold_extract_optab_supported_p direct_optab_supported_p |
#define direct_fold_left_optab_supported_p direct_optab_supported_p |
#define direct_fold_len_extract_optab_supported_p direct_optab_supported_p |
#define direct_gather_load_optab_supported_p convert_optab_supported_p |
#define direct_len_load_optab_supported_p direct_optab_supported_p |
#define direct_len_store_optab_supported_p direct_optab_supported_p |
#define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p |
#define direct_mask_fold_left_optab_supported_p direct_optab_supported_p |
#define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p |
#define direct_mask_len_load_optab_supported_p convert_optab_supported_p |
#define direct_mask_len_store_optab_supported_p convert_optab_supported_p |
#define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p |
#define direct_mask_load_optab_supported_p convert_optab_supported_p |
#define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p |
#define direct_mask_store_optab_supported_p convert_optab_supported_p |
#define direct_scatter_store_optab_supported_p convert_optab_supported_p |
#define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p |
#define direct_ternary_optab_supported_p direct_optab_supported_p |
#define direct_unary_convert_optab_supported_p convert_optab_supported_p |
#define direct_unary_optab_supported_p direct_optab_supported_p |
#define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p |
#define direct_vec_cond_optab_supported_p convert_optab_supported_p |
#define direct_vec_extract_optab_supported_p convert_optab_supported_p |
#define direct_vec_set_optab_supported_p direct_optab_supported_p |
#define direct_while_optab_supported_p convert_optab_supported_p |
#define expand_binary_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 2) |
#define expand_check_ptrs_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 4) |
#define expand_cond_binary_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 4) |
#define expand_cond_len_binary_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 6) |
#define expand_cond_len_ternary_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 7) |
#define expand_cond_len_unary_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 5) |
#define expand_cond_ternary_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 5) |
#define expand_cond_unary_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 3) |
#define expand_fold_extract_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 3) |
#define expand_fold_left_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 2) |
#define expand_fold_len_extract_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 5) |
#define expand_len_load_optab_fn expand_partial_load_optab_fn |
#define expand_len_store_optab_fn expand_partial_store_optab_fn |
#define expand_mask_fold_left_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 3) |
#define expand_mask_len_fold_left_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 5) |
#define expand_mask_len_load_optab_fn expand_partial_load_optab_fn |
#define expand_mask_len_store_optab_fn expand_partial_store_optab_fn |
#define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn |
#define expand_mask_load_optab_fn expand_partial_load_optab_fn |
#define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn |
#define expand_mask_store_optab_fn expand_partial_store_optab_fn |
#define expand_ternary_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 3) |
#define expand_unary_convert_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_convert_optab_fn (FN, STMT, OPTAB, 1) |
Expanders for optabs that can use expand_convert_optab_fn.
#define expand_unary_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_direct_optab_fn (FN, STMT, OPTAB, 1) |
Expanders for optabs that can use expand_direct_optab_fn.
Referenced by expand_CLRSB(), expand_CLZ(), expand_CTZ(), expand_FFS(), expand_PARITY(), and expand_POPCOUNT().
#define expand_vec_extract_optab_fn | ( | FN, | |
STMT, | |||
OPTAB ) expand_convert_optab_fn (FN, STMT, OPTAB, 2) |
#define fold_extract_direct { 2, 2, false } |
#define fold_left_direct { 1, 1, false } |
#define fold_len_extract_direct { 2, 2, false } |
#define FOR_EACH_CODE_MAPPING | ( | T | ) |
Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX that maps to a tree code CODE. There is also an IFN_COND_LEN_##SUFFIX for each such IFN_COND_##SUFFIX.
Referenced by conditional_internal_fn_code(), get_conditional_internal_fn(), and get_conditional_len_internal_fn().
#define FOR_EACH_COND_FN_PAIR | ( | T | ) |
#define gather_load_direct { 3, 1, false } |
#define INIT_PATTERN_VALUE 0xFE |
Expand the IFN_DEFERRED_INIT function: LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL); Initialize the LHS with zero/pattern according to its second argument INIT_TYPE: if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize; if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern to initialize; The LHS variable is initialized including paddings. The reasons to choose 0xFE for pattern initialization are: 1. It is a non-canonical virtual address on x86_64, and at the high end of the i386 kernel address space. 2. It is a very large float value (-1.694739530317379e+38). 3. It is also an unusual number for integers.
Referenced by expand_DEFERRED_INIT().
#define len_load_direct { -1, -1, false } |
#define len_store_direct { 3, 3, false } |
#define load_lanes_direct { -1, -1, false } |
#define mask_fold_left_direct { 1, 1, false } |
#define mask_len_fold_left_direct { 1, 1, false } |
#define mask_len_load_direct { -1, 4, false } |
#define mask_len_store_direct { 4, 5, false } |
#define mask_load_direct { -1, 2, false } |
#define mask_load_lanes_direct { -1, -1, false } |
#define mask_store_direct { 3, 2, false } |
#define mask_store_lanes_direct { 0, 0, false } |
#define not_direct { -2, -2, false } |
Create static initializers for the information returned by direct_internal_fn.
#define scatter_store_direct { 3, 1, false } |
#define store_lanes_direct { 0, 0, false } |
#define ternary_direct { 0, 0, true } |
#define unary_convert_direct { -1, 0, true } |
#define unary_direct { 0, 0, true } |
#define vec_cond_direct { 2, 0, false } |
#define vec_cond_mask_direct { 1, 0, false } |
#define vec_cond_mask_len_direct { 1, 1, false } |
#define vec_extract_direct { 0, -1, false } |
#define vec_set_direct { 3, 3, false } |
#define while_direct { 0, 2, false } |
|
static |
Add mask and len arguments according to the STMT.
References create_convert_operand_from(), create_input_operand(), expand_binop(), expand_normal(), GEN_INT, GET_MODE_PRECISION(), ggc_alloc(), gimple_call_arg(), gimple_call_internal_fn(), HOST_WIDE_INT_1U, internal_fn_len_index(), internal_fn_mask_index(), mask_rtx(), NULL_RTX, OPTAB_WIDEN, SCALAR_INT_MODE_P, poly_int< N, C >::to_constant(), TREE_TYPE, TYPE_MODE, TYPE_UNSIGNED, TYPE_VECTOR_SUBPARTS(), and VECTOR_BOOLEAN_TYPE_P.
Referenced by expand_gather_load_optab_fn(), expand_partial_load_optab_fn(), expand_partial_store_optab_fn(), and expand_scatter_store_optab_fn().
bool associative_binary_fn_p | ( | internal_fn | fn | ) |
Return true if FN is an associative binary operation.
References ggc_alloc().
Referenced by associative_binary_op_p().
bool can_interpret_as_conditional_op_p | ( | gimple * | stmt, |
tree * | cond_out, | ||
tree_code * | code_out, | ||
tree(&) | ops[3], | ||
tree * | else_out, | ||
tree * | len, | ||
tree * | bias ) |
Return true if STMT can be interpreted as a conditional tree code operation of the form: LHS = COND ? OP (RHS1, ...) : ELSE; operating elementwise if the operands are vectors. This includes the case of an all-true COND, so that the operation always happens. There is an alternative approach to interpret the STMT when the operands are vectors which is the operation predicated by both conditional mask and loop control length, the equivalent C code: for (int i = 0; i < NUNTIS; i++) { if (i < LEN + BIAS && COND[i]) LHS[i] = A[i] CODE B[i]; else LHS[i] = ELSE[i]; } When returning true, set: - *COND_OUT to the condition COND, or to NULL_TREE if the condition is known to be all-true - *CODE_OUT to the tree code - OPS[I] to operand I of *CODE_OUT - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the condition is known to be all true. - *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE. - *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE.
References separate_ops::code, conditional_internal_fn_code(), ggc_alloc(), gimple_assign_rhs1(), gimple_assign_rhs2(), gimple_assign_rhs3(), gimple_assign_rhs_code(), gimple_call_arg(), gimple_call_internal_fn(), gimple_call_internal_p(), gimple_call_num_args(), i, integer_truep(), internal_fn_len_index(), and NULL_TREE.
Referenced by convert_mult_to_fma(), and convert_mult_to_fma_1().
|
static |
Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand mode MODE can be expanded without using a libcall.
References CONSTANT_P, convert_modes(), delete_insns_since(), expand_mult(), find_widening_optab_handler, gen_raw_REG(), get_last_insn(), ggc_alloc(), last, LAST_VIRTUAL_REGISTER, expand_operand::mode, and NULL_RTX.
Referenced by expand_mul_overflow().
bool commutative_binary_fn_p | ( | internal_fn | fn | ) |
Return true if FN is a binary operation and if FN is commutative.
References ggc_alloc().
Referenced by commutative_binary_op_p(), and first_commutative_argument().
bool commutative_ternary_fn_p | ( | internal_fn | fn | ) |
Return true if FN is a ternary operation and if its first two arguments are commutative.
References ggc_alloc().
Referenced by commutative_ternary_op_p(), and first_commutative_argument().
tree_code conditional_internal_fn_code | ( | internal_fn | ifn | ) |
If IFN implements the conditional form of a tree code, return that tree code, otherwise return ERROR_MARK.
References CASE, FOR_EACH_CODE_MAPPING, and ggc_alloc().
Referenced by can_interpret_as_conditional_op_p(), check_reduction_path(), internal_fn_mask_index(), try_conditional_simplification(), vect_transform_loop(), vect_transform_reduction(), vectorizable_reduction(), and vectorize_fold_left_reduction().
Helper for expand_DIVMOD. Return true if the sequence starting with INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes.
References CALL_P, FOR_EACH_SUBRTX, GET_CODE, ggc_alloc(), INSN_P, NEXT_INSN(), and PATTERN().
Referenced by expand_DIVMOD().
|
static |
Return true if OPTAB is supported for TYPES, where the first type is the destination and the second type is the source. Used for convert optabs.
References convert_optab_handler(), ggc_alloc(), and TYPE_MODE.
|
static |
Return the optab used by internal function FN.
References gcc_unreachable, and IFN_LAST.
optab direct_internal_fn_optab | ( | internal_fn | fn, |
tree_pair | types ) |
Return the optab used by internal function FN.
References gcc_unreachable, and IFN_LAST.
Referenced by internal_check_ptrs_fn_supported_p(), internal_gather_scatter_fn_supported_p(), internal_len_load_store_bias(), and supportable_widening_operation().
bool direct_internal_fn_supported_p | ( | gcall * | stmt, |
optimization_type | opt_type ) |
Return true if the STMT is supported when the optimization type is OPT_TYPE, given that STMT is a call to a direct internal function.
References direct_internal_fn_supported_p(), direct_internal_fn_types(), ggc_alloc(), and gimple_call_internal_fn().
bool direct_internal_fn_supported_p | ( | internal_fn | fn, |
tree | type, | ||
optimization_type | opt_type ) |
Return true if FN is supported for type TYPE when the optimization type is OPT_TYPE. The caller knows that the "type0" and "type1" fields of FN's direct_internal_fn_info structure are the same.
References direct_internal_fn(), direct_internal_fn_supported_p(), gcc_checking_assert, ggc_alloc(), direct_internal_fn_info::type0, and direct_internal_fn_info::type1.
bool direct_internal_fn_supported_p | ( | internal_fn | fn, |
tree_pair | types, | ||
optimization_type | opt_type ) |
Return true if FN is supported for the types in TYPES when the optimization type is OPT_TYPE. The types are those associated with the "type0" and "type1" fields of FN's direct_internal_fn_info structure.
References gcc_unreachable, and IFN_LAST.
Referenced by build_call_internal(), build_cltz_expr(), build_popcount_expr(), can_produce_all_loop_masks_p(), cond_removal_in_builtin_zero_pattern(), convert_mult_to_fma(), direct_internal_fn_supported_p(), direct_internal_fn_supported_p(), direct_internal_fn_supported_p(), directly_supported_p(), fold_builtin_bit_query(), get_masked_reduction_fn(), gimple_expand_vec_cond_expr(), internal_fn_reciprocal(), match_saturation_arith(), match_single_bit_test(), maybe_build_call_expr_loc(), optimize_count_trailing_zeroes(), reassociate_bb(), replacement_internal_fn(), try_vectorize_loop_1(), use_mask_by_cond_expr_p(), vect_analyze_loop_2(), vect_gen_while(), vect_pattern_validate_optab(), vect_recog_abd_pattern(), vect_recog_average_pattern(), vect_recog_ctz_ffs_pattern(), vect_recog_divmod_pattern(), vect_recog_mulhs_pattern(), vect_recog_popcount_clz_ctz_ffs_pattern(), vect_recog_pow_pattern(), vect_recog_sat_add_pattern(), vectorizable_bb_reduc_epilogue(), vectorizable_call(), vectorizable_condition(), vectorizable_early_exit(), vectorizable_internal_function(), vectorizable_live_operation(), vectorizable_operation(), vectorizable_reduction(), and vectorized_internal_fn_supported_p().
tree_pair direct_internal_fn_types | ( | internal_fn | fn, |
gcall * | call ) |
CALL is a call whose return type and arguments are in principle compatible with FN (which satisfies direct_internal_fn_p). Return the types that should be used to determine whether the target supports FN.
References direct_internal_fn(), gimple_call_arg(), gimple_call_lhs(), separate_ops::op0, separate_ops::op1, TREE_TYPE, direct_internal_fn_info::type0, and direct_internal_fn_info::type1.
tree_pair direct_internal_fn_types | ( | internal_fn | fn, |
tree | return_type, | ||
tree * | args ) |
RETURN_TYPE and ARGS are a return type and argument list that are in principle compatible with FN (which satisfies direct_internal_fn_p). Return the types that should be used to determine whether the target supports FN.
References direct_internal_fn(), ggc_alloc(), TREE_TYPE, direct_internal_fn_info::type0, and direct_internal_fn_info::type1.
Referenced by build_call_internal(), direct_internal_fn_supported_p(), expand_convert_optab_fn(), expand_direct_optab_fn(), internal_fn_reciprocal(), maybe_build_call_expr_loc(), and replacement_internal_fn().
|
static |
Return true if OPTAB is supported for TYPES (whose modes should be the same) when the optimization type is OPT_TYPE. Used for simple direct optabs.
References direct_optab_handler(), gcc_checking_assert, ggc_alloc(), and TYPE_MODE.
|
static |
|
static |
Expand ADD_OVERFLOW STMT.
References expand_arith_overflow(), and ggc_alloc().
void expand_addsub_overflow | ( | location_t | loc, |
tree_code | code, | ||
tree | lhs, | ||
tree | arg0, | ||
tree | arg1, | ||
bool | unsr_p, | ||
bool | uns0_p, | ||
bool | uns1_p, | ||
bool | is_ubsan, | ||
tree * | datap ) |
Add sub/add overflow checking to the statement STMT. CODE says whether the operation is +, or -.
References add_reg_br_prob_note(), any_condjump_p(), BITS_PER_WORD, cfun, const0_rtx, CONST_SCALAR_INT_P, create_fixed_operand(), create_input_operand(), create_output_operand(), delete_insns_since(), do_compare_rtx_and_jump(), do_pending_stack_adjust(), emit_jump(), emit_label(), error_mark_node, profile_probability::even(), expand_arith_overflow_result_store(), expand_arith_set_overflow(), expand_binop(), expand_expr(), expand_normal(), expand_ubsan_result_store(), expand_unop(), EXPAND_WRITE, find_reg_note(), gcc_assert, gen_label_rtx(), gen_reg_rtx(), get_last_insn(), GET_MODE_PRECISION(), get_range_pos_neg(), ggc_alloc(), immed_wide_int_const(), JUMP_P, last, maybe_expand_insn(), wi::min_value(), expand_operand::mode, NULL, NULL_RTX, optab_handler(), OPTAB_LIB_WIDEN, pop_temp_slots(), PROFILE_ABSENT, profile_status_for_fn, push_temp_slots(), REG_P, SCALAR_INT_TYPE_MODE, SIGNED, expand_operand::target, TREE_TYPE, ubsan_build_overflow_builtin(), profile_probability::very_likely(), profile_probability::very_unlikely(), and write_complex_part().
Referenced by expand_arith_overflow(), expand_doubleword_mod(), expand_UBSAN_CHECK_ADD(), expand_UBSAN_CHECK_SUB(), and expand_vector_ubsan_overflow().
|
static |
References gcc_unreachable.
Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion.
References BITS_PER_WORD, build_nonstandard_integer_type(), separate_ops::code, const0_rtx, expand_addsub_overflow(), expand_arith_overflow_result_store(), expand_expr(), expand_expr_real_2(), expand_mul_overflow(), expand_neg_overflow(), EXPAND_NORMAL, EXPAND_WRITE, fold_convert_loc(), gcc_unreachable, get_min_precision(), GET_MODE_PRECISION(), get_range_pos_neg(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), gimple_location(), integer_zerop(), separate_ops::location, MAX, MIN, NULL, NULL_RTX, NULL_TREE, separate_ops::op0, separate_ops::op1, separate_ops::op2, SCALAR_INT_TYPE_MODE, SIGNED, smallest_int_mode_for_size(), targetm, TREE_TYPE, separate_ops::type, type(), TYPE_PRECISION, TYPE_UNSIGNED, UNSIGNED, and write_complex_part().
Referenced by expand_ADD_OVERFLOW(), expand_MUL_OVERFLOW(), and expand_SUB_OVERFLOW().
|
static |
Helper for expand_*_overflow. Store RES into the __real__ part of TARGET. If RES has larger MODE than __real__ part of TARGET, set the __imag__ part to 1 if RES doesn't fit into it. Similarly if LHS has smaller precision than its mode.
References convert_modes(), do_compare_rtx_and_jump(), emit_label(), expand_arith_set_overflow(), expand_shift(), expand_simple_binop(), gcc_assert, gen_label_rtx(), GET_MODE, GET_MODE_INNER, GET_MODE_PRECISION(), ggc_alloc(), immed_wide_int_const(), expand_operand::mode, NULL, NULL_RTX, OPTAB_LIB_WIDEN, wi::shifted_mask(), expand_operand::target, TREE_TYPE, TYPE_PRECISION, TYPE_UNSIGNED, profile_probability::very_likely(), and write_complex_part().
Referenced by expand_addsub_overflow(), expand_arith_overflow(), expand_mul_overflow(), and expand_neg_overflow().
Helper for expand_*_overflow. Set the __imag__ part to true (1 except for signed:1 type, in which case store -1).
References const1_rtx, constm1_rtx, expand_operand::target, TREE_TYPE, TYPE_PRECISION, TYPE_UNSIGNED, and write_complex_part().
Referenced by expand_addsub_overflow(), expand_arith_overflow_result_store(), expand_mul_overflow(), and expand_neg_overflow().
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
void expand_ASSUME | ( | internal_fn | , |
gcall * | ) |
|
static |
Expand atomic add fetch and cmp with 0.
References expand_ifn_atomic_op_fetch_cmp_0().
|
static |
Expand atomic and fetch and cmp with 0.
References expand_ifn_atomic_op_fetch_cmp_0().
|
static |
Expand atomic bit test and complement.
References expand_ifn_atomic_bit_test_and().
|
static |
Expand atomic bit test and reset.
References expand_ifn_atomic_bit_test_and().
|
static |
Expand atomic bit test and set.
References expand_ifn_atomic_bit_test_and().
|
static |
Expand atomic bit test and set.
References expand_ifn_atomic_compare_exchange().
|
static |
Expand atomic or fetch and cmp with 0.
References expand_ifn_atomic_op_fetch_cmp_0().
|
static |
Expand atomic sub fetch and cmp with 0.
References expand_ifn_atomic_op_fetch_cmp_0().
|
static |
Expand atomic xor fetch and cmp with 0.
References expand_ifn_atomic_op_fetch_cmp_0().
void expand_BITINTTOFLOAT | ( | internal_fn | , |
gcall * | stmt ) |
References DECIMAL_FLOAT_MODE_P, emit_library_call_value(), emit_move_insn(), expand_expr(), expand_normal(), EXPAND_WRITE, GET_MODE_NAME, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), init_one_libfunc(), LCT_PURE, expand_operand::mode, NULL_RTX, ptr_mode, expand_operand::target, TREE_TYPE, and TYPE_MODE.
|
static |
References as_combined_fn(), expand_assignment(), fold_const_call(), gcc_checking_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), NULL_TREE, TREE_CODE, and TREE_TYPE.
Referenced by expand_CLRSB(), expand_CLZ(), expand_CTZ(), expand_FFS(), expand_PARITY(), and expand_POPCOUNT().
|
static |
Return a memory reference of type TYPE for argument INDEX of STMT. Use argument INDEX + 1 to derive the second (TBAA) operand.
References build_aligned_type(), build_int_cst(), copy_node(), fold_build2, get_gimple_for_ssa_name(), ggc_alloc(), gimple_assign_rhs1(), gimple_assign_single_p(), gimple_call_arg(), integer_zerop(), offset, TMR_OFFSET, wi::to_poly_wide(), TREE_CODE, TREE_OPERAND, tree_to_shwi(), TREE_TYPE, type(), TYPE_ALIGN, types_compatible_p(), and wide_int_to_tree().
Referenced by expand_partial_load_optab_fn(), and expand_partial_store_optab_fn().
void expand_CLRSB | ( | internal_fn | fn, |
gcall * | stmt ) |
References expand_bitquery(), expand_unary_optab_fn, and ggc_alloc().
void expand_CLZ | ( | internal_fn | fn, |
gcall * | stmt ) |
References expand_bitquery(), expand_unary_optab_fn, and ggc_alloc().
|
static |
References gcc_unreachable.
|
static |
Coroutines, all should have been processed at this stage.
References gcc_unreachable.
|
static |
References gcc_unreachable.
|
static |
References gcc_unreachable.
|
static |
Expand a call to a convert-like optab using the operands in STMT. FN has a single output operand and NARGS input operands.
References convert_optab_handler(), direct_internal_fn_types(), expand_fn_using_insn(), and TYPE_MODE.
void expand_CTZ | ( | internal_fn | fn, |
gcall * | stmt ) |
References expand_bitquery(), expand_unary_optab_fn, and ggc_alloc().
|
static |
References AUTO_INIT_PATTERN, AUTO_INIT_UNINITIALIZED, build1(), build_call_expr(), build_fold_addr_expr, build_int_cst(), build_nonstandard_integer_type(), build_zero_cst(), builtin_decl_implicit(), DECL_P, DECL_SIZE_UNIT, expand_assignment(), expand_builtin_memset(), fold_build1, wi::from_buffer(), gcc_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), handled_component_p(), have_insn_for(), INIT_PATTERN_VALUE, int_mode_for_size(), integer_type_node, integer_zero_node, integer_zerop(), is_gimple_reg_type(), mark_addressable(), mem_ref_refers_to_non_mem_p(), non_mem_decl_p(), NULL_RTX, SET, TREE_CODE, tree_fits_uhwi_p(), tree_int_cst_equal(), TREE_INT_CST_LOW, TREE_OPERAND, tree_to_uhwi(), TREE_TYPE, TYPE_MODE, expand_operand::value, and wide_int_to_tree().
|
static |
Expand a call to FN using the operands in STMT. FN has a single output operand and NARGS input operands.
References direct_internal_fn_types(), direct_optab_handler(), expand_fn_using_insn(), and TYPE_MODE.
|
static |
Expand DIVMOD() using: a) optab handler for udivmod/sdivmod if it is available. b) If optab_handler doesn't exist, generate call to target-specific divmod libfunc.
References build2(), separate_ops::code, CONST_INT_P, contains_call_div_mod(), copy_rtx(), delete_insns_since(), emit_insn(), emit_move_insn(), end_sequence(), expand_doubleword_divmod(), expand_expr(), expand_expr_real_2(), EXPAND_NORMAL, expand_normal(), expand_twoval_binop(), EXPAND_WRITE, gcc_assert, gcc_unreachable, gen_reg_rtx(), get_insns(), get_last_insn(), GET_MODE_SIZE(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), gimple_location(), insns, INTVAL, is_int_mode(), last, separate_ops::location, make_tree(), expand_operand::mode, NULL, NULL_RTX, NULL_TREE, separate_ops::op0, separate_ops::op1, separate_ops::op2, optab_handler(), optab_libfunc(), optimize_insn_for_speed_p(), pow2p_hwi(), set_dst_reg_note(), start_sequence(), expand_operand::target, targetm, TREE_CODE, TREE_TYPE, separate_ops::type, type(), TYPE_MODE, TYPE_UNSIGNED, and word_mode.
void expand_DIVMODBITINT | ( | internal_fn | , |
gcall * | stmt ) |
References emit_library_call_value_1(), expand_normal(), ggc_alloc(), gimple_call_arg(), i, init_one_libfunc(), LCT_NORMAL, NULL_RTX, and ptr_mode.
|
static |
This should get expanded in the lower pass.
References error_at(), and gimple_location().
void expand_FFS | ( | internal_fn | fn, |
gcall * | stmt ) |
References expand_bitquery(), expand_unary_optab_fn, and ggc_alloc().
void expand_FLOATTOBITINT | ( | internal_fn | , |
gcall * | stmt ) |
|
static |
Expand STMT using instruction ICODE. The instruction has NOUTPUTS output operands and NINPUTS input operands, where NOUTPUTS is either 0 or 1. The output operand (if any) comes first, followed by the NINPUTS input operands.
References convert_move(), convert_to_mode(), create_convert_operand_from(), create_input_operand(), create_output_operand(), create_undefined_input_operand(), emit_move_insn(), expand_binop(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, gcc_checking_assert, GEN_INT, GET_CODE, GET_MODE, GET_MODE_PRECISION(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), HOST_WIDE_INT_1U, i, insn_data, INTEGRAL_TYPE_P, NULL_RTX, OPTAB_WIDEN, rtx_equal_p(), SCALAR_INT_MODE_P, SSA_NAME_IS_DEFAULT_DEF, SSA_NAME_VAR, SUBREG_PROMOTED_SIGN, SUBREG_PROMOTED_VAR_P, SUBREG_REG, poly_int< N, C >::to_constant(), TREE_CODE, TREE_TYPE, TYPE_MODE, TYPE_UNSIGNED, TYPE_VECTOR_SUBPARTS(), VAR_P, and VECTOR_BOOLEAN_TYPE_P.
Referenced by expand_convert_optab_fn(), and expand_direct_optab_fn().
|
static |
Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB.
References add_mask_and_len_args(), convert_optab_handler(), create_address_operand(), create_input_operand(), create_integer_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), i, NULL_RTX, offset, rtx_equal_p(), tree_to_shwi(), TREE_TYPE, TYPE_MODE, TYPE_UNSIGNED, and expand_operand::value.
|
static |
The position of an OpenACC execution engine along one compute axis.
References const0_rtx, emit_insn(), emit_move_insn(), expand_expr(), EXPAND_NORMAL, EXPAND_WRITE, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), NULL_RTX, expand_operand::target, and targetm.
|
static |
The size of an OpenACC compute dimension.
References emit_insn(), emit_move_insn(), expand_expr(), EXPAND_NORMAL, EXPAND_WRITE, GEN_INT, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), NULL_RTX, expand_operand::target, and targetm.
|
static |
This is expanded by oacc_device_lower pass.
References gcc_unreachable.
|
static |
This is expanded by oacc_device_lower pass.
References gcc_unreachable.
|
static |
This is expanded by oacc_device_lower pass.
References gcc_unreachable.
|
static |
This should get expanded in adjust_simduid_builtins.
References gcc_unreachable.
|
static |
This should get expanded in adjust_simduid_builtins.
References gcc_unreachable.
|
static |
This should get expanded in adjust_simduid_builtins.
References gcc_unreachable.
|
static |
This should get expanded in adjust_simduid_builtins.
References gcc_unreachable.
|
static |
This should get expanded in adjust_simduid_builtins.
References gcc_unreachable.
|
static |
This should get expanded in omp_device_lower pass.
References gcc_unreachable.
|
static |
Allocate per-lane storage and begin non-uniform execution region.
References create_input_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, gen_reg_rtx(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), NULL_RTX, rtx_equal_p(), expand_operand::target, targetm, and expand_operand::value.
|
static |
Deallocate per-lane storage and leave non-uniform execution region.
References create_input_operand(), expand_insn(), expand_normal(), gcc_assert, gcc_checking_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), and targetm.
|
static |
Lane index on SIMT targets: thread index in the warp on NVPTX. On targets without SIMT execution this should be expanded in omp_device_lower pass.
References emit_insn(), expand_expr(), EXPAND_WRITE, gcc_assert, ggc_alloc(), gimple_call_lhs(), NULL_RTX, expand_operand::target, and targetm.
|
static |
Lane index of the first SIMT lane that supplies a non-zero argument. This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the lane that executed the last iteration for handling OpenMP lastprivate.
References create_input_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), expand_operand::mode, NULL_RTX, rtx_equal_p(), expand_operand::target, targetm, TREE_TYPE, TYPE_MODE, and expand_operand::value.
|
static |
Non-transparent predicate used in SIMT lowering of OpenMP "ordered".
References create_input_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), expand_operand::mode, NULL_RTX, rtx_equal_p(), expand_operand::target, targetm, TREE_TYPE, TYPE_MODE, and expand_operand::value.
|
static |
This should get expanded in omp_device_lower pass.
References gcc_unreachable.
|
static |
"Or" boolean reduction across SIMT lanes: return non-zero in all lanes if any lane supplies a non-zero argument.
References create_input_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), expand_operand::mode, NULL_RTX, rtx_equal_p(), expand_operand::target, targetm, TREE_TYPE, TYPE_MODE, and expand_operand::value.
|
static |
Exchange between SIMT lanes with a "butterfly" pattern: source lane index is destination lane index XOR given offset.
References create_input_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), expand_operand::mode, NULL_RTX, rtx_equal_p(), expand_operand::target, targetm, TREE_TYPE, TYPE_MODE, and expand_operand::value.
|
static |
Exchange between SIMT lanes according to given source lane index.
References create_input_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), expand_operand::mode, NULL_RTX, rtx_equal_p(), expand_operand::target, targetm, TREE_TYPE, TYPE_MODE, and expand_operand::value.
|
static |
This should get expanded in omp_device_lower pass.
References gcc_unreachable.
|
static |
This should get expanded in omp_device_lower pass.
References gcc_unreachable.
|
static |
For hwasan stack tagging: Clear tags on the dynamically allocated space. For use after an object dynamically allocated on the stack goes out of scope.
References emit_library_call_value(), expand_expr(), EXPAND_NORMAL, expand_simple_binop(), gcc_assert, ggc_alloc(), gimple_call_arg(), HWASAN_STACK_BACKGROUND, init_one_libfunc(), LCT_NORMAL, NULL_RTX, OPTAB_WIDEN, ptr_mode, stack_pointer_rtx, and virtual_stack_dynamic_rtx.
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
|
static |
For hwasan stack tagging: Return a tag to be used for a dynamic allocation.
References emit_move_insn(), expand_expr(), EXPAND_NORMAL, expand_simple_binop(), gcc_assert, gen_int_mode(), GET_MODE, ggc_alloc(), gimple_call_lhs(), hwasan_current_frame_tag(), hwasan_frame_base(), hwasan_increment_frame_tag(), hwasan_truncate_to_tag_size(), expand_operand::mode, NULL_RTX, OPTAB_WIDEN, expand_operand::target, and targetm.
|
static |
For hwasan stack tagging: Tag a region of space in the shadow stack according to the base pointer of an object on the stack. N.b. the length provided in the internal call is required to be aligned to HWASAN_TAG_GRANULE_SIZE.
References emit_library_call(), expand_normal(), gcc_assert, gcc_checking_assert, ggc_alloc(), gimple_call_arg(), HWASAN_STACK_BACKGROUND, init_one_libfunc(), LCT_NORMAL, NULL_RTX, ptr_mode, targetm, TREE_CODE, and tree_to_shwi().
|
static |
For hwasan stack tagging: Store a tag into a pointer.
References emit_move_insn(), expand_expr(), EXPAND_NORMAL, expand_normal(), gcc_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), NULL_RTX, ptr_mode, expand_operand::target, and targetm.
Expand STMT, which is a call to internal function FN.
References expand_internal_call(), and gimple_call_internal_fn().
void expand_internal_call | ( | internal_fn | fn, |
gcall * | stmt ) |
Expand STMT as though it were a call to internal function FN.
References internal_fn_expanders.
Referenced by expand_call_stmt(), and expand_internal_call().
|
static |
Expand LAUNDER to assignment, lhs = arg0.
References expand_assignment(), gimple_call_arg(), and gimple_call_lhs().
|
static |
Expand LOAD_LANES call STMT using optab OPTAB.
References create_fixed_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, get_multi_vector_move(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), MEM_P, NULL_RTX, PUT_MODE(), rtx_equal_p(), expand_operand::target, TREE_TYPE, type(), TYPE_MODE, and expand_operand::value.
|
static |
This should get folded in tree-vectorizer.cc.
References gcc_unreachable.
|
static |
This should get folded in tree-vectorizer.cc.
References gcc_unreachable.
void expand_MASK_CALL | ( | internal_fn | , |
gcall * | ) |
References gcc_unreachable.
|
static |
Expand MUL_OVERFLOW STMT.
References expand_arith_overflow(), and ggc_alloc().
|
static |
Add mul overflow checking to the statement STMT.
References add_reg_br_prob_note(), any_condjump_p(), build_nonstandard_integer_type(), can_mult_highpart_p(), can_widen_mult_without_libcall(), cfun, separate_ops::code, const0_rtx, const1_rtx, constm1_rtx, convert_modes(), create_fixed_operand(), create_input_operand(), create_output_operand(), delete_insns_since(), do_compare_rtx_and_jump(), do_pending_stack_adjust(), emit_jump(), emit_label(), emit_move_insn(), error_mark_node, profile_probability::even(), expand_arith_overflow_result_store(), expand_arith_set_overflow(), expand_binop(), expand_expr(), expand_expr_real_2(), EXPAND_NORMAL, expand_normal(), expand_shift(), expand_simple_binop(), expand_ubsan_result_store(), expand_unop(), EXPAND_WRITE, find_reg_note(), force_reg(), gcc_assert, gcc_unreachable, gen_label_rtx(), gen_reg_rtx(), get_last_insn(), get_min_precision(), GET_MODE_2XWIDER_MODE(), GET_MODE_PRECISION(), get_range_pos_neg(), ggc_alloc(), gimple_assign_rhs_code(), immed_wide_int_const(), int_mode_for_size(), integer_pow2p(), is_gimple_assign(), JUMP_P, last, separate_ops::location, make_tree(), maybe_expand_insn(), wi::min_value(), expand_operand::mode, NULL, NULL_RTX, NULL_TREE, separate_ops::op0, separate_ops::op1, separate_ops::op2, optab_handler(), OPTAB_LIB_WIDEN, OPTAB_WIDEN, optimize_insn_for_speed_p(), pop_temp_slots(), PROFILE_ABSENT, profile_status_for_fn, push_temp_slots(), SCALAR_INT_TYPE_MODE, SIGNED, single_imm_use(), expand_operand::target, targetm, tree_log2(), TREE_TYPE, separate_ops::type, type(), TYPE_PRECISION, ubsan_build_overflow_builtin(), profile_probability::unlikely(), UNSIGNED, profile_probability::very_likely(), profile_probability::very_unlikely(), and write_complex_part().
Referenced by expand_arith_overflow(), expand_UBSAN_CHECK_MUL(), and expand_vector_ubsan_overflow().
void expand_MULBITINT | ( | internal_fn | , |
gcall * | stmt ) |
References emit_library_call_value_1(), expand_normal(), ggc_alloc(), gimple_call_arg(), i, init_one_libfunc(), LCT_NORMAL, NULL_RTX, and ptr_mode.
|
static |
Add negate overflow checking to the statement STMT.
References add_reg_br_prob_note(), any_condjump_p(), cfun, const0_rtx, create_fixed_operand(), create_input_operand(), create_output_operand(), delete_insns_since(), do_compare_rtx_and_jump(), do_pending_stack_adjust(), emit_jump(), emit_label(), expand_arith_overflow_result_store(), expand_arith_set_overflow(), expand_expr(), expand_normal(), expand_ubsan_result_store(), expand_unop(), EXPAND_WRITE, find_reg_note(), gen_label_rtx(), gen_reg_rtx(), get_last_insn(), ggc_alloc(), JUMP_P, last, maybe_expand_insn(), expand_operand::mode, NULL, NULL_RTX, NULL_TREE, optab_handler(), pop_temp_slots(), PROFILE_ABSENT, profile_status_for_fn, push_temp_slots(), SCALAR_INT_TYPE_MODE, expand_operand::target, TREE_TYPE, TYPE_MIN_VALUE, ubsan_build_overflow_builtin(), profile_probability::very_likely(), profile_probability::very_unlikely(), and write_complex_part().
Referenced by expand_arith_overflow(), expand_UBSAN_CHECK_SUB(), and expand_vector_ubsan_overflow().
|
static |
Expand a NOP.
void expand_PARITY | ( | internal_fn | fn, |
gcall * | stmt ) |
References expand_bitquery(), expand_unary_optab_fn, and ggc_alloc().
|
static |
Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab * OPTAB.
References add_mask_and_len_args(), clear_mem_offset(), convert_optab_handler(), create_fixed_operand(), create_output_operand(), direct_optab_handler(), emit_move_insn(), expand_call_mem_ref(), expand_expr(), expand_insn(), EXPAND_WRITE, gcc_assert, get_multi_vector_move(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), i, internal_fn_mask_index(), MEM_P, NULL_RTX, NULL_TREE, rtx_equal_p(), set_mem_expr(), expand_operand::target, TREE_TYPE, type(), TYPE_MODE, and expand_operand::value.
|
static |
Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab * OPTAB.
References add_mask_and_len_args(), clear_mem_offset(), convert_optab_handler(), create_fixed_operand(), create_input_operand(), direct_optab_handler(), expand_call_mem_ref(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, get_multi_vector_move(), ggc_alloc(), gimple_call_arg(), i, internal_fn_mask_index(), internal_fn_stored_value_index(), MEM_P, NULL_RTX, NULL_TREE, set_mem_expr(), TREE_TYPE, type(), and TYPE_MODE.
void expand_PHI | ( | internal_fn | , |
gcall * | ) |
References gcc_unreachable.
void expand_POPCOUNT | ( | internal_fn | fn, |
gcall * | stmt ) |
References const0_rtx, const1_rtx, constm1_rtx, convert_to_mode(), do_pending_stack_adjust(), emit_insn(), emit_move_insn(), emit_store_flag(), end_sequence(), expand_bitquery(), expand_normal(), expand_simple_binop(), expand_unary_optab_fn, get_insns(), GET_MODE, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), gimple_call_num_args(), integer_zerop(), expand_operand::mode, NULL_RTX, OPTAB_DIRECT, optimize_insn_for_speed_p(), seq_cost(), start_sequence(), TREE_TYPE, and TYPE_MODE.
void expand_RAWMEMCHR | ( | internal_fn | , |
gcall * | stmt ) |
Expand IFN_RAWMEMCHR internal function.
References create_fixed_operand(), create_input_operand(), create_output_operand(), direct_optab_handler(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, get_memory_rtx(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), expand_operand::mode, NULL, NULL_RTX, rtx_equal_p(), TREE_TYPE, TYPE_MODE, and expand_operand::value.
|
static |
Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB.
References add_mask_and_len_args(), convert_optab_handler(), create_address_operand(), create_input_operand(), create_integer_operand(), expand_insn(), expand_normal(), ggc_alloc(), gimple_call_arg(), gimple_call_internal_fn(), i, internal_fn_stored_value_index(), offset, tree_to_shwi(), TREE_TYPE, TYPE_MODE, and TYPE_UNSIGNED.
|
static |
Set errno to EDOM.
References emit_move_insn(), gcc_unreachable, gen_int_mode(), gen_rtx_MEM(), GET_MODE, ggc_alloc(), and word_mode.
void expand_SHUFFLEVECTOR | ( | internal_fn | , |
gcall * | ) |
References gcc_unreachable.
void expand_SPACESHIP | ( | internal_fn | , |
gcall * | stmt ) |
References create_input_operand(), create_output_operand(), do_pending_stack_adjust(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), NULL_RTX, separate_ops::op1, separate_ops::op2, optab_handler(), rtx_equal_p(), expand_operand::target, TREE_TYPE, TYPE_MODE, and expand_operand::value.
|
static |
Expand STORE_LANES call STMT using optab OPTAB.
References create_fixed_operand(), create_input_operand(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, get_multi_vector_move(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), MEM_P, NULL_RTX, PUT_MODE(), expand_operand::target, TREE_TYPE, type(), and TYPE_MODE.
|
static |
Expand SUB_OVERFLOW STMT.
References expand_arith_overflow(), and ggc_alloc().
|
static |
This should get expanded in the tsan pass.
References gcc_unreachable.
|
static |
Expand UADDC STMT.
References create_input_operand(), create_output_operand(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gen_reg_rtx(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), expand_operand::mode, NULL_RTX, separate_ops::op1, separate_ops::op2, optab_handler(), expand_operand::target, TREE_TYPE, TYPE_MODE, and write_complex_part().
Referenced by expand_USUBC().
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
|
static |
Expand UBSAN_CHECK_ADD call STMT.
References expand_addsub_overflow(), expand_vector_ubsan_overflow(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), gimple_location(), NULL, TREE_TYPE, and VECTOR_TYPE_P.
|
static |
Expand UBSAN_CHECK_MUL call STMT.
References expand_mul_overflow(), expand_vector_ubsan_overflow(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), gimple_location(), NULL, TREE_TYPE, and VECTOR_TYPE_P.
|
static |
Expand UBSAN_CHECK_SUB call STMT.
References expand_addsub_overflow(), expand_neg_overflow(), expand_vector_ubsan_overflow(), ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), gimple_location(), integer_zerop(), NULL, TREE_TYPE, and VECTOR_TYPE_P.
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
|
static |
Helper for expand_*_overflow. Store RES into TARGET.
References convert_move(), do_compare_rtx_and_jump(), emit_move_insn(), expand_shift(), expand_simple_binop(), gen_reg_rtx(), GET_CODE, GET_MODE_PRECISION(), ggc_alloc(), immed_wide_int_const(), expand_operand::mode, NULL, NULL_RTX, OPTAB_LIB_WIDEN, wi::shifted_mask(), SUBREG_PROMOTED_SIGN, SUBREG_PROMOTED_VAR_P, SUBREG_REG, expand_operand::target, TREE_CODE, TREE_TYPE, TYPE_PRECISION, TYPE_UNSIGNED, and profile_probability::very_unlikely().
Referenced by expand_addsub_overflow(), expand_mul_overflow(), and expand_neg_overflow().
|
static |
This should get expanded in the sanopt pass.
References gcc_unreachable.
|
static |
Expand the IFN_UNIQUE function according to its first argument.
References const0_rtx, emit_insn(), expand_expr(), expand_normal(), EXPAND_WRITE, gcc_unreachable, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), NULL_RTX, expand_operand::target, targetm, and TREE_INT_CST_LOW.
|
static |
Expand USUBC STMT.
References expand_UADDC().
|
static |
IFN_VA_ARG is supposed to be expanded at pass_stdarg. So this dummy function should never be called.
References gcc_unreachable.
|
static |
Expand VCOND_MASK optab internal function. The expansion of STMT happens based on OPTAB table associated.
References convert_optab_handler(), create_input_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, gcc_assert, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), expand_operand::mode, NULL_RTX, rtx_equal_p(), expand_operand::target, TREE_TYPE, TYPE_MODE, and expand_operand::value.
|
static |
Expand VCOND, VCONDU and VCONDEQ optab internal functions. The expansion of STMT happens based on OPTAB table associated.
References convert_optab_handler(), create_fixed_operand(), create_input_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), int_cst_value(), expand_operand::mode, NULL_RTX, operand_equal_p(), rtx_equal_p(), expand_operand::target, TREE_TYPE, TYPE_MODE, TYPE_UNSIGNED, expand_operand::value, vector_compare_rtx(), and XEXP.
|
static |
IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector. So this dummy function should never be called.
References gcc_unreachable.
|
static |
Expand VEC_SET internal functions.
References create_convert_operand_from(), create_fixed_operand(), create_input_operand(), emit_move_insn(), expand_expr(), expand_normal(), EXPAND_WRITE, gcc_unreachable, gen_reg_rtx(), GET_MODE_INNER, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), maybe_expand_insn(), NULL_RTX, optab_handler(), expand_operand::target, TREE_TYPE, TYPE_MODE, and expand_operand::value.
|
static |
Expand UBSAN_CHECK_* internal function if it has vector operands.
References assign_temp(), bitsize_int, build4_loc(), build_array_type_nelts(), build_int_cst(), separate_ops::code, const0_rtx, do_compare_rtx_and_jump(), do_pending_stack_adjust(), emit_label(), emit_move_insn(), expand_addsub_overflow(), expand_expr(), expand_expr_real_2(), expand_mul_overflow(), expand_neg_overflow(), EXPAND_NORMAL, expand_normal(), EXPAND_WRITE, fold_build1_loc(), fold_build3_loc(), gcc_unreachable, gen_int_mode(), gen_label_rtx(), gen_reg_rtx(), GET_MODE, ggc_alloc(), i, integer_zerop(), poly_int< N, C >::is_constant(), separate_ops::location, make_tree(), MEM_P, NULL, NULL_RTX, NULL_TREE, separate_ops::op0, separate_ops::op1, separate_ops::op2, optab_default, optab_for_tree_code(), optab_handler(), sizetype, TREE_CODE, tree_to_uhwi(), TREE_TYPE, separate_ops::type, TYPE_MODE, TYPE_SIZE, TYPE_VECTOR_SUBPARTS(), uniform_vector_p(), unknown_optab, VECTOR_MODE_P, and profile_probability::very_likely().
Referenced by expand_UBSAN_CHECK_ADD(), expand_UBSAN_CHECK_MUL(), and expand_UBSAN_CHECK_SUB().
|
static |
Expand WHILE_ULT call STMT using optab OPTAB.
References convert_optab_handler(), create_input_operand(), create_integer_operand(), create_output_operand(), emit_move_insn(), expand_expr(), expand_insn(), expand_normal(), EXPAND_WRITE, ggc_alloc(), gimple_call_arg(), gimple_call_lhs(), i, NULL_RTX, rtx_equal_p(), TREE_TYPE, TYPE_MODE, TYPE_VECTOR_SUBPARTS(), and VECTOR_MODE_P.
int first_commutative_argument | ( | internal_fn | fn | ) |
If FN is commutative in two consecutive arguments, return the index of the first, otherwise return -1.
References commutative_binary_fn_p(), commutative_ternary_fn_p(), and ggc_alloc().
internal_fn get_conditional_internal_fn | ( | internal_fn | fn | ) |
Return a function that only performs internal function FN when a certain condition is met and that uses a given fallback value otherwise. In other words, the returned function FN' is such that: LHS = FN' (COND, A1, ... An, ELSE) is equivalent to the C expression: LHS = COND ? FN (A1, ..., An) : ELSE; operating elementwise if the operands are vectors. Return IFN_LAST if no such function exists.
References CASE, FOR_EACH_COND_FN_PAIR, and IFN_LAST.
internal_fn get_conditional_internal_fn | ( | tree_code | code | ) |
Return a function that only performs CODE when a certain condition is met and that uses a given fallback value otherwise. For example, if CODE is a binary operation associated with conditional function FN: LHS = FN (COND, A, B, ELSE) is equivalent to the C expression: LHS = COND ? A CODE B : ELSE; operating elementwise if the operands are vectors. Return IFN_LAST if no such function exists.
References CASE, separate_ops::code, FOR_EACH_CODE_MAPPING, and IFN_LAST.
internal_fn get_conditional_len_internal_fn | ( | tree_code | code | ) |
Like get_conditional_internal_fn, but return a function that additionally restricts the operation to the leading elements of a vector. The number of elements to process is given by a length and bias pair, as for IFN_LOAD_LEN. The values of the remaining elements are taken from the fallback ("else") argument. For example, if CODE is a binary operation associated with FN: LHS = FN (COND, A, B, ELSE, LEN, BIAS) is equivalent to the C code: for (int i = 0; i < NUNITS; i++) { if (i < LEN + BIAS && COND[i]) LHS[i] = A[i] CODE B[i]; else LHS[i] = ELSE[i]; }
References CASE, separate_ops::code, FOR_EACH_CODE_MAPPING, and IFN_LAST.
Referenced by vectorizable_operation().
internal_fn get_len_internal_fn | ( | internal_fn | fn | ) |
If there exists an internal function like IFN that operates on vectors, but with additional length and bias parameters, return the internal_fn for that function, otherwise return IFN_LAST.
References IFN_LAST.
Referenced by convert_conditional_op(), and vectorizable_call().
Return minimum precision needed to represent all values of ARG in SIGNed integral type.
References CONVERT_EXPR_CODE_P, CONVERT_EXPR_P, wi::ext(), g, get_global_range_query(), get_min_precision(), get_range_pos_neg(), ggc_alloc(), gimple_assign_rhs1(), gimple_assign_rhs_code(), INTEGRAL_TYPE_P, is_gimple_assign(), MAX, MIN, wi::min_precision(), wi::neg_p(), r, range_query::range_of_expr(), SIGNED, SSA_NAME_DEF_STMT, wi::to_wide(), wi::to_widest(), TREE_CODE, TREE_OPERAND, TREE_TYPE, TYPE_PRECISION, TYPE_SIGN, TYPE_UNSIGNED, and UNSIGNED.
Referenced by expand_arith_overflow(), expand_mul_overflow(), and get_min_precision().
|
static |
ARRAY_TYPE is an array of vector modes. Return the associated insn for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none.
References convert_optab_handler(), gcc_assert, ggc_alloc(), TREE_CODE, TREE_TYPE, and TYPE_MODE.
Referenced by expand_load_lanes_optab_fn(), expand_partial_load_optab_fn(), expand_partial_store_optab_fn(), and expand_store_lanes_optab_fn().
internal_fn get_unconditional_internal_fn | ( | internal_fn | ifn | ) |
If IFN implements the conditional form of an unconditional internal function, return that unconditional function, otherwise return IFN_LAST.
References CASE, FOR_EACH_COND_FN_PAIR, and IFN_LAST.
Referenced by internal_fn_mask_index(), and try_conditional_simplification().
void init_internal_fns | ( | ) |
Initialize internal function tables.
References IFN_LAST, and internal_fn_fnspec_array.
Referenced by build_common_builtin_nodes().
bool internal_check_ptrs_fn_supported_p | ( | internal_fn | ifn, |
tree | type, | ||
poly_uint64 | length, | ||
unsigned int | align ) |
Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN for pointers of type TYPE when the accesses have LENGTH bytes and their common byte alignment is ALIGN.
References direct_internal_fn_optab(), direct_optab_handler(), GEN_INT, ggc_alloc(), immed_wide_int_const(), insn_operand_matches(), and TYPE_MODE.
Referenced by create_ifn_alias_checks().
int internal_fn_else_index | ( | internal_fn | fn | ) |
If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the argument that is used when the condition is false. Return -1 otherwise.
References ggc_alloc().
Referenced by check_reduction_path(), and vect_transform_reduction().
int internal_fn_len_index | ( | internal_fn | fn | ) |
If FN takes a vector len argument, return the index of that argument, otherwise return -1.
References ggc_alloc().
Referenced by add_mask_and_len_args(), can_interpret_as_conditional_op_p(), gimple_fold_partial_load_store_mem_ref(), initialize_ao_ref_for_dse(), try_conditional_simplification(), vectorizable_call(), and vn_reference_lookup_3().
int internal_fn_mask_index | ( | internal_fn | fn | ) |
If FN takes a vector mask argument, return the index of that argument, otherwise return -1.
References conditional_internal_fn_code(), get_unconditional_internal_fn(), ggc_alloc(), and IFN_LAST.
Referenced by add_mask_and_len_args(), exist_non_indexing_operands_for_use_p(), expand_partial_load_optab_fn(), expand_partial_store_optab_fn(), gimple_fold_partial_load_store_mem_ref(), ifcvt_can_hoist(), vect_get_load_store_mask(), vect_get_smallest_scalar_type(), vect_recog_mask_conversion_pattern(), vectorizable_call(), vectorizable_load(), vectorizable_reduction(), vectorizable_store(), and vn_reference_lookup_3().
int internal_fn_stored_value_index | ( | internal_fn | fn | ) |
If FN takes a value that should be stored to memory, return the index of that argument, otherwise return -1.
References ggc_alloc().
Referenced by call_may_clobber_ref_p_1(), exist_non_indexing_operands_for_use_p(), expand_partial_store_optab_fn(), expand_scatter_store_optab_fn(), get_mem_type_for_internal_fn(), gimple_fold_partial_store(), initialize_ao_ref_for_dse(), vect_check_store_rhs(), vect_get_smallest_scalar_type(), vect_get_store_rhs(), vect_recog_mask_conversion_pattern(), and vn_reference_lookup_3().
bool internal_gather_scatter_fn_p | ( | internal_fn | fn | ) |
Return true if IFN is some form of gather load or scatter store.
References ggc_alloc().
Referenced by exist_non_indexing_operands_for_use_p(), and vect_check_gather_scatter().
bool internal_gather_scatter_fn_supported_p | ( | internal_fn | ifn, |
tree | vector_type, | ||
tree | memory_element_type, | ||
tree | offset_vector_type, | ||
int | scale ) |
Return true if the target supports gather load or scatter store function IFN. For loads, VECTOR_TYPE is the vector type of the load result, while for stores it is the vector type of the stored data argument. MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded or stored. OFFSET_VECTOR_TYPE is the vector type that holds the offset from the shared base address of each loaded or stored element. SCALE is the amount by which these offsets should be multiplied *after* they have been extended to address width.
References convert_optab_handler(), direct_internal_fn_optab(), GEN_INT, ggc_alloc(), insn_operand_matches(), internal_load_fn_p(), tree_int_cst_equal(), TREE_TYPE, TYPE_MODE, TYPE_SIZE, TYPE_UNSIGNED, and TYPE_VECTOR_SUBPARTS().
Referenced by check_load_store_for_partial_vectors(), and vect_gather_scatter_fn_p().
signed char internal_len_load_store_bias | ( | internal_fn | ifn, |
machine_mode | mode ) |
Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD or IFN_{LEN_,MASK_LEN_,}STORE. For now we only support the biases of 0 and -1 (in case 0 is not an allowable length for {len_,mask_len_}load or {len_,mask_len_}store). If none of the biases match what the backend provides, return VECT_PARTIAL_BIAS_UNSUPPORTED.
References convert_optab_handler(), direct_internal_fn_optab(), direct_optab_handler(), GEN_INT, ggc_alloc(), insn_operand_matches(), targetm, and VECT_PARTIAL_BIAS_UNSUPPORTED.
Referenced by vect_verify_loop_lens().
bool internal_load_fn_p | ( | internal_fn | fn | ) |
Return true if IFN is some form of load from memory.
References ggc_alloc().
Referenced by internal_gather_scatter_fn_supported_p(), vect_get_smallest_scalar_type(), vectorizable_call(), and vectorizable_load().
bool internal_store_fn_p | ( | internal_fn | fn | ) |
Return true if IFN is some form of store to memory.
References ggc_alloc().
Referenced by vect_build_slp_tree_1(), vect_check_store_rhs(), vect_get_smallest_scalar_type(), vect_recog_mask_conversion_pattern(), vectorizable_call(), vectorizable_store(), and vn_reference_lookup_3().
|
extern |
Given an internal_fn IFN that is a widening function, return its corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD.
References gcc_assert, gcc_unreachable, and widening_fn_p().
Referenced by supportable_widening_operation().
|
extern |
Geven an internal_fn IFN that is a widening function, return its corresponding LO and HI internal_fns.
References gcc_assert, gcc_unreachable, and widening_fn_p().
Referenced by supportable_widening_operation().
internal_fn lookup_internal_fn | ( | const char * | name | ) |
Return the internal function called NAME, or IFN_LAST if there's no such function.
References ggc_alloc(), i, IFN_LAST, and internal_fn_name().
|
static |
Return true if load/store lanes optab OPTAB is supported for array type TYPES.first when the optimization type is OPT_TYPE.
References convert_optab_handler(), gcc_assert, ggc_alloc(), TREE_CODE, TREE_TYPE, and TYPE_MODE.
Return true if IFN_SET_EDOM is supported.
Referenced by use_internal_fn().
bool vectorized_internal_fn_supported_p | ( | internal_fn | ifn, |
tree | type ) |
If TYPE is a vector type, return true if IFN is a direct internal function that is supported for that type. If TYPE is a scalar type, return true if IFN is a direct internal function that is supported for the target's preferred vector version of TYPE.
References build_vector_type_for_mode(), direct_internal_fn_supported_p(), opt_mode< T >::exists(), ggc_alloc(), OPTIMIZE_FOR_SPEED, related_vector_mode(), targetm, TYPE_MODE, VECTOR_MODE_P, and VECTOR_TYPE_P.
Referenced by convert_scalar_cond_reduction(), and ifcvt_can_predicate().
bool widening_fn_p | ( | code_helper | code | ) |
Return true if this CODE describes an internal_fn that returns a vector with elements twice as wide as the element size of the input vectors.
References as_internal_fn(), separate_ops::code, and internal_fn_p().
Referenced by lookup_evenodd_internal_fn(), lookup_hilo_internal_fn(), supportable_widening_operation(), and vectorizable_conversion().
const direct_internal_fn_info direct_internal_fn_array[IFN_LAST+1] |
Referenced by direct_internal_fn(), direct_internal_fn_p(), and vectorizable_internal_fn_p().
|
static |
Routines to expand each internal function, indexed by function number. Each routine has the prototype: expand_<NAME> (gcall *stmt) where STMT is the statement that performs the call.
Referenced by expand_internal_call().
const int internal_fn_flags_array[] |
The ECF_* flags of each internal function, indexed by function number.
Referenced by internal_fn_flags().
const_tree internal_fn_fnspec_array[IFN_LAST+1] |
Fnspec of each internal function, indexed by function number.
Referenced by init_internal_fns(), and internal_fn_fnspec().
Internal functions. Copyright (C) 2011-2024 Free Software Foundation, Inc. This file is part of GCC. GCC is free software; you can redistribute it and/or modify it under the terms of the GNU General Public License as published by the Free Software Foundation; either version 3, or (at your option) any later version. GCC is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for more details. You should have received a copy of the GNU General Public License along with GCC; see the file COPYING3. If not see <http://www.gnu.org/licenses/>.
For lang_hooks.types.type_for_mode.
The names of each internal function, indexed by function number.
Referenced by internal_fn_name().