GCC Middle and Back End API Reference
tree-vectorizer.h
Go to the documentation of this file.
1/* Vectorizer
2 Copyright (C) 2003-2025 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4
5This file is part of GCC.
6
7GCC is free software; you can redistribute it and/or modify it under
8the terms of the GNU General Public License as published by the Free
9Software Foundation; either version 3, or (at your option) any later
10version.
11
12GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13WARRANTY; without even the implied warranty of MERCHANTABILITY or
14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15for more details.
16
17You should have received a copy of the GNU General Public License
18along with GCC; see the file COPYING3. If not see
19<http://www.gnu.org/licenses/>. */
20
21#ifndef GCC_TREE_VECTORIZER_H
22#define GCC_TREE_VECTORIZER_H
23
25typedef struct _slp_tree *slp_tree;
26
27#include "tree-data-ref.h"
28#include "tree-hash-traits.h"
29#include "target.h"
30#include "internal-fn.h"
31#include "tree-ssa-operands.h"
32#include "gimple-match.h"
33#include "dominance.h"
34
35/* Used for naming of new temporaries. */
42
43/* Defines type of operation. */
49
50/* Define type of available alignment support. */
58
59/* Define type of peeling support to indicate how peeling for alignment can help
60 make vectorization supported. */
66
67/* Define type of def-use cross-iteration cycle. */
81
82/* Define operation type of linear/non-linear induction variable. */
90
91/* Define type of reduction. */
97
98 /* Retain a scalar phi and use a FOLD_EXTRACT_LAST within the loop
99 to implement:
100
101 for (int i = 0; i < VF; ++i)
102 res = cond[i] ? val[i] : res; */
104
105 /* Use a folding reduction within the loop to implement:
106
107 for (int i = 0; i < VF; ++i)
108 res = res OP val[i];
109
110 (with no reassocation). */
112};
113
114#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
115 || ((D) == vect_double_reduction_def) \
116 || ((D) == vect_nested_cycle))
117
118/* Structure to encapsulate information about a group of like
119 instructions to be presented to the target cost model. */
129
131
132/* Maps base addresses to an innermost_loop_behavior and the stmt it was
133 derived from that gives the maximum known alignment for that base. */
135 std::pair<stmt_vec_info, innermost_loop_behavior *> >
137
138/* Represents elements [START, START + LENGTH) of cyclical array OPS*
139 (i.e. OPS repeated to give at least START + LENGTH elements) */
141{
142 tree op (unsigned int i) const;
143 bool all_same_p () const;
144
146 unsigned int start;
147 unsigned int length;
148};
149
150/* Return element I of the slice. */
151inline tree
152vect_scalar_ops_slice::op (unsigned int i) const
153{
154 return (*ops)[(i + start) % ops->length ()];
155}
156
157/* Hash traits for vect_scalar_ops_slice. */
158struct vect_scalar_ops_slice_hash : typed_noop_remove<vect_scalar_ops_slice>
159{
162
163 static const bool empty_zero_p = true;
164
165 static void mark_deleted (value_type &s) { s.length = ~0U; }
166 static void mark_empty (value_type &s) { s.length = 0; }
167 static bool is_deleted (const value_type &s) { return s.length == ~0U; }
168 static bool is_empty (const value_type &s) { return s.length == 0; }
169 static hashval_t hash (const value_type &);
170 static bool equal (const value_type &, const compare_type &);
171};
172
173/* Describes how we're going to vectorize an individual load or store,
174 or a group of loads or stores. */
177
178 /* An access to an invariant address. This is used only for loads. */
180
181 /* A simple contiguous access. */
183
184 /* A contiguous access that goes down in memory rather than up,
185 with no additional permutation. This is used only for stores
186 of invariants. */
188
189 /* A simple contiguous access in which the elements need to be reversed
190 after loading or before storing. */
192
193 /* An access that uses IFN_LOAD_LANES or IFN_STORE_LANES. */
195
196 /* An access in which each scalar element is loaded or stored
197 individually. */
199
200 /* A hybrid of VMAT_CONTIGUOUS and VMAT_ELEMENTWISE, used for grouped
201 SLP accesses. Each unrolled iteration uses a contiguous load
202 or store for the whole group, but the groups from separate iterations
203 are combined in the same way as for VMAT_ELEMENTWISE. */
205
206 /* The access uses gather loads or scatter stores. */
210};
211
212/* Returns whether MAT is any of the VMAT_GATHER_SCATTER_* kinds. */
213
214inline bool
221
222/*-----------------------------------------------------------------*/
223/* Info on vectorized defs. */
224/*-----------------------------------------------------------------*/
248
249/************************************************************************
250 SLP
251 ************************************************************************/
256
257struct vect_data {
258 virtual ~vect_data () = default;
259};
260
261/* Analysis data from vectorizable_simd_clone_call for
262 call_simd_clone_vec_info_type. */
264 virtual ~vect_simd_clone_data () = default;
267
268 /* Selected SIMD clone's function info. First vector element
269 is SIMD clone's function decl, followed by a pair of trees (base + step)
270 for linear arguments (pair of NULLs for other arguments). */
272};
273
274/* Analysis data from vectorizable_load and vectorizable_store for
275 load_vec_info_type and store_vec_info_type. */
279 virtual ~vect_load_store_data () = default;
280
284 internal_fn lanes_ifn; // VMAT_LOAD_STORE_LANES
286 union {
287 internal_fn ifn; // VMAT_GATHER_SCATTER_IFN
288 tree decl; // VMAT_GATHER_SCATTER_DECL
289 } gs;
290 tree strided_offset_vectype; // VMAT_GATHER_SCATTER_IFN, originally strided
292 unsigned n_perms; // SLP_TREE_LOAD_PERMUTATION
293};
294
295/* A computation tree of an SLP instance. Each node corresponds to a group of
296 stmts to be packed in a SIMD stmt. */
297struct _slp_tree {
298 _slp_tree ();
299 ~_slp_tree ();
300
301 void push_vec_def (gimple *def);
302 void push_vec_def (tree def) { vec_defs.quick_push (def); }
303
304 /* Nodes that contain def-stmts of this node statements operands. */
306
307 /* A group of scalar stmts to be vectorized together. */
309 /* A group of scalar operands to be vectorized together. */
311 /* The representative that should be used for analysis and
312 code generation. */
314
315 struct {
316 /* SLP cycle the node resides in, or -1. */
317 int id;
318 /* The SLP operand index with the edge on the SLP cycle, or -1. */
321
322 /* Load permutation relative to the stores, NULL if there is no
323 permutation. */
325 /* Lane permutation of the operands scalar lanes encoded as pairs
326 of { operand number, lane number }. The number of elements
327 denotes the number of output lanes. */
329
331 /* Vectorized defs. */
333
334 /* Reference count in the SLP graph. */
335 unsigned int refcnt;
336 /* The maximum number of vector elements for the subtree rooted
337 at this node. */
339 /* The DEF type of this node. */
341 /* The number of scalar lanes produced by this node. */
342 unsigned int lanes;
343 /* The operation of this node. */
345 /* For gather/scatter memory operations the scale each offset element
346 should be multiplied by before being added to the base. */
348 /* For gather/scatter memory operations the loop-invariant base value. */
350 /* Whether uses of this load or feeders of this store are suitable
351 for load/store-lanes. */
353 /* For BB vect, flag to indicate this load node should be vectorized
354 as to avoid STLF fails because of related stores. */
356
358
359 /* The kind of operation as determined by analysis and optional
360 kind specific data. */
363
364 template <class T>
365 T& get_data (T& else_) { return data ? *static_cast <T *> (data) : else_; }
366
367 /* If not NULL this is a cached failed SLP discovery attempt with
368 the lanes that failed during SLP discovery as 'false'. This is
369 a copy of the matches array. */
370 bool *failed;
371
372 /* Allocate from slp_tree_pool. */
373 static void *operator new (size_t);
374
375 /* Return memory to slp_tree_pool. */
376 static void operator delete (void *, size_t);
377
378 /* Linked list of nodes to release when we free the slp_tree_pool. */
381};
382
383/* The enum describes the type of operations that an SLP instance
384 can perform. */
385
394
395/* SLP instance is a sequence of stmts in a loop that can be packed into
396 SIMD stmts. */
397typedef class _slp_instance {
398public:
399 /* The root of SLP tree. */
401
402 /* For vector constructors, the constructor stmt that the SLP tree is built
403 from, NULL otherwise. */
405
406 /* For slp_inst_kind_bb_reduc the defs that were not vectorized, NULL
407 otherwise. */
409
410 /* The group of nodes that contain loads of this SLP instance. */
412
413 /* The SLP node containing the reduction PHIs. */
415
416 /* Vector cost of this entry to the SLP graph. */
418
419 /* If this instance is the main entry of a subgraph the set of
420 entries into the same subgraph, including itself. */
422
423 /* The type of operation the SLP instance is performing. */
425
428
429
430/* Access Functions. */
431#define SLP_INSTANCE_TREE(S) (S)->root
432#define SLP_INSTANCE_LOADS(S) (S)->loads
433#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
434#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
435#define SLP_INSTANCE_KIND(S) (S)->kind
436
437#define SLP_TREE_CHILDREN(S) (S)->children
438#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
439#define SLP_TREE_SCALAR_OPS(S) (S)->ops
440#define SLP_TREE_REF_COUNT(S) (S)->refcnt
441#define SLP_TREE_VEC_DEFS(S) (S)->vec_defs
442#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
443#define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
444#define SLP_TREE_DEF_TYPE(S) (S)->def_type
445#define SLP_TREE_VECTYPE(S) (S)->vectype
446#define SLP_TREE_REPRESENTATIVE(S) (S)->representative
447#define SLP_TREE_LANES(S) (S)->lanes
448#define SLP_TREE_CODE(S) (S)->code
449#define SLP_TREE_TYPE(S) (S)->type
450#define SLP_TREE_GS_SCALE(S) (S)->gs_scale
451#define SLP_TREE_GS_BASE(S) (S)->gs_base
452#define SLP_TREE_REDUC_IDX(S) (S)->cycle_info.reduc_idx
453#define SLP_TREE_PERMUTE_P(S) ((S)->code == VEC_PERM_EXPR)
454
457{
460 return static_cast<vect_load_store_data *> (node->data)->memory_access_type;
461 return VMAT_UNINITIALIZED;
462}
463
470
471/* Key for map that records association between
472 scalar conditions and corresponding loop mask, and
473 is populated by vect_record_loop_mask. */
474
476{
477 scalar_cond_masked_key (tree t, unsigned ncopies_)
478 : ncopies (ncopies_)
479 {
481 }
482
484
485 unsigned ncopies;
490};
491
492template<>
494{
497
498 static inline hashval_t
500 {
502 h.add_int (v.code);
503 inchash::add_expr (v.op0, h, 0);
504 inchash::add_expr (v.op1, h, 0);
505 h.add_int (v.ncopies);
506 h.add_flag (v.inverted_p);
507 return h.end ();
508 }
509
510 static inline bool
512 {
513 return (existing.ncopies == candidate.ncopies
514 && existing.code == candidate.code
515 && existing.inverted_p == candidate.inverted_p
516 && operand_equal_p (existing.op0, candidate.op0, 0)
517 && operand_equal_p (existing.op1, candidate.op1, 0));
518 }
519
520 static const bool empty_zero_p = true;
521
522 static inline void
524 {
525 v.ncopies = 0;
526 v.inverted_p = false;
527 }
528
529 static inline bool
531 {
532 return v.ncopies == 0;
533 }
534
535 static inline void mark_deleted (value_type &) {}
536
537 static inline bool is_deleted (const value_type &)
538 {
539 return false;
540 }
541
542 static inline void remove (value_type &) {}
543};
544
546
547/* Key and map that records association between vector conditions and
548 corresponding loop mask, and is populated by prepare_vec_mask. */
549
552
553/* Describes two objects whose addresses must be unequal for the vectorized
554 loop to be valid. */
555typedef std::pair<tree, tree> vec_object_pair;
556
557/* Records that vectorization is only possible if abs (EXPR) >= MIN_VALUE.
558 UNSIGNED_P is true if we can assume that abs (EXPR) == EXPR. */
569
570/* Vectorizer state shared between different analyses like vector sizes
571 of the same CFG region. */
573public:
576
577 void save_datarefs();
578 void check_datarefs();
579
580 /* All data references. Freed by free_data_refs, so not an auto_vec. */
583
584 /* The loop nest in which the data dependences are computed. */
586
587 /* All data dependences. Freed by free_dependence_relations, so not
588 an auto_vec. */
590};
591
592/* Vectorizer state common between loop and basic-block vectorization. */
593class vec_info {
594public:
596 enum vec_kind { bb, loop };
597
599 ~vec_info ();
600
613
614 /* The type of vectorization. */
616
617 /* Shared vectorizer state. */
619
620 /* The mapping of GIMPLE UID to stmt_vec_info. */
622 /* Whether the above mapping is complete. */
624
625 /* Whether we've done a transform we think OK to not update virtual
626 SSA form. */
628
629 /* The SLP graph. */
631
632 /* Maps base addresses to an innermost_loop_behavior that gives the maximum
633 known alignment for that base. */
635
636 /* All interleaving chains of stores, represented by the first
637 stmt in the chain. */
639
640 /* The set of vector modes used in the vectorized region. */
642
643 /* The argument we should pass to related_vector_mode when looking up
644 the vector mode for a scalar mode, or VOIDmode if we haven't yet
645 made any decisions about which vector modes to use. */
646 machine_mode vector_mode;
647
648 /* The basic blocks in the vectorization region. For _loop_vec_info,
649 the memory is internally managed, while for _bb_vec_info, it points
650 to element space of an external auto_vec<>. This inconsistency is
651 not a good class design pattern. TODO: improve it with an unified
652 auto_vec<> whose lifetime is confined to vec_info object. */
654
655 /* The count of the basic blocks in the vectorization region. */
656 unsigned int nbbs;
657
658 /* Used to keep a sequence of def stmts of a pattern stmt that are loop
659 invariant if they exists.
660 The sequence is emitted in the loop preheader should the loop be vectorized
661 and are reset when undoing patterns. */
663
664private:
666 void set_vinfo_for_stmt (gimple *, stmt_vec_info, bool = true);
667 void free_stmt_vec_infos ();
669};
670
671class _loop_vec_info;
672class _bb_vec_info;
673
674template<>
675template<>
676inline bool
681
682template<>
683template<>
684inline bool
689
690/* In general, we can divide the vector statements in a vectorized loop
691 into related groups ("rgroups") and say that for each rgroup there is
692 some nS such that the rgroup operates on nS values from one scalar
693 iteration followed by nS values from the next. That is, if VF is the
694 vectorization factor of the loop, the rgroup operates on a sequence:
695
696 (1,1) (1,2) ... (1,nS) (2,1) ... (2,nS) ... (VF,1) ... (VF,nS)
697
698 where (i,j) represents a scalar value with index j in a scalar
699 iteration with index i.
700
701 [ We use the term "rgroup" to emphasise that this grouping isn't
702 necessarily the same as the grouping of statements used elsewhere.
703 For example, if we implement a group of scalar loads using gather
704 loads, we'll use a separate gather load for each scalar load, and
705 thus each gather load will belong to its own rgroup. ]
706
707 In general this sequence will occupy nV vectors concatenated
708 together. If these vectors have nL lanes each, the total number
709 of scalar values N is given by:
710
711 N = nS * VF = nV * nL
712
713 None of nS, VF, nV and nL are required to be a power of 2. nS and nV
714 are compile-time constants but VF and nL can be variable (if the target
715 supports variable-length vectors).
716
717 In classical vectorization, each iteration of the vector loop would
718 handle exactly VF iterations of the original scalar loop. However,
719 in vector loops that are able to operate on partial vectors, a
720 particular iteration of the vector loop might handle fewer than VF
721 iterations of the scalar loop. The vector lanes that correspond to
722 iterations of the scalar loop are said to be "active" and the other
723 lanes are said to be "inactive".
724
725 In such vector loops, many rgroups need to be controlled to ensure
726 that they have no effect for the inactive lanes. Conceptually, each
727 such rgroup needs a sequence of booleans in the same order as above,
728 but with each (i,j) replaced by a boolean that indicates whether
729 iteration i is active. This sequence occupies nV vector controls
730 that again have nL lanes each. Thus the control sequence as a whole
731 consists of VF independent booleans that are each repeated nS times.
732
733 Taking mask-based approach as a partially-populated vectors example.
734 We make the simplifying assumption that if a sequence of nV masks is
735 suitable for one (nS,nL) pair, we can reuse it for (nS/2,nL/2) by
736 VIEW_CONVERTing it. This holds for all current targets that support
737 fully-masked loops. For example, suppose the scalar loop is:
738
739 float *f;
740 double *d;
741 for (int i = 0; i < n; ++i)
742 {
743 f[i * 2 + 0] += 1.0f;
744 f[i * 2 + 1] += 2.0f;
745 d[i] += 3.0;
746 }
747
748 and suppose that vectors have 256 bits. The vectorized f accesses
749 will belong to one rgroup and the vectorized d access to another:
750
751 f rgroup: nS = 2, nV = 1, nL = 8
752 d rgroup: nS = 1, nV = 1, nL = 4
753 VF = 4
754
755 [ In this simple example the rgroups do correspond to the normal
756 SLP grouping scheme. ]
757
758 If only the first three lanes are active, the masks we need are:
759
760 f rgroup: 1 1 | 1 1 | 1 1 | 0 0
761 d rgroup: 1 | 1 | 1 | 0
762
763 Here we can use a mask calculated for f's rgroup for d's, but not
764 vice versa.
765
766 Thus for each value of nV, it is enough to provide nV masks, with the
767 mask being calculated based on the highest nL (or, equivalently, based
768 on the highest nS) required by any rgroup with that nV. We therefore
769 represent the entire collection of masks as a two-level table, with the
770 first level being indexed by nV - 1 (since nV == 0 doesn't exist) and
771 the second being indexed by the mask index 0 <= i < nV. */
772
773/* The controls (like masks or lengths) needed by rgroups with nV vectors,
774 according to the description above. */
776 /* The largest nS for all rgroups that use these controls.
777 For vect_partial_vectors_avx512 this is the constant nscalars_per_iter
778 for all members of the group. */
780
781 /* For the largest nS recorded above, the loop controls divide each scalar
782 into FACTOR equal-sized pieces. This is useful if we need to split
783 element-based accesses into byte-based accesses.
784 For vect_partial_vectors_avx512 this records nV instead. */
785 unsigned int factor;
786
787 /* This is a vector type with MAX_NSCALARS_PER_ITER * VF / nV elements.
788 For mask-based controls, it is the type of the masks in CONTROLS.
789 For length-based controls, it can be any vector type that has the
790 specified number of elements; the type of the elements doesn't matter. */
792
793 /* When there is no uniformly used LOOP_VINFO_RGROUP_COMPARE_TYPE this
794 is the rgroup specific type used. */
796
797 /* A vector of nV controls, in iteration order. */
799
800 /* In case of len_load and len_store with a bias there is only one
801 rgroup. This holds the adjusted loop length for the this rgroup. */
803};
804
806{
807 bool is_empty () const { return mask_set.is_empty (); }
808
809 /* Set to record vectype, nvector pairs. */
812
813 /* rgroup_controls used for the partial vector scheme. */
815};
816
818
820
821/* Abstraction around info on reductions which is still in stmt_vec_info
822 but will be duplicated or moved elsewhere. */
824{
825public:
826 /* The def type of the main reduction PHI, vect_reduction_def or
827 vect_double_reduction_def. */
829
830 /* The reduction type as detected by
831 vect_is_simple_reduction and vectorizable_reduction. */
833
834 /* The original scalar reduction code, to be used in the epilogue. */
836
837 /* A vector internal function we should use in the epilogue. */
839
840 /* For loop reduction with multiple vectorized results (ncopies > 1), a
841 lane-reducing operation participating in it may not use all of those
842 results, this field specifies result index starting from which any
843 following land-reducing operation would be assigned to. */
844 unsigned int reduc_result_pos;
845
846 /* Whether we force a single cycle PHI during reduction vectorization. */
848
849 /* The vector type for performing the actual reduction operation. */
851
852 /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used. */
854
855 /* If not NULL the value to be added to compute final reduction value. */
857
858 /* If non-null, the reduction is being performed by an epilogue loop
859 and we have decided to reuse this accumulator from the main loop. */
861
862 /* If the vector code is performing N scalar reductions in parallel,
863 this variable gives the initial scalar values of those N reductions. */
865
866 /* If the vector code is performing N scalar reductions in parallel, this
867 variable gives the vectorized code's final (scalar) result for each of
868 those N reductions. In other words, REDUC_SCALAR_RESULTS[I] replaces
869 the original scalar code's loop-closed SSA PHI for reduction number I. */
871};
872
874
875#define VECT_REDUC_INFO_DEF_TYPE(I) ((I)->def_type)
876#define VECT_REDUC_INFO_TYPE(I) ((I)->reduc_type)
877#define VECT_REDUC_INFO_CODE(I) ((I)->reduc_code)
878#define VECT_REDUC_INFO_FN(I) ((I)->reduc_fn)
879#define VECT_REDUC_INFO_SCALAR_RESULTS(I) ((I)->reduc_scalar_results)
880#define VECT_REDUC_INFO_INITIAL_VALUES(I) ((I)->reduc_initial_values)
881#define VECT_REDUC_INFO_REUSED_ACCUMULATOR(I) ((I)->reused_accumulator)
882#define VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL(I) ((I)->induc_cond_initial_val)
883#define VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT(I) ((I)->reduc_epilogue_adjustment)
884#define VECT_REDUC_INFO_VECTYPE(I) ((I)->reduc_vectype)
885#define VECT_REDUC_INFO_FORCE_SINGLE_CYCLE(I) ((I)->force_single_cycle)
886#define VECT_REDUC_INFO_RESULT_POS(I) ((I)->reduc_result_pos)
887
888/* Information about a reduction accumulator from the main loop that could
889 conceivably be reused as the input to a reduction in an epilogue loop. */
891 /* The final value of the accumulator, which forms the input to the
892 reduction operation. */
894
895 /* The stmt_vec_info that describes the reduction (i.e. the one for
896 which is_reduc_info is true). */
898};
899
900/*-----------------------------------------------------------------*/
901/* Info on vectorized loops. */
902/*-----------------------------------------------------------------*/
903typedef class _loop_vec_info : public vec_info {
904public:
907
908 /* The loop to which this info struct refers to. */
909 class loop *loop;
910
911 /* Number of latch executions. */
913 /* Number of iterations. */
915 /* Number of iterations of the original loop. */
917 /* Condition under which this loop is analyzed and versioned. */
919
920 /* The cost of the vector code. */
922
923 /* The cost of the scalar code. */
925
926 /* Threshold of number of iterations below which vectorization will not be
927 performed. It is calculated from MIN_PROFITABLE_ITERS and
928 param_min_vect_loop_bound. */
929 unsigned int th;
930
931 /* When applying loop versioning, the vector form should only be used
932 if the number of scalar iterations is >= this value, on top of all
933 the other requirements. Ignored when loop versioning is not being
934 used. */
936
937 /* Unrolling factor */
939
940 /* If this loop is an epilogue loop whose main loop can be skipped,
941 MAIN_LOOP_EDGE is the edge from the main loop to this loop's
942 preheader. SKIP_MAIN_LOOP_EDGE is then the edge that skips the
943 main loop and goes straight to this loop's preheader.
944
945 Both fields are null otherwise. */
948
949 /* If this loop is an epilogue loop that might be skipped after executing
950 the main loop, this edge is the one that skips the epilogue. */
952
953 /* Reduction descriptors of this loop. Referenced to from SLP nodes
954 by index. */
956
957 /* The vectorized form of a standard reduction replaces the original
958 scalar code's final result (a loop-closed SSA PHI) with the result
959 of a vector-to-scalar reduction operation. After vectorization,
960 this variable maps these vector-to-scalar results to information
961 about the reductions that generated them. */
963
964 /* The number of times that the target suggested we unroll the vector loop
965 in order to promote more ILP. This value will be used to re-analyze the
966 loop for vectorization and if successful the value will be folded into
967 vectorization_factor (and therefore exactly divides
968 vectorization_factor). */
970
971 /* Maximum runtime vectorization factor, or MAX_VECTORIZATION_FACTOR
972 if there is no particular limit. */
973 unsigned HOST_WIDE_INT max_vectorization_factor;
974
975 /* The masks that a fully-masked loop should use to avoid operating
976 on inactive scalars. */
978
979 /* The lengths that a loop with length should use to avoid operating
980 on inactive scalars. */
982
983 /* Set of scalar conditions that have loop mask applied. */
985
986 /* Set of vector conditions that have loop mask applied. */
988
989 /* If we are using a loop mask to align memory addresses, this variable
990 contains the number of vector elements that we should skip in the
991 first iteration of the vector loop (i.e. the number of leading
992 elements that should be false in the first mask). */
994
995 /* If we are using a loop mask to align memory addresses and we're in an
996 early break loop then this variable contains the number of elements that
997 were skipped during the initial iteration of the loop. */
999
1000 /* The type that the loop control IV should be converted to before
1001 testing which of the VF scalars are active and inactive.
1002 Only meaningful if LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
1004
1005 /* For #pragma omp simd if (x) loops the x expression. If constant 0,
1006 the loop should not be vectorized, if constant non-zero, simd_if_cond
1007 shouldn't be set and loop vectorized normally, if SSA_NAME, the loop
1008 should be versioned on that condition, using scalar loop if the condition
1009 is false and vectorized loop otherwise. */
1011
1012 /* The type that the vector loop control IV should have when
1013 LOOP_VINFO_USING_PARTIAL_VECTORS_P is true. */
1015
1016 /* The style used for implementing partial vectors when
1017 LOOP_VINFO_USING_PARTIAL_VECTORS_P is true. */
1019
1020 /* Unknown DRs according to which loop was peeled. */
1022
1023 /* peeling_for_alignment indicates whether peeling for alignment will take
1024 place, and what the peeling factor should be:
1025 peeling_for_alignment = X means:
1026 If X=0: Peeling for alignment will not be applied.
1027 If X>0: Peel first X iterations.
1028 If X=-1: Generate a runtime test to calculate the number of iterations
1029 to be peeled, using the dataref recorded in the field
1030 unaligned_dr. */
1032
1033 /* The mask used to check the alignment of pointers or arrays. */
1035
1036 /* The maximum speculative read amount in VLA modes for runtime check. */
1038
1039 /* Indicates whether the loop has any non-linear IV. */
1041
1042 /* Data Dependence Relations defining address ranges that are candidates
1043 for a run-time aliasing check. */
1045
1046 /* Data Dependence Relations defining address ranges together with segment
1047 lengths from which the run-time aliasing check is built. */
1049
1050 /* Check that the addresses of each pair of objects is unequal. */
1052
1053 /* List of values that are required to be nonzero. This is used to check
1054 whether things like "x[i * n] += 1;" are safe and eventually gets added
1055 to the checks for lower bounds below. */
1057
1058 /* List of values that need to be checked for a minimum value. */
1060
1061 /* Statements in the loop that have data references that are candidates for a
1062 runtime (loop versioning) misalignment check. */
1064
1065 /* Reduction cycles detected in the loop. Used in loop-aware SLP. */
1067
1068 /* All reduction chains in the loop, represented by the first
1069 stmt in the chain. */
1071
1072 /* Defs that could not be analyzed such as OMP SIMD calls without
1073 a LHS. */
1075
1076 /* Cost vector for a single scalar iteration. */
1078
1079 /* Map of IV base/step expressions to inserted name in the preheader. */
1081
1082 /* Map of OpenMP "omp simd array" scan variables to corresponding
1083 rhs of the store of the initializer. */
1085
1086 /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
1087 applied to the loop, i.e., no unrolling is needed, this is 1. */
1089
1090 /* The factor used to over weight those statements in an inner loop
1091 relative to the loop being vectorized. */
1093
1094 /* Is the loop vectorizable? */
1096
1097 /* Records whether we still have the option of vectorizing this loop
1098 using partially-populated vectors; in other words, whether it is
1099 still possible for one iteration of the vector loop to handle
1100 fewer than VF scalars. */
1102
1103 /* Records whether we must use niter masking for correctness reasons. */
1105
1106 /* True if we've decided to use partially-populated vectors, so that
1107 the vector loop can handle fewer than VF scalars. */
1109
1110 /* True if we've decided to use a decrementing loop control IV that counts
1111 scalars. This can be done for any loop that:
1112
1113 (a) uses length "controls"; and
1114 (b) can iterate more than once. */
1116
1117 /* True if we've decided to use output of select_vl to adjust IV of
1118 both loop control and data reference pointer. This is only true
1119 for single-rgroup control. */
1121
1122 /* True if we've decided to use partially-populated vectors for the
1123 epilogue of loop. */
1125
1126 /* True if we've decided to use peeling with versioning together, which allows
1127 unaligned unsupported data refs to be uniformly aligned after a certain
1128 amount of peeling (mutual alignment). Otherwise, we use versioning alone
1129 so these data refs must be already aligned to a power-of-two boundary
1130 without peeling. */
1132
1133 /* The bias for len_load and len_store. For now, only 0 and -1 are
1134 supported. -1 must be used when a backend does not support
1135 len_load/len_store with a length of zero. */
1137
1138 /* When we have grouped data accesses with gaps, we may introduce invalid
1139 memory accesses. We peel the last iteration of the loop to prevent
1140 this. */
1142
1143 /* When the number of iterations is not a multiple of the vector size
1144 we need to peel off iterations at the end to form an epilogue loop. */
1146
1147 /* When the loop has early breaks that we can vectorize we need to peel
1148 the loop for the break finding loop. */
1150
1151 /* List of loop additional IV conditionals found in the loop. */
1153
1154 /* Main loop IV cond. */
1156
1157 /* True if we have an unroll factor requested by the user through pragma GCC
1158 unroll. */
1160
1161 /* True if there are no loop carried data dependencies in the loop.
1162 If loop->safelen <= 1, then this is always true, either the loop
1163 didn't have any loop carried data dependencies, or the loop is being
1164 vectorized guarded with some runtime alias checks, or couldn't
1165 be vectorized at all, but then this field shouldn't be used.
1166 For loop->safelen >= 2, the user has asserted that there are no
1167 backward dependencies, but there still could be loop carried forward
1168 dependencies in such loops. This flag will be false if normal
1169 vectorizer data dependency analysis would fail or require versioning
1170 for alias, but because of loop->safelen >= 2 it has been vectorized
1171 even without versioning for alias. E.g. in:
1172 #pragma omp simd
1173 for (int i = 0; i < m; i++)
1174 a[i] = a[i + k] * c;
1175 (or #pragma simd or #pragma ivdep) we can vectorize this and it will
1176 DTRT even for k > 0 && k < m, but without safelen we would not
1177 vectorize this, so this field would be false. */
1179
1180 /* Mark loops having masked stores. */
1182
1183 /* Queued scaling factor for the scalar loop. */
1185
1186 /* If if-conversion versioned this loop before conversion, this is the
1187 loop version without if-conversion. */
1189
1190 /* For loops being epilogues of already vectorized loops
1191 this points to the main vectorized loop. Otherwise NULL. */
1193
1194 /* For loops being epilogues of already vectorized loops
1195 this points to the preceeding vectorized (possibly epilogue) loop.
1196 Otherwise NULL. */
1198
1199 /* Used to store loop_vec_infos of the epilogue of this loop during
1200 analysis. */
1202
1203 /* If this is an epilogue loop the DR advancement applied. */
1205
1206 /* The controlling loop IV for the current loop when vectorizing. This IV
1207 controls the natural exits of the loop. */
1209
1210 /* The controlling loop IV for the epilogue loop when vectorizing. This IV
1211 controls the natural exits of the loop. */
1213
1214 /* The controlling loop IV for the scalar loop being vectorized. This IV
1215 controls the natural exits of the loop. */
1217
1218 /* Used to store the list of stores needing to be moved if doing early
1219 break vectorization as they would violate the scalar loop semantics if
1220 vectorized in their current location. These are stored in order that they
1221 need to be moved. */
1223
1224 /* The final basic block where to move statements to. In the case of
1225 multiple exits this could be pretty far away. */
1227
1228 /* Statements whose VUSES need updating if early break vectorization is to
1229 happen. */
1231
1232 /* Record statements that are needed to be live for early break vectorization
1233 but may not have an LC PHI node materialized yet in the exits. */
1236
1237/* Access Functions. */
1238#define LOOP_VINFO_LOOP(L) (L)->loop
1239#define LOOP_VINFO_IV_EXIT(L) (L)->vec_loop_iv_exit
1240#define LOOP_VINFO_EPILOGUE_IV_EXIT(L) (L)->vec_epilogue_loop_iv_exit
1241#define LOOP_VINFO_SCALAR_IV_EXIT(L) (L)->scalar_loop_iv_exit
1242#define LOOP_VINFO_BBS(L) (L)->bbs
1243#define LOOP_VINFO_NBBS(L) (L)->nbbs
1244#define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1
1245#define LOOP_VINFO_NITERS(L) (L)->num_iters
1246/* Since LOOP_VINFO_NITERS and LOOP_VINFO_NITERSM1 can change after
1247 prologue peeling retain total unchanged scalar loop iterations for
1248 cost model. */
1249#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
1250#define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions
1251#define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th
1252#define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold
1253#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
1254#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
1255#define LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P(L) (L)->must_use_partial_vectors_p
1256#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
1257#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
1258#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
1259#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \
1260 (L)->epil_using_partial_vectors_p
1261#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment
1262#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
1263#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
1264#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
1265#define LOOP_VINFO_MASKS(L) (L)->masks
1266#define LOOP_VINFO_LENS(L) (L)->lens
1267#define LOOP_VINFO_MASK_SKIP_NITERS(L) (L)->mask_skip_niters
1268#define LOOP_VINFO_MASK_NITERS_PFA_OFFSET(L) (L)->mask_skip_niters_pfa_offset
1269#define LOOP_VINFO_RGROUP_COMPARE_TYPE(L) (L)->rgroup_compare_type
1270#define LOOP_VINFO_RGROUP_IV_TYPE(L) (L)->rgroup_iv_type
1271#define LOOP_VINFO_PARTIAL_VECTORS_STYLE(L) (L)->partial_vector_style
1272#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
1273#define LOOP_VINFO_MAX_SPEC_READ_AMOUNT(L) (L)->max_spec_read_amount
1274#define LOOP_VINFO_LOOP_NEST(L) (L)->shared->loop_nest
1275#define LOOP_VINFO_DATAREFS(L) (L)->shared->datarefs
1276#define LOOP_VINFO_DDRS(L) (L)->shared->ddrs
1277#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
1278#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
1279#define LOOP_VINFO_NON_LINEAR_IV(L) (L)->nonlinear_iv
1280#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
1281#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
1282#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
1283#define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs
1284#define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs
1285#define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero
1286#define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds
1287#define LOOP_VINFO_USER_UNROLL(L) (L)->user_unroll
1288#define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
1289#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
1290#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
1291#define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
1292#define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
1293#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
1294#define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter
1295#define LOOP_VINFO_EARLY_BREAKS(L) (L)->early_breaks
1296#define LOOP_VINFO_EARLY_BRK_STORES(L) (L)->early_break_stores
1297#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L) \
1298 (single_pred ((L)->loop->latch) != (L)->vec_loop_iv_exit->src)
1299#define LOOP_VINFO_EARLY_BREAKS_LIVE_IVS(L) \
1300 (L)->early_break_live_ivs
1301#define LOOP_VINFO_EARLY_BRK_DEST_BB(L) (L)->early_break_dest_bb
1302#define LOOP_VINFO_EARLY_BRK_VUSES(L) (L)->early_break_vuses
1303#define LOOP_VINFO_LOOP_CONDS(L) (L)->conds
1304#define LOOP_VINFO_LOOP_IV_COND(L) (L)->loop_iv_cond
1305#define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
1306#define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop
1307#define LOOP_VINFO_SCALAR_LOOP_SCALING(L) (L)->scalar_loop_scaling
1308#define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store
1309#define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
1310#define LOOP_VINFO_MAIN_LOOP_INFO(L) (L)->main_loop_info
1311#define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info
1312#define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
1313#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
1314#define LOOP_VINFO_INV_PATTERN_DEF_SEQ(L) (L)->inv_pattern_def_seq
1315#define LOOP_VINFO_DRS_ADVANCED_BY(L) (L)->drs_advanced_by
1316#define LOOP_VINFO_ALTERNATE_DEFS(L) (L)->alternate_defs
1317
1318#define LOOP_VINFO_FULLY_MASKED_P(L) \
1319 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1320 && !LOOP_VINFO_MASKS (L).is_empty ())
1321
1322#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \
1323 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1324 && !LOOP_VINFO_LENS (L).is_empty ())
1325
1326#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
1327 ((L)->may_misalign_stmts.length () > 0)
1328#define LOOP_REQUIRES_VERSIONING_FOR_SPEC_READ(L) \
1329 (maybe_gt ((L)->max_spec_read_amount, 0U))
1330#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
1331 ((L)->comp_alias_ddrs.length () > 0 \
1332 || (L)->check_unequal_addrs.length () > 0 \
1333 || (L)->lower_bounds.length () > 0)
1334#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \
1335 (LOOP_VINFO_NITERS_ASSUMPTIONS (L))
1336#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \
1337 (LOOP_VINFO_SIMD_IF_COND (L))
1338#define LOOP_REQUIRES_VERSIONING(L) \
1339 (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \
1340 || LOOP_REQUIRES_VERSIONING_FOR_SPEC_READ (L) \
1341 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \
1342 || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
1343 || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
1344
1345#define LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING(L) \
1346 ((L)->may_misalign_stmts.length () > 0 \
1347 && !LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (L))
1348
1349#define LOOP_VINFO_NITERS_KNOWN_P(L) \
1350 (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
1351
1352#define LOOP_VINFO_EPILOGUE_P(L) \
1353 (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL)
1354
1355#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \
1356 (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L)))
1357
1358/* Wrapper for loop_vec_info, for tracking success/failure, where a non-NULL
1359 value signifies success, and a NULL value signifies failure, supporting
1360 propagating an opt_problem * describing the failure back up the call
1361 stack. */
1362typedef opt_pointer_wrapper <loop_vec_info> opt_loop_vec_info;
1363
1364inline loop_vec_info
1366{
1367 return (loop_vec_info) loop->aux;
1368}
1369
1380
1389
1390#define BB_VINFO_BBS(B) (B)->bbs
1391#define BB_VINFO_NBBS(B) (B)->nbbs
1392#define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
1393#define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
1394#define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
1395#define BB_VINFO_DDRS(B) (B)->shared->ddrs
1396
1397/* Indicates whether/how a variable is used in the scope of loop/basic
1398 block. */
1401
1402 /* The def is only used outside the loop. */
1404 /* The def is in the inner loop, and the use is in the outer loop, and the
1405 use is a reduction stmt. */
1407 /* The def is in the inner loop, and the use is in the outer loop (and is
1408 not part of reduction). */
1410
1411 /* defs that feed computations that end up (only) in a reduction. These
1412 defs may be used by non-reduction stmts, but eventually, any
1413 computations/values that are affected by these defs are used to compute
1414 a reduction (i.e. don't get stored to memory, for example). We use this
1415 to identify computations that we can change the order in which they are
1416 computed. */
1418
1420};
1421
1422/* The type of vectorization. pure_slp means the stmt is covered by the
1423 SLP graph, not_vect means it is not. This is mostly used by BB
1424 vectorization. */
1429
1430/* Says whether a statement is a load, a store of a vectorized statement
1431 result, or a store of an invariant value. */
1437
1439public:
1440 /* The data reference itself. */
1442 /* The statement that contains the data reference. */
1444 /* The analysis group this DR belongs to when doing BB vectorization.
1445 DRs of the same group belong to the same conditional execution context. */
1446 unsigned group;
1447 /* The misalignment in bytes of the reference, or -1 if not known. */
1449 /* The byte alignment that we'd ideally like the reference to have,
1450 and the value that misalignment is measured against. */
1452 /* If true the alignment of base_decl needs to be increased. */
1454
1455 /* Set by early break vectorization when this DR needs peeling for alignment
1456 for correctness. */
1458
1459 /* Set by early break vectorization when this DR's scalar accesses are known
1460 to be inbounds of a known bounds loop. */
1462
1464
1465 /* Stores current vectorized loop's offset. To be added to the DR's
1466 offset to calculate current offset of data reference. */
1468};
1469
1470typedef struct data_reference *dr_p;
1471
1473public:
1474
1475 /* Indicates whether this stmts is part of a computation whose result is
1476 used outside the loop. */
1477 bool live;
1478
1479 /* Stmt is part of some pattern (computation idiom) */
1481
1482 /* True if the statement was created during pattern recognition as
1483 part of the replacement for RELATED_STMT. This implies that the
1484 statement isn't part of any basic block, although for convenience
1485 its gimple_bb is the same as for RELATED_STMT. */
1487
1488 /* Is this statement vectorizable or should it be skipped in (partial)
1489 vectorization. */
1491
1492 /* The stmt to which this info struct refers to. */
1494
1495 /* The vector type to be used for the LHS of this statement. */
1497
1498 /* The following is relevant only for stmts that contain a non-scalar
1499 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
1500 at most one such data-ref. */
1501
1503
1504 /* Information about the data-ref relative to this loop
1505 nest (the loop that is being considered for vectorization). */
1507
1508 /* For loop PHI nodes, the base and evolution part of it. This makes sure
1509 this information is still available in vect_update_ivs_after_vectorizer
1510 where we may not be able to re-analyze the PHI nodes evolution as
1511 peeling for the prologue loop can make it unanalyzable. The evolution
1512 part is still correct after peeling, but the base may have changed from
1513 the version here. */
1517
1518 /* Used for various bookkeeping purposes, generally holding a pointer to
1519 some other stmt S that is in some way "related" to this stmt.
1520 Current use of this field is:
1521 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
1522 true): S is the "pattern stmt" that represents (and replaces) the
1523 sequence of stmts that constitutes the pattern. Similarly, the
1524 related_stmt of the "pattern stmt" points back to this stmt (which is
1525 the last stmt in the original sequence of stmts that constitutes the
1526 pattern). */
1528
1529 /* Used to keep a sequence of def stmts of a pattern stmt if such exists.
1530 The sequence is attached to the original statement rather than the
1531 pattern statement. */
1533
1534 /* Classify the def of this stmt. */
1536
1537 /* Whether the stmt is SLPed, loop-based vectorized, or both. */
1539
1540 /* Interleaving and reduction chains info. */
1541 /* First element in the group. */
1543 /* Pointer to the next element in the group. */
1545 /* The size of the group. */
1546 unsigned int size;
1547 /* For loads only, the gap from the previous load. For consecutive loads, GAP
1548 is 1. */
1549 unsigned int gap;
1550
1551 /* The minimum negative dependence distance this stmt participates in
1552 or zero if none. */
1553 unsigned int min_neg_dist;
1554
1555 /* Not all stmts in the loop need to be vectorized. e.g, the increment
1556 of the loop induction variable and computation of array indexes. relevant
1557 indicates whether the stmt needs to be vectorized. */
1559
1560 /* For loads if this is a gather, for stores if this is a scatter. */
1562
1563 /* True if this is an access with loop-invariant stride. */
1565
1566 /* For both loads and stores. */
1568
1569 /* On a reduction PHI the reduction type as detected by
1570 vect_is_simple_reduction. */
1572
1573 /* On a reduction PHI, the original reduction code as detected by
1574 vect_is_simple_reduction. */
1576
1577 /* On a stmt participating in a reduction the index of the operand
1578 on the reduction SSA cycle. */
1580
1581 /* On a reduction PHI the def returned by vect_is_simple_reduction.
1582 On the def returned by vect_is_simple_reduction the corresponding PHI. */
1584
1585 /* If nonzero, the lhs of the statement could be truncated to this
1586 many bits without affecting any users of the result. */
1588
1589 /* If nonzero, all non-boolean input operands have the same precision,
1590 and they could each be truncated to this many bits without changing
1591 the result. */
1593
1594 /* If OPERATION_BITS is nonzero, the statement could be performed on
1595 an integer with the sign and number of bits given by OPERATION_SIGN
1596 and OPERATION_BITS without changing the result. */
1599
1600 /* If the statement produces a boolean result, this value describes
1601 how we should choose the associated vector type. The possible
1602 values are:
1603
1604 - an integer precision N if we should use the vector mask type
1605 associated with N-bit integers. This is only used if all relevant
1606 input booleans also want the vector mask type for N-bit integers,
1607 or if we can convert them into that form by pattern-matching.
1608
1609 - ~0U if we considered choosing a vector mask type but decided
1610 to treat the boolean as a normal integer type instead.
1611
1612 - 0 otherwise. This means either that the operation isn't one that
1613 could have a vector mask type (and so should have a normal vector
1614 type instead) or that we simply haven't made a choice either way. */
1615 unsigned int mask_precision;
1616
1617 /* True if this is only suitable for SLP vectorization. */
1619
1620 /* True if this is a pattern that can only be handled by SLP
1621 vectorization. */
1623};
1624
1625/* Information about a gather/scatter call. */
1627 /* The internal function to use for the gather/scatter operation,
1628 or IFN_LAST if a built-in function should be used instead. */
1630
1631 /* The FUNCTION_DECL for the built-in gather/scatter function,
1632 or null if an internal function should be used instead. */
1634
1635 /* The loop-invariant base value. */
1637
1638 /* The TBBA alias pointer the value of which determines the alignment
1639 of the scalar accesses. */
1641
1642 /* The original scalar offset, which is a non-loop-invariant SSA_NAME. */
1644
1645 /* Each offset element should be multiplied by this amount before
1646 being added to the base. */
1648
1649 /* The type of the vectorized offset. */
1651
1652 /* The type of the scalar elements after loading or before storing. */
1654
1655 /* The type of the scalar elements being loaded or stored. */
1657};
1658
1659/* Access Functions. */
1660#define STMT_VINFO_STMT(S) (S)->stmt
1661#define STMT_VINFO_RELEVANT(S) (S)->relevant
1662#define STMT_VINFO_LIVE_P(S) (S)->live
1663#define STMT_VINFO_VECTYPE(S) (S)->vectype
1664#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
1665#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
1666#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
1667#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
1668#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
1669#define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
1670
1671#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop
1672#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address
1673#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init
1674#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset
1675#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step
1676#define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment
1677#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \
1678 (S)->dr_wrt_vec_loop.base_misalignment
1679#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \
1680 (S)->dr_wrt_vec_loop.offset_alignment
1681#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
1682 (S)->dr_wrt_vec_loop.step_alignment
1683
1684#define STMT_VINFO_DR_INFO(S) \
1685 (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
1686
1687#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
1688#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
1689#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
1690#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
1691#define STMT_VINFO_GROUPED_ACCESS(S) \
1692 ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
1693#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
1694#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
1695#define STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE(S) (S)->loop_phi_evolution_type
1696#define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
1697#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
1698#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
1699#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
1700#define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
1701#define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p
1702#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
1703
1704#define DR_GROUP_FIRST_ELEMENT(S) \
1705 (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
1706#define DR_GROUP_NEXT_ELEMENT(S) \
1707 (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
1708#define DR_GROUP_SIZE(S) \
1709 (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
1710#define DR_GROUP_GAP(S) \
1711 (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
1712
1713#define REDUC_GROUP_FIRST_ELEMENT(S) \
1714 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
1715#define REDUC_GROUP_NEXT_ELEMENT(S) \
1716 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
1717#define REDUC_GROUP_SIZE(S) \
1718 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
1719
1720#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
1721
1722#define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
1723#define STMT_SLP_TYPE(S) (S)->slp_type
1724
1725
1726/* Contains the scalar or vector costs for a vec_info. */
1728{
1729public:
1730 vector_costs (vec_info *, bool);
1731 virtual ~vector_costs () {}
1732
1733 /* Update the costs in response to adding COUNT copies of a statement.
1734
1735 - WHERE specifies whether the cost occurs in the loop prologue,
1736 the loop body, or the loop epilogue.
1737 - KIND is the kind of statement, which is always meaningful.
1738 - STMT_INFO or NODE, if nonnull, describe the statement that will be
1739 vectorized.
1740 - VECTYPE, if nonnull, is the vector type that the vectorized
1741 statement will operate on. Note that this should be used in
1742 preference to STMT_VINFO_VECTYPE (STMT_INFO) since the latter
1743 is not correct for SLP.
1744 - for unaligned_load and unaligned_store statements, MISALIGN is
1745 the byte misalignment of the load or store relative to the target's
1746 preferred alignment for VECTYPE, or DR_MISALIGNMENT_UNKNOWN
1747 if the misalignment is not known.
1748
1749 Return the calculated cost as well as recording it. The return
1750 value is used for dumping purposes. */
1751 virtual unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
1752 stmt_vec_info stmt_info,
1753 slp_tree node,
1754 tree vectype, int misalign,
1756
1757 /* Finish calculating the cost of the code. The results can be
1758 read back using the functions below.
1759
1760 If the costs describe vector code, SCALAR_COSTS gives the costs
1761 of the corresponding scalar code, otherwise it is null. */
1762 virtual void finish_cost (const vector_costs *scalar_costs);
1763
1764 /* The costs in THIS and OTHER both describe ways of vectorizing
1765 a main loop. Return true if the costs described by THIS are
1766 cheaper than the costs described by OTHER. Return false if any
1767 of the following are true:
1768
1769 - THIS and OTHER are of equal cost
1770 - OTHER is better than THIS
1771 - we can't be sure about the relative costs of THIS and OTHER. */
1772 virtual bool better_main_loop_than_p (const vector_costs *other) const;
1773
1774 /* Likewise, but the costs in THIS and OTHER both describe ways of
1775 vectorizing an epilogue loop of MAIN_LOOP. */
1776 virtual bool better_epilogue_loop_than_p (const vector_costs *other,
1777 loop_vec_info main_loop) const;
1778
1779 unsigned int prologue_cost () const;
1780 unsigned int body_cost () const;
1781 unsigned int epilogue_cost () const;
1782 unsigned int outside_cost () const;
1783 unsigned int total_cost () const;
1784 unsigned int suggested_unroll_factor () const;
1785 machine_mode suggested_epilogue_mode (int &masked) const;
1787
1788protected:
1790 unsigned int);
1792 unsigned int);
1793 int compare_inside_loop_cost (const vector_costs *) const;
1794 int compare_outside_loop_cost (const vector_costs *) const;
1795
1796 /* The region of code that we're considering vectorizing. */
1798
1799 /* True if we're costing the scalar code, false if we're costing
1800 the vector code. */
1802
1803 /* The costs of the three regions, indexed by vect_cost_model_location. */
1804 unsigned int m_costs[3];
1805
1806 /* The suggested unrolling factor determined at finish_cost. */
1808
1809 /* The suggested mode to be used for a vectorized epilogue or VOIDmode,
1810 determined at finish_cost. m_masked_epilogue specifies whether the
1811 epilogue should use masked vectorization, regardless of the
1812 --param vect-partial-vector-usage default. If -1 then the
1813 --param setting takes precedence. If the user explicitly specified
1814 --param vect-partial-vector-usage then that takes precedence. */
1817
1818 /* True if finish_cost has been called. */
1820};
1821
1822/* Create costs for VINFO. COSTING_FOR_SCALAR is true if the costs
1823 are for scalar code, false if they are for vector code. */
1824
1825inline
1836
1837/* Return the cost of the prologue code (in abstract units). */
1838
1839inline unsigned int
1845
1846/* Return the cost of the body code (in abstract units). */
1847
1848inline unsigned int
1854
1855/* Return the cost of the epilogue code (in abstract units). */
1856
1857inline unsigned int
1863
1864/* Return the cost of the prologue and epilogue code (in abstract units). */
1865
1866inline unsigned int
1868{
1869 return prologue_cost () + epilogue_cost ();
1870}
1871
1872/* Return the cost of the prologue, body and epilogue code
1873 (in abstract units). */
1874
1875inline unsigned int
1877{
1878 return body_cost () + outside_cost ();
1879}
1880
1881/* Return the suggested unroll factor. */
1882
1883inline unsigned int
1889
1890/* Return the suggested epilogue mode. */
1891
1892inline machine_mode
1894{
1896 masked_p = m_masked_epilogue;
1898}
1899
1900#define VECT_MAX_COST 1000
1901
1902/* The maximum number of intermediate steps required in multi-step type
1903 conversion. */
1904#define MAX_INTERM_CVT_STEPS 3
1905
1906#define MAX_VECTORIZATION_FACTOR INT_MAX
1907
1908/* Nonzero if TYPE represents a (scalar) boolean type or type
1909 in the middle-end compatible with it (unsigned precision 1 integral
1910 types). Used to determine which types should be vectorized as
1911 VECTOR_BOOLEAN_TYPE_P. */
1912
1913#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \
1914 (TREE_CODE (TYPE) == BOOLEAN_TYPE \
1915 || ((TREE_CODE (TYPE) == INTEGER_TYPE \
1916 || TREE_CODE (TYPE) == ENUMERAL_TYPE) \
1917 && TYPE_PRECISION (TYPE) == 1 \
1918 && TYPE_UNSIGNED (TYPE)))
1919
1920inline bool
1922{
1923 return (loop->inner
1924 && (loop->inner == (gimple_bb (stmt_info->stmt))->loop_father));
1925}
1926
1927/* PHI is either a scalar reduction phi or a scalar induction phi.
1928 Return the initial value of the variable on entry to the containing
1929 loop. */
1930
1931inline tree
1933{
1934 basic_block bb = gimple_bb (phi);
1936 gcc_assert (pe->dest == bb);
1937 return PHI_ARG_DEF_FROM_EDGE (phi, pe);
1938}
1939
1940/* Return true if STMT_INFO should produce a vector mask type rather than
1941 a normal nonmask type. */
1942
1943inline bool
1945{
1946 return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
1947}
1948
1949/* Return TRUE if a statement represented by STMT_INFO is a part of a
1950 pattern. */
1951
1952inline bool
1954{
1955 return stmt_info->pattern_stmt_p;
1956}
1957
1958/* If STMT_INFO is a pattern statement, return the statement that it
1959 replaces, otherwise return STMT_INFO itself. */
1960
1961inline stmt_vec_info
1963{
1964 if (is_pattern_stmt_p (stmt_info))
1965 return STMT_VINFO_RELATED_STMT (stmt_info);
1966 return stmt_info;
1967}
1968
1969/* Return the later statement between STMT1_INFO and STMT2_INFO. */
1970
1971inline stmt_vec_info
1973{
1974 gimple *stmt1 = vect_orig_stmt (stmt1_info)->stmt;
1975 gimple *stmt2 = vect_orig_stmt (stmt2_info)->stmt;
1976 if (gimple_bb (stmt1) == gimple_bb (stmt2))
1977 {
1978 if (gimple_uid (stmt1) > gimple_uid (stmt2))
1979 return stmt1_info;
1980 else
1981 return stmt2_info;
1982 }
1983 /* ??? We should be really calling this function only with stmts
1984 in the same BB but we can recover if there's a domination
1985 relationship between them. */
1987 gimple_bb (stmt1), gimple_bb (stmt2)))
1988 return stmt1_info;
1990 gimple_bb (stmt2), gimple_bb (stmt1)))
1991 return stmt2_info;
1992 gcc_unreachable ();
1993}
1994
1995/* If STMT_INFO has been replaced by a pattern statement, return the
1996 replacement statement, otherwise return STMT_INFO itself. */
1997
1998inline stmt_vec_info
2000{
2001 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
2002 return STMT_VINFO_RELATED_STMT (stmt_info);
2003 return stmt_info;
2004}
2005
2006/* Return true if BB is a loop header. */
2007
2008inline bool
2010{
2011 if (bb == (bb->loop_father)->header)
2012 return true;
2013
2014 return false;
2015}
2016
2017/* Return pow2 (X). */
2018
2019inline int
2021{
2022 int i, res = 1;
2023
2024 for (i = 0; i < x; i++)
2025 res *= 2;
2026
2027 return res;
2028}
2029
2030/* Alias targetm.vectorize.builtin_vectorization_cost. */
2031
2032inline int
2034 tree vectype, int misalign)
2035{
2036 return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
2037 vectype, misalign);
2038}
2039
2040/* Get cost by calling cost target builtin. */
2041
2042inline
2044{
2045 return builtin_vectorization_cost (type_of_cost, NULL, 0);
2046}
2047
2048/* Alias targetm.vectorize.init_cost. */
2049
2050inline vector_costs *
2051init_cost (vec_info *vinfo, bool costing_for_scalar)
2052{
2053 return targetm.vectorize.create_costs (vinfo, costing_for_scalar);
2054}
2055
2056extern void dump_stmt_cost (FILE *, int, enum vect_cost_for_stmt,
2057 stmt_vec_info, slp_tree, tree, int, unsigned,
2059
2060/* Dump and add costs. */
2061
2062inline unsigned
2064 enum vect_cost_for_stmt kind,
2065 stmt_vec_info stmt_info, slp_tree node,
2066 tree vectype, int misalign,
2067 enum vect_cost_model_location where)
2068{
2069 /* Even though a vector type might be set on stmt do not pass that on when
2070 costing the scalar IL. A SLP node shouldn't have been recorded. */
2071 if (costs->costing_for_scalar ())
2072 {
2073 vectype = NULL_TREE;
2074 gcc_checking_assert (node == NULL);
2075 }
2076 unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, node, vectype,
2077 misalign, where);
2078 if (dump_file && (dump_flags & TDF_DETAILS))
2079 dump_stmt_cost (dump_file, count, kind, stmt_info, node, vectype, misalign,
2080 cost, where);
2081 return cost;
2082}
2083
2084inline unsigned
2086 enum vect_cost_model_location where)
2087{
2089 || kind == scalar_stmt);
2090 return add_stmt_cost (costs, count, kind, NULL, NULL, NULL_TREE, 0, where);
2091}
2092
2093inline unsigned
2095{
2096 return add_stmt_cost (costs, i->count, i->kind, i->stmt_info, i->node,
2097 i->vectype, i->misalign, i->where);
2098}
2099
2100inline void
2102{
2103 stmt_info_for_cost *cost;
2104 unsigned i;
2105 FOR_EACH_VEC_ELT (*cost_vec, i, cost)
2106 add_stmt_cost (costs, cost->count, cost->kind, cost->stmt_info,
2107 cost->node, cost->vectype, cost->misalign, cost->where);
2108}
2109
2110/*-----------------------------------------------------------------*/
2111/* Info on data references alignment. */
2112/*-----------------------------------------------------------------*/
2113#define DR_MISALIGNMENT_UNKNOWN (-1)
2114#define DR_MISALIGNMENT_UNINITIALIZED (-2)
2115
2116inline void
2118{
2119 dr_info->misalignment = val;
2120}
2121
2122extern int dr_misalignment (dr_vec_info *dr_info, tree vectype,
2123 poly_int64 offset = 0);
2124
2125#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
2126
2127/* Only defined once DR_MISALIGNMENT is defined. */
2128inline const poly_uint64
2130{
2133 return dr_info->target_alignment;
2134}
2135#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR)
2136#define DR_SCALAR_KNOWN_BOUNDS(DR) (DR)->scalar_access_known_in_bounds
2137
2138/* Return if the stmt_vec_info requires peeling for alignment. */
2139inline bool
2141{
2143 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2145 else
2146 dr_info = STMT_VINFO_DR_INFO (stmt_info);
2147
2148 return dr_info->safe_speculative_read_required;
2149}
2150
2151/* Set the safe_speculative_read_required for the the stmt_vec_info, if group
2152 access then set on the fist element otherwise set on DR directly. */
2153inline void
2155 bool requires_alignment)
2156{
2158 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2160 else
2161 dr_info = STMT_VINFO_DR_INFO (stmt_info);
2162
2163 dr_info->safe_speculative_read_required = requires_alignment;
2164}
2165
2166inline void
2168{
2169 dr_info->target_alignment = val;
2170}
2171#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL)
2172
2173/* Return true if data access DR_INFO is aligned to the targets
2174 preferred alignment for VECTYPE (which may be less than a full vector). */
2175
2176inline bool
2178{
2179 return (dr_misalignment (dr_info, vectype) == 0);
2180}
2181
2182/* Return TRUE if the (mis-)alignment of the data access is known with
2183 respect to the targets preferred alignment for VECTYPE, and FALSE
2184 otherwise. */
2185
2186inline bool
2191
2192/* Return the minimum alignment in bytes that the vectorized version
2193 of DR_INFO is guaranteed to have. */
2194
2195inline unsigned int
2197 poly_int64 offset = 0)
2198{
2199 int misalignment = dr_misalignment (dr_info, vectype, offset);
2200 if (misalignment == DR_MISALIGNMENT_UNKNOWN)
2202 else if (misalignment == 0)
2203 return known_alignment (DR_TARGET_ALIGNMENT (dr_info));
2204 return misalignment & -misalignment;
2205}
2206
2207/* Return the behavior of DR_INFO with respect to the vectorization context
2208 (which for outer loop vectorization might not be the behavior recorded
2209 in DR_INFO itself). */
2210
2213{
2214 stmt_vec_info stmt_info = dr_info->stmt;
2215 loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
2216 if (loop_vinfo == NULL
2217 || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info))
2218 return &DR_INNERMOST (dr_info->dr);
2219 else
2220 return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
2221}
2222
2223/* Return the offset calculated by adding the offset of this DR_INFO to the
2224 corresponding data_reference's offset. If CHECK_OUTER then use
2225 vect_dr_behavior to select the appropriate data_reference to use. */
2226
2227inline tree
2229 dr_vec_info *dr_info, bool check_outer = false)
2230{
2232 if (check_outer)
2233 base = vect_dr_behavior (vinfo, dr_info);
2234 else
2235 base = &dr_info->dr->innermost;
2236
2237 tree offset = base->offset;
2238
2239 if (!dr_info->offset)
2240 return offset;
2241
2242 offset = fold_convert (sizetype, offset);
2243 return fold_build2 (PLUS_EXPR, TREE_TYPE (dr_info->offset), offset,
2244 dr_info->offset);
2245}
2246
2247
2248/* Return the vect cost model for LOOP. */
2249inline enum vect_cost_model
2251{
2252 if (loop != NULL
2254 && flag_simd_cost_model != VECT_COST_MODEL_DEFAULT)
2255 return flag_simd_cost_model;
2256 return flag_vect_cost_model;
2257}
2258
2259/* Return true if the vect cost model is unlimited. */
2260inline bool
2265
2266/* Return true if the loop described by LOOP_VINFO is fully-masked and
2267 if the first iteration should use a partial mask in order to achieve
2268 alignment. */
2269
2270inline bool
2272{
2273 /* With early break vectorization we don't know whether the accesses will stay
2274 inside the loop or not. TODO: The early break adjustment code can be
2275 implemented the same way as vectorizable_linear_induction. However we
2276 can't test this today so reject it. */
2277 return (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
2278 && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
2279 && !(LOOP_VINFO_NON_LINEAR_IV (loop_vinfo)
2280 && LOOP_VINFO_EARLY_BREAKS (loop_vinfo)));
2281}
2282
2283/* Return the number of vectors of type VECTYPE that are needed to get
2284 NUNITS elements. NUNITS should be based on the vectorization factor,
2285 so it is always a known multiple of the number of elements in VECTYPE. */
2286
2287inline unsigned int
2289{
2290 return exact_div (nunits, TYPE_VECTOR_SUBPARTS (vectype)).to_constant ();
2291}
2292
2293/* Return the number of vectors in the context of vectorization region VINFO,
2294 needed for a group of statements and a vector type as specified by NODE. */
2295
2296inline unsigned int
2298{
2299 poly_uint64 vf;
2300
2301 if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
2302 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2303 else
2304 vf = 1;
2305
2306 vf *= SLP_TREE_LANES (node);
2307 tree vectype = SLP_TREE_VECTYPE (node);
2308
2309 return vect_get_num_vectors (vf, vectype);
2310}
2311
2312/* Update maximum unit count *MAX_NUNITS so that it accounts for
2313 NUNITS. *MAX_NUNITS can be 1 if we haven't yet recorded anything. */
2314
2315inline void
2317{
2318 /* All unit counts have the form vec_info::vector_size * X for some
2319 rational X, so two unit sizes must have a common multiple.
2320 Everything is a multiple of the initial value of 1. */
2321 *max_nunits = force_common_multiple (*max_nunits, nunits);
2322}
2323
2324/* Update maximum unit count *MAX_NUNITS so that it accounts for
2325 the number of units in vector type VECTYPE. *MAX_NUNITS can be 1
2326 if we haven't yet recorded any vector types. */
2327
2328inline void
2330{
2331 vect_update_max_nunits (max_nunits, TYPE_VECTOR_SUBPARTS (vectype));
2332}
2333
2334/* Return the vectorization factor that should be used for costing
2335 purposes while vectorizing the loop described by LOOP_VINFO.
2336 Pick a reasonable estimate if the vectorization factor isn't
2337 known at compile time. */
2338
2339inline unsigned int
2341{
2342 return estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
2343}
2344
2345/* Estimate the number of elements in VEC_TYPE for costing purposes.
2346 Pick a reasonable estimate if the exact number isn't known at
2347 compile time. */
2348
2349inline unsigned int
2351{
2352 return estimated_poly_value (TYPE_VECTOR_SUBPARTS (vec_type));
2353}
2354
2355/* Return the maximum possible vectorization factor for LOOP_VINFO. */
2356
2357inline unsigned HOST_WIDE_INT
2359{
2360 unsigned HOST_WIDE_INT vf;
2361 if (LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&vf))
2362 return vf;
2364}
2365
2366/* Return the size of the value accessed by unvectorized data reference
2367 DR_INFO. This is only valid once STMT_VINFO_VECTYPE has been calculated
2368 for the associated gimple statement, since that guarantees that DR_INFO
2369 accesses either a scalar or a scalar equivalent. ("Scalar equivalent"
2370 here includes things like V1SI, which can be vectorized in the same way
2371 as a plain SI.) */
2372
2373inline unsigned int
2378
2379/* Return true if LOOP_VINFO requires a runtime check for whether the
2380 vector loop is profitable. */
2381
2382inline bool
2384{
2385 unsigned int th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
2386 return (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2387 && th >= vect_vf_for_cost (loop_vinfo));
2388}
2389
2390/* Return true if CODE is a lane-reducing opcode. */
2391
2392inline bool
2394{
2395 return code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR;
2396}
2397
2398/* Return true if STMT is a lane-reducing statement. */
2399
2400inline bool
2402{
2403 if (auto *assign = dyn_cast <gassign *> (stmt))
2405 return false;
2406}
2407
2408/* Source location + hotness information. */
2410
2411/* A macro for calling:
2412 dump_begin_scope (MSG, vect_location);
2413 via an RAII object, thus printing "=== MSG ===\n" to the dumpfile etc,
2414 and then calling
2415 dump_end_scope ();
2416 once the object goes out of scope, thus capturing the nesting of
2417 the scopes.
2418
2419 These scopes affect dump messages within them: dump messages at the
2420 top level implicitly default to MSG_PRIORITY_USER_FACING, whereas those
2421 in a nested scope implicitly default to MSG_PRIORITY_INTERNALS. */
2422
2423#define DUMP_VECT_SCOPE(MSG) \
2424 AUTO_DUMP_SCOPE (MSG, vect_location)
2425
2426/* A sentinel class for ensuring that the "vect_location" global gets
2427 reset at the end of a scope.
2428
2429 The "vect_location" global is used during dumping and contains a
2430 location_t, which could contain references to a tree block via the
2431 ad-hoc data. This data is used for tracking inlining information,
2432 but it's not a GC root; it's simply assumed that such locations never
2433 get accessed if the blocks are optimized away.
2434
2435 Hence we need to ensure that such locations are purged at the end
2436 of any operations using them (e.g. via this class). */
2437
2439{
2440 public:
2442};
2443
2444/*-----------------------------------------------------------------*/
2445/* Function prototypes. */
2446/*-----------------------------------------------------------------*/
2447
2448/* Simple loop peeling and versioning utilities for vectorizer's purposes -
2449 in tree-vect-loop-manip.cc. */
2450extern void vect_set_loop_condition (class loop *, edge, loop_vec_info,
2451 tree, tree, tree, bool);
2452extern bool slpeel_can_duplicate_loop_p (const class loop *, const_edge,
2453 const_edge);
2455 class loop *, edge,
2456 edge, edge *, bool = true,
2459extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
2460 tree *, tree *, tree *, int, bool, bool,
2461 tree *);
2467extern edge vec_init_loop_exit_info (class loop *);
2469
2470/* In tree-vect-stmts.cc. */
2471extern tree get_related_vectype_for_scalar_type (machine_mode, tree,
2472 poly_uint64 = 0);
2473extern tree get_vectype_for_scalar_type (vec_info *, tree, unsigned int = 0);
2475extern tree get_mask_type_for_scalar_type (vec_info *, tree, unsigned int = 0);
2478extern bool vect_chooses_same_modes_p (vec_info *, machine_mode);
2479extern bool vect_chooses_same_modes_p (machine_mode, machine_mode);
2481extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
2482 stmt_vec_info * = NULL, gimple ** = NULL);
2483extern bool vect_is_simple_use (vec_info *, slp_tree,
2484 unsigned, tree *, slp_tree *,
2485 enum vect_def_type *,
2486 tree *, stmt_vec_info * = NULL);
2492 int*, vec<tree> *);
2494 code_helper *, int *,
2495 vec<tree> *);
2497 tree, tree,
2498 vec<std::pair<tree, tree_code> > &,
2499 tree = NULL_TREE,
2500 slp_tree = NULL);
2502
2503extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2505 tree, int, enum vect_cost_model_location);
2506extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2508 tree, int, enum vect_cost_model_location);
2509extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2510 enum vect_cost_for_stmt,
2512extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2514 slp_tree, tree, int,
2516
2517/* Overload of record_stmt_cost with VECTYPE derived from STMT_INFO. */
2518
2519inline unsigned
2521 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
2522 int misalign, enum vect_cost_model_location where)
2523{
2524 return record_stmt_cost (body_cost_vec, count, kind, stmt_info,
2525 STMT_VINFO_VECTYPE (stmt_info), misalign, where);
2526}
2527
2528/* Overload of record_stmt_cost with VECTYPE derived from SLP node. */
2529
2530inline unsigned
2532 enum vect_cost_for_stmt kind, slp_tree node,
2533 int misalign, enum vect_cost_model_location where)
2534{
2535 return record_stmt_cost (body_cost_vec, count, kind, node,
2536 SLP_TREE_VECTYPE (node), misalign, where);
2537}
2538
2545 tree, vec<tree> *,
2546 tree = NULL, vec<tree> * = NULL,
2547 tree = NULL, vec<tree> * = NULL,
2548 tree = NULL, vec<tree> * = NULL);
2551extern tree vect_get_slp_vect_def (slp_tree, unsigned);
2560 dr_alignment_support, int, bool,
2561 unsigned int *, unsigned int *,
2563 stmt_vector_for_cost *, bool);
2566 unsigned int *, stmt_vector_for_cost *);
2567extern bool vect_supportable_shift (vec_info *, enum tree_code, tree);
2570extern void optimize_mask_stores (class loop*);
2572 const char * = nullptr);
2576 tree *, unsigned int = 0);
2578
2579/* In tree-if-conv.cc. */
2580extern bool ref_within_array_bound (gimple *, tree);
2581
2582/* In tree-vect-data-refs.cc. */
2585 (vec_info *, dr_vec_info *, tree, int,
2586 bool = false);
2595extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
2596 tree, int, internal_fn *, tree *,
2597 vec<int> * = nullptr);
2600 vec<int> * = nullptr);
2605 vec<int> *, int);
2606extern opt_result vect_analyze_data_refs (vec_info *, bool *);
2609 stmt_vec_info, tree, class loop *, tree,
2611 gimple **, bool,
2612 tree = NULL_TREE);
2615extern void vect_copy_ref_info (tree, tree);
2617extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT);
2618extern internal_fn vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, bool);
2619extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
2620extern internal_fn vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT,
2621 bool, vec<int> * = nullptr);
2625 class loop **);
2626extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
2628 const char * = NULL);
2631 tree);
2632
2633/* In tree-vect-loop.cc. */
2634extern tree neutral_op_for_reduction (tree, code_helper, tree, bool = true);
2637/* Used in tree-vect-loop-manip.cc */
2639/* Used in gimple-loop-interchange.c and tree-parloops.cc. */
2641 enum tree_code);
2643/* Drive for loop analysis stage. */
2645 vec_info_shared *);
2648 tree *, bool);
2649extern tree vect_halve_mask_nunits (tree, machine_mode);
2650extern tree vect_double_mask_nunits (tree, machine_mode);
2652 unsigned int, tree, tree);
2655 unsigned int, tree, unsigned int);
2656extern void vect_record_loop_len (loop_vec_info, vec_loop_lens *, unsigned int,
2657 tree, unsigned int);
2659 vec_loop_lens *, unsigned int, tree,
2660 unsigned int, unsigned int);
2663 unsigned int, tree, tree, unsigned int,
2664 unsigned int);
2668
2669/* Drive for loop transformation stage. */
2670extern class loop *vect_transform_loop (loop_vec_info, gimple *);
2680extern opt_result vect_analyze_loop_form (class loop *, gimple *,
2683 const vect_loop_form_info *,
2684 loop_vec_info = nullptr);
2686 slp_tree, slp_instance, int,
2687 bool, stmt_vector_for_cost *);
2697 slp_tree);
2709extern bool vect_emulated_vector_p (tree);
2712extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
2717
2718/* Nonlinear induction. */
2721
2722/* In tree-vect-slp.cc. */
2723extern void vect_slp_init (void);
2724extern void vect_slp_fini (void);
2728 bool, unsigned *,
2729 unsigned * = nullptr, bool = false);
2733extern void vect_schedule_slp (vec_info *, const vec<slp_instance> &);
2734extern opt_result vect_analyze_slp (vec_info *, unsigned, bool);
2737extern void vect_optimize_slp (vec_info *);
2738extern void vect_gather_slp_loads (vec_info *);
2739extern tree vect_get_slp_scalar_def (slp_tree, unsigned);
2740extern void vect_get_slp_defs (slp_tree, vec<tree> *);
2741extern void vect_get_slp_defs (vec_info *, slp_tree, vec<vec<tree> > *,
2742 unsigned n = -1U);
2743extern bool vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop);
2744extern bool vect_slp_function (function *);
2748extern bool can_duplicate_and_interleave_p (vec_info *, unsigned int, tree,
2749 unsigned int * = NULL,
2750 tree * = NULL, tree * = NULL);
2752 const vec<tree> &, unsigned int, vec<tree> &);
2755extern void vect_free_slp_tree (slp_tree);
2756extern bool compatible_calls_p (gcall *, gcall *, bool);
2757extern int vect_slp_child_index_for_operand (const gimple *, int op, bool);
2758
2761extern tree vect_get_mask_load_else (int, tree);
2762
2763/* In tree-vect-patterns.cc. */
2764extern void
2766extern bool vect_get_range_info (tree, wide_int*, wide_int*);
2767
2768/* Pattern recognition functions.
2769 Additional pattern recognition functions can (and will) be added
2770 in the future. */
2772
2773/* In tree-vectorizer.cc. */
2774unsigned vectorize_loops (void);
2776gimple *vect_loop_vectorized_call (class loop *, gcond **cond = NULL);
2778
2779/* SLP Pattern matcher types, tree-vect-slp-patterns.cc. */
2780
2781/* Forward declaration of possible two operands operation that can be matched
2782 by the complex numbers pattern matchers. */
2783enum _complex_operation : unsigned;
2784
2785/* All possible load permute values that could result from the partial data-flow
2786 analysis. */
2796
2797/* Cache from nodes to the load permutation they represent. */
2798typedef hash_map <slp_tree, complex_perm_kinds_t>
2800
2801/* Cache from nodes pair to being compatible or not. */
2802typedef pair_hash <nofree_ptr_hash <_slp_tree>,
2803 nofree_ptr_hash <_slp_tree>> slp_node_hash;
2804typedef hash_map <slp_node_hash, bool> slp_compat_nodes_map_t;
2805
2806
2807/* Vector pattern matcher base class. All SLP pattern matchers must inherit
2808 from this type. */
2809
2811{
2812 protected:
2813 /* The number of arguments that the IFN requires. */
2814 unsigned m_num_args;
2815
2816 /* The internal function that will be used when a pattern is created. */
2818
2819 /* The current node being inspected. */
2821
2822 /* The list of operands to be the children for the node produced when the
2823 internal function is created. */
2825
2826 /* Default constructor where NODE is the root of the tree to inspect. */
2828 {
2829 this->m_ifn = ifn;
2830 this->m_node = node;
2831 this->m_ops.create (0);
2832 if (m_ops)
2833 this->m_ops.safe_splice (*m_ops);
2834 }
2835
2836 public:
2837
2838 /* Create a new instance of the pattern matcher class of the given type. */
2841
2842 /* Build the pattern from the data collected so far. */
2843 virtual void build (vec_info *) = 0;
2844
2845 /* Default destructor. */
2846 virtual ~vect_pattern ()
2847 {
2848 this->m_ops.release ();
2849 }
2850};
2851
2852/* Function pointer to create a new pattern matcher from a generic type. */
2853typedef vect_pattern* (*vect_pattern_decl_t) (slp_tree_to_load_perm_map_t *,
2855 slp_tree *);
2856
2857/* List of supported pattern matchers. */
2859
2860/* Number of supported pattern matchers. */
2861extern size_t num__slp_patterns;
2862
2863/* ----------------------------------------------------------------------
2864 Target support routines
2865 -----------------------------------------------------------------------
2866 The following routines are provided to simplify costing decisions in
2867 target code. Please add more as needed. */
2868
2869/* Return true if an operaton of kind KIND for STMT_INFO represents
2870 the extraction of an element from a vector in preparation for
2871 storing the element to memory. */
2872inline bool
2874{
2875 return (kind == vec_to_scalar
2876 && STMT_VINFO_DATA_REF (stmt_info)
2877 && DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)));
2878}
2879
2880/* Return true if STMT_INFO represents part of a reduction. */
2881inline bool
2883{
2884 return STMT_VINFO_REDUC_IDX (stmt_info) != -1;
2885}
2886
2887/* Return true if SLP_NODE represents part of a reduction. */
2888inline bool
2890{
2891 return SLP_TREE_REDUC_IDX (slp_node) != -1;
2892}
2893
2894/* If STMT_INFO describes a reduction, return the vect_reduction_type
2895 of the reduction it describes, otherwise return -1. */
2896inline int
2898{
2899 if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
2900 {
2901 vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, node);
2902 if (reduc_info)
2903 return int (VECT_REDUC_INFO_TYPE (reduc_info));
2904 }
2905 return -1;
2906}
2907
2908/* If STMT_INFO is a COND_EXPR that includes an embedded comparison, return the
2909 scalar type of the values being compared. Return null otherwise. */
2910inline tree
2912{
2913 if (auto *assign = dyn_cast<gassign *> (stmt_info->stmt))
2914 if (gimple_assign_rhs_code (assign) == COND_EXPR)
2915 {
2916 tree cond = gimple_assign_rhs1 (assign);
2917 if (COMPARISON_CLASS_P (cond))
2918 return TREE_TYPE (TREE_OPERAND (cond, 0));
2919 }
2920 return NULL_TREE;
2921}
2922
2923/* If STMT_INFO is a comparison or contains an embedded comparison, return the
2924 scalar type of the values being compared. Return null otherwise. */
2925inline tree
2927{
2928 if (auto *assign = dyn_cast<gassign *> (stmt_info->stmt))
2930 return TREE_TYPE (gimple_assign_rhs1 (assign));
2931 return vect_embedded_comparison_type (stmt_info);
2932}
2933
2934/* Return true if STMT_INFO extends the result of a load. */
2935inline bool
2937{
2938 /* Although this is quite large for an inline function, this part
2939 at least should be inline. */
2940 gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
2941 if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
2942 return false;
2943
2944 tree rhs = gimple_assign_rhs1 (stmt_info->stmt);
2945 tree lhs_type = TREE_TYPE (gimple_assign_lhs (assign));
2946 tree rhs_type = TREE_TYPE (rhs);
2947 if (!INTEGRAL_TYPE_P (lhs_type)
2948 || !INTEGRAL_TYPE_P (rhs_type)
2949 || TYPE_PRECISION (lhs_type) <= TYPE_PRECISION (rhs_type))
2950 return false;
2951
2952 stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
2953 return (def_stmt_info
2954 && STMT_VINFO_DATA_REF (def_stmt_info)
2955 && DR_IS_READ (STMT_VINFO_DATA_REF (def_stmt_info)));
2956}
2957
2958/* Return true if STMT_INFO is an integer truncation. */
2959inline bool
2961{
2962 gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
2963 if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
2964 return false;
2965
2966 tree lhs_type = TREE_TYPE (gimple_assign_lhs (assign));
2967 tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (assign));
2968 return (INTEGRAL_TYPE_P (lhs_type)
2969 && INTEGRAL_TYPE_P (rhs_type)
2970 && TYPE_PRECISION (lhs_type) < TYPE_PRECISION (rhs_type));
2971}
2972
2973/* Build a GIMPLE_ASSIGN or GIMPLE_CALL with the tree_code,
2974 or internal_fn contained in ch, respectively. */
2976#endif /* GCC_TREE_VECTORIZER_H */
edge loop_preheader_edge(const class loop *loop)
Definition cfgloop.cc:1760
class loop * loop_p
Definition cfgloop.h:98
Definition tree-vectorizer.h:1382
vec< slp_root > roots
Definition tree-vectorizer.h:1387
_bb_vec_info(vec< basic_block > bbs, vec_info_shared *)
Definition tree-vect-slp.cc:7969
~_bb_vec_info()
Definition tree-vect-slp.cc:8006
Definition tree-vectorizer.h:903
tree mask_skip_niters_pfa_offset
Definition tree-vectorizer.h:998
bool peeling_for_gaps
Definition tree-vectorizer.h:1141
bool can_use_partial_vectors_p
Definition tree-vectorizer.h:1101
poly_uint64 ptr_mask
Definition tree-vectorizer.h:1034
signed char partial_load_store_bias
Definition tree-vectorizer.h:1136
class vector_costs * vector_costs
Definition tree-vectorizer.h:921
class dr_vec_info * unaligned_dr
Definition tree-vectorizer.h:1021
_loop_vec_info(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:801
edge skip_main_loop_edge
Definition tree-vectorizer.h:947
auto_vec< stmt_vec_info > reduction_chains
Definition tree-vectorizer.h:1070
unsigned int th
Definition tree-vectorizer.h:929
_loop_vec_info * epilogue_vinfo
Definition tree-vectorizer.h:1201
_loop_vec_info * main_loop_info
Definition tree-vectorizer.h:1192
auto_vec< gimple * > early_break_vuses
Definition tree-vectorizer.h:1230
bool using_partial_vectors_p
Definition tree-vectorizer.h:1108
bool no_data_dependencies
Definition tree-vectorizer.h:1178
class loop * scalar_loop
Definition tree-vectorizer.h:1188
tree drs_advanced_by
Definition tree-vectorizer.h:1204
scalar_cond_masked_set_type scalar_cond_masked_set
Definition tree-vectorizer.h:984
auto_vec< ddr_p > may_alias_ddrs
Definition tree-vectorizer.h:1044
tree num_itersm1
Definition tree-vectorizer.h:912
unsigned int inner_loop_cost_factor
Definition tree-vectorizer.h:1092
bool vectorizable
Definition tree-vectorizer.h:1095
vect_partial_vector_style partial_vector_style
Definition tree-vectorizer.h:1018
poly_uint64 slp_unrolling_factor
Definition tree-vectorizer.h:1088
poly_uint64 max_spec_read_amount
Definition tree-vectorizer.h:1037
auto_vec< vect_reduc_info > reduc_infos
Definition tree-vectorizer.h:955
edge vec_epilogue_loop_iv_exit
Definition tree-vectorizer.h:1212
tree num_iters
Definition tree-vectorizer.h:914
hash_map< tree, tree > * scan_map
Definition tree-vectorizer.h:1084
int peeling_for_alignment
Definition tree-vectorizer.h:1031
vec_cond_masked_set_type vec_cond_masked_set
Definition tree-vectorizer.h:987
bool epil_using_partial_vectors_p
Definition tree-vectorizer.h:1124
bool early_breaks
Definition tree-vectorizer.h:1149
poly_uint64 versioning_threshold
Definition tree-vectorizer.h:935
auto_vec< vec_lower_bound > lower_bounds
Definition tree-vectorizer.h:1059
profile_probability scalar_loop_scaling
Definition tree-vectorizer.h:1184
auto_vec< gimple * > early_break_stores
Definition tree-vectorizer.h:1222
edge main_loop_edge
Definition tree-vectorizer.h:946
edge skip_this_loop_edge
Definition tree-vectorizer.h:951
bool must_use_partial_vectors_p
Definition tree-vectorizer.h:1104
auto_vec< stmt_info_for_cost > scalar_cost_vec
Definition tree-vectorizer.h:1077
gcond * loop_iv_cond
Definition tree-vectorizer.h:1155
basic_block early_break_dest_bb
Definition tree-vectorizer.h:1226
bool using_select_vl_p
Definition tree-vectorizer.h:1120
tree rgroup_compare_type
Definition tree-vectorizer.h:1003
poly_uint64 vectorization_factor
Definition tree-vectorizer.h:938
bool peeling_for_niter
Definition tree-vectorizer.h:1145
auto_vec< dr_with_seg_len_pair_t > comp_alias_ddrs
Definition tree-vectorizer.h:1048
edge vec_loop_iv_exit
Definition tree-vectorizer.h:1208
tree num_iters_unchanged
Definition tree-vectorizer.h:916
vec_loop_lens lens
Definition tree-vectorizer.h:981
unsigned HOST_WIDE_INT max_vectorization_factor
Definition tree-vectorizer.h:973
bool allow_mutual_alignment
Definition tree-vectorizer.h:1131
unsigned int suggested_unroll_factor
Definition tree-vectorizer.h:969
hash_map< tree_operand_hash, tree > * ivexpr_map
Definition tree-vectorizer.h:1080
auto_vec< stmt_vec_info > may_misalign_stmts
Definition tree-vectorizer.h:1063
tree rgroup_iv_type
Definition tree-vectorizer.h:1014
vec_loop_masks masks
Definition tree-vectorizer.h:977
tree num_iters_assumptions
Definition tree-vectorizer.h:918
tree simd_if_cond
Definition tree-vectorizer.h:1010
auto_vec< stmt_vec_info > alternate_defs
Definition tree-vectorizer.h:1074
auto_vec< tree > check_nonzero
Definition tree-vectorizer.h:1056
bool user_unroll
Definition tree-vectorizer.h:1159
auto_vec< stmt_vec_info > early_break_live_ivs
Definition tree-vectorizer.h:1234
tree mask_skip_niters
Definition tree-vectorizer.h:993
bool has_mask_store
Definition tree-vectorizer.h:1181
auto_vec< stmt_vec_info > reductions
Definition tree-vectorizer.h:1066
edge scalar_loop_iv_exit
Definition tree-vectorizer.h:1216
class loop * loop
Definition tree-vectorizer.h:909
bool nonlinear_iv
Definition tree-vectorizer.h:1040
~_loop_vec_info()
Definition tree-vect-loop.cc:926
bool using_decrementing_iv_p
Definition tree-vectorizer.h:1115
auto_vec< gcond * > conds
Definition tree-vectorizer.h:1152
_loop_vec_info * orig_loop_info
Definition tree-vectorizer.h:1197
hash_map< tree, vect_reusable_accumulator > reusable_accumulators
Definition tree-vectorizer.h:962
class vector_costs * scalar_costs
Definition tree-vectorizer.h:924
auto_vec< vec_object_pair > check_unequal_addrs
Definition tree-vectorizer.h:1051
Definition tree-vectorizer.h:397
slp_instance_kind kind
Definition tree-vectorizer.h:424
vec< stmt_vec_info > root_stmts
Definition tree-vectorizer.h:404
slp_tree root
Definition tree-vectorizer.h:400
dump_user_location_t location() const
Definition tree-vect-slp.cc:203
vec< slp_tree > loads
Definition tree-vectorizer.h:411
vec< _slp_instance * > subgraph_entries
Definition tree-vectorizer.h:421
vec< tree > remain_defs
Definition tree-vectorizer.h:408
slp_tree reduc_phis
Definition tree-vectorizer.h:414
stmt_vector_for_cost cost_vec
Definition tree-vectorizer.h:417
Definition tree-vectorizer.h:1472
unsigned int min_input_precision
Definition tree-vectorizer.h:1592
code_helper reduc_code
Definition tree-vectorizer.h:1575
tree vectype
Definition tree-vectorizer.h:1496
bool in_pattern_p
Definition tree-vectorizer.h:1480
stmt_vec_info first_element
Definition tree-vectorizer.h:1542
unsigned int mask_precision
Definition tree-vectorizer.h:1615
int reduc_idx
Definition tree-vectorizer.h:1579
gimple * stmt
Definition tree-vectorizer.h:1493
tree loop_phi_evolution_base_unchanged
Definition tree-vectorizer.h:1514
enum vect_induction_op_type loop_phi_evolution_type
Definition tree-vectorizer.h:1516
innermost_loop_behavior dr_wrt_vec_loop
Definition tree-vectorizer.h:1506
bool slp_vect_only_p
Definition tree-vectorizer.h:1618
bool live
Definition tree-vectorizer.h:1477
unsigned int gap
Definition tree-vectorizer.h:1549
unsigned int min_output_precision
Definition tree-vectorizer.h:1587
unsigned int min_neg_dist
Definition tree-vectorizer.h:1553
bool vectorizable
Definition tree-vectorizer.h:1490
bool pattern_stmt_p
Definition tree-vectorizer.h:1486
bool strided_p
Definition tree-vectorizer.h:1564
enum vect_relevant relevant
Definition tree-vectorizer.h:1558
unsigned int size
Definition tree-vectorizer.h:1546
signop operation_sign
Definition tree-vectorizer.h:1598
gimple_seq pattern_def_seq
Definition tree-vectorizer.h:1532
stmt_vec_info related_stmt
Definition tree-vectorizer.h:1527
unsigned simd_lane_access_p
Definition tree-vectorizer.h:1567
bool slp_vect_pattern_only_p
Definition tree-vectorizer.h:1622
stmt_vec_info reduc_def
Definition tree-vectorizer.h:1583
bool gather_scatter_p
Definition tree-vectorizer.h:1561
enum vect_def_type def_type
Definition tree-vectorizer.h:1535
stmt_vec_info next_element
Definition tree-vectorizer.h:1544
unsigned int operation_precision
Definition tree-vectorizer.h:1597
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:1571
enum slp_vect_type slp_type
Definition tree-vectorizer.h:1538
dr_vec_info dr_aux
Definition tree-vectorizer.h:1502
tree loop_phi_evolution_part
Definition tree-vectorizer.h:1515
Definition tree-vectorizer.h:2439
~auto_purge_vect_location()
Definition tree-vectorizer.cc:94
Definition vec.h:1667
Definition tree.h:82
Definition tree-vectorizer.h:1438
bool scalar_access_known_in_bounds
Definition tree-vectorizer.h:1461
unsigned group
Definition tree-vectorizer.h:1446
tree base_decl
Definition tree-vectorizer.h:1463
bool base_misaligned
Definition tree-vectorizer.h:1453
tree offset
Definition tree-vectorizer.h:1467
int misalignment
Definition tree-vectorizer.h:1448
poly_uint64 target_alignment
Definition tree-vectorizer.h:1451
data_reference * dr
Definition tree-vectorizer.h:1441
bool safe_speculative_read_required
Definition tree-vectorizer.h:1457
stmt_vec_info stmt
Definition tree-vectorizer.h:1443
Definition dumpfile.h:340
Definition hash-map.h:40
Definition hash-set.h:37
Definition inchash.h:38
void add_int(unsigned v)
Definition inchash.h:55
hashval_t end() const
Definition inchash.h:49
void add_flag(bool flag)
Definition inchash.h:124
Definition cfgloop.h:120
class loop * inner
Definition cfgloop.h:148
unsigned force_vectorize
Definition cfgloop.h:227
void * aux
Definition cfgloop.h:154
Definition opt-problem.h:179
Definition profile-count.h:149
Definition tree-vectorizer.h:572
vec< data_reference_p > datarefs
Definition tree-vectorizer.h:581
vec< data_reference > datarefs_copy
Definition tree-vectorizer.h:582
auto_vec< loop_p > loop_nest
Definition tree-vectorizer.h:585
void check_datarefs()
Definition tree-vectorizer.cc:507
void save_datarefs()
Definition tree-vectorizer.cc:497
~vec_info_shared()
Definition tree-vectorizer.cc:489
vec< ddr_p > ddrs
Definition tree-vectorizer.h:589
vec_info_shared()
Definition tree-vectorizer.cc:482
Definition tree-vectorizer.h:593
stmt_vec_info lookup_stmt(gimple *)
Definition tree-vectorizer.cc:571
basic_block * bbs
Definition tree-vectorizer.h:653
auto_vec< stmt_vec_info > grouped_stores
Definition tree-vectorizer.h:638
~vec_info()
Definition tree-vectorizer.cc:474
auto_vec< slp_instance > slp_instances
Definition tree-vectorizer.h:630
void replace_stmt(gimple_stmt_iterator *, stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:660
void remove_stmt(stmt_vec_info)
Definition tree-vectorizer.cc:644
void insert_on_entry(stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:702
void set_vinfo_for_stmt(gimple *, stmt_vec_info, bool=true)
Definition tree-vectorizer.cc:745
vec_base_alignments base_alignments
Definition tree-vectorizer.h:634
machine_mode vector_mode
Definition tree-vectorizer.h:646
stmt_vec_info resync_stmt_addr(gimple *)
Definition tree-vectorizer.cc:551
stmt_vec_info new_stmt_vec_info(gimple *stmt)
Definition tree-vectorizer.cc:713
class dr_vec_info * lookup_dr(data_reference *)
Definition tree-vectorizer.cc:613
vec< stmt_vec_info > stmt_vec_infos
Definition tree-vectorizer.h:621
void move_dr(stmt_vec_info, stmt_vec_info)
Definition tree-vectorizer.cc:625
bool any_known_not_updated_vssa
Definition tree-vectorizer.h:627
vec_kind
Definition tree-vectorizer.h:596
@ loop
Definition tree-vectorizer.h:596
@ bb
Definition tree-vectorizer.h:596
void free_stmt_vec_infos()
Definition tree-vectorizer.cc:766
vec_info_shared * shared
Definition tree-vectorizer.h:618
gimple_seq inv_pattern_def_seq
Definition tree-vectorizer.h:662
unsigned int nbbs
Definition tree-vectorizer.h:656
stmt_vec_info add_stmt(gimple *)
Definition tree-vectorizer.cc:522
vec_kind kind
Definition tree-vectorizer.h:615
mode_set used_vector_modes
Definition tree-vectorizer.h:641
hash_set< int_hash< machine_mode, E_VOIDmode, E_BLKmode > > mode_set
Definition tree-vectorizer.h:595
void insert_seq_on_entry(stmt_vec_info, gimple_seq)
Definition tree-vectorizer.cc:676
stmt_vec_info lookup_def(tree)
Definition tree-vectorizer.cc:588
void free_stmt_vec_info(stmt_vec_info)
Definition tree-vectorizer.cc:777
stmt_vec_info lookup_single_use(tree)
Definition tree-vectorizer.cc:601
vec_info(vec_kind, vec_info_shared *)
Definition tree-vectorizer.cc:463
stmt_vec_info add_pattern_stmt(gimple *, stmt_vec_info)
Definition tree-vectorizer.cc:534
bool stmt_vec_info_ro
Definition tree-vectorizer.h:623
vec_lower_bound()
Definition tree-vectorizer.h:561
poly_uint64 min_value
Definition tree-vectorizer.h:567
bool unsigned_p
Definition tree-vectorizer.h:566
vec_lower_bound(tree e, bool u, poly_uint64 m)
Definition tree-vectorizer.h:562
tree expr
Definition tree-vectorizer.h:565
Definition vec-perm-indices.h:51
Definition tree-vectorizer.h:2811
static vect_pattern * recognize(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
internal_fn m_ifn
Definition tree-vectorizer.h:2817
vec< slp_tree > m_ops
Definition tree-vectorizer.h:2824
virtual ~vect_pattern()
Definition tree-vectorizer.h:2846
virtual void build(vec_info *)=0
vect_pattern(slp_tree *node, vec< slp_tree > *m_ops, internal_fn ifn)
Definition tree-vectorizer.h:2827
slp_tree * m_node
Definition tree-vectorizer.h:2820
unsigned m_num_args
Definition tree-vectorizer.h:2814
Definition tree-vectorizer.h:824
tree reduc_epilogue_adjustment
Definition tree-vectorizer.h:856
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:832
struct vect_reusable_accumulator * reused_accumulator
Definition tree-vectorizer.h:860
internal_fn reduc_fn
Definition tree-vectorizer.h:838
code_helper reduc_code
Definition tree-vectorizer.h:835
auto_vec< tree > reduc_initial_values
Definition tree-vectorizer.h:864
tree induc_cond_initial_val
Definition tree-vectorizer.h:853
tree reduc_vectype
Definition tree-vectorizer.h:850
bool force_single_cycle
Definition tree-vectorizer.h:847
enum vect_def_type def_type
Definition tree-vectorizer.h:828
unsigned int reduc_result_pos
Definition tree-vectorizer.h:844
auto_vec< tree > reduc_scalar_results
Definition tree-vectorizer.h:870
Definition tree-vectorizer.h:1728
unsigned int adjust_cost_for_freq(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1874
virtual ~vector_costs()
Definition tree-vectorizer.h:1731
bool m_costing_for_scalar
Definition tree-vectorizer.h:1801
vector_costs(vec_info *, bool)
Definition tree-vectorizer.h:1826
bool costing_for_scalar() const
Definition tree-vectorizer.h:1786
vec_info * m_vinfo
Definition tree-vectorizer.h:1797
virtual unsigned int add_stmt_cost(int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, vect_cost_model_location where)
Definition tree-vectorizer.cc:1836
machine_mode m_suggested_epilogue_mode
Definition tree-vectorizer.h:1815
virtual bool better_main_loop_than_p(const vector_costs *other) const
Definition tree-vectorizer.cc:1894
unsigned int m_costs[3]
Definition tree-vectorizer.h:1804
unsigned int outside_cost() const
Definition tree-vectorizer.h:1867
unsigned int record_stmt_cost(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1860
virtual bool better_epilogue_loop_than_p(const vector_costs *other, loop_vec_info main_loop) const
Definition tree-vectorizer.cc:1913
unsigned int epilogue_cost() const
Definition tree-vectorizer.h:1858
int m_masked_epilogue
Definition tree-vectorizer.h:1816
machine_mode suggested_epilogue_mode(int &masked) const
Definition tree-vectorizer.h:1893
unsigned int m_suggested_unroll_factor
Definition tree-vectorizer.h:1807
unsigned int suggested_unroll_factor() const
Definition tree-vectorizer.h:1884
bool m_finished
Definition tree-vectorizer.h:1819
int compare_outside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:2089
unsigned int body_cost() const
Definition tree-vectorizer.h:1849
virtual void finish_cost(const vector_costs *scalar_costs)
Definition tree-vectorizer.cc:1849
unsigned int prologue_cost() const
Definition tree-vectorizer.h:1840
unsigned int total_cost() const
Definition tree-vectorizer.h:1876
int compare_inside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:1996
struct basic_block_def * basic_block
Definition coretypes.h:357
class edge_def * edge
Definition coretypes.h:354
gimple * gimple_seq
Definition coretypes.h:100
const class edge_def * const_edge
Definition coretypes.h:355
const union tree_node * const_tree
Definition coretypes.h:98
union tree_node * tree
Definition coretypes.h:97
static unsigned int count[debug_counter_number_of_counters]
Definition dbgcnt.cc:50
bool dominated_by_p(enum cdi_direction dir, const_basic_block bb1, const_basic_block bb2)
Definition dominance.cc:1125
@ CDI_DOMINATORS
Definition dominance.h:25
dump_flags_t dump_flags
Definition dumpfile.cc:65
FILE * dump_file
Definition dumpfile.cc:63
@ TDF_DETAILS
Definition dumpfile.h:92
vect_cost_model
Definition flag-types.h:279
@ VECT_COST_MODEL_UNLIMITED
Definition flag-types.h:283
@ VECT_COST_MODEL_DEFAULT
Definition flag-types.h:284
bool operand_equal_p(const_tree arg0, const_tree arg1, unsigned int flags)
Definition fold-const.cc:4340
#define fold_convert(T1, T2)
Definition fold-const.h:96
#define fold_build2(c, t1, t2, t3)
Definition fold-const.h:76
static struct token T
Definition gengtype-parse.cc:45
internal_fn
Definition genmatch.cc:1015
tree_code
Definition genmatch.cc:1002
unsigned gimple_uid(const gimple *g)
Definition gimple.h:2091
enum tree_code gimple_assign_rhs_code(const gassign *gs)
Definition gimple.h:2857
tree gimple_assign_rhs1(const gassign *gs)
Definition gimple.h:2667
basic_block gimple_bb(const gimple *g)
Definition gimple.h:1867
tree gimple_assign_lhs(const gassign *gs)
Definition gimple.h:2616
#define init_cost
Definition ira-costs.cc:63
T dyn_cast(U *p)
Definition is-a.h:280
void add_expr(const_tree t, inchash::hash &hstate, unsigned int flags)
Definition fold-const.cc:4354
opt_pointer_wrapper< tree > opt_tree
Definition opt-problem.h:287
poly_int< NUM_POLY_INT_COEFFS, unsigned HOST_WIDE_INT > poly_uint64
Definition poly-int-types.h:25
poly_int< NUM_POLY_INT_COEFFS, HOST_WIDE_INT > poly_int64
Definition poly-int-types.h:24
i
Definition poly-int.h:776
signop
Definition signop.h:28
Definition tree-vectorizer.h:297
stmt_vec_info representative
Definition tree-vectorizer.h:313
tree gs_base
Definition tree-vectorizer.h:349
unsigned int lanes
Definition tree-vectorizer.h:342
bool avoid_stlf_fail
Definition tree-vectorizer.h:355
load_permutation_t load_permutation
Definition tree-vectorizer.h:324
slp_tree prev_node
Definition tree-vectorizer.h:380
enum tree_code code
Definition tree-vectorizer.h:344
bool ldst_lanes
Definition tree-vectorizer.h:352
vec< tree > vec_defs
Definition tree-vectorizer.h:332
~_slp_tree()
Definition tree-vect-slp.cc:138
T & get_data(T &else_)
Definition tree-vectorizer.h:365
int gs_scale
Definition tree-vectorizer.h:347
unsigned int refcnt
Definition tree-vectorizer.h:335
enum stmt_vec_info_type type
Definition tree-vectorizer.h:361
slp_tree next_node
Definition tree-vectorizer.h:379
int vertex
Definition tree-vectorizer.h:357
enum vect_def_type def_type
Definition tree-vectorizer.h:340
bool * failed
Definition tree-vectorizer.h:370
vec< tree > ops
Definition tree-vectorizer.h:310
void push_vec_def(gimple *def)
Definition tree-vect-slp.cc:161
int reduc_idx
Definition tree-vectorizer.h:319
poly_uint64 max_nunits
Definition tree-vectorizer.h:338
lane_permutation_t lane_permutation
Definition tree-vectorizer.h:328
_slp_tree()
Definition tree-vect-slp.cc:105
int id
Definition tree-vectorizer.h:317
struct _slp_tree::@120146070312105203102335350023136130264031166255 cycle_info
tree vectype
Definition tree-vectorizer.h:330
void push_vec_def(tree def)
Definition tree-vectorizer.h:302
vec< slp_tree > children
Definition tree-vectorizer.h:305
vect_data * data
Definition tree-vectorizer.h:362
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:308
class loop * loop_father
Definition basic-block.h:126
Definition ira-costs.cc:54
Definition tree-data-ref.h:151
struct innermost_loop_behavior innermost
Definition tree-data-ref.h:173
Definition loop-invariant.cc:88
static const bool empty_zero_p
Definition tree-vectorizer.h:520
static void mark_empty(value_type &v)
Definition tree-vectorizer.h:523
scalar_cond_masked_key value_type
Definition tree-vectorizer.h:496
static bool equal(value_type existing, value_type candidate)
Definition tree-vectorizer.h:511
static void remove(value_type &)
Definition tree-vectorizer.h:542
static hashval_t hash(value_type v)
Definition tree-vectorizer.h:499
static void mark_deleted(value_type &)
Definition tree-vectorizer.h:535
static bool is_deleted(const value_type &)
Definition tree-vectorizer.h:537
static bool is_empty(value_type v)
Definition tree-vectorizer.h:530
scalar_cond_masked_key compare_type
Definition tree-vectorizer.h:495
Definition hash-traits.h:466
Definition graphite.h:338
data_reference_p dr
Definition graphite.h:343
Definition function.h:249
Definition gimple.h:907
Definition tree-vectorizer.h:1626
tree decl
Definition tree-vectorizer.h:1633
tree element_type
Definition tree-vectorizer.h:1653
tree base
Definition tree-vectorizer.h:1636
tree offset_vectype
Definition tree-vectorizer.h:1650
tree alias_ptr
Definition tree-vectorizer.h:1640
int scale
Definition tree-vectorizer.h:1647
tree offset
Definition tree-vectorizer.h:1643
internal_fn ifn
Definition tree-vectorizer.h:1629
tree memory_type
Definition tree-vectorizer.h:1656
Definition gimple.h:352
Definition gimple.h:861
Definition gimple-iterator.h:26
Definition gimple.h:221
Definition gimple.h:461
Definition tree-data-ref.h:50
tree offset
Definition tree-data-ref.h:52
Definition hash-traits.h:122
static bool test(U *p)
Definition hash-traits.h:345
Definition tree-vectorizer.h:775
tree compare_type
Definition tree-vectorizer.h:795
unsigned int max_nscalars_per_iter
Definition tree-vectorizer.h:779
unsigned int factor
Definition tree-vectorizer.h:785
tree type
Definition tree-vectorizer.h:791
tree bias_adjusted_ctrl
Definition tree-vectorizer.h:802
vec< tree > controls
Definition tree-vectorizer.h:798
unsigned ncopies
Definition tree-vectorizer.h:485
tree_code code
Definition tree-vectorizer.h:487
scalar_cond_masked_key(tree t, unsigned ncopies_)
Definition tree-vectorizer.h:477
void get_cond_ops_from_tree(tree)
Definition tree-vectorizer.cc:1786
tree op0
Definition tree-vectorizer.h:488
tree op1
Definition tree-vectorizer.h:489
bool inverted_p
Definition tree-vectorizer.h:486
vec< tree > remain
Definition tree-vectorizer.h:1378
slp_instance_kind kind
Definition tree-vectorizer.h:1375
vec< stmt_vec_info > roots
Definition tree-vectorizer.h:1377
slp_root(slp_instance_kind kind_, vec< stmt_vec_info > stmts_, vec< stmt_vec_info > roots_, vec< tree > remain_=vNULL)
Definition tree-vectorizer.h:1372
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:1376
Definition tree-vectorizer.h:120
tree vectype
Definition tree-vectorizer.h:126
int misalign
Definition tree-vectorizer.h:127
slp_tree node
Definition tree-vectorizer.h:125
enum vect_cost_model_location where
Definition tree-vectorizer.h:123
enum vect_cost_for_stmt kind
Definition tree-vectorizer.h:122
stmt_vec_info stmt_info
Definition tree-vectorizer.h:124
int count
Definition tree-vectorizer.h:121
Definition tree-hash-traits.h:25
Definition hash-traits.h:75
Definition tree-vectorizer.h:806
hash_set< pair_hash< nofree_ptr_hash< tree_node >, int_hash< unsigned, 0 > > > mask_set
Definition tree-vectorizer.h:811
auto_vec< rgroup_controls > rgc_vec
Definition tree-vectorizer.h:814
bool is_empty() const
Definition tree-vectorizer.h:807
Definition vec.h:450
Definition tree-vectorizer.h:257
virtual ~vect_data()=default
Definition tree-vectorizer.h:276
int misalignment
Definition tree-vectorizer.h:283
tree decl
Definition tree-vectorizer.h:288
virtual ~vect_load_store_data()=default
vect_load_store_data()=default
internal_fn ifn
Definition tree-vectorizer.h:287
unsigned n_perms
Definition tree-vectorizer.h:292
poly_int64 poffset
Definition tree-vectorizer.h:285
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:281
vect_load_store_data(vect_load_store_data &&other)=default
tree strided_offset_vectype
Definition tree-vectorizer.h:290
union vect_load_store_data::@336230030006311336100276173266177172342371306160 gs
auto_vec< int > elsvals
Definition tree-vectorizer.h:291
dr_alignment_support alignment_support_scheme
Definition tree-vectorizer.h:282
internal_fn lanes_ifn
Definition tree-vectorizer.h:284
Definition tree-vectorizer.h:2672
gcond * inner_loop_cond
Definition tree-vectorizer.h:2677
auto_vec< gcond * > conds
Definition tree-vectorizer.h:2676
tree number_of_iterationsm1
Definition tree-vectorizer.h:2674
tree assumptions
Definition tree-vectorizer.h:2675
edge loop_exit
Definition tree-vectorizer.h:2678
tree number_of_iterations
Definition tree-vectorizer.h:2673
Definition tree-vectorizer.h:890
tree reduc_input
Definition tree-vectorizer.h:893
vect_reduc_info reduc_info
Definition tree-vectorizer.h:897
Definition tree-vectorizer.h:159
static void mark_deleted(value_type &s)
Definition tree-vectorizer.h:165
static bool is_empty(const value_type &s)
Definition tree-vectorizer.h:168
static bool is_deleted(const value_type &s)
Definition tree-vectorizer.h:167
static const bool empty_zero_p
Definition tree-vectorizer.h:163
static void mark_empty(value_type &s)
Definition tree-vectorizer.h:166
static hashval_t hash(const value_type &)
Definition tree-vect-slp.cc:8187
vect_scalar_ops_slice value_type
Definition tree-vectorizer.h:160
static bool equal(const value_type &, const compare_type &)
Definition tree-vect-slp.cc:8196
vect_scalar_ops_slice compare_type
Definition tree-vectorizer.h:161
Definition tree-vectorizer.h:141
unsigned int length
Definition tree-vectorizer.h:147
unsigned int start
Definition tree-vectorizer.h:146
bool all_same_p() const
Definition tree-vect-slp.cc:8178
tree op(unsigned int i) const
Definition tree-vectorizer.h:152
vec< tree > * ops
Definition tree-vectorizer.h:145
virtual ~vect_simd_clone_data()=default
vect_simd_clone_data(vect_simd_clone_data &&other)=default
vect_simd_clone_data()=default
auto_vec< tree > simd_clone_info
Definition tree-vectorizer.h:271
#define NULL
Definition system.h:50
#define gcc_assert(EXPR)
Definition system.h:814
#define gcc_unreachable()
Definition system.h:841
#define false
Definition system.h:888
#define gcc_checking_assert(EXPR)
Definition system.h:821
HOST_WIDE_INT estimated_poly_value(poly_int64 x, poly_value_estimate_kind kind=POLY_VALUE_LIKELY)
Definition target.h:337
struct gcc_target targetm
vect_cost_model_location
Definition target.h:211
@ vect_body
Definition target.h:213
@ vect_prologue
Definition target.h:212
@ vect_epilogue
Definition target.h:214
vect_cost_for_stmt
Definition target.h:189
@ scalar_stmt
Definition target.h:190
@ cond_branch_taken
Definition target.h:203
@ vec_to_scalar
Definition target.h:200
@ cond_branch_not_taken
Definition target.h:202
@ tcc_comparison
Definition tree-core.h:233
@ vec_kind
Definition tree-core.h:1035
#define DR_IS_WRITE(DR)
Definition tree-data-ref.h:192
#define DR_IS_READ(DR)
Definition tree-data-ref.h:191
#define DR_REF(DR)
Definition tree-data-ref.h:185
#define DR_INNERMOST(DR)
Definition tree-data-ref.h:203
static tree candidate(unsigned uid)
Definition tree-sra.cc:326
#define PHI_ARG_DEF_FROM_EDGE(PHI, E)
Definition tree-ssa-operands.h:84
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset)
Definition tree-vect-data-refs.cc:1303
vect_pattern_decl_t slp_patterns[]
Definition tree-vect-slp-patterns.cc:1703
_complex_operation
Definition tree-vect-slp-patterns.cc:119
size_t num__slp_patterns
Definition tree-vect-slp-patterns.cc:1714
dump_user_location_t vect_location
Definition tree-vectorizer.cc:88
stmt_vec_info vect_orig_stmt(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1962
#define LOOP_VINFO_COST_MODEL_THRESHOLD(L)
Definition tree-vectorizer.h:1251
bool vect_can_vectorize_without_simd_p(tree_code)
Definition tree-vect-loop.cc:8838
tree cse_and_gimplify_to_preheader(loop_vec_info, tree)
Definition tree-vect-loop.cc:950
int builtin_vectorization_cost(enum vect_cost_for_stmt type_of_cost, tree vectype, int misalign)
Definition tree-vectorizer.h:2033
loop_vec_info loop_vec_info_for_loop(class loop *loop)
Definition tree-vectorizer.h:1365
bool vect_is_extending_load(class vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2936
bool vect_transform_stmt(vec_info *, stmt_vec_info, gimple_stmt_iterator *, slp_tree, slp_instance)
Definition tree-vect-stmts.cc:12720
bool vect_is_simple_use(tree, vec_info *, enum vect_def_type *, stmt_vec_info *=NULL, gimple **=NULL)
Definition tree-vect-stmts.cc:13187
bool vect_rgroup_iv_might_wrap_p(loop_vec_info, rgroup_controls *)
Definition tree-vect-loop.cc:11673
tree vect_halve_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:10528
void dump_stmt_cost(FILE *, int, enum vect_cost_for_stmt, stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location)
Definition tree-vectorizer.cc:102
vec< unsigned > load_permutation_t
Definition tree-vectorizer.h:254
bool vect_slp_analyze_instance_alignment(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:3142
internal_fn vect_store_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:6045
tree vect_get_new_vect_var(tree, enum vect_var_kind, const char *)
Definition tree-vect-data-refs.cc:5311
void vect_mark_pattern_stmts(vec_info *, stmt_vec_info, gimple *, tree)
Definition tree-vect-patterns.cc:7010
class loop * vect_loop_versioning(loop_vec_info, gimple *)
Definition tree-vect-loop-manip.cc:4121
tree vect_get_slp_vect_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10542
opt_tree vect_get_mask_type_for_stmt(stmt_vec_info, unsigned int=0)
#define LOOP_VINFO_VECT_FACTOR(L)
Definition tree-vectorizer.h:1263
void vect_free_slp_instance(slp_instance)
Definition tree-vect-slp.cc:215
#define SLP_TREE_LANES(S)
Definition tree-vectorizer.h:447
bool vectorizable_early_exit(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:12296
bool vect_slp_function(function *)
Definition tree-vect-slp.cc:10075
std::pair< tree, tree > vec_object_pair
Definition tree-vectorizer.h:555
tree vect_gen_perm_mask_any(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9143
tree vect_setup_realignment(vec_info *, stmt_vec_info, tree, gimple_stmt_iterator *, tree *, enum dr_alignment_support, tree, class loop **)
Definition tree-vect-data-refs.cc:6121
_bb_vec_info * bb_vec_info
#define LOOP_VINFO_FULLY_MASKED_P(L)
Definition tree-vectorizer.h:1318
int vect_get_known_peeling_cost(loop_vec_info, int, int *, stmt_vector_for_cost *, stmt_vector_for_cost *, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:4020
opt_result vect_analyze_data_refs(vec_info *, bool *)
Definition tree-vect-data-refs.cc:5058
bool compatible_calls_p(gcall *, gcall *, bool)
Definition tree-vect-slp.cc:997
bool vect_transform_slp_perm_load(vec_info *, slp_tree, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, unsigned *, unsigned *=nullptr, bool=false)
tree vect_gen_while_not(gimple_seq *, tree, tree, tree)
Definition tree-vect-stmts.cc:14041
#define VECT_REDUC_INFO_TYPE(I)
Definition tree-vectorizer.h:876
tree vect_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2926
bool is_simple_and_all_uses_invariant(stmt_vec_info, loop_vec_info)
Definition tree-vect-stmts.cc:319
#define STMT_VINFO_GROUPED_ACCESS(S)
Definition tree-vectorizer.h:1691
void vect_schedule_slp(vec_info *, const vec< slp_instance > &)
Definition tree-vect-slp.cc:11965
tree vect_peel_nonlinear_iv_init(gimple_seq *, tree, tree, tree, enum vect_induction_op_type)
Definition tree-vect-loop.cc:8964
bool check_reduction_path(dump_user_location_t, loop_p, gphi *, tree, enum tree_code)
Definition tree-vect-loop.cc:3702
bool can_duplicate_and_interleave_p(vec_info *, unsigned int, tree, unsigned int *=NULL, tree *=NULL, tree *=NULL)
Definition tree-vect-slp.cc:427
auto_vec< unsigned, 16 > auto_load_permutation_t
Definition tree-vectorizer.h:255
#define SLP_TREE_VECTYPE(S)
Definition tree-vectorizer.h:445
tree get_vectype_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13010
#define LOOP_VINFO_LOOP(L)
Definition tree-vectorizer.h:1238
_complex_perm_kinds
Definition tree-vectorizer.h:2787
@ PERM_EVENEVEN
Definition tree-vectorizer.h:2792
@ PERM_TOP
Definition tree-vectorizer.h:2794
@ PERM_ODDEVEN
Definition tree-vectorizer.h:2790
@ PERM_EVENODD
Definition tree-vectorizer.h:2789
@ PERM_UNKNOWN
Definition tree-vectorizer.h:2788
@ PERM_ODDODD
Definition tree-vectorizer.h:2791
#define SLP_TREE_TYPE(S)
Definition tree-vectorizer.h:449
void vect_record_base_alignments(vec_info *)
Definition tree-vect-data-refs.cc:1384
vec_load_store_type
Definition tree-vectorizer.h:1432
@ VLS_LOAD
Definition tree-vectorizer.h:1433
@ VLS_STORE
Definition tree-vectorizer.h:1434
@ VLS_STORE_INVARIANT
Definition tree-vectorizer.h:1435
tree get_mask_type_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13085
pair_hash< tree_operand_hash, tree_operand_hash > tree_cond_mask_hash
Definition tree-vectorizer.h:550
_loop_vec_info * loop_vec_info
vect_reduc_info info_for_reduction(loop_vec_info, slp_tree)
Definition tree-vect-loop.cc:5118
tree vect_double_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:10538
vect_var_kind
Definition tree-vectorizer.h:36
@ vect_pointer_var
Definition tree-vectorizer.h:38
@ vect_scalar_var
Definition tree-vectorizer.h:39
@ vect_mask_var
Definition tree-vectorizer.h:40
@ vect_simple_var
Definition tree-vectorizer.h:37
int vect_get_place_in_interleaving_chain(stmt_vec_info, stmt_vec_info)
Definition tree-vect-slp.cc:398
class loop * vect_do_peeling(loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, tree *)
Definition tree-vect-loop-manip.cc:3135
void optimize_mask_stores(class loop *)
Definition tree-vect-loop.cc:11437
class _stmt_vec_info * stmt_vec_info
Definition tree-vectorizer.h:24
#define STMT_VINFO_DATA_REF(S)
Definition tree-vectorizer.h:1665
tree vect_get_loop_len(loop_vec_info, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:10725
bool vect_transform_cycle_phi(loop_vec_info, stmt_vec_info, slp_tree, slp_instance)
Definition tree-vect-loop.cc:8236
unsigned int vect_get_num_copies(vec_info *vinfo, slp_tree node)
Definition tree-vectorizer.h:2297
bool vect_is_store_elt_extraction(vect_cost_for_stmt kind, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2873
opt_result vect_get_vector_types_for_stmt(vec_info *, stmt_vec_info, tree *, tree *, unsigned int=0)
Definition tree-vect-stmts.cc:14065
const poly_uint64 dr_target_alignment(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2129
void vect_update_inits_of_drs(loop_vec_info, tree, tree_code)
Definition tree-vect-loop-manip.cc:2615
struct data_reference * dr_p
Definition tree-vectorizer.h:1470
bool vect_can_advance_ivs_p(loop_vec_info)
Definition tree-vect-loop-manip.cc:2211
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vect-data-refs.cc:1303
tree vect_init_vector(vec_info *, stmt_vec_info, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1203
void vect_copy_ref_info(tree, tree)
Definition tree-vect-data-refs.cc:5871
bool vect_transform_reduction(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree)
Definition tree-vect-loop.cc:7894
bool vectorizable_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:8449
bool vectorizable_lane_reducing(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:6790
tree vect_get_new_ssa_name(tree, enum vect_var_kind, const char *=NULL)
Definition tree-vect-data-refs.cc:5349
#define STMT_VINFO_VECTYPE(S)
Definition tree-vectorizer.h:1663
opt_result vect_prune_runtime_alias_test_list(loop_vec_info)
Definition tree-vect-data-refs.cc:4171
void vect_describe_gather_scatter_call(stmt_vec_info, gather_scatter_info *)
Definition tree-vect-data-refs.cc:4535
bool needs_fold_left_reduction_p(tree, code_helper)
Definition tree-vect-loop.cc:3463
bool vect_nop_conversion_p(stmt_vec_info)
Definition tree-vect-stmts.cc:5595
class vect_reduc_info_s * vect_reduc_info
Definition tree-vectorizer.h:873
void vect_finish_stmt_generation(vec_info *, stmt_vec_info, gimple *, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1328
#define LOOP_VINFO_NON_LINEAR_IV(L)
Definition tree-vectorizer.h:1279
unsigned HOST_WIDE_INT vect_max_vf(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2358
internal_fn vect_load_lanes_supported(tree, unsigned HOST_WIDE_INT, bool, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:6475
loop_vec_info vect_create_loop_vinfo(class loop *, vec_info_shared *, const vect_loop_form_info *, loop_vec_info=nullptr)
Definition tree-vect-loop.cc:1730
bool vect_stmt_dominates_stmt_p(gimple *, gimple *)
Definition tree-vectorizer.cc:793
tree vect_get_slp_scalar_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10524
bool vect_supportable_shift(vec_info *, enum tree_code, tree)
Definition tree-vect-stmts.cc:5773
void vect_optimize_slp(vec_info *)
Definition tree-vect-slp.cc:7840
void vect_iv_increment_position(edge, gimple_stmt_iterator *, bool *)
Definition tree-vect-loop-manip.cc:457
hash_set< scalar_cond_masked_key > scalar_cond_masked_set_type
Definition tree-vectorizer.h:545
tree vect_phi_initial_value(gphi *phi)
Definition tree-vectorizer.h:1932
opt_pointer_wrapper< loop_vec_info > opt_loop_vec_info
Definition tree-vectorizer.h:1362
bool vectorizable_reduction(loop_vec_info, stmt_vec_info, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:6950
unsigned record_stmt_cost(stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location)
Definition tree-vect-stmts.cc:110
bool vect_emulated_vector_p(tree)
Definition tree-vect-loop.cc:8827
tree get_same_sized_vectype(tree, tree)
Definition tree-vect-stmts.cc:13121
peeling_support
Definition tree-vectorizer.h:61
@ peeling_known_supported
Definition tree-vectorizer.h:62
@ peeling_maybe_supported
Definition tree-vectorizer.h:63
@ peeling_unsupported
Definition tree-vectorizer.h:64
gimple * vect_gimple_build(tree, code_helper, tree, tree=NULL_TREE)
Definition tree-vect-patterns.cc:7332
void add_stmt_costs(vector_costs *costs, stmt_vector_for_cost *cost_vec)
Definition tree-vectorizer.h:2101
void vect_finish_replace_stmt(vec_info *, stmt_vec_info, gimple *)
Definition tree-vect-stmts.cc:1312
tree prepare_vec_mask(loop_vec_info, tree, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1605
void vect_slp_fini(void)
Definition tree-vect-slp.cc:80
#define LOOP_VINFO_NITERS_KNOWN_P(L)
Definition tree-vectorizer.h:1349
stmt_vec_info get_later_stmt(stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
Definition tree-vectorizer.h:1972
vec< std::pair< unsigned, unsigned > > lane_permutation_t
Definition tree-vectorizer.h:252
tree vect_gen_loop_len_mask(loop_vec_info, gimple_stmt_iterator *, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:10792
widest_int vect_iv_limit_for_partial_vectors(loop_vec_info loop_vinfo)
Definition tree-vect-loop.cc:11632
void vect_get_vec_defs(vec_info *, slp_tree, tree, vec< tree > *, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL)
Definition tree-vect-stmts.cc:1265
void vect_gather_slp_loads(vec_info *)
Definition tree-vect-slp.cc:7859
vect_partial_vector_style
Definition tree-vectorizer.h:464
@ vect_partial_vectors_none
Definition tree-vectorizer.h:465
@ vect_partial_vectors_len
Definition tree-vectorizer.h:468
@ vect_partial_vectors_avx512
Definition tree-vectorizer.h:467
@ vect_partial_vectors_while_ult
Definition tree-vectorizer.h:466
bool vect_gather_scatter_fn_p(vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4443
class _slp_instance * slp_instance
bool vect_maybe_update_slp_op_vectype(slp_tree, tree)
Definition tree-vect-stmts.cc:13338
#define STMT_VINFO_REDUC_IDX(S)
Definition tree-vectorizer.h:1669
class loop * slpeel_tree_duplicate_loop_to_edge_cfg(class loop *, edge, class loop *, edge, edge, edge *, bool=true, vec< basic_block > *=NULL)
Definition tree-vect-loop-manip.cc:1478
unsigned int vect_vf_for_cost(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2340
unsigned int vect_nunits_for_cost(tree vec_type)
Definition tree-vectorizer.h:2350
bool vect_get_range_info(tree, wide_int *, wide_int *)
Definition tree-vect-patterns.cc:69
bool vect_grouped_load_supported(tree, bool, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:6358
#define STMT_VINFO_IN_PATTERN_P(S)
Definition tree-vectorizer.h:1687
tree vect_create_addr_base_for_vector_ref(vec_info *, stmt_vec_info, gimple_seq *, tree)
Definition tree-vect-data-refs.cc:5424
stmt_vec_info vect_find_last_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3529
tree vect_get_main_loop_result(loop_vec_info, tree, tree)
Definition tree-vect-loop-manip.cc:3026
#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L)
Definition tree-vectorizer.h:1278
#define STMT_VINFO_DR_INFO(S)
Definition tree-vectorizer.h:1684
hash_map< slp_node_hash, bool > slp_compat_nodes_map_t
Definition tree-vectorizer.h:2804
tree get_related_vectype_for_scalar_type(machine_mode, tree, poly_uint64=0)
Definition tree-vect-stmts.cc:12892
bool lane_reducing_op_p(code_helper code)
Definition tree-vectorizer.h:2393
bool nested_in_vect_loop_p(class loop *loop, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1921
bool vect_slp_analyze_instance_dependence(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:1164
bool vect_detect_hybrid_slp(loop_vec_info)
auto_vec< std::pair< data_reference *, tree > > drs_init_vec
Definition tree-vectorizer.h:819
edge vec_init_loop_exit_info(class loop *)
Definition tree-vect-loop.cc:745
opt_result vect_analyze_data_ref_accesses(vec_info *, vec< int > *)
Definition tree-vect-data-refs.cc:3636
int vect_get_stmt_cost(enum vect_cost_for_stmt type_of_cost)
Definition tree-vectorizer.h:2043
opt_result vect_analyze_stmt(vec_info *, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:12570
dump_user_location_t find_loop_location(class loop *)
Definition tree-vect-loop-manip.cc:2070
bool vectorizable_recurr(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:8658
bool vect_slp_analyze_operations(vec_info *)
Definition tree-vect-slp.cc:8723
tree neutral_op_for_reduction(tree, code_helper, tree, bool=true)
Definition tree-vect-loop.cc:3405
void vect_free_slp_tree(slp_tree)
Definition tree-vect-slp.cc:175
unsigned vectorize_loops(void)
bool vect_can_force_dr_alignment_p(const_tree, poly_uint64)
Definition tree-vect-data-refs.cc:6504
tree bump_vector_ptr(vec_info *, tree, gimple *, gimple_stmt_iterator *, stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:5807
bool vect_apply_runtime_profitability_check_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2383
unsigned int vect_get_num_vectors(poly_uint64 nunits, tree vectype)
Definition tree-vectorizer.h:2288
#define DR_TARGET_ALIGNMENT(DR)
Definition tree-vectorizer.h:2135
void vect_free_loop_info_assumptions(class loop *)
Definition tree-vectorizer.cc:855
void set_dr_target_alignment(dr_vec_info *dr_info, poly_uint64 val)
Definition tree-vectorizer.h:2167
stmt_vec_info vect_stmt_to_vectorize(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1999
bool supportable_indirect_convert_operation(code_helper, tree, tree, vec< std::pair< tree, tree_code > > &, tree=NULL_TREE, slp_tree=NULL)
Definition tree-vect-stmts.cc:13882
opt_loop_vec_info vect_analyze_loop(class loop *, gimple *, vec_info_shared *)
Definition tree-vect-loop.cc:2992
void dr_set_safe_speculative_read_required(stmt_vec_info stmt_info, bool requires_alignment)
Definition tree-vectorizer.h:2154
#define STMT_VINFO_RELATED_STMT(S)
Definition tree-vectorizer.h:1688
bool is_pattern_stmt_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1953
#define MAX_VECTORIZATION_FACTOR
Definition tree-vectorizer.h:1906
int vect_pow2(int x)
Definition tree-vectorizer.h:2020
bool reduction_fn_for_scalar_code(code_helper, internal_fn *)
Definition tree-vect-loop.cc:3343
void vect_pattern_recog(vec_info *)
Definition tree-vect-patterns.cc:7296
int vect_slp_child_index_for_operand(const gimple *, int op, bool)
Definition tree-vect-slp.cc:617
vect_induction_op_type
Definition tree-vectorizer.h:83
@ vect_step_op_add
Definition tree-vectorizer.h:84
@ vect_step_op_shl
Definition tree-vectorizer.h:87
@ vect_step_op_mul
Definition tree-vectorizer.h:86
@ vect_step_op_neg
Definition tree-vectorizer.h:85
@ vect_step_op_shr
Definition tree-vectorizer.h:88
void vect_set_loop_condition(class loop *, edge, loop_vec_info, tree, tree, tree, bool)
Definition tree-vect-loop-manip.cc:1393
bool lane_reducing_stmt_p(gimple *stmt)
Definition tree-vectorizer.h:2401
void vect_prepare_for_masked_peels(loop_vec_info)
Definition tree-vect-loop-manip.cc:2645
void vect_get_store_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:1004
stmt_vec_info vect_find_first_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3547
enum _complex_perm_kinds complex_perm_kinds_t
bool vect_is_integer_truncation(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2960
auto_vec< std::pair< unsigned, unsigned >, 16 > auto_lane_permutation_t
Definition tree-vectorizer.h:253
bool vect_use_loop_mask_for_alignment_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2271
tree vect_build_loop_niters(loop_vec_info, bool *=NULL)
Definition tree-vect-loop-manip.cc:2691
bool vect_get_loop_mask_type(loop_vec_info)
void vect_update_max_nunits(poly_uint64 *max_nunits, poly_uint64 nunits)
Definition tree-vectorizer.h:2316
slp_instance_kind
Definition tree-vectorizer.h:386
@ slp_inst_kind_ctor
Definition tree-vectorizer.h:391
@ slp_inst_kind_reduc_group
Definition tree-vectorizer.h:388
@ slp_inst_kind_store
Definition tree-vectorizer.h:387
@ slp_inst_kind_reduc_chain
Definition tree-vectorizer.h:389
@ slp_inst_kind_gcond
Definition tree-vectorizer.h:392
@ slp_inst_kind_bb_reduc
Definition tree-vectorizer.h:390
enum vect_cost_model loop_cost_model(loop_p loop)
Definition tree-vectorizer.h:2250
unsigned int vect_known_alignment_in_bytes(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vectorizer.h:2196
#define DR_MISALIGNMENT_UNKNOWN
Definition tree-vectorizer.h:2113
bool mat_gather_scatter_p(vect_memory_access_type mat)
Definition tree-vectorizer.h:215
hash_map< slp_tree, complex_perm_kinds_t > slp_tree_to_load_perm_map_t
Definition tree-vectorizer.h:2799
opt_result vect_enhance_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2332
bool vect_is_reduction(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2882
#define LOOP_VINFO_EARLY_BREAKS(L)
Definition tree-vectorizer.h:1295
enum dr_alignment_support vect_supportable_dr_alignment(vec_info *, dr_vec_info *, tree, int, bool=false)
Definition tree-vect-data-refs.cc:6528
bool vectorizable_induction(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9440
#define SLP_TREE_REDUC_IDX(S)
Definition tree-vectorizer.h:452
innermost_loop_behavior * vect_dr_behavior(vec_info *vinfo, dr_vec_info *dr_info)
Definition tree-vectorizer.h:2212
tree perm_mask_for_reverse(tree)
Definition tree-vect-stmts.cc:1791
vect_memory_access_type SLP_TREE_MEMORY_ACCESS_TYPE(slp_tree node)
Definition tree-vectorizer.h:456
stmt_vec_info_type
Definition tree-vectorizer.h:225
@ comparison_vec_info_type
Definition tree-vectorizer.h:235
@ condition_vec_info_type
Definition tree-vectorizer.h:234
@ type_conversion_vec_info_type
Definition tree-vectorizer.h:240
@ load_vec_info_type
Definition tree-vectorizer.h:227
@ loop_exit_ctrl_vec_info_type
Definition tree-vectorizer.h:245
@ permute_info_type
Definition tree-vectorizer.h:246
@ recurr_info_type
Definition tree-vectorizer.h:244
@ op_vec_info_type
Definition tree-vectorizer.h:230
@ reduc_vec_info_type
Definition tree-vectorizer.h:236
@ assignment_vec_info_type
Definition tree-vectorizer.h:233
@ undef_vec_info_type
Definition tree-vectorizer.h:226
@ shift_vec_info_type
Definition tree-vectorizer.h:229
@ type_demotion_vec_info_type
Definition tree-vectorizer.h:239
@ store_vec_info_type
Definition tree-vectorizer.h:228
@ cycle_phi_info_type
Definition tree-vectorizer.h:241
@ phi_info_type
Definition tree-vectorizer.h:243
@ lc_phi_info_type
Definition tree-vectorizer.h:242
@ type_promotion_vec_info_type
Definition tree-vectorizer.h:238
@ call_simd_clone_vec_info_type
Definition tree-vectorizer.h:232
@ call_vec_info_type
Definition tree-vectorizer.h:231
@ induc_vec_info_type
Definition tree-vectorizer.h:237
void vect_get_slp_defs(slp_tree, vec< tree > *)
Definition tree-vect-slp.cc:10550
void vect_record_loop_len(loop_vec_info, vec_loop_lens *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:10690
struct _slp_tree * slp_tree
Definition tree-vectorizer.h:25
gimple_seq vect_gen_len(tree, tree, tree, tree)
Definition tree-vect-stmts.cc:14229
bool dr_safe_speculative_read_required(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2140
bool vect_check_gather_scatter(stmt_vec_info, tree, loop_vec_info, gather_scatter_info *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4562
tree vect_embedded_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2911
gimple * vect_loop_vectorized_call(class loop *, gcond **cond=NULL)
Definition tree-vectorizer.cc:870
void duplicate_and_interleave(vec_info *, gimple_seq *, tree, const vec< tree > &, unsigned int, vec< tree > &)
Definition tree-vect-slp.cc:10207
bool vect_grouped_store_supported(tree, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:5927
void vect_get_load_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool)
Definition tree-vect-stmts.cc:1056
vect_def_type
Definition tree-vectorizer.h:68
@ vect_induction_def
Definition tree-vectorizer.h:73
@ vect_uninitialized_def
Definition tree-vectorizer.h:69
@ vect_nested_cycle
Definition tree-vectorizer.h:76
@ vect_constant_def
Definition tree-vectorizer.h:70
@ vect_external_def
Definition tree-vectorizer.h:71
@ vect_condition_def
Definition tree-vectorizer.h:78
@ vect_internal_def
Definition tree-vectorizer.h:72
@ vect_reduction_def
Definition tree-vectorizer.h:74
@ vect_double_reduction_def
Definition tree-vectorizer.h:75
@ vect_first_order_recurrence
Definition tree-vectorizer.h:77
@ vect_unknown_def_type
Definition tree-vectorizer.h:79
opt_result vect_analyze_data_ref_dependences(loop_vec_info, unsigned int *)
Definition tree-vect-data-refs.cc:887
vect_relevant
Definition tree-vectorizer.h:1399
@ vect_used_in_outer
Definition tree-vectorizer.h:1409
@ vect_used_in_scope
Definition tree-vectorizer.h:1419
@ vect_unused_in_scope
Definition tree-vectorizer.h:1400
@ vect_used_by_reduction
Definition tree-vectorizer.h:1417
@ vect_used_in_outer_by_reduction
Definition tree-vectorizer.h:1406
@ vect_used_only_live
Definition tree-vectorizer.h:1403
tree vect_get_mask_load_else(int, tree)
Definition tree-vect-stmts.cc:2691
bool is_loop_header_bb_p(basic_block bb)
Definition tree-vectorizer.h:2009
opt_result vect_analyze_loop_form(class loop *, gimple *, vect_loop_form_info *)
Definition tree-vect-loop.cc:1523
#define STMT_VINFO_DR_WRT_VEC_LOOP(S)
Definition tree-vectorizer.h:1671
bool vectorizable_live_operation(vec_info *, stmt_vec_info, slp_tree, slp_instance, int, bool, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:10124
slp_tree vect_create_new_slp_node(unsigned, tree_code)
Definition tree-vect-slp.cc:230
opt_result vect_analyze_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:3074
bool unlimited_cost_model(loop_p loop)
Definition tree-vectorizer.h:2261
opt_result vect_mark_stmts_to_be_vectorized(loop_vec_info, bool *)
Definition tree-vect-stmts.cc:698
tree vect_gen_while(gimple_seq *, tree, tree, tree, const char *=nullptr)
Definition tree-vect-stmts.cc:14017
tree vect_get_store_rhs(stmt_vec_info)
Definition tree-vect-stmts.cc:1872
class loop * vect_transform_loop(loop_vec_info, gimple *)
Definition tree-vect-loop.cc:11044
void vect_slp_init(void)
Definition tree-vect-slp.cc:74
unsigned int vect_get_scalar_dr_size(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2374
opt_result vect_determine_partial_vectors_and_peeling(loop_vec_info)
Definition tree-vect-loop.cc:2113
bool ref_within_array_bound(gimple *, tree)
Definition tree-if-conv.cc:852
dr_alignment_support
Definition tree-vectorizer.h:51
@ dr_unaligned_supported
Definition tree-vectorizer.h:53
@ dr_explicit_realign_optimized
Definition tree-vectorizer.h:55
@ dr_aligned
Definition tree-vectorizer.h:56
@ dr_explicit_realign
Definition tree-vectorizer.h:54
@ dr_unaligned_unsupported
Definition tree-vectorizer.h:52
int compare_step_with_zero(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:1780
bool supportable_narrowing_operation(code_helper, tree, tree, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:13709
operation_type
Definition tree-vectorizer.h:44
@ ternary_op
Definition tree-vectorizer.h:47
@ unary_op
Definition tree-vectorizer.h:45
@ binary_op
Definition tree-vectorizer.h:46
void vect_record_loop_mask(loop_vec_info, vec_loop_masks *, unsigned int, tree, tree)
Definition tree-vect-loop.cc:10550
vect_memory_access_type
Definition tree-vectorizer.h:175
@ VMAT_STRIDED_SLP
Definition tree-vectorizer.h:204
@ VMAT_GATHER_SCATTER_LEGACY
Definition tree-vectorizer.h:207
@ VMAT_GATHER_SCATTER_EMULATED
Definition tree-vectorizer.h:209
@ VMAT_INVARIANT
Definition tree-vectorizer.h:179
@ VMAT_UNINITIALIZED
Definition tree-vectorizer.h:176
@ VMAT_ELEMENTWISE
Definition tree-vectorizer.h:198
@ VMAT_GATHER_SCATTER_IFN
Definition tree-vectorizer.h:208
@ VMAT_CONTIGUOUS_DOWN
Definition tree-vectorizer.h:187
@ VMAT_CONTIGUOUS
Definition tree-vectorizer.h:182
@ VMAT_LOAD_STORE_LANES
Definition tree-vectorizer.h:194
@ VMAT_CONTIGUOUS_REVERSE
Definition tree-vectorizer.h:191
bool vect_chooses_same_modes_p(vec_info *, machine_mode)
Definition tree-vect-stmts.cc:13139
bool slpeel_can_duplicate_loop_p(const class loop *, const_edge, const_edge)
Definition tree-vect-loop-manip.cc:2038
tree vect_get_loop_mask(loop_vec_info, gimple_stmt_iterator *, vec_loop_masks *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:10572
slp_vect_type
Definition tree-vectorizer.h:1425
@ not_vect
Definition tree-vectorizer.h:1426
@ pure_slp
Definition tree-vectorizer.h:1427
vect_reduction_type
Definition tree-vectorizer.h:92
@ TREE_CODE_REDUCTION
Definition tree-vectorizer.h:93
@ FOLD_LEFT_REDUCTION
Definition tree-vectorizer.h:111
@ EXTRACT_LAST_REDUCTION
Definition tree-vectorizer.h:103
@ INTEGER_INDUC_COND_REDUCTION
Definition tree-vectorizer.h:95
@ CONST_COND_REDUCTION
Definition tree-vectorizer.h:96
@ COND_REDUCTION
Definition tree-vectorizer.h:94
bool supportable_widening_operation(vec_info *, code_helper, stmt_vec_info, tree, tree, code_helper *, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:13378
auto_vec< rgroup_controls > vec_loop_lens
Definition tree-vectorizer.h:817
void set_dr_misalignment(dr_vec_info *dr_info, int val)
Definition tree-vectorizer.h:2117
bool vect_use_mask_type_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1944
pair_hash< nofree_ptr_hash< _slp_tree >, nofree_ptr_hash< _slp_tree > > slp_node_hash
Definition tree-vectorizer.h:2803
hash_map< tree_operand_hash, std::pair< stmt_vec_info, innermost_loop_behavior * > > vec_base_alignments
Definition tree-vectorizer.h:136
unsigned add_stmt_cost(vector_costs *costs, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where)
Definition tree-vectorizer.h:2063
tree vect_get_smallest_scalar_type(stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:161
vec< stmt_info_for_cost > stmt_vector_for_cost
Definition tree-vectorizer.h:130
bool known_alignment_for_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2187
void vect_remove_stores(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:12862
#define DR_GROUP_FIRST_ELEMENT(S)
Definition tree-vectorizer.h:1704
hash_set< tree_cond_mask_hash > vec_cond_masked_set_type
Definition tree-vectorizer.h:551
opt_result vect_find_stmt_data_reference(loop_p, gimple *, vec< data_reference_p > *, vec< int > *, int)
Definition tree-vect-data-refs.cc:4883
tree get_dr_vinfo_offset(vec_info *vinfo, dr_vec_info *dr_info, bool check_outer=false)
Definition tree-vectorizer.h:2228
bool vect_slp_if_converted_bb(basic_block bb, loop_p orig_loop)
Definition tree-vect-slp.cc:10064
bool vectorizable_slp_permutation(vec_info *, gimple_stmt_iterator *, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-slp.cc:11350
tree vect_gen_perm_mask_checked(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9158
opt_result vect_analyze_slp(vec_info *, unsigned, bool)
Definition tree-vect-slp.cc:5173
vect_pattern *(* vect_pattern_decl_t)(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
Definition tree-vectorizer.h:2853
tree vect_create_destination_var(tree, tree)
Definition tree-vect-data-refs.cc:5893
bool vect_transform_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:8489
tree vect_create_data_ref_ptr(vec_info *, stmt_vec_info, tree, class loop *, tree, tree *, gimple_stmt_iterator *, gimple **, bool, tree=NULL_TREE)
Definition tree-vect-data-refs.cc:5547
int vect_reduc_type(vec_info *vinfo, slp_tree node)
Definition tree-vectorizer.h:2897
void vect_gen_vector_loop_niters(loop_vec_info, tree, tree *, tree *, bool)
Definition tree-vect-loop-manip.cc:2782
bool vectorizable_phi(bb_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:8516
bool vect_make_slp_decision(loop_vec_info)
Definition tree-vect-slp.cc:7917
bool aligned_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2177
unsigned HOST_WIDE_INT tree_to_uhwi(const_tree t)
Definition tree.cc:6671
#define TYPE_ALIGN_UNIT(NODE)
Definition tree.h:2454
poly_uint64 TYPE_VECTOR_SUBPARTS(const_tree node)
Definition tree.h:4365
#define TYPE_PRECISION(NODE)
Definition tree.h:2360
#define TREE_OPERAND(NODE, I)
Definition tree.h:1319
#define sizetype
Definition tree.h:5242
#define TREE_CODE_CLASS(CODE)
Definition tree.h:203
#define TYPE_SIZE_UNIT(NODE)
Definition tree.h:2357
#define CONVERT_EXPR_CODE_P(CODE)
Definition tree.h:554
#define TREE_TYPE(NODE)
Definition tree.h:513
#define INTEGRAL_TYPE_P(TYPE)
Definition tree.h:614
#define COMPARISON_CLASS_P(NODE)
Definition tree.h:257
#define NULL_TREE
Definition tree.h:318
#define FOR_EACH_VEC_ELT(V, I, P)
Definition vec.h:1895
constexpr vnull vNULL
Definition vec.h:569
generic_wide_int< widest_int_storage< WIDEST_INT_MAX_PRECISION > > widest_int
Definition wide-int.h:345
generic_wide_int< wide_int_storage > wide_int
Definition wide-int.h:343