GCC Middle and Back End API Reference
tree-vectorizer.h
Go to the documentation of this file.
1/* Vectorizer
2 Copyright (C) 2003-2025 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4
5This file is part of GCC.
6
7GCC is free software; you can redistribute it and/or modify it under
8the terms of the GNU General Public License as published by the Free
9Software Foundation; either version 3, or (at your option) any later
10version.
11
12GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13WARRANTY; without even the implied warranty of MERCHANTABILITY or
14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15for more details.
16
17You should have received a copy of the GNU General Public License
18along with GCC; see the file COPYING3. If not see
19<http://www.gnu.org/licenses/>. */
20
21#ifndef GCC_TREE_VECTORIZER_H
22#define GCC_TREE_VECTORIZER_H
23
25typedef struct _slp_tree *slp_tree;
26
27#include "tree-data-ref.h"
28#include "tree-hash-traits.h"
29#include "target.h"
30#include "internal-fn.h"
31#include "tree-ssa-operands.h"
32#include "gimple-match.h"
33#include "dominance.h"
34
35/* Used for naming of new temporaries. */
42
43/* Defines type of operation. */
49
50/* Define type of available alignment support. */
58
59/* Define type of peeling support to indicate how peeling for alignment can help
60 make vectorization supported. */
66
67/* Define type of def-use cross-iteration cycle. */
81
82/* Define operation type of linear/non-linear induction variable. */
90
91/* Define type of reduction. */
97
98 /* Retain a scalar phi and use a FOLD_EXTRACT_LAST within the loop
99 to implement:
100
101 for (int i = 0; i < VF; ++i)
102 res = cond[i] ? val[i] : res; */
104
105 /* Use a folding reduction within the loop to implement:
106
107 for (int i = 0; i < VF; ++i)
108 res = res OP val[i];
109
110 (with no reassocation). */
112};
113
114#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
115 || ((D) == vect_double_reduction_def) \
116 || ((D) == vect_nested_cycle))
117
118/* Structure to encapsulate information about a group of like
119 instructions to be presented to the target cost model. */
129
131
132/* Maps base addresses to an innermost_loop_behavior and the stmt it was
133 derived from that gives the maximum known alignment for that base. */
135 std::pair<stmt_vec_info, innermost_loop_behavior *> >
137
138/* Represents elements [START, START + LENGTH) of cyclical array OPS*
139 (i.e. OPS repeated to give at least START + LENGTH elements) */
141{
142 tree op (unsigned int i) const;
143 bool all_same_p () const;
144
146 unsigned int start;
147 unsigned int length;
148};
149
150/* Return element I of the slice. */
151inline tree
152vect_scalar_ops_slice::op (unsigned int i) const
153{
154 return (*ops)[(i + start) % ops->length ()];
155}
156
157/* Hash traits for vect_scalar_ops_slice. */
158struct vect_scalar_ops_slice_hash : typed_noop_remove<vect_scalar_ops_slice>
159{
162
163 static const bool empty_zero_p = true;
164
165 static void mark_deleted (value_type &s) { s.length = ~0U; }
166 static void mark_empty (value_type &s) { s.length = 0; }
167 static bool is_deleted (const value_type &s) { return s.length == ~0U; }
168 static bool is_empty (const value_type &s) { return s.length == 0; }
169 static hashval_t hash (const value_type &);
170 static bool equal (const value_type &, const compare_type &);
171};
172
173/* Describes how we're going to vectorize an individual load or store,
174 or a group of loads or stores. */
177
178 /* An access to an invariant address. This is used only for loads. */
180
181 /* A simple contiguous access. */
183
184 /* A contiguous access that goes down in memory rather than up,
185 with no additional permutation. This is used only for stores
186 of invariants. */
188
189 /* A simple contiguous access in which the elements need to be reversed
190 after loading or before storing. */
192
193 /* An access that uses IFN_LOAD_LANES or IFN_STORE_LANES. */
195
196 /* An access in which each scalar element is loaded or stored
197 individually. */
199
200 /* A hybrid of VMAT_CONTIGUOUS and VMAT_ELEMENTWISE, used for grouped
201 SLP accesses. Each unrolled iteration uses a contiguous load
202 or store for the whole group, but the groups from separate iterations
203 are combined in the same way as for VMAT_ELEMENTWISE. */
205
206 /* The access uses gather loads or scatter stores. */
210};
211
212/* Returns whether MAT is any of the VMAT_GATHER_SCATTER_* kinds. */
213
214inline bool
221
222/*-----------------------------------------------------------------*/
223/* Info on vectorized defs. */
224/*-----------------------------------------------------------------*/
248
249/************************************************************************
250 SLP
251 ************************************************************************/
256
257struct vect_data {
258 virtual ~vect_data () = default;
259};
260
261/* Analysis data from vectorizable_simd_clone_call for
262 call_simd_clone_vec_info_type. */
264 virtual ~vect_simd_clone_data () = default;
267
268 /* Selected SIMD clone's function info. First vector element
269 is SIMD clone's function decl, followed by a pair of trees (base + step)
270 for linear arguments (pair of NULLs for other arguments). */
272};
273
274/* Analysis data from vectorizable_load and vectorizable_store for
275 load_vec_info_type and store_vec_info_type. */
279 virtual ~vect_load_store_data () = default;
280
284 internal_fn lanes_ifn; // VMAT_LOAD_STORE_LANES
286 union {
287 internal_fn ifn; // VMAT_GATHER_SCATTER_IFN
288 tree decl; // VMAT_GATHER_SCATTER_DECL
289 } gs;
290 tree strided_offset_vectype; // VMAT_GATHER_SCATTER_IFN, originally strided
291 tree ls_type; // VMAT_GATHER_SCATTER_IFN
293 /* True if the load requires a load permutation. */
294 bool slp_perm; // SLP_TREE_LOAD_PERMUTATION
295 unsigned n_perms; // SLP_TREE_LOAD_PERMUTATION
296};
297
298/* A computation tree of an SLP instance. Each node corresponds to a group of
299 stmts to be packed in a SIMD stmt. */
300struct _slp_tree {
301 _slp_tree ();
302 ~_slp_tree ();
303
304 void push_vec_def (gimple *def);
305 void push_vec_def (tree def) { vec_defs.quick_push (def); }
306
307 /* Nodes that contain def-stmts of this node statements operands. */
309
310 /* A group of scalar stmts to be vectorized together. */
312 /* A group of scalar operands to be vectorized together. */
314 /* The representative that should be used for analysis and
315 code generation. */
317
318 struct {
319 /* SLP cycle the node resides in, or -1. */
320 int id;
321 /* The SLP operand index with the edge on the SLP cycle, or -1. */
324
325 /* Load permutation relative to the stores, NULL if there is no
326 permutation. */
328 /* Lane permutation of the operands scalar lanes encoded as pairs
329 of { operand number, lane number }. The number of elements
330 denotes the number of output lanes. */
332
334 /* Vectorized defs. */
336
337 /* Reference count in the SLP graph. */
338 unsigned int refcnt;
339 /* The maximum number of vector elements for the subtree rooted
340 at this node. */
342 /* The DEF type of this node. */
344 /* The number of scalar lanes produced by this node. */
345 unsigned int lanes;
346 /* The operation of this node. */
348 /* For gather/scatter memory operations the scale each offset element
349 should be multiplied by before being added to the base. */
351 /* For gather/scatter memory operations the loop-invariant base value. */
353 /* Whether uses of this load or feeders of this store are suitable
354 for load/store-lanes. */
356 /* For BB vect, flag to indicate this load node should be vectorized
357 as to avoid STLF fails because of related stores. */
359
361
362 /* The kind of operation as determined by analysis and optional
363 kind specific data. */
366
367 template <class T>
368 T& get_data (T& else_) { return data ? *static_cast <T *> (data) : else_; }
369
370 /* If not NULL this is a cached failed SLP discovery attempt with
371 the lanes that failed during SLP discovery as 'false'. This is
372 a copy of the matches array. */
373 bool *failed;
374
375 /* Allocate from slp_tree_pool. */
376 static void *operator new (size_t);
377
378 /* Return memory to slp_tree_pool. */
379 static void operator delete (void *, size_t);
380
381 /* Linked list of nodes to release when we free the slp_tree_pool. */
384};
385
386/* The enum describes the type of operations that an SLP instance
387 can perform. */
388
397
398/* SLP instance is a sequence of stmts in a loop that can be packed into
399 SIMD stmts. */
400typedef class _slp_instance {
401public:
402 /* The root of SLP tree. */
404
405 /* For vector constructors, the constructor stmt that the SLP tree is built
406 from, NULL otherwise. */
408
409 /* For slp_inst_kind_bb_reduc the defs that were not vectorized, NULL
410 otherwise. */
412
413 /* The group of nodes that contain loads of this SLP instance. */
415
416 /* The SLP node containing the reduction PHIs. */
418
419 /* Vector cost of this entry to the SLP graph. */
421
422 /* If this instance is the main entry of a subgraph the set of
423 entries into the same subgraph, including itself. */
425
426 /* The type of operation the SLP instance is performing. */
428
431
432
433/* Access Functions. */
434#define SLP_INSTANCE_TREE(S) (S)->root
435#define SLP_INSTANCE_LOADS(S) (S)->loads
436#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
437#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
438#define SLP_INSTANCE_KIND(S) (S)->kind
439
440#define SLP_TREE_CHILDREN(S) (S)->children
441#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
442#define SLP_TREE_SCALAR_OPS(S) (S)->ops
443#define SLP_TREE_REF_COUNT(S) (S)->refcnt
444#define SLP_TREE_VEC_DEFS(S) (S)->vec_defs
445#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
446#define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
447#define SLP_TREE_DEF_TYPE(S) (S)->def_type
448#define SLP_TREE_VECTYPE(S) (S)->vectype
449#define SLP_TREE_REPRESENTATIVE(S) (S)->representative
450#define SLP_TREE_LANES(S) (S)->lanes
451#define SLP_TREE_CODE(S) (S)->code
452#define SLP_TREE_TYPE(S) (S)->type
453#define SLP_TREE_GS_SCALE(S) (S)->gs_scale
454#define SLP_TREE_GS_BASE(S) (S)->gs_base
455#define SLP_TREE_REDUC_IDX(S) (S)->cycle_info.reduc_idx
456#define SLP_TREE_PERMUTE_P(S) ((S)->code == VEC_PERM_EXPR)
457
460{
463 return static_cast<vect_load_store_data *> (node->data)->memory_access_type;
464 return VMAT_UNINITIALIZED;
465}
466
473
474/* Key for map that records association between
475 scalar conditions and corresponding loop mask, and
476 is populated by vect_record_loop_mask. */
477
479{
480 scalar_cond_masked_key (tree t, unsigned ncopies_)
481 : ncopies (ncopies_)
482 {
484 }
485
487
488 unsigned ncopies;
493};
494
495template<>
497{
500
501 static inline hashval_t
503 {
505 h.add_int (v.code);
506 inchash::add_expr (v.op0, h, 0);
507 inchash::add_expr (v.op1, h, 0);
508 h.add_int (v.ncopies);
509 h.add_flag (v.inverted_p);
510 return h.end ();
511 }
512
513 static inline bool
515 {
516 return (existing.ncopies == candidate.ncopies
517 && existing.code == candidate.code
518 && existing.inverted_p == candidate.inverted_p
519 && operand_equal_p (existing.op0, candidate.op0, 0)
520 && operand_equal_p (existing.op1, candidate.op1, 0));
521 }
522
523 static const bool empty_zero_p = true;
524
525 static inline void
527 {
528 v.ncopies = 0;
529 v.inverted_p = false;
530 }
531
532 static inline bool
534 {
535 return v.ncopies == 0;
536 }
537
538 static inline void mark_deleted (value_type &) {}
539
540 static inline bool is_deleted (const value_type &)
541 {
542 return false;
543 }
544
545 static inline void remove (value_type &) {}
546};
547
549
550/* Key and map that records association between vector conditions and
551 corresponding loop mask, and is populated by prepare_vec_mask. */
552
555
556/* Describes two objects whose addresses must be unequal for the vectorized
557 loop to be valid. */
558typedef std::pair<tree, tree> vec_object_pair;
559
560/* Records that vectorization is only possible if abs (EXPR) >= MIN_VALUE.
561 UNSIGNED_P is true if we can assume that abs (EXPR) == EXPR. */
572
573/* Vectorizer state shared between different analyses like vector sizes
574 of the same CFG region. */
576public:
579
580 void save_datarefs();
581 void check_datarefs();
582
583 /* All data references. Freed by free_data_refs, so not an auto_vec. */
586
587 /* The loop nest in which the data dependences are computed. */
589
590 /* All data dependences. Freed by free_dependence_relations, so not
591 an auto_vec. */
593};
594
595/* Vectorizer state common between loop and basic-block vectorization. */
596class vec_info {
597public:
599 enum vec_kind { bb, loop };
600
602 ~vec_info ();
603
616
617 /* The type of vectorization. */
619
620 /* Shared vectorizer state. */
622
623 /* The mapping of GIMPLE UID to stmt_vec_info. */
625 /* Whether the above mapping is complete. */
627
628 /* Whether we've done a transform we think OK to not update virtual
629 SSA form. */
631
632 /* The SLP graph. */
634
635 /* Maps base addresses to an innermost_loop_behavior that gives the maximum
636 known alignment for that base. */
638
639 /* All interleaving chains of stores, represented by the first
640 stmt in the chain. */
642
643 /* The set of vector modes used in the vectorized region. */
645
646 /* The argument we should pass to related_vector_mode when looking up
647 the vector mode for a scalar mode, or VOIDmode if we haven't yet
648 made any decisions about which vector modes to use. */
649 machine_mode vector_mode;
650
651 /* The basic blocks in the vectorization region. For _loop_vec_info,
652 the memory is internally managed, while for _bb_vec_info, it points
653 to element space of an external auto_vec<>. This inconsistency is
654 not a good class design pattern. TODO: improve it with an unified
655 auto_vec<> whose lifetime is confined to vec_info object. */
657
658 /* The count of the basic blocks in the vectorization region. */
659 unsigned int nbbs;
660
661 /* Used to keep a sequence of def stmts of a pattern stmt that are loop
662 invariant if they exists.
663 The sequence is emitted in the loop preheader should the loop be vectorized
664 and are reset when undoing patterns. */
666
667private:
669 void set_vinfo_for_stmt (gimple *, stmt_vec_info, bool = true);
670 void free_stmt_vec_infos ();
672};
673
674class _loop_vec_info;
675class _bb_vec_info;
676
677template<>
678template<>
679inline bool
684
685template<>
686template<>
687inline bool
692
693/* In general, we can divide the vector statements in a vectorized loop
694 into related groups ("rgroups") and say that for each rgroup there is
695 some nS such that the rgroup operates on nS values from one scalar
696 iteration followed by nS values from the next. That is, if VF is the
697 vectorization factor of the loop, the rgroup operates on a sequence:
698
699 (1,1) (1,2) ... (1,nS) (2,1) ... (2,nS) ... (VF,1) ... (VF,nS)
700
701 where (i,j) represents a scalar value with index j in a scalar
702 iteration with index i.
703
704 [ We use the term "rgroup" to emphasise that this grouping isn't
705 necessarily the same as the grouping of statements used elsewhere.
706 For example, if we implement a group of scalar loads using gather
707 loads, we'll use a separate gather load for each scalar load, and
708 thus each gather load will belong to its own rgroup. ]
709
710 In general this sequence will occupy nV vectors concatenated
711 together. If these vectors have nL lanes each, the total number
712 of scalar values N is given by:
713
714 N = nS * VF = nV * nL
715
716 None of nS, VF, nV and nL are required to be a power of 2. nS and nV
717 are compile-time constants but VF and nL can be variable (if the target
718 supports variable-length vectors).
719
720 In classical vectorization, each iteration of the vector loop would
721 handle exactly VF iterations of the original scalar loop. However,
722 in vector loops that are able to operate on partial vectors, a
723 particular iteration of the vector loop might handle fewer than VF
724 iterations of the scalar loop. The vector lanes that correspond to
725 iterations of the scalar loop are said to be "active" and the other
726 lanes are said to be "inactive".
727
728 In such vector loops, many rgroups need to be controlled to ensure
729 that they have no effect for the inactive lanes. Conceptually, each
730 such rgroup needs a sequence of booleans in the same order as above,
731 but with each (i,j) replaced by a boolean that indicates whether
732 iteration i is active. This sequence occupies nV vector controls
733 that again have nL lanes each. Thus the control sequence as a whole
734 consists of VF independent booleans that are each repeated nS times.
735
736 Taking mask-based approach as a partially-populated vectors example.
737 We make the simplifying assumption that if a sequence of nV masks is
738 suitable for one (nS,nL) pair, we can reuse it for (nS/2,nL/2) by
739 VIEW_CONVERTing it. This holds for all current targets that support
740 fully-masked loops. For example, suppose the scalar loop is:
741
742 float *f;
743 double *d;
744 for (int i = 0; i < n; ++i)
745 {
746 f[i * 2 + 0] += 1.0f;
747 f[i * 2 + 1] += 2.0f;
748 d[i] += 3.0;
749 }
750
751 and suppose that vectors have 256 bits. The vectorized f accesses
752 will belong to one rgroup and the vectorized d access to another:
753
754 f rgroup: nS = 2, nV = 1, nL = 8
755 d rgroup: nS = 1, nV = 1, nL = 4
756 VF = 4
757
758 [ In this simple example the rgroups do correspond to the normal
759 SLP grouping scheme. ]
760
761 If only the first three lanes are active, the masks we need are:
762
763 f rgroup: 1 1 | 1 1 | 1 1 | 0 0
764 d rgroup: 1 | 1 | 1 | 0
765
766 Here we can use a mask calculated for f's rgroup for d's, but not
767 vice versa.
768
769 Thus for each value of nV, it is enough to provide nV masks, with the
770 mask being calculated based on the highest nL (or, equivalently, based
771 on the highest nS) required by any rgroup with that nV. We therefore
772 represent the entire collection of masks as a two-level table, with the
773 first level being indexed by nV - 1 (since nV == 0 doesn't exist) and
774 the second being indexed by the mask index 0 <= i < nV. */
775
776/* The controls (like masks or lengths) needed by rgroups with nV vectors,
777 according to the description above. */
779 /* The largest nS for all rgroups that use these controls.
780 For vect_partial_vectors_avx512 this is the constant nscalars_per_iter
781 for all members of the group. */
783
784 /* For the largest nS recorded above, the loop controls divide each scalar
785 into FACTOR equal-sized pieces. This is useful if we need to split
786 element-based accesses into byte-based accesses.
787 For vect_partial_vectors_avx512 this records nV instead. */
788 unsigned int factor;
789
790 /* This is a vector type with MAX_NSCALARS_PER_ITER * VF / nV elements.
791 For mask-based controls, it is the type of the masks in CONTROLS.
792 For length-based controls, it can be any vector type that has the
793 specified number of elements; the type of the elements doesn't matter. */
795
796 /* When there is no uniformly used LOOP_VINFO_RGROUP_COMPARE_TYPE this
797 is the rgroup specific type used. */
799
800 /* A vector of nV controls, in iteration order. */
802
803 /* In case of len_load and len_store with a bias there is only one
804 rgroup. This holds the adjusted loop length for the this rgroup. */
806};
807
809{
810 bool is_empty () const { return mask_set.is_empty (); }
811
812 /* Set to record vectype, nvector pairs. */
815
816 /* rgroup_controls used for the partial vector scheme. */
818};
819
821
823
824/* Abstraction around info on reductions which is still in stmt_vec_info
825 but will be duplicated or moved elsewhere. */
827{
828public:
829 /* The def type of the main reduction PHI, vect_reduction_def or
830 vect_double_reduction_def. */
832
833 /* The reduction type as detected by
834 vect_is_simple_reduction and vectorizable_reduction. */
836
837 /* The original scalar reduction code, to be used in the epilogue. */
839
840 /* A vector internal function we should use in the epilogue. */
842
843 /* For loop reduction with multiple vectorized results (ncopies > 1), a
844 lane-reducing operation participating in it may not use all of those
845 results, this field specifies result index starting from which any
846 following land-reducing operation would be assigned to. */
847 unsigned int reduc_result_pos;
848
849 /* Whether this represents a reduction chain. */
851
852 /* Whether we force a single cycle PHI during reduction vectorization. */
854
855 /* The vector type for performing the actual reduction operation. */
857
858 /* The vector type we should use for the final reduction in the epilogue
859 when we reduce a mask. */
861
862 /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used. */
864
865 /* If not NULL the value to be added to compute final reduction value. */
867
868 /* If non-null, the reduction is being performed by an epilogue loop
869 and we have decided to reuse this accumulator from the main loop. */
871
872 /* If the vector code is performing N scalar reductions in parallel,
873 this variable gives the initial scalar values of those N reductions. */
875
876 /* If the vector code is performing N scalar reductions in parallel, this
877 variable gives the vectorized code's final (scalar) result for each of
878 those N reductions. In other words, REDUC_SCALAR_RESULTS[I] replaces
879 the original scalar code's loop-closed SSA PHI for reduction number I. */
881};
882
884
885#define VECT_REDUC_INFO_DEF_TYPE(I) ((I)->def_type)
886#define VECT_REDUC_INFO_TYPE(I) ((I)->reduc_type)
887#define VECT_REDUC_INFO_CODE(I) ((I)->reduc_code)
888#define VECT_REDUC_INFO_FN(I) ((I)->reduc_fn)
889#define VECT_REDUC_INFO_SCALAR_RESULTS(I) ((I)->reduc_scalar_results)
890#define VECT_REDUC_INFO_INITIAL_VALUES(I) ((I)->reduc_initial_values)
891#define VECT_REDUC_INFO_REUSED_ACCUMULATOR(I) ((I)->reused_accumulator)
892#define VECT_REDUC_INFO_INDUC_COND_INITIAL_VAL(I) ((I)->induc_cond_initial_val)
893#define VECT_REDUC_INFO_EPILOGUE_ADJUSTMENT(I) ((I)->reduc_epilogue_adjustment)
894#define VECT_REDUC_INFO_VECTYPE(I) ((I)->reduc_vectype)
895#define VECT_REDUC_INFO_VECTYPE_FOR_MASK(I) ((I)->reduc_vectype_for_mask)
896#define VECT_REDUC_INFO_FORCE_SINGLE_CYCLE(I) ((I)->force_single_cycle)
897#define VECT_REDUC_INFO_RESULT_POS(I) ((I)->reduc_result_pos)
898
899/* Information about a reduction accumulator from the main loop that could
900 conceivably be reused as the input to a reduction in an epilogue loop. */
902 /* The final value of the accumulator, which forms the input to the
903 reduction operation. */
905
906 /* The stmt_vec_info that describes the reduction (i.e. the one for
907 which is_reduc_info is true). */
909};
910
911/*-----------------------------------------------------------------*/
912/* Info on vectorized loops. */
913/*-----------------------------------------------------------------*/
914typedef class _loop_vec_info : public vec_info {
915public:
918
919 /* The loop to which this info struct refers to. */
920 class loop *loop;
921
922 /* Number of latch executions. */
924 /* Number of iterations. */
926 /* Number of iterations of the original loop. */
928 /* Condition under which this loop is analyzed and versioned. */
930
931 /* The cost of the vector code. */
933
934 /* The cost of the scalar code. */
936
937 /* Threshold of number of iterations below which vectorization will not be
938 performed. It is calculated from MIN_PROFITABLE_ITERS and
939 param_min_vect_loop_bound. */
940 unsigned int th;
941
942 /* When applying loop versioning, the vector form should only be used
943 if the number of scalar iterations is >= this value, on top of all
944 the other requirements. Ignored when loop versioning is not being
945 used. */
947
948 /* Unrolling factor. In case of suitable super-word parallelism
949 it can be that no unrolling is needed, and thus this is 1. */
951
952 /* If this loop is an epilogue loop whose main loop can be skipped,
953 MAIN_LOOP_EDGE is the edge from the main loop to this loop's
954 preheader. SKIP_MAIN_LOOP_EDGE is then the edge that skips the
955 main loop and goes straight to this loop's preheader.
956
957 Both fields are null otherwise. */
960
961 /* If this loop is an epilogue loop that might be skipped after executing
962 the main loop, this edge is the one that skips the epilogue. */
964
965 /* Reduction descriptors of this loop. Referenced to from SLP nodes
966 by index. */
968
969 /* The vectorized form of a standard reduction replaces the original
970 scalar code's final result (a loop-closed SSA PHI) with the result
971 of a vector-to-scalar reduction operation. After vectorization,
972 this variable maps these vector-to-scalar results to information
973 about the reductions that generated them. */
975
976 /* The number of times that the target suggested we unroll the vector loop
977 in order to promote more ILP. This value will be used to re-analyze the
978 loop for vectorization and if successful the value will be folded into
979 vectorization_factor (and therefore exactly divides
980 vectorization_factor). */
982
983 /* Maximum runtime vectorization factor, or MAX_VECTORIZATION_FACTOR
984 if there is no particular limit. */
985 unsigned HOST_WIDE_INT max_vectorization_factor;
986
987 /* The masks that a fully-masked loop should use to avoid operating
988 on inactive scalars. */
990
991 /* The lengths that a loop with length should use to avoid operating
992 on inactive scalars. */
994
995 /* Set of scalar conditions that have loop mask applied. */
997
998 /* Set of vector conditions that have loop mask applied. */
1000
1001 /* If we are using a loop mask to align memory addresses, this variable
1002 contains the number of vector elements that we should skip in the
1003 first iteration of the vector loop (i.e. the number of leading
1004 elements that should be false in the first mask). */
1006
1007 /* If we are using a loop mask to align memory addresses and we're in an
1008 early break loop then this variable contains the number of elements that
1009 were skipped during the initial iteration of the loop. */
1011
1012 /* The type that the loop control IV should be converted to before
1013 testing which of the VF scalars are active and inactive.
1014 Only meaningful if LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
1016
1017 /* For #pragma omp simd if (x) loops the x expression. If constant 0,
1018 the loop should not be vectorized, if constant non-zero, simd_if_cond
1019 shouldn't be set and loop vectorized normally, if SSA_NAME, the loop
1020 should be versioned on that condition, using scalar loop if the condition
1021 is false and vectorized loop otherwise. */
1023
1024 /* The type that the vector loop control IV should have when
1025 LOOP_VINFO_USING_PARTIAL_VECTORS_P is true. */
1027
1028 /* The style used for implementing partial vectors when
1029 LOOP_VINFO_USING_PARTIAL_VECTORS_P is true. */
1031
1032 /* Unknown DRs according to which loop was peeled. */
1034
1035 /* peeling_for_alignment indicates whether peeling for alignment will take
1036 place, and what the peeling factor should be:
1037 peeling_for_alignment = X means:
1038 If X=0: Peeling for alignment will not be applied.
1039 If X>0: Peel first X iterations.
1040 If X=-1: Generate a runtime test to calculate the number of iterations
1041 to be peeled, using the dataref recorded in the field
1042 unaligned_dr. */
1044
1045 /* The mask used to check the alignment of pointers or arrays. */
1047
1048 /* The maximum speculative read amount in VLA modes for runtime check. */
1050
1051 /* Indicates whether the loop has any non-linear IV. */
1053
1054 /* Data Dependence Relations defining address ranges that are candidates
1055 for a run-time aliasing check. */
1057
1058 /* Data Dependence Relations defining address ranges together with segment
1059 lengths from which the run-time aliasing check is built. */
1061
1062 /* Check that the addresses of each pair of objects is unequal. */
1064
1065 /* List of values that are required to be nonzero. This is used to check
1066 whether things like "x[i * n] += 1;" are safe and eventually gets added
1067 to the checks for lower bounds below. */
1069
1070 /* List of values that need to be checked for a minimum value. */
1072
1073 /* Statements in the loop that have data references that are candidates for a
1074 runtime (loop versioning) misalignment check. */
1076
1077 /* Reduction cycles detected in the loop. Used in loop-aware SLP. */
1079
1080 /* Defs that could not be analyzed such as OMP SIMD calls without
1081 a LHS. */
1083
1084 /* Cost vector for a single scalar iteration. */
1086
1087 /* Map of IV base/step expressions to inserted name in the preheader. */
1089
1090 /* Map of OpenMP "omp simd array" scan variables to corresponding
1091 rhs of the store of the initializer. */
1093
1094 /* The factor used to over weight those statements in an inner loop
1095 relative to the loop being vectorized. */
1097
1098 /* Is the loop vectorizable? */
1100
1101 /* Records whether we still have the option of vectorizing this loop
1102 using partially-populated vectors; in other words, whether it is
1103 still possible for one iteration of the vector loop to handle
1104 fewer than VF scalars. */
1106
1107 /* Records whether we must use niter masking for correctness reasons. */
1109
1110 /* True if we've decided to use partially-populated vectors, so that
1111 the vector loop can handle fewer than VF scalars. */
1113
1114 /* True if we've decided to use a decrementing loop control IV that counts
1115 scalars. This can be done for any loop that:
1116
1117 (a) uses length "controls"; and
1118 (b) can iterate more than once. */
1120
1121 /* True if we've decided to use output of select_vl to adjust IV of
1122 both loop control and data reference pointer. This is only true
1123 for single-rgroup control. */
1125
1126 /* True if we've decided to use partially-populated vectors for the
1127 epilogue of loop. */
1129
1130 /* True if we've decided to use peeling with versioning together, which allows
1131 unaligned unsupported data refs to be uniformly aligned after a certain
1132 amount of peeling (mutual alignment). Otherwise, we use versioning alone
1133 so these data refs must be already aligned to a power-of-two boundary
1134 without peeling. */
1136
1137 /* The bias for len_load and len_store. For now, only 0 and -1 are
1138 supported. -1 must be used when a backend does not support
1139 len_load/len_store with a length of zero. */
1141
1142 /* When we have grouped data accesses with gaps, we may introduce invalid
1143 memory accesses. We peel the last iteration of the loop to prevent
1144 this. */
1146
1147 /* When the number of iterations is not a multiple of the vector size
1148 we need to peel off iterations at the end to form an epilogue loop. */
1150
1151 /* When the loop has early breaks that we can vectorize we need to peel
1152 the loop for the break finding loop. */
1154
1155 /* List of loop additional IV conditionals found in the loop. */
1157
1158 /* Main loop IV cond. */
1160
1161 /* True if we have an unroll factor requested by the user through pragma GCC
1162 unroll. */
1164
1165 /* True if there are no loop carried data dependencies in the loop.
1166 If loop->safelen <= 1, then this is always true, either the loop
1167 didn't have any loop carried data dependencies, or the loop is being
1168 vectorized guarded with some runtime alias checks, or couldn't
1169 be vectorized at all, but then this field shouldn't be used.
1170 For loop->safelen >= 2, the user has asserted that there are no
1171 backward dependencies, but there still could be loop carried forward
1172 dependencies in such loops. This flag will be false if normal
1173 vectorizer data dependency analysis would fail or require versioning
1174 for alias, but because of loop->safelen >= 2 it has been vectorized
1175 even without versioning for alias. E.g. in:
1176 #pragma omp simd
1177 for (int i = 0; i < m; i++)
1178 a[i] = a[i + k] * c;
1179 (or #pragma simd or #pragma ivdep) we can vectorize this and it will
1180 DTRT even for k > 0 && k < m, but without safelen we would not
1181 vectorize this, so this field would be false. */
1183
1184 /* Mark loops having masked stores. */
1186
1187 /* Queued scaling factor for the scalar loop. */
1189
1190 /* If if-conversion versioned this loop before conversion, this is the
1191 loop version without if-conversion. */
1193
1194 /* For loops being epilogues of already vectorized loops
1195 this points to the main vectorized loop. Otherwise NULL. */
1197
1198 /* For loops being epilogues of already vectorized loops
1199 this points to the preceeding vectorized (possibly epilogue) loop.
1200 Otherwise NULL. */
1202
1203 /* Used to store loop_vec_infos of the epilogue of this loop during
1204 analysis. */
1206
1207 /* If this is an epilogue loop the DR advancement applied. */
1209
1210 /* The controlling loop IV for the current loop when vectorizing. This IV
1211 controls the natural exits of the loop. */
1213
1214 /* The controlling loop IV for the epilogue loop when vectorizing. This IV
1215 controls the natural exits of the loop. */
1217
1218 /* The controlling loop IV for the scalar loop being vectorized. This IV
1219 controls the natural exits of the loop. */
1221
1222 /* Used to store the list of stores needing to be moved if doing early
1223 break vectorization as they would violate the scalar loop semantics if
1224 vectorized in their current location. These are stored in order that they
1225 need to be moved. */
1227
1228 /* The final basic block where to move statements to. In the case of
1229 multiple exits this could be pretty far away. */
1231
1232 /* Statements whose VUSES need updating if early break vectorization is to
1233 happen. */
1235
1236 /* Record statements that are needed to be live for early break vectorization
1237 but may not have an LC PHI node materialized yet in the exits. */
1240
1241/* Access Functions. */
1242#define LOOP_VINFO_LOOP(L) (L)->loop
1243#define LOOP_VINFO_IV_EXIT(L) (L)->vec_loop_iv_exit
1244#define LOOP_VINFO_EPILOGUE_IV_EXIT(L) (L)->vec_epilogue_loop_iv_exit
1245#define LOOP_VINFO_SCALAR_IV_EXIT(L) (L)->scalar_loop_iv_exit
1246#define LOOP_VINFO_BBS(L) (L)->bbs
1247#define LOOP_VINFO_NBBS(L) (L)->nbbs
1248#define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1
1249#define LOOP_VINFO_NITERS(L) (L)->num_iters
1250/* Since LOOP_VINFO_NITERS and LOOP_VINFO_NITERSM1 can change after
1251 prologue peeling retain total unchanged scalar loop iterations for
1252 cost model. */
1253#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
1254#define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions
1255#define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th
1256#define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold
1257#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
1258#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
1259#define LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P(L) (L)->must_use_partial_vectors_p
1260#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
1261#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
1262#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
1263#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \
1264 (L)->epil_using_partial_vectors_p
1265#define LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT(L) (L)->allow_mutual_alignment
1266#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
1267#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
1268#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
1269#define LOOP_VINFO_MASKS(L) (L)->masks
1270#define LOOP_VINFO_LENS(L) (L)->lens
1271#define LOOP_VINFO_MASK_SKIP_NITERS(L) (L)->mask_skip_niters
1272#define LOOP_VINFO_MASK_NITERS_PFA_OFFSET(L) (L)->mask_skip_niters_pfa_offset
1273#define LOOP_VINFO_RGROUP_COMPARE_TYPE(L) (L)->rgroup_compare_type
1274#define LOOP_VINFO_RGROUP_IV_TYPE(L) (L)->rgroup_iv_type
1275#define LOOP_VINFO_PARTIAL_VECTORS_STYLE(L) (L)->partial_vector_style
1276#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
1277#define LOOP_VINFO_MAX_SPEC_READ_AMOUNT(L) (L)->max_spec_read_amount
1278#define LOOP_VINFO_LOOP_NEST(L) (L)->shared->loop_nest
1279#define LOOP_VINFO_DATAREFS(L) (L)->shared->datarefs
1280#define LOOP_VINFO_DDRS(L) (L)->shared->ddrs
1281#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
1282#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
1283#define LOOP_VINFO_NON_LINEAR_IV(L) (L)->nonlinear_iv
1284#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
1285#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
1286#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
1287#define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs
1288#define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs
1289#define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero
1290#define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds
1291#define LOOP_VINFO_USER_UNROLL(L) (L)->user_unroll
1292#define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
1293#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
1294#define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
1295#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
1296#define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter
1297#define LOOP_VINFO_EARLY_BREAKS(L) (L)->early_breaks
1298#define LOOP_VINFO_EARLY_BRK_STORES(L) (L)->early_break_stores
1299#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L) \
1300 (single_pred ((L)->loop->latch) != (L)->vec_loop_iv_exit->src)
1301#define LOOP_VINFO_EARLY_BREAKS_LIVE_IVS(L) \
1302 (L)->early_break_live_ivs
1303#define LOOP_VINFO_EARLY_BRK_DEST_BB(L) (L)->early_break_dest_bb
1304#define LOOP_VINFO_EARLY_BRK_VUSES(L) (L)->early_break_vuses
1305#define LOOP_VINFO_LOOP_CONDS(L) (L)->conds
1306#define LOOP_VINFO_LOOP_IV_COND(L) (L)->loop_iv_cond
1307#define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
1308#define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop
1309#define LOOP_VINFO_SCALAR_LOOP_SCALING(L) (L)->scalar_loop_scaling
1310#define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store
1311#define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
1312#define LOOP_VINFO_MAIN_LOOP_INFO(L) (L)->main_loop_info
1313#define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info
1314#define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
1315#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
1316#define LOOP_VINFO_INV_PATTERN_DEF_SEQ(L) (L)->inv_pattern_def_seq
1317#define LOOP_VINFO_DRS_ADVANCED_BY(L) (L)->drs_advanced_by
1318#define LOOP_VINFO_ALTERNATE_DEFS(L) (L)->alternate_defs
1319
1320#define LOOP_VINFO_FULLY_MASKED_P(L) \
1321 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1322 && !LOOP_VINFO_MASKS (L).is_empty ())
1323
1324#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \
1325 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1326 && !LOOP_VINFO_LENS (L).is_empty ())
1327
1328#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
1329 ((L)->may_misalign_stmts.length () > 0)
1330#define LOOP_REQUIRES_VERSIONING_FOR_SPEC_READ(L) \
1331 (maybe_gt ((L)->max_spec_read_amount, 0U))
1332#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
1333 ((L)->comp_alias_ddrs.length () > 0 \
1334 || (L)->check_unequal_addrs.length () > 0 \
1335 || (L)->lower_bounds.length () > 0)
1336#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \
1337 (LOOP_VINFO_NITERS_ASSUMPTIONS (L))
1338#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \
1339 (LOOP_VINFO_SIMD_IF_COND (L))
1340#define LOOP_REQUIRES_VERSIONING(L) \
1341 (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \
1342 || LOOP_REQUIRES_VERSIONING_FOR_SPEC_READ (L) \
1343 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \
1344 || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
1345 || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
1346
1347#define LOOP_VINFO_USE_VERSIONING_WITHOUT_PEELING(L) \
1348 ((L)->may_misalign_stmts.length () > 0 \
1349 && !LOOP_VINFO_ALLOW_MUTUAL_ALIGNMENT (L))
1350
1351#define LOOP_VINFO_NITERS_KNOWN_P(L) \
1352 (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
1353
1354#define LOOP_VINFO_EPILOGUE_P(L) \
1355 (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL)
1356
1357#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \
1358 (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L)))
1359
1360/* Wrapper for loop_vec_info, for tracking success/failure, where a non-NULL
1361 value signifies success, and a NULL value signifies failure, supporting
1362 propagating an opt_problem * describing the failure back up the call
1363 stack. */
1364typedef opt_pointer_wrapper <loop_vec_info> opt_loop_vec_info;
1365
1366inline loop_vec_info
1368{
1369 return (loop_vec_info) loop->aux;
1370}
1371
1382
1391
1392#define BB_VINFO_BBS(B) (B)->bbs
1393#define BB_VINFO_NBBS(B) (B)->nbbs
1394#define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
1395#define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
1396#define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
1397#define BB_VINFO_DDRS(B) (B)->shared->ddrs
1398
1399/* Indicates whether/how a variable is used in the scope of loop/basic
1400 block. */
1403
1404 /* The def is only used outside the loop. */
1406 /* The def is in the inner loop, and the use is in the outer loop, and the
1407 use is a reduction stmt. */
1409 /* The def is in the inner loop, and the use is in the outer loop (and is
1410 not part of reduction). */
1412
1413 /* defs that feed computations that end up (only) in a reduction. These
1414 defs may be used by non-reduction stmts, but eventually, any
1415 computations/values that are affected by these defs are used to compute
1416 a reduction (i.e. don't get stored to memory, for example). We use this
1417 to identify computations that we can change the order in which they are
1418 computed. */
1420
1422};
1423
1424/* The type of vectorization. pure_slp means the stmt is covered by the
1425 SLP graph, not_vect means it is not. This is mostly used by BB
1426 vectorization. */
1431
1432/* Says whether a statement is a load, a store of a vectorized statement
1433 result, or a store of an invariant value. */
1439
1441public:
1442 /* The data reference itself. */
1444 /* The statement that contains the data reference. */
1446 /* The analysis group this DR belongs to when doing BB vectorization.
1447 DRs of the same group belong to the same conditional execution context. */
1448 unsigned group;
1449 /* The misalignment in bytes of the reference, or -1 if not known. */
1451 /* The byte alignment that we'd ideally like the reference to have,
1452 and the value that misalignment is measured against. */
1454 /* If true the alignment of base_decl needs to be increased. */
1456
1457 /* Set by early break vectorization when this DR needs peeling for alignment
1458 for correctness. */
1460
1461 /* Set by early break vectorization when this DR's scalar accesses are known
1462 to be inbounds of a known bounds loop. */
1464
1466
1467 /* Stores current vectorized loop's offset. To be added to the DR's
1468 offset to calculate current offset of data reference. */
1470};
1471
1472typedef struct data_reference *dr_p;
1473
1475public:
1476
1477 /* Indicates whether this stmts is part of a computation whose result is
1478 used outside the loop. */
1479 bool live;
1480
1481 /* Stmt is part of some pattern (computation idiom) */
1483
1484 /* True if the statement was created during pattern recognition as
1485 part of the replacement for RELATED_STMT. This implies that the
1486 statement isn't part of any basic block, although for convenience
1487 its gimple_bb is the same as for RELATED_STMT. */
1489
1490 /* Is this statement vectorizable or should it be skipped in (partial)
1491 vectorization. */
1493
1494 /* The stmt to which this info struct refers to. */
1496
1497 /* The vector type to be used for the LHS of this statement. */
1499
1500 /* The following is relevant only for stmts that contain a non-scalar
1501 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
1502 at most one such data-ref. */
1503
1505
1506 /* Information about the data-ref relative to this loop
1507 nest (the loop that is being considered for vectorization). */
1509
1510 /* For loop PHI nodes, the base and evolution part of it. This makes sure
1511 this information is still available in vect_update_ivs_after_vectorizer
1512 where we may not be able to re-analyze the PHI nodes evolution as
1513 peeling for the prologue loop can make it unanalyzable. The evolution
1514 part is still correct after peeling, but the base may have changed from
1515 the version here. */
1519
1520 /* Used for various bookkeeping purposes, generally holding a pointer to
1521 some other stmt S that is in some way "related" to this stmt.
1522 Current use of this field is:
1523 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
1524 true): S is the "pattern stmt" that represents (and replaces) the
1525 sequence of stmts that constitutes the pattern. Similarly, the
1526 related_stmt of the "pattern stmt" points back to this stmt (which is
1527 the last stmt in the original sequence of stmts that constitutes the
1528 pattern). */
1530
1531 /* Used to keep a sequence of def stmts of a pattern stmt if such exists.
1532 The sequence is attached to the original statement rather than the
1533 pattern statement. */
1535
1536 /* Classify the def of this stmt. */
1538
1539 /* Whether the stmt is SLPed, loop-based vectorized, or both. */
1541
1542 /* Interleaving chains info. */
1543 /* First element in the group. */
1545 /* Pointer to the next element in the group. */
1547 /* The size of the group. */
1548 unsigned int size;
1549 /* For loads only, the gap from the previous load. For consecutive loads, GAP
1550 is 1. */
1551 unsigned int gap;
1552
1553 /* The minimum negative dependence distance this stmt participates in
1554 or zero if none. */
1555 unsigned int min_neg_dist;
1556
1557 /* Not all stmts in the loop need to be vectorized. e.g, the increment
1558 of the loop induction variable and computation of array indexes. relevant
1559 indicates whether the stmt needs to be vectorized. */
1561
1562 /* For loads if this is a gather, for stores if this is a scatter. */
1564
1565 /* True if this is an access with loop-invariant stride. */
1567
1568 /* For both loads and stores. */
1570
1571 /* On a reduction PHI the reduction type as detected by
1572 vect_is_simple_reduction. */
1574
1575 /* On a reduction PHI, the original reduction code as detected by
1576 vect_is_simple_reduction. */
1578
1579 /* On a stmt participating in a reduction the index of the operand
1580 on the reduction SSA cycle. */
1582
1583 /* On a reduction PHI the def returned by vect_is_simple_reduction.
1584 On the def returned by vect_is_simple_reduction the corresponding PHI. */
1586
1587 /* If nonzero, the lhs of the statement could be truncated to this
1588 many bits without affecting any users of the result. */
1590
1591 /* If nonzero, all non-boolean input operands have the same precision,
1592 and they could each be truncated to this many bits without changing
1593 the result. */
1595
1596 /* If OPERATION_BITS is nonzero, the statement could be performed on
1597 an integer with the sign and number of bits given by OPERATION_SIGN
1598 and OPERATION_BITS without changing the result. */
1601
1602 /* If the statement produces a boolean result, this value describes
1603 how we should choose the associated vector type. The possible
1604 values are:
1605
1606 - an integer precision N if we should use the vector mask type
1607 associated with N-bit integers. This is only used if all relevant
1608 input booleans also want the vector mask type for N-bit integers,
1609 or if we can convert them into that form by pattern-matching.
1610
1611 - ~0U if we considered choosing a vector mask type but decided
1612 to treat the boolean as a normal integer type instead.
1613
1614 - 0 otherwise. This means either that the operation isn't one that
1615 could have a vector mask type (and so should have a normal vector
1616 type instead) or that we simply haven't made a choice either way. */
1617 unsigned int mask_precision;
1618
1619 /* True if this is only suitable for SLP vectorization. */
1621
1622 /* True if this is a pattern that can only be handled by SLP
1623 vectorization. */
1625};
1626
1627/* Information about a gather/scatter call. */
1629 /* The internal function to use for the gather/scatter operation,
1630 or IFN_LAST if a built-in function should be used instead. */
1632
1633 /* The FUNCTION_DECL for the built-in gather/scatter function,
1634 or null if an internal function should be used instead. */
1636
1637 /* The loop-invariant base value. */
1639
1640 /* The TBBA alias pointer the value of which determines the alignment
1641 of the scalar accesses. */
1643
1644 /* The original scalar offset, which is a non-loop-invariant SSA_NAME. */
1646
1647 /* Each offset element should be multiplied by this amount before
1648 being added to the base. */
1650
1651 /* The type of the vectorized offset. */
1653
1654 /* The type of the scalar elements after loading or before storing. */
1656
1657 /* The type of the scalar elements being loaded or stored. */
1659};
1660
1661/* Access Functions. */
1662#define STMT_VINFO_STMT(S) (S)->stmt
1663#define STMT_VINFO_RELEVANT(S) (S)->relevant
1664#define STMT_VINFO_LIVE_P(S) (S)->live
1665#define STMT_VINFO_VECTYPE(S) (S)->vectype
1666#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
1667#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
1668#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
1669#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
1670#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
1671#define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
1672
1673#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop
1674#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address
1675#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init
1676#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset
1677#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step
1678#define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment
1679#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \
1680 (S)->dr_wrt_vec_loop.base_misalignment
1681#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \
1682 (S)->dr_wrt_vec_loop.offset_alignment
1683#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
1684 (S)->dr_wrt_vec_loop.step_alignment
1685
1686#define STMT_VINFO_DR_INFO(S) \
1687 (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
1688
1689#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
1690#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
1691#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
1692#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
1693#define STMT_VINFO_GROUPED_ACCESS(S) \
1694 ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
1695#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
1696#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
1697#define STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE(S) (S)->loop_phi_evolution_type
1698#define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
1699#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
1700#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
1701#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
1702#define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
1703#define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p
1704#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
1705
1706#define DR_GROUP_FIRST_ELEMENT(S) \
1707 (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
1708#define DR_GROUP_NEXT_ELEMENT(S) \
1709 (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
1710#define DR_GROUP_SIZE(S) \
1711 (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
1712#define DR_GROUP_GAP(S) \
1713 (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
1714
1715#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
1716
1717#define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
1718#define STMT_SLP_TYPE(S) (S)->slp_type
1719
1720
1721/* Contains the scalar or vector costs for a vec_info. */
1723{
1724public:
1725 vector_costs (vec_info *, bool);
1726 virtual ~vector_costs () {}
1727
1728 /* Update the costs in response to adding COUNT copies of a statement.
1729
1730 - WHERE specifies whether the cost occurs in the loop prologue,
1731 the loop body, or the loop epilogue.
1732 - KIND is the kind of statement, which is always meaningful.
1733 - STMT_INFO or NODE, if nonnull, describe the statement that will be
1734 vectorized.
1735 - VECTYPE, if nonnull, is the vector type that the vectorized
1736 statement will operate on. Note that this should be used in
1737 preference to STMT_VINFO_VECTYPE (STMT_INFO) since the latter
1738 is not correct for SLP.
1739 - for unaligned_load and unaligned_store statements, MISALIGN is
1740 the byte misalignment of the load or store relative to the target's
1741 preferred alignment for VECTYPE, or DR_MISALIGNMENT_UNKNOWN
1742 if the misalignment is not known.
1743
1744 Return the calculated cost as well as recording it. The return
1745 value is used for dumping purposes. */
1746 virtual unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
1747 stmt_vec_info stmt_info,
1748 slp_tree node,
1749 tree vectype, int misalign,
1751
1752 /* Finish calculating the cost of the code. The results can be
1753 read back using the functions below.
1754
1755 If the costs describe vector code, SCALAR_COSTS gives the costs
1756 of the corresponding scalar code, otherwise it is null. */
1757 virtual void finish_cost (const vector_costs *scalar_costs);
1758
1759 /* The costs in THIS and OTHER both describe ways of vectorizing
1760 a main loop. Return true if the costs described by THIS are
1761 cheaper than the costs described by OTHER. Return false if any
1762 of the following are true:
1763
1764 - THIS and OTHER are of equal cost
1765 - OTHER is better than THIS
1766 - we can't be sure about the relative costs of THIS and OTHER. */
1767 virtual bool better_main_loop_than_p (const vector_costs *other) const;
1768
1769 /* Likewise, but the costs in THIS and OTHER both describe ways of
1770 vectorizing an epilogue loop of MAIN_LOOP. */
1771 virtual bool better_epilogue_loop_than_p (const vector_costs *other,
1772 loop_vec_info main_loop) const;
1773
1774 unsigned int prologue_cost () const;
1775 unsigned int body_cost () const;
1776 unsigned int epilogue_cost () const;
1777 unsigned int outside_cost () const;
1778 unsigned int total_cost () const;
1779 unsigned int suggested_unroll_factor () const;
1780 machine_mode suggested_epilogue_mode (int &masked) const;
1782
1783protected:
1785 unsigned int);
1787 unsigned int);
1788 int compare_inside_loop_cost (const vector_costs *) const;
1789 int compare_outside_loop_cost (const vector_costs *) const;
1790
1791 /* The region of code that we're considering vectorizing. */
1793
1794 /* True if we're costing the scalar code, false if we're costing
1795 the vector code. */
1797
1798 /* The costs of the three regions, indexed by vect_cost_model_location. */
1799 unsigned int m_costs[3];
1800
1801 /* The suggested unrolling factor determined at finish_cost. */
1803
1804 /* The suggested mode to be used for a vectorized epilogue or VOIDmode,
1805 determined at finish_cost. m_masked_epilogue specifies whether the
1806 epilogue should use masked vectorization, regardless of the
1807 --param vect-partial-vector-usage default. If -1 then the
1808 --param setting takes precedence. If the user explicitly specified
1809 --param vect-partial-vector-usage then that takes precedence. */
1812
1813 /* True if finish_cost has been called. */
1815};
1816
1817/* Create costs for VINFO. COSTING_FOR_SCALAR is true if the costs
1818 are for scalar code, false if they are for vector code. */
1819
1820inline
1831
1832/* Return the cost of the prologue code (in abstract units). */
1833
1834inline unsigned int
1840
1841/* Return the cost of the body code (in abstract units). */
1842
1843inline unsigned int
1849
1850/* Return the cost of the epilogue code (in abstract units). */
1851
1852inline unsigned int
1858
1859/* Return the cost of the prologue and epilogue code (in abstract units). */
1860
1861inline unsigned int
1863{
1864 return prologue_cost () + epilogue_cost ();
1865}
1866
1867/* Return the cost of the prologue, body and epilogue code
1868 (in abstract units). */
1869
1870inline unsigned int
1872{
1873 return body_cost () + outside_cost ();
1874}
1875
1876/* Return the suggested unroll factor. */
1877
1878inline unsigned int
1884
1885/* Return the suggested epilogue mode. */
1886
1887inline machine_mode
1889{
1891 masked_p = m_masked_epilogue;
1893}
1894
1895#define VECT_MAX_COST 1000
1896
1897/* The maximum number of intermediate steps required in multi-step type
1898 conversion. */
1899#define MAX_INTERM_CVT_STEPS 3
1900
1901#define MAX_VECTORIZATION_FACTOR INT_MAX
1902
1903/* Nonzero if TYPE represents a (scalar) boolean type or type
1904 in the middle-end compatible with it (unsigned precision 1 integral
1905 types). Used to determine which types should be vectorized as
1906 VECTOR_BOOLEAN_TYPE_P. */
1907
1908#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \
1909 (TREE_CODE (TYPE) == BOOLEAN_TYPE \
1910 || ((TREE_CODE (TYPE) == INTEGER_TYPE \
1911 || TREE_CODE (TYPE) == ENUMERAL_TYPE) \
1912 && TYPE_PRECISION (TYPE) == 1 \
1913 && TYPE_UNSIGNED (TYPE)))
1914
1915inline bool
1917{
1918 return (loop->inner
1919 && (loop->inner == (gimple_bb (stmt_info->stmt))->loop_father));
1920}
1921
1922/* PHI is either a scalar reduction phi or a scalar induction phi.
1923 Return the initial value of the variable on entry to the containing
1924 loop. */
1925
1926inline tree
1928{
1929 basic_block bb = gimple_bb (phi);
1931 gcc_assert (pe->dest == bb);
1932 return PHI_ARG_DEF_FROM_EDGE (phi, pe);
1933}
1934
1935/* Return true if STMT_INFO should produce a vector mask type rather than
1936 a normal nonmask type. */
1937
1938inline bool
1940{
1941 return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
1942}
1943
1944/* Return TRUE if a statement represented by STMT_INFO is a part of a
1945 pattern. */
1946
1947inline bool
1949{
1950 return stmt_info->pattern_stmt_p;
1951}
1952
1953/* If STMT_INFO is a pattern statement, return the statement that it
1954 replaces, otherwise return STMT_INFO itself. */
1955
1956inline stmt_vec_info
1958{
1959 if (is_pattern_stmt_p (stmt_info))
1960 return STMT_VINFO_RELATED_STMT (stmt_info);
1961 return stmt_info;
1962}
1963
1964/* Return the later statement between STMT1_INFO and STMT2_INFO. */
1965
1966inline stmt_vec_info
1968{
1969 gimple *stmt1 = vect_orig_stmt (stmt1_info)->stmt;
1970 gimple *stmt2 = vect_orig_stmt (stmt2_info)->stmt;
1971 if (gimple_bb (stmt1) == gimple_bb (stmt2))
1972 {
1973 if (gimple_uid (stmt1) > gimple_uid (stmt2))
1974 return stmt1_info;
1975 else
1976 return stmt2_info;
1977 }
1978 /* ??? We should be really calling this function only with stmts
1979 in the same BB but we can recover if there's a domination
1980 relationship between them. */
1982 gimple_bb (stmt1), gimple_bb (stmt2)))
1983 return stmt1_info;
1985 gimple_bb (stmt2), gimple_bb (stmt1)))
1986 return stmt2_info;
1987 gcc_unreachable ();
1988}
1989
1990/* If STMT_INFO has been replaced by a pattern statement, return the
1991 replacement statement, otherwise return STMT_INFO itself. */
1992
1993inline stmt_vec_info
1995{
1996 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
1997 return STMT_VINFO_RELATED_STMT (stmt_info);
1998 return stmt_info;
1999}
2000
2001/* Return true if BB is a loop header. */
2002
2003inline bool
2005{
2006 if (bb == (bb->loop_father)->header)
2007 return true;
2008
2009 return false;
2010}
2011
2012/* Return pow2 (X). */
2013
2014inline int
2016{
2017 int i, res = 1;
2018
2019 for (i = 0; i < x; i++)
2020 res *= 2;
2021
2022 return res;
2023}
2024
2025/* Alias targetm.vectorize.builtin_vectorization_cost. */
2026
2027inline int
2029 tree vectype, int misalign)
2030{
2031 return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
2032 vectype, misalign);
2033}
2034
2035/* Get cost by calling cost target builtin. */
2036
2037inline
2039{
2040 return builtin_vectorization_cost (type_of_cost, NULL, 0);
2041}
2042
2043/* Alias targetm.vectorize.init_cost. */
2044
2045inline vector_costs *
2046init_cost (vec_info *vinfo, bool costing_for_scalar)
2047{
2048 return targetm.vectorize.create_costs (vinfo, costing_for_scalar);
2049}
2050
2051extern void dump_stmt_cost (FILE *, int, enum vect_cost_for_stmt,
2052 stmt_vec_info, slp_tree, tree, int, unsigned,
2054
2055/* Dump and add costs. */
2056
2057inline unsigned
2059 enum vect_cost_for_stmt kind,
2060 stmt_vec_info stmt_info, slp_tree node,
2061 tree vectype, int misalign,
2062 enum vect_cost_model_location where)
2063{
2064 /* Even though a vector type might be set on stmt do not pass that on when
2065 costing the scalar IL. A SLP node shouldn't have been recorded. */
2066 if (costs->costing_for_scalar ())
2067 {
2068 vectype = NULL_TREE;
2070 }
2071 unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, node, vectype,
2072 misalign, where);
2073 if (dump_file && (dump_flags & TDF_DETAILS))
2074 dump_stmt_cost (dump_file, count, kind, stmt_info, node, vectype, misalign,
2075 cost, where);
2076 return cost;
2077}
2078
2079inline unsigned
2081 enum vect_cost_model_location where)
2082{
2084 || kind == scalar_stmt);
2085 return add_stmt_cost (costs, count, kind, NULL, NULL, NULL_TREE, 0, where);
2086}
2087
2088inline unsigned
2090{
2091 return add_stmt_cost (costs, i->count, i->kind, i->stmt_info, i->node,
2092 i->vectype, i->misalign, i->where);
2093}
2094
2095inline void
2097{
2098 stmt_info_for_cost *cost;
2099 unsigned i;
2100 FOR_EACH_VEC_ELT (*cost_vec, i, cost)
2101 add_stmt_cost (costs, cost->count, cost->kind, cost->stmt_info,
2102 cost->node, cost->vectype, cost->misalign, cost->where);
2103}
2104
2105/*-----------------------------------------------------------------*/
2106/* Info on data references alignment. */
2107/*-----------------------------------------------------------------*/
2108#define DR_MISALIGNMENT_UNKNOWN (-1)
2109#define DR_MISALIGNMENT_UNINITIALIZED (-2)
2110
2111inline void
2113{
2114 dr_info->misalignment = val;
2115}
2116
2117extern int dr_misalignment (dr_vec_info *dr_info, tree vectype,
2118 poly_int64 offset = 0);
2119
2120#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
2121
2122/* Only defined once DR_MISALIGNMENT is defined. */
2123inline const poly_uint64
2125{
2128 return dr_info->target_alignment;
2129}
2130#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR)
2131#define DR_SCALAR_KNOWN_BOUNDS(DR) (DR)->scalar_access_known_in_bounds
2132
2133/* Return if the stmt_vec_info requires peeling for alignment. */
2134inline bool
2136{
2138 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2140 else
2141 dr_info = STMT_VINFO_DR_INFO (stmt_info);
2142
2143 return dr_info->safe_speculative_read_required;
2144}
2145
2146/* Set the safe_speculative_read_required for the the stmt_vec_info, if group
2147 access then set on the fist element otherwise set on DR directly. */
2148inline void
2150 bool requires_alignment)
2151{
2153 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2155 else
2156 dr_info = STMT_VINFO_DR_INFO (stmt_info);
2157
2158 dr_info->safe_speculative_read_required = requires_alignment;
2159}
2160
2161inline void
2163{
2164 dr_info->target_alignment = val;
2165}
2166#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL)
2167
2168/* Return true if data access DR_INFO is aligned to the targets
2169 preferred alignment for VECTYPE (which may be less than a full vector). */
2170
2171inline bool
2173{
2174 return (dr_misalignment (dr_info, vectype) == 0);
2175}
2176
2177/* Return TRUE if the (mis-)alignment of the data access is known with
2178 respect to the targets preferred alignment for VECTYPE, and FALSE
2179 otherwise. */
2180
2181inline bool
2186
2187/* Return the minimum alignment in bytes that the vectorized version
2188 of DR_INFO is guaranteed to have. */
2189
2190inline unsigned int
2192 poly_int64 offset = 0)
2193{
2194 int misalignment = dr_misalignment (dr_info, vectype, offset);
2195 if (misalignment == DR_MISALIGNMENT_UNKNOWN)
2197 else if (misalignment == 0)
2198 return known_alignment (DR_TARGET_ALIGNMENT (dr_info));
2199 return misalignment & -misalignment;
2200}
2201
2202/* Return the behavior of DR_INFO with respect to the vectorization context
2203 (which for outer loop vectorization might not be the behavior recorded
2204 in DR_INFO itself). */
2205
2208{
2209 stmt_vec_info stmt_info = dr_info->stmt;
2210 loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
2211 if (loop_vinfo == NULL
2212 || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info))
2213 return &DR_INNERMOST (dr_info->dr);
2214 else
2215 return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
2216}
2217
2218/* Return the offset calculated by adding the offset of this DR_INFO to the
2219 corresponding data_reference's offset. If CHECK_OUTER then use
2220 vect_dr_behavior to select the appropriate data_reference to use. */
2221
2222inline tree
2224 dr_vec_info *dr_info, bool check_outer = false)
2225{
2227 if (check_outer)
2228 base = vect_dr_behavior (vinfo, dr_info);
2229 else
2230 base = &dr_info->dr->innermost;
2231
2232 tree offset = base->offset;
2233
2234 if (!dr_info->offset)
2235 return offset;
2236
2237 offset = fold_convert (sizetype, offset);
2238 return fold_build2 (PLUS_EXPR, TREE_TYPE (dr_info->offset), offset,
2239 dr_info->offset);
2240}
2241
2242
2243/* Return the vect cost model for LOOP. */
2244inline enum vect_cost_model
2246{
2247 if (loop != NULL
2249 && flag_simd_cost_model != VECT_COST_MODEL_DEFAULT)
2250 return flag_simd_cost_model;
2251 return flag_vect_cost_model;
2252}
2253
2254/* Return true if the vect cost model is unlimited. */
2255inline bool
2260
2261/* Return true if the loop described by LOOP_VINFO is fully-masked and
2262 if the first iteration should use a partial mask in order to achieve
2263 alignment. */
2264
2265inline bool
2267{
2268 /* With early break vectorization we don't know whether the accesses will stay
2269 inside the loop or not. TODO: The early break adjustment code can be
2270 implemented the same way as vectorizable_linear_induction. However we
2271 can't test this today so reject it. */
2272 return (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
2273 && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
2274 && !(LOOP_VINFO_NON_LINEAR_IV (loop_vinfo)
2275 && LOOP_VINFO_EARLY_BREAKS (loop_vinfo)));
2276}
2277
2278/* Return the number of vectors of type VECTYPE that are needed to get
2279 NUNITS elements. NUNITS should be based on the vectorization factor,
2280 so it is always a known multiple of the number of elements in VECTYPE. */
2281
2282inline unsigned int
2284{
2285 return exact_div (nunits, TYPE_VECTOR_SUBPARTS (vectype)).to_constant ();
2286}
2287
2288/* Return the number of vectors in the context of vectorization region VINFO,
2289 needed for a group of statements and a vector type as specified by NODE. */
2290
2291inline unsigned int
2293{
2294 poly_uint64 vf;
2295
2296 if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
2297 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2298 else
2299 vf = 1;
2300
2301 vf *= SLP_TREE_LANES (node);
2302 tree vectype = SLP_TREE_VECTYPE (node);
2303
2304 return vect_get_num_vectors (vf, vectype);
2305}
2306
2307/* Update maximum unit count *MAX_NUNITS so that it accounts for
2308 NUNITS. *MAX_NUNITS can be 1 if we haven't yet recorded anything. */
2309
2310inline void
2312{
2313 /* All unit counts have the form vec_info::vector_size * X for some
2314 rational X, so two unit sizes must have a common multiple.
2315 Everything is a multiple of the initial value of 1. */
2316 *max_nunits = force_common_multiple (*max_nunits, nunits);
2317}
2318
2319/* Update maximum unit count *MAX_NUNITS so that it accounts for
2320 the number of units in vector type VECTYPE. *MAX_NUNITS can be 1
2321 if we haven't yet recorded any vector types. */
2322
2323inline void
2325{
2326 vect_update_max_nunits (max_nunits, TYPE_VECTOR_SUBPARTS (vectype));
2327}
2328
2329/* Return the vectorization factor that should be used for costing
2330 purposes while vectorizing the loop described by LOOP_VINFO.
2331 Pick a reasonable estimate if the vectorization factor isn't
2332 known at compile time. */
2333
2334inline unsigned int
2336{
2337 return estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
2338}
2339
2340/* Estimate the number of elements in VEC_TYPE for costing purposes.
2341 Pick a reasonable estimate if the exact number isn't known at
2342 compile time. */
2343
2344inline unsigned int
2346{
2347 return estimated_poly_value (TYPE_VECTOR_SUBPARTS (vec_type));
2348}
2349
2350/* Return the maximum possible vectorization factor for LOOP_VINFO. */
2351
2352inline unsigned HOST_WIDE_INT
2354{
2355 unsigned HOST_WIDE_INT vf;
2356 if (LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&vf))
2357 return vf;
2359}
2360
2361/* Return the size of the value accessed by unvectorized data reference
2362 DR_INFO. This is only valid once STMT_VINFO_VECTYPE has been calculated
2363 for the associated gimple statement, since that guarantees that DR_INFO
2364 accesses either a scalar or a scalar equivalent. ("Scalar equivalent"
2365 here includes things like V1SI, which can be vectorized in the same way
2366 as a plain SI.) */
2367
2368inline unsigned int
2373
2374/* Return true if LOOP_VINFO requires a runtime check for whether the
2375 vector loop is profitable. */
2376
2377inline bool
2379{
2380 unsigned int th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
2381 return (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2382 && th >= vect_vf_for_cost (loop_vinfo));
2383}
2384
2385/* Return true if CODE is a lane-reducing opcode. */
2386
2387inline bool
2389{
2390 return code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR;
2391}
2392
2393/* Return true if STMT is a lane-reducing statement. */
2394
2395inline bool
2397{
2398 if (auto *assign = dyn_cast <gassign *> (stmt))
2400 return false;
2401}
2402
2403/* Source location + hotness information. */
2405
2406/* A macro for calling:
2407 dump_begin_scope (MSG, vect_location);
2408 via an RAII object, thus printing "=== MSG ===\n" to the dumpfile etc,
2409 and then calling
2410 dump_end_scope ();
2411 once the object goes out of scope, thus capturing the nesting of
2412 the scopes.
2413
2414 These scopes affect dump messages within them: dump messages at the
2415 top level implicitly default to MSG_PRIORITY_USER_FACING, whereas those
2416 in a nested scope implicitly default to MSG_PRIORITY_INTERNALS. */
2417
2418#define DUMP_VECT_SCOPE(MSG) \
2419 AUTO_DUMP_SCOPE (MSG, vect_location)
2420
2421/* A sentinel class for ensuring that the "vect_location" global gets
2422 reset at the end of a scope.
2423
2424 The "vect_location" global is used during dumping and contains a
2425 location_t, which could contain references to a tree block via the
2426 ad-hoc data. This data is used for tracking inlining information,
2427 but it's not a GC root; it's simply assumed that such locations never
2428 get accessed if the blocks are optimized away.
2429
2430 Hence we need to ensure that such locations are purged at the end
2431 of any operations using them (e.g. via this class). */
2432
2434{
2435 public:
2437};
2438
2439/*-----------------------------------------------------------------*/
2440/* Function prototypes. */
2441/*-----------------------------------------------------------------*/
2442
2443/* Simple loop peeling and versioning utilities for vectorizer's purposes -
2444 in tree-vect-loop-manip.cc. */
2445extern void vect_set_loop_condition (class loop *, edge, loop_vec_info,
2446 tree, tree, tree, bool);
2447extern bool slpeel_can_duplicate_loop_p (const class loop *, const_edge,
2448 const_edge);
2450 class loop *, edge,
2451 edge, edge *, bool = true,
2454extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
2455 tree *, tree *, tree *, int, bool, bool,
2456 tree *);
2462extern edge vec_init_loop_exit_info (class loop *);
2464
2465/* In tree-vect-stmts.cc. */
2466extern tree get_related_vectype_for_scalar_type (machine_mode, tree,
2467 poly_uint64 = 0);
2468extern tree get_vectype_for_scalar_type (vec_info *, tree, unsigned int = 0);
2470extern tree get_mask_type_for_scalar_type (vec_info *, tree, unsigned int = 0);
2473extern bool vect_chooses_same_modes_p (vec_info *, machine_mode);
2474extern bool vect_chooses_same_modes_p (machine_mode, machine_mode);
2476extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
2477 stmt_vec_info * = NULL, gimple ** = NULL);
2478extern bool vect_is_simple_use (vec_info *, slp_tree,
2479 unsigned, tree *, slp_tree *,
2480 enum vect_def_type *,
2481 tree *, stmt_vec_info * = NULL);
2487 int*, vec<tree> *);
2489 code_helper *, int *,
2490 vec<tree> *);
2492 tree, tree,
2493 vec<std::pair<tree, tree_code> > &,
2494 tree = NULL_TREE,
2495 slp_tree = NULL);
2497
2498extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2500 tree, int, enum vect_cost_model_location);
2501extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2503 tree, int, enum vect_cost_model_location);
2504extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2505 enum vect_cost_for_stmt,
2507extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2509 slp_tree, tree, int,
2511
2512/* Overload of record_stmt_cost with VECTYPE derived from STMT_INFO. */
2513
2514inline unsigned
2516 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
2517 int misalign, enum vect_cost_model_location where)
2518{
2519 return record_stmt_cost (body_cost_vec, count, kind, stmt_info,
2520 STMT_VINFO_VECTYPE (stmt_info), misalign, where);
2521}
2522
2523/* Overload of record_stmt_cost with VECTYPE derived from SLP node. */
2524
2525inline unsigned
2527 enum vect_cost_for_stmt kind, slp_tree node,
2528 int misalign, enum vect_cost_model_location where)
2529{
2530 return record_stmt_cost (body_cost_vec, count, kind, node,
2531 SLP_TREE_VECTYPE (node), misalign, where);
2532}
2533
2540 tree, vec<tree> *,
2541 tree = NULL, vec<tree> * = NULL,
2542 tree = NULL, vec<tree> * = NULL,
2543 tree = NULL, vec<tree> * = NULL);
2546extern tree vect_get_slp_vect_def (slp_tree, unsigned);
2555 dr_alignment_support, int, bool,
2556 unsigned int *, unsigned int *,
2558 stmt_vector_for_cost *, bool);
2561 unsigned int *, stmt_vector_for_cost *);
2562extern bool vect_supportable_shift (vec_info *, enum tree_code, tree);
2565extern void optimize_mask_stores (class loop*);
2567 const char * = nullptr);
2571 tree *, unsigned int = 0);
2573
2574/* In tree-if-conv.cc. */
2575extern bool ref_within_array_bound (gimple *, tree);
2576
2577/* In tree-vect-data-refs.cc. */
2580 (vec_info *, dr_vec_info *, tree, int,
2581 bool = false);
2590extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
2591 tree, int, internal_fn *, tree *,
2592 vec<int> * = nullptr);
2595 vec<int> * = nullptr);
2600 vec<int> *, int);
2601extern opt_result vect_analyze_data_refs (vec_info *, bool *);
2604 stmt_vec_info, tree, class loop *, tree,
2606 gimple **, bool,
2607 tree = NULL_TREE);
2610extern void vect_copy_ref_info (tree, tree);
2612extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT);
2613extern internal_fn vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, bool);
2614extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
2615extern internal_fn vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT,
2616 bool, vec<int> * = nullptr);
2620 class loop **);
2621extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
2623 const char * = NULL);
2626 tree);
2627
2628/* In tree-vect-loop.cc. */
2629extern tree neutral_op_for_reduction (tree, code_helper, tree, bool = true);
2632/* Used in tree-vect-loop-manip.cc */
2634/* Used in gimple-loop-interchange.c and tree-parloops.cc. */
2636 enum tree_code);
2638/* Drive for loop analysis stage. */
2640 vec_info_shared *);
2643 tree *, bool);
2644extern tree vect_halve_mask_nunits (tree, machine_mode);
2645extern tree vect_double_mask_nunits (tree, machine_mode);
2647 unsigned int, tree, tree);
2650 unsigned int, tree, unsigned int);
2651extern void vect_record_loop_len (loop_vec_info, vec_loop_lens *, unsigned int,
2652 tree, unsigned int);
2654 vec_loop_lens *, unsigned int, tree,
2655 unsigned int, unsigned int);
2658 unsigned int, tree, tree, unsigned int,
2659 unsigned int);
2663
2664/* Drive for loop transformation stage. */
2665extern class loop *vect_transform_loop (loop_vec_info, gimple *);
2675extern opt_result vect_analyze_loop_form (class loop *, gimple *,
2678 const vect_loop_form_info *,
2679 loop_vec_info = nullptr);
2681 slp_tree, slp_instance, int,
2682 bool, stmt_vector_for_cost *);
2692 slp_tree);
2704extern bool vect_emulated_vector_p (tree);
2707extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
2712
2713/* Nonlinear induction. */
2716
2717/* In tree-vect-slp.cc. */
2718extern void vect_slp_init (void);
2719extern void vect_slp_fini (void);
2723 bool, unsigned *,
2724 unsigned * = nullptr, bool = false);
2728extern void vect_schedule_slp (vec_info *, const vec<slp_instance> &);
2729extern opt_result vect_analyze_slp (vec_info *, unsigned, bool);
2732extern void vect_optimize_slp (vec_info *);
2733extern void vect_gather_slp_loads (vec_info *);
2734extern tree vect_get_slp_scalar_def (slp_tree, unsigned);
2735extern void vect_get_slp_defs (slp_tree, vec<tree> *);
2736extern void vect_get_slp_defs (vec_info *, slp_tree, vec<vec<tree> > *,
2737 unsigned n = -1U);
2738extern bool vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop);
2739extern bool vect_slp_function (function *);
2743extern bool can_duplicate_and_interleave_p (vec_info *, unsigned int, tree,
2744 unsigned int * = NULL,
2745 tree * = NULL, tree * = NULL);
2747 const vec<tree> &, unsigned int, vec<tree> &);
2750extern void vect_free_slp_tree (slp_tree);
2751extern bool compatible_calls_p (gcall *, gcall *, bool);
2752extern int vect_slp_child_index_for_operand (const gimple *, int op, bool);
2753
2756extern tree vect_get_mask_load_else (int, tree);
2757
2758/* In tree-vect-patterns.cc. */
2759extern void
2761extern bool vect_get_range_info (tree, wide_int*, wide_int*);
2762
2763/* Pattern recognition functions.
2764 Additional pattern recognition functions can (and will) be added
2765 in the future. */
2767
2768/* In tree-vectorizer.cc. */
2769unsigned vectorize_loops (void);
2771gimple *vect_loop_vectorized_call (class loop *, gcond **cond = NULL);
2773
2774/* SLP Pattern matcher types, tree-vect-slp-patterns.cc. */
2775
2776/* Forward declaration of possible two operands operation that can be matched
2777 by the complex numbers pattern matchers. */
2778enum _complex_operation : unsigned;
2779
2780/* All possible load permute values that could result from the partial data-flow
2781 analysis. */
2791
2792/* Cache from nodes to the load permutation they represent. */
2793typedef hash_map <slp_tree, complex_perm_kinds_t>
2795
2796/* Cache from nodes pair to being compatible or not. */
2797typedef pair_hash <nofree_ptr_hash <_slp_tree>,
2798 nofree_ptr_hash <_slp_tree>> slp_node_hash;
2799typedef hash_map <slp_node_hash, bool> slp_compat_nodes_map_t;
2800
2801
2802/* Vector pattern matcher base class. All SLP pattern matchers must inherit
2803 from this type. */
2804
2806{
2807 protected:
2808 /* The number of arguments that the IFN requires. */
2809 unsigned m_num_args;
2810
2811 /* The internal function that will be used when a pattern is created. */
2813
2814 /* The current node being inspected. */
2816
2817 /* The list of operands to be the children for the node produced when the
2818 internal function is created. */
2820
2821 /* Default constructor where NODE is the root of the tree to inspect. */
2823 {
2824 this->m_ifn = ifn;
2825 this->m_node = node;
2826 this->m_ops.create (0);
2827 if (m_ops)
2828 this->m_ops.safe_splice (*m_ops);
2829 }
2830
2831 public:
2832
2833 /* Create a new instance of the pattern matcher class of the given type. */
2836
2837 /* Build the pattern from the data collected so far. */
2838 virtual void build (vec_info *) = 0;
2839
2840 /* Default destructor. */
2841 virtual ~vect_pattern ()
2842 {
2843 this->m_ops.release ();
2844 }
2845};
2846
2847/* Function pointer to create a new pattern matcher from a generic type. */
2848typedef vect_pattern* (*vect_pattern_decl_t) (slp_tree_to_load_perm_map_t *,
2850 slp_tree *);
2851
2852/* List of supported pattern matchers. */
2854
2855/* Number of supported pattern matchers. */
2856extern size_t num__slp_patterns;
2857
2858/* ----------------------------------------------------------------------
2859 Target support routines
2860 -----------------------------------------------------------------------
2861 The following routines are provided to simplify costing decisions in
2862 target code. Please add more as needed. */
2863
2864/* Return true if an operaton of kind KIND for STMT_INFO represents
2865 the extraction of an element from a vector in preparation for
2866 storing the element to memory. */
2867inline bool
2869{
2870 return (kind == vec_to_scalar
2871 && STMT_VINFO_DATA_REF (stmt_info)
2872 && DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)));
2873}
2874
2875/* Return true if STMT_INFO represents part of a reduction. */
2876inline bool
2878{
2879 return STMT_VINFO_REDUC_IDX (stmt_info) != -1;
2880}
2881
2882/* Return true if SLP_NODE represents part of a reduction. */
2883inline bool
2885{
2886 return SLP_TREE_REDUC_IDX (slp_node) != -1;
2887}
2888
2889/* If STMT_INFO describes a reduction, return the vect_reduction_type
2890 of the reduction it describes, otherwise return -1. */
2891inline int
2893{
2894 if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
2895 {
2896 vect_reduc_info reduc_info = info_for_reduction (loop_vinfo, node);
2897 if (reduc_info)
2898 return int (VECT_REDUC_INFO_TYPE (reduc_info));
2899 }
2900 return -1;
2901}
2902
2903/* If STMT_INFO is a COND_EXPR that includes an embedded comparison, return the
2904 scalar type of the values being compared. Return null otherwise. */
2905inline tree
2907{
2908 if (auto *assign = dyn_cast<gassign *> (stmt_info->stmt))
2909 if (gimple_assign_rhs_code (assign) == COND_EXPR)
2910 {
2911 tree cond = gimple_assign_rhs1 (assign);
2912 if (COMPARISON_CLASS_P (cond))
2913 return TREE_TYPE (TREE_OPERAND (cond, 0));
2914 }
2915 return NULL_TREE;
2916}
2917
2918/* If STMT_INFO is a comparison or contains an embedded comparison, return the
2919 scalar type of the values being compared. Return null otherwise. */
2920inline tree
2922{
2923 if (auto *assign = dyn_cast<gassign *> (stmt_info->stmt))
2925 return TREE_TYPE (gimple_assign_rhs1 (assign));
2926 return vect_embedded_comparison_type (stmt_info);
2927}
2928
2929/* Return true if STMT_INFO extends the result of a load. */
2930inline bool
2932{
2933 /* Although this is quite large for an inline function, this part
2934 at least should be inline. */
2935 gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
2936 if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
2937 return false;
2938
2939 tree rhs = gimple_assign_rhs1 (stmt_info->stmt);
2940 tree lhs_type = TREE_TYPE (gimple_assign_lhs (assign));
2941 tree rhs_type = TREE_TYPE (rhs);
2942 if (!INTEGRAL_TYPE_P (lhs_type)
2943 || !INTEGRAL_TYPE_P (rhs_type)
2944 || TYPE_PRECISION (lhs_type) <= TYPE_PRECISION (rhs_type))
2945 return false;
2946
2947 stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
2948 return (def_stmt_info
2949 && STMT_VINFO_DATA_REF (def_stmt_info)
2950 && DR_IS_READ (STMT_VINFO_DATA_REF (def_stmt_info)));
2951}
2952
2953/* Return true if STMT_INFO is an integer truncation. */
2954inline bool
2956{
2957 gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
2958 if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
2959 return false;
2960
2961 tree lhs_type = TREE_TYPE (gimple_assign_lhs (assign));
2962 tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (assign));
2963 return (INTEGRAL_TYPE_P (lhs_type)
2964 && INTEGRAL_TYPE_P (rhs_type)
2965 && TYPE_PRECISION (lhs_type) < TYPE_PRECISION (rhs_type));
2966}
2967
2968/* Build a GIMPLE_ASSIGN or GIMPLE_CALL with the tree_code,
2969 or internal_fn contained in ch, respectively. */
2971#endif /* GCC_TREE_VECTORIZER_H */
edge loop_preheader_edge(const class loop *loop)
Definition cfgloop.cc:1760
class loop * loop_p
Definition cfgloop.h:98
Definition tree-vectorizer.h:1384
vec< slp_root > roots
Definition tree-vectorizer.h:1389
_bb_vec_info(vec< basic_block > bbs, vec_info_shared *)
Definition tree-vect-slp.cc:8234
~_bb_vec_info()
Definition tree-vect-slp.cc:8271
Definition tree-vectorizer.h:914
tree mask_skip_niters_pfa_offset
Definition tree-vectorizer.h:1010
bool peeling_for_gaps
Definition tree-vectorizer.h:1145
bool can_use_partial_vectors_p
Definition tree-vectorizer.h:1105
poly_uint64 ptr_mask
Definition tree-vectorizer.h:1046
signed char partial_load_store_bias
Definition tree-vectorizer.h:1140
class vector_costs * vector_costs
Definition tree-vectorizer.h:932
class dr_vec_info * unaligned_dr
Definition tree-vectorizer.h:1033
_loop_vec_info(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:711
edge skip_main_loop_edge
Definition tree-vectorizer.h:959
unsigned int th
Definition tree-vectorizer.h:940
_loop_vec_info * epilogue_vinfo
Definition tree-vectorizer.h:1205
_loop_vec_info * main_loop_info
Definition tree-vectorizer.h:1196
auto_vec< gimple * > early_break_vuses
Definition tree-vectorizer.h:1234
bool using_partial_vectors_p
Definition tree-vectorizer.h:1112
bool no_data_dependencies
Definition tree-vectorizer.h:1182
class loop * scalar_loop
Definition tree-vectorizer.h:1192
tree drs_advanced_by
Definition tree-vectorizer.h:1208
scalar_cond_masked_set_type scalar_cond_masked_set
Definition tree-vectorizer.h:996
auto_vec< ddr_p > may_alias_ddrs
Definition tree-vectorizer.h:1056
tree num_itersm1
Definition tree-vectorizer.h:923
unsigned int inner_loop_cost_factor
Definition tree-vectorizer.h:1096
bool vectorizable
Definition tree-vectorizer.h:1099
vect_partial_vector_style partial_vector_style
Definition tree-vectorizer.h:1030
poly_uint64 max_spec_read_amount
Definition tree-vectorizer.h:1049
auto_vec< vect_reduc_info > reduc_infos
Definition tree-vectorizer.h:967
edge vec_epilogue_loop_iv_exit
Definition tree-vectorizer.h:1216
tree num_iters
Definition tree-vectorizer.h:925
hash_map< tree, tree > * scan_map
Definition tree-vectorizer.h:1092
int peeling_for_alignment
Definition tree-vectorizer.h:1043
vec_cond_masked_set_type vec_cond_masked_set
Definition tree-vectorizer.h:999
bool epil_using_partial_vectors_p
Definition tree-vectorizer.h:1128
bool early_breaks
Definition tree-vectorizer.h:1153
poly_uint64 versioning_threshold
Definition tree-vectorizer.h:946
auto_vec< vec_lower_bound > lower_bounds
Definition tree-vectorizer.h:1071
profile_probability scalar_loop_scaling
Definition tree-vectorizer.h:1188
auto_vec< gimple * > early_break_stores
Definition tree-vectorizer.h:1226
edge main_loop_edge
Definition tree-vectorizer.h:958
edge skip_this_loop_edge
Definition tree-vectorizer.h:963
bool must_use_partial_vectors_p
Definition tree-vectorizer.h:1108
auto_vec< stmt_info_for_cost > scalar_cost_vec
Definition tree-vectorizer.h:1085
gcond * loop_iv_cond
Definition tree-vectorizer.h:1159
basic_block early_break_dest_bb
Definition tree-vectorizer.h:1230
bool using_select_vl_p
Definition tree-vectorizer.h:1124
tree rgroup_compare_type
Definition tree-vectorizer.h:1015
poly_uint64 vectorization_factor
Definition tree-vectorizer.h:950
bool peeling_for_niter
Definition tree-vectorizer.h:1149
auto_vec< dr_with_seg_len_pair_t > comp_alias_ddrs
Definition tree-vectorizer.h:1060
edge vec_loop_iv_exit
Definition tree-vectorizer.h:1212
tree num_iters_unchanged
Definition tree-vectorizer.h:927
vec_loop_lens lens
Definition tree-vectorizer.h:993
unsigned HOST_WIDE_INT max_vectorization_factor
Definition tree-vectorizer.h:985
bool allow_mutual_alignment
Definition tree-vectorizer.h:1135
unsigned int suggested_unroll_factor
Definition tree-vectorizer.h:981
hash_map< tree_operand_hash, tree > * ivexpr_map
Definition tree-vectorizer.h:1088
auto_vec< stmt_vec_info > may_misalign_stmts
Definition tree-vectorizer.h:1075
tree rgroup_iv_type
Definition tree-vectorizer.h:1026
vec_loop_masks masks
Definition tree-vectorizer.h:989
tree num_iters_assumptions
Definition tree-vectorizer.h:929
tree simd_if_cond
Definition tree-vectorizer.h:1022
auto_vec< stmt_vec_info > alternate_defs
Definition tree-vectorizer.h:1082
auto_vec< tree > check_nonzero
Definition tree-vectorizer.h:1068
bool user_unroll
Definition tree-vectorizer.h:1163
auto_vec< stmt_vec_info > early_break_live_ivs
Definition tree-vectorizer.h:1238
tree mask_skip_niters
Definition tree-vectorizer.h:1005
bool has_mask_store
Definition tree-vectorizer.h:1185
auto_vec< stmt_vec_info > reductions
Definition tree-vectorizer.h:1078
edge scalar_loop_iv_exit
Definition tree-vectorizer.h:1220
class loop * loop
Definition tree-vectorizer.h:920
bool nonlinear_iv
Definition tree-vectorizer.h:1052
~_loop_vec_info()
Definition tree-vect-loop.cc:835
bool using_decrementing_iv_p
Definition tree-vectorizer.h:1119
auto_vec< gcond * > conds
Definition tree-vectorizer.h:1156
_loop_vec_info * orig_loop_info
Definition tree-vectorizer.h:1201
hash_map< tree, vect_reusable_accumulator > reusable_accumulators
Definition tree-vectorizer.h:974
class vector_costs * scalar_costs
Definition tree-vectorizer.h:935
auto_vec< vec_object_pair > check_unequal_addrs
Definition tree-vectorizer.h:1063
Definition tree-vectorizer.h:400
slp_instance_kind kind
Definition tree-vectorizer.h:427
vec< stmt_vec_info > root_stmts
Definition tree-vectorizer.h:407
slp_tree root
Definition tree-vectorizer.h:403
dump_user_location_t location() const
Definition tree-vect-slp.cc:206
vec< slp_tree > loads
Definition tree-vectorizer.h:414
vec< _slp_instance * > subgraph_entries
Definition tree-vectorizer.h:424
vec< tree > remain_defs
Definition tree-vectorizer.h:411
slp_tree reduc_phis
Definition tree-vectorizer.h:417
stmt_vector_for_cost cost_vec
Definition tree-vectorizer.h:420
Definition tree-vectorizer.h:1474
unsigned int min_input_precision
Definition tree-vectorizer.h:1594
code_helper reduc_code
Definition tree-vectorizer.h:1577
tree vectype
Definition tree-vectorizer.h:1498
bool in_pattern_p
Definition tree-vectorizer.h:1482
stmt_vec_info first_element
Definition tree-vectorizer.h:1544
unsigned int mask_precision
Definition tree-vectorizer.h:1617
int reduc_idx
Definition tree-vectorizer.h:1581
gimple * stmt
Definition tree-vectorizer.h:1495
tree loop_phi_evolution_base_unchanged
Definition tree-vectorizer.h:1516
enum vect_induction_op_type loop_phi_evolution_type
Definition tree-vectorizer.h:1518
innermost_loop_behavior dr_wrt_vec_loop
Definition tree-vectorizer.h:1508
bool slp_vect_only_p
Definition tree-vectorizer.h:1620
bool live
Definition tree-vectorizer.h:1479
unsigned int gap
Definition tree-vectorizer.h:1551
unsigned int min_output_precision
Definition tree-vectorizer.h:1589
unsigned int min_neg_dist
Definition tree-vectorizer.h:1555
bool vectorizable
Definition tree-vectorizer.h:1492
bool pattern_stmt_p
Definition tree-vectorizer.h:1488
bool strided_p
Definition tree-vectorizer.h:1566
enum vect_relevant relevant
Definition tree-vectorizer.h:1560
unsigned int size
Definition tree-vectorizer.h:1548
signop operation_sign
Definition tree-vectorizer.h:1600
gimple_seq pattern_def_seq
Definition tree-vectorizer.h:1534
stmt_vec_info related_stmt
Definition tree-vectorizer.h:1529
unsigned simd_lane_access_p
Definition tree-vectorizer.h:1569
bool slp_vect_pattern_only_p
Definition tree-vectorizer.h:1624
stmt_vec_info reduc_def
Definition tree-vectorizer.h:1585
bool gather_scatter_p
Definition tree-vectorizer.h:1563
enum vect_def_type def_type
Definition tree-vectorizer.h:1537
stmt_vec_info next_element
Definition tree-vectorizer.h:1546
unsigned int operation_precision
Definition tree-vectorizer.h:1599
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:1573
enum slp_vect_type slp_type
Definition tree-vectorizer.h:1540
dr_vec_info dr_aux
Definition tree-vectorizer.h:1504
tree loop_phi_evolution_part
Definition tree-vectorizer.h:1517
Definition tree-vectorizer.h:2434
~auto_purge_vect_location()
Definition tree-vectorizer.cc:94
Definition vec.h:1667
Definition tree.h:82
Definition tree-vectorizer.h:1440
bool scalar_access_known_in_bounds
Definition tree-vectorizer.h:1463
unsigned group
Definition tree-vectorizer.h:1448
tree base_decl
Definition tree-vectorizer.h:1465
bool base_misaligned
Definition tree-vectorizer.h:1455
tree offset
Definition tree-vectorizer.h:1469
int misalignment
Definition tree-vectorizer.h:1450
poly_uint64 target_alignment
Definition tree-vectorizer.h:1453
data_reference * dr
Definition tree-vectorizer.h:1443
bool safe_speculative_read_required
Definition tree-vectorizer.h:1459
stmt_vec_info stmt
Definition tree-vectorizer.h:1445
Definition dumpfile.h:340
Definition hash-map.h:40
Definition hash-set.h:37
Definition inchash.h:38
void add_int(unsigned v)
Definition inchash.h:55
hashval_t end() const
Definition inchash.h:49
void add_flag(bool flag)
Definition inchash.h:124
Definition cfgloop.h:120
class loop * inner
Definition cfgloop.h:148
unsigned force_vectorize
Definition cfgloop.h:227
void * aux
Definition cfgloop.h:154
Definition opt-problem.h:179
Definition profile-count.h:149
Definition tree-vectorizer.h:575
vec< data_reference_p > datarefs
Definition tree-vectorizer.h:584
vec< data_reference > datarefs_copy
Definition tree-vectorizer.h:585
auto_vec< loop_p > loop_nest
Definition tree-vectorizer.h:588
void check_datarefs()
Definition tree-vectorizer.cc:507
void save_datarefs()
Definition tree-vectorizer.cc:497
~vec_info_shared()
Definition tree-vectorizer.cc:489
vec< ddr_p > ddrs
Definition tree-vectorizer.h:592
vec_info_shared()
Definition tree-vectorizer.cc:482
Definition tree-vectorizer.h:596
stmt_vec_info lookup_stmt(gimple *)
Definition tree-vectorizer.cc:571
basic_block * bbs
Definition tree-vectorizer.h:656
auto_vec< stmt_vec_info > grouped_stores
Definition tree-vectorizer.h:641
~vec_info()
Definition tree-vectorizer.cc:474
auto_vec< slp_instance > slp_instances
Definition tree-vectorizer.h:633
void replace_stmt(gimple_stmt_iterator *, stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:660
void remove_stmt(stmt_vec_info)
Definition tree-vectorizer.cc:644
void insert_on_entry(stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:702
void set_vinfo_for_stmt(gimple *, stmt_vec_info, bool=true)
Definition tree-vectorizer.cc:745
vec_base_alignments base_alignments
Definition tree-vectorizer.h:637
machine_mode vector_mode
Definition tree-vectorizer.h:649
stmt_vec_info resync_stmt_addr(gimple *)
Definition tree-vectorizer.cc:551
stmt_vec_info new_stmt_vec_info(gimple *stmt)
Definition tree-vectorizer.cc:713
class dr_vec_info * lookup_dr(data_reference *)
Definition tree-vectorizer.cc:613
vec< stmt_vec_info > stmt_vec_infos
Definition tree-vectorizer.h:624
void move_dr(stmt_vec_info, stmt_vec_info)
Definition tree-vectorizer.cc:625
bool any_known_not_updated_vssa
Definition tree-vectorizer.h:630
vec_kind
Definition tree-vectorizer.h:599
@ loop
Definition tree-vectorizer.h:599
@ bb
Definition tree-vectorizer.h:599
void free_stmt_vec_infos()
Definition tree-vectorizer.cc:766
vec_info_shared * shared
Definition tree-vectorizer.h:621
gimple_seq inv_pattern_def_seq
Definition tree-vectorizer.h:665
unsigned int nbbs
Definition tree-vectorizer.h:659
stmt_vec_info add_stmt(gimple *)
Definition tree-vectorizer.cc:522
vec_kind kind
Definition tree-vectorizer.h:618
mode_set used_vector_modes
Definition tree-vectorizer.h:644
hash_set< int_hash< machine_mode, E_VOIDmode, E_BLKmode > > mode_set
Definition tree-vectorizer.h:598
void insert_seq_on_entry(stmt_vec_info, gimple_seq)
Definition tree-vectorizer.cc:676
stmt_vec_info lookup_def(tree)
Definition tree-vectorizer.cc:588
void free_stmt_vec_info(stmt_vec_info)
Definition tree-vectorizer.cc:777
stmt_vec_info lookup_single_use(tree)
Definition tree-vectorizer.cc:601
vec_info(vec_kind, vec_info_shared *)
Definition tree-vectorizer.cc:463
stmt_vec_info add_pattern_stmt(gimple *, stmt_vec_info)
Definition tree-vectorizer.cc:534
bool stmt_vec_info_ro
Definition tree-vectorizer.h:626
vec_lower_bound()
Definition tree-vectorizer.h:564
poly_uint64 min_value
Definition tree-vectorizer.h:570
bool unsigned_p
Definition tree-vectorizer.h:569
vec_lower_bound(tree e, bool u, poly_uint64 m)
Definition tree-vectorizer.h:565
tree expr
Definition tree-vectorizer.h:568
Definition vec-perm-indices.h:51
Definition tree-vectorizer.h:2806
static vect_pattern * recognize(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
internal_fn m_ifn
Definition tree-vectorizer.h:2812
vec< slp_tree > m_ops
Definition tree-vectorizer.h:2819
virtual ~vect_pattern()
Definition tree-vectorizer.h:2841
virtual void build(vec_info *)=0
vect_pattern(slp_tree *node, vec< slp_tree > *m_ops, internal_fn ifn)
Definition tree-vectorizer.h:2822
slp_tree * m_node
Definition tree-vectorizer.h:2815
unsigned m_num_args
Definition tree-vectorizer.h:2809
Definition tree-vectorizer.h:827
tree reduc_epilogue_adjustment
Definition tree-vectorizer.h:866
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:835
struct vect_reusable_accumulator * reused_accumulator
Definition tree-vectorizer.h:870
internal_fn reduc_fn
Definition tree-vectorizer.h:841
tree reduc_vectype_for_mask
Definition tree-vectorizer.h:860
bool is_reduc_chain
Definition tree-vectorizer.h:850
code_helper reduc_code
Definition tree-vectorizer.h:838
auto_vec< tree > reduc_initial_values
Definition tree-vectorizer.h:874
tree induc_cond_initial_val
Definition tree-vectorizer.h:863
tree reduc_vectype
Definition tree-vectorizer.h:856
bool force_single_cycle
Definition tree-vectorizer.h:853
enum vect_def_type def_type
Definition tree-vectorizer.h:831
unsigned int reduc_result_pos
Definition tree-vectorizer.h:847
auto_vec< tree > reduc_scalar_results
Definition tree-vectorizer.h:880
Definition tree-vectorizer.h:1723
unsigned int adjust_cost_for_freq(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1874
virtual ~vector_costs()
Definition tree-vectorizer.h:1726
bool m_costing_for_scalar
Definition tree-vectorizer.h:1796
vector_costs(vec_info *, bool)
Definition tree-vectorizer.h:1821
bool costing_for_scalar() const
Definition tree-vectorizer.h:1781
vec_info * m_vinfo
Definition tree-vectorizer.h:1792
virtual unsigned int add_stmt_cost(int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, vect_cost_model_location where)
Definition tree-vectorizer.cc:1836
machine_mode m_suggested_epilogue_mode
Definition tree-vectorizer.h:1810
virtual bool better_main_loop_than_p(const vector_costs *other) const
Definition tree-vectorizer.cc:1894
unsigned int m_costs[3]
Definition tree-vectorizer.h:1799
unsigned int outside_cost() const
Definition tree-vectorizer.h:1862
unsigned int record_stmt_cost(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1860
virtual bool better_epilogue_loop_than_p(const vector_costs *other, loop_vec_info main_loop) const
Definition tree-vectorizer.cc:1913
unsigned int epilogue_cost() const
Definition tree-vectorizer.h:1853
int m_masked_epilogue
Definition tree-vectorizer.h:1811
machine_mode suggested_epilogue_mode(int &masked) const
Definition tree-vectorizer.h:1888
unsigned int m_suggested_unroll_factor
Definition tree-vectorizer.h:1802
unsigned int suggested_unroll_factor() const
Definition tree-vectorizer.h:1879
bool m_finished
Definition tree-vectorizer.h:1814
int compare_outside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:2089
unsigned int body_cost() const
Definition tree-vectorizer.h:1844
virtual void finish_cost(const vector_costs *scalar_costs)
Definition tree-vectorizer.cc:1849
unsigned int prologue_cost() const
Definition tree-vectorizer.h:1835
unsigned int total_cost() const
Definition tree-vectorizer.h:1871
int compare_inside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:1996
struct basic_block_def * basic_block
Definition coretypes.h:370
class edge_def * edge
Definition coretypes.h:367
gimple * gimple_seq
Definition coretypes.h:100
const class edge_def * const_edge
Definition coretypes.h:368
const union tree_node * const_tree
Definition coretypes.h:98
union tree_node * tree
Definition coretypes.h:97
static unsigned int count[debug_counter_number_of_counters]
Definition dbgcnt.cc:50
bool dominated_by_p(enum cdi_direction dir, const_basic_block bb1, const_basic_block bb2)
Definition dominance.cc:1125
@ CDI_DOMINATORS
Definition dominance.h:25
dump_flags_t dump_flags
Definition dumpfile.cc:65
FILE * dump_file
Definition dumpfile.cc:63
@ TDF_DETAILS
Definition dumpfile.h:92
vect_cost_model
Definition flag-types.h:279
@ VECT_COST_MODEL_UNLIMITED
Definition flag-types.h:283
@ VECT_COST_MODEL_DEFAULT
Definition flag-types.h:284
bool operand_equal_p(const_tree arg0, const_tree arg1, unsigned int flags)
Definition fold-const.cc:4355
#define fold_convert(T1, T2)
Definition fold-const.h:96
#define fold_build2(c, t1, t2, t3)
Definition fold-const.h:76
static struct token T
Definition gengtype-parse.cc:45
internal_fn
Definition genmatch.cc:1015
tree_code
Definition genmatch.cc:1002
unsigned gimple_uid(const gimple *g)
Definition gimple.h:2091
enum tree_code gimple_assign_rhs_code(const gassign *gs)
Definition gimple.h:2857
tree gimple_assign_rhs1(const gassign *gs)
Definition gimple.h:2667
basic_block gimple_bb(const gimple *g)
Definition gimple.h:1867
tree gimple_assign_lhs(const gassign *gs)
Definition gimple.h:2616
#define init_cost
Definition ira-costs.cc:63
T dyn_cast(U *p)
Definition is-a.h:280
Definition custom-sarif-properties/state-graphs.h:33
void add_expr(const_tree t, inchash::hash &hstate, unsigned int flags)
Definition fold-const.cc:4369
opt_pointer_wrapper< tree > opt_tree
Definition opt-problem.h:287
poly_int< NUM_POLY_INT_COEFFS, unsigned HOST_WIDE_INT > poly_uint64
Definition poly-int-types.h:25
poly_int< NUM_POLY_INT_COEFFS, HOST_WIDE_INT > poly_int64
Definition poly-int-types.h:24
i
Definition poly-int.h:776
signop
Definition signop.h:28
Definition tree-vectorizer.h:300
stmt_vec_info representative
Definition tree-vectorizer.h:316
tree gs_base
Definition tree-vectorizer.h:352
unsigned int lanes
Definition tree-vectorizer.h:345
bool avoid_stlf_fail
Definition tree-vectorizer.h:358
load_permutation_t load_permutation
Definition tree-vectorizer.h:327
slp_tree prev_node
Definition tree-vectorizer.h:383
enum tree_code code
Definition tree-vectorizer.h:347
bool ldst_lanes
Definition tree-vectorizer.h:355
vec< tree > vec_defs
Definition tree-vectorizer.h:335
~_slp_tree()
Definition tree-vect-slp.cc:141
T & get_data(T &else_)
Definition tree-vectorizer.h:368
int gs_scale
Definition tree-vectorizer.h:350
unsigned int refcnt
Definition tree-vectorizer.h:338
enum stmt_vec_info_type type
Definition tree-vectorizer.h:364
slp_tree next_node
Definition tree-vectorizer.h:382
int vertex
Definition tree-vectorizer.h:360
enum vect_def_type def_type
Definition tree-vectorizer.h:343
bool * failed
Definition tree-vectorizer.h:373
vec< tree > ops
Definition tree-vectorizer.h:313
void push_vec_def(gimple *def)
Definition tree-vect-slp.cc:164
int reduc_idx
Definition tree-vectorizer.h:322
poly_uint64 max_nunits
Definition tree-vectorizer.h:341
lane_permutation_t lane_permutation
Definition tree-vectorizer.h:331
_slp_tree()
Definition tree-vect-slp.cc:108
int id
Definition tree-vectorizer.h:320
struct _slp_tree::@120146070312105203102335350023136130264031166255 cycle_info
tree vectype
Definition tree-vectorizer.h:333
void push_vec_def(tree def)
Definition tree-vectorizer.h:305
vec< slp_tree > children
Definition tree-vectorizer.h:308
vect_data * data
Definition tree-vectorizer.h:365
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:311
class loop * loop_father
Definition basic-block.h:126
Definition ira-costs.cc:54
Definition tree-data-ref.h:151
struct innermost_loop_behavior innermost
Definition tree-data-ref.h:173
Definition loop-invariant.cc:88
static const bool empty_zero_p
Definition tree-vectorizer.h:523
static void mark_empty(value_type &v)
Definition tree-vectorizer.h:526
scalar_cond_masked_key value_type
Definition tree-vectorizer.h:499
static bool equal(value_type existing, value_type candidate)
Definition tree-vectorizer.h:514
static void remove(value_type &)
Definition tree-vectorizer.h:545
static hashval_t hash(value_type v)
Definition tree-vectorizer.h:502
static void mark_deleted(value_type &)
Definition tree-vectorizer.h:538
static bool is_deleted(const value_type &)
Definition tree-vectorizer.h:540
static bool is_empty(value_type v)
Definition tree-vectorizer.h:533
scalar_cond_masked_key compare_type
Definition tree-vectorizer.h:498
Definition hash-traits.h:466
Definition graphite.h:338
data_reference_p dr
Definition graphite.h:343
Definition function.h:249
Definition gimple.h:907
Definition tree-vectorizer.h:1628
tree decl
Definition tree-vectorizer.h:1635
tree element_type
Definition tree-vectorizer.h:1655
tree base
Definition tree-vectorizer.h:1638
tree offset_vectype
Definition tree-vectorizer.h:1652
tree alias_ptr
Definition tree-vectorizer.h:1642
int scale
Definition tree-vectorizer.h:1649
tree offset
Definition tree-vectorizer.h:1645
internal_fn ifn
Definition tree-vectorizer.h:1631
tree memory_type
Definition tree-vectorizer.h:1658
Definition gimple.h:352
Definition gimple.h:861
Definition gimple-iterator.h:26
Definition gimple.h:221
Definition gimple.h:461
Definition tree-data-ref.h:50
tree offset
Definition tree-data-ref.h:52
Definition hash-traits.h:122
static bool test(U *p)
Definition hash-traits.h:345
Definition tree-vectorizer.h:778
tree compare_type
Definition tree-vectorizer.h:798
unsigned int max_nscalars_per_iter
Definition tree-vectorizer.h:782
unsigned int factor
Definition tree-vectorizer.h:788
tree type
Definition tree-vectorizer.h:794
tree bias_adjusted_ctrl
Definition tree-vectorizer.h:805
vec< tree > controls
Definition tree-vectorizer.h:801
unsigned ncopies
Definition tree-vectorizer.h:488
tree_code code
Definition tree-vectorizer.h:490
scalar_cond_masked_key(tree t, unsigned ncopies_)
Definition tree-vectorizer.h:480
void get_cond_ops_from_tree(tree)
Definition tree-vectorizer.cc:1786
tree op0
Definition tree-vectorizer.h:491
tree op1
Definition tree-vectorizer.h:492
bool inverted_p
Definition tree-vectorizer.h:489
vec< tree > remain
Definition tree-vectorizer.h:1380
slp_instance_kind kind
Definition tree-vectorizer.h:1377
vec< stmt_vec_info > roots
Definition tree-vectorizer.h:1379
slp_root(slp_instance_kind kind_, vec< stmt_vec_info > stmts_, vec< stmt_vec_info > roots_, vec< tree > remain_=vNULL)
Definition tree-vectorizer.h:1374
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:1378
Definition tree-vectorizer.h:120
tree vectype
Definition tree-vectorizer.h:126
int misalign
Definition tree-vectorizer.h:127
slp_tree node
Definition tree-vectorizer.h:125
enum vect_cost_model_location where
Definition tree-vectorizer.h:123
enum vect_cost_for_stmt kind
Definition tree-vectorizer.h:122
stmt_vec_info stmt_info
Definition tree-vectorizer.h:124
int count
Definition tree-vectorizer.h:121
Definition tree-hash-traits.h:25
Definition hash-traits.h:75
Definition tree-vectorizer.h:809
hash_set< pair_hash< nofree_ptr_hash< tree_node >, int_hash< unsigned, 0 > > > mask_set
Definition tree-vectorizer.h:814
auto_vec< rgroup_controls > rgc_vec
Definition tree-vectorizer.h:817
bool is_empty() const
Definition tree-vectorizer.h:810
Definition vec.h:450
Definition tree-vectorizer.h:257
virtual ~vect_data()=default
Definition tree-vectorizer.h:276
int misalignment
Definition tree-vectorizer.h:283
tree decl
Definition tree-vectorizer.h:288
virtual ~vect_load_store_data()=default
vect_load_store_data()=default
internal_fn ifn
Definition tree-vectorizer.h:287
unsigned n_perms
Definition tree-vectorizer.h:295
poly_int64 poffset
Definition tree-vectorizer.h:285
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:281
vect_load_store_data(vect_load_store_data &&other)=default
tree strided_offset_vectype
Definition tree-vectorizer.h:290
tree ls_type
Definition tree-vectorizer.h:291
union vect_load_store_data::@336230030006311336100276173266177172342371306160 gs
auto_vec< int > elsvals
Definition tree-vectorizer.h:292
dr_alignment_support alignment_support_scheme
Definition tree-vectorizer.h:282
internal_fn lanes_ifn
Definition tree-vectorizer.h:284
bool slp_perm
Definition tree-vectorizer.h:294
Definition tree-vectorizer.h:2667
gcond * inner_loop_cond
Definition tree-vectorizer.h:2672
auto_vec< gcond * > conds
Definition tree-vectorizer.h:2671
tree number_of_iterationsm1
Definition tree-vectorizer.h:2669
tree assumptions
Definition tree-vectorizer.h:2670
edge loop_exit
Definition tree-vectorizer.h:2673
tree number_of_iterations
Definition tree-vectorizer.h:2668
Definition tree-vectorizer.h:901
tree reduc_input
Definition tree-vectorizer.h:904
vect_reduc_info reduc_info
Definition tree-vectorizer.h:908
Definition tree-vectorizer.h:159
static void mark_deleted(value_type &s)
Definition tree-vectorizer.h:165
static bool is_empty(const value_type &s)
Definition tree-vectorizer.h:168
static bool is_deleted(const value_type &s)
Definition tree-vectorizer.h:167
static const bool empty_zero_p
Definition tree-vectorizer.h:163
static void mark_empty(value_type &s)
Definition tree-vectorizer.h:166
static hashval_t hash(const value_type &)
Definition tree-vect-slp.cc:8452
vect_scalar_ops_slice value_type
Definition tree-vectorizer.h:160
static bool equal(const value_type &, const compare_type &)
Definition tree-vect-slp.cc:8461
vect_scalar_ops_slice compare_type
Definition tree-vectorizer.h:161
Definition tree-vectorizer.h:141
unsigned int length
Definition tree-vectorizer.h:147
unsigned int start
Definition tree-vectorizer.h:146
bool all_same_p() const
Definition tree-vect-slp.cc:8443
tree op(unsigned int i) const
Definition tree-vectorizer.h:152
vec< tree > * ops
Definition tree-vectorizer.h:145
virtual ~vect_simd_clone_data()=default
vect_simd_clone_data(vect_simd_clone_data &&other)=default
vect_simd_clone_data()=default
auto_vec< tree > simd_clone_info
Definition tree-vectorizer.h:271
#define NULL
Definition system.h:50
#define gcc_assert(EXPR)
Definition system.h:814
#define gcc_unreachable()
Definition system.h:841
#define false
Definition system.h:888
#define gcc_checking_assert(EXPR)
Definition system.h:821
HOST_WIDE_INT estimated_poly_value(poly_int64 x, poly_value_estimate_kind kind=POLY_VALUE_LIKELY)
Definition target.h:337
struct gcc_target targetm
vect_cost_model_location
Definition target.h:211
@ vect_body
Definition target.h:213
@ vect_prologue
Definition target.h:212
@ vect_epilogue
Definition target.h:214
vect_cost_for_stmt
Definition target.h:189
@ scalar_stmt
Definition target.h:190
@ cond_branch_taken
Definition target.h:203
@ vec_to_scalar
Definition target.h:200
@ cond_branch_not_taken
Definition target.h:202
@ tcc_comparison
Definition tree-core.h:240
@ vec_kind
Definition tree-core.h:1042
#define DR_IS_WRITE(DR)
Definition tree-data-ref.h:192
#define DR_IS_READ(DR)
Definition tree-data-ref.h:191
#define DR_REF(DR)
Definition tree-data-ref.h:185
#define DR_INNERMOST(DR)
Definition tree-data-ref.h:203
static tree candidate(unsigned uid)
Definition tree-sra.cc:326
#define PHI_ARG_DEF_FROM_EDGE(PHI, E)
Definition tree-ssa-operands.h:84
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset)
Definition tree-vect-data-refs.cc:1303
vect_pattern_decl_t slp_patterns[]
Definition tree-vect-slp-patterns.cc:1703
_complex_operation
Definition tree-vect-slp-patterns.cc:119
size_t num__slp_patterns
Definition tree-vect-slp-patterns.cc:1714
dump_user_location_t vect_location
Definition tree-vectorizer.cc:88
stmt_vec_info vect_orig_stmt(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1957
#define LOOP_VINFO_COST_MODEL_THRESHOLD(L)
Definition tree-vectorizer.h:1255
bool vect_can_vectorize_without_simd_p(tree_code)
Definition tree-vect-loop.cc:8809
tree cse_and_gimplify_to_preheader(loop_vec_info, tree)
Definition tree-vect-loop.cc:859
int builtin_vectorization_cost(enum vect_cost_for_stmt type_of_cost, tree vectype, int misalign)
Definition tree-vectorizer.h:2028
loop_vec_info loop_vec_info_for_loop(class loop *loop)
Definition tree-vectorizer.h:1367
bool vect_is_extending_load(class vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2931
bool vect_transform_stmt(vec_info *, stmt_vec_info, gimple_stmt_iterator *, slp_tree, slp_instance)
Definition tree-vect-stmts.cc:12926
bool vect_is_simple_use(tree, vec_info *, enum vect_def_type *, stmt_vec_info *=NULL, gimple **=NULL)
Definition tree-vect-stmts.cc:13393
bool vect_rgroup_iv_might_wrap_p(loop_vec_info, rgroup_controls *)
Definition tree-vect-loop.cc:11644
tree vect_halve_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:10499
void dump_stmt_cost(FILE *, int, enum vect_cost_for_stmt, stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location)
Definition tree-vectorizer.cc:102
vec< unsigned > load_permutation_t
Definition tree-vectorizer.h:254
bool vect_slp_analyze_instance_alignment(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:3142
internal_fn vect_store_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:6045
tree vect_get_new_vect_var(tree, enum vect_var_kind, const char *)
Definition tree-vect-data-refs.cc:5311
void vect_mark_pattern_stmts(vec_info *, stmt_vec_info, gimple *, tree)
Definition tree-vect-patterns.cc:7435
class loop * vect_loop_versioning(loop_vec_info, gimple *)
Definition tree-vect-loop-manip.cc:4121
tree vect_get_slp_vect_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10807
opt_tree vect_get_mask_type_for_stmt(stmt_vec_info, unsigned int=0)
#define LOOP_VINFO_VECT_FACTOR(L)
Definition tree-vectorizer.h:1267
void vect_free_slp_instance(slp_instance)
Definition tree-vect-slp.cc:218
#define SLP_TREE_LANES(S)
Definition tree-vectorizer.h:450
bool vectorizable_early_exit(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:12502
bool vect_slp_function(function *)
Definition tree-vect-slp.cc:10340
std::pair< tree, tree > vec_object_pair
Definition tree-vectorizer.h:558
tree vect_gen_perm_mask_any(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9360
tree vect_setup_realignment(vec_info *, stmt_vec_info, tree, gimple_stmt_iterator *, tree *, enum dr_alignment_support, tree, class loop **)
Definition tree-vect-data-refs.cc:6121
_bb_vec_info * bb_vec_info
#define LOOP_VINFO_FULLY_MASKED_P(L)
Definition tree-vectorizer.h:1320
int vect_get_known_peeling_cost(loop_vec_info, int, int *, stmt_vector_for_cost *, stmt_vector_for_cost *, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:3908
opt_result vect_analyze_data_refs(vec_info *, bool *)
Definition tree-vect-data-refs.cc:5058
bool compatible_calls_p(gcall *, gcall *, bool)
Definition tree-vect-slp.cc:1000
bool vect_transform_slp_perm_load(vec_info *, slp_tree, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, unsigned *, unsigned *=nullptr, bool=false)
tree vect_gen_while_not(gimple_seq *, tree, tree, tree)
Definition tree-vect-stmts.cc:14247
#define VECT_REDUC_INFO_TYPE(I)
Definition tree-vectorizer.h:886
tree vect_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2921
bool is_simple_and_all_uses_invariant(stmt_vec_info, loop_vec_info)
Definition tree-vect-stmts.cc:322
#define STMT_VINFO_GROUPED_ACCESS(S)
Definition tree-vectorizer.h:1693
void vect_schedule_slp(vec_info *, const vec< slp_instance > &)
Definition tree-vect-slp.cc:12230
tree vect_peel_nonlinear_iv_init(gimple_seq *, tree, tree, tree, enum vect_induction_op_type)
Definition tree-vect-loop.cc:8935
bool check_reduction_path(dump_user_location_t, loop_p, gphi *, tree, enum tree_code)
Definition tree-vect-loop.cc:3626
bool can_duplicate_and_interleave_p(vec_info *, unsigned int, tree, unsigned int *=NULL, tree *=NULL, tree *=NULL)
Definition tree-vect-slp.cc:430
auto_vec< unsigned, 16 > auto_load_permutation_t
Definition tree-vectorizer.h:255
#define SLP_TREE_VECTYPE(S)
Definition tree-vectorizer.h:448
tree get_vectype_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13216
#define LOOP_VINFO_LOOP(L)
Definition tree-vectorizer.h:1242
_complex_perm_kinds
Definition tree-vectorizer.h:2782
@ PERM_EVENEVEN
Definition tree-vectorizer.h:2787
@ PERM_TOP
Definition tree-vectorizer.h:2789
@ PERM_ODDEVEN
Definition tree-vectorizer.h:2785
@ PERM_EVENODD
Definition tree-vectorizer.h:2784
@ PERM_UNKNOWN
Definition tree-vectorizer.h:2783
@ PERM_ODDODD
Definition tree-vectorizer.h:2786
#define SLP_TREE_TYPE(S)
Definition tree-vectorizer.h:452
void vect_record_base_alignments(vec_info *)
Definition tree-vect-data-refs.cc:1384
vec_load_store_type
Definition tree-vectorizer.h:1434
@ VLS_LOAD
Definition tree-vectorizer.h:1435
@ VLS_STORE
Definition tree-vectorizer.h:1436
@ VLS_STORE_INVARIANT
Definition tree-vectorizer.h:1437
tree get_mask_type_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13291
pair_hash< tree_operand_hash, tree_operand_hash > tree_cond_mask_hash
Definition tree-vectorizer.h:553
_loop_vec_info * loop_vec_info
vect_reduc_info info_for_reduction(loop_vec_info, slp_tree)
Definition tree-vect-loop.cc:5016
tree vect_double_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:10509
vect_var_kind
Definition tree-vectorizer.h:36
@ vect_pointer_var
Definition tree-vectorizer.h:38
@ vect_scalar_var
Definition tree-vectorizer.h:39
@ vect_mask_var
Definition tree-vectorizer.h:40
@ vect_simple_var
Definition tree-vectorizer.h:37
int vect_get_place_in_interleaving_chain(stmt_vec_info, stmt_vec_info)
Definition tree-vect-slp.cc:401
class loop * vect_do_peeling(loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, tree *)
Definition tree-vect-loop-manip.cc:3135
void optimize_mask_stores(class loop *)
Definition tree-vect-loop.cc:11408
class _stmt_vec_info * stmt_vec_info
Definition tree-vectorizer.h:24
#define STMT_VINFO_DATA_REF(S)
Definition tree-vectorizer.h:1667
tree vect_get_loop_len(loop_vec_info, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:10696
bool vect_transform_cycle_phi(loop_vec_info, stmt_vec_info, slp_tree, slp_instance)
Definition tree-vect-loop.cc:8205
unsigned int vect_get_num_copies(vec_info *vinfo, slp_tree node)
Definition tree-vectorizer.h:2292
bool vect_is_store_elt_extraction(vect_cost_for_stmt kind, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2868
opt_result vect_get_vector_types_for_stmt(vec_info *, stmt_vec_info, tree *, tree *, unsigned int=0)
Definition tree-vect-stmts.cc:14271
const poly_uint64 dr_target_alignment(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2124
void vect_update_inits_of_drs(loop_vec_info, tree, tree_code)
Definition tree-vect-loop-manip.cc:2615
struct data_reference * dr_p
Definition tree-vectorizer.h:1472
bool vect_can_advance_ivs_p(loop_vec_info)
Definition tree-vect-loop-manip.cc:2211
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vect-data-refs.cc:1303
tree vect_init_vector(vec_info *, stmt_vec_info, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1206
void vect_copy_ref_info(tree, tree)
Definition tree-vect-data-refs.cc:5871
bool vect_transform_reduction(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree)
Definition tree-vect-loop.cc:7861
bool vectorizable_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:8420
bool vectorizable_lane_reducing(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:6747
tree vect_get_new_ssa_name(tree, enum vect_var_kind, const char *=NULL)
Definition tree-vect-data-refs.cc:5349
#define STMT_VINFO_VECTYPE(S)
Definition tree-vectorizer.h:1665
opt_result vect_prune_runtime_alias_test_list(loop_vec_info)
Definition tree-vect-data-refs.cc:4171
void vect_describe_gather_scatter_call(stmt_vec_info, gather_scatter_info *)
Definition tree-vect-data-refs.cc:4535
bool needs_fold_left_reduction_p(tree, code_helper)
Definition tree-vect-loop.cc:3387
bool vect_nop_conversion_p(stmt_vec_info)
Definition tree-vect-stmts.cc:5790
class vect_reduc_info_s * vect_reduc_info
Definition tree-vectorizer.h:883
void vect_finish_stmt_generation(vec_info *, stmt_vec_info, gimple *, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1331
#define LOOP_VINFO_NON_LINEAR_IV(L)
Definition tree-vectorizer.h:1283
unsigned HOST_WIDE_INT vect_max_vf(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2353
internal_fn vect_load_lanes_supported(tree, unsigned HOST_WIDE_INT, bool, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:6475
loop_vec_info vect_create_loop_vinfo(class loop *, vec_info_shared *, const vect_loop_form_info *, loop_vec_info=nullptr)
Definition tree-vect-loop.cc:1639
bool vect_stmt_dominates_stmt_p(gimple *, gimple *)
Definition tree-vectorizer.cc:793
tree vect_get_slp_scalar_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10789
bool vect_supportable_shift(vec_info *, enum tree_code, tree)
Definition tree-vect-stmts.cc:5969
void vect_optimize_slp(vec_info *)
Definition tree-vect-slp.cc:8105
void vect_iv_increment_position(edge, gimple_stmt_iterator *, bool *)
Definition tree-vect-loop-manip.cc:457
hash_set< scalar_cond_masked_key > scalar_cond_masked_set_type
Definition tree-vectorizer.h:548
tree vect_phi_initial_value(gphi *phi)
Definition tree-vectorizer.h:1927
opt_pointer_wrapper< loop_vec_info > opt_loop_vec_info
Definition tree-vectorizer.h:1364
bool vectorizable_reduction(loop_vec_info, stmt_vec_info, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:6907
unsigned record_stmt_cost(stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location)
Definition tree-vect-stmts.cc:113
bool vect_emulated_vector_p(tree)
Definition tree-vect-loop.cc:8798
tree get_same_sized_vectype(tree, tree)
Definition tree-vect-stmts.cc:13327
peeling_support
Definition tree-vectorizer.h:61
@ peeling_known_supported
Definition tree-vectorizer.h:62
@ peeling_maybe_supported
Definition tree-vectorizer.h:63
@ peeling_unsupported
Definition tree-vectorizer.h:64
gimple * vect_gimple_build(tree, code_helper, tree, tree=NULL_TREE)
Definition tree-vect-patterns.cc:7760
void add_stmt_costs(vector_costs *costs, stmt_vector_for_cost *cost_vec)
Definition tree-vectorizer.h:2096
void vect_finish_replace_stmt(vec_info *, stmt_vec_info, gimple *)
Definition tree-vect-stmts.cc:1315
tree prepare_vec_mask(loop_vec_info, tree, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1608
void vect_slp_fini(void)
Definition tree-vect-slp.cc:83
#define LOOP_VINFO_NITERS_KNOWN_P(L)
Definition tree-vectorizer.h:1351
stmt_vec_info get_later_stmt(stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
Definition tree-vectorizer.h:1967
vec< std::pair< unsigned, unsigned > > lane_permutation_t
Definition tree-vectorizer.h:252
tree vect_gen_loop_len_mask(loop_vec_info, gimple_stmt_iterator *, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:10763
widest_int vect_iv_limit_for_partial_vectors(loop_vec_info loop_vinfo)
Definition tree-vect-loop.cc:11603
void vect_get_vec_defs(vec_info *, slp_tree, tree, vec< tree > *, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL)
Definition tree-vect-stmts.cc:1268
void vect_gather_slp_loads(vec_info *)
Definition tree-vect-slp.cc:8124
vect_partial_vector_style
Definition tree-vectorizer.h:467
@ vect_partial_vectors_none
Definition tree-vectorizer.h:468
@ vect_partial_vectors_len
Definition tree-vectorizer.h:471
@ vect_partial_vectors_avx512
Definition tree-vectorizer.h:470
@ vect_partial_vectors_while_ult
Definition tree-vectorizer.h:469
bool vect_gather_scatter_fn_p(vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4443
class _slp_instance * slp_instance
bool vect_maybe_update_slp_op_vectype(slp_tree, tree)
Definition tree-vect-stmts.cc:13544
#define STMT_VINFO_REDUC_IDX(S)
Definition tree-vectorizer.h:1671
class loop * slpeel_tree_duplicate_loop_to_edge_cfg(class loop *, edge, class loop *, edge, edge, edge *, bool=true, vec< basic_block > *=NULL)
Definition tree-vect-loop-manip.cc:1478
unsigned int vect_vf_for_cost(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2335
unsigned int vect_nunits_for_cost(tree vec_type)
Definition tree-vectorizer.h:2345
bool vect_get_range_info(tree, wide_int *, wide_int *)
Definition tree-vect-patterns.cc:69
bool vect_grouped_load_supported(tree, bool, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:6358
#define STMT_VINFO_IN_PATTERN_P(S)
Definition tree-vectorizer.h:1689
tree vect_create_addr_base_for_vector_ref(vec_info *, stmt_vec_info, gimple_seq *, tree)
Definition tree-vect-data-refs.cc:5424
stmt_vec_info vect_find_last_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3532
tree vect_get_main_loop_result(loop_vec_info, tree, tree)
Definition tree-vect-loop-manip.cc:3026
#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L)
Definition tree-vectorizer.h:1282
#define STMT_VINFO_DR_INFO(S)
Definition tree-vectorizer.h:1686
hash_map< slp_node_hash, bool > slp_compat_nodes_map_t
Definition tree-vectorizer.h:2799
tree get_related_vectype_for_scalar_type(machine_mode, tree, poly_uint64=0)
Definition tree-vect-stmts.cc:13098
bool lane_reducing_op_p(code_helper code)
Definition tree-vectorizer.h:2388
bool nested_in_vect_loop_p(class loop *loop, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1916
bool vect_slp_analyze_instance_dependence(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:1164
bool vect_detect_hybrid_slp(loop_vec_info)
auto_vec< std::pair< data_reference *, tree > > drs_init_vec
Definition tree-vectorizer.h:822
edge vec_init_loop_exit_info(class loop *)
Definition tree-vect-loop.cc:655
opt_result vect_analyze_data_ref_accesses(vec_info *, vec< int > *)
Definition tree-vect-data-refs.cc:3636
int vect_get_stmt_cost(enum vect_cost_for_stmt type_of_cost)
Definition tree-vectorizer.h:2038
opt_result vect_analyze_stmt(vec_info *, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:12776
dump_user_location_t find_loop_location(class loop *)
Definition tree-vect-loop-manip.cc:2070
bool vectorizable_recurr(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:8629
bool vect_slp_analyze_operations(vec_info *)
Definition tree-vect-slp.cc:8988
tree neutral_op_for_reduction(tree, code_helper, tree, bool=true)
Definition tree-vect-loop.cc:3329
void vect_free_slp_tree(slp_tree)
Definition tree-vect-slp.cc:178
unsigned vectorize_loops(void)
bool vect_can_force_dr_alignment_p(const_tree, poly_uint64)
Definition tree-vect-data-refs.cc:6504
tree bump_vector_ptr(vec_info *, tree, gimple *, gimple_stmt_iterator *, stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:5807
bool vect_apply_runtime_profitability_check_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2378
unsigned int vect_get_num_vectors(poly_uint64 nunits, tree vectype)
Definition tree-vectorizer.h:2283
#define DR_TARGET_ALIGNMENT(DR)
Definition tree-vectorizer.h:2130
void vect_free_loop_info_assumptions(class loop *)
Definition tree-vectorizer.cc:855
void set_dr_target_alignment(dr_vec_info *dr_info, poly_uint64 val)
Definition tree-vectorizer.h:2162
stmt_vec_info vect_stmt_to_vectorize(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1994
bool supportable_indirect_convert_operation(code_helper, tree, tree, vec< std::pair< tree, tree_code > > &, tree=NULL_TREE, slp_tree=NULL)
Definition tree-vect-stmts.cc:14088
opt_loop_vec_info vect_analyze_loop(class loop *, gimple *, vec_info_shared *)
Definition tree-vect-loop.cc:2894
void dr_set_safe_speculative_read_required(stmt_vec_info stmt_info, bool requires_alignment)
Definition tree-vectorizer.h:2149
#define STMT_VINFO_RELATED_STMT(S)
Definition tree-vectorizer.h:1690
bool is_pattern_stmt_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1948
#define MAX_VECTORIZATION_FACTOR
Definition tree-vectorizer.h:1901
int vect_pow2(int x)
Definition tree-vectorizer.h:2015
bool reduction_fn_for_scalar_code(code_helper, internal_fn *)
Definition tree-vect-loop.cc:3245
void vect_pattern_recog(vec_info *)
Definition tree-vect-patterns.cc:7724
int vect_slp_child_index_for_operand(const gimple *, int op, bool)
Definition tree-vect-slp.cc:620
vect_induction_op_type
Definition tree-vectorizer.h:83
@ vect_step_op_add
Definition tree-vectorizer.h:84
@ vect_step_op_shl
Definition tree-vectorizer.h:87
@ vect_step_op_mul
Definition tree-vectorizer.h:86
@ vect_step_op_neg
Definition tree-vectorizer.h:85
@ vect_step_op_shr
Definition tree-vectorizer.h:88
void vect_set_loop_condition(class loop *, edge, loop_vec_info, tree, tree, tree, bool)
Definition tree-vect-loop-manip.cc:1393
bool lane_reducing_stmt_p(gimple *stmt)
Definition tree-vectorizer.h:2396
void vect_prepare_for_masked_peels(loop_vec_info)
Definition tree-vect-loop-manip.cc:2645
void vect_get_store_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:1007
stmt_vec_info vect_find_first_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3550
enum _complex_perm_kinds complex_perm_kinds_t
bool vect_is_integer_truncation(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2955
auto_vec< std::pair< unsigned, unsigned >, 16 > auto_lane_permutation_t
Definition tree-vectorizer.h:253
bool vect_use_loop_mask_for_alignment_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2266
tree vect_build_loop_niters(loop_vec_info, bool *=NULL)
Definition tree-vect-loop-manip.cc:2691
bool vect_get_loop_mask_type(loop_vec_info)
void vect_update_max_nunits(poly_uint64 *max_nunits, poly_uint64 nunits)
Definition tree-vectorizer.h:2311
slp_instance_kind
Definition tree-vectorizer.h:389
@ slp_inst_kind_ctor
Definition tree-vectorizer.h:394
@ slp_inst_kind_reduc_group
Definition tree-vectorizer.h:391
@ slp_inst_kind_store
Definition tree-vectorizer.h:390
@ slp_inst_kind_reduc_chain
Definition tree-vectorizer.h:392
@ slp_inst_kind_gcond
Definition tree-vectorizer.h:395
@ slp_inst_kind_bb_reduc
Definition tree-vectorizer.h:393
enum vect_cost_model loop_cost_model(loop_p loop)
Definition tree-vectorizer.h:2245
unsigned int vect_known_alignment_in_bytes(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vectorizer.h:2191
#define DR_MISALIGNMENT_UNKNOWN
Definition tree-vectorizer.h:2108
bool mat_gather_scatter_p(vect_memory_access_type mat)
Definition tree-vectorizer.h:215
hash_map< slp_tree, complex_perm_kinds_t > slp_tree_to_load_perm_map_t
Definition tree-vectorizer.h:2794
opt_result vect_enhance_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2332
bool vect_is_reduction(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2877
#define LOOP_VINFO_EARLY_BREAKS(L)
Definition tree-vectorizer.h:1297
enum dr_alignment_support vect_supportable_dr_alignment(vec_info *, dr_vec_info *, tree, int, bool=false)
Definition tree-vect-data-refs.cc:6529
bool vectorizable_induction(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9411
#define SLP_TREE_REDUC_IDX(S)
Definition tree-vectorizer.h:455
innermost_loop_behavior * vect_dr_behavior(vec_info *vinfo, dr_vec_info *dr_info)
Definition tree-vectorizer.h:2207
tree perm_mask_for_reverse(tree)
Definition tree-vect-stmts.cc:1883
vect_memory_access_type SLP_TREE_MEMORY_ACCESS_TYPE(slp_tree node)
Definition tree-vectorizer.h:459
stmt_vec_info_type
Definition tree-vectorizer.h:225
@ comparison_vec_info_type
Definition tree-vectorizer.h:235
@ condition_vec_info_type
Definition tree-vectorizer.h:234
@ type_conversion_vec_info_type
Definition tree-vectorizer.h:240
@ load_vec_info_type
Definition tree-vectorizer.h:227
@ loop_exit_ctrl_vec_info_type
Definition tree-vectorizer.h:245
@ permute_info_type
Definition tree-vectorizer.h:246
@ recurr_info_type
Definition tree-vectorizer.h:244
@ op_vec_info_type
Definition tree-vectorizer.h:230
@ reduc_vec_info_type
Definition tree-vectorizer.h:236
@ assignment_vec_info_type
Definition tree-vectorizer.h:233
@ undef_vec_info_type
Definition tree-vectorizer.h:226
@ shift_vec_info_type
Definition tree-vectorizer.h:229
@ type_demotion_vec_info_type
Definition tree-vectorizer.h:239
@ store_vec_info_type
Definition tree-vectorizer.h:228
@ cycle_phi_info_type
Definition tree-vectorizer.h:241
@ phi_info_type
Definition tree-vectorizer.h:243
@ lc_phi_info_type
Definition tree-vectorizer.h:242
@ type_promotion_vec_info_type
Definition tree-vectorizer.h:238
@ call_simd_clone_vec_info_type
Definition tree-vectorizer.h:232
@ call_vec_info_type
Definition tree-vectorizer.h:231
@ induc_vec_info_type
Definition tree-vectorizer.h:237
void vect_get_slp_defs(slp_tree, vec< tree > *)
Definition tree-vect-slp.cc:10815
void vect_record_loop_len(loop_vec_info, vec_loop_lens *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:10661
struct _slp_tree * slp_tree
Definition tree-vectorizer.h:25
gimple_seq vect_gen_len(tree, tree, tree, tree)
Definition tree-vect-stmts.cc:14435
bool dr_safe_speculative_read_required(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2135
bool vect_check_gather_scatter(stmt_vec_info, tree, loop_vec_info, gather_scatter_info *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4562
tree vect_embedded_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2906
gimple * vect_loop_vectorized_call(class loop *, gcond **cond=NULL)
Definition tree-vectorizer.cc:870
void duplicate_and_interleave(vec_info *, gimple_seq *, tree, const vec< tree > &, unsigned int, vec< tree > &)
Definition tree-vect-slp.cc:10472
bool vect_grouped_store_supported(tree, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:5927
void vect_get_load_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool)
Definition tree-vect-stmts.cc:1059
vect_def_type
Definition tree-vectorizer.h:68
@ vect_induction_def
Definition tree-vectorizer.h:73
@ vect_uninitialized_def
Definition tree-vectorizer.h:69
@ vect_nested_cycle
Definition tree-vectorizer.h:76
@ vect_constant_def
Definition tree-vectorizer.h:70
@ vect_external_def
Definition tree-vectorizer.h:71
@ vect_condition_def
Definition tree-vectorizer.h:78
@ vect_internal_def
Definition tree-vectorizer.h:72
@ vect_reduction_def
Definition tree-vectorizer.h:74
@ vect_double_reduction_def
Definition tree-vectorizer.h:75
@ vect_first_order_recurrence
Definition tree-vectorizer.h:77
@ vect_unknown_def_type
Definition tree-vectorizer.h:79
opt_result vect_analyze_data_ref_dependences(loop_vec_info, unsigned int *)
Definition tree-vect-data-refs.cc:887
vect_relevant
Definition tree-vectorizer.h:1401
@ vect_used_in_outer
Definition tree-vectorizer.h:1411
@ vect_used_in_scope
Definition tree-vectorizer.h:1421
@ vect_unused_in_scope
Definition tree-vectorizer.h:1402
@ vect_used_by_reduction
Definition tree-vectorizer.h:1419
@ vect_used_in_outer_by_reduction
Definition tree-vectorizer.h:1408
@ vect_used_only_live
Definition tree-vectorizer.h:1405
tree vect_get_mask_load_else(int, tree)
Definition tree-vect-stmts.cc:2883
bool is_loop_header_bb_p(basic_block bb)
Definition tree-vectorizer.h:2004
opt_result vect_analyze_loop_form(class loop *, gimple *, vect_loop_form_info *)
Definition tree-vect-loop.cc:1432
#define STMT_VINFO_DR_WRT_VEC_LOOP(S)
Definition tree-vectorizer.h:1673
bool vectorizable_live_operation(vec_info *, stmt_vec_info, slp_tree, slp_instance, int, bool, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:10095
slp_tree vect_create_new_slp_node(unsigned, tree_code)
Definition tree-vect-slp.cc:233
opt_result vect_analyze_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:3074
bool unlimited_cost_model(loop_p loop)
Definition tree-vectorizer.h:2256
opt_result vect_mark_stmts_to_be_vectorized(loop_vec_info, bool *)
Definition tree-vect-stmts.cc:701
tree vect_gen_while(gimple_seq *, tree, tree, tree, const char *=nullptr)
Definition tree-vect-stmts.cc:14223
tree vect_get_store_rhs(stmt_vec_info)
Definition tree-vect-stmts.cc:1964
class loop * vect_transform_loop(loop_vec_info, gimple *)
Definition tree-vect-loop.cc:11015
void vect_slp_init(void)
Definition tree-vect-slp.cc:77
unsigned int vect_get_scalar_dr_size(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2369
opt_result vect_determine_partial_vectors_and_peeling(loop_vec_info)
Definition tree-vect-loop.cc:2022
bool ref_within_array_bound(gimple *, tree)
Definition tree-if-conv.cc:852
dr_alignment_support
Definition tree-vectorizer.h:51
@ dr_unaligned_supported
Definition tree-vectorizer.h:53
@ dr_explicit_realign_optimized
Definition tree-vectorizer.h:55
@ dr_aligned
Definition tree-vectorizer.h:56
@ dr_explicit_realign
Definition tree-vectorizer.h:54
@ dr_unaligned_unsupported
Definition tree-vectorizer.h:52
int compare_step_with_zero(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:1872
bool supportable_narrowing_operation(code_helper, tree, tree, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:13915
operation_type
Definition tree-vectorizer.h:44
@ ternary_op
Definition tree-vectorizer.h:47
@ unary_op
Definition tree-vectorizer.h:45
@ binary_op
Definition tree-vectorizer.h:46
void vect_record_loop_mask(loop_vec_info, vec_loop_masks *, unsigned int, tree, tree)
Definition tree-vect-loop.cc:10521
vect_memory_access_type
Definition tree-vectorizer.h:175
@ VMAT_STRIDED_SLP
Definition tree-vectorizer.h:204
@ VMAT_GATHER_SCATTER_LEGACY
Definition tree-vectorizer.h:207
@ VMAT_GATHER_SCATTER_EMULATED
Definition tree-vectorizer.h:209
@ VMAT_INVARIANT
Definition tree-vectorizer.h:179
@ VMAT_UNINITIALIZED
Definition tree-vectorizer.h:176
@ VMAT_ELEMENTWISE
Definition tree-vectorizer.h:198
@ VMAT_GATHER_SCATTER_IFN
Definition tree-vectorizer.h:208
@ VMAT_CONTIGUOUS_DOWN
Definition tree-vectorizer.h:187
@ VMAT_CONTIGUOUS
Definition tree-vectorizer.h:182
@ VMAT_LOAD_STORE_LANES
Definition tree-vectorizer.h:194
@ VMAT_CONTIGUOUS_REVERSE
Definition tree-vectorizer.h:191
bool vect_chooses_same_modes_p(vec_info *, machine_mode)
Definition tree-vect-stmts.cc:13345
bool slpeel_can_duplicate_loop_p(const class loop *, const_edge, const_edge)
Definition tree-vect-loop-manip.cc:2038
tree vect_get_loop_mask(loop_vec_info, gimple_stmt_iterator *, vec_loop_masks *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:10543
slp_vect_type
Definition tree-vectorizer.h:1427
@ not_vect
Definition tree-vectorizer.h:1428
@ pure_slp
Definition tree-vectorizer.h:1429
vect_reduction_type
Definition tree-vectorizer.h:92
@ TREE_CODE_REDUCTION
Definition tree-vectorizer.h:93
@ FOLD_LEFT_REDUCTION
Definition tree-vectorizer.h:111
@ EXTRACT_LAST_REDUCTION
Definition tree-vectorizer.h:103
@ INTEGER_INDUC_COND_REDUCTION
Definition tree-vectorizer.h:95
@ CONST_COND_REDUCTION
Definition tree-vectorizer.h:96
@ COND_REDUCTION
Definition tree-vectorizer.h:94
bool supportable_widening_operation(vec_info *, code_helper, stmt_vec_info, tree, tree, code_helper *, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:13584
auto_vec< rgroup_controls > vec_loop_lens
Definition tree-vectorizer.h:820
void set_dr_misalignment(dr_vec_info *dr_info, int val)
Definition tree-vectorizer.h:2112
bool vect_use_mask_type_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1939
pair_hash< nofree_ptr_hash< _slp_tree >, nofree_ptr_hash< _slp_tree > > slp_node_hash
Definition tree-vectorizer.h:2798
hash_map< tree_operand_hash, std::pair< stmt_vec_info, innermost_loop_behavior * > > vec_base_alignments
Definition tree-vectorizer.h:136
unsigned add_stmt_cost(vector_costs *costs, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where)
Definition tree-vectorizer.h:2058
tree vect_get_smallest_scalar_type(stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:161
vec< stmt_info_for_cost > stmt_vector_for_cost
Definition tree-vectorizer.h:130
bool known_alignment_for_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2182
void vect_remove_stores(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:13068
#define DR_GROUP_FIRST_ELEMENT(S)
Definition tree-vectorizer.h:1706
hash_set< tree_cond_mask_hash > vec_cond_masked_set_type
Definition tree-vectorizer.h:554
opt_result vect_find_stmt_data_reference(loop_p, gimple *, vec< data_reference_p > *, vec< int > *, int)
Definition tree-vect-data-refs.cc:4883
tree get_dr_vinfo_offset(vec_info *vinfo, dr_vec_info *dr_info, bool check_outer=false)
Definition tree-vectorizer.h:2223
bool vect_slp_if_converted_bb(basic_block bb, loop_p orig_loop)
Definition tree-vect-slp.cc:10329
bool vectorizable_slp_permutation(vec_info *, gimple_stmt_iterator *, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-slp.cc:11615
tree vect_gen_perm_mask_checked(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9375
opt_result vect_analyze_slp(vec_info *, unsigned, bool)
Definition tree-vect-slp.cc:5489
vect_pattern *(* vect_pattern_decl_t)(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
Definition tree-vectorizer.h:2848
tree vect_create_destination_var(tree, tree)
Definition tree-vect-data-refs.cc:5893
bool vect_transform_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:8460
tree vect_create_data_ref_ptr(vec_info *, stmt_vec_info, tree, class loop *, tree, tree *, gimple_stmt_iterator *, gimple **, bool, tree=NULL_TREE)
Definition tree-vect-data-refs.cc:5547
int vect_reduc_type(vec_info *vinfo, slp_tree node)
Definition tree-vectorizer.h:2892
void vect_gen_vector_loop_niters(loop_vec_info, tree, tree *, tree *, bool)
Definition tree-vect-loop-manip.cc:2782
bool vectorizable_phi(bb_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:8487
bool vect_make_slp_decision(loop_vec_info)
Definition tree-vect-slp.cc:8182
bool aligned_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2172
unsigned HOST_WIDE_INT tree_to_uhwi(const_tree t)
Definition tree.cc:6672
#define TYPE_ALIGN_UNIT(NODE)
Definition tree.h:2454
poly_uint64 TYPE_VECTOR_SUBPARTS(const_tree node)
Definition tree.h:4365
#define TYPE_PRECISION(NODE)
Definition tree.h:2360
#define TREE_OPERAND(NODE, I)
Definition tree.h:1319
#define sizetype
Definition tree.h:5242
#define TREE_CODE_CLASS(CODE)
Definition tree.h:203
#define TYPE_SIZE_UNIT(NODE)
Definition tree.h:2357
#define CONVERT_EXPR_CODE_P(CODE)
Definition tree.h:554
#define TREE_TYPE(NODE)
Definition tree.h:513
#define INTEGRAL_TYPE_P(TYPE)
Definition tree.h:614
#define COMPARISON_CLASS_P(NODE)
Definition tree.h:257
#define NULL_TREE
Definition tree.h:318
#define FOR_EACH_VEC_ELT(V, I, P)
Definition vec.h:1895
constexpr vnull vNULL
Definition vec.h:569
generic_wide_int< widest_int_storage< WIDEST_INT_MAX_PRECISION > > widest_int
Definition wide-int.h:345
generic_wide_int< wide_int_storage > wide_int
Definition wide-int.h:343