GCC Middle and Back End API Reference
tree-vectorizer.h
Go to the documentation of this file.
1/* Vectorizer
2 Copyright (C) 2003-2025 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4
5This file is part of GCC.
6
7GCC is free software; you can redistribute it and/or modify it under
8the terms of the GNU General Public License as published by the Free
9Software Foundation; either version 3, or (at your option) any later
10version.
11
12GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13WARRANTY; without even the implied warranty of MERCHANTABILITY or
14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15for more details.
16
17You should have received a copy of the GNU General Public License
18along with GCC; see the file COPYING3. If not see
19<http://www.gnu.org/licenses/>. */
20
21#ifndef GCC_TREE_VECTORIZER_H
22#define GCC_TREE_VECTORIZER_H
23
25typedef struct _slp_tree *slp_tree;
26
27#include "tree-data-ref.h"
28#include "tree-hash-traits.h"
29#include "target.h"
30#include "internal-fn.h"
31#include "tree-ssa-operands.h"
32#include "gimple-match.h"
33#include "dominance.h"
34
35/* Used for naming of new temporaries. */
42
43/* Defines type of operation. */
49
50/* Define type of available alignment support. */
58
59/* Define type of def-use cross-iteration cycle. */
73
74/* Define operation type of linear/non-linear induction variable. */
82
83/* Define type of reduction. */
89
90 /* Retain a scalar phi and use a FOLD_EXTRACT_LAST within the loop
91 to implement:
92
93 for (int i = 0; i < VF; ++i)
94 res = cond[i] ? val[i] : res; */
96
97 /* Use a folding reduction within the loop to implement:
98
99 for (int i = 0; i < VF; ++i)
100 res = res OP val[i];
101
102 (with no reassocation). */
104};
105
106#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
107 || ((D) == vect_double_reduction_def) \
108 || ((D) == vect_nested_cycle))
109
110/* Structure to encapsulate information about a group of like
111 instructions to be presented to the target cost model. */
121
123
124/* Maps base addresses to an innermost_loop_behavior and the stmt it was
125 derived from that gives the maximum known alignment for that base. */
127 std::pair<stmt_vec_info, innermost_loop_behavior *> >
129
130/* Represents elements [START, START + LENGTH) of cyclical array OPS*
131 (i.e. OPS repeated to give at least START + LENGTH elements) */
133{
134 tree op (unsigned int i) const;
135 bool all_same_p () const;
136
138 unsigned int start;
139 unsigned int length;
140};
141
142/* Return element I of the slice. */
143inline tree
144vect_scalar_ops_slice::op (unsigned int i) const
145{
146 return (*ops)[(i + start) % ops->length ()];
147}
148
149/* Hash traits for vect_scalar_ops_slice. */
150struct vect_scalar_ops_slice_hash : typed_noop_remove<vect_scalar_ops_slice>
151{
154
155 static const bool empty_zero_p = true;
156
157 static void mark_deleted (value_type &s) { s.length = ~0U; }
158 static void mark_empty (value_type &s) { s.length = 0; }
159 static bool is_deleted (const value_type &s) { return s.length == ~0U; }
160 static bool is_empty (const value_type &s) { return s.length == 0; }
161 static hashval_t hash (const value_type &);
162 static bool equal (const value_type &, const compare_type &);
163};
164
165/* Describes how we're going to vectorize an individual load or store,
166 or a group of loads or stores. */
168 /* An access to an invariant address. This is used only for loads. */
170
171 /* A simple contiguous access. */
173
174 /* A contiguous access that goes down in memory rather than up,
175 with no additional permutation. This is used only for stores
176 of invariants. */
178
179 /* A simple contiguous access in which the elements need to be permuted
180 after loading or before storing. Only used for loop vectorization;
181 SLP uses separate permutes. */
183
184 /* A simple contiguous access in which the elements need to be reversed
185 after loading or before storing. */
187
188 /* An access that uses IFN_LOAD_LANES or IFN_STORE_LANES. */
190
191 /* An access in which each scalar element is loaded or stored
192 individually. */
194
195 /* A hybrid of VMAT_CONTIGUOUS and VMAT_ELEMENTWISE, used for grouped
196 SLP accesses. Each unrolled iteration uses a contiguous load
197 or store for the whole group, but the groups from separate iterations
198 are combined in the same way as for VMAT_ELEMENTWISE. */
200
201 /* The access uses gather loads or scatter stores. */
203};
204
205/************************************************************************
206 SLP
207 ************************************************************************/
212
213/* A computation tree of an SLP instance. Each node corresponds to a group of
214 stmts to be packed in a SIMD stmt. */
215struct _slp_tree {
216 _slp_tree ();
217 ~_slp_tree ();
218
219 void push_vec_def (gimple *def);
220 void push_vec_def (tree def) { vec_defs.quick_push (def); }
221
222 /* Nodes that contain def-stmts of this node statements operands. */
224
225 /* A group of scalar stmts to be vectorized together. */
227 /* A group of scalar operands to be vectorized together. */
229 /* The representative that should be used for analysis and
230 code generation. */
232
233 /* Load permutation relative to the stores, NULL if there is no
234 permutation. */
236 /* Lane permutation of the operands scalar lanes encoded as pairs
237 of { operand number, lane number }. The number of elements
238 denotes the number of output lanes. */
240
241 /* Selected SIMD clone's function info. First vector element
242 is SIMD clone's function decl, followed by a pair of trees (base + step)
243 for linear arguments (pair of NULLs for other arguments). */
245
247 /* Vectorized defs. */
249 /* Number of vector stmts that are created to replace the group of scalar
250 stmts. It is calculated during the transformation phase as the number of
251 scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
252 divided by vector size. */
253 unsigned int vec_stmts_size;
254
255 /* Reference count in the SLP graph. */
256 unsigned int refcnt;
257 /* The maximum number of vector elements for the subtree rooted
258 at this node. */
260 /* The DEF type of this node. */
262 /* The number of scalar lanes produced by this node. */
263 unsigned int lanes;
264 /* The operation of this node. */
266 /* Whether uses of this load or feeders of this store are suitable
267 for load/store-lanes. */
269 /* For BB vect, flag to indicate this load node should be vectorized
270 as to avoid STLF fails because of related stores. */
272
274
275 /* Classifies how the load or store is going to be implemented
276 for loop vectorization. */
278
279 /* If not NULL this is a cached failed SLP discovery attempt with
280 the lanes that failed during SLP discovery as 'false'. This is
281 a copy of the matches array. */
282 bool *failed;
283
284 /* Allocate from slp_tree_pool. */
285 static void *operator new (size_t);
286
287 /* Return memory to slp_tree_pool. */
288 static void operator delete (void *, size_t);
289
290 /* Linked list of nodes to release when we free the slp_tree_pool. */
293};
294
295/* The enum describes the type of operations that an SLP instance
296 can perform. */
297
306
307/* SLP instance is a sequence of stmts in a loop that can be packed into
308 SIMD stmts. */
309typedef class _slp_instance {
310public:
311 /* The root of SLP tree. */
313
314 /* For vector constructors, the constructor stmt that the SLP tree is built
315 from, NULL otherwise. */
317
318 /* For slp_inst_kind_bb_reduc the defs that were not vectorized, NULL
319 otherwise. */
321
322 /* The group of nodes that contain loads of this SLP instance. */
324
325 /* The SLP node containing the reduction PHIs. */
327
328 /* Vector cost of this entry to the SLP graph. */
330
331 /* If this instance is the main entry of a subgraph the set of
332 entries into the same subgraph, including itself. */
334
335 /* The type of operation the SLP instance is performing. */
337
340
341
342/* Access Functions. */
343#define SLP_INSTANCE_TREE(S) (S)->root
344#define SLP_INSTANCE_LOADS(S) (S)->loads
345#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
346#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
347#define SLP_INSTANCE_KIND(S) (S)->kind
348
349#define SLP_TREE_CHILDREN(S) (S)->children
350#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
351#define SLP_TREE_SCALAR_OPS(S) (S)->ops
352#define SLP_TREE_REF_COUNT(S) (S)->refcnt
353#define SLP_TREE_VEC_DEFS(S) (S)->vec_defs
354#define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
355#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
356#define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
357#define SLP_TREE_SIMD_CLONE_INFO(S) (S)->simd_clone_info
358#define SLP_TREE_DEF_TYPE(S) (S)->def_type
359#define SLP_TREE_VECTYPE(S) (S)->vectype
360#define SLP_TREE_REPRESENTATIVE(S) (S)->representative
361#define SLP_TREE_LANES(S) (S)->lanes
362#define SLP_TREE_CODE(S) (S)->code
363#define SLP_TREE_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
364
371
372/* Key for map that records association between
373 scalar conditions and corresponding loop mask, and
374 is populated by vect_record_loop_mask. */
375
377{
378 scalar_cond_masked_key (tree t, unsigned ncopies_)
379 : ncopies (ncopies_)
380 {
382 }
383
385
386 unsigned ncopies;
391};
392
393template<>
395{
398
399 static inline hashval_t
401 {
403 h.add_int (v.code);
404 inchash::add_expr (v.op0, h, 0);
405 inchash::add_expr (v.op1, h, 0);
406 h.add_int (v.ncopies);
407 h.add_flag (v.inverted_p);
408 return h.end ();
409 }
410
411 static inline bool
413 {
414 return (existing.ncopies == candidate.ncopies
415 && existing.code == candidate.code
416 && existing.inverted_p == candidate.inverted_p
417 && operand_equal_p (existing.op0, candidate.op0, 0)
418 && operand_equal_p (existing.op1, candidate.op1, 0));
419 }
420
421 static const bool empty_zero_p = true;
422
423 static inline void
425 {
426 v.ncopies = 0;
427 v.inverted_p = false;
428 }
429
430 static inline bool
432 {
433 return v.ncopies == 0;
434 }
435
436 static inline void mark_deleted (value_type &) {}
437
438 static inline bool is_deleted (const value_type &)
439 {
440 return false;
441 }
442
443 static inline void remove (value_type &) {}
444};
445
447
448/* Key and map that records association between vector conditions and
449 corresponding loop mask, and is populated by prepare_vec_mask. */
450
453
454/* Describes two objects whose addresses must be unequal for the vectorized
455 loop to be valid. */
456typedef std::pair<tree, tree> vec_object_pair;
457
458/* Records that vectorization is only possible if abs (EXPR) >= MIN_VALUE.
459 UNSIGNED_P is true if we can assume that abs (EXPR) == EXPR. */
470
471/* Vectorizer state shared between different analyses like vector sizes
472 of the same CFG region. */
474public:
477
478 void save_datarefs();
479 void check_datarefs();
480
481 /* All data references. Freed by free_data_refs, so not an auto_vec. */
484
485 /* The loop nest in which the data dependences are computed. */
487
488 /* All data dependences. Freed by free_dependence_relations, so not
489 an auto_vec. */
491};
492
493/* Vectorizer state common between loop and basic-block vectorization. */
494class vec_info {
495public:
497 enum vec_kind { bb, loop };
498
500 ~vec_info ();
501
514
515 /* The type of vectorization. */
517
518 /* Shared vectorizer state. */
520
521 /* The mapping of GIMPLE UID to stmt_vec_info. */
523 /* Whether the above mapping is complete. */
525
526 /* Whether we've done a transform we think OK to not update virtual
527 SSA form. */
529
530 /* The SLP graph. */
532
533 /* Maps base addresses to an innermost_loop_behavior that gives the maximum
534 known alignment for that base. */
536
537 /* All interleaving chains of stores, represented by the first
538 stmt in the chain. */
540
541 /* The set of vector modes used in the vectorized region. */
543
544 /* The argument we should pass to related_vector_mode when looking up
545 the vector mode for a scalar mode, or VOIDmode if we haven't yet
546 made any decisions about which vector modes to use. */
547 machine_mode vector_mode;
548
549 /* The basic blocks in the vectorization region. For _loop_vec_info,
550 the memory is internally managed, while for _bb_vec_info, it points
551 to element space of an external auto_vec<>. This inconsistency is
552 not a good class design pattern. TODO: improve it with an unified
553 auto_vec<> whose lifetime is confined to vec_info object. */
555
556 /* The count of the basic blocks in the vectorization region. */
557 unsigned int nbbs;
558
559 /* Used to keep a sequence of def stmts of a pattern stmt that are loop
560 invariant if they exists.
561 The sequence is emitted in the loop preheader should the loop be vectorized
562 and are reset when undoing patterns. */
564
565private:
567 void set_vinfo_for_stmt (gimple *, stmt_vec_info, bool = true);
568 void free_stmt_vec_infos ();
570};
571
572class _loop_vec_info;
573class _bb_vec_info;
574
575template<>
576template<>
577inline bool
582
583template<>
584template<>
585inline bool
590
591/* In general, we can divide the vector statements in a vectorized loop
592 into related groups ("rgroups") and say that for each rgroup there is
593 some nS such that the rgroup operates on nS values from one scalar
594 iteration followed by nS values from the next. That is, if VF is the
595 vectorization factor of the loop, the rgroup operates on a sequence:
596
597 (1,1) (1,2) ... (1,nS) (2,1) ... (2,nS) ... (VF,1) ... (VF,nS)
598
599 where (i,j) represents a scalar value with index j in a scalar
600 iteration with index i.
601
602 [ We use the term "rgroup" to emphasise that this grouping isn't
603 necessarily the same as the grouping of statements used elsewhere.
604 For example, if we implement a group of scalar loads using gather
605 loads, we'll use a separate gather load for each scalar load, and
606 thus each gather load will belong to its own rgroup. ]
607
608 In general this sequence will occupy nV vectors concatenated
609 together. If these vectors have nL lanes each, the total number
610 of scalar values N is given by:
611
612 N = nS * VF = nV * nL
613
614 None of nS, VF, nV and nL are required to be a power of 2. nS and nV
615 are compile-time constants but VF and nL can be variable (if the target
616 supports variable-length vectors).
617
618 In classical vectorization, each iteration of the vector loop would
619 handle exactly VF iterations of the original scalar loop. However,
620 in vector loops that are able to operate on partial vectors, a
621 particular iteration of the vector loop might handle fewer than VF
622 iterations of the scalar loop. The vector lanes that correspond to
623 iterations of the scalar loop are said to be "active" and the other
624 lanes are said to be "inactive".
625
626 In such vector loops, many rgroups need to be controlled to ensure
627 that they have no effect for the inactive lanes. Conceptually, each
628 such rgroup needs a sequence of booleans in the same order as above,
629 but with each (i,j) replaced by a boolean that indicates whether
630 iteration i is active. This sequence occupies nV vector controls
631 that again have nL lanes each. Thus the control sequence as a whole
632 consists of VF independent booleans that are each repeated nS times.
633
634 Taking mask-based approach as a partially-populated vectors example.
635 We make the simplifying assumption that if a sequence of nV masks is
636 suitable for one (nS,nL) pair, we can reuse it for (nS/2,nL/2) by
637 VIEW_CONVERTing it. This holds for all current targets that support
638 fully-masked loops. For example, suppose the scalar loop is:
639
640 float *f;
641 double *d;
642 for (int i = 0; i < n; ++i)
643 {
644 f[i * 2 + 0] += 1.0f;
645 f[i * 2 + 1] += 2.0f;
646 d[i] += 3.0;
647 }
648
649 and suppose that vectors have 256 bits. The vectorized f accesses
650 will belong to one rgroup and the vectorized d access to another:
651
652 f rgroup: nS = 2, nV = 1, nL = 8
653 d rgroup: nS = 1, nV = 1, nL = 4
654 VF = 4
655
656 [ In this simple example the rgroups do correspond to the normal
657 SLP grouping scheme. ]
658
659 If only the first three lanes are active, the masks we need are:
660
661 f rgroup: 1 1 | 1 1 | 1 1 | 0 0
662 d rgroup: 1 | 1 | 1 | 0
663
664 Here we can use a mask calculated for f's rgroup for d's, but not
665 vice versa.
666
667 Thus for each value of nV, it is enough to provide nV masks, with the
668 mask being calculated based on the highest nL (or, equivalently, based
669 on the highest nS) required by any rgroup with that nV. We therefore
670 represent the entire collection of masks as a two-level table, with the
671 first level being indexed by nV - 1 (since nV == 0 doesn't exist) and
672 the second being indexed by the mask index 0 <= i < nV. */
673
674/* The controls (like masks or lengths) needed by rgroups with nV vectors,
675 according to the description above. */
677 /* The largest nS for all rgroups that use these controls.
678 For vect_partial_vectors_avx512 this is the constant nscalars_per_iter
679 for all members of the group. */
681
682 /* For the largest nS recorded above, the loop controls divide each scalar
683 into FACTOR equal-sized pieces. This is useful if we need to split
684 element-based accesses into byte-based accesses.
685 For vect_partial_vectors_avx512 this records nV instead. */
686 unsigned int factor;
687
688 /* This is a vector type with MAX_NSCALARS_PER_ITER * VF / nV elements.
689 For mask-based controls, it is the type of the masks in CONTROLS.
690 For length-based controls, it can be any vector type that has the
691 specified number of elements; the type of the elements doesn't matter. */
693
694 /* When there is no uniformly used LOOP_VINFO_RGROUP_COMPARE_TYPE this
695 is the rgroup specific type used. */
697
698 /* A vector of nV controls, in iteration order. */
700
701 /* In case of len_load and len_store with a bias there is only one
702 rgroup. This holds the adjusted loop length for the this rgroup. */
704};
705
707{
708 bool is_empty () const { return mask_set.is_empty (); }
709
710 /* Set to record vectype, nvector pairs. */
713
714 /* rgroup_controls used for the partial vector scheme. */
716};
717
719
721
722/* Information about a reduction accumulator from the main loop that could
723 conceivably be reused as the input to a reduction in an epilogue loop. */
725 /* The final value of the accumulator, which forms the input to the
726 reduction operation. */
728
729 /* The stmt_vec_info that describes the reduction (i.e. the one for
730 which is_reduc_info is true). */
732};
733
734/*-----------------------------------------------------------------*/
735/* Info on vectorized loops. */
736/*-----------------------------------------------------------------*/
737typedef class _loop_vec_info : public vec_info {
738public:
741
742 /* The loop to which this info struct refers to. */
743 class loop *loop;
744
745 /* Number of latch executions. */
747 /* Number of iterations. */
749 /* Number of iterations of the original loop. */
751 /* Condition under which this loop is analyzed and versioned. */
753
754 /* The cost of the vector code. */
756
757 /* The cost of the scalar code. */
759
760 /* Threshold of number of iterations below which vectorization will not be
761 performed. It is calculated from MIN_PROFITABLE_ITERS and
762 param_min_vect_loop_bound. */
763 unsigned int th;
764
765 /* When applying loop versioning, the vector form should only be used
766 if the number of scalar iterations is >= this value, on top of all
767 the other requirements. Ignored when loop versioning is not being
768 used. */
770
771 /* Unrolling factor */
773
774 /* If this loop is an epilogue loop whose main loop can be skipped,
775 MAIN_LOOP_EDGE is the edge from the main loop to this loop's
776 preheader. SKIP_MAIN_LOOP_EDGE is then the edge that skips the
777 main loop and goes straight to this loop's preheader.
778
779 Both fields are null otherwise. */
782
783 /* If this loop is an epilogue loop that might be skipped after executing
784 the main loop, this edge is the one that skips the epilogue. */
786
787 /* The vectorized form of a standard reduction replaces the original
788 scalar code's final result (a loop-closed SSA PHI) with the result
789 of a vector-to-scalar reduction operation. After vectorization,
790 this variable maps these vector-to-scalar results to information
791 about the reductions that generated them. */
793
794 /* The number of times that the target suggested we unroll the vector loop
795 in order to promote more ILP. This value will be used to re-analyze the
796 loop for vectorization and if successful the value will be folded into
797 vectorization_factor (and therefore exactly divides
798 vectorization_factor). */
800
801 /* Maximum runtime vectorization factor, or MAX_VECTORIZATION_FACTOR
802 if there is no particular limit. */
803 unsigned HOST_WIDE_INT max_vectorization_factor;
804
805 /* The masks that a fully-masked loop should use to avoid operating
806 on inactive scalars. */
808
809 /* The lengths that a loop with length should use to avoid operating
810 on inactive scalars. */
812
813 /* Set of scalar conditions that have loop mask applied. */
815
816 /* Set of vector conditions that have loop mask applied. */
818
819 /* If we are using a loop mask to align memory addresses, this variable
820 contains the number of vector elements that we should skip in the
821 first iteration of the vector loop (i.e. the number of leading
822 elements that should be false in the first mask). */
824
825 /* If we are using a loop mask to align memory addresses and we're in an
826 early break loop then this variable contains the number of elements that
827 were skipped during the initial iteration of the loop. */
829
830 /* The type that the loop control IV should be converted to before
831 testing which of the VF scalars are active and inactive.
832 Only meaningful if LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
834
835 /* For #pragma omp simd if (x) loops the x expression. If constant 0,
836 the loop should not be vectorized, if constant non-zero, simd_if_cond
837 shouldn't be set and loop vectorized normally, if SSA_NAME, the loop
838 should be versioned on that condition, using scalar loop if the condition
839 is false and vectorized loop otherwise. */
841
842 /* The type that the vector loop control IV should have when
843 LOOP_VINFO_USING_PARTIAL_VECTORS_P is true. */
845
846 /* The style used for implementing partial vectors when
847 LOOP_VINFO_USING_PARTIAL_VECTORS_P is true. */
849
850 /* Unknown DRs according to which loop was peeled. */
852
853 /* peeling_for_alignment indicates whether peeling for alignment will take
854 place, and what the peeling factor should be:
855 peeling_for_alignment = X means:
856 If X=0: Peeling for alignment will not be applied.
857 If X>0: Peel first X iterations.
858 If X=-1: Generate a runtime test to calculate the number of iterations
859 to be peeled, using the dataref recorded in the field
860 unaligned_dr. */
862
863 /* The mask used to check the alignment of pointers or arrays. */
865
866 /* Indicates whether the loop has any non-linear IV. */
868
869 /* Data Dependence Relations defining address ranges that are candidates
870 for a run-time aliasing check. */
872
873 /* Data Dependence Relations defining address ranges together with segment
874 lengths from which the run-time aliasing check is built. */
876
877 /* Check that the addresses of each pair of objects is unequal. */
879
880 /* List of values that are required to be nonzero. This is used to check
881 whether things like "x[i * n] += 1;" are safe and eventually gets added
882 to the checks for lower bounds below. */
884
885 /* List of values that need to be checked for a minimum value. */
887
888 /* Statements in the loop that have data references that are candidates for a
889 runtime (loop versioning) misalignment check. */
891
892 /* Reduction cycles detected in the loop. Used in loop-aware SLP. */
894
895 /* All reduction chains in the loop, represented by the first
896 stmt in the chain. */
898
899 /* Cost vector for a single scalar iteration. */
901
902 /* Map of IV base/step expressions to inserted name in the preheader. */
904
905 /* Map of OpenMP "omp simd array" scan variables to corresponding
906 rhs of the store of the initializer. */
908
909 /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
910 applied to the loop, i.e., no unrolling is needed, this is 1. */
912
913 /* The factor used to over weight those statements in an inner loop
914 relative to the loop being vectorized. */
916
917 /* Is the loop vectorizable? */
919
920 /* Records whether we still have the option of vectorizing this loop
921 using partially-populated vectors; in other words, whether it is
922 still possible for one iteration of the vector loop to handle
923 fewer than VF scalars. */
925
926 /* Records whether we must use niter masking for correctness reasons. */
928
929 /* True if we've decided to use partially-populated vectors, so that
930 the vector loop can handle fewer than VF scalars. */
932
933 /* True if we've decided to use a decrementing loop control IV that counts
934 scalars. This can be done for any loop that:
935
936 (a) uses length "controls"; and
937 (b) can iterate more than once. */
939
940 /* True if we've decided to use output of select_vl to adjust IV of
941 both loop control and data reference pointer. This is only true
942 for single-rgroup control. */
944
945 /* True if we've decided to use partially-populated vectors for the
946 epilogue of loop. */
948
949 /* The bias for len_load and len_store. For now, only 0 and -1 are
950 supported. -1 must be used when a backend does not support
951 len_load/len_store with a length of zero. */
953
954 /* When we have grouped data accesses with gaps, we may introduce invalid
955 memory accesses. We peel the last iteration of the loop to prevent
956 this. */
958
959 /* When the number of iterations is not a multiple of the vector size
960 we need to peel off iterations at the end to form an epilogue loop. */
962
963 /* When the loop has early breaks that we can vectorize we need to peel
964 the loop for the break finding loop. */
966
967 /* List of loop additional IV conditionals found in the loop. */
969
970 /* Main loop IV cond. */
972
973 /* True if there are no loop carried data dependencies in the loop.
974 If loop->safelen <= 1, then this is always true, either the loop
975 didn't have any loop carried data dependencies, or the loop is being
976 vectorized guarded with some runtime alias checks, or couldn't
977 be vectorized at all, but then this field shouldn't be used.
978 For loop->safelen >= 2, the user has asserted that there are no
979 backward dependencies, but there still could be loop carried forward
980 dependencies in such loops. This flag will be false if normal
981 vectorizer data dependency analysis would fail or require versioning
982 for alias, but because of loop->safelen >= 2 it has been vectorized
983 even without versioning for alias. E.g. in:
984 #pragma omp simd
985 for (int i = 0; i < m; i++)
986 a[i] = a[i + k] * c;
987 (or #pragma simd or #pragma ivdep) we can vectorize this and it will
988 DTRT even for k > 0 && k < m, but without safelen we would not
989 vectorize this, so this field would be false. */
991
992 /* Mark loops having masked stores. */
994
995 /* Queued scaling factor for the scalar loop. */
997
998 /* If if-conversion versioned this loop before conversion, this is the
999 loop version without if-conversion. */
1001
1002 /* For loops being epilogues of already vectorized loops
1003 this points to the main vectorized loop. Otherwise NULL. */
1005
1006 /* For loops being epilogues of already vectorized loops
1007 this points to the preceeding vectorized (possibly epilogue) loop.
1008 Otherwise NULL. */
1010
1011 /* Used to store loop_vec_infos of the epilogue of this loop during
1012 analysis. */
1014
1015 /* If this is an epilogue loop the DR advancement applied. */
1017
1018 /* The controlling loop IV for the current loop when vectorizing. This IV
1019 controls the natural exits of the loop. */
1021
1022 /* The controlling loop IV for the epilogue loop when vectorizing. This IV
1023 controls the natural exits of the loop. */
1025
1026 /* The controlling loop IV for the scalar loop being vectorized. This IV
1027 controls the natural exits of the loop. */
1029
1030 /* Used to store the list of stores needing to be moved if doing early
1031 break vectorization as they would violate the scalar loop semantics if
1032 vectorized in their current location. These are stored in order that they
1033 need to be moved. */
1035
1036 /* The final basic block where to move statements to. In the case of
1037 multiple exits this could be pretty far away. */
1039
1040 /* Statements whose VUSES need updating if early break vectorization is to
1041 happen. */
1043
1044 /* Record statements that are needed to be live for early break vectorization
1045 but may not have an LC PHI node materialized yet in the exits. */
1048
1049/* Access Functions. */
1050#define LOOP_VINFO_LOOP(L) (L)->loop
1051#define LOOP_VINFO_IV_EXIT(L) (L)->vec_loop_iv_exit
1052#define LOOP_VINFO_EPILOGUE_IV_EXIT(L) (L)->vec_epilogue_loop_iv_exit
1053#define LOOP_VINFO_SCALAR_IV_EXIT(L) (L)->scalar_loop_iv_exit
1054#define LOOP_VINFO_BBS(L) (L)->bbs
1055#define LOOP_VINFO_NBBS(L) (L)->nbbs
1056#define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1
1057#define LOOP_VINFO_NITERS(L) (L)->num_iters
1058/* Since LOOP_VINFO_NITERS and LOOP_VINFO_NITERSM1 can change after
1059 prologue peeling retain total unchanged scalar loop iterations for
1060 cost model. */
1061#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
1062#define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions
1063#define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th
1064#define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold
1065#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
1066#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
1067#define LOOP_VINFO_MUST_USE_PARTIAL_VECTORS_P(L) (L)->must_use_partial_vectors_p
1068#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
1069#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
1070#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
1071#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \
1072 (L)->epil_using_partial_vectors_p
1073#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
1074#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
1075#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
1076#define LOOP_VINFO_MASKS(L) (L)->masks
1077#define LOOP_VINFO_LENS(L) (L)->lens
1078#define LOOP_VINFO_MASK_SKIP_NITERS(L) (L)->mask_skip_niters
1079#define LOOP_VINFO_MASK_NITERS_PFA_OFFSET(L) (L)->mask_skip_niters_pfa_offset
1080#define LOOP_VINFO_RGROUP_COMPARE_TYPE(L) (L)->rgroup_compare_type
1081#define LOOP_VINFO_RGROUP_IV_TYPE(L) (L)->rgroup_iv_type
1082#define LOOP_VINFO_PARTIAL_VECTORS_STYLE(L) (L)->partial_vector_style
1083#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
1084#define LOOP_VINFO_LOOP_NEST(L) (L)->shared->loop_nest
1085#define LOOP_VINFO_DATAREFS(L) (L)->shared->datarefs
1086#define LOOP_VINFO_DDRS(L) (L)->shared->ddrs
1087#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
1088#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
1089#define LOOP_VINFO_NON_LINEAR_IV(L) (L)->nonlinear_iv
1090#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
1091#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
1092#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
1093#define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs
1094#define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs
1095#define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero
1096#define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds
1097#define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
1098#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
1099#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
1100#define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
1101#define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
1102#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
1103#define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter
1104#define LOOP_VINFO_EARLY_BREAKS(L) (L)->early_breaks
1105#define LOOP_VINFO_EARLY_BRK_STORES(L) (L)->early_break_stores
1106#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L) \
1107 (single_pred ((L)->loop->latch) != (L)->vec_loop_iv_exit->src)
1108#define LOOP_VINFO_EARLY_BREAKS_LIVE_IVS(L) \
1109 (L)->early_break_live_ivs
1110#define LOOP_VINFO_EARLY_BRK_DEST_BB(L) (L)->early_break_dest_bb
1111#define LOOP_VINFO_EARLY_BRK_VUSES(L) (L)->early_break_vuses
1112#define LOOP_VINFO_LOOP_CONDS(L) (L)->conds
1113#define LOOP_VINFO_LOOP_IV_COND(L) (L)->loop_iv_cond
1114#define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
1115#define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop
1116#define LOOP_VINFO_SCALAR_LOOP_SCALING(L) (L)->scalar_loop_scaling
1117#define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store
1118#define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
1119#define LOOP_VINFO_MAIN_LOOP_INFO(L) (L)->main_loop_info
1120#define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info
1121#define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
1122#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
1123#define LOOP_VINFO_INV_PATTERN_DEF_SEQ(L) (L)->inv_pattern_def_seq
1124#define LOOP_VINFO_DRS_ADVANCED_BY(L) (L)->drs_advanced_by
1125
1126#define LOOP_VINFO_FULLY_MASKED_P(L) \
1127 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1128 && !LOOP_VINFO_MASKS (L).is_empty ())
1129
1130#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \
1131 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1132 && !LOOP_VINFO_LENS (L).is_empty ())
1133
1134#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
1135 ((L)->may_misalign_stmts.length () > 0)
1136#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
1137 ((L)->comp_alias_ddrs.length () > 0 \
1138 || (L)->check_unequal_addrs.length () > 0 \
1139 || (L)->lower_bounds.length () > 0)
1140#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \
1141 (LOOP_VINFO_NITERS_ASSUMPTIONS (L))
1142#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \
1143 (LOOP_VINFO_SIMD_IF_COND (L))
1144#define LOOP_REQUIRES_VERSIONING(L) \
1145 (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \
1146 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \
1147 || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
1148 || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
1149
1150#define LOOP_VINFO_NITERS_KNOWN_P(L) \
1151 (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
1152
1153#define LOOP_VINFO_EPILOGUE_P(L) \
1154 (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL)
1155
1156#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \
1157 (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L)))
1158
1159/* Wrapper for loop_vec_info, for tracking success/failure, where a non-NULL
1160 value signifies success, and a NULL value signifies failure, supporting
1161 propagating an opt_problem * describing the failure back up the call
1162 stack. */
1163typedef opt_pointer_wrapper <loop_vec_info> opt_loop_vec_info;
1164
1165inline loop_vec_info
1167{
1168 return (loop_vec_info) loop->aux;
1169}
1170
1181
1190
1191#define BB_VINFO_BBS(B) (B)->bbs
1192#define BB_VINFO_NBBS(B) (B)->nbbs
1193#define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
1194#define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
1195#define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
1196#define BB_VINFO_DDRS(B) (B)->shared->ddrs
1197
1198/*-----------------------------------------------------------------*/
1199/* Info on vectorized defs. */
1200/*-----------------------------------------------------------------*/
1223
1224/* Indicates whether/how a variable is used in the scope of loop/basic
1225 block. */
1228
1229 /* The def is only used outside the loop. */
1231 /* The def is in the inner loop, and the use is in the outer loop, and the
1232 use is a reduction stmt. */
1234 /* The def is in the inner loop, and the use is in the outer loop (and is
1235 not part of reduction). */
1237
1238 /* defs that feed computations that end up (only) in a reduction. These
1239 defs may be used by non-reduction stmts, but eventually, any
1240 computations/values that are affected by these defs are used to compute
1241 a reduction (i.e. don't get stored to memory, for example). We use this
1242 to identify computations that we can change the order in which they are
1243 computed. */
1245
1247};
1248
1249/* The type of vectorization that can be applied to the stmt: regular loop-based
1250 vectorization; pure SLP - the stmt is a part of SLP instances and does not
1251 have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
1252 a part of SLP instance and also must be loop-based vectorized, since it has
1253 uses outside SLP sequences.
1254
1255 In the loop context the meanings of pure and hybrid SLP are slightly
1256 different. By saying that pure SLP is applied to the loop, we mean that we
1257 exploit only intra-iteration parallelism in the loop; i.e., the loop can be
1258 vectorized without doing any conceptual unrolling, cause we don't pack
1259 together stmts from different iterations, only within a single iteration.
1260 Loop hybrid SLP means that we exploit both intra-iteration and
1261 inter-iteration parallelism (e.g., number of elements in the vector is 4
1262 and the slp-group-size is 2, in which case we don't have enough parallelism
1263 within an iteration, so we obtain the rest of the parallelism from subsequent
1264 iterations by unrolling the loop by 2). */
1270
1271/* Says whether a statement is a load, a store of a vectorized statement
1272 result, or a store of an invariant value. */
1278
1280public:
1281 /* The data reference itself. */
1283 /* The statement that contains the data reference. */
1285 /* The analysis group this DR belongs to when doing BB vectorization.
1286 DRs of the same group belong to the same conditional execution context. */
1287 unsigned group;
1288 /* The misalignment in bytes of the reference, or -1 if not known. */
1290 /* The byte alignment that we'd ideally like the reference to have,
1291 and the value that misalignment is measured against. */
1293 /* If true the alignment of base_decl needs to be increased. */
1295
1296 /* Set by early break vectorization when this DR needs peeling for alignment
1297 for correctness. */
1299
1300 /* Set by early break vectorization when this DR's scalar accesses are known
1301 to be inbounds of a known bounds loop. */
1303
1305
1306 /* Stores current vectorized loop's offset. To be added to the DR's
1307 offset to calculate current offset of data reference. */
1309};
1310
1311typedef struct data_reference *dr_p;
1312
1314public:
1315
1317
1318 /* Indicates whether this stmts is part of a computation whose result is
1319 used outside the loop. */
1320 bool live;
1321
1322 /* Stmt is part of some pattern (computation idiom) */
1324
1325 /* True if the statement was created during pattern recognition as
1326 part of the replacement for RELATED_STMT. This implies that the
1327 statement isn't part of any basic block, although for convenience
1328 its gimple_bb is the same as for RELATED_STMT. */
1330
1331 /* Is this statement vectorizable or should it be skipped in (partial)
1332 vectorization. */
1334
1335 /* The stmt to which this info struct refers to. */
1337
1338 /* The vector type to be used for the LHS of this statement. */
1340
1341 /* The vectorized stmts. */
1343
1344 /* The following is relevant only for stmts that contain a non-scalar
1345 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
1346 at most one such data-ref. */
1347
1349
1350 /* Information about the data-ref relative to this loop
1351 nest (the loop that is being considered for vectorization). */
1353
1354 /* For loop PHI nodes, the base and evolution part of it. This makes sure
1355 this information is still available in vect_update_ivs_after_vectorizer
1356 where we may not be able to re-analyze the PHI nodes evolution as
1357 peeling for the prologue loop can make it unanalyzable. The evolution
1358 part is still correct after peeling, but the base may have changed from
1359 the version here. */
1363
1364 /* Used for various bookkeeping purposes, generally holding a pointer to
1365 some other stmt S that is in some way "related" to this stmt.
1366 Current use of this field is:
1367 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
1368 true): S is the "pattern stmt" that represents (and replaces) the
1369 sequence of stmts that constitutes the pattern. Similarly, the
1370 related_stmt of the "pattern stmt" points back to this stmt (which is
1371 the last stmt in the original sequence of stmts that constitutes the
1372 pattern). */
1374
1375 /* Used to keep a sequence of def stmts of a pattern stmt if such exists.
1376 The sequence is attached to the original statement rather than the
1377 pattern statement. */
1379
1380 /* Selected SIMD clone's function info. First vector element
1381 is SIMD clone's function decl, followed by a pair of trees (base + step)
1382 for linear arguments (pair of NULLs for other arguments). */
1384
1385 /* Classify the def of this stmt. */
1387
1388 /* Whether the stmt is SLPed, loop-based vectorized, or both. */
1390
1391 /* Interleaving and reduction chains info. */
1392 /* First element in the group. */
1394 /* Pointer to the next element in the group. */
1396 /* The size of the group. */
1397 unsigned int size;
1398 /* For stores, number of stores from this group seen. We vectorize the last
1399 one. */
1400 unsigned int store_count;
1401 /* For loads only, the gap from the previous load. For consecutive loads, GAP
1402 is 1. */
1403 unsigned int gap;
1404
1405 /* The minimum negative dependence distance this stmt participates in
1406 or zero if none. */
1407 unsigned int min_neg_dist;
1408
1409 /* Not all stmts in the loop need to be vectorized. e.g, the increment
1410 of the loop induction variable and computation of array indexes. relevant
1411 indicates whether the stmt needs to be vectorized. */
1413
1414 /* For loads if this is a gather, for stores if this is a scatter. */
1416
1417 /* True if this is an access with loop-invariant stride. */
1419
1420 /* For both loads and stores. */
1422
1423 /* Classifies how the load or store is going to be implemented
1424 for loop vectorization. */
1426
1427 /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used. */
1429
1430 /* If not NULL the value to be added to compute final reduction value. */
1432
1433 /* On a reduction PHI the reduction type as detected by
1434 vect_is_simple_reduction and vectorizable_reduction. */
1436
1437 /* The original reduction code, to be used in the epilogue. */
1439 /* An internal function we should use in the epilogue. */
1441
1442 /* On a stmt participating in the reduction the index of the operand
1443 on the reduction SSA cycle. */
1445
1446 /* On a reduction PHI the def returned by vect_force_simple_reduction.
1447 On the def returned by vect_force_simple_reduction the
1448 corresponding PHI. */
1450
1451 /* The vector input type relevant for reduction vectorization. */
1453
1454 /* The vector type for performing the actual reduction. */
1456
1457 /* For loop reduction with multiple vectorized results (ncopies > 1), a
1458 lane-reducing operation participating in it may not use all of those
1459 results, this field specifies result index starting from which any
1460 following land-reducing operation would be assigned to. */
1461 unsigned int reduc_result_pos;
1462
1463 /* If IS_REDUC_INFO is true and if the vector code is performing
1464 N scalar reductions in parallel, this variable gives the initial
1465 scalar values of those N reductions. */
1467
1468 /* If IS_REDUC_INFO is true and if the vector code is performing
1469 N scalar reductions in parallel, this variable gives the vectorized code's
1470 final (scalar) result for each of those N reductions. In other words,
1471 REDUC_SCALAR_RESULTS[I] replaces the original scalar code's loop-closed
1472 SSA PHI for reduction number I. */
1474
1475 /* Only meaningful if IS_REDUC_INFO. If non-null, the reduction is
1476 being performed by an epilogue loop and we have decided to reuse
1477 this accumulator from the main loop. */
1479
1480 /* Whether we force a single cycle PHI during reduction vectorization. */
1482
1483 /* Whether on this stmt reduction meta is recorded. */
1485
1486 /* If nonzero, the lhs of the statement could be truncated to this
1487 many bits without affecting any users of the result. */
1489
1490 /* If nonzero, all non-boolean input operands have the same precision,
1491 and they could each be truncated to this many bits without changing
1492 the result. */
1494
1495 /* If OPERATION_BITS is nonzero, the statement could be performed on
1496 an integer with the sign and number of bits given by OPERATION_SIGN
1497 and OPERATION_BITS without changing the result. */
1500
1501 /* If the statement produces a boolean result, this value describes
1502 how we should choose the associated vector type. The possible
1503 values are:
1504
1505 - an integer precision N if we should use the vector mask type
1506 associated with N-bit integers. This is only used if all relevant
1507 input booleans also want the vector mask type for N-bit integers,
1508 or if we can convert them into that form by pattern-matching.
1509
1510 - ~0U if we considered choosing a vector mask type but decided
1511 to treat the boolean as a normal integer type instead.
1512
1513 - 0 otherwise. This means either that the operation isn't one that
1514 could have a vector mask type (and so should have a normal vector
1515 type instead) or that we simply haven't made a choice either way. */
1516 unsigned int mask_precision;
1517
1518 /* True if this is only suitable for SLP vectorization. */
1520
1521 /* True if this is a pattern that can only be handled by SLP
1522 vectorization. */
1524};
1525
1526/* Information about a gather/scatter call. */
1528 /* The internal function to use for the gather/scatter operation,
1529 or IFN_LAST if a built-in function should be used instead. */
1531
1532 /* The FUNCTION_DECL for the built-in gather/scatter function,
1533 or null if an internal function should be used instead. */
1535
1536 /* The loop-invariant base value. */
1538
1539 /* The original scalar offset, which is a non-loop-invariant SSA_NAME. */
1541
1542 /* Each offset element should be multiplied by this amount before
1543 being added to the base. */
1545
1546 /* The definition type for the vectorized offset. */
1548
1549 /* The type of the vectorized offset. */
1551
1552 /* The type of the scalar elements after loading or before storing. */
1554
1555 /* The type of the scalar elements being loaded or stored. */
1557};
1558
1559/* Access Functions. */
1560#define STMT_VINFO_TYPE(S) (S)->type
1561#define STMT_VINFO_STMT(S) (S)->stmt
1562#define STMT_VINFO_RELEVANT(S) (S)->relevant
1563#define STMT_VINFO_LIVE_P(S) (S)->live
1564#define STMT_VINFO_VECTYPE(S) (S)->vectype
1565#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
1566#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
1567#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
1568#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
1569#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
1570#define STMT_VINFO_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
1571#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
1572#define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
1573#define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
1574#define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
1575#define STMT_VINFO_FORCE_SINGLE_CYCLE(S) (S)->force_single_cycle
1576
1577#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop
1578#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address
1579#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init
1580#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset
1581#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step
1582#define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment
1583#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \
1584 (S)->dr_wrt_vec_loop.base_misalignment
1585#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \
1586 (S)->dr_wrt_vec_loop.offset_alignment
1587#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
1588 (S)->dr_wrt_vec_loop.step_alignment
1589
1590#define STMT_VINFO_DR_INFO(S) \
1591 (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
1592
1593#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
1594#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
1595#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
1596#define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info
1597#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
1598#define STMT_VINFO_GROUPED_ACCESS(S) \
1599 ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
1600#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
1601#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
1602#define STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE(S) (S)->loop_phi_evolution_type
1603#define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
1604#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
1605#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
1606#define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn
1607#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
1608#define STMT_VINFO_REDUC_VECTYPE(S) (S)->reduc_vectype
1609#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
1610#define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
1611#define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p
1612
1613#define DR_GROUP_FIRST_ELEMENT(S) \
1614 (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
1615#define DR_GROUP_NEXT_ELEMENT(S) \
1616 (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
1617#define DR_GROUP_SIZE(S) \
1618 (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
1619#define DR_GROUP_STORE_COUNT(S) \
1620 (gcc_checking_assert ((S)->dr_aux.dr), (S)->store_count)
1621#define DR_GROUP_GAP(S) \
1622 (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
1623
1624#define REDUC_GROUP_FIRST_ELEMENT(S) \
1625 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
1626#define REDUC_GROUP_NEXT_ELEMENT(S) \
1627 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
1628#define REDUC_GROUP_SIZE(S) \
1629 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
1630
1631#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
1632
1633#define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
1634#define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
1635#define STMT_SLP_TYPE(S) (S)->slp_type
1636
1637/* Contains the scalar or vector costs for a vec_info. */
1639{
1640public:
1641 vector_costs (vec_info *, bool);
1642 virtual ~vector_costs () {}
1643
1644 /* Update the costs in response to adding COUNT copies of a statement.
1645
1646 - WHERE specifies whether the cost occurs in the loop prologue,
1647 the loop body, or the loop epilogue.
1648 - KIND is the kind of statement, which is always meaningful.
1649 - STMT_INFO or NODE, if nonnull, describe the statement that will be
1650 vectorized.
1651 - VECTYPE, if nonnull, is the vector type that the vectorized
1652 statement will operate on. Note that this should be used in
1653 preference to STMT_VINFO_VECTYPE (STMT_INFO) since the latter
1654 is not correct for SLP.
1655 - for unaligned_load and unaligned_store statements, MISALIGN is
1656 the byte misalignment of the load or store relative to the target's
1657 preferred alignment for VECTYPE, or DR_MISALIGNMENT_UNKNOWN
1658 if the misalignment is not known.
1659
1660 Return the calculated cost as well as recording it. The return
1661 value is used for dumping purposes. */
1662 virtual unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
1663 stmt_vec_info stmt_info,
1664 slp_tree node,
1665 tree vectype, int misalign,
1667
1668 /* Finish calculating the cost of the code. The results can be
1669 read back using the functions below.
1670
1671 If the costs describe vector code, SCALAR_COSTS gives the costs
1672 of the corresponding scalar code, otherwise it is null. */
1673 virtual void finish_cost (const vector_costs *scalar_costs);
1674
1675 /* The costs in THIS and OTHER both describe ways of vectorizing
1676 a main loop. Return true if the costs described by THIS are
1677 cheaper than the costs described by OTHER. Return false if any
1678 of the following are true:
1679
1680 - THIS and OTHER are of equal cost
1681 - OTHER is better than THIS
1682 - we can't be sure about the relative costs of THIS and OTHER. */
1683 virtual bool better_main_loop_than_p (const vector_costs *other) const;
1684
1685 /* Likewise, but the costs in THIS and OTHER both describe ways of
1686 vectorizing an epilogue loop of MAIN_LOOP. */
1687 virtual bool better_epilogue_loop_than_p (const vector_costs *other,
1688 loop_vec_info main_loop) const;
1689
1690 unsigned int prologue_cost () const;
1691 unsigned int body_cost () const;
1692 unsigned int epilogue_cost () const;
1693 unsigned int outside_cost () const;
1694 unsigned int total_cost () const;
1695 unsigned int suggested_unroll_factor () const;
1696 machine_mode suggested_epilogue_mode () const;
1697
1698protected:
1700 unsigned int);
1702 unsigned int);
1703 int compare_inside_loop_cost (const vector_costs *) const;
1704 int compare_outside_loop_cost (const vector_costs *) const;
1705
1706 /* The region of code that we're considering vectorizing. */
1708
1709 /* True if we're costing the scalar code, false if we're costing
1710 the vector code. */
1712
1713 /* The costs of the three regions, indexed by vect_cost_model_location. */
1714 unsigned int m_costs[3];
1715
1716 /* The suggested unrolling factor determined at finish_cost. */
1718
1719 /* The suggested mode to be used for a vectorized epilogue or VOIDmode,
1720 determined at finish_cost. */
1722
1723 /* True if finish_cost has been called. */
1725};
1726
1727/* Create costs for VINFO. COSTING_FOR_SCALAR is true if the costs
1728 are for scalar code, false if they are for vector code. */
1729
1730inline
1731vector_costs::vector_costs (vec_info *vinfo, bool costing_for_scalar)
1732 : m_vinfo (vinfo),
1733 m_costing_for_scalar (costing_for_scalar),
1734 m_costs (),
1736 m_suggested_epilogue_mode(VOIDmode),
1738{
1739}
1740
1741/* Return the cost of the prologue code (in abstract units). */
1742
1743inline unsigned int
1749
1750/* Return the cost of the body code (in abstract units). */
1751
1752inline unsigned int
1758
1759/* Return the cost of the epilogue code (in abstract units). */
1760
1761inline unsigned int
1767
1768/* Return the cost of the prologue and epilogue code (in abstract units). */
1769
1770inline unsigned int
1772{
1773 return prologue_cost () + epilogue_cost ();
1774}
1775
1776/* Return the cost of the prologue, body and epilogue code
1777 (in abstract units). */
1778
1779inline unsigned int
1781{
1782 return body_cost () + outside_cost ();
1783}
1784
1785/* Return the suggested unroll factor. */
1786
1787inline unsigned int
1793
1794/* Return the suggested epilogue mode. */
1795
1796inline machine_mode
1802
1803#define VECT_MAX_COST 1000
1804
1805/* The maximum number of intermediate steps required in multi-step type
1806 conversion. */
1807#define MAX_INTERM_CVT_STEPS 3
1808
1809#define MAX_VECTORIZATION_FACTOR INT_MAX
1810
1811/* Nonzero if TYPE represents a (scalar) boolean type or type
1812 in the middle-end compatible with it (unsigned precision 1 integral
1813 types). Used to determine which types should be vectorized as
1814 VECTOR_BOOLEAN_TYPE_P. */
1815
1816#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \
1817 (TREE_CODE (TYPE) == BOOLEAN_TYPE \
1818 || ((TREE_CODE (TYPE) == INTEGER_TYPE \
1819 || TREE_CODE (TYPE) == ENUMERAL_TYPE) \
1820 && TYPE_PRECISION (TYPE) == 1 \
1821 && TYPE_UNSIGNED (TYPE)))
1822
1823inline bool
1825{
1826 return (loop->inner
1827 && (loop->inner == (gimple_bb (stmt_info->stmt))->loop_father));
1828}
1829
1830/* PHI is either a scalar reduction phi or a scalar induction phi.
1831 Return the initial value of the variable on entry to the containing
1832 loop. */
1833
1834inline tree
1836{
1837 basic_block bb = gimple_bb (phi);
1839 gcc_assert (pe->dest == bb);
1840 return PHI_ARG_DEF_FROM_EDGE (phi, pe);
1841}
1842
1843/* Return true if STMT_INFO should produce a vector mask type rather than
1844 a normal nonmask type. */
1845
1846inline bool
1848{
1849 return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
1850}
1851
1852/* Return TRUE if a statement represented by STMT_INFO is a part of a
1853 pattern. */
1854
1855inline bool
1857{
1858 return stmt_info->pattern_stmt_p;
1859}
1860
1861/* If STMT_INFO is a pattern statement, return the statement that it
1862 replaces, otherwise return STMT_INFO itself. */
1863
1864inline stmt_vec_info
1866{
1867 if (is_pattern_stmt_p (stmt_info))
1868 return STMT_VINFO_RELATED_STMT (stmt_info);
1869 return stmt_info;
1870}
1871
1872/* Return the later statement between STMT1_INFO and STMT2_INFO. */
1873
1874inline stmt_vec_info
1876{
1877 gimple *stmt1 = vect_orig_stmt (stmt1_info)->stmt;
1878 gimple *stmt2 = vect_orig_stmt (stmt2_info)->stmt;
1879 if (gimple_bb (stmt1) == gimple_bb (stmt2))
1880 {
1881 if (gimple_uid (stmt1) > gimple_uid (stmt2))
1882 return stmt1_info;
1883 else
1884 return stmt2_info;
1885 }
1886 /* ??? We should be really calling this function only with stmts
1887 in the same BB but we can recover if there's a domination
1888 relationship between them. */
1890 gimple_bb (stmt1), gimple_bb (stmt2)))
1891 return stmt1_info;
1893 gimple_bb (stmt2), gimple_bb (stmt1)))
1894 return stmt2_info;
1895 gcc_unreachable ();
1896}
1897
1898/* If STMT_INFO has been replaced by a pattern statement, return the
1899 replacement statement, otherwise return STMT_INFO itself. */
1900
1901inline stmt_vec_info
1903{
1904 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
1905 return STMT_VINFO_RELATED_STMT (stmt_info);
1906 return stmt_info;
1907}
1908
1909/* Return true if BB is a loop header. */
1910
1911inline bool
1913{
1914 if (bb == (bb->loop_father)->header)
1915 return true;
1916
1917 return false;
1918}
1919
1920/* Return pow2 (X). */
1921
1922inline int
1924{
1925 int i, res = 1;
1926
1927 for (i = 0; i < x; i++)
1928 res *= 2;
1929
1930 return res;
1931}
1932
1933/* Alias targetm.vectorize.builtin_vectorization_cost. */
1934
1935inline int
1937 tree vectype, int misalign)
1938{
1939 return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
1940 vectype, misalign);
1941}
1942
1943/* Get cost by calling cost target builtin. */
1944
1945inline
1947{
1948 return builtin_vectorization_cost (type_of_cost, NULL, 0);
1949}
1950
1951/* Alias targetm.vectorize.init_cost. */
1952
1953inline vector_costs *
1954init_cost (vec_info *vinfo, bool costing_for_scalar)
1955{
1956 return targetm.vectorize.create_costs (vinfo, costing_for_scalar);
1957}
1958
1959extern void dump_stmt_cost (FILE *, int, enum vect_cost_for_stmt,
1960 stmt_vec_info, slp_tree, tree, int, unsigned,
1962
1963/* Dump and add costs. */
1964
1965inline unsigned
1967 enum vect_cost_for_stmt kind,
1968 stmt_vec_info stmt_info, slp_tree node,
1969 tree vectype, int misalign,
1970 enum vect_cost_model_location where)
1971{
1972 unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, node, vectype,
1973 misalign, where);
1974 if (dump_file && (dump_flags & TDF_DETAILS))
1975 dump_stmt_cost (dump_file, count, kind, stmt_info, node, vectype, misalign,
1976 cost, where);
1977 return cost;
1978}
1979
1980inline unsigned
1982 enum vect_cost_model_location where)
1983{
1985 || kind == scalar_stmt);
1986 return add_stmt_cost (costs, count, kind, NULL, NULL, NULL_TREE, 0, where);
1987}
1988
1989inline unsigned
1991{
1992 return add_stmt_cost (costs, i->count, i->kind, i->stmt_info, i->node,
1993 i->vectype, i->misalign, i->where);
1994}
1995
1996inline void
1998{
1999 stmt_info_for_cost *cost;
2000 unsigned i;
2001 FOR_EACH_VEC_ELT (*cost_vec, i, cost)
2002 add_stmt_cost (costs, cost->count, cost->kind, cost->stmt_info,
2003 cost->node, cost->vectype, cost->misalign, cost->where);
2004}
2005
2006/*-----------------------------------------------------------------*/
2007/* Info on data references alignment. */
2008/*-----------------------------------------------------------------*/
2009#define DR_MISALIGNMENT_UNKNOWN (-1)
2010#define DR_MISALIGNMENT_UNINITIALIZED (-2)
2011
2012inline void
2014{
2015 dr_info->misalignment = val;
2016}
2017
2018extern int dr_misalignment (dr_vec_info *dr_info, tree vectype,
2019 poly_int64 offset = 0);
2020
2021#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
2022
2023/* Only defined once DR_MISALIGNMENT is defined. */
2024inline const poly_uint64
2026{
2029 return dr_info->target_alignment;
2030}
2031#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR)
2032#define DR_SCALAR_KNOWN_BOUNDS(DR) (DR)->scalar_access_known_in_bounds
2033
2034/* Return if the stmt_vec_info requires peeling for alignment. */
2035inline bool
2037{
2039 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2041 else
2042 dr_info = STMT_VINFO_DR_INFO (stmt_info);
2043
2044 return dr_info->safe_speculative_read_required;
2045}
2046
2047/* Set the safe_speculative_read_required for the the stmt_vec_info, if group
2048 access then set on the fist element otherwise set on DR directly. */
2049inline void
2051 bool requires_alignment)
2052{
2054 if (STMT_VINFO_GROUPED_ACCESS (stmt_info))
2056 else
2057 dr_info = STMT_VINFO_DR_INFO (stmt_info);
2058
2059 dr_info->safe_speculative_read_required = requires_alignment;
2060}
2061
2062inline void
2064{
2065 dr_info->target_alignment = val;
2066}
2067#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL)
2068
2069/* Return true if data access DR_INFO is aligned to the targets
2070 preferred alignment for VECTYPE (which may be less than a full vector). */
2071
2072inline bool
2074{
2075 return (dr_misalignment (dr_info, vectype) == 0);
2076}
2077
2078/* Return TRUE if the (mis-)alignment of the data access is known with
2079 respect to the targets preferred alignment for VECTYPE, and FALSE
2080 otherwise. */
2081
2082inline bool
2087
2088/* Return the minimum alignment in bytes that the vectorized version
2089 of DR_INFO is guaranteed to have. */
2090
2091inline unsigned int
2093 poly_int64 offset = 0)
2094{
2095 int misalignment = dr_misalignment (dr_info, vectype, offset);
2096 if (misalignment == DR_MISALIGNMENT_UNKNOWN)
2098 else if (misalignment == 0)
2099 return known_alignment (DR_TARGET_ALIGNMENT (dr_info));
2100 return misalignment & -misalignment;
2101}
2102
2103/* Return the behavior of DR_INFO with respect to the vectorization context
2104 (which for outer loop vectorization might not be the behavior recorded
2105 in DR_INFO itself). */
2106
2109{
2110 stmt_vec_info stmt_info = dr_info->stmt;
2111 loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
2112 if (loop_vinfo == NULL
2113 || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info))
2114 return &DR_INNERMOST (dr_info->dr);
2115 else
2116 return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
2117}
2118
2119/* Return the offset calculated by adding the offset of this DR_INFO to the
2120 corresponding data_reference's offset. If CHECK_OUTER then use
2121 vect_dr_behavior to select the appropriate data_reference to use. */
2122
2123inline tree
2125 dr_vec_info *dr_info, bool check_outer = false)
2126{
2128 if (check_outer)
2129 base = vect_dr_behavior (vinfo, dr_info);
2130 else
2131 base = &dr_info->dr->innermost;
2132
2133 tree offset = base->offset;
2134
2135 if (!dr_info->offset)
2136 return offset;
2137
2138 offset = fold_convert (sizetype, offset);
2139 return fold_build2 (PLUS_EXPR, TREE_TYPE (dr_info->offset), offset,
2140 dr_info->offset);
2141}
2142
2143
2144/* Return the vect cost model for LOOP. */
2145inline enum vect_cost_model
2147{
2148 if (loop != NULL
2150 && flag_simd_cost_model != VECT_COST_MODEL_DEFAULT)
2151 return flag_simd_cost_model;
2152 return flag_vect_cost_model;
2153}
2154
2155/* Return true if the vect cost model is unlimited. */
2156inline bool
2161
2162/* Return true if the loop described by LOOP_VINFO is fully-masked and
2163 if the first iteration should use a partial mask in order to achieve
2164 alignment. */
2165
2166inline bool
2168{
2169 /* With early break vectorization we don't know whether the accesses will stay
2170 inside the loop or not. TODO: The early break adjustment code can be
2171 implemented the same way as vectorizable_linear_induction. However we
2172 can't test this today so reject it. */
2173 return (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
2174 && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo)
2175 && !(LOOP_VINFO_NON_LINEAR_IV (loop_vinfo)
2176 && LOOP_VINFO_EARLY_BREAKS (loop_vinfo)));
2177}
2178
2179/* Return the number of vectors of type VECTYPE that are needed to get
2180 NUNITS elements. NUNITS should be based on the vectorization factor,
2181 so it is always a known multiple of the number of elements in VECTYPE. */
2182
2183inline unsigned int
2185{
2186 return exact_div (nunits, TYPE_VECTOR_SUBPARTS (vectype)).to_constant ();
2187}
2188
2189/* Return the number of vectors in the context of vectorization region VINFO,
2190 needed for a group of statements, whose size is specified by lanes of NODE,
2191 if NULL, it is 1. The statements are supposed to be interleaved together
2192 with no gap, and all operate on vectors of type VECTYPE, if NULL, the
2193 vectype of NODE is used. */
2194
2195inline unsigned int
2197{
2198 poly_uint64 vf;
2199
2200 if (loop_vec_info loop_vinfo = dyn_cast <loop_vec_info> (vinfo))
2201 vf = LOOP_VINFO_VECT_FACTOR (loop_vinfo);
2202 else
2203 vf = 1;
2204
2205 if (node)
2206 {
2207 vf *= SLP_TREE_LANES (node);
2208 if (!vectype)
2209 vectype = SLP_TREE_VECTYPE (node);
2210 }
2211
2212 return vect_get_num_vectors (vf, vectype);
2213}
2214
2215/* Return the number of copies needed for loop vectorization when
2216 a statement operates on vectors of type VECTYPE. This is the
2217 vectorization factor divided by the number of elements in
2218 VECTYPE and is always known at compile time. */
2219
2220inline unsigned int
2222{
2223 return vect_get_num_copies (loop_vinfo, NULL, vectype);
2224}
2225
2226/* Update maximum unit count *MAX_NUNITS so that it accounts for
2227 NUNITS. *MAX_NUNITS can be 1 if we haven't yet recorded anything. */
2228
2229inline void
2231{
2232 /* All unit counts have the form vec_info::vector_size * X for some
2233 rational X, so two unit sizes must have a common multiple.
2234 Everything is a multiple of the initial value of 1. */
2235 *max_nunits = force_common_multiple (*max_nunits, nunits);
2236}
2237
2238/* Update maximum unit count *MAX_NUNITS so that it accounts for
2239 the number of units in vector type VECTYPE. *MAX_NUNITS can be 1
2240 if we haven't yet recorded any vector types. */
2241
2242inline void
2244{
2245 vect_update_max_nunits (max_nunits, TYPE_VECTOR_SUBPARTS (vectype));
2246}
2247
2248/* Return the vectorization factor that should be used for costing
2249 purposes while vectorizing the loop described by LOOP_VINFO.
2250 Pick a reasonable estimate if the vectorization factor isn't
2251 known at compile time. */
2252
2253inline unsigned int
2255{
2256 return estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
2257}
2258
2259/* Estimate the number of elements in VEC_TYPE for costing purposes.
2260 Pick a reasonable estimate if the exact number isn't known at
2261 compile time. */
2262
2263inline unsigned int
2265{
2266 return estimated_poly_value (TYPE_VECTOR_SUBPARTS (vec_type));
2267}
2268
2269/* Return the maximum possible vectorization factor for LOOP_VINFO. */
2270
2271inline unsigned HOST_WIDE_INT
2273{
2274 unsigned HOST_WIDE_INT vf;
2275 if (LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&vf))
2276 return vf;
2278}
2279
2280/* Return the size of the value accessed by unvectorized data reference
2281 DR_INFO. This is only valid once STMT_VINFO_VECTYPE has been calculated
2282 for the associated gimple statement, since that guarantees that DR_INFO
2283 accesses either a scalar or a scalar equivalent. ("Scalar equivalent"
2284 here includes things like V1SI, which can be vectorized in the same way
2285 as a plain SI.) */
2286
2287inline unsigned int
2292
2293/* Return true if LOOP_VINFO requires a runtime check for whether the
2294 vector loop is profitable. */
2295
2296inline bool
2298{
2299 unsigned int th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
2300 return (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2301 && th >= vect_vf_for_cost (loop_vinfo));
2302}
2303
2304/* Return true if CODE is a lane-reducing opcode. */
2305
2306inline bool
2308{
2309 return code == DOT_PROD_EXPR || code == WIDEN_SUM_EXPR || code == SAD_EXPR;
2310}
2311
2312/* Return true if STMT is a lane-reducing statement. */
2313
2314inline bool
2316{
2317 if (auto *assign = dyn_cast <gassign *> (stmt))
2319 return false;
2320}
2321
2322/* Source location + hotness information. */
2324
2325/* A macro for calling:
2326 dump_begin_scope (MSG, vect_location);
2327 via an RAII object, thus printing "=== MSG ===\n" to the dumpfile etc,
2328 and then calling
2329 dump_end_scope ();
2330 once the object goes out of scope, thus capturing the nesting of
2331 the scopes.
2332
2333 These scopes affect dump messages within them: dump messages at the
2334 top level implicitly default to MSG_PRIORITY_USER_FACING, whereas those
2335 in a nested scope implicitly default to MSG_PRIORITY_INTERNALS. */
2336
2337#define DUMP_VECT_SCOPE(MSG) \
2338 AUTO_DUMP_SCOPE (MSG, vect_location)
2339
2340/* A sentinel class for ensuring that the "vect_location" global gets
2341 reset at the end of a scope.
2342
2343 The "vect_location" global is used during dumping and contains a
2344 location_t, which could contain references to a tree block via the
2345 ad-hoc data. This data is used for tracking inlining information,
2346 but it's not a GC root; it's simply assumed that such locations never
2347 get accessed if the blocks are optimized away.
2348
2349 Hence we need to ensure that such locations are purged at the end
2350 of any operations using them (e.g. via this class). */
2351
2353{
2354 public:
2356};
2357
2358/*-----------------------------------------------------------------*/
2359/* Function prototypes. */
2360/*-----------------------------------------------------------------*/
2361
2362/* Simple loop peeling and versioning utilities for vectorizer's purposes -
2363 in tree-vect-loop-manip.cc. */
2364extern void vect_set_loop_condition (class loop *, edge, loop_vec_info,
2365 tree, tree, tree, bool);
2366extern bool slpeel_can_duplicate_loop_p (const class loop *, const_edge,
2367 const_edge);
2369 class loop *, edge,
2370 edge, edge *, bool = true,
2373extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
2374 tree *, tree *, tree *, int, bool, bool,
2375 tree *);
2381extern edge vec_init_loop_exit_info (class loop *);
2383
2384/* In tree-vect-stmts.cc. */
2385extern tree get_related_vectype_for_scalar_type (machine_mode, tree,
2386 poly_uint64 = 0);
2387extern tree get_vectype_for_scalar_type (vec_info *, tree, unsigned int = 0);
2389extern tree get_mask_type_for_scalar_type (vec_info *, tree, unsigned int = 0);
2392extern bool vect_chooses_same_modes_p (vec_info *, machine_mode);
2394extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
2395 stmt_vec_info * = NULL, gimple ** = NULL);
2396extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
2397 tree *, stmt_vec_info * = NULL,
2398 gimple ** = NULL);
2400 unsigned, tree *, slp_tree *,
2401 enum vect_def_type *,
2402 tree *, stmt_vec_info * = NULL);
2408 int*, vec<tree> *);
2410 code_helper *, int *,
2411 vec<tree> *);
2413 tree, tree,
2414 vec<std::pair<tree, tree_code> > &,
2415 tree = NULL_TREE,
2416 slp_tree = NULL);
2418
2419extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2421 tree, int, enum vect_cost_model_location);
2422extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2424 tree, int, enum vect_cost_model_location);
2425extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2426 enum vect_cost_for_stmt,
2428extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2430 slp_tree, tree, int,
2432
2433/* Overload of record_stmt_cost with VECTYPE derived from STMT_INFO. */
2434
2435inline unsigned
2437 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
2438 int misalign, enum vect_cost_model_location where)
2439{
2440 return record_stmt_cost (body_cost_vec, count, kind, stmt_info,
2441 STMT_VINFO_VECTYPE (stmt_info), misalign, where);
2442}
2443
2444/* Overload of record_stmt_cost with VECTYPE derived from STMT_INFO and
2445 SLP node specified. */
2446
2447inline unsigned
2449 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
2450 slp_tree node,
2451 int misalign, enum vect_cost_model_location where)
2452{
2453 return record_stmt_cost (body_cost_vec, count, kind, stmt_info, node,
2454 STMT_VINFO_VECTYPE (stmt_info), misalign, where);
2455}
2456
2463 tree op, vec<tree> *, tree = NULL);
2465 tree, vec<tree> *,
2466 tree = NULL, vec<tree> * = NULL,
2467 tree = NULL, vec<tree> * = NULL,
2468 tree = NULL, vec<tree> * = NULL);
2470 tree, tree, vec<tree> *,
2471 tree = NULL, tree = NULL, vec<tree> * = NULL,
2472 tree = NULL, tree = NULL, vec<tree> * = NULL,
2473 tree = NULL, tree = NULL, vec<tree> * = NULL);
2476extern tree vect_get_slp_vect_def (slp_tree, unsigned);
2483 slp_tree,
2486 dr_alignment_support, int, bool,
2487 unsigned int *, unsigned int *,
2489 stmt_vector_for_cost *, bool);
2492 unsigned int *, stmt_vector_for_cost *);
2493extern bool vect_supportable_shift (vec_info *, enum tree_code, tree);
2496extern void optimize_mask_stores (class loop*);
2498 const char * = nullptr);
2502 tree *, unsigned int = 0);
2504
2505/* In tree-if-conv.cc. */
2506extern bool ref_within_array_bound (gimple *, tree);
2507
2508/* In tree-vect-data-refs.cc. */
2511 (vec_info *, dr_vec_info *, tree, int);
2520extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
2521 tree, int, internal_fn *, tree *,
2522 vec<int> * = nullptr);
2525 vec<int> * = nullptr);
2528 vec<int> *, int);
2532 stmt_vec_info, tree, class loop *, tree,
2534 gimple **, bool,
2535 tree = NULL_TREE);
2538extern void vect_copy_ref_info (tree, tree);
2540extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT);
2541extern internal_fn vect_store_lanes_supported (tree, unsigned HOST_WIDE_INT, bool);
2542extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
2543extern internal_fn vect_load_lanes_supported (tree, unsigned HOST_WIDE_INT,
2544 bool, vec<int> * = nullptr);
2546 unsigned int, stmt_vec_info,
2551 class loop **);
2553 int, gimple_stmt_iterator *);
2556extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
2558 const char * = NULL);
2561 tree);
2562
2563/* In tree-vect-loop.cc. */
2564extern tree neutral_op_for_reduction (tree, code_helper, tree, bool = true);
2567/* Used in tree-vect-loop-manip.cc */
2569/* Used in gimple-loop-interchange.c and tree-parloops.cc. */
2571 enum tree_code);
2573/* Drive for loop analysis stage. */
2575 vec_info_shared *);
2578 tree *, bool);
2579extern tree vect_halve_mask_nunits (tree, machine_mode);
2580extern tree vect_double_mask_nunits (tree, machine_mode);
2582 unsigned int, tree, tree);
2585 unsigned int, tree, unsigned int);
2586extern void vect_record_loop_len (loop_vec_info, vec_loop_lens *, unsigned int,
2587 tree, unsigned int);
2589 vec_loop_lens *, unsigned int, tree,
2590 unsigned int, unsigned int);
2593 unsigned int, tree, tree, unsigned int,
2594 unsigned int);
2598
2599/* Drive for loop transformation stage. */
2600extern class loop *vect_transform_loop (loop_vec_info, gimple *);
2610extern opt_result vect_analyze_loop_form (class loop *, gimple *,
2613 const vect_loop_form_info *,
2614 loop_vec_info = nullptr);
2616 slp_tree, slp_instance, int,
2617 bool, stmt_vector_for_cost *);
2624 gimple **, slp_tree,
2628 slp_tree);
2640extern bool vect_emulated_vector_p (tree);
2643extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
2648
2649/* Nonlinear induction. */
2652
2653/* In tree-vect-slp.cc. */
2654extern void vect_slp_init (void);
2655extern void vect_slp_fini (void);
2659 bool, unsigned *,
2660 unsigned * = nullptr, bool = false);
2662extern void vect_schedule_slp (vec_info *, const vec<slp_instance> &);
2663extern opt_result vect_analyze_slp (vec_info *, unsigned, bool);
2666extern void vect_optimize_slp (vec_info *);
2667extern void vect_gather_slp_loads (vec_info *);
2668extern tree vect_get_slp_scalar_def (slp_tree, unsigned);
2669extern void vect_get_slp_defs (slp_tree, vec<tree> *);
2670extern void vect_get_slp_defs (vec_info *, slp_tree, vec<vec<tree> > *,
2671 unsigned n = -1U);
2672extern bool vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop);
2673extern bool vect_slp_function (function *);
2677extern bool can_duplicate_and_interleave_p (vec_info *, unsigned int, tree,
2678 unsigned int * = NULL,
2679 tree * = NULL, tree * = NULL);
2681 const vec<tree> &, unsigned int, vec<tree> &);
2684extern void vect_free_slp_tree (slp_tree);
2685extern bool compatible_calls_p (gcall *, gcall *);
2686extern int vect_slp_child_index_for_operand (const gimple *, int op, bool);
2687
2690extern tree vect_get_mask_load_else (int, tree);
2691
2692/* In tree-vect-patterns.cc. */
2693extern void
2695extern bool vect_get_range_info (tree, wide_int*, wide_int*);
2696
2697/* Pattern recognition functions.
2698 Additional pattern recognition functions can (and will) be added
2699 in the future. */
2701
2702/* In tree-vectorizer.cc. */
2703unsigned vectorize_loops (void);
2705gimple *vect_loop_vectorized_call (class loop *, gcond **cond = NULL);
2707
2708/* SLP Pattern matcher types, tree-vect-slp-patterns.cc. */
2709
2710/* Forward declaration of possible two operands operation that can be matched
2711 by the complex numbers pattern matchers. */
2712enum _complex_operation : unsigned;
2713
2714/* All possible load permute values that could result from the partial data-flow
2715 analysis. */
2725
2726/* Cache from nodes to the load permutation they represent. */
2727typedef hash_map <slp_tree, complex_perm_kinds_t>
2729
2730/* Cache from nodes pair to being compatible or not. */
2731typedef pair_hash <nofree_ptr_hash <_slp_tree>,
2732 nofree_ptr_hash <_slp_tree>> slp_node_hash;
2733typedef hash_map <slp_node_hash, bool> slp_compat_nodes_map_t;
2734
2735
2736/* Vector pattern matcher base class. All SLP pattern matchers must inherit
2737 from this type. */
2738
2740{
2741 protected:
2742 /* The number of arguments that the IFN requires. */
2743 unsigned m_num_args;
2744
2745 /* The internal function that will be used when a pattern is created. */
2747
2748 /* The current node being inspected. */
2750
2751 /* The list of operands to be the children for the node produced when the
2752 internal function is created. */
2754
2755 /* Default constructor where NODE is the root of the tree to inspect. */
2757 {
2758 this->m_ifn = ifn;
2759 this->m_node = node;
2760 this->m_ops.create (0);
2761 if (m_ops)
2762 this->m_ops.safe_splice (*m_ops);
2763 }
2764
2765 public:
2766
2767 /* Create a new instance of the pattern matcher class of the given type. */
2770
2771 /* Build the pattern from the data collected so far. */
2772 virtual void build (vec_info *) = 0;
2773
2774 /* Default destructor. */
2775 virtual ~vect_pattern ()
2776 {
2777 this->m_ops.release ();
2778 }
2779};
2780
2781/* Function pointer to create a new pattern matcher from a generic type. */
2782typedef vect_pattern* (*vect_pattern_decl_t) (slp_tree_to_load_perm_map_t *,
2784 slp_tree *);
2785
2786/* List of supported pattern matchers. */
2788
2789/* Number of supported pattern matchers. */
2790extern size_t num__slp_patterns;
2791
2792/* ----------------------------------------------------------------------
2793 Target support routines
2794 -----------------------------------------------------------------------
2795 The following routines are provided to simplify costing decisions in
2796 target code. Please add more as needed. */
2797
2798/* Return true if an operaton of kind KIND for STMT_INFO represents
2799 the extraction of an element from a vector in preparation for
2800 storing the element to memory. */
2801inline bool
2803{
2804 return (kind == vec_to_scalar
2805 && STMT_VINFO_DATA_REF (stmt_info)
2806 && DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)));
2807}
2808
2809/* Return true if STMT_INFO represents part of a reduction. */
2810inline bool
2812{
2813 return STMT_VINFO_REDUC_IDX (stmt_info) >= 0;
2814}
2815
2816/* Returns the memory acccess type being used to vectorize the statement. If
2817 SLP this is read from NODE, otherwise it's read from the STMT_VINFO. */
2818
2821{
2822 if (node)
2823 return SLP_TREE_MEMORY_ACCESS_TYPE (node);
2824 else
2825 return STMT_VINFO_MEMORY_ACCESS_TYPE (stmt_info);
2826}
2827
2828/* If STMT_INFO describes a reduction, return the vect_reduction_type
2829 of the reduction it describes, otherwise return -1. */
2830inline int
2832{
2833 if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
2834 if (STMT_VINFO_REDUC_DEF (stmt_info))
2835 {
2836 stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
2837 return int (STMT_VINFO_REDUC_TYPE (reduc_info));
2838 }
2839 return -1;
2840}
2841
2842/* If STMT_INFO is a COND_EXPR that includes an embedded comparison, return the
2843 scalar type of the values being compared. Return null otherwise. */
2844inline tree
2846{
2847 if (auto *assign = dyn_cast<gassign *> (stmt_info->stmt))
2848 if (gimple_assign_rhs_code (assign) == COND_EXPR)
2849 {
2850 tree cond = gimple_assign_rhs1 (assign);
2851 if (COMPARISON_CLASS_P (cond))
2852 return TREE_TYPE (TREE_OPERAND (cond, 0));
2853 }
2854 return NULL_TREE;
2855}
2856
2857/* If STMT_INFO is a comparison or contains an embedded comparison, return the
2858 scalar type of the values being compared. Return null otherwise. */
2859inline tree
2861{
2862 if (auto *assign = dyn_cast<gassign *> (stmt_info->stmt))
2864 return TREE_TYPE (gimple_assign_rhs1 (assign));
2865 return vect_embedded_comparison_type (stmt_info);
2866}
2867
2868/* Return true if STMT_INFO extends the result of a load. */
2869inline bool
2871{
2872 /* Although this is quite large for an inline function, this part
2873 at least should be inline. */
2874 gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
2875 if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
2876 return false;
2877
2878 tree rhs = gimple_assign_rhs1 (stmt_info->stmt);
2879 tree lhs_type = TREE_TYPE (gimple_assign_lhs (assign));
2880 tree rhs_type = TREE_TYPE (rhs);
2881 if (!INTEGRAL_TYPE_P (lhs_type)
2882 || !INTEGRAL_TYPE_P (rhs_type)
2883 || TYPE_PRECISION (lhs_type) <= TYPE_PRECISION (rhs_type))
2884 return false;
2885
2886 stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
2887 return (def_stmt_info
2888 && STMT_VINFO_DATA_REF (def_stmt_info)
2889 && DR_IS_READ (STMT_VINFO_DATA_REF (def_stmt_info)));
2890}
2891
2892/* Return true if STMT_INFO is an integer truncation. */
2893inline bool
2895{
2896 gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
2897 if (!assign || !CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (assign)))
2898 return false;
2899
2900 tree lhs_type = TREE_TYPE (gimple_assign_lhs (assign));
2901 tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (assign));
2902 return (INTEGRAL_TYPE_P (lhs_type)
2903 && INTEGRAL_TYPE_P (rhs_type)
2904 && TYPE_PRECISION (lhs_type) < TYPE_PRECISION (rhs_type));
2905}
2906
2907/* Build a GIMPLE_ASSIGN or GIMPLE_CALL with the tree_code,
2908 or internal_fn contained in ch, respectively. */
2910#endif /* GCC_TREE_VECTORIZER_H */
edge loop_preheader_edge(const class loop *loop)
Definition cfgloop.cc:1760
class loop * loop_p
Definition cfgloop.h:98
Definition tree-vectorizer.h:1183
vec< slp_root > roots
Definition tree-vectorizer.h:1188
_bb_vec_info(vec< basic_block > bbs, vec_info_shared *)
Definition tree-vect-slp.cc:7737
~_bb_vec_info()
Definition tree-vect-slp.cc:7774
Definition tree-vectorizer.h:737
tree mask_skip_niters_pfa_offset
Definition tree-vectorizer.h:828
bool peeling_for_gaps
Definition tree-vectorizer.h:957
bool can_use_partial_vectors_p
Definition tree-vectorizer.h:924
signed char partial_load_store_bias
Definition tree-vectorizer.h:952
class vector_costs * vector_costs
Definition tree-vectorizer.h:755
class dr_vec_info * unaligned_dr
Definition tree-vectorizer.h:851
_loop_vec_info(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:1034
edge skip_main_loop_edge
Definition tree-vectorizer.h:781
auto_vec< stmt_vec_info > reduction_chains
Definition tree-vectorizer.h:897
unsigned int th
Definition tree-vectorizer.h:763
_loop_vec_info * epilogue_vinfo
Definition tree-vectorizer.h:1013
_loop_vec_info * main_loop_info
Definition tree-vectorizer.h:1004
auto_vec< gimple * > early_break_vuses
Definition tree-vectorizer.h:1042
bool using_partial_vectors_p
Definition tree-vectorizer.h:931
bool no_data_dependencies
Definition tree-vectorizer.h:990
class loop * scalar_loop
Definition tree-vectorizer.h:1000
tree drs_advanced_by
Definition tree-vectorizer.h:1016
scalar_cond_masked_set_type scalar_cond_masked_set
Definition tree-vectorizer.h:814
auto_vec< ddr_p > may_alias_ddrs
Definition tree-vectorizer.h:871
tree num_itersm1
Definition tree-vectorizer.h:746
unsigned int inner_loop_cost_factor
Definition tree-vectorizer.h:915
bool vectorizable
Definition tree-vectorizer.h:918
vect_partial_vector_style partial_vector_style
Definition tree-vectorizer.h:848
poly_uint64 slp_unrolling_factor
Definition tree-vectorizer.h:911
edge vec_epilogue_loop_iv_exit
Definition tree-vectorizer.h:1024
tree num_iters
Definition tree-vectorizer.h:748
hash_map< tree, tree > * scan_map
Definition tree-vectorizer.h:907
int peeling_for_alignment
Definition tree-vectorizer.h:861
vec_cond_masked_set_type vec_cond_masked_set
Definition tree-vectorizer.h:817
bool epil_using_partial_vectors_p
Definition tree-vectorizer.h:947
bool early_breaks
Definition tree-vectorizer.h:965
poly_uint64 versioning_threshold
Definition tree-vectorizer.h:769
auto_vec< vec_lower_bound > lower_bounds
Definition tree-vectorizer.h:886
profile_probability scalar_loop_scaling
Definition tree-vectorizer.h:996
auto_vec< gimple * > early_break_stores
Definition tree-vectorizer.h:1034
edge main_loop_edge
Definition tree-vectorizer.h:780
edge skip_this_loop_edge
Definition tree-vectorizer.h:785
bool must_use_partial_vectors_p
Definition tree-vectorizer.h:927
auto_vec< stmt_info_for_cost > scalar_cost_vec
Definition tree-vectorizer.h:900
gcond * loop_iv_cond
Definition tree-vectorizer.h:971
basic_block early_break_dest_bb
Definition tree-vectorizer.h:1038
bool using_select_vl_p
Definition tree-vectorizer.h:943
tree rgroup_compare_type
Definition tree-vectorizer.h:833
poly_uint64 vectorization_factor
Definition tree-vectorizer.h:772
bool peeling_for_niter
Definition tree-vectorizer.h:961
auto_vec< dr_with_seg_len_pair_t > comp_alias_ddrs
Definition tree-vectorizer.h:875
edge vec_loop_iv_exit
Definition tree-vectorizer.h:1020
tree num_iters_unchanged
Definition tree-vectorizer.h:750
vec_loop_lens lens
Definition tree-vectorizer.h:811
unsigned HOST_WIDE_INT max_vectorization_factor
Definition tree-vectorizer.h:803
unsigned int suggested_unroll_factor
Definition tree-vectorizer.h:799
hash_map< tree_operand_hash, tree > * ivexpr_map
Definition tree-vectorizer.h:903
auto_vec< stmt_vec_info > may_misalign_stmts
Definition tree-vectorizer.h:890
tree rgroup_iv_type
Definition tree-vectorizer.h:844
vec_loop_masks masks
Definition tree-vectorizer.h:807
tree num_iters_assumptions
Definition tree-vectorizer.h:752
tree simd_if_cond
Definition tree-vectorizer.h:840
auto_vec< tree > check_nonzero
Definition tree-vectorizer.h:883
auto_vec< stmt_vec_info > early_break_live_ivs
Definition tree-vectorizer.h:1046
tree mask_skip_niters
Definition tree-vectorizer.h:823
bool has_mask_store
Definition tree-vectorizer.h:993
auto_vec< stmt_vec_info > reductions
Definition tree-vectorizer.h:893
edge scalar_loop_iv_exit
Definition tree-vectorizer.h:1028
class loop * loop
Definition tree-vectorizer.h:743
bool nonlinear_iv
Definition tree-vectorizer.h:867
~_loop_vec_info()
Definition tree-vect-loop.cc:1156
bool using_decrementing_iv_p
Definition tree-vectorizer.h:938
auto_vec< gcond * > conds
Definition tree-vectorizer.h:968
_loop_vec_info * orig_loop_info
Definition tree-vectorizer.h:1009
hash_map< tree, vect_reusable_accumulator > reusable_accumulators
Definition tree-vectorizer.h:792
int ptr_mask
Definition tree-vectorizer.h:864
class vector_costs * scalar_costs
Definition tree-vectorizer.h:758
auto_vec< vec_object_pair > check_unequal_addrs
Definition tree-vectorizer.h:878
Definition tree-vectorizer.h:309
slp_instance_kind kind
Definition tree-vectorizer.h:336
vec< stmt_vec_info > root_stmts
Definition tree-vectorizer.h:316
slp_tree root
Definition tree-vectorizer.h:312
dump_user_location_t location() const
Definition tree-vect-slp.cc:201
vec< slp_tree > loads
Definition tree-vectorizer.h:323
vec< _slp_instance * > subgraph_entries
Definition tree-vectorizer.h:333
vec< tree > remain_defs
Definition tree-vectorizer.h:320
slp_tree reduc_phis
Definition tree-vectorizer.h:326
stmt_vector_for_cost cost_vec
Definition tree-vectorizer.h:329
Definition tree-vectorizer.h:1313
unsigned int min_input_precision
Definition tree-vectorizer.h:1493
code_helper reduc_code
Definition tree-vectorizer.h:1438
unsigned int store_count
Definition tree-vectorizer.h:1400
tree vectype
Definition tree-vectorizer.h:1339
vec< tree > simd_clone_info
Definition tree-vectorizer.h:1383
tree induc_cond_initial_val
Definition tree-vectorizer.h:1428
bool in_pattern_p
Definition tree-vectorizer.h:1323
stmt_vec_info first_element
Definition tree-vectorizer.h:1393
unsigned int mask_precision
Definition tree-vectorizer.h:1516
vec< tree > reduc_initial_values
Definition tree-vectorizer.h:1466
int reduc_idx
Definition tree-vectorizer.h:1444
gimple * stmt
Definition tree-vectorizer.h:1336
tree loop_phi_evolution_base_unchanged
Definition tree-vectorizer.h:1360
enum vect_induction_op_type loop_phi_evolution_type
Definition tree-vectorizer.h:1362
innermost_loop_behavior dr_wrt_vec_loop
Definition tree-vectorizer.h:1352
vec< gimple * > vec_stmts
Definition tree-vectorizer.h:1342
bool slp_vect_only_p
Definition tree-vectorizer.h:1519
bool live
Definition tree-vectorizer.h:1320
unsigned int gap
Definition tree-vectorizer.h:1403
unsigned int min_output_precision
Definition tree-vectorizer.h:1488
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:1425
internal_fn reduc_fn
Definition tree-vectorizer.h:1440
enum stmt_vec_info_type type
Definition tree-vectorizer.h:1316
unsigned int min_neg_dist
Definition tree-vectorizer.h:1407
vec< tree > reduc_scalar_results
Definition tree-vectorizer.h:1473
tree reduc_vectype_in
Definition tree-vectorizer.h:1452
bool vectorizable
Definition tree-vectorizer.h:1333
bool pattern_stmt_p
Definition tree-vectorizer.h:1329
bool strided_p
Definition tree-vectorizer.h:1418
enum vect_relevant relevant
Definition tree-vectorizer.h:1412
unsigned int size
Definition tree-vectorizer.h:1397
signop operation_sign
Definition tree-vectorizer.h:1499
bool force_single_cycle
Definition tree-vectorizer.h:1481
bool is_reduc_info
Definition tree-vectorizer.h:1484
gimple_seq pattern_def_seq
Definition tree-vectorizer.h:1378
stmt_vec_info related_stmt
Definition tree-vectorizer.h:1373
unsigned simd_lane_access_p
Definition tree-vectorizer.h:1421
bool slp_vect_pattern_only_p
Definition tree-vectorizer.h:1523
tree reduc_vectype
Definition tree-vectorizer.h:1455
stmt_vec_info reduc_def
Definition tree-vectorizer.h:1449
bool gather_scatter_p
Definition tree-vectorizer.h:1415
enum vect_def_type def_type
Definition tree-vectorizer.h:1386
stmt_vec_info next_element
Definition tree-vectorizer.h:1395
unsigned int operation_precision
Definition tree-vectorizer.h:1498
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:1435
tree reduc_epilogue_adjustment
Definition tree-vectorizer.h:1431
unsigned int reduc_result_pos
Definition tree-vectorizer.h:1461
enum slp_vect_type slp_type
Definition tree-vectorizer.h:1389
dr_vec_info dr_aux
Definition tree-vectorizer.h:1348
tree loop_phi_evolution_part
Definition tree-vectorizer.h:1361
vect_reusable_accumulator * reused_accumulator
Definition tree-vectorizer.h:1478
Definition tree-vectorizer.h:2353
~auto_purge_vect_location()
Definition tree-vectorizer.cc:94
Definition vec.h:1667
Definition tree.h:81
Definition tree-vectorizer.h:1279
bool scalar_access_known_in_bounds
Definition tree-vectorizer.h:1302
unsigned group
Definition tree-vectorizer.h:1287
tree base_decl
Definition tree-vectorizer.h:1304
bool base_misaligned
Definition tree-vectorizer.h:1294
tree offset
Definition tree-vectorizer.h:1308
int misalignment
Definition tree-vectorizer.h:1289
poly_uint64 target_alignment
Definition tree-vectorizer.h:1292
data_reference * dr
Definition tree-vectorizer.h:1282
bool safe_speculative_read_required
Definition tree-vectorizer.h:1298
stmt_vec_info stmt
Definition tree-vectorizer.h:1284
Definition dumpfile.h:340
Definition hash-map.h:40
Definition hash-set.h:37
Definition inchash.h:38
void add_int(unsigned v)
Definition inchash.h:55
hashval_t end() const
Definition inchash.h:49
void add_flag(bool flag)
Definition inchash.h:124
Definition cfgloop.h:120
class loop * inner
Definition cfgloop.h:148
unsigned force_vectorize
Definition cfgloop.h:227
void * aux
Definition cfgloop.h:154
Definition opt-problem.h:179
Definition profile-count.h:147
Definition tree-vectorizer.h:473
vec< data_reference_p > datarefs
Definition tree-vectorizer.h:482
vec< data_reference > datarefs_copy
Definition tree-vectorizer.h:483
auto_vec< loop_p > loop_nest
Definition tree-vectorizer.h:486
void check_datarefs()
Definition tree-vectorizer.cc:507
void save_datarefs()
Definition tree-vectorizer.cc:497
~vec_info_shared()
Definition tree-vectorizer.cc:489
vec< ddr_p > ddrs
Definition tree-vectorizer.h:490
vec_info_shared()
Definition tree-vectorizer.cc:482
Definition tree-vectorizer.h:494
stmt_vec_info lookup_stmt(gimple *)
Definition tree-vectorizer.cc:571
basic_block * bbs
Definition tree-vectorizer.h:554
auto_vec< stmt_vec_info > grouped_stores
Definition tree-vectorizer.h:539
~vec_info()
Definition tree-vectorizer.cc:474
auto_vec< slp_instance > slp_instances
Definition tree-vectorizer.h:531
void replace_stmt(gimple_stmt_iterator *, stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:660
void remove_stmt(stmt_vec_info)
Definition tree-vectorizer.cc:644
void insert_on_entry(stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:702
void set_vinfo_for_stmt(gimple *, stmt_vec_info, bool=true)
Definition tree-vectorizer.cc:749
vec_base_alignments base_alignments
Definition tree-vectorizer.h:535
machine_mode vector_mode
Definition tree-vectorizer.h:547
stmt_vec_info resync_stmt_addr(gimple *)
Definition tree-vectorizer.cc:551
stmt_vec_info new_stmt_vec_info(gimple *stmt)
Definition tree-vectorizer.cc:713
class dr_vec_info * lookup_dr(data_reference *)
Definition tree-vectorizer.cc:613
vec< stmt_vec_info > stmt_vec_infos
Definition tree-vectorizer.h:522
void move_dr(stmt_vec_info, stmt_vec_info)
Definition tree-vectorizer.cc:625
bool any_known_not_updated_vssa
Definition tree-vectorizer.h:528
vec_kind
Definition tree-vectorizer.h:497
@ loop
Definition tree-vectorizer.h:497
@ bb
Definition tree-vectorizer.h:497
void free_stmt_vec_infos()
Definition tree-vectorizer.cc:770
vec_info_shared * shared
Definition tree-vectorizer.h:519
gimple_seq inv_pattern_def_seq
Definition tree-vectorizer.h:563
unsigned int nbbs
Definition tree-vectorizer.h:557
stmt_vec_info add_stmt(gimple *)
Definition tree-vectorizer.cc:522
vec_kind kind
Definition tree-vectorizer.h:516
mode_set used_vector_modes
Definition tree-vectorizer.h:542
hash_set< int_hash< machine_mode, E_VOIDmode, E_BLKmode > > mode_set
Definition tree-vectorizer.h:496
void insert_seq_on_entry(stmt_vec_info, gimple_seq)
Definition tree-vectorizer.cc:676
stmt_vec_info lookup_def(tree)
Definition tree-vectorizer.cc:588
void free_stmt_vec_info(stmt_vec_info)
Definition tree-vectorizer.cc:781
stmt_vec_info lookup_single_use(tree)
Definition tree-vectorizer.cc:601
vec_info(vec_kind, vec_info_shared *)
Definition tree-vectorizer.cc:463
stmt_vec_info add_pattern_stmt(gimple *, stmt_vec_info)
Definition tree-vectorizer.cc:534
bool stmt_vec_info_ro
Definition tree-vectorizer.h:524
vec_lower_bound()
Definition tree-vectorizer.h:462
poly_uint64 min_value
Definition tree-vectorizer.h:468
bool unsigned_p
Definition tree-vectorizer.h:467
vec_lower_bound(tree e, bool u, poly_uint64 m)
Definition tree-vectorizer.h:463
tree expr
Definition tree-vectorizer.h:466
Definition vec-perm-indices.h:51
Definition tree-vectorizer.h:2740
static vect_pattern * recognize(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
internal_fn m_ifn
Definition tree-vectorizer.h:2746
vec< slp_tree > m_ops
Definition tree-vectorizer.h:2753
virtual ~vect_pattern()
Definition tree-vectorizer.h:2775
virtual void build(vec_info *)=0
vect_pattern(slp_tree *node, vec< slp_tree > *m_ops, internal_fn ifn)
Definition tree-vectorizer.h:2756
slp_tree * m_node
Definition tree-vectorizer.h:2749
unsigned m_num_args
Definition tree-vectorizer.h:2743
Definition tree-vectorizer.h:1639
unsigned int adjust_cost_for_freq(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1873
virtual ~vector_costs()
Definition tree-vectorizer.h:1642
bool m_costing_for_scalar
Definition tree-vectorizer.h:1711
vector_costs(vec_info *, bool)
Definition tree-vectorizer.h:1731
vec_info * m_vinfo
Definition tree-vectorizer.h:1707
virtual unsigned int add_stmt_cost(int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, vect_cost_model_location where)
Definition tree-vectorizer.cc:1835
machine_mode suggested_epilogue_mode() const
Definition tree-vectorizer.h:1797
machine_mode m_suggested_epilogue_mode
Definition tree-vectorizer.h:1721
virtual bool better_main_loop_than_p(const vector_costs *other) const
Definition tree-vectorizer.cc:1893
unsigned int m_costs[3]
Definition tree-vectorizer.h:1714
unsigned int outside_cost() const
Definition tree-vectorizer.h:1771
unsigned int record_stmt_cost(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1859
virtual bool better_epilogue_loop_than_p(const vector_costs *other, loop_vec_info main_loop) const
Definition tree-vectorizer.cc:1912
unsigned int epilogue_cost() const
Definition tree-vectorizer.h:1762
unsigned int m_suggested_unroll_factor
Definition tree-vectorizer.h:1717
unsigned int suggested_unroll_factor() const
Definition tree-vectorizer.h:1788
bool m_finished
Definition tree-vectorizer.h:1724
int compare_outside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:2088
unsigned int body_cost() const
Definition tree-vectorizer.h:1753
virtual void finish_cost(const vector_costs *scalar_costs)
Definition tree-vectorizer.cc:1848
unsigned int prologue_cost() const
Definition tree-vectorizer.h:1744
unsigned int total_cost() const
Definition tree-vectorizer.h:1780
int compare_inside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:1995
struct basic_block_def * basic_block
Definition coretypes.h:355
class edge_def * edge
Definition coretypes.h:352
gimple * gimple_seq
Definition coretypes.h:100
const class edge_def * const_edge
Definition coretypes.h:353
const union tree_node * const_tree
Definition coretypes.h:98
union tree_node * tree
Definition coretypes.h:97
static unsigned int count[debug_counter_number_of_counters]
Definition dbgcnt.cc:50
bool dominated_by_p(enum cdi_direction dir, const_basic_block bb1, const_basic_block bb2)
Definition dominance.cc:1125
@ CDI_DOMINATORS
Definition dominance.h:25
dump_flags_t dump_flags
Definition dumpfile.cc:65
FILE * dump_file
Definition dumpfile.cc:63
@ TDF_DETAILS
Definition dumpfile.h:92
vect_cost_model
Definition flag-types.h:279
@ VECT_COST_MODEL_UNLIMITED
Definition flag-types.h:283
@ VECT_COST_MODEL_DEFAULT
Definition flag-types.h:284
bool operand_equal_p(const_tree arg0, const_tree arg1, unsigned int flags)
Definition fold-const.cc:4332
#define fold_convert(T1, T2)
Definition fold-const.h:96
#define fold_build2(c, t1, t2, t3)
Definition fold-const.h:76
internal_fn
Definition genmatch.cc:1015
tree_code
Definition genmatch.cc:1002
unsigned gimple_uid(const gimple *g)
Definition gimple.h:2087
enum tree_code gimple_assign_rhs_code(const gassign *gs)
Definition gimple.h:2853
tree gimple_assign_rhs1(const gassign *gs)
Definition gimple.h:2663
basic_block gimple_bb(const gimple *g)
Definition gimple.h:1863
tree gimple_assign_lhs(const gassign *gs)
Definition gimple.h:2612
#define init_cost
Definition ira-costs.cc:63
T dyn_cast(U *p)
Definition is-a.h:280
void add_expr(const_tree t, inchash::hash &hstate, unsigned int flags)
Definition fold-const.cc:4346
opt_pointer_wrapper< tree > opt_tree
Definition opt-problem.h:287
poly_int< NUM_POLY_INT_COEFFS, unsigned HOST_WIDE_INT > poly_uint64
Definition poly-int-types.h:25
poly_int< NUM_POLY_INT_COEFFS, HOST_WIDE_INT > poly_int64
Definition poly-int-types.h:24
i
Definition poly-int.h:776
signop
Definition signop.h:28
Definition tree-vectorizer.h:215
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:277
stmt_vec_info representative
Definition tree-vectorizer.h:231
unsigned int lanes
Definition tree-vectorizer.h:263
bool avoid_stlf_fail
Definition tree-vectorizer.h:271
load_permutation_t load_permutation
Definition tree-vectorizer.h:235
slp_tree prev_node
Definition tree-vectorizer.h:292
enum tree_code code
Definition tree-vectorizer.h:265
bool ldst_lanes
Definition tree-vectorizer.h:268
unsigned int vec_stmts_size
Definition tree-vectorizer.h:253
vec< tree > vec_defs
Definition tree-vectorizer.h:248
~_slp_tree()
Definition tree-vect-slp.cc:137
unsigned int refcnt
Definition tree-vectorizer.h:256
slp_tree next_node
Definition tree-vectorizer.h:291
int vertex
Definition tree-vectorizer.h:273
enum vect_def_type def_type
Definition tree-vectorizer.h:261
bool * failed
Definition tree-vectorizer.h:282
vec< tree > simd_clone_info
Definition tree-vectorizer.h:244
vec< tree > ops
Definition tree-vectorizer.h:228
void push_vec_def(gimple *def)
Definition tree-vect-slp.cc:159
poly_uint64 max_nunits
Definition tree-vectorizer.h:259
lane_permutation_t lane_permutation
Definition tree-vectorizer.h:239
_slp_tree()
Definition tree-vect-slp.cc:107
tree vectype
Definition tree-vectorizer.h:246
void push_vec_def(tree def)
Definition tree-vectorizer.h:220
vec< slp_tree > children
Definition tree-vectorizer.h:223
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:226
class loop * loop_father
Definition basic-block.h:126
Definition ira-costs.cc:54
Definition tree-data-ref.h:151
struct innermost_loop_behavior innermost
Definition tree-data-ref.h:173
Definition loop-invariant.cc:88
static const bool empty_zero_p
Definition tree-vectorizer.h:421
static void mark_empty(value_type &v)
Definition tree-vectorizer.h:424
scalar_cond_masked_key value_type
Definition tree-vectorizer.h:397
static bool equal(value_type existing, value_type candidate)
Definition tree-vectorizer.h:412
static void remove(value_type &)
Definition tree-vectorizer.h:443
static hashval_t hash(value_type v)
Definition tree-vectorizer.h:400
static void mark_deleted(value_type &)
Definition tree-vectorizer.h:436
static bool is_deleted(const value_type &)
Definition tree-vectorizer.h:438
static bool is_empty(value_type v)
Definition tree-vectorizer.h:431
scalar_cond_masked_key compare_type
Definition tree-vectorizer.h:396
Definition hash-traits.h:466
Definition graphite.h:338
data_reference_p dr
Definition graphite.h:343
Definition function.h:249
Definition gimple.h:904
Definition tree-vectorizer.h:1527
tree decl
Definition tree-vectorizer.h:1534
tree element_type
Definition tree-vectorizer.h:1553
tree base
Definition tree-vectorizer.h:1537
tree offset_vectype
Definition tree-vectorizer.h:1550
int scale
Definition tree-vectorizer.h:1544
enum vect_def_type offset_dt
Definition tree-vectorizer.h:1547
tree offset
Definition tree-vectorizer.h:1540
internal_fn ifn
Definition tree-vectorizer.h:1530
tree memory_type
Definition tree-vectorizer.h:1556
Definition gimple.h:352
Definition gimple.h:858
Definition gimple-iterator.h:26
Definition gimple.h:221
Definition gimple.h:461
Definition tree-data-ref.h:50
tree offset
Definition tree-data-ref.h:52
Definition hash-traits.h:122
static bool test(U *p)
Definition hash-traits.h:345
Definition tree-vectorizer.h:676
tree compare_type
Definition tree-vectorizer.h:696
unsigned int max_nscalars_per_iter
Definition tree-vectorizer.h:680
unsigned int factor
Definition tree-vectorizer.h:686
tree type
Definition tree-vectorizer.h:692
tree bias_adjusted_ctrl
Definition tree-vectorizer.h:703
vec< tree > controls
Definition tree-vectorizer.h:699
unsigned ncopies
Definition tree-vectorizer.h:386
tree_code code
Definition tree-vectorizer.h:388
scalar_cond_masked_key(tree t, unsigned ncopies_)
Definition tree-vectorizer.h:378
void get_cond_ops_from_tree(tree)
Definition tree-vectorizer.cc:1785
tree op0
Definition tree-vectorizer.h:389
tree op1
Definition tree-vectorizer.h:390
bool inverted_p
Definition tree-vectorizer.h:387
vec< tree > remain
Definition tree-vectorizer.h:1179
slp_instance_kind kind
Definition tree-vectorizer.h:1176
vec< stmt_vec_info > roots
Definition tree-vectorizer.h:1178
slp_root(slp_instance_kind kind_, vec< stmt_vec_info > stmts_, vec< stmt_vec_info > roots_, vec< tree > remain_=vNULL)
Definition tree-vectorizer.h:1173
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:1177
Definition tree-vectorizer.h:112
tree vectype
Definition tree-vectorizer.h:118
int misalign
Definition tree-vectorizer.h:119
slp_tree node
Definition tree-vectorizer.h:117
enum vect_cost_model_location where
Definition tree-vectorizer.h:115
enum vect_cost_for_stmt kind
Definition tree-vectorizer.h:114
stmt_vec_info stmt_info
Definition tree-vectorizer.h:116
int count
Definition tree-vectorizer.h:113
Definition tree-hash-traits.h:25
Definition hash-traits.h:75
Definition tree-vectorizer.h:707
hash_set< pair_hash< nofree_ptr_hash< tree_node >, int_hash< unsigned, 0 > > > mask_set
Definition tree-vectorizer.h:712
auto_vec< rgroup_controls > rgc_vec
Definition tree-vectorizer.h:715
bool is_empty() const
Definition tree-vectorizer.h:708
Definition vec.h:450
Definition tree-vectorizer.h:2602
gcond * inner_loop_cond
Definition tree-vectorizer.h:2607
auto_vec< gcond * > conds
Definition tree-vectorizer.h:2606
tree number_of_iterationsm1
Definition tree-vectorizer.h:2604
tree assumptions
Definition tree-vectorizer.h:2605
edge loop_exit
Definition tree-vectorizer.h:2608
tree number_of_iterations
Definition tree-vectorizer.h:2603
Definition tree-vectorizer.h:724
stmt_vec_info reduc_info
Definition tree-vectorizer.h:731
tree reduc_input
Definition tree-vectorizer.h:727
Definition tree-vectorizer.h:151
static void mark_deleted(value_type &s)
Definition tree-vectorizer.h:157
static bool is_empty(const value_type &s)
Definition tree-vectorizer.h:160
static bool is_deleted(const value_type &s)
Definition tree-vectorizer.h:159
static const bool empty_zero_p
Definition tree-vectorizer.h:155
static void mark_empty(value_type &s)
Definition tree-vectorizer.h:158
static hashval_t hash(const value_type &)
Definition tree-vect-slp.cc:7966
vect_scalar_ops_slice value_type
Definition tree-vectorizer.h:152
static bool equal(const value_type &, const compare_type &)
Definition tree-vect-slp.cc:7975
vect_scalar_ops_slice compare_type
Definition tree-vectorizer.h:153
Definition tree-vectorizer.h:133
unsigned int length
Definition tree-vectorizer.h:139
unsigned int start
Definition tree-vectorizer.h:138
bool all_same_p() const
Definition tree-vect-slp.cc:7957
tree op(unsigned int i) const
Definition tree-vectorizer.h:144
vec< tree > * ops
Definition tree-vectorizer.h:137
#define NULL
Definition system.h:50
#define gcc_assert(EXPR)
Definition system.h:814
#define gcc_unreachable()
Definition system.h:841
#define false
Definition system.h:888
#define gcc_checking_assert(EXPR)
Definition system.h:821
HOST_WIDE_INT estimated_poly_value(poly_int64 x, poly_value_estimate_kind kind=POLY_VALUE_LIKELY)
Definition target.h:338
struct gcc_target targetm
vect_cost_model_location
Definition target.h:212
@ vect_body
Definition target.h:214
@ vect_prologue
Definition target.h:213
@ vect_epilogue
Definition target.h:215
vect_cost_for_stmt
Definition target.h:190
@ scalar_stmt
Definition target.h:191
@ cond_branch_taken
Definition target.h:204
@ vec_to_scalar
Definition target.h:201
@ cond_branch_not_taken
Definition target.h:203
@ tcc_comparison
Definition tree-core.h:233
@ vec_kind
Definition tree-core.h:1031
#define DR_IS_WRITE(DR)
Definition tree-data-ref.h:192
#define DR_IS_READ(DR)
Definition tree-data-ref.h:191
#define DR_REF(DR)
Definition tree-data-ref.h:185
#define DR_INNERMOST(DR)
Definition tree-data-ref.h:203
static tree candidate(unsigned uid)
Definition tree-sra.cc:326
#define PHI_ARG_DEF_FROM_EDGE(PHI, E)
Definition tree-ssa-operands.h:84
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset)
Definition tree-vect-data-refs.cc:1303
vect_pattern_decl_t slp_patterns[]
Definition tree-vect-slp-patterns.cc:1658
_complex_operation
Definition tree-vect-slp-patterns.cc:119
size_t num__slp_patterns
Definition tree-vect-slp-patterns.cc:1669
dump_user_location_t vect_location
Definition tree-vectorizer.cc:88
stmt_vec_info vect_orig_stmt(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1865
#define LOOP_VINFO_COST_MODEL_THRESHOLD(L)
Definition tree-vectorizer.h:1063
bool vect_can_vectorize_without_simd_p(tree_code)
Definition tree-vect-loop.cc:9385
tree cse_and_gimplify_to_preheader(loop_vec_info, tree)
Definition tree-vect-loop.cc:1178
int builtin_vectorization_cost(enum vect_cost_for_stmt type_of_cost, tree vectype, int misalign)
Definition tree-vectorizer.h:1936
loop_vec_info loop_vec_info_for_loop(class loop *loop)
Definition tree-vectorizer.h:1166
bool vect_is_extending_load(class vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2870
bool vect_transform_stmt(vec_info *, stmt_vec_info, gimple_stmt_iterator *, slp_tree, slp_instance)
Definition tree-vect-stmts.cc:14053
bool vect_is_simple_use(tree, vec_info *, enum vect_def_type *, stmt_vec_info *=NULL, gimple **=NULL)
Definition tree-vect-stmts.cc:14533
bool vect_rgroup_iv_might_wrap_p(loop_vec_info, rgroup_controls *)
Definition tree-vect-loop.cc:12663
tree vect_halve_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11437
void dump_stmt_cost(FILE *, int, enum vect_cost_for_stmt, stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location)
Definition tree-vectorizer.cc:102
vec< unsigned > load_permutation_t
Definition tree-vectorizer.h:210
bool vect_slp_analyze_instance_alignment(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:3052
internal_fn vect_store_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:5954
tree vect_get_new_vect_var(tree, enum vect_var_kind, const char *)
Definition tree-vect-data-refs.cc:5213
void vect_mark_pattern_stmts(vec_info *, stmt_vec_info, gimple *, tree)
Definition tree-vect-patterns.cc:6910
class loop * vect_loop_versioning(loop_vec_info, gimple *)
Definition tree-vect-loop-manip.cc:4003
tree vect_get_slp_vect_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10324
opt_tree vect_get_mask_type_for_stmt(stmt_vec_info, unsigned int=0)
#define LOOP_VINFO_VECT_FACTOR(L)
Definition tree-vectorizer.h:1074
void vect_free_slp_instance(slp_instance)
Definition tree-vect-slp.cc:213
#define SLP_TREE_LANES(S)
Definition tree-vectorizer.h:361
bool vect_slp_function(function *)
Definition tree-vect-slp.cc:9857
std::pair< tree, tree > vec_object_pair
Definition tree-vectorizer.h:456
tree vect_gen_perm_mask_any(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:10052
_bb_vec_info * bb_vec_info
#define LOOP_VINFO_FULLY_MASKED_P(L)
Definition tree-vectorizer.h:1126
int vect_get_known_peeling_cost(loop_vec_info, int, int *, stmt_vector_for_cost *, stmt_vector_for_cost *, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:4499
bool vect_transform_slp_perm_load(vec_info *, slp_tree, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, unsigned *, unsigned *=nullptr, bool=false)
tree vect_gen_while_not(gimple_seq *, tree, tree, tree)
Definition tree-vect-stmts.cc:15460
bool vectorizable_early_exit(vec_info *, stmt_vec_info, gimple_stmt_iterator *, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:13476
tree vect_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2860
bool is_simple_and_all_uses_invariant(stmt_vec_info, loop_vec_info)
Definition tree-vect-stmts.cc:326
bool vectorizable_phi(vec_info *, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9076
#define STMT_VINFO_GROUPED_ACCESS(S)
Definition tree-vectorizer.h:1598
#define STMT_VINFO_MEMORY_ACCESS_TYPE(S)
Definition tree-vectorizer.h:1570
void vect_schedule_slp(vec_info *, const vec< slp_instance > &)
Definition tree-vect-slp.cc:11765
tree vect_peel_nonlinear_iv_init(gimple_seq *, tree, tree, tree, enum vect_induction_op_type)
Definition tree-vect-loop.cc:9511
bool check_reduction_path(dump_user_location_t, loop_p, gphi *, tree, enum tree_code)
Definition tree-vect-loop.cc:4178
bool can_duplicate_and_interleave_p(vec_info *, unsigned int, tree, unsigned int *=NULL, tree *=NULL, tree *=NULL)
Definition tree-vect-slp.cc:425
auto_vec< unsigned, 16 > auto_load_permutation_t
Definition tree-vectorizer.h:211
#define SLP_TREE_VECTYPE(S)
Definition tree-vectorizer.h:359
tree get_vectype_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:14373
#define LOOP_VINFO_LOOP(L)
Definition tree-vectorizer.h:1050
_complex_perm_kinds
Definition tree-vectorizer.h:2716
@ PERM_EVENEVEN
Definition tree-vectorizer.h:2721
@ PERM_TOP
Definition tree-vectorizer.h:2723
@ PERM_ODDEVEN
Definition tree-vectorizer.h:2719
@ PERM_EVENODD
Definition tree-vectorizer.h:2718
@ PERM_UNKNOWN
Definition tree-vectorizer.h:2717
@ PERM_ODDODD
Definition tree-vectorizer.h:2720
void vect_record_base_alignments(vec_info *)
Definition tree-vect-data-refs.cc:1384
void vect_get_vec_defs(vec_info *, stmt_vec_info, slp_tree, unsigned, tree, vec< tree > *, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL)
Definition tree-vect-stmts.cc:1375
vec_load_store_type
Definition tree-vectorizer.h:1273
@ VLS_LOAD
Definition tree-vectorizer.h:1274
@ VLS_STORE
Definition tree-vectorizer.h:1275
@ VLS_STORE_INVARIANT
Definition tree-vectorizer.h:1276
tree get_mask_type_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:14448
pair_hash< tree_operand_hash, tree_operand_hash > tree_cond_mask_hash
Definition tree-vectorizer.h:451
_loop_vec_info * loop_vec_info
tree vect_double_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11447
vect_var_kind
Definition tree-vectorizer.h:36
@ vect_pointer_var
Definition tree-vectorizer.h:38
@ vect_scalar_var
Definition tree-vectorizer.h:39
@ vect_mask_var
Definition tree-vectorizer.h:40
@ vect_simple_var
Definition tree-vectorizer.h:37
int vect_get_place_in_interleaving_chain(stmt_vec_info, stmt_vec_info)
Definition tree-vect-slp.cc:396
class loop * vect_do_peeling(loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, tree *)
Definition tree-vect-loop-manip.cc:3128
void optimize_mask_stores(class loop *)
Definition tree-vect-loop.cc:12427
class _stmt_vec_info * stmt_vec_info
Definition tree-vectorizer.h:24
#define STMT_VINFO_DATA_REF(S)
Definition tree-vectorizer.h:1567
tree vect_get_loop_len(loop_vec_info, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:11634
bool vect_transform_cycle_phi(loop_vec_info, stmt_vec_info, slp_tree, slp_instance)
Definition tree-vect-loop.cc:8809
bool vect_is_store_elt_extraction(vect_cost_for_stmt kind, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2802
void vect_detect_hybrid_slp(loop_vec_info)
Definition tree-vect-slp.cc:7652
opt_result vect_get_vector_types_for_stmt(vec_info *, stmt_vec_info, tree *, tree *, unsigned int=0)
Definition tree-vect-stmts.cc:15484
const poly_uint64 dr_target_alignment(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2025
void vect_update_inits_of_drs(loop_vec_info, tree, tree_code)
Definition tree-vect-loop-manip.cc:2622
struct data_reference * dr_p
Definition tree-vectorizer.h:1311
bool vect_can_advance_ivs_p(loop_vec_info)
Definition tree-vect-loop-manip.cc:2211
opt_result vect_analyze_data_refs(vec_info *, poly_uint64 *, bool *)
Definition tree-vect-data-refs.cc:4954
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vect-data-refs.cc:1303
tree vect_init_vector(vec_info *, stmt_vec_info, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1204
void vect_copy_ref_info(tree, tree)
Definition tree-vect-data-refs.cc:5780
void vect_record_grouped_load_vectors(vec_info *, stmt_vec_info, vec< tree >)
Definition tree-vect-data-refs.cc:7188
bool vect_transform_reduction(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, slp_tree)
Definition tree-vect-loop.cc:8462
bool vectorizable_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:9021
bool vectorizable_lane_reducing(loop_vec_info, stmt_vec_info, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:7339
tree vect_get_new_ssa_name(tree, enum vect_var_kind, const char *=NULL)
Definition tree-vect-data-refs.cc:5251
#define STMT_VINFO_VECTYPE(S)
Definition tree-vectorizer.h:1564
opt_result vect_prune_runtime_alias_test_list(loop_vec_info)
Definition tree-vect-data-refs.cc:4075
bool needs_fold_left_reduction_p(tree, code_helper)
Definition tree-vect-loop.cc:3943
bool vect_nop_conversion_p(stmt_vec_info)
Definition tree-vect-stmts.cc:6137
void vect_finish_stmt_generation(vec_info *, stmt_vec_info, gimple *, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1436
#define LOOP_VINFO_NON_LINEAR_IV(L)
Definition tree-vectorizer.h:1089
unsigned HOST_WIDE_INT vect_max_vf(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2272
internal_fn vect_load_lanes_supported(tree, unsigned HOST_WIDE_INT, bool, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:6583
loop_vec_info vect_create_loop_vinfo(class loop *, vec_info_shared *, const vect_loop_form_info *, loop_vec_info=nullptr)
Definition tree-vect-loop.cc:1947
bool vect_stmt_dominates_stmt_p(gimple *, gimple *)
Definition tree-vectorizer.cc:801
tree vect_get_slp_scalar_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:10306
bool vect_supportable_shift(vec_info *, enum tree_code, tree)
Definition tree-vect-stmts.cc:6335
void vect_optimize_slp(vec_info *)
Definition tree-vect-slp.cc:7433
void vect_iv_increment_position(edge, gimple_stmt_iterator *, bool *)
Definition tree-vect-loop-manip.cc:457
hash_set< scalar_cond_masked_key > scalar_cond_masked_set_type
Definition tree-vectorizer.h:446
tree vect_phi_initial_value(gphi *phi)
Definition tree-vectorizer.h:1835
opt_pointer_wrapper< loop_vec_info > opt_loop_vec_info
Definition tree-vectorizer.h:1163
bool vectorizable_reduction(loop_vec_info, stmt_vec_info, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:7501
unsigned record_stmt_cost(stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location)
Definition tree-vect-stmts.cc:118
bool vect_emulated_vector_p(tree)
Definition tree-vect-loop.cc:9374
tree get_same_sized_vectype(tree, tree)
Definition tree-vect-stmts.cc:14484
gimple * vect_gimple_build(tree, code_helper, tree, tree=NULL_TREE)
Definition tree-vect-patterns.cc:7214
void add_stmt_costs(vector_costs *costs, stmt_vector_for_cost *cost_vec)
Definition tree-vectorizer.h:1997
void vect_finish_replace_stmt(vec_info *, stmt_vec_info, gimple *)
Definition tree-vect-stmts.cc:1420
tree prepare_vec_mask(loop_vec_info, tree, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1693
void vect_slp_fini(void)
Definition tree-vect-slp.cc:82
#define LOOP_VINFO_NITERS_KNOWN_P(L)
Definition tree-vectorizer.h:1150
stmt_vec_info get_later_stmt(stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
Definition tree-vectorizer.h:1875
vec< std::pair< unsigned, unsigned > > lane_permutation_t
Definition tree-vectorizer.h:208
tree vect_gen_loop_len_mask(loop_vec_info, gimple_stmt_iterator *, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:11701
widest_int vect_iv_limit_for_partial_vectors(loop_vec_info loop_vinfo)
Definition tree-vect-loop.cc:12622
void vect_get_vec_defs_for_operand(vec_info *vinfo, stmt_vec_info, unsigned, tree op, vec< tree > *, tree=NULL)
Definition tree-vect-stmts.cc:1276
void vect_gather_slp_loads(vec_info *)
Definition tree-vect-slp.cc:7452
vect_partial_vector_style
Definition tree-vectorizer.h:365
@ vect_partial_vectors_none
Definition tree-vectorizer.h:366
@ vect_partial_vectors_len
Definition tree-vectorizer.h:369
@ vect_partial_vectors_avx512
Definition tree-vectorizer.h:368
@ vect_partial_vectors_while_ult
Definition tree-vectorizer.h:367
bool vect_gather_scatter_fn_p(vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4346
class _slp_instance * slp_instance
bool vect_maybe_update_slp_op_vectype(slp_tree, tree)
Definition tree-vect-stmts.cc:14758
bool compatible_calls_p(gcall *, gcall *)
Definition tree-vect-slp.cc:989
#define STMT_VINFO_REDUC_IDX(S)
Definition tree-vectorizer.h:1574
class loop * slpeel_tree_duplicate_loop_to_edge_cfg(class loop *, edge, class loop *, edge, edge, edge *, bool=true, vec< basic_block > *=NULL)
Definition tree-vect-loop-manip.cc:1478
unsigned int vect_vf_for_cost(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2254
unsigned int vect_nunits_for_cost(tree vec_type)
Definition tree-vectorizer.h:2264
bool vect_get_range_info(tree, wide_int *, wide_int *)
Definition tree-vect-patterns.cc:69
bool vect_grouped_load_supported(tree, bool, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:6466
#define STMT_VINFO_IN_PATTERN_P(S)
Definition tree-vectorizer.h:1593
tree vect_create_addr_base_for_vector_ref(vec_info *, stmt_vec_info, gimple_seq *, tree)
Definition tree-vect-data-refs.cc:5326
stmt_vec_info vect_find_last_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3391
tree vect_get_main_loop_result(loop_vec_info, tree, tree)
Definition tree-vect-loop-manip.cc:3019
#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L)
Definition tree-vectorizer.h:1088
#define STMT_VINFO_DR_INFO(S)
Definition tree-vectorizer.h:1590
hash_map< slp_node_hash, bool > slp_compat_nodes_map_t
Definition tree-vectorizer.h:2733
tree get_related_vectype_for_scalar_type(machine_mode, tree, poly_uint64=0)
Definition tree-vect-stmts.cc:14255
bool lane_reducing_op_p(code_helper code)
Definition tree-vectorizer.h:2307
bool nested_in_vect_loop_p(class loop *loop, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1824
bool vect_slp_analyze_instance_dependence(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:1164
auto_vec< std::pair< data_reference *, tree > > drs_init_vec
Definition tree-vectorizer.h:720
edge vec_init_loop_exit_info(class loop *)
Definition tree-vect-loop.cc:978
opt_result vect_analyze_data_ref_accesses(vec_info *, vec< int > *)
Definition tree-vect-data-refs.cc:3546
int vect_get_stmt_cost(enum vect_cost_for_stmt type_of_cost)
Definition tree-vectorizer.h:1946
void vect_transform_grouped_load(vec_info *, stmt_vec_info, vec< tree >, int, gimple_stmt_iterator *)
Definition tree-vect-data-refs.cc:7157
dump_user_location_t find_loop_location(class loop *)
Definition tree-vect-loop-manip.cc:2070
bool vect_slp_analyze_operations(vec_info *)
Definition tree-vect-slp.cc:8504
tree neutral_op_for_reduction(tree, code_helper, tree, bool=true)
Definition tree-vect-loop.cc:3885
void vect_free_slp_tree(slp_tree)
Definition tree-vect-slp.cc:173
unsigned vectorize_loops(void)
bool vect_can_force_dr_alignment_p(const_tree, poly_uint64)
Definition tree-vect-data-refs.cc:7239
tree bump_vector_ptr(vec_info *, tree, gimple *, gimple_stmt_iterator *, stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:5711
bool vect_apply_runtime_profitability_check_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2297
unsigned int vect_get_num_vectors(poly_uint64 nunits, tree vectype)
Definition tree-vectorizer.h:2184
#define DR_TARGET_ALIGNMENT(DR)
Definition tree-vectorizer.h:2031
void vect_free_loop_info_assumptions(class loop *)
Definition tree-vectorizer.cc:863
void set_dr_target_alignment(dr_vec_info *dr_info, poly_uint64 val)
Definition tree-vectorizer.h:2063
stmt_vec_info vect_stmt_to_vectorize(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1902
bool supportable_indirect_convert_operation(code_helper, tree, tree, vec< std::pair< tree, tree_code > > &, tree=NULL_TREE, slp_tree=NULL)
Definition tree-vect-stmts.cc:15300
opt_loop_vec_info vect_analyze_loop(class loop *, gimple *, vec_info_shared *)
Definition tree-vect-loop.cc:3507
void dr_set_safe_speculative_read_required(stmt_vec_info stmt_info, bool requires_alignment)
Definition tree-vectorizer.h:2050
int vect_reduc_type(vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2831
#define STMT_VINFO_RELATED_STMT(S)
Definition tree-vectorizer.h:1594
bool is_pattern_stmt_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1856
#define SLP_TREE_MEMORY_ACCESS_TYPE(S)
Definition tree-vectorizer.h:363
#define MAX_VECTORIZATION_FACTOR
Definition tree-vectorizer.h:1809
int vect_pow2(int x)
Definition tree-vectorizer.h:1923
bool reduction_fn_for_scalar_code(code_helper, internal_fn *)
Definition tree-vect-loop.cc:3823
void vect_pattern_recog(vec_info *)
Definition tree-vect-patterns.cc:7178
int vect_slp_child_index_for_operand(const gimple *, int op, bool)
Definition tree-vect-slp.cc:618
vect_induction_op_type
Definition tree-vectorizer.h:75
@ vect_step_op_add
Definition tree-vectorizer.h:76
@ vect_step_op_shl
Definition tree-vectorizer.h:79
@ vect_step_op_mul
Definition tree-vectorizer.h:78
@ vect_step_op_neg
Definition tree-vectorizer.h:77
@ vect_step_op_shr
Definition tree-vectorizer.h:80
void vect_set_loop_condition(class loop *, edge, loop_vec_info, tree, tree, tree, bool)
Definition tree-vect-loop-manip.cc:1393
bool lane_reducing_stmt_p(gimple *stmt)
Definition tree-vectorizer.h:2315
void vect_prepare_for_masked_peels(loop_vec_info)
Definition tree-vect-loop-manip.cc:2652
void vect_get_store_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:1015
stmt_vec_info vect_find_first_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:3409
enum _complex_perm_kinds complex_perm_kinds_t
bool vect_is_integer_truncation(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2894
auto_vec< std::pair< unsigned, unsigned >, 16 > auto_lane_permutation_t
Definition tree-vectorizer.h:209
bool vect_use_loop_mask_for_alignment_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2167
tree vect_build_loop_niters(loop_vec_info, bool *=NULL)
Definition tree-vect-loop-manip.cc:2698
bool vect_get_loop_mask_type(loop_vec_info)
void vect_update_max_nunits(poly_uint64 *max_nunits, poly_uint64 nunits)
Definition tree-vectorizer.h:2230
slp_instance_kind
Definition tree-vectorizer.h:298
@ slp_inst_kind_ctor
Definition tree-vectorizer.h:303
@ slp_inst_kind_reduc_group
Definition tree-vectorizer.h:300
@ slp_inst_kind_store
Definition tree-vectorizer.h:299
@ slp_inst_kind_reduc_chain
Definition tree-vectorizer.h:301
@ slp_inst_kind_gcond
Definition tree-vectorizer.h:304
@ slp_inst_kind_bb_reduc
Definition tree-vectorizer.h:302
enum vect_cost_model loop_cost_model(loop_p loop)
Definition tree-vectorizer.h:2146
unsigned int vect_known_alignment_in_bytes(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vectorizer.h:2092
#define DR_MISALIGNMENT_UNKNOWN
Definition tree-vectorizer.h:2009
hash_map< slp_tree, complex_perm_kinds_t > slp_tree_to_load_perm_map_t
Definition tree-vectorizer.h:2728
opt_result vect_enhance_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2290
bool vect_is_reduction(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2811
#define LOOP_VINFO_EARLY_BREAKS(L)
Definition tree-vectorizer.h:1104
innermost_loop_behavior * vect_dr_behavior(vec_info *vinfo, dr_vec_info *dr_info)
Definition tree-vectorizer.h:2108
void vect_permute_store_chain(vec_info *, vec< tree > &, unsigned int, stmt_vec_info, gimple_stmt_iterator *, vec< tree > *)
Definition tree-vect-data-refs.cc:6040
tree perm_mask_for_reverse(tree)
Definition tree-vect-stmts.cc:1864
stmt_vec_info_type
Definition tree-vectorizer.h:1201
@ comparison_vec_info_type
Definition tree-vectorizer.h:1211
@ condition_vec_info_type
Definition tree-vectorizer.h:1210
@ type_conversion_vec_info_type
Definition tree-vectorizer.h:1216
@ load_vec_info_type
Definition tree-vectorizer.h:1203
@ loop_exit_ctrl_vec_info_type
Definition tree-vectorizer.h:1221
@ recurr_info_type
Definition tree-vectorizer.h:1220
@ op_vec_info_type
Definition tree-vectorizer.h:1206
@ reduc_vec_info_type
Definition tree-vectorizer.h:1212
@ assignment_vec_info_type
Definition tree-vectorizer.h:1209
@ undef_vec_info_type
Definition tree-vectorizer.h:1202
@ shift_vec_info_type
Definition tree-vectorizer.h:1205
@ type_demotion_vec_info_type
Definition tree-vectorizer.h:1215
@ store_vec_info_type
Definition tree-vectorizer.h:1204
@ cycle_phi_info_type
Definition tree-vectorizer.h:1217
@ phi_info_type
Definition tree-vectorizer.h:1219
@ lc_phi_info_type
Definition tree-vectorizer.h:1218
@ type_promotion_vec_info_type
Definition tree-vectorizer.h:1214
@ call_simd_clone_vec_info_type
Definition tree-vectorizer.h:1208
@ call_vec_info_type
Definition tree-vectorizer.h:1207
@ induc_vec_info_type
Definition tree-vectorizer.h:1213
void vect_get_slp_defs(slp_tree, vec< tree > *)
Definition tree-vect-slp.cc:10332
void vect_record_loop_len(loop_vec_info, vec_loop_lens *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11599
struct _slp_tree * slp_tree
Definition tree-vectorizer.h:25
gimple_seq vect_gen_len(tree, tree, tree, tree)
Definition tree-vect-stmts.cc:15648
bool dr_safe_speculative_read_required(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2036
tree vect_embedded_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2845
gimple * vect_loop_vectorized_call(class loop *, gcond **cond=NULL)
Definition tree-vectorizer.cc:878
void duplicate_and_interleave(vec_info *, gimple_seq *, tree, const vec< tree > &, unsigned int, vec< tree > &)
Definition tree-vect-slp.cc:9989
bool vect_grouped_store_supported(tree, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:5836
tree vect_setup_realignment(vec_info *, stmt_vec_info, gimple_stmt_iterator *, tree *, enum dr_alignment_support, tree, class loop **)
Definition tree-vect-data-refs.cc:6228
void vect_get_load_cost(vec_info *, stmt_vec_info, slp_tree, int, dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool)
Definition tree-vect-stmts.cc:1065
vect_def_type
Definition tree-vectorizer.h:60
@ vect_induction_def
Definition tree-vectorizer.h:65
@ vect_uninitialized_def
Definition tree-vectorizer.h:61
@ vect_nested_cycle
Definition tree-vectorizer.h:68
@ vect_constant_def
Definition tree-vectorizer.h:62
@ vect_external_def
Definition tree-vectorizer.h:63
@ vect_condition_def
Definition tree-vectorizer.h:70
@ vect_internal_def
Definition tree-vectorizer.h:64
@ vect_reduction_def
Definition tree-vectorizer.h:66
@ vect_double_reduction_def
Definition tree-vectorizer.h:67
@ vect_first_order_recurrence
Definition tree-vectorizer.h:69
@ vect_unknown_def_type
Definition tree-vectorizer.h:71
opt_result vect_analyze_data_ref_dependences(loop_vec_info, unsigned int *)
Definition tree-vect-data-refs.cc:887
vect_relevant
Definition tree-vectorizer.h:1226
@ vect_used_in_outer
Definition tree-vectorizer.h:1236
@ vect_used_in_scope
Definition tree-vectorizer.h:1246
@ vect_unused_in_scope
Definition tree-vectorizer.h:1227
@ vect_used_by_reduction
Definition tree-vectorizer.h:1244
@ vect_used_in_outer_by_reduction
Definition tree-vectorizer.h:1233
@ vect_used_only_live
Definition tree-vectorizer.h:1230
tree vect_get_mask_load_else(int, tree)
Definition tree-vect-stmts.cc:2964
bool is_loop_header_bb_p(basic_block bb)
Definition tree-vectorizer.h:1912
opt_result vect_analyze_loop_form(class loop *, gimple *, vect_loop_form_info *)
Definition tree-vect-loop.cc:1751
#define STMT_VINFO_DR_WRT_VEC_LOOP(S)
Definition tree-vectorizer.h:1577
bool vectorizable_live_operation(vec_info *, stmt_vec_info, slp_tree, slp_instance, int, bool, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:11001
slp_tree vect_create_new_slp_node(unsigned, tree_code)
Definition tree-vect-slp.cc:228
opt_result vect_analyze_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2984
bool unlimited_cost_model(loop_p loop)
Definition tree-vectorizer.h:2157
bool vectorizable_recurr(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9218
unsigned int vect_get_num_copies(vec_info *vinfo, slp_tree node, tree vectype=NULL)
Definition tree-vectorizer.h:2196
opt_result vect_mark_stmts_to_be_vectorized(loop_vec_info, bool *)
Definition tree-vect-stmts.cc:699
tree vect_gen_while(gimple_seq *, tree, tree, tree, const char *=nullptr)
Definition tree-vect-stmts.cc:15436
tree vect_get_store_rhs(stmt_vec_info)
Definition tree-vect-stmts.cc:1945
class loop * vect_transform_loop(loop_vec_info, gimple *)
Definition tree-vect-loop.cc:12042
void vect_slp_init(void)
Definition tree-vect-slp.cc:76
bool vect_check_gather_scatter(stmt_vec_info, loop_vec_info, gather_scatter_info *, vec< int > *=nullptr)
Definition tree-vect-data-refs.cc:4451
unsigned int vect_get_scalar_dr_size(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2288
enum dr_alignment_support vect_supportable_dr_alignment(vec_info *, dr_vec_info *, tree, int)
Definition tree-vect-data-refs.cc:7262
opt_result vect_determine_partial_vectors_and_peeling(loop_vec_info)
Definition tree-vect-loop.cc:2634
bool ref_within_array_bound(gimple *, tree)
Definition tree-if-conv.cc:848
dr_alignment_support
Definition tree-vectorizer.h:51
@ dr_unaligned_supported
Definition tree-vectorizer.h:53
@ dr_explicit_realign_optimized
Definition tree-vectorizer.h:55
@ dr_aligned
Definition tree-vectorizer.h:56
@ dr_explicit_realign
Definition tree-vectorizer.h:54
@ dr_unaligned_unsupported
Definition tree-vectorizer.h:52
int compare_step_with_zero(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:1853
bool supportable_narrowing_operation(code_helper, tree, tree, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:15127
#define STMT_VINFO_REDUC_TYPE(S)
Definition tree-vectorizer.h:1604
operation_type
Definition tree-vectorizer.h:44
@ ternary_op
Definition tree-vectorizer.h:47
@ unary_op
Definition tree-vectorizer.h:45
@ binary_op
Definition tree-vectorizer.h:46
void vect_record_loop_mask(loop_vec_info, vec_loop_masks *, unsigned int, tree, tree)
Definition tree-vect-loop.cc:11459
vect_memory_access_type
Definition tree-vectorizer.h:167
@ VMAT_STRIDED_SLP
Definition tree-vectorizer.h:199
@ VMAT_INVARIANT
Definition tree-vectorizer.h:169
@ VMAT_GATHER_SCATTER
Definition tree-vectorizer.h:202
@ VMAT_ELEMENTWISE
Definition tree-vectorizer.h:193
@ VMAT_CONTIGUOUS_DOWN
Definition tree-vectorizer.h:177
@ VMAT_CONTIGUOUS
Definition tree-vectorizer.h:172
@ VMAT_CONTIGUOUS_PERMUTE
Definition tree-vectorizer.h:182
@ VMAT_LOAD_STORE_LANES
Definition tree-vectorizer.h:189
@ VMAT_CONTIGUOUS_REVERSE
Definition tree-vectorizer.h:186
bool vect_chooses_same_modes_p(vec_info *, machine_mode)
Definition tree-vect-stmts.cc:14502
bool slpeel_can_duplicate_loop_p(const class loop *, const_edge, const_edge)
Definition tree-vect-loop-manip.cc:2038
opt_result vect_analyze_stmt(vec_info *, stmt_vec_info, bool *, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:13787
tree vect_get_loop_mask(loop_vec_info, gimple_stmt_iterator *, vec_loop_masks *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11481
slp_vect_type
Definition tree-vectorizer.h:1265
@ loop_vect
Definition tree-vectorizer.h:1266
@ hybrid
Definition tree-vectorizer.h:1268
@ pure_slp
Definition tree-vectorizer.h:1267
vect_reduction_type
Definition tree-vectorizer.h:84
@ TREE_CODE_REDUCTION
Definition tree-vectorizer.h:85
@ FOLD_LEFT_REDUCTION
Definition tree-vectorizer.h:103
@ EXTRACT_LAST_REDUCTION
Definition tree-vectorizer.h:95
@ INTEGER_INDUC_COND_REDUCTION
Definition tree-vectorizer.h:87
@ CONST_COND_REDUCTION
Definition tree-vectorizer.h:88
@ COND_REDUCTION
Definition tree-vectorizer.h:86
bool supportable_widening_operation(vec_info *, code_helper, stmt_vec_info, tree, tree, code_helper *, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:14798
auto_vec< rgroup_controls > vec_loop_lens
Definition tree-vectorizer.h:718
void set_dr_misalignment(dr_vec_info *dr_info, int val)
Definition tree-vectorizer.h:2013
bool vect_use_mask_type_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1847
pair_hash< nofree_ptr_hash< _slp_tree >, nofree_ptr_hash< _slp_tree > > slp_node_hash
Definition tree-vectorizer.h:2732
hash_map< tree_operand_hash, std::pair< stmt_vec_info, innermost_loop_behavior * > > vec_base_alignments
Definition tree-vectorizer.h:128
vect_memory_access_type vect_mem_access_type(stmt_vec_info stmt_info, slp_tree node)
Definition tree-vectorizer.h:2820
unsigned add_stmt_cost(vector_costs *costs, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where)
Definition tree-vectorizer.h:1966
tree vect_get_smallest_scalar_type(stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:161
vec< stmt_info_for_cost > stmt_vector_for_cost
Definition tree-vectorizer.h:122
stmt_vec_info info_for_reduction(vec_info *, stmt_vec_info)
Definition tree-vect-loop.cc:5597
bool known_alignment_for_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2083
void vect_remove_stores(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:14225
#define DR_GROUP_FIRST_ELEMENT(S)
Definition tree-vectorizer.h:1613
hash_set< tree_cond_mask_hash > vec_cond_masked_set_type
Definition tree-vectorizer.h:452
#define STMT_VINFO_REDUC_DEF(S)
Definition tree-vectorizer.h:1607
opt_result vect_find_stmt_data_reference(loop_p, gimple *, vec< data_reference_p > *, vec< int > *, int)
Definition tree-vect-data-refs.cc:4779
tree get_dr_vinfo_offset(vec_info *vinfo, dr_vec_info *dr_info, bool check_outer=false)
Definition tree-vectorizer.h:2124
bool vect_slp_if_converted_bb(basic_block bb, loop_p orig_loop)
Definition tree-vect-slp.cc:9846
tree vect_gen_perm_mask_checked(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:10067
bool vectorizable_induction(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9996
opt_result vect_analyze_slp(vec_info *, unsigned, bool)
Definition tree-vect-slp.cc:4812
vect_pattern *(* vect_pattern_decl_t)(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
Definition tree-vectorizer.h:2782
tree vect_create_destination_var(tree, tree)
Definition tree-vect-data-refs.cc:5802
bool vect_transform_lc_phi(loop_vec_info, stmt_vec_info, slp_tree)
Definition tree-vect-loop.cc:9049
tree vect_create_data_ref_ptr(vec_info *, stmt_vec_info, tree, class loop *, tree, tree *, gimple_stmt_iterator *, gimple **, bool, tree=NULL_TREE)
Definition tree-vect-data-refs.cc:5449
void vect_gen_vector_loop_niters(loop_vec_info, tree, tree *, tree *, bool)
Definition tree-vect-loop-manip.cc:2789
bool vect_make_slp_decision(loop_vec_info)
Definition tree-vect-slp.cc:7511
bool aligned_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:2073
unsigned HOST_WIDE_INT tree_to_uhwi(const_tree t)
Definition tree.cc:6594
#define TYPE_ALIGN_UNIT(NODE)
Definition tree.h:2400
poly_uint64 TYPE_VECTOR_SUBPARTS(const_tree node)
Definition tree.h:4311
#define TYPE_PRECISION(NODE)
Definition tree.h:2306
#define TREE_OPERAND(NODE, I)
Definition tree.h:1300
#define sizetype
Definition tree.h:5182
#define TREE_CODE_CLASS(CODE)
Definition tree.h:202
#define TYPE_SIZE_UNIT(NODE)
Definition tree.h:2303
#define CONVERT_EXPR_CODE_P(CODE)
Definition tree.h:553
#define TREE_TYPE(NODE)
Definition tree.h:512
#define INTEGRAL_TYPE_P(TYPE)
Definition tree.h:613
#define COMPARISON_CLASS_P(NODE)
Definition tree.h:256
#define NULL_TREE
Definition tree.h:317
#define FOR_EACH_VEC_ELT(V, I, P)
Definition vec.h:1895
constexpr vnull vNULL
Definition vec.h:569
generic_wide_int< widest_int_storage< WIDEST_INT_MAX_PRECISION > > widest_int
Definition wide-int.h:345
generic_wide_int< wide_int_storage > wide_int
Definition wide-int.h:343