GCC Middle and Back End API Reference
tree-vectorizer.h
Go to the documentation of this file.
1/* Vectorizer
2 Copyright (C) 2003-2024 Free Software Foundation, Inc.
3 Contributed by Dorit Naishlos <dorit@il.ibm.com>
4
5This file is part of GCC.
6
7GCC is free software; you can redistribute it and/or modify it under
8the terms of the GNU General Public License as published by the Free
9Software Foundation; either version 3, or (at your option) any later
10version.
11
12GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13WARRANTY; without even the implied warranty of MERCHANTABILITY or
14FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15for more details.
16
17You should have received a copy of the GNU General Public License
18along with GCC; see the file COPYING3. If not see
19<http://www.gnu.org/licenses/>. */
20
21#ifndef GCC_TREE_VECTORIZER_H
22#define GCC_TREE_VECTORIZER_H
23
25typedef struct _slp_tree *slp_tree;
26
27#include "tree-data-ref.h"
28#include "tree-hash-traits.h"
29#include "target.h"
30#include "internal-fn.h"
31#include "tree-ssa-operands.h"
32#include "gimple-match.h"
33
34/* Used for naming of new temporaries. */
41
42/* Defines type of operation. */
48
49/* Define type of available alignment support. */
57
58/* Define type of def-use cross-iteration cycle. */
72
73/* Define operation type of linear/non-linear induction variable. */
81
82/* Define type of reduction. */
88
89 /* Retain a scalar phi and use a FOLD_EXTRACT_LAST within the loop
90 to implement:
91
92 for (int i = 0; i < VF; ++i)
93 res = cond[i] ? val[i] : res; */
95
96 /* Use a folding reduction within the loop to implement:
97
98 for (int i = 0; i < VF; ++i)
99 res = res OP val[i];
100
101 (with no reassocation). */
104
105#define VECTORIZABLE_CYCLE_DEF(D) (((D) == vect_reduction_def) \
106 || ((D) == vect_double_reduction_def) \
107 || ((D) == vect_nested_cycle))
108
109/* Structure to encapsulate information about a group of like
110 instructions to be presented to the target cost model. */
120
122
123/* Maps base addresses to an innermost_loop_behavior and the stmt it was
124 derived from that gives the maximum known alignment for that base. */
126 std::pair<stmt_vec_info, innermost_loop_behavior *> >
128
129/* Represents elements [START, START + LENGTH) of cyclical array OPS*
130 (i.e. OPS repeated to give at least START + LENGTH elements) */
132{
133 tree op (unsigned int i) const;
134 bool all_same_p () const;
135
137 unsigned int start;
138 unsigned int length;
139};
140
141/* Return element I of the slice. */
142inline tree
143vect_scalar_ops_slice::op (unsigned int i) const
144{
145 return (*ops)[(i + start) % ops->length ()];
146}
147
148/* Hash traits for vect_scalar_ops_slice. */
149struct vect_scalar_ops_slice_hash : typed_noop_remove<vect_scalar_ops_slice>
150{
153
154 static const bool empty_zero_p = true;
155
156 static void mark_deleted (value_type &s) { s.length = ~0U; }
157 static void mark_empty (value_type &s) { s.length = 0; }
158 static bool is_deleted (const value_type &s) { return s.length == ~0U; }
159 static bool is_empty (const value_type &s) { return s.length == 0; }
160 static hashval_t hash (const value_type &);
161 static bool equal (const value_type &, const compare_type &);
162};
163
164/************************************************************************
165 SLP
166 ************************************************************************/
171
172/* A computation tree of an SLP instance. Each node corresponds to a group of
173 stmts to be packed in a SIMD stmt. */
174struct _slp_tree {
175 _slp_tree ();
176 ~_slp_tree ();
177
178 void push_vec_def (gimple *def);
179 void push_vec_def (tree def) { vec_defs.quick_push (def); }
180
181 /* Nodes that contain def-stmts of this node statements operands. */
183
184 /* A group of scalar stmts to be vectorized together. */
186 /* A group of scalar operands to be vectorized together. */
188 /* The representative that should be used for analysis and
189 code generation. */
191
192 /* Load permutation relative to the stores, NULL if there is no
193 permutation. */
195 /* Lane permutation of the operands scalar lanes encoded as pairs
196 of { operand number, lane number }. The number of elements
197 denotes the number of output lanes. */
199
200 /* Selected SIMD clone's function info. First vector element
201 is SIMD clone's function decl, followed by a pair of trees (base + step)
202 for linear arguments (pair of NULLs for other arguments). */
204
206 /* Vectorized defs. */
208 /* Number of vector stmts that are created to replace the group of scalar
209 stmts. It is calculated during the transformation phase as the number of
210 scalar elements in one scalar iteration (GROUP_SIZE) multiplied by VF
211 divided by vector size. */
212 unsigned int vec_stmts_size;
213
214 /* Reference count in the SLP graph. */
215 unsigned int refcnt;
216 /* The maximum number of vector elements for the subtree rooted
217 at this node. */
219 /* The DEF type of this node. */
221 /* The number of scalar lanes produced by this node. */
222 unsigned int lanes;
223 /* The operation of this node. */
225
227
228 /* If not NULL this is a cached failed SLP discovery attempt with
229 the lanes that failed during SLP discovery as 'false'. This is
230 a copy of the matches array. */
231 bool *failed;
232
233 /* Allocate from slp_tree_pool. */
234 static void *operator new (size_t);
235
236 /* Return memory to slp_tree_pool. */
237 static void operator delete (void *, size_t);
238
239 /* Linked list of nodes to release when we free the slp_tree_pool. */
242};
243
244/* The enum describes the type of operations that an SLP instance
245 can perform. */
246
254
255/* SLP instance is a sequence of stmts in a loop that can be packed into
256 SIMD stmts. */
257typedef class _slp_instance {
258public:
259 /* The root of SLP tree. */
261
262 /* For vector constructors, the constructor stmt that the SLP tree is built
263 from, NULL otherwise. */
265
266 /* For slp_inst_kind_bb_reduc the defs that were not vectorized, NULL
267 otherwise. */
269
270 /* The unrolling factor required to vectorized this SLP instance. */
272
273 /* The group of nodes that contain loads of this SLP instance. */
275
276 /* The SLP node containing the reduction PHIs. */
278
279 /* Vector cost of this entry to the SLP graph. */
281
282 /* If this instance is the main entry of a subgraph the set of
283 entries into the same subgraph, including itself. */
285
286 /* The type of operation the SLP instance is performing. */
288
291
292
293/* Access Functions. */
294#define SLP_INSTANCE_TREE(S) (S)->root
295#define SLP_INSTANCE_UNROLLING_FACTOR(S) (S)->unrolling_factor
296#define SLP_INSTANCE_LOADS(S) (S)->loads
297#define SLP_INSTANCE_ROOT_STMTS(S) (S)->root_stmts
298#define SLP_INSTANCE_REMAIN_DEFS(S) (S)->remain_defs
299#define SLP_INSTANCE_KIND(S) (S)->kind
300
301#define SLP_TREE_CHILDREN(S) (S)->children
302#define SLP_TREE_SCALAR_STMTS(S) (S)->stmts
303#define SLP_TREE_SCALAR_OPS(S) (S)->ops
304#define SLP_TREE_REF_COUNT(S) (S)->refcnt
305#define SLP_TREE_VEC_DEFS(S) (S)->vec_defs
306#define SLP_TREE_NUMBER_OF_VEC_STMTS(S) (S)->vec_stmts_size
307#define SLP_TREE_LOAD_PERMUTATION(S) (S)->load_permutation
308#define SLP_TREE_LANE_PERMUTATION(S) (S)->lane_permutation
309#define SLP_TREE_SIMD_CLONE_INFO(S) (S)->simd_clone_info
310#define SLP_TREE_DEF_TYPE(S) (S)->def_type
311#define SLP_TREE_VECTYPE(S) (S)->vectype
312#define SLP_TREE_REPRESENTATIVE(S) (S)->representative
313#define SLP_TREE_LANES(S) (S)->lanes
314#define SLP_TREE_CODE(S) (S)->code
315
322
323/* Key for map that records association between
324 scalar conditions and corresponding loop mask, and
325 is populated by vect_record_loop_mask. */
326
343
344template<>
346{
349
350 static inline hashval_t
352 {
354 h.add_int (v.code);
355 inchash::add_expr (v.op0, h, 0);
356 inchash::add_expr (v.op1, h, 0);
357 h.add_int (v.ncopies);
358 h.add_flag (v.inverted_p);
359 return h.end ();
360 }
361
362 static inline bool
364 {
365 return (existing.ncopies == candidate.ncopies
366 && existing.code == candidate.code
367 && existing.inverted_p == candidate.inverted_p
368 && operand_equal_p (existing.op0, candidate.op0, 0)
369 && operand_equal_p (existing.op1, candidate.op1, 0));
370 }
371
372 static const bool empty_zero_p = true;
373
374 static inline void
376 {
377 v.ncopies = 0;
378 v.inverted_p = false;
379 }
380
381 static inline bool
383 {
384 return v.ncopies == 0;
385 }
386
387 static inline void mark_deleted (value_type &) {}
388
389 static inline bool is_deleted (const value_type &)
390 {
391 return false;
392 }
393
394 static inline void remove (value_type &) {}
395};
396
398
399/* Key and map that records association between vector conditions and
400 corresponding loop mask, and is populated by prepare_vec_mask. */
401
404
405/* Describes two objects whose addresses must be unequal for the vectorized
406 loop to be valid. */
407typedef std::pair<tree, tree> vec_object_pair;
408
409/* Records that vectorization is only possible if abs (EXPR) >= MIN_VALUE.
410 UNSIGNED_P is true if we can assume that abs (EXPR) == EXPR. */
421
422/* Vectorizer state shared between different analyses like vector sizes
423 of the same CFG region. */
425public:
428
429 void save_datarefs();
430 void check_datarefs();
431
432 /* The number of scalar stmts. */
433 unsigned n_stmts;
434
435 /* All data references. Freed by free_data_refs, so not an auto_vec. */
438
439 /* The loop nest in which the data dependences are computed. */
441
442 /* All data dependences. Freed by free_dependence_relations, so not
443 an auto_vec. */
445};
446
447/* Vectorizer state common between loop and basic-block vectorization. */
448class vec_info {
449public:
451 enum vec_kind { bb, loop };
452
454 ~vec_info ();
455
467
468 /* The type of vectorization. */
470
471 /* Shared vectorizer state. */
473
474 /* The mapping of GIMPLE UID to stmt_vec_info. */
476 /* Whether the above mapping is complete. */
478
479 /* Whether we've done a transform we think OK to not update virtual
480 SSA form. */
482
483 /* The SLP graph. */
485
486 /* Maps base addresses to an innermost_loop_behavior that gives the maximum
487 known alignment for that base. */
489
490 /* All interleaving chains of stores, represented by the first
491 stmt in the chain. */
493
494 /* The set of vector modes used in the vectorized region. */
496
497 /* The argument we should pass to related_vector_mode when looking up
498 the vector mode for a scalar mode, or VOIDmode if we haven't yet
499 made any decisions about which vector modes to use. */
500 machine_mode vector_mode;
501
502private:
504 void set_vinfo_for_stmt (gimple *, stmt_vec_info, bool = true);
505 void free_stmt_vec_infos ();
507};
508
509class _loop_vec_info;
510class _bb_vec_info;
511
512template<>
513template<>
514inline bool
519
520template<>
521template<>
522inline bool
527
528/* In general, we can divide the vector statements in a vectorized loop
529 into related groups ("rgroups") and say that for each rgroup there is
530 some nS such that the rgroup operates on nS values from one scalar
531 iteration followed by nS values from the next. That is, if VF is the
532 vectorization factor of the loop, the rgroup operates on a sequence:
533
534 (1,1) (1,2) ... (1,nS) (2,1) ... (2,nS) ... (VF,1) ... (VF,nS)
535
536 where (i,j) represents a scalar value with index j in a scalar
537 iteration with index i.
538
539 [ We use the term "rgroup" to emphasise that this grouping isn't
540 necessarily the same as the grouping of statements used elsewhere.
541 For example, if we implement a group of scalar loads using gather
542 loads, we'll use a separate gather load for each scalar load, and
543 thus each gather load will belong to its own rgroup. ]
544
545 In general this sequence will occupy nV vectors concatenated
546 together. If these vectors have nL lanes each, the total number
547 of scalar values N is given by:
548
549 N = nS * VF = nV * nL
550
551 None of nS, VF, nV and nL are required to be a power of 2. nS and nV
552 are compile-time constants but VF and nL can be variable (if the target
553 supports variable-length vectors).
554
555 In classical vectorization, each iteration of the vector loop would
556 handle exactly VF iterations of the original scalar loop. However,
557 in vector loops that are able to operate on partial vectors, a
558 particular iteration of the vector loop might handle fewer than VF
559 iterations of the scalar loop. The vector lanes that correspond to
560 iterations of the scalar loop are said to be "active" and the other
561 lanes are said to be "inactive".
562
563 In such vector loops, many rgroups need to be controlled to ensure
564 that they have no effect for the inactive lanes. Conceptually, each
565 such rgroup needs a sequence of booleans in the same order as above,
566 but with each (i,j) replaced by a boolean that indicates whether
567 iteration i is active. This sequence occupies nV vector controls
568 that again have nL lanes each. Thus the control sequence as a whole
569 consists of VF independent booleans that are each repeated nS times.
570
571 Taking mask-based approach as a partially-populated vectors example.
572 We make the simplifying assumption that if a sequence of nV masks is
573 suitable for one (nS,nL) pair, we can reuse it for (nS/2,nL/2) by
574 VIEW_CONVERTing it. This holds for all current targets that support
575 fully-masked loops. For example, suppose the scalar loop is:
576
577 float *f;
578 double *d;
579 for (int i = 0; i < n; ++i)
580 {
581 f[i * 2 + 0] += 1.0f;
582 f[i * 2 + 1] += 2.0f;
583 d[i] += 3.0;
584 }
585
586 and suppose that vectors have 256 bits. The vectorized f accesses
587 will belong to one rgroup and the vectorized d access to another:
588
589 f rgroup: nS = 2, nV = 1, nL = 8
590 d rgroup: nS = 1, nV = 1, nL = 4
591 VF = 4
592
593 [ In this simple example the rgroups do correspond to the normal
594 SLP grouping scheme. ]
595
596 If only the first three lanes are active, the masks we need are:
597
598 f rgroup: 1 1 | 1 1 | 1 1 | 0 0
599 d rgroup: 1 | 1 | 1 | 0
600
601 Here we can use a mask calculated for f's rgroup for d's, but not
602 vice versa.
603
604 Thus for each value of nV, it is enough to provide nV masks, with the
605 mask being calculated based on the highest nL (or, equivalently, based
606 on the highest nS) required by any rgroup with that nV. We therefore
607 represent the entire collection of masks as a two-level table, with the
608 first level being indexed by nV - 1 (since nV == 0 doesn't exist) and
609 the second being indexed by the mask index 0 <= i < nV. */
610
611/* The controls (like masks or lengths) needed by rgroups with nV vectors,
612 according to the description above. */
614 /* The largest nS for all rgroups that use these controls.
615 For vect_partial_vectors_avx512 this is the constant nscalars_per_iter
616 for all members of the group. */
618
619 /* For the largest nS recorded above, the loop controls divide each scalar
620 into FACTOR equal-sized pieces. This is useful if we need to split
621 element-based accesses into byte-based accesses.
622 For vect_partial_vectors_avx512 this records nV instead. */
623 unsigned int factor;
624
625 /* This is a vector type with MAX_NSCALARS_PER_ITER * VF / nV elements.
626 For mask-based controls, it is the type of the masks in CONTROLS.
627 For length-based controls, it can be any vector type that has the
628 specified number of elements; the type of the elements doesn't matter. */
630
631 /* When there is no uniformly used LOOP_VINFO_RGROUP_COMPARE_TYPE this
632 is the rgroup specific type used. */
634
635 /* A vector of nV controls, in iteration order. */
637
638 /* In case of len_load and len_store with a bias there is only one
639 rgroup. This holds the adjusted loop length for the this rgroup. */
641};
642
644{
645 bool is_empty () const { return mask_set.is_empty (); }
646
647 /* Set to record vectype, nvector pairs. */
650
651 /* rgroup_controls used for the partial vector scheme. */
653};
654
656
658
659/* Information about a reduction accumulator from the main loop that could
660 conceivably be reused as the input to a reduction in an epilogue loop. */
662 /* The final value of the accumulator, which forms the input to the
663 reduction operation. */
665
666 /* The stmt_vec_info that describes the reduction (i.e. the one for
667 which is_reduc_info is true). */
669};
670
671/*-----------------------------------------------------------------*/
672/* Info on vectorized loops. */
673/*-----------------------------------------------------------------*/
674typedef class _loop_vec_info : public vec_info {
675public:
678
679 /* The loop to which this info struct refers to. */
680 class loop *loop;
681
682 /* The loop basic blocks. */
684
685 /* Number of latch executions. */
687 /* Number of iterations. */
689 /* Number of iterations of the original loop. */
691 /* Condition under which this loop is analyzed and versioned. */
693
694 /* The cost of the vector code. */
696
697 /* The cost of the scalar code. */
699
700 /* Threshold of number of iterations below which vectorization will not be
701 performed. It is calculated from MIN_PROFITABLE_ITERS and
702 param_min_vect_loop_bound. */
703 unsigned int th;
704
705 /* When applying loop versioning, the vector form should only be used
706 if the number of scalar iterations is >= this value, on top of all
707 the other requirements. Ignored when loop versioning is not being
708 used. */
710
711 /* Unrolling factor */
713
714 /* If this loop is an epilogue loop whose main loop can be skipped,
715 MAIN_LOOP_EDGE is the edge from the main loop to this loop's
716 preheader. SKIP_MAIN_LOOP_EDGE is then the edge that skips the
717 main loop and goes straight to this loop's preheader.
718
719 Both fields are null otherwise. */
722
723 /* If this loop is an epilogue loop that might be skipped after executing
724 the main loop, this edge is the one that skips the epilogue. */
726
727 /* The vectorized form of a standard reduction replaces the original
728 scalar code's final result (a loop-closed SSA PHI) with the result
729 of a vector-to-scalar reduction operation. After vectorization,
730 this variable maps these vector-to-scalar results to information
731 about the reductions that generated them. */
733
734 /* The number of times that the target suggested we unroll the vector loop
735 in order to promote more ILP. This value will be used to re-analyze the
736 loop for vectorization and if successful the value will be folded into
737 vectorization_factor (and therefore exactly divides
738 vectorization_factor). */
740
741 /* Maximum runtime vectorization factor, or MAX_VECTORIZATION_FACTOR
742 if there is no particular limit. */
744
745 /* The masks that a fully-masked loop should use to avoid operating
746 on inactive scalars. */
748
749 /* The lengths that a loop with length should use to avoid operating
750 on inactive scalars. */
752
753 /* Set of scalar conditions that have loop mask applied. */
755
756 /* Set of vector conditions that have loop mask applied. */
758
759 /* If we are using a loop mask to align memory addresses, this variable
760 contains the number of vector elements that we should skip in the
761 first iteration of the vector loop (i.e. the number of leading
762 elements that should be false in the first mask). */
764
765 /* The type that the loop control IV should be converted to before
766 testing which of the VF scalars are active and inactive.
767 Only meaningful if LOOP_VINFO_USING_PARTIAL_VECTORS_P. */
769
770 /* For #pragma omp simd if (x) loops the x expression. If constant 0,
771 the loop should not be vectorized, if constant non-zero, simd_if_cond
772 shouldn't be set and loop vectorized normally, if SSA_NAME, the loop
773 should be versioned on that condition, using scalar loop if the condition
774 is false and vectorized loop otherwise. */
776
777 /* The type that the vector loop control IV should have when
778 LOOP_VINFO_USING_PARTIAL_VECTORS_P is true. */
780
781 /* The style used for implementing partial vectors when
782 LOOP_VINFO_USING_PARTIAL_VECTORS_P is true. */
784
785 /* Unknown DRs according to which loop was peeled. */
787
788 /* peeling_for_alignment indicates whether peeling for alignment will take
789 place, and what the peeling factor should be:
790 peeling_for_alignment = X means:
791 If X=0: Peeling for alignment will not be applied.
792 If X>0: Peel first X iterations.
793 If X=-1: Generate a runtime test to calculate the number of iterations
794 to be peeled, using the dataref recorded in the field
795 unaligned_dr. */
797
798 /* The mask used to check the alignment of pointers or arrays. */
800
801 /* Data Dependence Relations defining address ranges that are candidates
802 for a run-time aliasing check. */
804
805 /* Data Dependence Relations defining address ranges together with segment
806 lengths from which the run-time aliasing check is built. */
808
809 /* Check that the addresses of each pair of objects is unequal. */
811
812 /* List of values that are required to be nonzero. This is used to check
813 whether things like "x[i * n] += 1;" are safe and eventually gets added
814 to the checks for lower bounds below. */
816
817 /* List of values that need to be checked for a minimum value. */
819
820 /* Statements in the loop that have data references that are candidates for a
821 runtime (loop versioning) misalignment check. */
823
824 /* Reduction cycles detected in the loop. Used in loop-aware SLP. */
826
827 /* All reduction chains in the loop, represented by the first
828 stmt in the chain. */
830
831 /* Cost vector for a single scalar iteration. */
833
834 /* Map of IV base/step expressions to inserted name in the preheader. */
836
837 /* Map of OpenMP "omp simd array" scan variables to corresponding
838 rhs of the store of the initializer. */
840
841 /* The unrolling factor needed to SLP the loop. In case of that pure SLP is
842 applied to the loop, i.e., no unrolling is needed, this is 1. */
844
845 /* The factor used to over weight those statements in an inner loop
846 relative to the loop being vectorized. */
848
849 /* Is the loop vectorizable? */
851
852 /* Records whether we still have the option of vectorizing this loop
853 using partially-populated vectors; in other words, whether it is
854 still possible for one iteration of the vector loop to handle
855 fewer than VF scalars. */
857
858 /* True if we've decided to use partially-populated vectors, so that
859 the vector loop can handle fewer than VF scalars. */
861
862 /* True if we've decided to use a decrementing loop control IV that counts
863 scalars. This can be done for any loop that:
864
865 (a) uses length "controls"; and
866 (b) can iterate more than once. */
868
869 /* True if we've decided to use output of select_vl to adjust IV of
870 both loop control and data reference pointer. This is only true
871 for single-rgroup control. */
873
874 /* True if we've decided to use partially-populated vectors for the
875 epilogue of loop. */
877
878 /* The bias for len_load and len_store. For now, only 0 and -1 are
879 supported. -1 must be used when a backend does not support
880 len_load/len_store with a length of zero. */
882
883 /* When we have grouped data accesses with gaps, we may introduce invalid
884 memory accesses. We peel the last iteration of the loop to prevent
885 this. */
887
888 /* When the number of iterations is not a multiple of the vector size
889 we need to peel off iterations at the end to form an epilogue loop. */
891
892 /* When the loop has early breaks that we can vectorize we need to peel
893 the loop for the break finding loop. */
895
896 /* List of loop additional IV conditionals found in the loop. */
898
899 /* Main loop IV cond. */
901
902 /* True if there are no loop carried data dependencies in the loop.
903 If loop->safelen <= 1, then this is always true, either the loop
904 didn't have any loop carried data dependencies, or the loop is being
905 vectorized guarded with some runtime alias checks, or couldn't
906 be vectorized at all, but then this field shouldn't be used.
907 For loop->safelen >= 2, the user has asserted that there are no
908 backward dependencies, but there still could be loop carried forward
909 dependencies in such loops. This flag will be false if normal
910 vectorizer data dependency analysis would fail or require versioning
911 for alias, but because of loop->safelen >= 2 it has been vectorized
912 even without versioning for alias. E.g. in:
913 #pragma omp simd
914 for (int i = 0; i < m; i++)
915 a[i] = a[i + k] * c;
916 (or #pragma simd or #pragma ivdep) we can vectorize this and it will
917 DTRT even for k > 0 && k < m, but without safelen we would not
918 vectorize this, so this field would be false. */
920
921 /* Mark loops having masked stores. */
923
924 /* Queued scaling factor for the scalar loop. */
926
927 /* If if-conversion versioned this loop before conversion, this is the
928 loop version without if-conversion. */
930
931 /* For loops being epilogues of already vectorized loops
932 this points to the original vectorized loop. Otherwise NULL. */
934
935 /* Used to store loop_vec_infos of epilogues of this loop during
936 analysis. */
938
939 /* The controlling loop IV for the current loop when vectorizing. This IV
940 controls the natural exits of the loop. */
942
943 /* The controlling loop IV for the epilogue loop when vectorizing. This IV
944 controls the natural exits of the loop. */
946
947 /* The controlling loop IV for the scalar loop being vectorized. This IV
948 controls the natural exits of the loop. */
950
951 /* Used to store the list of stores needing to be moved if doing early
952 break vectorization as they would violate the scalar loop semantics if
953 vectorized in their current location. These are stored in order that they
954 need to be moved. */
956
957 /* The final basic block where to move statements to. In the case of
958 multiple exits this could be pretty far away. */
960
961 /* Statements whose VUSES need updating if early break vectorization is to
962 happen. */
965
966/* Access Functions. */
967#define LOOP_VINFO_LOOP(L) (L)->loop
968#define LOOP_VINFO_IV_EXIT(L) (L)->vec_loop_iv_exit
969#define LOOP_VINFO_EPILOGUE_IV_EXIT(L) (L)->vec_epilogue_loop_iv_exit
970#define LOOP_VINFO_SCALAR_IV_EXIT(L) (L)->scalar_loop_iv_exit
971#define LOOP_VINFO_BBS(L) (L)->bbs
972#define LOOP_VINFO_NITERSM1(L) (L)->num_itersm1
973#define LOOP_VINFO_NITERS(L) (L)->num_iters
974/* Since LOOP_VINFO_NITERS and LOOP_VINFO_NITERSM1 can change after
975 prologue peeling retain total unchanged scalar loop iterations for
976 cost model. */
977#define LOOP_VINFO_NITERS_UNCHANGED(L) (L)->num_iters_unchanged
978#define LOOP_VINFO_NITERS_ASSUMPTIONS(L) (L)->num_iters_assumptions
979#define LOOP_VINFO_COST_MODEL_THRESHOLD(L) (L)->th
980#define LOOP_VINFO_VERSIONING_THRESHOLD(L) (L)->versioning_threshold
981#define LOOP_VINFO_VECTORIZABLE_P(L) (L)->vectorizable
982#define LOOP_VINFO_CAN_USE_PARTIAL_VECTORS_P(L) (L)->can_use_partial_vectors_p
983#define LOOP_VINFO_USING_PARTIAL_VECTORS_P(L) (L)->using_partial_vectors_p
984#define LOOP_VINFO_USING_DECREMENTING_IV_P(L) (L)->using_decrementing_iv_p
985#define LOOP_VINFO_USING_SELECT_VL_P(L) (L)->using_select_vl_p
986#define LOOP_VINFO_EPIL_USING_PARTIAL_VECTORS_P(L) \
987 (L)->epil_using_partial_vectors_p
988#define LOOP_VINFO_PARTIAL_LOAD_STORE_BIAS(L) (L)->partial_load_store_bias
989#define LOOP_VINFO_VECT_FACTOR(L) (L)->vectorization_factor
990#define LOOP_VINFO_MAX_VECT_FACTOR(L) (L)->max_vectorization_factor
991#define LOOP_VINFO_MASKS(L) (L)->masks
992#define LOOP_VINFO_LENS(L) (L)->lens
993#define LOOP_VINFO_MASK_SKIP_NITERS(L) (L)->mask_skip_niters
994#define LOOP_VINFO_RGROUP_COMPARE_TYPE(L) (L)->rgroup_compare_type
995#define LOOP_VINFO_RGROUP_IV_TYPE(L) (L)->rgroup_iv_type
996#define LOOP_VINFO_PARTIAL_VECTORS_STYLE(L) (L)->partial_vector_style
997#define LOOP_VINFO_PTR_MASK(L) (L)->ptr_mask
998#define LOOP_VINFO_N_STMTS(L) (L)->shared->n_stmts
999#define LOOP_VINFO_LOOP_NEST(L) (L)->shared->loop_nest
1000#define LOOP_VINFO_DATAREFS(L) (L)->shared->datarefs
1001#define LOOP_VINFO_DDRS(L) (L)->shared->ddrs
1002#define LOOP_VINFO_INT_NITERS(L) (TREE_INT_CST_LOW ((L)->num_iters))
1003#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L) (L)->peeling_for_alignment
1004#define LOOP_VINFO_UNALIGNED_DR(L) (L)->unaligned_dr
1005#define LOOP_VINFO_MAY_MISALIGN_STMTS(L) (L)->may_misalign_stmts
1006#define LOOP_VINFO_MAY_ALIAS_DDRS(L) (L)->may_alias_ddrs
1007#define LOOP_VINFO_COMP_ALIAS_DDRS(L) (L)->comp_alias_ddrs
1008#define LOOP_VINFO_CHECK_UNEQUAL_ADDRS(L) (L)->check_unequal_addrs
1009#define LOOP_VINFO_CHECK_NONZERO(L) (L)->check_nonzero
1010#define LOOP_VINFO_LOWER_BOUNDS(L) (L)->lower_bounds
1011#define LOOP_VINFO_GROUPED_STORES(L) (L)->grouped_stores
1012#define LOOP_VINFO_SLP_INSTANCES(L) (L)->slp_instances
1013#define LOOP_VINFO_SLP_UNROLLING_FACTOR(L) (L)->slp_unrolling_factor
1014#define LOOP_VINFO_REDUCTIONS(L) (L)->reductions
1015#define LOOP_VINFO_REDUCTION_CHAINS(L) (L)->reduction_chains
1016#define LOOP_VINFO_PEELING_FOR_GAPS(L) (L)->peeling_for_gaps
1017#define LOOP_VINFO_PEELING_FOR_NITER(L) (L)->peeling_for_niter
1018#define LOOP_VINFO_EARLY_BREAKS(L) (L)->early_breaks
1019#define LOOP_VINFO_EARLY_BRK_STORES(L) (L)->early_break_stores
1020#define LOOP_VINFO_EARLY_BREAKS_VECT_PEELED(L) \
1021 (single_pred ((L)->loop->latch) != (L)->vec_loop_iv_exit->src)
1022#define LOOP_VINFO_EARLY_BRK_DEST_BB(L) (L)->early_break_dest_bb
1023#define LOOP_VINFO_EARLY_BRK_VUSES(L) (L)->early_break_vuses
1024#define LOOP_VINFO_LOOP_CONDS(L) (L)->conds
1025#define LOOP_VINFO_LOOP_IV_COND(L) (L)->loop_iv_cond
1026#define LOOP_VINFO_NO_DATA_DEPENDENCIES(L) (L)->no_data_dependencies
1027#define LOOP_VINFO_SCALAR_LOOP(L) (L)->scalar_loop
1028#define LOOP_VINFO_SCALAR_LOOP_SCALING(L) (L)->scalar_loop_scaling
1029#define LOOP_VINFO_HAS_MASK_STORE(L) (L)->has_mask_store
1030#define LOOP_VINFO_SCALAR_ITERATION_COST(L) (L)->scalar_cost_vec
1031#define LOOP_VINFO_ORIG_LOOP_INFO(L) (L)->orig_loop_info
1032#define LOOP_VINFO_SIMD_IF_COND(L) (L)->simd_if_cond
1033#define LOOP_VINFO_INNER_LOOP_COST_FACTOR(L) (L)->inner_loop_cost_factor
1034
1035#define LOOP_VINFO_FULLY_MASKED_P(L) \
1036 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1037 && !LOOP_VINFO_MASKS (L).is_empty ())
1038
1039#define LOOP_VINFO_FULLY_WITH_LENGTH_P(L) \
1040 (LOOP_VINFO_USING_PARTIAL_VECTORS_P (L) \
1041 && !LOOP_VINFO_LENS (L).is_empty ())
1042
1043#define LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT(L) \
1044 ((L)->may_misalign_stmts.length () > 0)
1045#define LOOP_REQUIRES_VERSIONING_FOR_ALIAS(L) \
1046 ((L)->comp_alias_ddrs.length () > 0 \
1047 || (L)->check_unequal_addrs.length () > 0 \
1048 || (L)->lower_bounds.length () > 0)
1049#define LOOP_REQUIRES_VERSIONING_FOR_NITERS(L) \
1050 (LOOP_VINFO_NITERS_ASSUMPTIONS (L))
1051#define LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND(L) \
1052 (LOOP_VINFO_SIMD_IF_COND (L))
1053#define LOOP_REQUIRES_VERSIONING(L) \
1054 (LOOP_REQUIRES_VERSIONING_FOR_ALIGNMENT (L) \
1055 || LOOP_REQUIRES_VERSIONING_FOR_ALIAS (L) \
1056 || LOOP_REQUIRES_VERSIONING_FOR_NITERS (L) \
1057 || LOOP_REQUIRES_VERSIONING_FOR_SIMD_IF_COND (L))
1058
1059#define LOOP_VINFO_NITERS_KNOWN_P(L) \
1060 (tree_fits_shwi_p ((L)->num_iters) && tree_to_shwi ((L)->num_iters) > 0)
1061
1062#define LOOP_VINFO_EPILOGUE_P(L) \
1063 (LOOP_VINFO_ORIG_LOOP_INFO (L) != NULL)
1064
1065#define LOOP_VINFO_ORIG_MAX_VECT_FACTOR(L) \
1066 (LOOP_VINFO_MAX_VECT_FACTOR (LOOP_VINFO_ORIG_LOOP_INFO (L)))
1067
1068/* Wrapper for loop_vec_info, for tracking success/failure, where a non-NULL
1069 value signifies success, and a NULL value signifies failure, supporting
1070 propagating an opt_problem * describing the failure back up the call
1071 stack. */
1073
1074inline loop_vec_info
1076{
1077 return (loop_vec_info) loop->aux;
1078}
1079
1090
1091typedef class _bb_vec_info : public vec_info
1092{
1093public:
1095 ~_bb_vec_info ();
1096
1097 /* The region we are operating on. bbs[0] is the entry, excluding
1098 its PHI nodes. In the future we might want to track an explicit
1099 entry edge to cover bbs[0] PHI nodes and have a region entry
1100 insert location. */
1102
1105
1106#define BB_VINFO_BB(B) (B)->bb
1107#define BB_VINFO_GROUPED_STORES(B) (B)->grouped_stores
1108#define BB_VINFO_SLP_INSTANCES(B) (B)->slp_instances
1109#define BB_VINFO_DATAREFS(B) (B)->shared->datarefs
1110#define BB_VINFO_DDRS(B) (B)->shared->ddrs
1111
1112/*-----------------------------------------------------------------*/
1113/* Info on vectorized defs. */
1114/*-----------------------------------------------------------------*/
1137
1138/* Indicates whether/how a variable is used in the scope of loop/basic
1139 block. */
1142
1143 /* The def is only used outside the loop. */
1145 /* The def is in the inner loop, and the use is in the outer loop, and the
1146 use is a reduction stmt. */
1148 /* The def is in the inner loop, and the use is in the outer loop (and is
1149 not part of reduction). */
1151
1152 /* defs that feed computations that end up (only) in a reduction. These
1153 defs may be used by non-reduction stmts, but eventually, any
1154 computations/values that are affected by these defs are used to compute
1155 a reduction (i.e. don't get stored to memory, for example). We use this
1156 to identify computations that we can change the order in which they are
1157 computed. */
1159
1162
1163/* The type of vectorization that can be applied to the stmt: regular loop-based
1164 vectorization; pure SLP - the stmt is a part of SLP instances and does not
1165 have uses outside SLP instances; or hybrid SLP and loop-based - the stmt is
1166 a part of SLP instance and also must be loop-based vectorized, since it has
1167 uses outside SLP sequences.
1168
1169 In the loop context the meanings of pure and hybrid SLP are slightly
1170 different. By saying that pure SLP is applied to the loop, we mean that we
1171 exploit only intra-iteration parallelism in the loop; i.e., the loop can be
1172 vectorized without doing any conceptual unrolling, cause we don't pack
1173 together stmts from different iterations, only within a single iteration.
1174 Loop hybrid SLP means that we exploit both intra-iteration and
1175 inter-iteration parallelism (e.g., number of elements in the vector is 4
1176 and the slp-group-size is 2, in which case we don't have enough parallelism
1177 within an iteration, so we obtain the rest of the parallelism from subsequent
1178 iterations by unrolling the loop by 2). */
1184
1185/* Says whether a statement is a load, a store of a vectorized statement
1186 result, or a store of an invariant value. */
1192
1193/* Describes how we're going to vectorize an individual load or store,
1194 or a group of loads or stores. */
1196 /* An access to an invariant address. This is used only for loads. */
1198
1199 /* A simple contiguous access. */
1201
1202 /* A contiguous access that goes down in memory rather than up,
1203 with no additional permutation. This is used only for stores
1204 of invariants. */
1206
1207 /* A simple contiguous access in which the elements need to be permuted
1208 after loading or before storing. Only used for loop vectorization;
1209 SLP uses separate permutes. */
1211
1212 /* A simple contiguous access in which the elements need to be reversed
1213 after loading or before storing. */
1215
1216 /* An access that uses IFN_LOAD_LANES or IFN_STORE_LANES. */
1218
1219 /* An access in which each scalar element is loaded or stored
1220 individually. */
1222
1223 /* A hybrid of VMAT_CONTIGUOUS and VMAT_ELEMENTWISE, used for grouped
1224 SLP accesses. Each unrolled iteration uses a contiguous load
1225 or store for the whole group, but the groups from separate iterations
1226 are combined in the same way as for VMAT_ELEMENTWISE. */
1228
1229 /* The access uses gather loads or scatter stores. */
1232
1234public:
1235 /* The data reference itself. */
1237 /* The statement that contains the data reference. */
1239 /* The analysis group this DR belongs to when doing BB vectorization.
1240 DRs of the same group belong to the same conditional execution context. */
1241 unsigned group;
1242 /* The misalignment in bytes of the reference, or -1 if not known. */
1244 /* The byte alignment that we'd ideally like the reference to have,
1245 and the value that misalignment is measured against. */
1247 /* If true the alignment of base_decl needs to be increased. */
1250
1251 /* Stores current vectorized loop's offset. To be added to the DR's
1252 offset to calculate current offset of data reference. */
1254};
1255
1256typedef struct data_reference *dr_p;
1257
1259public:
1260
1262
1263 /* Indicates whether this stmts is part of a computation whose result is
1264 used outside the loop. */
1265 bool live;
1266
1267 /* Stmt is part of some pattern (computation idiom) */
1269
1270 /* True if the statement was created during pattern recognition as
1271 part of the replacement for RELATED_STMT. This implies that the
1272 statement isn't part of any basic block, although for convenience
1273 its gimple_bb is the same as for RELATED_STMT. */
1275
1276 /* Is this statement vectorizable or should it be skipped in (partial)
1277 vectorization. */
1279
1280 /* The stmt to which this info struct refers to. */
1282
1283 /* The vector type to be used for the LHS of this statement. */
1285
1286 /* The vectorized stmts. */
1288
1289 /* The following is relevant only for stmts that contain a non-scalar
1290 data-ref (array/pointer/struct access). A GIMPLE stmt is expected to have
1291 at most one such data-ref. */
1292
1294
1295 /* Information about the data-ref relative to this loop
1296 nest (the loop that is being considered for vectorization). */
1298
1299 /* For loop PHI nodes, the base and evolution part of it. This makes sure
1300 this information is still available in vect_update_ivs_after_vectorizer
1301 where we may not be able to re-analyze the PHI nodes evolution as
1302 peeling for the prologue loop can make it unanalyzable. The evolution
1303 part is still correct after peeling, but the base may have changed from
1304 the version here. */
1308
1309 /* Used for various bookkeeping purposes, generally holding a pointer to
1310 some other stmt S that is in some way "related" to this stmt.
1311 Current use of this field is:
1312 If this stmt is part of a pattern (i.e. the field 'in_pattern_p' is
1313 true): S is the "pattern stmt" that represents (and replaces) the
1314 sequence of stmts that constitutes the pattern. Similarly, the
1315 related_stmt of the "pattern stmt" points back to this stmt (which is
1316 the last stmt in the original sequence of stmts that constitutes the
1317 pattern). */
1319
1320 /* Used to keep a sequence of def stmts of a pattern stmt if such exists.
1321 The sequence is attached to the original statement rather than the
1322 pattern statement. */
1324
1325 /* Selected SIMD clone's function info. First vector element
1326 is SIMD clone's function decl, followed by a pair of trees (base + step)
1327 for linear arguments (pair of NULLs for other arguments). */
1329
1330 /* Classify the def of this stmt. */
1332
1333 /* Whether the stmt is SLPed, loop-based vectorized, or both. */
1335
1336 /* Interleaving and reduction chains info. */
1337 /* First element in the group. */
1339 /* Pointer to the next element in the group. */
1341 /* The size of the group. */
1342 unsigned int size;
1343 /* For stores, number of stores from this group seen. We vectorize the last
1344 one. */
1345 unsigned int store_count;
1346 /* For loads only, the gap from the previous load. For consecutive loads, GAP
1347 is 1. */
1348 unsigned int gap;
1349
1350 /* The minimum negative dependence distance this stmt participates in
1351 or zero if none. */
1352 unsigned int min_neg_dist;
1353
1354 /* Not all stmts in the loop need to be vectorized. e.g, the increment
1355 of the loop induction variable and computation of array indexes. relevant
1356 indicates whether the stmt needs to be vectorized. */
1358
1359 /* For loads if this is a gather, for stores if this is a scatter. */
1361
1362 /* True if this is an access with loop-invariant stride. */
1364
1365 /* For both loads and stores. */
1367
1368 /* Classifies how the load or store is going to be implemented
1369 for loop vectorization. */
1371
1372 /* For INTEGER_INDUC_COND_REDUCTION, the initial value to be used. */
1374
1375 /* If not NULL the value to be added to compute final reduction value. */
1377
1378 /* On a reduction PHI the reduction type as detected by
1379 vect_is_simple_reduction and vectorizable_reduction. */
1381
1382 /* The original reduction code, to be used in the epilogue. */
1384 /* An internal function we should use in the epilogue. */
1386
1387 /* On a stmt participating in the reduction the index of the operand
1388 on the reduction SSA cycle. */
1390
1391 /* On a reduction PHI the def returned by vect_force_simple_reduction.
1392 On the def returned by vect_force_simple_reduction the
1393 corresponding PHI. */
1395
1396 /* The vector input type relevant for reduction vectorization. */
1398
1399 /* The vector type for performing the actual reduction. */
1401
1402 /* If IS_REDUC_INFO is true and if the vector code is performing
1403 N scalar reductions in parallel, this variable gives the initial
1404 scalar values of those N reductions. */
1406
1407 /* If IS_REDUC_INFO is true and if the vector code is performing
1408 N scalar reductions in parallel, this variable gives the vectorized code's
1409 final (scalar) result for each of those N reductions. In other words,
1410 REDUC_SCALAR_RESULTS[I] replaces the original scalar code's loop-closed
1411 SSA PHI for reduction number I. */
1413
1414 /* Only meaningful if IS_REDUC_INFO. If non-null, the reduction is
1415 being performed by an epilogue loop and we have decided to reuse
1416 this accumulator from the main loop. */
1418
1419 /* Whether we force a single cycle PHI during reduction vectorization. */
1421
1422 /* Whether on this stmt reduction meta is recorded. */
1424
1425 /* If nonzero, the lhs of the statement could be truncated to this
1426 many bits without affecting any users of the result. */
1428
1429 /* If nonzero, all non-boolean input operands have the same precision,
1430 and they could each be truncated to this many bits without changing
1431 the result. */
1433
1434 /* If OPERATION_BITS is nonzero, the statement could be performed on
1435 an integer with the sign and number of bits given by OPERATION_SIGN
1436 and OPERATION_BITS without changing the result. */
1439
1440 /* If the statement produces a boolean result, this value describes
1441 how we should choose the associated vector type. The possible
1442 values are:
1443
1444 - an integer precision N if we should use the vector mask type
1445 associated with N-bit integers. This is only used if all relevant
1446 input booleans also want the vector mask type for N-bit integers,
1447 or if we can convert them into that form by pattern-matching.
1448
1449 - ~0U if we considered choosing a vector mask type but decided
1450 to treat the boolean as a normal integer type instead.
1451
1452 - 0 otherwise. This means either that the operation isn't one that
1453 could have a vector mask type (and so should have a normal vector
1454 type instead) or that we simply haven't made a choice either way. */
1455 unsigned int mask_precision;
1456
1457 /* True if this is only suitable for SLP vectorization. */
1459
1460 /* True if this is a pattern that can only be handled by SLP
1461 vectorization. */
1463};
1464
1465/* Information about a gather/scatter call. */
1467 /* The internal function to use for the gather/scatter operation,
1468 or IFN_LAST if a built-in function should be used instead. */
1470
1471 /* The FUNCTION_DECL for the built-in gather/scatter function,
1472 or null if an internal function should be used instead. */
1474
1475 /* The loop-invariant base value. */
1477
1478 /* The original scalar offset, which is a non-loop-invariant SSA_NAME. */
1480
1481 /* Each offset element should be multiplied by this amount before
1482 being added to the base. */
1484
1485 /* The definition type for the vectorized offset. */
1487
1488 /* The type of the vectorized offset. */
1490
1491 /* The type of the scalar elements after loading or before storing. */
1493
1494 /* The type of the scalar elements being loaded or stored. */
1496};
1497
1498/* Access Functions. */
1499#define STMT_VINFO_TYPE(S) (S)->type
1500#define STMT_VINFO_STMT(S) (S)->stmt
1501#define STMT_VINFO_RELEVANT(S) (S)->relevant
1502#define STMT_VINFO_LIVE_P(S) (S)->live
1503#define STMT_VINFO_VECTYPE(S) (S)->vectype
1504#define STMT_VINFO_VEC_STMTS(S) (S)->vec_stmts
1505#define STMT_VINFO_VECTORIZABLE(S) (S)->vectorizable
1506#define STMT_VINFO_DATA_REF(S) ((S)->dr_aux.dr + 0)
1507#define STMT_VINFO_GATHER_SCATTER_P(S) (S)->gather_scatter_p
1508#define STMT_VINFO_STRIDED_P(S) (S)->strided_p
1509#define STMT_VINFO_MEMORY_ACCESS_TYPE(S) (S)->memory_access_type
1510#define STMT_VINFO_SIMD_LANE_ACCESS_P(S) (S)->simd_lane_access_p
1511#define STMT_VINFO_VEC_INDUC_COND_INITIAL_VAL(S) (S)->induc_cond_initial_val
1512#define STMT_VINFO_REDUC_EPILOGUE_ADJUSTMENT(S) (S)->reduc_epilogue_adjustment
1513#define STMT_VINFO_REDUC_IDX(S) (S)->reduc_idx
1514#define STMT_VINFO_FORCE_SINGLE_CYCLE(S) (S)->force_single_cycle
1515
1516#define STMT_VINFO_DR_WRT_VEC_LOOP(S) (S)->dr_wrt_vec_loop
1517#define STMT_VINFO_DR_BASE_ADDRESS(S) (S)->dr_wrt_vec_loop.base_address
1518#define STMT_VINFO_DR_INIT(S) (S)->dr_wrt_vec_loop.init
1519#define STMT_VINFO_DR_OFFSET(S) (S)->dr_wrt_vec_loop.offset
1520#define STMT_VINFO_DR_STEP(S) (S)->dr_wrt_vec_loop.step
1521#define STMT_VINFO_DR_BASE_ALIGNMENT(S) (S)->dr_wrt_vec_loop.base_alignment
1522#define STMT_VINFO_DR_BASE_MISALIGNMENT(S) \
1523 (S)->dr_wrt_vec_loop.base_misalignment
1524#define STMT_VINFO_DR_OFFSET_ALIGNMENT(S) \
1525 (S)->dr_wrt_vec_loop.offset_alignment
1526#define STMT_VINFO_DR_STEP_ALIGNMENT(S) \
1527 (S)->dr_wrt_vec_loop.step_alignment
1528
1529#define STMT_VINFO_DR_INFO(S) \
1530 (gcc_checking_assert ((S)->dr_aux.stmt == (S)), &(S)->dr_aux)
1531
1532#define STMT_VINFO_IN_PATTERN_P(S) (S)->in_pattern_p
1533#define STMT_VINFO_RELATED_STMT(S) (S)->related_stmt
1534#define STMT_VINFO_PATTERN_DEF_SEQ(S) (S)->pattern_def_seq
1535#define STMT_VINFO_SIMD_CLONE_INFO(S) (S)->simd_clone_info
1536#define STMT_VINFO_DEF_TYPE(S) (S)->def_type
1537#define STMT_VINFO_GROUPED_ACCESS(S) \
1538 ((S)->dr_aux.dr && DR_GROUP_FIRST_ELEMENT(S))
1539#define STMT_VINFO_LOOP_PHI_EVOLUTION_BASE_UNCHANGED(S) (S)->loop_phi_evolution_base_unchanged
1540#define STMT_VINFO_LOOP_PHI_EVOLUTION_PART(S) (S)->loop_phi_evolution_part
1541#define STMT_VINFO_LOOP_PHI_EVOLUTION_TYPE(S) (S)->loop_phi_evolution_type
1542#define STMT_VINFO_MIN_NEG_DIST(S) (S)->min_neg_dist
1543#define STMT_VINFO_REDUC_TYPE(S) (S)->reduc_type
1544#define STMT_VINFO_REDUC_CODE(S) (S)->reduc_code
1545#define STMT_VINFO_REDUC_FN(S) (S)->reduc_fn
1546#define STMT_VINFO_REDUC_DEF(S) (S)->reduc_def
1547#define STMT_VINFO_REDUC_VECTYPE(S) (S)->reduc_vectype
1548#define STMT_VINFO_REDUC_VECTYPE_IN(S) (S)->reduc_vectype_in
1549#define STMT_VINFO_SLP_VECT_ONLY(S) (S)->slp_vect_only_p
1550#define STMT_VINFO_SLP_VECT_ONLY_PATTERN(S) (S)->slp_vect_pattern_only_p
1551
1552#define DR_GROUP_FIRST_ELEMENT(S) \
1553 (gcc_checking_assert ((S)->dr_aux.dr), (S)->first_element)
1554#define DR_GROUP_NEXT_ELEMENT(S) \
1555 (gcc_checking_assert ((S)->dr_aux.dr), (S)->next_element)
1556#define DR_GROUP_SIZE(S) \
1557 (gcc_checking_assert ((S)->dr_aux.dr), (S)->size)
1558#define DR_GROUP_STORE_COUNT(S) \
1559 (gcc_checking_assert ((S)->dr_aux.dr), (S)->store_count)
1560#define DR_GROUP_GAP(S) \
1561 (gcc_checking_assert ((S)->dr_aux.dr), (S)->gap)
1562
1563#define REDUC_GROUP_FIRST_ELEMENT(S) \
1564 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->first_element)
1565#define REDUC_GROUP_NEXT_ELEMENT(S) \
1566 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->next_element)
1567#define REDUC_GROUP_SIZE(S) \
1568 (gcc_checking_assert (!(S)->dr_aux.dr), (S)->size)
1569
1570#define STMT_VINFO_RELEVANT_P(S) ((S)->relevant != vect_unused_in_scope)
1571
1572#define HYBRID_SLP_STMT(S) ((S)->slp_type == hybrid)
1573#define PURE_SLP_STMT(S) ((S)->slp_type == pure_slp)
1574#define STMT_SLP_TYPE(S) (S)->slp_type
1575
1576/* Contains the scalar or vector costs for a vec_info. */
1578{
1579public:
1580 vector_costs (vec_info *, bool);
1581 virtual ~vector_costs () {}
1582
1583 /* Update the costs in response to adding COUNT copies of a statement.
1584
1585 - WHERE specifies whether the cost occurs in the loop prologue,
1586 the loop body, or the loop epilogue.
1587 - KIND is the kind of statement, which is always meaningful.
1588 - STMT_INFO or NODE, if nonnull, describe the statement that will be
1589 vectorized.
1590 - VECTYPE, if nonnull, is the vector type that the vectorized
1591 statement will operate on. Note that this should be used in
1592 preference to STMT_VINFO_VECTYPE (STMT_INFO) since the latter
1593 is not correct for SLP.
1594 - for unaligned_load and unaligned_store statements, MISALIGN is
1595 the byte misalignment of the load or store relative to the target's
1596 preferred alignment for VECTYPE, or DR_MISALIGNMENT_UNKNOWN
1597 if the misalignment is not known.
1598
1599 Return the calculated cost as well as recording it. The return
1600 value is used for dumping purposes. */
1601 virtual unsigned int add_stmt_cost (int count, vect_cost_for_stmt kind,
1602 stmt_vec_info stmt_info,
1603 slp_tree node,
1604 tree vectype, int misalign,
1606
1607 /* Finish calculating the cost of the code. The results can be
1608 read back using the functions below.
1609
1610 If the costs describe vector code, SCALAR_COSTS gives the costs
1611 of the corresponding scalar code, otherwise it is null. */
1612 virtual void finish_cost (const vector_costs *scalar_costs);
1613
1614 /* The costs in THIS and OTHER both describe ways of vectorizing
1615 a main loop. Return true if the costs described by THIS are
1616 cheaper than the costs described by OTHER. Return false if any
1617 of the following are true:
1618
1619 - THIS and OTHER are of equal cost
1620 - OTHER is better than THIS
1621 - we can't be sure about the relative costs of THIS and OTHER. */
1622 virtual bool better_main_loop_than_p (const vector_costs *other) const;
1623
1624 /* Likewise, but the costs in THIS and OTHER both describe ways of
1625 vectorizing an epilogue loop of MAIN_LOOP. */
1626 virtual bool better_epilogue_loop_than_p (const vector_costs *other,
1627 loop_vec_info main_loop) const;
1628
1629 unsigned int prologue_cost () const;
1630 unsigned int body_cost () const;
1631 unsigned int epilogue_cost () const;
1632 unsigned int outside_cost () const;
1633 unsigned int total_cost () const;
1634 unsigned int suggested_unroll_factor () const;
1635
1636protected:
1638 unsigned int);
1640 unsigned int);
1641 int compare_inside_loop_cost (const vector_costs *) const;
1642 int compare_outside_loop_cost (const vector_costs *) const;
1643
1644 /* The region of code that we're considering vectorizing. */
1646
1647 /* True if we're costing the scalar code, false if we're costing
1648 the vector code. */
1650
1651 /* The costs of the three regions, indexed by vect_cost_model_location. */
1652 unsigned int m_costs[3];
1653
1654 /* The suggested unrolling factor determined at finish_cost. */
1656
1657 /* True if finish_cost has been called. */
1659};
1660
1661/* Create costs for VINFO. COSTING_FOR_SCALAR is true if the costs
1662 are for scalar code, false if they are for vector code. */
1663
1664inline
1666 : m_vinfo (vinfo),
1667 m_costing_for_scalar (costing_for_scalar),
1668 m_costs (),
1669 m_suggested_unroll_factor(1),
1670 m_finished (false)
1671{
1672}
1673
1674/* Return the cost of the prologue code (in abstract units). */
1675
1676inline unsigned int
1682
1683/* Return the cost of the body code (in abstract units). */
1684
1685inline unsigned int
1691
1692/* Return the cost of the epilogue code (in abstract units). */
1693
1694inline unsigned int
1700
1701/* Return the cost of the prologue and epilogue code (in abstract units). */
1702
1703inline unsigned int
1705{
1706 return prologue_cost () + epilogue_cost ();
1707}
1708
1709/* Return the cost of the prologue, body and epilogue code
1710 (in abstract units). */
1711
1712inline unsigned int
1714{
1715 return body_cost () + outside_cost ();
1716}
1717
1718/* Return the suggested unroll factor. */
1719
1720inline unsigned int
1726
1727#define VECT_MAX_COST 1000
1728
1729/* The maximum number of intermediate steps required in multi-step type
1730 conversion. */
1731#define MAX_INTERM_CVT_STEPS 3
1732
1733#define MAX_VECTORIZATION_FACTOR INT_MAX
1734
1735/* Nonzero if TYPE represents a (scalar) boolean type or type
1736 in the middle-end compatible with it (unsigned precision 1 integral
1737 types). Used to determine which types should be vectorized as
1738 VECTOR_BOOLEAN_TYPE_P. */
1739
1740#define VECT_SCALAR_BOOLEAN_TYPE_P(TYPE) \
1741 (TREE_CODE (TYPE) == BOOLEAN_TYPE \
1742 || ((TREE_CODE (TYPE) == INTEGER_TYPE \
1743 || TREE_CODE (TYPE) == ENUMERAL_TYPE) \
1744 && TYPE_PRECISION (TYPE) == 1 \
1745 && TYPE_UNSIGNED (TYPE)))
1746
1747inline bool
1749{
1750 return (loop->inner
1751 && (loop->inner == (gimple_bb (stmt_info->stmt))->loop_father));
1752}
1753
1754/* PHI is either a scalar reduction phi or a scalar induction phi.
1755 Return the initial value of the variable on entry to the containing
1756 loop. */
1757
1758inline tree
1760{
1761 basic_block bb = gimple_bb (phi);
1763 gcc_assert (pe->dest == bb);
1764 return PHI_ARG_DEF_FROM_EDGE (phi, pe);
1765}
1766
1767/* Return true if STMT_INFO should produce a vector mask type rather than
1768 a normal nonmask type. */
1769
1770inline bool
1772{
1773 return stmt_info->mask_precision && stmt_info->mask_precision != ~0U;
1774}
1775
1776/* Return TRUE if a statement represented by STMT_INFO is a part of a
1777 pattern. */
1778
1779inline bool
1781{
1782 return stmt_info->pattern_stmt_p;
1783}
1784
1785/* If STMT_INFO is a pattern statement, return the statement that it
1786 replaces, otherwise return STMT_INFO itself. */
1787
1788inline stmt_vec_info
1790{
1791 if (is_pattern_stmt_p (stmt_info))
1792 return STMT_VINFO_RELATED_STMT (stmt_info);
1793 return stmt_info;
1794}
1795
1796/* Return the later statement between STMT1_INFO and STMT2_INFO. */
1797
1798inline stmt_vec_info
1807
1808/* If STMT_INFO has been replaced by a pattern statement, return the
1809 replacement statement, otherwise return STMT_INFO itself. */
1810
1811inline stmt_vec_info
1813{
1814 if (STMT_VINFO_IN_PATTERN_P (stmt_info))
1815 return STMT_VINFO_RELATED_STMT (stmt_info);
1816 return stmt_info;
1817}
1818
1819/* Return true if BB is a loop header. */
1820
1821inline bool
1823{
1824 if (bb == (bb->loop_father)->header)
1825 return true;
1826
1827 return false;
1828}
1829
1830/* Return pow2 (X). */
1831
1832inline int
1834{
1835 int i, res = 1;
1836
1837 for (i = 0; i < x; i++)
1838 res *= 2;
1839
1840 return res;
1841}
1842
1843/* Alias targetm.vectorize.builtin_vectorization_cost. */
1844
1845inline int
1847 tree vectype, int misalign)
1848{
1849 return targetm.vectorize.builtin_vectorization_cost (type_of_cost,
1850 vectype, misalign);
1851}
1852
1853/* Get cost by calling cost target builtin. */
1854
1855inline
1860
1861/* Alias targetm.vectorize.init_cost. */
1862
1863inline vector_costs *
1865{
1866 return targetm.vectorize.create_costs (vinfo, costing_for_scalar);
1867}
1868
1869extern void dump_stmt_cost (FILE *, int, enum vect_cost_for_stmt,
1870 stmt_vec_info, slp_tree, tree, int, unsigned,
1872
1873/* Alias targetm.vectorize.add_stmt_cost. */
1874
1875inline unsigned
1877 enum vect_cost_for_stmt kind,
1878 stmt_vec_info stmt_info, slp_tree node,
1879 tree vectype, int misalign,
1880 enum vect_cost_model_location where)
1881{
1882 unsigned cost = costs->add_stmt_cost (count, kind, stmt_info, node, vectype,
1883 misalign, where);
1884 if (dump_file && (dump_flags & TDF_DETAILS))
1885 dump_stmt_cost (dump_file, count, kind, stmt_info, node, vectype, misalign,
1886 cost, where);
1887 return cost;
1888}
1889
1890inline unsigned
1892 enum vect_cost_model_location where)
1893{
1895 || kind == scalar_stmt);
1896 return add_stmt_cost (costs, count, kind, NULL, NULL, NULL_TREE, 0, where);
1897}
1898
1899/* Alias targetm.vectorize.add_stmt_cost. */
1900
1901inline unsigned
1903{
1904 return add_stmt_cost (costs, i->count, i->kind, i->stmt_info, i->node,
1905 i->vectype, i->misalign, i->where);
1906}
1907
1908/* Alias targetm.vectorize.finish_cost. */
1909
1910inline void
1912 unsigned *prologue_cost, unsigned *body_cost,
1913 unsigned *epilogue_cost, unsigned *suggested_unroll_factor = NULL)
1914{
1915 costs->finish_cost (scalar_costs);
1916 *prologue_cost = costs->prologue_cost ();
1917 *body_cost = costs->body_cost ();
1918 *epilogue_cost = costs->epilogue_cost ();
1919 if (suggested_unroll_factor)
1920 *suggested_unroll_factor = costs->suggested_unroll_factor ();
1921}
1922
1923inline void
1925{
1926 stmt_info_for_cost *cost;
1927 unsigned i;
1928 FOR_EACH_VEC_ELT (*cost_vec, i, cost)
1929 add_stmt_cost (costs, cost->count, cost->kind, cost->stmt_info,
1930 cost->node, cost->vectype, cost->misalign, cost->where);
1931}
1932
1933/*-----------------------------------------------------------------*/
1934/* Info on data references alignment. */
1935/*-----------------------------------------------------------------*/
1936#define DR_MISALIGNMENT_UNKNOWN (-1)
1937#define DR_MISALIGNMENT_UNINITIALIZED (-2)
1938
1939inline void
1941{
1942 dr_info->misalignment = val;
1943}
1944
1945extern int dr_misalignment (dr_vec_info *dr_info, tree vectype,
1946 poly_int64 offset = 0);
1947
1948#define SET_DR_MISALIGNMENT(DR, VAL) set_dr_misalignment (DR, VAL)
1949
1950/* Only defined once DR_MISALIGNMENT is defined. */
1951inline const poly_uint64
1953{
1956 return dr_info->target_alignment;
1957}
1958#define DR_TARGET_ALIGNMENT(DR) dr_target_alignment (DR)
1959
1960inline void
1962{
1963 dr_info->target_alignment = val;
1964}
1965#define SET_DR_TARGET_ALIGNMENT(DR, VAL) set_dr_target_alignment (DR, VAL)
1966
1967/* Return true if data access DR_INFO is aligned to the targets
1968 preferred alignment for VECTYPE (which may be less than a full vector). */
1969
1970inline bool
1972{
1973 return (dr_misalignment (dr_info, vectype) == 0);
1974}
1975
1976/* Return TRUE if the (mis-)alignment of the data access is known with
1977 respect to the targets preferred alignment for VECTYPE, and FALSE
1978 otherwise. */
1979
1980inline bool
1985
1986/* Return the minimum alignment in bytes that the vectorized version
1987 of DR_INFO is guaranteed to have. */
1988
1989inline unsigned int
1991{
1992 int misalignment = dr_misalignment (dr_info, vectype);
1993 if (misalignment == DR_MISALIGNMENT_UNKNOWN)
1995 else if (misalignment == 0)
1997 return misalignment & -misalignment;
1998}
1999
2000/* Return the behavior of DR_INFO with respect to the vectorization context
2001 (which for outer loop vectorization might not be the behavior recorded
2002 in DR_INFO itself). */
2003
2006{
2007 stmt_vec_info stmt_info = dr_info->stmt;
2008 loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo);
2009 if (loop_vinfo == NULL
2010 || !nested_in_vect_loop_p (LOOP_VINFO_LOOP (loop_vinfo), stmt_info))
2011 return &DR_INNERMOST (dr_info->dr);
2012 else
2013 return &STMT_VINFO_DR_WRT_VEC_LOOP (stmt_info);
2014}
2015
2016/* Return the offset calculated by adding the offset of this DR_INFO to the
2017 corresponding data_reference's offset. If CHECK_OUTER then use
2018 vect_dr_behavior to select the appropriate data_reference to use. */
2019
2020inline tree
2022 dr_vec_info *dr_info, bool check_outer = false)
2023{
2025 if (check_outer)
2026 base = vect_dr_behavior (vinfo, dr_info);
2027 else
2028 base = &dr_info->dr->innermost;
2029
2030 tree offset = base->offset;
2031
2032 if (!dr_info->offset)
2033 return offset;
2034
2036 return fold_build2 (PLUS_EXPR, TREE_TYPE (dr_info->offset), offset,
2037 dr_info->offset);
2038}
2039
2040
2041/* Return the vect cost model for LOOP. */
2042inline enum vect_cost_model
2051
2052/* Return true if the vect cost model is unlimited. */
2053inline bool
2058
2059/* Return true if the loop described by LOOP_VINFO is fully-masked and
2060 if the first iteration should use a partial mask in order to achieve
2061 alignment. */
2062
2063inline bool
2065{
2066 return (LOOP_VINFO_FULLY_MASKED_P (loop_vinfo)
2067 && LOOP_VINFO_PEELING_FOR_ALIGNMENT (loop_vinfo));
2068}
2069
2070/* Return the number of vectors of type VECTYPE that are needed to get
2071 NUNITS elements. NUNITS should be based on the vectorization factor,
2072 so it is always a known multiple of the number of elements in VECTYPE. */
2073
2074inline unsigned int
2076{
2077 return exact_div (nunits, TYPE_VECTOR_SUBPARTS (vectype)).to_constant ();
2078}
2079
2080/* Return the number of copies needed for loop vectorization when
2081 a statement operates on vectors of type VECTYPE. This is the
2082 vectorization factor divided by the number of elements in
2083 VECTYPE and is always known at compile time. */
2084
2085inline unsigned int
2087{
2088 return vect_get_num_vectors (LOOP_VINFO_VECT_FACTOR (loop_vinfo), vectype);
2089}
2090
2091/* Update maximum unit count *MAX_NUNITS so that it accounts for
2092 NUNITS. *MAX_NUNITS can be 1 if we haven't yet recorded anything. */
2093
2094inline void
2096{
2097 /* All unit counts have the form vec_info::vector_size * X for some
2098 rational X, so two unit sizes must have a common multiple.
2099 Everything is a multiple of the initial value of 1. */
2100 *max_nunits = force_common_multiple (*max_nunits, nunits);
2101}
2102
2103/* Update maximum unit count *MAX_NUNITS so that it accounts for
2104 the number of units in vector type VECTYPE. *MAX_NUNITS can be 1
2105 if we haven't yet recorded any vector types. */
2106
2107inline void
2109{
2110 vect_update_max_nunits (max_nunits, TYPE_VECTOR_SUBPARTS (vectype));
2111}
2112
2113/* Return the vectorization factor that should be used for costing
2114 purposes while vectorizing the loop described by LOOP_VINFO.
2115 Pick a reasonable estimate if the vectorization factor isn't
2116 known at compile time. */
2117
2118inline unsigned int
2120{
2121 return estimated_poly_value (LOOP_VINFO_VECT_FACTOR (loop_vinfo));
2122}
2123
2124/* Estimate the number of elements in VEC_TYPE for costing purposes.
2125 Pick a reasonable estimate if the exact number isn't known at
2126 compile time. */
2127
2128inline unsigned int
2130{
2131 return estimated_poly_value (TYPE_VECTOR_SUBPARTS (vec_type));
2132}
2133
2134/* Return the maximum possible vectorization factor for LOOP_VINFO. */
2135
2136inline unsigned HOST_WIDE_INT
2138{
2139 unsigned HOST_WIDE_INT vf;
2140 if (LOOP_VINFO_VECT_FACTOR (loop_vinfo).is_constant (&vf))
2141 return vf;
2143}
2144
2145/* Return the size of the value accessed by unvectorized data reference
2146 DR_INFO. This is only valid once STMT_VINFO_VECTYPE has been calculated
2147 for the associated gimple statement, since that guarantees that DR_INFO
2148 accesses either a scalar or a scalar equivalent. ("Scalar equivalent"
2149 here includes things like V1SI, which can be vectorized in the same way
2150 as a plain SI.) */
2151
2152inline unsigned int
2157
2158/* Return true if LOOP_VINFO requires a runtime check for whether the
2159 vector loop is profitable. */
2160
2161inline bool
2163{
2164 unsigned int th = LOOP_VINFO_COST_MODEL_THRESHOLD (loop_vinfo);
2165 return (!LOOP_VINFO_NITERS_KNOWN_P (loop_vinfo)
2166 && th >= vect_vf_for_cost (loop_vinfo));
2167}
2168
2169/* Source location + hotness information. */
2171
2172/* A macro for calling:
2173 dump_begin_scope (MSG, vect_location);
2174 via an RAII object, thus printing "=== MSG ===\n" to the dumpfile etc,
2175 and then calling
2176 dump_end_scope ();
2177 once the object goes out of scope, thus capturing the nesting of
2178 the scopes.
2179
2180 These scopes affect dump messages within them: dump messages at the
2181 top level implicitly default to MSG_PRIORITY_USER_FACING, whereas those
2182 in a nested scope implicitly default to MSG_PRIORITY_INTERNALS. */
2183
2184#define DUMP_VECT_SCOPE(MSG) \
2185 AUTO_DUMP_SCOPE (MSG, vect_location)
2186
2187/* A sentinel class for ensuring that the "vect_location" global gets
2188 reset at the end of a scope.
2189
2190 The "vect_location" global is used during dumping and contains a
2191 location_t, which could contain references to a tree block via the
2192 ad-hoc data. This data is used for tracking inlining information,
2193 but it's not a GC root; it's simply assumed that such locations never
2194 get accessed if the blocks are optimized away.
2195
2196 Hence we need to ensure that such locations are purged at the end
2197 of any operations using them (e.g. via this class). */
2198
2200{
2201 public:
2203};
2204
2205/*-----------------------------------------------------------------*/
2206/* Function prototypes. */
2207/*-----------------------------------------------------------------*/
2208
2209/* Simple loop peeling and versioning utilities for vectorizer's purposes -
2210 in tree-vect-loop-manip.cc. */
2211extern void vect_set_loop_condition (class loop *, edge, loop_vec_info,
2212 tree, tree, tree, bool);
2213extern bool slpeel_can_duplicate_loop_p (const class loop *, const_edge,
2214 const_edge);
2216 class loop *, edge,
2217 edge, edge *, bool = true,
2220extern class loop *vect_do_peeling (loop_vec_info, tree, tree,
2221 tree *, tree *, tree *, int, bool, bool,
2222 tree *);
2228extern edge vec_init_loop_exit_info (class loop *);
2230
2231/* In tree-vect-stmts.cc. */
2232extern tree get_related_vectype_for_scalar_type (machine_mode, tree,
2233 poly_uint64 = 0);
2234extern tree get_vectype_for_scalar_type (vec_info *, tree, unsigned int = 0);
2236extern tree get_mask_type_for_scalar_type (vec_info *, tree, unsigned int = 0);
2239extern bool vect_chooses_same_modes_p (vec_info *, machine_mode);
2241extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
2242 stmt_vec_info * = NULL, gimple ** = NULL);
2243extern bool vect_is_simple_use (tree, vec_info *, enum vect_def_type *,
2244 tree *, stmt_vec_info * = NULL,
2245 gimple ** = NULL);
2247 unsigned, tree *, slp_tree *,
2248 enum vect_def_type *,
2249 tree *, stmt_vec_info * = NULL);
2255 int*, vec<tree> *);
2257 code_helper *, int *,
2258 vec<tree> *);
2259
2260extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2262 tree, int, enum vect_cost_model_location);
2263extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2265 tree, int, enum vect_cost_model_location);
2266extern unsigned record_stmt_cost (stmt_vector_for_cost *, int,
2267 enum vect_cost_for_stmt,
2269
2270/* Overload of record_stmt_cost with VECTYPE derived from STMT_INFO. */
2271
2272inline unsigned
2274 enum vect_cost_for_stmt kind, stmt_vec_info stmt_info,
2275 int misalign, enum vect_cost_model_location where)
2276{
2277 return record_stmt_cost (body_cost_vec, count, kind, stmt_info,
2278 STMT_VINFO_VECTYPE (stmt_info), misalign, where);
2279}
2280
2287 tree op, vec<tree> *, tree = NULL);
2289 tree, vec<tree> *,
2290 tree = NULL, vec<tree> * = NULL,
2291 tree = NULL, vec<tree> * = NULL,
2292 tree = NULL, vec<tree> * = NULL);
2294 tree, tree, vec<tree> *,
2295 tree = NULL, tree = NULL, vec<tree> * = NULL,
2296 tree = NULL, tree = NULL, vec<tree> * = NULL,
2297 tree = NULL, tree = NULL, vec<tree> * = NULL);
2300extern tree vect_get_slp_vect_def (slp_tree, unsigned);
2307 slp_tree,
2309extern void vect_get_load_cost (vec_info *, stmt_vec_info, int,
2310 dr_alignment_support, int, bool,
2311 unsigned int *, unsigned int *,
2313 stmt_vector_for_cost *, bool);
2314extern void vect_get_store_cost (vec_info *, stmt_vec_info, int,
2316 unsigned int *, stmt_vector_for_cost *);
2317extern bool vect_supportable_shift (vec_info *, enum tree_code, tree);
2320extern void optimize_mask_stores (class loop*);
2322 const char * = nullptr);
2326 tree *, unsigned int = 0);
2328
2329/* In tree-if-conv.cc. */
2330extern bool ref_within_array_bound (gimple *, tree);
2331
2332/* In tree-vect-data-refs.cc. */
2335 (vec_info *, dr_vec_info *, tree, int);
2344extern bool vect_gather_scatter_fn_p (vec_info *, bool, bool, tree, tree,
2345 tree, int, internal_fn *, tree *);
2350 vec<int> *, int);
2354 stmt_vec_info, tree, class loop *, tree,
2356 gimple **, bool,
2357 tree = NULL_TREE);
2360extern void vect_copy_ref_info (tree, tree);
2362extern bool vect_grouped_store_supported (tree, unsigned HOST_WIDE_INT);
2364extern bool vect_grouped_load_supported (tree, bool, unsigned HOST_WIDE_INT);
2367 unsigned int, stmt_vec_info,
2372 class loop **);
2374 int, gimple_stmt_iterator *);
2377extern tree vect_get_new_vect_var (tree, enum vect_var_kind, const char *);
2379 const char * = NULL);
2382 tree);
2383
2384/* In tree-vect-loop.cc. */
2385extern tree neutral_op_for_reduction (tree, code_helper, tree, bool = true);
2388/* Used in tree-vect-loop-manip.cc */
2390/* Used in gimple-loop-interchange.c and tree-parloops.cc. */
2392 enum tree_code);
2394/* Drive for loop analysis stage. */
2398 tree *, bool);
2399extern tree vect_halve_mask_nunits (tree, machine_mode);
2400extern tree vect_double_mask_nunits (tree, machine_mode);
2402 unsigned int, tree, tree);
2405 unsigned int, tree, unsigned int);
2406extern void vect_record_loop_len (loop_vec_info, vec_loop_lens *, unsigned int,
2407 tree, unsigned int);
2409 vec_loop_lens *, unsigned int, tree,
2410 unsigned int, unsigned int);
2414
2415/* Drive for loop transformation stage. */
2416extern class loop *vect_transform_loop (loop_vec_info, gimple *);
2428 const vect_loop_form_info *,
2429 loop_vec_info = nullptr);
2431 slp_tree, slp_instance, int,
2432 bool, stmt_vector_for_cost *);
2437 gimple **, slp_tree,
2441 gimple **, slp_tree);
2443 gimple **,
2446 gimple **, slp_tree);
2451extern bool vect_emulated_vector_p (tree);
2454extern int vect_get_known_peeling_cost (loop_vec_info, int, int *,
2459
2460/* Nonlinear induction. */
2463
2464/* In tree-vect-slp.cc. */
2465extern void vect_slp_init (void);
2466extern void vect_slp_fini (void);
2470 bool, unsigned *,
2471 unsigned * = nullptr, bool = false);
2473extern void vect_schedule_slp (vec_info *, const vec<slp_instance> &);
2474extern opt_result vect_analyze_slp (vec_info *, unsigned);
2477extern void vect_optimize_slp (vec_info *);
2478extern void vect_gather_slp_loads (vec_info *);
2479extern void vect_get_slp_defs (slp_tree, vec<tree> *);
2480extern void vect_get_slp_defs (vec_info *, slp_tree, vec<vec<tree> > *,
2481 unsigned n = -1U);
2482extern bool vect_slp_if_converted_bb (basic_block bb, loop_p orig_loop);
2483extern bool vect_slp_function (function *);
2487extern bool can_duplicate_and_interleave_p (vec_info *, unsigned int, tree,
2488 unsigned int * = NULL,
2489 tree * = NULL, tree * = NULL);
2491 const vec<tree> &, unsigned int, vec<tree> &);
2494extern void vect_free_slp_tree (slp_tree);
2495extern bool compatible_calls_p (gcall *, gcall *);
2496extern int vect_slp_child_index_for_operand (const gimple *, int op, bool);
2497
2498/* In tree-vect-patterns.cc. */
2499extern void
2501extern bool vect_get_range_info (tree, wide_int*, wide_int*);
2502
2503/* Pattern recognition functions.
2504 Additional pattern recognition functions can (and will) be added
2505 in the future. */
2507
2508/* In tree-vectorizer.cc. */
2509unsigned vectorize_loops (void);
2511gimple *vect_loop_vectorized_call (class loop *, gcond **cond = NULL);
2513
2514/* SLP Pattern matcher types, tree-vect-slp-patterns.cc. */
2515
2516/* Forward declaration of possible two operands operation that can be matched
2517 by the complex numbers pattern matchers. */
2519
2520/* All possible load permute values that could result from the partial data-flow
2521 analysis. */
2531
2532/* Cache from nodes to the load permutation they represent. */
2535
2536/* Cache from nodes pair to being compatible or not. */
2540
2541
2542/* Vector pattern matcher base class. All SLP pattern matchers must inherit
2543 from this type. */
2544
2546{
2547 protected:
2548 /* The number of arguments that the IFN requires. */
2549 unsigned m_num_args;
2550
2551 /* The internal function that will be used when a pattern is created. */
2553
2554 /* The current node being inspected. */
2556
2557 /* The list of operands to be the children for the node produced when the
2558 internal function is created. */
2560
2561 /* Default constructor where NODE is the root of the tree to inspect. */
2563 {
2564 this->m_ifn = ifn;
2565 this->m_node = node;
2566 this->m_ops.create (0);
2567 if (m_ops)
2568 this->m_ops.safe_splice (*m_ops);
2569 }
2570
2571 public:
2572
2573 /* Create a new instance of the pattern matcher class of the given type. */
2576
2577 /* Build the pattern from the data collected so far. */
2578 virtual void build (vec_info *) = 0;
2579
2580 /* Default destructor. */
2581 virtual ~vect_pattern ()
2582 {
2583 this->m_ops.release ();
2584 }
2585};
2586
2587/* Function pointer to create a new pattern matcher from a generic type. */
2588typedef vect_pattern* (*vect_pattern_decl_t) (slp_tree_to_load_perm_map_t *,
2590 slp_tree *);
2591
2592/* List of supported pattern matchers. */
2594
2595/* Number of supported pattern matchers. */
2596extern size_t num__slp_patterns;
2597
2598/* ----------------------------------------------------------------------
2599 Target support routines
2600 -----------------------------------------------------------------------
2601 The following routines are provided to simplify costing decisions in
2602 target code. Please add more as needed. */
2603
2604/* Return true if an operaton of kind KIND for STMT_INFO represents
2605 the extraction of an element from a vector in preparation for
2606 storing the element to memory. */
2607inline bool
2609{
2610 return (kind == vec_to_scalar
2611 && STMT_VINFO_DATA_REF (stmt_info)
2612 && DR_IS_WRITE (STMT_VINFO_DATA_REF (stmt_info)));
2613}
2614
2615/* Return true if STMT_INFO represents part of a reduction. */
2616inline bool
2618{
2619 return STMT_VINFO_REDUC_IDX (stmt_info) >= 0;
2620}
2621
2622/* If STMT_INFO describes a reduction, return the vect_reduction_type
2623 of the reduction it describes, otherwise return -1. */
2624inline int
2626{
2627 if (loop_vec_info loop_vinfo = dyn_cast<loop_vec_info> (vinfo))
2628 if (STMT_VINFO_REDUC_DEF (stmt_info))
2629 {
2630 stmt_vec_info reduc_info = info_for_reduction (loop_vinfo, stmt_info);
2631 return int (STMT_VINFO_REDUC_TYPE (reduc_info));
2632 }
2633 return -1;
2634}
2635
2636/* If STMT_INFO is a COND_EXPR that includes an embedded comparison, return the
2637 scalar type of the values being compared. Return null otherwise. */
2638inline tree
2640{
2641 if (auto *assign = dyn_cast<gassign *> (stmt_info->stmt))
2643 {
2645 if (COMPARISON_CLASS_P (cond))
2646 return TREE_TYPE (TREE_OPERAND (cond, 0));
2647 }
2648 return NULL_TREE;
2649}
2650
2651/* If STMT_INFO is a comparison or contains an embedded comparison, return the
2652 scalar type of the values being compared. Return null otherwise. */
2653inline tree
2655{
2656 if (auto *assign = dyn_cast<gassign *> (stmt_info->stmt))
2659 return vect_embedded_comparison_type (stmt_info);
2660}
2661
2662/* Return true if STMT_INFO extends the result of a load. */
2663inline bool
2665{
2666 /* Although this is quite large for an inline function, this part
2667 at least should be inline. */
2668 gassign *assign = dyn_cast <gassign *> (stmt_info->stmt);
2670 return false;
2671
2672 tree rhs = gimple_assign_rhs1 (stmt_info->stmt);
2674 tree rhs_type = TREE_TYPE (rhs);
2678 return false;
2679
2680 stmt_vec_info def_stmt_info = vinfo->lookup_def (rhs);
2681 return (def_stmt_info
2684}
2685
2686/* Return true if STMT_INFO is an integer truncation. */
2687inline bool
2700
2701/* Build a GIMPLE_ASSIGN or GIMPLE_CALL with the tree_code,
2702 or internal_fn contained in ch, respectively. */
2704#endif /* GCC_TREE_VECTORIZER_H */
edge loop_preheader_edge(const class loop *loop)
Definition cfgloop.cc:1760
class loop * loop_p
Definition cfgloop.h:98
Definition tree-vectorizer.h:1092
vec< slp_root > roots
Definition tree-vectorizer.h:1103
_bb_vec_info(vec< basic_block > bbs, vec_info_shared *)
Definition tree-vect-slp.cc:6055
vec< basic_block > bbs
Definition tree-vectorizer.h:1101
~_bb_vec_info()
Definition tree-vect-slp.cc:6086
Definition tree-vectorizer.h:674
bool peeling_for_gaps
Definition tree-vectorizer.h:886
bool can_use_partial_vectors_p
Definition tree-vectorizer.h:856
signed char partial_load_store_bias
Definition tree-vectorizer.h:881
class vector_costs * vector_costs
Definition tree-vectorizer.h:695
class dr_vec_info * unaligned_dr
Definition tree-vectorizer.h:786
_loop_vec_info(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:1028
edge skip_main_loop_edge
Definition tree-vectorizer.h:721
auto_vec< stmt_vec_info > reduction_chains
Definition tree-vectorizer.h:829
unsigned int th
Definition tree-vectorizer.h:703
auto_vec< gimple * > early_break_vuses
Definition tree-vectorizer.h:963
bool using_partial_vectors_p
Definition tree-vectorizer.h:860
bool no_data_dependencies
Definition tree-vectorizer.h:919
class loop * scalar_loop
Definition tree-vectorizer.h:929
vec< _loop_vec_info * > epilogue_vinfos
Definition tree-vectorizer.h:937
scalar_cond_masked_set_type scalar_cond_masked_set
Definition tree-vectorizer.h:754
auto_vec< ddr_p > may_alias_ddrs
Definition tree-vectorizer.h:803
tree num_itersm1
Definition tree-vectorizer.h:686
unsigned int inner_loop_cost_factor
Definition tree-vectorizer.h:847
bool vectorizable
Definition tree-vectorizer.h:850
vect_partial_vector_style partial_vector_style
Definition tree-vectorizer.h:783
poly_uint64 slp_unrolling_factor
Definition tree-vectorizer.h:843
edge vec_epilogue_loop_iv_exit
Definition tree-vectorizer.h:945
basic_block * bbs
Definition tree-vectorizer.h:683
tree num_iters
Definition tree-vectorizer.h:688
hash_map< tree, tree > * scan_map
Definition tree-vectorizer.h:839
int peeling_for_alignment
Definition tree-vectorizer.h:796
vec_cond_masked_set_type vec_cond_masked_set
Definition tree-vectorizer.h:757
bool epil_using_partial_vectors_p
Definition tree-vectorizer.h:876
bool early_breaks
Definition tree-vectorizer.h:894
poly_uint64 versioning_threshold
Definition tree-vectorizer.h:709
auto_vec< vec_lower_bound > lower_bounds
Definition tree-vectorizer.h:818
profile_probability scalar_loop_scaling
Definition tree-vectorizer.h:925
auto_vec< gimple * > early_break_stores
Definition tree-vectorizer.h:955
edge main_loop_edge
Definition tree-vectorizer.h:720
edge skip_this_loop_edge
Definition tree-vectorizer.h:725
auto_vec< stmt_info_for_cost > scalar_cost_vec
Definition tree-vectorizer.h:832
gcond * loop_iv_cond
Definition tree-vectorizer.h:900
basic_block early_break_dest_bb
Definition tree-vectorizer.h:959
bool using_select_vl_p
Definition tree-vectorizer.h:872
tree rgroup_compare_type
Definition tree-vectorizer.h:768
poly_uint64 vectorization_factor
Definition tree-vectorizer.h:712
bool peeling_for_niter
Definition tree-vectorizer.h:890
auto_vec< dr_with_seg_len_pair_t > comp_alias_ddrs
Definition tree-vectorizer.h:807
edge vec_loop_iv_exit
Definition tree-vectorizer.h:941
tree num_iters_unchanged
Definition tree-vectorizer.h:690
vec_loop_lens lens
Definition tree-vectorizer.h:751
unsigned HOST_WIDE_INT max_vectorization_factor
Definition tree-vectorizer.h:743
unsigned int suggested_unroll_factor
Definition tree-vectorizer.h:739
hash_map< tree_operand_hash, tree > * ivexpr_map
Definition tree-vectorizer.h:835
auto_vec< stmt_vec_info > may_misalign_stmts
Definition tree-vectorizer.h:822
tree rgroup_iv_type
Definition tree-vectorizer.h:779
vec_loop_masks masks
Definition tree-vectorizer.h:747
tree num_iters_assumptions
Definition tree-vectorizer.h:692
tree simd_if_cond
Definition tree-vectorizer.h:775
auto_vec< tree > check_nonzero
Definition tree-vectorizer.h:815
tree mask_skip_niters
Definition tree-vectorizer.h:763
bool has_mask_store
Definition tree-vectorizer.h:922
auto_vec< stmt_vec_info > reductions
Definition tree-vectorizer.h:825
edge scalar_loop_iv_exit
Definition tree-vectorizer.h:949
class loop * loop
Definition tree-vectorizer.h:680
~_loop_vec_info()
Definition tree-vect-loop.cc:1146
bool using_decrementing_iv_p
Definition tree-vectorizer.h:867
auto_vec< gcond * > conds
Definition tree-vectorizer.h:897
_loop_vec_info * orig_loop_info
Definition tree-vectorizer.h:933
hash_map< tree, vect_reusable_accumulator > reusable_accumulators
Definition tree-vectorizer.h:732
int ptr_mask
Definition tree-vectorizer.h:799
class vector_costs * scalar_costs
Definition tree-vectorizer.h:698
auto_vec< vec_object_pair > check_unequal_addrs
Definition tree-vectorizer.h:810
Definition tree-vectorizer.h:257
poly_uint64 unrolling_factor
Definition tree-vectorizer.h:271
slp_instance_kind kind
Definition tree-vectorizer.h:287
vec< stmt_vec_info > root_stmts
Definition tree-vectorizer.h:264
slp_tree root
Definition tree-vectorizer.h:260
dump_user_location_t location() const
Definition tree-vect-slp.cc:197
vec< slp_tree > loads
Definition tree-vectorizer.h:274
vec< _slp_instance * > subgraph_entries
Definition tree-vectorizer.h:284
vec< tree > remain_defs
Definition tree-vectorizer.h:268
slp_tree reduc_phis
Definition tree-vectorizer.h:277
stmt_vector_for_cost cost_vec
Definition tree-vectorizer.h:280
Definition tree-vectorizer.h:1258
unsigned int min_input_precision
Definition tree-vectorizer.h:1432
code_helper reduc_code
Definition tree-vectorizer.h:1383
unsigned int store_count
Definition tree-vectorizer.h:1345
tree vectype
Definition tree-vectorizer.h:1284
vec< tree > simd_clone_info
Definition tree-vectorizer.h:1328
tree induc_cond_initial_val
Definition tree-vectorizer.h:1373
bool in_pattern_p
Definition tree-vectorizer.h:1268
stmt_vec_info first_element
Definition tree-vectorizer.h:1338
unsigned int mask_precision
Definition tree-vectorizer.h:1455
vec< tree > reduc_initial_values
Definition tree-vectorizer.h:1405
int reduc_idx
Definition tree-vectorizer.h:1389
gimple * stmt
Definition tree-vectorizer.h:1281
tree loop_phi_evolution_base_unchanged
Definition tree-vectorizer.h:1305
enum vect_induction_op_type loop_phi_evolution_type
Definition tree-vectorizer.h:1307
innermost_loop_behavior dr_wrt_vec_loop
Definition tree-vectorizer.h:1297
vec< gimple * > vec_stmts
Definition tree-vectorizer.h:1287
bool slp_vect_only_p
Definition tree-vectorizer.h:1458
bool live
Definition tree-vectorizer.h:1265
unsigned int gap
Definition tree-vectorizer.h:1348
unsigned int min_output_precision
Definition tree-vectorizer.h:1427
vect_memory_access_type memory_access_type
Definition tree-vectorizer.h:1370
internal_fn reduc_fn
Definition tree-vectorizer.h:1385
enum stmt_vec_info_type type
Definition tree-vectorizer.h:1261
unsigned int min_neg_dist
Definition tree-vectorizer.h:1352
vec< tree > reduc_scalar_results
Definition tree-vectorizer.h:1412
tree reduc_vectype_in
Definition tree-vectorizer.h:1397
bool vectorizable
Definition tree-vectorizer.h:1278
bool pattern_stmt_p
Definition tree-vectorizer.h:1274
bool strided_p
Definition tree-vectorizer.h:1363
enum vect_relevant relevant
Definition tree-vectorizer.h:1357
unsigned int size
Definition tree-vectorizer.h:1342
signop operation_sign
Definition tree-vectorizer.h:1438
bool force_single_cycle
Definition tree-vectorizer.h:1420
bool is_reduc_info
Definition tree-vectorizer.h:1423
gimple_seq pattern_def_seq
Definition tree-vectorizer.h:1323
stmt_vec_info related_stmt
Definition tree-vectorizer.h:1318
unsigned simd_lane_access_p
Definition tree-vectorizer.h:1366
bool slp_vect_pattern_only_p
Definition tree-vectorizer.h:1462
tree reduc_vectype
Definition tree-vectorizer.h:1400
stmt_vec_info reduc_def
Definition tree-vectorizer.h:1394
bool gather_scatter_p
Definition tree-vectorizer.h:1360
enum vect_def_type def_type
Definition tree-vectorizer.h:1331
stmt_vec_info next_element
Definition tree-vectorizer.h:1340
unsigned int operation_precision
Definition tree-vectorizer.h:1437
enum vect_reduction_type reduc_type
Definition tree-vectorizer.h:1380
tree reduc_epilogue_adjustment
Definition tree-vectorizer.h:1376
enum slp_vect_type slp_type
Definition tree-vectorizer.h:1334
dr_vec_info dr_aux
Definition tree-vectorizer.h:1293
tree loop_phi_evolution_part
Definition tree-vectorizer.h:1306
vect_reusable_accumulator * reused_accumulator
Definition tree-vectorizer.h:1417
Definition tree-vectorizer.h:2200
~auto_purge_vect_location()
Definition tree-vectorizer.cc:94
Definition vec.h:1656
Definition tree.h:81
Definition tree-vectorizer.h:1233
unsigned group
Definition tree-vectorizer.h:1241
tree base_decl
Definition tree-vectorizer.h:1249
bool base_misaligned
Definition tree-vectorizer.h:1248
tree offset
Definition tree-vectorizer.h:1253
int misalignment
Definition tree-vectorizer.h:1243
poly_uint64 target_alignment
Definition tree-vectorizer.h:1246
data_reference * dr
Definition tree-vectorizer.h:1236
stmt_vec_info stmt
Definition tree-vectorizer.h:1238
Definition dumpfile.h:340
Definition genmatch.cc:845
Definition hash-map.h:40
Definition hash-set.h:37
bool is_empty() const
Definition hash-set.h:107
Definition inchash.h:38
Definition cfgloop.h:120
class loop * inner
Definition cfgloop.h:148
unsigned force_vectorize
Definition cfgloop.h:227
void * aux
Definition cfgloop.h:154
Definition opt-problem.h:230
Definition opt-problem.h:179
Definition profile-count.h:147
Definition tree-vectorizer.h:424
vec< data_reference_p > datarefs
Definition tree-vectorizer.h:436
vec< data_reference > datarefs_copy
Definition tree-vectorizer.h:437
auto_vec< loop_p > loop_nest
Definition tree-vectorizer.h:440
void check_datarefs()
Definition tree-vectorizer.cc:505
unsigned n_stmts
Definition tree-vectorizer.h:433
void save_datarefs()
Definition tree-vectorizer.cc:495
~vec_info_shared()
Definition tree-vectorizer.cc:487
vec< ddr_p > ddrs
Definition tree-vectorizer.h:444
vec_info_shared()
Definition tree-vectorizer.cc:479
Definition tree-vectorizer.h:448
stmt_vec_info lookup_stmt(gimple *)
Definition tree-vectorizer.cc:545
auto_vec< stmt_vec_info > grouped_stores
Definition tree-vectorizer.h:492
~vec_info()
Definition tree-vectorizer.cc:471
auto_vec< slp_instance > slp_instances
Definition tree-vectorizer.h:484
void replace_stmt(gimple_stmt_iterator *, stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:630
void remove_stmt(stmt_vec_info)
Definition tree-vectorizer.cc:614
void insert_on_entry(stmt_vec_info, gimple *)
Definition tree-vectorizer.cc:673
void set_vinfo_for_stmt(gimple *, stmt_vec_info, bool=true)
Definition tree-vectorizer.cc:720
vec_base_alignments base_alignments
Definition tree-vectorizer.h:488
machine_mode vector_mode
Definition tree-vectorizer.h:500
stmt_vec_info new_stmt_vec_info(gimple *stmt)
Definition tree-vectorizer.cc:684
class dr_vec_info * lookup_dr(data_reference *)
Definition tree-vectorizer.cc:587
vec< stmt_vec_info > stmt_vec_infos
Definition tree-vectorizer.h:475
void move_dr(stmt_vec_info, stmt_vec_info)
Definition tree-vectorizer.cc:599
bool any_known_not_updated_vssa
Definition tree-vectorizer.h:481
vec_kind
Definition tree-vectorizer.h:451
@ loop
Definition tree-vectorizer.h:451
@ bb
Definition tree-vectorizer.h:451
void free_stmt_vec_infos()
Definition tree-vectorizer.cc:741
vec_info_shared * shared
Definition tree-vectorizer.h:472
stmt_vec_info add_stmt(gimple *)
Definition tree-vectorizer.cc:520
vec_kind kind
Definition tree-vectorizer.h:469
mode_set used_vector_modes
Definition tree-vectorizer.h:495
hash_set< int_hash< machine_mode, E_VOIDmode, E_BLKmode > > mode_set
Definition tree-vectorizer.h:450
void insert_seq_on_entry(stmt_vec_info, gimple_seq)
Definition tree-vectorizer.cc:646
stmt_vec_info lookup_def(tree)
Definition tree-vectorizer.cc:562
void free_stmt_vec_info(stmt_vec_info)
Definition tree-vectorizer.cc:752
stmt_vec_info lookup_single_use(tree)
Definition tree-vectorizer.cc:575
vec_info(vec_kind, vec_info_shared *)
Definition tree-vectorizer.cc:463
stmt_vec_info add_pattern_stmt(gimple *, stmt_vec_info)
Definition tree-vectorizer.cc:532
bool stmt_vec_info_ro
Definition tree-vectorizer.h:477
Definition tree-vectorizer.h:411
vec_lower_bound()
Definition tree-vectorizer.h:413
poly_uint64 min_value
Definition tree-vectorizer.h:419
bool unsigned_p
Definition tree-vectorizer.h:418
vec_lower_bound(tree e, bool u, poly_uint64 m)
Definition tree-vectorizer.h:414
tree expr
Definition tree-vectorizer.h:417
Definition vec-perm-indices.h:51
Definition tree-vectorizer.h:2546
static vect_pattern * recognize(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
internal_fn m_ifn
Definition tree-vectorizer.h:2552
vec< slp_tree > m_ops
Definition tree-vectorizer.h:2559
virtual ~vect_pattern()
Definition tree-vectorizer.h:2581
virtual void build(vec_info *)=0
vect_pattern(slp_tree *node, vec< slp_tree > *m_ops, internal_fn ifn)
Definition tree-vectorizer.h:2562
slp_tree * m_node
Definition tree-vectorizer.h:2555
unsigned m_num_args
Definition tree-vectorizer.h:2549
Definition tree-vectorizer.h:1578
unsigned int adjust_cost_for_freq(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1839
virtual ~vector_costs()
Definition tree-vectorizer.h:1581
bool m_costing_for_scalar
Definition tree-vectorizer.h:1649
vector_costs(vec_info *, bool)
Definition tree-vectorizer.h:1665
vec_info * m_vinfo
Definition tree-vectorizer.h:1645
virtual unsigned int add_stmt_cost(int count, vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, vect_cost_model_location where)
Definition tree-vectorizer.cc:1801
virtual bool better_main_loop_than_p(const vector_costs *other) const
Definition tree-vectorizer.cc:1859
unsigned int m_costs[3]
Definition tree-vectorizer.h:1652
unsigned int outside_cost() const
Definition tree-vectorizer.h:1704
unsigned int record_stmt_cost(stmt_vec_info, vect_cost_model_location, unsigned int)
Definition tree-vectorizer.cc:1825
virtual bool better_epilogue_loop_than_p(const vector_costs *other, loop_vec_info main_loop) const
Definition tree-vectorizer.cc:1878
unsigned int epilogue_cost() const
Definition tree-vectorizer.h:1695
unsigned int m_suggested_unroll_factor
Definition tree-vectorizer.h:1655
unsigned int suggested_unroll_factor() const
Definition tree-vectorizer.h:1721
bool m_finished
Definition tree-vectorizer.h:1658
int compare_outside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:2054
unsigned int body_cost() const
Definition tree-vectorizer.h:1686
virtual void finish_cost(const vector_costs *scalar_costs)
Definition tree-vectorizer.cc:1814
unsigned int prologue_cost() const
Definition tree-vectorizer.h:1677
unsigned int total_cost() const
Definition tree-vectorizer.h:1713
int compare_inside_loop_cost(const vector_costs *) const
Definition tree-vectorizer.cc:1961
class edge_def * edge
Definition coretypes.h:342
const class edge_def * const_edge
Definition coretypes.h:343
const union tree_node * const_tree
Definition coretypes.h:98
union tree_node * tree
Definition coretypes.h:97
static unsigned int count[debug_counter_number_of_counters]
Definition dbgcnt.cc:50
dump_flags_t dump_flags
Definition dumpfile.cc:64
FILE * dump_file
Definition dumpfile.cc:62
@ TDF_DETAILS
Definition dumpfile.h:92
vect_cost_model
Definition flag-types.h:276
@ VECT_COST_MODEL_UNLIMITED
Definition flag-types.h:280
@ VECT_COST_MODEL_DEFAULT
Definition flag-types.h:281
bool operand_equal_p(const_tree arg0, const_tree arg1, unsigned int flags)
Definition fold-const.cc:4223
#define fold_convert(T1, T2)
Definition fold-const.h:92
#define fold_build2(c, t1, t2, t3)
Definition fold-const.h:72
internal_fn
Definition genmatch.cc:360
tree_code
Definition genmatch.cc:347
T * ggc_alloc(ALONE_CXX_MEM_STAT_INFO)
Definition ggc.h:184
unsigned gimple_uid(const gimple *g)
Definition gimple.h:2083
enum tree_code gimple_assign_rhs_code(const gassign *gs)
Definition gimple.h:2849
tree gimple_assign_rhs1(const gassign *gs)
Definition gimple.h:2659
basic_block gimple_bb(const gimple *g)
Definition gimple.h:1859
tree gimple_assign_lhs(const gassign *gs)
Definition gimple.h:2608
#define init_cost
Definition ira-costs.cc:63
void add_expr(const_tree t, inchash::hash &hstate, unsigned int flags)
Definition fold-const.cc:4237
i
Definition poly-int.h:772
rtx offset
Definition postreload.cc:691
signop
Definition signop.h:28
Definition tree-vectorizer.h:174
stmt_vec_info representative
Definition tree-vectorizer.h:190
unsigned int lanes
Definition tree-vectorizer.h:222
load_permutation_t load_permutation
Definition tree-vectorizer.h:194
slp_tree prev_node
Definition tree-vectorizer.h:241
enum tree_code code
Definition tree-vectorizer.h:224
unsigned int vec_stmts_size
Definition tree-vectorizer.h:212
vec< tree > vec_defs
Definition tree-vectorizer.h:207
~_slp_tree()
Definition tree-vect-slp.cc:133
unsigned int refcnt
Definition tree-vectorizer.h:215
slp_tree next_node
Definition tree-vectorizer.h:240
int vertex
Definition tree-vectorizer.h:226
enum vect_def_type def_type
Definition tree-vectorizer.h:220
bool * failed
Definition tree-vectorizer.h:231
vec< tree > simd_clone_info
Definition tree-vectorizer.h:203
vec< tree > ops
Definition tree-vectorizer.h:187
void push_vec_def(gimple *def)
Definition tree-vect-slp.cc:155
poly_uint64 max_nunits
Definition tree-vectorizer.h:218
lane_permutation_t lane_permutation
Definition tree-vectorizer.h:198
_slp_tree()
Definition tree-vect-slp.cc:106
tree vectype
Definition tree-vectorizer.h:205
void push_vec_def(tree def)
Definition tree-vectorizer.h:179
vec< slp_tree > children
Definition tree-vectorizer.h:182
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:185
Definition basic-block.h:117
class loop * loop_father
Definition basic-block.h:126
Definition ira-costs.cc:54
Definition tree-data-ref.h:151
struct innermost_loop_behavior innermost
Definition tree-data-ref.h:173
Definition loop-invariant.cc:88
static void mark_empty(value_type &v)
Definition tree-vectorizer.h:375
scalar_cond_masked_key value_type
Definition tree-vectorizer.h:348
static bool equal(value_type existing, value_type candidate)
Definition tree-vectorizer.h:363
static void remove(value_type &)
Definition tree-vectorizer.h:394
static hashval_t hash(value_type v)
Definition tree-vectorizer.h:351
static void mark_deleted(value_type &)
Definition tree-vectorizer.h:387
static bool is_deleted(const value_type &)
Definition tree-vectorizer.h:389
static bool is_empty(value_type v)
Definition tree-vectorizer.h:382
scalar_cond_masked_key compare_type
Definition tree-vectorizer.h:347
Definition hash-traits.h:466
Definition graphite.h:338
data_reference_p dr
Definition graphite.h:343
Definition function.h:249
Definition gimple.h:903
Definition tree-vectorizer.h:1466
tree decl
Definition tree-vectorizer.h:1473
tree element_type
Definition tree-vectorizer.h:1492
tree base
Definition tree-vectorizer.h:1476
tree offset_vectype
Definition tree-vectorizer.h:1489
int scale
Definition tree-vectorizer.h:1483
enum vect_def_type offset_dt
Definition tree-vectorizer.h:1486
tree offset
Definition tree-vectorizer.h:1479
internal_fn ifn
Definition tree-vectorizer.h:1469
tree memory_type
Definition tree-vectorizer.h:1495
Definition gimple.h:352
Definition gimple.h:857
Definition gimple-iterator.h:26
Definition gimple.h:225
Definition gimple.h:461
Definition tree-data-ref.h:50
tree offset
Definition tree-data-ref.h:52
Definition hash-traits.h:122
static bool test(U *p)
Definition hash-traits.h:345
Definition tree-vectorizer.h:613
tree compare_type
Definition tree-vectorizer.h:633
unsigned int max_nscalars_per_iter
Definition tree-vectorizer.h:617
unsigned int factor
Definition tree-vectorizer.h:623
tree type
Definition tree-vectorizer.h:629
tree bias_adjusted_ctrl
Definition tree-vectorizer.h:640
vec< tree > controls
Definition tree-vectorizer.h:636
Definition tree-vectorizer.h:328
unsigned ncopies
Definition tree-vectorizer.h:337
tree_code code
Definition tree-vectorizer.h:339
scalar_cond_masked_key(tree t, unsigned ncopies_)
Definition tree-vectorizer.h:329
void get_cond_ops_from_tree(tree)
Definition tree-vectorizer.cc:1751
tree op0
Definition tree-vectorizer.h:340
tree op1
Definition tree-vectorizer.h:341
bool inverted_p
Definition tree-vectorizer.h:338
Definition tree-vectorizer.h:1081
vec< tree > remain
Definition tree-vectorizer.h:1088
slp_instance_kind kind
Definition tree-vectorizer.h:1085
vec< stmt_vec_info > roots
Definition tree-vectorizer.h:1087
slp_root(slp_instance_kind kind_, vec< stmt_vec_info > stmts_, vec< stmt_vec_info > roots_, vec< tree > remain_=vNULL)
Definition tree-vectorizer.h:1082
vec< stmt_vec_info > stmts
Definition tree-vectorizer.h:1086
Definition tree-vectorizer.h:111
tree vectype
Definition tree-vectorizer.h:117
int misalign
Definition tree-vectorizer.h:118
slp_tree node
Definition tree-vectorizer.h:116
enum vect_cost_model_location where
Definition tree-vectorizer.h:114
enum vect_cost_for_stmt kind
Definition tree-vectorizer.h:113
stmt_vec_info stmt_info
Definition tree-vectorizer.h:115
int count
Definition tree-vectorizer.h:112
Definition tree-hash-traits.h:25
Definition hash-traits.h:75
Definition tree-vectorizer.h:644
hash_set< pair_hash< nofree_ptr_hash< tree_node >, int_hash< unsigned, 0 > > > mask_set
Definition tree-vectorizer.h:649
auto_vec< rgroup_controls > rgc_vec
Definition tree-vectorizer.h:652
bool is_empty() const
Definition tree-vectorizer.h:645
Definition vec.h:450
Definition tree-vectorizer.h:2418
gcond * inner_loop_cond
Definition tree-vectorizer.h:2423
auto_vec< gcond * > conds
Definition tree-vectorizer.h:2422
tree number_of_iterationsm1
Definition tree-vectorizer.h:2420
tree assumptions
Definition tree-vectorizer.h:2421
edge loop_exit
Definition tree-vectorizer.h:2424
tree number_of_iterations
Definition tree-vectorizer.h:2419
Definition tree-vectorizer.h:661
stmt_vec_info reduc_info
Definition tree-vectorizer.h:668
tree reduc_input
Definition tree-vectorizer.h:664
Definition tree-vectorizer.h:150
static void mark_deleted(value_type &s)
Definition tree-vectorizer.h:156
static bool is_empty(const value_type &s)
Definition tree-vectorizer.h:159
static bool is_deleted(const value_type &s)
Definition tree-vectorizer.h:158
static const bool empty_zero_p
Definition tree-vectorizer.h:154
static void mark_empty(value_type &s)
Definition tree-vectorizer.h:157
static hashval_t hash(const value_type &)
Definition tree-vect-slp.cc:6232
vect_scalar_ops_slice value_type
Definition tree-vectorizer.h:151
static bool equal(const value_type &, const compare_type &)
Definition tree-vect-slp.cc:6241
vect_scalar_ops_slice compare_type
Definition tree-vectorizer.h:152
Definition tree-vectorizer.h:132
unsigned int length
Definition tree-vectorizer.h:138
unsigned int start
Definition tree-vectorizer.h:137
bool all_same_p() const
Definition tree-vect-slp.cc:6223
tree op(unsigned int i) const
Definition tree-vectorizer.h:143
vec< tree > * ops
Definition tree-vectorizer.h:136
#define NULL
Definition system.h:50
#define gcc_assert(EXPR)
Definition system.h:821
#define false
Definition system.h:895
#define gcc_checking_assert(EXPR)
Definition system.h:828
HOST_WIDE_INT estimated_poly_value(poly_int64 x, poly_value_estimate_kind kind=POLY_VALUE_LIKELY)
Definition target.h:306
struct gcc_target targetm
vect_cost_model_location
Definition target.h:209
@ vect_body
Definition target.h:211
@ vect_prologue
Definition target.h:210
@ vect_epilogue
Definition target.h:212
vect_cost_for_stmt
Definition target.h:187
@ scalar_stmt
Definition target.h:188
@ cond_branch_taken
Definition target.h:201
@ vec_to_scalar
Definition target.h:198
@ cond_branch_not_taken
Definition target.h:200
@ tcc_comparison
Definition tree-core.h:233
#define DR_IS_WRITE(DR)
Definition tree-data-ref.h:192
#define DR_IS_READ(DR)
Definition tree-data-ref.h:191
#define DR_REF(DR)
Definition tree-data-ref.h:185
#define DR_INNERMOST(DR)
Definition tree-data-ref.h:203
static tree candidate(unsigned uid)
Definition tree-sra.cc:325
#define PHI_ARG_DEF_FROM_EDGE(PHI, E)
Definition tree-ssa-operands.h:88
_complex_operation
Definition tree-vect-slp-patterns.cc:119
stmt_vec_info vect_orig_stmt(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1789
#define LOOP_VINFO_COST_MODEL_THRESHOLD(L)
Definition tree-vectorizer.h:979
bool vect_can_vectorize_without_simd_p(tree_code)
Definition tree-vect-loop.cc:9304
tree cse_and_gimplify_to_preheader(loop_vec_info, tree)
Definition tree-vect-loop.cc:1169
void finish_cost(vector_costs *costs, const vector_costs *scalar_costs, unsigned *prologue_cost, unsigned *body_cost, unsigned *epilogue_cost, unsigned *suggested_unroll_factor=NULL)
Definition tree-vectorizer.h:1911
int builtin_vectorization_cost(enum vect_cost_for_stmt type_of_cost, tree vectype, int misalign)
Definition tree-vectorizer.h:1846
loop_vec_info loop_vec_info_for_loop(class loop *loop)
Definition tree-vectorizer.h:1075
bool vect_is_extending_load(class vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2664
bool vect_transform_stmt(vec_info *, stmt_vec_info, gimple_stmt_iterator *, slp_tree, slp_instance)
Definition tree-vect-stmts.cc:13350
bool vect_is_simple_use(tree, vec_info *, enum vect_def_type *, stmt_vec_info *=NULL, gimple **=NULL)
Definition tree-vect-stmts.cc:13823
bool vect_rgroup_iv_might_wrap_p(loop_vec_info, rgroup_controls *)
Definition tree-vect-loop.cc:12536
tree vect_halve_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11138
void dump_stmt_cost(FILE *, int, enum vect_cost_for_stmt, stmt_vec_info, slp_tree, tree, int, unsigned, enum vect_cost_model_location)
Definition tree-vectorizer.cc:102
vec< unsigned > load_permutation_t
Definition tree-vectorizer.h:169
bool vect_slp_analyze_instance_alignment(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:2895
internal_fn vect_store_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:5757
tree vect_get_new_vect_var(tree, enum vect_var_kind, const char *)
Definition tree-vect-data-refs.cc:5016
void vect_mark_pattern_stmts(vec_info *, stmt_vec_info, gimple *, tree)
Definition tree-vect-patterns.cc:7009
class loop * vect_loop_versioning(loop_vec_info, gimple *)
Definition tree-vect-loop-manip.cc:3975
bool vect_check_gather_scatter(stmt_vec_info, loop_vec_info, gather_scatter_info *)
Definition tree-vect-data-refs.cc:4271
void vect_get_store_cost(vec_info *, stmt_vec_info, int, dr_alignment_support, int, unsigned int *, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:987
tree vect_get_slp_vect_def(slp_tree, unsigned)
Definition tree-vect-slp.cc:8510
internal_fn vect_load_lanes_supported(tree, unsigned HOST_WIDE_INT, bool)
Definition tree-vect-data-refs.cc:6384
opt_tree vect_get_mask_type_for_stmt(stmt_vec_info, unsigned int=0)
#define LOOP_VINFO_VECT_FACTOR(L)
Definition tree-vectorizer.h:989
void vect_free_slp_instance(slp_instance)
Definition tree-vect-slp.cc:209
bool vect_slp_function(function *)
Definition tree-vect-slp.cc:8062
std::pair< tree, tree > vec_object_pair
Definition tree-vectorizer.h:407
tree vect_gen_perm_mask_any(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9700
bool vect_transform_reduction(loop_vec_info, stmt_vec_info, gimple_stmt_iterator *, gimple **, slp_tree)
Definition tree-vect-loop.cc:8436
_bb_vec_info * bb_vec_info
#define LOOP_VINFO_FULLY_MASKED_P(L)
Definition tree-vectorizer.h:1035
int vect_get_known_peeling_cost(loop_vec_info, int, int *, stmt_vector_for_cost *, stmt_vector_for_cost *, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:4519
bool vect_transform_slp_perm_load(vec_info *, slp_tree, const vec< tree > &, gimple_stmt_iterator *, poly_uint64, bool, unsigned *, unsigned *=nullptr, bool=false)
tree vect_gen_while_not(gimple_seq *, tree, tree, tree)
Definition tree-vect-stmts.cc:14588
tree vect_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2654
bool is_simple_and_all_uses_invariant(stmt_vec_info, loop_vec_info)
Definition tree-vect-stmts.cc:308
bool vectorizable_phi(vec_info *, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:8997
#define STMT_VINFO_GROUPED_ACCESS(S)
Definition tree-vectorizer.h:1537
void vect_schedule_slp(vec_info *, const vec< slp_instance > &)
Definition tree-vect-slp.cc:9796
tree vect_peel_nonlinear_iv_init(gimple_seq *, tree, tree, tree, enum vect_induction_op_type)
Definition tree-vect-loop.cc:9430
bool check_reduction_path(dump_user_location_t, loop_p, gphi *, tree, enum tree_code)
Definition tree-vect-loop.cc:4199
bool can_duplicate_and_interleave_p(vec_info *, unsigned int, tree, unsigned int *=NULL, tree *=NULL, tree *=NULL)
Definition tree-vect-slp.cc:421
auto_vec< unsigned, 16 > auto_load_permutation_t
Definition tree-vectorizer.h:170
tree get_vectype_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13663
#define LOOP_VINFO_LOOP(L)
Definition tree-vectorizer.h:967
_complex_perm_kinds
Definition tree-vectorizer.h:2522
@ PERM_EVENEVEN
Definition tree-vectorizer.h:2527
@ PERM_TOP
Definition tree-vectorizer.h:2529
@ PERM_ODDEVEN
Definition tree-vectorizer.h:2525
@ PERM_EVENODD
Definition tree-vectorizer.h:2524
@ PERM_UNKNOWN
Definition tree-vectorizer.h:2523
@ PERM_ODDODD
Definition tree-vectorizer.h:2526
void vect_record_base_alignments(vec_info *)
Definition tree-vect-data-refs.cc:1271
void vect_get_vec_defs(vec_info *, stmt_vec_info, slp_tree, unsigned, tree, vec< tree > *, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL, tree=NULL, vec< tree > *=NULL)
Definition tree-vect-stmts.cc:1335
vec_load_store_type
Definition tree-vectorizer.h:1187
@ VLS_LOAD
Definition tree-vectorizer.h:1188
@ VLS_STORE
Definition tree-vectorizer.h:1189
@ VLS_STORE_INVARIANT
Definition tree-vectorizer.h:1190
tree get_mask_type_for_scalar_type(vec_info *, tree, unsigned int=0)
Definition tree-vect-stmts.cc:13738
pair_hash< tree_operand_hash, tree_operand_hash > tree_cond_mask_hash
Definition tree-vectorizer.h:402
vect_pattern_decl_t slp_patterns[]
Definition tree-vect-slp-patterns.cc:1636
_loop_vec_info * loop_vec_info
tree vect_double_mask_nunits(tree, machine_mode)
Definition tree-vect-loop.cc:11148
vect_var_kind
Definition tree-vectorizer.h:35
@ vect_pointer_var
Definition tree-vectorizer.h:37
@ vect_scalar_var
Definition tree-vectorizer.h:38
@ vect_mask_var
Definition tree-vectorizer.h:39
@ vect_simple_var
Definition tree-vectorizer.h:36
int vect_get_place_in_interleaving_chain(stmt_vec_info, stmt_vec_info)
Definition tree-vect-slp.cc:392
class loop * vect_do_peeling(loop_vec_info, tree, tree, tree *, tree *, tree *, int, bool, bool, tree *)
Definition tree-vect-loop-manip.cc:3114
void optimize_mask_stores(class loop *)
Definition tree-vect-loop.cc:12301
class _stmt_vec_info * stmt_vec_info
Definition tree-vectorizer.h:24
#define STMT_VINFO_DATA_REF(S)
Definition tree-vectorizer.h:1506
tree vect_get_loop_len(loop_vec_info, gimple_stmt_iterator *, vec_loop_lens *, unsigned int, tree, unsigned int, unsigned int)
Definition tree-vect-loop.cc:11335
bool vect_is_store_elt_extraction(vect_cost_for_stmt kind, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2608
void vect_detect_hybrid_slp(loop_vec_info)
Definition tree-vect-slp.cc:5970
opt_result vect_get_vector_types_for_stmt(vec_info *, stmt_vec_info, tree *, tree *, unsigned int=0)
Definition tree-vect-stmts.cc:14612
const poly_uint64 dr_target_alignment(dr_vec_info *dr_info)
Definition tree-vectorizer.h:1952
opt_result vect_analyze_loop_form(class loop *, vect_loop_form_info *)
Definition tree-vect-loop.cc:1737
void vect_update_inits_of_drs(loop_vec_info, tree, tree_code)
Definition tree-vect-loop-manip.cc:2608
struct data_reference * dr_p
Definition tree-vectorizer.h:1256
bool vect_can_advance_ivs_p(loop_vec_info)
Definition tree-vect-loop-manip.cc:2197
opt_result vect_analyze_data_refs(vec_info *, poly_uint64 *, bool *)
Definition tree-vect-data-refs.cc:4757
int dr_misalignment(dr_vec_info *dr_info, tree vectype, poly_int64 offset=0)
Definition tree-vect-data-refs.cc:1194
tree vect_init_vector(vec_info *, stmt_vec_info, tree, tree, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1174
void vect_copy_ref_info(tree, tree)
Definition tree-vect-data-refs.cc:5583
void vect_record_grouped_load_vectors(vec_info *, stmt_vec_info, vec< tree >)
Definition tree-vect-data-refs.cc:6989
bool vectorizable_lc_phi(loop_vec_info, stmt_vec_info, gimple **, slp_tree)
Definition tree-vect-loop.cc:8939
tree vect_get_new_ssa_name(tree, enum vect_var_kind, const char *=NULL)
Definition tree-vect-data-refs.cc:5054
#define STMT_VINFO_VECTYPE(S)
Definition tree-vectorizer.h:1503
opt_result vect_prune_runtime_alias_test_list(loop_vec_info)
Definition tree-vect-data-refs.cc:3908
bool needs_fold_left_reduction_p(tree, code_helper)
Definition tree-vect-loop.cc:3970
bool vect_nop_conversion_p(stmt_vec_info)
Definition tree-vect-stmts.cc:5861
void vect_finish_stmt_generation(vec_info *, stmt_vec_info, gimple *, gimple_stmt_iterator *)
Definition tree-vect-stmts.cc:1396
unsigned HOST_WIDE_INT vect_max_vf(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2137
loop_vec_info vect_create_loop_vinfo(class loop *, vec_info_shared *, const vect_loop_form_info *, loop_vec_info=nullptr)
Definition tree-vect-loop.cc:1914
bool vect_stmt_dominates_stmt_p(gimple *, gimple *)
Definition tree-vectorizer.cc:772
bool vect_supportable_shift(vec_info *, enum tree_code, tree)
Definition tree-vect-stmts.cc:6056
void vect_optimize_slp(vec_info *)
Definition tree-vect-slp.cc:5802
void vect_iv_increment_position(edge, gimple_stmt_iterator *, bool *)
Definition tree-vect-loop-manip.cc:457
hash_set< scalar_cond_masked_key > scalar_cond_masked_set_type
Definition tree-vectorizer.h:397
tree vect_phi_initial_value(gphi *phi)
Definition tree-vectorizer.h:1759
opt_pointer_wrapper< loop_vec_info > opt_loop_vec_info
Definition tree-vectorizer.h:1072
bool vectorizable_reduction(loop_vec_info, stmt_vec_info, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:7431
unsigned record_stmt_cost(stmt_vector_for_cost *, int, enum vect_cost_for_stmt, stmt_vec_info, tree, int, enum vect_cost_model_location)
Definition tree-vect-stmts.cc:117
bool vect_emulated_vector_p(tree)
Definition tree-vect-loop.cc:9293
tree get_same_sized_vectype(tree, tree)
Definition tree-vect-stmts.cc:13774
gimple * vect_gimple_build(tree, code_helper, tree, tree=NULL_TREE)
Definition tree-vect-patterns.cc:7350
void add_stmt_costs(vector_costs *costs, stmt_vector_for_cost *cost_vec)
Definition tree-vectorizer.h:1924
void vect_finish_replace_stmt(vec_info *, stmt_vec_info, gimple *)
Definition tree-vect-stmts.cc:1380
void vect_slp_fini(void)
Definition tree-vect-slp.cc:81
#define LOOP_VINFO_NITERS_KNOWN_P(L)
Definition tree-vectorizer.h:1059
stmt_vec_info get_later_stmt(stmt_vec_info stmt1_info, stmt_vec_info stmt2_info)
Definition tree-vectorizer.h:1799
vec< std::pair< unsigned, unsigned > > lane_permutation_t
Definition tree-vectorizer.h:167
widest_int vect_iv_limit_for_partial_vectors(loop_vec_info loop_vinfo)
Definition tree-vect-loop.cc:12495
void vect_get_vec_defs_for_operand(vec_info *vinfo, stmt_vec_info, unsigned, tree op, vec< tree > *, tree=NULL)
Definition tree-vect-stmts.cc:1246
void vect_gather_slp_loads(vec_info *)
Definition tree-vect-slp.cc:5812
vect_partial_vector_style
Definition tree-vectorizer.h:316
@ vect_partial_vectors_none
Definition tree-vectorizer.h:317
@ vect_partial_vectors_len
Definition tree-vectorizer.h:320
@ vect_partial_vectors_avx512
Definition tree-vectorizer.h:319
@ vect_partial_vectors_while_ult
Definition tree-vectorizer.h:318
class _slp_instance * slp_instance
bool vect_maybe_update_slp_op_vectype(slp_tree, tree)
Definition tree-vect-stmts.cc:14035
bool compatible_calls_p(gcall *, gcall *)
Definition tree-vect-slp.cc:952
#define STMT_VINFO_REDUC_IDX(S)
Definition tree-vectorizer.h:1513
class loop * slpeel_tree_duplicate_loop_to_edge_cfg(class loop *, edge, class loop *, edge, edge, edge *, bool=true, vec< basic_block > *=NULL)
Definition tree-vect-loop-manip.cc:1474
unsigned int vect_vf_for_cost(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2119
unsigned int vect_nunits_for_cost(tree vec_type)
Definition tree-vectorizer.h:2129
bool vect_get_range_info(tree, wide_int *, wide_int *)
Definition tree-vect-patterns.cc:67
bool vect_grouped_load_supported(tree, bool, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:6269
#define STMT_VINFO_IN_PATTERN_P(S)
Definition tree-vectorizer.h:1532
dump_user_location_t vect_location
Definition tree-vectorizer.cc:88
tree vect_create_addr_base_for_vector_ref(vec_info *, stmt_vec_info, gimple_seq *, tree)
Definition tree-vect-data-refs.cc:5129
stmt_vec_info vect_find_last_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:2959
tree vect_get_main_loop_result(loop_vec_info, tree, tree)
Definition tree-vect-loop-manip.cc:3005
#define LOOP_VINFO_PEELING_FOR_ALIGNMENT(L)
Definition tree-vectorizer.h:1003
#define STMT_VINFO_DR_INFO(S)
Definition tree-vectorizer.h:1529
hash_map< slp_node_hash, bool > slp_compat_nodes_map_t
Definition tree-vectorizer.h:2539
bool vect_gather_scatter_fn_p(vec_info *, bool, bool, tree, tree, tree, int, internal_fn *, tree *)
Definition tree-vect-data-refs.cc:4170
tree get_related_vectype_for_scalar_type(machine_mode, tree, poly_uint64=0)
Definition tree-vect-stmts.cc:13545
bool nested_in_vect_loop_p(class loop *loop, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1748
bool vect_slp_analyze_instance_dependence(vec_info *, slp_instance)
Definition tree-vect-data-refs.cc:1146
auto_vec< std::pair< data_reference *, tree > > drs_init_vec
Definition tree-vectorizer.h:657
edge vec_init_loop_exit_info(class loop *)
Definition tree-vect-loop.cc:972
opt_result vect_analyze_data_ref_accesses(vec_info *, vec< int > *)
Definition tree-vect-data-refs.cc:3386
int vect_get_stmt_cost(enum vect_cost_for_stmt type_of_cost)
Definition tree-vectorizer.h:1856
void vect_transform_grouped_load(vec_info *, stmt_vec_info, vec< tree >, int, gimple_stmt_iterator *)
Definition tree-vect-data-refs.cc:6958
dump_user_location_t find_loop_location(class loop *)
Definition tree-vect-loop-manip.cc:2056
opt_result vect_analyze_slp(vec_info *, unsigned)
Definition tree-vect-slp.cc:3639
bool vect_slp_analyze_operations(vec_info *)
Definition tree-vect-slp.cc:6758
tree neutral_op_for_reduction(tree, code_helper, tree, bool=true)
Definition tree-vect-loop.cc:3912
void vect_free_slp_tree(slp_tree)
Definition tree-vect-slp.cc:169
unsigned vectorize_loops(void)
bool vect_can_force_dr_alignment_p(const_tree, poly_uint64)
Definition tree-vect-data-refs.cc:7040
tree bump_vector_ptr(vec_info *, tree, gimple *, gimple_stmt_iterator *, stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:5514
bool vect_apply_runtime_profitability_check_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2162
unsigned int vect_get_num_vectors(poly_uint64 nunits, tree vectype)
Definition tree-vectorizer.h:2075
#define DR_TARGET_ALIGNMENT(DR)
Definition tree-vectorizer.h:1958
void vect_free_loop_info_assumptions(class loop *)
Definition tree-vectorizer.cc:834
void set_dr_target_alignment(dr_vec_info *dr_info, poly_uint64 val)
Definition tree-vectorizer.h:1961
stmt_vec_info vect_stmt_to_vectorize(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1812
int vect_reduc_type(vec_info *vinfo, stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2625
#define STMT_VINFO_RELATED_STMT(S)
Definition tree-vectorizer.h:1533
bool is_pattern_stmt_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1780
#define MAX_VECTORIZATION_FACTOR
Definition tree-vectorizer.h:1733
int vect_pow2(int x)
Definition tree-vectorizer.h:1833
bool reduction_fn_for_scalar_code(code_helper, internal_fn *)
Definition tree-vect-loop.cc:3850
void vect_pattern_recog(vec_info *)
Definition tree-vect-patterns.cc:7290
int vect_slp_child_index_for_operand(const gimple *, int op, bool)
Definition tree-vect-slp.cc:598
vect_induction_op_type
Definition tree-vectorizer.h:74
@ vect_step_op_add
Definition tree-vectorizer.h:75
@ vect_step_op_shl
Definition tree-vectorizer.h:78
@ vect_step_op_mul
Definition tree-vectorizer.h:77
@ vect_step_op_neg
Definition tree-vectorizer.h:76
@ vect_step_op_shr
Definition tree-vectorizer.h:79
void vect_set_loop_condition(class loop *, edge, loop_vec_info, tree, tree, tree, bool)
Definition tree-vect-loop-manip.cc:1389
void vect_prepare_for_masked_peels(loop_vec_info)
Definition tree-vect-loop-manip.cc:2638
stmt_vec_info vect_find_first_scalar_stmt_in_slp(slp_tree)
Definition tree-vect-slp.cc:2976
enum _complex_perm_kinds complex_perm_kinds_t
bool vect_is_integer_truncation(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2688
auto_vec< std::pair< unsigned, unsigned >, 16 > auto_lane_permutation_t
Definition tree-vectorizer.h:168
bool vect_use_loop_mask_for_alignment_p(loop_vec_info loop_vinfo)
Definition tree-vectorizer.h:2064
tree vect_build_loop_niters(loop_vec_info, bool *=NULL)
Definition tree-vect-loop-manip.cc:2684
bool vect_get_loop_mask_type(loop_vec_info)
void vect_update_max_nunits(poly_uint64 *max_nunits, poly_uint64 nunits)
Definition tree-vectorizer.h:2095
slp_instance_kind
Definition tree-vectorizer.h:247
@ slp_inst_kind_ctor
Definition tree-vectorizer.h:252
@ slp_inst_kind_reduc_group
Definition tree-vectorizer.h:249
@ slp_inst_kind_store
Definition tree-vectorizer.h:248
@ slp_inst_kind_reduc_chain
Definition tree-vectorizer.h:250
@ slp_inst_kind_bb_reduc
Definition tree-vectorizer.h:251
enum vect_cost_model loop_cost_model(loop_p loop)
Definition tree-vectorizer.h:2043
#define DR_MISALIGNMENT_UNKNOWN
Definition tree-vectorizer.h:1936
hash_map< slp_tree, complex_perm_kinds_t > slp_tree_to_load_perm_map_t
Definition tree-vectorizer.h:2534
opt_result vect_enhance_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2141
bool vect_is_reduction(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2617
innermost_loop_behavior * vect_dr_behavior(vec_info *vinfo, dr_vec_info *dr_info)
Definition tree-vectorizer.h:2005
void vect_permute_store_chain(vec_info *, vec< tree > &, unsigned int, stmt_vec_info, gimple_stmt_iterator *, vec< tree > *)
Definition tree-vect-data-refs.cc:5843
tree perm_mask_for_reverse(tree)
Definition tree-vect-stmts.cc:1809
stmt_vec_info_type
Definition tree-vectorizer.h:1115
@ comparison_vec_info_type
Definition tree-vectorizer.h:1125
@ condition_vec_info_type
Definition tree-vectorizer.h:1124
@ type_conversion_vec_info_type
Definition tree-vectorizer.h:1130
@ load_vec_info_type
Definition tree-vectorizer.h:1117
@ loop_exit_ctrl_vec_info_type
Definition tree-vectorizer.h:1135
@ recurr_info_type
Definition tree-vectorizer.h:1134
@ op_vec_info_type
Definition tree-vectorizer.h:1120
@ reduc_vec_info_type
Definition tree-vectorizer.h:1126
@ assignment_vec_info_type
Definition tree-vectorizer.h:1123
@ undef_vec_info_type
Definition tree-vectorizer.h:1116
@ shift_vec_info_type
Definition tree-vectorizer.h:1119
@ type_demotion_vec_info_type
Definition tree-vectorizer.h:1129
@ store_vec_info_type
Definition tree-vectorizer.h:1118
@ cycle_phi_info_type
Definition tree-vectorizer.h:1131
@ phi_info_type
Definition tree-vectorizer.h:1133
@ lc_phi_info_type
Definition tree-vectorizer.h:1132
@ type_promotion_vec_info_type
Definition tree-vectorizer.h:1128
@ call_simd_clone_vec_info_type
Definition tree-vectorizer.h:1122
@ call_vec_info_type
Definition tree-vectorizer.h:1121
@ induc_vec_info_type
Definition tree-vectorizer.h:1127
void vect_get_slp_defs(slp_tree, vec< tree > *)
Definition tree-vect-slp.cc:8518
void vect_record_loop_len(loop_vec_info, vec_loop_lens *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11300
struct _slp_tree * slp_tree
Definition tree-vectorizer.h:25
gimple_seq vect_gen_len(tree, tree, tree, tree)
Definition tree-vect-stmts.cc:14776
bool vect_transform_cycle_phi(loop_vec_info, stmt_vec_info, gimple **, slp_tree, slp_instance)
Definition tree-vect-loop.cc:8661
tree vect_embedded_comparison_type(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:2639
gimple * vect_loop_vectorized_call(class loop *, gcond **cond=NULL)
Definition tree-vectorizer.cc:849
void duplicate_and_interleave(vec_info *, gimple_seq *, tree, const vec< tree > &, unsigned int, vec< tree > &)
Definition tree-vect-slp.cc:8194
bool vect_grouped_store_supported(tree, unsigned HOST_WIDE_INT)
Definition tree-vect-data-refs.cc:5639
tree vect_setup_realignment(vec_info *, stmt_vec_info, gimple_stmt_iterator *, tree *, enum dr_alignment_support, tree, class loop **)
Definition tree-vect-data-refs.cc:6031
vect_def_type
Definition tree-vectorizer.h:59
@ vect_induction_def
Definition tree-vectorizer.h:64
@ vect_uninitialized_def
Definition tree-vectorizer.h:60
@ vect_nested_cycle
Definition tree-vectorizer.h:67
@ vect_constant_def
Definition tree-vectorizer.h:61
@ vect_external_def
Definition tree-vectorizer.h:62
@ vect_condition_def
Definition tree-vectorizer.h:69
@ vect_internal_def
Definition tree-vectorizer.h:63
@ vect_reduction_def
Definition tree-vectorizer.h:65
@ vect_double_reduction_def
Definition tree-vectorizer.h:66
@ vect_first_order_recurrence
Definition tree-vectorizer.h:68
@ vect_unknown_def_type
Definition tree-vectorizer.h:70
opt_result vect_analyze_data_ref_dependences(loop_vec_info, unsigned int *)
Definition tree-vect-data-refs.cc:871
vect_relevant
Definition tree-vectorizer.h:1140
@ vect_used_in_outer
Definition tree-vectorizer.h:1150
@ vect_used_in_scope
Definition tree-vectorizer.h:1160
@ vect_unused_in_scope
Definition tree-vectorizer.h:1141
@ vect_used_by_reduction
Definition tree-vectorizer.h:1158
@ vect_used_in_outer_by_reduction
Definition tree-vectorizer.h:1147
@ vect_used_only_live
Definition tree-vectorizer.h:1144
bool is_loop_header_bb_p(basic_block bb)
Definition tree-vectorizer.h:1822
#define STMT_VINFO_DR_WRT_VEC_LOOP(S)
Definition tree-vectorizer.h:1516
bool vectorizable_live_operation(vec_info *, stmt_vec_info, slp_tree, slp_instance, int, bool, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:10708
slp_tree vect_create_new_slp_node(unsigned, tree_code)
Definition tree-vect-slp.cc:224
opt_result vect_analyze_data_refs_alignment(loop_vec_info)
Definition tree-vect-data-refs.cc:2828
unsigned int vect_known_alignment_in_bytes(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:1990
bool unlimited_cost_model(loop_p loop)
Definition tree-vectorizer.h:2054
bool vectorizable_recurr(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9140
opt_result vect_mark_stmts_to_be_vectorized(loop_vec_info, bool *)
Definition tree-vect-stmts.cc:673
tree vect_gen_while(gimple_seq *, tree, tree, tree, const char *=nullptr)
Definition tree-vect-stmts.cc:14564
tree vect_get_store_rhs(stmt_vec_info)
Definition tree-vect-stmts.cc:1890
class loop * vect_transform_loop(loop_vec_info, gimple *)
Definition tree-vect-loop.cc:11830
void vect_slp_init(void)
Definition tree-vect-slp.cc:75
unsigned int vect_get_num_copies(loop_vec_info loop_vinfo, tree vectype)
Definition tree-vectorizer.h:2086
unsigned int vect_get_scalar_dr_size(dr_vec_info *dr_info)
Definition tree-vectorizer.h:2153
enum dr_alignment_support vect_supportable_dr_alignment(vec_info *, dr_vec_info *, tree, int)
Definition tree-vect-data-refs.cc:7063
opt_result vect_determine_partial_vectors_and_peeling(loop_vec_info)
Definition tree-vect-loop.cc:2644
bool ref_within_array_bound(gimple *, tree)
Definition tree-if-conv.cc:848
dr_alignment_support
Definition tree-vectorizer.h:50
@ dr_unaligned_supported
Definition tree-vectorizer.h:52
@ dr_explicit_realign_optimized
Definition tree-vectorizer.h:54
@ dr_aligned
Definition tree-vectorizer.h:55
@ dr_explicit_realign
Definition tree-vectorizer.h:53
@ dr_unaligned_unsupported
Definition tree-vectorizer.h:51
bool supportable_narrowing_operation(code_helper, tree, tree, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:14403
#define STMT_VINFO_REDUC_TYPE(S)
Definition tree-vectorizer.h:1543
operation_type
Definition tree-vectorizer.h:43
@ ternary_op
Definition tree-vectorizer.h:46
@ unary_op
Definition tree-vectorizer.h:44
@ binary_op
Definition tree-vectorizer.h:45
void vect_record_loop_mask(loop_vec_info, vec_loop_masks *, unsigned int, tree, tree)
Definition tree-vect-loop.cc:11160
vect_memory_access_type
Definition tree-vectorizer.h:1195
@ VMAT_STRIDED_SLP
Definition tree-vectorizer.h:1227
@ VMAT_INVARIANT
Definition tree-vectorizer.h:1197
@ VMAT_GATHER_SCATTER
Definition tree-vectorizer.h:1230
@ VMAT_ELEMENTWISE
Definition tree-vectorizer.h:1221
@ VMAT_CONTIGUOUS_DOWN
Definition tree-vectorizer.h:1205
@ VMAT_CONTIGUOUS
Definition tree-vectorizer.h:1200
@ VMAT_CONTIGUOUS_PERMUTE
Definition tree-vectorizer.h:1210
@ VMAT_LOAD_STORE_LANES
Definition tree-vectorizer.h:1217
@ VMAT_CONTIGUOUS_REVERSE
Definition tree-vectorizer.h:1214
bool vect_chooses_same_modes_p(vec_info *, machine_mode)
Definition tree-vect-stmts.cc:13792
bool slpeel_can_duplicate_loop_p(const class loop *, const_edge, const_edge)
Definition tree-vect-loop-manip.cc:2024
opt_result vect_analyze_stmt(vec_info *, stmt_vec_info, bool *, slp_tree, slp_instance, stmt_vector_for_cost *)
Definition tree-vect-stmts.cc:13103
tree vect_get_loop_mask(loop_vec_info, gimple_stmt_iterator *, vec_loop_masks *, unsigned int, tree, unsigned int)
Definition tree-vect-loop.cc:11182
slp_vect_type
Definition tree-vectorizer.h:1179
@ loop_vect
Definition tree-vectorizer.h:1180
@ hybrid
Definition tree-vectorizer.h:1182
@ pure_slp
Definition tree-vectorizer.h:1181
vect_reduction_type
Definition tree-vectorizer.h:83
@ TREE_CODE_REDUCTION
Definition tree-vectorizer.h:84
@ FOLD_LEFT_REDUCTION
Definition tree-vectorizer.h:102
@ EXTRACT_LAST_REDUCTION
Definition tree-vectorizer.h:94
@ INTEGER_INDUC_COND_REDUCTION
Definition tree-vectorizer.h:86
@ CONST_COND_REDUCTION
Definition tree-vectorizer.h:87
@ COND_REDUCTION
Definition tree-vectorizer.h:85
bool supportable_widening_operation(vec_info *, code_helper, stmt_vec_info, tree, tree, code_helper *, code_helper *, int *, vec< tree > *)
Definition tree-vect-stmts.cc:14072
auto_vec< rgroup_controls > vec_loop_lens
Definition tree-vectorizer.h:655
void set_dr_misalignment(dr_vec_info *dr_info, int val)
Definition tree-vectorizer.h:1940
bool vect_use_mask_type_p(stmt_vec_info stmt_info)
Definition tree-vectorizer.h:1771
pair_hash< nofree_ptr_hash< _slp_tree >, nofree_ptr_hash< _slp_tree > > slp_node_hash
Definition tree-vectorizer.h:2538
hash_map< tree_operand_hash, std::pair< stmt_vec_info, innermost_loop_behavior * > > vec_base_alignments
Definition tree-vectorizer.h:127
unsigned add_stmt_cost(vector_costs *costs, int count, enum vect_cost_for_stmt kind, stmt_vec_info stmt_info, slp_tree node, tree vectype, int misalign, enum vect_cost_model_location where)
Definition tree-vectorizer.h:1876
tree vect_get_smallest_scalar_type(stmt_vec_info, tree)
Definition tree-vect-data-refs.cc:147
vec< stmt_info_for_cost > stmt_vector_for_cost
Definition tree-vectorizer.h:121
stmt_vec_info info_for_reduction(vec_info *, stmt_vec_info)
Definition tree-vect-loop.cc:5677
bool known_alignment_for_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:1981
void vect_remove_stores(vec_info *, stmt_vec_info)
Definition tree-vect-stmts.cc:13515
#define DR_GROUP_FIRST_ELEMENT(S)
Definition tree-vectorizer.h:1552
void vect_get_load_cost(vec_info *, stmt_vec_info, int, dr_alignment_support, int, bool, unsigned int *, unsigned int *, stmt_vector_for_cost *, stmt_vector_for_cost *, bool)
Definition tree-vect-stmts.cc:1037
hash_set< tree_cond_mask_hash > vec_cond_masked_set_type
Definition tree-vectorizer.h:403
#define STMT_VINFO_REDUC_DEF(S)
Definition tree-vectorizer.h:1546
opt_result vect_find_stmt_data_reference(loop_p, gimple *, vec< data_reference_p > *, vec< int > *, int)
Definition tree-vect-data-refs.cc:4582
opt_loop_vec_info vect_analyze_loop(class loop *, vec_info_shared *)
Definition tree-vect-loop.cc:3567
tree get_dr_vinfo_offset(vec_info *vinfo, dr_vec_info *dr_info, bool check_outer=false)
Definition tree-vectorizer.h:2021
bool vect_slp_if_converted_bb(basic_block bb, loop_p orig_loop)
Definition tree-vect-slp.cc:8051
size_t num__slp_patterns
Definition tree-vect-slp-patterns.cc:1647
tree vect_gen_perm_mask_checked(tree, const vec_perm_indices &)
Definition tree-vect-stmts.cc:9715
bool vectorizable_induction(loop_vec_info, stmt_vec_info, gimple **, slp_tree, stmt_vector_for_cost *)
Definition tree-vect-loop.cc:9909
vect_pattern *(* vect_pattern_decl_t)(slp_tree_to_load_perm_map_t *, slp_compat_nodes_map_t *, slp_tree *)
Definition tree-vectorizer.h:2588
tree vect_create_destination_var(tree, tree)
Definition tree-vect-data-refs.cc:5605
tree vect_create_data_ref_ptr(vec_info *, stmt_vec_info, tree, class loop *, tree, tree *, gimple_stmt_iterator *, gimple **, bool, tree=NULL_TREE)
Definition tree-vect-data-refs.cc:5252
void vect_gen_vector_loop_niters(loop_vec_info, tree, tree *, tree *, bool)
Definition tree-vect-loop-manip.cc:2775
bool vect_make_slp_decision(loop_vec_info)
Definition tree-vect-slp.cc:5830
bool aligned_access_p(dr_vec_info *dr_info, tree vectype)
Definition tree-vectorizer.h:1971
unsigned HOST_WIDE_INT tree_to_uhwi(const_tree t)
Definition tree.cc:6471
#define TYPE_ALIGN_UNIT(NODE)
Definition tree.h:2339
poly_uint64 TYPE_VECTOR_SUBPARTS(const_tree node)
Definition tree.h:4243
#define TYPE_PRECISION(NODE)
Definition tree.h:2245
#define TREE_OPERAND(NODE, I)
Definition tree.h:1285
#define sizetype
Definition tree.h:5102
#define TREE_CODE_CLASS(CODE)
Definition tree.h:202
#define TYPE_SIZE_UNIT(NODE)
Definition tree.h:2242
#define CONVERT_EXPR_CODE_P(CODE)
Definition tree.h:553
#define TREE_TYPE(NODE)
Definition tree.h:512
#define INTEGRAL_TYPE_P(TYPE)
Definition tree.h:613
#define COMPARISON_CLASS_P(NODE)
Definition tree.h:256
#define NULL_TREE
Definition tree.h:317
#define FOR_EACH_VEC_ELT(V, I, P)
Definition vec.h:1884
constexpr vnull vNULL
Definition vec.h:569