Line data Source code
1 : /* Tree-based target query functions relating to optabs
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "target.h"
25 : #include "insn-codes.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "optabs.h"
30 : #include "optabs-tree.h"
31 : #include "stor-layout.h"
32 : #include "internal-fn.h"
33 :
34 : /* Return the optab used for computing the operation given by the tree code,
35 : CODE and the tree EXP. This function is not always usable (for example, it
36 : cannot give complete results for multiplication or division) but probably
37 : ought to be relied on more widely throughout the expander. */
38 : optab
39 10344858 : optab_for_tree_code (enum tree_code code, const_tree type,
40 : enum optab_subtype subtype)
41 : {
42 10344858 : bool trapv;
43 10344858 : switch (code)
44 : {
45 : case BIT_AND_EXPR:
46 : return and_optab;
47 :
48 231076 : case BIT_IOR_EXPR:
49 231076 : return ior_optab;
50 :
51 7684 : case BIT_NOT_EXPR:
52 7684 : return one_cmpl_optab;
53 :
54 52366 : case BIT_XOR_EXPR:
55 52366 : return xor_optab;
56 :
57 2792 : case MULT_HIGHPART_EXPR:
58 2792 : return TYPE_UNSIGNED (type) ? umul_highpart_optab : smul_highpart_optab;
59 :
60 66 : case CEIL_MOD_EXPR:
61 66 : case FLOOR_MOD_EXPR:
62 66 : case ROUND_MOD_EXPR:
63 : /* {s,u}mod_optab implements TRUNC_MOD_EXPR. For scalar modes,
64 : expansion has code to adjust TRUNC_MOD_EXPR into the desired other
65 : modes, but for vector modes it does not. The adjustment code
66 : should be instead emitted in tree-vect-patterns.cc. */
67 66 : if (VECTOR_TYPE_P (type))
68 : return unknown_optab;
69 : /* FALLTHRU */
70 39629 : case TRUNC_MOD_EXPR:
71 39629 : return TYPE_UNSIGNED (type) ? umod_optab : smod_optab;
72 :
73 18 : case CEIL_DIV_EXPR:
74 18 : case FLOOR_DIV_EXPR:
75 18 : case ROUND_DIV_EXPR:
76 : /* {,u}{s,u}div_optab implements {TRUNC,EXACT}_DIV_EXPR or RDIV_EXPR.
77 : For scalar modes, expansion has code to adjust TRUNC_DIV_EXPR
78 : into the desired other modes, but for vector modes it does not.
79 : The adjustment code should be instead emitted in
80 : tree-vect-patterns.cc. */
81 18 : if (VECTOR_TYPE_P (type))
82 : return unknown_optab;
83 : /* FALLTHRU */
84 33667 : case RDIV_EXPR:
85 33667 : gcc_assert (FLOAT_TYPE_P (type)
86 : || ALL_FIXED_POINT_MODE_P (TYPE_MODE (type)));
87 : /* FALLTHRU */
88 80688 : case TRUNC_DIV_EXPR:
89 80688 : case EXACT_DIV_EXPR:
90 80688 : if (TYPE_SATURATING (type))
91 0 : return TYPE_UNSIGNED (type) ? usdiv_optab : ssdiv_optab;
92 80688 : return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab;
93 :
94 747877 : case LSHIFT_EXPR:
95 747877 : if (VECTOR_TYPE_P (type))
96 : {
97 747837 : if (subtype == optab_vector)
98 362865 : return TYPE_SATURATING (type) ? unknown_optab : vashl_optab;
99 :
100 384972 : gcc_assert (subtype == optab_scalar);
101 : }
102 385012 : if (TYPE_SATURATING (type))
103 0 : return TYPE_UNSIGNED (type) ? usashl_optab : ssashl_optab;
104 : return ashl_optab;
105 :
106 235442 : case RSHIFT_EXPR:
107 235442 : if (VECTOR_TYPE_P (type))
108 : {
109 235405 : if (subtype == optab_vector)
110 112503 : return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab;
111 :
112 122902 : gcc_assert (subtype == optab_scalar);
113 : }
114 122939 : return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab;
115 :
116 491 : case LROTATE_EXPR:
117 491 : if (VECTOR_TYPE_P (type))
118 : {
119 491 : if (subtype == optab_vector)
120 : return vrotl_optab;
121 :
122 189 : gcc_assert (subtype == optab_scalar);
123 : }
124 : return rotl_optab;
125 :
126 14489 : case RROTATE_EXPR:
127 14489 : if (VECTOR_TYPE_P (type))
128 : {
129 14489 : if (subtype == optab_vector)
130 : return vrotr_optab;
131 :
132 7384 : gcc_assert (subtype == optab_scalar);
133 : }
134 : return rotr_optab;
135 :
136 90735 : case MAX_EXPR:
137 90735 : return TYPE_UNSIGNED (type) ? umax_optab : smax_optab;
138 :
139 46756 : case MIN_EXPR:
140 46756 : return TYPE_UNSIGNED (type) ? umin_optab : smin_optab;
141 :
142 : case POINTER_PLUS_EXPR:
143 : return add_optab;
144 :
145 : case POINTER_DIFF_EXPR:
146 : return sub_optab;
147 :
148 0 : case REALIGN_LOAD_EXPR:
149 0 : return vec_realign_load_optab;
150 :
151 2146 : case WIDEN_SUM_EXPR:
152 2146 : return TYPE_UNSIGNED (type) ? usum_widen_optab : ssum_widen_optab;
153 :
154 1855 : case DOT_PROD_EXPR:
155 1855 : {
156 1855 : if (subtype == optab_vector_mixed_sign)
157 : return usdot_prod_optab;
158 :
159 1333 : return (TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab);
160 : }
161 :
162 919 : case SAD_EXPR:
163 919 : return TYPE_UNSIGNED (type) ? usad_optab : ssad_optab;
164 :
165 33600 : case WIDEN_MULT_PLUS_EXPR:
166 33600 : return (TYPE_UNSIGNED (type)
167 33600 : ? (TYPE_SATURATING (type)
168 23919 : ? usmadd_widen_optab : umadd_widen_optab)
169 9681 : : (TYPE_SATURATING (type)
170 9681 : ? ssmadd_widen_optab : smadd_widen_optab));
171 :
172 871 : case WIDEN_MULT_MINUS_EXPR:
173 871 : return (TYPE_UNSIGNED (type)
174 871 : ? (TYPE_SATURATING (type)
175 476 : ? usmsub_widen_optab : umsub_widen_optab)
176 395 : : (TYPE_SATURATING (type)
177 395 : ? ssmsub_widen_optab : smsub_widen_optab));
178 :
179 89344 : case VEC_WIDEN_MULT_HI_EXPR:
180 89344 : return (TYPE_UNSIGNED (type)
181 89344 : ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab);
182 :
183 89337 : case VEC_WIDEN_MULT_LO_EXPR:
184 89337 : return (TYPE_UNSIGNED (type)
185 89337 : ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab);
186 :
187 184679 : case VEC_WIDEN_MULT_EVEN_EXPR:
188 184679 : return (TYPE_UNSIGNED (type)
189 184679 : ? vec_widen_umult_even_optab : vec_widen_smult_even_optab);
190 :
191 184679 : case VEC_WIDEN_MULT_ODD_EXPR:
192 184679 : return (TYPE_UNSIGNED (type)
193 184679 : ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab);
194 :
195 9426 : case VEC_WIDEN_LSHIFT_HI_EXPR:
196 9426 : return (TYPE_UNSIGNED (type)
197 9426 : ? vec_widen_ushiftl_hi_optab : vec_widen_sshiftl_hi_optab);
198 :
199 9426 : case VEC_WIDEN_LSHIFT_LO_EXPR:
200 9426 : return (TYPE_UNSIGNED (type)
201 9426 : ? vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab);
202 :
203 61758 : case VEC_UNPACK_HI_EXPR:
204 61758 : return (TYPE_UNSIGNED (type)
205 61758 : ? vec_unpacku_hi_optab : vec_unpacks_hi_optab);
206 :
207 61736 : case VEC_UNPACK_LO_EXPR:
208 61736 : return (TYPE_UNSIGNED (type)
209 61736 : ? vec_unpacku_lo_optab : vec_unpacks_lo_optab);
210 :
211 10108 : case VEC_UNPACK_FLOAT_HI_EXPR:
212 : /* The signedness is determined from input operand. */
213 10108 : return (TYPE_UNSIGNED (type)
214 10108 : ? vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab);
215 :
216 10108 : case VEC_UNPACK_FLOAT_LO_EXPR:
217 : /* The signedness is determined from input operand. */
218 10108 : return (TYPE_UNSIGNED (type)
219 10108 : ? vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab);
220 :
221 255 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
222 : /* The signedness is determined from output operand. */
223 255 : return (TYPE_UNSIGNED (type)
224 255 : ? vec_unpack_ufix_trunc_hi_optab
225 : : vec_unpack_sfix_trunc_hi_optab);
226 :
227 255 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
228 : /* The signedness is determined from output operand. */
229 255 : return (TYPE_UNSIGNED (type)
230 255 : ? vec_unpack_ufix_trunc_lo_optab
231 : : vec_unpack_sfix_trunc_lo_optab);
232 :
233 65202 : case VEC_PACK_TRUNC_EXPR:
234 65202 : return vec_pack_trunc_optab;
235 :
236 0 : case VEC_PACK_SAT_EXPR:
237 0 : return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab;
238 :
239 978 : case VEC_PACK_FIX_TRUNC_EXPR:
240 : /* The signedness is determined from output operand. */
241 978 : return (TYPE_UNSIGNED (type)
242 978 : ? vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab);
243 :
244 314 : case VEC_PACK_FLOAT_EXPR:
245 : /* The signedness is determined from input operand. */
246 314 : return (TYPE_UNSIGNED (type)
247 314 : ? vec_packu_float_optab : vec_packs_float_optab);
248 :
249 0 : case VEC_DUPLICATE_EXPR:
250 0 : return vec_duplicate_optab;
251 :
252 0 : case VEC_SERIES_EXPR:
253 0 : return vec_series_optab;
254 :
255 6609474 : default:
256 6609474 : break;
257 : }
258 :
259 6609474 : trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type);
260 6609474 : switch (code)
261 : {
262 4534657 : case PLUS_EXPR:
263 4534657 : if (TYPE_SATURATING (type))
264 0 : return TYPE_UNSIGNED (type) ? usadd_optab : ssadd_optab;
265 4534657 : return trapv ? addv_optab : add_optab;
266 :
267 956298 : case MINUS_EXPR:
268 956298 : if (TYPE_SATURATING (type))
269 0 : return TYPE_UNSIGNED (type) ? ussub_optab : sssub_optab;
270 956298 : return trapv ? subv_optab : sub_optab;
271 :
272 943721 : case MULT_EXPR:
273 943721 : if (TYPE_SATURATING (type))
274 0 : return TYPE_UNSIGNED (type) ? usmul_optab : ssmul_optab;
275 943721 : return trapv ? smulv_optab : smul_optab;
276 :
277 68450 : case NEGATE_EXPR:
278 68450 : if (TYPE_SATURATING (type))
279 0 : return TYPE_UNSIGNED (type) ? usneg_optab : ssneg_optab;
280 68450 : return trapv ? negv_optab : neg_optab;
281 :
282 3080 : case ABS_EXPR:
283 3080 : return trapv ? absv_optab : abs_optab;
284 :
285 : case ABSU_EXPR:
286 : return abs_optab;
287 : default:
288 : return unknown_optab;
289 : }
290 : }
291 :
292 : /* Check whether an operation represented by CODE is a 'half' widening operation
293 : in which the input vector type has half the number of bits of the output
294 : vector type e.g. V8QI->V8HI.
295 :
296 : This is handled by widening the inputs using NOP_EXPRs then using a
297 : non-widening stmt e.g. MINUS_EXPR. RTL fusing converts these to the widening
298 : hardware instructions if supported.
299 :
300 : The more typical case (handled in supportable_widening_operation) is where
301 : the input vector type has the same number of bits as the output vector type.
302 : In this case half the elements of the input vectors must be processed at a
303 : time into respective vector outputs with elements twice as wide i.e. a
304 : 'hi'/'lo' pair using codes such as VEC_WIDEN_MINUS_HI/LO.
305 :
306 : Supported widening operations:
307 : WIDEN_MULT_EXPR
308 : WIDEN_LSHIFT_EXPR
309 :
310 : Output:
311 : - CODE1 - The non-widened code, which will be used after the inputs are
312 : converted to the wide type. */
313 : bool
314 243 : supportable_half_widening_operation (enum tree_code code, tree vectype_out,
315 : tree vectype_in, enum tree_code *code1)
316 : {
317 243 : machine_mode m1,m2;
318 243 : enum tree_code dummy_code;
319 243 : optab op;
320 :
321 243 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
322 :
323 243 : m1 = TYPE_MODE (vectype_out);
324 243 : m2 = TYPE_MODE (vectype_in);
325 :
326 243 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
327 : return false;
328 :
329 243 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in),
330 486 : TYPE_VECTOR_SUBPARTS (vectype_out)))
331 : return false;
332 :
333 243 : switch (code)
334 : {
335 0 : case WIDEN_LSHIFT_EXPR:
336 0 : *code1 = LSHIFT_EXPR;
337 0 : break;
338 243 : case WIDEN_MULT_EXPR:
339 243 : *code1 = MULT_EXPR;
340 243 : break;
341 : default:
342 : return false;
343 : }
344 :
345 243 : if (!supportable_convert_operation (NOP_EXPR, vectype_out, vectype_in,
346 : &dummy_code))
347 : return false;
348 :
349 209 : op = optab_for_tree_code (*code1, vectype_out, optab_vector);
350 209 : return (optab_handler (op, TYPE_MODE (vectype_out)) != CODE_FOR_nothing);
351 : }
352 :
353 : /* Function supportable_convert_operation
354 :
355 : Check whether an operation represented by the code CODE is a
356 : convert operation that is supported by the target platform in
357 : vector form (i.e., when operating on arguments of type VECTYPE_IN
358 : producing a result of type VECTYPE_OUT).
359 :
360 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
361 : This function checks if these operations are supported
362 : by the target platform directly (via vector tree-codes).
363 :
364 : Output:
365 : - CODE1 is code of vector operation to be used when
366 : vectorizing the operation, if available. */
367 :
368 : bool
369 87007 : supportable_convert_operation (enum tree_code code,
370 : tree vectype_out, tree vectype_in,
371 : enum tree_code *code1)
372 : {
373 87007 : machine_mode m1,m2;
374 87007 : bool truncp;
375 :
376 87007 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
377 :
378 87007 : m1 = TYPE_MODE (vectype_out);
379 87007 : m2 = TYPE_MODE (vectype_in);
380 :
381 87007 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
382 : return false;
383 :
384 : /* First check if we can done conversion directly. */
385 86840 : if ((code == FIX_TRUNC_EXPR
386 1406 : && can_fix_p (m1,m2,TYPE_UNSIGNED (vectype_out), &truncp)
387 : != CODE_FOR_nothing)
388 87202 : || (code == FLOAT_EXPR
389 9523 : && can_float_p (m1,m2,TYPE_UNSIGNED (vectype_in))
390 : != CODE_FOR_nothing))
391 : {
392 9992 : *code1 = code;
393 9992 : return true;
394 : }
395 :
396 153696 : if (GET_MODE_UNIT_PRECISION (m1) > GET_MODE_UNIT_PRECISION (m2)
397 76848 : && can_extend_p (m1, m2, TYPE_UNSIGNED (vectype_in)))
398 : {
399 6641 : *code1 = code;
400 6641 : return true;
401 : }
402 :
403 140414 : if (GET_MODE_UNIT_PRECISION (m1) < GET_MODE_UNIT_PRECISION (m2)
404 75791 : && convert_optab_handler (trunc_optab, m1, m2) != CODE_FOR_nothing)
405 : {
406 3430 : *code1 = code;
407 3430 : return true;
408 : }
409 :
410 : return false;
411 : }
412 :
413 : /* Return true iff vec_cmp_optab/vec_cmpu_optab can handle a vector comparison
414 : for code CODE, comparing operands of type VALUE_TYPE and producing a result
415 : of type MASK_TYPE. */
416 :
417 : static bool
418 1362582 : vec_cmp_icode_p (tree value_type, tree mask_type, enum tree_code code)
419 : {
420 1362582 : enum rtx_code rcode = get_rtx_code_1 (code, TYPE_UNSIGNED (value_type));
421 1362582 : if (rcode == UNKNOWN)
422 : return false;
423 :
424 1362582 : return can_vec_cmp_compare_p (rcode, TYPE_MODE (value_type),
425 2725164 : TYPE_MODE (mask_type));
426 : }
427 :
428 : /* Return true iff vec_cmpeq_optab can handle a vector comparison for code
429 : CODE, comparing operands of type VALUE_TYPE and producing a result of type
430 : MASK_TYPE. */
431 :
432 : static bool
433 370159 : vec_cmp_eq_icode_p (tree value_type, tree mask_type, enum tree_code code)
434 : {
435 370159 : if (code != EQ_EXPR && code != NE_EXPR)
436 : return false;
437 :
438 178808 : return get_vec_cmp_eq_icode (TYPE_MODE (value_type), TYPE_MODE (mask_type))
439 178808 : != CODE_FOR_nothing;
440 : }
441 :
442 : /* Return TRUE if appropriate vector insn is available
443 : for vector comparison expr with vector type VALUE_TYPE
444 : and resulting mask with MASK_TYPE. */
445 :
446 : bool
447 1362582 : expand_vec_cmp_expr_p (tree value_type, tree mask_type, enum tree_code code)
448 : {
449 1362582 : return vec_cmp_icode_p (value_type, mask_type, code)
450 1362582 : || vec_cmp_eq_icode_p (value_type, mask_type, code);
451 : }
452 :
453 : /* Return TRUE iff, appropriate vector insns are available
454 : for vector cond expr with vector type VALUE_TYPE and a comparison
455 : with operand vector types in CMP_OP_TYPE. */
456 :
457 : bool
458 107416 : expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
459 : {
460 107416 : if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)
461 322248 : && get_vcond_mask_icode (TYPE_MODE (value_type),
462 107416 : TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing)
463 : return true;
464 :
465 : return false;
466 : }
467 :
468 : /* Use the current target and options to initialize
469 : TREE_OPTIMIZATION_OPTABS (OPTNODE). */
470 :
471 : void
472 1655542 : init_tree_optimization_optabs (tree optnode)
473 : {
474 : /* Quick exit if we have already computed optabs for this target. */
475 1655542 : if (TREE_OPTIMIZATION_BASE_OPTABS (optnode) == this_target_optabs)
476 : return;
477 :
478 : /* Forget any previous information and set up for the current target. */
479 24460 : TREE_OPTIMIZATION_BASE_OPTABS (optnode) = this_target_optabs;
480 24460 : struct target_optabs *tmp_optabs = (struct target_optabs *)
481 24460 : TREE_OPTIMIZATION_OPTABS (optnode);
482 24460 : if (tmp_optabs)
483 436 : memset (tmp_optabs, 0, sizeof (struct target_optabs));
484 : else
485 24024 : tmp_optabs = ggc_cleared_alloc<target_optabs> ();
486 :
487 : /* Generate a new set of optabs into tmp_optabs. */
488 24460 : init_all_optabs (tmp_optabs);
489 :
490 : /* If the optabs changed, record it. */
491 24460 : if (memcmp (tmp_optabs, this_target_optabs, sizeof (struct target_optabs)))
492 10444 : TREE_OPTIMIZATION_OPTABS (optnode) = tmp_optabs;
493 : else
494 : {
495 14016 : TREE_OPTIMIZATION_OPTABS (optnode) = NULL;
496 14016 : ggc_free (tmp_optabs);
497 : }
498 : }
499 :
500 : /* Return TRUE if the target has support for vector right shift of an
501 : operand of type TYPE. If OT_TYPE is OPTAB_DEFAULT, check for existence
502 : of a shift by either a scalar or a vector. Otherwise, check only
503 : for a shift that matches OT_TYPE. */
504 :
505 : bool
506 297570 : target_supports_op_p (tree type, enum tree_code code,
507 : enum optab_subtype ot_subtype)
508 : {
509 297570 : optab ot = optab_for_tree_code (code, type, ot_subtype);
510 297570 : return ot != unknown_optab && can_implement_p (ot, TYPE_MODE (type));
511 : }
512 :
513 : /* Return true if the target has support for masked load/store.
514 : We can support masked load/store by either mask{load,store}
515 : or mask_len_{load,store}.
516 : This helper function checks whether target supports masked
517 : load/store and return corresponding IFN in the last argument
518 : (IFN_MASK_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
519 : If there is support and ELSVALS is nonzero store the possible else values
520 : in the vector it points to. */
521 :
522 : bool
523 296790 : target_supports_mask_load_store_p (machine_mode mode, machine_mode mask_mode,
524 : bool is_load, internal_fn *ifn,
525 : vec<int> *elsvals)
526 : {
527 296790 : optab op = is_load ? maskload_optab : maskstore_optab;
528 72902 : optab len_op = is_load ? mask_len_load_optab : mask_len_store_optab;
529 296790 : enum insn_code icode;
530 296790 : if ((icode = convert_optab_handler (op, mode, mask_mode))
531 : != CODE_FOR_nothing)
532 : {
533 96684 : if (ifn)
534 2153 : *ifn = is_load ? IFN_MASK_LOAD : IFN_MASK_STORE;
535 96684 : if (elsvals && is_load)
536 62020 : get_supported_else_vals (icode,
537 62020 : internal_fn_else_index (IFN_MASK_LOAD),
538 : *elsvals);
539 96684 : return true;
540 : }
541 200106 : else if ((icode = convert_optab_handler (len_op, mode, mask_mode))
542 : != CODE_FOR_nothing)
543 : {
544 0 : if (ifn)
545 0 : *ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
546 0 : if (elsvals && is_load)
547 0 : get_supported_else_vals (icode,
548 0 : internal_fn_else_index (IFN_MASK_LEN_LOAD),
549 : *elsvals);
550 0 : return true;
551 : }
552 : return false;
553 : }
554 :
555 : /* Return true if target supports vector masked load/store for mode.
556 : An additional output in the last argument which is the IFN pointer.
557 : We set IFN as MASK_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
558 : which optab is supported in the target.
559 : If there is support and ELSVALS is nonzero store the possible else values
560 : in the vector it points to. */
561 :
562 : bool
563 276155 : can_vec_mask_load_store_p (machine_mode mode,
564 : machine_mode mask_mode,
565 : bool is_load,
566 : internal_fn *ifn,
567 : vec<int> *elsvals)
568 : {
569 276155 : machine_mode vmode;
570 :
571 : /* If mode is vector mode, check it directly. */
572 276155 : if (VECTOR_MODE_P (mode))
573 264494 : return target_supports_mask_load_store_p (mode, mask_mode, is_load, ifn,
574 264494 : elsvals);
575 :
576 : /* Otherwise, return true if there is some vector mode with
577 : the mask load/store supported. */
578 :
579 : /* See if there is any chance the mask load or store might be
580 : vectorized. If not, punt. */
581 11661 : scalar_mode smode;
582 11661 : if (!is_a <scalar_mode> (mode, &smode))
583 : return false;
584 :
585 11661 : vmode = targetm.vectorize.preferred_simd_mode (smode);
586 2262 : if (VECTOR_MODE_P (vmode)
587 11483 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
588 23144 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
589 : elsvals))
590 2238 : return true;
591 :
592 9423 : auto_vector_modes vector_modes;
593 9423 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
594 55128 : for (machine_mode base_mode : vector_modes)
595 26859 : if (related_vector_mode (base_mode, smode).exists (&vmode)
596 47672 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
597 20813 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
598 : elsvals))
599 0 : return true;
600 : return false;
601 9423 : }
602 :
603 : /* Return true if the target has support for len load/store.
604 : We can support len load/store by either len_{load,store}
605 : or mask_len_{load,store}.
606 : This helper function checks whether target supports len
607 : load/store and return corresponding IFN in the last argument
608 : (IFN_LEN_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
609 : If there is support and ELSVALS is nonzero store thepossible
610 : else values in the vector it points to. */
611 :
612 : static bool
613 518806 : target_supports_len_load_store_p (machine_mode mode, bool is_load,
614 : internal_fn *ifn, vec<int> *elsvals)
615 : {
616 518806 : optab op = is_load ? len_load_optab : len_store_optab;
617 127172 : optab masked_op = is_load ? mask_len_load_optab : mask_len_store_optab;
618 518806 : internal_fn which_ifn;
619 :
620 518806 : enum insn_code icode;
621 518806 : if ((icode = direct_optab_handler (op, mode)) != CODE_FOR_nothing)
622 : {
623 0 : which_ifn = is_load ? IFN_LEN_LOAD : IFN_LEN_STORE;
624 : }
625 518806 : machine_mode mask_mode;
626 518806 : if (!icode
627 1037612 : && targetm.vectorize.get_mask_mode (mode).exists (&mask_mode)
628 518806 : && ((icode = convert_optab_handler (masked_op, mode, mask_mode))
629 : != CODE_FOR_nothing))
630 0 : which_ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
631 :
632 518806 : if (icode && elsvals && is_load)
633 0 : get_supported_else_vals (icode, internal_fn_else_index (which_ifn),
634 : *elsvals);
635 :
636 518806 : if (icode && ifn)
637 0 : *ifn = which_ifn;
638 518806 : return icode;
639 : }
640 :
641 : /* If target supports vector load/store with length for vector mode MODE,
642 : return the corresponding vector mode, otherwise return opt_machine_mode ().
643 : There are two flavors for vector load/store with length, one is to measure
644 : length with bytes, the other is to measure length with lanes.
645 : As len_{load,store} optabs point out, for the flavor with bytes, we use
646 : VnQI to wrap the other supportable same size vector modes.
647 : An additional output in the last argument which is the IFN pointer.
648 : We set IFN as LEN_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
649 : which optab is supported in the target.
650 : If there is support and ELSVALS is nonzero store the possible else values
651 : in the vector it points to. */
652 :
653 : opt_machine_mode
654 259404 : get_len_load_store_mode (machine_mode mode, bool is_load, internal_fn *ifn,
655 : vec<int> *elsvals)
656 : {
657 259404 : gcc_assert (VECTOR_MODE_P (mode));
658 :
659 : /* Check if length in lanes supported for this mode directly. */
660 259404 : if (target_supports_len_load_store_p (mode, is_load, ifn, elsvals))
661 0 : return mode;
662 :
663 : /* Check if length in bytes supported for same vector size VnQI. */
664 259404 : machine_mode vmode;
665 518808 : poly_uint64 nunits = GET_MODE_SIZE (mode);
666 259404 : if (related_vector_mode (mode, QImode, nunits).exists (&vmode)
667 259402 : && target_supports_len_load_store_p (vmode, is_load, ifn, elsvals))
668 0 : return vmode;
669 :
670 259404 : return opt_machine_mode ();
671 : }
|
