Line data Source code
1 : /* Tree-based target query functions relating to optabs
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "target.h"
25 : #include "insn-codes.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "optabs.h"
30 : #include "optabs-tree.h"
31 : #include "stor-layout.h"
32 : #include "internal-fn.h"
33 :
34 : /* Return the optab used for computing the operation given by the tree code,
35 : CODE and the tree EXP. This function is not always usable (for example, it
36 : cannot give complete results for multiplication or division) but probably
37 : ought to be relied on more widely throughout the expander. */
38 : optab
39 10662526 : optab_for_tree_code (enum tree_code code, const_tree type,
40 : enum optab_subtype subtype)
41 : {
42 10662526 : bool trapv;
43 10662526 : switch (code)
44 : {
45 : case BIT_AND_EXPR:
46 : return and_optab;
47 :
48 207006 : case BIT_IOR_EXPR:
49 207006 : return ior_optab;
50 :
51 6709 : case BIT_NOT_EXPR:
52 6709 : return one_cmpl_optab;
53 :
54 54916 : case BIT_XOR_EXPR:
55 54916 : return xor_optab;
56 :
57 5003 : case MULT_HIGHPART_EXPR:
58 5003 : return TYPE_UNSIGNED (type) ? umul_highpart_optab : smul_highpart_optab;
59 :
60 78 : case CEIL_MOD_EXPR:
61 78 : case FLOOR_MOD_EXPR:
62 78 : case ROUND_MOD_EXPR:
63 : /* {s,u}mod_optab implements TRUNC_MOD_EXPR. For scalar modes,
64 : expansion has code to adjust TRUNC_MOD_EXPR into the desired other
65 : modes, but for vector modes it does not. The adjustment code
66 : should be instead emitted in tree-vect-patterns.cc. */
67 78 : if (VECTOR_TYPE_P (type))
68 : return unknown_optab;
69 : /* FALLTHRU */
70 48177 : case TRUNC_MOD_EXPR:
71 48177 : return TYPE_UNSIGNED (type) ? umod_optab : smod_optab;
72 :
73 12 : case CEIL_DIV_EXPR:
74 12 : case FLOOR_DIV_EXPR:
75 12 : case ROUND_DIV_EXPR:
76 : /* {,u}{s,u}div_optab implements {TRUNC,EXACT}_DIV_EXPR or RDIV_EXPR.
77 : For scalar modes, expansion has code to adjust TRUNC_DIV_EXPR
78 : into the desired other modes, but for vector modes it does not.
79 : The adjustment code should be instead emitted in
80 : tree-vect-patterns.cc. */
81 12 : if (VECTOR_TYPE_P (type))
82 : return unknown_optab;
83 : /* FALLTHRU */
84 36566 : case RDIV_EXPR:
85 36566 : gcc_assert (FLOAT_TYPE_P (type)
86 : || ALL_FIXED_POINT_MODE_P (TYPE_MODE (type)));
87 : /* FALLTHRU */
88 92325 : case TRUNC_DIV_EXPR:
89 92325 : case EXACT_DIV_EXPR:
90 92325 : if (TYPE_SATURATING (type))
91 0 : return TYPE_UNSIGNED (type) ? usdiv_optab : ssdiv_optab;
92 92325 : return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab;
93 :
94 756667 : case LSHIFT_EXPR:
95 756667 : if (VECTOR_TYPE_P (type))
96 : {
97 756653 : if (subtype == optab_vector)
98 412825 : return TYPE_SATURATING (type) ? unknown_optab : vashl_optab;
99 :
100 343828 : gcc_assert (subtype == optab_scalar);
101 : }
102 343842 : if (TYPE_SATURATING (type))
103 0 : return TYPE_UNSIGNED (type) ? usashl_optab : ssashl_optab;
104 : return ashl_optab;
105 :
106 200466 : case RSHIFT_EXPR:
107 200466 : if (VECTOR_TYPE_P (type))
108 : {
109 200445 : if (subtype == optab_vector)
110 141378 : return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab;
111 :
112 59067 : gcc_assert (subtype == optab_scalar);
113 : }
114 59088 : return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab;
115 :
116 1249 : case LROTATE_EXPR:
117 1249 : if (VECTOR_TYPE_P (type))
118 : {
119 1249 : if (subtype == optab_vector)
120 : return vrotl_optab;
121 :
122 475 : gcc_assert (subtype == optab_scalar);
123 : }
124 : return rotl_optab;
125 :
126 15838 : case RROTATE_EXPR:
127 15838 : if (VECTOR_TYPE_P (type))
128 : {
129 15838 : if (subtype == optab_vector)
130 : return vrotr_optab;
131 :
132 7830 : gcc_assert (subtype == optab_scalar);
133 : }
134 : return rotr_optab;
135 :
136 107273 : case MAX_EXPR:
137 107273 : return TYPE_UNSIGNED (type) ? umax_optab : smax_optab;
138 :
139 54689 : case MIN_EXPR:
140 54689 : return TYPE_UNSIGNED (type) ? umin_optab : smin_optab;
141 :
142 : case POINTER_PLUS_EXPR:
143 : return add_optab;
144 :
145 : case POINTER_DIFF_EXPR:
146 : return sub_optab;
147 :
148 0 : case REALIGN_LOAD_EXPR:
149 0 : return vec_realign_load_optab;
150 :
151 3931 : case WIDEN_SUM_EXPR:
152 3931 : return (TYPE_UNSIGNED (type)
153 3931 : ? reduc_widen_usum_optab : reduc_widen_ssum_optab);
154 :
155 1925 : case DOT_PROD_EXPR:
156 1925 : {
157 1925 : if (subtype == optab_vector_mixed_sign)
158 : return usdot_prod_optab;
159 :
160 1388 : return (TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab);
161 : }
162 :
163 973 : case SAD_EXPR:
164 973 : return TYPE_UNSIGNED (type) ? usad_optab : ssad_optab;
165 :
166 30878 : case WIDEN_MULT_PLUS_EXPR:
167 30878 : return (TYPE_UNSIGNED (type)
168 30878 : ? (TYPE_SATURATING (type)
169 20908 : ? usmadd_widen_optab : umadd_widen_optab)
170 9970 : : (TYPE_SATURATING (type)
171 9970 : ? ssmadd_widen_optab : smadd_widen_optab));
172 :
173 556 : case WIDEN_MULT_MINUS_EXPR:
174 556 : return (TYPE_UNSIGNED (type)
175 556 : ? (TYPE_SATURATING (type)
176 444 : ? usmsub_widen_optab : umsub_widen_optab)
177 112 : : (TYPE_SATURATING (type)
178 112 : ? ssmsub_widen_optab : smsub_widen_optab));
179 :
180 102636 : case VEC_WIDEN_MULT_HI_EXPR:
181 102636 : return (TYPE_UNSIGNED (type)
182 102636 : ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab);
183 :
184 102629 : case VEC_WIDEN_MULT_LO_EXPR:
185 102629 : return (TYPE_UNSIGNED (type)
186 102629 : ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab);
187 :
188 207410 : case VEC_WIDEN_MULT_EVEN_EXPR:
189 207410 : return (TYPE_UNSIGNED (type)
190 207410 : ? vec_widen_umult_even_optab : vec_widen_smult_even_optab);
191 :
192 207410 : case VEC_WIDEN_MULT_ODD_EXPR:
193 207410 : return (TYPE_UNSIGNED (type)
194 207410 : ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab);
195 :
196 9852 : case VEC_WIDEN_LSHIFT_HI_EXPR:
197 9852 : return (TYPE_UNSIGNED (type)
198 9852 : ? vec_widen_ushiftl_hi_optab : vec_widen_sshiftl_hi_optab);
199 :
200 9852 : case VEC_WIDEN_LSHIFT_LO_EXPR:
201 9852 : return (TYPE_UNSIGNED (type)
202 9852 : ? vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab);
203 :
204 66338 : case VEC_UNPACK_HI_EXPR:
205 66338 : return (TYPE_UNSIGNED (type)
206 66338 : ? vec_unpacku_hi_optab : vec_unpacks_hi_optab);
207 :
208 66335 : case VEC_UNPACK_LO_EXPR:
209 66335 : return (TYPE_UNSIGNED (type)
210 66335 : ? vec_unpacku_lo_optab : vec_unpacks_lo_optab);
211 :
212 10275 : case VEC_UNPACK_FLOAT_HI_EXPR:
213 : /* The signedness is determined from input operand. */
214 10275 : return (TYPE_UNSIGNED (type)
215 10275 : ? vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab);
216 :
217 10281 : case VEC_UNPACK_FLOAT_LO_EXPR:
218 : /* The signedness is determined from input operand. */
219 10281 : return (TYPE_UNSIGNED (type)
220 10281 : ? vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab);
221 :
222 271 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
223 : /* The signedness is determined from output operand. */
224 271 : return (TYPE_UNSIGNED (type)
225 271 : ? vec_unpack_ufix_trunc_hi_optab
226 : : vec_unpack_sfix_trunc_hi_optab);
227 :
228 271 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
229 : /* The signedness is determined from output operand. */
230 271 : return (TYPE_UNSIGNED (type)
231 271 : ? vec_unpack_ufix_trunc_lo_optab
232 : : vec_unpack_sfix_trunc_lo_optab);
233 :
234 68860 : case VEC_PACK_TRUNC_EXPR:
235 68860 : return vec_pack_trunc_optab;
236 :
237 0 : case VEC_PACK_SAT_EXPR:
238 0 : return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab;
239 :
240 1021 : case VEC_PACK_FIX_TRUNC_EXPR:
241 : /* The signedness is determined from output operand. */
242 1021 : return (TYPE_UNSIGNED (type)
243 1021 : ? vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab);
244 :
245 975 : case VEC_PACK_FLOAT_EXPR:
246 : /* The signedness is determined from input operand. */
247 975 : return (TYPE_UNSIGNED (type)
248 975 : ? vec_packu_float_optab : vec_packs_float_optab);
249 :
250 0 : case VEC_DUPLICATE_EXPR:
251 0 : return vec_duplicate_optab;
252 :
253 0 : case VEC_SERIES_EXPR:
254 0 : return vec_series_optab;
255 :
256 6759914 : default:
257 6759914 : break;
258 : }
259 :
260 6759914 : trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type);
261 6759914 : switch (code)
262 : {
263 4602028 : case PLUS_EXPR:
264 4602028 : if (TYPE_SATURATING (type))
265 0 : return TYPE_UNSIGNED (type) ? usadd_optab : ssadd_optab;
266 4602028 : return trapv ? addv_optab : add_optab;
267 :
268 972712 : case MINUS_EXPR:
269 972712 : if (TYPE_SATURATING (type))
270 0 : return TYPE_UNSIGNED (type) ? ussub_optab : sssub_optab;
271 972712 : return trapv ? subv_optab : sub_optab;
272 :
273 994202 : case MULT_EXPR:
274 994202 : if (TYPE_SATURATING (type))
275 0 : return TYPE_UNSIGNED (type) ? usmul_optab : ssmul_optab;
276 994202 : return trapv ? smulv_optab : smul_optab;
277 :
278 78662 : case NEGATE_EXPR:
279 78662 : if (TYPE_SATURATING (type))
280 0 : return TYPE_UNSIGNED (type) ? usneg_optab : ssneg_optab;
281 78662 : return trapv ? negv_optab : neg_optab;
282 :
283 3968 : case ABS_EXPR:
284 3968 : return trapv ? absv_optab : abs_optab;
285 :
286 : case ABSU_EXPR:
287 : return abs_optab;
288 : default:
289 : return unknown_optab;
290 : }
291 : }
292 :
293 : /* Check whether an operation represented by CODE is a 'half' widening operation
294 : in which the input vector type has half the number of bits of the output
295 : vector type e.g. V8QI->V8HI.
296 :
297 : This is handled by widening the inputs using NOP_EXPRs then using a
298 : non-widening stmt e.g. MINUS_EXPR. RTL fusing converts these to the widening
299 : hardware instructions if supported.
300 :
301 : The more typical case (handled in supportable_widening_operation) is where
302 : the input vector type has the same number of bits as the output vector type.
303 : In this case half the elements of the input vectors must be processed at a
304 : time into respective vector outputs with elements twice as wide i.e. a
305 : 'hi'/'lo' pair using codes such as VEC_WIDEN_MINUS_HI/LO.
306 :
307 : Supported widening operations:
308 : WIDEN_MULT_EXPR
309 : WIDEN_LSHIFT_EXPR
310 :
311 : Output:
312 : - CODE1 - The non-widened code, which will be used after the inputs are
313 : converted to the wide type. */
314 : bool
315 359 : supportable_half_widening_operation (enum tree_code code, tree vectype_out,
316 : tree vectype_in, enum tree_code *code1)
317 : {
318 359 : machine_mode m1,m2;
319 359 : optab op;
320 :
321 359 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
322 :
323 359 : m1 = TYPE_MODE (vectype_out);
324 359 : m2 = TYPE_MODE (vectype_in);
325 :
326 359 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
327 : return false;
328 :
329 359 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in),
330 718 : TYPE_VECTOR_SUBPARTS (vectype_out)))
331 : return false;
332 :
333 359 : switch (code)
334 : {
335 0 : case WIDEN_LSHIFT_EXPR:
336 0 : *code1 = LSHIFT_EXPR;
337 0 : break;
338 359 : case WIDEN_MULT_EXPR:
339 359 : *code1 = MULT_EXPR;
340 359 : break;
341 : default:
342 : return false;
343 : }
344 :
345 359 : if (!supportable_convert_operation (NOP_EXPR, vectype_out, vectype_in))
346 : return false;
347 :
348 285 : op = optab_for_tree_code (*code1, vectype_out, optab_vector);
349 285 : return (optab_handler (op, TYPE_MODE (vectype_out)) != CODE_FOR_nothing);
350 : }
351 :
352 : /* Function supportable_convert_operation
353 :
354 : Check whether an operation represented by the code CODE is a
355 : convert operation that is supported by the target platform in
356 : vector form (i.e., when operating on arguments of type VECTYPE_IN
357 : producing a result of type VECTYPE_OUT).
358 :
359 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
360 : This function checks if these operations are supported
361 : by the target platform directly (via vector tree-codes). */
362 :
363 : bool
364 97975 : supportable_convert_operation (enum tree_code code,
365 : tree vectype_out, tree vectype_in)
366 : {
367 97975 : machine_mode m1,m2;
368 97975 : bool truncp;
369 :
370 97975 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
371 :
372 97975 : m1 = TYPE_MODE (vectype_out);
373 97975 : m2 = TYPE_MODE (vectype_in);
374 :
375 97975 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
376 : return false;
377 :
378 97795 : if (m1 == m2
379 64756 : && (CONVERT_EXPR_CODE_P (code) || code == VIEW_CONVERT_EXPR))
380 : return true;
381 :
382 : /* First check if we can done conversion directly. */
383 33039 : if ((code == FIX_TRUNC_EXPR
384 1560 : && can_fix_p (m1,m2,TYPE_UNSIGNED (vectype_out), &truncp)
385 : != CODE_FOR_nothing)
386 33475 : || (code == FLOAT_EXPR
387 14175 : && can_float_p (m1,m2,TYPE_UNSIGNED (vectype_in))
388 : != CODE_FOR_nothing))
389 : return true;
390 :
391 42824 : if (GET_MODE_UNIT_PRECISION (m1) > GET_MODE_UNIT_PRECISION (m2)
392 21412 : && can_extend_p (m1, m2, TYPE_UNSIGNED (vectype_in)))
393 : return true;
394 :
395 27446 : if (GET_MODE_UNIT_PRECISION (m1) < GET_MODE_UNIT_PRECISION (m2)
396 20502 : && convert_optab_handler (trunc_optab, m1, m2) != CODE_FOR_nothing)
397 4191 : return true;
398 :
399 : return false;
400 : }
401 :
402 : /* Return true iff vec_cmp_optab/vec_cmpu_optab can handle a vector comparison
403 : for code CODE, comparing operands of type VALUE_TYPE and producing a result
404 : of type MASK_TYPE. */
405 :
406 : static bool
407 1467936 : vec_cmp_icode_p (tree value_type, tree mask_type, enum tree_code code)
408 : {
409 1467936 : enum rtx_code rcode = get_rtx_code_1 (code, TYPE_UNSIGNED (value_type));
410 1467936 : if (rcode == UNKNOWN)
411 : return false;
412 :
413 1467936 : return can_vec_cmp_compare_p (rcode, TYPE_MODE (value_type),
414 2935872 : TYPE_MODE (mask_type));
415 : }
416 :
417 : /* Return true iff vec_cmpeq_optab can handle a vector comparison for code
418 : CODE, comparing operands of type VALUE_TYPE and producing a result of type
419 : MASK_TYPE. */
420 :
421 : static bool
422 419579 : vec_cmp_eq_icode_p (tree value_type, tree mask_type, enum tree_code code)
423 : {
424 419579 : if (code != EQ_EXPR && code != NE_EXPR)
425 : return false;
426 :
427 216036 : return get_vec_cmp_eq_icode (TYPE_MODE (value_type), TYPE_MODE (mask_type))
428 216036 : != CODE_FOR_nothing;
429 : }
430 :
431 : /* Return TRUE if appropriate vector insn is available
432 : for vector comparison expr with vector type VALUE_TYPE
433 : and resulting mask with MASK_TYPE. */
434 :
435 : bool
436 1467936 : expand_vec_cmp_expr_p (tree value_type, tree mask_type, enum tree_code code)
437 : {
438 1467936 : return vec_cmp_icode_p (value_type, mask_type, code)
439 1467936 : || vec_cmp_eq_icode_p (value_type, mask_type, code);
440 : }
441 :
442 : /* Return TRUE iff, appropriate vector insns are available
443 : for vector cond expr with vector type VALUE_TYPE and a comparison
444 : with operand vector types in CMP_OP_TYPE. */
445 :
446 : bool
447 115684 : expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
448 : {
449 115684 : if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)
450 347052 : && get_vcond_mask_icode (TYPE_MODE (value_type),
451 115684 : TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing)
452 112950 : return true;
453 :
454 : return false;
455 : }
456 :
457 : /* Use the current target and options to initialize
458 : TREE_OPTIMIZATION_OPTABS (OPTNODE). */
459 :
460 : void
461 1660643 : init_tree_optimization_optabs (tree optnode)
462 : {
463 : /* Quick exit if we have already computed optabs for this target. */
464 1660643 : if (TREE_OPTIMIZATION_BASE_OPTABS (optnode) == this_target_optabs)
465 : return;
466 :
467 : /* Forget any previous information and set up for the current target. */
468 24783 : TREE_OPTIMIZATION_BASE_OPTABS (optnode) = this_target_optabs;
469 24783 : struct target_optabs *tmp_optabs = (struct target_optabs *)
470 24783 : TREE_OPTIMIZATION_OPTABS (optnode);
471 24783 : if (tmp_optabs)
472 436 : memset (tmp_optabs, 0, sizeof (struct target_optabs));
473 : else
474 24347 : tmp_optabs = ggc_cleared_alloc<target_optabs> ();
475 :
476 : /* Generate a new set of optabs into tmp_optabs. */
477 24783 : init_all_optabs (tmp_optabs);
478 :
479 : /* If the optabs changed, record it. */
480 24783 : if (memcmp (tmp_optabs, this_target_optabs, sizeof (struct target_optabs)))
481 10512 : TREE_OPTIMIZATION_OPTABS (optnode) = tmp_optabs;
482 : else
483 : {
484 14271 : TREE_OPTIMIZATION_OPTABS (optnode) = NULL;
485 14271 : ggc_free (tmp_optabs);
486 : }
487 : }
488 :
489 : /* Return TRUE if the target has support for vector right shift of an
490 : operand of type TYPE. If OT_TYPE is OPTAB_DEFAULT, check for existence
491 : of a shift by either a scalar or a vector. Otherwise, check only
492 : for a shift that matches OT_TYPE. */
493 :
494 : bool
495 323091 : target_supports_op_p (tree type, enum tree_code code,
496 : enum optab_subtype ot_subtype)
497 : {
498 323091 : optab ot = optab_for_tree_code (code, type, ot_subtype);
499 323091 : return ot != unknown_optab && can_implement_p (ot, TYPE_MODE (type));
500 : }
501 :
502 : /* Return true if the target has support for masked load/store.
503 : We can support masked load/store by either mask{load,store}
504 : or mask_len_{load,store}.
505 : This helper function checks whether target supports masked
506 : load/store and return corresponding IFN in the last argument
507 : (IFN_MASK_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
508 : If there is support and ELSVALS is nonzero store the possible else values
509 : in the vector it points to. */
510 :
511 : bool
512 307611 : target_supports_mask_load_store_p (machine_mode mode, machine_mode mask_mode,
513 : bool is_load, internal_fn *ifn,
514 : vec<int> *elsvals)
515 : {
516 307611 : optab op = is_load ? maskload_optab : maskstore_optab;
517 74469 : optab len_op = is_load ? mask_len_load_optab : mask_len_store_optab;
518 307611 : enum insn_code icode;
519 307611 : if ((icode = convert_optab_handler (op, mode, mask_mode))
520 : != CODE_FOR_nothing)
521 : {
522 97571 : if (ifn)
523 2024 : *ifn = is_load ? IFN_MASK_LOAD : IFN_MASK_STORE;
524 97571 : if (elsvals && is_load)
525 62955 : get_supported_else_vals (icode,
526 62955 : internal_fn_else_index (IFN_MASK_LOAD),
527 : *elsvals);
528 : return true;
529 : }
530 210040 : else if ((icode = convert_optab_handler (len_op, mode, mask_mode))
531 : != CODE_FOR_nothing)
532 : {
533 0 : if (ifn)
534 0 : *ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
535 0 : if (elsvals && is_load)
536 0 : get_supported_else_vals (icode,
537 0 : internal_fn_else_index (IFN_MASK_LEN_LOAD),
538 : *elsvals);
539 : return true;
540 : }
541 : return false;
542 : }
543 :
544 : /* Return true if target supports vector masked load/store for mode.
545 : An additional output in the last argument which is the IFN pointer.
546 : We set IFN as MASK_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
547 : which optab is supported in the target.
548 : If there is support and ELSVALS is nonzero store the possible else values
549 : in the vector it points to. */
550 :
551 : bool
552 286284 : can_vec_mask_load_store_p (machine_mode mode,
553 : machine_mode mask_mode,
554 : bool is_load,
555 : internal_fn *ifn,
556 : vec<int> *elsvals)
557 : {
558 286284 : machine_mode vmode;
559 :
560 : /* If mode is vector mode, check it directly. */
561 286284 : if (VECTOR_MODE_P (mode))
562 274372 : return target_supports_mask_load_store_p (mode, mask_mode, is_load, ifn,
563 274372 : elsvals);
564 :
565 : /* Otherwise, return true if there is some vector mode with
566 : the mask load/store supported. */
567 :
568 : /* See if there is any chance the mask load or store might be
569 : vectorized. If not, punt. */
570 11912 : scalar_mode smode;
571 11912 : if (!is_a <scalar_mode> (mode, &smode))
572 : return false;
573 :
574 11912 : vmode = targetm.vectorize.preferred_simd_mode (smode);
575 2231 : if (VECTOR_MODE_P (vmode)
576 11734 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
577 23646 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
578 : elsvals))
579 2154 : return true;
580 :
581 9758 : auto_vector_modes vector_modes;
582 9758 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
583 57141 : for (machine_mode base_mode : vector_modes)
584 27867 : if (related_vector_mode (base_mode, smode).exists (&vmode)
585 49372 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
586 21505 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
587 : elsvals))
588 0 : return true;
589 : return false;
590 9758 : }
591 :
592 : /* Return true if the target has support for len load/store.
593 : We can support len load/store by either len_{load,store}
594 : or mask_len_{load,store}.
595 : This helper function checks whether target supports len
596 : load/store and return corresponding IFN in the last argument
597 : (IFN_LEN_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
598 : If there is support and ELSVALS is nonzero store thepossible
599 : else values in the vector it points to. */
600 :
601 : static bool
602 538860 : target_supports_len_load_store_p (machine_mode mode, bool is_load,
603 : internal_fn *ifn, vec<int> *elsvals)
604 : {
605 538860 : optab op = is_load ? len_load_optab : len_store_optab;
606 130122 : optab masked_op = is_load ? mask_len_load_optab : mask_len_store_optab;
607 538860 : internal_fn which_ifn;
608 :
609 538860 : enum insn_code icode;
610 538860 : if ((icode = direct_optab_handler (op, mode)) != CODE_FOR_nothing)
611 : {
612 0 : which_ifn = is_load ? IFN_LEN_LOAD : IFN_LEN_STORE;
613 : }
614 : machine_mode mask_mode;
615 : if (!icode
616 1077720 : && targetm.vectorize.get_mask_mode (mode).exists (&mask_mode)
617 538860 : && ((icode = convert_optab_handler (masked_op, mode, mask_mode))
618 : != CODE_FOR_nothing))
619 0 : which_ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
620 :
621 538860 : if (icode && elsvals && is_load)
622 0 : get_supported_else_vals (icode, internal_fn_else_index (which_ifn),
623 : *elsvals);
624 :
625 538860 : if (icode && ifn)
626 0 : *ifn = which_ifn;
627 538860 : return icode;
628 : }
629 :
630 : /* If target supports vector load/store with length for vector mode MODE,
631 : return the corresponding vector mode, otherwise return opt_machine_mode ().
632 : There are two flavors for vector load/store with length, one is to measure
633 : length with bytes, the other is to measure length with lanes.
634 : As len_{load,store} optabs point out, for the flavor with bytes, we use
635 : VnQI to wrap the other supportable same size vector modes.
636 : An additional output in the last argument which is the IFN pointer.
637 : We set IFN as LEN_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
638 : which optab is supported in the target.
639 : If there is support and ELSVALS is nonzero store the possible else values
640 : in the vector it points to. */
641 :
642 : opt_machine_mode
643 269431 : get_len_load_store_mode (machine_mode mode, bool is_load, internal_fn *ifn,
644 : vec<int> *elsvals)
645 : {
646 269431 : gcc_assert (VECTOR_MODE_P (mode));
647 :
648 : /* Check if length in lanes supported for this mode directly. */
649 269431 : if (target_supports_len_load_store_p (mode, is_load, ifn, elsvals))
650 0 : return mode;
651 :
652 : /* Check if length in bytes supported for same vector size VnQI. */
653 269431 : machine_mode vmode;
654 538862 : poly_uint64 nunits = GET_MODE_SIZE (mode);
655 269431 : if (related_vector_mode (mode, QImode, nunits).exists (&vmode)
656 269429 : && target_supports_len_load_store_p (vmode, is_load, ifn, elsvals))
657 0 : return vmode;
658 :
659 269431 : return opt_machine_mode ();
660 : }
|
