Line data Source code
1 : /* Tree-based target query functions relating to optabs
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "target.h"
25 : #include "insn-codes.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "optabs.h"
30 : #include "optabs-tree.h"
31 : #include "stor-layout.h"
32 : #include "internal-fn.h"
33 :
34 : /* Return the optab used for computing the operation given by the tree code,
35 : CODE and the tree EXP. This function is not always usable (for example, it
36 : cannot give complete results for multiplication or division) but probably
37 : ought to be relied on more widely throughout the expander. */
38 : optab
39 10492679 : optab_for_tree_code (enum tree_code code, const_tree type,
40 : enum optab_subtype subtype)
41 : {
42 10492679 : bool trapv;
43 10492679 : switch (code)
44 : {
45 : case BIT_AND_EXPR:
46 : return and_optab;
47 :
48 233880 : case BIT_IOR_EXPR:
49 233880 : return ior_optab;
50 :
51 6830 : case BIT_NOT_EXPR:
52 6830 : return one_cmpl_optab;
53 :
54 53115 : case BIT_XOR_EXPR:
55 53115 : return xor_optab;
56 :
57 2904 : case MULT_HIGHPART_EXPR:
58 2904 : return TYPE_UNSIGNED (type) ? umul_highpart_optab : smul_highpart_optab;
59 :
60 78 : case CEIL_MOD_EXPR:
61 78 : case FLOOR_MOD_EXPR:
62 78 : case ROUND_MOD_EXPR:
63 : /* {s,u}mod_optab implements TRUNC_MOD_EXPR. For scalar modes,
64 : expansion has code to adjust TRUNC_MOD_EXPR into the desired other
65 : modes, but for vector modes it does not. The adjustment code
66 : should be instead emitted in tree-vect-patterns.cc. */
67 78 : if (VECTOR_TYPE_P (type))
68 : return unknown_optab;
69 : /* FALLTHRU */
70 47908 : case TRUNC_MOD_EXPR:
71 47908 : return TYPE_UNSIGNED (type) ? umod_optab : smod_optab;
72 :
73 12 : case CEIL_DIV_EXPR:
74 12 : case FLOOR_DIV_EXPR:
75 12 : case ROUND_DIV_EXPR:
76 : /* {,u}{s,u}div_optab implements {TRUNC,EXACT}_DIV_EXPR or RDIV_EXPR.
77 : For scalar modes, expansion has code to adjust TRUNC_DIV_EXPR
78 : into the desired other modes, but for vector modes it does not.
79 : The adjustment code should be instead emitted in
80 : tree-vect-patterns.cc. */
81 12 : if (VECTOR_TYPE_P (type))
82 : return unknown_optab;
83 : /* FALLTHRU */
84 35613 : case RDIV_EXPR:
85 35613 : gcc_assert (FLOAT_TYPE_P (type)
86 : || ALL_FIXED_POINT_MODE_P (TYPE_MODE (type)));
87 : /* FALLTHRU */
88 91698 : case TRUNC_DIV_EXPR:
89 91698 : case EXACT_DIV_EXPR:
90 91698 : if (TYPE_SATURATING (type))
91 0 : return TYPE_UNSIGNED (type) ? usdiv_optab : ssdiv_optab;
92 91698 : return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab;
93 :
94 707856 : case LSHIFT_EXPR:
95 707856 : if (VECTOR_TYPE_P (type))
96 : {
97 707816 : if (subtype == optab_vector)
98 375211 : return TYPE_SATURATING (type) ? unknown_optab : vashl_optab;
99 :
100 332605 : gcc_assert (subtype == optab_scalar);
101 : }
102 332645 : if (TYPE_SATURATING (type))
103 0 : return TYPE_UNSIGNED (type) ? usashl_optab : ssashl_optab;
104 : return ashl_optab;
105 :
106 172587 : case RSHIFT_EXPR:
107 172587 : if (VECTOR_TYPE_P (type))
108 : {
109 172550 : if (subtype == optab_vector)
110 117446 : return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab;
111 :
112 55104 : gcc_assert (subtype == optab_scalar);
113 : }
114 55141 : return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab;
115 :
116 1247 : case LROTATE_EXPR:
117 1247 : if (VECTOR_TYPE_P (type))
118 : {
119 1247 : if (subtype == optab_vector)
120 : return vrotl_optab;
121 :
122 474 : gcc_assert (subtype == optab_scalar);
123 : }
124 : return rotl_optab;
125 :
126 15727 : case RROTATE_EXPR:
127 15727 : if (VECTOR_TYPE_P (type))
128 : {
129 15727 : if (subtype == optab_vector)
130 : return vrotr_optab;
131 :
132 7778 : gcc_assert (subtype == optab_scalar);
133 : }
134 : return rotr_optab;
135 :
136 106837 : case MAX_EXPR:
137 106837 : return TYPE_UNSIGNED (type) ? umax_optab : smax_optab;
138 :
139 56419 : case MIN_EXPR:
140 56419 : return TYPE_UNSIGNED (type) ? umin_optab : smin_optab;
141 :
142 : case POINTER_PLUS_EXPR:
143 : return add_optab;
144 :
145 : case POINTER_DIFF_EXPR:
146 : return sub_optab;
147 :
148 0 : case REALIGN_LOAD_EXPR:
149 0 : return vec_realign_load_optab;
150 :
151 3868 : case WIDEN_SUM_EXPR:
152 3868 : return (TYPE_UNSIGNED (type)
153 3868 : ? reduc_widen_usum_optab : reduc_widen_ssum_optab);
154 :
155 1889 : case DOT_PROD_EXPR:
156 1889 : {
157 1889 : if (subtype == optab_vector_mixed_sign)
158 : return usdot_prod_optab;
159 :
160 1367 : return (TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab);
161 : }
162 :
163 919 : case SAD_EXPR:
164 919 : return TYPE_UNSIGNED (type) ? usad_optab : ssad_optab;
165 :
166 30752 : case WIDEN_MULT_PLUS_EXPR:
167 30752 : return (TYPE_UNSIGNED (type)
168 30752 : ? (TYPE_SATURATING (type)
169 20994 : ? usmadd_widen_optab : umadd_widen_optab)
170 9758 : : (TYPE_SATURATING (type)
171 9758 : ? ssmadd_widen_optab : smadd_widen_optab));
172 :
173 599 : case WIDEN_MULT_MINUS_EXPR:
174 599 : return (TYPE_UNSIGNED (type)
175 599 : ? (TYPE_SATURATING (type)
176 463 : ? usmsub_widen_optab : umsub_widen_optab)
177 136 : : (TYPE_SATURATING (type)
178 136 : ? ssmsub_widen_optab : smsub_widen_optab));
179 :
180 98132 : case VEC_WIDEN_MULT_HI_EXPR:
181 98132 : return (TYPE_UNSIGNED (type)
182 98132 : ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab);
183 :
184 98125 : case VEC_WIDEN_MULT_LO_EXPR:
185 98125 : return (TYPE_UNSIGNED (type)
186 98125 : ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab);
187 :
188 200557 : case VEC_WIDEN_MULT_EVEN_EXPR:
189 200557 : return (TYPE_UNSIGNED (type)
190 200557 : ? vec_widen_umult_even_optab : vec_widen_smult_even_optab);
191 :
192 200557 : case VEC_WIDEN_MULT_ODD_EXPR:
193 200557 : return (TYPE_UNSIGNED (type)
194 200557 : ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab);
195 :
196 9740 : case VEC_WIDEN_LSHIFT_HI_EXPR:
197 9740 : return (TYPE_UNSIGNED (type)
198 9740 : ? vec_widen_ushiftl_hi_optab : vec_widen_sshiftl_hi_optab);
199 :
200 9740 : case VEC_WIDEN_LSHIFT_LO_EXPR:
201 9740 : return (TYPE_UNSIGNED (type)
202 9740 : ? vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab);
203 :
204 67256 : case VEC_UNPACK_HI_EXPR:
205 67256 : return (TYPE_UNSIGNED (type)
206 67256 : ? vec_unpacku_hi_optab : vec_unpacks_hi_optab);
207 :
208 67248 : case VEC_UNPACK_LO_EXPR:
209 67248 : return (TYPE_UNSIGNED (type)
210 67248 : ? vec_unpacku_lo_optab : vec_unpacks_lo_optab);
211 :
212 10177 : case VEC_UNPACK_FLOAT_HI_EXPR:
213 : /* The signedness is determined from input operand. */
214 10177 : return (TYPE_UNSIGNED (type)
215 10177 : ? vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab);
216 :
217 10183 : case VEC_UNPACK_FLOAT_LO_EXPR:
218 : /* The signedness is determined from input operand. */
219 10183 : return (TYPE_UNSIGNED (type)
220 10183 : ? vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab);
221 :
222 271 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
223 : /* The signedness is determined from output operand. */
224 271 : return (TYPE_UNSIGNED (type)
225 271 : ? vec_unpack_ufix_trunc_hi_optab
226 : : vec_unpack_sfix_trunc_hi_optab);
227 :
228 271 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
229 : /* The signedness is determined from output operand. */
230 271 : return (TYPE_UNSIGNED (type)
231 271 : ? vec_unpack_ufix_trunc_lo_optab
232 : : vec_unpack_sfix_trunc_lo_optab);
233 :
234 67047 : case VEC_PACK_TRUNC_EXPR:
235 67047 : return vec_pack_trunc_optab;
236 :
237 0 : case VEC_PACK_SAT_EXPR:
238 0 : return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab;
239 :
240 1018 : case VEC_PACK_FIX_TRUNC_EXPR:
241 : /* The signedness is determined from output operand. */
242 1018 : return (TYPE_UNSIGNED (type)
243 1018 : ? vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab);
244 :
245 975 : case VEC_PACK_FLOAT_EXPR:
246 : /* The signedness is determined from input operand. */
247 975 : return (TYPE_UNSIGNED (type)
248 975 : ? vec_packu_float_optab : vec_packs_float_optab);
249 :
250 0 : case VEC_DUPLICATE_EXPR:
251 0 : return vec_duplicate_optab;
252 :
253 0 : case VEC_SERIES_EXPR:
254 0 : return vec_series_optab;
255 :
256 6679654 : default:
257 6679654 : break;
258 : }
259 :
260 6679654 : trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type);
261 6679654 : switch (code)
262 : {
263 4557777 : case PLUS_EXPR:
264 4557777 : if (TYPE_SATURATING (type))
265 0 : return TYPE_UNSIGNED (type) ? usadd_optab : ssadd_optab;
266 4557777 : return trapv ? addv_optab : add_optab;
267 :
268 965215 : case MINUS_EXPR:
269 965215 : if (TYPE_SATURATING (type))
270 0 : return TYPE_UNSIGNED (type) ? ussub_optab : sssub_optab;
271 965215 : return trapv ? subv_optab : sub_optab;
272 :
273 966811 : case MULT_EXPR:
274 966811 : if (TYPE_SATURATING (type))
275 0 : return TYPE_UNSIGNED (type) ? usmul_optab : ssmul_optab;
276 966811 : return trapv ? smulv_optab : smul_optab;
277 :
278 78504 : case NEGATE_EXPR:
279 78504 : if (TYPE_SATURATING (type))
280 0 : return TYPE_UNSIGNED (type) ? usneg_optab : ssneg_optab;
281 78504 : return trapv ? negv_optab : neg_optab;
282 :
283 3879 : case ABS_EXPR:
284 3879 : return trapv ? absv_optab : abs_optab;
285 :
286 : case ABSU_EXPR:
287 : return abs_optab;
288 : default:
289 : return unknown_optab;
290 : }
291 : }
292 :
293 : /* Check whether an operation represented by CODE is a 'half' widening operation
294 : in which the input vector type has half the number of bits of the output
295 : vector type e.g. V8QI->V8HI.
296 :
297 : This is handled by widening the inputs using NOP_EXPRs then using a
298 : non-widening stmt e.g. MINUS_EXPR. RTL fusing converts these to the widening
299 : hardware instructions if supported.
300 :
301 : The more typical case (handled in supportable_widening_operation) is where
302 : the input vector type has the same number of bits as the output vector type.
303 : In this case half the elements of the input vectors must be processed at a
304 : time into respective vector outputs with elements twice as wide i.e. a
305 : 'hi'/'lo' pair using codes such as VEC_WIDEN_MINUS_HI/LO.
306 :
307 : Supported widening operations:
308 : WIDEN_MULT_EXPR
309 : WIDEN_LSHIFT_EXPR
310 :
311 : Output:
312 : - CODE1 - The non-widened code, which will be used after the inputs are
313 : converted to the wide type. */
314 : bool
315 411 : supportable_half_widening_operation (enum tree_code code, tree vectype_out,
316 : tree vectype_in, enum tree_code *code1)
317 : {
318 411 : machine_mode m1,m2;
319 411 : optab op;
320 :
321 411 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
322 :
323 411 : m1 = TYPE_MODE (vectype_out);
324 411 : m2 = TYPE_MODE (vectype_in);
325 :
326 411 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
327 : return false;
328 :
329 411 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in),
330 822 : TYPE_VECTOR_SUBPARTS (vectype_out)))
331 : return false;
332 :
333 411 : switch (code)
334 : {
335 0 : case WIDEN_LSHIFT_EXPR:
336 0 : *code1 = LSHIFT_EXPR;
337 0 : break;
338 411 : case WIDEN_MULT_EXPR:
339 411 : *code1 = MULT_EXPR;
340 411 : break;
341 : default:
342 : return false;
343 : }
344 :
345 411 : if (!supportable_convert_operation (NOP_EXPR, vectype_out, vectype_in))
346 : return false;
347 :
348 330 : op = optab_for_tree_code (*code1, vectype_out, optab_vector);
349 330 : return (optab_handler (op, TYPE_MODE (vectype_out)) != CODE_FOR_nothing);
350 : }
351 :
352 : /* Function supportable_convert_operation
353 :
354 : Check whether an operation represented by the code CODE is a
355 : convert operation that is supported by the target platform in
356 : vector form (i.e., when operating on arguments of type VECTYPE_IN
357 : producing a result of type VECTYPE_OUT).
358 :
359 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
360 : This function checks if these operations are supported
361 : by the target platform directly (via vector tree-codes). */
362 :
363 : bool
364 96232 : supportable_convert_operation (enum tree_code code,
365 : tree vectype_out, tree vectype_in)
366 : {
367 96232 : machine_mode m1,m2;
368 96232 : bool truncp;
369 :
370 96232 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
371 :
372 96232 : m1 = TYPE_MODE (vectype_out);
373 96232 : m2 = TYPE_MODE (vectype_in);
374 :
375 96232 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
376 : return false;
377 :
378 96048 : if (m1 == m2
379 64074 : && (CONVERT_EXPR_CODE_P (code) || code == VIEW_CONVERT_EXPR))
380 : return true;
381 :
382 : /* First check if we can done conversion directly. */
383 31974 : if ((code == FIX_TRUNC_EXPR
384 1561 : && can_fix_p (m1,m2,TYPE_UNSIGNED (vectype_out), &truncp)
385 : != CODE_FOR_nothing)
386 32392 : || (code == FLOAT_EXPR
387 13125 : && can_float_p (m1,m2,TYPE_UNSIGNED (vectype_in))
388 : != CODE_FOR_nothing))
389 : return true;
390 :
391 41834 : if (GET_MODE_UNIT_PRECISION (m1) > GET_MODE_UNIT_PRECISION (m2)
392 20917 : && can_extend_p (m1, m2, TYPE_UNSIGNED (vectype_in)))
393 : return true;
394 :
395 26226 : if (GET_MODE_UNIT_PRECISION (m1) < GET_MODE_UNIT_PRECISION (m2)
396 19877 : && convert_optab_handler (trunc_optab, m1, m2) != CODE_FOR_nothing)
397 4171 : return true;
398 :
399 : return false;
400 : }
401 :
402 : /* Return true iff vec_cmp_optab/vec_cmpu_optab can handle a vector comparison
403 : for code CODE, comparing operands of type VALUE_TYPE and producing a result
404 : of type MASK_TYPE. */
405 :
406 : static bool
407 1477218 : vec_cmp_icode_p (tree value_type, tree mask_type, enum tree_code code)
408 : {
409 1477218 : enum rtx_code rcode = get_rtx_code_1 (code, TYPE_UNSIGNED (value_type));
410 1477218 : if (rcode == UNKNOWN)
411 : return false;
412 :
413 1477218 : return can_vec_cmp_compare_p (rcode, TYPE_MODE (value_type),
414 2954436 : TYPE_MODE (mask_type));
415 : }
416 :
417 : /* Return true iff vec_cmpeq_optab can handle a vector comparison for code
418 : CODE, comparing operands of type VALUE_TYPE and producing a result of type
419 : MASK_TYPE. */
420 :
421 : static bool
422 413227 : vec_cmp_eq_icode_p (tree value_type, tree mask_type, enum tree_code code)
423 : {
424 413227 : if (code != EQ_EXPR && code != NE_EXPR)
425 : return false;
426 :
427 212061 : return get_vec_cmp_eq_icode (TYPE_MODE (value_type), TYPE_MODE (mask_type))
428 212061 : != CODE_FOR_nothing;
429 : }
430 :
431 : /* Return TRUE if appropriate vector insn is available
432 : for vector comparison expr with vector type VALUE_TYPE
433 : and resulting mask with MASK_TYPE. */
434 :
435 : bool
436 1477218 : expand_vec_cmp_expr_p (tree value_type, tree mask_type, enum tree_code code)
437 : {
438 1477218 : return vec_cmp_icode_p (value_type, mask_type, code)
439 1477218 : || vec_cmp_eq_icode_p (value_type, mask_type, code);
440 : }
441 :
442 : /* Return TRUE iff, appropriate vector insns are available
443 : for vector cond expr with vector type VALUE_TYPE and a comparison
444 : with operand vector types in CMP_OP_TYPE. */
445 :
446 : bool
447 114622 : expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
448 : {
449 114622 : if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)
450 343866 : && get_vcond_mask_icode (TYPE_MODE (value_type),
451 114622 : TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing)
452 111894 : return true;
453 :
454 : return false;
455 : }
456 :
457 : /* Use the current target and options to initialize
458 : TREE_OPTIMIZATION_OPTABS (OPTNODE). */
459 :
460 : void
461 1657632 : init_tree_optimization_optabs (tree optnode)
462 : {
463 : /* Quick exit if we have already computed optabs for this target. */
464 1657632 : if (TREE_OPTIMIZATION_BASE_OPTABS (optnode) == this_target_optabs)
465 : return;
466 :
467 : /* Forget any previous information and set up for the current target. */
468 24739 : TREE_OPTIMIZATION_BASE_OPTABS (optnode) = this_target_optabs;
469 24739 : struct target_optabs *tmp_optabs = (struct target_optabs *)
470 24739 : TREE_OPTIMIZATION_OPTABS (optnode);
471 24739 : if (tmp_optabs)
472 436 : memset (tmp_optabs, 0, sizeof (struct target_optabs));
473 : else
474 24303 : tmp_optabs = ggc_cleared_alloc<target_optabs> ();
475 :
476 : /* Generate a new set of optabs into tmp_optabs. */
477 24739 : init_all_optabs (tmp_optabs);
478 :
479 : /* If the optabs changed, record it. */
480 24739 : if (memcmp (tmp_optabs, this_target_optabs, sizeof (struct target_optabs)))
481 10601 : TREE_OPTIMIZATION_OPTABS (optnode) = tmp_optabs;
482 : else
483 : {
484 14138 : TREE_OPTIMIZATION_OPTABS (optnode) = NULL;
485 14138 : ggc_free (tmp_optabs);
486 : }
487 : }
488 :
489 : /* Return TRUE if the target has support for vector right shift of an
490 : operand of type TYPE. If OT_TYPE is OPTAB_DEFAULT, check for existence
491 : of a shift by either a scalar or a vector. Otherwise, check only
492 : for a shift that matches OT_TYPE. */
493 :
494 : bool
495 320653 : target_supports_op_p (tree type, enum tree_code code,
496 : enum optab_subtype ot_subtype)
497 : {
498 320653 : optab ot = optab_for_tree_code (code, type, ot_subtype);
499 320653 : return ot != unknown_optab && can_implement_p (ot, TYPE_MODE (type));
500 : }
501 :
502 : /* Return true if the target has support for masked load/store.
503 : We can support masked load/store by either mask{load,store}
504 : or mask_len_{load,store}.
505 : This helper function checks whether target supports masked
506 : load/store and return corresponding IFN in the last argument
507 : (IFN_MASK_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
508 : If there is support and ELSVALS is nonzero store the possible else values
509 : in the vector it points to. */
510 :
511 : bool
512 305750 : target_supports_mask_load_store_p (machine_mode mode, machine_mode mask_mode,
513 : bool is_load, internal_fn *ifn,
514 : vec<int> *elsvals)
515 : {
516 305750 : optab op = is_load ? maskload_optab : maskstore_optab;
517 73893 : optab len_op = is_load ? mask_len_load_optab : mask_len_store_optab;
518 305750 : enum insn_code icode;
519 305750 : if ((icode = convert_optab_handler (op, mode, mask_mode))
520 : != CODE_FOR_nothing)
521 : {
522 97481 : if (ifn)
523 2024 : *ifn = is_load ? IFN_MASK_LOAD : IFN_MASK_STORE;
524 97481 : if (elsvals && is_load)
525 62796 : get_supported_else_vals (icode,
526 62796 : internal_fn_else_index (IFN_MASK_LOAD),
527 : *elsvals);
528 : return true;
529 : }
530 208269 : else if ((icode = convert_optab_handler (len_op, mode, mask_mode))
531 : != CODE_FOR_nothing)
532 : {
533 0 : if (ifn)
534 0 : *ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
535 0 : if (elsvals && is_load)
536 0 : get_supported_else_vals (icode,
537 0 : internal_fn_else_index (IFN_MASK_LEN_LOAD),
538 : *elsvals);
539 : return true;
540 : }
541 : return false;
542 : }
543 :
544 : /* Return true if target supports vector masked load/store for mode.
545 : An additional output in the last argument which is the IFN pointer.
546 : We set IFN as MASK_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
547 : which optab is supported in the target.
548 : If there is support and ELSVALS is nonzero store the possible else values
549 : in the vector it points to. */
550 :
551 : bool
552 284518 : can_vec_mask_load_store_p (machine_mode mode,
553 : machine_mode mask_mode,
554 : bool is_load,
555 : internal_fn *ifn,
556 : vec<int> *elsvals)
557 : {
558 284518 : machine_mode vmode;
559 :
560 : /* If mode is vector mode, check it directly. */
561 284518 : if (VECTOR_MODE_P (mode))
562 272468 : return target_supports_mask_load_store_p (mode, mask_mode, is_load, ifn,
563 272468 : elsvals);
564 :
565 : /* Otherwise, return true if there is some vector mode with
566 : the mask load/store supported. */
567 :
568 : /* See if there is any chance the mask load or store might be
569 : vectorized. If not, punt. */
570 12050 : scalar_mode smode;
571 12050 : if (!is_a <scalar_mode> (mode, &smode))
572 : return false;
573 :
574 12050 : vmode = targetm.vectorize.preferred_simd_mode (smode);
575 2252 : if (VECTOR_MODE_P (vmode)
576 11872 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
577 23922 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
578 : elsvals))
579 2335 : return true;
580 :
581 9715 : auto_vector_modes vector_modes;
582 9715 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
583 56882 : for (machine_mode base_mode : vector_modes)
584 27737 : if (related_vector_mode (base_mode, smode).exists (&vmode)
585 49147 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
586 21410 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
587 : elsvals))
588 0 : return true;
589 : return false;
590 9715 : }
591 :
592 : /* Return true if the target has support for len load/store.
593 : We can support len load/store by either len_{load,store}
594 : or mask_len_{load,store}.
595 : This helper function checks whether target supports len
596 : load/store and return corresponding IFN in the last argument
597 : (IFN_LEN_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
598 : If there is support and ELSVALS is nonzero store thepossible
599 : else values in the vector it points to. */
600 :
601 : static bool
602 535052 : target_supports_len_load_store_p (machine_mode mode, bool is_load,
603 : internal_fn *ifn, vec<int> *elsvals)
604 : {
605 535052 : optab op = is_load ? len_load_optab : len_store_optab;
606 129218 : optab masked_op = is_load ? mask_len_load_optab : mask_len_store_optab;
607 535052 : internal_fn which_ifn;
608 :
609 535052 : enum insn_code icode;
610 535052 : if ((icode = direct_optab_handler (op, mode)) != CODE_FOR_nothing)
611 : {
612 0 : which_ifn = is_load ? IFN_LEN_LOAD : IFN_LEN_STORE;
613 : }
614 : machine_mode mask_mode;
615 : if (!icode
616 1070104 : && targetm.vectorize.get_mask_mode (mode).exists (&mask_mode)
617 535052 : && ((icode = convert_optab_handler (masked_op, mode, mask_mode))
618 : != CODE_FOR_nothing))
619 0 : which_ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
620 :
621 535052 : if (icode && elsvals && is_load)
622 0 : get_supported_else_vals (icode, internal_fn_else_index (which_ifn),
623 : *elsvals);
624 :
625 535052 : if (icode && ifn)
626 0 : *ifn = which_ifn;
627 535052 : return icode;
628 : }
629 :
630 : /* If target supports vector load/store with length for vector mode MODE,
631 : return the corresponding vector mode, otherwise return opt_machine_mode ().
632 : There are two flavors for vector load/store with length, one is to measure
633 : length with bytes, the other is to measure length with lanes.
634 : As len_{load,store} optabs point out, for the flavor with bytes, we use
635 : VnQI to wrap the other supportable same size vector modes.
636 : An additional output in the last argument which is the IFN pointer.
637 : We set IFN as LEN_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
638 : which optab is supported in the target.
639 : If there is support and ELSVALS is nonzero store the possible else values
640 : in the vector it points to. */
641 :
642 : opt_machine_mode
643 267527 : get_len_load_store_mode (machine_mode mode, bool is_load, internal_fn *ifn,
644 : vec<int> *elsvals)
645 : {
646 267527 : gcc_assert (VECTOR_MODE_P (mode));
647 :
648 : /* Check if length in lanes supported for this mode directly. */
649 267527 : if (target_supports_len_load_store_p (mode, is_load, ifn, elsvals))
650 0 : return mode;
651 :
652 : /* Check if length in bytes supported for same vector size VnQI. */
653 267527 : machine_mode vmode;
654 535054 : poly_uint64 nunits = GET_MODE_SIZE (mode);
655 267527 : if (related_vector_mode (mode, QImode, nunits).exists (&vmode)
656 267525 : && target_supports_len_load_store_p (vmode, is_load, ifn, elsvals))
657 0 : return vmode;
658 :
659 267527 : return opt_machine_mode ();
660 : }
|
