Line data Source code
1 : /* Tree-based target query functions relating to optabs
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "target.h"
25 : #include "insn-codes.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "optabs.h"
30 : #include "optabs-tree.h"
31 : #include "stor-layout.h"
32 : #include "internal-fn.h"
33 :
34 : /* Return the optab used for computing the operation given by the tree code,
35 : CODE and the tree EXP. This function is not always usable (for example, it
36 : cannot give complete results for multiplication or division) but probably
37 : ought to be relied on more widely throughout the expander. */
38 : optab
39 10717986 : optab_for_tree_code (enum tree_code code, const_tree type,
40 : enum optab_subtype subtype)
41 : {
42 10717986 : bool trapv;
43 10717986 : switch (code)
44 : {
45 : case BIT_AND_EXPR:
46 : return and_optab;
47 :
48 207480 : case BIT_IOR_EXPR:
49 207480 : return ior_optab;
50 :
51 6677 : case BIT_NOT_EXPR:
52 6677 : return one_cmpl_optab;
53 :
54 55961 : case BIT_XOR_EXPR:
55 55961 : return xor_optab;
56 :
57 4395 : case MULT_HIGHPART_EXPR:
58 4395 : return TYPE_UNSIGNED (type) ? umul_highpart_optab : smul_highpart_optab;
59 :
60 78 : case CEIL_MOD_EXPR:
61 78 : case FLOOR_MOD_EXPR:
62 78 : case ROUND_MOD_EXPR:
63 : /* {s,u}mod_optab implements TRUNC_MOD_EXPR. For scalar modes,
64 : expansion has code to adjust TRUNC_MOD_EXPR into the desired other
65 : modes, but for vector modes it does not. The adjustment code
66 : should be instead emitted in tree-vect-patterns.cc. */
67 78 : if (VECTOR_TYPE_P (type))
68 : return unknown_optab;
69 : /* FALLTHRU */
70 49146 : case TRUNC_MOD_EXPR:
71 49146 : return TYPE_UNSIGNED (type) ? umod_optab : smod_optab;
72 :
73 12 : case CEIL_DIV_EXPR:
74 12 : case FLOOR_DIV_EXPR:
75 12 : case ROUND_DIV_EXPR:
76 : /* {,u}{s,u}div_optab implements {TRUNC,EXACT}_DIV_EXPR or RDIV_EXPR.
77 : For scalar modes, expansion has code to adjust TRUNC_DIV_EXPR
78 : into the desired other modes, but for vector modes it does not.
79 : The adjustment code should be instead emitted in
80 : tree-vect-patterns.cc. */
81 12 : if (VECTOR_TYPE_P (type))
82 : return unknown_optab;
83 : /* FALLTHRU */
84 36544 : case RDIV_EXPR:
85 36544 : gcc_assert (FLOAT_TYPE_P (type)
86 : || ALL_FIXED_POINT_MODE_P (TYPE_MODE (type)));
87 : /* FALLTHRU */
88 93328 : case TRUNC_DIV_EXPR:
89 93328 : case EXACT_DIV_EXPR:
90 93328 : if (TYPE_SATURATING (type))
91 0 : return TYPE_UNSIGNED (type) ? usdiv_optab : ssdiv_optab;
92 93328 : return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab;
93 :
94 767762 : case LSHIFT_EXPR:
95 767762 : if (VECTOR_TYPE_P (type))
96 : {
97 767748 : if (subtype == optab_vector)
98 418261 : return TYPE_SATURATING (type) ? unknown_optab : vashl_optab;
99 :
100 349487 : gcc_assert (subtype == optab_scalar);
101 : }
102 349501 : if (TYPE_SATURATING (type))
103 0 : return TYPE_UNSIGNED (type) ? usashl_optab : ssashl_optab;
104 : return ashl_optab;
105 :
106 214735 : case RSHIFT_EXPR:
107 214735 : if (VECTOR_TYPE_P (type))
108 : {
109 214726 : if (subtype == optab_vector)
110 153421 : return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab;
111 :
112 61305 : gcc_assert (subtype == optab_scalar);
113 : }
114 61314 : return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab;
115 :
116 1249 : case LROTATE_EXPR:
117 1249 : if (VECTOR_TYPE_P (type))
118 : {
119 1249 : if (subtype == optab_vector)
120 : return vrotl_optab;
121 :
122 475 : gcc_assert (subtype == optab_scalar);
123 : }
124 : return rotl_optab;
125 :
126 15694 : case RROTATE_EXPR:
127 15694 : if (VECTOR_TYPE_P (type))
128 : {
129 15694 : if (subtype == optab_vector)
130 : return vrotr_optab;
131 :
132 7758 : gcc_assert (subtype == optab_scalar);
133 : }
134 : return rotr_optab;
135 :
136 108137 : case MAX_EXPR:
137 108137 : return TYPE_UNSIGNED (type) ? umax_optab : smax_optab;
138 :
139 54071 : case MIN_EXPR:
140 54071 : return TYPE_UNSIGNED (type) ? umin_optab : smin_optab;
141 :
142 : case POINTER_PLUS_EXPR:
143 : return add_optab;
144 :
145 : case POINTER_DIFF_EXPR:
146 : return sub_optab;
147 :
148 0 : case REALIGN_LOAD_EXPR:
149 0 : return vec_realign_load_optab;
150 :
151 4037 : case WIDEN_SUM_EXPR:
152 4037 : return (TYPE_UNSIGNED (type)
153 4037 : ? reduc_widen_usum_optab : reduc_widen_ssum_optab);
154 :
155 1921 : case DOT_PROD_EXPR:
156 1921 : {
157 1921 : if (subtype == optab_vector_mixed_sign)
158 : return usdot_prod_optab;
159 :
160 1384 : return (TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab);
161 : }
162 :
163 973 : case SAD_EXPR:
164 973 : return TYPE_UNSIGNED (type) ? usad_optab : ssad_optab;
165 :
166 31186 : case WIDEN_MULT_PLUS_EXPR:
167 31186 : return (TYPE_UNSIGNED (type)
168 31186 : ? (TYPE_SATURATING (type)
169 21212 : ? usmadd_widen_optab : umadd_widen_optab)
170 9974 : : (TYPE_SATURATING (type)
171 9974 : ? ssmadd_widen_optab : smadd_widen_optab));
172 :
173 565 : case WIDEN_MULT_MINUS_EXPR:
174 565 : return (TYPE_UNSIGNED (type)
175 565 : ? (TYPE_SATURATING (type)
176 453 : ? usmsub_widen_optab : umsub_widen_optab)
177 112 : : (TYPE_SATURATING (type)
178 112 : ? ssmsub_widen_optab : smsub_widen_optab));
179 :
180 103633 : case VEC_WIDEN_MULT_HI_EXPR:
181 103633 : return (TYPE_UNSIGNED (type)
182 103633 : ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab);
183 :
184 103626 : case VEC_WIDEN_MULT_LO_EXPR:
185 103626 : return (TYPE_UNSIGNED (type)
186 103626 : ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab);
187 :
188 209500 : case VEC_WIDEN_MULT_EVEN_EXPR:
189 209500 : return (TYPE_UNSIGNED (type)
190 209500 : ? vec_widen_umult_even_optab : vec_widen_smult_even_optab);
191 :
192 209460 : case VEC_WIDEN_MULT_ODD_EXPR:
193 209460 : return (TYPE_UNSIGNED (type)
194 209460 : ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab);
195 :
196 9813 : case VEC_WIDEN_LSHIFT_HI_EXPR:
197 9813 : return (TYPE_UNSIGNED (type)
198 9813 : ? vec_widen_ushiftl_hi_optab : vec_widen_sshiftl_hi_optab);
199 :
200 9813 : case VEC_WIDEN_LSHIFT_LO_EXPR:
201 9813 : return (TYPE_UNSIGNED (type)
202 9813 : ? vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab);
203 :
204 66766 : case VEC_UNPACK_HI_EXPR:
205 66766 : return (TYPE_UNSIGNED (type)
206 66766 : ? vec_unpacku_hi_optab : vec_unpacks_hi_optab);
207 :
208 66763 : case VEC_UNPACK_LO_EXPR:
209 66763 : return (TYPE_UNSIGNED (type)
210 66763 : ? vec_unpacku_lo_optab : vec_unpacks_lo_optab);
211 :
212 10289 : case VEC_UNPACK_FLOAT_HI_EXPR:
213 : /* The signedness is determined from input operand. */
214 10289 : return (TYPE_UNSIGNED (type)
215 10289 : ? vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab);
216 :
217 10295 : case VEC_UNPACK_FLOAT_LO_EXPR:
218 : /* The signedness is determined from input operand. */
219 10295 : return (TYPE_UNSIGNED (type)
220 10295 : ? vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab);
221 :
222 271 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
223 : /* The signedness is determined from output operand. */
224 271 : return (TYPE_UNSIGNED (type)
225 271 : ? vec_unpack_ufix_trunc_hi_optab
226 : : vec_unpack_sfix_trunc_hi_optab);
227 :
228 271 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
229 : /* The signedness is determined from output operand. */
230 271 : return (TYPE_UNSIGNED (type)
231 271 : ? vec_unpack_ufix_trunc_lo_optab
232 : : vec_unpack_sfix_trunc_lo_optab);
233 :
234 68760 : case VEC_PACK_TRUNC_EXPR:
235 68760 : return vec_pack_trunc_optab;
236 :
237 0 : case VEC_PACK_SAT_EXPR:
238 0 : return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab;
239 :
240 1021 : case VEC_PACK_FIX_TRUNC_EXPR:
241 : /* The signedness is determined from output operand. */
242 1021 : return (TYPE_UNSIGNED (type)
243 1021 : ? vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab);
244 :
245 975 : case VEC_PACK_FLOAT_EXPR:
246 : /* The signedness is determined from input operand. */
247 975 : return (TYPE_UNSIGNED (type)
248 975 : ? vec_packu_float_optab : vec_packs_float_optab);
249 :
250 0 : case VEC_DUPLICATE_EXPR:
251 0 : return vec_duplicate_optab;
252 :
253 0 : case VEC_SERIES_EXPR:
254 0 : return vec_series_optab;
255 :
256 6784291 : default:
257 6784291 : break;
258 : }
259 :
260 6784291 : trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type);
261 6784291 : switch (code)
262 : {
263 4608898 : case PLUS_EXPR:
264 4608898 : if (TYPE_SATURATING (type))
265 0 : return TYPE_UNSIGNED (type) ? usadd_optab : ssadd_optab;
266 4608898 : return trapv ? addv_optab : add_optab;
267 :
268 981211 : case MINUS_EXPR:
269 981211 : if (TYPE_SATURATING (type))
270 0 : return TYPE_UNSIGNED (type) ? ussub_optab : sssub_optab;
271 981211 : return trapv ? subv_optab : sub_optab;
272 :
273 1001327 : case MULT_EXPR:
274 1001327 : if (TYPE_SATURATING (type))
275 0 : return TYPE_UNSIGNED (type) ? usmul_optab : ssmul_optab;
276 1001327 : return trapv ? smulv_optab : smul_optab;
277 :
278 79679 : case NEGATE_EXPR:
279 79679 : if (TYPE_SATURATING (type))
280 0 : return TYPE_UNSIGNED (type) ? usneg_optab : ssneg_optab;
281 79679 : return trapv ? negv_optab : neg_optab;
282 :
283 3974 : case ABS_EXPR:
284 3974 : return trapv ? absv_optab : abs_optab;
285 :
286 : case ABSU_EXPR:
287 : return abs_optab;
288 : default:
289 : return unknown_optab;
290 : }
291 : }
292 :
293 : /* Check whether an operation represented by CODE is a 'half' widening operation
294 : in which the input vector type has half the number of bits of the output
295 : vector type e.g. V8QI->V8HI.
296 :
297 : This is handled by widening the inputs using NOP_EXPRs then using a
298 : non-widening stmt e.g. MINUS_EXPR. RTL fusing converts these to the widening
299 : hardware instructions if supported.
300 :
301 : The more typical case (handled in supportable_widening_operation) is where
302 : the input vector type has the same number of bits as the output vector type.
303 : In this case half the elements of the input vectors must be processed at a
304 : time into respective vector outputs with elements twice as wide i.e. a
305 : 'hi'/'lo' pair using codes such as VEC_WIDEN_MINUS_HI/LO.
306 :
307 : Supported widening operations:
308 : WIDEN_MULT_EXPR
309 : WIDEN_LSHIFT_EXPR
310 :
311 : Output:
312 : - CODE1 - The non-widened code, which will be used after the inputs are
313 : converted to the wide type. */
314 : bool
315 361 : supportable_half_widening_operation (enum tree_code code, tree vectype_out,
316 : tree vectype_in, enum tree_code *code1)
317 : {
318 361 : machine_mode m1,m2;
319 361 : optab op;
320 :
321 361 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
322 :
323 361 : m1 = TYPE_MODE (vectype_out);
324 361 : m2 = TYPE_MODE (vectype_in);
325 :
326 361 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
327 : return false;
328 :
329 361 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in),
330 722 : TYPE_VECTOR_SUBPARTS (vectype_out)))
331 : return false;
332 :
333 361 : switch (code)
334 : {
335 0 : case WIDEN_LSHIFT_EXPR:
336 0 : *code1 = LSHIFT_EXPR;
337 0 : break;
338 361 : case WIDEN_MULT_EXPR:
339 361 : *code1 = MULT_EXPR;
340 361 : break;
341 : default:
342 : return false;
343 : }
344 :
345 361 : if (!supportable_convert_operation (NOP_EXPR, vectype_out, vectype_in))
346 : return false;
347 :
348 287 : op = optab_for_tree_code (*code1, vectype_out, optab_vector);
349 287 : return (optab_handler (op, TYPE_MODE (vectype_out)) != CODE_FOR_nothing);
350 : }
351 :
352 : /* Function supportable_convert_operation
353 :
354 : Check whether an operation represented by the code CODE is a
355 : convert operation that is supported by the target platform in
356 : vector form (i.e., when operating on arguments of type VECTYPE_IN
357 : producing a result of type VECTYPE_OUT).
358 :
359 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
360 : This function checks if these operations are supported
361 : by the target platform directly (via vector tree-codes). */
362 :
363 : bool
364 98595 : supportable_convert_operation (enum tree_code code,
365 : tree vectype_out, tree vectype_in)
366 : {
367 98595 : machine_mode m1,m2;
368 98595 : bool truncp;
369 :
370 98595 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
371 :
372 98595 : m1 = TYPE_MODE (vectype_out);
373 98595 : m2 = TYPE_MODE (vectype_in);
374 :
375 98595 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
376 : return false;
377 :
378 98415 : if (m1 == m2
379 65257 : && (CONVERT_EXPR_CODE_P (code) || code == VIEW_CONVERT_EXPR))
380 : return true;
381 :
382 : /* First check if we can done conversion directly. */
383 33158 : if ((code == FIX_TRUNC_EXPR
384 1560 : && can_fix_p (m1,m2,TYPE_UNSIGNED (vectype_out), &truncp)
385 : != CODE_FOR_nothing)
386 33594 : || (code == FLOAT_EXPR
387 14270 : && can_float_p (m1,m2,TYPE_UNSIGNED (vectype_in))
388 : != CODE_FOR_nothing))
389 : return true;
390 :
391 42866 : if (GET_MODE_UNIT_PRECISION (m1) > GET_MODE_UNIT_PRECISION (m2)
392 21433 : && can_extend_p (m1, m2, TYPE_UNSIGNED (vectype_in)))
393 : return true;
394 :
395 27436 : if (GET_MODE_UNIT_PRECISION (m1) < GET_MODE_UNIT_PRECISION (m2)
396 20487 : && convert_optab_handler (trunc_optab, m1, m2) != CODE_FOR_nothing)
397 4180 : return true;
398 :
399 : return false;
400 : }
401 :
402 : /* Return true iff vec_cmp_optab/vec_cmpu_optab can handle a vector comparison
403 : for code CODE, comparing operands of type VALUE_TYPE and producing a result
404 : of type MASK_TYPE. */
405 :
406 : static bool
407 1467484 : vec_cmp_icode_p (tree value_type, tree mask_type, enum tree_code code)
408 : {
409 1467484 : enum rtx_code rcode = get_rtx_code_1 (code, TYPE_UNSIGNED (value_type));
410 1467484 : if (rcode == UNKNOWN)
411 : return false;
412 :
413 1467484 : return can_vec_cmp_compare_p (rcode, TYPE_MODE (value_type),
414 2934968 : TYPE_MODE (mask_type));
415 : }
416 :
417 : /* Return true iff vec_cmpeq_optab can handle a vector comparison for code
418 : CODE, comparing operands of type VALUE_TYPE and producing a result of type
419 : MASK_TYPE. */
420 :
421 : static bool
422 417963 : vec_cmp_eq_icode_p (tree value_type, tree mask_type, enum tree_code code)
423 : {
424 417963 : if (code != EQ_EXPR && code != NE_EXPR)
425 : return false;
426 :
427 215371 : return get_vec_cmp_eq_icode (TYPE_MODE (value_type), TYPE_MODE (mask_type))
428 215371 : != CODE_FOR_nothing;
429 : }
430 :
431 : /* Return TRUE if appropriate vector insn is available
432 : for vector comparison expr with vector type VALUE_TYPE
433 : and resulting mask with MASK_TYPE. */
434 :
435 : bool
436 1467484 : expand_vec_cmp_expr_p (tree value_type, tree mask_type, enum tree_code code)
437 : {
438 1467484 : return vec_cmp_icode_p (value_type, mask_type, code)
439 1467484 : || vec_cmp_eq_icode_p (value_type, mask_type, code);
440 : }
441 :
442 : /* Return TRUE iff, appropriate vector insns are available
443 : for vector cond expr with vector type VALUE_TYPE and a comparison
444 : with operand vector types in CMP_OP_TYPE. */
445 :
446 : bool
447 116105 : expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
448 : {
449 116105 : if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)
450 348315 : && get_vcond_mask_icode (TYPE_MODE (value_type),
451 116105 : TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing)
452 113371 : return true;
453 :
454 : return false;
455 : }
456 :
457 : /* Use the current target and options to initialize
458 : TREE_OPTIMIZATION_OPTABS (OPTNODE). */
459 :
460 : void
461 1666180 : init_tree_optimization_optabs (tree optnode)
462 : {
463 : /* Quick exit if we have already computed optabs for this target. */
464 1666180 : if (TREE_OPTIMIZATION_BASE_OPTABS (optnode) == this_target_optabs)
465 : return;
466 :
467 : /* Forget any previous information and set up for the current target. */
468 25229 : TREE_OPTIMIZATION_BASE_OPTABS (optnode) = this_target_optabs;
469 25229 : struct target_optabs *tmp_optabs = (struct target_optabs *)
470 25229 : TREE_OPTIMIZATION_OPTABS (optnode);
471 25229 : if (tmp_optabs)
472 436 : memset (tmp_optabs, 0, sizeof (struct target_optabs));
473 : else
474 24793 : tmp_optabs = ggc_cleared_alloc<target_optabs> ();
475 :
476 : /* Generate a new set of optabs into tmp_optabs. */
477 25229 : init_all_optabs (tmp_optabs);
478 :
479 : /* If the optabs changed, record it. */
480 25229 : if (memcmp (tmp_optabs, this_target_optabs, sizeof (struct target_optabs)))
481 10926 : TREE_OPTIMIZATION_OPTABS (optnode) = tmp_optabs;
482 : else
483 : {
484 14303 : TREE_OPTIMIZATION_OPTABS (optnode) = NULL;
485 14303 : ggc_free (tmp_optabs);
486 : }
487 : }
488 :
489 : /* Return TRUE if the target has support for vector right shift of an
490 : operand of type TYPE. If OT_TYPE is OPTAB_DEFAULT, check for existence
491 : of a shift by either a scalar or a vector. Otherwise, check only
492 : for a shift that matches OT_TYPE. */
493 :
494 : bool
495 328274 : target_supports_op_p (tree type, enum tree_code code,
496 : enum optab_subtype ot_subtype)
497 : {
498 328274 : optab ot = optab_for_tree_code (code, type, ot_subtype);
499 328274 : return ot != unknown_optab && can_implement_p (ot, TYPE_MODE (type));
500 : }
501 :
502 : /* Return true if the target has support for masked load/store.
503 : We can support masked load/store by either mask{load,store}
504 : or mask_len_{load,store}.
505 : This helper function checks whether target supports masked
506 : load/store and return corresponding IFN in the last argument
507 : (IFN_MASK_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
508 : If there is support and ELSVALS is nonzero store the possible else values
509 : in the vector it points to. */
510 :
511 : bool
512 311786 : target_supports_mask_load_store_p (machine_mode mode, machine_mode mask_mode,
513 : bool is_load, internal_fn *ifn,
514 : vec<int> *elsvals)
515 : {
516 311786 : optab op = is_load ? maskload_optab : maskstore_optab;
517 75995 : optab len_op = is_load ? mask_len_load_optab : mask_len_store_optab;
518 311786 : enum insn_code icode;
519 311786 : if ((icode = convert_optab_handler (op, mode, mask_mode))
520 : != CODE_FOR_nothing)
521 : {
522 99157 : if (ifn)
523 2075 : *ifn = is_load ? IFN_MASK_LOAD : IFN_MASK_STORE;
524 99157 : if (elsvals && is_load)
525 63857 : get_supported_else_vals (icode,
526 63857 : internal_fn_else_index (IFN_MASK_LOAD),
527 : *elsvals);
528 : return true;
529 : }
530 212629 : else if ((icode = convert_optab_handler (len_op, mode, mask_mode))
531 : != CODE_FOR_nothing)
532 : {
533 0 : if (ifn)
534 0 : *ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
535 0 : if (elsvals && is_load)
536 0 : get_supported_else_vals (icode,
537 0 : internal_fn_else_index (IFN_MASK_LEN_LOAD),
538 : *elsvals);
539 : return true;
540 : }
541 : return false;
542 : }
543 :
544 : /* Return true if target supports vector masked load/store for mode.
545 : An additional output in the last argument which is the IFN pointer.
546 : We set IFN as MASK_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
547 : which optab is supported in the target.
548 : If there is support and ELSVALS is nonzero store the possible else values
549 : in the vector it points to. */
550 :
551 : bool
552 290622 : can_vec_mask_load_store_p (machine_mode mode,
553 : machine_mode mask_mode,
554 : bool is_load,
555 : internal_fn *ifn,
556 : vec<int> *elsvals)
557 : {
558 290622 : machine_mode vmode;
559 :
560 : /* If mode is vector mode, check it directly. */
561 290622 : if (VECTOR_MODE_P (mode))
562 278774 : return target_supports_mask_load_store_p (mode, mask_mode, is_load, ifn,
563 278774 : elsvals);
564 :
565 : /* Otherwise, return true if there is some vector mode with
566 : the mask load/store supported. */
567 :
568 : /* See if there is any chance the mask load or store might be
569 : vectorized. If not, punt. */
570 11848 : scalar_mode smode;
571 11848 : if (!is_a <scalar_mode> (mode, &smode))
572 : return false;
573 :
574 11848 : vmode = targetm.vectorize.preferred_simd_mode (smode);
575 2231 : if (VECTOR_MODE_P (vmode)
576 11670 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
577 23518 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
578 : elsvals))
579 2164 : return true;
580 :
581 9684 : auto_vector_modes vector_modes;
582 9684 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
583 56699 : for (machine_mode base_mode : vector_modes)
584 27647 : if (related_vector_mode (base_mode, smode).exists (&vmode)
585 48989 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
586 21342 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
587 : elsvals))
588 0 : return true;
589 : return false;
590 9684 : }
591 :
592 : /* Return true if the target has support for len load/store.
593 : We can support len load/store by either len_{load,store}
594 : or mask_len_{load,store}.
595 : This helper function checks whether target supports len
596 : load/store and return corresponding IFN in the last argument
597 : (IFN_LEN_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
598 : If there is support and ELSVALS is nonzero store thepossible
599 : else values in the vector it points to. */
600 :
601 : static bool
602 547492 : target_supports_len_load_store_p (machine_mode mode, bool is_load,
603 : internal_fn *ifn, vec<int> *elsvals)
604 : {
605 547492 : optab op = is_load ? len_load_optab : len_store_optab;
606 133180 : optab masked_op = is_load ? mask_len_load_optab : mask_len_store_optab;
607 547492 : internal_fn which_ifn;
608 :
609 547492 : enum insn_code icode;
610 547492 : if ((icode = direct_optab_handler (op, mode)) != CODE_FOR_nothing)
611 : {
612 0 : which_ifn = is_load ? IFN_LEN_LOAD : IFN_LEN_STORE;
613 : }
614 : machine_mode mask_mode;
615 : if (!icode
616 1094984 : && targetm.vectorize.get_mask_mode (mode).exists (&mask_mode)
617 547492 : && ((icode = convert_optab_handler (masked_op, mode, mask_mode))
618 : != CODE_FOR_nothing))
619 0 : which_ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
620 :
621 547492 : if (icode && elsvals && is_load)
622 0 : get_supported_else_vals (icode, internal_fn_else_index (which_ifn),
623 : *elsvals);
624 :
625 547492 : if (icode && ifn)
626 0 : *ifn = which_ifn;
627 547492 : return icode;
628 : }
629 :
630 : /* If target supports vector load/store with length for vector mode MODE,
631 : return the corresponding vector mode, otherwise return opt_machine_mode ().
632 : There are two flavors for vector load/store with length, one is to measure
633 : length with bytes, the other is to measure length with lanes.
634 : As len_{load,store} optabs point out, for the flavor with bytes, we use
635 : VnQI to wrap the other supportable same size vector modes.
636 : An additional output in the last argument which is the IFN pointer.
637 : We set IFN as LEN_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
638 : which optab is supported in the target.
639 : If there is support and ELSVALS is nonzero store the possible else values
640 : in the vector it points to. */
641 :
642 : opt_machine_mode
643 273747 : get_len_load_store_mode (machine_mode mode, bool is_load, internal_fn *ifn,
644 : vec<int> *elsvals)
645 : {
646 273747 : gcc_assert (VECTOR_MODE_P (mode));
647 :
648 : /* Check if length in lanes supported for this mode directly. */
649 273747 : if (target_supports_len_load_store_p (mode, is_load, ifn, elsvals))
650 0 : return mode;
651 :
652 : /* Check if length in bytes supported for same vector size VnQI. */
653 273747 : machine_mode vmode;
654 547494 : poly_uint64 nunits = GET_MODE_SIZE (mode);
655 273747 : if (related_vector_mode (mode, QImode, nunits).exists (&vmode)
656 273745 : && target_supports_len_load_store_p (vmode, is_load, ifn, elsvals))
657 0 : return vmode;
658 :
659 273747 : return opt_machine_mode ();
660 : }
|