Line data Source code
1 : /* Tree-based target query functions relating to optabs
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "target.h"
25 : #include "insn-codes.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "optabs.h"
30 : #include "optabs-tree.h"
31 : #include "stor-layout.h"
32 : #include "internal-fn.h"
33 :
34 : /* Return the optab used for computing the operation given by the tree code,
35 : CODE and the tree EXP. This function is not always usable (for example, it
36 : cannot give complete results for multiplication or division) but probably
37 : ought to be relied on more widely throughout the expander. */
38 : optab
39 10032987 : optab_for_tree_code (enum tree_code code, const_tree type,
40 : enum optab_subtype subtype)
41 : {
42 10032987 : bool trapv;
43 10032987 : switch (code)
44 : {
45 : case BIT_AND_EXPR:
46 : return and_optab;
47 :
48 226793 : case BIT_IOR_EXPR:
49 226793 : return ior_optab;
50 :
51 7016 : case BIT_NOT_EXPR:
52 7016 : return one_cmpl_optab;
53 :
54 50532 : case BIT_XOR_EXPR:
55 50532 : return xor_optab;
56 :
57 2720 : case MULT_HIGHPART_EXPR:
58 2720 : return TYPE_UNSIGNED (type) ? umul_highpart_optab : smul_highpart_optab;
59 :
60 42 : case CEIL_MOD_EXPR:
61 42 : case FLOOR_MOD_EXPR:
62 42 : case ROUND_MOD_EXPR:
63 : /* {s,u}mod_optab implements TRUNC_MOD_EXPR. For scalar modes,
64 : expansion has code to adjust TRUNC_MOD_EXPR into the desired other
65 : modes, but for vector modes it does not. The adjustment code
66 : should be instead emitted in tree-vect-patterns.cc. */
67 42 : if (VECTOR_TYPE_P (type))
68 : return unknown_optab;
69 : /* FALLTHRU */
70 39152 : case TRUNC_MOD_EXPR:
71 39152 : return TYPE_UNSIGNED (type) ? umod_optab : smod_optab;
72 :
73 18 : case CEIL_DIV_EXPR:
74 18 : case FLOOR_DIV_EXPR:
75 18 : case ROUND_DIV_EXPR:
76 : /* {,u}{s,u}div_optab implements {TRUNC,EXACT}_DIV_EXPR or RDIV_EXPR.
77 : For scalar modes, expansion has code to adjust TRUNC_DIV_EXPR
78 : into the desired other modes, but for vector modes it does not.
79 : The adjustment code should be instead emitted in
80 : tree-vect-patterns.cc. */
81 18 : if (VECTOR_TYPE_P (type))
82 : return unknown_optab;
83 : /* FALLTHRU */
84 33398 : case RDIV_EXPR:
85 33398 : gcc_assert (FLOAT_TYPE_P (type)
86 : || ALL_FIXED_POINT_MODE_P (TYPE_MODE (type)));
87 : /* FALLTHRU */
88 80964 : case TRUNC_DIV_EXPR:
89 80964 : case EXACT_DIV_EXPR:
90 80964 : if (TYPE_SATURATING (type))
91 0 : return TYPE_UNSIGNED (type) ? usdiv_optab : ssdiv_optab;
92 80964 : return TYPE_UNSIGNED (type) ? udiv_optab : sdiv_optab;
93 :
94 692956 : case LSHIFT_EXPR:
95 692956 : if (VECTOR_TYPE_P (type))
96 : {
97 692922 : if (subtype == optab_vector)
98 334669 : return TYPE_SATURATING (type) ? unknown_optab : vashl_optab;
99 :
100 358253 : gcc_assert (subtype == optab_scalar);
101 : }
102 358287 : if (TYPE_SATURATING (type))
103 0 : return TYPE_UNSIGNED (type) ? usashl_optab : ssashl_optab;
104 : return ashl_optab;
105 :
106 226915 : case RSHIFT_EXPR:
107 226915 : if (VECTOR_TYPE_P (type))
108 : {
109 226888 : if (subtype == optab_vector)
110 108804 : return TYPE_UNSIGNED (type) ? vlshr_optab : vashr_optab;
111 :
112 118084 : gcc_assert (subtype == optab_scalar);
113 : }
114 118111 : return TYPE_UNSIGNED (type) ? lshr_optab : ashr_optab;
115 :
116 387 : case LROTATE_EXPR:
117 387 : if (VECTOR_TYPE_P (type))
118 : {
119 387 : if (subtype == optab_vector)
120 : return vrotl_optab;
121 :
122 155 : gcc_assert (subtype == optab_scalar);
123 : }
124 : return rotl_optab;
125 :
126 12497 : case RROTATE_EXPR:
127 12497 : if (VECTOR_TYPE_P (type))
128 : {
129 12497 : if (subtype == optab_vector)
130 : return vrotr_optab;
131 :
132 6340 : gcc_assert (subtype == optab_scalar);
133 : }
134 : return rotr_optab;
135 :
136 88862 : case MAX_EXPR:
137 88862 : return TYPE_UNSIGNED (type) ? umax_optab : smax_optab;
138 :
139 45457 : case MIN_EXPR:
140 45457 : return TYPE_UNSIGNED (type) ? umin_optab : smin_optab;
141 :
142 : case POINTER_PLUS_EXPR:
143 : return add_optab;
144 :
145 : case POINTER_DIFF_EXPR:
146 : return sub_optab;
147 :
148 0 : case REALIGN_LOAD_EXPR:
149 0 : return vec_realign_load_optab;
150 :
151 1619 : case WIDEN_SUM_EXPR:
152 1619 : return TYPE_UNSIGNED (type) ? usum_widen_optab : ssum_widen_optab;
153 :
154 1388 : case DOT_PROD_EXPR:
155 1388 : {
156 1388 : if (subtype == optab_vector_mixed_sign)
157 : return usdot_prod_optab;
158 :
159 1015 : return (TYPE_UNSIGNED (type) ? udot_prod_optab : sdot_prod_optab);
160 : }
161 :
162 540 : case SAD_EXPR:
163 540 : return TYPE_UNSIGNED (type) ? usad_optab : ssad_optab;
164 :
165 33333 : case WIDEN_MULT_PLUS_EXPR:
166 33333 : return (TYPE_UNSIGNED (type)
167 33333 : ? (TYPE_SATURATING (type)
168 23731 : ? usmadd_widen_optab : umadd_widen_optab)
169 9602 : : (TYPE_SATURATING (type)
170 9602 : ? ssmadd_widen_optab : smadd_widen_optab));
171 :
172 861 : case WIDEN_MULT_MINUS_EXPR:
173 861 : return (TYPE_UNSIGNED (type)
174 861 : ? (TYPE_SATURATING (type)
175 470 : ? usmsub_widen_optab : umsub_widen_optab)
176 391 : : (TYPE_SATURATING (type)
177 391 : ? ssmsub_widen_optab : smsub_widen_optab));
178 :
179 79358 : case VEC_WIDEN_MULT_HI_EXPR:
180 79358 : return (TYPE_UNSIGNED (type)
181 79358 : ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab);
182 :
183 79351 : case VEC_WIDEN_MULT_LO_EXPR:
184 79351 : return (TYPE_UNSIGNED (type)
185 79351 : ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab);
186 :
187 169453 : case VEC_WIDEN_MULT_EVEN_EXPR:
188 169453 : return (TYPE_UNSIGNED (type)
189 169453 : ? vec_widen_umult_even_optab : vec_widen_smult_even_optab);
190 :
191 169453 : case VEC_WIDEN_MULT_ODD_EXPR:
192 169453 : return (TYPE_UNSIGNED (type)
193 169453 : ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab);
194 :
195 9309 : case VEC_WIDEN_LSHIFT_HI_EXPR:
196 9309 : return (TYPE_UNSIGNED (type)
197 9309 : ? vec_widen_ushiftl_hi_optab : vec_widen_sshiftl_hi_optab);
198 :
199 9309 : case VEC_WIDEN_LSHIFT_LO_EXPR:
200 9309 : return (TYPE_UNSIGNED (type)
201 9309 : ? vec_widen_ushiftl_lo_optab : vec_widen_sshiftl_lo_optab);
202 :
203 53165 : case VEC_UNPACK_HI_EXPR:
204 53165 : return (TYPE_UNSIGNED (type)
205 53165 : ? vec_unpacku_hi_optab : vec_unpacks_hi_optab);
206 :
207 53143 : case VEC_UNPACK_LO_EXPR:
208 53143 : return (TYPE_UNSIGNED (type)
209 53143 : ? vec_unpacku_lo_optab : vec_unpacks_lo_optab);
210 :
211 8099 : case VEC_UNPACK_FLOAT_HI_EXPR:
212 : /* The signedness is determined from input operand. */
213 8099 : return (TYPE_UNSIGNED (type)
214 8099 : ? vec_unpacku_float_hi_optab : vec_unpacks_float_hi_optab);
215 :
216 8099 : case VEC_UNPACK_FLOAT_LO_EXPR:
217 : /* The signedness is determined from input operand. */
218 8099 : return (TYPE_UNSIGNED (type)
219 8099 : ? vec_unpacku_float_lo_optab : vec_unpacks_float_lo_optab);
220 :
221 237 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
222 : /* The signedness is determined from output operand. */
223 237 : return (TYPE_UNSIGNED (type)
224 237 : ? vec_unpack_ufix_trunc_hi_optab
225 : : vec_unpack_sfix_trunc_hi_optab);
226 :
227 237 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
228 : /* The signedness is determined from output operand. */
229 237 : return (TYPE_UNSIGNED (type)
230 237 : ? vec_unpack_ufix_trunc_lo_optab
231 : : vec_unpack_sfix_trunc_lo_optab);
232 :
233 56409 : case VEC_PACK_TRUNC_EXPR:
234 56409 : return vec_pack_trunc_optab;
235 :
236 0 : case VEC_PACK_SAT_EXPR:
237 0 : return TYPE_UNSIGNED (type) ? vec_pack_usat_optab : vec_pack_ssat_optab;
238 :
239 877 : case VEC_PACK_FIX_TRUNC_EXPR:
240 : /* The signedness is determined from output operand. */
241 877 : return (TYPE_UNSIGNED (type)
242 877 : ? vec_pack_ufix_trunc_optab : vec_pack_sfix_trunc_optab);
243 :
244 241 : case VEC_PACK_FLOAT_EXPR:
245 : /* The signedness is determined from input operand. */
246 241 : return (TYPE_UNSIGNED (type)
247 241 : ? vec_packu_float_optab : vec_packs_float_optab);
248 :
249 0 : case VEC_DUPLICATE_EXPR:
250 0 : return vec_duplicate_optab;
251 :
252 0 : case VEC_SERIES_EXPR:
253 0 : return vec_series_optab;
254 :
255 6411092 : default:
256 6411092 : break;
257 : }
258 :
259 6411092 : trapv = INTEGRAL_TYPE_P (type) && TYPE_OVERFLOW_TRAPS (type);
260 6411092 : switch (code)
261 : {
262 4450764 : case PLUS_EXPR:
263 4450764 : if (TYPE_SATURATING (type))
264 0 : return TYPE_UNSIGNED (type) ? usadd_optab : ssadd_optab;
265 4450764 : return trapv ? addv_optab : add_optab;
266 :
267 911301 : case MINUS_EXPR:
268 911301 : if (TYPE_SATURATING (type))
269 0 : return TYPE_UNSIGNED (type) ? ussub_optab : sssub_optab;
270 911301 : return trapv ? subv_optab : sub_optab;
271 :
272 890962 : case MULT_EXPR:
273 890962 : if (TYPE_SATURATING (type))
274 0 : return TYPE_UNSIGNED (type) ? usmul_optab : ssmul_optab;
275 890962 : return trapv ? smulv_optab : smul_optab;
276 :
277 65731 : case NEGATE_EXPR:
278 65731 : if (TYPE_SATURATING (type))
279 0 : return TYPE_UNSIGNED (type) ? usneg_optab : ssneg_optab;
280 65731 : return trapv ? negv_optab : neg_optab;
281 :
282 2138 : case ABS_EXPR:
283 2138 : return trapv ? absv_optab : abs_optab;
284 :
285 : case ABSU_EXPR:
286 : return abs_optab;
287 : default:
288 : return unknown_optab;
289 : }
290 : }
291 :
292 : /* Check whether an operation represented by CODE is a 'half' widening operation
293 : in which the input vector type has half the number of bits of the output
294 : vector type e.g. V8QI->V8HI.
295 :
296 : This is handled by widening the inputs using NOP_EXPRs then using a
297 : non-widening stmt e.g. MINUS_EXPR. RTL fusing converts these to the widening
298 : hardware instructions if supported.
299 :
300 : The more typical case (handled in supportable_widening_operation) is where
301 : the input vector type has the same number of bits as the output vector type.
302 : In this case half the elements of the input vectors must be processed at a
303 : time into respective vector outputs with elements twice as wide i.e. a
304 : 'hi'/'lo' pair using codes such as VEC_WIDEN_MINUS_HI/LO.
305 :
306 : Supported widening operations:
307 : WIDEN_MULT_EXPR
308 : WIDEN_LSHIFT_EXPR
309 :
310 : Output:
311 : - CODE1 - The non-widened code, which will be used after the inputs are
312 : converted to the wide type. */
313 : bool
314 243 : supportable_half_widening_operation (enum tree_code code, tree vectype_out,
315 : tree vectype_in, enum tree_code *code1)
316 : {
317 243 : machine_mode m1,m2;
318 243 : enum tree_code dummy_code;
319 243 : optab op;
320 :
321 243 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
322 :
323 243 : m1 = TYPE_MODE (vectype_out);
324 243 : m2 = TYPE_MODE (vectype_in);
325 :
326 243 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
327 : return false;
328 :
329 243 : if (maybe_ne (TYPE_VECTOR_SUBPARTS (vectype_in),
330 486 : TYPE_VECTOR_SUBPARTS (vectype_out)))
331 : return false;
332 :
333 243 : switch (code)
334 : {
335 0 : case WIDEN_LSHIFT_EXPR:
336 0 : *code1 = LSHIFT_EXPR;
337 0 : break;
338 243 : case WIDEN_MULT_EXPR:
339 243 : *code1 = MULT_EXPR;
340 243 : break;
341 : default:
342 : return false;
343 : }
344 :
345 243 : if (!supportable_convert_operation (NOP_EXPR, vectype_out, vectype_in,
346 : &dummy_code))
347 : return false;
348 :
349 209 : op = optab_for_tree_code (*code1, vectype_out, optab_vector);
350 209 : return (optab_handler (op, TYPE_MODE (vectype_out)) != CODE_FOR_nothing);
351 : }
352 :
353 : /* Function supportable_convert_operation
354 :
355 : Check whether an operation represented by the code CODE is a
356 : convert operation that is supported by the target platform in
357 : vector form (i.e., when operating on arguments of type VECTYPE_IN
358 : producing a result of type VECTYPE_OUT).
359 :
360 : Convert operations we currently support directly are FIX_TRUNC and FLOAT.
361 : This function checks if these operations are supported
362 : by the target platform directly (via vector tree-codes).
363 :
364 : Output:
365 : - CODE1 is code of vector operation to be used when
366 : vectorizing the operation, if available. */
367 :
368 : bool
369 72332 : supportable_convert_operation (enum tree_code code,
370 : tree vectype_out, tree vectype_in,
371 : enum tree_code *code1)
372 : {
373 72332 : machine_mode m1,m2;
374 72332 : bool truncp;
375 :
376 72332 : gcc_assert (VECTOR_TYPE_P (vectype_out) && VECTOR_TYPE_P (vectype_in));
377 :
378 72332 : m1 = TYPE_MODE (vectype_out);
379 72332 : m2 = TYPE_MODE (vectype_in);
380 :
381 72332 : if (!VECTOR_MODE_P (m1) || !VECTOR_MODE_P (m2))
382 : return false;
383 :
384 : /* First check if we can done conversion directly. */
385 72175 : if ((code == FIX_TRUNC_EXPR
386 1164 : && can_fix_p (m1,m2,TYPE_UNSIGNED (vectype_out), &truncp)
387 : != CODE_FOR_nothing)
388 72475 : || (code == FLOAT_EXPR
389 6528 : && can_float_p (m1,m2,TYPE_UNSIGNED (vectype_in))
390 : != CODE_FOR_nothing))
391 : {
392 6959 : *code1 = code;
393 6959 : return true;
394 : }
395 :
396 130432 : if (GET_MODE_UNIT_PRECISION (m1) > GET_MODE_UNIT_PRECISION (m2)
397 65216 : && can_extend_p (m1, m2, TYPE_UNSIGNED (vectype_in)))
398 : {
399 6449 : *code1 = code;
400 6449 : return true;
401 : }
402 :
403 117534 : if (GET_MODE_UNIT_PRECISION (m1) < GET_MODE_UNIT_PRECISION (m2)
404 64192 : && convert_optab_handler (trunc_optab, m1, m2) != CODE_FOR_nothing)
405 : {
406 3422 : *code1 = code;
407 3422 : return true;
408 : }
409 :
410 : return false;
411 : }
412 :
413 : /* Return true iff vec_cmp_optab/vec_cmpu_optab can handle a vector comparison
414 : for code CODE, comparing operands of type VALUE_TYPE and producing a result
415 : of type MASK_TYPE. */
416 :
417 : static bool
418 1316706 : vec_cmp_icode_p (tree value_type, tree mask_type, enum tree_code code)
419 : {
420 1316706 : enum rtx_code rcode = get_rtx_code_1 (code, TYPE_UNSIGNED (value_type));
421 1316706 : if (rcode == UNKNOWN)
422 : return false;
423 :
424 1316706 : return can_vec_cmp_compare_p (rcode, TYPE_MODE (value_type),
425 2633412 : TYPE_MODE (mask_type));
426 : }
427 :
428 : /* Return true iff vec_cmpeq_optab can handle a vector comparison for code
429 : CODE, comparing operands of type VALUE_TYPE and producing a result of type
430 : MASK_TYPE. */
431 :
432 : static bool
433 362051 : vec_cmp_eq_icode_p (tree value_type, tree mask_type, enum tree_code code)
434 : {
435 362051 : if (code != EQ_EXPR && code != NE_EXPR)
436 : return false;
437 :
438 180155 : return get_vec_cmp_eq_icode (TYPE_MODE (value_type), TYPE_MODE (mask_type))
439 180155 : != CODE_FOR_nothing;
440 : }
441 :
442 : /* Return TRUE if appropriate vector insn is available
443 : for vector comparison expr with vector type VALUE_TYPE
444 : and resulting mask with MASK_TYPE. */
445 :
446 : bool
447 1316706 : expand_vec_cmp_expr_p (tree value_type, tree mask_type, enum tree_code code)
448 : {
449 1316706 : return vec_cmp_icode_p (value_type, mask_type, code)
450 1316706 : || vec_cmp_eq_icode_p (value_type, mask_type, code);
451 : }
452 :
453 : /* Return TRUE iff, appropriate vector insns are available
454 : for vector cond expr with vector type VALUE_TYPE and a comparison
455 : with operand vector types in CMP_OP_TYPE. */
456 :
457 : bool
458 95304 : expand_vec_cond_expr_p (tree value_type, tree cmp_op_type)
459 : {
460 95304 : if (VECTOR_BOOLEAN_TYPE_P (cmp_op_type)
461 285912 : && get_vcond_mask_icode (TYPE_MODE (value_type),
462 95304 : TYPE_MODE (cmp_op_type)) != CODE_FOR_nothing)
463 : return true;
464 :
465 : return false;
466 : }
467 :
468 : /* Use the current target and options to initialize
469 : TREE_OPTIMIZATION_OPTABS (OPTNODE). */
470 :
471 : void
472 1650796 : init_tree_optimization_optabs (tree optnode)
473 : {
474 : /* Quick exit if we have already computed optabs for this target. */
475 1650796 : if (TREE_OPTIMIZATION_BASE_OPTABS (optnode) == this_target_optabs)
476 : return;
477 :
478 : /* Forget any previous information and set up for the current target. */
479 23889 : TREE_OPTIMIZATION_BASE_OPTABS (optnode) = this_target_optabs;
480 23889 : struct target_optabs *tmp_optabs = (struct target_optabs *)
481 23889 : TREE_OPTIMIZATION_OPTABS (optnode);
482 23889 : if (tmp_optabs)
483 436 : memset (tmp_optabs, 0, sizeof (struct target_optabs));
484 : else
485 23453 : tmp_optabs = ggc_cleared_alloc<target_optabs> ();
486 :
487 : /* Generate a new set of optabs into tmp_optabs. */
488 23889 : init_all_optabs (tmp_optabs);
489 :
490 : /* If the optabs changed, record it. */
491 23889 : if (memcmp (tmp_optabs, this_target_optabs, sizeof (struct target_optabs)))
492 10469 : TREE_OPTIMIZATION_OPTABS (optnode) = tmp_optabs;
493 : else
494 : {
495 13420 : TREE_OPTIMIZATION_OPTABS (optnode) = NULL;
496 13420 : ggc_free (tmp_optabs);
497 : }
498 : }
499 :
500 : /* Return TRUE if the target has support for vector right shift of an
501 : operand of type TYPE. If OT_TYPE is OPTAB_DEFAULT, check for existence
502 : of a shift by either a scalar or a vector. Otherwise, check only
503 : for a shift that matches OT_TYPE. */
504 :
505 : bool
506 272964 : target_supports_op_p (tree type, enum tree_code code,
507 : enum optab_subtype ot_subtype)
508 : {
509 272964 : optab ot = optab_for_tree_code (code, type, ot_subtype);
510 272964 : return ot != unknown_optab && can_implement_p (ot, TYPE_MODE (type));
511 : }
512 :
513 : /* Return true if the target has support for masked load/store.
514 : We can support masked load/store by either mask{load,store}
515 : or mask_len_{load,store}.
516 : This helper function checks whether target supports masked
517 : load/store and return corresponding IFN in the last argument
518 : (IFN_MASK_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
519 : If there is support and ELSVALS is nonzero store the possible else values
520 : in the vector it points to. */
521 :
522 : bool
523 234013 : target_supports_mask_load_store_p (machine_mode mode, machine_mode mask_mode,
524 : bool is_load, internal_fn *ifn,
525 : vec<int> *elsvals)
526 : {
527 234013 : optab op = is_load ? maskload_optab : maskstore_optab;
528 54594 : optab len_op = is_load ? mask_len_load_optab : mask_len_store_optab;
529 234013 : enum insn_code icode;
530 234013 : if ((icode = convert_optab_handler (op, mode, mask_mode))
531 : != CODE_FOR_nothing)
532 : {
533 71853 : if (ifn)
534 2163 : *ifn = is_load ? IFN_MASK_LOAD : IFN_MASK_STORE;
535 71853 : if (elsvals && is_load)
536 45974 : get_supported_else_vals (icode,
537 45974 : internal_fn_else_index (IFN_MASK_LOAD),
538 : *elsvals);
539 71853 : return true;
540 : }
541 162160 : else if ((icode = convert_optab_handler (len_op, mode, mask_mode))
542 : != CODE_FOR_nothing)
543 : {
544 0 : if (ifn)
545 0 : *ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
546 0 : if (elsvals && is_load)
547 0 : get_supported_else_vals (icode,
548 0 : internal_fn_else_index (IFN_MASK_LEN_LOAD),
549 : *elsvals);
550 0 : return true;
551 : }
552 : return false;
553 : }
554 :
555 : /* Return true if target supports vector masked load/store for mode.
556 : An additional output in the last argument which is the IFN pointer.
557 : We set IFN as MASK_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
558 : which optab is supported in the target.
559 : If there is support and ELSVALS is nonzero store the possible else values
560 : in the vector it points to. */
561 :
562 : bool
563 212832 : can_vec_mask_load_store_p (machine_mode mode,
564 : machine_mode mask_mode,
565 : bool is_load,
566 : internal_fn *ifn,
567 : vec<int> *elsvals)
568 : {
569 212832 : machine_mode vmode;
570 :
571 : /* If mode is vector mode, check it directly. */
572 212832 : if (VECTOR_MODE_P (mode))
573 200896 : return target_supports_mask_load_store_p (mode, mask_mode, is_load, ifn,
574 200896 : elsvals);
575 :
576 : /* Otherwise, return true if there is some vector mode with
577 : the mask load/store supported. */
578 :
579 : /* See if there is any chance the mask load or store might be
580 : vectorized. If not, punt. */
581 11936 : scalar_mode smode;
582 11936 : if (!is_a <scalar_mode> (mode, &smode))
583 : return false;
584 :
585 11936 : vmode = targetm.vectorize.preferred_simd_mode (smode);
586 2263 : if (VECTOR_MODE_P (vmode)
587 11758 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
588 23694 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
589 : elsvals))
590 2237 : return true;
591 :
592 9699 : auto_vector_modes vector_modes;
593 9699 : targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
594 56783 : for (machine_mode base_mode : vector_modes)
595 27686 : if (related_vector_mode (base_mode, smode).exists (&vmode)
596 49045 : && targetm.vectorize.get_mask_mode (vmode).exists (&mask_mode)
597 21359 : && target_supports_mask_load_store_p (vmode, mask_mode, is_load, ifn,
598 : elsvals))
599 0 : return true;
600 : return false;
601 9699 : }
602 :
603 : /* Return true if the target has support for len load/store.
604 : We can support len load/store by either len_{load,store}
605 : or mask_len_{load,store}.
606 : This helper function checks whether target supports len
607 : load/store and return corresponding IFN in the last argument
608 : (IFN_LEN_{LOAD,STORE} or IFN_MASK_LEN_{LOAD,STORE}).
609 : If there is support and ELSVALS is nonzero store thepossible
610 : else values in the vector it points to. */
611 :
612 : static bool
613 393602 : target_supports_len_load_store_p (machine_mode mode, bool is_load,
614 : internal_fn *ifn, vec<int> *elsvals)
615 : {
616 393602 : optab op = is_load ? len_load_optab : len_store_optab;
617 91440 : optab masked_op = is_load ? mask_len_load_optab : mask_len_store_optab;
618 393602 : internal_fn which_ifn;
619 :
620 393602 : enum insn_code icode;
621 393602 : if ((icode = direct_optab_handler (op, mode)) != CODE_FOR_nothing)
622 : {
623 0 : which_ifn = is_load ? IFN_LEN_LOAD : IFN_LEN_STORE;
624 : }
625 393602 : machine_mode mask_mode;
626 393602 : if (!icode
627 787204 : && targetm.vectorize.get_mask_mode (mode).exists (&mask_mode)
628 393602 : && ((icode = convert_optab_handler (masked_op, mode, mask_mode))
629 : != CODE_FOR_nothing))
630 0 : which_ifn = is_load ? IFN_MASK_LEN_LOAD : IFN_MASK_LEN_STORE;
631 :
632 393602 : if (icode && elsvals && is_load)
633 0 : get_supported_else_vals (icode, internal_fn_else_index (which_ifn),
634 : *elsvals);
635 :
636 393602 : if (icode && ifn)
637 0 : *ifn = which_ifn;
638 393602 : return icode;
639 : }
640 :
641 : /* If target supports vector load/store with length for vector mode MODE,
642 : return the corresponding vector mode, otherwise return opt_machine_mode ().
643 : There are two flavors for vector load/store with length, one is to measure
644 : length with bytes, the other is to measure length with lanes.
645 : As len_{load,store} optabs point out, for the flavor with bytes, we use
646 : VnQI to wrap the other supportable same size vector modes.
647 : An additional output in the last argument which is the IFN pointer.
648 : We set IFN as LEN_{LOAD,STORE} or MASK_LEN_{LOAD,STORE} according
649 : which optab is supported in the target.
650 : If there is support and ELSVALS is nonzero store the possible else values
651 : in the vector it points to. */
652 :
653 : opt_machine_mode
654 196802 : get_len_load_store_mode (machine_mode mode, bool is_load, internal_fn *ifn,
655 : vec<int> *elsvals)
656 : {
657 196802 : gcc_assert (VECTOR_MODE_P (mode));
658 :
659 : /* Check if length in lanes supported for this mode directly. */
660 196802 : if (target_supports_len_load_store_p (mode, is_load, ifn, elsvals))
661 0 : return mode;
662 :
663 : /* Check if length in bytes supported for same vector size VnQI. */
664 196802 : machine_mode vmode;
665 393604 : poly_uint64 nunits = GET_MODE_SIZE (mode);
666 196802 : if (related_vector_mode (mode, QImode, nunits).exists (&vmode)
667 196800 : && target_supports_len_load_store_p (vmode, is_load, ifn, elsvals))
668 0 : return vmode;
669 :
670 196802 : return opt_machine_mode ();
671 : }
|
