Line data Source code
1 : /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "predict.h"
30 : #include "tm_p.h"
31 : #include "optabs.h"
32 : #include "expmed.h"
33 : #include "emit-rtl.h"
34 : #include "recog.h"
35 : #include "diagnostic-core.h"
36 : #include "rtx-vector-builder.h"
37 :
38 : /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 : is properly defined. */
40 : #include "stor-layout.h"
41 : #include "except.h"
42 : #include "dojump.h"
43 : #include "explow.h"
44 : #include "expr.h"
45 : #include "optabs-tree.h"
46 : #include "libfuncs.h"
47 : #include "internal-fn.h"
48 : #include "langhooks.h"
49 : #include "gimple.h"
50 : #include "ssa.h"
51 : #include "tree-ssa-live.h"
52 : #include "tree-outof-ssa.h"
53 :
54 : static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
55 : machine_mode *);
56 : static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
57 : static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
58 :
59 : static rtx emit_conditional_move_1 (rtx, rtx, rtx, rtx, machine_mode);
60 :
61 : /* Debug facility for use in GDB. */
62 : void debug_optab_libfuncs (void);
63 :
64 : /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
65 : the result of operation CODE applied to OP0 (and OP1 if it is a binary
66 : operation). OP0_MODE is OP0's mode.
67 :
68 : If the last insn does not set TARGET, don't do anything, but return true.
69 :
70 : If the last insn or a previous insn sets TARGET and TARGET is one of OP0
71 : or OP1, don't add the REG_EQUAL note but return false. Our caller can then
72 : try again, ensuring that TARGET is not one of the operands. */
73 :
74 : static bool
75 1198207 : add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
76 : rtx op1, machine_mode op0_mode)
77 : {
78 1198207 : rtx_insn *last_insn;
79 1198207 : rtx set;
80 1198207 : rtx note;
81 :
82 2396414 : gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
83 :
84 1198207 : if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
85 : && GET_RTX_CLASS (code) != RTX_BIN_ARITH
86 : && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
87 : && GET_RTX_CLASS (code) != RTX_COMPARE
88 : && GET_RTX_CLASS (code) != RTX_UNARY)
89 : return true;
90 :
91 1191012 : if (GET_CODE (target) == ZERO_EXTRACT)
92 : return true;
93 :
94 : for (last_insn = insns;
95 2586661 : NEXT_INSN (last_insn) != NULL_RTX;
96 : last_insn = NEXT_INSN (last_insn))
97 : ;
98 :
99 : /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
100 : a value changing in the insn, so the note would be invalid for CSE. */
101 1191012 : if (reg_overlap_mentioned_p (target, op0)
102 1191012 : || (op1 && reg_overlap_mentioned_p (target, op1)))
103 : {
104 48807 : if (MEM_P (target)
105 48807 : && (rtx_equal_p (target, op0)
106 41467 : || (op1 && rtx_equal_p (target, op1))))
107 : {
108 : /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
109 : over expanding it as temp = MEM op X, MEM = temp. If the target
110 : supports MEM = MEM op X instructions, it is sometimes too hard
111 : to reconstruct that form later, especially if X is also a memory,
112 : and due to multiple occurrences of addresses the address might
113 : be forced into register unnecessarily.
114 : Note that not emitting the REG_EQUIV note might inhibit
115 : CSE in some cases. */
116 3305 : set = single_set (last_insn);
117 3305 : if (set
118 3305 : && GET_CODE (SET_SRC (set)) == code
119 3294 : && MEM_P (SET_DEST (set))
120 6599 : && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
121 0 : || (op1 && rtx_equal_p (SET_DEST (set),
122 0 : XEXP (SET_SRC (set), 1)))))
123 3294 : return true;
124 : }
125 45513 : return false;
126 : }
127 :
128 1142205 : set = set_for_reg_notes (last_insn);
129 1142205 : if (set == NULL_RTX)
130 : return true;
131 :
132 689919 : if (! rtx_equal_p (SET_DEST (set), target)
133 : /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
134 689919 : && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
135 0 : || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
136 311 : return true;
137 :
138 689608 : if (GET_RTX_CLASS (code) == RTX_UNARY)
139 40284 : switch (code)
140 : {
141 2986 : case FFS:
142 2986 : case CLZ:
143 2986 : case CTZ:
144 2986 : case CLRSB:
145 2986 : case POPCOUNT:
146 2986 : case PARITY:
147 2986 : case BSWAP:
148 2986 : if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
149 : {
150 230 : note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
151 230 : if (GET_MODE_UNIT_SIZE (op0_mode)
152 460 : > GET_MODE_UNIT_SIZE (GET_MODE (target)))
153 230 : note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
154 : note, op0_mode);
155 : else
156 0 : note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
157 : note, op0_mode);
158 : break;
159 : }
160 : /* FALLTHRU */
161 40054 : default:
162 40054 : note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
163 40054 : break;
164 : }
165 : else
166 649324 : note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
167 :
168 689608 : set_unique_reg_note (last_insn, REG_EQUAL, note);
169 :
170 689608 : return true;
171 : }
172 :
173 : /* Given two input operands, OP0 and OP1, determine what the correct from_mode
174 : for a widening operation would be. In most cases this would be OP0, but if
175 : that's a constant it'll be VOIDmode, which isn't useful. */
176 :
177 : static machine_mode
178 32219 : widened_mode (machine_mode to_mode, rtx op0, rtx op1)
179 : {
180 32219 : machine_mode m0 = GET_MODE (op0);
181 32219 : machine_mode m1 = GET_MODE (op1);
182 32219 : machine_mode result;
183 :
184 32219 : if (m0 == VOIDmode && m1 == VOIDmode)
185 : return to_mode;
186 64438 : else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
187 : result = m1;
188 : else
189 : result = m0;
190 :
191 96657 : if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
192 0 : return to_mode;
193 :
194 : return result;
195 : }
196 :
197 : /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
198 : says whether OP is signed or unsigned. NO_EXTEND is true if we need
199 : not actually do a sign-extend or zero-extend, but can leave the
200 : higher-order bits of the result rtx undefined, for example, in the case
201 : of logical operations, but not right shifts. */
202 :
203 : static rtx
204 538 : widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
205 : int unsignedp, bool no_extend)
206 : {
207 538 : rtx result;
208 538 : scalar_int_mode int_mode;
209 :
210 : /* If we don't have to extend and this is a constant, return it. */
211 538 : if (no_extend && GET_MODE (op) == VOIDmode)
212 : return op;
213 :
214 : /* If we must extend do so. If OP is a SUBREG for a promoted object, also
215 : extend since it will be more efficient to do so unless the signedness of
216 : a promoted object differs from our extension. */
217 7 : if (! no_extend
218 538 : || !is_a <scalar_int_mode> (mode, &int_mode)
219 7 : || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
220 0 : && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
221 531 : return convert_modes (mode, oldmode, op, unsignedp);
222 :
223 : /* If MODE is no wider than a single word, we return a lowpart or paradoxical
224 : SUBREG. */
225 14 : if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
226 7 : return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
227 :
228 : /* Otherwise, get an object of MODE, clobber it, and set the low-order
229 : part to OP. */
230 :
231 0 : result = gen_reg_rtx (int_mode);
232 0 : emit_clobber (result);
233 0 : emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
234 0 : return result;
235 : }
236 :
237 : /* Expand vector widening operations.
238 :
239 : There are two different classes of operations handled here:
240 : 1) Operations whose result is wider than all the arguments to the operation.
241 : Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
242 : In this case OP0 and optionally OP1 would be initialized,
243 : but WIDE_OP wouldn't (not relevant for this case).
244 : 2) Operations whose result is of the same size as the last argument to the
245 : operation, but wider than all the other arguments to the operation.
246 : Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
247 : In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
248 :
249 : E.g, when called to expand the following operations, this is how
250 : the arguments will be initialized:
251 : nops OP0 OP1 WIDE_OP
252 : widening-sum 2 oprnd0 - oprnd1
253 : widening-dot-product 3 oprnd0 oprnd1 oprnd2
254 : widening-mult 2 oprnd0 oprnd1 -
255 : type-promotion (vec-unpack) 1 oprnd0 - - */
256 :
257 : rtx
258 20624 : expand_widen_pattern_expr (const_sepops ops, rtx op0, rtx op1, rtx wide_op,
259 : rtx target, int unsignedp)
260 : {
261 20624 : class expand_operand eops[4];
262 20624 : tree oprnd0, oprnd1, oprnd2;
263 20624 : machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
264 20624 : optab widen_pattern_optab;
265 20624 : enum insn_code icode;
266 20624 : int nops = TREE_CODE_LENGTH (ops->code);
267 20624 : int op;
268 20624 : bool sbool = false;
269 :
270 20624 : oprnd0 = ops->op0;
271 20624 : oprnd1 = nops >= 2 ? ops->op1 : NULL_TREE;
272 1520 : oprnd2 = nops >= 3 ? ops->op2 : NULL_TREE;
273 :
274 20624 : tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
275 20624 : if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
276 20624 : || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
277 : /* The sign is from the result type rather than operand's type
278 : for these ops. */
279 34 : widen_pattern_optab
280 34 : = optab_for_tree_code (ops->code, ops->type, optab_default);
281 20590 : else if ((ops->code == VEC_UNPACK_HI_EXPR
282 20590 : || ops->code == VEC_UNPACK_LO_EXPR)
283 17346 : && VECTOR_BOOLEAN_TYPE_P (ops->type)
284 4804 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
285 4804 : && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
286 21074 : && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
287 : {
288 : /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
289 : the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
290 : vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
291 : the pattern number of elements in the wider vector. */
292 484 : widen_pattern_optab
293 239 : = (ops->code == VEC_UNPACK_HI_EXPR
294 484 : ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
295 : sbool = true;
296 : }
297 20106 : else if (ops->code == DOT_PROD_EXPR)
298 : {
299 395 : enum optab_subtype subtype = optab_default;
300 395 : signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0));
301 395 : signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1));
302 395 : if (sign1 == sign2)
303 : ;
304 79 : else if (sign1 == SIGNED && sign2 == UNSIGNED)
305 : {
306 : subtype = optab_vector_mixed_sign;
307 : /* Same as optab_vector_mixed_sign but flip the operands. */
308 : std::swap (op0, op1);
309 : }
310 65 : else if (sign1 == UNSIGNED && sign2 == SIGNED)
311 : subtype = optab_vector_mixed_sign;
312 : else
313 0 : gcc_unreachable ();
314 :
315 395 : widen_pattern_optab
316 395 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype);
317 : }
318 : else
319 19711 : widen_pattern_optab
320 19711 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
321 20624 : if (ops->code == WIDEN_MULT_PLUS_EXPR
322 20624 : || ops->code == WIDEN_MULT_MINUS_EXPR
323 20624 : || ops->code == DOT_PROD_EXPR)
324 395 : icode = find_widening_optab_handler (widen_pattern_optab,
325 : TYPE_MODE (TREE_TYPE (ops->op2)),
326 : tmode0);
327 20229 : else if (ops->code == WIDEN_SUM_EXPR)
328 0 : icode = find_widening_optab_handler (widen_pattern_optab,
329 : TYPE_MODE (TREE_TYPE (ops->op1)),
330 : tmode0);
331 : else
332 20229 : icode = optab_handler (widen_pattern_optab, tmode0);
333 20624 : gcc_assert (icode != CODE_FOR_nothing);
334 :
335 20624 : if (nops >= 2)
336 1520 : tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
337 19104 : else if (sbool)
338 : {
339 484 : nops = 2;
340 484 : op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
341 484 : tmode1 = tmode0;
342 : }
343 :
344 : /* The last operand is of a wider mode than the rest of the operands. */
345 20624 : if (nops == 2)
346 : wmode = tmode1;
347 19129 : else if (nops == 3)
348 : {
349 509 : gcc_assert (tmode1 == tmode0);
350 509 : gcc_assert (op1);
351 509 : wmode = TYPE_MODE (TREE_TYPE (oprnd2));
352 : }
353 :
354 20624 : op = 0;
355 20624 : create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
356 20624 : create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
357 20624 : if (op1)
358 2004 : create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
359 20624 : if (wide_op)
360 509 : create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
361 20624 : expand_insn (icode, op, eops);
362 20624 : return eops[0].value;
363 : }
364 :
365 : /* Generate code to perform an operation specified by TERNARY_OPTAB
366 : on operands OP0, OP1 and OP2, with result having machine-mode MODE.
367 :
368 : UNSIGNEDP is for the case where we have to widen the operands
369 : to perform the operation. It says to use zero-extension.
370 :
371 : If TARGET is nonzero, the value
372 : is generated there, if it is convenient to do so.
373 : In all cases an rtx is returned for the locus of the value;
374 : this may or may not be TARGET. */
375 :
376 : rtx
377 0 : expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
378 : rtx op1, rtx op2, rtx target, int unsignedp)
379 : {
380 0 : class expand_operand ops[4];
381 0 : enum insn_code icode = optab_handler (ternary_optab, mode);
382 :
383 0 : gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
384 :
385 0 : create_output_operand (&ops[0], target, mode);
386 0 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
387 0 : create_convert_operand_from (&ops[2], op1, mode, unsignedp);
388 0 : create_convert_operand_from (&ops[3], op2, mode, unsignedp);
389 0 : expand_insn (icode, 4, ops);
390 0 : return ops[0].value;
391 : }
392 :
393 :
394 : /* Like expand_binop, but return a constant rtx if the result can be
395 : calculated at compile time. The arguments and return value are
396 : otherwise the same as for expand_binop. */
397 :
398 : rtx
399 1121 : simplify_expand_binop (machine_mode mode, optab binoptab,
400 : rtx op0, rtx op1, rtx target, int unsignedp,
401 : enum optab_methods methods)
402 : {
403 1121 : if (CONSTANT_P (op0) && CONSTANT_P (op1))
404 : {
405 0 : rtx x = simplify_binary_operation (optab_to_code (binoptab),
406 : mode, op0, op1);
407 0 : if (x)
408 : return x;
409 : }
410 :
411 1121 : return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
412 : }
413 :
414 : /* Like simplify_expand_binop, but always put the result in TARGET.
415 : Return true if the expansion succeeded. */
416 :
417 : bool
418 1121 : force_expand_binop (machine_mode mode, optab binoptab,
419 : rtx op0, rtx op1, rtx target, int unsignedp,
420 : enum optab_methods methods)
421 : {
422 1121 : rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
423 : target, unsignedp, methods);
424 1121 : if (x == 0)
425 : return false;
426 1121 : if (x != target)
427 0 : emit_move_insn (target, x);
428 : return true;
429 : }
430 :
431 : /* Create a new vector value in VMODE with all elements set to OP. If OP
432 : is not a constant, the mode of it must be the element mode of VMODE
433 : (if the element is BImode, additionally OP is allowed to be in QImode).
434 : If OP is a constant, then the return value will be a constant. */
435 :
436 : rtx
437 58 : expand_vector_broadcast (machine_mode vmode, rtx op)
438 : {
439 58 : int n;
440 58 : rtvec vec;
441 :
442 58 : gcc_checking_assert (VECTOR_MODE_P (vmode));
443 58 : gcc_checking_assert (CONST_INT_P (op)
444 : || GET_MODE_INNER (vmode) == GET_MODE (op)
445 : || (GET_MODE_INNER (vmode) == BImode
446 : && GET_MODE (op) == QImode));
447 :
448 58 : if (valid_for_const_vector_p (vmode, op))
449 58 : return gen_const_vec_duplicate (vmode, op);
450 :
451 0 : insn_code icode = optab_handler (vec_duplicate_optab, vmode);
452 0 : if (icode != CODE_FOR_nothing)
453 : {
454 0 : class expand_operand ops[2];
455 0 : create_output_operand (&ops[0], NULL_RTX, vmode);
456 0 : create_input_operand (&ops[1], op, GET_MODE (op));
457 0 : expand_insn (icode, 2, ops);
458 0 : return ops[0].value;
459 : }
460 :
461 0 : if (!GET_MODE_NUNITS (vmode).is_constant (&n))
462 : return NULL;
463 :
464 : /* ??? If the target doesn't have a vec_init, then we have no easy way
465 : of performing this operation. Most of this sort of generic support
466 : is hidden away in the vector lowering support in gimple. */
467 0 : icode = convert_optab_handler (vec_init_optab, vmode,
468 0 : GET_MODE_INNER (vmode));
469 0 : if (icode == CODE_FOR_nothing)
470 : return NULL;
471 :
472 0 : vec = rtvec_alloc (n);
473 0 : for (int i = 0; i < n; ++i)
474 0 : RTVEC_ELT (vec, i) = op;
475 0 : rtx ret = gen_reg_rtx (vmode);
476 0 : emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
477 :
478 0 : return ret;
479 : }
480 :
481 : /* This subroutine of expand_doubleword_shift handles the cases in which
482 : the effective shift value is >= BITS_PER_WORD. The arguments and return
483 : value are the same as for the parent routine, except that SUPERWORD_OP1
484 : is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
485 : INTO_TARGET may be null if the caller has decided to calculate it. */
486 :
487 : static bool
488 0 : expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
489 : rtx outof_target, rtx into_target,
490 : int unsignedp, enum optab_methods methods)
491 : {
492 0 : if (into_target != 0)
493 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
494 : into_target, unsignedp, methods))
495 : return false;
496 :
497 0 : if (outof_target != 0)
498 : {
499 : /* For a signed right shift, we must fill OUTOF_TARGET with copies
500 : of the sign bit, otherwise we must fill it with zeros. */
501 0 : if (binoptab != ashr_optab)
502 0 : emit_move_insn (outof_target, CONST0_RTX (word_mode));
503 : else
504 0 : if (!force_expand_binop (word_mode, binoptab, outof_input,
505 : gen_int_shift_amount (word_mode,
506 0 : BITS_PER_WORD - 1),
507 : outof_target, unsignedp, methods))
508 : return false;
509 : }
510 : return true;
511 : }
512 :
513 : /* This subroutine of expand_doubleword_shift handles the cases in which
514 : the effective shift value is < BITS_PER_WORD. The arguments and return
515 : value are the same as for the parent routine. */
516 :
517 : static bool
518 0 : expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
519 : rtx outof_input, rtx into_input, rtx op1,
520 : rtx outof_target, rtx into_target,
521 : int unsignedp, enum optab_methods methods,
522 : unsigned HOST_WIDE_INT shift_mask)
523 : {
524 0 : optab reverse_unsigned_shift, unsigned_shift;
525 0 : rtx tmp, carries;
526 :
527 0 : reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
528 0 : unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
529 :
530 : /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
531 : We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
532 : the opposite direction to BINOPTAB. */
533 0 : if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
534 : {
535 0 : carries = outof_input;
536 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
537 : op1_mode), op1_mode);
538 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
539 : 0, true, methods);
540 : }
541 : else
542 : {
543 : /* We must avoid shifting by BITS_PER_WORD bits since that is either
544 : the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
545 : has unknown behavior. Do a single shift first, then shift by the
546 : remainder. It's OK to use ~OP1 as the remainder if shift counts
547 : are truncated to the mode size. */
548 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
549 : outof_input, const1_rtx, 0,
550 : unsignedp, methods);
551 0 : if (carries == const0_rtx)
552 : tmp = const0_rtx;
553 0 : else if (shift_mask == BITS_PER_WORD - 1)
554 0 : tmp = expand_unop (op1_mode, one_cmpl_optab, op1, 0, true);
555 : else
556 : {
557 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
558 : op1_mode), op1_mode);
559 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
560 : 0, true, methods);
561 : }
562 : }
563 0 : if (tmp == 0 || carries == 0)
564 : return false;
565 0 : if (carries != const0_rtx && tmp != const0_rtx)
566 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
567 : carries, tmp, 0, unsignedp, methods);
568 0 : if (carries == 0)
569 : return false;
570 :
571 0 : if (into_input != const0_rtx)
572 : {
573 : /* Shift INTO_INPUT logically by OP1. This is the last use of
574 : INTO_INPUT so the result can go directly into INTO_TARGET if
575 : convenient. */
576 0 : tmp = simplify_expand_binop (word_mode, unsigned_shift, into_input,
577 : op1, into_target, unsignedp, methods);
578 0 : if (tmp == 0)
579 : return false;
580 :
581 : /* Now OR/PLUS in the bits carried over from OUTOF_INPUT. */
582 0 : if (!force_expand_binop (word_mode, add_optab, tmp, carries,
583 : into_target, unsignedp, methods))
584 : return false;
585 : }
586 : else
587 0 : emit_move_insn (into_target, carries);
588 :
589 : /* Use a standard word_mode shift for the out-of half. */
590 0 : if (outof_target != 0)
591 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
592 : outof_target, unsignedp, methods))
593 : return false;
594 :
595 : return true;
596 : }
597 :
598 :
599 : /* Try implementing expand_doubleword_shift using conditional moves.
600 : The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
601 : otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
602 : are the shift counts to use in the former and latter case. All other
603 : arguments are the same as the parent routine. */
604 :
605 : static bool
606 0 : expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
607 : enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
608 : rtx outof_input, rtx into_input,
609 : rtx subword_op1, rtx superword_op1,
610 : rtx outof_target, rtx into_target,
611 : int unsignedp, enum optab_methods methods,
612 : unsigned HOST_WIDE_INT shift_mask)
613 : {
614 0 : rtx outof_superword, into_superword;
615 :
616 : /* Put the superword version of the output into OUTOF_SUPERWORD and
617 : INTO_SUPERWORD. */
618 0 : outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
619 0 : if (outof_target != 0 && subword_op1 == superword_op1)
620 : {
621 : /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
622 : OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
623 0 : into_superword = outof_target;
624 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
625 : outof_superword, 0, unsignedp, methods))
626 : return false;
627 : }
628 : else
629 : {
630 0 : into_superword = gen_reg_rtx (word_mode);
631 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
632 : outof_superword, into_superword,
633 : unsignedp, methods))
634 : return false;
635 : }
636 :
637 : /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
638 0 : if (!expand_subword_shift (op1_mode, binoptab,
639 : outof_input, into_input, subword_op1,
640 : outof_target, into_target,
641 : unsignedp, methods, shift_mask))
642 : return false;
643 :
644 : /* Select between them. Do the INTO half first because INTO_SUPERWORD
645 : might be the current value of OUTOF_TARGET. */
646 0 : if (!emit_conditional_move (into_target, { cmp_code, cmp1, cmp2, op1_mode },
647 : into_target, into_superword, word_mode, false))
648 : return false;
649 :
650 0 : if (outof_target != 0)
651 0 : if (!emit_conditional_move (outof_target,
652 0 : { cmp_code, cmp1, cmp2, op1_mode },
653 : outof_target, outof_superword,
654 : word_mode, false))
655 : return false;
656 :
657 : return true;
658 : }
659 :
660 : /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
661 : OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
662 : input operand; the shift moves bits in the direction OUTOF_INPUT->
663 : INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
664 : of the target. OP1 is the shift count and OP1_MODE is its mode.
665 : If OP1 is constant, it will have been truncated as appropriate
666 : and is known to be nonzero.
667 :
668 : If SHIFT_MASK is zero, the result of word shifts is undefined when the
669 : shift count is outside the range [0, BITS_PER_WORD). This routine must
670 : avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
671 :
672 : If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
673 : masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
674 : fill with zeros or sign bits as appropriate.
675 :
676 : If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
677 : a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
678 : Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
679 : In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
680 : are undefined.
681 :
682 : BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
683 : may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
684 : OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
685 : function wants to calculate it itself.
686 :
687 : Return true if the shift could be successfully synthesized. */
688 :
689 : static bool
690 0 : expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
691 : rtx outof_input, rtx into_input, rtx op1,
692 : rtx outof_target, rtx into_target,
693 : int unsignedp, enum optab_methods methods,
694 : unsigned HOST_WIDE_INT shift_mask)
695 : {
696 0 : rtx superword_op1, tmp, cmp1, cmp2;
697 0 : enum rtx_code cmp_code;
698 :
699 : /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
700 : fill the result with sign or zero bits as appropriate. If so, the value
701 : of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
702 : this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
703 : and INTO_INPUT), then emit code to set up OUTOF_TARGET.
704 :
705 : This isn't worthwhile for constant shifts since the optimizers will
706 : cope better with in-range shift counts. */
707 0 : if (shift_mask >= BITS_PER_WORD
708 0 : && outof_target != 0
709 0 : && !CONSTANT_P (op1))
710 : {
711 0 : if (!expand_doubleword_shift (op1_mode, binoptab,
712 : outof_input, into_input, op1,
713 : 0, into_target,
714 : unsignedp, methods, shift_mask))
715 : return false;
716 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
717 : outof_target, unsignedp, methods))
718 : return false;
719 : return true;
720 : }
721 :
722 : /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
723 : is true when the effective shift value is less than BITS_PER_WORD.
724 : Set SUPERWORD_OP1 to the shift count that should be used to shift
725 : OUTOF_INPUT into INTO_TARGET when the condition is false. */
726 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
727 0 : if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
728 : {
729 : /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
730 : is a subword shift count. */
731 0 : cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
732 : 0, true, methods);
733 0 : cmp2 = CONST0_RTX (op1_mode);
734 0 : cmp_code = EQ;
735 0 : superword_op1 = op1;
736 : }
737 : else
738 : {
739 : /* Set CMP1 to OP1 - BITS_PER_WORD. */
740 0 : cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
741 : 0, true, methods);
742 0 : cmp2 = CONST0_RTX (op1_mode);
743 0 : cmp_code = LT;
744 0 : superword_op1 = cmp1;
745 : }
746 0 : if (cmp1 == 0)
747 : return false;
748 :
749 : /* If we can compute the condition at compile time, pick the
750 : appropriate subroutine. */
751 0 : tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
752 0 : if (tmp != 0 && CONST_INT_P (tmp))
753 : {
754 0 : if (tmp == const0_rtx)
755 0 : return expand_superword_shift (binoptab, outof_input, superword_op1,
756 : outof_target, into_target,
757 0 : unsignedp, methods);
758 : else
759 0 : return expand_subword_shift (op1_mode, binoptab,
760 : outof_input, into_input, op1,
761 : outof_target, into_target,
762 0 : unsignedp, methods, shift_mask);
763 : }
764 :
765 : /* Try using conditional moves to generate straight-line code. */
766 0 : if (HAVE_conditional_move)
767 : {
768 0 : rtx_insn *start = get_last_insn ();
769 0 : if (expand_doubleword_shift_condmove (op1_mode, binoptab,
770 : cmp_code, cmp1, cmp2,
771 : outof_input, into_input,
772 : op1, superword_op1,
773 : outof_target, into_target,
774 : unsignedp, methods, shift_mask))
775 : return true;
776 0 : delete_insns_since (start);
777 : }
778 :
779 : /* As a last resort, use branches to select the correct alternative. */
780 0 : rtx_code_label *subword_label = gen_label_rtx ();
781 0 : rtx_code_label *done_label = gen_label_rtx ();
782 :
783 0 : NO_DEFER_POP;
784 0 : do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
785 : 0, 0, subword_label,
786 : profile_probability::uninitialized ());
787 0 : OK_DEFER_POP;
788 :
789 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
790 : outof_target, into_target,
791 : unsignedp, methods))
792 : return false;
793 :
794 0 : emit_jump_insn (targetm.gen_jump (done_label));
795 0 : emit_barrier ();
796 0 : emit_label (subword_label);
797 :
798 0 : if (!expand_subword_shift (op1_mode, binoptab,
799 : outof_input, into_input, op1,
800 : outof_target, into_target,
801 : unsignedp, methods, shift_mask))
802 : return false;
803 :
804 0 : emit_label (done_label);
805 0 : return true;
806 : }
807 :
808 : /* Subroutine of expand_binop. Perform a double word multiplication of
809 : operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
810 : as the target's word_mode. This function return NULL_RTX if anything
811 : goes wrong, in which case it may have already emitted instructions
812 : which need to be deleted.
813 :
814 : If we want to multiply two two-word values and have normal and widening
815 : multiplies of single-word values, we can do this with three smaller
816 : multiplications.
817 :
818 : The multiplication proceeds as follows:
819 : _______________________
820 : [__op0_high_|__op0_low__]
821 : _______________________
822 : * [__op1_high_|__op1_low__]
823 : _______________________________________________
824 : _______________________
825 : (1) [__op0_low__*__op1_low__]
826 : _______________________
827 : (2a) [__op0_low__*__op1_high_]
828 : _______________________
829 : (2b) [__op0_high_*__op1_low__]
830 : _______________________
831 : (3) [__op0_high_*__op1_high_]
832 :
833 :
834 : This gives a 4-word result. Since we are only interested in the
835 : lower 2 words, partial result (3) and the upper words of (2a) and
836 : (2b) don't need to be calculated. Hence (2a) and (2b) can be
837 : calculated using non-widening multiplication.
838 :
839 : (1), however, needs to be calculated with an unsigned widening
840 : multiplication. If this operation is not directly supported we
841 : try using a signed widening multiplication and adjust the result.
842 : This adjustment works as follows:
843 :
844 : If both operands are positive then no adjustment is needed.
845 :
846 : If the operands have different signs, for example op0_low < 0 and
847 : op1_low >= 0, the instruction treats the most significant bit of
848 : op0_low as a sign bit instead of a bit with significance
849 : 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
850 : with 2**BITS_PER_WORD - op0_low, and two's complements the
851 : result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
852 : the result.
853 :
854 : Similarly, if both operands are negative, we need to add
855 : (op0_low + op1_low) * 2**BITS_PER_WORD.
856 :
857 : We use a trick to adjust quickly. We logically shift op0_low right
858 : (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
859 : op0_high (op1_high) before it is used to calculate 2b (2a). If no
860 : logical shift exists, we do an arithmetic right shift and subtract
861 : the 0 or -1. */
862 :
863 : static rtx
864 14501 : expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
865 : bool umulp, enum optab_methods methods)
866 : {
867 14501 : int low = (WORDS_BIG_ENDIAN ? 1 : 0);
868 14501 : int high = (WORDS_BIG_ENDIAN ? 0 : 1);
869 14501 : rtx wordm1 = (umulp ? NULL_RTX
870 0 : : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
871 14501 : rtx product, adjust, product_high, temp;
872 :
873 14501 : rtx op0_high = operand_subword_force (op0, high, mode);
874 14501 : rtx op0_low = operand_subword_force (op0, low, mode);
875 14501 : rtx op1_high = operand_subword_force (op1, high, mode);
876 14501 : rtx op1_low = operand_subword_force (op1, low, mode);
877 :
878 : /* If we're using an unsigned multiply to directly compute the product
879 : of the low-order words of the operands and perform any required
880 : adjustments of the operands, we begin by trying two more multiplications
881 : and then computing the appropriate sum.
882 :
883 : We have checked above that the required addition is provided.
884 : Full-word addition will normally always succeed, especially if
885 : it is provided at all, so we don't worry about its failure. The
886 : multiplication may well fail, however, so we do handle that. */
887 :
888 14501 : if (!umulp)
889 : {
890 : /* ??? This could be done with emit_store_flag where available. */
891 0 : temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
892 : NULL_RTX, 1, methods);
893 0 : if (temp)
894 0 : op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
895 : NULL_RTX, 0, OPTAB_DIRECT);
896 : else
897 : {
898 0 : temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
899 : NULL_RTX, 0, methods);
900 0 : if (!temp)
901 : return NULL_RTX;
902 0 : op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
903 : NULL_RTX, 0, OPTAB_DIRECT);
904 : }
905 :
906 0 : if (!op0_high)
907 : return NULL_RTX;
908 : }
909 :
910 14501 : adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
911 : NULL_RTX, 0, OPTAB_DIRECT);
912 14501 : if (!adjust)
913 : return NULL_RTX;
914 :
915 : /* OP0_HIGH should now be dead. */
916 :
917 14501 : if (!umulp)
918 : {
919 : /* ??? This could be done with emit_store_flag where available. */
920 0 : temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
921 : NULL_RTX, 1, methods);
922 0 : if (temp)
923 0 : op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
924 : NULL_RTX, 0, OPTAB_DIRECT);
925 : else
926 : {
927 0 : temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
928 : NULL_RTX, 0, methods);
929 0 : if (!temp)
930 : return NULL_RTX;
931 0 : op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
932 : NULL_RTX, 0, OPTAB_DIRECT);
933 : }
934 :
935 0 : if (!op1_high)
936 : return NULL_RTX;
937 : }
938 :
939 14501 : temp = expand_binop (word_mode, smul_optab, op1_high, op0_low,
940 : NULL_RTX, 0, OPTAB_DIRECT);
941 14501 : if (!temp)
942 : return NULL_RTX;
943 :
944 : /* OP1_HIGH should now be dead. */
945 :
946 14501 : adjust = expand_binop (word_mode, add_optab, adjust, temp,
947 : NULL_RTX, 0, OPTAB_DIRECT);
948 :
949 14501 : if (target && !REG_P (target))
950 8212 : target = NULL_RTX;
951 :
952 : /* *_widen_optab needs to determine operand mode, make sure at least
953 : one operand has non-VOID mode. */
954 14501 : if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
955 0 : op0_low = force_reg (word_mode, op0_low);
956 :
957 14501 : if (umulp)
958 14501 : product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
959 : target, 1, OPTAB_DIRECT);
960 : else
961 0 : product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
962 : target, 1, OPTAB_DIRECT);
963 :
964 14501 : if (!product)
965 : return NULL_RTX;
966 :
967 14501 : product_high = operand_subword (product, high, 1, mode);
968 14501 : adjust = expand_binop (word_mode, add_optab, product_high, adjust,
969 : NULL_RTX, 0, OPTAB_DIRECT);
970 14501 : emit_move_insn (product_high, adjust);
971 14501 : return product;
972 : }
973 :
974 : /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
975 : constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
976 : (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
977 : computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
978 : + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
979 : depends on the bit value, if 2, then carry from the addition needs to be
980 : added too, i.e. like:
981 : sum += __builtin_add_overflow (low, high, &sum)
982 :
983 : Optimize signed double-word OP0 % OP1 similarly, just apply some correction
984 : factor to the sum before doing unsigned remainder, in the form of
985 : sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
986 : then perform unsigned
987 : remainder = sum % OP1;
988 : and finally
989 : remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
990 :
991 : static rtx
992 22329 : expand_doubleword_mod (machine_mode mode, rtx op0, rtx op1, bool unsignedp)
993 : {
994 22329 : if (INTVAL (op1) <= 1 || (INTVAL (op1) & 1) == 0)
995 : return NULL_RTX;
996 :
997 22008 : rtx_insn *last = get_last_insn ();
998 748218 : for (int bit = BITS_PER_WORD; bit >= BITS_PER_WORD / 2; bit--)
999 : {
1000 703595 : wide_int w = wi::shifted_mask (bit, 1, false, 2 * BITS_PER_WORD);
1001 674714 : if (wi::ne_p (wi::umod_trunc (w, INTVAL (op1)), 1))
1002 673406 : continue;
1003 1308 : rtx sum = NULL_RTX, mask = NULL_RTX;
1004 1769 : if (bit == BITS_PER_WORD)
1005 : {
1006 : /* For signed modulo we need to add correction to the sum
1007 : and that might again overflow. */
1008 360 : if (!unsignedp)
1009 136 : continue;
1010 224 : if (optab_handler (uaddv4_optab, word_mode) == CODE_FOR_nothing)
1011 0 : continue;
1012 224 : tree wtype = lang_hooks.types.type_for_mode (word_mode, 1);
1013 224 : if (wtype == NULL_TREE)
1014 0 : continue;
1015 224 : tree ctype = build_complex_type (wtype);
1016 224 : if (TYPE_MODE (ctype) != GET_MODE_COMPLEX_MODE (word_mode))
1017 0 : continue;
1018 224 : machine_mode cmode = TYPE_MODE (ctype);
1019 224 : rtx op00 = operand_subword_force (op0, 0, mode);
1020 224 : rtx op01 = operand_subword_force (op0, 1, mode);
1021 224 : rtx cres = gen_rtx_CONCAT (cmode, gen_reg_rtx (word_mode),
1022 : gen_reg_rtx (word_mode));
1023 224 : tree lhs = make_tree (ctype, cres);
1024 224 : tree arg0 = make_tree (wtype, op00);
1025 224 : tree arg1 = make_tree (wtype, op01);
1026 224 : expand_addsub_overflow (UNKNOWN_LOCATION, PLUS_EXPR, lhs, arg0,
1027 : arg1, true, true, true, false, NULL);
1028 224 : sum = expand_simple_binop (word_mode, PLUS, XEXP (cres, 0),
1029 : XEXP (cres, 1), NULL_RTX, 1,
1030 : OPTAB_DIRECT);
1031 224 : if (sum == NULL_RTX)
1032 : return NULL_RTX;
1033 : }
1034 : else
1035 : {
1036 : /* Code below uses GEN_INT, so we need the masks to be representable
1037 : in HOST_WIDE_INTs. */
1038 948 : if (bit >= HOST_BITS_PER_WIDE_INT)
1039 0 : continue;
1040 : /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1041 : overflow. Consider 64-bit -1ULL for word size 32, if we add
1042 : 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1043 1170 : if (bit == BITS_PER_WORD - 1)
1044 147 : continue;
1045 :
1046 801 : int count = (2 * BITS_PER_WORD + bit - 1) / bit;
1047 801 : rtx sum_corr = NULL_RTX;
1048 :
1049 801 : if (!unsignedp)
1050 : {
1051 : /* For signed modulo, compute it as unsigned modulo of
1052 : sum with a correction added to it if OP0 is negative,
1053 : such that the result can be computed as unsigned
1054 : remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1055 496 : w = wi::min_value (2 * BITS_PER_WORD, SIGNED);
1056 406 : wide_int wmod1 = wi::umod_trunc (w, INTVAL (op1));
1057 406 : wide_int wmod2 = wi::smod_trunc (w, INTVAL (op1));
1058 : /* wmod2 == -wmod1. */
1059 406 : wmod2 = wmod2 + (INTVAL (op1) - 1);
1060 406 : if (wi::ne_p (wmod1, wmod2))
1061 : {
1062 406 : wide_int wcorr = wmod2 - wmod1;
1063 406 : if (wi::neg_p (w))
1064 406 : wcorr = wcorr + INTVAL (op1);
1065 : /* Now verify if the count sums can't overflow, and punt
1066 : if they could. */
1067 496 : w = wi::mask (bit, false, 2 * BITS_PER_WORD);
1068 406 : w = w * (count - 1);
1069 902 : w = w + wi::mask (2 * BITS_PER_WORD - (count - 1) * bit,
1070 812 : false, 2 * BITS_PER_WORD);
1071 406 : w = w + wcorr;
1072 496 : w = wi::lrshift (w, BITS_PER_WORD);
1073 406 : if (wi::ne_p (w, 0))
1074 0 : continue;
1075 :
1076 406 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1077 : mode);
1078 406 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1079 406 : GEN_INT (BITS_PER_WORD - 1),
1080 : NULL_RTX, 0, OPTAB_DIRECT);
1081 406 : if (mask == NULL_RTX)
1082 : return NULL_RTX;
1083 406 : sum_corr = immed_wide_int_const (wcorr, word_mode);
1084 406 : sum_corr = expand_simple_binop (word_mode, AND, mask,
1085 : sum_corr, NULL_RTX, 1,
1086 : OPTAB_DIRECT);
1087 406 : if (sum_corr == NULL_RTX)
1088 : return NULL_RTX;
1089 406 : }
1090 406 : }
1091 :
1092 3373 : for (int i = 0; i < count; i++)
1093 : {
1094 2572 : rtx v = op0;
1095 2572 : if (i)
1096 1771 : v = expand_simple_binop (mode, LSHIFTRT, v, GEN_INT (i * bit),
1097 : NULL_RTX, 1, OPTAB_DIRECT);
1098 2572 : if (v == NULL_RTX)
1099 : return NULL_RTX;
1100 2572 : v = force_lowpart_subreg (word_mode, v, mode);
1101 2572 : if (v == NULL_RTX)
1102 : return NULL_RTX;
1103 2572 : if (i != count - 1)
1104 1771 : v = expand_simple_binop (word_mode, AND, v,
1105 1771 : GEN_INT ((HOST_WIDE_INT_1U << bit)
1106 : - 1), NULL_RTX, 1,
1107 : OPTAB_DIRECT);
1108 2572 : if (v == NULL_RTX)
1109 : return NULL_RTX;
1110 2572 : if (sum == NULL_RTX)
1111 : sum = v;
1112 : else
1113 1771 : sum = expand_simple_binop (word_mode, PLUS, sum, v, NULL_RTX,
1114 : 1, OPTAB_DIRECT);
1115 2572 : if (sum == NULL_RTX)
1116 : return NULL_RTX;
1117 : }
1118 801 : if (sum_corr)
1119 : {
1120 406 : sum = expand_simple_binop (word_mode, PLUS, sum, sum_corr,
1121 : NULL_RTX, 1, OPTAB_DIRECT);
1122 406 : if (sum == NULL_RTX)
1123 : return NULL_RTX;
1124 : }
1125 : }
1126 1025 : rtx remainder = expand_divmod (1, TRUNC_MOD_EXPR, word_mode, sum,
1127 1025 : gen_int_mode (INTVAL (op1), word_mode),
1128 : NULL_RTX, 1, OPTAB_DIRECT);
1129 1025 : if (remainder == NULL_RTX)
1130 : return NULL_RTX;
1131 :
1132 1025 : if (!unsignedp)
1133 : {
1134 406 : if (mask == NULL_RTX)
1135 : {
1136 0 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1137 : mode);
1138 0 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1139 0 : GEN_INT (BITS_PER_WORD - 1),
1140 : NULL_RTX, 0, OPTAB_DIRECT);
1141 0 : if (mask == NULL_RTX)
1142 : return NULL_RTX;
1143 : }
1144 406 : mask = expand_simple_binop (word_mode, AND, mask,
1145 406 : gen_int_mode (1 - INTVAL (op1),
1146 : word_mode),
1147 : NULL_RTX, 1, OPTAB_DIRECT);
1148 406 : if (mask == NULL_RTX)
1149 : return NULL_RTX;
1150 406 : remainder = expand_simple_binop (word_mode, PLUS, remainder,
1151 : mask, NULL_RTX, 1, OPTAB_DIRECT);
1152 406 : if (remainder == NULL_RTX)
1153 : return NULL_RTX;
1154 : }
1155 :
1156 1025 : remainder = convert_modes (mode, word_mode, remainder, unsignedp);
1157 : /* Punt if we need any library calls. */
1158 1025 : if (last)
1159 929 : last = NEXT_INSN (last);
1160 : else
1161 96 : last = get_insns ();
1162 21089 : for (; last; last = NEXT_INSN (last))
1163 20064 : if (CALL_P (last))
1164 : return NULL_RTX;
1165 : return remainder;
1166 674714 : }
1167 : return NULL_RTX;
1168 : }
1169 :
1170 : /* Similarly to the above function, but compute both quotient and remainder.
1171 : Quotient can be computed from the remainder as:
1172 : rem = op0 % op1; // Handled using expand_doubleword_mod
1173 : quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1174 : // 2 * BITS_PER_WORD
1175 :
1176 : We can also handle cases where op1 is a multiple of power of two constant
1177 : and constant handled by expand_doubleword_mod.
1178 : op11 = 1 << __builtin_ctz (op1);
1179 : op12 = op1 / op11;
1180 : rem1 = op0 % op12; // Handled using expand_doubleword_mod
1181 : quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1182 : // 2 * BITS_PER_WORD
1183 : rem = (quot1 % op11) * op12 + rem1;
1184 : quot = quot1 / op11; */
1185 :
1186 : rtx
1187 22057 : expand_doubleword_divmod (machine_mode mode, rtx op0, rtx op1, rtx *rem,
1188 : bool unsignedp)
1189 : {
1190 22057 : *rem = NULL_RTX;
1191 :
1192 : /* Negative dividend should have been optimized into positive,
1193 : similarly modulo by 1 and modulo by power of two is optimized
1194 : differently too. */
1195 43040 : if (INTVAL (op1) <= 1 || pow2p_hwi (INTVAL (op1)))
1196 : return NULL_RTX;
1197 :
1198 22008 : rtx op11 = const1_rtx;
1199 22008 : rtx op12 = op1;
1200 22008 : if ((INTVAL (op1) & 1) == 0)
1201 : {
1202 4993 : int bit = ctz_hwi (INTVAL (op1));
1203 4993 : op11 = GEN_INT (HOST_WIDE_INT_1 << bit);
1204 4993 : op12 = GEN_INT (INTVAL (op1) >> bit);
1205 : }
1206 :
1207 22008 : rtx rem1 = expand_doubleword_mod (mode, op0, op12, unsignedp);
1208 22008 : if (rem1 == NULL_RTX)
1209 : return NULL_RTX;
1210 :
1211 1025 : int prec = 2 * BITS_PER_WORD;
1212 1025 : wide_int a = wide_int::from (INTVAL (op12), prec + 1, UNSIGNED);
1213 1025 : wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
1214 1025 : wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
1215 1025 : rtx inv = immed_wide_int_const (m, mode);
1216 :
1217 1025 : rtx_insn *last = get_last_insn ();
1218 1025 : rtx quot1 = expand_simple_binop (mode, MINUS, op0, rem1,
1219 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1220 1025 : if (quot1 == NULL_RTX)
1221 : return NULL_RTX;
1222 :
1223 1025 : quot1 = expand_simple_binop (mode, MULT, quot1, inv,
1224 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1225 1025 : if (quot1 == NULL_RTX)
1226 : return NULL_RTX;
1227 :
1228 1025 : if (op11 != const1_rtx)
1229 : {
1230 365 : rtx rem2 = expand_divmod (1, TRUNC_MOD_EXPR, mode, quot1, op11,
1231 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1232 365 : if (rem2 == NULL_RTX)
1233 : return NULL_RTX;
1234 :
1235 365 : rem2 = expand_simple_binop (mode, MULT, rem2, op12, NULL_RTX,
1236 : unsignedp, OPTAB_DIRECT);
1237 365 : if (rem2 == NULL_RTX)
1238 : return NULL_RTX;
1239 :
1240 365 : rem2 = expand_simple_binop (mode, PLUS, rem2, rem1, NULL_RTX,
1241 : unsignedp, OPTAB_DIRECT);
1242 365 : if (rem2 == NULL_RTX)
1243 : return NULL_RTX;
1244 :
1245 365 : rtx quot2 = expand_divmod (0, TRUNC_DIV_EXPR, mode, quot1, op11,
1246 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1247 365 : if (quot2 == NULL_RTX)
1248 : return NULL_RTX;
1249 :
1250 : rem1 = rem2;
1251 : quot1 = quot2;
1252 : }
1253 :
1254 : /* Punt if we need any library calls. */
1255 1025 : if (last)
1256 1025 : last = NEXT_INSN (last);
1257 : else
1258 0 : last = get_insns ();
1259 16360 : for (; last; last = NEXT_INSN (last))
1260 15335 : if (CALL_P (last))
1261 : return NULL_RTX;
1262 :
1263 1025 : *rem = rem1;
1264 1025 : return quot1;
1265 1025 : }
1266 :
1267 : /* Wrapper around expand_binop which takes an rtx code to specify
1268 : the operation to perform, not an optab pointer. All other
1269 : arguments are the same. */
1270 : rtx
1271 4008151 : expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
1272 : rtx op1, rtx target, int unsignedp,
1273 : enum optab_methods methods)
1274 : {
1275 4008151 : optab binop = code_to_optab (code);
1276 4008151 : gcc_assert (binop);
1277 :
1278 4008151 : return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1279 : }
1280 :
1281 : /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1282 : binop. Order them according to commutative_operand_precedence and, if
1283 : possible, try to put TARGET or a pseudo first. */
1284 : static bool
1285 12198111 : swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1286 : {
1287 12198111 : int op0_prec = commutative_operand_precedence (op0);
1288 12198111 : int op1_prec = commutative_operand_precedence (op1);
1289 :
1290 12198111 : if (op0_prec < op1_prec)
1291 : return true;
1292 :
1293 12007011 : if (op0_prec > op1_prec)
1294 : return false;
1295 :
1296 : /* With equal precedence, both orders are ok, but it is better if the
1297 : first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1298 2226660 : if (target == 0 || REG_P (target))
1299 2485106 : return (REG_P (op1) && !REG_P (op0)) || target == op1;
1300 : else
1301 88343 : return rtx_equal_p (op1, target);
1302 : }
1303 :
1304 : /* Return true if BINOPTAB implements a shift operation. */
1305 :
1306 : static bool
1307 23967288 : shift_optab_p (optab binoptab)
1308 : {
1309 9601155 : switch (optab_to_code (binoptab))
1310 : {
1311 : case ASHIFT:
1312 : case SS_ASHIFT:
1313 : case US_ASHIFT:
1314 : case ASHIFTRT:
1315 : case LSHIFTRT:
1316 : case ROTATE:
1317 : case ROTATERT:
1318 : return true;
1319 :
1320 12821306 : default:
1321 0 : return false;
1322 : }
1323 : }
1324 :
1325 : /* Return true if BINOPTAB implements a commutative binary operation. */
1326 :
1327 : static bool
1328 14527097 : commutative_optab_p (optab binoptab)
1329 : {
1330 14527097 : return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1331 : || binoptab == smul_widen_optab
1332 2413107 : || binoptab == umul_widen_optab
1333 : || binoptab == smul_highpart_optab
1334 2380888 : || binoptab == umul_highpart_optab
1335 2328986 : || binoptab == vec_widen_sadd_optab
1336 2328986 : || binoptab == vec_widen_uadd_optab
1337 : || binoptab == vec_widen_sadd_hi_optab
1338 2328986 : || binoptab == vec_widen_sadd_lo_optab
1339 : || binoptab == vec_widen_uadd_hi_optab
1340 2328986 : || binoptab == vec_widen_uadd_lo_optab
1341 : || binoptab == vec_widen_sadd_even_optab
1342 2328986 : || binoptab == vec_widen_sadd_odd_optab
1343 : || binoptab == vec_widen_uadd_even_optab
1344 16856083 : || binoptab == vec_widen_uadd_odd_optab);
1345 : }
1346 :
1347 : /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1348 : optimizing, and if the operand is a constant that costs more than
1349 : 1 instruction, force the constant into a register and return that
1350 : register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1351 :
1352 : static rtx
1353 27350205 : avoid_expensive_constant (machine_mode mode, optab binoptab,
1354 : int opn, rtx x, bool unsignedp)
1355 : {
1356 27350205 : bool speed = optimize_insn_for_speed_p ();
1357 :
1358 27350205 : if (mode != VOIDmode
1359 27350205 : && optimize
1360 20619556 : && CONSTANT_P (x)
1361 34472109 : && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1362 7121904 : > set_src_cost (x, mode, speed)))
1363 : {
1364 0 : if (CONST_INT_P (x))
1365 : {
1366 0 : HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1367 0 : if (intval != INTVAL (x))
1368 0 : x = GEN_INT (intval);
1369 : }
1370 : else
1371 0 : x = convert_modes (mode, VOIDmode, x, unsignedp);
1372 0 : x = force_reg (mode, x);
1373 : }
1374 27350205 : return x;
1375 : }
1376 :
1377 : /* Helper function for expand_binop: handle the case where there
1378 : is an insn ICODE that directly implements the indicated operation.
1379 : Returns null if this is not possible. */
1380 : static rtx
1381 14335476 : expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1382 : rtx op0, rtx op1,
1383 : rtx target, int unsignedp, enum optab_methods methods)
1384 : {
1385 14335476 : machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1386 14335476 : machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1387 14335476 : machine_mode mode0, mode1, tmp_mode;
1388 14335476 : class expand_operand ops[3];
1389 14335476 : bool commutative_p;
1390 14335476 : rtx_insn *pat;
1391 14335476 : rtx xop0 = op0, xop1 = op1;
1392 14335476 : bool canonicalize_op1 = false;
1393 14335476 : rtx_insn *last = get_last_insn ();
1394 :
1395 : /* If it is a commutative operator and the modes would match
1396 : if we would swap the operands, we can save the conversions. */
1397 14335476 : commutative_p = commutative_optab_p (binoptab);
1398 14335476 : if (commutative_p
1399 12137031 : && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1400 13994 : && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0)
1401 : std::swap (xop0, xop1);
1402 :
1403 : /* If we are optimizing, force expensive constants into a register. */
1404 14335476 : xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1405 14335476 : if (!shift_optab_p (binoptab))
1406 12821306 : xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1407 : else
1408 : /* Shifts and rotates often use a different mode for op1 from op0;
1409 : for VOIDmode constants we don't know the mode, so force it
1410 : to be canonicalized using convert_modes. */
1411 : canonicalize_op1 = true;
1412 :
1413 : /* In case the insn wants input operands in modes different from
1414 : those of the actual operands, convert the operands. It would
1415 : seem that we don't need to convert CONST_INTs, but we do, so
1416 : that they're properly zero-extended, sign-extended or truncated
1417 : for their mode. */
1418 :
1419 14335476 : mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1420 14335476 : if (xmode0 != VOIDmode && xmode0 != mode0)
1421 : {
1422 3433 : xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1423 3433 : mode0 = xmode0;
1424 : }
1425 :
1426 14335476 : mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1427 14335476 : ? GET_MODE (xop1) : mode);
1428 14335476 : if (xmode1 != VOIDmode && xmode1 != mode1)
1429 : {
1430 1566105 : xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1431 1566105 : mode1 = xmode1;
1432 : }
1433 :
1434 : /* If operation is commutative,
1435 : try to make the first operand a register.
1436 : Even better, try to make it the same as the target.
1437 : Also try to make the last operand a constant. */
1438 14335476 : if (commutative_p
1439 14335476 : && swap_commutative_operands_with_target (target, xop0, xop1))
1440 : std::swap (xop0, xop1);
1441 :
1442 : /* Now, if insn's predicates don't allow our operands, put them into
1443 : pseudo regs. */
1444 :
1445 14335476 : if (binoptab == vec_pack_trunc_optab
1446 14335476 : || binoptab == vec_pack_usat_optab
1447 14325178 : || binoptab == vec_pack_ssat_optab
1448 14325178 : || binoptab == vec_pack_ufix_trunc_optab
1449 14325051 : || binoptab == vec_pack_sfix_trunc_optab
1450 14325051 : || binoptab == vec_packu_float_optab
1451 14324831 : || binoptab == vec_packs_float_optab)
1452 : {
1453 : /* The mode of the result is different then the mode of the
1454 : arguments. */
1455 10665 : tmp_mode = insn_data[(int) icode].operand[0].mode;
1456 2575 : if (VECTOR_MODE_P (mode)
1457 30435 : && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1458 : return NULL_RTX;
1459 : }
1460 : else
1461 : tmp_mode = mode;
1462 :
1463 14335476 : create_output_operand (&ops[0], target, tmp_mode);
1464 14335476 : create_input_operand (&ops[1], xop0, mode0);
1465 14335476 : create_input_operand (&ops[2], xop1, mode1);
1466 14335476 : pat = maybe_gen_insn (icode, 3, ops);
1467 14335476 : if (pat)
1468 : {
1469 : /* If PAT is composed of more than one insn, try to add an appropriate
1470 : REG_EQUAL note to it. If we can't because TEMP conflicts with an
1471 : operand, call expand_binop again, this time without a target. */
1472 13238265 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1473 15483284 : && ! add_equal_note (pat, ops[0].value,
1474 : optab_to_code (binoptab),
1475 : ops[1].value, ops[2].value, mode0))
1476 : {
1477 44406 : delete_insns_since (last);
1478 44406 : return expand_binop_directly (icode, mode, binoptab, op0, op1,
1479 44406 : NULL_RTX, unsignedp, methods);
1480 : }
1481 :
1482 14291068 : emit_insn (pat);
1483 14291068 : return ops[0].value;
1484 : }
1485 :
1486 : return NULL_RTX;
1487 : }
1488 :
1489 : /* Generate code to perform an operation specified by BINOPTAB
1490 : on operands OP0 and OP1, with result having machine-mode MODE.
1491 :
1492 : UNSIGNEDP is for the case where we have to widen the operands
1493 : to perform the operation. It says to use zero-extension.
1494 :
1495 : If TARGET is nonzero, the value
1496 : is generated there, if it is convenient to do so.
1497 : In all cases an rtx is returned for the locus of the value;
1498 : this may or may not be TARGET. */
1499 :
1500 : rtx
1501 14482853 : expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1502 : rtx target, int unsignedp, enum optab_methods methods)
1503 : {
1504 12478320 : enum optab_methods next_methods
1505 14482853 : = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1506 14482853 : ? OPTAB_WIDEN : methods);
1507 14482853 : enum mode_class mclass;
1508 14482853 : enum insn_code icode;
1509 14482853 : machine_mode wider_mode;
1510 14482853 : scalar_int_mode int_mode;
1511 14482853 : rtx libfunc;
1512 14482853 : rtx temp;
1513 14482853 : rtx_insn *entry_last = get_last_insn ();
1514 14482853 : rtx_insn *last;
1515 :
1516 14482853 : mclass = GET_MODE_CLASS (mode);
1517 :
1518 : /* If subtracting an integer constant, convert this into an addition of
1519 : the negated constant. */
1520 :
1521 14482853 : if (binoptab == sub_optab && CONST_INT_P (op1))
1522 : {
1523 1033483 : op1 = negate_rtx (mode, op1);
1524 1033483 : binoptab = add_optab;
1525 : }
1526 : /* For shifts, constant invalid op1 might be expanded from different
1527 : mode than MODE. As those are invalid, force them to a register
1528 : to avoid further problems during expansion. */
1529 13449370 : else if (CONST_INT_P (op1)
1530 15940246 : && shift_optab_p (binoptab)
1531 17821549 : && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1532 : {
1533 108 : op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1534 108 : op1 = force_reg (GET_MODE_INNER (mode), op1);
1535 : }
1536 :
1537 : /* Record where to delete back to if we backtrack. */
1538 14482853 : last = get_last_insn ();
1539 :
1540 : /* If we can do it with a three-operand insn, do so. */
1541 :
1542 14482853 : if (methods != OPTAB_MUST_WIDEN)
1543 : {
1544 14482853 : if (convert_optab_p (binoptab))
1545 : {
1546 32219 : machine_mode from_mode = widened_mode (mode, op0, op1);
1547 32219 : icode = find_widening_optab_handler (binoptab, mode, from_mode);
1548 : }
1549 : else
1550 14450634 : icode = optab_handler (binoptab, mode);
1551 14482853 : if (icode != CODE_FOR_nothing)
1552 : {
1553 14291069 : temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1554 : target, unsignedp, methods);
1555 14291069 : if (temp)
1556 : return temp;
1557 1 : delete_insns_since (last);
1558 : }
1559 : }
1560 :
1561 : /* If we were trying to rotate, and that didn't work, try rotating
1562 : the other direction before falling back to shifts and bitwise-or. */
1563 191785 : if (((binoptab == rotl_optab
1564 23 : && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1565 191784 : || (binoptab == rotr_optab
1566 11 : && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1567 191786 : && is_int_mode (mode, &int_mode))
1568 : {
1569 1 : optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1570 1 : rtx newop1;
1571 1 : unsigned int bits = GET_MODE_PRECISION (int_mode);
1572 :
1573 1 : if (CONST_INT_P (op1))
1574 0 : newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1575 1 : else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1576 0 : newop1 = negate_rtx (GET_MODE (op1), op1);
1577 : else
1578 1 : newop1 = expand_binop (GET_MODE (op1), sub_optab,
1579 1 : gen_int_mode (bits, GET_MODE (op1)), op1,
1580 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1581 :
1582 1 : temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1583 : target, unsignedp, methods);
1584 1 : if (temp)
1585 14482853 : return temp;
1586 1 : delete_insns_since (last);
1587 : }
1588 :
1589 : /* If this is a multiply, see if we can do a widening operation that
1590 : takes operands of this mode and makes a wider mode. */
1591 :
1592 191785 : if (binoptab == smul_optab
1593 25284 : && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1594 210167 : && (convert_optab_handler ((unsignedp
1595 : ? umul_widen_optab
1596 : : smul_widen_optab),
1597 : wider_mode, mode) != CODE_FOR_nothing))
1598 : {
1599 : /* *_widen_optab needs to determine operand mode, make sure at least
1600 : one operand has non-VOID mode. */
1601 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1602 0 : op0 = force_reg (mode, op0);
1603 0 : temp = expand_binop (wider_mode,
1604 : unsignedp ? umul_widen_optab : smul_widen_optab,
1605 : op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1606 :
1607 0 : if (temp != 0)
1608 : {
1609 0 : if (GET_MODE_CLASS (mode) == MODE_INT
1610 0 : && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1611 0 : return gen_lowpart (mode, temp);
1612 : else
1613 0 : return convert_to_mode (mode, temp, unsignedp);
1614 : }
1615 : }
1616 :
1617 : /* If this is a vector shift by a scalar, see if we can do a vector
1618 : shift by a vector. If so, broadcast the scalar into a vector. */
1619 191785 : if (mclass == MODE_VECTOR_INT)
1620 : {
1621 33 : optab otheroptab = unknown_optab;
1622 :
1623 33 : if (binoptab == ashl_optab)
1624 : otheroptab = vashl_optab;
1625 : else if (binoptab == ashr_optab)
1626 : otheroptab = vashr_optab;
1627 : else if (binoptab == lshr_optab)
1628 : otheroptab = vlshr_optab;
1629 : else if (binoptab == rotl_optab)
1630 : otheroptab = vrotl_optab;
1631 : else if (binoptab == rotr_optab)
1632 : otheroptab = vrotr_optab;
1633 :
1634 : if (otheroptab
1635 33 : && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1636 : {
1637 : /* The scalar may be wider or narrower than the vector element.
1638 : Truncate or extend it to the proper size to fit in the
1639 : broadcast vector. */
1640 0 : scalar_mode inner_mode = GET_MODE_INNER (mode);
1641 0 : if (!CONST_INT_P (op1))
1642 : {
1643 0 : auto mode1 = as_a <scalar_int_mode> (GET_MODE (op1));
1644 0 : int size1 = GET_MODE_BITSIZE (mode1);
1645 0 : int inner_size = GET_MODE_BITSIZE (inner_mode);
1646 :
1647 0 : if (size1 != inner_size)
1648 : {
1649 0 : auto unary = size1 > inner_size ? TRUNCATE : ZERO_EXTEND;
1650 0 : op1 = force_reg (inner_mode,
1651 : simplify_gen_unary (unary, inner_mode,
1652 : op1, mode1));
1653 : }
1654 : }
1655 :
1656 0 : rtx vop1 = expand_vector_broadcast (mode, op1);
1657 0 : if (vop1)
1658 : {
1659 0 : temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1660 : target, unsignedp, methods);
1661 0 : if (temp)
1662 14482853 : return temp;
1663 0 : delete_insns_since (last);
1664 : }
1665 : }
1666 : }
1667 :
1668 : /* Look for a wider mode of the same class for which we think we
1669 : can open-code the operation. Check for a widening multiply at the
1670 : wider mode as well. */
1671 :
1672 191785 : if (CLASS_HAS_WIDER_MODES_P (mclass)
1673 191752 : && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1674 116023 : FOR_EACH_WIDER_MODE (wider_mode, mode)
1675 : {
1676 62492 : machine_mode next_mode;
1677 62492 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1678 62492 : || (binoptab == smul_optab
1679 78314 : && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1680 19456 : && (find_widening_optab_handler ((unsignedp
1681 : ? umul_widen_optab
1682 : : smul_widen_optab),
1683 : next_mode, mode)
1684 : != CODE_FOR_nothing)))
1685 : {
1686 164 : rtx xop0 = op0, xop1 = op1;
1687 164 : bool no_extend = false;
1688 :
1689 : /* For certain integer operations, we need not actually extend
1690 : the narrow operands, as long as we will truncate
1691 : the results to the same narrowness. */
1692 :
1693 164 : if ((binoptab == ior_optab || binoptab == and_optab
1694 164 : || binoptab == xor_optab
1695 164 : || binoptab == add_optab || binoptab == sub_optab
1696 : || binoptab == smul_optab || binoptab == ashl_optab)
1697 89 : && mclass == MODE_INT)
1698 : {
1699 7 : no_extend = true;
1700 7 : xop0 = avoid_expensive_constant (mode, binoptab, 0,
1701 : xop0, unsignedp);
1702 7 : if (binoptab != ashl_optab)
1703 0 : xop1 = avoid_expensive_constant (mode, binoptab, 1,
1704 : xop1, unsignedp);
1705 : }
1706 :
1707 164 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1708 :
1709 : /* The second operand of a shift must always be extended. */
1710 328 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1711 164 : no_extend && binoptab != ashl_optab);
1712 :
1713 164 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1714 : unsignedp, OPTAB_DIRECT);
1715 164 : if (temp)
1716 : {
1717 164 : if (mclass != MODE_INT
1718 164 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1719 : {
1720 157 : if (target == 0)
1721 49 : target = gen_reg_rtx (mode);
1722 157 : convert_move (target, temp, 0);
1723 157 : return target;
1724 : }
1725 : else
1726 7 : return gen_lowpart (mode, temp);
1727 : }
1728 : else
1729 0 : delete_insns_since (last);
1730 : }
1731 : }
1732 :
1733 : /* If operation is commutative,
1734 : try to make the first operand a register.
1735 : Even better, try to make it the same as the target.
1736 : Also try to make the last operand a constant. */
1737 191621 : if (commutative_optab_p (binoptab)
1738 191621 : && swap_commutative_operands_with_target (target, op0, op1))
1739 : std::swap (op0, op1);
1740 :
1741 : /* These can be done a word at a time. */
1742 191621 : if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1743 0 : && is_int_mode (mode, &int_mode)
1744 0 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1745 191621 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1746 : {
1747 0 : int i;
1748 0 : rtx_insn *insns;
1749 :
1750 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1751 : won't be accurate, so use a new target. */
1752 0 : if (target == 0
1753 0 : || target == op0
1754 0 : || target == op1
1755 0 : || reg_overlap_mentioned_p (target, op0)
1756 0 : || reg_overlap_mentioned_p (target, op1)
1757 0 : || !valid_multiword_target_p (target))
1758 0 : target = gen_reg_rtx (int_mode);
1759 :
1760 0 : start_sequence ();
1761 :
1762 : /* Do the actual arithmetic. */
1763 0 : machine_mode op0_mode = GET_MODE (op0);
1764 0 : machine_mode op1_mode = GET_MODE (op1);
1765 0 : if (op0_mode == VOIDmode)
1766 0 : op0_mode = int_mode;
1767 0 : if (op1_mode == VOIDmode)
1768 0 : op1_mode = int_mode;
1769 0 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1770 : {
1771 0 : rtx target_piece = operand_subword (target, i, 1, int_mode);
1772 0 : rtx x = expand_binop (word_mode, binoptab,
1773 0 : operand_subword_force (op0, i, op0_mode),
1774 0 : operand_subword_force (op1, i, op1_mode),
1775 : target_piece, unsignedp, next_methods);
1776 :
1777 0 : if (x == 0)
1778 : break;
1779 :
1780 0 : if (target_piece != x)
1781 0 : emit_move_insn (target_piece, x);
1782 : }
1783 :
1784 0 : insns = end_sequence ();
1785 :
1786 0 : if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1787 : {
1788 0 : emit_insn (insns);
1789 0 : return target;
1790 : }
1791 : }
1792 :
1793 : /* Synthesize double word shifts from single word shifts. */
1794 191621 : if ((binoptab == lshr_optab || binoptab == ashl_optab
1795 191621 : || binoptab == ashr_optab)
1796 0 : && is_int_mode (mode, &int_mode)
1797 0 : && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1798 0 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1799 0 : && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1800 0 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1801 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1802 191621 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1803 : {
1804 0 : unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1805 0 : scalar_int_mode op1_mode;
1806 :
1807 0 : double_shift_mask = targetm.shift_truncation_mask (int_mode);
1808 0 : shift_mask = targetm.shift_truncation_mask (word_mode);
1809 0 : op1_mode = (GET_MODE (op1) != VOIDmode
1810 0 : ? as_a <scalar_int_mode> (GET_MODE (op1))
1811 : : word_mode);
1812 :
1813 : /* Apply the truncation to constant shifts. */
1814 0 : if (double_shift_mask > 0 && CONST_INT_P (op1))
1815 0 : op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1816 :
1817 0 : if (op1 == CONST0_RTX (op1_mode))
1818 14482853 : return op0;
1819 :
1820 : /* Make sure that this is a combination that expand_doubleword_shift
1821 : can handle. See the comments there for details. */
1822 0 : if (double_shift_mask == 0
1823 0 : || (shift_mask == BITS_PER_WORD - 1
1824 0 : && double_shift_mask == BITS_PER_WORD * 2 - 1))
1825 : {
1826 0 : rtx_insn *insns;
1827 0 : rtx into_target, outof_target;
1828 0 : rtx into_input, outof_input;
1829 0 : int left_shift, outof_word;
1830 :
1831 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1832 : won't be accurate, so use a new target. */
1833 0 : if (target == 0
1834 0 : || target == op0
1835 0 : || target == op1
1836 0 : || reg_overlap_mentioned_p (target, op0)
1837 0 : || reg_overlap_mentioned_p (target, op1)
1838 0 : || !valid_multiword_target_p (target))
1839 0 : target = gen_reg_rtx (int_mode);
1840 :
1841 0 : start_sequence ();
1842 :
1843 : /* OUTOF_* is the word we are shifting bits away from, and
1844 : INTO_* is the word that we are shifting bits towards, thus
1845 : they differ depending on the direction of the shift and
1846 : WORDS_BIG_ENDIAN. */
1847 :
1848 0 : left_shift = binoptab == ashl_optab;
1849 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1850 :
1851 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1852 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1853 :
1854 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1855 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1856 :
1857 0 : if (expand_doubleword_shift (op1_mode, binoptab,
1858 : outof_input, into_input, op1,
1859 : outof_target, into_target,
1860 : unsignedp, next_methods, shift_mask))
1861 : {
1862 0 : insns = end_sequence ();
1863 :
1864 0 : emit_insn (insns);
1865 0 : return target;
1866 : }
1867 0 : end_sequence ();
1868 : }
1869 : }
1870 :
1871 : /* Synthesize double word rotates from single word shifts. */
1872 191621 : if ((binoptab == rotl_optab || binoptab == rotr_optab)
1873 34 : && is_int_mode (mode, &int_mode)
1874 1 : && CONST_INT_P (op1)
1875 0 : && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1876 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1877 191621 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1878 : {
1879 0 : rtx_insn *insns;
1880 0 : rtx into_target, outof_target;
1881 0 : rtx into_input, outof_input;
1882 0 : rtx inter;
1883 0 : int shift_count, left_shift, outof_word;
1884 :
1885 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1886 : won't be accurate, so use a new target. Do this also if target is not
1887 : a REG, first because having a register instead may open optimization
1888 : opportunities, and second because if target and op0 happen to be MEMs
1889 : designating the same location, we would risk clobbering it too early
1890 : in the code sequence we generate below. */
1891 0 : if (target == 0
1892 0 : || target == op0
1893 0 : || target == op1
1894 0 : || !REG_P (target)
1895 0 : || reg_overlap_mentioned_p (target, op0)
1896 0 : || reg_overlap_mentioned_p (target, op1)
1897 0 : || !valid_multiword_target_p (target))
1898 0 : target = gen_reg_rtx (int_mode);
1899 :
1900 0 : start_sequence ();
1901 :
1902 0 : shift_count = INTVAL (op1);
1903 :
1904 : /* OUTOF_* is the word we are shifting bits away from, and
1905 : INTO_* is the word that we are shifting bits towards, thus
1906 : they differ depending on the direction of the shift and
1907 : WORDS_BIG_ENDIAN. */
1908 :
1909 0 : left_shift = (binoptab == rotl_optab);
1910 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1911 :
1912 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1913 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1914 :
1915 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1916 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1917 :
1918 0 : if (shift_count == BITS_PER_WORD)
1919 : {
1920 : /* This is just a word swap. */
1921 0 : emit_move_insn (outof_target, into_input);
1922 0 : emit_move_insn (into_target, outof_input);
1923 0 : inter = const0_rtx;
1924 : }
1925 : else
1926 : {
1927 0 : rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1928 0 : HOST_WIDE_INT first_shift_count, second_shift_count;
1929 0 : optab reverse_unsigned_shift, unsigned_shift;
1930 :
1931 0 : reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1932 0 : ? lshr_optab : ashl_optab);
1933 :
1934 0 : unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1935 0 : ? ashl_optab : lshr_optab);
1936 :
1937 0 : if (shift_count > BITS_PER_WORD)
1938 : {
1939 0 : first_shift_count = shift_count - BITS_PER_WORD;
1940 0 : second_shift_count = 2 * BITS_PER_WORD - shift_count;
1941 : }
1942 : else
1943 : {
1944 0 : first_shift_count = BITS_PER_WORD - shift_count;
1945 0 : second_shift_count = shift_count;
1946 : }
1947 0 : rtx first_shift_count_rtx
1948 0 : = gen_int_shift_amount (word_mode, first_shift_count);
1949 0 : rtx second_shift_count_rtx
1950 0 : = gen_int_shift_amount (word_mode, second_shift_count);
1951 :
1952 0 : into_temp1 = expand_binop (word_mode, unsigned_shift,
1953 : outof_input, first_shift_count_rtx,
1954 : NULL_RTX, unsignedp, next_methods);
1955 0 : into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1956 : into_input, second_shift_count_rtx,
1957 : NULL_RTX, unsignedp, next_methods);
1958 :
1959 0 : if (into_temp1 != 0 && into_temp2 != 0)
1960 0 : inter = expand_binop (word_mode, add_optab, into_temp1, into_temp2,
1961 : into_target, unsignedp, next_methods);
1962 : else
1963 : inter = 0;
1964 :
1965 0 : if (inter != 0 && inter != into_target)
1966 0 : emit_move_insn (into_target, inter);
1967 :
1968 0 : outof_temp1 = expand_binop (word_mode, unsigned_shift,
1969 : into_input, first_shift_count_rtx,
1970 : NULL_RTX, unsignedp, next_methods);
1971 0 : outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1972 : outof_input, second_shift_count_rtx,
1973 : NULL_RTX, unsignedp, next_methods);
1974 :
1975 0 : if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1976 0 : inter = expand_binop (word_mode, add_optab,
1977 : outof_temp1, outof_temp2,
1978 : outof_target, unsignedp, next_methods);
1979 :
1980 0 : if (inter != 0 && inter != outof_target)
1981 0 : emit_move_insn (outof_target, inter);
1982 : }
1983 :
1984 0 : insns = end_sequence ();
1985 :
1986 0 : if (inter != 0)
1987 : {
1988 0 : emit_insn (insns);
1989 0 : return target;
1990 : }
1991 : }
1992 :
1993 : /* These can be done a word at a time by propagating carries. */
1994 191621 : if ((binoptab == add_optab || binoptab == sub_optab)
1995 13508 : && is_int_mode (mode, &int_mode)
1996 0 : && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
1997 191621 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1998 : {
1999 0 : unsigned int i;
2000 0 : optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
2001 0 : const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
2002 0 : rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
2003 0 : rtx xop0, xop1, xtarget;
2004 :
2005 : /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
2006 : value is one of those, use it. Otherwise, use 1 since it is the
2007 : one easiest to get. */
2008 : #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
2009 0 : int normalizep = STORE_FLAG_VALUE;
2010 : #else
2011 : int normalizep = 1;
2012 : #endif
2013 :
2014 : /* Prepare the operands. */
2015 0 : xop0 = force_reg (int_mode, op0);
2016 0 : xop1 = force_reg (int_mode, op1);
2017 :
2018 0 : xtarget = gen_reg_rtx (int_mode);
2019 :
2020 0 : if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
2021 : target = xtarget;
2022 :
2023 : /* Indicate for flow that the entire target reg is being set. */
2024 0 : if (REG_P (target))
2025 0 : emit_clobber (xtarget);
2026 :
2027 : /* Do the actual arithmetic. */
2028 0 : for (i = 0; i < nwords; i++)
2029 : {
2030 0 : int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
2031 0 : rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
2032 0 : rtx op0_piece = operand_subword_force (xop0, index, int_mode);
2033 0 : rtx op1_piece = operand_subword_force (xop1, index, int_mode);
2034 0 : rtx x;
2035 :
2036 : /* Main add/subtract of the input operands. */
2037 0 : x = expand_binop (word_mode, binoptab,
2038 : op0_piece, op1_piece,
2039 : target_piece, unsignedp, next_methods);
2040 0 : if (x == 0)
2041 : break;
2042 :
2043 0 : if (i + 1 < nwords)
2044 : {
2045 : /* Store carry from main add/subtract. */
2046 0 : carry_out = gen_reg_rtx (word_mode);
2047 0 : carry_out = emit_store_flag_force (carry_out,
2048 : (binoptab == add_optab
2049 : ? LT : GT),
2050 : x, op0_piece,
2051 : word_mode, 1, normalizep);
2052 : }
2053 :
2054 0 : if (i > 0)
2055 : {
2056 0 : rtx newx;
2057 :
2058 : /* Add/subtract previous carry to main result. */
2059 0 : newx = expand_binop (word_mode,
2060 : normalizep == 1 ? binoptab : otheroptab,
2061 : x, carry_in,
2062 : NULL_RTX, 1, next_methods);
2063 :
2064 0 : if (i + 1 < nwords)
2065 : {
2066 : /* Get out carry from adding/subtracting carry in. */
2067 0 : rtx carry_tmp = gen_reg_rtx (word_mode);
2068 0 : carry_tmp = emit_store_flag_force (carry_tmp,
2069 : (binoptab == add_optab
2070 : ? LT : GT),
2071 : newx, x,
2072 : word_mode, 1, normalizep);
2073 :
2074 : /* Logical-ior the two poss. carry together. */
2075 0 : carry_out = expand_binop (word_mode, ior_optab,
2076 : carry_out, carry_tmp,
2077 : carry_out, 0, next_methods);
2078 0 : if (carry_out == 0)
2079 : break;
2080 : }
2081 0 : emit_move_insn (target_piece, newx);
2082 : }
2083 : else
2084 : {
2085 0 : if (x != target_piece)
2086 0 : emit_move_insn (target_piece, x);
2087 : }
2088 :
2089 : carry_in = carry_out;
2090 : }
2091 :
2092 0 : if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
2093 : {
2094 0 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
2095 0 : || ! rtx_equal_p (target, xtarget))
2096 : {
2097 0 : rtx_insn *temp = emit_move_insn (target, xtarget);
2098 :
2099 0 : set_dst_reg_note (temp, REG_EQUAL,
2100 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2101 : int_mode, copy_rtx (xop0),
2102 : copy_rtx (xop1)),
2103 : target);
2104 : }
2105 : else
2106 : target = xtarget;
2107 :
2108 0 : return target;
2109 : }
2110 :
2111 : else
2112 0 : delete_insns_since (last);
2113 : }
2114 :
2115 : /* Attempt to synthesize double word multiplies using a sequence of word
2116 : mode multiplications. We first attempt to generate a sequence using a
2117 : more efficient unsigned widening multiply, and if that fails we then
2118 : try using a signed widening multiply. */
2119 :
2120 191621 : if (binoptab == smul_optab
2121 25264 : && is_int_mode (mode, &int_mode)
2122 17623 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2123 14501 : && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2124 206122 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2125 : {
2126 14501 : rtx product = NULL_RTX;
2127 14501 : if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
2128 : != CODE_FOR_nothing)
2129 : {
2130 14501 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2131 : true, methods);
2132 14501 : if (!product)
2133 0 : delete_insns_since (last);
2134 : }
2135 :
2136 0 : if (product == NULL_RTX
2137 0 : && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
2138 : != CODE_FOR_nothing))
2139 : {
2140 0 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2141 : false, methods);
2142 0 : if (!product)
2143 0 : delete_insns_since (last);
2144 : }
2145 :
2146 14501 : if (product != NULL_RTX)
2147 : {
2148 14501 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2149 : {
2150 22638 : rtx_insn *move = emit_move_insn (target ? target : product,
2151 : product);
2152 14501 : set_dst_reg_note (move,
2153 : REG_EQUAL,
2154 : gen_rtx_fmt_ee (MULT, int_mode,
2155 : copy_rtx (op0),
2156 : copy_rtx (op1)),
2157 : target ? target : product);
2158 : }
2159 14501 : return product;
2160 : }
2161 : }
2162 :
2163 : /* Attempt to synthetize double word modulo by constant divisor. */
2164 177120 : if ((binoptab == umod_optab
2165 177120 : || binoptab == smod_optab
2166 177120 : || binoptab == udiv_optab
2167 : || binoptab == sdiv_optab)
2168 113850 : && optimize
2169 89325 : && CONST_INT_P (op1)
2170 24071 : && is_int_mode (mode, &int_mode)
2171 26301 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2172 33008 : && optab_handler ((binoptab == umod_optab || binoptab == udiv_optab)
2173 : ? udivmod_optab : sdivmod_optab,
2174 : int_mode) == CODE_FOR_nothing
2175 21356 : && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
2176 21356 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
2177 21356 : && optimize_insn_for_speed_p ())
2178 : {
2179 21149 : rtx res = NULL_RTX;
2180 21149 : if ((binoptab == umod_optab || binoptab == smod_optab)
2181 5121 : && (INTVAL (op1) & 1) == 0)
2182 321 : res = expand_doubleword_mod (int_mode, op0, op1,
2183 : binoptab == umod_optab);
2184 : else
2185 : {
2186 20828 : rtx quot = expand_doubleword_divmod (int_mode, op0, op1, &res,
2187 : binoptab == umod_optab
2188 : || binoptab == udiv_optab);
2189 20828 : if (quot == NULL_RTX)
2190 20197 : res = NULL_RTX;
2191 631 : else if (binoptab == udiv_optab || binoptab == sdiv_optab)
2192 424 : res = quot;
2193 : }
2194 21149 : if (res != NULL_RTX)
2195 : {
2196 631 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2197 : {
2198 746 : rtx_insn *move = emit_move_insn (target ? target : res,
2199 : res);
2200 631 : set_dst_reg_note (move, REG_EQUAL,
2201 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2202 : int_mode, copy_rtx (op0), op1),
2203 : target ? target : res);
2204 : }
2205 631 : return res;
2206 : }
2207 : else
2208 20518 : delete_insns_since (last);
2209 : }
2210 :
2211 : /* It can't be open-coded in this mode.
2212 : Use a library call if one is available and caller says that's ok. */
2213 :
2214 176489 : libfunc = optab_libfunc (binoptab, mode);
2215 176489 : if (libfunc
2216 111438 : && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2217 : {
2218 30657 : rtx_insn *insns;
2219 30657 : rtx op1x = op1;
2220 30657 : machine_mode op1_mode = mode;
2221 30657 : rtx value;
2222 :
2223 30657 : start_sequence ();
2224 :
2225 30657 : if (shift_optab_p (binoptab))
2226 : {
2227 0 : op1_mode = targetm.libgcc_shift_count_mode ();
2228 : /* Specify unsigned here,
2229 : since negative shift counts are meaningless. */
2230 0 : op1x = convert_to_mode (op1_mode, op1, 1);
2231 : }
2232 :
2233 30657 : if (GET_MODE (op0) != VOIDmode
2234 30607 : && GET_MODE (op0) != mode)
2235 0 : op0 = convert_to_mode (mode, op0, unsignedp);
2236 :
2237 : /* Pass 1 for NO_QUEUE so we don't lose any increments
2238 : if the libcall is cse'd or moved. */
2239 30657 : value = emit_library_call_value (libfunc,
2240 : NULL_RTX, LCT_CONST, mode,
2241 : op0, mode, op1x, op1_mode);
2242 :
2243 30657 : insns = end_sequence ();
2244 :
2245 30657 : bool trapv = trapv_binoptab_p (binoptab);
2246 30657 : target = gen_reg_rtx (mode);
2247 30657 : emit_libcall_block_1 (insns, target, value,
2248 : trapv ? NULL_RTX
2249 30141 : : gen_rtx_fmt_ee (optab_to_code (binoptab),
2250 : mode, op0, op1), trapv);
2251 :
2252 30657 : return target;
2253 : }
2254 :
2255 145832 : delete_insns_since (last);
2256 :
2257 : /* It can't be done in this mode. Can we do it in a wider mode? */
2258 :
2259 145832 : if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2260 : || methods == OPTAB_MUST_WIDEN))
2261 : {
2262 : /* Caller says, don't even try. */
2263 132380 : delete_insns_since (entry_last);
2264 132380 : return 0;
2265 : }
2266 :
2267 : /* Compute the value of METHODS to pass to recursive calls.
2268 : Don't allow widening to be tried recursively. */
2269 :
2270 13452 : methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2271 :
2272 : /* Look for a wider mode of the same class for which it appears we can do
2273 : the operation. */
2274 :
2275 13452 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2276 : {
2277 : /* This code doesn't make sense for conversion optabs, since we
2278 : wouldn't then want to extend the operands to be the same size
2279 : as the result. */
2280 13452 : gcc_assert (!convert_optab_p (binoptab));
2281 42214 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2282 : {
2283 28763 : if (optab_handler (binoptab, wider_mode)
2284 28763 : || (methods == OPTAB_LIB
2285 2 : && optab_libfunc (binoptab, wider_mode)))
2286 : {
2287 1 : rtx xop0 = op0, xop1 = op1;
2288 1 : bool no_extend = false;
2289 :
2290 : /* For certain integer operations, we need not actually extend
2291 : the narrow operands, as long as we will truncate
2292 : the results to the same narrowness. */
2293 :
2294 1 : if ((binoptab == ior_optab || binoptab == and_optab
2295 1 : || binoptab == xor_optab
2296 1 : || binoptab == add_optab || binoptab == sub_optab
2297 : || binoptab == smul_optab || binoptab == ashl_optab)
2298 0 : && mclass == MODE_INT)
2299 1 : no_extend = true;
2300 :
2301 1 : xop0 = widen_operand (xop0, wider_mode, mode,
2302 : unsignedp, no_extend);
2303 :
2304 : /* The second operand of a shift must always be extended. */
2305 2 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2306 1 : no_extend && binoptab != ashl_optab);
2307 :
2308 1 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2309 : unsignedp, methods);
2310 1 : if (temp)
2311 : {
2312 1 : if (mclass != MODE_INT
2313 1 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2314 : {
2315 0 : if (target == 0)
2316 0 : target = gen_reg_rtx (mode);
2317 0 : convert_move (target, temp, 0);
2318 0 : return target;
2319 : }
2320 : else
2321 1 : return gen_lowpart (mode, temp);
2322 : }
2323 : else
2324 0 : delete_insns_since (last);
2325 : }
2326 : }
2327 : }
2328 :
2329 13451 : delete_insns_since (entry_last);
2330 13451 : return 0;
2331 : }
2332 :
2333 : /* Expand a binary operator which has both signed and unsigned forms.
2334 : UOPTAB is the optab for unsigned operations, and SOPTAB is for
2335 : signed operations.
2336 :
2337 : If we widen unsigned operands, we may use a signed wider operation instead
2338 : of an unsigned wider operation, since the result would be the same. */
2339 :
2340 : rtx
2341 94584 : sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
2342 : rtx op0, rtx op1, rtx target, int unsignedp,
2343 : enum optab_methods methods)
2344 : {
2345 94584 : rtx temp;
2346 94584 : optab direct_optab = unsignedp ? uoptab : soptab;
2347 94584 : bool save_enable;
2348 :
2349 : /* Do it without widening, if possible. */
2350 94584 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2351 : unsignedp, OPTAB_DIRECT);
2352 94584 : if (temp || methods == OPTAB_DIRECT)
2353 : return temp;
2354 :
2355 : /* Try widening to a signed int. Disable any direct use of any
2356 : signed insn in the current mode. */
2357 8711 : save_enable = swap_optab_enable (soptab, mode, false);
2358 :
2359 8711 : temp = expand_binop (mode, soptab, op0, op1, target,
2360 : unsignedp, OPTAB_WIDEN);
2361 :
2362 : /* For unsigned operands, try widening to an unsigned int. */
2363 8711 : if (!temp && unsignedp)
2364 4618 : temp = expand_binop (mode, uoptab, op0, op1, target,
2365 : unsignedp, OPTAB_WIDEN);
2366 8711 : if (temp || methods == OPTAB_WIDEN)
2367 4984 : goto egress;
2368 :
2369 : /* Use the right width libcall if that exists. */
2370 3727 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2371 : unsignedp, OPTAB_LIB);
2372 3727 : if (temp || methods == OPTAB_LIB)
2373 3727 : goto egress;
2374 :
2375 : /* Must widen and use a libcall, use either signed or unsigned. */
2376 0 : temp = expand_binop (mode, soptab, op0, op1, target,
2377 : unsignedp, methods);
2378 0 : if (!temp && unsignedp)
2379 0 : temp = expand_binop (mode, uoptab, op0, op1, target,
2380 : unsignedp, methods);
2381 :
2382 0 : egress:
2383 : /* Undo the fiddling above. */
2384 8711 : if (save_enable)
2385 0 : swap_optab_enable (soptab, mode, true);
2386 : return temp;
2387 : }
2388 :
2389 : /* Generate code to perform an operation specified by UNOPPTAB
2390 : on operand OP0, with two results to TARG0 and TARG1.
2391 : We assume that the order of the operands for the instruction
2392 : is TARG0, TARG1, OP0.
2393 :
2394 : Either TARG0 or TARG1 may be zero, but what that means is that
2395 : the result is not actually wanted. We will generate it into
2396 : a dummy pseudo-reg and discard it. They may not both be zero.
2397 :
2398 : Returns true if this operation can be performed; false if not. */
2399 :
2400 : bool
2401 6 : expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2402 : int unsignedp)
2403 : {
2404 6 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2405 6 : enum mode_class mclass;
2406 6 : machine_mode wider_mode;
2407 6 : rtx_insn *entry_last = get_last_insn ();
2408 6 : rtx_insn *last;
2409 :
2410 6 : mclass = GET_MODE_CLASS (mode);
2411 :
2412 6 : if (!targ0)
2413 0 : targ0 = gen_reg_rtx (mode);
2414 6 : if (!targ1)
2415 0 : targ1 = gen_reg_rtx (mode);
2416 :
2417 : /* Record where to go back to if we fail. */
2418 6 : last = get_last_insn ();
2419 :
2420 6 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2421 : {
2422 6 : class expand_operand ops[3];
2423 6 : enum insn_code icode = optab_handler (unoptab, mode);
2424 :
2425 6 : create_fixed_operand (&ops[0], targ0);
2426 6 : create_fixed_operand (&ops[1], targ1);
2427 6 : create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2428 6 : if (maybe_expand_insn (icode, 3, ops))
2429 6 : return true;
2430 : }
2431 :
2432 : /* It can't be done in this mode. Can we do it in a wider mode? */
2433 :
2434 0 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2435 : {
2436 0 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2437 : {
2438 0 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2439 : {
2440 0 : rtx t0 = gen_reg_rtx (wider_mode);
2441 0 : rtx t1 = gen_reg_rtx (wider_mode);
2442 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2443 :
2444 0 : if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2445 : {
2446 0 : convert_move (targ0, t0, unsignedp);
2447 0 : convert_move (targ1, t1, unsignedp);
2448 0 : return true;
2449 : }
2450 : else
2451 0 : delete_insns_since (last);
2452 : }
2453 : }
2454 : }
2455 :
2456 0 : delete_insns_since (entry_last);
2457 0 : return false;
2458 : }
2459 :
2460 : /* Generate code to perform an operation specified by BINOPTAB
2461 : on operands OP0 and OP1, with two results to TARG1 and TARG2.
2462 : We assume that the order of the operands for the instruction
2463 : is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2464 : [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2465 :
2466 : Either TARG0 or TARG1 may be zero, but what that means is that
2467 : the result is not actually wanted. We will generate it into
2468 : a dummy pseudo-reg and discard it. They may not both be zero.
2469 :
2470 : Returns true if this operation can be performed; false if not. */
2471 :
2472 : bool
2473 101728 : expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2474 : int unsignedp)
2475 : {
2476 101728 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2477 101728 : enum mode_class mclass;
2478 101728 : machine_mode wider_mode;
2479 101728 : rtx_insn *entry_last = get_last_insn ();
2480 101728 : rtx_insn *last;
2481 :
2482 101728 : mclass = GET_MODE_CLASS (mode);
2483 :
2484 101728 : if (!targ0)
2485 31201 : targ0 = gen_reg_rtx (mode);
2486 101728 : if (!targ1)
2487 59064 : targ1 = gen_reg_rtx (mode);
2488 :
2489 : /* Record where to go back to if we fail. */
2490 101728 : last = get_last_insn ();
2491 :
2492 101728 : if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2493 : {
2494 96708 : class expand_operand ops[4];
2495 96708 : enum insn_code icode = optab_handler (binoptab, mode);
2496 96708 : machine_mode mode0 = insn_data[icode].operand[1].mode;
2497 96708 : machine_mode mode1 = insn_data[icode].operand[2].mode;
2498 96708 : rtx xop0 = op0, xop1 = op1;
2499 :
2500 : /* If we are optimizing, force expensive constants into a register. */
2501 96708 : xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2502 96708 : xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2503 :
2504 96708 : create_fixed_operand (&ops[0], targ0);
2505 96708 : create_convert_operand_from (&ops[1], xop0, mode, unsignedp);
2506 96708 : create_convert_operand_from (&ops[2], xop1, mode, unsignedp);
2507 96708 : create_fixed_operand (&ops[3], targ1);
2508 96708 : if (maybe_expand_insn (icode, 4, ops))
2509 96708 : return true;
2510 0 : delete_insns_since (last);
2511 : }
2512 :
2513 : /* It can't be done in this mode. Can we do it in a wider mode? */
2514 :
2515 5020 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2516 : {
2517 15886 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2518 : {
2519 10866 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2520 : {
2521 0 : rtx t0 = gen_reg_rtx (wider_mode);
2522 0 : rtx t1 = gen_reg_rtx (wider_mode);
2523 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2524 0 : rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2525 :
2526 0 : if (expand_twoval_binop (binoptab, cop0, cop1,
2527 : t0, t1, unsignedp))
2528 : {
2529 0 : convert_move (targ0, t0, unsignedp);
2530 0 : convert_move (targ1, t1, unsignedp);
2531 0 : return true;
2532 : }
2533 : else
2534 0 : delete_insns_since (last);
2535 : }
2536 : }
2537 : }
2538 :
2539 5020 : delete_insns_since (entry_last);
2540 5020 : return false;
2541 : }
2542 :
2543 : /* Expand the two-valued library call indicated by BINOPTAB, but
2544 : preserve only one of the values. If TARG0 is non-NULL, the first
2545 : value is placed into TARG0; otherwise the second value is placed
2546 : into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2547 : value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2548 : This routine assumes that the value returned by the library call is
2549 : as if the return value was of an integral mode twice as wide as the
2550 : mode of OP0. Returns 1 if the call was successful. */
2551 :
2552 : bool
2553 0 : expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2554 : rtx targ0, rtx targ1, enum rtx_code code)
2555 : {
2556 0 : machine_mode mode;
2557 0 : machine_mode libval_mode;
2558 0 : rtx libval;
2559 0 : rtx_insn *insns;
2560 0 : rtx libfunc;
2561 :
2562 : /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2563 0 : gcc_assert (!targ0 != !targ1);
2564 :
2565 0 : mode = GET_MODE (op0);
2566 0 : libfunc = optab_libfunc (binoptab, mode);
2567 0 : if (!libfunc)
2568 : return false;
2569 :
2570 : /* The value returned by the library function will have twice as
2571 : many bits as the nominal MODE. */
2572 0 : auto return_size = 2 * GET_MODE_BITSIZE (mode);
2573 0 : if (!smallest_int_mode_for_size (return_size).exists (&libval_mode))
2574 0 : return false;
2575 :
2576 0 : start_sequence ();
2577 0 : libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2578 : libval_mode,
2579 : op0, mode,
2580 : op1, mode);
2581 : /* Get the part of VAL containing the value that we want. */
2582 0 : libval = simplify_gen_subreg (mode, libval, libval_mode,
2583 0 : targ0 ? 0 : GET_MODE_SIZE (mode));
2584 0 : insns = end_sequence ();
2585 : /* Move the into the desired location. */
2586 0 : emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2587 : gen_rtx_fmt_ee (code, mode, op0, op1));
2588 :
2589 0 : return true;
2590 : }
2591 :
2592 :
2593 : /* Wrapper around expand_unop which takes an rtx code to specify
2594 : the operation to perform, not an optab pointer. All other
2595 : arguments are the same. */
2596 : rtx
2597 16076 : expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2598 : rtx target, int unsignedp)
2599 : {
2600 16076 : optab unop = code_to_optab (code);
2601 16076 : gcc_assert (unop);
2602 :
2603 16076 : return expand_unop (mode, unop, op0, target, unsignedp);
2604 : }
2605 :
2606 : /* Try calculating
2607 : (clz:narrow x)
2608 : as
2609 : (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2610 :
2611 : A similar operation can be used for clrsb. UNOPTAB says which operation
2612 : we are trying to expand. */
2613 : static rtx
2614 274 : widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2615 : {
2616 274 : opt_scalar_int_mode wider_mode_iter;
2617 1127 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2618 : {
2619 853 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2620 853 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2621 : {
2622 0 : rtx xop0, temp;
2623 0 : rtx_insn *last;
2624 :
2625 0 : last = get_last_insn ();
2626 :
2627 0 : if (target == 0)
2628 0 : target = gen_reg_rtx (mode);
2629 0 : xop0 = widen_operand (op0, wider_mode, mode,
2630 : unoptab != clrsb_optab, false);
2631 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2632 : unoptab != clrsb_optab);
2633 0 : if (temp != 0)
2634 0 : temp = expand_binop
2635 0 : (wider_mode, sub_optab, temp,
2636 0 : gen_int_mode (GET_MODE_PRECISION (wider_mode)
2637 0 : - GET_MODE_PRECISION (mode),
2638 : wider_mode),
2639 : target, true, OPTAB_DIRECT);
2640 0 : if (temp == 0)
2641 0 : delete_insns_since (last);
2642 :
2643 0 : return temp;
2644 : }
2645 : }
2646 : return 0;
2647 : }
2648 :
2649 : /* Attempt to emit (clrsb:mode op0) as
2650 : (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2651 : (const_int -1))
2652 : if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2653 : or as
2654 : (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2655 : (ashr:mode op0 (const_int prec-1)))
2656 : (const_int 1)))
2657 : otherwise. */
2658 :
2659 : static rtx
2660 91 : expand_clrsb_using_clz (scalar_int_mode mode, rtx op0, rtx target)
2661 : {
2662 91 : if (optimize_insn_for_size_p ()
2663 91 : || optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2664 4 : return NULL_RTX;
2665 :
2666 87 : start_sequence ();
2667 87 : HOST_WIDE_INT val = 0;
2668 87 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) != 2
2669 87 : || val != GET_MODE_PRECISION (mode))
2670 : val = 0;
2671 : else
2672 : val = 1;
2673 :
2674 85 : rtx temp2 = op0;
2675 85 : if (!val)
2676 : {
2677 85 : temp2 = expand_binop (mode, ashl_optab, op0, const1_rtx,
2678 : NULL_RTX, 0, OPTAB_DIRECT);
2679 85 : if (!temp2)
2680 : {
2681 0 : fail:
2682 0 : end_sequence ();
2683 0 : return NULL_RTX;
2684 : }
2685 : }
2686 :
2687 87 : rtx temp = expand_binop (mode, ashr_optab, op0,
2688 87 : GEN_INT (GET_MODE_PRECISION (mode) - 1),
2689 : NULL_RTX, 0, OPTAB_DIRECT);
2690 87 : if (!temp)
2691 0 : goto fail;
2692 :
2693 87 : temp = expand_binop (mode, xor_optab, temp2, temp, NULL_RTX, 0,
2694 : OPTAB_DIRECT);
2695 87 : if (!temp)
2696 0 : goto fail;
2697 :
2698 87 : if (!val)
2699 : {
2700 85 : temp = expand_binop (mode, ior_optab, temp, const1_rtx,
2701 : NULL_RTX, 0, OPTAB_DIRECT);
2702 85 : if (!temp)
2703 0 : goto fail;
2704 : }
2705 87 : temp = expand_unop_direct (mode, clz_optab, temp, val ? NULL_RTX : target,
2706 : true);
2707 87 : if (!temp)
2708 0 : goto fail;
2709 87 : if (val)
2710 : {
2711 2 : temp = expand_binop (mode, add_optab, temp, constm1_rtx,
2712 : target, 0, OPTAB_DIRECT);
2713 2 : if (!temp)
2714 0 : goto fail;
2715 : }
2716 :
2717 87 : rtx_insn *seq = end_sequence ();
2718 :
2719 87 : add_equal_note (seq, temp, CLRSB, op0, NULL_RTX, mode);
2720 87 : emit_insn (seq);
2721 87 : return temp;
2722 : }
2723 :
2724 : static rtx expand_ffs (scalar_int_mode, rtx, rtx);
2725 :
2726 : /* Try calculating clz, ctz or ffs of a double-word quantity as two clz, ctz or
2727 : ffs operations on word-sized quantities, choosing which based on whether the
2728 : high (for clz) or low (for ctz and ffs) word is nonzero. */
2729 : static rtx
2730 230 : expand_doubleword_clz_ctz_ffs (scalar_int_mode mode, rtx op0, rtx target,
2731 : optab unoptab)
2732 : {
2733 230 : rtx xop0 = force_reg (mode, op0);
2734 230 : rtx subhi = gen_highpart (word_mode, xop0);
2735 230 : rtx sublo = gen_lowpart (word_mode, xop0);
2736 230 : rtx_code_label *hi0_label = gen_label_rtx ();
2737 230 : rtx_code_label *after_label = gen_label_rtx ();
2738 230 : rtx_insn *seq;
2739 230 : rtx temp, result;
2740 230 : int addend = 0;
2741 :
2742 : /* If we were not given a target, use a word_mode register, not a
2743 : 'mode' register. The result will fit, and nobody is expecting
2744 : anything bigger (the return type of __builtin_clz* is int). */
2745 230 : if (!target)
2746 0 : target = gen_reg_rtx (word_mode);
2747 :
2748 : /* In any case, write to a word_mode scratch in both branches of the
2749 : conditional, so we can ensure there is a single move insn setting
2750 : 'target' to tag a REG_EQUAL note on. */
2751 230 : result = gen_reg_rtx (word_mode);
2752 :
2753 230 : if (unoptab != clz_optab)
2754 47 : std::swap (subhi, sublo);
2755 :
2756 230 : start_sequence ();
2757 :
2758 : /* If the high word is not equal to zero,
2759 : then clz of the full value is clz of the high word. */
2760 230 : emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2761 : word_mode, true, hi0_label);
2762 :
2763 230 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2764 230 : temp = expand_unop_direct (word_mode, unoptab, subhi, result, true);
2765 : else
2766 : {
2767 0 : gcc_assert (unoptab == ffs_optab);
2768 0 : temp = expand_ffs (word_mode, subhi, result);
2769 : }
2770 230 : if (!temp)
2771 0 : goto fail;
2772 :
2773 230 : if (temp != result)
2774 0 : convert_move (result, temp, true);
2775 :
2776 230 : emit_jump_insn (targetm.gen_jump (after_label));
2777 230 : emit_barrier ();
2778 :
2779 : /* Else clz of the full value is clz of the low word plus the number
2780 : of bits in the high word. Similarly for ctz/ffs of the high word,
2781 : except that ffs should be 0 when both words are zero. */
2782 230 : emit_label (hi0_label);
2783 :
2784 230 : if (unoptab == ffs_optab)
2785 : {
2786 0 : convert_move (result, const0_rtx, true);
2787 0 : emit_cmp_and_jump_insns (sublo, CONST0_RTX (word_mode), EQ, 0,
2788 : word_mode, true, after_label);
2789 : }
2790 :
2791 230 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2792 230 : temp = expand_unop_direct (word_mode, unoptab, sublo, NULL_RTX, true);
2793 : else
2794 : {
2795 0 : gcc_assert (unoptab == ffs_optab);
2796 0 : temp = expand_unop_direct (word_mode, ctz_optab, sublo, NULL_RTX, true);
2797 0 : addend = 1;
2798 : }
2799 :
2800 230 : if (!temp)
2801 0 : goto fail;
2802 :
2803 460 : temp = expand_binop (word_mode, add_optab, temp,
2804 230 : gen_int_mode (GET_MODE_BITSIZE (word_mode) + addend,
2805 : word_mode),
2806 : result, true, OPTAB_DIRECT);
2807 230 : if (!temp)
2808 0 : goto fail;
2809 230 : if (temp != result)
2810 0 : convert_move (result, temp, true);
2811 :
2812 230 : emit_label (after_label);
2813 230 : convert_move (target, result, true);
2814 :
2815 230 : seq = end_sequence ();
2816 :
2817 230 : add_equal_note (seq, target, optab_to_code (unoptab), xop0, NULL_RTX, mode);
2818 230 : emit_insn (seq);
2819 230 : return target;
2820 :
2821 0 : fail:
2822 0 : end_sequence ();
2823 0 : return 0;
2824 : }
2825 :
2826 : /* Try calculating popcount of a double-word quantity as two popcount's of
2827 : word-sized quantities and summing up the results. */
2828 : static rtx
2829 0 : expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2830 : {
2831 0 : rtx t0, t1, t;
2832 0 : rtx_insn *seq;
2833 :
2834 0 : start_sequence ();
2835 :
2836 0 : t0 = expand_unop_direct (word_mode, popcount_optab,
2837 : operand_subword_force (op0, 0, mode), NULL_RTX,
2838 : true);
2839 0 : t1 = expand_unop_direct (word_mode, popcount_optab,
2840 : operand_subword_force (op0, 1, mode), NULL_RTX,
2841 : true);
2842 0 : if (!t0 || !t1)
2843 : {
2844 0 : end_sequence ();
2845 0 : return NULL_RTX;
2846 : }
2847 :
2848 : /* If we were not given a target, use a word_mode register, not a
2849 : 'mode' register. The result will fit, and nobody is expecting
2850 : anything bigger (the return type of __builtin_popcount* is int). */
2851 0 : if (!target)
2852 0 : target = gen_reg_rtx (word_mode);
2853 :
2854 0 : t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2855 :
2856 0 : seq = end_sequence ();
2857 :
2858 0 : add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2859 0 : emit_insn (seq);
2860 0 : return t;
2861 : }
2862 :
2863 : /* Try calculating
2864 : (parity:wide x)
2865 : as
2866 : (parity:narrow (low (x) ^ high (x))) */
2867 : static rtx
2868 0 : expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2869 : {
2870 0 : rtx t = expand_binop (word_mode, xor_optab,
2871 : operand_subword_force (op0, 0, mode),
2872 : operand_subword_force (op0, 1, mode),
2873 : NULL_RTX, 0, OPTAB_DIRECT);
2874 0 : return expand_unop (word_mode, parity_optab, t, target, true);
2875 : }
2876 :
2877 : /* Try calculating
2878 : (bswap:narrow x)
2879 : as
2880 : (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2881 : static rtx
2882 155 : widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2883 : {
2884 155 : rtx x;
2885 155 : rtx_insn *last;
2886 155 : opt_scalar_int_mode wider_mode_iter;
2887 :
2888 613 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2889 458 : if (optab_handler (bswap_optab, wider_mode_iter.require ())
2890 : != CODE_FOR_nothing)
2891 : break;
2892 :
2893 155 : if (!wider_mode_iter.exists ())
2894 : return NULL_RTX;
2895 :
2896 0 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2897 0 : last = get_last_insn ();
2898 :
2899 0 : x = widen_operand (op0, wider_mode, mode, true, true);
2900 0 : x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2901 :
2902 0 : gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2903 : && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2904 0 : if (x != 0)
2905 0 : x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2906 0 : GET_MODE_BITSIZE (wider_mode)
2907 0 : - GET_MODE_BITSIZE (mode),
2908 : NULL_RTX, true);
2909 :
2910 0 : if (x != 0)
2911 : {
2912 0 : if (target == 0)
2913 0 : target = gen_reg_rtx (mode);
2914 0 : emit_move_insn (target, gen_lowpart (mode, x));
2915 : }
2916 : else
2917 0 : delete_insns_since (last);
2918 :
2919 : return target;
2920 : }
2921 :
2922 : /* Try calculating bswap as two bswaps of two word-sized operands. */
2923 :
2924 : static rtx
2925 155 : expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2926 : {
2927 155 : rtx t0, t1;
2928 :
2929 155 : t1 = expand_unop (word_mode, bswap_optab,
2930 : operand_subword_force (op, 0, mode), NULL_RTX, true);
2931 155 : t0 = expand_unop (word_mode, bswap_optab,
2932 : operand_subword_force (op, 1, mode), NULL_RTX, true);
2933 :
2934 155 : if (target == 0 || !valid_multiword_target_p (target))
2935 1 : target = gen_reg_rtx (mode);
2936 155 : if (REG_P (target))
2937 155 : emit_clobber (target);
2938 155 : emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2939 155 : emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2940 :
2941 155 : return target;
2942 : }
2943 :
2944 : /* Try calculating (parity x) as (and (popcount x) 1), where
2945 : popcount can also be done in a wider mode. */
2946 : static rtx
2947 11 : expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2948 : {
2949 11 : enum mode_class mclass = GET_MODE_CLASS (mode);
2950 11 : opt_scalar_int_mode wider_mode_iter;
2951 11 : FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2952 : {
2953 11 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2954 11 : if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2955 : {
2956 11 : rtx xop0, temp;
2957 11 : rtx_insn *last;
2958 :
2959 11 : last = get_last_insn ();
2960 :
2961 11 : if (target == 0 || GET_MODE (target) != wider_mode)
2962 6 : target = gen_reg_rtx (wider_mode);
2963 :
2964 11 : xop0 = widen_operand (op0, wider_mode, mode, true, false);
2965 11 : temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2966 : true);
2967 11 : if (temp != 0)
2968 11 : temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2969 : target, true, OPTAB_DIRECT);
2970 :
2971 11 : if (temp)
2972 : {
2973 11 : if (mclass != MODE_INT
2974 11 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2975 0 : return convert_to_mode (mode, temp, 0);
2976 : else
2977 11 : return gen_lowpart (mode, temp);
2978 : }
2979 : else
2980 0 : delete_insns_since (last);
2981 : }
2982 : }
2983 : return 0;
2984 : }
2985 :
2986 : /* Try calculating ctz(x) as K - clz(x & -x) ,
2987 : where K is GET_MODE_PRECISION(mode) - 1.
2988 :
2989 : Both __builtin_ctz and __builtin_clz are undefined at zero, so we
2990 : don't have to worry about what the hardware does in that case. (If
2991 : the clz instruction produces the usual value at 0, which is K, the
2992 : result of this code sequence will be -1; expand_ffs, below, relies
2993 : on this. It might be nice to have it be K instead, for consistency
2994 : with the (very few) processors that provide a ctz with a defined
2995 : value, but that would take one more instruction, and it would be
2996 : less convenient for expand_ffs anyway. */
2997 :
2998 : static rtx
2999 50 : expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
3000 : {
3001 50 : rtx_insn *seq;
3002 50 : rtx temp;
3003 :
3004 50 : if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
3005 : return 0;
3006 :
3007 0 : start_sequence ();
3008 :
3009 0 : temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
3010 0 : if (temp)
3011 0 : temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
3012 : true, OPTAB_DIRECT);
3013 0 : if (temp)
3014 0 : temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
3015 0 : if (temp)
3016 0 : temp = expand_binop (mode, sub_optab,
3017 0 : gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
3018 : temp, target,
3019 : true, OPTAB_DIRECT);
3020 0 : if (temp == 0)
3021 : {
3022 0 : end_sequence ();
3023 0 : return 0;
3024 : }
3025 :
3026 0 : seq = end_sequence ();
3027 :
3028 0 : add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
3029 0 : emit_insn (seq);
3030 0 : return temp;
3031 : }
3032 :
3033 :
3034 : /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
3035 : else with the sequence used by expand_clz.
3036 :
3037 : The ffs builtin promises to return zero for a zero value and ctz/clz
3038 : may have an undefined value in that case. If they do not give us a
3039 : convenient value, we have to generate a test and branch. */
3040 : static rtx
3041 0 : expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
3042 : {
3043 0 : HOST_WIDE_INT val = 0;
3044 0 : bool defined_at_zero = false;
3045 0 : rtx temp;
3046 0 : rtx_insn *seq;
3047 :
3048 0 : if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
3049 : {
3050 0 : start_sequence ();
3051 :
3052 0 : temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
3053 0 : if (!temp)
3054 0 : goto fail;
3055 :
3056 0 : defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
3057 : }
3058 0 : else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
3059 : {
3060 0 : start_sequence ();
3061 0 : temp = expand_ctz (mode, op0, 0);
3062 0 : if (!temp)
3063 0 : goto fail;
3064 :
3065 0 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
3066 : {
3067 0 : defined_at_zero = true;
3068 0 : val = (GET_MODE_PRECISION (mode) - 1) - val;
3069 : }
3070 : }
3071 : else
3072 : return 0;
3073 :
3074 0 : if (defined_at_zero && val == -1)
3075 : /* No correction needed at zero. */;
3076 : else
3077 : {
3078 : /* We don't try to do anything clever with the situation found
3079 : on some processors (eg Alpha) where ctz(0:mode) ==
3080 : bitsize(mode). If someone can think of a way to send N to -1
3081 : and leave alone all values in the range 0..N-1 (where N is a
3082 : power of two), cheaper than this test-and-branch, please add it.
3083 :
3084 : The test-and-branch is done after the operation itself, in case
3085 : the operation sets condition codes that can be recycled for this.
3086 : (This is true on i386, for instance.) */
3087 :
3088 0 : rtx_code_label *nonzero_label = gen_label_rtx ();
3089 0 : emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
3090 : mode, true, nonzero_label);
3091 :
3092 0 : convert_move (temp, GEN_INT (-1), false);
3093 0 : emit_label (nonzero_label);
3094 : }
3095 :
3096 : /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3097 : to produce a value in the range 0..bitsize. */
3098 0 : temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
3099 : target, false, OPTAB_DIRECT);
3100 0 : if (!temp)
3101 0 : goto fail;
3102 :
3103 0 : seq = end_sequence ();
3104 :
3105 0 : add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
3106 0 : emit_insn (seq);
3107 0 : return temp;
3108 :
3109 0 : fail:
3110 0 : end_sequence ();
3111 0 : return 0;
3112 : }
3113 :
3114 : /* Expand a floating point absolute value or negation operation via a
3115 : logical operation on the sign bit. MODE is the mode of the operands
3116 : and FMODE is the scalar inner mode. */
3117 :
3118 : static rtx
3119 1101 : expand_absneg_bit (rtx_code code, machine_mode mode,
3120 : scalar_float_mode fmode, rtx op0, rtx target)
3121 : {
3122 1101 : int bitpos, word, nwords, i;
3123 1101 : machine_mode new_mode;
3124 1101 : scalar_int_mode imode;
3125 1101 : rtx temp;
3126 1101 : rtx_insn *insns;
3127 :
3128 1101 : auto op = code == NEG ? neg_optab : abs_optab;
3129 1101 : if (!get_absneg_bit_mode (op, mode, fmode, &bitpos).exists (&new_mode))
3130 0 : return NULL_RTX;
3131 :
3132 2202 : imode = as_a<scalar_int_mode> (GET_MODE_INNER (new_mode));
3133 2252 : if (VECTOR_MODE_P (mode) || GET_MODE_SIZE (fmode) <= UNITS_PER_WORD)
3134 : {
3135 : word = 0;
3136 : nwords = 1;
3137 : }
3138 : else
3139 : {
3140 17 : if (FLOAT_WORDS_BIG_ENDIAN)
3141 : word = (GET_MODE_BITSIZE (fmode) - bitpos) / BITS_PER_WORD;
3142 : else
3143 17 : word = bitpos / BITS_PER_WORD;
3144 17 : bitpos = bitpos % BITS_PER_WORD;
3145 34 : nwords = (GET_MODE_BITSIZE (fmode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3146 : }
3147 :
3148 1101 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3149 1101 : if (code == ABS)
3150 840 : mask = ~mask;
3151 :
3152 1101 : if (target == 0
3153 1101 : || target == op0
3154 633 : || reg_overlap_mentioned_p (target, op0)
3155 1730 : || (nwords > 1 && !valid_multiword_target_p (target)))
3156 472 : target = gen_reg_rtx (mode);
3157 :
3158 1101 : if (nwords > 1)
3159 : {
3160 17 : start_sequence ();
3161 :
3162 51 : for (i = 0; i < nwords; ++i)
3163 : {
3164 34 : rtx targ_piece = operand_subword (target, i, 1, mode);
3165 34 : rtx op0_piece = operand_subword_force (op0, i, mode);
3166 :
3167 34 : if (i == word)
3168 : {
3169 46 : temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
3170 : op0_piece,
3171 17 : immed_wide_int_const (mask, imode),
3172 : targ_piece, 1, OPTAB_LIB_WIDEN);
3173 17 : if (temp != targ_piece)
3174 0 : emit_move_insn (targ_piece, temp);
3175 : }
3176 : else
3177 17 : emit_move_insn (targ_piece, op0_piece);
3178 : }
3179 :
3180 17 : insns = end_sequence ();
3181 :
3182 17 : emit_insn (insns);
3183 : }
3184 : else
3185 : {
3186 1084 : rtx mask_rtx = immed_wide_int_const (mask, imode);
3187 1084 : if (VECTOR_MODE_P (new_mode))
3188 0 : mask_rtx = gen_const_vec_duplicate (new_mode, mask_rtx);
3189 1333 : temp = expand_binop (new_mode, code == ABS ? and_optab : xor_optab,
3190 1084 : gen_lowpart (new_mode, op0), mask_rtx,
3191 1084 : gen_lowpart (new_mode, target), 1, OPTAB_LIB_WIDEN);
3192 1084 : target = force_lowpart_subreg (mode, temp, new_mode);
3193 :
3194 1084 : set_dst_reg_note (get_last_insn (), REG_EQUAL,
3195 : gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
3196 : target);
3197 : }
3198 :
3199 1101 : return target;
3200 1101 : }
3201 :
3202 : /* As expand_unop, but will fail rather than attempt the operation in a
3203 : different mode or with a libcall. */
3204 : static rtx
3205 163291 : expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
3206 : int unsignedp)
3207 : {
3208 163291 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
3209 : {
3210 160906 : class expand_operand ops[2];
3211 160906 : enum insn_code icode = optab_handler (unoptab, mode);
3212 160906 : rtx_insn *last = get_last_insn ();
3213 160906 : rtx_insn *pat;
3214 :
3215 160906 : create_output_operand (&ops[0], target, mode);
3216 160906 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
3217 160906 : pat = maybe_gen_insn (icode, 2, ops);
3218 160906 : if (pat)
3219 : {
3220 160634 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3221 200061 : && ! add_equal_note (pat, ops[0].value,
3222 : optab_to_code (unoptab),
3223 : ops[1].value, NULL_RTX, mode))
3224 : {
3225 1078 : delete_insns_since (last);
3226 160906 : return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3227 : }
3228 :
3229 159828 : emit_insn (pat);
3230 :
3231 159828 : return ops[0].value;
3232 : }
3233 : }
3234 : return 0;
3235 : }
3236 :
3237 : /* Generate code to perform an operation specified by UNOPTAB
3238 : on operand OP0, with result having machine-mode MODE.
3239 :
3240 : UNSIGNEDP is for the case where we have to widen the operands
3241 : to perform the operation. It says to use zero-extension.
3242 :
3243 : If TARGET is nonzero, the value
3244 : is generated there, if it is convenient to do so.
3245 : In all cases an rtx is returned for the locus of the value;
3246 : this may or may not be TARGET. */
3247 :
3248 : rtx
3249 162744 : expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
3250 : int unsignedp)
3251 : {
3252 162744 : enum mode_class mclass = GET_MODE_CLASS (mode);
3253 162744 : machine_mode wider_mode;
3254 162744 : scalar_int_mode int_mode;
3255 162744 : scalar_float_mode float_mode;
3256 162744 : rtx temp;
3257 162744 : rtx libfunc;
3258 :
3259 162744 : temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3260 162744 : if (temp)
3261 : return temp;
3262 :
3263 : /* It can't be done in this mode. Can we open-code it in a wider mode? */
3264 :
3265 : /* Widening (or narrowing) clz needs special treatment. */
3266 2385 : if (unoptab == clz_optab)
3267 : {
3268 183 : if (is_a <scalar_int_mode> (mode, &int_mode))
3269 : {
3270 183 : temp = widen_leading (int_mode, op0, target, unoptab);
3271 183 : if (temp)
3272 : return temp;
3273 :
3274 366 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3275 183 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3276 : {
3277 183 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target,
3278 : unoptab);
3279 183 : if (temp)
3280 : return temp;
3281 : }
3282 : }
3283 :
3284 0 : goto try_libcall;
3285 : }
3286 :
3287 2202 : if (unoptab == clrsb_optab)
3288 : {
3289 91 : if (is_a <scalar_int_mode> (mode, &int_mode))
3290 : {
3291 91 : temp = widen_leading (int_mode, op0, target, unoptab);
3292 91 : if (temp)
3293 : return temp;
3294 91 : temp = expand_clrsb_using_clz (int_mode, op0, target);
3295 91 : if (temp)
3296 : return temp;
3297 : }
3298 4 : goto try_libcall;
3299 : }
3300 :
3301 2111 : if (unoptab == popcount_optab
3302 617 : && is_a <scalar_int_mode> (mode, &int_mode)
3303 649 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3304 2 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3305 2111 : && optimize_insn_for_speed_p ())
3306 : {
3307 0 : temp = expand_doubleword_popcount (int_mode, op0, target);
3308 0 : if (temp)
3309 : return temp;
3310 : }
3311 :
3312 2111 : if (unoptab == parity_optab
3313 11 : && is_a <scalar_int_mode> (mode, &int_mode)
3314 11 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3315 0 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3316 0 : || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
3317 2111 : && optimize_insn_for_speed_p ())
3318 : {
3319 0 : temp = expand_doubleword_parity (int_mode, op0, target);
3320 0 : if (temp)
3321 : return temp;
3322 : }
3323 :
3324 : /* Widening (or narrowing) bswap needs special treatment. */
3325 2111 : if (unoptab == bswap_optab)
3326 : {
3327 : /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3328 : or ROTATERT. First try these directly; if this fails, then try the
3329 : obvious pair of shifts with allowed widening, as this will probably
3330 : be always more efficient than the other fallback methods. */
3331 155 : if (mode == HImode)
3332 : {
3333 0 : rtx_insn *last;
3334 0 : rtx temp1, temp2;
3335 :
3336 0 : if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3337 : {
3338 0 : temp = expand_binop (mode, rotl_optab, op0,
3339 : gen_int_shift_amount (mode, 8),
3340 : target, unsignedp, OPTAB_DIRECT);
3341 0 : if (temp)
3342 : return temp;
3343 : }
3344 :
3345 0 : if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3346 : {
3347 0 : temp = expand_binop (mode, rotr_optab, op0,
3348 : gen_int_shift_amount (mode, 8),
3349 : target, unsignedp, OPTAB_DIRECT);
3350 0 : if (temp)
3351 : return temp;
3352 : }
3353 :
3354 0 : last = get_last_insn ();
3355 :
3356 0 : temp1 = expand_binop (mode, ashl_optab, op0,
3357 : gen_int_shift_amount (mode, 8), NULL_RTX,
3358 : unsignedp, OPTAB_WIDEN);
3359 0 : temp2 = expand_binop (mode, lshr_optab, op0,
3360 : gen_int_shift_amount (mode, 8), NULL_RTX,
3361 : unsignedp, OPTAB_WIDEN);
3362 0 : if (temp1 && temp2)
3363 : {
3364 0 : temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3365 : unsignedp, OPTAB_WIDEN);
3366 0 : if (temp)
3367 : return temp;
3368 : }
3369 :
3370 0 : delete_insns_since (last);
3371 : }
3372 :
3373 155 : if (is_a <scalar_int_mode> (mode, &int_mode))
3374 : {
3375 155 : temp = widen_bswap (int_mode, op0, target);
3376 155 : if (temp)
3377 : return temp;
3378 :
3379 : /* We do not provide a 128-bit bswap in libgcc so force the use of
3380 : a double bswap for 64-bit targets. */
3381 310 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3382 155 : && (UNITS_PER_WORD == 8
3383 148 : || optab_handler (unoptab, word_mode) != CODE_FOR_nothing))
3384 : {
3385 155 : temp = expand_doubleword_bswap (mode, op0, target);
3386 155 : if (temp)
3387 : return temp;
3388 : }
3389 : }
3390 :
3391 0 : goto try_libcall;
3392 : }
3393 :
3394 : /* Neg should be tried via expand_absneg_bit before widening. */
3395 1956 : if (optab_to_code (unoptab) == NEG)
3396 : {
3397 : /* Try negating floating point values by flipping the sign bit. */
3398 704 : if (is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3399 : {
3400 261 : temp = expand_absneg_bit (NEG, mode, float_mode, op0, target);
3401 261 : if (temp)
3402 : return temp;
3403 : }
3404 :
3405 : /* If there is no negation pattern, and we have no negative zero,
3406 : try subtracting from zero. */
3407 91 : if (!HONOR_SIGNED_ZEROS (mode))
3408 : {
3409 91 : temp = expand_binop (mode, (unoptab == negv_optab
3410 : ? subv_optab : sub_optab),
3411 : CONST0_RTX (mode), op0, target,
3412 : unsignedp, OPTAB_DIRECT);
3413 91 : if (temp)
3414 : return temp;
3415 : }
3416 : }
3417 :
3418 : /* ABS also needs to be handled similarly. */
3419 1695 : if (optab_to_code (unoptab) == ABS
3420 2621 : && is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3421 : {
3422 840 : temp = expand_absneg_bit (ABS, mode, float_mode, op0, target);
3423 840 : if (temp)
3424 : return temp;
3425 : }
3426 :
3427 855 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3428 3824 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3429 : {
3430 2969 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3431 : {
3432 0 : rtx xop0 = op0;
3433 0 : rtx_insn *last = get_last_insn ();
3434 :
3435 : /* For certain operations, we need not actually extend
3436 : the narrow operand, as long as we will truncate the
3437 : results to the same narrowness. */
3438 :
3439 0 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3440 0 : (unoptab == neg_optab
3441 0 : || unoptab == one_cmpl_optab)
3442 0 : && mclass == MODE_INT);
3443 :
3444 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3445 : unsignedp);
3446 :
3447 0 : if (temp)
3448 : {
3449 0 : if (mclass != MODE_INT
3450 0 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3451 : {
3452 0 : if (target == 0)
3453 0 : target = gen_reg_rtx (mode);
3454 0 : convert_move (target, temp, 0);
3455 0 : return target;
3456 : }
3457 : else
3458 0 : return gen_lowpart (mode, temp);
3459 : }
3460 : else
3461 0 : delete_insns_since (last);
3462 : }
3463 : }
3464 :
3465 : /* These can be done a word at a time. */
3466 855 : if (unoptab == one_cmpl_optab
3467 0 : && is_int_mode (mode, &int_mode)
3468 0 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
3469 855 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3470 : {
3471 0 : int i;
3472 0 : rtx_insn *insns;
3473 :
3474 0 : if (target == 0
3475 0 : || target == op0
3476 0 : || reg_overlap_mentioned_p (target, op0)
3477 0 : || !valid_multiword_target_p (target))
3478 0 : target = gen_reg_rtx (int_mode);
3479 :
3480 0 : start_sequence ();
3481 :
3482 : /* Do the actual arithmetic. */
3483 0 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
3484 : {
3485 0 : rtx target_piece = operand_subword (target, i, 1, int_mode);
3486 0 : rtx x = expand_unop (word_mode, unoptab,
3487 0 : operand_subword_force (op0, i, int_mode),
3488 : target_piece, unsignedp);
3489 :
3490 0 : if (target_piece != x)
3491 0 : emit_move_insn (target_piece, x);
3492 : }
3493 :
3494 0 : insns = end_sequence ();
3495 :
3496 0 : emit_insn (insns);
3497 0 : return target;
3498 : }
3499 :
3500 : /* Emit ~op0 as op0 ^ -1. */
3501 855 : if (unoptab == one_cmpl_optab
3502 0 : && (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
3503 855 : && optab_handler (xor_optab, mode) != CODE_FOR_nothing)
3504 : {
3505 0 : temp = expand_binop (mode, xor_optab, op0, CONSTM1_RTX (mode),
3506 : target, unsignedp, OPTAB_DIRECT);
3507 0 : if (temp)
3508 : return temp;
3509 : }
3510 :
3511 : /* Try calculating parity (x) as popcount (x) % 2. */
3512 855 : if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
3513 : {
3514 11 : temp = expand_parity (int_mode, op0, target);
3515 11 : if (temp)
3516 : return temp;
3517 : }
3518 :
3519 : /* Try implementing ffs (x) in terms of clz (x). */
3520 844 : if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
3521 : {
3522 0 : temp = expand_ffs (int_mode, op0, target);
3523 0 : if (temp)
3524 : return temp;
3525 : }
3526 :
3527 : /* Try implementing ctz (x) in terms of clz (x). */
3528 844 : if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
3529 : {
3530 50 : temp = expand_ctz (int_mode, op0, target);
3531 50 : if (temp)
3532 : return temp;
3533 : }
3534 :
3535 844 : if ((unoptab == ctz_optab || unoptab == ffs_optab)
3536 50 : && optimize_insn_for_speed_p ()
3537 47 : && is_a <scalar_int_mode> (mode, &int_mode)
3538 94 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3539 891 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3540 0 : || optab_handler (ctz_optab, word_mode) != CODE_FOR_nothing))
3541 : {
3542 47 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target, unoptab);
3543 47 : if (temp)
3544 : return temp;
3545 : }
3546 :
3547 801 : try_libcall:
3548 : /* Now try a library call in this mode. */
3549 801 : libfunc = optab_libfunc (unoptab, mode);
3550 801 : if (libfunc)
3551 : {
3552 518 : rtx_insn *insns;
3553 518 : rtx value;
3554 518 : rtx eq_value;
3555 518 : machine_mode outmode = mode;
3556 :
3557 : /* All of these functions return small values. Thus we choose to
3558 : have them return something that isn't a double-word. */
3559 518 : if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3560 : || unoptab == clrsb_optab || unoptab == popcount_optab
3561 518 : || unoptab == parity_optab)
3562 427 : outmode
3563 427 : = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3564 : optab_libfunc (unoptab, mode)));
3565 :
3566 518 : start_sequence ();
3567 :
3568 : /* Pass 1 for NO_QUEUE so we don't lose any increments
3569 : if the libcall is cse'd or moved. */
3570 518 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3571 : op0, mode);
3572 518 : insns = end_sequence ();
3573 :
3574 518 : target = gen_reg_rtx (outmode);
3575 518 : bool trapv = trapv_unoptab_p (unoptab);
3576 518 : if (trapv)
3577 : eq_value = NULL_RTX;
3578 : else
3579 : {
3580 427 : eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3581 1281 : if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3582 397 : eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3583 90 : else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3584 0 : eq_value = simplify_gen_unary (ZERO_EXTEND,
3585 : outmode, eq_value, mode);
3586 : }
3587 518 : emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3588 :
3589 518 : return target;
3590 : }
3591 :
3592 : /* It can't be done in this mode. Can we do it in a wider mode? */
3593 :
3594 283 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3595 : {
3596 626 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3597 : {
3598 540 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3599 540 : || optab_libfunc (unoptab, wider_mode))
3600 : {
3601 197 : rtx xop0 = op0;
3602 197 : rtx_insn *last = get_last_insn ();
3603 :
3604 : /* For certain operations, we need not actually extend
3605 : the narrow operand, as long as we will truncate the
3606 : results to the same narrowness. */
3607 197 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3608 197 : (unoptab == neg_optab
3609 197 : || unoptab == one_cmpl_optab
3610 197 : || unoptab == bswap_optab)
3611 197 : && mclass == MODE_INT);
3612 :
3613 197 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3614 : unsignedp);
3615 :
3616 : /* If we are generating clz using wider mode, adjust the
3617 : result. Similarly for clrsb. */
3618 197 : if ((unoptab == clz_optab || unoptab == clrsb_optab)
3619 1 : && temp != 0)
3620 : {
3621 1 : scalar_int_mode wider_int_mode
3622 1 : = as_a <scalar_int_mode> (wider_mode);
3623 1 : int_mode = as_a <scalar_int_mode> (mode);
3624 1 : temp = expand_binop
3625 1 : (wider_mode, sub_optab, temp,
3626 1 : gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3627 2 : - GET_MODE_PRECISION (int_mode),
3628 : wider_int_mode),
3629 : target, true, OPTAB_DIRECT);
3630 : }
3631 :
3632 : /* Likewise for bswap. */
3633 197 : if (unoptab == bswap_optab && temp != 0)
3634 : {
3635 0 : scalar_int_mode wider_int_mode
3636 0 : = as_a <scalar_int_mode> (wider_mode);
3637 0 : int_mode = as_a <scalar_int_mode> (mode);
3638 0 : gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3639 : == GET_MODE_BITSIZE (wider_int_mode)
3640 : && GET_MODE_PRECISION (int_mode)
3641 : == GET_MODE_BITSIZE (int_mode));
3642 :
3643 0 : temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3644 0 : GET_MODE_BITSIZE (wider_int_mode)
3645 0 : - GET_MODE_BITSIZE (int_mode),
3646 : NULL_RTX, true);
3647 : }
3648 :
3649 197 : if (temp)
3650 : {
3651 197 : if (mclass != MODE_INT)
3652 : {
3653 0 : if (target == 0)
3654 0 : target = gen_reg_rtx (mode);
3655 0 : convert_move (target, temp, 0);
3656 0 : return target;
3657 : }
3658 : else
3659 197 : return gen_lowpart (mode, temp);
3660 : }
3661 : else
3662 0 : delete_insns_since (last);
3663 : }
3664 : }
3665 : }
3666 :
3667 : /* One final attempt at implementing negation via subtraction,
3668 : this time allowing widening of the operand. */
3669 86 : if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3670 : {
3671 0 : rtx temp;
3672 0 : temp = expand_binop (mode,
3673 : unoptab == negv_optab ? subv_optab : sub_optab,
3674 : CONST0_RTX (mode), op0,
3675 : target, unsignedp, OPTAB_LIB_WIDEN);
3676 0 : if (temp)
3677 : return temp;
3678 : }
3679 :
3680 : return 0;
3681 : }
3682 :
3683 : /* Emit code to compute the absolute value of OP0, with result to
3684 : TARGET if convenient. (TARGET may be 0.) The return value says
3685 : where the result actually is to be found.
3686 :
3687 : MODE is the mode of the operand; the mode of the result is
3688 : different but can be deduced from MODE.
3689 :
3690 : */
3691 :
3692 : rtx
3693 24323 : expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3694 : int result_unsignedp)
3695 : {
3696 24323 : rtx temp;
3697 :
3698 24323 : if (GET_MODE_CLASS (mode) != MODE_INT
3699 4195 : || ! flag_trapv)
3700 : result_unsignedp = 1;
3701 :
3702 : /* First try to do it with a special abs instruction. */
3703 24409 : temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3704 : op0, target, 0);
3705 24323 : if (temp != 0)
3706 : return temp;
3707 :
3708 : /* If we have a MAX insn, we can do this as MAX (x, -x). */
3709 86 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3710 86 : && !HONOR_SIGNED_ZEROS (mode))
3711 : {
3712 86 : rtx_insn *last = get_last_insn ();
3713 :
3714 172 : temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3715 : op0, NULL_RTX, 0);
3716 86 : if (temp != 0)
3717 86 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3718 : OPTAB_WIDEN);
3719 :
3720 86 : if (temp != 0)
3721 : return temp;
3722 :
3723 0 : delete_insns_since (last);
3724 : }
3725 :
3726 : /* If this machine has expensive jumps, we can do integer absolute
3727 : value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3728 : where W is the width of MODE. */
3729 :
3730 0 : scalar_int_mode int_mode;
3731 0 : if (is_int_mode (mode, &int_mode)
3732 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
3733 : false) >= 2)
3734 : {
3735 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3736 0 : GET_MODE_PRECISION (int_mode) - 1,
3737 : NULL_RTX, 0);
3738 :
3739 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3740 : OPTAB_LIB_WIDEN);
3741 0 : if (temp != 0)
3742 0 : temp = expand_binop (int_mode,
3743 : result_unsignedp ? sub_optab : subv_optab,
3744 : temp, extended, target, 0, OPTAB_LIB_WIDEN);
3745 :
3746 0 : if (temp != 0)
3747 : return temp;
3748 : }
3749 :
3750 : return NULL_RTX;
3751 : }
3752 :
3753 : rtx
3754 24299 : expand_abs (machine_mode mode, rtx op0, rtx target,
3755 : int result_unsignedp, int safe)
3756 : {
3757 24299 : rtx temp;
3758 24299 : rtx_code_label *op1;
3759 :
3760 24299 : if (GET_MODE_CLASS (mode) != MODE_INT
3761 4171 : || ! flag_trapv)
3762 24213 : result_unsignedp = 1;
3763 :
3764 24299 : temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3765 24299 : if (temp != 0)
3766 : return temp;
3767 :
3768 : /* If that does not win, use conditional jump and negate. */
3769 :
3770 : /* It is safe to use the target if it is the same
3771 : as the source if this is also a pseudo register */
3772 0 : if (op0 == target && REG_P (op0)
3773 0 : && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3774 : safe = 1;
3775 :
3776 0 : op1 = gen_label_rtx ();
3777 0 : if (target == 0 || ! safe
3778 0 : || GET_MODE (target) != mode
3779 0 : || (MEM_P (target) && MEM_VOLATILE_P (target))
3780 0 : || (REG_P (target)
3781 0 : && REGNO (target) < FIRST_PSEUDO_REGISTER))
3782 0 : target = gen_reg_rtx (mode);
3783 :
3784 0 : emit_move_insn (target, op0);
3785 0 : NO_DEFER_POP;
3786 :
3787 0 : do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3788 : NULL_RTX, NULL, op1,
3789 : profile_probability::uninitialized ());
3790 :
3791 0 : op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3792 : target, target, 0);
3793 0 : if (op0 != target)
3794 0 : emit_move_insn (target, op0);
3795 0 : emit_label (op1);
3796 0 : OK_DEFER_POP;
3797 0 : return target;
3798 : }
3799 :
3800 : /* Emit code to compute the one's complement absolute value of OP0
3801 : (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3802 : (TARGET may be NULL_RTX.) The return value says where the result
3803 : actually is to be found.
3804 :
3805 : MODE is the mode of the operand; the mode of the result is
3806 : different but can be deduced from MODE. */
3807 :
3808 : rtx
3809 0 : expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3810 : {
3811 0 : rtx temp;
3812 :
3813 : /* Not applicable for floating point modes. */
3814 0 : if (FLOAT_MODE_P (mode))
3815 : return NULL_RTX;
3816 :
3817 : /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3818 0 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3819 : {
3820 0 : rtx_insn *last = get_last_insn ();
3821 :
3822 0 : temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3823 0 : if (temp != 0)
3824 0 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3825 : OPTAB_WIDEN);
3826 :
3827 0 : if (temp != 0)
3828 : return temp;
3829 :
3830 0 : delete_insns_since (last);
3831 : }
3832 :
3833 : /* If this machine has expensive jumps, we can do one's complement
3834 : absolute value of X as (((signed) x >> (W-1)) ^ x). */
3835 :
3836 0 : scalar_int_mode int_mode;
3837 0 : if (is_int_mode (mode, &int_mode)
3838 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
3839 : false) >= 2)
3840 : {
3841 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3842 0 : GET_MODE_PRECISION (int_mode) - 1,
3843 : NULL_RTX, 0);
3844 :
3845 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3846 : OPTAB_LIB_WIDEN);
3847 :
3848 0 : if (temp != 0)
3849 : return temp;
3850 : }
3851 :
3852 : return NULL_RTX;
3853 : }
3854 :
3855 : /* A subroutine of expand_copysign, perform the copysign operation using the
3856 : abs and neg primitives advertised to exist on the target. The assumption
3857 : is that we have a split register file, and leaving op0 in fp registers,
3858 : and not playing with subregs so much, will help the register allocator. */
3859 :
3860 : static rtx
3861 11605 : expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3862 : int bitpos, bool op0_is_abs)
3863 : {
3864 11605 : scalar_int_mode imode;
3865 11605 : enum insn_code icode;
3866 11605 : rtx sign;
3867 11605 : rtx_code_label *label;
3868 :
3869 11605 : if (target == op1)
3870 132 : target = NULL_RTX;
3871 :
3872 : /* Check if the back end provides an insn that handles signbit for the
3873 : argument's mode. */
3874 11605 : icode = optab_handler (signbit_optab, mode);
3875 11605 : if (icode != CODE_FOR_nothing)
3876 : {
3877 11565 : imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
3878 11565 : sign = gen_reg_rtx (imode);
3879 11565 : emit_unop_insn (icode, sign, op1, UNKNOWN);
3880 : }
3881 : else
3882 : {
3883 100 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3884 : {
3885 40 : if (!int_mode_for_mode (mode).exists (&imode))
3886 0 : return NULL_RTX;
3887 40 : op1 = gen_lowpart (imode, op1);
3888 : }
3889 : else
3890 : {
3891 0 : int word;
3892 :
3893 0 : imode = word_mode;
3894 0 : if (FLOAT_WORDS_BIG_ENDIAN)
3895 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3896 : else
3897 0 : word = bitpos / BITS_PER_WORD;
3898 0 : bitpos = bitpos % BITS_PER_WORD;
3899 0 : op1 = operand_subword_force (op1, word, mode);
3900 : }
3901 :
3902 40 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3903 80 : sign = expand_binop (imode, and_optab, op1,
3904 80 : immed_wide_int_const (mask, imode),
3905 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
3906 40 : }
3907 :
3908 11605 : if (!op0_is_abs)
3909 : {
3910 151 : op0 = expand_unop (mode, abs_optab, op0, target, 0);
3911 151 : if (op0 == NULL)
3912 : return NULL_RTX;
3913 : target = op0;
3914 : }
3915 : else
3916 : {
3917 11454 : if (target == NULL_RTX)
3918 84 : target = copy_to_reg (op0);
3919 : else
3920 11370 : emit_move_insn (target, op0);
3921 : }
3922 :
3923 11605 : label = gen_label_rtx ();
3924 11605 : emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3925 :
3926 11605 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
3927 11454 : op0 = simplify_unary_operation (NEG, mode, op0, mode);
3928 : else
3929 151 : op0 = expand_unop (mode, neg_optab, op0, target, 0);
3930 11605 : if (op0 != target)
3931 11454 : emit_move_insn (target, op0);
3932 :
3933 11605 : emit_label (label);
3934 :
3935 11605 : return target;
3936 : }
3937 :
3938 :
3939 : /* A subroutine of expand_copysign, perform the entire copysign operation
3940 : with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3941 : is true if op0 is known to have its sign bit clear. */
3942 :
3943 : static rtx
3944 68 : expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3945 : int bitpos, bool op0_is_abs)
3946 : {
3947 68 : scalar_int_mode imode;
3948 68 : int word, nwords, i;
3949 68 : rtx temp;
3950 68 : rtx_insn *insns;
3951 :
3952 156 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3953 : {
3954 68 : if (!int_mode_for_mode (mode).exists (&imode))
3955 0 : return NULL_RTX;
3956 : word = 0;
3957 : nwords = 1;
3958 : }
3959 : else
3960 : {
3961 0 : imode = word_mode;
3962 :
3963 0 : if (FLOAT_WORDS_BIG_ENDIAN)
3964 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3965 : else
3966 0 : word = bitpos / BITS_PER_WORD;
3967 0 : bitpos = bitpos % BITS_PER_WORD;
3968 0 : nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3969 : }
3970 :
3971 68 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3972 :
3973 68 : if (target == 0
3974 68 : || target == op0
3975 68 : || target == op1
3976 64 : || reg_overlap_mentioned_p (target, op0)
3977 64 : || reg_overlap_mentioned_p (target, op1)
3978 132 : || (nwords > 1 && !valid_multiword_target_p (target)))
3979 4 : target = gen_reg_rtx (mode);
3980 :
3981 68 : if (nwords > 1)
3982 : {
3983 0 : start_sequence ();
3984 :
3985 0 : for (i = 0; i < nwords; ++i)
3986 : {
3987 0 : rtx targ_piece = operand_subword (target, i, 1, mode);
3988 0 : rtx op0_piece = operand_subword_force (op0, i, mode);
3989 :
3990 0 : if (i == word)
3991 : {
3992 0 : if (!op0_is_abs)
3993 0 : op0_piece
3994 0 : = expand_binop (imode, and_optab, op0_piece,
3995 0 : immed_wide_int_const (~mask, imode),
3996 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
3997 0 : op1 = expand_binop (imode, and_optab,
3998 0 : operand_subword_force (op1, i, mode),
3999 0 : immed_wide_int_const (mask, imode),
4000 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4001 :
4002 0 : temp = expand_binop (imode, ior_optab, op0_piece, op1,
4003 : targ_piece, 1, OPTAB_LIB_WIDEN);
4004 0 : if (temp != targ_piece)
4005 0 : emit_move_insn (targ_piece, temp);
4006 : }
4007 : else
4008 0 : emit_move_insn (targ_piece, op0_piece);
4009 : }
4010 :
4011 0 : insns = end_sequence ();
4012 :
4013 0 : emit_insn (insns);
4014 : }
4015 : else
4016 : {
4017 136 : op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
4018 68 : immed_wide_int_const (mask, imode),
4019 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4020 :
4021 68 : op0 = gen_lowpart (imode, op0);
4022 68 : if (!op0_is_abs)
4023 136 : op0 = expand_binop (imode, and_optab, op0,
4024 136 : immed_wide_int_const (~mask, imode),
4025 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4026 :
4027 68 : temp = expand_binop (imode, ior_optab, op0, op1,
4028 68 : gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
4029 68 : target = force_lowpart_subreg (mode, temp, imode);
4030 : }
4031 :
4032 68 : return target;
4033 68 : }
4034 :
4035 : /* Expand the C99 copysign operation. OP0 and OP1 must be the same
4036 : scalar floating point mode. Return NULL if we do not know how to
4037 : expand the operation inline. */
4038 :
4039 : rtx
4040 11673 : expand_copysign (rtx op0, rtx op1, rtx target)
4041 : {
4042 11673 : scalar_float_mode mode;
4043 11673 : const struct real_format *fmt;
4044 11673 : bool op0_is_abs;
4045 11673 : rtx temp;
4046 :
4047 11673 : mode = as_a <scalar_float_mode> (GET_MODE (op0));
4048 11673 : gcc_assert (GET_MODE (op1) == mode);
4049 :
4050 : /* First try to do it with a special instruction. */
4051 11673 : temp = expand_binop (mode, copysign_optab, op0, op1,
4052 : target, 0, OPTAB_DIRECT);
4053 11673 : if (temp)
4054 : return temp;
4055 :
4056 11673 : fmt = REAL_MODE_FORMAT (mode);
4057 11673 : if (fmt == NULL || !fmt->has_signed_zero)
4058 : return NULL_RTX;
4059 :
4060 11673 : op0_is_abs = false;
4061 11673 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
4062 : {
4063 11454 : if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
4064 0 : op0 = simplify_unary_operation (ABS, mode, op0, mode);
4065 : op0_is_abs = true;
4066 : }
4067 :
4068 11673 : if (fmt->signbit_ro >= 0
4069 11673 : && (CONST_DOUBLE_AS_FLOAT_P (op0)
4070 219 : || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
4071 151 : && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
4072 : {
4073 23210 : temp = expand_copysign_absneg (mode, op0, op1, target,
4074 11605 : fmt->signbit_ro, op0_is_abs);
4075 11605 : if (temp)
4076 : return temp;
4077 : }
4078 :
4079 68 : if (fmt->signbit_rw < 0)
4080 : return NULL_RTX;
4081 68 : return expand_copysign_bit (mode, op0, op1, target,
4082 68 : fmt->signbit_rw, op0_is_abs);
4083 : }
4084 :
4085 : /* Generate an instruction whose insn-code is INSN_CODE,
4086 : with two operands: an output TARGET and an input OP0.
4087 : TARGET *must* be nonzero, and the output is always stored there.
4088 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4089 : the value that is stored into TARGET.
4090 :
4091 : Return false if expansion failed. */
4092 :
4093 : bool
4094 2149946 : maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
4095 : enum rtx_code code)
4096 : {
4097 2149946 : class expand_operand ops[2];
4098 2149946 : rtx_insn *pat;
4099 :
4100 2149946 : create_output_operand (&ops[0], target, GET_MODE (target));
4101 2149946 : create_input_operand (&ops[1], op0, GET_MODE (op0));
4102 2149946 : pat = maybe_gen_insn (icode, 2, ops);
4103 2149946 : if (!pat)
4104 : return false;
4105 :
4106 1147056 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
4107 2172601 : && code != UNKNOWN)
4108 10925 : add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
4109 10925 : GET_MODE (op0));
4110 :
4111 2149945 : emit_insn (pat);
4112 :
4113 2149945 : if (ops[0].value != target)
4114 38208 : emit_move_insn (target, ops[0].value);
4115 : return true;
4116 : }
4117 : /* Generate an instruction whose insn-code is INSN_CODE,
4118 : with two operands: an output TARGET and an input OP0.
4119 : TARGET *must* be nonzero, and the output is always stored there.
4120 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4121 : the value that is stored into TARGET. */
4122 :
4123 : void
4124 2103759 : emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
4125 : {
4126 2103759 : bool ok = maybe_emit_unop_insn (icode, target, op0, code);
4127 2103759 : gcc_assert (ok);
4128 2103759 : }
4129 :
4130 : struct no_conflict_data
4131 : {
4132 : rtx target;
4133 : rtx_insn *first, *insn;
4134 : bool must_stay;
4135 : };
4136 :
4137 : /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4138 : the currently examined clobber / store has to stay in the list of
4139 : insns that constitute the actual libcall block. */
4140 : static void
4141 57169 : no_conflict_move_test (rtx dest, const_rtx set, void *p0)
4142 : {
4143 57169 : struct no_conflict_data *p= (struct no_conflict_data *) p0;
4144 :
4145 : /* If this inns directly contributes to setting the target, it must stay. */
4146 57169 : if (reg_overlap_mentioned_p (p->target, dest))
4147 0 : p->must_stay = true;
4148 : /* If we haven't committed to keeping any other insns in the list yet,
4149 : there is nothing more to check. */
4150 57169 : else if (p->insn == p->first)
4151 : return;
4152 : /* If this insn sets / clobbers a register that feeds one of the insns
4153 : already in the list, this insn has to stay too. */
4154 27395 : else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
4155 27395 : || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
4156 27395 : || reg_used_between_p (dest, p->first, p->insn)
4157 : /* Likewise if this insn depends on a register set by a previous
4158 : insn in the list, or if it sets a result (presumably a hard
4159 : register) that is set or clobbered by a previous insn.
4160 : N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4161 : SET_DEST perform the former check on the address, and the latter
4162 : check on the MEM. */
4163 54790 : || (GET_CODE (set) == SET
4164 27395 : && (modified_in_p (SET_SRC (set), p->first)
4165 27393 : || modified_in_p (SET_DEST (set), p->first)
4166 27393 : || modified_between_p (SET_SRC (set), p->first, p->insn)
4167 27393 : || modified_between_p (SET_DEST (set), p->first, p->insn))))
4168 2 : p->must_stay = true;
4169 : }
4170 :
4171 :
4172 : /* Emit code to make a call to a constant function or a library call.
4173 :
4174 : INSNS is a list containing all insns emitted in the call.
4175 : These insns leave the result in RESULT. Our block is to copy RESULT
4176 : to TARGET, which is logically equivalent to EQUIV.
4177 :
4178 : We first emit any insns that set a pseudo on the assumption that these are
4179 : loading constants into registers; doing so allows them to be safely cse'ed
4180 : between blocks. Then we emit all the other insns in the block, followed by
4181 : an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4182 : note with an operand of EQUIV. */
4183 :
4184 : static void
4185 116367 : emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
4186 : bool equiv_may_trap)
4187 : {
4188 116367 : rtx final_dest = target;
4189 116367 : rtx_insn *next, *last, *insn;
4190 :
4191 : /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4192 : into a MEM later. Protect the libcall block from this change. */
4193 116367 : if (! REG_P (target) || REG_USERVAR_P (target))
4194 1808 : target = gen_reg_rtx (GET_MODE (target));
4195 :
4196 : /* If we're using non-call exceptions, a libcall corresponding to an
4197 : operation that may trap may also trap. */
4198 : /* ??? See the comment in front of make_reg_eh_region_note. */
4199 116367 : if (cfun->can_throw_non_call_exceptions
4200 116367 : && (equiv_may_trap || may_trap_p (equiv)))
4201 : {
4202 1036 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4203 832 : if (CALL_P (insn))
4204 : {
4205 204 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
4206 204 : if (note)
4207 : {
4208 204 : int lp_nr = INTVAL (XEXP (note, 0));
4209 204 : if (lp_nr == 0 || lp_nr == INT_MIN)
4210 204 : remove_note (insn, note);
4211 : }
4212 : }
4213 : }
4214 : else
4215 : {
4216 : /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4217 : reg note to indicate that this call cannot throw or execute a nonlocal
4218 : goto (unless there is already a REG_EH_REGION note, in which case
4219 : we update it). */
4220 514470 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4221 398307 : if (CALL_P (insn))
4222 116163 : make_reg_eh_region_note_nothrow_nononlocal (insn);
4223 : }
4224 :
4225 : /* First emit all insns that set pseudos. Remove them from the list as
4226 : we go. Avoid insns that set pseudos which were referenced in previous
4227 : insns. These can be generated by move_by_pieces, for example,
4228 : to update an address. Similarly, avoid insns that reference things
4229 : set in previous insns. */
4230 :
4231 515506 : for (insn = insns; insn; insn = next)
4232 : {
4233 399139 : rtx set = single_set (insn);
4234 :
4235 399139 : next = NEXT_INSN (insn);
4236 :
4237 384644 : if (set != 0 && REG_P (SET_DEST (set))
4238 746619 : && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
4239 : {
4240 47074 : struct no_conflict_data data;
4241 :
4242 47074 : data.target = const0_rtx;
4243 47074 : data.first = insns;
4244 47074 : data.insn = insn;
4245 47074 : data.must_stay = 0;
4246 47074 : note_stores (insn, no_conflict_move_test, &data);
4247 47074 : if (! data.must_stay)
4248 : {
4249 47072 : if (PREV_INSN (insn))
4250 27393 : SET_NEXT_INSN (PREV_INSN (insn)) = next;
4251 : else
4252 : insns = next;
4253 :
4254 47072 : if (next)
4255 47072 : SET_PREV_INSN (next) = PREV_INSN (insn);
4256 :
4257 47072 : add_insn (insn);
4258 : }
4259 : }
4260 :
4261 : /* Some ports use a loop to copy large arguments onto the stack.
4262 : Don't move anything outside such a loop. */
4263 399139 : if (LABEL_P (insn))
4264 : break;
4265 : }
4266 :
4267 : /* Write the remaining insns followed by the final copy. */
4268 468434 : for (insn = insns; insn; insn = next)
4269 : {
4270 352067 : next = NEXT_INSN (insn);
4271 :
4272 352067 : add_insn (insn);
4273 : }
4274 :
4275 116367 : last = emit_move_insn (target, result);
4276 116367 : if (equiv)
4277 115760 : set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
4278 :
4279 116367 : if (final_dest != target)
4280 1808 : emit_move_insn (final_dest, target);
4281 116367 : }
4282 :
4283 : void
4284 85192 : emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
4285 : {
4286 85192 : emit_libcall_block_1 (insns, target, result, equiv, false);
4287 85192 : }
4288 :
4289 : /* True if we can perform a comparison of mode MODE straightforwardly.
4290 : PURPOSE describes how this comparison will be used. CODE is the rtx
4291 : comparison code we will be using.
4292 :
4293 : ??? Actually, CODE is slightly weaker than that. A target is still
4294 : required to implement all of the normal bcc operations, but not
4295 : required to implement all (or any) of the unordered bcc operations. */
4296 :
4297 : bool
4298 17933451 : can_compare_p (enum rtx_code code, machine_mode mode,
4299 : enum can_compare_purpose purpose)
4300 : {
4301 17933451 : rtx test;
4302 17933451 : test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
4303 19978118 : do
4304 : {
4305 19978118 : enum insn_code icode;
4306 :
4307 19978118 : if (purpose == ccp_jump
4308 19781807 : && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
4309 38418624 : && insn_operand_matches (icode, 0, test))
4310 : return true;
4311 3322470 : if (purpose == ccp_store_flag
4312 196311 : && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
4313 3451338 : && insn_operand_matches (icode, 1, test))
4314 : return true;
4315 :
4316 3320020 : mode = GET_MODE_WIDER_MODE (mode).else_void ();
4317 3320020 : PUT_MODE (test, mode);
4318 : }
4319 3320020 : while (mode != VOIDmode);
4320 :
4321 : return false;
4322 : }
4323 :
4324 : /* Return whether RTL code CODE corresponds to an unsigned optab. */
4325 :
4326 : static bool
4327 1316706 : unsigned_optab_p (enum rtx_code code)
4328 : {
4329 1316706 : return code == LTU || code == LEU || code == GTU || code == GEU;
4330 : }
4331 :
4332 : /* Return whether the backend-emitted comparison for code CODE, comparing
4333 : operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4334 : operand OPNO of pattern ICODE. */
4335 :
4336 : static bool
4337 954655 : insn_predicate_matches_p (enum insn_code icode, unsigned int opno,
4338 : enum rtx_code code, machine_mode mask_mode,
4339 : machine_mode value_mode)
4340 : {
4341 954655 : rtx reg1 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 1);
4342 954655 : rtx reg2 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 2);
4343 954655 : rtx test = alloca_rtx_fmt_ee (code, mask_mode, reg1, reg2);
4344 954655 : return insn_operand_matches (icode, opno, test);
4345 : }
4346 :
4347 : /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4348 : for code CODE, comparing operands of mode VALUE_MODE and producing a result
4349 : with MASK_MODE. */
4350 :
4351 : bool
4352 1316706 : can_vec_cmp_compare_p (enum rtx_code code, machine_mode value_mode,
4353 : machine_mode mask_mode)
4354 : {
4355 1316706 : enum insn_code icode
4356 1316706 : = get_vec_cmp_icode (value_mode, mask_mode, unsigned_optab_p (code));
4357 1316706 : if (icode == CODE_FOR_nothing)
4358 : return false;
4359 :
4360 954655 : return insn_predicate_matches_p (icode, 1, code, mask_mode, value_mode);
4361 : }
4362 :
4363 : /* Return whether the backend can emit vector set instructions for inserting
4364 : element into vector at variable index position. */
4365 :
4366 : bool
4367 141 : can_vec_set_var_idx_p (machine_mode vec_mode)
4368 : {
4369 141 : if (!VECTOR_MODE_P (vec_mode))
4370 : return false;
4371 :
4372 139 : machine_mode inner_mode = GET_MODE_INNER (vec_mode);
4373 :
4374 139 : rtx reg1 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 1);
4375 139 : rtx reg2 = alloca_raw_REG (inner_mode, LAST_VIRTUAL_REGISTER + 2);
4376 :
4377 139 : enum insn_code icode = optab_handler (vec_set_optab, vec_mode);
4378 :
4379 139 : const struct insn_data_d *data = &insn_data[icode];
4380 139 : machine_mode idx_mode = data->operand[2].mode;
4381 :
4382 139 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4383 :
4384 138 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4385 138 : && insn_operand_matches (icode, 1, reg2)
4386 277 : && insn_operand_matches (icode, 2, reg3);
4387 : }
4388 :
4389 : /* Return whether the backend can emit a vec_extract instruction with
4390 : a non-constant index. */
4391 : bool
4392 18411 : can_vec_extract_var_idx_p (machine_mode vec_mode, machine_mode extr_mode)
4393 : {
4394 18411 : if (!VECTOR_MODE_P (vec_mode))
4395 : return false;
4396 :
4397 15835 : rtx reg1 = alloca_raw_REG (extr_mode, LAST_VIRTUAL_REGISTER + 1);
4398 15835 : rtx reg2 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 2);
4399 :
4400 15835 : enum insn_code icode = convert_optab_handler (vec_extract_optab,
4401 : vec_mode, extr_mode);
4402 :
4403 15835 : const struct insn_data_d *data = &insn_data[icode];
4404 15835 : machine_mode idx_mode = data->operand[2].mode;
4405 :
4406 15835 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4407 :
4408 15008 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4409 15008 : && insn_operand_matches (icode, 1, reg2)
4410 30843 : && insn_operand_matches (icode, 2, reg3);
4411 : }
4412 :
4413 : /* This function is called when we are going to emit a compare instruction that
4414 : compares the values found in X and Y, using the rtl operator COMPARISON.
4415 :
4416 : If they have mode BLKmode, then SIZE specifies the size of both operands.
4417 :
4418 : UNSIGNEDP nonzero says that the operands are unsigned;
4419 : this matters if they need to be widened (as given by METHODS).
4420 :
4421 : *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4422 : if we failed to produce one.
4423 :
4424 : *PMODE is the mode of the inputs (in case they are const_int).
4425 :
4426 : *OPTAB is the optab to check for OPTAB_DIRECT support. Defaults to
4427 : cbranch_optab.
4428 :
4429 : This function performs all the setup necessary so that the caller only has
4430 : to emit a single comparison insn. This setup can involve doing a BLKmode
4431 : comparison or emitting a library call to perform the comparison if no insn
4432 : is available to handle it.
4433 : The values which are passed in through pointers can be modified; the caller
4434 : should perform the comparison on the modified values. Constant
4435 : comparisons must have already been folded. */
4436 :
4437 : static void
4438 6806487 : prepare_cmp_insn (rtx x, rtx y, rtx *mask, enum rtx_code comparison, rtx size,
4439 : int unsignedp, enum optab_methods methods,
4440 : rtx *ptest, machine_mode *pmode, optab optab)
4441 : {
4442 6806487 : machine_mode mode = *pmode;
4443 6806487 : rtx libfunc, test;
4444 6806487 : machine_mode cmp_mode;
4445 :
4446 : /* The other methods are not needed. */
4447 6806487 : gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4448 : || methods == OPTAB_LIB_WIDEN);
4449 :
4450 6806487 : if (CONST_SCALAR_INT_P (y))
4451 4403709 : canonicalize_comparison (mode, &comparison, &y);
4452 :
4453 : /* If we are optimizing, force expensive constants into a register. */
4454 1 : if (CONSTANT_P (x) && optimize
4455 1 : && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
4456 : > COSTS_N_INSNS (1))
4457 6806487 : && can_create_pseudo_p ())
4458 0 : x = force_reg (mode, x);
4459 :
4460 4914591 : if (CONSTANT_P (y) && optimize
4461 3892306 : && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
4462 : > COSTS_N_INSNS (1))
4463 6979034 : && can_create_pseudo_p ())
4464 172547 : y = force_reg (mode, y);
4465 :
4466 : /* Don't let both operands fail to indicate the mode. */
4467 6806487 : if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4468 0 : x = force_reg (mode, x);
4469 6806487 : if (mode == VOIDmode)
4470 4662 : mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4471 :
4472 : /* Handle all BLKmode compares. */
4473 :
4474 6806487 : if (mode == BLKmode)
4475 : {
4476 0 : machine_mode result_mode;
4477 0 : enum insn_code cmp_code;
4478 0 : rtx result;
4479 0 : rtx opalign
4480 0 : = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4481 :
4482 0 : gcc_assert (size);
4483 :
4484 : /* Try to use a memory block compare insn - either cmpstr
4485 : or cmpmem will do. */
4486 0 : opt_scalar_int_mode cmp_mode_iter;
4487 0 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
4488 : {
4489 0 : scalar_int_mode cmp_mode = cmp_mode_iter.require ();
4490 0 : cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4491 0 : if (cmp_code == CODE_FOR_nothing)
4492 0 : cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4493 0 : if (cmp_code == CODE_FOR_nothing)
4494 0 : cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4495 0 : if (cmp_code == CODE_FOR_nothing)
4496 0 : continue;
4497 :
4498 : /* Must make sure the size fits the insn's mode. */
4499 0 : if (CONST_INT_P (size)
4500 0 : ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
4501 0 : : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
4502 0 : > GET_MODE_BITSIZE (cmp_mode)))
4503 0 : continue;
4504 :
4505 0 : result_mode = insn_data[cmp_code].operand[0].mode;
4506 0 : result = gen_reg_rtx (result_mode);
4507 0 : size = convert_to_mode (cmp_mode, size, 1);
4508 0 : emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4509 :
4510 0 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4511 0 : *pmode = result_mode;
4512 0 : return;
4513 : }
4514 :
4515 0 : if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4516 0 : goto fail;
4517 :
4518 : /* Otherwise call a library function. */
4519 0 : result = emit_block_comp_via_libcall (x, y, size);
4520 :
4521 0 : x = result;
4522 0 : y = const0_rtx;
4523 0 : mode = TYPE_MODE (integer_type_node);
4524 0 : methods = OPTAB_LIB_WIDEN;
4525 0 : unsignedp = false;
4526 : }
4527 :
4528 : /* Don't allow operands to the compare to trap, as that can put the
4529 : compare and branch in different basic blocks. */
4530 6806487 : if (cfun->can_throw_non_call_exceptions)
4531 : {
4532 1135646 : if (!can_create_pseudo_p () && (may_trap_p (x) || may_trap_p (y)))
4533 0 : goto fail;
4534 1135646 : if (may_trap_p (x))
4535 25098 : x = copy_to_reg (x);
4536 1135646 : if (may_trap_p (y))
4537 1522 : y = copy_to_reg (y);
4538 : }
4539 :
4540 6806487 : if (GET_MODE_CLASS (mode) == MODE_CC)
4541 : {
4542 496 : enum insn_code icode = optab_handler (cbranch_optab, CCmode);
4543 496 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4544 496 : if (icode != CODE_FOR_nothing
4545 496 : && insn_operand_matches (icode, 0, test))
4546 : {
4547 496 : *ptest = test;
4548 496 : return;
4549 : }
4550 : else
4551 0 : goto fail;
4552 : }
4553 :
4554 6805991 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4555 6891454 : FOR_EACH_WIDER_MODE_FROM (cmp_mode, mode)
4556 : {
4557 6844347 : enum insn_code icode;
4558 6844347 : icode = optab_handler (optab, cmp_mode);
4559 6844347 : if (icode != CODE_FOR_nothing
4560 6844347 : && insn_operand_matches (icode, 0, test))
4561 : {
4562 6758884 : rtx_insn *last = get_last_insn ();
4563 6758884 : rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4564 6758884 : rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4565 6758884 : if (op0 && op1
4566 6758884 : && insn_operand_matches (icode, 1, op0)
4567 13517768 : && insn_operand_matches (icode, 2, op1))
4568 : {
4569 6758884 : XEXP (test, 0) = op0;
4570 6758884 : XEXP (test, 1) = op1;
4571 6758884 : *ptest = test;
4572 6758884 : *pmode = cmp_mode;
4573 6758884 : return;
4574 : }
4575 0 : delete_insns_since (last);
4576 : }
4577 :
4578 85463 : if (methods == OPTAB_DIRECT)
4579 : break;
4580 : }
4581 :
4582 47107 : if (methods != OPTAB_LIB_WIDEN)
4583 2925 : goto fail;
4584 :
4585 44182 : if (SCALAR_FLOAT_MODE_P (mode))
4586 : {
4587 : /* Small trick if UNORDERED isn't implemented by the hardware. */
4588 44182 : if (comparison == UNORDERED && rtx_equal_p (x, y))
4589 : {
4590 761 : prepare_cmp_insn (x, y, mask, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
4591 : ptest, pmode, optab);
4592 761 : if (*ptest)
4593 : return;
4594 : }
4595 :
4596 44182 : prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4597 : }
4598 : else
4599 : {
4600 0 : rtx result;
4601 0 : machine_mode ret_mode;
4602 :
4603 : /* Handle a libcall just for the mode we are using. */
4604 0 : libfunc = optab_libfunc (cmp_optab, mode);
4605 0 : gcc_assert (libfunc);
4606 :
4607 : /* If we want unsigned, and this mode has a distinct unsigned
4608 : comparison routine, use that. */
4609 0 : if (unsignedp)
4610 : {
4611 0 : rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4612 0 : if (ulibfunc)
4613 0 : libfunc = ulibfunc;
4614 : }
4615 :
4616 0 : ret_mode = targetm.libgcc_cmp_return_mode ();
4617 0 : result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4618 : ret_mode, x, mode, y, mode);
4619 :
4620 : /* There are two kinds of comparison routines. Biased routines
4621 : return 0/1/2, and unbiased routines return -1/0/1. Other parts
4622 : of gcc expect that the comparison operation is equivalent
4623 : to the modified comparison. For signed comparisons compare the
4624 : result against 1 in the biased case, and zero in the unbiased
4625 : case. For unsigned comparisons always compare against 1 after
4626 : biasing the unbiased result by adding 1. This gives us a way to
4627 : represent LTU.
4628 : The comparisons in the fixed-point helper library are always
4629 : biased. */
4630 0 : x = result;
4631 0 : y = const1_rtx;
4632 :
4633 0 : if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4634 : {
4635 : if (unsignedp)
4636 : x = plus_constant (ret_mode, result, 1);
4637 : else
4638 : y = const0_rtx;
4639 : }
4640 :
4641 0 : *pmode = ret_mode;
4642 0 : prepare_cmp_insn (x, y, mask, comparison, NULL_RTX, unsignedp, methods,
4643 : ptest, pmode, optab);
4644 : }
4645 :
4646 : return;
4647 :
4648 2925 : fail:
4649 2925 : *ptest = NULL_RTX;
4650 : }
4651 :
4652 : /* Before emitting an insn with code ICODE, make sure that X, which is going
4653 : to be used for operand OPNUM of the insn, is converted from mode MODE to
4654 : WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4655 : that it is accepted by the operand predicate. Return the new value. */
4656 :
4657 : rtx
4658 14928334 : prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4659 : machine_mode wider_mode, int unsignedp)
4660 : {
4661 14928334 : if (mode != wider_mode)
4662 7266 : x = convert_modes (wider_mode, mode, x, unsignedp);
4663 :
4664 14928334 : if (!insn_operand_matches (icode, opnum, x))
4665 : {
4666 62581 : machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4667 62581 : if (reload_completed)
4668 : return NULL_RTX;
4669 62581 : if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4670 : return NULL_RTX;
4671 62271 : x = copy_to_mode_reg (op_mode, x);
4672 : }
4673 :
4674 : return x;
4675 : }
4676 :
4677 : /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4678 : we can do the branch. */
4679 :
4680 : static void
4681 6347415 : emit_cmp_and_jump_insn_1 (rtx test, rtx cond, rtx len, rtx bias,
4682 : machine_mode mode, rtx label, direct_optab cmp_optab,
4683 : profile_probability prob, bool test_branch)
4684 : {
4685 6347415 : machine_mode optab_mode;
4686 6347415 : enum mode_class mclass;
4687 6347415 : enum insn_code icode;
4688 6347415 : rtx_insn *insn;
4689 :
4690 6347415 : mclass = GET_MODE_CLASS (mode);
4691 6347415 : optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4692 6347415 : icode = optab_handler (cmp_optab, optab_mode);
4693 :
4694 6347415 : gcc_assert (icode != CODE_FOR_nothing);
4695 6347415 : gcc_assert (test_branch || insn_operand_matches (icode, 0, test));
4696 6347415 : gcc_assert (cond == NULL_RTX || (cond != NULL_RTX && !test_branch));
4697 6347415 : if (test_branch)
4698 0 : insn = emit_jump_insn (GEN_FCN (icode) (XEXP (test, 0),
4699 0 : XEXP (test, 1), label));
4700 6347415 : else if (len)
4701 : {
4702 0 : gcc_assert (cond);
4703 0 : gcc_assert (bias);
4704 0 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4705 0 : XEXP (test, 1), len, bias,
4706 : label));
4707 : }
4708 6347415 : else if (cond)
4709 0 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4710 0 : XEXP (test, 1), label));
4711 : else
4712 6347415 : insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4713 6347415 : XEXP (test, 1), label));
4714 :
4715 6347415 : if (prob.initialized_p ()
4716 4791236 : && profile_status_for_fn (cfun) != PROFILE_ABSENT
4717 4691750 : && insn
4718 4691750 : && JUMP_P (insn)
4719 4691022 : && any_condjump_p (insn)
4720 11038437 : && !find_reg_note (insn, REG_BR_PROB, 0))
4721 4690101 : add_reg_br_prob_note (insn, prob);
4722 6347415 : }
4723 :
4724 : /* PTEST points to a comparison that compares its first operand with zero.
4725 : Check to see if it can be performed as a bit-test-and-branch instead.
4726 : On success, return the instruction that performs the bit-test-and-branch
4727 : and replace the second operand of *PTEST with the bit number to test.
4728 : On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
4729 :
4730 : Note that the comparison described by *PTEST should not be taken
4731 : literally after a successful return. *PTEST is just a convenient
4732 : place to store the two operands of the bit-and-test.
4733 :
4734 : VAL must contain the original tree expression for the first operand
4735 : of *PTEST. */
4736 :
4737 : static enum insn_code
4738 2585242 : validate_test_and_branch (tree val, rtx *ptest, machine_mode *pmode, optab *res)
4739 : {
4740 2585242 : if (!val || TREE_CODE (val) != SSA_NAME)
4741 : return CODE_FOR_nothing;
4742 :
4743 2445117 : machine_mode mode = TYPE_MODE (TREE_TYPE (val));
4744 2445117 : rtx test = *ptest;
4745 2445117 : direct_optab optab;
4746 :
4747 2445117 : if (GET_CODE (test) == EQ)
4748 : optab = tbranch_eq_optab;
4749 1287392 : else if (GET_CODE (test) == NE)
4750 : optab = tbranch_ne_optab;
4751 : else
4752 : return CODE_FOR_nothing;
4753 :
4754 2218061 : *res = optab;
4755 :
4756 : /* If the target supports the testbit comparison directly, great. */
4757 2218061 : auto icode = direct_optab_handler (optab, mode);
4758 2218061 : if (icode == CODE_FOR_nothing)
4759 : return icode;
4760 :
4761 0 : if (tree_zero_one_valued_p (val))
4762 : {
4763 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 : 0;
4764 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4765 0 : *ptest = test;
4766 0 : *pmode = mode;
4767 0 : return icode;
4768 : }
4769 :
4770 0 : wide_int wcst = get_nonzero_bits (val);
4771 0 : if (wcst == -1)
4772 : return CODE_FOR_nothing;
4773 :
4774 0 : int bitpos;
4775 :
4776 0 : if ((bitpos = wi::exact_log2 (wcst)) == -1)
4777 : return CODE_FOR_nothing;
4778 :
4779 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 - bitpos : bitpos;
4780 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4781 0 : *ptest = test;
4782 0 : *pmode = mode;
4783 0 : return icode;
4784 0 : }
4785 :
4786 : /* Generate code to compare X with Y so that the condition codes are
4787 : set and to jump to LABEL if the condition is true. If X is a
4788 : constant and Y is not a constant, then the comparison is swapped to
4789 : ensure that the comparison RTL has the canonical form.
4790 :
4791 : UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4792 : need to be widened. UNSIGNEDP is also used to select the proper
4793 : branch condition code.
4794 :
4795 : If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4796 :
4797 : MODE is the mode of the inputs (in case they are const_int).
4798 :
4799 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4800 : It will be potentially converted into an unsigned variant based on
4801 : UNSIGNEDP to select a proper jump instruction.
4802 :
4803 : PROB is the probability of jumping to LABEL. If the comparison is against
4804 : zero then VAL contains the expression from which the non-zero RTL is
4805 : derived. */
4806 :
4807 : void
4808 6347415 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4809 : machine_mode mode, int unsignedp, tree val, rtx label,
4810 : profile_probability prob)
4811 : {
4812 6347415 : rtx op0 = x, op1 = y;
4813 6347415 : rtx test;
4814 :
4815 : /* Swap operands and condition to ensure canonical RTL. */
4816 6347415 : if (swap_commutative_operands_p (x, y)
4817 6347415 : && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4818 : {
4819 0 : op0 = y, op1 = x;
4820 0 : comparison = swap_condition (comparison);
4821 : }
4822 :
4823 : /* If OP0 is still a constant, then both X and Y must be constants
4824 : or the opposite comparison is not supported. Force X into a register
4825 : to create canonical RTL. */
4826 6347415 : if (CONSTANT_P (op0))
4827 18317 : op0 = force_reg (mode, op0);
4828 :
4829 6347415 : if (unsignedp)
4830 3799096 : comparison = unsigned_condition (comparison);
4831 :
4832 : /* cbranch is no longer preferred for vectors, so when using a vector mode
4833 : check vec_cbranch variants instead. */
4834 6347415 : if (!VECTOR_MODE_P (GET_MODE (op0)))
4835 6328220 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
4836 : OPTAB_LIB_WIDEN, &test, &mode, cbranch_optab);
4837 :
4838 : /* Check if we're comparing a truth type with 0, and if so check if
4839 : the target supports tbranch. */
4840 6347415 : machine_mode tmode = mode;
4841 6347415 : direct_optab optab;
4842 6347415 : if (op1 == CONST0_RTX (GET_MODE (op1)))
4843 : {
4844 2586890 : if (!VECTOR_MODE_P (GET_MODE (op1))
4845 2585242 : && validate_test_and_branch (val, &test, &tmode,
4846 : &optab) != CODE_FOR_nothing)
4847 : {
4848 0 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, tmode,
4849 : label, optab, prob, true);
4850 0 : return;
4851 : }
4852 :
4853 : /* If we are comparing equality with 0, check if VAL is another equality
4854 : comparison and if the target supports it directly. */
4855 2586890 : gimple *def_stmt = NULL;
4856 2447673 : if (val && TREE_CODE (val) == SSA_NAME
4857 2446752 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (val))
4858 2370 : && (comparison == NE || comparison == EQ)
4859 2589260 : && (def_stmt = get_gimple_for_ssa_name (val)))
4860 : {
4861 1666 : tree masked_op = NULL_TREE;
4862 1666 : tree len_op = NULL_TREE;
4863 1666 : tree len_bias = NULL_TREE;
4864 : /* First determine if the operation should be masked or unmasked. */
4865 1666 : if (is_gimple_assign (def_stmt)
4866 1666 : && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
4867 : {
4868 : /* See if one side if a comparison, if so use the other side as
4869 : the mask. */
4870 2 : gimple *mask_def = NULL;
4871 2 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
4872 2 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
4873 2 : if (TREE_CODE (rhs1) == SSA_NAME
4874 2 : && (mask_def = get_gimple_for_ssa_name (rhs1))
4875 2 : && is_gimple_assign (mask_def)
4876 4 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
4877 : masked_op = rhs2;
4878 0 : else if (TREE_CODE (rhs2) == SSA_NAME
4879 0 : && (mask_def = get_gimple_for_ssa_name (rhs2))
4880 0 : && is_gimple_assign (mask_def)
4881 0 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
4882 : masked_op = rhs1;
4883 :
4884 2 : if (masked_op)
4885 1666 : def_stmt = mask_def;
4886 : }
4887 : /* Else check to see if we're a LEN target. */
4888 1664 : else if (is_gimple_call (def_stmt)
4889 0 : && gimple_call_internal_p (def_stmt)
4890 1664 : && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
4891 : {
4892 : /* Example to consume:
4893 :
4894 : a = _59 != vect__4.17_75;
4895 : vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
4896 : if (vcmp != { 0, ... })
4897 :
4898 : and transform into
4899 :
4900 : if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)). */
4901 0 : gcall *call = dyn_cast <gcall *> (def_stmt);
4902 0 : tree true_branch = gimple_call_arg (call, 1);
4903 0 : tree false_branch = gimple_call_arg (call, 2);
4904 0 : if (integer_minus_onep (true_branch)
4905 0 : && integer_zerop (false_branch))
4906 : {
4907 0 : len_op = gimple_call_arg (call, 3);
4908 0 : len_bias = gimple_call_arg (call, 4);
4909 0 : tree arg0 = gimple_call_arg (call, 0);
4910 :
4911 0 : if (TREE_CODE (arg0) == SSA_NAME)
4912 0 : def_stmt = get_gimple_for_ssa_name (arg0);
4913 : }
4914 : }
4915 :
4916 1666 : enum insn_code icode;
4917 1666 : if (def_stmt
4918 1666 : && is_gimple_assign (def_stmt)
4919 3332 : && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
4920 : == tcc_comparison)
4921 : {
4922 1478 : class expand_operand ops[5];
4923 1478 : rtx_insn *tmp = NULL;
4924 1478 : start_sequence ();
4925 1478 : tree t_op0 = gimple_assign_rhs1 (def_stmt);
4926 1478 : tree t_op1 = gimple_assign_rhs2 (def_stmt);
4927 1478 : rtx op0c = expand_normal (t_op0);
4928 1478 : rtx op1c = expand_normal (t_op1);
4929 1478 : machine_mode mode2 = GET_MODE (op0c);
4930 :
4931 1478 : int nops = masked_op ? 3 : (len_op ? 5 : 2);
4932 1478 : int offset = masked_op || len_op ? 1 : 0;
4933 1478 : create_input_operand (&ops[offset + 0], op0c, mode2);
4934 1478 : create_input_operand (&ops[offset + 1], op1c, mode2);
4935 1478 : if (masked_op)
4936 : {
4937 0 : rtx mask_op = expand_normal (masked_op);
4938 0 : auto mask_mode = GET_MODE (mask_op);
4939 0 : create_input_operand (&ops[0], mask_op, mask_mode);
4940 : }
4941 1478 : else if (len_op)
4942 : {
4943 0 : rtx len_rtx = expand_normal (len_op);
4944 0 : rtx len_bias_rtx = expand_normal (len_bias);
4945 0 : tree lhs = gimple_get_lhs (def_stmt);
4946 0 : auto mask_mode = TYPE_MODE (TREE_TYPE (lhs));
4947 : /* ??? We could use something like internal_fn's
4948 : add_mask_else_and_len_args here. Currently it
4949 : only supports a fixed, consecutive order of
4950 : mask and len, though. */
4951 0 : create_input_operand (&ops[0], CONSTM1_RTX (mask_mode),
4952 : mask_mode);
4953 0 : create_convert_operand_from
4954 0 : (&ops[3], len_rtx, TYPE_MODE (TREE_TYPE (len_op)),
4955 0 : TYPE_UNSIGNED (TREE_TYPE (len_op)));
4956 0 : create_input_operand (&ops[4], len_bias_rtx, QImode);
4957 : }
4958 :
4959 1478 : int unsignedp2 = TYPE_UNSIGNED (TREE_TYPE (t_op0));
4960 1478 : auto inner_code = gimple_assign_rhs_code (def_stmt);
4961 1478 : rtx test2 = NULL_RTX;
4962 :
4963 1478 : enum rtx_code comparison2 = get_rtx_code (inner_code, unsignedp2);
4964 1478 : if (unsignedp2)
4965 488 : comparison2 = unsigned_condition (comparison2);
4966 1478 : if (comparison == NE)
4967 1010 : optab = masked_op ? cond_vec_cbranch_any_optab
4968 505 : : len_op ? cond_len_vec_cbranch_any_optab
4969 : : vec_cbranch_any_optab;
4970 : else
4971 1946 : optab = masked_op ? cond_vec_cbranch_all_optab
4972 973 : : len_op ? cond_len_vec_cbranch_all_optab
4973 : : vec_cbranch_all_optab;
4974 :
4975 1478 : if ((icode = optab_handler (optab, mode2))
4976 : != CODE_FOR_nothing
4977 1478 : && maybe_legitimize_operands (icode, 1, nops, ops))
4978 : {
4979 0 : test2 = gen_rtx_fmt_ee (comparison2, VOIDmode,
4980 : ops[offset + 0].value,
4981 : ops[offset + 1].value);
4982 0 : if (insn_operand_matches (icode, 0, test2))
4983 : {
4984 0 : rtx mask
4985 0 : = (masked_op || len_op) ? ops[0].value : NULL_RTX;
4986 0 : rtx len = len_op ? ops[3].value : NULL_RTX;
4987 0 : rtx bias = len_op ? ops[4].value : NULL_RTX;
4988 0 : emit_cmp_and_jump_insn_1 (test2, mask, len, bias, mode2,
4989 : label, optab, prob, false);
4990 0 : tmp = get_insns ();
4991 : }
4992 : }
4993 :
4994 0 : end_sequence ();
4995 1478 : if (tmp)
4996 : {
4997 0 : emit_insn (tmp);
4998 0 : return;
4999 : }
5000 : }
5001 : }
5002 : }
5003 :
5004 : /* cbranch should only be used for VECTOR_BOOLEAN_TYPE_P values. */
5005 6347415 : direct_optab base_optab = cbranch_optab;
5006 6347415 : if (VECTOR_MODE_P (GET_MODE (op0)))
5007 : {
5008 : /* If cbranch is provided, use it. If we get here it means we have an
5009 : instruction in between what created the boolean value and the gcond
5010 : that is not a masking operation. This can happen for instance during
5011 : unrolling of early-break where we have an OR-reduction to reduce the
5012 : masks. In this case knowing we have a mask can let us generate better
5013 : code. If it's not there there then check the vector specific
5014 : optabs. */
5015 19195 : if (optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
5016 : {
5017 0 : if (comparison == NE)
5018 : base_optab = vec_cbranch_any_optab;
5019 : else
5020 0 : base_optab = vec_cbranch_all_optab;
5021 :
5022 0 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5023 : OPTAB_DIRECT, &test, &mode, base_optab);
5024 :
5025 0 : enum insn_code icode = optab_handler (base_optab, mode);
5026 :
5027 : /* If the new cbranch isn't supported, degrade back to old one. */
5028 0 : if (icode == CODE_FOR_nothing
5029 0 : || !test
5030 0 : || !insn_operand_matches (icode, 0, test))
5031 : base_optab = cbranch_optab;
5032 : }
5033 :
5034 19195 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5035 : OPTAB_LIB_WIDEN, &test, &mode, base_optab);
5036 : }
5037 :
5038 6347415 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, mode, label,
5039 : base_optab, prob, false);
5040 : }
5041 :
5042 : /* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown. */
5043 :
5044 : void
5045 148475 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
5046 : machine_mode mode, int unsignedp, rtx label,
5047 : profile_probability prob)
5048 : {
5049 148475 : emit_cmp_and_jump_insns (x, y, comparison, size, mode, unsignedp, NULL,
5050 : label, prob);
5051 148475 : }
5052 :
5053 :
5054 : /* Emit a library call comparison between floating point X and Y.
5055 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
5056 :
5057 : static void
5058 44182 : prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
5059 : rtx *ptest, machine_mode *pmode)
5060 : {
5061 44182 : enum rtx_code swapped = swap_condition (comparison);
5062 44182 : enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
5063 44182 : machine_mode orig_mode = GET_MODE (x);
5064 44182 : machine_mode mode;
5065 44182 : rtx true_rtx, false_rtx;
5066 44182 : rtx value, target, equiv;
5067 44182 : rtx_insn *insns;
5068 44182 : rtx libfunc = 0;
5069 44182 : bool reversed_p = false;
5070 44182 : scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
5071 :
5072 44182 : FOR_EACH_WIDER_MODE_FROM (mode, orig_mode)
5073 : {
5074 44182 : if (code_to_optab (comparison)
5075 44182 : && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
5076 : break;
5077 :
5078 0 : if (code_to_optab (swapped)
5079 0 : && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
5080 : {
5081 : std::swap (x, y);
5082 : comparison = swapped;
5083 : break;
5084 : }
5085 :
5086 0 : if (code_to_optab (reversed)
5087 0 : && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
5088 : {
5089 : comparison = reversed;
5090 : reversed_p = true;
5091 : break;
5092 : }
5093 : }
5094 :
5095 44182 : gcc_assert (mode != VOIDmode);
5096 :
5097 44182 : if (mode != orig_mode)
5098 : {
5099 0 : x = convert_to_mode (mode, x, 0);
5100 0 : y = convert_to_mode (mode, y, 0);
5101 : }
5102 :
5103 : /* Attach a REG_EQUAL note describing the semantics of the libcall to
5104 : the RTL. The allows the RTL optimizers to delete the libcall if the
5105 : condition can be determined at compile-time. */
5106 44182 : if (comparison == UNORDERED
5107 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5108 : {
5109 5610 : true_rtx = const_true_rtx;
5110 5610 : false_rtx = const0_rtx;
5111 : }
5112 : else
5113 : {
5114 38572 : switch (comparison)
5115 : {
5116 3354 : case EQ:
5117 3354 : true_rtx = const0_rtx;
5118 3354 : false_rtx = const_true_rtx;
5119 3354 : break;
5120 :
5121 23459 : case NE:
5122 23459 : true_rtx = const_true_rtx;
5123 23459 : false_rtx = const0_rtx;
5124 23459 : break;
5125 :
5126 2673 : case GT:
5127 2673 : true_rtx = const1_rtx;
5128 2673 : false_rtx = const0_rtx;
5129 2673 : break;
5130 :
5131 3085 : case GE:
5132 3085 : true_rtx = const0_rtx;
5133 3085 : false_rtx = constm1_rtx;
5134 3085 : break;
5135 :
5136 2431 : case LT:
5137 2431 : true_rtx = constm1_rtx;
5138 2431 : false_rtx = const0_rtx;
5139 2431 : break;
5140 :
5141 3570 : case LE:
5142 3570 : true_rtx = const0_rtx;
5143 3570 : false_rtx = const1_rtx;
5144 3570 : break;
5145 :
5146 0 : default:
5147 0 : gcc_unreachable ();
5148 : }
5149 : }
5150 :
5151 44182 : if (comparison == UNORDERED)
5152 : {
5153 5610 : rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
5154 5610 : equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
5155 5610 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5156 : temp, const_true_rtx, equiv);
5157 : }
5158 : else
5159 : {
5160 38572 : equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
5161 38572 : if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5162 38572 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5163 : equiv, true_rtx, false_rtx);
5164 : }
5165 :
5166 44182 : start_sequence ();
5167 44182 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5168 : cmp_mode, x, mode, y, mode);
5169 44182 : insns = end_sequence ();
5170 :
5171 44182 : target = gen_reg_rtx (cmp_mode);
5172 44182 : emit_libcall_block (insns, target, value, equiv);
5173 :
5174 44182 : if (comparison == UNORDERED
5175 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
5176 44182 : || reversed_p)
5177 11220 : *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
5178 : else
5179 38572 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
5180 :
5181 44182 : *pmode = cmp_mode;
5182 44182 : }
5183 :
5184 : /* Generate code to indirectly jump to a location given in the rtx LOC. */
5185 :
5186 : void
5187 1306 : emit_indirect_jump (rtx loc)
5188 : {
5189 1306 : if (!targetm.have_indirect_jump ())
5190 0 : sorry ("indirect jumps are not available on this target");
5191 : else
5192 : {
5193 1306 : class expand_operand ops[1];
5194 1306 : create_address_operand (&ops[0], loc);
5195 1306 : expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
5196 1306 : emit_barrier ();
5197 : }
5198 1306 : }
5199 :
5200 :
5201 : /* Emit a conditional move instruction if the machine supports one for that
5202 : condition and machine mode.
5203 :
5204 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5205 : the mode to use should they be constants. If it is VOIDmode, they cannot
5206 : both be constants.
5207 :
5208 : OP2 should be stored in TARGET if the comparison is true, otherwise OP3
5209 : should be stored there. MODE is the mode to use should they be constants.
5210 : If it is VOIDmode, they cannot both be constants.
5211 :
5212 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5213 : is not supported. */
5214 :
5215 : rtx
5216 443555 : emit_conditional_move (rtx target, struct rtx_comparison comp,
5217 : rtx op2, rtx op3,
5218 : machine_mode mode, int unsignedp)
5219 : {
5220 443555 : rtx comparison;
5221 443555 : rtx_insn *last;
5222 443555 : enum insn_code icode;
5223 443555 : enum rtx_code reversed;
5224 :
5225 : /* If the two source operands are identical, that's just a move. */
5226 :
5227 443555 : if (rtx_equal_p (op2, op3))
5228 : {
5229 24002 : if (!target)
5230 0 : target = gen_reg_rtx (mode);
5231 :
5232 24002 : emit_move_insn (target, op3);
5233 24002 : return target;
5234 : }
5235 :
5236 : /* If one operand is constant, make it the second one. Only do this
5237 : if the other operand is not constant as well. */
5238 :
5239 419553 : if (swap_commutative_operands_p (comp.op0, comp.op1))
5240 : {
5241 412 : std::swap (comp.op0, comp.op1);
5242 412 : comp.code = swap_condition (comp.code);
5243 : }
5244 :
5245 : /* get_condition will prefer to generate LT and GT even if the old
5246 : comparison was against zero, so undo that canonicalization here since
5247 : comparisons against zero are cheaper. */
5248 :
5249 419553 : if (comp.code == LT && comp.op1 == const1_rtx)
5250 2375 : comp.code = LE, comp.op1 = const0_rtx;
5251 417178 : else if (comp.code == GT && comp.op1 == constm1_rtx)
5252 7144 : comp.code = GE, comp.op1 = const0_rtx;
5253 :
5254 419553 : if (comp.mode == VOIDmode)
5255 335320 : comp.mode = GET_MODE (comp.op0);
5256 :
5257 419553 : enum rtx_code orig_code = comp.code;
5258 419553 : bool swapped = false;
5259 419553 : if (swap_commutative_operands_p (op2, op3)
5260 419553 : && ((reversed =
5261 127534 : reversed_comparison_code_parts (comp.code, comp.op0, comp.op1, NULL))
5262 : != UNKNOWN))
5263 : {
5264 : std::swap (op2, op3);
5265 : comp.code = reversed;
5266 : swapped = true;
5267 : }
5268 :
5269 419553 : if (mode == VOIDmode)
5270 0 : mode = GET_MODE (op2);
5271 :
5272 419553 : icode = direct_optab_handler (movcc_optab, mode);
5273 :
5274 419553 : if (icode == CODE_FOR_nothing)
5275 : return NULL_RTX;
5276 :
5277 412862 : if (!target)
5278 0 : target = gen_reg_rtx (mode);
5279 :
5280 412862 : for (int pass = 0; ; pass++)
5281 : {
5282 444584 : comp.code = unsignedp ? unsigned_condition (comp.code) : comp.code;
5283 889168 : comparison =
5284 444584 : simplify_gen_relational (comp.code, VOIDmode,
5285 : comp.mode, comp.op0, comp.op1);
5286 :
5287 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5288 : punt and let the caller figure out how best to deal with this
5289 : situation. */
5290 444584 : if (COMPARISON_P (comparison))
5291 : {
5292 443718 : saved_pending_stack_adjust save;
5293 443718 : save_pending_stack_adjust (&save);
5294 443718 : last = get_last_insn ();
5295 443718 : do_pending_stack_adjust ();
5296 443718 : machine_mode cmpmode = comp.mode;
5297 443718 : rtx orig_op0 = XEXP (comparison, 0);
5298 443718 : rtx orig_op1 = XEXP (comparison, 1);
5299 443718 : rtx op2p = op2;
5300 443718 : rtx op3p = op3;
5301 : /* If we are optimizing, force expensive constants into a register
5302 : but preserve an eventual equality with op2/op3. */
5303 1 : if (CONSTANT_P (orig_op0) && optimize
5304 1 : && cmpmode == mode
5305 1 : && (rtx_cost (orig_op0, mode, COMPARE, 0,
5306 1 : optimize_insn_for_speed_p ())
5307 : > COSTS_N_INSNS (1))
5308 443718 : && can_create_pseudo_p ())
5309 : {
5310 0 : if (rtx_equal_p (orig_op0, op2))
5311 0 : op2p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5312 0 : else if (rtx_equal_p (orig_op0, op3))
5313 0 : op3p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5314 : }
5315 271408 : if (CONSTANT_P (orig_op1) && optimize
5316 250473 : && cmpmode == mode
5317 76964 : && (rtx_cost (orig_op1, mode, COMPARE, 0,
5318 76964 : optimize_insn_for_speed_p ())
5319 : > COSTS_N_INSNS (1))
5320 443877 : && can_create_pseudo_p ())
5321 : {
5322 159 : if (rtx_equal_p (orig_op1, op2))
5323 102 : op2p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5324 57 : else if (rtx_equal_p (orig_op1, op3))
5325 33 : op3p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5326 : }
5327 443718 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5328 443718 : GET_CODE (comparison), NULL_RTX, unsignedp,
5329 : OPTAB_WIDEN, &comparison, &cmpmode, cbranch_optab);
5330 443718 : if (comparison)
5331 : {
5332 441554 : rtx res = emit_conditional_move_1 (target, comparison,
5333 : op2p, op3p, mode);
5334 441554 : if (res != NULL_RTX)
5335 379008 : return res;
5336 : }
5337 64710 : delete_insns_since (last);
5338 64710 : restore_pending_stack_adjust (&save);
5339 : }
5340 :
5341 65576 : if (pass == 1)
5342 : return NULL_RTX;
5343 :
5344 : /* If the preferred op2/op3 order is not usable, retry with other
5345 : operand order, perhaps it will expand successfully. */
5346 33898 : if (swapped)
5347 : comp.code = orig_code;
5348 66268 : else if ((reversed =
5349 33134 : reversed_comparison_code_parts (orig_code, comp.op0, comp.op1,
5350 : NULL))
5351 : != UNKNOWN)
5352 : comp.code = reversed;
5353 : else
5354 : return NULL_RTX;
5355 : std::swap (op2, op3);
5356 : }
5357 : }
5358 :
5359 : /* Helper function that, in addition to COMPARISON, also tries
5360 : the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
5361 : to when we pass the specific constituents of a comparison, no
5362 : additional insns are emitted for it. It might still be necessary
5363 : to emit more than one insn for the final conditional move, though. */
5364 :
5365 : rtx
5366 87566 : emit_conditional_move (rtx target, rtx comparison, rtx rev_comparison,
5367 : rtx op2, rtx op3, machine_mode mode)
5368 : {
5369 87566 : rtx res = emit_conditional_move_1 (target, comparison, op2, op3, mode);
5370 :
5371 87566 : if (res != NULL_RTX)
5372 : return res;
5373 :
5374 4975 : return emit_conditional_move_1 (target, rev_comparison, op3, op2, mode);
5375 : }
5376 :
5377 : /* Helper for emitting a conditional move. */
5378 :
5379 : static rtx
5380 534095 : emit_conditional_move_1 (rtx target, rtx comparison,
5381 : rtx op2, rtx op3, machine_mode mode)
5382 : {
5383 534095 : enum insn_code icode;
5384 :
5385 534095 : if (comparison == NULL_RTX || !COMPARISON_P (comparison))
5386 : return NULL_RTX;
5387 :
5388 : /* If the two source operands are identical, that's just a move.
5389 : As the comparison comes in non-canonicalized, we must make
5390 : sure not to discard any possible side effects. If there are
5391 : side effects, just let the target handle it. */
5392 534095 : if (!side_effects_p (comparison) && rtx_equal_p (op2, op3))
5393 : {
5394 0 : if (!target)
5395 0 : target = gen_reg_rtx (mode);
5396 :
5397 0 : emit_move_insn (target, op3);
5398 0 : return target;
5399 : }
5400 :
5401 534095 : if (mode == VOIDmode)
5402 0 : mode = GET_MODE (op2);
5403 :
5404 534095 : icode = direct_optab_handler (movcc_optab, mode);
5405 :
5406 534095 : if (icode == CODE_FOR_nothing)
5407 : return NULL_RTX;
5408 :
5409 534095 : if (!target)
5410 0 : target = gen_reg_rtx (mode);
5411 :
5412 534095 : class expand_operand ops[4];
5413 :
5414 534095 : create_output_operand (&ops[0], target, mode);
5415 534095 : create_fixed_operand (&ops[1], comparison);
5416 534095 : create_input_operand (&ops[2], op2, mode);
5417 534095 : create_input_operand (&ops[3], op3, mode);
5418 :
5419 534095 : if (maybe_expand_insn (icode, 4, ops))
5420 : {
5421 461599 : if (ops[0].value != target)
5422 55 : convert_move (target, ops[0].value, false);
5423 461599 : return target;
5424 : }
5425 :
5426 : return NULL_RTX;
5427 : }
5428 :
5429 :
5430 : /* Emit a conditional negate or bitwise complement using the
5431 : negcc or notcc optabs if available. Return NULL_RTX if such operations
5432 : are not available. Otherwise return the RTX holding the result.
5433 : TARGET is the desired destination of the result. COMP is the comparison
5434 : on which to negate. If COND is true move into TARGET the negation
5435 : or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5436 : CODE is either NEG or NOT. MODE is the machine mode in which the
5437 : operation is performed. */
5438 :
5439 : rtx
5440 1246 : emit_conditional_neg_or_complement (rtx target, rtx_code code,
5441 : machine_mode mode, rtx cond, rtx op1,
5442 : rtx op2)
5443 : {
5444 1246 : optab op = unknown_optab;
5445 1246 : if (code == NEG)
5446 : op = negcc_optab;
5447 69 : else if (code == NOT)
5448 : op = notcc_optab;
5449 : else
5450 0 : gcc_unreachable ();
5451 :
5452 1246 : insn_code icode = direct_optab_handler (op, mode);
5453 :
5454 1246 : if (icode == CODE_FOR_nothing)
5455 : return NULL_RTX;
5456 :
5457 0 : if (!target)
5458 0 : target = gen_reg_rtx (mode);
5459 :
5460 0 : rtx_insn *last = get_last_insn ();
5461 0 : class expand_operand ops[4];
5462 :
5463 0 : create_output_operand (&ops[0], target, mode);
5464 0 : create_fixed_operand (&ops[1], cond);
5465 0 : create_input_operand (&ops[2], op1, mode);
5466 0 : create_input_operand (&ops[3], op2, mode);
5467 :
5468 0 : if (maybe_expand_insn (icode, 4, ops))
5469 : {
5470 0 : if (ops[0].value != target)
5471 0 : convert_move (target, ops[0].value, false);
5472 :
5473 0 : return target;
5474 : }
5475 0 : delete_insns_since (last);
5476 0 : return NULL_RTX;
5477 : }
5478 :
5479 : /* Emit a conditional addition instruction if the machine supports one for that
5480 : condition and machine mode.
5481 :
5482 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5483 : the mode to use should they be constants. If it is VOIDmode, they cannot
5484 : both be constants.
5485 :
5486 : OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5487 : should be stored there. MODE is the mode to use should they be constants.
5488 : If it is VOIDmode, they cannot both be constants.
5489 :
5490 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5491 : is not supported. */
5492 :
5493 : rtx
5494 15160 : emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
5495 : machine_mode cmode, rtx op2, rtx op3,
5496 : machine_mode mode, int unsignedp)
5497 : {
5498 15160 : rtx comparison;
5499 15160 : rtx_insn *last;
5500 15160 : enum insn_code icode;
5501 :
5502 : /* If one operand is constant, make it the second one. Only do this
5503 : if the other operand is not constant as well. */
5504 :
5505 15160 : if (swap_commutative_operands_p (op0, op1))
5506 : {
5507 0 : std::swap (op0, op1);
5508 0 : code = swap_condition (code);
5509 : }
5510 :
5511 : /* get_condition will prefer to generate LT and GT even if the old
5512 : comparison was against zero, so undo that canonicalization here since
5513 : comparisons against zero are cheaper. */
5514 15160 : if (code == LT && op1 == const1_rtx)
5515 36 : code = LE, op1 = const0_rtx;
5516 15124 : else if (code == GT && op1 == constm1_rtx)
5517 89 : code = GE, op1 = const0_rtx;
5518 :
5519 15160 : if (cmode == VOIDmode)
5520 15160 : cmode = GET_MODE (op0);
5521 :
5522 15160 : if (mode == VOIDmode)
5523 0 : mode = GET_MODE (op2);
5524 :
5525 15160 : icode = optab_handler (addcc_optab, mode);
5526 :
5527 15160 : if (icode == CODE_FOR_nothing)
5528 : return 0;
5529 :
5530 14593 : if (!target)
5531 0 : target = gen_reg_rtx (mode);
5532 :
5533 14593 : code = unsignedp ? unsigned_condition (code) : code;
5534 14593 : comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
5535 :
5536 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5537 : return NULL and let the caller figure out how best to deal with this
5538 : situation. */
5539 14593 : if (!COMPARISON_P (comparison))
5540 : return NULL_RTX;
5541 :
5542 14593 : do_pending_stack_adjust ();
5543 14593 : last = get_last_insn ();
5544 14593 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5545 14593 : GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
5546 : &comparison, &cmode, cbranch_optab);
5547 14593 : if (comparison)
5548 : {
5549 14593 : class expand_operand ops[4];
5550 :
5551 14593 : create_output_operand (&ops[0], target, mode);
5552 14593 : create_fixed_operand (&ops[1], comparison);
5553 14593 : create_input_operand (&ops[2], op2, mode);
5554 14593 : create_input_operand (&ops[3], op3, mode);
5555 14593 : if (maybe_expand_insn (icode, 4, ops))
5556 : {
5557 1274 : if (ops[0].value != target)
5558 0 : convert_move (target, ops[0].value, false);
5559 1274 : return target;
5560 : }
5561 : }
5562 13319 : delete_insns_since (last);
5563 13319 : return NULL_RTX;
5564 : }
5565 :
5566 : /* These functions attempt to generate an insn body, rather than
5567 : emitting the insn, but if the gen function already emits them, we
5568 : make no attempt to turn them back into naked patterns. */
5569 :
5570 : /* Generate and return an insn body to add Y to X. */
5571 :
5572 : rtx_insn *
5573 247 : gen_add2_insn (rtx x, rtx y)
5574 : {
5575 247 : enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
5576 :
5577 247 : gcc_assert (insn_operand_matches (icode, 0, x));
5578 247 : gcc_assert (insn_operand_matches (icode, 1, x));
5579 247 : gcc_assert (insn_operand_matches (icode, 2, y));
5580 :
5581 247 : return GEN_FCN (icode) (x, x, y);
5582 : }
5583 :
5584 : /* Generate and return an insn body to add r1 and c,
5585 : storing the result in r0. */
5586 :
5587 : rtx_insn *
5588 0 : gen_add3_insn (rtx r0, rtx r1, rtx c)
5589 : {
5590 0 : enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
5591 :
5592 0 : if (icode == CODE_FOR_nothing
5593 0 : || !insn_operand_matches (icode, 0, r0)
5594 0 : || !insn_operand_matches (icode, 1, r1)
5595 0 : || !insn_operand_matches (icode, 2, c))
5596 0 : return NULL;
5597 :
5598 0 : return GEN_FCN (icode) (r0, r1, c);
5599 : }
5600 :
5601 : bool
5602 3580 : have_add2_insn (rtx x, rtx y)
5603 : {
5604 3580 : enum insn_code icode;
5605 :
5606 3580 : gcc_assert (GET_MODE (x) != VOIDmode);
5607 :
5608 3580 : icode = optab_handler (add_optab, GET_MODE (x));
5609 :
5610 3580 : if (icode == CODE_FOR_nothing)
5611 : return false;
5612 :
5613 3580 : if (!insn_operand_matches (icode, 0, x)
5614 3580 : || !insn_operand_matches (icode, 1, x)
5615 7160 : || !insn_operand_matches (icode, 2, y))
5616 170 : return false;
5617 :
5618 : return true;
5619 : }
5620 :
5621 : /* Generate and return an insn body to add Y to X. */
5622 :
5623 : rtx_insn *
5624 0 : gen_addptr3_insn (rtx x, rtx y, rtx z)
5625 : {
5626 0 : enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
5627 :
5628 0 : gcc_assert (insn_operand_matches (icode, 0, x));
5629 0 : gcc_assert (insn_operand_matches (icode, 1, y));
5630 0 : gcc_assert (insn_operand_matches (icode, 2, z));
5631 :
5632 0 : return GEN_FCN (icode) (x, y, z);
5633 : }
5634 :
5635 : /* Return true if the target implements an addptr pattern and X, Y,
5636 : and Z are valid for the pattern predicates. */
5637 :
5638 : bool
5639 627195 : have_addptr3_insn (rtx x, rtx y, rtx z)
5640 : {
5641 627195 : enum insn_code icode;
5642 :
5643 627195 : gcc_assert (GET_MODE (x) != VOIDmode);
5644 :
5645 627195 : icode = optab_handler (addptr3_optab, GET_MODE (x));
5646 :
5647 627195 : if (icode == CODE_FOR_nothing)
5648 : return false;
5649 :
5650 0 : if (!insn_operand_matches (icode, 0, x)
5651 0 : || !insn_operand_matches (icode, 1, y)
5652 0 : || !insn_operand_matches (icode, 2, z))
5653 0 : return false;
5654 :
5655 : return true;
5656 : }
5657 :
5658 : /* Generate and return an insn body to subtract Y from X. */
5659 :
5660 : rtx_insn *
5661 11 : gen_sub2_insn (rtx x, rtx y)
5662 : {
5663 11 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
5664 :
5665 11 : gcc_assert (insn_operand_matches (icode, 0, x));
5666 11 : gcc_assert (insn_operand_matches (icode, 1, x));
5667 11 : gcc_assert (insn_operand_matches (icode, 2, y));
5668 :
5669 11 : return GEN_FCN (icode) (x, x, y);
5670 : }
5671 :
5672 : /* Generate and return an insn body to subtract r1 and c,
5673 : storing the result in r0. */
5674 :
5675 : rtx_insn *
5676 1470 : gen_sub3_insn (rtx r0, rtx r1, rtx c)
5677 : {
5678 1470 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
5679 :
5680 1470 : if (icode == CODE_FOR_nothing
5681 1470 : || !insn_operand_matches (icode, 0, r0)
5682 1470 : || !insn_operand_matches (icode, 1, r1)
5683 2940 : || !insn_operand_matches (icode, 2, c))
5684 0 : return NULL;
5685 :
5686 1470 : return GEN_FCN (icode) (r0, r1, c);
5687 : }
5688 :
5689 : bool
5690 0 : have_sub2_insn (rtx x, rtx y)
5691 : {
5692 0 : enum insn_code icode;
5693 :
5694 0 : gcc_assert (GET_MODE (x) != VOIDmode);
5695 :
5696 0 : icode = optab_handler (sub_optab, GET_MODE (x));
5697 :
5698 0 : if (icode == CODE_FOR_nothing)
5699 : return false;
5700 :
5701 0 : if (!insn_operand_matches (icode, 0, x)
5702 0 : || !insn_operand_matches (icode, 1, x)
5703 0 : || !insn_operand_matches (icode, 2, y))
5704 0 : return false;
5705 :
5706 : return true;
5707 : }
5708 :
5709 : /* Generate the body of an insn to extend Y (with mode MFROM)
5710 : into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5711 :
5712 : rtx_insn *
5713 30455 : gen_extend_insn (rtx x, rtx y, machine_mode mto,
5714 : machine_mode mfrom, int unsignedp)
5715 : {
5716 30455 : enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
5717 30455 : return GEN_FCN (icode) (x, y);
5718 : }
5719 :
5720 : /* Generate code to convert FROM to floating point
5721 : and store in TO. FROM must be fixed point and not VOIDmode.
5722 : UNSIGNEDP nonzero means regard FROM as unsigned.
5723 : Normally this is done by correcting the final value
5724 : if it is negative. */
5725 :
5726 : void
5727 143838 : expand_float (rtx to, rtx from, int unsignedp)
5728 : {
5729 143838 : enum insn_code icode;
5730 143838 : rtx target = to;
5731 143838 : scalar_mode from_mode, to_mode;
5732 143838 : machine_mode fmode, imode;
5733 143838 : bool can_do_signed = false;
5734 :
5735 : /* Crash now, because we won't be able to decide which mode to use. */
5736 143838 : gcc_assert (GET_MODE (from) != VOIDmode);
5737 :
5738 : /* Look for an insn to do the conversion. Do it in the specified
5739 : modes if possible; otherwise convert either input, output or both to
5740 : wider mode. If the integer mode is wider than the mode of FROM,
5741 : we can do the conversion signed even if the input is unsigned. */
5742 :
5743 165862 : FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
5744 259643 : FOR_EACH_MODE_FROM (imode, GET_MODE (from))
5745 : {
5746 237619 : int doing_unsigned = unsignedp;
5747 :
5748 267706 : if (fmode != GET_MODE (to)
5749 237619 : && (significand_size (fmode)
5750 94416 : < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
5751 30087 : continue;
5752 :
5753 207532 : icode = can_float_p (fmode, imode, unsignedp);
5754 207532 : if (icode == CODE_FOR_nothing && unsignedp)
5755 : {
5756 35650 : enum insn_code scode = can_float_p (fmode, imode, 0);
5757 35650 : if (scode != CODE_FOR_nothing)
5758 6322 : can_do_signed = true;
5759 35650 : if (imode != GET_MODE (from))
5760 24581 : icode = scode, doing_unsigned = 0;
5761 : }
5762 :
5763 207532 : if (icode != CODE_FOR_nothing)
5764 : {
5765 135259 : if (imode != GET_MODE (from))
5766 10040 : from = convert_to_mode (imode, from, unsignedp);
5767 :
5768 135259 : if (fmode != GET_MODE (to))
5769 1589 : target = gen_reg_rtx (fmode);
5770 :
5771 265819 : emit_unop_insn (icode, target, from,
5772 : doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5773 :
5774 135259 : if (target != to)
5775 1589 : convert_move (to, target, 0);
5776 135259 : return;
5777 : }
5778 : }
5779 :
5780 : /* Unsigned integer, and no way to convert directly. Convert as signed,
5781 : then unconditionally adjust the result. */
5782 8579 : if (unsignedp
5783 8579 : && can_do_signed
5784 1098 : && is_a <scalar_mode> (GET_MODE (to), &to_mode)
5785 9677 : && is_a <scalar_mode> (GET_MODE (from), &from_mode))
5786 : {
5787 1098 : opt_scalar_mode fmode_iter;
5788 1098 : rtx_code_label *label = gen_label_rtx ();
5789 1098 : rtx temp;
5790 1098 : REAL_VALUE_TYPE offset;
5791 :
5792 : /* Look for a usable floating mode FMODE wider than the source and at
5793 : least as wide as the target. Using FMODE will avoid rounding woes
5794 : with unsigned values greater than the signed maximum value. */
5795 :
5796 2105 : FOR_EACH_MODE_FROM (fmode_iter, to_mode)
5797 : {
5798 2105 : scalar_mode fmode = fmode_iter.require ();
5799 3112 : if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
5800 2105 : && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
5801 : break;
5802 : }
5803 :
5804 1098 : if (!fmode_iter.exists (&fmode))
5805 : {
5806 : /* There is no such mode. Pretend the target is wide enough. */
5807 0 : fmode = to_mode;
5808 :
5809 : /* Avoid double-rounding when TO is narrower than FROM. */
5810 0 : if ((significand_size (fmode) + 1)
5811 0 : < GET_MODE_PRECISION (from_mode))
5812 : {
5813 0 : rtx temp1;
5814 0 : rtx_code_label *neglabel = gen_label_rtx ();
5815 :
5816 : /* Don't use TARGET if it isn't a register, is a hard register,
5817 : or is the wrong mode. */
5818 0 : if (!REG_P (target)
5819 0 : || REGNO (target) < FIRST_PSEUDO_REGISTER
5820 0 : || GET_MODE (target) != fmode)
5821 0 : target = gen_reg_rtx (fmode);
5822 :
5823 0 : imode = from_mode;
5824 0 : do_pending_stack_adjust ();
5825 :
5826 : /* Test whether the sign bit is set. */
5827 0 : emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
5828 : 0, neglabel);
5829 :
5830 : /* The sign bit is not set. Convert as signed. */
5831 0 : expand_float (target, from, 0);
5832 0 : emit_jump_insn (targetm.gen_jump (label));
5833 0 : emit_barrier ();
5834 :
5835 : /* The sign bit is set.
5836 : Convert to a usable (positive signed) value by shifting right
5837 : one bit, while remembering if a nonzero bit was shifted
5838 : out; i.e., compute (from & 1) | (from >> 1). */
5839 :
5840 0 : emit_label (neglabel);
5841 0 : temp = expand_binop (imode, and_optab, from, const1_rtx,
5842 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
5843 0 : temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
5844 0 : temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
5845 : OPTAB_LIB_WIDEN);
5846 0 : expand_float (target, temp, 0);
5847 :
5848 : /* Multiply by 2 to undo the shift above. */
5849 0 : temp = expand_binop (fmode, add_optab, target, target,
5850 : target, 0, OPTAB_LIB_WIDEN);
5851 0 : if (temp != target)
5852 0 : emit_move_insn (target, temp);
5853 :
5854 0 : do_pending_stack_adjust ();
5855 0 : emit_label (label);
5856 0 : goto done;
5857 : }
5858 : }
5859 :
5860 : /* If we are about to do some arithmetic to correct for an
5861 : unsigned operand, do it in a pseudo-register. */
5862 :
5863 1098 : if (to_mode != fmode
5864 1098 : || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
5865 399 : target = gen_reg_rtx (fmode);
5866 :
5867 : /* Convert as signed integer to floating. */
5868 1098 : expand_float (target, from, 0);
5869 :
5870 : /* If FROM is negative (and therefore TO is negative),
5871 : correct its value by 2**bitwidth. */
5872 :
5873 1098 : do_pending_stack_adjust ();
5874 1098 : emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
5875 : 0, label);
5876 :
5877 :
5878 1098 : real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
5879 1098 : temp = expand_binop (fmode, add_optab, target,
5880 : const_double_from_real_value (offset, fmode),
5881 : target, 0, OPTAB_LIB_WIDEN);
5882 1098 : if (temp != target)
5883 0 : emit_move_insn (target, temp);
5884 :
5885 1098 : do_pending_stack_adjust ();
5886 1098 : emit_label (label);
5887 1098 : goto done;
5888 : }
5889 :
5890 : /* No hardware instruction available; call a library routine. */
5891 7481 : {
5892 7481 : rtx libfunc;
5893 7481 : rtx_insn *insns;
5894 7481 : rtx value;
5895 7481 : convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
5896 :
5897 7481 : if (is_narrower_int_mode (GET_MODE (from), SImode))
5898 972 : from = convert_to_mode (SImode, from, unsignedp);
5899 :
5900 7481 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5901 7481 : gcc_assert (libfunc);
5902 :
5903 7481 : start_sequence ();
5904 :
5905 14962 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5906 7481 : GET_MODE (to), from, GET_MODE (from));
5907 7481 : insns = end_sequence ();
5908 :
5909 7481 : emit_libcall_block (insns, target, value,
5910 7481 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
5911 : GET_MODE (to), from));
5912 : }
5913 :
5914 8579 : done:
5915 :
5916 : /* Copy result to requested destination
5917 : if we have been computing in a temp location. */
5918 :
5919 8579 : if (target != to)
5920 : {
5921 399 : if (GET_MODE (target) == GET_MODE (to))
5922 4 : emit_move_insn (to, target);
5923 : else
5924 395 : convert_move (to, target, 0);
5925 : }
5926 : }
5927 :
5928 : /* Generate code to convert FROM to fixed point and store in TO. FROM
5929 : must be floating point. */
5930 :
5931 : void
5932 58913 : expand_fix (rtx to, rtx from, int unsignedp)
5933 : {
5934 58913 : enum insn_code icode;
5935 58913 : rtx target = to;
5936 58913 : machine_mode fmode, imode;
5937 58913 : opt_scalar_mode fmode_iter;
5938 58913 : bool must_trunc = false;
5939 :
5940 : /* We first try to find a pair of modes, one real and one integer, at
5941 : least as wide as FROM and TO, respectively, in which we can open-code
5942 : this conversion. If the integer mode is wider than the mode of TO,
5943 : we can do the conversion either signed or unsigned. */
5944 :
5945 89069 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5946 213129 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5947 : {
5948 182973 : int doing_unsigned = unsignedp;
5949 :
5950 182973 : icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
5951 182973 : if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
5952 61517 : icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
5953 :
5954 140227 : if (icode != CODE_FOR_nothing)
5955 : {
5956 45956 : rtx_insn *last = get_last_insn ();
5957 45956 : rtx from1 = from;
5958 45956 : if (fmode != GET_MODE (from))
5959 : {
5960 1094 : if (REAL_MODE_FORMAT (GET_MODE (from))
5961 : == &arm_bfloat_half_format
5962 1094 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
5963 : /* The BF -> SF conversions can be just a shift, doesn't
5964 : need to handle sNANs. */
5965 : {
5966 38 : int save_flag_finite_math_only = flag_finite_math_only;
5967 38 : flag_finite_math_only = true;
5968 38 : from1 = convert_to_mode (fmode, from, 0);
5969 38 : flag_finite_math_only = save_flag_finite_math_only;
5970 : }
5971 : else
5972 1056 : from1 = convert_to_mode (fmode, from, 0);
5973 : }
5974 :
5975 45956 : if (must_trunc)
5976 : {
5977 0 : rtx temp = gen_reg_rtx (GET_MODE (from1));
5978 0 : from1 = expand_unop (GET_MODE (from1), ftrunc_optab, from1,
5979 : temp, 0);
5980 : }
5981 :
5982 45956 : if (imode != GET_MODE (to))
5983 10803 : target = gen_reg_rtx (imode);
5984 :
5985 87646 : if (maybe_emit_unop_insn (icode, target, from1,
5986 : doing_unsigned ? UNSIGNED_FIX : FIX))
5987 : {
5988 45956 : if (target != to)
5989 10803 : convert_move (to, target, unsignedp);
5990 49952 : return;
5991 : }
5992 0 : delete_insns_since (last);
5993 : }
5994 : }
5995 :
5996 : /* For an unsigned conversion, there is one more way to do it.
5997 : If we have a signed conversion, we generate code that compares
5998 : the real value to the largest representable positive number. If if
5999 : is smaller, the conversion is done normally. Otherwise, subtract
6000 : one plus the highest signed number, convert, and add it back.
6001 :
6002 : We only need to check all real modes, since we know we didn't find
6003 : anything with a wider integer mode.
6004 :
6005 : This code used to extend FP value into mode wider than the destination.
6006 : This is needed for decimal float modes which cannot accurately
6007 : represent one plus the highest signed number of the same size, but
6008 : not for binary modes. Consider, for instance conversion from SFmode
6009 : into DImode.
6010 :
6011 : The hot path through the code is dealing with inputs smaller than 2^63
6012 : and doing just the conversion, so there is no bits to lose.
6013 :
6014 : In the other path we know the value is positive in the range 2^63..2^64-1
6015 : inclusive. (as for other input overflow happens and result is undefined)
6016 : So we know that the most important bit set in mantissa corresponds to
6017 : 2^63. The subtraction of 2^63 should not generate any rounding as it
6018 : simply clears out that bit. The rest is trivial. */
6019 :
6020 12957 : scalar_int_mode to_mode;
6021 12957 : if (unsignedp
6022 7353 : && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
6023 20310 : && HWI_COMPUTABLE_MODE_P (to_mode))
6024 8964 : FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
6025 : {
6026 6774 : scalar_mode fmode = fmode_iter.require ();
6027 6774 : if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
6028 : 0, &must_trunc)
6029 6774 : && (!DECIMAL_FLOAT_MODE_P (fmode)
6030 0 : || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
6031 : {
6032 3992 : int bitsize;
6033 3992 : REAL_VALUE_TYPE offset;
6034 3992 : rtx limit;
6035 3992 : rtx_code_label *lab1, *lab2;
6036 3992 : rtx_insn *insn;
6037 :
6038 3992 : bitsize = GET_MODE_PRECISION (to_mode);
6039 3992 : real_2expN (&offset, bitsize - 1, fmode);
6040 3992 : limit = const_double_from_real_value (offset, fmode);
6041 3992 : lab1 = gen_label_rtx ();
6042 3992 : lab2 = gen_label_rtx ();
6043 :
6044 3992 : if (fmode != GET_MODE (from))
6045 : {
6046 184 : if (REAL_MODE_FORMAT (GET_MODE (from))
6047 : == &arm_bfloat_half_format
6048 184 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
6049 : /* The BF -> SF conversions can be just a shift, doesn't
6050 : need to handle sNANs. */
6051 : {
6052 4 : int save_flag_finite_math_only = flag_finite_math_only;
6053 4 : flag_finite_math_only = true;
6054 4 : from = convert_to_mode (fmode, from, 0);
6055 4 : flag_finite_math_only = save_flag_finite_math_only;
6056 : }
6057 : else
6058 180 : from = convert_to_mode (fmode, from, 0);
6059 : }
6060 :
6061 : /* See if we need to do the subtraction. */
6062 3992 : do_pending_stack_adjust ();
6063 3992 : emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
6064 3992 : GET_MODE (from), 0, lab1);
6065 :
6066 : /* If not, do the signed "fix" and branch around fixup code. */
6067 3992 : expand_fix (to, from, 0);
6068 3992 : emit_jump_insn (targetm.gen_jump (lab2));
6069 3992 : emit_barrier ();
6070 :
6071 : /* Otherwise, subtract 2**(N-1), convert to signed number,
6072 : then add 2**(N-1). Do the addition using XOR since this
6073 : will often generate better code. */
6074 3992 : emit_label (lab1);
6075 3992 : target = expand_binop (GET_MODE (from), sub_optab, from, limit,
6076 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
6077 3992 : expand_fix (to, target, 0);
6078 3992 : target = expand_binop (to_mode, xor_optab, to,
6079 : gen_int_mode
6080 3992 : (HOST_WIDE_INT_1 << (bitsize - 1),
6081 : to_mode),
6082 : to, 1, OPTAB_LIB_WIDEN);
6083 :
6084 3992 : if (target != to)
6085 0 : emit_move_insn (to, target);
6086 :
6087 3992 : emit_label (lab2);
6088 :
6089 3992 : if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
6090 : {
6091 : /* Make a place for a REG_NOTE and add it. */
6092 3992 : insn = emit_move_insn (to, to);
6093 3992 : set_dst_reg_note (insn, REG_EQUAL,
6094 : gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
6095 : copy_rtx (from)),
6096 : to);
6097 : }
6098 :
6099 3992 : return;
6100 : }
6101 : }
6102 :
6103 : #ifdef HAVE_SFmode
6104 8965 : if (REAL_MODE_FORMAT (GET_MODE (from)) == &arm_bfloat_half_format
6105 8965 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
6106 : /* We don't have BF -> TI library functions, use BF -> SF -> TI
6107 : instead but the BF -> SF conversion can be just a shift, doesn't
6108 : need to handle sNANs. */
6109 : {
6110 4 : int save_flag_finite_math_only = flag_finite_math_only;
6111 4 : flag_finite_math_only = true;
6112 4 : from = convert_to_mode (SFmode, from, 0);
6113 4 : flag_finite_math_only = save_flag_finite_math_only;
6114 4 : expand_fix (to, from, unsignedp);
6115 4 : return;
6116 : }
6117 : #endif
6118 :
6119 : /* We can't do it with an insn, so use a library call. But first ensure
6120 : that the mode of TO is at least as wide as SImode, since those are the
6121 : only library calls we know about. */
6122 :
6123 8961 : if (is_narrower_int_mode (GET_MODE (to), SImode))
6124 : {
6125 1953 : target = gen_reg_rtx (SImode);
6126 :
6127 1953 : expand_fix (target, from, unsignedp);
6128 : }
6129 : else
6130 : {
6131 7008 : rtx_insn *insns;
6132 7008 : rtx value;
6133 7008 : rtx libfunc;
6134 :
6135 7008 : convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
6136 7008 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
6137 7008 : gcc_assert (libfunc);
6138 :
6139 7008 : start_sequence ();
6140 :
6141 14016 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
6142 7008 : GET_MODE (to), from, GET_MODE (from));
6143 7008 : insns = end_sequence ();
6144 :
6145 7008 : emit_libcall_block (insns, target, value,
6146 7008 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
6147 : GET_MODE (to), from));
6148 : }
6149 :
6150 8961 : if (target != to)
6151 : {
6152 1953 : if (GET_MODE (to) == GET_MODE (target))
6153 0 : emit_move_insn (to, target);
6154 : else
6155 1953 : convert_move (to, target, 0);
6156 : }
6157 : }
6158 :
6159 :
6160 : /* Promote integer arguments for a libcall if necessary.
6161 : emit_library_call_value cannot do the promotion because it does not
6162 : know if it should do a signed or unsigned promotion. This is because
6163 : there are no tree types defined for libcalls. */
6164 :
6165 : static rtx
6166 0 : prepare_libcall_arg (rtx arg, int uintp)
6167 : {
6168 0 : scalar_int_mode mode;
6169 0 : machine_mode arg_mode;
6170 0 : if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
6171 : {
6172 : /* If we need to promote the integer function argument we need to do
6173 : it here instead of inside emit_library_call_value because in
6174 : emit_library_call_value we don't know if we should do a signed or
6175 : unsigned promotion. */
6176 :
6177 0 : int unsigned_p = 0;
6178 0 : arg_mode = promote_function_mode (NULL_TREE, mode,
6179 : &unsigned_p, NULL_TREE, 0);
6180 0 : if (arg_mode != mode)
6181 0 : return convert_to_mode (arg_mode, arg, uintp);
6182 : }
6183 : return arg;
6184 : }
6185 :
6186 : /* Generate code to convert FROM or TO a fixed-point.
6187 : If UINTP is true, either TO or FROM is an unsigned integer.
6188 : If SATP is true, we need to saturate the result. */
6189 :
6190 : void
6191 0 : expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
6192 : {
6193 0 : machine_mode to_mode = GET_MODE (to);
6194 0 : machine_mode from_mode = GET_MODE (from);
6195 0 : convert_optab tab;
6196 0 : enum rtx_code this_code;
6197 0 : enum insn_code code;
6198 0 : rtx_insn *insns;
6199 0 : rtx value;
6200 0 : rtx libfunc;
6201 :
6202 0 : if (to_mode == from_mode)
6203 : {
6204 0 : emit_move_insn (to, from);
6205 0 : return;
6206 : }
6207 :
6208 0 : if (uintp)
6209 : {
6210 0 : tab = satp ? satfractuns_optab : fractuns_optab;
6211 0 : this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
6212 : }
6213 : else
6214 : {
6215 0 : tab = satp ? satfract_optab : fract_optab;
6216 0 : this_code = satp ? SAT_FRACT : FRACT_CONVERT;
6217 : }
6218 0 : code = convert_optab_handler (tab, to_mode, from_mode);
6219 0 : if (code != CODE_FOR_nothing)
6220 : {
6221 0 : emit_unop_insn (code, to, from, this_code);
6222 0 : return;
6223 : }
6224 :
6225 0 : libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
6226 0 : gcc_assert (libfunc);
6227 :
6228 0 : from = prepare_libcall_arg (from, uintp);
6229 0 : from_mode = GET_MODE (from);
6230 :
6231 0 : start_sequence ();
6232 0 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
6233 : from, from_mode);
6234 0 : insns = end_sequence ();
6235 :
6236 0 : emit_libcall_block (insns, to, value,
6237 : gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
6238 : }
6239 :
6240 : /* Generate code to convert FROM to fixed point and store in TO. FROM
6241 : must be floating point, TO must be signed. Use the conversion optab
6242 : TAB to do the conversion. */
6243 :
6244 : bool
6245 496 : expand_sfix_optab (rtx to, rtx from, convert_optab tab)
6246 : {
6247 496 : enum insn_code icode;
6248 496 : rtx target = to;
6249 496 : machine_mode fmode, imode;
6250 :
6251 : /* We first try to find a pair of modes, one real and one integer, at
6252 : least as wide as FROM and TO, respectively, in which we can open-code
6253 : this conversion. If the integer mode is wider than the mode of TO,
6254 : we can do the conversion either signed or unsigned. */
6255 :
6256 2054 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
6257 8810 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
6258 : {
6259 7252 : icode = convert_optab_handler (tab, imode, fmode,
6260 : insn_optimization_type ());
6261 7252 : if (icode != CODE_FOR_nothing)
6262 : {
6263 64 : rtx_insn *last = get_last_insn ();
6264 64 : if (fmode != GET_MODE (from))
6265 64 : from = convert_to_mode (fmode, from, 0);
6266 :
6267 64 : if (imode != GET_MODE (to))
6268 0 : target = gen_reg_rtx (imode);
6269 :
6270 64 : if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
6271 : {
6272 0 : delete_insns_since (last);
6273 0 : continue;
6274 : }
6275 64 : if (target != to)
6276 0 : convert_move (to, target, 0);
6277 64 : return true;
6278 : }
6279 : }
6280 :
6281 : return false;
6282 : }
6283 :
6284 : /* Report whether we have an instruction to perform the operation
6285 : specified by CODE on operands of mode MODE. */
6286 : bool
6287 93201135 : have_insn_for (enum rtx_code code, machine_mode mode)
6288 : {
6289 93201135 : return (code_to_optab (code)
6290 93201135 : && (optab_handler (code_to_optab (code), mode)
6291 93201135 : != CODE_FOR_nothing));
6292 : }
6293 :
6294 : /* Print information about the current contents of the optabs on
6295 : STDERR. */
6296 :
6297 : DEBUG_FUNCTION void
6298 0 : debug_optab_libfuncs (void)
6299 : {
6300 0 : int i, j, k;
6301 :
6302 : /* Dump the arithmetic optabs. */
6303 0 : for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6304 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6305 : {
6306 0 : rtx l = optab_libfunc ((optab) i, (machine_mode) j);
6307 0 : if (l)
6308 : {
6309 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6310 0 : fprintf (stderr, "%s\t%s:\t%s\n",
6311 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6312 0 : GET_MODE_NAME (j),
6313 : XSTR (l, 0));
6314 : }
6315 : }
6316 :
6317 : /* Dump the conversion optabs. */
6318 0 : for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6319 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6320 0 : for (k = 0; k < NUM_MACHINE_MODES; ++k)
6321 : {
6322 0 : rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
6323 : (machine_mode) k);
6324 0 : if (l)
6325 : {
6326 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6327 0 : fprintf (stderr, "%s\t%s\t%s:\t%s\n",
6328 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6329 0 : GET_MODE_NAME (j),
6330 0 : GET_MODE_NAME (k),
6331 : XSTR (l, 0));
6332 : }
6333 : }
6334 0 : }
6335 :
6336 : /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6337 : CODE. Return 0 on failure. */
6338 :
6339 : rtx_insn *
6340 0 : gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6341 : {
6342 0 : machine_mode mode = GET_MODE (op1);
6343 0 : enum insn_code icode;
6344 0 : rtx_insn *insn;
6345 0 : rtx trap_rtx;
6346 :
6347 0 : if (mode == VOIDmode)
6348 : return 0;
6349 :
6350 0 : icode = optab_handler (ctrap_optab, mode);
6351 0 : if (icode == CODE_FOR_nothing)
6352 : return 0;
6353 :
6354 : /* Some targets only accept a zero trap code. */
6355 0 : if (!insn_operand_matches (icode, 3, tcode))
6356 : return 0;
6357 :
6358 0 : do_pending_stack_adjust ();
6359 0 : start_sequence ();
6360 0 : prepare_cmp_insn (op1, op2, NULL, code, NULL_RTX, false, OPTAB_DIRECT,
6361 : &trap_rtx, &mode, cbranch_optab);
6362 0 : if (!trap_rtx)
6363 : insn = NULL;
6364 : else
6365 0 : insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6366 : tcode);
6367 :
6368 : /* If that failed, then give up. */
6369 0 : if (insn == 0)
6370 : {
6371 0 : end_sequence ();
6372 0 : return 0;
6373 : }
6374 :
6375 0 : emit_insn (insn);
6376 0 : insn = end_sequence ();
6377 0 : return insn;
6378 : }
6379 :
6380 : /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
6381 : or unsigned operation code. */
6382 :
6383 : enum rtx_code
6384 1472133 : get_rtx_code_1 (enum tree_code tcode, bool unsignedp)
6385 : {
6386 1472133 : enum rtx_code code;
6387 1472133 : switch (tcode)
6388 : {
6389 : case EQ_EXPR:
6390 : code = EQ;
6391 : break;
6392 746114 : case NE_EXPR:
6393 746114 : code = NE;
6394 746114 : break;
6395 139614 : case LT_EXPR:
6396 139614 : code = unsignedp ? LTU : LT;
6397 : break;
6398 72007 : case LE_EXPR:
6399 72007 : code = unsignedp ? LEU : LE;
6400 : break;
6401 114809 : case GT_EXPR:
6402 114809 : code = unsignedp ? GTU : GT;
6403 : break;
6404 76512 : case GE_EXPR:
6405 76512 : code = unsignedp ? GEU : GE;
6406 : break;
6407 :
6408 1073 : case UNORDERED_EXPR:
6409 1073 : code = UNORDERED;
6410 1073 : break;
6411 957 : case ORDERED_EXPR:
6412 957 : code = ORDERED;
6413 957 : break;
6414 406 : case UNLT_EXPR:
6415 406 : code = UNLT;
6416 406 : break;
6417 4309 : case UNLE_EXPR:
6418 4309 : code = UNLE;
6419 4309 : break;
6420 421 : case UNGT_EXPR:
6421 421 : code = UNGT;
6422 421 : break;
6423 3406 : case UNGE_EXPR:
6424 3406 : code = UNGE;
6425 3406 : break;
6426 186 : case UNEQ_EXPR:
6427 186 : code = UNEQ;
6428 186 : break;
6429 445 : case LTGT_EXPR:
6430 445 : code = LTGT;
6431 445 : break;
6432 :
6433 32788 : case BIT_AND_EXPR:
6434 32788 : code = AND;
6435 32788 : break;
6436 :
6437 21 : case BIT_IOR_EXPR:
6438 21 : code = IOR;
6439 21 : break;
6440 :
6441 0 : default:
6442 0 : code = UNKNOWN;
6443 0 : break;
6444 : }
6445 1472133 : return code;
6446 : }
6447 :
6448 : /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6449 : or unsigned operation code. */
6450 :
6451 : enum rtx_code
6452 155427 : get_rtx_code (enum tree_code tcode, bool unsignedp)
6453 : {
6454 155427 : enum rtx_code code = get_rtx_code_1 (tcode, unsignedp);
6455 155427 : gcc_assert (code != UNKNOWN);
6456 155427 : return code;
6457 : }
6458 :
6459 : /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6460 : select signed or unsigned operators. OPNO holds the index of the
6461 : first comparison operand for insn ICODE. Do not generate the
6462 : compare instruction itself. */
6463 :
6464 : rtx
6465 22755 : vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
6466 : tree t_op0, tree t_op1, bool unsignedp,
6467 : enum insn_code icode, unsigned int opno)
6468 : {
6469 22755 : class expand_operand ops[2];
6470 22755 : rtx rtx_op0, rtx_op1;
6471 22755 : machine_mode m0, m1;
6472 22755 : enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6473 :
6474 22755 : gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6475 :
6476 : /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6477 : has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6478 : cases, use the original mode. */
6479 22755 : rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6480 : EXPAND_STACK_PARM);
6481 22755 : m0 = GET_MODE (rtx_op0);
6482 22755 : if (m0 == VOIDmode)
6483 0 : m0 = TYPE_MODE (TREE_TYPE (t_op0));
6484 :
6485 22755 : rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6486 : EXPAND_STACK_PARM);
6487 22755 : m1 = GET_MODE (rtx_op1);
6488 22755 : if (m1 == VOIDmode)
6489 0 : m1 = TYPE_MODE (TREE_TYPE (t_op1));
6490 :
6491 22755 : create_input_operand (&ops[0], rtx_op0, m0);
6492 22755 : create_input_operand (&ops[1], rtx_op1, m1);
6493 22755 : if (!maybe_legitimize_operands (icode, opno, 2, ops))
6494 0 : gcc_unreachable ();
6495 22755 : return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
6496 : }
6497 :
6498 : /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6499 : the first vec_perm operand, assuming the second operand (for left shift
6500 : first operand) is a constant vector of zeros. Return the shift distance
6501 : in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6502 : mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6503 : shift or vec_shl_optab for left shift. */
6504 : static rtx
6505 2695 : shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
6506 : optab shift_optab)
6507 : {
6508 2695 : unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
6509 2695 : poly_int64 first = sel[0];
6510 5390 : if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
6511 : return NULL_RTX;
6512 :
6513 2694 : if (shift_optab == vec_shl_optab)
6514 : {
6515 807 : unsigned int nelt;
6516 1614 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6517 2695 : return NULL_RTX;
6518 807 : unsigned firstidx = 0;
6519 5547 : for (unsigned int i = 0; i < nelt; i++)
6520 : {
6521 4794 : if (known_eq (sel[i], nelt))
6522 : {
6523 777 : if (i == 0 || firstidx)
6524 : return NULL_RTX;
6525 : firstidx = i;
6526 : }
6527 8034 : else if (firstidx
6528 5464 : ? maybe_ne (sel[i], nelt + i - firstidx)
6529 4017 : : maybe_ge (sel[i], nelt))
6530 : return NULL_RTX;
6531 : }
6532 :
6533 753 : if (firstidx == 0)
6534 : return NULL_RTX;
6535 753 : first = firstidx;
6536 : }
6537 1887 : else if (!sel.series_p (0, 1, first, 1))
6538 : {
6539 152 : unsigned int nelt;
6540 304 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6541 177 : return NULL_RTX;
6542 252 : for (unsigned int i = 1; i < nelt; i++)
6543 : {
6544 222 : poly_int64 expected = i + first;
6545 : /* Indices into the second vector are all equivalent. */
6546 222 : if (maybe_lt (sel[i], nelt)
6547 363 : ? maybe_ne (sel[i], expected)
6548 141 : : maybe_lt (expected, nelt))
6549 177 : return NULL_RTX;
6550 : }
6551 : }
6552 :
6553 2518 : return gen_int_shift_amount (mode, first * bitsize);
6554 : }
6555 :
6556 : /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6557 :
6558 : static rtx
6559 10 : expand_vec_perm_1 (enum insn_code icode, rtx target,
6560 : rtx v0, rtx v1, rtx sel)
6561 : {
6562 10 : machine_mode tmode = GET_MODE (target);
6563 10 : machine_mode smode = GET_MODE (sel);
6564 10 : class expand_operand ops[4];
6565 :
6566 10 : gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
6567 : || related_int_vector_mode (tmode).require () == smode);
6568 10 : create_output_operand (&ops[0], target, tmode);
6569 10 : create_input_operand (&ops[3], sel, smode);
6570 :
6571 : /* Make an effort to preserve v0 == v1. The target expander is able to
6572 : rely on this to determine if we're permuting a single input operand. */
6573 10 : if (rtx_equal_p (v0, v1))
6574 : {
6575 6 : if (!insn_operand_matches (icode, 1, v0))
6576 0 : v0 = force_reg (tmode, v0);
6577 6 : gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6578 6 : gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6579 :
6580 6 : create_fixed_operand (&ops[1], v0);
6581 6 : create_fixed_operand (&ops[2], v0);
6582 : }
6583 : else
6584 : {
6585 4 : create_input_operand (&ops[1], v0, tmode);
6586 4 : create_input_operand (&ops[2], v1, tmode);
6587 : }
6588 :
6589 10 : if (maybe_expand_insn (icode, 4, ops))
6590 10 : return ops[0].value;
6591 : return NULL_RTX;
6592 : }
6593 :
6594 : /* Check if vec_perm mask SEL is a constant equivalent to an and operation of
6595 : the non-zero vec_perm operand with some mask consisting of 0xffs and 0x00s,
6596 : assuming the other vec_perm operand is a constant vector of zeros. Return
6597 : the mask for the equivalent and operation, or NULL_RTX if the vec_perm can
6598 : not be modeled as an and. MODE is the mode of the value being anded.
6599 : ZERO_OP0_P is true if the first operand of the vec_perm is a constant vector
6600 : of zeros or false if the second operand of the vec_perm is a constant vector
6601 : of zeros. */
6602 : rtx
6603 0 : vec_perm_and_mask (machine_mode mode, const vec_perm_indices &sel,
6604 : bool zero_op0_p)
6605 : {
6606 0 : unsigned int nelt;
6607 0 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6608 : return NULL_RTX;
6609 :
6610 0 : rtx_vector_builder builder (mode, nelt, 1);
6611 0 : machine_mode emode = GET_MODE_INNER (mode);
6612 :
6613 0 : for (unsigned int i = 0; i < nelt; i++)
6614 : {
6615 0 : if (zero_op0_p)
6616 : {
6617 0 : if (known_eq (sel[i], nelt + i))
6618 0 : builder.quick_push (CONSTM1_RTX (emode));
6619 0 : else if (known_lt (sel[i], nelt))
6620 0 : builder.quick_push (CONST0_RTX (emode));
6621 : else
6622 0 : return NULL_RTX;
6623 : }
6624 : else
6625 : {
6626 0 : if (known_eq (sel[i], i))
6627 0 : builder.quick_push (CONSTM1_RTX (emode));
6628 0 : else if (known_ge (sel[i], nelt))
6629 0 : builder.quick_push (CONST0_RTX (emode));
6630 : else
6631 : return NULL_RTX;
6632 : }
6633 : }
6634 :
6635 0 : return builder.build ();
6636 0 : }
6637 :
6638 : /* Implement a permutation of vectors v0 and v1 using the permutation
6639 : vector in SEL and return the result. Use TARGET to hold the result
6640 : if nonnull and convenient.
6641 :
6642 : MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6643 : is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6644 : to have a particular mode. */
6645 :
6646 : rtx
6647 75236 : expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
6648 : const vec_perm_builder &sel, machine_mode sel_mode,
6649 : rtx target)
6650 : {
6651 75236 : if (!target || !register_operand (target, mode))
6652 60816 : target = gen_reg_rtx (mode);
6653 :
6654 : /* Set QIMODE to a different vector mode with byte elements.
6655 : If no such mode, or if MODE already has byte elements, use VOIDmode. */
6656 75236 : machine_mode qimode;
6657 75236 : if (!qimode_for_vec_perm (mode).exists (&qimode))
6658 7024 : qimode = VOIDmode;
6659 :
6660 75236 : rtx_insn *last = get_last_insn ();
6661 :
6662 75236 : bool single_arg_p = rtx_equal_p (v0, v1);
6663 : /* Always specify two input vectors here and leave the target to handle
6664 : cases in which the inputs are equal. Not all backends can cope with
6665 : the single-input representation when testing for a double-input
6666 : target instruction. */
6667 150472 : vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
6668 :
6669 : /* See if this can be handled with a vec_shr or vec_shl. We only do this
6670 : if the second (for vec_shr) or first (for vec_shl) vector is all
6671 : zeroes. */
6672 75236 : insn_code shift_code = CODE_FOR_nothing;
6673 75236 : insn_code shift_code_qi = CODE_FOR_nothing;
6674 75236 : optab shift_optab = unknown_optab;
6675 75236 : rtx v2 = v0;
6676 75236 : if (v1 == CONST0_RTX (GET_MODE (v1)))
6677 : shift_optab = vec_shr_optab;
6678 73284 : else if (v0 == CONST0_RTX (GET_MODE (v0)))
6679 : {
6680 : shift_optab = vec_shl_optab;
6681 : v2 = v1;
6682 : }
6683 : if (shift_optab != unknown_optab)
6684 : {
6685 3083 : shift_code = optab_handler (shift_optab, mode);
6686 5580 : shift_code_qi = ((qimode != VOIDmode && qimode != mode)
6687 5873 : ? optab_handler (shift_optab, qimode)
6688 : : CODE_FOR_nothing);
6689 : }
6690 3083 : if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
6691 : {
6692 2695 : rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices, shift_optab);
6693 2695 : if (shift_amt)
6694 : {
6695 2518 : class expand_operand ops[3];
6696 2518 : if (shift_amt == const0_rtx)
6697 2518 : return v2;
6698 2518 : if (shift_code != CODE_FOR_nothing)
6699 : {
6700 2518 : create_output_operand (&ops[0], target, mode);
6701 2518 : create_input_operand (&ops[1], v2, mode);
6702 2518 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6703 2518 : if (maybe_expand_insn (shift_code, 3, ops))
6704 2518 : return ops[0].value;
6705 : }
6706 0 : if (shift_code_qi != CODE_FOR_nothing)
6707 : {
6708 0 : rtx tmp = gen_reg_rtx (qimode);
6709 0 : create_output_operand (&ops[0], tmp, qimode);
6710 0 : create_input_operand (&ops[1], gen_lowpart (qimode, v2), qimode);
6711 0 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6712 0 : if (maybe_expand_insn (shift_code_qi, 3, ops))
6713 0 : return gen_lowpart (mode, ops[0].value);
6714 : }
6715 : }
6716 : }
6717 :
6718 72718 : if (targetm.vectorize.vec_perm_const != NULL)
6719 : {
6720 72718 : if (single_arg_p)
6721 18580 : v1 = v0;
6722 :
6723 72718 : gcc_checking_assert (GET_MODE (v0) == GET_MODE (v1));
6724 72718 : machine_mode op_mode = GET_MODE (v0);
6725 72718 : if (targetm.vectorize.vec_perm_const (mode, op_mode, target, v0, v1,
6726 : indices))
6727 : return target;
6728 : }
6729 :
6730 : /* Fall back to a constant byte-based permutation. */
6731 0 : vec_perm_indices qimode_indices;
6732 0 : rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
6733 0 : if (qimode != VOIDmode)
6734 : {
6735 0 : qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
6736 0 : target_qi = gen_reg_rtx (qimode);
6737 0 : v0_qi = gen_lowpart (qimode, v0);
6738 0 : v1_qi = gen_lowpart (qimode, v1);
6739 0 : if (targetm.vectorize.vec_perm_const != NULL
6740 0 : && targetm.vectorize.vec_perm_const (qimode, qimode, target_qi, v0_qi,
6741 : v1_qi, qimode_indices))
6742 0 : return gen_lowpart (mode, target_qi);
6743 : }
6744 :
6745 0 : v0 = force_reg (mode, v0);
6746 0 : if (single_arg_p)
6747 0 : v1 = v0;
6748 0 : v1 = force_reg (mode, v1);
6749 :
6750 : /* Otherwise expand as a fully variable permutation. */
6751 :
6752 : /* The optabs are only defined for selectors with the same width
6753 : as the values being permuted. */
6754 0 : machine_mode required_sel_mode;
6755 0 : if (!related_int_vector_mode (mode).exists (&required_sel_mode))
6756 : {
6757 0 : delete_insns_since (last);
6758 0 : return NULL_RTX;
6759 : }
6760 :
6761 : /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6762 : If that isn't the mode we want then we need to prove that using
6763 : REQUIRED_SEL_MODE is OK. */
6764 0 : if (sel_mode != required_sel_mode)
6765 : {
6766 0 : if (!selector_fits_mode_p (required_sel_mode, indices))
6767 : {
6768 0 : delete_insns_since (last);
6769 0 : return NULL_RTX;
6770 : }
6771 : sel_mode = required_sel_mode;
6772 : }
6773 :
6774 0 : insn_code icode = direct_optab_handler (vec_perm_optab, mode);
6775 0 : if (icode != CODE_FOR_nothing)
6776 : {
6777 0 : rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
6778 0 : rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
6779 0 : if (tmp)
6780 : return tmp;
6781 : }
6782 :
6783 0 : if (qimode != VOIDmode
6784 0 : && selector_fits_mode_p (qimode, qimode_indices))
6785 : {
6786 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
6787 0 : if (icode != CODE_FOR_nothing)
6788 : {
6789 0 : rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
6790 0 : rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
6791 0 : if (tmp)
6792 0 : return gen_lowpart (mode, tmp);
6793 : }
6794 : }
6795 :
6796 0 : delete_insns_since (last);
6797 0 : return NULL_RTX;
6798 0 : }
6799 :
6800 : /* Implement a permutation of vectors v0 and v1 using the permutation
6801 : vector in SEL and return the result. Use TARGET to hold the result
6802 : if nonnull and convenient.
6803 :
6804 : MODE is the mode of the vectors being permuted (V0 and V1).
6805 : SEL must have the integer equivalent of MODE and is known to be
6806 : unsuitable for permutes with a constant permutation vector. */
6807 :
6808 : rtx
6809 10 : expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6810 : {
6811 10 : enum insn_code icode;
6812 10 : unsigned int i, u;
6813 10 : rtx tmp, sel_qi;
6814 :
6815 10 : u = GET_MODE_UNIT_SIZE (mode);
6816 :
6817 10 : if (!target || GET_MODE (target) != mode)
6818 0 : target = gen_reg_rtx (mode);
6819 :
6820 10 : icode = direct_optab_handler (vec_perm_optab, mode);
6821 10 : if (icode != CODE_FOR_nothing)
6822 : {
6823 10 : tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6824 10 : if (tmp)
6825 : return tmp;
6826 : }
6827 :
6828 : /* As a special case to aid several targets, lower the element-based
6829 : permutation to a byte-based permutation and try again. */
6830 0 : machine_mode qimode;
6831 0 : if (!qimode_for_vec_perm (mode).exists (&qimode)
6832 0 : || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
6833 0 : return NULL_RTX;
6834 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
6835 0 : if (icode == CODE_FOR_nothing)
6836 : return NULL_RTX;
6837 :
6838 : /* Multiply each element by its byte size. */
6839 0 : machine_mode selmode = GET_MODE (sel);
6840 0 : if (u == 2)
6841 0 : sel = expand_simple_binop (selmode, PLUS, sel, sel,
6842 : NULL, 0, OPTAB_DIRECT);
6843 : else
6844 0 : sel = expand_simple_binop (selmode, ASHIFT, sel,
6845 0 : gen_int_shift_amount (selmode, exact_log2 (u)),
6846 : NULL, 0, OPTAB_DIRECT);
6847 0 : gcc_assert (sel != NULL);
6848 :
6849 : /* Broadcast the low byte each element into each of its bytes.
6850 : The encoding has U interleaved stepped patterns, one for each
6851 : byte of an element. */
6852 0 : vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
6853 0 : unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
6854 0 : for (i = 0; i < 3; ++i)
6855 0 : for (unsigned int j = 0; j < u; ++j)
6856 0 : const_sel.quick_push (i * u + low_byte_in_u);
6857 0 : sel = gen_lowpart (qimode, sel);
6858 0 : sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
6859 0 : gcc_assert (sel != NULL);
6860 :
6861 : /* Add the byte offset to each byte element. */
6862 : /* Note that the definition of the indicies here is memory ordering,
6863 : so there should be no difference between big and little endian. */
6864 0 : rtx_vector_builder byte_indices (qimode, u, 1);
6865 0 : for (i = 0; i < u; ++i)
6866 0 : byte_indices.quick_push (GEN_INT (i));
6867 0 : tmp = byte_indices.build ();
6868 0 : sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
6869 : sel, 0, OPTAB_DIRECT);
6870 0 : gcc_assert (sel_qi != NULL);
6871 :
6872 0 : tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6873 0 : tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
6874 0 : gen_lowpart (qimode, v1), sel_qi);
6875 0 : if (tmp)
6876 0 : tmp = gen_lowpart (mode, tmp);
6877 0 : return tmp;
6878 0 : }
6879 :
6880 : /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
6881 : Use TARGET for the result if nonnull and convenient. */
6882 :
6883 : rtx
6884 0 : expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
6885 : {
6886 0 : class expand_operand ops[3];
6887 0 : enum insn_code icode;
6888 0 : machine_mode emode = GET_MODE_INNER (vmode);
6889 :
6890 0 : icode = direct_optab_handler (vec_series_optab, vmode);
6891 0 : gcc_assert (icode != CODE_FOR_nothing);
6892 :
6893 0 : create_output_operand (&ops[0], target, vmode);
6894 0 : create_input_operand (&ops[1], op0, emode);
6895 0 : create_input_operand (&ops[2], op1, emode);
6896 :
6897 0 : expand_insn (icode, 3, ops);
6898 0 : return ops[0].value;
6899 : }
6900 :
6901 : /* Generate insns for a vector comparison into a mask. */
6902 :
6903 : rtx
6904 22755 : expand_vec_cmp_expr (tree type, tree exp, rtx target)
6905 : {
6906 22755 : class expand_operand ops[4];
6907 22755 : enum insn_code icode;
6908 22755 : rtx comparison;
6909 22755 : machine_mode mask_mode = TYPE_MODE (type);
6910 22755 : machine_mode vmode;
6911 22755 : bool unsignedp;
6912 22755 : tree op0a, op0b;
6913 22755 : enum tree_code tcode;
6914 :
6915 22755 : op0a = TREE_OPERAND (exp, 0);
6916 22755 : op0b = TREE_OPERAND (exp, 1);
6917 22755 : tcode = TREE_CODE (exp);
6918 :
6919 22755 : unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
6920 22755 : vmode = TYPE_MODE (TREE_TYPE (op0a));
6921 :
6922 22755 : icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
6923 22755 : if (icode == CODE_FOR_nothing)
6924 : {
6925 244 : if (tcode == EQ_EXPR || tcode == NE_EXPR)
6926 244 : icode = get_vec_cmp_eq_icode (vmode, mask_mode);
6927 244 : if (icode == CODE_FOR_nothing)
6928 : return 0;
6929 : }
6930 :
6931 22755 : comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
6932 : unsignedp, icode, 2);
6933 22755 : create_output_operand (&ops[0], target, mask_mode);
6934 22755 : create_fixed_operand (&ops[1], comparison);
6935 22755 : create_fixed_operand (&ops[2], XEXP (comparison, 0));
6936 22755 : create_fixed_operand (&ops[3], XEXP (comparison, 1));
6937 22755 : expand_insn (icode, 4, ops);
6938 22755 : return ops[0].value;
6939 : }
6940 :
6941 : /* Expand a highpart multiply. */
6942 :
6943 : rtx
6944 1726 : expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
6945 : rtx target, bool uns_p)
6946 : {
6947 1726 : class expand_operand eops[3];
6948 1726 : enum insn_code icode;
6949 1726 : int method, i;
6950 1726 : machine_mode wmode;
6951 1726 : rtx m1, m2;
6952 1726 : optab tab1, tab2;
6953 :
6954 1726 : method = can_mult_highpart_p (mode, uns_p);
6955 1726 : switch (method)
6956 : {
6957 : case 0:
6958 : return NULL_RTX;
6959 1016 : case 1:
6960 1016 : tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6961 1016 : return expand_binop (mode, tab1, op0, op1, target, uns_p,
6962 1016 : OPTAB_LIB_WIDEN);
6963 0 : case 2:
6964 0 : return expmed_mult_highpart_optab (as_a <scalar_int_mode> (mode),
6965 0 : op0, op1, target, uns_p, INT_MAX);
6966 703 : case 3:
6967 703 : tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6968 502 : tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6969 : break;
6970 7 : case 4:
6971 7 : tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6972 7 : tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6973 : if (BYTES_BIG_ENDIAN)
6974 : std::swap (tab1, tab2);
6975 : break;
6976 0 : default:
6977 0 : gcc_unreachable ();
6978 : }
6979 :
6980 710 : icode = optab_handler (tab1, mode);
6981 710 : wmode = insn_data[icode].operand[0].mode;
6982 2130 : gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
6983 : GET_MODE_NUNITS (mode)));
6984 2130 : gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
6985 :
6986 710 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6987 710 : create_input_operand (&eops[1], op0, mode);
6988 710 : create_input_operand (&eops[2], op1, mode);
6989 710 : expand_insn (icode, 3, eops);
6990 710 : m1 = gen_lowpart (mode, eops[0].value);
6991 :
6992 710 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
6993 710 : create_input_operand (&eops[1], op0, mode);
6994 710 : create_input_operand (&eops[2], op1, mode);
6995 710 : expand_insn (optab_handler (tab2, mode), 3, eops);
6996 710 : m2 = gen_lowpart (mode, eops[0].value);
6997 :
6998 1420 : vec_perm_builder sel;
6999 710 : if (method == 3)
7000 : {
7001 : /* The encoding has 2 interleaved stepped patterns. */
7002 1406 : sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
7003 4921 : for (i = 0; i < 6; ++i)
7004 12654 : sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
7005 6327 : + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
7006 : }
7007 : else
7008 : {
7009 : /* The encoding has a single interleaved stepped pattern. */
7010 14 : sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
7011 28 : for (i = 0; i < 3; ++i)
7012 21 : sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
7013 : }
7014 :
7015 710 : return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
7016 : }
7017 :
7018 : /* Helper function to find the MODE_CC set in a sync_compare_and_swap
7019 : pattern. */
7020 :
7021 : static void
7022 0 : find_cc_set (rtx x, const_rtx pat, void *data)
7023 : {
7024 0 : if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
7025 0 : && GET_CODE (pat) == SET)
7026 : {
7027 0 : rtx *p_cc_reg = (rtx *) data;
7028 0 : gcc_assert (!*p_cc_reg);
7029 0 : *p_cc_reg = x;
7030 : }
7031 0 : }
7032 :
7033 : /* This is a helper function for the other atomic operations. This function
7034 : emits a loop that contains SEQ that iterates until a compare-and-swap
7035 : operation at the end succeeds. MEM is the memory to be modified. SEQ is
7036 : a set of instructions that takes a value from OLD_REG as an input and
7037 : produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
7038 : set to the current contents of MEM. After SEQ, a compare-and-swap will
7039 : attempt to update MEM with NEW_REG. The function returns true when the
7040 : loop was generated successfully. */
7041 :
7042 : static bool
7043 5070 : expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
7044 : {
7045 5070 : machine_mode mode = GET_MODE (mem);
7046 5070 : rtx_code_label *label;
7047 5070 : rtx cmp_reg, success, oldval;
7048 :
7049 : /* The loop we want to generate looks like
7050 :
7051 : cmp_reg = mem;
7052 : label:
7053 : old_reg = cmp_reg;
7054 : seq;
7055 : (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
7056 : if (success)
7057 : goto label;
7058 :
7059 : Note that we only do the plain load from memory once. Subsequent
7060 : iterations use the value loaded by the compare-and-swap pattern. */
7061 :
7062 5070 : label = gen_label_rtx ();
7063 5070 : cmp_reg = gen_reg_rtx (mode);
7064 :
7065 5070 : emit_move_insn (cmp_reg, mem);
7066 5070 : emit_label (label);
7067 5070 : emit_move_insn (old_reg, cmp_reg);
7068 5070 : if (seq)
7069 5057 : emit_insn (seq);
7070 :
7071 5070 : success = NULL_RTX;
7072 5070 : oldval = cmp_reg;
7073 5070 : if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
7074 : new_reg, false, MEMMODEL_SYNC_SEQ_CST,
7075 : MEMMODEL_RELAXED))
7076 : return false;
7077 :
7078 5070 : if (oldval != cmp_reg)
7079 0 : emit_move_insn (cmp_reg, oldval);
7080 :
7081 : /* Mark this jump predicted not taken. */
7082 5070 : emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
7083 5070 : GET_MODE (success), 1, label,
7084 : profile_probability::guessed_never ());
7085 5070 : return true;
7086 : }
7087 :
7088 :
7089 : /* This function tries to emit an atomic_exchange intruction. VAL is written
7090 : to *MEM using memory model MODEL. The previous contents of *MEM are returned,
7091 : using TARGET if possible. */
7092 :
7093 : static rtx
7094 3893 : maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7095 : {
7096 3893 : machine_mode mode = GET_MODE (mem);
7097 3893 : enum insn_code icode;
7098 :
7099 : /* If the target supports the exchange directly, great. */
7100 3893 : icode = direct_optab_handler (atomic_exchange_optab, mode);
7101 3893 : if (icode != CODE_FOR_nothing)
7102 : {
7103 3876 : class expand_operand ops[4];
7104 :
7105 3876 : create_output_operand (&ops[0], target, mode);
7106 3876 : create_fixed_operand (&ops[1], mem);
7107 3876 : create_input_operand (&ops[2], val, mode);
7108 3876 : create_integer_operand (&ops[3], model);
7109 3876 : if (maybe_expand_insn (icode, 4, ops))
7110 3876 : return ops[0].value;
7111 : }
7112 :
7113 : return NULL_RTX;
7114 : }
7115 :
7116 : /* This function tries to implement an atomic exchange operation using
7117 : __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
7118 : The previous contents of *MEM are returned, using TARGET if possible.
7119 : Since this instructionn is an acquire barrier only, stronger memory
7120 : models may require additional barriers to be emitted. */
7121 :
7122 : static rtx
7123 2 : maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
7124 : enum memmodel model)
7125 : {
7126 2 : machine_mode mode = GET_MODE (mem);
7127 2 : enum insn_code icode;
7128 2 : rtx_insn *last_insn = get_last_insn ();
7129 :
7130 2 : icode = optab_handler (sync_lock_test_and_set_optab, mode);
7131 :
7132 : /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7133 : exists, and the memory model is stronger than acquire, add a release
7134 : barrier before the instruction. */
7135 :
7136 2 : if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
7137 0 : expand_mem_thread_fence (model);
7138 :
7139 2 : if (icode != CODE_FOR_nothing)
7140 : {
7141 0 : class expand_operand ops[3];
7142 0 : create_output_operand (&ops[0], target, mode);
7143 0 : create_fixed_operand (&ops[1], mem);
7144 0 : create_input_operand (&ops[2], val, mode);
7145 0 : if (maybe_expand_insn (icode, 3, ops))
7146 0 : return ops[0].value;
7147 : }
7148 :
7149 : /* If an external test-and-set libcall is provided, use that instead of
7150 : any external compare-and-swap that we might get from the compare-and-
7151 : swap-loop expansion later. */
7152 2 : if (!can_compare_and_swap_p (mode, false))
7153 : {
7154 2 : rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
7155 2 : if (libfunc != NULL)
7156 : {
7157 0 : rtx addr;
7158 :
7159 0 : addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7160 0 : return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7161 : mode, addr, ptr_mode,
7162 0 : val, mode);
7163 : }
7164 : }
7165 :
7166 : /* If the test_and_set can't be emitted, eliminate any barrier that might
7167 : have been emitted. */
7168 2 : delete_insns_since (last_insn);
7169 2 : return NULL_RTX;
7170 : }
7171 :
7172 : /* This function tries to implement an atomic exchange operation using a
7173 : compare_and_swap loop. VAL is written to *MEM. The previous contents of
7174 : *MEM are returned, using TARGET if possible. No memory model is required
7175 : since a compare_and_swap loop is seq-cst. */
7176 :
7177 : static rtx
7178 17 : maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
7179 : {
7180 17 : machine_mode mode = GET_MODE (mem);
7181 :
7182 17 : if (can_compare_and_swap_p (mode, true))
7183 : {
7184 13 : if (!target || !register_operand (target, mode))
7185 1 : target = gen_reg_rtx (mode);
7186 13 : if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
7187 : return target;
7188 : }
7189 :
7190 : return NULL_RTX;
7191 : }
7192 :
7193 : /* This function tries to implement an atomic test-and-set operation
7194 : using the atomic_test_and_set instruction pattern. A boolean value
7195 : is returned from the operation, using TARGET if possible. */
7196 :
7197 : static rtx
7198 268 : maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7199 : {
7200 268 : machine_mode pat_bool_mode;
7201 268 : class expand_operand ops[3];
7202 :
7203 268 : if (!targetm.have_atomic_test_and_set ())
7204 : return NULL_RTX;
7205 :
7206 : /* While we always get QImode from __atomic_test_and_set, we get
7207 : other memory modes from __sync_lock_test_and_set. Note that we
7208 : use no endian adjustment here. This matches the 4.6 behavior
7209 : in the Sparc backend. */
7210 0 : enum insn_code icode = targetm.code_for_atomic_test_and_set;
7211 0 : gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
7212 0 : if (GET_MODE (mem) != QImode)
7213 0 : mem = adjust_address_nv (mem, QImode, 0);
7214 :
7215 0 : pat_bool_mode = insn_data[icode].operand[0].mode;
7216 0 : create_output_operand (&ops[0], target, pat_bool_mode);
7217 0 : create_fixed_operand (&ops[1], mem);
7218 0 : create_integer_operand (&ops[2], model);
7219 :
7220 0 : if (maybe_expand_insn (icode, 3, ops))
7221 0 : return ops[0].value;
7222 : return NULL_RTX;
7223 : }
7224 :
7225 : /* This function expands the legacy _sync_lock test_and_set operation which is
7226 : generally an atomic exchange. Some limited targets only allow the
7227 : constant 1 to be stored. This is an ACQUIRE operation.
7228 :
7229 : TARGET is an optional place to stick the return value.
7230 : MEM is where VAL is stored. */
7231 :
7232 : rtx
7233 326 : expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7234 : {
7235 326 : rtx ret;
7236 :
7237 : /* Try an atomic_exchange first. */
7238 326 : ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
7239 326 : if (ret)
7240 : return ret;
7241 :
7242 2 : ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
7243 : MEMMODEL_SYNC_ACQUIRE);
7244 2 : if (ret)
7245 : return ret;
7246 :
7247 2 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7248 2 : if (ret)
7249 : return ret;
7250 :
7251 : /* If there are no other options, try atomic_test_and_set if the value
7252 : being stored is 1. */
7253 2 : if (val == const1_rtx)
7254 2 : ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
7255 :
7256 : return ret;
7257 : }
7258 :
7259 : /* This function expands the atomic test_and_set operation:
7260 : atomically store a boolean TRUE into MEM and return the previous value.
7261 :
7262 : MEMMODEL is the memory model variant to use.
7263 : TARGET is an optional place to stick the return value. */
7264 :
7265 : rtx
7266 266 : expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7267 : {
7268 266 : machine_mode mode = GET_MODE (mem);
7269 266 : rtx ret, trueval, subtarget;
7270 :
7271 266 : ret = maybe_emit_atomic_test_and_set (target, mem, model);
7272 266 : if (ret)
7273 : return ret;
7274 :
7275 : /* Be binary compatible with non-default settings of trueval, and different
7276 : cpu revisions. E.g. one revision may have atomic-test-and-set, but
7277 : another only has atomic-exchange. */
7278 266 : if (targetm.atomic_test_and_set_trueval == 1)
7279 : {
7280 266 : trueval = const1_rtx;
7281 266 : subtarget = target ? target : gen_reg_rtx (mode);
7282 : }
7283 : else
7284 : {
7285 0 : trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7286 0 : subtarget = gen_reg_rtx (mode);
7287 : }
7288 :
7289 : /* Try the atomic-exchange optab... */
7290 266 : ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
7291 :
7292 : /* ... then an atomic-compare-and-swap loop ... */
7293 266 : if (!ret)
7294 0 : ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
7295 :
7296 : /* ... before trying the vaguely defined legacy lock_test_and_set. */
7297 0 : if (!ret)
7298 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
7299 :
7300 : /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7301 : things with the value 1. Thus we try again without trueval. */
7302 266 : if (!ret && targetm.atomic_test_and_set_trueval != 1)
7303 : {
7304 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
7305 :
7306 0 : if (ret)
7307 : {
7308 : /* Rectify the not-one trueval. */
7309 0 : ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7310 0 : gcc_assert (ret);
7311 : }
7312 : }
7313 :
7314 : return ret;
7315 : }
7316 :
7317 : /* This function expands the atomic exchange operation:
7318 : atomically store VAL in MEM and return the previous value in MEM.
7319 :
7320 : MEMMODEL is the memory model variant to use.
7321 : TARGET is an optional place to stick the return value. */
7322 :
7323 : rtx
7324 2838 : expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7325 : {
7326 2838 : machine_mode mode = GET_MODE (mem);
7327 2838 : rtx ret;
7328 :
7329 : /* If loads are not atomic for the required size and we are not called to
7330 : provide a __sync builtin, do not do anything so that we stay consistent
7331 : with atomic loads of the same size. */
7332 2838 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7333 : return NULL_RTX;
7334 :
7335 2807 : ret = maybe_emit_atomic_exchange (target, mem, val, model);
7336 :
7337 : /* Next try a compare-and-swap loop for the exchange. */
7338 2807 : if (!ret)
7339 13 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7340 :
7341 : return ret;
7342 : }
7343 :
7344 : /* This function expands the atomic compare exchange operation:
7345 :
7346 : *PTARGET_BOOL is an optional place to store the boolean success/failure.
7347 : *PTARGET_OVAL is an optional place to store the old value from memory.
7348 : Both target parameters may be NULL or const0_rtx to indicate that we do
7349 : not care about that return value. Both target parameters are updated on
7350 : success to the actual location of the corresponding result.
7351 :
7352 : MEMMODEL is the memory model variant to use.
7353 :
7354 : The return value of the function is true for success. */
7355 :
7356 : bool
7357 28476 : expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7358 : rtx mem, rtx expected, rtx desired,
7359 : bool is_weak, enum memmodel succ_model,
7360 : enum memmodel fail_model)
7361 : {
7362 28476 : machine_mode mode = GET_MODE (mem);
7363 28476 : class expand_operand ops[8];
7364 28476 : enum insn_code icode;
7365 28476 : rtx target_oval, target_bool = NULL_RTX;
7366 28476 : rtx libfunc;
7367 :
7368 : /* If loads are not atomic for the required size and we are not called to
7369 : provide a __sync builtin, do not do anything so that we stay consistent
7370 : with atomic loads of the same size. */
7371 28476 : if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
7372 : return false;
7373 :
7374 : /* Load expected into a register for the compare and swap. */
7375 26725 : if (MEM_P (expected))
7376 7360 : expected = copy_to_reg (expected);
7377 :
7378 : /* Make sure we always have some place to put the return oldval.
7379 : Further, make sure that place is distinct from the input expected,
7380 : just in case we need that path down below. */
7381 26725 : if (ptarget_oval && *ptarget_oval == const0_rtx)
7382 : ptarget_oval = NULL;
7383 :
7384 26499 : if (ptarget_oval == NULL
7385 26499 : || (target_oval = *ptarget_oval) == NULL
7386 5300 : || reg_overlap_mentioned_p (expected, target_oval))
7387 21503 : target_oval = gen_reg_rtx (mode);
7388 :
7389 26725 : icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7390 26725 : if (icode != CODE_FOR_nothing)
7391 : {
7392 26723 : machine_mode bool_mode = insn_data[icode].operand[0].mode;
7393 :
7394 26723 : if (ptarget_bool && *ptarget_bool == const0_rtx)
7395 : ptarget_bool = NULL;
7396 :
7397 : /* Make sure we always have a place for the bool operand. */
7398 26468 : if (ptarget_bool == NULL
7399 26468 : || (target_bool = *ptarget_bool) == NULL
7400 7243 : || GET_MODE (target_bool) != bool_mode)
7401 19480 : target_bool = gen_reg_rtx (bool_mode);
7402 :
7403 : /* Emit the compare_and_swap. */
7404 26723 : create_output_operand (&ops[0], target_bool, bool_mode);
7405 26723 : create_output_operand (&ops[1], target_oval, mode);
7406 26723 : create_fixed_operand (&ops[2], mem);
7407 26723 : create_input_operand (&ops[3], expected, mode);
7408 26723 : create_input_operand (&ops[4], desired, mode);
7409 26723 : create_integer_operand (&ops[5], is_weak);
7410 26723 : create_integer_operand (&ops[6], succ_model);
7411 26723 : create_integer_operand (&ops[7], fail_model);
7412 26723 : if (maybe_expand_insn (icode, 8, ops))
7413 : {
7414 : /* Return success/failure. */
7415 26723 : target_bool = ops[0].value;
7416 26723 : target_oval = ops[1].value;
7417 26723 : goto success;
7418 : }
7419 : }
7420 :
7421 : /* Otherwise fall back to the original __sync_val_compare_and_swap
7422 : which is always seq-cst. */
7423 2 : icode = optab_handler (sync_compare_and_swap_optab, mode);
7424 2 : if (icode != CODE_FOR_nothing)
7425 : {
7426 0 : rtx cc_reg;
7427 :
7428 0 : create_output_operand (&ops[0], target_oval, mode);
7429 0 : create_fixed_operand (&ops[1], mem);
7430 0 : create_input_operand (&ops[2], expected, mode);
7431 0 : create_input_operand (&ops[3], desired, mode);
7432 0 : if (!maybe_expand_insn (icode, 4, ops))
7433 0 : return false;
7434 :
7435 0 : target_oval = ops[0].value;
7436 :
7437 : /* If the caller isn't interested in the boolean return value,
7438 : skip the computation of it. */
7439 0 : if (ptarget_bool == NULL)
7440 0 : goto success;
7441 :
7442 : /* Otherwise, work out if the compare-and-swap succeeded. */
7443 0 : cc_reg = NULL_RTX;
7444 0 : if (have_insn_for (COMPARE, CCmode))
7445 0 : note_stores (get_last_insn (), find_cc_set, &cc_reg);
7446 0 : if (cc_reg)
7447 : {
7448 0 : target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7449 : const0_rtx, VOIDmode, 0, 1);
7450 0 : goto success;
7451 : }
7452 0 : goto success_bool_from_val;
7453 : }
7454 :
7455 : /* Also check for library support for __sync_val_compare_and_swap. */
7456 2 : libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7457 2 : if (libfunc != NULL)
7458 : {
7459 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7460 0 : rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7461 : mode, addr, ptr_mode,
7462 : expected, mode, desired, mode);
7463 0 : emit_move_insn (target_oval, target);
7464 :
7465 : /* Compute the boolean return value only if requested. */
7466 0 : if (ptarget_bool)
7467 0 : goto success_bool_from_val;
7468 : else
7469 0 : goto success;
7470 : }
7471 :
7472 : /* Failure. */
7473 : return false;
7474 :
7475 0 : success_bool_from_val:
7476 0 : target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7477 : expected, VOIDmode, 1, 1);
7478 26723 : success:
7479 : /* Make sure that the oval output winds up where the caller asked. */
7480 26723 : if (ptarget_oval)
7481 26499 : *ptarget_oval = target_oval;
7482 26723 : if (ptarget_bool)
7483 26468 : *ptarget_bool = target_bool;
7484 : return true;
7485 : }
7486 :
7487 : /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7488 :
7489 : static void
7490 0 : expand_asm_memory_blockage (void)
7491 : {
7492 0 : rtx asm_op, clob;
7493 :
7494 0 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7495 : rtvec_alloc (0), rtvec_alloc (0),
7496 : rtvec_alloc (0), UNKNOWN_LOCATION);
7497 0 : MEM_VOLATILE_P (asm_op) = 1;
7498 :
7499 0 : clob = gen_rtx_SCRATCH (VOIDmode);
7500 0 : clob = gen_rtx_MEM (BLKmode, clob);
7501 0 : clob = gen_rtx_CLOBBER (VOIDmode, clob);
7502 :
7503 0 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7504 0 : }
7505 :
7506 : /* Do not propagate memory accesses across this point. */
7507 :
7508 : static void
7509 115852 : expand_memory_blockage (void)
7510 : {
7511 115852 : if (targetm.have_memory_blockage ())
7512 115852 : emit_insn (targetm.gen_memory_blockage ());
7513 : else
7514 0 : expand_asm_memory_blockage ();
7515 115852 : }
7516 :
7517 : /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7518 : same time clobbering the register set specified by REGS. */
7519 :
7520 : void
7521 130 : expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs)
7522 : {
7523 130 : rtx asm_op, clob_mem;
7524 :
7525 130 : unsigned int num_of_regs = 0;
7526 12090 : for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7527 11960 : if (TEST_HARD_REG_BIT (regs, i))
7528 1078 : num_of_regs++;
7529 :
7530 130 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7531 : rtvec_alloc (0), rtvec_alloc (0),
7532 : rtvec_alloc (0), UNKNOWN_LOCATION);
7533 130 : MEM_VOLATILE_P (asm_op) = 1;
7534 :
7535 130 : rtvec v = rtvec_alloc (num_of_regs + 2);
7536 :
7537 130 : clob_mem = gen_rtx_SCRATCH (VOIDmode);
7538 130 : clob_mem = gen_rtx_MEM (BLKmode, clob_mem);
7539 130 : clob_mem = gen_rtx_CLOBBER (VOIDmode, clob_mem);
7540 :
7541 130 : RTVEC_ELT (v, 0) = asm_op;
7542 130 : RTVEC_ELT (v, 1) = clob_mem;
7543 :
7544 130 : if (num_of_regs > 0)
7545 : {
7546 : unsigned int j = 2;
7547 12090 : for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7548 11960 : if (TEST_HARD_REG_BIT (regs, i))
7549 : {
7550 1078 : RTVEC_ELT (v, j) = gen_rtx_CLOBBER (VOIDmode, regno_reg_rtx[i]);
7551 1078 : j++;
7552 : }
7553 130 : gcc_assert (j == (num_of_regs + 2));
7554 : }
7555 :
7556 130 : emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
7557 130 : }
7558 :
7559 : /* This routine will either emit the mem_thread_fence pattern or issue a
7560 : sync_synchronize to generate a fence for memory model MEMMODEL. */
7561 :
7562 : void
7563 966 : expand_mem_thread_fence (enum memmodel model)
7564 : {
7565 966 : if (is_mm_relaxed (model))
7566 : return;
7567 957 : if (targetm.have_mem_thread_fence ())
7568 : {
7569 957 : emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
7570 957 : expand_memory_blockage ();
7571 : }
7572 0 : else if (targetm.have_memory_barrier ())
7573 0 : emit_insn (targetm.gen_memory_barrier ());
7574 0 : else if (synchronize_libfunc != NULL_RTX)
7575 0 : emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
7576 : else
7577 0 : expand_memory_blockage ();
7578 : }
7579 :
7580 : /* Emit a signal fence with given memory model. */
7581 :
7582 : void
7583 60 : expand_mem_signal_fence (enum memmodel model)
7584 : {
7585 : /* No machine barrier is required to implement a signal fence, but
7586 : a compiler memory barrier must be issued, except for relaxed MM. */
7587 60 : if (!is_mm_relaxed (model))
7588 51 : expand_memory_blockage ();
7589 60 : }
7590 :
7591 : /* This function expands the atomic load operation:
7592 : return the atomically loaded value in MEM.
7593 :
7594 : MEMMODEL is the memory model variant to use.
7595 : TARGET is an option place to stick the return value. */
7596 :
7597 : rtx
7598 70221 : expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7599 : {
7600 70221 : machine_mode mode = GET_MODE (mem);
7601 70221 : enum insn_code icode;
7602 :
7603 : /* If the target supports the load directly, great. */
7604 70221 : icode = direct_optab_handler (atomic_load_optab, mode);
7605 70221 : if (icode != CODE_FOR_nothing)
7606 : {
7607 66393 : class expand_operand ops[3];
7608 66393 : rtx_insn *last = get_last_insn ();
7609 66393 : if (is_mm_seq_cst (model))
7610 38205 : expand_memory_blockage ();
7611 :
7612 66393 : create_output_operand (&ops[0], target, mode);
7613 66393 : create_fixed_operand (&ops[1], mem);
7614 66393 : create_integer_operand (&ops[2], model);
7615 66393 : if (maybe_expand_insn (icode, 3, ops))
7616 : {
7617 66393 : if (!is_mm_relaxed (model))
7618 51865 : expand_memory_blockage ();
7619 66393 : return ops[0].value;
7620 : }
7621 0 : delete_insns_since (last);
7622 : }
7623 :
7624 : /* If the size of the object is greater than word size on this target,
7625 : then we assume that a load will not be atomic. We could try to
7626 : emulate a load with a compare-and-swap operation, but the store that
7627 : doing this could result in would be incorrect if this is a volatile
7628 : atomic load or targetting read-only-mapped memory. */
7629 3828 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7630 : /* If there is no atomic load, leave the library call. */
7631 : return NULL_RTX;
7632 :
7633 : /* Otherwise assume loads are atomic, and emit the proper barriers. */
7634 0 : if (!target || target == const0_rtx)
7635 0 : target = gen_reg_rtx (mode);
7636 :
7637 : /* For SEQ_CST, emit a barrier before the load. */
7638 0 : if (is_mm_seq_cst (model))
7639 0 : expand_mem_thread_fence (model);
7640 :
7641 0 : emit_move_insn (target, mem);
7642 :
7643 : /* Emit the appropriate barrier after the load. */
7644 0 : expand_mem_thread_fence (model);
7645 :
7646 0 : return target;
7647 : }
7648 :
7649 : /* This function expands the atomic store operation:
7650 : Atomically store VAL in MEM.
7651 : MEMMODEL is the memory model variant to use.
7652 : USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7653 : function returns const0_rtx if a pattern was emitted. */
7654 :
7655 : rtx
7656 17587 : expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7657 : {
7658 17587 : machine_mode mode = GET_MODE (mem);
7659 17587 : enum insn_code icode;
7660 17587 : class expand_operand ops[3];
7661 :
7662 : /* If the target supports the store directly, great. */
7663 17587 : icode = direct_optab_handler (atomic_store_optab, mode);
7664 17587 : if (icode != CODE_FOR_nothing)
7665 : {
7666 15989 : rtx_insn *last = get_last_insn ();
7667 15989 : if (!is_mm_relaxed (model))
7668 13240 : expand_memory_blockage ();
7669 15989 : create_fixed_operand (&ops[0], mem);
7670 15989 : create_input_operand (&ops[1], val, mode);
7671 15989 : create_integer_operand (&ops[2], model);
7672 15989 : if (maybe_expand_insn (icode, 3, ops))
7673 : {
7674 15989 : if (is_mm_seq_cst (model))
7675 11534 : expand_memory_blockage ();
7676 15989 : return const0_rtx;
7677 : }
7678 0 : delete_insns_since (last);
7679 : }
7680 :
7681 : /* If using __sync_lock_release is a viable alternative, try it.
7682 : Note that this will not be set to true if we are expanding a generic
7683 : __atomic_store_n. */
7684 1598 : if (use_release)
7685 : {
7686 2 : icode = direct_optab_handler (sync_lock_release_optab, mode);
7687 2 : if (icode != CODE_FOR_nothing)
7688 : {
7689 0 : create_fixed_operand (&ops[0], mem);
7690 0 : create_input_operand (&ops[1], const0_rtx, mode);
7691 0 : if (maybe_expand_insn (icode, 2, ops))
7692 : {
7693 : /* lock_release is only a release barrier. */
7694 0 : if (is_mm_seq_cst (model))
7695 0 : expand_mem_thread_fence (model);
7696 0 : return const0_rtx;
7697 : }
7698 : }
7699 : }
7700 :
7701 : /* If the size of the object is greater than word size on this target,
7702 : a default store will not be atomic. */
7703 1598 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7704 : {
7705 : /* If loads are atomic or we are called to provide a __sync builtin,
7706 : we can try a atomic_exchange and throw away the result. Otherwise,
7707 : don't do anything so that we do not create an inconsistency between
7708 : loads and stores. */
7709 1598 : if (can_atomic_load_p (mode) || is_mm_sync (model))
7710 : {
7711 2 : rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7712 2 : if (!target)
7713 2 : target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
7714 : val);
7715 2 : if (target)
7716 0 : return const0_rtx;
7717 : }
7718 1598 : return NULL_RTX;
7719 : }
7720 :
7721 : /* Otherwise assume stores are atomic, and emit the proper barriers. */
7722 0 : expand_mem_thread_fence (model);
7723 :
7724 0 : emit_move_insn (mem, val);
7725 :
7726 : /* For SEQ_CST, also emit a barrier after the store. */
7727 0 : if (is_mm_seq_cst (model))
7728 0 : expand_mem_thread_fence (model);
7729 :
7730 0 : return const0_rtx;
7731 : }
7732 :
7733 :
7734 : /* Structure containing the pointers and values required to process the
7735 : various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7736 :
7737 : struct atomic_op_functions
7738 : {
7739 : direct_optab mem_fetch_before;
7740 : direct_optab mem_fetch_after;
7741 : direct_optab mem_no_result;
7742 : optab fetch_before;
7743 : optab fetch_after;
7744 : direct_optab no_result;
7745 : enum rtx_code reverse_code;
7746 : };
7747 :
7748 :
7749 : /* Fill in structure pointed to by OP with the various optab entries for an
7750 : operation of type CODE. */
7751 :
7752 : static void
7753 34908 : get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7754 : {
7755 34908 : gcc_assert (op!= NULL);
7756 :
7757 : /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7758 : in the source code during compilation, and the optab entries are not
7759 : computable until runtime. Fill in the values at runtime. */
7760 34908 : switch (code)
7761 : {
7762 22645 : case PLUS:
7763 22645 : op->mem_fetch_before = atomic_fetch_add_optab;
7764 22645 : op->mem_fetch_after = atomic_add_fetch_optab;
7765 22645 : op->mem_no_result = atomic_add_optab;
7766 22645 : op->fetch_before = sync_old_add_optab;
7767 22645 : op->fetch_after = sync_new_add_optab;
7768 22645 : op->no_result = sync_add_optab;
7769 22645 : op->reverse_code = MINUS;
7770 22645 : break;
7771 4802 : case MINUS:
7772 4802 : op->mem_fetch_before = atomic_fetch_sub_optab;
7773 4802 : op->mem_fetch_after = atomic_sub_fetch_optab;
7774 4802 : op->mem_no_result = atomic_sub_optab;
7775 4802 : op->fetch_before = sync_old_sub_optab;
7776 4802 : op->fetch_after = sync_new_sub_optab;
7777 4802 : op->no_result = sync_sub_optab;
7778 4802 : op->reverse_code = PLUS;
7779 4802 : break;
7780 2194 : case XOR:
7781 2194 : op->mem_fetch_before = atomic_fetch_xor_optab;
7782 2194 : op->mem_fetch_after = atomic_xor_fetch_optab;
7783 2194 : op->mem_no_result = atomic_xor_optab;
7784 2194 : op->fetch_before = sync_old_xor_optab;
7785 2194 : op->fetch_after = sync_new_xor_optab;
7786 2194 : op->no_result = sync_xor_optab;
7787 2194 : op->reverse_code = XOR;
7788 2194 : break;
7789 2068 : case AND:
7790 2068 : op->mem_fetch_before = atomic_fetch_and_optab;
7791 2068 : op->mem_fetch_after = atomic_and_fetch_optab;
7792 2068 : op->mem_no_result = atomic_and_optab;
7793 2068 : op->fetch_before = sync_old_and_optab;
7794 2068 : op->fetch_after = sync_new_and_optab;
7795 2068 : op->no_result = sync_and_optab;
7796 2068 : op->reverse_code = UNKNOWN;
7797 2068 : break;
7798 2496 : case IOR:
7799 2496 : op->mem_fetch_before = atomic_fetch_or_optab;
7800 2496 : op->mem_fetch_after = atomic_or_fetch_optab;
7801 2496 : op->mem_no_result = atomic_or_optab;
7802 2496 : op->fetch_before = sync_old_ior_optab;
7803 2496 : op->fetch_after = sync_new_ior_optab;
7804 2496 : op->no_result = sync_ior_optab;
7805 2496 : op->reverse_code = UNKNOWN;
7806 2496 : break;
7807 703 : case NOT:
7808 703 : op->mem_fetch_before = atomic_fetch_nand_optab;
7809 703 : op->mem_fetch_after = atomic_nand_fetch_optab;
7810 703 : op->mem_no_result = atomic_nand_optab;
7811 703 : op->fetch_before = sync_old_nand_optab;
7812 703 : op->fetch_after = sync_new_nand_optab;
7813 703 : op->no_result = sync_nand_optab;
7814 703 : op->reverse_code = UNKNOWN;
7815 703 : break;
7816 0 : default:
7817 0 : gcc_unreachable ();
7818 : }
7819 34908 : }
7820 :
7821 : /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7822 : using memory order MODEL. If AFTER is true the operation needs to return
7823 : the value of *MEM after the operation, otherwise the previous value.
7824 : TARGET is an optional place to place the result. The result is unused if
7825 : it is const0_rtx.
7826 : Return the result if there is a better sequence, otherwise NULL_RTX. */
7827 :
7828 : static rtx
7829 34906 : maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7830 : enum memmodel model, bool after)
7831 : {
7832 : /* If the value is prefetched, or not used, it may be possible to replace
7833 : the sequence with a native exchange operation. */
7834 34906 : if (!after || target == const0_rtx)
7835 : {
7836 : /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7837 26814 : if (code == AND && val == const0_rtx)
7838 : {
7839 327 : if (target == const0_rtx)
7840 150 : target = gen_reg_rtx (GET_MODE (mem));
7841 327 : return maybe_emit_atomic_exchange (target, mem, val, model);
7842 : }
7843 :
7844 : /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7845 26487 : if (code == IOR && val == constm1_rtx)
7846 : {
7847 165 : if (target == const0_rtx)
7848 164 : target = gen_reg_rtx (GET_MODE (mem));
7849 165 : return maybe_emit_atomic_exchange (target, mem, val, model);
7850 : }
7851 : }
7852 :
7853 : return NULL_RTX;
7854 : }
7855 :
7856 : /* Try to emit an instruction for a specific operation varaition.
7857 : OPTAB contains the OP functions.
7858 : TARGET is an optional place to return the result. const0_rtx means unused.
7859 : MEM is the memory location to operate on.
7860 : VAL is the value to use in the operation.
7861 : USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7862 : MODEL is the memory model, if used.
7863 : AFTER is true if the returned result is the value after the operation. */
7864 :
7865 : static rtx
7866 64346 : maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
7867 : rtx val, bool use_memmodel, enum memmodel model, bool after)
7868 : {
7869 64346 : machine_mode mode = GET_MODE (mem);
7870 64346 : class expand_operand ops[4];
7871 64346 : enum insn_code icode;
7872 64346 : int op_counter = 0;
7873 64346 : int num_ops;
7874 :
7875 : /* Check to see if there is a result returned. */
7876 64346 : if (target == const0_rtx)
7877 : {
7878 14624 : if (use_memmodel)
7879 : {
7880 14265 : icode = direct_optab_handler (optab->mem_no_result, mode);
7881 14265 : create_integer_operand (&ops[2], model);
7882 14265 : num_ops = 3;
7883 : }
7884 : else
7885 : {
7886 359 : icode = direct_optab_handler (optab->no_result, mode);
7887 359 : num_ops = 2;
7888 : }
7889 : }
7890 : /* Otherwise, we need to generate a result. */
7891 : else
7892 : {
7893 49722 : if (use_memmodel)
7894 : {
7895 31090 : icode = direct_optab_handler (after ? optab->mem_fetch_after
7896 : : optab->mem_fetch_before, mode);
7897 31090 : create_integer_operand (&ops[3], model);
7898 31090 : num_ops = 4;
7899 : }
7900 : else
7901 : {
7902 18632 : icode = optab_handler (after ? optab->fetch_after
7903 : : optab->fetch_before, mode);
7904 18632 : num_ops = 3;
7905 : }
7906 49722 : create_output_operand (&ops[op_counter++], target, mode);
7907 : }
7908 64346 : if (icode == CODE_FOR_nothing)
7909 : return NULL_RTX;
7910 :
7911 26364 : create_fixed_operand (&ops[op_counter++], mem);
7912 : /* VAL may have been promoted to a wider mode. Shrink it if so. */
7913 26364 : create_convert_operand_to (&ops[op_counter++], val, mode, true);
7914 :
7915 26364 : if (maybe_expand_insn (icode, num_ops, ops))
7916 26364 : return (target == const0_rtx ? const0_rtx : ops[0].value);
7917 :
7918 : return NULL_RTX;
7919 : }
7920 :
7921 :
7922 : /* This function expands an atomic fetch_OP or OP_fetch operation:
7923 : TARGET is an option place to stick the return value. const0_rtx indicates
7924 : the result is unused.
7925 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
7926 : CODE is the operation being performed (OP)
7927 : MEMMODEL is the memory model variant to use.
7928 : AFTER is true to return the result of the operation (OP_fetch).
7929 : AFTER is false to return the value before the operation (fetch_OP).
7930 :
7931 : This function will *only* generate instructions if there is a direct
7932 : optab. No compare and swap loops or libcalls will be generated. */
7933 :
7934 : static rtx
7935 34906 : expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
7936 : enum rtx_code code, enum memmodel model,
7937 : bool after)
7938 : {
7939 34906 : machine_mode mode = GET_MODE (mem);
7940 34906 : struct atomic_op_functions optab;
7941 34906 : rtx result;
7942 34906 : bool unused_result = (target == const0_rtx);
7943 :
7944 34906 : get_atomic_op_for_code (&optab, code);
7945 :
7946 : /* Check to see if there are any better instructions. */
7947 34906 : result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
7948 34906 : if (result)
7949 : return result;
7950 :
7951 : /* Check for the case where the result isn't used and try those patterns. */
7952 34414 : if (unused_result)
7953 : {
7954 : /* Try the memory model variant first. */
7955 14265 : result = maybe_emit_op (&optab, target, mem, val, true, model, true);
7956 14265 : if (result)
7957 : return result;
7958 :
7959 : /* Next try the old style withuot a memory model. */
7960 359 : result = maybe_emit_op (&optab, target, mem, val, false, model, true);
7961 359 : if (result)
7962 : return result;
7963 :
7964 : /* There is no no-result pattern, so try patterns with a result. */
7965 : target = NULL_RTX;
7966 : }
7967 :
7968 : /* Try the __atomic version. */
7969 20508 : result = maybe_emit_op (&optab, target, mem, val, true, model, after);
7970 20508 : if (result)
7971 : return result;
7972 :
7973 : /* Try the older __sync version. */
7974 11865 : result = maybe_emit_op (&optab, target, mem, val, false, model, after);
7975 11865 : if (result)
7976 : return result;
7977 :
7978 : /* If the fetch value can be calculated from the other variation of fetch,
7979 : try that operation. */
7980 11865 : if (after || unused_result || optab.reverse_code != UNKNOWN)
7981 : {
7982 : /* Try the __atomic version, then the older __sync version. */
7983 10582 : result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
7984 10582 : if (!result)
7985 6767 : result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
7986 :
7987 6767 : if (result)
7988 : {
7989 : /* If the result isn't used, no need to do compensation code. */
7990 3815 : if (unused_result)
7991 : return result;
7992 :
7993 : /* Issue compensation code. Fetch_after == fetch_before OP val.
7994 : Fetch_before == after REVERSE_OP val. */
7995 3815 : if (!after)
7996 0 : code = optab.reverse_code;
7997 3815 : if (code == NOT)
7998 : {
7999 0 : result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
8000 : true, OPTAB_LIB_WIDEN);
8001 0 : result = expand_simple_unop (mode, NOT, result, target, true);
8002 : }
8003 : else
8004 3815 : result = expand_simple_binop (mode, code, result, val, target,
8005 : true, OPTAB_LIB_WIDEN);
8006 3815 : return result;
8007 : }
8008 : }
8009 :
8010 : /* No direct opcode can be generated. */
8011 : return NULL_RTX;
8012 : }
8013 :
8014 :
8015 :
8016 : /* This function expands an atomic fetch_OP or OP_fetch operation:
8017 : TARGET is an option place to stick the return value. const0_rtx indicates
8018 : the result is unused.
8019 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
8020 : CODE is the operation being performed (OP)
8021 : MEMMODEL is the memory model variant to use.
8022 : AFTER is true to return the result of the operation (OP_fetch).
8023 : AFTER is false to return the value before the operation (fetch_OP). */
8024 : rtx
8025 32514 : expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
8026 : enum memmodel model, bool after)
8027 : {
8028 32514 : machine_mode mode = GET_MODE (mem);
8029 32514 : rtx result;
8030 32514 : bool unused_result = (target == const0_rtx);
8031 :
8032 : /* If loads are not atomic for the required size and we are not called to
8033 : provide a __sync builtin, do not do anything so that we stay consistent
8034 : with atomic loads of the same size. */
8035 32514 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
8036 : return NULL_RTX;
8037 :
8038 31915 : result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
8039 : after);
8040 :
8041 31915 : if (result)
8042 : return result;
8043 :
8044 : /* Add/sub can be implemented by doing the reverse operation with -(val). */
8045 8011 : if (code == PLUS || code == MINUS)
8046 : {
8047 2991 : rtx tmp;
8048 2991 : enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
8049 :
8050 2991 : start_sequence ();
8051 2991 : tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
8052 2991 : result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
8053 : model, after);
8054 2991 : if (result)
8055 : {
8056 : /* PLUS worked so emit the insns and return. */
8057 2952 : tmp = end_sequence ();
8058 2952 : emit_insn (tmp);
8059 2952 : return result;
8060 : }
8061 :
8062 : /* PLUS did not work, so throw away the negation code and continue. */
8063 39 : end_sequence ();
8064 : }
8065 :
8066 : /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
8067 5059 : if (!can_compare_and_swap_p (mode, false))
8068 : {
8069 2 : rtx libfunc;
8070 2 : bool fixup = false;
8071 2 : enum rtx_code orig_code = code;
8072 2 : struct atomic_op_functions optab;
8073 :
8074 2 : get_atomic_op_for_code (&optab, code);
8075 2 : libfunc = optab_libfunc (after ? optab.fetch_after
8076 : : optab.fetch_before, mode);
8077 2 : if (libfunc == NULL
8078 2 : && (after || unused_result || optab.reverse_code != UNKNOWN))
8079 : {
8080 2 : fixup = true;
8081 2 : if (!after)
8082 1 : code = optab.reverse_code;
8083 2 : libfunc = optab_libfunc (after ? optab.fetch_before
8084 : : optab.fetch_after, mode);
8085 : }
8086 2 : if (libfunc != NULL)
8087 : {
8088 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
8089 0 : result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
8090 : addr, ptr_mode, val, mode);
8091 :
8092 0 : if (!unused_result && fixup)
8093 0 : result = expand_simple_binop (mode, code, result, val, target,
8094 : true, OPTAB_LIB_WIDEN);
8095 0 : return result;
8096 : }
8097 :
8098 : /* We need the original code for any further attempts. */
8099 2 : code = orig_code;
8100 : }
8101 :
8102 : /* If nothing else has succeeded, default to a compare and swap loop. */
8103 5059 : if (can_compare_and_swap_p (mode, true))
8104 : {
8105 5057 : rtx_insn *insn;
8106 5057 : rtx t0 = gen_reg_rtx (mode), t1;
8107 :
8108 5057 : start_sequence ();
8109 :
8110 : /* If the result is used, get a register for it. */
8111 5057 : if (!unused_result)
8112 : {
8113 4721 : if (!target || !register_operand (target, mode))
8114 12 : target = gen_reg_rtx (mode);
8115 : /* If fetch_before, copy the value now. */
8116 4721 : if (!after)
8117 2068 : emit_move_insn (target, t0);
8118 : }
8119 : else
8120 336 : target = const0_rtx;
8121 :
8122 5057 : t1 = t0;
8123 5057 : if (code == NOT)
8124 : {
8125 687 : t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
8126 : true, OPTAB_LIB_WIDEN);
8127 687 : t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
8128 : }
8129 : else
8130 4370 : t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
8131 : OPTAB_LIB_WIDEN);
8132 :
8133 : /* For after, copy the value now. */
8134 5057 : if (!unused_result && after)
8135 2653 : emit_move_insn (target, t1);
8136 5057 : insn = end_sequence ();
8137 :
8138 5057 : if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
8139 : return target;
8140 : }
8141 :
8142 : return NULL_RTX;
8143 : }
8144 :
8145 : /* Return true if OPERAND is suitable for operand number OPNO of
8146 : instruction ICODE. */
8147 :
8148 : bool
8149 152114749 : insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
8150 : {
8151 152114749 : return (!insn_data[(int) icode].operand[opno].predicate
8152 302719825 : || (insn_data[(int) icode].operand[opno].predicate
8153 150605076 : (operand, insn_data[(int) icode].operand[opno].mode)));
8154 : }
8155 :
8156 : /* TARGET is a target of a multiword operation that we are going to
8157 : implement as a series of word-mode operations. Return true if
8158 : TARGET is suitable for this purpose. */
8159 :
8160 : bool
8161 188 : valid_multiword_target_p (rtx target)
8162 : {
8163 188 : machine_mode mode;
8164 188 : int i, size;
8165 :
8166 188 : mode = GET_MODE (target);
8167 376 : if (!GET_MODE_SIZE (mode).is_constant (&size))
8168 : return false;
8169 860 : for (i = 0; i < size; i += UNITS_PER_WORD)
8170 376 : if (!validate_subreg (word_mode, mode, target, i))
8171 : return false;
8172 : return true;
8173 : }
8174 :
8175 : /* Make OP describe an input operand that has value INTVAL and that has
8176 : no inherent mode. This function should only be used for operands that
8177 : are always expand-time constants. The backend may request that INTVAL
8178 : be copied into a different kind of rtx, but it must specify the mode
8179 : of that rtx if so. */
8180 :
8181 : void
8182 1961598 : create_integer_operand (class expand_operand *op, poly_int64 intval)
8183 : {
8184 1961598 : create_expand_operand (op, EXPAND_INTEGER,
8185 : gen_int_mode (intval, MAX_MODE_INT),
8186 : VOIDmode, false, intval);
8187 1961598 : }
8188 :
8189 : /* Like maybe_legitimize_operand, but do not change the code of the
8190 : current rtx value. */
8191 :
8192 : static bool
8193 50734181 : maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
8194 : class expand_operand *op)
8195 : {
8196 : /* See if the operand matches in its current form. */
8197 50734181 : if (insn_operand_matches (icode, opno, op->value))
8198 : return true;
8199 :
8200 : /* If the operand is a memory whose address has no side effects,
8201 : try forcing the address into a non-virtual pseudo register.
8202 : The check for side effects is important because copy_to_mode_reg
8203 : cannot handle things like auto-modified addresses. */
8204 1512474 : if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
8205 : {
8206 13094 : rtx addr, mem;
8207 :
8208 13094 : mem = op->value;
8209 13094 : addr = XEXP (mem, 0);
8210 984 : if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
8211 13900 : && !side_effects_p (addr))
8212 : {
8213 8491 : rtx_insn *last;
8214 8491 : machine_mode mode;
8215 :
8216 8491 : last = get_last_insn ();
8217 8491 : mode = get_address_mode (mem);
8218 8491 : mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
8219 8491 : if (insn_operand_matches (icode, opno, mem))
8220 : {
8221 0 : op->value = mem;
8222 0 : return true;
8223 : }
8224 8491 : delete_insns_since (last);
8225 : }
8226 : }
8227 :
8228 : return false;
8229 : }
8230 :
8231 : /* Try to make OP match operand OPNO of instruction ICODE. Return true
8232 : on success, storing the new operand value back in OP. */
8233 :
8234 : static bool
8235 58290215 : maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
8236 : class expand_operand *op)
8237 : {
8238 58290215 : machine_mode mode, imode, tmode;
8239 :
8240 58290215 : mode = op->mode;
8241 58290215 : switch (op->type)
8242 : {
8243 4216606 : case EXPAND_FIXED:
8244 4216606 : {
8245 4216606 : temporary_volatile_ok v (true);
8246 4216606 : return maybe_legitimize_operand_same_code (icode, opno, op);
8247 4216606 : }
8248 :
8249 18813717 : case EXPAND_OUTPUT:
8250 18813717 : gcc_assert (mode != VOIDmode);
8251 18813717 : if (op->value
8252 11650507 : && op->value != const0_rtx
8253 11650061 : && GET_MODE (op->value) == mode
8254 30428076 : && maybe_legitimize_operand_same_code (icode, opno, op))
8255 : return true;
8256 :
8257 7320267 : op->value = gen_reg_rtx (mode);
8258 7320267 : op->target = 0;
8259 7320267 : break;
8260 :
8261 34903216 : case EXPAND_INPUT:
8262 34903216 : input:
8263 34903216 : gcc_assert (mode != VOIDmode);
8264 34903216 : gcc_assert (GET_MODE (op->value) == VOIDmode
8265 : || GET_MODE (op->value) == mode);
8266 34903216 : if (maybe_legitimize_operand_same_code (icode, opno, op))
8267 : return true;
8268 :
8269 1247146 : op->value = copy_to_mode_reg (mode, op->value);
8270 1247146 : break;
8271 :
8272 197217 : case EXPAND_CONVERT_TO:
8273 197217 : gcc_assert (mode != VOIDmode);
8274 197217 : op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8275 197217 : goto input;
8276 :
8277 603857 : case EXPAND_CONVERT_FROM:
8278 603857 : if (GET_MODE (op->value) != VOIDmode)
8279 397878 : mode = GET_MODE (op->value);
8280 : else
8281 : /* The caller must tell us what mode this value has. */
8282 205979 : gcc_assert (mode != VOIDmode);
8283 :
8284 603857 : imode = insn_data[(int) icode].operand[opno].mode;
8285 603857 : tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
8286 603857 : ? GET_MODE_INNER (imode) : imode);
8287 603857 : if (tmode != VOIDmode && tmode != mode)
8288 : {
8289 1512 : op->value = convert_modes (tmode, mode, op->value, op->unsigned_p);
8290 1512 : mode = tmode;
8291 : }
8292 603857 : if (imode != VOIDmode && imode != mode)
8293 : {
8294 58 : gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
8295 58 : op->value = expand_vector_broadcast (imode, op->value);
8296 58 : mode = imode;
8297 : }
8298 603857 : goto input;
8299 :
8300 3345 : case EXPAND_ADDRESS:
8301 3345 : op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
8302 : op->value);
8303 3345 : goto input;
8304 :
8305 1611229 : case EXPAND_INTEGER:
8306 1611229 : mode = insn_data[(int) icode].operand[opno].mode;
8307 1611229 : if (mode != VOIDmode
8308 1611229 : && known_eq (trunc_int_for_mode (op->int_value, mode),
8309 : op->int_value))
8310 : {
8311 1254553 : op->value = gen_int_mode (op->int_value, mode);
8312 1254553 : goto input;
8313 : }
8314 : break;
8315 :
8316 249 : case EXPAND_UNDEFINED_INPUT:
8317 : /* See if the predicate accepts a SCRATCH rtx, which in this context
8318 : indicates an undefined value. Use an uninitialized register if not. */
8319 249 : if (!insn_operand_matches (icode, opno, op->value))
8320 : {
8321 249 : op->value = gen_reg_rtx (op->mode);
8322 249 : goto input;
8323 : }
8324 : return true;
8325 : }
8326 8924089 : return insn_operand_matches (icode, opno, op->value);
8327 : }
8328 :
8329 : /* Make OP describe an input operand that should have the same value
8330 : as VALUE, after any mode conversion that the target might request.
8331 : TYPE is the type of VALUE. */
8332 :
8333 : void
8334 2518 : create_convert_operand_from_type (class expand_operand *op,
8335 : rtx value, tree type)
8336 : {
8337 2518 : create_convert_operand_from (op, value, TYPE_MODE (type),
8338 2518 : TYPE_UNSIGNED (type));
8339 2518 : }
8340 :
8341 : /* Return true if the requirements on operands OP1 and OP2 of instruction
8342 : ICODE are similar enough for the result of legitimizing OP1 to be
8343 : reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
8344 : with OP1 and OP2 respectively. */
8345 :
8346 : static inline bool
8347 64575358 : can_reuse_operands_p (enum insn_code icode,
8348 : unsigned int opno1, unsigned int opno2,
8349 : const class expand_operand *op1,
8350 : const class expand_operand *op2)
8351 : {
8352 : /* Check requirements that are common to all types. */
8353 64575358 : if (op1->type != op2->type
8354 20021863 : || op1->mode != op2->mode
8355 18537358 : || (insn_data[(int) icode].operand[opno1].mode
8356 18537358 : != insn_data[(int) icode].operand[opno2].mode))
8357 : return false;
8358 :
8359 : /* Check the requirements for specific types. */
8360 16754863 : switch (op1->type)
8361 : {
8362 : case EXPAND_OUTPUT:
8363 : case EXPAND_UNDEFINED_INPUT:
8364 : /* Outputs and undefined intputs must remain distinct. */
8365 : return false;
8366 :
8367 : case EXPAND_FIXED:
8368 : case EXPAND_INPUT:
8369 : case EXPAND_ADDRESS:
8370 : case EXPAND_INTEGER:
8371 : return true;
8372 :
8373 99616 : case EXPAND_CONVERT_TO:
8374 99616 : case EXPAND_CONVERT_FROM:
8375 99616 : return op1->unsigned_p == op2->unsigned_p;
8376 : }
8377 0 : gcc_unreachable ();
8378 : }
8379 :
8380 : /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8381 : of instruction ICODE. Return true on success, leaving the new operand
8382 : values in the OPS themselves. Emit no code on failure. */
8383 :
8384 : bool
8385 19532820 : maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8386 : unsigned int nops, class expand_operand *ops)
8387 : {
8388 19532820 : rtx_insn *last = get_last_insn ();
8389 19532820 : rtx *orig_values = XALLOCAVEC (rtx, nops);
8390 78039892 : for (unsigned int i = 0; i < nops; i++)
8391 : {
8392 58659229 : orig_values[i] = ops[i].value;
8393 :
8394 : /* First try reusing the result of an earlier legitimization.
8395 : This avoids duplicate rtl and ensures that tied operands
8396 : remain tied.
8397 :
8398 : This search is linear, but NOPS is bounded at compile time
8399 : to a small number (current a single digit). */
8400 58659229 : unsigned int j = 0;
8401 122865573 : for (; j < i; ++j)
8402 64575358 : if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
8403 16735060 : && rtx_equal_p (orig_values[j], orig_values[i])
8404 383763 : && ops[j].value
8405 64944376 : && insn_operand_matches (icode, opno + i, ops[j].value))
8406 : {
8407 369014 : ops[i].value = copy_rtx (ops[j].value);
8408 369014 : break;
8409 : }
8410 :
8411 : /* Otherwise try legitimizing the operand on its own. */
8412 58659229 : if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
8413 : {
8414 152157 : delete_insns_since (last);
8415 152157 : return false;
8416 : }
8417 : }
8418 : return true;
8419 : }
8420 :
8421 : /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8422 : as its operands. Return the instruction pattern on success,
8423 : and emit any necessary set-up code. Return null and emit no
8424 : code on failure. */
8425 :
8426 : rtx_insn *
8427 19510063 : maybe_gen_insn (enum insn_code icode, unsigned int nops,
8428 : class expand_operand *ops)
8429 : {
8430 19510063 : gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8431 19510063 : if (!maybe_legitimize_operands (icode, 0, nops, ops))
8432 : return NULL;
8433 :
8434 19357906 : switch (nops)
8435 : {
8436 0 : case 0:
8437 0 : return GEN_FCN (icode) ();
8438 1309 : case 1:
8439 1309 : return GEN_FCN (icode) (ops[0].value);
8440 2848639 : case 2:
8441 2848639 : return GEN_FCN (icode) (ops[0].value, ops[1].value);
8442 14546942 : case 3:
8443 14546942 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8444 1581921 : case 4:
8445 1581921 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8446 1581921 : ops[3].value);
8447 189379 : case 5:
8448 189379 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8449 189379 : ops[3].value, ops[4].value);
8450 262 : case 6:
8451 262 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8452 262 : ops[3].value, ops[4].value, ops[5].value);
8453 0 : case 7:
8454 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8455 0 : ops[3].value, ops[4].value, ops[5].value,
8456 0 : ops[6].value);
8457 26723 : case 8:
8458 26723 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8459 26723 : ops[3].value, ops[4].value, ops[5].value,
8460 26723 : ops[6].value, ops[7].value);
8461 162731 : case 9:
8462 162731 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8463 162731 : ops[3].value, ops[4].value, ops[5].value,
8464 162731 : ops[6].value, ops[7].value, ops[8].value);
8465 0 : case 10:
8466 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8467 0 : ops[3].value, ops[4].value, ops[5].value,
8468 0 : ops[6].value, ops[7].value, ops[8].value,
8469 0 : ops[9].value);
8470 0 : case 11:
8471 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8472 0 : ops[3].value, ops[4].value, ops[5].value,
8473 0 : ops[6].value, ops[7].value, ops[8].value,
8474 0 : ops[9].value, ops[10].value);
8475 : }
8476 0 : gcc_unreachable ();
8477 : }
8478 :
8479 : /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8480 : as its operands. Return true on success and emit no code on failure. */
8481 :
8482 : bool
8483 2862429 : maybe_expand_insn (enum insn_code icode, unsigned int nops,
8484 : class expand_operand *ops)
8485 : {
8486 2862429 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8487 2862429 : if (pat)
8488 : {
8489 2070310 : emit_insn (pat);
8490 2070310 : return true;
8491 : }
8492 : return false;
8493 : }
8494 :
8495 : /* Like maybe_expand_insn, but for jumps. */
8496 :
8497 : bool
8498 1306 : maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8499 : class expand_operand *ops)
8500 : {
8501 1306 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8502 1306 : if (pat)
8503 : {
8504 1306 : emit_jump_insn (pat);
8505 1306 : return true;
8506 : }
8507 : return false;
8508 : }
8509 :
8510 : /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8511 : as its operands. */
8512 :
8513 : void
8514 628996 : expand_insn (enum insn_code icode, unsigned int nops,
8515 : class expand_operand *ops)
8516 : {
8517 628996 : if (!maybe_expand_insn (icode, nops, ops))
8518 0 : gcc_unreachable ();
8519 628996 : }
8520 :
8521 : /* Like expand_insn, but for jumps. */
8522 :
8523 : void
8524 1306 : expand_jump_insn (enum insn_code icode, unsigned int nops,
8525 : class expand_operand *ops)
8526 : {
8527 1306 : if (!maybe_expand_jump_insn (icode, nops, ops))
8528 0 : gcc_unreachable ();
8529 1306 : }
|
