Line data Source code
1 : /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "predict.h"
30 : #include "tm_p.h"
31 : #include "optabs.h"
32 : #include "expmed.h"
33 : #include "emit-rtl.h"
34 : #include "recog.h"
35 : #include "diagnostic-core.h"
36 : #include "rtx-vector-builder.h"
37 :
38 : /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 : is properly defined. */
40 : #include "stor-layout.h"
41 : #include "except.h"
42 : #include "dojump.h"
43 : #include "explow.h"
44 : #include "expr.h"
45 : #include "optabs-tree.h"
46 : #include "libfuncs.h"
47 : #include "internal-fn.h"
48 : #include "langhooks.h"
49 : #include "gimple.h"
50 : #include "ssa.h"
51 : #include "tree-ssa-live.h"
52 : #include "tree-outof-ssa.h"
53 :
54 : static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
55 : machine_mode *);
56 : static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
57 : static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
58 :
59 : static rtx emit_conditional_move_1 (rtx, rtx, rtx, rtx, machine_mode);
60 :
61 : /* Debug facility for use in GDB. */
62 : void debug_optab_libfuncs (void);
63 :
64 : /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
65 : the result of operation CODE applied to OP0 (and OP1 if it is a binary
66 : operation). OP0_MODE is OP0's mode.
67 :
68 : If the last insn does not set TARGET, don't do anything, but return true.
69 :
70 : If the last insn or a previous insn sets TARGET and TARGET is one of OP0
71 : or OP1, don't add the REG_EQUAL note but return false. Our caller can then
72 : try again, ensuring that TARGET is not one of the operands. */
73 :
74 : static bool
75 1176855 : add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
76 : rtx op1, machine_mode op0_mode)
77 : {
78 1176855 : rtx_insn *last_insn;
79 1176855 : rtx set;
80 1176855 : rtx note;
81 :
82 2353710 : gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
83 :
84 1176855 : if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
85 : && GET_RTX_CLASS (code) != RTX_BIN_ARITH
86 : && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
87 : && GET_RTX_CLASS (code) != RTX_COMPARE
88 : && GET_RTX_CLASS (code) != RTX_UNARY)
89 : return true;
90 :
91 1169700 : if (GET_CODE (target) == ZERO_EXTRACT)
92 : return true;
93 :
94 : for (last_insn = insns;
95 2542505 : NEXT_INSN (last_insn) != NULL_RTX;
96 : last_insn = NEXT_INSN (last_insn))
97 : ;
98 :
99 : /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
100 : a value changing in the insn, so the note would be invalid for CSE. */
101 1169700 : if (reg_overlap_mentioned_p (target, op0)
102 1169700 : || (op1 && reg_overlap_mentioned_p (target, op1)))
103 : {
104 49251 : if (MEM_P (target)
105 49251 : && (rtx_equal_p (target, op0)
106 41872 : || (op1 && rtx_equal_p (target, op1))))
107 : {
108 : /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
109 : over expanding it as temp = MEM op X, MEM = temp. If the target
110 : supports MEM = MEM op X instructions, it is sometimes too hard
111 : to reconstruct that form later, especially if X is also a memory,
112 : and due to multiple occurrences of addresses the address might
113 : be forced into register unnecessarily.
114 : Note that not emitting the REG_EQUIV note might inhibit
115 : CSE in some cases. */
116 3333 : set = single_set (last_insn);
117 3333 : if (set
118 3333 : && GET_CODE (SET_SRC (set)) == code
119 3322 : && MEM_P (SET_DEST (set))
120 6655 : && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
121 0 : || (op1 && rtx_equal_p (SET_DEST (set),
122 0 : XEXP (SET_SRC (set), 1)))))
123 3322 : return true;
124 : }
125 45929 : return false;
126 : }
127 :
128 1120449 : set = set_for_reg_notes (last_insn);
129 1120449 : if (set == NULL_RTX)
130 : return true;
131 :
132 671424 : if (! rtx_equal_p (SET_DEST (set), target)
133 : /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
134 671424 : && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
135 0 : || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
136 387 : return true;
137 :
138 671037 : if (GET_RTX_CLASS (code) == RTX_UNARY)
139 40080 : switch (code)
140 : {
141 3052 : case FFS:
142 3052 : case CLZ:
143 3052 : case CTZ:
144 3052 : case CLRSB:
145 3052 : case POPCOUNT:
146 3052 : case PARITY:
147 3052 : case BSWAP:
148 3052 : if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
149 : {
150 252 : note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
151 252 : if (GET_MODE_UNIT_SIZE (op0_mode)
152 504 : > GET_MODE_UNIT_SIZE (GET_MODE (target)))
153 252 : note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
154 : note, op0_mode);
155 : else
156 0 : note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
157 : note, op0_mode);
158 : break;
159 : }
160 : /* FALLTHRU */
161 39828 : default:
162 39828 : note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
163 39828 : break;
164 : }
165 : else
166 630957 : note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
167 :
168 671037 : set_unique_reg_note (last_insn, REG_EQUAL, note);
169 :
170 671037 : return true;
171 : }
172 :
173 : /* Given two input operands, OP0 and OP1, determine what the correct from_mode
174 : for a widening operation would be. In most cases this would be OP0, but if
175 : that's a constant it'll be VOIDmode, which isn't useful. */
176 :
177 : static machine_mode
178 31558 : widened_mode (machine_mode to_mode, rtx op0, rtx op1)
179 : {
180 31558 : machine_mode m0 = GET_MODE (op0);
181 31558 : machine_mode m1 = GET_MODE (op1);
182 31558 : machine_mode result;
183 :
184 31558 : if (m0 == VOIDmode && m1 == VOIDmode)
185 : return to_mode;
186 63116 : else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
187 : result = m1;
188 : else
189 : result = m0;
190 :
191 94674 : if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
192 0 : return to_mode;
193 :
194 : return result;
195 : }
196 :
197 : /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
198 : says whether OP is signed or unsigned. NO_EXTEND is true if we need
199 : not actually do a sign-extend or zero-extend, but can leave the
200 : higher-order bits of the result rtx undefined, for example, in the case
201 : of logical operations, but not right shifts. */
202 :
203 : static rtx
204 539 : widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
205 : int unsignedp, bool no_extend)
206 : {
207 539 : rtx result;
208 539 : scalar_int_mode int_mode;
209 :
210 : /* If we don't have to extend and this is a constant, return it. */
211 539 : if (no_extend && GET_MODE (op) == VOIDmode)
212 : return op;
213 :
214 : /* If we must extend do so. If OP is a SUBREG for a promoted object, also
215 : extend since it will be more efficient to do so unless the signedness of
216 : a promoted object differs from our extension. */
217 7 : if (! no_extend
218 539 : || !is_a <scalar_int_mode> (mode, &int_mode)
219 7 : || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
220 0 : && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
221 532 : return convert_modes (mode, oldmode, op, unsignedp);
222 :
223 : /* If MODE is no wider than a single word, we return a lowpart or paradoxical
224 : SUBREG. */
225 14 : if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
226 7 : return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
227 :
228 : /* Otherwise, get an object of MODE, clobber it, and set the low-order
229 : part to OP. */
230 :
231 0 : result = gen_reg_rtx (int_mode);
232 0 : emit_clobber (result);
233 0 : emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
234 0 : return result;
235 : }
236 :
237 : /* Expand vector widening operations.
238 :
239 : There are two different classes of operations handled here:
240 : 1) Operations whose result is wider than all the arguments to the operation.
241 : Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
242 : In this case OP0 and optionally OP1 would be initialized,
243 : but WIDE_OP wouldn't (not relevant for this case).
244 : 2) Operations whose result is of the same size as the last argument to the
245 : operation, but wider than all the other arguments to the operation.
246 : Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
247 : In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
248 :
249 : E.g, when called to expand the following operations, this is how
250 : the arguments will be initialized:
251 : nops OP0 OP1 WIDE_OP
252 : widening-sum 2 oprnd0 - oprnd1
253 : widening-dot-product 3 oprnd0 oprnd1 oprnd2
254 : widening-mult 2 oprnd0 oprnd1 -
255 : type-promotion (vec-unpack) 1 oprnd0 - - */
256 :
257 : rtx
258 20979 : expand_widen_pattern_expr (const_sepops ops, rtx op0, rtx op1, rtx wide_op,
259 : rtx target, int unsignedp)
260 : {
261 20979 : class expand_operand eops[4];
262 20979 : tree oprnd0, oprnd1, oprnd2;
263 20979 : machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
264 20979 : optab widen_pattern_optab;
265 20979 : enum insn_code icode;
266 20979 : int nops = TREE_CODE_LENGTH (ops->code);
267 20979 : int op;
268 20979 : bool sbool = false;
269 :
270 20979 : oprnd0 = ops->op0;
271 20979 : oprnd1 = nops >= 2 ? ops->op1 : NULL_TREE;
272 1525 : oprnd2 = nops >= 3 ? ops->op2 : NULL_TREE;
273 :
274 20979 : tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
275 20979 : if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
276 20979 : || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
277 : /* The sign is from the result type rather than operand's type
278 : for these ops. */
279 34 : widen_pattern_optab
280 34 : = optab_for_tree_code (ops->code, ops->type, optab_default);
281 20945 : else if ((ops->code == VEC_UNPACK_HI_EXPR
282 20945 : || ops->code == VEC_UNPACK_LO_EXPR)
283 17690 : && VECTOR_BOOLEAN_TYPE_P (ops->type)
284 4534 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
285 4534 : && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
286 21429 : && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
287 : {
288 : /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
289 : the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
290 : vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
291 : the pattern number of elements in the wider vector. */
292 484 : widen_pattern_optab
293 239 : = (ops->code == VEC_UNPACK_HI_EXPR
294 484 : ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
295 : sbool = true;
296 : }
297 20461 : else if (ops->code == DOT_PROD_EXPR)
298 : {
299 419 : enum optab_subtype subtype = optab_default;
300 419 : signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0));
301 419 : signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1));
302 419 : if (sign1 == sign2)
303 : ;
304 79 : else if (sign1 == SIGNED && sign2 == UNSIGNED)
305 : {
306 : subtype = optab_vector_mixed_sign;
307 : /* Same as optab_vector_mixed_sign but flip the operands. */
308 : std::swap (op0, op1);
309 : }
310 60 : else if (sign1 == UNSIGNED && sign2 == SIGNED)
311 : subtype = optab_vector_mixed_sign;
312 : else
313 0 : gcc_unreachable ();
314 :
315 419 : widen_pattern_optab
316 419 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype);
317 : }
318 : else
319 20042 : widen_pattern_optab
320 20042 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
321 20979 : if (ops->code == WIDEN_MULT_PLUS_EXPR
322 20979 : || ops->code == WIDEN_MULT_MINUS_EXPR
323 20979 : || ops->code == DOT_PROD_EXPR)
324 419 : icode = find_widening_optab_handler (widen_pattern_optab,
325 : TYPE_MODE (TREE_TYPE (ops->op2)),
326 : tmode0);
327 20560 : else if (ops->code == WIDEN_SUM_EXPR)
328 0 : icode = find_widening_optab_handler (widen_pattern_optab,
329 : TYPE_MODE (TREE_TYPE (ops->op1)),
330 : tmode0);
331 : else
332 20560 : icode = optab_handler (widen_pattern_optab, tmode0);
333 20979 : gcc_assert (icode != CODE_FOR_nothing);
334 :
335 20979 : if (nops >= 2)
336 1525 : tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
337 19454 : else if (sbool)
338 : {
339 484 : nops = 2;
340 484 : op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
341 484 : tmode1 = tmode0;
342 : }
343 :
344 : /* The last operand is of a wider mode than the rest of the operands. */
345 20979 : if (nops == 2)
346 : wmode = tmode1;
347 19502 : else if (nops == 3)
348 : {
349 532 : gcc_assert (tmode1 == tmode0);
350 532 : gcc_assert (op1);
351 532 : wmode = TYPE_MODE (TREE_TYPE (oprnd2));
352 : }
353 :
354 20979 : op = 0;
355 20979 : create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
356 20979 : create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
357 20979 : if (op1)
358 2009 : create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
359 20979 : if (wide_op)
360 532 : create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
361 20979 : expand_insn (icode, op, eops);
362 20979 : return eops[0].value;
363 : }
364 :
365 : /* Generate code to perform an operation specified by TERNARY_OPTAB
366 : on operands OP0, OP1 and OP2, with result having machine-mode MODE.
367 :
368 : UNSIGNEDP is for the case where we have to widen the operands
369 : to perform the operation. It says to use zero-extension.
370 :
371 : If TARGET is nonzero, the value
372 : is generated there, if it is convenient to do so.
373 : In all cases an rtx is returned for the locus of the value;
374 : this may or may not be TARGET. */
375 :
376 : rtx
377 0 : expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
378 : rtx op1, rtx op2, rtx target, int unsignedp)
379 : {
380 0 : class expand_operand ops[4];
381 0 : enum insn_code icode = optab_handler (ternary_optab, mode);
382 :
383 0 : gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
384 :
385 0 : create_output_operand (&ops[0], target, mode);
386 0 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
387 0 : create_convert_operand_from (&ops[2], op1, mode, unsignedp);
388 0 : create_convert_operand_from (&ops[3], op2, mode, unsignedp);
389 0 : expand_insn (icode, 4, ops);
390 0 : return ops[0].value;
391 : }
392 :
393 :
394 : /* Like expand_binop, but return a constant rtx if the result can be
395 : calculated at compile time. The arguments and return value are
396 : otherwise the same as for expand_binop. */
397 :
398 : rtx
399 1113 : simplify_expand_binop (machine_mode mode, optab binoptab,
400 : rtx op0, rtx op1, rtx target, int unsignedp,
401 : enum optab_methods methods)
402 : {
403 1113 : if (CONSTANT_P (op0) && CONSTANT_P (op1))
404 : {
405 0 : rtx x = simplify_binary_operation (optab_to_code (binoptab),
406 : mode, op0, op1);
407 0 : if (x)
408 : return x;
409 : }
410 :
411 1113 : return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
412 : }
413 :
414 : /* Like simplify_expand_binop, but always put the result in TARGET.
415 : Return true if the expansion succeeded. */
416 :
417 : bool
418 1113 : force_expand_binop (machine_mode mode, optab binoptab,
419 : rtx op0, rtx op1, rtx target, int unsignedp,
420 : enum optab_methods methods)
421 : {
422 1113 : rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
423 : target, unsignedp, methods);
424 1113 : if (x == 0)
425 : return false;
426 1113 : if (x != target)
427 0 : emit_move_insn (target, x);
428 : return true;
429 : }
430 :
431 : /* Create a new vector value in VMODE with all elements set to OP. If OP
432 : is not a constant, the mode of it must be the element mode of VMODE
433 : (if the element is BImode, additionally OP is allowed to be in QImode).
434 : If OP is a constant, then the return value will be a constant. */
435 :
436 : rtx
437 58 : expand_vector_broadcast (machine_mode vmode, rtx op)
438 : {
439 58 : int n;
440 58 : rtvec vec;
441 :
442 58 : gcc_checking_assert (VECTOR_MODE_P (vmode));
443 58 : gcc_checking_assert (CONST_INT_P (op)
444 : || GET_MODE_INNER (vmode) == GET_MODE (op)
445 : || (GET_MODE_INNER (vmode) == BImode
446 : && GET_MODE (op) == QImode));
447 :
448 58 : if (valid_for_const_vector_p (vmode, op))
449 58 : return gen_const_vec_duplicate (vmode, op);
450 :
451 0 : insn_code icode = optab_handler (vec_duplicate_optab, vmode);
452 0 : if (icode != CODE_FOR_nothing)
453 : {
454 0 : class expand_operand ops[2];
455 0 : create_output_operand (&ops[0], NULL_RTX, vmode);
456 0 : create_input_operand (&ops[1], op, GET_MODE (op));
457 0 : expand_insn (icode, 2, ops);
458 0 : return ops[0].value;
459 : }
460 :
461 0 : if (!GET_MODE_NUNITS (vmode).is_constant (&n))
462 : return NULL;
463 :
464 : /* ??? If the target doesn't have a vec_init, then we have no easy way
465 : of performing this operation. Most of this sort of generic support
466 : is hidden away in the vector lowering support in gimple. */
467 0 : icode = convert_optab_handler (vec_init_optab, vmode,
468 0 : GET_MODE_INNER (vmode));
469 0 : if (icode == CODE_FOR_nothing)
470 : return NULL;
471 :
472 0 : vec = rtvec_alloc (n);
473 0 : for (int i = 0; i < n; ++i)
474 0 : RTVEC_ELT (vec, i) = op;
475 0 : rtx ret = gen_reg_rtx (vmode);
476 0 : emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
477 :
478 0 : return ret;
479 : }
480 :
481 : /* This subroutine of expand_doubleword_shift handles the cases in which
482 : the effective shift value is >= BITS_PER_WORD. The arguments and return
483 : value are the same as for the parent routine, except that SUPERWORD_OP1
484 : is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
485 : INTO_TARGET may be null if the caller has decided to calculate it. */
486 :
487 : static bool
488 0 : expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
489 : rtx outof_target, rtx into_target,
490 : int unsignedp, enum optab_methods methods)
491 : {
492 0 : if (into_target != 0)
493 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
494 : into_target, unsignedp, methods))
495 : return false;
496 :
497 0 : if (outof_target != 0)
498 : {
499 : /* For a signed right shift, we must fill OUTOF_TARGET with copies
500 : of the sign bit, otherwise we must fill it with zeros. */
501 0 : if (binoptab != ashr_optab)
502 0 : emit_move_insn (outof_target, CONST0_RTX (word_mode));
503 : else
504 0 : if (!force_expand_binop (word_mode, binoptab, outof_input,
505 : gen_int_shift_amount (word_mode,
506 0 : BITS_PER_WORD - 1),
507 : outof_target, unsignedp, methods))
508 : return false;
509 : }
510 : return true;
511 : }
512 :
513 : /* This subroutine of expand_doubleword_shift handles the cases in which
514 : the effective shift value is < BITS_PER_WORD. The arguments and return
515 : value are the same as for the parent routine. */
516 :
517 : static bool
518 0 : expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
519 : rtx outof_input, rtx into_input, rtx op1,
520 : rtx outof_target, rtx into_target,
521 : int unsignedp, enum optab_methods methods,
522 : unsigned HOST_WIDE_INT shift_mask)
523 : {
524 0 : optab reverse_unsigned_shift, unsigned_shift;
525 0 : rtx tmp, carries;
526 :
527 0 : reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
528 0 : unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
529 :
530 : /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
531 : We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
532 : the opposite direction to BINOPTAB. */
533 0 : if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
534 : {
535 0 : carries = outof_input;
536 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
537 : op1_mode), op1_mode);
538 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
539 : 0, true, methods);
540 : }
541 : else
542 : {
543 : /* We must avoid shifting by BITS_PER_WORD bits since that is either
544 : the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
545 : has unknown behavior. Do a single shift first, then shift by the
546 : remainder. It's OK to use ~OP1 as the remainder if shift counts
547 : are truncated to the mode size. */
548 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
549 : outof_input, const1_rtx, 0,
550 : unsignedp, methods);
551 0 : if (carries == const0_rtx)
552 : tmp = const0_rtx;
553 0 : else if (shift_mask == BITS_PER_WORD - 1)
554 0 : tmp = expand_unop (op1_mode, one_cmpl_optab, op1, 0, true);
555 : else
556 : {
557 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
558 : op1_mode), op1_mode);
559 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
560 : 0, true, methods);
561 : }
562 : }
563 0 : if (tmp == 0 || carries == 0)
564 : return false;
565 0 : if (carries != const0_rtx && tmp != const0_rtx)
566 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
567 : carries, tmp, 0, unsignedp, methods);
568 0 : if (carries == 0)
569 : return false;
570 :
571 0 : if (into_input != const0_rtx)
572 : {
573 : /* Shift INTO_INPUT logically by OP1. This is the last use of
574 : INTO_INPUT so the result can go directly into INTO_TARGET if
575 : convenient. */
576 0 : tmp = simplify_expand_binop (word_mode, unsigned_shift, into_input,
577 : op1, into_target, unsignedp, methods);
578 0 : if (tmp == 0)
579 : return false;
580 :
581 : /* Now OR/PLUS in the bits carried over from OUTOF_INPUT. */
582 0 : if (!force_expand_binop (word_mode, add_optab, tmp, carries,
583 : into_target, unsignedp, methods))
584 : return false;
585 : }
586 : else
587 0 : emit_move_insn (into_target, carries);
588 :
589 : /* Use a standard word_mode shift for the out-of half. */
590 0 : if (outof_target != 0)
591 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
592 : outof_target, unsignedp, methods))
593 : return false;
594 :
595 : return true;
596 : }
597 :
598 :
599 : /* Try implementing expand_doubleword_shift using conditional moves.
600 : The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
601 : otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
602 : are the shift counts to use in the former and latter case. All other
603 : arguments are the same as the parent routine. */
604 :
605 : static bool
606 0 : expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
607 : enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
608 : rtx outof_input, rtx into_input,
609 : rtx subword_op1, rtx superword_op1,
610 : rtx outof_target, rtx into_target,
611 : int unsignedp, enum optab_methods methods,
612 : unsigned HOST_WIDE_INT shift_mask)
613 : {
614 0 : rtx outof_superword, into_superword;
615 :
616 : /* Put the superword version of the output into OUTOF_SUPERWORD and
617 : INTO_SUPERWORD. */
618 0 : outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
619 0 : if (outof_target != 0 && subword_op1 == superword_op1)
620 : {
621 : /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
622 : OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
623 0 : into_superword = outof_target;
624 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
625 : outof_superword, 0, unsignedp, methods))
626 : return false;
627 : }
628 : else
629 : {
630 0 : into_superword = gen_reg_rtx (word_mode);
631 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
632 : outof_superword, into_superword,
633 : unsignedp, methods))
634 : return false;
635 : }
636 :
637 : /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
638 0 : if (!expand_subword_shift (op1_mode, binoptab,
639 : outof_input, into_input, subword_op1,
640 : outof_target, into_target,
641 : unsignedp, methods, shift_mask))
642 : return false;
643 :
644 : /* Select between them. Do the INTO half first because INTO_SUPERWORD
645 : might be the current value of OUTOF_TARGET. */
646 0 : if (!emit_conditional_move (into_target, { cmp_code, cmp1, cmp2, op1_mode },
647 : into_target, into_superword, word_mode, false))
648 : return false;
649 :
650 0 : if (outof_target != 0)
651 0 : if (!emit_conditional_move (outof_target,
652 0 : { cmp_code, cmp1, cmp2, op1_mode },
653 : outof_target, outof_superword,
654 : word_mode, false))
655 : return false;
656 :
657 : return true;
658 : }
659 :
660 : /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
661 : OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
662 : input operand; the shift moves bits in the direction OUTOF_INPUT->
663 : INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
664 : of the target. OP1 is the shift count and OP1_MODE is its mode.
665 : If OP1 is constant, it will have been truncated as appropriate
666 : and is known to be nonzero.
667 :
668 : If SHIFT_MASK is zero, the result of word shifts is undefined when the
669 : shift count is outside the range [0, BITS_PER_WORD). This routine must
670 : avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
671 :
672 : If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
673 : masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
674 : fill with zeros or sign bits as appropriate.
675 :
676 : If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
677 : a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
678 : Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
679 : In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
680 : are undefined.
681 :
682 : BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
683 : may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
684 : OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
685 : function wants to calculate it itself.
686 :
687 : Return true if the shift could be successfully synthesized. */
688 :
689 : static bool
690 0 : expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
691 : rtx outof_input, rtx into_input, rtx op1,
692 : rtx outof_target, rtx into_target,
693 : int unsignedp, enum optab_methods methods,
694 : unsigned HOST_WIDE_INT shift_mask)
695 : {
696 0 : rtx superword_op1, tmp, cmp1, cmp2;
697 0 : enum rtx_code cmp_code;
698 :
699 : /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
700 : fill the result with sign or zero bits as appropriate. If so, the value
701 : of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
702 : this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
703 : and INTO_INPUT), then emit code to set up OUTOF_TARGET.
704 :
705 : This isn't worthwhile for constant shifts since the optimizers will
706 : cope better with in-range shift counts. */
707 0 : if (shift_mask >= BITS_PER_WORD
708 0 : && outof_target != 0
709 0 : && !CONSTANT_P (op1))
710 : {
711 0 : if (!expand_doubleword_shift (op1_mode, binoptab,
712 : outof_input, into_input, op1,
713 : 0, into_target,
714 : unsignedp, methods, shift_mask))
715 : return false;
716 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
717 : outof_target, unsignedp, methods))
718 : return false;
719 : return true;
720 : }
721 :
722 : /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
723 : is true when the effective shift value is less than BITS_PER_WORD.
724 : Set SUPERWORD_OP1 to the shift count that should be used to shift
725 : OUTOF_INPUT into INTO_TARGET when the condition is false. */
726 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
727 0 : if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
728 : {
729 : /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
730 : is a subword shift count. */
731 0 : cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
732 : 0, true, methods);
733 0 : cmp2 = CONST0_RTX (op1_mode);
734 0 : cmp_code = EQ;
735 0 : superword_op1 = op1;
736 : }
737 : else
738 : {
739 : /* Set CMP1 to OP1 - BITS_PER_WORD. */
740 0 : cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
741 : 0, true, methods);
742 0 : cmp2 = CONST0_RTX (op1_mode);
743 0 : cmp_code = LT;
744 0 : superword_op1 = cmp1;
745 : }
746 0 : if (cmp1 == 0)
747 : return false;
748 :
749 : /* If we can compute the condition at compile time, pick the
750 : appropriate subroutine. */
751 0 : tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
752 0 : if (tmp != 0 && CONST_INT_P (tmp))
753 : {
754 0 : if (tmp == const0_rtx)
755 0 : return expand_superword_shift (binoptab, outof_input, superword_op1,
756 : outof_target, into_target,
757 0 : unsignedp, methods);
758 : else
759 0 : return expand_subword_shift (op1_mode, binoptab,
760 : outof_input, into_input, op1,
761 : outof_target, into_target,
762 0 : unsignedp, methods, shift_mask);
763 : }
764 :
765 : /* Try using conditional moves to generate straight-line code. */
766 0 : if (HAVE_conditional_move)
767 : {
768 0 : rtx_insn *start = get_last_insn ();
769 0 : if (expand_doubleword_shift_condmove (op1_mode, binoptab,
770 : cmp_code, cmp1, cmp2,
771 : outof_input, into_input,
772 : op1, superword_op1,
773 : outof_target, into_target,
774 : unsignedp, methods, shift_mask))
775 : return true;
776 0 : delete_insns_since (start);
777 : }
778 :
779 : /* As a last resort, use branches to select the correct alternative. */
780 0 : rtx_code_label *subword_label = gen_label_rtx ();
781 0 : rtx_code_label *done_label = gen_label_rtx ();
782 :
783 0 : NO_DEFER_POP;
784 0 : do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
785 : 0, 0, subword_label,
786 : profile_probability::uninitialized ());
787 0 : OK_DEFER_POP;
788 :
789 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
790 : outof_target, into_target,
791 : unsignedp, methods))
792 : return false;
793 :
794 0 : emit_jump_insn (targetm.gen_jump (done_label));
795 0 : emit_barrier ();
796 0 : emit_label (subword_label);
797 :
798 0 : if (!expand_subword_shift (op1_mode, binoptab,
799 : outof_input, into_input, op1,
800 : outof_target, into_target,
801 : unsignedp, methods, shift_mask))
802 : return false;
803 :
804 0 : emit_label (done_label);
805 0 : return true;
806 : }
807 :
808 : /* Subroutine of expand_binop. Perform a double word multiplication of
809 : operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
810 : as the target's word_mode. This function return NULL_RTX if anything
811 : goes wrong, in which case it may have already emitted instructions
812 : which need to be deleted.
813 :
814 : If we want to multiply two two-word values and have normal and widening
815 : multiplies of single-word values, we can do this with three smaller
816 : multiplications.
817 :
818 : The multiplication proceeds as follows:
819 : _______________________
820 : [__op0_high_|__op0_low__]
821 : _______________________
822 : * [__op1_high_|__op1_low__]
823 : _______________________________________________
824 : _______________________
825 : (1) [__op0_low__*__op1_low__]
826 : _______________________
827 : (2a) [__op0_low__*__op1_high_]
828 : _______________________
829 : (2b) [__op0_high_*__op1_low__]
830 : _______________________
831 : (3) [__op0_high_*__op1_high_]
832 :
833 :
834 : This gives a 4-word result. Since we are only interested in the
835 : lower 2 words, partial result (3) and the upper words of (2a) and
836 : (2b) don't need to be calculated. Hence (2a) and (2b) can be
837 : calculated using non-widening multiplication.
838 :
839 : (1), however, needs to be calculated with an unsigned widening
840 : multiplication. If this operation is not directly supported we
841 : try using a signed widening multiplication and adjust the result.
842 : This adjustment works as follows:
843 :
844 : If both operands are positive then no adjustment is needed.
845 :
846 : If the operands have different signs, for example op0_low < 0 and
847 : op1_low >= 0, the instruction treats the most significant bit of
848 : op0_low as a sign bit instead of a bit with significance
849 : 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
850 : with 2**BITS_PER_WORD - op0_low, and two's complements the
851 : result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
852 : the result.
853 :
854 : Similarly, if both operands are negative, we need to add
855 : (op0_low + op1_low) * 2**BITS_PER_WORD.
856 :
857 : We use a trick to adjust quickly. We logically shift op0_low right
858 : (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
859 : op0_high (op1_high) before it is used to calculate 2b (2a). If no
860 : logical shift exists, we do an arithmetic right shift and subtract
861 : the 0 or -1. */
862 :
863 : static rtx
864 14218 : expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
865 : bool umulp, enum optab_methods methods)
866 : {
867 14218 : int low = (WORDS_BIG_ENDIAN ? 1 : 0);
868 14218 : int high = (WORDS_BIG_ENDIAN ? 0 : 1);
869 14218 : rtx wordm1 = (umulp ? NULL_RTX
870 0 : : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
871 14218 : rtx product, adjust, product_high, temp;
872 :
873 14218 : rtx op0_high = operand_subword_force (op0, high, mode);
874 14218 : rtx op0_low = operand_subword_force (op0, low, mode);
875 14218 : rtx op1_high = operand_subword_force (op1, high, mode);
876 14218 : rtx op1_low = operand_subword_force (op1, low, mode);
877 :
878 : /* If we're using an unsigned multiply to directly compute the product
879 : of the low-order words of the operands and perform any required
880 : adjustments of the operands, we begin by trying two more multiplications
881 : and then computing the appropriate sum.
882 :
883 : We have checked above that the required addition is provided.
884 : Full-word addition will normally always succeed, especially if
885 : it is provided at all, so we don't worry about its failure. The
886 : multiplication may well fail, however, so we do handle that. */
887 :
888 14218 : if (!umulp)
889 : {
890 : /* ??? This could be done with emit_store_flag where available. */
891 0 : temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
892 : NULL_RTX, 1, methods);
893 0 : if (temp)
894 0 : op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
895 : NULL_RTX, 0, OPTAB_DIRECT);
896 : else
897 : {
898 0 : temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
899 : NULL_RTX, 0, methods);
900 0 : if (!temp)
901 : return NULL_RTX;
902 0 : op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
903 : NULL_RTX, 0, OPTAB_DIRECT);
904 : }
905 :
906 0 : if (!op0_high)
907 : return NULL_RTX;
908 : }
909 :
910 14218 : if (op1_low == const1_rtx)
911 : adjust = op0_high;
912 14178 : else if (op1_low == const0_rtx)
913 : adjust = const0_rtx;
914 14103 : else if (op1_low == const2_rtx)
915 0 : adjust = expand_binop (word_mode, add_optab, op0_high, op0_high,
916 : NULL_RTX, 0, OPTAB_DIRECT);
917 : else
918 14103 : adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
919 : NULL_RTX, 0, OPTAB_DIRECT);
920 14218 : if (!adjust)
921 : return NULL_RTX;
922 :
923 : /* OP0_HIGH should now be dead. */
924 :
925 14218 : if (!umulp)
926 : {
927 : /* ??? This could be done with emit_store_flag where available. */
928 0 : temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
929 : NULL_RTX, 1, methods);
930 0 : if (temp)
931 0 : op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
932 : NULL_RTX, 0, OPTAB_DIRECT);
933 : else
934 : {
935 0 : temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
936 : NULL_RTX, 0, methods);
937 0 : if (!temp)
938 : return NULL_RTX;
939 0 : op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
940 : NULL_RTX, 0, OPTAB_DIRECT);
941 : }
942 :
943 0 : if (!op1_high)
944 : return NULL_RTX;
945 : }
946 :
947 14218 : if (op1_high == const1_rtx)
948 : temp = op0_low;
949 14217 : else if (op1_high == const0_rtx)
950 : temp = const0_rtx;
951 13446 : else if (op1_high == const2_rtx)
952 0 : temp = expand_binop (word_mode, add_optab, op0_low, op0_low,
953 : NULL_RTX, 0, OPTAB_DIRECT);
954 : else
955 13446 : temp = expand_binop (word_mode, smul_optab, op0_low, op1_high,
956 : NULL_RTX, 0, OPTAB_DIRECT);
957 14218 : if (!temp)
958 : return NULL_RTX;
959 :
960 : /* OP1_HIGH should now be dead. */
961 :
962 14218 : adjust = expand_binop (word_mode, add_optab, adjust, temp,
963 : NULL_RTX, 0, OPTAB_DIRECT);
964 :
965 14218 : if (target && !REG_P (target))
966 8135 : target = NULL_RTX;
967 :
968 : /* *_widen_optab needs to determine operand mode, make sure at least
969 : one operand has non-VOID mode. */
970 14218 : if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
971 0 : op0_low = force_reg (word_mode, op0_low);
972 :
973 14218 : if (op1_low == const1_rtx)
974 40 : product = convert_modes (mode, word_mode, op0_low, umulp);
975 14178 : else if (umulp)
976 14178 : product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
977 : target, 1, OPTAB_DIRECT);
978 : else
979 0 : product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
980 : target, 1, OPTAB_DIRECT);
981 :
982 14218 : if (!product)
983 : return NULL_RTX;
984 :
985 14218 : product_high = operand_subword (product, high, 1, mode);
986 14218 : adjust = expand_binop (word_mode, add_optab, product_high, adjust,
987 : NULL_RTX, 0, OPTAB_DIRECT);
988 14218 : emit_move_insn (product_high, adjust);
989 14218 : return product;
990 : }
991 :
992 : /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
993 : constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
994 : (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
995 : computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
996 : + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
997 : depends on the bit value, if 2, then carry from the addition needs to be
998 : added too, i.e. like:
999 : sum += __builtin_add_overflow (low, high, &sum)
1000 :
1001 : Optimize signed double-word OP0 % OP1 similarly, just apply some correction
1002 : factor to the sum before doing unsigned remainder, in the form of
1003 : sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
1004 : then perform unsigned
1005 : remainder = sum % OP1;
1006 : and finally
1007 : remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
1008 :
1009 : static rtx
1010 20765 : expand_doubleword_mod (machine_mode mode, rtx op0, rtx op1, bool unsignedp)
1011 : {
1012 20765 : if (INTVAL (op1) <= 1 || (INTVAL (op1) & 1) == 0)
1013 : return NULL_RTX;
1014 :
1015 20444 : rtx_insn *last = get_last_insn ();
1016 694416 : for (int bit = BITS_PER_WORD; bit >= BITS_PER_WORD / 2; bit--)
1017 : {
1018 652827 : wide_int w = wi::shifted_mask (bit, 1, false, 2 * BITS_PER_WORD);
1019 622686 : if (wi::ne_p (wi::umod_trunc (w, INTVAL (op1)), 1))
1020 621401 : continue;
1021 1285 : rtx sum = NULL_RTX, mask = NULL_RTX;
1022 1753 : if (bit == BITS_PER_WORD)
1023 : {
1024 : /* For signed modulo we need to add correction to the sum
1025 : and that might again overflow. */
1026 363 : if (!unsignedp)
1027 137 : continue;
1028 226 : if (optab_handler (uaddv4_optab, word_mode) == CODE_FOR_nothing)
1029 0 : continue;
1030 226 : tree wtype = lang_hooks.types.type_for_mode (word_mode, 1);
1031 226 : if (wtype == NULL_TREE)
1032 0 : continue;
1033 226 : tree ctype = build_complex_type (wtype);
1034 226 : if (TYPE_MODE (ctype) != GET_MODE_COMPLEX_MODE (word_mode))
1035 0 : continue;
1036 226 : machine_mode cmode = TYPE_MODE (ctype);
1037 226 : rtx op00 = operand_subword_force (op0, 0, mode);
1038 226 : rtx op01 = operand_subword_force (op0, 1, mode);
1039 226 : rtx cres = gen_rtx_CONCAT (cmode, gen_reg_rtx (word_mode),
1040 : gen_reg_rtx (word_mode));
1041 226 : tree lhs = make_tree (ctype, cres);
1042 226 : tree arg0 = make_tree (wtype, op00);
1043 226 : tree arg1 = make_tree (wtype, op01);
1044 226 : expand_addsub_overflow (UNKNOWN_LOCATION, PLUS_EXPR, lhs, arg0,
1045 : arg1, true, true, true, false, NULL);
1046 226 : sum = expand_simple_binop (word_mode, PLUS, XEXP (cres, 0),
1047 : XEXP (cres, 1), NULL_RTX, 1,
1048 : OPTAB_DIRECT);
1049 226 : if (sum == NULL_RTX)
1050 : return NULL_RTX;
1051 : }
1052 : else
1053 : {
1054 : /* Code below uses GEN_INT, so we need the masks to be representable
1055 : in HOST_WIDE_INTs. */
1056 922 : if (bit >= HOST_BITS_PER_WIDE_INT)
1057 0 : continue;
1058 : /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1059 : overflow. Consider 64-bit -1ULL for word size 32, if we add
1060 : 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1061 1150 : if (bit == BITS_PER_WORD - 1)
1062 147 : continue;
1063 :
1064 775 : int count = (2 * BITS_PER_WORD + bit - 1) / bit;
1065 775 : rtx sum_corr = NULL_RTX;
1066 :
1067 775 : if (!unsignedp)
1068 : {
1069 : /* For signed modulo, compute it as unsigned modulo of
1070 : sum with a correction added to it if OP0 is negative,
1071 : such that the result can be computed as unsigned
1072 : remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1073 498 : w = wi::min_value (2 * BITS_PER_WORD, SIGNED);
1074 407 : wide_int wmod1 = wi::umod_trunc (w, INTVAL (op1));
1075 407 : wide_int wmod2 = wi::smod_trunc (w, INTVAL (op1));
1076 : /* wmod2 == -wmod1. */
1077 407 : wmod2 = wmod2 + (INTVAL (op1) - 1);
1078 407 : if (wi::ne_p (wmod1, wmod2))
1079 : {
1080 407 : wide_int wcorr = wmod2 - wmod1;
1081 407 : if (wi::neg_p (w))
1082 407 : wcorr = wcorr + INTVAL (op1);
1083 : /* Now verify if the count sums can't overflow, and punt
1084 : if they could. */
1085 498 : w = wi::mask (bit, false, 2 * BITS_PER_WORD);
1086 407 : w = w * (count - 1);
1087 905 : w = w + wi::mask (2 * BITS_PER_WORD - (count - 1) * bit,
1088 814 : false, 2 * BITS_PER_WORD);
1089 407 : w = w + wcorr;
1090 498 : w = wi::lrshift (w, BITS_PER_WORD);
1091 407 : if (wi::ne_p (w, 0))
1092 0 : continue;
1093 :
1094 407 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1095 : mode);
1096 407 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1097 407 : GEN_INT (BITS_PER_WORD - 1),
1098 : NULL_RTX, 0, OPTAB_DIRECT);
1099 407 : if (mask == NULL_RTX)
1100 : return NULL_RTX;
1101 407 : sum_corr = immed_wide_int_const (wcorr, word_mode);
1102 407 : sum_corr = expand_simple_binop (word_mode, AND, mask,
1103 : sum_corr, NULL_RTX, 1,
1104 : OPTAB_DIRECT);
1105 407 : if (sum_corr == NULL_RTX)
1106 : return NULL_RTX;
1107 407 : }
1108 407 : }
1109 :
1110 3274 : for (int i = 0; i < count; i++)
1111 : {
1112 2499 : rtx v = op0;
1113 2499 : if (i)
1114 1724 : v = expand_simple_binop (mode, LSHIFTRT, v, GEN_INT (i * bit),
1115 : NULL_RTX, 1, OPTAB_DIRECT);
1116 2499 : if (v == NULL_RTX)
1117 : return NULL_RTX;
1118 2499 : v = force_lowpart_subreg (word_mode, v, mode);
1119 2499 : if (v == NULL_RTX)
1120 : return NULL_RTX;
1121 2499 : if (i != count - 1)
1122 1724 : v = expand_simple_binop (word_mode, AND, v,
1123 1724 : GEN_INT ((HOST_WIDE_INT_1U << bit)
1124 : - 1), NULL_RTX, 1,
1125 : OPTAB_DIRECT);
1126 2499 : if (v == NULL_RTX)
1127 : return NULL_RTX;
1128 2499 : if (sum == NULL_RTX)
1129 : sum = v;
1130 : else
1131 1724 : sum = expand_simple_binop (word_mode, PLUS, sum, v, NULL_RTX,
1132 : 1, OPTAB_DIRECT);
1133 2499 : if (sum == NULL_RTX)
1134 : return NULL_RTX;
1135 : }
1136 775 : if (sum_corr)
1137 : {
1138 407 : sum = expand_simple_binop (word_mode, PLUS, sum, sum_corr,
1139 : NULL_RTX, 1, OPTAB_DIRECT);
1140 407 : if (sum == NULL_RTX)
1141 : return NULL_RTX;
1142 : }
1143 : }
1144 1001 : rtx remainder = expand_divmod (1, TRUNC_MOD_EXPR, word_mode, sum,
1145 1001 : gen_int_mode (INTVAL (op1), word_mode),
1146 : NULL_RTX, 1, OPTAB_DIRECT);
1147 1001 : if (remainder == NULL_RTX)
1148 : return NULL_RTX;
1149 :
1150 1001 : if (!unsignedp)
1151 : {
1152 407 : if (mask == NULL_RTX)
1153 : {
1154 0 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1155 : mode);
1156 0 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1157 0 : GEN_INT (BITS_PER_WORD - 1),
1158 : NULL_RTX, 0, OPTAB_DIRECT);
1159 0 : if (mask == NULL_RTX)
1160 : return NULL_RTX;
1161 : }
1162 407 : mask = expand_simple_binop (word_mode, AND, mask,
1163 407 : gen_int_mode (1 - INTVAL (op1),
1164 : word_mode),
1165 : NULL_RTX, 1, OPTAB_DIRECT);
1166 407 : if (mask == NULL_RTX)
1167 : return NULL_RTX;
1168 407 : remainder = expand_simple_binop (word_mode, PLUS, remainder,
1169 : mask, NULL_RTX, 1, OPTAB_DIRECT);
1170 407 : if (remainder == NULL_RTX)
1171 : return NULL_RTX;
1172 : }
1173 :
1174 1001 : remainder = convert_modes (mode, word_mode, remainder, unsignedp);
1175 : /* Punt if we need any library calls. */
1176 1001 : if (last)
1177 905 : last = NEXT_INSN (last);
1178 : else
1179 96 : last = get_insns ();
1180 20545 : for (; last; last = NEXT_INSN (last))
1181 19544 : if (CALL_P (last))
1182 : return NULL_RTX;
1183 : return remainder;
1184 622686 : }
1185 : return NULL_RTX;
1186 : }
1187 :
1188 : /* Similarly to the above function, but compute both quotient and remainder.
1189 : Quotient can be computed from the remainder as:
1190 : rem = op0 % op1; // Handled using expand_doubleword_mod
1191 : quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1192 : // 2 * BITS_PER_WORD
1193 :
1194 : We can also handle cases where op1 is a multiple of power of two constant
1195 : and constant handled by expand_doubleword_mod.
1196 : op11 = 1 << __builtin_ctz (op1);
1197 : op12 = op1 / op11;
1198 : rem1 = op0 % op12; // Handled using expand_doubleword_mod
1199 : quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1200 : // 2 * BITS_PER_WORD
1201 : rem = (quot1 % op11) * op12 + rem1;
1202 : quot = quot1 / op11; */
1203 :
1204 : rtx
1205 20493 : expand_doubleword_divmod (machine_mode mode, rtx op0, rtx op1, rtx *rem,
1206 : bool unsignedp)
1207 : {
1208 20493 : *rem = NULL_RTX;
1209 :
1210 : /* Negative dividend should have been optimized into positive,
1211 : similarly modulo by 1 and modulo by power of two is optimized
1212 : differently too. */
1213 39936 : if (INTVAL (op1) <= 1 || pow2p_hwi (INTVAL (op1)))
1214 : return NULL_RTX;
1215 :
1216 20444 : rtx op11 = const1_rtx;
1217 20444 : rtx op12 = op1;
1218 20444 : if ((INTVAL (op1) & 1) == 0)
1219 : {
1220 3428 : int bit = ctz_hwi (INTVAL (op1));
1221 3428 : op11 = GEN_INT (HOST_WIDE_INT_1 << bit);
1222 3428 : op12 = GEN_INT (INTVAL (op1) >> bit);
1223 : }
1224 :
1225 20444 : rtx rem1 = expand_doubleword_mod (mode, op0, op12, unsignedp);
1226 20444 : if (rem1 == NULL_RTX)
1227 : return NULL_RTX;
1228 :
1229 1001 : int prec = 2 * BITS_PER_WORD;
1230 1001 : wide_int a = wide_int::from (INTVAL (op12), prec + 1, UNSIGNED);
1231 1001 : wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
1232 1001 : wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
1233 1001 : rtx inv = immed_wide_int_const (m, mode);
1234 :
1235 1001 : rtx_insn *last = get_last_insn ();
1236 1001 : rtx quot1 = expand_simple_binop (mode, MINUS, op0, rem1,
1237 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1238 1001 : if (quot1 == NULL_RTX)
1239 : return NULL_RTX;
1240 :
1241 1001 : quot1 = expand_simple_binop (mode, MULT, quot1, inv,
1242 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1243 1001 : if (quot1 == NULL_RTX)
1244 : return NULL_RTX;
1245 :
1246 1001 : if (op11 != const1_rtx)
1247 : {
1248 340 : rtx rem2 = expand_divmod (1, TRUNC_MOD_EXPR, mode, quot1, op11,
1249 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1250 340 : if (rem2 == NULL_RTX)
1251 : return NULL_RTX;
1252 :
1253 340 : rem2 = expand_simple_binop (mode, MULT, rem2, op12, NULL_RTX,
1254 : unsignedp, OPTAB_DIRECT);
1255 340 : if (rem2 == NULL_RTX)
1256 : return NULL_RTX;
1257 :
1258 340 : rem2 = expand_simple_binop (mode, PLUS, rem2, rem1, NULL_RTX,
1259 : unsignedp, OPTAB_DIRECT);
1260 340 : if (rem2 == NULL_RTX)
1261 : return NULL_RTX;
1262 :
1263 340 : rtx quot2 = expand_divmod (0, TRUNC_DIV_EXPR, mode, quot1, op11,
1264 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1265 340 : if (quot2 == NULL_RTX)
1266 : return NULL_RTX;
1267 :
1268 : rem1 = rem2;
1269 : quot1 = quot2;
1270 : }
1271 :
1272 : /* Punt if we need any library calls. */
1273 1001 : if (last)
1274 1001 : last = NEXT_INSN (last);
1275 : else
1276 0 : last = get_insns ();
1277 15453 : for (; last; last = NEXT_INSN (last))
1278 14452 : if (CALL_P (last))
1279 : return NULL_RTX;
1280 :
1281 1001 : *rem = rem1;
1282 1001 : return quot1;
1283 1001 : }
1284 :
1285 : /* Wrapper around expand_binop which takes an rtx code to specify
1286 : the operation to perform, not an optab pointer. All other
1287 : arguments are the same. */
1288 : rtx
1289 4017373 : expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
1290 : rtx op1, rtx target, int unsignedp,
1291 : enum optab_methods methods)
1292 : {
1293 4017373 : optab binop = code_to_optab (code);
1294 4017373 : gcc_assert (binop);
1295 :
1296 4017373 : return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1297 : }
1298 :
1299 : /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1300 : binop. Order them according to commutative_operand_precedence and, if
1301 : possible, try to put TARGET or a pseudo first. */
1302 : static bool
1303 12133172 : swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1304 : {
1305 12133172 : int op0_prec = commutative_operand_precedence (op0);
1306 12133172 : int op1_prec = commutative_operand_precedence (op1);
1307 :
1308 12133172 : if (op0_prec < op1_prec)
1309 : return true;
1310 :
1311 11944767 : if (op0_prec > op1_prec)
1312 : return false;
1313 :
1314 : /* With equal precedence, both orders are ok, but it is better if the
1315 : first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1316 2217016 : if (target == 0 || REG_P (target))
1317 2479309 : return (REG_P (op1) && !REG_P (op0)) || target == op1;
1318 : else
1319 87354 : return rtx_equal_p (op1, target);
1320 : }
1321 :
1322 : /* Return true if BINOPTAB implements a shift operation. */
1323 :
1324 : static bool
1325 23811932 : shift_optab_p (optab binoptab)
1326 : {
1327 9536315 : switch (optab_to_code (binoptab))
1328 : {
1329 : case ASHIFT:
1330 : case SS_ASHIFT:
1331 : case US_ASHIFT:
1332 : case ASHIFTRT:
1333 : case LSHIFTRT:
1334 : case ROTATE:
1335 : case ROTATERT:
1336 : return true;
1337 :
1338 12739710 : default:
1339 0 : return false;
1340 : }
1341 : }
1342 :
1343 : /* Return true if BINOPTAB implements a commutative binary operation. */
1344 :
1345 : static bool
1346 14434205 : commutative_optab_p (optab binoptab)
1347 : {
1348 14434205 : return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1349 : || binoptab == smul_widen_optab
1350 2382862 : || binoptab == umul_widen_optab
1351 : || binoptab == smul_highpart_optab
1352 2351304 : || binoptab == umul_highpart_optab
1353 2301033 : || binoptab == vec_widen_sadd_optab
1354 2301033 : || binoptab == vec_widen_uadd_optab
1355 : || binoptab == vec_widen_sadd_hi_optab
1356 2301033 : || binoptab == vec_widen_sadd_lo_optab
1357 : || binoptab == vec_widen_uadd_hi_optab
1358 2301033 : || binoptab == vec_widen_uadd_lo_optab
1359 : || binoptab == vec_widen_sadd_even_optab
1360 2301033 : || binoptab == vec_widen_sadd_odd_optab
1361 : || binoptab == vec_widen_uadd_even_optab
1362 16735238 : || binoptab == vec_widen_uadd_odd_optab);
1363 : }
1364 :
1365 : /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1366 : optimizing, and if the operand is a constant that costs more than
1367 : 1 instruction, force the constant into a register and return that
1368 : register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1369 :
1370 : static rtx
1371 27178628 : avoid_expensive_constant (machine_mode mode, optab binoptab,
1372 : int opn, rtx x, bool unsignedp)
1373 : {
1374 27178628 : bool speed = optimize_insn_for_speed_p ();
1375 :
1376 27178628 : if (mode != VOIDmode
1377 27178628 : && optimize
1378 20415464 : && CONSTANT_P (x)
1379 34234624 : && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1380 7055996 : > set_src_cost (x, mode, speed)))
1381 : {
1382 0 : if (CONST_INT_P (x))
1383 : {
1384 0 : HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1385 0 : if (intval != INTVAL (x))
1386 0 : x = GEN_INT (intval);
1387 : }
1388 : else
1389 0 : x = convert_modes (mode, VOIDmode, x, unsignedp);
1390 0 : x = force_reg (mode, x);
1391 : }
1392 27178628 : return x;
1393 : }
1394 :
1395 : /* Helper function for expand_binop: handle the case where there
1396 : is an insn ICODE that directly implements the indicated operation.
1397 : Returns null if this is not possible. */
1398 : static rtx
1399 14245157 : expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1400 : rtx op0, rtx op1,
1401 : rtx target, int unsignedp, enum optab_methods methods)
1402 : {
1403 14245157 : machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1404 14245157 : machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1405 14245157 : machine_mode mode0, mode1, tmp_mode;
1406 14245157 : class expand_operand ops[3];
1407 14245157 : bool commutative_p;
1408 14245157 : rtx_insn *pat;
1409 14245157 : rtx xop0 = op0, xop1 = op1;
1410 14245157 : bool canonicalize_op1 = false;
1411 14245157 : rtx_insn *last = get_last_insn ();
1412 :
1413 : /* If it is a commutative operator and the modes would match
1414 : if we would swap the operands, we can save the conversions. */
1415 14245157 : commutative_p = commutative_optab_p (binoptab);
1416 14245157 : if (commutative_p
1417 12073301 : && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1418 13233 : && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0)
1419 : std::swap (xop0, xop1);
1420 :
1421 : /* If we are optimizing, force expensive constants into a register. */
1422 14245157 : xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1423 14245157 : if (!shift_optab_p (binoptab))
1424 12739710 : xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1425 : else
1426 : /* Shifts and rotates often use a different mode for op1 from op0;
1427 : for VOIDmode constants we don't know the mode, so force it
1428 : to be canonicalized using convert_modes. */
1429 : canonicalize_op1 = true;
1430 :
1431 : /* In case the insn wants input operands in modes different from
1432 : those of the actual operands, convert the operands. It would
1433 : seem that we don't need to convert CONST_INTs, but we do, so
1434 : that they're properly zero-extended, sign-extended or truncated
1435 : for their mode. */
1436 :
1437 14245157 : mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1438 14245157 : if (xmode0 != VOIDmode && xmode0 != mode0)
1439 : {
1440 3011 : xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1441 3011 : mode0 = xmode0;
1442 : }
1443 :
1444 14245157 : mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1445 14245157 : ? GET_MODE (xop1) : mode);
1446 14245157 : if (xmode1 != VOIDmode && xmode1 != mode1)
1447 : {
1448 1561386 : xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1449 1561386 : mode1 = xmode1;
1450 : }
1451 :
1452 : /* If operation is commutative,
1453 : try to make the first operand a register.
1454 : Even better, try to make it the same as the target.
1455 : Also try to make the last operand a constant. */
1456 14245157 : if (commutative_p
1457 14245157 : && swap_commutative_operands_with_target (target, xop0, xop1))
1458 : std::swap (xop0, xop1);
1459 :
1460 : /* Now, if insn's predicates don't allow our operands, put them into
1461 : pseudo regs. */
1462 :
1463 14245157 : if (binoptab == vec_pack_trunc_optab
1464 14245157 : || binoptab == vec_pack_usat_optab
1465 14234900 : || binoptab == vec_pack_ssat_optab
1466 14234900 : || binoptab == vec_pack_ufix_trunc_optab
1467 14234762 : || binoptab == vec_pack_sfix_trunc_optab
1468 14234762 : || binoptab == vec_packu_float_optab
1469 14234542 : || binoptab == vec_packs_float_optab)
1470 : {
1471 : /* The mode of the result is different then the mode of the
1472 : arguments. */
1473 10635 : tmp_mode = insn_data[(int) icode].operand[0].mode;
1474 2598 : if (VECTOR_MODE_P (mode)
1475 30321 : && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1476 : return NULL_RTX;
1477 : }
1478 : else
1479 : tmp_mode = mode;
1480 :
1481 14245157 : create_output_operand (&ops[0], target, tmp_mode);
1482 14245157 : create_input_operand (&ops[1], xop0, mode0);
1483 14245157 : create_input_operand (&ops[2], xop1, mode1);
1484 14245157 : pat = maybe_gen_insn (icode, 3, ops);
1485 14245157 : if (pat)
1486 : {
1487 : /* If PAT is composed of more than one insn, try to add an appropriate
1488 : REG_EQUAL note to it. If we can't because TEMP conflicts with an
1489 : operand, call expand_binop again, this time without a target. */
1490 13151715 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1491 15371713 : && ! add_equal_note (pat, ops[0].value,
1492 : optab_to_code (binoptab),
1493 : ops[1].value, ops[2].value, mode0))
1494 : {
1495 44831 : delete_insns_since (last);
1496 44831 : return expand_binop_directly (icode, mode, binoptab, op0, op1,
1497 44831 : NULL_RTX, unsignedp, methods);
1498 : }
1499 :
1500 14200324 : emit_insn (pat);
1501 14200324 : return ops[0].value;
1502 : }
1503 :
1504 : return NULL_RTX;
1505 : }
1506 :
1507 : /* Generate code to perform an operation specified by BINOPTAB
1508 : on operands OP0 and OP1, with result having machine-mode MODE.
1509 :
1510 : UNSIGNEDP is for the case where we have to widen the operands
1511 : to perform the operation. It says to use zero-extension.
1512 :
1513 : If TARGET is nonzero, the value
1514 : is generated there, if it is convenient to do so.
1515 : In all cases an rtx is returned for the locus of the value;
1516 : this may or may not be TARGET. */
1517 :
1518 : rtx
1519 14389536 : expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1520 : rtx target, int unsignedp, enum optab_methods methods)
1521 : {
1522 12400358 : enum optab_methods next_methods
1523 14389536 : = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1524 14389536 : ? OPTAB_WIDEN : methods);
1525 14389536 : enum mode_class mclass;
1526 14389536 : enum insn_code icode;
1527 14389536 : machine_mode wider_mode;
1528 14389536 : scalar_int_mode int_mode;
1529 14389536 : rtx libfunc;
1530 14389536 : rtx temp;
1531 14389536 : rtx_insn *entry_last = get_last_insn ();
1532 14389536 : rtx_insn *last;
1533 :
1534 14389536 : mclass = GET_MODE_CLASS (mode);
1535 :
1536 : /* If subtracting an integer constant, convert this into an addition of
1537 : the negated constant. */
1538 :
1539 14389536 : if (binoptab == sub_optab && CONST_INT_P (op1))
1540 : {
1541 1035583 : op1 = negate_rtx (mode, op1);
1542 1035583 : binoptab = add_optab;
1543 : }
1544 : /* For shifts, constant invalid op1 might be expanded from different
1545 : mode than MODE. As those are invalid, force them to a register
1546 : to avoid further problems during expansion. */
1547 13353953 : else if (CONST_INT_P (op1)
1548 15836703 : && shift_optab_p (binoptab)
1549 17695454 : && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1550 : {
1551 138 : op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1552 138 : op1 = force_reg (GET_MODE_INNER (mode), op1);
1553 : }
1554 :
1555 : /* Record where to delete back to if we backtrack. */
1556 14389536 : last = get_last_insn ();
1557 :
1558 : /* If we can do it with a three-operand insn, do so. */
1559 :
1560 14389536 : if (methods != OPTAB_MUST_WIDEN)
1561 : {
1562 14389536 : if (convert_optab_p (binoptab))
1563 : {
1564 31558 : machine_mode from_mode = widened_mode (mode, op0, op1);
1565 31558 : icode = find_widening_optab_handler (binoptab, mode, from_mode);
1566 : }
1567 : else
1568 14357978 : icode = optab_handler (binoptab, mode);
1569 14389536 : if (icode != CODE_FOR_nothing)
1570 : {
1571 14200325 : temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1572 : target, unsignedp, methods);
1573 14200325 : if (temp)
1574 : return temp;
1575 1 : delete_insns_since (last);
1576 : }
1577 : }
1578 :
1579 : /* If we were trying to rotate, and that didn't work, try rotating
1580 : the other direction before falling back to shifts and bitwise-or. */
1581 189212 : if (((binoptab == rotl_optab
1582 23 : && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1583 189211 : || (binoptab == rotr_optab
1584 11 : && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1585 189213 : && is_int_mode (mode, &int_mode))
1586 : {
1587 1 : optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1588 1 : rtx newop1;
1589 1 : unsigned int bits = GET_MODE_PRECISION (int_mode);
1590 :
1591 1 : if (CONST_INT_P (op1))
1592 0 : newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1593 1 : else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1594 0 : newop1 = negate_rtx (GET_MODE (op1), op1);
1595 : else
1596 1 : newop1 = expand_binop (GET_MODE (op1), sub_optab,
1597 1 : gen_int_mode (bits, GET_MODE (op1)), op1,
1598 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1599 :
1600 1 : temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1601 : target, unsignedp, methods);
1602 1 : if (temp)
1603 14389536 : return temp;
1604 1 : delete_insns_since (last);
1605 : }
1606 :
1607 : /* If this is a multiply, see if we can do a widening operation that
1608 : takes operands of this mode and makes a wider mode. */
1609 :
1610 189212 : if (binoptab == smul_optab
1611 25001 : && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1612 207313 : && (convert_optab_handler ((unsignedp
1613 : ? umul_widen_optab
1614 : : smul_widen_optab),
1615 : wider_mode, mode) != CODE_FOR_nothing))
1616 : {
1617 : /* *_widen_optab needs to determine operand mode, make sure at least
1618 : one operand has non-VOID mode. */
1619 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1620 0 : op0 = force_reg (mode, op0);
1621 0 : temp = expand_binop (wider_mode,
1622 : unsignedp ? umul_widen_optab : smul_widen_optab,
1623 : op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1624 :
1625 0 : if (temp != 0)
1626 : {
1627 0 : if (GET_MODE_CLASS (mode) == MODE_INT
1628 0 : && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1629 0 : return gen_lowpart (mode, temp);
1630 : else
1631 0 : return convert_to_mode (mode, temp, unsignedp);
1632 : }
1633 : }
1634 :
1635 : /* If this is a vector shift by a scalar, see if we can do a vector
1636 : shift by a vector. If so, broadcast the scalar into a vector. */
1637 189212 : if (mclass == MODE_VECTOR_INT)
1638 : {
1639 33 : optab otheroptab = unknown_optab;
1640 :
1641 33 : if (binoptab == ashl_optab)
1642 : otheroptab = vashl_optab;
1643 : else if (binoptab == ashr_optab)
1644 : otheroptab = vashr_optab;
1645 : else if (binoptab == lshr_optab)
1646 : otheroptab = vlshr_optab;
1647 : else if (binoptab == rotl_optab)
1648 : otheroptab = vrotl_optab;
1649 : else if (binoptab == rotr_optab)
1650 : otheroptab = vrotr_optab;
1651 :
1652 : if (otheroptab
1653 33 : && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1654 : {
1655 : /* The scalar may be wider or narrower than the vector element.
1656 : Truncate or extend it to the proper size to fit in the
1657 : broadcast vector. */
1658 0 : scalar_mode inner_mode = GET_MODE_INNER (mode);
1659 0 : if (!CONST_INT_P (op1))
1660 : {
1661 0 : auto mode1 = as_a <scalar_int_mode> (GET_MODE (op1));
1662 0 : int size1 = GET_MODE_BITSIZE (mode1);
1663 0 : int inner_size = GET_MODE_BITSIZE (inner_mode);
1664 :
1665 0 : if (size1 != inner_size)
1666 : {
1667 0 : auto unary = size1 > inner_size ? TRUNCATE : ZERO_EXTEND;
1668 0 : op1 = force_reg (inner_mode,
1669 : simplify_gen_unary (unary, inner_mode,
1670 : op1, mode1));
1671 : }
1672 : }
1673 :
1674 0 : rtx vop1 = expand_vector_broadcast (mode, op1);
1675 0 : if (vop1)
1676 : {
1677 0 : temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1678 : target, unsignedp, methods);
1679 0 : if (temp)
1680 14389536 : return temp;
1681 0 : delete_insns_since (last);
1682 : }
1683 : }
1684 : }
1685 :
1686 : /* Look for a wider mode of the same class for which we think we
1687 : can open-code the operation. Check for a widening multiply at the
1688 : wider mode as well. */
1689 :
1690 189212 : if (CLASS_HAS_WIDER_MODES_P (mclass)
1691 189179 : && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1692 112535 : FOR_EACH_WIDER_MODE (wider_mode, mode)
1693 : {
1694 60097 : machine_mode next_mode;
1695 60097 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1696 60097 : || (binoptab == smul_optab
1697 75448 : && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1698 18985 : && (find_widening_optab_handler ((unsignedp
1699 : ? umul_widen_optab
1700 : : smul_widen_optab),
1701 : next_mode, mode)
1702 : != CODE_FOR_nothing)))
1703 : {
1704 164 : rtx xop0 = op0, xop1 = op1;
1705 164 : bool no_extend = false;
1706 :
1707 : /* For certain integer operations, we need not actually extend
1708 : the narrow operands, as long as we will truncate
1709 : the results to the same narrowness. */
1710 :
1711 164 : if ((binoptab == ior_optab || binoptab == and_optab
1712 164 : || binoptab == xor_optab
1713 164 : || binoptab == add_optab || binoptab == sub_optab
1714 : || binoptab == smul_optab || binoptab == ashl_optab)
1715 89 : && mclass == MODE_INT)
1716 : {
1717 7 : no_extend = true;
1718 7 : xop0 = avoid_expensive_constant (mode, binoptab, 0,
1719 : xop0, unsignedp);
1720 7 : if (binoptab != ashl_optab)
1721 0 : xop1 = avoid_expensive_constant (mode, binoptab, 1,
1722 : xop1, unsignedp);
1723 : }
1724 :
1725 164 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1726 :
1727 : /* The second operand of a shift must always be extended. */
1728 328 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1729 164 : no_extend && binoptab != ashl_optab);
1730 :
1731 164 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1732 : unsignedp, OPTAB_DIRECT);
1733 164 : if (temp)
1734 : {
1735 164 : if (mclass != MODE_INT
1736 164 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1737 : {
1738 157 : if (target == 0)
1739 49 : target = gen_reg_rtx (mode);
1740 157 : convert_move (target, temp, 0);
1741 157 : return target;
1742 : }
1743 : else
1744 7 : return gen_lowpart (mode, temp);
1745 : }
1746 : else
1747 0 : delete_insns_since (last);
1748 : }
1749 : }
1750 :
1751 : /* If operation is commutative,
1752 : try to make the first operand a register.
1753 : Even better, try to make it the same as the target.
1754 : Also try to make the last operand a constant. */
1755 189048 : if (commutative_optab_p (binoptab)
1756 189048 : && swap_commutative_operands_with_target (target, op0, op1))
1757 : std::swap (op0, op1);
1758 :
1759 : /* These can be done a word at a time. */
1760 189048 : if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1761 0 : && is_int_mode (mode, &int_mode)
1762 0 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1763 189048 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1764 : {
1765 0 : int i;
1766 0 : rtx_insn *insns;
1767 :
1768 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1769 : won't be accurate, so use a new target. */
1770 0 : if (target == 0
1771 0 : || target == op0
1772 0 : || target == op1
1773 0 : || reg_overlap_mentioned_p (target, op0)
1774 0 : || reg_overlap_mentioned_p (target, op1)
1775 0 : || !valid_multiword_target_p (target))
1776 0 : target = gen_reg_rtx (int_mode);
1777 :
1778 0 : start_sequence ();
1779 :
1780 : /* Do the actual arithmetic. */
1781 0 : machine_mode op0_mode = GET_MODE (op0);
1782 0 : machine_mode op1_mode = GET_MODE (op1);
1783 0 : if (op0_mode == VOIDmode)
1784 0 : op0_mode = int_mode;
1785 0 : if (op1_mode == VOIDmode)
1786 0 : op1_mode = int_mode;
1787 0 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1788 : {
1789 0 : rtx target_piece = operand_subword (target, i, 1, int_mode);
1790 0 : rtx x = expand_binop (word_mode, binoptab,
1791 0 : operand_subword_force (op0, i, op0_mode),
1792 0 : operand_subword_force (op1, i, op1_mode),
1793 : target_piece, unsignedp, next_methods);
1794 :
1795 0 : if (x == 0)
1796 : break;
1797 :
1798 0 : if (target_piece != x)
1799 0 : emit_move_insn (target_piece, x);
1800 : }
1801 :
1802 0 : insns = end_sequence ();
1803 :
1804 0 : if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1805 : {
1806 0 : emit_insn (insns);
1807 0 : return target;
1808 : }
1809 : }
1810 :
1811 : /* Synthesize double word shifts from single word shifts. */
1812 189048 : if ((binoptab == lshr_optab || binoptab == ashl_optab
1813 189048 : || binoptab == ashr_optab)
1814 0 : && is_int_mode (mode, &int_mode)
1815 0 : && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1816 0 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1817 0 : && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1818 0 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1819 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1820 189048 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1821 : {
1822 0 : unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1823 0 : scalar_int_mode op1_mode;
1824 :
1825 0 : double_shift_mask = targetm.shift_truncation_mask (int_mode);
1826 0 : shift_mask = targetm.shift_truncation_mask (word_mode);
1827 0 : op1_mode = (GET_MODE (op1) != VOIDmode
1828 0 : ? as_a <scalar_int_mode> (GET_MODE (op1))
1829 : : word_mode);
1830 :
1831 : /* Apply the truncation to constant shifts. */
1832 0 : if (double_shift_mask > 0 && CONST_INT_P (op1))
1833 0 : op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1834 :
1835 0 : if (op1 == CONST0_RTX (op1_mode))
1836 14389536 : return op0;
1837 :
1838 : /* Make sure that this is a combination that expand_doubleword_shift
1839 : can handle. See the comments there for details. */
1840 0 : if (double_shift_mask == 0
1841 0 : || (shift_mask == BITS_PER_WORD - 1
1842 0 : && double_shift_mask == BITS_PER_WORD * 2 - 1))
1843 : {
1844 0 : rtx_insn *insns;
1845 0 : rtx into_target, outof_target;
1846 0 : rtx into_input, outof_input;
1847 0 : int left_shift, outof_word;
1848 :
1849 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1850 : won't be accurate, so use a new target. */
1851 0 : if (target == 0
1852 0 : || target == op0
1853 0 : || target == op1
1854 0 : || reg_overlap_mentioned_p (target, op0)
1855 0 : || reg_overlap_mentioned_p (target, op1)
1856 0 : || !valid_multiword_target_p (target))
1857 0 : target = gen_reg_rtx (int_mode);
1858 :
1859 0 : start_sequence ();
1860 :
1861 : /* OUTOF_* is the word we are shifting bits away from, and
1862 : INTO_* is the word that we are shifting bits towards, thus
1863 : they differ depending on the direction of the shift and
1864 : WORDS_BIG_ENDIAN. */
1865 :
1866 0 : left_shift = binoptab == ashl_optab;
1867 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1868 :
1869 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1870 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1871 :
1872 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1873 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1874 :
1875 0 : if (expand_doubleword_shift (op1_mode, binoptab,
1876 : outof_input, into_input, op1,
1877 : outof_target, into_target,
1878 : unsignedp, next_methods, shift_mask))
1879 : {
1880 0 : insns = end_sequence ();
1881 :
1882 0 : emit_insn (insns);
1883 0 : return target;
1884 : }
1885 0 : end_sequence ();
1886 : }
1887 : }
1888 :
1889 : /* Synthesize double word rotates from single word shifts. */
1890 189048 : if ((binoptab == rotl_optab || binoptab == rotr_optab)
1891 34 : && is_int_mode (mode, &int_mode)
1892 1 : && CONST_INT_P (op1)
1893 0 : && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1894 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1895 189048 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1896 : {
1897 0 : rtx_insn *insns;
1898 0 : rtx into_target, outof_target;
1899 0 : rtx into_input, outof_input;
1900 0 : rtx inter;
1901 0 : int shift_count, left_shift, outof_word;
1902 :
1903 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1904 : won't be accurate, so use a new target. Do this also if target is not
1905 : a REG, first because having a register instead may open optimization
1906 : opportunities, and second because if target and op0 happen to be MEMs
1907 : designating the same location, we would risk clobbering it too early
1908 : in the code sequence we generate below. */
1909 0 : if (target == 0
1910 0 : || target == op0
1911 0 : || target == op1
1912 0 : || !REG_P (target)
1913 0 : || reg_overlap_mentioned_p (target, op0)
1914 0 : || reg_overlap_mentioned_p (target, op1)
1915 0 : || !valid_multiword_target_p (target))
1916 0 : target = gen_reg_rtx (int_mode);
1917 :
1918 0 : start_sequence ();
1919 :
1920 0 : shift_count = INTVAL (op1);
1921 :
1922 : /* OUTOF_* is the word we are shifting bits away from, and
1923 : INTO_* is the word that we are shifting bits towards, thus
1924 : they differ depending on the direction of the shift and
1925 : WORDS_BIG_ENDIAN. */
1926 :
1927 0 : left_shift = (binoptab == rotl_optab);
1928 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1929 :
1930 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1931 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1932 :
1933 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1934 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1935 :
1936 0 : if (shift_count == BITS_PER_WORD)
1937 : {
1938 : /* This is just a word swap. */
1939 0 : emit_move_insn (outof_target, into_input);
1940 0 : emit_move_insn (into_target, outof_input);
1941 0 : inter = const0_rtx;
1942 : }
1943 : else
1944 : {
1945 0 : rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1946 0 : HOST_WIDE_INT first_shift_count, second_shift_count;
1947 0 : optab reverse_unsigned_shift, unsigned_shift;
1948 :
1949 0 : reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1950 0 : ? lshr_optab : ashl_optab);
1951 :
1952 0 : unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1953 0 : ? ashl_optab : lshr_optab);
1954 :
1955 0 : if (shift_count > BITS_PER_WORD)
1956 : {
1957 0 : first_shift_count = shift_count - BITS_PER_WORD;
1958 0 : second_shift_count = 2 * BITS_PER_WORD - shift_count;
1959 : }
1960 : else
1961 : {
1962 0 : first_shift_count = BITS_PER_WORD - shift_count;
1963 0 : second_shift_count = shift_count;
1964 : }
1965 0 : rtx first_shift_count_rtx
1966 0 : = gen_int_shift_amount (word_mode, first_shift_count);
1967 0 : rtx second_shift_count_rtx
1968 0 : = gen_int_shift_amount (word_mode, second_shift_count);
1969 :
1970 0 : into_temp1 = expand_binop (word_mode, unsigned_shift,
1971 : outof_input, first_shift_count_rtx,
1972 : NULL_RTX, unsignedp, next_methods);
1973 0 : into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1974 : into_input, second_shift_count_rtx,
1975 : NULL_RTX, unsignedp, next_methods);
1976 :
1977 0 : if (into_temp1 != 0 && into_temp2 != 0)
1978 0 : inter = expand_binop (word_mode, add_optab, into_temp1, into_temp2,
1979 : into_target, unsignedp, next_methods);
1980 : else
1981 : inter = 0;
1982 :
1983 0 : if (inter != 0 && inter != into_target)
1984 0 : emit_move_insn (into_target, inter);
1985 :
1986 0 : outof_temp1 = expand_binop (word_mode, unsigned_shift,
1987 : into_input, first_shift_count_rtx,
1988 : NULL_RTX, unsignedp, next_methods);
1989 0 : outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1990 : outof_input, second_shift_count_rtx,
1991 : NULL_RTX, unsignedp, next_methods);
1992 :
1993 0 : if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1994 0 : inter = expand_binop (word_mode, add_optab,
1995 : outof_temp1, outof_temp2,
1996 : outof_target, unsignedp, next_methods);
1997 :
1998 0 : if (inter != 0 && inter != outof_target)
1999 0 : emit_move_insn (outof_target, inter);
2000 : }
2001 :
2002 0 : insns = end_sequence ();
2003 :
2004 0 : if (inter != 0)
2005 : {
2006 0 : emit_insn (insns);
2007 0 : return target;
2008 : }
2009 : }
2010 :
2011 : /* These can be done a word at a time by propagating carries. */
2012 189048 : if ((binoptab == add_optab || binoptab == sub_optab)
2013 13518 : && is_int_mode (mode, &int_mode)
2014 0 : && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
2015 189048 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
2016 : {
2017 0 : unsigned int i;
2018 0 : optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
2019 0 : const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
2020 0 : rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
2021 0 : rtx xop0, xop1, xtarget;
2022 :
2023 : /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
2024 : value is one of those, use it. Otherwise, use 1 since it is the
2025 : one easiest to get. */
2026 : #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
2027 0 : int normalizep = STORE_FLAG_VALUE;
2028 : #else
2029 : int normalizep = 1;
2030 : #endif
2031 :
2032 : /* Prepare the operands. */
2033 0 : xop0 = force_reg (int_mode, op0);
2034 0 : xop1 = force_reg (int_mode, op1);
2035 :
2036 0 : xtarget = gen_reg_rtx (int_mode);
2037 :
2038 0 : if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
2039 : target = xtarget;
2040 :
2041 : /* Indicate for flow that the entire target reg is being set. */
2042 0 : if (REG_P (target))
2043 0 : emit_clobber (xtarget);
2044 :
2045 : /* Do the actual arithmetic. */
2046 0 : for (i = 0; i < nwords; i++)
2047 : {
2048 0 : int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
2049 0 : rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
2050 0 : rtx op0_piece = operand_subword_force (xop0, index, int_mode);
2051 0 : rtx op1_piece = operand_subword_force (xop1, index, int_mode);
2052 0 : rtx x;
2053 :
2054 : /* Main add/subtract of the input operands. */
2055 0 : x = expand_binop (word_mode, binoptab,
2056 : op0_piece, op1_piece,
2057 : target_piece, unsignedp, next_methods);
2058 0 : if (x == 0)
2059 : break;
2060 :
2061 0 : if (i + 1 < nwords)
2062 : {
2063 : /* Store carry from main add/subtract. */
2064 0 : carry_out = gen_reg_rtx (word_mode);
2065 0 : carry_out = emit_store_flag_force (carry_out,
2066 : (binoptab == add_optab
2067 : ? LT : GT),
2068 : x, op0_piece,
2069 : word_mode, 1, normalizep);
2070 : }
2071 :
2072 0 : if (i > 0)
2073 : {
2074 0 : rtx newx;
2075 :
2076 : /* Add/subtract previous carry to main result. */
2077 0 : newx = expand_binop (word_mode,
2078 : normalizep == 1 ? binoptab : otheroptab,
2079 : x, carry_in,
2080 : NULL_RTX, 1, next_methods);
2081 :
2082 0 : if (i + 1 < nwords)
2083 : {
2084 : /* Get out carry from adding/subtracting carry in. */
2085 0 : rtx carry_tmp = gen_reg_rtx (word_mode);
2086 0 : carry_tmp = emit_store_flag_force (carry_tmp,
2087 : (binoptab == add_optab
2088 : ? LT : GT),
2089 : newx, x,
2090 : word_mode, 1, normalizep);
2091 :
2092 : /* Logical-ior the two poss. carry together. */
2093 0 : carry_out = expand_binop (word_mode, ior_optab,
2094 : carry_out, carry_tmp,
2095 : carry_out, 0, next_methods);
2096 0 : if (carry_out == 0)
2097 : break;
2098 : }
2099 0 : emit_move_insn (target_piece, newx);
2100 : }
2101 : else
2102 : {
2103 0 : if (x != target_piece)
2104 0 : emit_move_insn (target_piece, x);
2105 : }
2106 :
2107 : carry_in = carry_out;
2108 : }
2109 :
2110 0 : if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
2111 : {
2112 0 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
2113 0 : || ! rtx_equal_p (target, xtarget))
2114 : {
2115 0 : rtx_insn *temp = emit_move_insn (target, xtarget);
2116 :
2117 0 : set_dst_reg_note (temp, REG_EQUAL,
2118 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2119 : int_mode, copy_rtx (xop0),
2120 : copy_rtx (xop1)),
2121 : target);
2122 : }
2123 : else
2124 : target = xtarget;
2125 :
2126 0 : return target;
2127 : }
2128 :
2129 : else
2130 0 : delete_insns_since (last);
2131 : }
2132 :
2133 : /* Attempt to synthesize double word multiplies using a sequence of word
2134 : mode multiplications. We first attempt to generate a sequence using a
2135 : more efficient unsigned widening multiply, and if that fails we then
2136 : try using a signed widening multiply. */
2137 :
2138 189048 : if (binoptab == smul_optab
2139 24981 : && is_int_mode (mode, &int_mode)
2140 17115 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2141 14218 : && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2142 203266 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2143 : {
2144 14218 : rtx product = NULL_RTX;
2145 14218 : if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
2146 : != CODE_FOR_nothing)
2147 : {
2148 14218 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2149 : true, methods);
2150 14218 : if (!product)
2151 0 : delete_insns_since (last);
2152 : }
2153 :
2154 0 : if (product == NULL_RTX
2155 0 : && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
2156 : != CODE_FOR_nothing))
2157 : {
2158 0 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2159 : false, methods);
2160 0 : if (!product)
2161 0 : delete_insns_since (last);
2162 : }
2163 :
2164 14218 : if (product != NULL_RTX)
2165 : {
2166 14218 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2167 : {
2168 22278 : rtx_insn *move = emit_move_insn (target ? target : product,
2169 : product);
2170 14218 : set_dst_reg_note (move,
2171 : REG_EQUAL,
2172 : gen_rtx_fmt_ee (MULT, int_mode,
2173 : copy_rtx (op0),
2174 : copy_rtx (op1)),
2175 : target ? target : product);
2176 : }
2177 14218 : return product;
2178 : }
2179 : }
2180 :
2181 : /* Attempt to synthetize double word modulo by constant divisor. */
2182 174830 : if ((binoptab == umod_optab
2183 174830 : || binoptab == smod_optab
2184 174830 : || binoptab == udiv_optab
2185 : || binoptab == sdiv_optab)
2186 112487 : && optimize
2187 87790 : && CONST_INT_P (op1)
2188 22539 : && is_int_mode (mode, &int_mode)
2189 24840 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2190 31032 : && optab_handler ((binoptab == umod_optab || binoptab == udiv_optab)
2191 : ? udivmod_optab : sdivmod_optab,
2192 : int_mode) == CODE_FOR_nothing
2193 19819 : && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
2194 19819 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
2195 19819 : && optimize_insn_for_speed_p ())
2196 : {
2197 19612 : rtx res = NULL_RTX;
2198 19612 : if ((binoptab == umod_optab || binoptab == smod_optab)
2199 5121 : && (INTVAL (op1) & 1) == 0)
2200 321 : res = expand_doubleword_mod (int_mode, op0, op1,
2201 : binoptab == umod_optab);
2202 : else
2203 : {
2204 19291 : rtx quot = expand_doubleword_divmod (int_mode, op0, op1, &res,
2205 : binoptab == umod_optab
2206 : || binoptab == udiv_optab);
2207 19291 : if (quot == NULL_RTX)
2208 18657 : res = NULL_RTX;
2209 634 : else if (binoptab == udiv_optab || binoptab == sdiv_optab)
2210 427 : res = quot;
2211 : }
2212 19612 : if (res != NULL_RTX)
2213 : {
2214 634 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2215 : {
2216 751 : rtx_insn *move = emit_move_insn (target ? target : res,
2217 : res);
2218 634 : set_dst_reg_note (move, REG_EQUAL,
2219 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2220 : int_mode, copy_rtx (op0), op1),
2221 : target ? target : res);
2222 : }
2223 634 : return res;
2224 : }
2225 : else
2226 18978 : delete_insns_since (last);
2227 : }
2228 :
2229 : /* It can't be open-coded in this mode.
2230 : Use a library call if one is available and caller says that's ok. */
2231 :
2232 174196 : libfunc = optab_libfunc (binoptab, mode);
2233 174196 : if (libfunc
2234 110070 : && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2235 : {
2236 30460 : rtx_insn *insns;
2237 30460 : rtx op1x = op1;
2238 30460 : machine_mode op1_mode = mode;
2239 30460 : rtx value;
2240 :
2241 30460 : start_sequence ();
2242 :
2243 30460 : if (shift_optab_p (binoptab))
2244 : {
2245 0 : op1_mode = targetm.libgcc_shift_count_mode ();
2246 : /* Specify unsigned here,
2247 : since negative shift counts are meaningless. */
2248 0 : op1x = convert_to_mode (op1_mode, op1, 1);
2249 : }
2250 :
2251 30460 : if (GET_MODE (op0) != VOIDmode
2252 30408 : && GET_MODE (op0) != mode)
2253 0 : op0 = convert_to_mode (mode, op0, unsignedp);
2254 :
2255 : /* Pass 1 for NO_QUEUE so we don't lose any increments
2256 : if the libcall is cse'd or moved. */
2257 30460 : value = emit_library_call_value (libfunc,
2258 : NULL_RTX, LCT_CONST, mode,
2259 : op0, mode, op1x, op1_mode);
2260 :
2261 30460 : insns = end_sequence ();
2262 :
2263 30460 : bool trapv = trapv_binoptab_p (binoptab);
2264 30460 : target = gen_reg_rtx (mode);
2265 30460 : emit_libcall_block_1 (insns, target, value,
2266 : trapv ? NULL_RTX
2267 29943 : : gen_rtx_fmt_ee (optab_to_code (binoptab),
2268 : mode, op0, op1), trapv);
2269 :
2270 30460 : return target;
2271 : }
2272 :
2273 143736 : delete_insns_since (last);
2274 :
2275 : /* It can't be done in this mode. Can we do it in a wider mode? */
2276 :
2277 143736 : if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2278 : || methods == OPTAB_MUST_WIDEN))
2279 : {
2280 : /* Caller says, don't even try. */
2281 131166 : delete_insns_since (entry_last);
2282 131166 : return 0;
2283 : }
2284 :
2285 : /* Compute the value of METHODS to pass to recursive calls.
2286 : Don't allow widening to be tried recursively. */
2287 :
2288 12570 : methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2289 :
2290 : /* Look for a wider mode of the same class for which it appears we can do
2291 : the operation. */
2292 :
2293 12570 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2294 : {
2295 : /* This code doesn't make sense for conversion optabs, since we
2296 : wouldn't then want to extend the operands to be the same size
2297 : as the result. */
2298 12570 : gcc_assert (!convert_optab_p (binoptab));
2299 39616 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2300 : {
2301 27047 : if (optab_handler (binoptab, wider_mode)
2302 27047 : || (methods == OPTAB_LIB
2303 2 : && optab_libfunc (binoptab, wider_mode)))
2304 : {
2305 1 : rtx xop0 = op0, xop1 = op1;
2306 1 : bool no_extend = false;
2307 :
2308 : /* For certain integer operations, we need not actually extend
2309 : the narrow operands, as long as we will truncate
2310 : the results to the same narrowness. */
2311 :
2312 1 : if ((binoptab == ior_optab || binoptab == and_optab
2313 1 : || binoptab == xor_optab
2314 1 : || binoptab == add_optab || binoptab == sub_optab
2315 : || binoptab == smul_optab || binoptab == ashl_optab)
2316 0 : && mclass == MODE_INT)
2317 1 : no_extend = true;
2318 :
2319 1 : xop0 = widen_operand (xop0, wider_mode, mode,
2320 : unsignedp, no_extend);
2321 :
2322 : /* The second operand of a shift must always be extended. */
2323 2 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2324 1 : no_extend && binoptab != ashl_optab);
2325 :
2326 1 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2327 : unsignedp, methods);
2328 1 : if (temp)
2329 : {
2330 1 : if (mclass != MODE_INT
2331 1 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2332 : {
2333 0 : if (target == 0)
2334 0 : target = gen_reg_rtx (mode);
2335 0 : convert_move (target, temp, 0);
2336 0 : return target;
2337 : }
2338 : else
2339 1 : return gen_lowpart (mode, temp);
2340 : }
2341 : else
2342 0 : delete_insns_since (last);
2343 : }
2344 : }
2345 : }
2346 :
2347 12569 : delete_insns_since (entry_last);
2348 12569 : return 0;
2349 : }
2350 :
2351 : /* Expand a binary operator which has both signed and unsigned forms.
2352 : UOPTAB is the optab for unsigned operations, and SOPTAB is for
2353 : signed operations.
2354 :
2355 : If we widen unsigned operands, we may use a signed wider operation instead
2356 : of an unsigned wider operation, since the result would be the same. */
2357 :
2358 : rtx
2359 94311 : sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
2360 : rtx op0, rtx op1, rtx target, int unsignedp,
2361 : enum optab_methods methods)
2362 : {
2363 94311 : rtx temp;
2364 94311 : optab direct_optab = unsignedp ? uoptab : soptab;
2365 94311 : bool save_enable;
2366 :
2367 : /* Do it without widening, if possible. */
2368 94311 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2369 : unsignedp, OPTAB_DIRECT);
2370 94311 : if (temp || methods == OPTAB_DIRECT)
2371 : return temp;
2372 :
2373 : /* Try widening to a signed int. Disable any direct use of any
2374 : signed insn in the current mode. */
2375 8270 : save_enable = swap_optab_enable (soptab, mode, false);
2376 :
2377 8270 : temp = expand_binop (mode, soptab, op0, op1, target,
2378 : unsignedp, OPTAB_WIDEN);
2379 :
2380 : /* For unsigned operands, try widening to an unsigned int. */
2381 8270 : if (!temp && unsignedp)
2382 4177 : temp = expand_binop (mode, uoptab, op0, op1, target,
2383 : unsignedp, OPTAB_WIDEN);
2384 8270 : if (temp || methods == OPTAB_WIDEN)
2385 4763 : goto egress;
2386 :
2387 : /* Use the right width libcall if that exists. */
2388 3507 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2389 : unsignedp, OPTAB_LIB);
2390 3507 : if (temp || methods == OPTAB_LIB)
2391 3507 : goto egress;
2392 :
2393 : /* Must widen and use a libcall, use either signed or unsigned. */
2394 0 : temp = expand_binop (mode, soptab, op0, op1, target,
2395 : unsignedp, methods);
2396 0 : if (!temp && unsignedp)
2397 0 : temp = expand_binop (mode, uoptab, op0, op1, target,
2398 : unsignedp, methods);
2399 :
2400 0 : egress:
2401 : /* Undo the fiddling above. */
2402 8270 : if (save_enable)
2403 0 : swap_optab_enable (soptab, mode, true);
2404 : return temp;
2405 : }
2406 :
2407 : /* Generate code to perform an operation specified by UNOPPTAB
2408 : on operand OP0, with two results to TARG0 and TARG1.
2409 : We assume that the order of the operands for the instruction
2410 : is TARG0, TARG1, OP0.
2411 :
2412 : Either TARG0 or TARG1 may be zero, but what that means is that
2413 : the result is not actually wanted. We will generate it into
2414 : a dummy pseudo-reg and discard it. They may not both be zero.
2415 :
2416 : Returns true if this operation can be performed; false if not. */
2417 :
2418 : bool
2419 6 : expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2420 : int unsignedp)
2421 : {
2422 6 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2423 6 : enum mode_class mclass;
2424 6 : machine_mode wider_mode;
2425 6 : rtx_insn *entry_last = get_last_insn ();
2426 6 : rtx_insn *last;
2427 :
2428 6 : mclass = GET_MODE_CLASS (mode);
2429 :
2430 6 : if (!targ0)
2431 0 : targ0 = gen_reg_rtx (mode);
2432 6 : if (!targ1)
2433 0 : targ1 = gen_reg_rtx (mode);
2434 :
2435 : /* Record where to go back to if we fail. */
2436 6 : last = get_last_insn ();
2437 :
2438 6 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2439 : {
2440 6 : class expand_operand ops[3];
2441 6 : enum insn_code icode = optab_handler (unoptab, mode);
2442 :
2443 6 : create_fixed_operand (&ops[0], targ0);
2444 6 : create_fixed_operand (&ops[1], targ1);
2445 6 : create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2446 6 : if (maybe_expand_insn (icode, 3, ops))
2447 6 : return true;
2448 : }
2449 :
2450 : /* It can't be done in this mode. Can we do it in a wider mode? */
2451 :
2452 0 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2453 : {
2454 0 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2455 : {
2456 0 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2457 : {
2458 0 : rtx t0 = gen_reg_rtx (wider_mode);
2459 0 : rtx t1 = gen_reg_rtx (wider_mode);
2460 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2461 :
2462 0 : if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2463 : {
2464 0 : convert_move (targ0, t0, unsignedp);
2465 0 : convert_move (targ1, t1, unsignedp);
2466 0 : return true;
2467 : }
2468 : else
2469 0 : delete_insns_since (last);
2470 : }
2471 : }
2472 : }
2473 :
2474 0 : delete_insns_since (entry_last);
2475 0 : return false;
2476 : }
2477 :
2478 : /* Generate code to perform an operation specified by BINOPTAB
2479 : on operands OP0 and OP1, with two results to TARG1 and TARG2.
2480 : We assume that the order of the operands for the instruction
2481 : is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2482 : [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2483 :
2484 : Either TARG0 or TARG1 may be zero, but what that means is that
2485 : the result is not actually wanted. We will generate it into
2486 : a dummy pseudo-reg and discard it. They may not both be zero.
2487 :
2488 : Returns true if this operation can be performed; false if not. */
2489 :
2490 : bool
2491 101676 : expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2492 : int unsignedp)
2493 : {
2494 101676 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2495 101676 : enum mode_class mclass;
2496 101676 : machine_mode wider_mode;
2497 101676 : rtx_insn *entry_last = get_last_insn ();
2498 101676 : rtx_insn *last;
2499 :
2500 101676 : mclass = GET_MODE_CLASS (mode);
2501 :
2502 101676 : if (!targ0)
2503 31336 : targ0 = gen_reg_rtx (mode);
2504 101676 : if (!targ1)
2505 58873 : targ1 = gen_reg_rtx (mode);
2506 :
2507 : /* Record where to go back to if we fail. */
2508 101676 : last = get_last_insn ();
2509 :
2510 101676 : if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2511 : {
2512 96877 : class expand_operand ops[4];
2513 96877 : enum insn_code icode = optab_handler (binoptab, mode);
2514 96877 : machine_mode mode0 = insn_data[icode].operand[1].mode;
2515 96877 : machine_mode mode1 = insn_data[icode].operand[2].mode;
2516 96877 : rtx xop0 = op0, xop1 = op1;
2517 :
2518 : /* If we are optimizing, force expensive constants into a register. */
2519 96877 : xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2520 96877 : xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2521 :
2522 96877 : create_fixed_operand (&ops[0], targ0);
2523 96877 : create_convert_operand_from (&ops[1], xop0, mode, unsignedp);
2524 96877 : create_convert_operand_from (&ops[2], xop1, mode, unsignedp);
2525 96877 : create_fixed_operand (&ops[3], targ1);
2526 96877 : if (maybe_expand_insn (icode, 4, ops))
2527 96877 : return true;
2528 0 : delete_insns_since (last);
2529 : }
2530 :
2531 : /* It can't be done in this mode. Can we do it in a wider mode? */
2532 :
2533 4799 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2534 : {
2535 15235 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2536 : {
2537 10436 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2538 : {
2539 0 : rtx t0 = gen_reg_rtx (wider_mode);
2540 0 : rtx t1 = gen_reg_rtx (wider_mode);
2541 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2542 0 : rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2543 :
2544 0 : if (expand_twoval_binop (binoptab, cop0, cop1,
2545 : t0, t1, unsignedp))
2546 : {
2547 0 : convert_move (targ0, t0, unsignedp);
2548 0 : convert_move (targ1, t1, unsignedp);
2549 0 : return true;
2550 : }
2551 : else
2552 0 : delete_insns_since (last);
2553 : }
2554 : }
2555 : }
2556 :
2557 4799 : delete_insns_since (entry_last);
2558 4799 : return false;
2559 : }
2560 :
2561 : /* Expand the two-valued library call indicated by BINOPTAB, but
2562 : preserve only one of the values. If TARG0 is non-NULL, the first
2563 : value is placed into TARG0; otherwise the second value is placed
2564 : into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2565 : value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2566 : This routine assumes that the value returned by the library call is
2567 : as if the return value was of an integral mode twice as wide as the
2568 : mode of OP0. Returns 1 if the call was successful. */
2569 :
2570 : bool
2571 0 : expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2572 : rtx targ0, rtx targ1, enum rtx_code code)
2573 : {
2574 0 : machine_mode mode;
2575 0 : machine_mode libval_mode;
2576 0 : rtx libval;
2577 0 : rtx_insn *insns;
2578 0 : rtx libfunc;
2579 :
2580 : /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2581 0 : gcc_assert (!targ0 != !targ1);
2582 :
2583 0 : mode = GET_MODE (op0);
2584 0 : libfunc = optab_libfunc (binoptab, mode);
2585 0 : if (!libfunc)
2586 : return false;
2587 :
2588 : /* The value returned by the library function will have twice as
2589 : many bits as the nominal MODE. */
2590 0 : auto return_size = 2 * GET_MODE_BITSIZE (mode);
2591 0 : if (!smallest_int_mode_for_size (return_size).exists (&libval_mode))
2592 0 : return false;
2593 :
2594 0 : start_sequence ();
2595 0 : libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2596 : libval_mode,
2597 : op0, mode,
2598 : op1, mode);
2599 : /* Get the part of VAL containing the value that we want. */
2600 0 : libval = simplify_gen_subreg (mode, libval, libval_mode,
2601 0 : targ0 ? 0 : GET_MODE_SIZE (mode));
2602 0 : insns = end_sequence ();
2603 : /* Move the into the desired location. */
2604 0 : emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2605 : gen_rtx_fmt_ee (code, mode, op0, op1));
2606 :
2607 0 : return true;
2608 : }
2609 :
2610 :
2611 : /* Wrapper around expand_unop which takes an rtx code to specify
2612 : the operation to perform, not an optab pointer. All other
2613 : arguments are the same. */
2614 : rtx
2615 16791 : expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2616 : rtx target, int unsignedp)
2617 : {
2618 16791 : optab unop = code_to_optab (code);
2619 16791 : gcc_assert (unop);
2620 :
2621 16791 : return expand_unop (mode, unop, op0, target, unsignedp);
2622 : }
2623 :
2624 : /* Try calculating
2625 : (clz:narrow x)
2626 : as
2627 : (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2628 :
2629 : A similar operation can be used for clrsb. UNOPTAB says which operation
2630 : we are trying to expand. */
2631 : static rtx
2632 296 : widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2633 : {
2634 296 : opt_scalar_int_mode wider_mode_iter;
2635 1215 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2636 : {
2637 919 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2638 919 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2639 : {
2640 0 : rtx xop0, temp;
2641 0 : rtx_insn *last;
2642 :
2643 0 : last = get_last_insn ();
2644 :
2645 0 : if (target == 0)
2646 0 : target = gen_reg_rtx (mode);
2647 0 : xop0 = widen_operand (op0, wider_mode, mode,
2648 : unoptab != clrsb_optab, false);
2649 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2650 : unoptab != clrsb_optab);
2651 0 : if (temp != 0)
2652 0 : temp = expand_binop
2653 0 : (wider_mode, sub_optab, temp,
2654 0 : gen_int_mode (GET_MODE_PRECISION (wider_mode)
2655 0 : - GET_MODE_PRECISION (mode),
2656 : wider_mode),
2657 : target, true, OPTAB_DIRECT);
2658 0 : if (temp == 0)
2659 0 : delete_insns_since (last);
2660 :
2661 0 : return temp;
2662 : }
2663 : }
2664 : return 0;
2665 : }
2666 :
2667 : /* Attempt to emit (clrsb:mode op0) as
2668 : (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2669 : (const_int -1))
2670 : if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2671 : or as
2672 : (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2673 : (ashr:mode op0 (const_int prec-1)))
2674 : (const_int 1)))
2675 : otherwise. */
2676 :
2677 : static rtx
2678 91 : expand_clrsb_using_clz (scalar_int_mode mode, rtx op0, rtx target)
2679 : {
2680 91 : if (optimize_insn_for_size_p ()
2681 91 : || optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2682 4 : return NULL_RTX;
2683 :
2684 87 : start_sequence ();
2685 87 : HOST_WIDE_INT val = 0;
2686 87 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) != 2
2687 87 : || val != GET_MODE_PRECISION (mode))
2688 : val = 0;
2689 : else
2690 : val = 1;
2691 :
2692 85 : rtx temp2 = op0;
2693 85 : if (!val)
2694 : {
2695 85 : temp2 = expand_binop (mode, ashl_optab, op0, const1_rtx,
2696 : NULL_RTX, 0, OPTAB_DIRECT);
2697 85 : if (!temp2)
2698 : {
2699 0 : fail:
2700 0 : end_sequence ();
2701 0 : return NULL_RTX;
2702 : }
2703 : }
2704 :
2705 87 : rtx temp = expand_binop (mode, ashr_optab, op0,
2706 87 : GEN_INT (GET_MODE_PRECISION (mode) - 1),
2707 : NULL_RTX, 0, OPTAB_DIRECT);
2708 87 : if (!temp)
2709 0 : goto fail;
2710 :
2711 87 : temp = expand_binop (mode, xor_optab, temp2, temp, NULL_RTX, 0,
2712 : OPTAB_DIRECT);
2713 87 : if (!temp)
2714 0 : goto fail;
2715 :
2716 87 : if (!val)
2717 : {
2718 85 : temp = expand_binop (mode, ior_optab, temp, const1_rtx,
2719 : NULL_RTX, 0, OPTAB_DIRECT);
2720 85 : if (!temp)
2721 0 : goto fail;
2722 : }
2723 87 : temp = expand_unop_direct (mode, clz_optab, temp, val ? NULL_RTX : target,
2724 : true);
2725 87 : if (!temp)
2726 0 : goto fail;
2727 87 : if (val)
2728 : {
2729 2 : temp = expand_binop (mode, add_optab, temp, constm1_rtx,
2730 : target, 0, OPTAB_DIRECT);
2731 2 : if (!temp)
2732 0 : goto fail;
2733 : }
2734 :
2735 87 : rtx_insn *seq = end_sequence ();
2736 :
2737 87 : add_equal_note (seq, temp, CLRSB, op0, NULL_RTX, mode);
2738 87 : emit_insn (seq);
2739 87 : return temp;
2740 : }
2741 :
2742 : static rtx expand_ffs (scalar_int_mode, rtx, rtx);
2743 :
2744 : /* Try calculating clz, ctz or ffs of a double-word quantity as two clz, ctz or
2745 : ffs operations on word-sized quantities, choosing which based on whether the
2746 : high (for clz) or low (for ctz and ffs) word is nonzero. */
2747 : static rtx
2748 252 : expand_doubleword_clz_ctz_ffs (scalar_int_mode mode, rtx op0, rtx target,
2749 : optab unoptab)
2750 : {
2751 252 : rtx xop0 = force_reg (mode, op0);
2752 252 : rtx subhi = gen_highpart (word_mode, xop0);
2753 252 : rtx sublo = gen_lowpart (word_mode, xop0);
2754 252 : rtx_code_label *hi0_label = gen_label_rtx ();
2755 252 : rtx_code_label *after_label = gen_label_rtx ();
2756 252 : rtx_insn *seq;
2757 252 : rtx temp, result;
2758 252 : int addend = 0;
2759 :
2760 : /* If we were not given a target, use a word_mode register, not a
2761 : 'mode' register. The result will fit, and nobody is expecting
2762 : anything bigger (the return type of __builtin_clz* is int). */
2763 252 : if (!target)
2764 0 : target = gen_reg_rtx (word_mode);
2765 :
2766 : /* In any case, write to a word_mode scratch in both branches of the
2767 : conditional, so we can ensure there is a single move insn setting
2768 : 'target' to tag a REG_EQUAL note on. */
2769 252 : result = gen_reg_rtx (word_mode);
2770 :
2771 252 : if (unoptab != clz_optab)
2772 47 : std::swap (subhi, sublo);
2773 :
2774 252 : start_sequence ();
2775 :
2776 : /* If the high word is not equal to zero,
2777 : then clz of the full value is clz of the high word. */
2778 252 : emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2779 : word_mode, true, hi0_label);
2780 :
2781 252 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2782 252 : temp = expand_unop_direct (word_mode, unoptab, subhi, result, true);
2783 : else
2784 : {
2785 0 : gcc_assert (unoptab == ffs_optab);
2786 0 : temp = expand_ffs (word_mode, subhi, result);
2787 : }
2788 252 : if (!temp)
2789 0 : goto fail;
2790 :
2791 252 : if (temp != result)
2792 0 : convert_move (result, temp, true);
2793 :
2794 252 : emit_jump_insn (targetm.gen_jump (after_label));
2795 252 : emit_barrier ();
2796 :
2797 : /* Else clz of the full value is clz of the low word plus the number
2798 : of bits in the high word. Similarly for ctz/ffs of the high word,
2799 : except that ffs should be 0 when both words are zero. */
2800 252 : emit_label (hi0_label);
2801 :
2802 252 : if (unoptab == ffs_optab)
2803 : {
2804 0 : convert_move (result, const0_rtx, true);
2805 0 : emit_cmp_and_jump_insns (sublo, CONST0_RTX (word_mode), EQ, 0,
2806 : word_mode, true, after_label);
2807 : }
2808 :
2809 252 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2810 252 : temp = expand_unop_direct (word_mode, unoptab, sublo, NULL_RTX, true);
2811 : else
2812 : {
2813 0 : gcc_assert (unoptab == ffs_optab);
2814 0 : temp = expand_unop_direct (word_mode, ctz_optab, sublo, NULL_RTX, true);
2815 0 : addend = 1;
2816 : }
2817 :
2818 252 : if (!temp)
2819 0 : goto fail;
2820 :
2821 504 : temp = expand_binop (word_mode, add_optab, temp,
2822 252 : gen_int_mode (GET_MODE_BITSIZE (word_mode) + addend,
2823 : word_mode),
2824 : result, true, OPTAB_DIRECT);
2825 252 : if (!temp)
2826 0 : goto fail;
2827 252 : if (temp != result)
2828 0 : convert_move (result, temp, true);
2829 :
2830 252 : emit_label (after_label);
2831 252 : convert_move (target, result, true);
2832 :
2833 252 : seq = end_sequence ();
2834 :
2835 252 : add_equal_note (seq, target, optab_to_code (unoptab), xop0, NULL_RTX, mode);
2836 252 : emit_insn (seq);
2837 252 : return target;
2838 :
2839 0 : fail:
2840 0 : end_sequence ();
2841 0 : return 0;
2842 : }
2843 :
2844 : /* Try calculating popcount of a double-word quantity as two popcount's of
2845 : word-sized quantities and summing up the results. */
2846 : static rtx
2847 0 : expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2848 : {
2849 0 : rtx t0, t1, t;
2850 0 : rtx_insn *seq;
2851 :
2852 0 : start_sequence ();
2853 :
2854 0 : t0 = expand_unop_direct (word_mode, popcount_optab,
2855 : operand_subword_force (op0, 0, mode), NULL_RTX,
2856 : true);
2857 0 : t1 = expand_unop_direct (word_mode, popcount_optab,
2858 : operand_subword_force (op0, 1, mode), NULL_RTX,
2859 : true);
2860 0 : if (!t0 || !t1)
2861 : {
2862 0 : end_sequence ();
2863 0 : return NULL_RTX;
2864 : }
2865 :
2866 : /* If we were not given a target, use a word_mode register, not a
2867 : 'mode' register. The result will fit, and nobody is expecting
2868 : anything bigger (the return type of __builtin_popcount* is int). */
2869 0 : if (!target)
2870 0 : target = gen_reg_rtx (word_mode);
2871 :
2872 0 : t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2873 :
2874 0 : seq = end_sequence ();
2875 :
2876 0 : add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2877 0 : emit_insn (seq);
2878 0 : return t;
2879 : }
2880 :
2881 : /* Try calculating
2882 : (parity:wide x)
2883 : as
2884 : (parity:narrow (low (x) ^ high (x))) */
2885 : static rtx
2886 0 : expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2887 : {
2888 0 : rtx t = expand_binop (word_mode, xor_optab,
2889 : operand_subword_force (op0, 0, mode),
2890 : operand_subword_force (op0, 1, mode),
2891 : NULL_RTX, 0, OPTAB_DIRECT);
2892 0 : return expand_unop (word_mode, parity_optab, t, target, true);
2893 : }
2894 :
2895 : /* Try calculating
2896 : (bswap:narrow x)
2897 : as
2898 : (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow))). */
2899 : static rtx
2900 155 : widen_bswap (scalar_int_mode mode, rtx op0, rtx target)
2901 : {
2902 155 : rtx x;
2903 155 : rtx_insn *last;
2904 155 : opt_scalar_int_mode wider_mode_iter;
2905 :
2906 613 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2907 458 : if (optab_handler (bswap_optab, wider_mode_iter.require ())
2908 : != CODE_FOR_nothing)
2909 : break;
2910 :
2911 155 : if (!wider_mode_iter.exists ())
2912 : return NULL_RTX;
2913 :
2914 0 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2915 0 : last = get_last_insn ();
2916 :
2917 0 : x = widen_operand (op0, wider_mode, mode, true, true);
2918 0 : x = expand_unop (wider_mode, bswap_optab, x, NULL_RTX, true);
2919 :
2920 0 : gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2921 : && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2922 0 : if (x != 0)
2923 0 : x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2924 0 : GET_MODE_BITSIZE (wider_mode)
2925 0 : - GET_MODE_BITSIZE (mode),
2926 : NULL_RTX, true);
2927 :
2928 0 : if (x != 0)
2929 : {
2930 0 : if (target == 0)
2931 0 : target = gen_reg_rtx (mode);
2932 0 : emit_move_insn (target, gen_lowpart (mode, x));
2933 : }
2934 : else
2935 0 : delete_insns_since (last);
2936 :
2937 : return target;
2938 : }
2939 :
2940 : /* Try calculating bswap as two bswaps of two word-sized operands. */
2941 :
2942 : static rtx
2943 155 : expand_doubleword_bswap (machine_mode mode, rtx op, rtx target)
2944 : {
2945 155 : rtx t0, t1;
2946 :
2947 155 : t1 = expand_unop (word_mode, bswap_optab,
2948 : operand_subword_force (op, 0, mode), NULL_RTX, true);
2949 155 : t0 = expand_unop (word_mode, bswap_optab,
2950 : operand_subword_force (op, 1, mode), NULL_RTX, true);
2951 :
2952 155 : if (target == 0 || !valid_multiword_target_p (target))
2953 1 : target = gen_reg_rtx (mode);
2954 155 : if (REG_P (target))
2955 155 : emit_clobber (target);
2956 155 : emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2957 155 : emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2958 :
2959 155 : return target;
2960 : }
2961 :
2962 : /* Try calculating (parity x) as (and (popcount x) 1), where
2963 : popcount can also be done in a wider mode. */
2964 : static rtx
2965 11 : expand_parity (scalar_int_mode mode, rtx op0, rtx target)
2966 : {
2967 11 : enum mode_class mclass = GET_MODE_CLASS (mode);
2968 11 : opt_scalar_int_mode wider_mode_iter;
2969 11 : FOR_EACH_MODE_FROM (wider_mode_iter, mode)
2970 : {
2971 11 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2972 11 : if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
2973 : {
2974 11 : rtx xop0, temp;
2975 11 : rtx_insn *last;
2976 :
2977 11 : last = get_last_insn ();
2978 :
2979 11 : if (target == 0 || GET_MODE (target) != wider_mode)
2980 6 : target = gen_reg_rtx (wider_mode);
2981 :
2982 11 : xop0 = widen_operand (op0, wider_mode, mode, true, false);
2983 11 : temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
2984 : true);
2985 11 : if (temp != 0)
2986 11 : temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
2987 : target, true, OPTAB_DIRECT);
2988 :
2989 11 : if (temp)
2990 : {
2991 11 : if (mclass != MODE_INT
2992 11 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2993 0 : return convert_to_mode (mode, temp, 0);
2994 : else
2995 11 : return gen_lowpart (mode, temp);
2996 : }
2997 : else
2998 0 : delete_insns_since (last);
2999 : }
3000 : }
3001 : return 0;
3002 : }
3003 :
3004 : /* Try calculating ctz(x) as K - clz(x & -x) ,
3005 : where K is GET_MODE_PRECISION(mode) - 1.
3006 :
3007 : Both __builtin_ctz and __builtin_clz are undefined at zero, so we
3008 : don't have to worry about what the hardware does in that case. (If
3009 : the clz instruction produces the usual value at 0, which is K, the
3010 : result of this code sequence will be -1; expand_ffs, below, relies
3011 : on this. It might be nice to have it be K instead, for consistency
3012 : with the (very few) processors that provide a ctz with a defined
3013 : value, but that would take one more instruction, and it would be
3014 : less convenient for expand_ffs anyway. */
3015 :
3016 : static rtx
3017 50 : expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
3018 : {
3019 50 : rtx_insn *seq;
3020 50 : rtx temp;
3021 :
3022 50 : if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
3023 : return 0;
3024 :
3025 0 : start_sequence ();
3026 :
3027 0 : temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
3028 0 : if (temp)
3029 0 : temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
3030 : true, OPTAB_DIRECT);
3031 0 : if (temp)
3032 0 : temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
3033 0 : if (temp)
3034 0 : temp = expand_binop (mode, sub_optab,
3035 0 : gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
3036 : temp, target,
3037 : true, OPTAB_DIRECT);
3038 0 : if (temp == 0)
3039 : {
3040 0 : end_sequence ();
3041 0 : return 0;
3042 : }
3043 :
3044 0 : seq = end_sequence ();
3045 :
3046 0 : add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
3047 0 : emit_insn (seq);
3048 0 : return temp;
3049 : }
3050 :
3051 :
3052 : /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
3053 : else with the sequence used by expand_clz.
3054 :
3055 : The ffs builtin promises to return zero for a zero value and ctz/clz
3056 : may have an undefined value in that case. If they do not give us a
3057 : convenient value, we have to generate a test and branch. */
3058 : static rtx
3059 0 : expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
3060 : {
3061 0 : HOST_WIDE_INT val = 0;
3062 0 : bool defined_at_zero = false;
3063 0 : rtx temp;
3064 0 : rtx_insn *seq;
3065 :
3066 0 : if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
3067 : {
3068 0 : start_sequence ();
3069 :
3070 0 : temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
3071 0 : if (!temp)
3072 0 : goto fail;
3073 :
3074 0 : defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
3075 : }
3076 0 : else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
3077 : {
3078 0 : start_sequence ();
3079 0 : temp = expand_ctz (mode, op0, 0);
3080 0 : if (!temp)
3081 0 : goto fail;
3082 :
3083 0 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
3084 : {
3085 0 : defined_at_zero = true;
3086 0 : val = (GET_MODE_PRECISION (mode) - 1) - val;
3087 : }
3088 : }
3089 : else
3090 : return 0;
3091 :
3092 0 : if (defined_at_zero && val == -1)
3093 : /* No correction needed at zero. */;
3094 : else
3095 : {
3096 : /* We don't try to do anything clever with the situation found
3097 : on some processors (eg Alpha) where ctz(0:mode) ==
3098 : bitsize(mode). If someone can think of a way to send N to -1
3099 : and leave alone all values in the range 0..N-1 (where N is a
3100 : power of two), cheaper than this test-and-branch, please add it.
3101 :
3102 : The test-and-branch is done after the operation itself, in case
3103 : the operation sets condition codes that can be recycled for this.
3104 : (This is true on i386, for instance.) */
3105 :
3106 0 : rtx_code_label *nonzero_label = gen_label_rtx ();
3107 0 : emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
3108 : mode, true, nonzero_label);
3109 :
3110 0 : convert_move (temp, GEN_INT (-1), false);
3111 0 : emit_label (nonzero_label);
3112 : }
3113 :
3114 : /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3115 : to produce a value in the range 0..bitsize. */
3116 0 : temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
3117 : target, false, OPTAB_DIRECT);
3118 0 : if (!temp)
3119 0 : goto fail;
3120 :
3121 0 : seq = end_sequence ();
3122 :
3123 0 : add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
3124 0 : emit_insn (seq);
3125 0 : return temp;
3126 :
3127 0 : fail:
3128 0 : end_sequence ();
3129 0 : return 0;
3130 : }
3131 :
3132 : /* Expand a floating point absolute value or negation operation via a
3133 : logical operation on the sign bit. MODE is the mode of the operands
3134 : and FMODE is the scalar inner mode. */
3135 :
3136 : static rtx
3137 1101 : expand_absneg_bit (rtx_code code, machine_mode mode,
3138 : scalar_float_mode fmode, rtx op0, rtx target)
3139 : {
3140 1101 : int bitpos, word, nwords, i;
3141 1101 : machine_mode new_mode;
3142 1101 : scalar_int_mode imode;
3143 1101 : rtx temp;
3144 1101 : rtx_insn *insns;
3145 :
3146 1101 : auto op = code == NEG ? neg_optab : abs_optab;
3147 1101 : if (!get_absneg_bit_mode (op, mode, fmode, &bitpos).exists (&new_mode))
3148 0 : return NULL_RTX;
3149 :
3150 2202 : imode = as_a<scalar_int_mode> (GET_MODE_INNER (new_mode));
3151 2252 : if (VECTOR_MODE_P (mode) || GET_MODE_SIZE (fmode) <= UNITS_PER_WORD)
3152 : {
3153 : word = 0;
3154 : nwords = 1;
3155 : }
3156 : else
3157 : {
3158 17 : if (FLOAT_WORDS_BIG_ENDIAN)
3159 : word = (GET_MODE_BITSIZE (fmode) - bitpos) / BITS_PER_WORD;
3160 : else
3161 17 : word = bitpos / BITS_PER_WORD;
3162 17 : bitpos = bitpos % BITS_PER_WORD;
3163 34 : nwords = (GET_MODE_BITSIZE (fmode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3164 : }
3165 :
3166 1101 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3167 1101 : if (code == ABS)
3168 840 : mask = ~mask;
3169 :
3170 1101 : if (target == 0
3171 1101 : || target == op0
3172 633 : || reg_overlap_mentioned_p (target, op0)
3173 1730 : || (nwords > 1 && !valid_multiword_target_p (target)))
3174 472 : target = gen_reg_rtx (mode);
3175 :
3176 1101 : if (nwords > 1)
3177 : {
3178 17 : start_sequence ();
3179 :
3180 51 : for (i = 0; i < nwords; ++i)
3181 : {
3182 34 : rtx targ_piece = operand_subword (target, i, 1, mode);
3183 34 : rtx op0_piece = operand_subword_force (op0, i, mode);
3184 :
3185 34 : if (i == word)
3186 : {
3187 46 : temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
3188 : op0_piece,
3189 17 : immed_wide_int_const (mask, imode),
3190 : targ_piece, 1, OPTAB_LIB_WIDEN);
3191 17 : if (temp != targ_piece)
3192 0 : emit_move_insn (targ_piece, temp);
3193 : }
3194 : else
3195 17 : emit_move_insn (targ_piece, op0_piece);
3196 : }
3197 :
3198 17 : insns = end_sequence ();
3199 :
3200 17 : emit_insn (insns);
3201 : }
3202 : else
3203 : {
3204 1084 : rtx mask_rtx = immed_wide_int_const (mask, imode);
3205 1084 : if (VECTOR_MODE_P (new_mode))
3206 0 : mask_rtx = gen_const_vec_duplicate (new_mode, mask_rtx);
3207 1333 : temp = expand_binop (new_mode, code == ABS ? and_optab : xor_optab,
3208 1084 : gen_lowpart (new_mode, op0), mask_rtx,
3209 1084 : gen_lowpart (new_mode, target), 1, OPTAB_LIB_WIDEN);
3210 1084 : target = force_lowpart_subreg (mode, temp, new_mode);
3211 :
3212 1084 : set_dst_reg_note (get_last_insn (), REG_EQUAL,
3213 : gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
3214 : target);
3215 : }
3216 :
3217 1101 : return target;
3218 1101 : }
3219 :
3220 : /* As expand_unop, but will fail rather than attempt the operation in a
3221 : different mode or with a libcall. */
3222 : static rtx
3223 164211 : expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
3224 : int unsignedp)
3225 : {
3226 164211 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
3227 : {
3228 161803 : class expand_operand ops[2];
3229 161803 : enum insn_code icode = optab_handler (unoptab, mode);
3230 161803 : rtx_insn *last = get_last_insn ();
3231 161803 : rtx_insn *pat;
3232 :
3233 161803 : create_output_operand (&ops[0], target, mode);
3234 161803 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
3235 161803 : pat = maybe_gen_insn (icode, 2, ops);
3236 161803 : if (pat)
3237 : {
3238 161517 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3239 200714 : && ! add_equal_note (pat, ops[0].value,
3240 : optab_to_code (unoptab),
3241 : ops[1].value, NULL_RTX, mode))
3242 : {
3243 1069 : delete_insns_since (last);
3244 161803 : return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3245 : }
3246 :
3247 160734 : emit_insn (pat);
3248 :
3249 160734 : return ops[0].value;
3250 : }
3251 : }
3252 : return 0;
3253 : }
3254 :
3255 : /* Generate code to perform an operation specified by UNOPTAB
3256 : on operand OP0, with result having machine-mode MODE.
3257 :
3258 : UNSIGNEDP is for the case where we have to widen the operands
3259 : to perform the operation. It says to use zero-extension.
3260 :
3261 : If TARGET is nonzero, the value
3262 : is generated there, if it is convenient to do so.
3263 : In all cases an rtx is returned for the locus of the value;
3264 : this may or may not be TARGET. */
3265 :
3266 : rtx
3267 163620 : expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
3268 : int unsignedp)
3269 : {
3270 163620 : enum mode_class mclass = GET_MODE_CLASS (mode);
3271 163620 : machine_mode wider_mode;
3272 163620 : scalar_int_mode int_mode;
3273 163620 : scalar_float_mode float_mode;
3274 163620 : rtx temp;
3275 163620 : rtx libfunc;
3276 :
3277 163620 : temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3278 163620 : if (temp)
3279 : return temp;
3280 :
3281 : /* It can't be done in this mode. Can we open-code it in a wider mode? */
3282 :
3283 : /* Widening (or narrowing) clz needs special treatment. */
3284 2408 : if (unoptab == clz_optab)
3285 : {
3286 205 : if (is_a <scalar_int_mode> (mode, &int_mode))
3287 : {
3288 205 : temp = widen_leading (int_mode, op0, target, unoptab);
3289 205 : if (temp)
3290 : return temp;
3291 :
3292 410 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3293 205 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3294 : {
3295 205 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target,
3296 : unoptab);
3297 205 : if (temp)
3298 : return temp;
3299 : }
3300 : }
3301 :
3302 0 : goto try_libcall;
3303 : }
3304 :
3305 2203 : if (unoptab == clrsb_optab)
3306 : {
3307 91 : if (is_a <scalar_int_mode> (mode, &int_mode))
3308 : {
3309 91 : temp = widen_leading (int_mode, op0, target, unoptab);
3310 91 : if (temp)
3311 : return temp;
3312 91 : temp = expand_clrsb_using_clz (int_mode, op0, target);
3313 91 : if (temp)
3314 : return temp;
3315 : }
3316 4 : goto try_libcall;
3317 : }
3318 :
3319 2112 : if (unoptab == popcount_optab
3320 618 : && is_a <scalar_int_mode> (mode, &int_mode)
3321 650 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3322 2 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3323 2112 : && optimize_insn_for_speed_p ())
3324 : {
3325 0 : temp = expand_doubleword_popcount (int_mode, op0, target);
3326 0 : if (temp)
3327 : return temp;
3328 : }
3329 :
3330 2112 : if (unoptab == parity_optab
3331 11 : && is_a <scalar_int_mode> (mode, &int_mode)
3332 11 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3333 0 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3334 0 : || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
3335 2112 : && optimize_insn_for_speed_p ())
3336 : {
3337 0 : temp = expand_doubleword_parity (int_mode, op0, target);
3338 0 : if (temp)
3339 : return temp;
3340 : }
3341 :
3342 : /* Widening (or narrowing) bswap needs special treatment. */
3343 2112 : if (unoptab == bswap_optab)
3344 : {
3345 : /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3346 : or ROTATERT. First try these directly; if this fails, then try the
3347 : obvious pair of shifts with allowed widening, as this will probably
3348 : be always more efficient than the other fallback methods. */
3349 155 : if (mode == HImode)
3350 : {
3351 0 : rtx_insn *last;
3352 0 : rtx temp1, temp2;
3353 :
3354 0 : if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3355 : {
3356 0 : temp = expand_binop (mode, rotl_optab, op0,
3357 : gen_int_shift_amount (mode, 8),
3358 : target, unsignedp, OPTAB_DIRECT);
3359 0 : if (temp)
3360 : return temp;
3361 : }
3362 :
3363 0 : if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3364 : {
3365 0 : temp = expand_binop (mode, rotr_optab, op0,
3366 : gen_int_shift_amount (mode, 8),
3367 : target, unsignedp, OPTAB_DIRECT);
3368 0 : if (temp)
3369 : return temp;
3370 : }
3371 :
3372 0 : last = get_last_insn ();
3373 :
3374 0 : temp1 = expand_binop (mode, ashl_optab, op0,
3375 : gen_int_shift_amount (mode, 8), NULL_RTX,
3376 : unsignedp, OPTAB_WIDEN);
3377 0 : temp2 = expand_binop (mode, lshr_optab, op0,
3378 : gen_int_shift_amount (mode, 8), NULL_RTX,
3379 : unsignedp, OPTAB_WIDEN);
3380 0 : if (temp1 && temp2)
3381 : {
3382 0 : temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3383 : unsignedp, OPTAB_WIDEN);
3384 0 : if (temp)
3385 : return temp;
3386 : }
3387 :
3388 0 : delete_insns_since (last);
3389 : }
3390 :
3391 155 : if (is_a <scalar_int_mode> (mode, &int_mode))
3392 : {
3393 155 : temp = widen_bswap (int_mode, op0, target);
3394 155 : if (temp)
3395 : return temp;
3396 :
3397 : /* We do not provide a 128-bit bswap in libgcc so force the use of
3398 : a double bswap for 64-bit targets. */
3399 310 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3400 155 : && (UNITS_PER_WORD == 8
3401 148 : || optab_handler (unoptab, word_mode) != CODE_FOR_nothing))
3402 : {
3403 155 : temp = expand_doubleword_bswap (mode, op0, target);
3404 155 : if (temp)
3405 : return temp;
3406 : }
3407 : }
3408 :
3409 0 : goto try_libcall;
3410 : }
3411 :
3412 : /* Neg should be tried via expand_absneg_bit before widening. */
3413 1957 : if (optab_to_code (unoptab) == NEG)
3414 : {
3415 : /* Try negating floating point values by flipping the sign bit. */
3416 704 : if (is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3417 : {
3418 261 : temp = expand_absneg_bit (NEG, mode, float_mode, op0, target);
3419 261 : if (temp)
3420 : return temp;
3421 : }
3422 :
3423 : /* If there is no negation pattern, and we have no negative zero,
3424 : try subtracting from zero. */
3425 91 : if (!HONOR_SIGNED_ZEROS (mode))
3426 : {
3427 91 : temp = expand_binop (mode, (unoptab == negv_optab
3428 : ? subv_optab : sub_optab),
3429 : CONST0_RTX (mode), op0, target,
3430 : unsignedp, OPTAB_DIRECT);
3431 91 : if (temp)
3432 : return temp;
3433 : }
3434 : }
3435 :
3436 : /* ABS also needs to be handled similarly. */
3437 1696 : if (optab_to_code (unoptab) == ABS
3438 2622 : && is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3439 : {
3440 840 : temp = expand_absneg_bit (ABS, mode, float_mode, op0, target);
3441 840 : if (temp)
3442 : return temp;
3443 : }
3444 :
3445 856 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3446 3829 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3447 : {
3448 2973 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3449 : {
3450 0 : rtx xop0 = op0;
3451 0 : rtx_insn *last = get_last_insn ();
3452 :
3453 : /* For certain operations, we need not actually extend
3454 : the narrow operand, as long as we will truncate the
3455 : results to the same narrowness. */
3456 :
3457 0 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3458 0 : (unoptab == neg_optab
3459 0 : || unoptab == one_cmpl_optab)
3460 0 : && mclass == MODE_INT);
3461 :
3462 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3463 : unsignedp);
3464 :
3465 0 : if (temp)
3466 : {
3467 0 : if (mclass != MODE_INT
3468 0 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3469 : {
3470 0 : if (target == 0)
3471 0 : target = gen_reg_rtx (mode);
3472 0 : convert_move (target, temp, 0);
3473 0 : return target;
3474 : }
3475 : else
3476 0 : return gen_lowpart (mode, temp);
3477 : }
3478 : else
3479 0 : delete_insns_since (last);
3480 : }
3481 : }
3482 :
3483 : /* These can be done a word at a time. */
3484 856 : if (unoptab == one_cmpl_optab
3485 0 : && is_int_mode (mode, &int_mode)
3486 0 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
3487 856 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3488 : {
3489 0 : int i;
3490 0 : rtx_insn *insns;
3491 :
3492 0 : if (target == 0
3493 0 : || target == op0
3494 0 : || reg_overlap_mentioned_p (target, op0)
3495 0 : || !valid_multiword_target_p (target))
3496 0 : target = gen_reg_rtx (int_mode);
3497 :
3498 0 : start_sequence ();
3499 :
3500 : /* Do the actual arithmetic. */
3501 0 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
3502 : {
3503 0 : rtx target_piece = operand_subword (target, i, 1, int_mode);
3504 0 : rtx x = expand_unop (word_mode, unoptab,
3505 0 : operand_subword_force (op0, i, int_mode),
3506 : target_piece, unsignedp);
3507 :
3508 0 : if (target_piece != x)
3509 0 : emit_move_insn (target_piece, x);
3510 : }
3511 :
3512 0 : insns = end_sequence ();
3513 :
3514 0 : emit_insn (insns);
3515 0 : return target;
3516 : }
3517 :
3518 : /* Emit ~op0 as op0 ^ -1. */
3519 856 : if (unoptab == one_cmpl_optab
3520 0 : && (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
3521 856 : && optab_handler (xor_optab, mode) != CODE_FOR_nothing)
3522 : {
3523 0 : temp = expand_binop (mode, xor_optab, op0, CONSTM1_RTX (mode),
3524 : target, unsignedp, OPTAB_DIRECT);
3525 0 : if (temp)
3526 : return temp;
3527 : }
3528 :
3529 : /* Try calculating parity (x) as popcount (x) % 2. */
3530 856 : if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
3531 : {
3532 11 : temp = expand_parity (int_mode, op0, target);
3533 11 : if (temp)
3534 : return temp;
3535 : }
3536 :
3537 : /* Try implementing ffs (x) in terms of clz (x). */
3538 845 : if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
3539 : {
3540 0 : temp = expand_ffs (int_mode, op0, target);
3541 0 : if (temp)
3542 : return temp;
3543 : }
3544 :
3545 : /* Try implementing ctz (x) in terms of clz (x). */
3546 845 : if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
3547 : {
3548 50 : temp = expand_ctz (int_mode, op0, target);
3549 50 : if (temp)
3550 : return temp;
3551 : }
3552 :
3553 845 : if ((unoptab == ctz_optab || unoptab == ffs_optab)
3554 50 : && optimize_insn_for_speed_p ()
3555 47 : && is_a <scalar_int_mode> (mode, &int_mode)
3556 94 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3557 892 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3558 0 : || optab_handler (ctz_optab, word_mode) != CODE_FOR_nothing))
3559 : {
3560 47 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target, unoptab);
3561 47 : if (temp)
3562 : return temp;
3563 : }
3564 :
3565 802 : try_libcall:
3566 : /* Now try a library call in this mode. */
3567 802 : libfunc = optab_libfunc (unoptab, mode);
3568 802 : if (libfunc)
3569 : {
3570 518 : rtx_insn *insns;
3571 518 : rtx value;
3572 518 : rtx eq_value;
3573 518 : machine_mode outmode = mode;
3574 :
3575 : /* All of these functions return small values. Thus we choose to
3576 : have them return something that isn't a double-word. */
3577 518 : if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3578 : || unoptab == clrsb_optab || unoptab == popcount_optab
3579 518 : || unoptab == parity_optab)
3580 427 : outmode
3581 427 : = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3582 : optab_libfunc (unoptab, mode)));
3583 :
3584 518 : start_sequence ();
3585 :
3586 : /* Pass 1 for NO_QUEUE so we don't lose any increments
3587 : if the libcall is cse'd or moved. */
3588 518 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3589 : op0, mode);
3590 518 : insns = end_sequence ();
3591 :
3592 518 : target = gen_reg_rtx (outmode);
3593 518 : bool trapv = trapv_unoptab_p (unoptab);
3594 518 : if (trapv)
3595 : eq_value = NULL_RTX;
3596 : else
3597 : {
3598 427 : eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3599 1281 : if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3600 397 : eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3601 90 : else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3602 0 : eq_value = simplify_gen_unary (ZERO_EXTEND,
3603 : outmode, eq_value, mode);
3604 : }
3605 518 : emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3606 :
3607 518 : return target;
3608 : }
3609 :
3610 : /* It can't be done in this mode. Can we do it in a wider mode? */
3611 :
3612 284 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3613 : {
3614 627 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3615 : {
3616 541 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3617 541 : || optab_libfunc (unoptab, wider_mode))
3618 : {
3619 198 : rtx xop0 = op0;
3620 198 : rtx_insn *last = get_last_insn ();
3621 :
3622 : /* For certain operations, we need not actually extend
3623 : the narrow operand, as long as we will truncate the
3624 : results to the same narrowness. */
3625 198 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3626 198 : (unoptab == neg_optab
3627 198 : || unoptab == one_cmpl_optab
3628 198 : || unoptab == bswap_optab)
3629 198 : && mclass == MODE_INT);
3630 :
3631 198 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3632 : unsignedp);
3633 :
3634 : /* If we are generating clz using wider mode, adjust the
3635 : result. Similarly for clrsb. */
3636 198 : if ((unoptab == clz_optab || unoptab == clrsb_optab)
3637 1 : && temp != 0)
3638 : {
3639 1 : scalar_int_mode wider_int_mode
3640 1 : = as_a <scalar_int_mode> (wider_mode);
3641 1 : int_mode = as_a <scalar_int_mode> (mode);
3642 1 : temp = expand_binop
3643 1 : (wider_mode, sub_optab, temp,
3644 1 : gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3645 2 : - GET_MODE_PRECISION (int_mode),
3646 : wider_int_mode),
3647 : target, true, OPTAB_DIRECT);
3648 : }
3649 :
3650 : /* Likewise for bswap. */
3651 198 : if (unoptab == bswap_optab && temp != 0)
3652 : {
3653 0 : scalar_int_mode wider_int_mode
3654 0 : = as_a <scalar_int_mode> (wider_mode);
3655 0 : int_mode = as_a <scalar_int_mode> (mode);
3656 0 : gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3657 : == GET_MODE_BITSIZE (wider_int_mode)
3658 : && GET_MODE_PRECISION (int_mode)
3659 : == GET_MODE_BITSIZE (int_mode));
3660 :
3661 0 : temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3662 0 : GET_MODE_BITSIZE (wider_int_mode)
3663 0 : - GET_MODE_BITSIZE (int_mode),
3664 : NULL_RTX, true);
3665 : }
3666 :
3667 198 : if (temp)
3668 : {
3669 198 : if (mclass != MODE_INT)
3670 : {
3671 0 : if (target == 0)
3672 0 : target = gen_reg_rtx (mode);
3673 0 : convert_move (target, temp, 0);
3674 0 : return target;
3675 : }
3676 : else
3677 198 : return gen_lowpart (mode, temp);
3678 : }
3679 : else
3680 0 : delete_insns_since (last);
3681 : }
3682 : }
3683 : }
3684 :
3685 : /* One final attempt at implementing negation via subtraction,
3686 : this time allowing widening of the operand. */
3687 86 : if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3688 : {
3689 0 : rtx temp;
3690 0 : temp = expand_binop (mode,
3691 : unoptab == negv_optab ? subv_optab : sub_optab,
3692 : CONST0_RTX (mode), op0,
3693 : target, unsignedp, OPTAB_LIB_WIDEN);
3694 0 : if (temp)
3695 : return temp;
3696 : }
3697 :
3698 : return 0;
3699 : }
3700 :
3701 : /* Emit code to compute the absolute value of OP0, with result to
3702 : TARGET if convenient. (TARGET may be 0.) The return value says
3703 : where the result actually is to be found.
3704 :
3705 : MODE is the mode of the operand; the mode of the result is
3706 : different but can be deduced from MODE.
3707 :
3708 : */
3709 :
3710 : rtx
3711 23439 : expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3712 : int result_unsignedp)
3713 : {
3714 23439 : rtx temp;
3715 :
3716 23439 : if (GET_MODE_CLASS (mode) != MODE_INT
3717 4076 : || ! flag_trapv)
3718 : result_unsignedp = 1;
3719 :
3720 : /* First try to do it with a special abs instruction. */
3721 23525 : temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3722 : op0, target, 0);
3723 23439 : if (temp != 0)
3724 : return temp;
3725 :
3726 : /* If we have a MAX insn, we can do this as MAX (x, -x). */
3727 86 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3728 86 : && !HONOR_SIGNED_ZEROS (mode))
3729 : {
3730 86 : rtx_insn *last = get_last_insn ();
3731 :
3732 172 : temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3733 : op0, NULL_RTX, 0);
3734 86 : if (temp != 0)
3735 86 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3736 : OPTAB_WIDEN);
3737 :
3738 86 : if (temp != 0)
3739 : return temp;
3740 :
3741 0 : delete_insns_since (last);
3742 : }
3743 :
3744 : /* If this machine has expensive jumps, we can do integer absolute
3745 : value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3746 : where W is the width of MODE. */
3747 :
3748 0 : scalar_int_mode int_mode;
3749 0 : if (is_int_mode (mode, &int_mode)
3750 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
3751 : false) >= 2)
3752 : {
3753 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3754 0 : GET_MODE_PRECISION (int_mode) - 1,
3755 : NULL_RTX, 0);
3756 :
3757 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3758 : OPTAB_LIB_WIDEN);
3759 0 : if (temp != 0)
3760 0 : temp = expand_binop (int_mode,
3761 : result_unsignedp ? sub_optab : subv_optab,
3762 : temp, extended, target, 0, OPTAB_LIB_WIDEN);
3763 :
3764 0 : if (temp != 0)
3765 : return temp;
3766 : }
3767 :
3768 : return NULL_RTX;
3769 : }
3770 :
3771 : rtx
3772 23415 : expand_abs (machine_mode mode, rtx op0, rtx target,
3773 : int result_unsignedp, int safe)
3774 : {
3775 23415 : rtx temp;
3776 23415 : rtx_code_label *op1;
3777 :
3778 23415 : if (GET_MODE_CLASS (mode) != MODE_INT
3779 4052 : || ! flag_trapv)
3780 23329 : result_unsignedp = 1;
3781 :
3782 23415 : temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3783 23415 : if (temp != 0)
3784 : return temp;
3785 :
3786 : /* If that does not win, use conditional jump and negate. */
3787 :
3788 : /* It is safe to use the target if it is the same
3789 : as the source if this is also a pseudo register */
3790 0 : if (op0 == target && REG_P (op0)
3791 0 : && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3792 : safe = 1;
3793 :
3794 0 : op1 = gen_label_rtx ();
3795 0 : if (target == 0 || ! safe
3796 0 : || GET_MODE (target) != mode
3797 0 : || (MEM_P (target) && MEM_VOLATILE_P (target))
3798 0 : || (REG_P (target)
3799 0 : && REGNO (target) < FIRST_PSEUDO_REGISTER))
3800 0 : target = gen_reg_rtx (mode);
3801 :
3802 0 : emit_move_insn (target, op0);
3803 0 : NO_DEFER_POP;
3804 :
3805 0 : do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3806 : NULL_RTX, NULL, op1,
3807 : profile_probability::uninitialized ());
3808 :
3809 0 : op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3810 : target, target, 0);
3811 0 : if (op0 != target)
3812 0 : emit_move_insn (target, op0);
3813 0 : emit_label (op1);
3814 0 : OK_DEFER_POP;
3815 0 : return target;
3816 : }
3817 :
3818 : /* Emit code to compute the one's complement absolute value of OP0
3819 : (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3820 : (TARGET may be NULL_RTX.) The return value says where the result
3821 : actually is to be found.
3822 :
3823 : MODE is the mode of the operand; the mode of the result is
3824 : different but can be deduced from MODE. */
3825 :
3826 : rtx
3827 0 : expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3828 : {
3829 0 : rtx temp;
3830 :
3831 : /* Not applicable for floating point modes. */
3832 0 : if (FLOAT_MODE_P (mode))
3833 : return NULL_RTX;
3834 :
3835 : /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3836 0 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3837 : {
3838 0 : rtx_insn *last = get_last_insn ();
3839 :
3840 0 : temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3841 0 : if (temp != 0)
3842 0 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3843 : OPTAB_WIDEN);
3844 :
3845 0 : if (temp != 0)
3846 : return temp;
3847 :
3848 0 : delete_insns_since (last);
3849 : }
3850 :
3851 : /* If this machine has expensive jumps, we can do one's complement
3852 : absolute value of X as (((signed) x >> (W-1)) ^ x). */
3853 :
3854 0 : scalar_int_mode int_mode;
3855 0 : if (is_int_mode (mode, &int_mode)
3856 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
3857 : false) >= 2)
3858 : {
3859 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3860 0 : GET_MODE_PRECISION (int_mode) - 1,
3861 : NULL_RTX, 0);
3862 :
3863 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3864 : OPTAB_LIB_WIDEN);
3865 :
3866 0 : if (temp != 0)
3867 : return temp;
3868 : }
3869 :
3870 : return NULL_RTX;
3871 : }
3872 :
3873 : /* A subroutine of expand_copysign, perform the copysign operation using the
3874 : abs and neg primitives advertised to exist on the target. The assumption
3875 : is that we have a split register file, and leaving op0 in fp registers,
3876 : and not playing with subregs so much, will help the register allocator. */
3877 :
3878 : static rtx
3879 11604 : expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3880 : int bitpos, bool op0_is_abs)
3881 : {
3882 11604 : scalar_int_mode imode;
3883 11604 : enum insn_code icode;
3884 11604 : rtx sign;
3885 11604 : rtx_code_label *label;
3886 :
3887 11604 : if (target == op1)
3888 132 : target = NULL_RTX;
3889 :
3890 : /* Check if the back end provides an insn that handles signbit for the
3891 : argument's mode. */
3892 11604 : icode = optab_handler (signbit_optab, mode);
3893 11604 : if (icode != CODE_FOR_nothing)
3894 : {
3895 11564 : imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
3896 11564 : sign = gen_reg_rtx (imode);
3897 11564 : emit_unop_insn (icode, sign, op1, UNKNOWN);
3898 : }
3899 : else
3900 : {
3901 100 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3902 : {
3903 40 : if (!int_mode_for_mode (mode).exists (&imode))
3904 0 : return NULL_RTX;
3905 40 : op1 = gen_lowpart (imode, op1);
3906 : }
3907 : else
3908 : {
3909 0 : int word;
3910 :
3911 0 : imode = word_mode;
3912 0 : if (FLOAT_WORDS_BIG_ENDIAN)
3913 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3914 : else
3915 0 : word = bitpos / BITS_PER_WORD;
3916 0 : bitpos = bitpos % BITS_PER_WORD;
3917 0 : op1 = operand_subword_force (op1, word, mode);
3918 : }
3919 :
3920 40 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3921 80 : sign = expand_binop (imode, and_optab, op1,
3922 80 : immed_wide_int_const (mask, imode),
3923 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
3924 40 : }
3925 :
3926 11604 : if (!op0_is_abs)
3927 : {
3928 150 : op0 = expand_unop (mode, abs_optab, op0, target, 0);
3929 150 : if (op0 == NULL)
3930 : return NULL_RTX;
3931 : target = op0;
3932 : }
3933 : else
3934 : {
3935 11454 : if (target == NULL_RTX)
3936 84 : target = copy_to_reg (op0);
3937 : else
3938 11370 : emit_move_insn (target, op0);
3939 : }
3940 :
3941 11604 : label = gen_label_rtx ();
3942 11604 : emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
3943 :
3944 11604 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
3945 11454 : op0 = simplify_unary_operation (NEG, mode, op0, mode);
3946 : else
3947 150 : op0 = expand_unop (mode, neg_optab, op0, target, 0);
3948 11604 : if (op0 != target)
3949 11454 : emit_move_insn (target, op0);
3950 :
3951 11604 : emit_label (label);
3952 :
3953 11604 : return target;
3954 : }
3955 :
3956 :
3957 : /* A subroutine of expand_copysign, perform the entire copysign operation
3958 : with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
3959 : is true if op0 is known to have its sign bit clear. */
3960 :
3961 : static rtx
3962 68 : expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
3963 : int bitpos, bool op0_is_abs)
3964 : {
3965 68 : scalar_int_mode imode;
3966 68 : int word, nwords, i;
3967 68 : rtx temp;
3968 68 : rtx_insn *insns;
3969 :
3970 156 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
3971 : {
3972 68 : if (!int_mode_for_mode (mode).exists (&imode))
3973 0 : return NULL_RTX;
3974 : word = 0;
3975 : nwords = 1;
3976 : }
3977 : else
3978 : {
3979 0 : imode = word_mode;
3980 :
3981 0 : if (FLOAT_WORDS_BIG_ENDIAN)
3982 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
3983 : else
3984 0 : word = bitpos / BITS_PER_WORD;
3985 0 : bitpos = bitpos % BITS_PER_WORD;
3986 0 : nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3987 : }
3988 :
3989 68 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3990 :
3991 68 : if (target == 0
3992 68 : || target == op0
3993 68 : || target == op1
3994 64 : || reg_overlap_mentioned_p (target, op0)
3995 64 : || reg_overlap_mentioned_p (target, op1)
3996 132 : || (nwords > 1 && !valid_multiword_target_p (target)))
3997 4 : target = gen_reg_rtx (mode);
3998 :
3999 68 : if (nwords > 1)
4000 : {
4001 0 : start_sequence ();
4002 :
4003 0 : for (i = 0; i < nwords; ++i)
4004 : {
4005 0 : rtx targ_piece = operand_subword (target, i, 1, mode);
4006 0 : rtx op0_piece = operand_subword_force (op0, i, mode);
4007 :
4008 0 : if (i == word)
4009 : {
4010 0 : if (!op0_is_abs)
4011 0 : op0_piece
4012 0 : = expand_binop (imode, and_optab, op0_piece,
4013 0 : immed_wide_int_const (~mask, imode),
4014 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4015 0 : op1 = expand_binop (imode, and_optab,
4016 0 : operand_subword_force (op1, i, mode),
4017 0 : immed_wide_int_const (mask, imode),
4018 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4019 :
4020 0 : temp = expand_binop (imode, ior_optab, op0_piece, op1,
4021 : targ_piece, 1, OPTAB_LIB_WIDEN);
4022 0 : if (temp != targ_piece)
4023 0 : emit_move_insn (targ_piece, temp);
4024 : }
4025 : else
4026 0 : emit_move_insn (targ_piece, op0_piece);
4027 : }
4028 :
4029 0 : insns = end_sequence ();
4030 :
4031 0 : emit_insn (insns);
4032 : }
4033 : else
4034 : {
4035 136 : op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
4036 68 : immed_wide_int_const (mask, imode),
4037 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4038 :
4039 68 : op0 = gen_lowpart (imode, op0);
4040 68 : if (!op0_is_abs)
4041 136 : op0 = expand_binop (imode, and_optab, op0,
4042 136 : immed_wide_int_const (~mask, imode),
4043 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4044 :
4045 68 : temp = expand_binop (imode, ior_optab, op0, op1,
4046 68 : gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
4047 68 : target = force_lowpart_subreg (mode, temp, imode);
4048 : }
4049 :
4050 68 : return target;
4051 68 : }
4052 :
4053 : /* Expand the C99 copysign operation. OP0 and OP1 must be the same
4054 : scalar floating point mode. Return NULL if we do not know how to
4055 : expand the operation inline. */
4056 :
4057 : rtx
4058 11672 : expand_copysign (rtx op0, rtx op1, rtx target)
4059 : {
4060 11672 : scalar_float_mode mode;
4061 11672 : const struct real_format *fmt;
4062 11672 : bool op0_is_abs;
4063 11672 : rtx temp;
4064 :
4065 11672 : mode = as_a <scalar_float_mode> (GET_MODE (op0));
4066 11672 : gcc_assert (GET_MODE (op1) == mode);
4067 :
4068 : /* First try to do it with a special instruction. */
4069 11672 : temp = expand_binop (mode, copysign_optab, op0, op1,
4070 : target, 0, OPTAB_DIRECT);
4071 11672 : if (temp)
4072 : return temp;
4073 :
4074 11672 : fmt = REAL_MODE_FORMAT (mode);
4075 11672 : if (fmt == NULL || !fmt->has_signed_zero)
4076 : return NULL_RTX;
4077 :
4078 11672 : op0_is_abs = false;
4079 11672 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
4080 : {
4081 11454 : if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
4082 0 : op0 = simplify_unary_operation (ABS, mode, op0, mode);
4083 : op0_is_abs = true;
4084 : }
4085 :
4086 11672 : if (fmt->signbit_ro >= 0
4087 11672 : && (CONST_DOUBLE_AS_FLOAT_P (op0)
4088 218 : || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
4089 150 : && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
4090 : {
4091 23208 : temp = expand_copysign_absneg (mode, op0, op1, target,
4092 11604 : fmt->signbit_ro, op0_is_abs);
4093 11604 : if (temp)
4094 : return temp;
4095 : }
4096 :
4097 68 : if (fmt->signbit_rw < 0)
4098 : return NULL_RTX;
4099 68 : return expand_copysign_bit (mode, op0, op1, target,
4100 68 : fmt->signbit_rw, op0_is_abs);
4101 : }
4102 :
4103 : /* Generate an instruction whose insn-code is INSN_CODE,
4104 : with two operands: an output TARGET and an input OP0.
4105 : TARGET *must* be nonzero, and the output is always stored there.
4106 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4107 : the value that is stored into TARGET.
4108 :
4109 : Return false if expansion failed. */
4110 :
4111 : bool
4112 2137035 : maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
4113 : enum rtx_code code)
4114 : {
4115 2137035 : class expand_operand ops[2];
4116 2137035 : rtx_insn *pat;
4117 :
4118 2137035 : create_output_operand (&ops[0], target, GET_MODE (target));
4119 2137035 : create_input_operand (&ops[1], op0, GET_MODE (op0));
4120 2137035 : pat = maybe_gen_insn (icode, 2, ops);
4121 2137035 : if (!pat)
4122 : return false;
4123 :
4124 1195525 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
4125 2159798 : && code != UNKNOWN)
4126 11047 : add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
4127 11047 : GET_MODE (op0));
4128 :
4129 2137034 : emit_insn (pat);
4130 :
4131 2137034 : if (ops[0].value != target)
4132 38310 : emit_move_insn (target, ops[0].value);
4133 : return true;
4134 : }
4135 : /* Generate an instruction whose insn-code is INSN_CODE,
4136 : with two operands: an output TARGET and an input OP0.
4137 : TARGET *must* be nonzero, and the output is always stored there.
4138 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4139 : the value that is stored into TARGET. */
4140 :
4141 : void
4142 2090801 : emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
4143 : {
4144 2090801 : bool ok = maybe_emit_unop_insn (icode, target, op0, code);
4145 2090801 : gcc_assert (ok);
4146 2090801 : }
4147 :
4148 : struct no_conflict_data
4149 : {
4150 : rtx target;
4151 : rtx_insn *first, *insn;
4152 : bool must_stay;
4153 : };
4154 :
4155 : /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4156 : the currently examined clobber / store has to stay in the list of
4157 : insns that constitute the actual libcall block. */
4158 : static void
4159 56737 : no_conflict_move_test (rtx dest, const_rtx set, void *p0)
4160 : {
4161 56737 : struct no_conflict_data *p= (struct no_conflict_data *) p0;
4162 :
4163 : /* If this inns directly contributes to setting the target, it must stay. */
4164 56737 : if (reg_overlap_mentioned_p (p->target, dest))
4165 0 : p->must_stay = true;
4166 : /* If we haven't committed to keeping any other insns in the list yet,
4167 : there is nothing more to check. */
4168 56737 : else if (p->insn == p->first)
4169 : return;
4170 : /* If this insn sets / clobbers a register that feeds one of the insns
4171 : already in the list, this insn has to stay too. */
4172 26960 : else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
4173 26960 : || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
4174 26960 : || reg_used_between_p (dest, p->first, p->insn)
4175 : /* Likewise if this insn depends on a register set by a previous
4176 : insn in the list, or if it sets a result (presumably a hard
4177 : register) that is set or clobbered by a previous insn.
4178 : N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4179 : SET_DEST perform the former check on the address, and the latter
4180 : check on the MEM. */
4181 53920 : || (GET_CODE (set) == SET
4182 26960 : && (modified_in_p (SET_SRC (set), p->first)
4183 26958 : || modified_in_p (SET_DEST (set), p->first)
4184 26958 : || modified_between_p (SET_SRC (set), p->first, p->insn)
4185 26958 : || modified_between_p (SET_DEST (set), p->first, p->insn))))
4186 2 : p->must_stay = true;
4187 : }
4188 :
4189 :
4190 : /* Emit code to make a call to a constant function or a library call.
4191 :
4192 : INSNS is a list containing all insns emitted in the call.
4193 : These insns leave the result in RESULT. Our block is to copy RESULT
4194 : to TARGET, which is logically equivalent to EQUIV.
4195 :
4196 : We first emit any insns that set a pseudo on the assumption that these are
4197 : loading constants into registers; doing so allows them to be safely cse'ed
4198 : between blocks. Then we emit all the other insns in the block, followed by
4199 : an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4200 : note with an operand of EQUIV. */
4201 :
4202 : static void
4203 115809 : emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
4204 : bool equiv_may_trap)
4205 : {
4206 115809 : rtx final_dest = target;
4207 115809 : rtx_insn *next, *last, *insn;
4208 :
4209 : /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4210 : into a MEM later. Protect the libcall block from this change. */
4211 115809 : if (! REG_P (target) || REG_USERVAR_P (target))
4212 1810 : target = gen_reg_rtx (GET_MODE (target));
4213 :
4214 : /* If we're using non-call exceptions, a libcall corresponding to an
4215 : operation that may trap may also trap. */
4216 : /* ??? See the comment in front of make_reg_eh_region_note. */
4217 115809 : if (cfun->can_throw_non_call_exceptions
4218 115809 : && (equiv_may_trap || may_trap_p (equiv)))
4219 : {
4220 1036 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4221 832 : if (CALL_P (insn))
4222 : {
4223 204 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
4224 204 : if (note)
4225 : {
4226 204 : int lp_nr = INTVAL (XEXP (note, 0));
4227 204 : if (lp_nr == 0 || lp_nr == INT_MIN)
4228 204 : remove_note (insn, note);
4229 : }
4230 : }
4231 : }
4232 : else
4233 : {
4234 : /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4235 : reg note to indicate that this call cannot throw or execute a nonlocal
4236 : goto (unless there is already a REG_EH_REGION note, in which case
4237 : we update it). */
4238 511613 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4239 396008 : if (CALL_P (insn))
4240 115605 : make_reg_eh_region_note_nothrow_nononlocal (insn);
4241 : }
4242 :
4243 : /* First emit all insns that set pseudos. Remove them from the list as
4244 : we go. Avoid insns that set pseudos which were referenced in previous
4245 : insns. These can be generated by move_by_pieces, for example,
4246 : to update an address. Similarly, avoid insns that reference things
4247 : set in previous insns. */
4248 :
4249 512649 : for (insn = insns; insn; insn = next)
4250 : {
4251 396840 : rtx set = single_set (insn);
4252 :
4253 396840 : next = NEXT_INSN (insn);
4254 :
4255 382346 : if (set != 0 && REG_P (SET_DEST (set))
4256 741974 : && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
4257 : {
4258 46644 : struct no_conflict_data data;
4259 :
4260 46644 : data.target = const0_rtx;
4261 46644 : data.first = insns;
4262 46644 : data.insn = insn;
4263 46644 : data.must_stay = 0;
4264 46644 : note_stores (insn, no_conflict_move_test, &data);
4265 46644 : if (! data.must_stay)
4266 : {
4267 46642 : if (PREV_INSN (insn))
4268 26958 : SET_NEXT_INSN (PREV_INSN (insn)) = next;
4269 : else
4270 : insns = next;
4271 :
4272 46642 : if (next)
4273 46642 : SET_PREV_INSN (next) = PREV_INSN (insn);
4274 :
4275 46642 : add_insn (insn);
4276 : }
4277 : }
4278 :
4279 : /* Some ports use a loop to copy large arguments onto the stack.
4280 : Don't move anything outside such a loop. */
4281 396840 : if (LABEL_P (insn))
4282 : break;
4283 : }
4284 :
4285 : /* Write the remaining insns followed by the final copy. */
4286 466007 : for (insn = insns; insn; insn = next)
4287 : {
4288 350198 : next = NEXT_INSN (insn);
4289 :
4290 350198 : add_insn (insn);
4291 : }
4292 :
4293 115809 : last = emit_move_insn (target, result);
4294 115809 : if (equiv)
4295 115201 : set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
4296 :
4297 115809 : if (final_dest != target)
4298 1810 : emit_move_insn (final_dest, target);
4299 115809 : }
4300 :
4301 : void
4302 84831 : emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
4303 : {
4304 84831 : emit_libcall_block_1 (insns, target, result, equiv, false);
4305 84831 : }
4306 :
4307 : /* True if we can perform a comparison of mode MODE straightforwardly.
4308 : PURPOSE describes how this comparison will be used. CODE is the rtx
4309 : comparison code we will be using.
4310 :
4311 : ??? Actually, CODE is slightly weaker than that. A target is still
4312 : required to implement all of the normal bcc operations, but not
4313 : required to implement all (or any) of the unordered bcc operations. */
4314 :
4315 : bool
4316 17778414 : can_compare_p (enum rtx_code code, machine_mode mode,
4317 : enum can_compare_purpose purpose)
4318 : {
4319 17778414 : rtx test;
4320 17778414 : test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
4321 19818755 : do
4322 : {
4323 19818755 : enum insn_code icode;
4324 :
4325 19818755 : if (purpose == ccp_jump
4326 19622409 : && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
4327 38103806 : && insn_operand_matches (icode, 0, test))
4328 : return true;
4329 3314150 : if (purpose == ccp_store_flag
4330 196346 : && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
4331 3443041 : && insn_operand_matches (icode, 1, test))
4332 : return true;
4333 :
4334 3311715 : mode = GET_MODE_WIDER_MODE (mode).else_void ();
4335 3311715 : PUT_MODE (test, mode);
4336 : }
4337 3311715 : while (mode != VOIDmode);
4338 :
4339 : return false;
4340 : }
4341 :
4342 : /* Return whether RTL code CODE corresponds to an unsigned optab. */
4343 :
4344 : static bool
4345 1362582 : unsigned_optab_p (enum rtx_code code)
4346 : {
4347 1362582 : return code == LTU || code == LEU || code == GTU || code == GEU;
4348 : }
4349 :
4350 : /* Return whether the backend-emitted comparison for code CODE, comparing
4351 : operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4352 : operand OPNO of pattern ICODE. */
4353 :
4354 : static bool
4355 992423 : insn_predicate_matches_p (enum insn_code icode, unsigned int opno,
4356 : enum rtx_code code, machine_mode mask_mode,
4357 : machine_mode value_mode)
4358 : {
4359 992423 : rtx reg1 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 1);
4360 992423 : rtx reg2 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 2);
4361 992423 : rtx test = alloca_rtx_fmt_ee (code, mask_mode, reg1, reg2);
4362 992423 : return insn_operand_matches (icode, opno, test);
4363 : }
4364 :
4365 : /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4366 : for code CODE, comparing operands of mode VALUE_MODE and producing a result
4367 : with MASK_MODE. */
4368 :
4369 : bool
4370 1362582 : can_vec_cmp_compare_p (enum rtx_code code, machine_mode value_mode,
4371 : machine_mode mask_mode)
4372 : {
4373 1362582 : enum insn_code icode
4374 1362582 : = get_vec_cmp_icode (value_mode, mask_mode, unsigned_optab_p (code));
4375 1362582 : if (icode == CODE_FOR_nothing)
4376 : return false;
4377 :
4378 992423 : return insn_predicate_matches_p (icode, 1, code, mask_mode, value_mode);
4379 : }
4380 :
4381 : /* Return whether the backend can emit vector set instructions for inserting
4382 : element into vector at variable index position. */
4383 :
4384 : bool
4385 165 : can_vec_set_var_idx_p (machine_mode vec_mode)
4386 : {
4387 165 : if (!VECTOR_MODE_P (vec_mode))
4388 : return false;
4389 :
4390 163 : machine_mode inner_mode = GET_MODE_INNER (vec_mode);
4391 :
4392 163 : rtx reg1 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 1);
4393 163 : rtx reg2 = alloca_raw_REG (inner_mode, LAST_VIRTUAL_REGISTER + 2);
4394 :
4395 163 : enum insn_code icode = optab_handler (vec_set_optab, vec_mode);
4396 :
4397 163 : const struct insn_data_d *data = &insn_data[icode];
4398 163 : machine_mode idx_mode = data->operand[2].mode;
4399 :
4400 163 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4401 :
4402 162 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4403 162 : && insn_operand_matches (icode, 1, reg2)
4404 325 : && insn_operand_matches (icode, 2, reg3);
4405 : }
4406 :
4407 : /* Return whether the backend can emit a vec_extract instruction with
4408 : a non-constant index. */
4409 : bool
4410 18500 : can_vec_extract_var_idx_p (machine_mode vec_mode, machine_mode extr_mode)
4411 : {
4412 18500 : if (!VECTOR_MODE_P (vec_mode))
4413 : return false;
4414 :
4415 15924 : rtx reg1 = alloca_raw_REG (extr_mode, LAST_VIRTUAL_REGISTER + 1);
4416 15924 : rtx reg2 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 2);
4417 :
4418 15924 : enum insn_code icode = convert_optab_handler (vec_extract_optab,
4419 : vec_mode, extr_mode);
4420 :
4421 15924 : const struct insn_data_d *data = &insn_data[icode];
4422 15924 : machine_mode idx_mode = data->operand[2].mode;
4423 :
4424 15924 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4425 :
4426 15064 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4427 15064 : && insn_operand_matches (icode, 1, reg2)
4428 30988 : && insn_operand_matches (icode, 2, reg3);
4429 : }
4430 :
4431 : /* This function is called when we are going to emit a compare instruction that
4432 : compares the values found in X and Y, using the rtl operator COMPARISON.
4433 :
4434 : If they have mode BLKmode, then SIZE specifies the size of both operands.
4435 :
4436 : UNSIGNEDP nonzero says that the operands are unsigned;
4437 : this matters if they need to be widened (as given by METHODS).
4438 :
4439 : *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4440 : if we failed to produce one.
4441 :
4442 : *PMODE is the mode of the inputs (in case they are const_int).
4443 :
4444 : *OPTAB is the optab to check for OPTAB_DIRECT support. Defaults to
4445 : cbranch_optab.
4446 :
4447 : This function performs all the setup necessary so that the caller only has
4448 : to emit a single comparison insn. This setup can involve doing a BLKmode
4449 : comparison or emitting a library call to perform the comparison if no insn
4450 : is available to handle it.
4451 : The values which are passed in through pointers can be modified; the caller
4452 : should perform the comparison on the modified values. Constant
4453 : comparisons must have already been folded. */
4454 :
4455 : static void
4456 6750026 : prepare_cmp_insn (rtx x, rtx y, rtx *mask, enum rtx_code comparison, rtx size,
4457 : int unsignedp, enum optab_methods methods,
4458 : rtx *ptest, machine_mode *pmode, optab optab)
4459 : {
4460 6750026 : machine_mode mode = *pmode;
4461 6750026 : rtx libfunc, test;
4462 6750026 : machine_mode cmp_mode;
4463 :
4464 : /* The other methods are not needed. */
4465 6750026 : gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4466 : || methods == OPTAB_LIB_WIDEN);
4467 :
4468 6750026 : if (CONST_SCALAR_INT_P (y))
4469 4370764 : canonicalize_comparison (mode, &comparison, &y);
4470 :
4471 : /* If we are optimizing, force expensive constants into a register. */
4472 1 : if (CONSTANT_P (x) && optimize
4473 1 : && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
4474 : > COSTS_N_INSNS (1))
4475 6750026 : && can_create_pseudo_p ())
4476 0 : x = force_reg (mode, x);
4477 :
4478 4882019 : if (CONSTANT_P (y) && optimize
4479 3855407 : && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
4480 : > COSTS_N_INSNS (1))
4481 6924367 : && can_create_pseudo_p ())
4482 174341 : y = force_reg (mode, y);
4483 :
4484 : /* Don't let both operands fail to indicate the mode. */
4485 6750026 : if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4486 0 : x = force_reg (mode, x);
4487 6750026 : if (mode == VOIDmode)
4488 4740 : mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4489 :
4490 : /* Handle all BLKmode compares. */
4491 :
4492 6750026 : if (mode == BLKmode)
4493 : {
4494 0 : machine_mode result_mode;
4495 0 : enum insn_code cmp_code;
4496 0 : rtx result;
4497 0 : rtx opalign
4498 0 : = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4499 :
4500 0 : gcc_assert (size);
4501 :
4502 : /* Try to use a memory block compare insn - either cmpstr
4503 : or cmpmem will do. */
4504 0 : opt_scalar_int_mode cmp_mode_iter;
4505 0 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
4506 : {
4507 0 : scalar_int_mode cmp_mode = cmp_mode_iter.require ();
4508 0 : cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4509 0 : if (cmp_code == CODE_FOR_nothing)
4510 0 : cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4511 0 : if (cmp_code == CODE_FOR_nothing)
4512 0 : cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4513 0 : if (cmp_code == CODE_FOR_nothing)
4514 0 : continue;
4515 :
4516 : /* Must make sure the size fits the insn's mode. */
4517 0 : if (CONST_INT_P (size)
4518 0 : ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
4519 0 : : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
4520 0 : > GET_MODE_BITSIZE (cmp_mode)))
4521 0 : continue;
4522 :
4523 0 : result_mode = insn_data[cmp_code].operand[0].mode;
4524 0 : result = gen_reg_rtx (result_mode);
4525 0 : size = convert_to_mode (cmp_mode, size, 1);
4526 0 : emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4527 :
4528 0 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4529 0 : *pmode = result_mode;
4530 0 : return;
4531 : }
4532 :
4533 0 : if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4534 0 : goto fail;
4535 :
4536 : /* Otherwise call a library function. */
4537 0 : result = emit_block_comp_via_libcall (x, y, size);
4538 :
4539 0 : x = result;
4540 0 : y = const0_rtx;
4541 0 : mode = TYPE_MODE (integer_type_node);
4542 0 : methods = OPTAB_LIB_WIDEN;
4543 0 : unsignedp = false;
4544 : }
4545 :
4546 : /* Don't allow operands to the compare to trap, as that can put the
4547 : compare and branch in different basic blocks. */
4548 6750026 : if (cfun->can_throw_non_call_exceptions)
4549 : {
4550 1134328 : if (!can_create_pseudo_p () && (may_trap_p (x) || may_trap_p (y)))
4551 0 : goto fail;
4552 1134328 : if (may_trap_p (x))
4553 25098 : x = copy_to_reg (x);
4554 1134328 : if (may_trap_p (y))
4555 1522 : y = copy_to_reg (y);
4556 : }
4557 :
4558 6750026 : if (GET_MODE_CLASS (mode) == MODE_CC)
4559 : {
4560 492 : enum insn_code icode = optab_handler (cbranch_optab, CCmode);
4561 492 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4562 492 : if (icode != CODE_FOR_nothing
4563 492 : && insn_operand_matches (icode, 0, test))
4564 : {
4565 492 : *ptest = test;
4566 492 : return;
4567 : }
4568 : else
4569 0 : goto fail;
4570 : }
4571 :
4572 6749534 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4573 6834525 : FOR_EACH_WIDER_MODE_FROM (cmp_mode, mode)
4574 : {
4575 6787824 : enum insn_code icode;
4576 6787824 : icode = optab_handler (optab, cmp_mode);
4577 6787824 : if (icode != CODE_FOR_nothing
4578 6787824 : && insn_operand_matches (icode, 0, test))
4579 : {
4580 6702833 : rtx_insn *last = get_last_insn ();
4581 6702833 : rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4582 6702833 : rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4583 6702833 : if (op0 && op1
4584 6702833 : && insn_operand_matches (icode, 1, op0)
4585 13405666 : && insn_operand_matches (icode, 2, op1))
4586 : {
4587 6702833 : XEXP (test, 0) = op0;
4588 6702833 : XEXP (test, 1) = op1;
4589 6702833 : *ptest = test;
4590 6702833 : *pmode = cmp_mode;
4591 6702833 : return;
4592 : }
4593 0 : delete_insns_since (last);
4594 : }
4595 :
4596 84991 : if (methods == OPTAB_DIRECT)
4597 : break;
4598 : }
4599 :
4600 46701 : if (methods != OPTAB_LIB_WIDEN)
4601 2897 : goto fail;
4602 :
4603 43804 : if (SCALAR_FLOAT_MODE_P (mode))
4604 : {
4605 : /* Small trick if UNORDERED isn't implemented by the hardware. */
4606 43804 : if (comparison == UNORDERED && rtx_equal_p (x, y))
4607 : {
4608 761 : prepare_cmp_insn (x, y, mask, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
4609 : ptest, pmode, optab);
4610 761 : if (*ptest)
4611 : return;
4612 : }
4613 :
4614 43804 : prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4615 : }
4616 : else
4617 : {
4618 0 : rtx result;
4619 0 : machine_mode ret_mode;
4620 :
4621 : /* Handle a libcall just for the mode we are using. */
4622 0 : libfunc = optab_libfunc (cmp_optab, mode);
4623 0 : gcc_assert (libfunc);
4624 :
4625 : /* If we want unsigned, and this mode has a distinct unsigned
4626 : comparison routine, use that. */
4627 0 : if (unsignedp)
4628 : {
4629 0 : rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4630 0 : if (ulibfunc)
4631 0 : libfunc = ulibfunc;
4632 : }
4633 :
4634 0 : ret_mode = targetm.libgcc_cmp_return_mode ();
4635 0 : result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4636 : ret_mode, x, mode, y, mode);
4637 :
4638 : /* There are two kinds of comparison routines. Biased routines
4639 : return 0/1/2, and unbiased routines return -1/0/1. Other parts
4640 : of gcc expect that the comparison operation is equivalent
4641 : to the modified comparison. For signed comparisons compare the
4642 : result against 1 in the biased case, and zero in the unbiased
4643 : case. For unsigned comparisons always compare against 1 after
4644 : biasing the unbiased result by adding 1. This gives us a way to
4645 : represent LTU.
4646 : The comparisons in the fixed-point helper library are always
4647 : biased. */
4648 0 : x = result;
4649 0 : y = const1_rtx;
4650 :
4651 0 : if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4652 : {
4653 : if (unsignedp)
4654 : x = plus_constant (ret_mode, result, 1);
4655 : else
4656 : y = const0_rtx;
4657 : }
4658 :
4659 0 : *pmode = ret_mode;
4660 0 : prepare_cmp_insn (x, y, mask, comparison, NULL_RTX, unsignedp, methods,
4661 : ptest, pmode, optab);
4662 : }
4663 :
4664 : return;
4665 :
4666 2897 : fail:
4667 2897 : *ptest = NULL_RTX;
4668 : }
4669 :
4670 : /* Before emitting an insn with code ICODE, make sure that X, which is going
4671 : to be used for operand OPNUM of the insn, is converted from mode MODE to
4672 : WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4673 : that it is accepted by the operand predicate. Return the new value. */
4674 :
4675 : rtx
4676 14816284 : prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4677 : machine_mode wider_mode, int unsignedp)
4678 : {
4679 14816284 : if (mode != wider_mode)
4680 7266 : x = convert_modes (wider_mode, mode, x, unsignedp);
4681 :
4682 14816284 : if (!insn_operand_matches (icode, opnum, x))
4683 : {
4684 62724 : machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4685 62724 : if (reload_completed)
4686 : return NULL_RTX;
4687 62724 : if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4688 : return NULL_RTX;
4689 62440 : x = copy_to_mode_reg (op_mode, x);
4690 : }
4691 :
4692 : return x;
4693 : }
4694 :
4695 : /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4696 : we can do the branch. */
4697 :
4698 : static void
4699 6300602 : emit_cmp_and_jump_insn_1 (rtx test, rtx cond, rtx len, rtx bias,
4700 : machine_mode mode, rtx label, direct_optab cmp_optab,
4701 : profile_probability prob, bool test_branch)
4702 : {
4703 6300602 : machine_mode optab_mode;
4704 6300602 : enum mode_class mclass;
4705 6300602 : enum insn_code icode;
4706 6300602 : rtx_insn *insn;
4707 :
4708 6300602 : mclass = GET_MODE_CLASS (mode);
4709 6300602 : optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4710 6300602 : icode = optab_handler (cmp_optab, optab_mode);
4711 :
4712 6300602 : gcc_assert (icode != CODE_FOR_nothing);
4713 6300602 : gcc_assert (test_branch || insn_operand_matches (icode, 0, test));
4714 6300602 : gcc_assert (cond == NULL_RTX || (cond != NULL_RTX && !test_branch));
4715 6300602 : if (test_branch)
4716 0 : insn = emit_jump_insn (GEN_FCN (icode) (XEXP (test, 0),
4717 0 : XEXP (test, 1), label));
4718 6300602 : else if (len)
4719 : {
4720 0 : gcc_assert (cond);
4721 0 : gcc_assert (bias);
4722 0 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4723 0 : XEXP (test, 1), len, bias,
4724 : label));
4725 : }
4726 6300602 : else if (cond)
4727 0 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4728 0 : XEXP (test, 1), label));
4729 : else
4730 6300602 : insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4731 6300602 : XEXP (test, 1), label));
4732 :
4733 6300602 : if (prob.initialized_p ()
4734 4731335 : && profile_status_for_fn (cfun) != PROFILE_ABSENT
4735 4631135 : && insn
4736 4631135 : && JUMP_P (insn)
4737 4630479 : && any_condjump_p (insn)
4738 10931081 : && !find_reg_note (insn, REG_BR_PROB, 0))
4739 4629558 : add_reg_br_prob_note (insn, prob);
4740 6300602 : }
4741 :
4742 : /* PTEST points to a comparison that compares its first operand with zero.
4743 : Check to see if it can be performed as a bit-test-and-branch instead.
4744 : On success, return the instruction that performs the bit-test-and-branch
4745 : and replace the second operand of *PTEST with the bit number to test.
4746 : On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
4747 :
4748 : Note that the comparison described by *PTEST should not be taken
4749 : literally after a successful return. *PTEST is just a convenient
4750 : place to store the two operands of the bit-and-test.
4751 :
4752 : VAL must contain the original tree expression for the first operand
4753 : of *PTEST. */
4754 :
4755 : static enum insn_code
4756 2583789 : validate_test_and_branch (tree val, rtx *ptest, machine_mode *pmode, optab *res)
4757 : {
4758 2583789 : if (!val || TREE_CODE (val) != SSA_NAME)
4759 : return CODE_FOR_nothing;
4760 :
4761 2438730 : machine_mode mode = TYPE_MODE (TREE_TYPE (val));
4762 2438730 : rtx test = *ptest;
4763 2438730 : direct_optab optab;
4764 :
4765 2438730 : if (GET_CODE (test) == EQ)
4766 : optab = tbranch_eq_optab;
4767 1285535 : else if (GET_CODE (test) == NE)
4768 : optab = tbranch_ne_optab;
4769 : else
4770 : return CODE_FOR_nothing;
4771 :
4772 2212004 : *res = optab;
4773 :
4774 : /* If the target supports the testbit comparison directly, great. */
4775 2212004 : auto icode = direct_optab_handler (optab, mode);
4776 2212004 : if (icode == CODE_FOR_nothing)
4777 : return icode;
4778 :
4779 0 : if (tree_zero_one_valued_p (val))
4780 : {
4781 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 : 0;
4782 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4783 0 : *ptest = test;
4784 0 : *pmode = mode;
4785 0 : return icode;
4786 : }
4787 :
4788 0 : wide_int wcst = get_nonzero_bits (val);
4789 0 : if (wcst == -1)
4790 : return CODE_FOR_nothing;
4791 :
4792 0 : int bitpos;
4793 :
4794 0 : if ((bitpos = wi::exact_log2 (wcst)) == -1)
4795 : return CODE_FOR_nothing;
4796 :
4797 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 - bitpos : bitpos;
4798 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4799 0 : *ptest = test;
4800 0 : *pmode = mode;
4801 0 : return icode;
4802 0 : }
4803 :
4804 : /* Generate code to compare X with Y so that the condition codes are
4805 : set and to jump to LABEL if the condition is true. If X is a
4806 : constant and Y is not a constant, then the comparison is swapped to
4807 : ensure that the comparison RTL has the canonical form.
4808 :
4809 : UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4810 : need to be widened. UNSIGNEDP is also used to select the proper
4811 : branch condition code.
4812 :
4813 : If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4814 :
4815 : MODE is the mode of the inputs (in case they are const_int).
4816 :
4817 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4818 : It will be potentially converted into an unsigned variant based on
4819 : UNSIGNEDP to select a proper jump instruction.
4820 :
4821 : PROB is the probability of jumping to LABEL. If the comparison is against
4822 : zero then VAL contains the expression from which the non-zero RTL is
4823 : derived. */
4824 :
4825 : void
4826 6300602 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4827 : machine_mode mode, int unsignedp, tree val, rtx label,
4828 : profile_probability prob)
4829 : {
4830 6300602 : rtx op0 = x, op1 = y;
4831 6300602 : rtx test;
4832 :
4833 : /* Swap operands and condition to ensure canonical RTL. */
4834 6300602 : if (swap_commutative_operands_p (x, y)
4835 6300602 : && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4836 : {
4837 0 : op0 = y, op1 = x;
4838 0 : comparison = swap_condition (comparison);
4839 : }
4840 :
4841 : /* If OP0 is still a constant, then both X and Y must be constants
4842 : or the opposite comparison is not supported. Force X into a register
4843 : to create canonical RTL. */
4844 6300602 : if (CONSTANT_P (op0))
4845 17699 : op0 = force_reg (mode, op0);
4846 :
4847 6300602 : if (unsignedp)
4848 3761433 : comparison = unsigned_condition (comparison);
4849 :
4850 : /* cbranch is no longer preferred for vectors, so when using a vector mode
4851 : check vec_cbranch variants instead. */
4852 6300602 : if (!VECTOR_MODE_P (GET_MODE (op0)))
4853 6281193 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
4854 : OPTAB_LIB_WIDEN, &test, &mode, cbranch_optab);
4855 :
4856 : /* Check if we're comparing a truth type with 0, and if so check if
4857 : the target supports tbranch. */
4858 6300602 : machine_mode tmode = mode;
4859 6300602 : direct_optab optab;
4860 6300602 : if (op1 == CONST0_RTX (GET_MODE (op1)))
4861 : {
4862 2585411 : if (!VECTOR_MODE_P (GET_MODE (op1))
4863 2583789 : && validate_test_and_branch (val, &test, &tmode,
4864 : &optab) != CODE_FOR_nothing)
4865 : {
4866 0 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, tmode,
4867 : label, optab, prob, true);
4868 0 : return;
4869 : }
4870 :
4871 : /* If we are comparing equality with 0, check if VAL is another equality
4872 : comparison and if the target supports it directly. */
4873 2585411 : gimple *def_stmt = NULL;
4874 2441261 : if (val && TREE_CODE (val) == SSA_NAME
4875 2440339 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (val))
4876 2348 : && (comparison == NE || comparison == EQ)
4877 2587759 : && (def_stmt = get_gimple_for_ssa_name (val)))
4878 : {
4879 1640 : tree masked_op = NULL_TREE;
4880 1640 : tree len_op = NULL_TREE;
4881 1640 : tree len_bias = NULL_TREE;
4882 : /* First determine if the operation should be masked or unmasked. */
4883 1640 : if (is_gimple_assign (def_stmt)
4884 1640 : && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
4885 : {
4886 : /* See if one side if a comparison, if so use the other side as
4887 : the mask. */
4888 2 : gimple *mask_def = NULL;
4889 2 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
4890 2 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
4891 2 : if (TREE_CODE (rhs1) == SSA_NAME
4892 2 : && (mask_def = get_gimple_for_ssa_name (rhs1))
4893 2 : && is_gimple_assign (mask_def)
4894 4 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
4895 : masked_op = rhs2;
4896 0 : else if (TREE_CODE (rhs2) == SSA_NAME
4897 0 : && (mask_def = get_gimple_for_ssa_name (rhs2))
4898 0 : && is_gimple_assign (mask_def)
4899 0 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
4900 : masked_op = rhs1;
4901 :
4902 2 : if (masked_op)
4903 1640 : def_stmt = mask_def;
4904 : }
4905 : /* Else check to see if we're a LEN target. */
4906 1638 : else if (is_gimple_call (def_stmt)
4907 0 : && gimple_call_internal_p (def_stmt)
4908 1638 : && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
4909 : {
4910 : /* Example to consume:
4911 :
4912 : a = _59 != vect__4.17_75;
4913 : vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
4914 : if (vcmp != { 0, ... })
4915 :
4916 : and transform into
4917 :
4918 : if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)). */
4919 0 : gcall *call = dyn_cast <gcall *> (def_stmt);
4920 0 : tree true_branch = gimple_call_arg (call, 1);
4921 0 : tree false_branch = gimple_call_arg (call, 2);
4922 0 : if (integer_minus_onep (true_branch)
4923 0 : && integer_zerop (false_branch))
4924 : {
4925 0 : len_op = gimple_call_arg (call, 3);
4926 0 : len_bias = gimple_call_arg (call, 4);
4927 0 : tree arg0 = gimple_call_arg (call, 0);
4928 :
4929 0 : if (TREE_CODE (arg0) == SSA_NAME)
4930 0 : def_stmt = get_gimple_for_ssa_name (arg0);
4931 : }
4932 : }
4933 :
4934 1640 : enum insn_code icode;
4935 1640 : if (def_stmt
4936 1640 : && is_gimple_assign (def_stmt)
4937 3280 : && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
4938 : == tcc_comparison)
4939 : {
4940 1453 : class expand_operand ops[5];
4941 1453 : rtx_insn *tmp = NULL;
4942 1453 : start_sequence ();
4943 1453 : tree t_op0 = gimple_assign_rhs1 (def_stmt);
4944 1453 : tree t_op1 = gimple_assign_rhs2 (def_stmt);
4945 1453 : rtx op0c = expand_normal (t_op0);
4946 1453 : rtx op1c = expand_normal (t_op1);
4947 1453 : machine_mode mode2 = GET_MODE (op0c);
4948 :
4949 1453 : int nops = masked_op ? 3 : (len_op ? 5 : 2);
4950 1453 : int offset = masked_op || len_op ? 1 : 0;
4951 1453 : create_input_operand (&ops[offset + 0], op0c, mode2);
4952 1453 : create_input_operand (&ops[offset + 1], op1c, mode2);
4953 1453 : if (masked_op)
4954 : {
4955 0 : rtx mask_op = expand_normal (masked_op);
4956 0 : auto mask_mode = GET_MODE (mask_op);
4957 0 : create_input_operand (&ops[0], mask_op, mask_mode);
4958 : }
4959 1453 : else if (len_op)
4960 : {
4961 0 : rtx len_rtx = expand_normal (len_op);
4962 0 : rtx len_bias_rtx = expand_normal (len_bias);
4963 0 : tree lhs = gimple_get_lhs (def_stmt);
4964 0 : auto mask_mode = TYPE_MODE (TREE_TYPE (lhs));
4965 : /* ??? We could use something like internal_fn's
4966 : add_mask_else_and_len_args here. Currently it
4967 : only supports a fixed, consecutive order of
4968 : mask and len, though. */
4969 0 : create_input_operand (&ops[0], CONSTM1_RTX (mask_mode),
4970 : mask_mode);
4971 0 : create_convert_operand_from
4972 0 : (&ops[3], len_rtx, TYPE_MODE (TREE_TYPE (len_op)),
4973 0 : TYPE_UNSIGNED (TREE_TYPE (len_op)));
4974 0 : create_input_operand (&ops[4], len_bias_rtx, QImode);
4975 : }
4976 :
4977 1453 : int unsignedp2 = TYPE_UNSIGNED (TREE_TYPE (t_op0));
4978 1453 : auto inner_code = gimple_assign_rhs_code (def_stmt);
4979 1453 : rtx test2 = NULL_RTX;
4980 :
4981 1453 : enum rtx_code comparison2 = get_rtx_code (inner_code, unsignedp2);
4982 1453 : if (unsignedp2)
4983 488 : comparison2 = unsigned_condition (comparison2);
4984 1453 : if (comparison == NE)
4985 962 : optab = masked_op ? cond_vec_cbranch_any_optab
4986 481 : : len_op ? cond_len_vec_cbranch_any_optab
4987 : : vec_cbranch_any_optab;
4988 : else
4989 1944 : optab = masked_op ? cond_vec_cbranch_all_optab
4990 972 : : len_op ? cond_len_vec_cbranch_all_optab
4991 : : vec_cbranch_all_optab;
4992 :
4993 1453 : if ((icode = optab_handler (optab, mode2))
4994 : != CODE_FOR_nothing
4995 1453 : && maybe_legitimize_operands (icode, 1, nops, ops))
4996 : {
4997 0 : test2 = gen_rtx_fmt_ee (comparison2, VOIDmode,
4998 : ops[offset + 0].value,
4999 : ops[offset + 1].value);
5000 0 : if (insn_operand_matches (icode, 0, test2))
5001 : {
5002 0 : rtx mask
5003 0 : = (masked_op || len_op) ? ops[0].value : NULL_RTX;
5004 0 : rtx len = len_op ? ops[3].value : NULL_RTX;
5005 0 : rtx bias = len_op ? ops[4].value : NULL_RTX;
5006 0 : emit_cmp_and_jump_insn_1 (test2, mask, len, bias, mode2,
5007 : label, optab, prob, false);
5008 0 : tmp = get_insns ();
5009 : }
5010 : }
5011 :
5012 0 : end_sequence ();
5013 1453 : if (tmp)
5014 : {
5015 0 : emit_insn (tmp);
5016 0 : return;
5017 : }
5018 : }
5019 : }
5020 : }
5021 :
5022 : /* cbranch should only be used for VECTOR_BOOLEAN_TYPE_P values. */
5023 6300602 : direct_optab base_optab = cbranch_optab;
5024 6300602 : if (VECTOR_MODE_P (GET_MODE (op0)))
5025 : {
5026 : /* If cbranch is provided, use it. If we get here it means we have an
5027 : instruction in between what created the boolean value and the gcond
5028 : that is not a masking operation. This can happen for instance during
5029 : unrolling of early-break where we have an OR-reduction to reduce the
5030 : masks. In this case knowing we have a mask can let us generate better
5031 : code. If it's not there there then check the vector specific
5032 : optabs. */
5033 19409 : if (optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
5034 : {
5035 0 : if (comparison == NE)
5036 : base_optab = vec_cbranch_any_optab;
5037 : else
5038 0 : base_optab = vec_cbranch_all_optab;
5039 :
5040 0 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5041 : OPTAB_DIRECT, &test, &mode, base_optab);
5042 :
5043 0 : enum insn_code icode = optab_handler (base_optab, mode);
5044 :
5045 : /* If the new cbranch isn't supported, degrade back to old one. */
5046 0 : if (icode == CODE_FOR_nothing
5047 0 : || !test
5048 0 : || !insn_operand_matches (icode, 0, test))
5049 : base_optab = cbranch_optab;
5050 : }
5051 :
5052 19409 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5053 : OPTAB_LIB_WIDEN, &test, &mode, base_optab);
5054 : }
5055 :
5056 6300602 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, mode, label,
5057 : base_optab, prob, false);
5058 : }
5059 :
5060 : /* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown. */
5061 :
5062 : void
5063 153611 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
5064 : machine_mode mode, int unsignedp, rtx label,
5065 : profile_probability prob)
5066 : {
5067 153611 : emit_cmp_and_jump_insns (x, y, comparison, size, mode, unsignedp, NULL,
5068 : label, prob);
5069 153611 : }
5070 :
5071 :
5072 : /* Emit a library call comparison between floating point X and Y.
5073 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
5074 :
5075 : static void
5076 43804 : prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
5077 : rtx *ptest, machine_mode *pmode)
5078 : {
5079 43804 : enum rtx_code swapped = swap_condition (comparison);
5080 43804 : enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
5081 43804 : machine_mode orig_mode = GET_MODE (x);
5082 43804 : machine_mode mode;
5083 43804 : rtx true_rtx, false_rtx;
5084 43804 : rtx value, target, equiv;
5085 43804 : rtx_insn *insns;
5086 43804 : rtx libfunc = 0;
5087 43804 : bool reversed_p = false;
5088 43804 : scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
5089 :
5090 43804 : FOR_EACH_WIDER_MODE_FROM (mode, orig_mode)
5091 : {
5092 43804 : if (code_to_optab (comparison)
5093 43804 : && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
5094 : break;
5095 :
5096 0 : if (code_to_optab (swapped)
5097 0 : && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
5098 : {
5099 : std::swap (x, y);
5100 : comparison = swapped;
5101 : break;
5102 : }
5103 :
5104 0 : if (code_to_optab (reversed)
5105 0 : && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
5106 : {
5107 : comparison = reversed;
5108 : reversed_p = true;
5109 : break;
5110 : }
5111 : }
5112 :
5113 43804 : gcc_assert (mode != VOIDmode);
5114 :
5115 43804 : if (mode != orig_mode)
5116 : {
5117 0 : x = convert_to_mode (mode, x, 0);
5118 0 : y = convert_to_mode (mode, y, 0);
5119 : }
5120 :
5121 : /* Attach a REG_EQUAL note describing the semantics of the libcall to
5122 : the RTL. The allows the RTL optimizers to delete the libcall if the
5123 : condition can be determined at compile-time. */
5124 43804 : if (comparison == UNORDERED
5125 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5126 : {
5127 5418 : true_rtx = const_true_rtx;
5128 5418 : false_rtx = const0_rtx;
5129 : }
5130 : else
5131 : {
5132 38386 : switch (comparison)
5133 : {
5134 3364 : case EQ:
5135 3364 : true_rtx = const0_rtx;
5136 3364 : false_rtx = const_true_rtx;
5137 3364 : break;
5138 :
5139 23461 : case NE:
5140 23461 : true_rtx = const_true_rtx;
5141 23461 : false_rtx = const0_rtx;
5142 23461 : break;
5143 :
5144 2471 : case GT:
5145 2471 : true_rtx = const1_rtx;
5146 2471 : false_rtx = const0_rtx;
5147 2471 : break;
5148 :
5149 3076 : case GE:
5150 3076 : true_rtx = const0_rtx;
5151 3076 : false_rtx = constm1_rtx;
5152 3076 : break;
5153 :
5154 2433 : case LT:
5155 2433 : true_rtx = constm1_rtx;
5156 2433 : false_rtx = const0_rtx;
5157 2433 : break;
5158 :
5159 3581 : case LE:
5160 3581 : true_rtx = const0_rtx;
5161 3581 : false_rtx = const1_rtx;
5162 3581 : break;
5163 :
5164 0 : default:
5165 0 : gcc_unreachable ();
5166 : }
5167 : }
5168 :
5169 43804 : if (comparison == UNORDERED)
5170 : {
5171 5418 : rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
5172 5418 : equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
5173 5418 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5174 : temp, const_true_rtx, equiv);
5175 : }
5176 : else
5177 : {
5178 38386 : equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
5179 38386 : if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5180 38386 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5181 : equiv, true_rtx, false_rtx);
5182 : }
5183 :
5184 43804 : start_sequence ();
5185 43804 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5186 : cmp_mode, x, mode, y, mode);
5187 43804 : insns = end_sequence ();
5188 :
5189 43804 : target = gen_reg_rtx (cmp_mode);
5190 43804 : emit_libcall_block (insns, target, value, equiv);
5191 :
5192 43804 : if (comparison == UNORDERED
5193 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
5194 43804 : || reversed_p)
5195 10836 : *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
5196 : else
5197 38386 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
5198 :
5199 43804 : *pmode = cmp_mode;
5200 43804 : }
5201 :
5202 : /* Generate code to indirectly jump to a location given in the rtx LOC. */
5203 :
5204 : void
5205 1308 : emit_indirect_jump (rtx loc)
5206 : {
5207 1308 : if (!targetm.have_indirect_jump ())
5208 0 : sorry ("indirect jumps are not available on this target");
5209 : else
5210 : {
5211 1308 : class expand_operand ops[1];
5212 1308 : create_address_operand (&ops[0], loc);
5213 1308 : expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
5214 1308 : emit_barrier ();
5215 : }
5216 1308 : }
5217 :
5218 :
5219 : /* Emit a conditional move instruction if the machine supports one for that
5220 : condition and machine mode.
5221 :
5222 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5223 : the mode to use should they be constants. If it is VOIDmode, they cannot
5224 : both be constants.
5225 :
5226 : OP2 should be stored in TARGET if the comparison is true, otherwise OP3
5227 : should be stored there. MODE is the mode to use should they be constants.
5228 : If it is VOIDmode, they cannot both be constants.
5229 :
5230 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5231 : is not supported. */
5232 :
5233 : rtx
5234 421987 : emit_conditional_move (rtx target, struct rtx_comparison comp,
5235 : rtx op2, rtx op3,
5236 : machine_mode mode, int unsignedp)
5237 : {
5238 421987 : rtx comparison;
5239 421987 : rtx_insn *last;
5240 421987 : enum insn_code icode;
5241 421987 : enum rtx_code reversed;
5242 :
5243 : /* If the two source operands are identical, that's just a move. */
5244 :
5245 421987 : if (rtx_equal_p (op2, op3))
5246 : {
5247 11888 : if (!target)
5248 0 : target = gen_reg_rtx (mode);
5249 :
5250 11888 : emit_move_insn (target, op3);
5251 11888 : return target;
5252 : }
5253 :
5254 : /* If one operand is constant, make it the second one. Only do this
5255 : if the other operand is not constant as well. */
5256 :
5257 410099 : if (swap_commutative_operands_p (comp.op0, comp.op1))
5258 : {
5259 407 : std::swap (comp.op0, comp.op1);
5260 407 : comp.code = swap_condition (comp.code);
5261 : }
5262 :
5263 : /* get_condition will prefer to generate LT and GT even if the old
5264 : comparison was against zero, so undo that canonicalization here since
5265 : comparisons against zero are cheaper. */
5266 :
5267 410099 : if (comp.code == LT && comp.op1 == const1_rtx)
5268 1978 : comp.code = LE, comp.op1 = const0_rtx;
5269 408121 : else if (comp.code == GT && comp.op1 == constm1_rtx)
5270 7214 : comp.code = GE, comp.op1 = const0_rtx;
5271 :
5272 410099 : if (comp.mode == VOIDmode)
5273 325911 : comp.mode = GET_MODE (comp.op0);
5274 :
5275 410099 : enum rtx_code orig_code = comp.code;
5276 410099 : bool swapped = false;
5277 410099 : if (swap_commutative_operands_p (op2, op3)
5278 410099 : && ((reversed =
5279 126902 : reversed_comparison_code_parts (comp.code, comp.op0, comp.op1, NULL))
5280 : != UNKNOWN))
5281 : {
5282 : std::swap (op2, op3);
5283 : comp.code = reversed;
5284 : swapped = true;
5285 : }
5286 :
5287 410099 : if (mode == VOIDmode)
5288 0 : mode = GET_MODE (op2);
5289 :
5290 410099 : icode = direct_optab_handler (movcc_optab, mode);
5291 :
5292 410099 : if (icode == CODE_FOR_nothing)
5293 : return NULL_RTX;
5294 :
5295 403421 : if (!target)
5296 0 : target = gen_reg_rtx (mode);
5297 :
5298 403421 : for (int pass = 0; ; pass++)
5299 : {
5300 434955 : comp.code = unsignedp ? unsigned_condition (comp.code) : comp.code;
5301 869910 : comparison =
5302 434955 : simplify_gen_relational (comp.code, VOIDmode,
5303 : comp.mode, comp.op0, comp.op1);
5304 :
5305 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5306 : punt and let the caller figure out how best to deal with this
5307 : situation. */
5308 434955 : if (COMPARISON_P (comparison))
5309 : {
5310 434089 : saved_pending_stack_adjust save;
5311 434089 : save_pending_stack_adjust (&save);
5312 434089 : last = get_last_insn ();
5313 434089 : do_pending_stack_adjust ();
5314 434089 : machine_mode cmpmode = comp.mode;
5315 434089 : rtx orig_op0 = XEXP (comparison, 0);
5316 434089 : rtx orig_op1 = XEXP (comparison, 1);
5317 434089 : rtx op2p = op2;
5318 434089 : rtx op3p = op3;
5319 : /* If we are optimizing, force expensive constants into a register
5320 : but preserve an eventual equality with op2/op3. */
5321 1 : if (CONSTANT_P (orig_op0) && optimize
5322 1 : && cmpmode == mode
5323 1 : && (rtx_cost (orig_op0, mode, COMPARE, 0,
5324 1 : optimize_insn_for_speed_p ())
5325 : > COSTS_N_INSNS (1))
5326 434089 : && can_create_pseudo_p ())
5327 : {
5328 0 : if (rtx_equal_p (orig_op0, op2))
5329 0 : op2p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5330 0 : else if (rtx_equal_p (orig_op0, op3))
5331 0 : op3p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5332 : }
5333 263614 : if (CONSTANT_P (orig_op1) && optimize
5334 242677 : && cmpmode == mode
5335 72882 : && (rtx_cost (orig_op1, mode, COMPARE, 0,
5336 72882 : optimize_insn_for_speed_p ())
5337 : > COSTS_N_INSNS (1))
5338 434257 : && can_create_pseudo_p ())
5339 : {
5340 168 : if (rtx_equal_p (orig_op1, op2))
5341 102 : op2p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5342 66 : else if (rtx_equal_p (orig_op1, op3))
5343 42 : op3p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5344 : }
5345 434089 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5346 434089 : GET_CODE (comparison), NULL_RTX, unsignedp,
5347 : OPTAB_WIDEN, &comparison, &cmpmode, cbranch_optab);
5348 434089 : if (comparison)
5349 : {
5350 431953 : rtx res = emit_conditional_move_1 (target, comparison,
5351 : op2p, op3p, mode);
5352 431953 : if (res != NULL_RTX)
5353 369783 : return res;
5354 : }
5355 64306 : delete_insns_since (last);
5356 64306 : restore_pending_stack_adjust (&save);
5357 : }
5358 :
5359 65172 : if (pass == 1)
5360 : return NULL_RTX;
5361 :
5362 : /* If the preferred op2/op3 order is not usable, retry with other
5363 : operand order, perhaps it will expand successfully. */
5364 33682 : if (swapped)
5365 : comp.code = orig_code;
5366 65836 : else if ((reversed =
5367 32918 : reversed_comparison_code_parts (orig_code, comp.op0, comp.op1,
5368 : NULL))
5369 : != UNKNOWN)
5370 : comp.code = reversed;
5371 : else
5372 : return NULL_RTX;
5373 : std::swap (op2, op3);
5374 : }
5375 : }
5376 :
5377 : /* Helper function that, in addition to COMPARISON, also tries
5378 : the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
5379 : to when we pass the specific constituents of a comparison, no
5380 : additional insns are emitted for it. It might still be necessary
5381 : to emit more than one insn for the final conditional move, though. */
5382 :
5383 : rtx
5384 88573 : emit_conditional_move (rtx target, rtx comparison, rtx rev_comparison,
5385 : rtx op2, rtx op3, machine_mode mode)
5386 : {
5387 88573 : rtx res = emit_conditional_move_1 (target, comparison, op2, op3, mode);
5388 :
5389 88573 : if (res != NULL_RTX)
5390 : return res;
5391 :
5392 4977 : return emit_conditional_move_1 (target, rev_comparison, op3, op2, mode);
5393 : }
5394 :
5395 : /* Helper for emitting a conditional move. */
5396 :
5397 : static rtx
5398 525503 : emit_conditional_move_1 (rtx target, rtx comparison,
5399 : rtx op2, rtx op3, machine_mode mode)
5400 : {
5401 525503 : enum insn_code icode;
5402 :
5403 525503 : if (comparison == NULL_RTX || !COMPARISON_P (comparison))
5404 : return NULL_RTX;
5405 :
5406 : /* If the two source operands are identical, that's just a move.
5407 : As the comparison comes in non-canonicalized, we must make
5408 : sure not to discard any possible side effects. If there are
5409 : side effects, just let the target handle it. */
5410 525503 : if (!side_effects_p (comparison) && rtx_equal_p (op2, op3))
5411 : {
5412 0 : if (!target)
5413 0 : target = gen_reg_rtx (mode);
5414 :
5415 0 : emit_move_insn (target, op3);
5416 0 : return target;
5417 : }
5418 :
5419 525503 : if (mode == VOIDmode)
5420 0 : mode = GET_MODE (op2);
5421 :
5422 525503 : icode = direct_optab_handler (movcc_optab, mode);
5423 :
5424 525503 : if (icode == CODE_FOR_nothing)
5425 : return NULL_RTX;
5426 :
5427 525503 : if (!target)
5428 0 : target = gen_reg_rtx (mode);
5429 :
5430 525503 : class expand_operand ops[4];
5431 :
5432 525503 : create_output_operand (&ops[0], target, mode);
5433 525503 : create_fixed_operand (&ops[1], comparison);
5434 525503 : create_input_operand (&ops[2], op2, mode);
5435 525503 : create_input_operand (&ops[3], op3, mode);
5436 :
5437 525503 : if (maybe_expand_insn (icode, 4, ops))
5438 : {
5439 453379 : if (ops[0].value != target)
5440 55 : convert_move (target, ops[0].value, false);
5441 453379 : return target;
5442 : }
5443 :
5444 : return NULL_RTX;
5445 : }
5446 :
5447 :
5448 : /* Emit a conditional negate or bitwise complement using the
5449 : negcc or notcc optabs if available. Return NULL_RTX if such operations
5450 : are not available. Otherwise return the RTX holding the result.
5451 : TARGET is the desired destination of the result. COMP is the comparison
5452 : on which to negate. If COND is true move into TARGET the negation
5453 : or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5454 : CODE is either NEG or NOT. MODE is the machine mode in which the
5455 : operation is performed. */
5456 :
5457 : rtx
5458 1257 : emit_conditional_neg_or_complement (rtx target, rtx_code code,
5459 : machine_mode mode, rtx cond, rtx op1,
5460 : rtx op2)
5461 : {
5462 1257 : optab op = unknown_optab;
5463 1257 : if (code == NEG)
5464 : op = negcc_optab;
5465 68 : else if (code == NOT)
5466 : op = notcc_optab;
5467 : else
5468 0 : gcc_unreachable ();
5469 :
5470 1257 : insn_code icode = direct_optab_handler (op, mode);
5471 :
5472 1257 : if (icode == CODE_FOR_nothing)
5473 : return NULL_RTX;
5474 :
5475 0 : if (!target)
5476 0 : target = gen_reg_rtx (mode);
5477 :
5478 0 : rtx_insn *last = get_last_insn ();
5479 0 : class expand_operand ops[4];
5480 :
5481 0 : create_output_operand (&ops[0], target, mode);
5482 0 : create_fixed_operand (&ops[1], cond);
5483 0 : create_input_operand (&ops[2], op1, mode);
5484 0 : create_input_operand (&ops[3], op2, mode);
5485 :
5486 0 : if (maybe_expand_insn (icode, 4, ops))
5487 : {
5488 0 : if (ops[0].value != target)
5489 0 : convert_move (target, ops[0].value, false);
5490 :
5491 0 : return target;
5492 : }
5493 0 : delete_insns_since (last);
5494 0 : return NULL_RTX;
5495 : }
5496 :
5497 : /* Emit a conditional addition instruction if the machine supports one for that
5498 : condition and machine mode.
5499 :
5500 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5501 : the mode to use should they be constants. If it is VOIDmode, they cannot
5502 : both be constants.
5503 :
5504 : OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5505 : should be stored there. MODE is the mode to use should they be constants.
5506 : If it is VOIDmode, they cannot both be constants.
5507 :
5508 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5509 : is not supported. */
5510 :
5511 : rtx
5512 15149 : emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
5513 : machine_mode cmode, rtx op2, rtx op3,
5514 : machine_mode mode, int unsignedp)
5515 : {
5516 15149 : rtx comparison;
5517 15149 : rtx_insn *last;
5518 15149 : enum insn_code icode;
5519 :
5520 : /* If one operand is constant, make it the second one. Only do this
5521 : if the other operand is not constant as well. */
5522 :
5523 15149 : if (swap_commutative_operands_p (op0, op1))
5524 : {
5525 0 : std::swap (op0, op1);
5526 0 : code = swap_condition (code);
5527 : }
5528 :
5529 : /* get_condition will prefer to generate LT and GT even if the old
5530 : comparison was against zero, so undo that canonicalization here since
5531 : comparisons against zero are cheaper. */
5532 15149 : if (code == LT && op1 == const1_rtx)
5533 36 : code = LE, op1 = const0_rtx;
5534 15113 : else if (code == GT && op1 == constm1_rtx)
5535 89 : code = GE, op1 = const0_rtx;
5536 :
5537 15149 : if (cmode == VOIDmode)
5538 15149 : cmode = GET_MODE (op0);
5539 :
5540 15149 : if (mode == VOIDmode)
5541 0 : mode = GET_MODE (op2);
5542 :
5543 15149 : icode = optab_handler (addcc_optab, mode);
5544 :
5545 15149 : if (icode == CODE_FOR_nothing)
5546 : return 0;
5547 :
5548 14574 : if (!target)
5549 0 : target = gen_reg_rtx (mode);
5550 :
5551 14574 : code = unsignedp ? unsigned_condition (code) : code;
5552 14574 : comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
5553 :
5554 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5555 : return NULL and let the caller figure out how best to deal with this
5556 : situation. */
5557 14574 : if (!COMPARISON_P (comparison))
5558 : return NULL_RTX;
5559 :
5560 14574 : do_pending_stack_adjust ();
5561 14574 : last = get_last_insn ();
5562 14574 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5563 14574 : GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
5564 : &comparison, &cmode, cbranch_optab);
5565 14574 : if (comparison)
5566 : {
5567 14574 : class expand_operand ops[4];
5568 :
5569 14574 : create_output_operand (&ops[0], target, mode);
5570 14574 : create_fixed_operand (&ops[1], comparison);
5571 14574 : create_input_operand (&ops[2], op2, mode);
5572 14574 : create_input_operand (&ops[3], op3, mode);
5573 14574 : if (maybe_expand_insn (icode, 4, ops))
5574 : {
5575 1270 : if (ops[0].value != target)
5576 0 : convert_move (target, ops[0].value, false);
5577 1270 : return target;
5578 : }
5579 : }
5580 13304 : delete_insns_since (last);
5581 13304 : return NULL_RTX;
5582 : }
5583 :
5584 : /* These functions attempt to generate an insn body, rather than
5585 : emitting the insn, but if the gen function already emits them, we
5586 : make no attempt to turn them back into naked patterns. */
5587 :
5588 : /* Generate and return an insn body to add Y to X. */
5589 :
5590 : rtx_insn *
5591 251 : gen_add2_insn (rtx x, rtx y)
5592 : {
5593 251 : enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
5594 :
5595 251 : gcc_assert (insn_operand_matches (icode, 0, x));
5596 251 : gcc_assert (insn_operand_matches (icode, 1, x));
5597 251 : gcc_assert (insn_operand_matches (icode, 2, y));
5598 :
5599 251 : return GEN_FCN (icode) (x, x, y);
5600 : }
5601 :
5602 : /* Generate and return an insn body to add r1 and c,
5603 : storing the result in r0. */
5604 :
5605 : rtx_insn *
5606 0 : gen_add3_insn (rtx r0, rtx r1, rtx c)
5607 : {
5608 0 : enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
5609 :
5610 0 : if (icode == CODE_FOR_nothing
5611 0 : || !insn_operand_matches (icode, 0, r0)
5612 0 : || !insn_operand_matches (icode, 1, r1)
5613 0 : || !insn_operand_matches (icode, 2, c))
5614 0 : return NULL;
5615 :
5616 0 : return GEN_FCN (icode) (r0, r1, c);
5617 : }
5618 :
5619 : bool
5620 3780 : have_add2_insn (rtx x, rtx y)
5621 : {
5622 3780 : enum insn_code icode;
5623 :
5624 3780 : gcc_assert (GET_MODE (x) != VOIDmode);
5625 :
5626 3780 : icode = optab_handler (add_optab, GET_MODE (x));
5627 :
5628 3780 : if (icode == CODE_FOR_nothing)
5629 : return false;
5630 :
5631 3780 : if (!insn_operand_matches (icode, 0, x)
5632 3780 : || !insn_operand_matches (icode, 1, x)
5633 7560 : || !insn_operand_matches (icode, 2, y))
5634 200 : return false;
5635 :
5636 : return true;
5637 : }
5638 :
5639 : /* Generate and return an insn body to add Y to X. */
5640 :
5641 : rtx_insn *
5642 0 : gen_addptr3_insn (rtx x, rtx y, rtx z)
5643 : {
5644 0 : enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
5645 :
5646 0 : gcc_assert (insn_operand_matches (icode, 0, x));
5647 0 : gcc_assert (insn_operand_matches (icode, 1, y));
5648 0 : gcc_assert (insn_operand_matches (icode, 2, z));
5649 :
5650 0 : return GEN_FCN (icode) (x, y, z);
5651 : }
5652 :
5653 : /* Return true if the target implements an addptr pattern and X, Y,
5654 : and Z are valid for the pattern predicates. */
5655 :
5656 : bool
5657 622150 : have_addptr3_insn (rtx x, rtx y, rtx z)
5658 : {
5659 622150 : enum insn_code icode;
5660 :
5661 622150 : gcc_assert (GET_MODE (x) != VOIDmode);
5662 :
5663 622150 : icode = optab_handler (addptr3_optab, GET_MODE (x));
5664 :
5665 622150 : if (icode == CODE_FOR_nothing)
5666 : return false;
5667 :
5668 0 : if (!insn_operand_matches (icode, 0, x)
5669 0 : || !insn_operand_matches (icode, 1, y)
5670 0 : || !insn_operand_matches (icode, 2, z))
5671 0 : return false;
5672 :
5673 : return true;
5674 : }
5675 :
5676 : /* Generate and return an insn body to subtract Y from X. */
5677 :
5678 : rtx_insn *
5679 11 : gen_sub2_insn (rtx x, rtx y)
5680 : {
5681 11 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
5682 :
5683 11 : gcc_assert (insn_operand_matches (icode, 0, x));
5684 11 : gcc_assert (insn_operand_matches (icode, 1, x));
5685 11 : gcc_assert (insn_operand_matches (icode, 2, y));
5686 :
5687 11 : return GEN_FCN (icode) (x, x, y);
5688 : }
5689 :
5690 : /* Generate and return an insn body to subtract r1 and c,
5691 : storing the result in r0. */
5692 :
5693 : rtx_insn *
5694 1532 : gen_sub3_insn (rtx r0, rtx r1, rtx c)
5695 : {
5696 1532 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
5697 :
5698 1532 : if (icode == CODE_FOR_nothing
5699 1532 : || !insn_operand_matches (icode, 0, r0)
5700 1532 : || !insn_operand_matches (icode, 1, r1)
5701 3064 : || !insn_operand_matches (icode, 2, c))
5702 0 : return NULL;
5703 :
5704 1532 : return GEN_FCN (icode) (r0, r1, c);
5705 : }
5706 :
5707 : bool
5708 0 : have_sub2_insn (rtx x, rtx y)
5709 : {
5710 0 : enum insn_code icode;
5711 :
5712 0 : gcc_assert (GET_MODE (x) != VOIDmode);
5713 :
5714 0 : icode = optab_handler (sub_optab, GET_MODE (x));
5715 :
5716 0 : if (icode == CODE_FOR_nothing)
5717 : return false;
5718 :
5719 0 : if (!insn_operand_matches (icode, 0, x)
5720 0 : || !insn_operand_matches (icode, 1, x)
5721 0 : || !insn_operand_matches (icode, 2, y))
5722 0 : return false;
5723 :
5724 : return true;
5725 : }
5726 :
5727 : /* Generate the body of an insn to extend Y (with mode MFROM)
5728 : into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5729 :
5730 : rtx_insn *
5731 30518 : gen_extend_insn (rtx x, rtx y, machine_mode mto,
5732 : machine_mode mfrom, int unsignedp)
5733 : {
5734 30518 : enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
5735 30518 : return GEN_FCN (icode) (x, y);
5736 : }
5737 :
5738 : /* Generate code to convert FROM to floating point
5739 : and store in TO. FROM must be fixed point and not VOIDmode.
5740 : UNSIGNEDP nonzero means regard FROM as unsigned.
5741 : Normally this is done by correcting the final value
5742 : if it is negative. */
5743 :
5744 : void
5745 143972 : expand_float (rtx to, rtx from, int unsignedp)
5746 : {
5747 143972 : enum insn_code icode;
5748 143972 : rtx target = to;
5749 143972 : scalar_mode from_mode, to_mode;
5750 143972 : machine_mode fmode, imode;
5751 143972 : bool can_do_signed = false;
5752 :
5753 : /* Crash now, because we won't be able to decide which mode to use. */
5754 143972 : gcc_assert (GET_MODE (from) != VOIDmode);
5755 :
5756 : /* Look for an insn to do the conversion. Do it in the specified
5757 : modes if possible; otherwise convert either input, output or both to
5758 : wider mode. If the integer mode is wider than the mode of FROM,
5759 : we can do the conversion signed even if the input is unsigned. */
5760 :
5761 166004 : FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
5762 260047 : FOR_EACH_MODE_FROM (imode, GET_MODE (from))
5763 : {
5764 238015 : int doing_unsigned = unsignedp;
5765 :
5766 268105 : if (fmode != GET_MODE (to)
5767 238015 : && (significand_size (fmode)
5768 94442 : < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
5769 30090 : continue;
5770 :
5771 207925 : icode = can_float_p (fmode, imode, unsignedp);
5772 207925 : if (icode == CODE_FOR_nothing && unsignedp)
5773 : {
5774 35674 : enum insn_code scode = can_float_p (fmode, imode, 0);
5775 35674 : if (scode != CODE_FOR_nothing)
5776 6337 : can_do_signed = true;
5777 35674 : if (imode != GET_MODE (from))
5778 24587 : icode = scode, doing_unsigned = 0;
5779 : }
5780 :
5781 207925 : if (icode != CODE_FOR_nothing)
5782 : {
5783 135388 : if (imode != GET_MODE (from))
5784 10169 : from = convert_to_mode (imode, from, unsignedp);
5785 :
5786 135388 : if (fmode != GET_MODE (to))
5787 1590 : target = gen_reg_rtx (fmode);
5788 :
5789 266072 : emit_unop_insn (icode, target, from,
5790 : doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5791 :
5792 135388 : if (target != to)
5793 1590 : convert_move (to, target, 0);
5794 135388 : return;
5795 : }
5796 : }
5797 :
5798 : /* Unsigned integer, and no way to convert directly. Convert as signed,
5799 : then unconditionally adjust the result. */
5800 8584 : if (unsignedp
5801 8584 : && can_do_signed
5802 1098 : && is_a <scalar_mode> (GET_MODE (to), &to_mode)
5803 9682 : && is_a <scalar_mode> (GET_MODE (from), &from_mode))
5804 : {
5805 1098 : opt_scalar_mode fmode_iter;
5806 1098 : rtx_code_label *label = gen_label_rtx ();
5807 1098 : rtx temp;
5808 1098 : REAL_VALUE_TYPE offset;
5809 :
5810 : /* Look for a usable floating mode FMODE wider than the source and at
5811 : least as wide as the target. Using FMODE will avoid rounding woes
5812 : with unsigned values greater than the signed maximum value. */
5813 :
5814 2105 : FOR_EACH_MODE_FROM (fmode_iter, to_mode)
5815 : {
5816 2105 : scalar_mode fmode = fmode_iter.require ();
5817 3112 : if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
5818 2105 : && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
5819 : break;
5820 : }
5821 :
5822 1098 : if (!fmode_iter.exists (&fmode))
5823 : {
5824 : /* There is no such mode. Pretend the target is wide enough. */
5825 0 : fmode = to_mode;
5826 :
5827 : /* Avoid double-rounding when TO is narrower than FROM. */
5828 0 : if ((significand_size (fmode) + 1)
5829 0 : < GET_MODE_PRECISION (from_mode))
5830 : {
5831 0 : rtx temp1;
5832 0 : rtx_code_label *neglabel = gen_label_rtx ();
5833 :
5834 : /* Don't use TARGET if it isn't a register, is a hard register,
5835 : or is the wrong mode. */
5836 0 : if (!REG_P (target)
5837 0 : || REGNO (target) < FIRST_PSEUDO_REGISTER
5838 0 : || GET_MODE (target) != fmode)
5839 0 : target = gen_reg_rtx (fmode);
5840 :
5841 0 : imode = from_mode;
5842 0 : do_pending_stack_adjust ();
5843 :
5844 : /* Test whether the sign bit is set. */
5845 0 : emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
5846 : 0, neglabel);
5847 :
5848 : /* The sign bit is not set. Convert as signed. */
5849 0 : expand_float (target, from, 0);
5850 0 : emit_jump_insn (targetm.gen_jump (label));
5851 0 : emit_barrier ();
5852 :
5853 : /* The sign bit is set.
5854 : Convert to a usable (positive signed) value by shifting right
5855 : one bit, while remembering if a nonzero bit was shifted
5856 : out; i.e., compute (from & 1) | (from >> 1). */
5857 :
5858 0 : emit_label (neglabel);
5859 0 : temp = expand_binop (imode, and_optab, from, const1_rtx,
5860 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
5861 0 : temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
5862 0 : temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
5863 : OPTAB_LIB_WIDEN);
5864 0 : expand_float (target, temp, 0);
5865 :
5866 : /* Multiply by 2 to undo the shift above. */
5867 0 : temp = expand_binop (fmode, add_optab, target, target,
5868 : target, 0, OPTAB_LIB_WIDEN);
5869 0 : if (temp != target)
5870 0 : emit_move_insn (target, temp);
5871 :
5872 0 : do_pending_stack_adjust ();
5873 0 : emit_label (label);
5874 0 : goto done;
5875 : }
5876 : }
5877 :
5878 : /* If we are about to do some arithmetic to correct for an
5879 : unsigned operand, do it in a pseudo-register. */
5880 :
5881 1098 : if (to_mode != fmode
5882 1098 : || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
5883 399 : target = gen_reg_rtx (fmode);
5884 :
5885 : /* Convert as signed integer to floating. */
5886 1098 : expand_float (target, from, 0);
5887 :
5888 : /* If FROM is negative (and therefore TO is negative),
5889 : correct its value by 2**bitwidth. */
5890 :
5891 1098 : do_pending_stack_adjust ();
5892 1098 : emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
5893 : 0, label);
5894 :
5895 :
5896 1098 : real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
5897 1098 : temp = expand_binop (fmode, add_optab, target,
5898 : const_double_from_real_value (offset, fmode),
5899 : target, 0, OPTAB_LIB_WIDEN);
5900 1098 : if (temp != target)
5901 0 : emit_move_insn (target, temp);
5902 :
5903 1098 : do_pending_stack_adjust ();
5904 1098 : emit_label (label);
5905 1098 : goto done;
5906 : }
5907 :
5908 : /* No hardware instruction available; call a library routine. */
5909 7486 : {
5910 7486 : rtx libfunc;
5911 7486 : rtx_insn *insns;
5912 7486 : rtx value;
5913 7486 : convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
5914 :
5915 7486 : if (is_narrower_int_mode (GET_MODE (from), SImode))
5916 974 : from = convert_to_mode (SImode, from, unsignedp);
5917 :
5918 7486 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
5919 7486 : gcc_assert (libfunc);
5920 :
5921 7486 : start_sequence ();
5922 :
5923 14972 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5924 7486 : GET_MODE (to), from, GET_MODE (from));
5925 7486 : insns = end_sequence ();
5926 :
5927 7486 : emit_libcall_block (insns, target, value,
5928 7486 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
5929 : GET_MODE (to), from));
5930 : }
5931 :
5932 8584 : done:
5933 :
5934 : /* Copy result to requested destination
5935 : if we have been computing in a temp location. */
5936 :
5937 8584 : if (target != to)
5938 : {
5939 399 : if (GET_MODE (target) == GET_MODE (to))
5940 4 : emit_move_insn (to, target);
5941 : else
5942 395 : convert_move (to, target, 0);
5943 : }
5944 : }
5945 :
5946 : /* Generate code to convert FROM to fixed point and store in TO. FROM
5947 : must be floating point. */
5948 :
5949 : void
5950 58969 : expand_fix (rtx to, rtx from, int unsignedp)
5951 : {
5952 58969 : enum insn_code icode;
5953 58969 : rtx target = to;
5954 58969 : machine_mode fmode, imode;
5955 58969 : opt_scalar_mode fmode_iter;
5956 58969 : bool must_trunc = false;
5957 :
5958 : /* We first try to find a pair of modes, one real and one integer, at
5959 : least as wide as FROM and TO, respectively, in which we can open-code
5960 : this conversion. If the integer mode is wider than the mode of TO,
5961 : we can do the conversion either signed or unsigned. */
5962 :
5963 89103 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
5964 213135 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
5965 : {
5966 183001 : int doing_unsigned = unsignedp;
5967 :
5968 183001 : icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
5969 183001 : if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
5970 61433 : icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
5971 :
5972 140195 : if (icode != CODE_FOR_nothing)
5973 : {
5974 46016 : rtx_insn *last = get_last_insn ();
5975 46016 : rtx from1 = from;
5976 46016 : if (fmode != GET_MODE (from))
5977 : {
5978 1095 : if (REAL_MODE_FORMAT (GET_MODE (from))
5979 : == &arm_bfloat_half_format
5980 1095 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
5981 : /* The BF -> SF conversions can be just a shift, doesn't
5982 : need to handle sNANs. */
5983 : {
5984 38 : int save_flag_finite_math_only = flag_finite_math_only;
5985 38 : flag_finite_math_only = true;
5986 38 : from1 = convert_to_mode (fmode, from, 0);
5987 38 : flag_finite_math_only = save_flag_finite_math_only;
5988 : }
5989 : else
5990 1057 : from1 = convert_to_mode (fmode, from, 0);
5991 : }
5992 :
5993 46016 : if (must_trunc)
5994 : {
5995 0 : rtx temp = gen_reg_rtx (GET_MODE (from1));
5996 0 : from1 = expand_unop (GET_MODE (from1), ftrunc_optab, from1,
5997 : temp, 0);
5998 : }
5999 :
6000 46016 : if (imode != GET_MODE (to))
6001 10829 : target = gen_reg_rtx (imode);
6002 :
6003 87766 : if (maybe_emit_unop_insn (icode, target, from1,
6004 : doing_unsigned ? UNSIGNED_FIX : FIX))
6005 : {
6006 46016 : if (target != to)
6007 10829 : convert_move (to, target, unsignedp);
6008 50004 : return;
6009 : }
6010 0 : delete_insns_since (last);
6011 : }
6012 : }
6013 :
6014 : /* For an unsigned conversion, there is one more way to do it.
6015 : If we have a signed conversion, we generate code that compares
6016 : the real value to the largest representable positive number. If if
6017 : is smaller, the conversion is done normally. Otherwise, subtract
6018 : one plus the highest signed number, convert, and add it back.
6019 :
6020 : We only need to check all real modes, since we know we didn't find
6021 : anything with a wider integer mode.
6022 :
6023 : This code used to extend FP value into mode wider than the destination.
6024 : This is needed for decimal float modes which cannot accurately
6025 : represent one plus the highest signed number of the same size, but
6026 : not for binary modes. Consider, for instance conversion from SFmode
6027 : into DImode.
6028 :
6029 : The hot path through the code is dealing with inputs smaller than 2^63
6030 : and doing just the conversion, so there is no bits to lose.
6031 :
6032 : In the other path we know the value is positive in the range 2^63..2^64-1
6033 : inclusive. (as for other input overflow happens and result is undefined)
6034 : So we know that the most important bit set in mantissa corresponds to
6035 : 2^63. The subtraction of 2^63 should not generate any rounding as it
6036 : simply clears out that bit. The rest is trivial. */
6037 :
6038 12953 : scalar_int_mode to_mode;
6039 12953 : if (unsignedp
6040 7345 : && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
6041 20298 : && HWI_COMPUTABLE_MODE_P (to_mode))
6042 8956 : FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
6043 : {
6044 6766 : scalar_mode fmode = fmode_iter.require ();
6045 6766 : if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
6046 : 0, &must_trunc)
6047 6766 : && (!DECIMAL_FLOAT_MODE_P (fmode)
6048 0 : || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
6049 : {
6050 3984 : int bitsize;
6051 3984 : REAL_VALUE_TYPE offset;
6052 3984 : rtx limit;
6053 3984 : rtx_code_label *lab1, *lab2;
6054 3984 : rtx_insn *insn;
6055 :
6056 3984 : bitsize = GET_MODE_PRECISION (to_mode);
6057 3984 : real_2expN (&offset, bitsize - 1, fmode);
6058 3984 : limit = const_double_from_real_value (offset, fmode);
6059 3984 : lab1 = gen_label_rtx ();
6060 3984 : lab2 = gen_label_rtx ();
6061 :
6062 3984 : if (fmode != GET_MODE (from))
6063 : {
6064 184 : if (REAL_MODE_FORMAT (GET_MODE (from))
6065 : == &arm_bfloat_half_format
6066 184 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
6067 : /* The BF -> SF conversions can be just a shift, doesn't
6068 : need to handle sNANs. */
6069 : {
6070 4 : int save_flag_finite_math_only = flag_finite_math_only;
6071 4 : flag_finite_math_only = true;
6072 4 : from = convert_to_mode (fmode, from, 0);
6073 4 : flag_finite_math_only = save_flag_finite_math_only;
6074 : }
6075 : else
6076 180 : from = convert_to_mode (fmode, from, 0);
6077 : }
6078 :
6079 : /* See if we need to do the subtraction. */
6080 3984 : do_pending_stack_adjust ();
6081 3984 : emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
6082 3984 : GET_MODE (from), 0, lab1);
6083 :
6084 : /* If not, do the signed "fix" and branch around fixup code. */
6085 3984 : expand_fix (to, from, 0);
6086 3984 : emit_jump_insn (targetm.gen_jump (lab2));
6087 3984 : emit_barrier ();
6088 :
6089 : /* Otherwise, subtract 2**(N-1), convert to signed number,
6090 : then add 2**(N-1). Do the addition using XOR since this
6091 : will often generate better code. */
6092 3984 : emit_label (lab1);
6093 3984 : target = expand_binop (GET_MODE (from), sub_optab, from, limit,
6094 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
6095 3984 : expand_fix (to, target, 0);
6096 3984 : target = expand_binop (to_mode, xor_optab, to,
6097 : gen_int_mode
6098 3984 : (HOST_WIDE_INT_1 << (bitsize - 1),
6099 : to_mode),
6100 : to, 1, OPTAB_LIB_WIDEN);
6101 :
6102 3984 : if (target != to)
6103 0 : emit_move_insn (to, target);
6104 :
6105 3984 : emit_label (lab2);
6106 :
6107 3984 : if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
6108 : {
6109 : /* Make a place for a REG_NOTE and add it. */
6110 3984 : insn = emit_move_insn (to, to);
6111 3984 : set_dst_reg_note (insn, REG_EQUAL,
6112 : gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
6113 : copy_rtx (from)),
6114 : to);
6115 : }
6116 :
6117 3984 : return;
6118 : }
6119 : }
6120 :
6121 : #ifdef HAVE_SFmode
6122 8969 : if (REAL_MODE_FORMAT (GET_MODE (from)) == &arm_bfloat_half_format
6123 8969 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
6124 : /* We don't have BF -> TI library functions, use BF -> SF -> TI
6125 : instead but the BF -> SF conversion can be just a shift, doesn't
6126 : need to handle sNANs. */
6127 : {
6128 4 : int save_flag_finite_math_only = flag_finite_math_only;
6129 4 : flag_finite_math_only = true;
6130 4 : from = convert_to_mode (SFmode, from, 0);
6131 4 : flag_finite_math_only = save_flag_finite_math_only;
6132 4 : expand_fix (to, from, unsignedp);
6133 4 : return;
6134 : }
6135 : #endif
6136 :
6137 : /* We can't do it with an insn, so use a library call. But first ensure
6138 : that the mode of TO is at least as wide as SImode, since those are the
6139 : only library calls we know about. */
6140 :
6141 8965 : if (is_narrower_int_mode (GET_MODE (to), SImode))
6142 : {
6143 1954 : target = gen_reg_rtx (SImode);
6144 :
6145 1954 : expand_fix (target, from, unsignedp);
6146 : }
6147 : else
6148 : {
6149 7011 : rtx_insn *insns;
6150 7011 : rtx value;
6151 7011 : rtx libfunc;
6152 :
6153 7011 : convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
6154 7011 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
6155 7011 : gcc_assert (libfunc);
6156 :
6157 7011 : start_sequence ();
6158 :
6159 14022 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
6160 7011 : GET_MODE (to), from, GET_MODE (from));
6161 7011 : insns = end_sequence ();
6162 :
6163 7011 : emit_libcall_block (insns, target, value,
6164 7011 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
6165 : GET_MODE (to), from));
6166 : }
6167 :
6168 8965 : if (target != to)
6169 : {
6170 1954 : if (GET_MODE (to) == GET_MODE (target))
6171 0 : emit_move_insn (to, target);
6172 : else
6173 1954 : convert_move (to, target, 0);
6174 : }
6175 : }
6176 :
6177 :
6178 : /* Promote integer arguments for a libcall if necessary.
6179 : emit_library_call_value cannot do the promotion because it does not
6180 : know if it should do a signed or unsigned promotion. This is because
6181 : there are no tree types defined for libcalls. */
6182 :
6183 : static rtx
6184 0 : prepare_libcall_arg (rtx arg, int uintp)
6185 : {
6186 0 : scalar_int_mode mode;
6187 0 : machine_mode arg_mode;
6188 0 : if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
6189 : {
6190 : /* If we need to promote the integer function argument we need to do
6191 : it here instead of inside emit_library_call_value because in
6192 : emit_library_call_value we don't know if we should do a signed or
6193 : unsigned promotion. */
6194 :
6195 0 : int unsigned_p = 0;
6196 0 : arg_mode = promote_function_mode (NULL_TREE, mode,
6197 : &unsigned_p, NULL_TREE, 0);
6198 0 : if (arg_mode != mode)
6199 0 : return convert_to_mode (arg_mode, arg, uintp);
6200 : }
6201 : return arg;
6202 : }
6203 :
6204 : /* Generate code to convert FROM or TO a fixed-point.
6205 : If UINTP is true, either TO or FROM is an unsigned integer.
6206 : If SATP is true, we need to saturate the result. */
6207 :
6208 : void
6209 0 : expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
6210 : {
6211 0 : machine_mode to_mode = GET_MODE (to);
6212 0 : machine_mode from_mode = GET_MODE (from);
6213 0 : convert_optab tab;
6214 0 : enum rtx_code this_code;
6215 0 : enum insn_code code;
6216 0 : rtx_insn *insns;
6217 0 : rtx value;
6218 0 : rtx libfunc;
6219 :
6220 0 : if (to_mode == from_mode)
6221 : {
6222 0 : emit_move_insn (to, from);
6223 0 : return;
6224 : }
6225 :
6226 0 : if (uintp)
6227 : {
6228 0 : tab = satp ? satfractuns_optab : fractuns_optab;
6229 0 : this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
6230 : }
6231 : else
6232 : {
6233 0 : tab = satp ? satfract_optab : fract_optab;
6234 0 : this_code = satp ? SAT_FRACT : FRACT_CONVERT;
6235 : }
6236 0 : code = convert_optab_handler (tab, to_mode, from_mode);
6237 0 : if (code != CODE_FOR_nothing)
6238 : {
6239 0 : emit_unop_insn (code, to, from, this_code);
6240 0 : return;
6241 : }
6242 :
6243 0 : libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
6244 0 : gcc_assert (libfunc);
6245 :
6246 0 : from = prepare_libcall_arg (from, uintp);
6247 0 : from_mode = GET_MODE (from);
6248 :
6249 0 : start_sequence ();
6250 0 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
6251 : from, from_mode);
6252 0 : insns = end_sequence ();
6253 :
6254 0 : emit_libcall_block (insns, to, value,
6255 : gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
6256 : }
6257 :
6258 : /* Generate code to convert FROM to fixed point and store in TO. FROM
6259 : must be floating point, TO must be signed. Use the conversion optab
6260 : TAB to do the conversion. */
6261 :
6262 : bool
6263 492 : expand_sfix_optab (rtx to, rtx from, convert_optab tab)
6264 : {
6265 492 : enum insn_code icode;
6266 492 : rtx target = to;
6267 492 : machine_mode fmode, imode;
6268 :
6269 : /* We first try to find a pair of modes, one real and one integer, at
6270 : least as wide as FROM and TO, respectively, in which we can open-code
6271 : this conversion. If the integer mode is wider than the mode of TO,
6272 : we can do the conversion either signed or unsigned. */
6273 :
6274 2038 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
6275 8738 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
6276 : {
6277 7192 : icode = convert_optab_handler (tab, imode, fmode,
6278 : insn_optimization_type ());
6279 7192 : if (icode != CODE_FOR_nothing)
6280 : {
6281 64 : rtx_insn *last = get_last_insn ();
6282 64 : if (fmode != GET_MODE (from))
6283 64 : from = convert_to_mode (fmode, from, 0);
6284 :
6285 64 : if (imode != GET_MODE (to))
6286 0 : target = gen_reg_rtx (imode);
6287 :
6288 64 : if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
6289 : {
6290 0 : delete_insns_since (last);
6291 0 : continue;
6292 : }
6293 64 : if (target != to)
6294 0 : convert_move (to, target, 0);
6295 64 : return true;
6296 : }
6297 : }
6298 :
6299 : return false;
6300 : }
6301 :
6302 : /* Report whether we have an instruction to perform the operation
6303 : specified by CODE on operands of mode MODE. */
6304 : bool
6305 91635110 : have_insn_for (enum rtx_code code, machine_mode mode)
6306 : {
6307 91635110 : return (code_to_optab (code)
6308 91635110 : && (optab_handler (code_to_optab (code), mode)
6309 91635110 : != CODE_FOR_nothing));
6310 : }
6311 :
6312 : /* Print information about the current contents of the optabs on
6313 : STDERR. */
6314 :
6315 : DEBUG_FUNCTION void
6316 0 : debug_optab_libfuncs (void)
6317 : {
6318 0 : int i, j, k;
6319 :
6320 : /* Dump the arithmetic optabs. */
6321 0 : for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6322 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6323 : {
6324 0 : rtx l = optab_libfunc ((optab) i, (machine_mode) j);
6325 0 : if (l)
6326 : {
6327 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6328 0 : fprintf (stderr, "%s\t%s:\t%s\n",
6329 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6330 0 : GET_MODE_NAME (j),
6331 : XSTR (l, 0));
6332 : }
6333 : }
6334 :
6335 : /* Dump the conversion optabs. */
6336 0 : for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6337 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6338 0 : for (k = 0; k < NUM_MACHINE_MODES; ++k)
6339 : {
6340 0 : rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
6341 : (machine_mode) k);
6342 0 : if (l)
6343 : {
6344 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6345 0 : fprintf (stderr, "%s\t%s\t%s:\t%s\n",
6346 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6347 0 : GET_MODE_NAME (j),
6348 0 : GET_MODE_NAME (k),
6349 : XSTR (l, 0));
6350 : }
6351 : }
6352 0 : }
6353 :
6354 : /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6355 : CODE. Return 0 on failure. */
6356 :
6357 : rtx_insn *
6358 0 : gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6359 : {
6360 0 : machine_mode mode = GET_MODE (op1);
6361 0 : enum insn_code icode;
6362 0 : rtx_insn *insn;
6363 0 : rtx trap_rtx;
6364 :
6365 0 : if (mode == VOIDmode)
6366 : return 0;
6367 :
6368 0 : icode = optab_handler (ctrap_optab, mode);
6369 0 : if (icode == CODE_FOR_nothing)
6370 : return 0;
6371 :
6372 : /* Some targets only accept a zero trap code. */
6373 0 : if (!insn_operand_matches (icode, 3, tcode))
6374 : return 0;
6375 :
6376 0 : do_pending_stack_adjust ();
6377 0 : start_sequence ();
6378 0 : prepare_cmp_insn (op1, op2, NULL, code, NULL_RTX, false, OPTAB_DIRECT,
6379 : &trap_rtx, &mode, cbranch_optab);
6380 0 : if (!trap_rtx)
6381 : insn = NULL;
6382 : else
6383 0 : insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6384 : tcode);
6385 :
6386 : /* If that failed, then give up. */
6387 0 : if (insn == 0)
6388 : {
6389 0 : end_sequence ();
6390 0 : return 0;
6391 : }
6392 :
6393 0 : emit_insn (insn);
6394 0 : insn = end_sequence ();
6395 0 : return insn;
6396 : }
6397 :
6398 : /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
6399 : or unsigned operation code. */
6400 :
6401 : enum rtx_code
6402 1519294 : get_rtx_code_1 (enum tree_code tcode, bool unsignedp)
6403 : {
6404 1519294 : enum rtx_code code;
6405 1519294 : switch (tcode)
6406 : {
6407 : case EQ_EXPR:
6408 : code = EQ;
6409 : break;
6410 766051 : case NE_EXPR:
6411 766051 : code = NE;
6412 766051 : break;
6413 157243 : case LT_EXPR:
6414 157243 : code = unsignedp ? LTU : LT;
6415 : break;
6416 72849 : case LE_EXPR:
6417 72849 : code = unsignedp ? LEU : LE;
6418 : break;
6419 118499 : case GT_EXPR:
6420 118499 : code = unsignedp ? GTU : GT;
6421 : break;
6422 78546 : case GE_EXPR:
6423 78546 : code = unsignedp ? GEU : GE;
6424 : break;
6425 :
6426 1235 : case UNORDERED_EXPR:
6427 1235 : code = UNORDERED;
6428 1235 : break;
6429 1034 : case ORDERED_EXPR:
6430 1034 : code = ORDERED;
6431 1034 : break;
6432 452 : case UNLT_EXPR:
6433 452 : code = UNLT;
6434 452 : break;
6435 4341 : case UNLE_EXPR:
6436 4341 : code = UNLE;
6437 4341 : break;
6438 457 : case UNGT_EXPR:
6439 457 : code = UNGT;
6440 457 : break;
6441 3438 : case UNGE_EXPR:
6442 3438 : code = UNGE;
6443 3438 : break;
6444 222 : case UNEQ_EXPR:
6445 222 : code = UNEQ;
6446 222 : break;
6447 497 : case LTGT_EXPR:
6448 497 : code = LTGT;
6449 497 : break;
6450 :
6451 32788 : case BIT_AND_EXPR:
6452 32788 : code = AND;
6453 32788 : break;
6454 :
6455 21 : case BIT_IOR_EXPR:
6456 21 : code = IOR;
6457 21 : break;
6458 :
6459 0 : default:
6460 0 : code = UNKNOWN;
6461 0 : break;
6462 : }
6463 1519294 : return code;
6464 : }
6465 :
6466 : /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6467 : or unsigned operation code. */
6468 :
6469 : enum rtx_code
6470 156712 : get_rtx_code (enum tree_code tcode, bool unsignedp)
6471 : {
6472 156712 : enum rtx_code code = get_rtx_code_1 (tcode, unsignedp);
6473 156712 : gcc_assert (code != UNKNOWN);
6474 156712 : return code;
6475 : }
6476 :
6477 : /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6478 : select signed or unsigned operators. OPNO holds the index of the
6479 : first comparison operand for insn ICODE. Do not generate the
6480 : compare instruction itself. */
6481 :
6482 : rtx
6483 24065 : vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
6484 : tree t_op0, tree t_op1, bool unsignedp,
6485 : enum insn_code icode, unsigned int opno)
6486 : {
6487 24065 : class expand_operand ops[2];
6488 24065 : rtx rtx_op0, rtx_op1;
6489 24065 : machine_mode m0, m1;
6490 24065 : enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6491 :
6492 24065 : gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6493 :
6494 : /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6495 : has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6496 : cases, use the original mode. */
6497 24065 : rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6498 : EXPAND_STACK_PARM);
6499 24065 : m0 = GET_MODE (rtx_op0);
6500 24065 : if (m0 == VOIDmode)
6501 0 : m0 = TYPE_MODE (TREE_TYPE (t_op0));
6502 :
6503 24065 : rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6504 : EXPAND_STACK_PARM);
6505 24065 : m1 = GET_MODE (rtx_op1);
6506 24065 : if (m1 == VOIDmode)
6507 0 : m1 = TYPE_MODE (TREE_TYPE (t_op1));
6508 :
6509 24065 : create_input_operand (&ops[0], rtx_op0, m0);
6510 24065 : create_input_operand (&ops[1], rtx_op1, m1);
6511 24065 : if (!maybe_legitimize_operands (icode, opno, 2, ops))
6512 0 : gcc_unreachable ();
6513 24065 : return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
6514 : }
6515 :
6516 : /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6517 : the first vec_perm operand, assuming the second operand (for left shift
6518 : first operand) is a constant vector of zeros. Return the shift distance
6519 : in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6520 : mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6521 : shift or vec_shl_optab for left shift. */
6522 : static rtx
6523 2801 : shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
6524 : optab shift_optab)
6525 : {
6526 2801 : unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
6527 2801 : poly_int64 first = sel[0];
6528 5602 : if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
6529 : return NULL_RTX;
6530 :
6531 2800 : if (shift_optab == vec_shl_optab)
6532 : {
6533 807 : unsigned int nelt;
6534 1614 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6535 2801 : return NULL_RTX;
6536 807 : unsigned firstidx = 0;
6537 5547 : for (unsigned int i = 0; i < nelt; i++)
6538 : {
6539 4794 : if (known_eq (sel[i], nelt))
6540 : {
6541 777 : if (i == 0 || firstidx)
6542 : return NULL_RTX;
6543 : firstidx = i;
6544 : }
6545 8034 : else if (firstidx
6546 5464 : ? maybe_ne (sel[i], nelt + i - firstidx)
6547 4017 : : maybe_ge (sel[i], nelt))
6548 : return NULL_RTX;
6549 : }
6550 :
6551 753 : if (firstidx == 0)
6552 : return NULL_RTX;
6553 753 : first = firstidx;
6554 : }
6555 1993 : else if (!sel.series_p (0, 1, first, 1))
6556 : {
6557 162 : unsigned int nelt;
6558 324 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6559 187 : return NULL_RTX;
6560 306 : for (unsigned int i = 1; i < nelt; i++)
6561 : {
6562 276 : poly_int64 expected = i + first;
6563 : /* Indices into the second vector are all equivalent. */
6564 276 : if (maybe_lt (sel[i], nelt)
6565 427 : ? maybe_ne (sel[i], expected)
6566 151 : : maybe_lt (expected, nelt))
6567 187 : return NULL_RTX;
6568 : }
6569 : }
6570 :
6571 2614 : return gen_int_shift_amount (mode, first * bitsize);
6572 : }
6573 :
6574 : /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6575 :
6576 : static rtx
6577 10 : expand_vec_perm_1 (enum insn_code icode, rtx target,
6578 : rtx v0, rtx v1, rtx sel)
6579 : {
6580 10 : machine_mode tmode = GET_MODE (target);
6581 10 : machine_mode smode = GET_MODE (sel);
6582 10 : class expand_operand ops[4];
6583 :
6584 10 : gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
6585 : || related_int_vector_mode (tmode).require () == smode);
6586 10 : create_output_operand (&ops[0], target, tmode);
6587 10 : create_input_operand (&ops[3], sel, smode);
6588 :
6589 : /* Make an effort to preserve v0 == v1. The target expander is able to
6590 : rely on this to determine if we're permuting a single input operand. */
6591 10 : if (rtx_equal_p (v0, v1))
6592 : {
6593 6 : if (!insn_operand_matches (icode, 1, v0))
6594 0 : v0 = force_reg (tmode, v0);
6595 6 : gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6596 6 : gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6597 :
6598 6 : create_fixed_operand (&ops[1], v0);
6599 6 : create_fixed_operand (&ops[2], v0);
6600 : }
6601 : else
6602 : {
6603 4 : create_input_operand (&ops[1], v0, tmode);
6604 4 : create_input_operand (&ops[2], v1, tmode);
6605 : }
6606 :
6607 10 : if (maybe_expand_insn (icode, 4, ops))
6608 10 : return ops[0].value;
6609 : return NULL_RTX;
6610 : }
6611 :
6612 : /* Check if vec_perm mask SEL is a constant equivalent to an and operation of
6613 : the non-zero vec_perm operand with some mask consisting of 0xffs and 0x00s,
6614 : assuming the other vec_perm operand is a constant vector of zeros. Return
6615 : the mask for the equivalent and operation, or NULL_RTX if the vec_perm can
6616 : not be modeled as an and. MODE is the mode of the value being anded.
6617 : ZERO_OP0_P is true if the first operand of the vec_perm is a constant vector
6618 : of zeros or false if the second operand of the vec_perm is a constant vector
6619 : of zeros. */
6620 : rtx
6621 0 : vec_perm_and_mask (machine_mode mode, const vec_perm_indices &sel,
6622 : bool zero_op0_p)
6623 : {
6624 0 : unsigned int nelt;
6625 0 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6626 : return NULL_RTX;
6627 :
6628 0 : rtx_vector_builder builder (mode, nelt, 1);
6629 0 : machine_mode emode = GET_MODE_INNER (mode);
6630 :
6631 0 : for (unsigned int i = 0; i < nelt; i++)
6632 : {
6633 0 : if (zero_op0_p)
6634 : {
6635 0 : if (known_eq (sel[i], nelt + i))
6636 0 : builder.quick_push (CONSTM1_RTX (emode));
6637 0 : else if (known_lt (sel[i], nelt))
6638 0 : builder.quick_push (CONST0_RTX (emode));
6639 : else
6640 0 : return NULL_RTX;
6641 : }
6642 : else
6643 : {
6644 0 : if (known_eq (sel[i], i))
6645 0 : builder.quick_push (CONSTM1_RTX (emode));
6646 0 : else if (known_ge (sel[i], nelt))
6647 0 : builder.quick_push (CONST0_RTX (emode));
6648 : else
6649 : return NULL_RTX;
6650 : }
6651 : }
6652 :
6653 0 : return builder.build ();
6654 0 : }
6655 :
6656 : /* Implement a permutation of vectors v0 and v1 using the permutation
6657 : vector in SEL and return the result. Use TARGET to hold the result
6658 : if nonnull and convenient.
6659 :
6660 : MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6661 : is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6662 : to have a particular mode. */
6663 :
6664 : rtx
6665 75082 : expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
6666 : const vec_perm_builder &sel, machine_mode sel_mode,
6667 : rtx target)
6668 : {
6669 75082 : if (!target || !register_operand (target, mode))
6670 60863 : target = gen_reg_rtx (mode);
6671 :
6672 : /* Set QIMODE to a different vector mode with byte elements.
6673 : If no such mode, or if MODE already has byte elements, use VOIDmode. */
6674 75082 : machine_mode qimode;
6675 75082 : if (!qimode_for_vec_perm (mode).exists (&qimode))
6676 7085 : qimode = VOIDmode;
6677 :
6678 75082 : rtx_insn *last = get_last_insn ();
6679 :
6680 75082 : bool single_arg_p = rtx_equal_p (v0, v1);
6681 : /* Always specify two input vectors here and leave the target to handle
6682 : cases in which the inputs are equal. Not all backends can cope with
6683 : the single-input representation when testing for a double-input
6684 : target instruction. */
6685 150164 : vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
6686 :
6687 : /* See if this can be handled with a vec_shr or vec_shl. We only do this
6688 : if the second (for vec_shr) or first (for vec_shl) vector is all
6689 : zeroes. */
6690 75082 : insn_code shift_code = CODE_FOR_nothing;
6691 75082 : insn_code shift_code_qi = CODE_FOR_nothing;
6692 75082 : optab shift_optab = unknown_optab;
6693 75082 : rtx v2 = v0;
6694 75082 : if (v1 == CONST0_RTX (GET_MODE (v1)))
6695 : shift_optab = vec_shr_optab;
6696 73024 : else if (v0 == CONST0_RTX (GET_MODE (v0)))
6697 : {
6698 : shift_optab = vec_shl_optab;
6699 : v2 = v1;
6700 : }
6701 : if (shift_optab != unknown_optab)
6702 : {
6703 3189 : shift_code = optab_handler (shift_optab, mode);
6704 5778 : shift_code_qi = ((qimode != VOIDmode && qimode != mode)
6705 6078 : ? optab_handler (shift_optab, qimode)
6706 : : CODE_FOR_nothing);
6707 : }
6708 3189 : if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
6709 : {
6710 2801 : rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices, shift_optab);
6711 2801 : if (shift_amt)
6712 : {
6713 2614 : class expand_operand ops[3];
6714 2614 : if (shift_amt == const0_rtx)
6715 2614 : return v2;
6716 2614 : if (shift_code != CODE_FOR_nothing)
6717 : {
6718 2614 : create_output_operand (&ops[0], target, mode);
6719 2614 : create_input_operand (&ops[1], v2, mode);
6720 2614 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6721 2614 : if (maybe_expand_insn (shift_code, 3, ops))
6722 2614 : return ops[0].value;
6723 : }
6724 0 : if (shift_code_qi != CODE_FOR_nothing)
6725 : {
6726 0 : rtx tmp = gen_reg_rtx (qimode);
6727 0 : create_output_operand (&ops[0], tmp, qimode);
6728 0 : create_input_operand (&ops[1], gen_lowpart (qimode, v2), qimode);
6729 0 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6730 0 : if (maybe_expand_insn (shift_code_qi, 3, ops))
6731 0 : return gen_lowpart (mode, ops[0].value);
6732 : }
6733 : }
6734 : }
6735 :
6736 72468 : if (targetm.vectorize.vec_perm_const != NULL)
6737 : {
6738 72468 : if (single_arg_p)
6739 18512 : v1 = v0;
6740 :
6741 72468 : gcc_checking_assert (GET_MODE (v0) == GET_MODE (v1));
6742 72468 : machine_mode op_mode = GET_MODE (v0);
6743 72468 : if (targetm.vectorize.vec_perm_const (mode, op_mode, target, v0, v1,
6744 : indices))
6745 : return target;
6746 : }
6747 :
6748 : /* Fall back to a constant byte-based permutation. */
6749 0 : vec_perm_indices qimode_indices;
6750 0 : rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
6751 0 : if (qimode != VOIDmode)
6752 : {
6753 0 : qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
6754 0 : target_qi = gen_reg_rtx (qimode);
6755 0 : v0_qi = gen_lowpart (qimode, v0);
6756 0 : v1_qi = gen_lowpart (qimode, v1);
6757 0 : if (targetm.vectorize.vec_perm_const != NULL
6758 0 : && targetm.vectorize.vec_perm_const (qimode, qimode, target_qi, v0_qi,
6759 : v1_qi, qimode_indices))
6760 0 : return gen_lowpart (mode, target_qi);
6761 : }
6762 :
6763 0 : v0 = force_reg (mode, v0);
6764 0 : if (single_arg_p)
6765 0 : v1 = v0;
6766 0 : v1 = force_reg (mode, v1);
6767 :
6768 : /* Otherwise expand as a fully variable permutation. */
6769 :
6770 : /* The optabs are only defined for selectors with the same width
6771 : as the values being permuted. */
6772 0 : machine_mode required_sel_mode;
6773 0 : if (!related_int_vector_mode (mode).exists (&required_sel_mode))
6774 : {
6775 0 : delete_insns_since (last);
6776 0 : return NULL_RTX;
6777 : }
6778 :
6779 : /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6780 : If that isn't the mode we want then we need to prove that using
6781 : REQUIRED_SEL_MODE is OK. */
6782 0 : if (sel_mode != required_sel_mode)
6783 : {
6784 0 : if (!selector_fits_mode_p (required_sel_mode, indices))
6785 : {
6786 0 : delete_insns_since (last);
6787 0 : return NULL_RTX;
6788 : }
6789 : sel_mode = required_sel_mode;
6790 : }
6791 :
6792 0 : insn_code icode = direct_optab_handler (vec_perm_optab, mode);
6793 0 : if (icode != CODE_FOR_nothing)
6794 : {
6795 0 : rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
6796 0 : rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
6797 0 : if (tmp)
6798 : return tmp;
6799 : }
6800 :
6801 0 : if (qimode != VOIDmode
6802 0 : && selector_fits_mode_p (qimode, qimode_indices))
6803 : {
6804 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
6805 0 : if (icode != CODE_FOR_nothing)
6806 : {
6807 0 : rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
6808 0 : rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
6809 0 : if (tmp)
6810 0 : return gen_lowpart (mode, tmp);
6811 : }
6812 : }
6813 :
6814 0 : delete_insns_since (last);
6815 0 : return NULL_RTX;
6816 0 : }
6817 :
6818 : /* Implement a permutation of vectors v0 and v1 using the permutation
6819 : vector in SEL and return the result. Use TARGET to hold the result
6820 : if nonnull and convenient.
6821 :
6822 : MODE is the mode of the vectors being permuted (V0 and V1).
6823 : SEL must have the integer equivalent of MODE and is known to be
6824 : unsuitable for permutes with a constant permutation vector. */
6825 :
6826 : rtx
6827 10 : expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6828 : {
6829 10 : enum insn_code icode;
6830 10 : unsigned int i, u;
6831 10 : rtx tmp, sel_qi;
6832 :
6833 10 : u = GET_MODE_UNIT_SIZE (mode);
6834 :
6835 10 : if (!target || GET_MODE (target) != mode)
6836 0 : target = gen_reg_rtx (mode);
6837 :
6838 10 : icode = direct_optab_handler (vec_perm_optab, mode);
6839 10 : if (icode != CODE_FOR_nothing)
6840 : {
6841 10 : tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6842 10 : if (tmp)
6843 : return tmp;
6844 : }
6845 :
6846 : /* As a special case to aid several targets, lower the element-based
6847 : permutation to a byte-based permutation and try again. */
6848 0 : machine_mode qimode;
6849 0 : if (!qimode_for_vec_perm (mode).exists (&qimode)
6850 0 : || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
6851 0 : return NULL_RTX;
6852 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
6853 0 : if (icode == CODE_FOR_nothing)
6854 : return NULL_RTX;
6855 :
6856 : /* Multiply each element by its byte size. */
6857 0 : machine_mode selmode = GET_MODE (sel);
6858 0 : if (u == 2)
6859 0 : sel = expand_simple_binop (selmode, PLUS, sel, sel,
6860 : NULL, 0, OPTAB_DIRECT);
6861 : else
6862 0 : sel = expand_simple_binop (selmode, ASHIFT, sel,
6863 0 : gen_int_shift_amount (selmode, exact_log2 (u)),
6864 : NULL, 0, OPTAB_DIRECT);
6865 0 : gcc_assert (sel != NULL);
6866 :
6867 : /* Broadcast the low byte each element into each of its bytes.
6868 : The encoding has U interleaved stepped patterns, one for each
6869 : byte of an element. */
6870 0 : vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
6871 0 : unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
6872 0 : for (i = 0; i < 3; ++i)
6873 0 : for (unsigned int j = 0; j < u; ++j)
6874 0 : const_sel.quick_push (i * u + low_byte_in_u);
6875 0 : sel = gen_lowpart (qimode, sel);
6876 0 : sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
6877 0 : gcc_assert (sel != NULL);
6878 :
6879 : /* Add the byte offset to each byte element. */
6880 : /* Note that the definition of the indicies here is memory ordering,
6881 : so there should be no difference between big and little endian. */
6882 0 : rtx_vector_builder byte_indices (qimode, u, 1);
6883 0 : for (i = 0; i < u; ++i)
6884 0 : byte_indices.quick_push (GEN_INT (i));
6885 0 : tmp = byte_indices.build ();
6886 0 : sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
6887 : sel, 0, OPTAB_DIRECT);
6888 0 : gcc_assert (sel_qi != NULL);
6889 :
6890 0 : tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
6891 0 : tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
6892 0 : gen_lowpart (qimode, v1), sel_qi);
6893 0 : if (tmp)
6894 0 : tmp = gen_lowpart (mode, tmp);
6895 0 : return tmp;
6896 0 : }
6897 :
6898 : /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
6899 : Use TARGET for the result if nonnull and convenient. */
6900 :
6901 : rtx
6902 0 : expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
6903 : {
6904 0 : class expand_operand ops[3];
6905 0 : enum insn_code icode;
6906 0 : machine_mode emode = GET_MODE_INNER (vmode);
6907 :
6908 0 : icode = direct_optab_handler (vec_series_optab, vmode);
6909 0 : gcc_assert (icode != CODE_FOR_nothing);
6910 :
6911 0 : create_output_operand (&ops[0], target, vmode);
6912 0 : create_input_operand (&ops[1], op0, emode);
6913 0 : create_input_operand (&ops[2], op1, emode);
6914 :
6915 0 : expand_insn (icode, 3, ops);
6916 0 : return ops[0].value;
6917 : }
6918 :
6919 : /* Generate insns for a vector comparison into a mask. */
6920 :
6921 : rtx
6922 24065 : expand_vec_cmp_expr (tree type, tree exp, rtx target)
6923 : {
6924 24065 : class expand_operand ops[4];
6925 24065 : enum insn_code icode;
6926 24065 : rtx comparison;
6927 24065 : machine_mode mask_mode = TYPE_MODE (type);
6928 24065 : machine_mode vmode;
6929 24065 : bool unsignedp;
6930 24065 : tree op0a, op0b;
6931 24065 : enum tree_code tcode;
6932 :
6933 24065 : op0a = TREE_OPERAND (exp, 0);
6934 24065 : op0b = TREE_OPERAND (exp, 1);
6935 24065 : tcode = TREE_CODE (exp);
6936 :
6937 24065 : unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
6938 24065 : vmode = TYPE_MODE (TREE_TYPE (op0a));
6939 :
6940 24065 : icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
6941 24065 : if (icode == CODE_FOR_nothing)
6942 : {
6943 522 : if (tcode == EQ_EXPR || tcode == NE_EXPR)
6944 522 : icode = get_vec_cmp_eq_icode (vmode, mask_mode);
6945 522 : if (icode == CODE_FOR_nothing)
6946 : return 0;
6947 : }
6948 :
6949 24065 : comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
6950 : unsignedp, icode, 2);
6951 24065 : create_output_operand (&ops[0], target, mask_mode);
6952 24065 : create_fixed_operand (&ops[1], comparison);
6953 24065 : create_fixed_operand (&ops[2], XEXP (comparison, 0));
6954 24065 : create_fixed_operand (&ops[3], XEXP (comparison, 1));
6955 24065 : expand_insn (icode, 4, ops);
6956 24065 : return ops[0].value;
6957 : }
6958 :
6959 : /* Expand a highpart multiply. */
6960 :
6961 : rtx
6962 1732 : expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
6963 : rtx target, bool uns_p)
6964 : {
6965 1732 : class expand_operand eops[3];
6966 1732 : enum insn_code icode;
6967 1732 : int method, i;
6968 1732 : machine_mode wmode;
6969 1732 : rtx m1, m2;
6970 1732 : optab tab1, tab2;
6971 :
6972 1732 : method = can_mult_highpart_p (mode, uns_p);
6973 1732 : switch (method)
6974 : {
6975 : case 0:
6976 : return NULL_RTX;
6977 1020 : case 1:
6978 1020 : tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
6979 1020 : return expand_binop (mode, tab1, op0, op1, target, uns_p,
6980 1020 : OPTAB_LIB_WIDEN);
6981 0 : case 2:
6982 0 : return expmed_mult_highpart_optab (as_a <scalar_int_mode> (mode),
6983 0 : op0, op1, target, uns_p, INT_MAX);
6984 704 : case 3:
6985 704 : tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
6986 502 : tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
6987 : break;
6988 8 : case 4:
6989 8 : tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
6990 8 : tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
6991 : if (BYTES_BIG_ENDIAN)
6992 : std::swap (tab1, tab2);
6993 : break;
6994 0 : default:
6995 0 : gcc_unreachable ();
6996 : }
6997 :
6998 712 : icode = optab_handler (tab1, mode);
6999 712 : wmode = insn_data[icode].operand[0].mode;
7000 2136 : gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
7001 : GET_MODE_NUNITS (mode)));
7002 2136 : gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
7003 :
7004 712 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7005 712 : create_input_operand (&eops[1], op0, mode);
7006 712 : create_input_operand (&eops[2], op1, mode);
7007 712 : expand_insn (icode, 3, eops);
7008 712 : m1 = gen_lowpart (mode, eops[0].value);
7009 :
7010 712 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7011 712 : create_input_operand (&eops[1], op0, mode);
7012 712 : create_input_operand (&eops[2], op1, mode);
7013 712 : expand_insn (optab_handler (tab2, mode), 3, eops);
7014 712 : m2 = gen_lowpart (mode, eops[0].value);
7015 :
7016 1424 : vec_perm_builder sel;
7017 712 : if (method == 3)
7018 : {
7019 : /* The encoding has 2 interleaved stepped patterns. */
7020 1408 : sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
7021 4928 : for (i = 0; i < 6; ++i)
7022 12672 : sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
7023 6336 : + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
7024 : }
7025 : else
7026 : {
7027 : /* The encoding has a single interleaved stepped pattern. */
7028 16 : sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
7029 32 : for (i = 0; i < 3; ++i)
7030 24 : sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
7031 : }
7032 :
7033 712 : return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
7034 : }
7035 :
7036 : /* Helper function to find the MODE_CC set in a sync_compare_and_swap
7037 : pattern. */
7038 :
7039 : static void
7040 0 : find_cc_set (rtx x, const_rtx pat, void *data)
7041 : {
7042 0 : if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
7043 0 : && GET_CODE (pat) == SET)
7044 : {
7045 0 : rtx *p_cc_reg = (rtx *) data;
7046 0 : gcc_assert (!*p_cc_reg);
7047 0 : *p_cc_reg = x;
7048 : }
7049 0 : }
7050 :
7051 : /* This is a helper function for the other atomic operations. This function
7052 : emits a loop that contains SEQ that iterates until a compare-and-swap
7053 : operation at the end succeeds. MEM is the memory to be modified. SEQ is
7054 : a set of instructions that takes a value from OLD_REG as an input and
7055 : produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
7056 : set to the current contents of MEM. After SEQ, a compare-and-swap will
7057 : attempt to update MEM with NEW_REG. The function returns true when the
7058 : loop was generated successfully. */
7059 :
7060 : static bool
7061 5070 : expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
7062 : {
7063 5070 : machine_mode mode = GET_MODE (mem);
7064 5070 : rtx_code_label *label;
7065 5070 : rtx cmp_reg, success, oldval;
7066 :
7067 : /* The loop we want to generate looks like
7068 :
7069 : cmp_reg = mem;
7070 : label:
7071 : old_reg = cmp_reg;
7072 : seq;
7073 : (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
7074 : if (success)
7075 : goto label;
7076 :
7077 : Note that we only do the plain load from memory once. Subsequent
7078 : iterations use the value loaded by the compare-and-swap pattern. */
7079 :
7080 5070 : label = gen_label_rtx ();
7081 5070 : cmp_reg = gen_reg_rtx (mode);
7082 :
7083 5070 : emit_move_insn (cmp_reg, mem);
7084 5070 : emit_label (label);
7085 5070 : emit_move_insn (old_reg, cmp_reg);
7086 5070 : if (seq)
7087 5057 : emit_insn (seq);
7088 :
7089 5070 : success = NULL_RTX;
7090 5070 : oldval = cmp_reg;
7091 5070 : if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
7092 : new_reg, false, MEMMODEL_SYNC_SEQ_CST,
7093 : MEMMODEL_RELAXED))
7094 : return false;
7095 :
7096 5070 : if (oldval != cmp_reg)
7097 0 : emit_move_insn (cmp_reg, oldval);
7098 :
7099 : /* Mark this jump predicted not taken. */
7100 5070 : emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
7101 5070 : GET_MODE (success), 1, label,
7102 : profile_probability::guessed_never ());
7103 5070 : return true;
7104 : }
7105 :
7106 :
7107 : /* This function tries to emit an atomic_exchange intruction. VAL is written
7108 : to *MEM using memory model MODEL. The previous contents of *MEM are returned,
7109 : using TARGET if possible. */
7110 :
7111 : static rtx
7112 3885 : maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7113 : {
7114 3885 : machine_mode mode = GET_MODE (mem);
7115 3885 : enum insn_code icode;
7116 :
7117 : /* If the target supports the exchange directly, great. */
7118 3885 : icode = direct_optab_handler (atomic_exchange_optab, mode);
7119 3885 : if (icode != CODE_FOR_nothing)
7120 : {
7121 3868 : class expand_operand ops[4];
7122 :
7123 3868 : create_output_operand (&ops[0], target, mode);
7124 3868 : create_fixed_operand (&ops[1], mem);
7125 3868 : create_input_operand (&ops[2], val, mode);
7126 3868 : create_integer_operand (&ops[3], model);
7127 3868 : if (maybe_expand_insn (icode, 4, ops))
7128 3868 : return ops[0].value;
7129 : }
7130 :
7131 : return NULL_RTX;
7132 : }
7133 :
7134 : /* This function tries to implement an atomic exchange operation using
7135 : __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
7136 : The previous contents of *MEM are returned, using TARGET if possible.
7137 : Since this instructionn is an acquire barrier only, stronger memory
7138 : models may require additional barriers to be emitted. */
7139 :
7140 : static rtx
7141 2 : maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
7142 : enum memmodel model)
7143 : {
7144 2 : machine_mode mode = GET_MODE (mem);
7145 2 : enum insn_code icode;
7146 2 : rtx_insn *last_insn = get_last_insn ();
7147 :
7148 2 : icode = optab_handler (sync_lock_test_and_set_optab, mode);
7149 :
7150 : /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7151 : exists, and the memory model is stronger than acquire, add a release
7152 : barrier before the instruction. */
7153 :
7154 2 : if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
7155 0 : expand_mem_thread_fence (model);
7156 :
7157 2 : if (icode != CODE_FOR_nothing)
7158 : {
7159 0 : class expand_operand ops[3];
7160 0 : create_output_operand (&ops[0], target, mode);
7161 0 : create_fixed_operand (&ops[1], mem);
7162 0 : create_input_operand (&ops[2], val, mode);
7163 0 : if (maybe_expand_insn (icode, 3, ops))
7164 0 : return ops[0].value;
7165 : }
7166 :
7167 : /* If an external test-and-set libcall is provided, use that instead of
7168 : any external compare-and-swap that we might get from the compare-and-
7169 : swap-loop expansion later. */
7170 2 : if (!can_compare_and_swap_p (mode, false))
7171 : {
7172 2 : rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
7173 2 : if (libfunc != NULL)
7174 : {
7175 0 : rtx addr;
7176 :
7177 0 : addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7178 0 : return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7179 : mode, addr, ptr_mode,
7180 0 : val, mode);
7181 : }
7182 : }
7183 :
7184 : /* If the test_and_set can't be emitted, eliminate any barrier that might
7185 : have been emitted. */
7186 2 : delete_insns_since (last_insn);
7187 2 : return NULL_RTX;
7188 : }
7189 :
7190 : /* This function tries to implement an atomic exchange operation using a
7191 : compare_and_swap loop. VAL is written to *MEM. The previous contents of
7192 : *MEM are returned, using TARGET if possible. No memory model is required
7193 : since a compare_and_swap loop is seq-cst. */
7194 :
7195 : static rtx
7196 17 : maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
7197 : {
7198 17 : machine_mode mode = GET_MODE (mem);
7199 :
7200 17 : if (can_compare_and_swap_p (mode, true))
7201 : {
7202 13 : if (!target || !register_operand (target, mode))
7203 1 : target = gen_reg_rtx (mode);
7204 13 : if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
7205 : return target;
7206 : }
7207 :
7208 : return NULL_RTX;
7209 : }
7210 :
7211 : /* This function tries to implement an atomic test-and-set operation
7212 : using the atomic_test_and_set instruction pattern. A boolean value
7213 : is returned from the operation, using TARGET if possible. */
7214 :
7215 : static rtx
7216 268 : maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7217 : {
7218 268 : machine_mode pat_bool_mode;
7219 268 : class expand_operand ops[3];
7220 :
7221 268 : if (!targetm.have_atomic_test_and_set ())
7222 : return NULL_RTX;
7223 :
7224 : /* While we always get QImode from __atomic_test_and_set, we get
7225 : other memory modes from __sync_lock_test_and_set. Note that we
7226 : use no endian adjustment here. This matches the 4.6 behavior
7227 : in the Sparc backend. */
7228 0 : enum insn_code icode = targetm.code_for_atomic_test_and_set;
7229 0 : gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
7230 0 : if (GET_MODE (mem) != QImode)
7231 0 : mem = adjust_address_nv (mem, QImode, 0);
7232 :
7233 0 : pat_bool_mode = insn_data[icode].operand[0].mode;
7234 0 : create_output_operand (&ops[0], target, pat_bool_mode);
7235 0 : create_fixed_operand (&ops[1], mem);
7236 0 : create_integer_operand (&ops[2], model);
7237 :
7238 0 : if (maybe_expand_insn (icode, 3, ops))
7239 0 : return ops[0].value;
7240 : return NULL_RTX;
7241 : }
7242 :
7243 : /* This function expands the legacy _sync_lock test_and_set operation which is
7244 : generally an atomic exchange. Some limited targets only allow the
7245 : constant 1 to be stored. This is an ACQUIRE operation.
7246 :
7247 : TARGET is an optional place to stick the return value.
7248 : MEM is where VAL is stored. */
7249 :
7250 : rtx
7251 326 : expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7252 : {
7253 326 : rtx ret;
7254 :
7255 : /* Try an atomic_exchange first. */
7256 326 : ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
7257 326 : if (ret)
7258 : return ret;
7259 :
7260 2 : ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
7261 : MEMMODEL_SYNC_ACQUIRE);
7262 2 : if (ret)
7263 : return ret;
7264 :
7265 2 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7266 2 : if (ret)
7267 : return ret;
7268 :
7269 : /* If there are no other options, try atomic_test_and_set if the value
7270 : being stored is 1. */
7271 2 : if (val == const1_rtx)
7272 2 : ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
7273 :
7274 : return ret;
7275 : }
7276 :
7277 : /* This function expands the atomic test_and_set operation:
7278 : atomically store a boolean TRUE into MEM and return the previous value.
7279 :
7280 : MEMMODEL is the memory model variant to use.
7281 : TARGET is an optional place to stick the return value. */
7282 :
7283 : rtx
7284 266 : expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7285 : {
7286 266 : machine_mode mode = GET_MODE (mem);
7287 266 : rtx ret, trueval, subtarget;
7288 :
7289 266 : ret = maybe_emit_atomic_test_and_set (target, mem, model);
7290 266 : if (ret)
7291 : return ret;
7292 :
7293 : /* Be binary compatible with non-default settings of trueval, and different
7294 : cpu revisions. E.g. one revision may have atomic-test-and-set, but
7295 : another only has atomic-exchange. */
7296 266 : if (targetm.atomic_test_and_set_trueval == 1)
7297 : {
7298 266 : trueval = const1_rtx;
7299 266 : subtarget = target ? target : gen_reg_rtx (mode);
7300 : }
7301 : else
7302 : {
7303 0 : trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7304 0 : subtarget = gen_reg_rtx (mode);
7305 : }
7306 :
7307 : /* Try the atomic-exchange optab... */
7308 266 : ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
7309 :
7310 : /* ... then an atomic-compare-and-swap loop ... */
7311 266 : if (!ret)
7312 0 : ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
7313 :
7314 : /* ... before trying the vaguely defined legacy lock_test_and_set. */
7315 0 : if (!ret)
7316 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
7317 :
7318 : /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7319 : things with the value 1. Thus we try again without trueval. */
7320 266 : if (!ret && targetm.atomic_test_and_set_trueval != 1)
7321 : {
7322 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
7323 :
7324 0 : if (ret)
7325 : {
7326 : /* Rectify the not-one trueval. */
7327 0 : ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7328 0 : gcc_assert (ret);
7329 : }
7330 : }
7331 :
7332 : return ret;
7333 : }
7334 :
7335 : /* This function expands the atomic exchange operation:
7336 : atomically store VAL in MEM and return the previous value in MEM.
7337 :
7338 : MEMMODEL is the memory model variant to use.
7339 : TARGET is an optional place to stick the return value. */
7340 :
7341 : rtx
7342 2830 : expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7343 : {
7344 2830 : machine_mode mode = GET_MODE (mem);
7345 2830 : rtx ret;
7346 :
7347 : /* If loads are not atomic for the required size and we are not called to
7348 : provide a __sync builtin, do not do anything so that we stay consistent
7349 : with atomic loads of the same size. */
7350 2830 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7351 : return NULL_RTX;
7352 :
7353 2799 : ret = maybe_emit_atomic_exchange (target, mem, val, model);
7354 :
7355 : /* Next try a compare-and-swap loop for the exchange. */
7356 2799 : if (!ret)
7357 13 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7358 :
7359 : return ret;
7360 : }
7361 :
7362 : /* This function expands the atomic compare exchange operation:
7363 :
7364 : *PTARGET_BOOL is an optional place to store the boolean success/failure.
7365 : *PTARGET_OVAL is an optional place to store the old value from memory.
7366 : Both target parameters may be NULL or const0_rtx to indicate that we do
7367 : not care about that return value. Both target parameters are updated on
7368 : success to the actual location of the corresponding result.
7369 :
7370 : MEMMODEL is the memory model variant to use.
7371 :
7372 : The return value of the function is true for success. */
7373 :
7374 : bool
7375 28480 : expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7376 : rtx mem, rtx expected, rtx desired,
7377 : bool is_weak, enum memmodel succ_model,
7378 : enum memmodel fail_model)
7379 : {
7380 28480 : machine_mode mode = GET_MODE (mem);
7381 28480 : class expand_operand ops[8];
7382 28480 : enum insn_code icode;
7383 28480 : rtx target_oval, target_bool = NULL_RTX;
7384 28480 : rtx libfunc;
7385 :
7386 : /* If loads are not atomic for the required size and we are not called to
7387 : provide a __sync builtin, do not do anything so that we stay consistent
7388 : with atomic loads of the same size. */
7389 28480 : if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
7390 : return false;
7391 :
7392 : /* Load expected into a register for the compare and swap. */
7393 26729 : if (MEM_P (expected))
7394 7363 : expected = copy_to_reg (expected);
7395 :
7396 : /* Make sure we always have some place to put the return oldval.
7397 : Further, make sure that place is distinct from the input expected,
7398 : just in case we need that path down below. */
7399 26729 : if (ptarget_oval && *ptarget_oval == const0_rtx)
7400 : ptarget_oval = NULL;
7401 :
7402 26503 : if (ptarget_oval == NULL
7403 26503 : || (target_oval = *ptarget_oval) == NULL
7404 5300 : || reg_overlap_mentioned_p (expected, target_oval))
7405 21507 : target_oval = gen_reg_rtx (mode);
7406 :
7407 26729 : icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7408 26729 : if (icode != CODE_FOR_nothing)
7409 : {
7410 26727 : machine_mode bool_mode = insn_data[icode].operand[0].mode;
7411 :
7412 26727 : if (ptarget_bool && *ptarget_bool == const0_rtx)
7413 : ptarget_bool = NULL;
7414 :
7415 : /* Make sure we always have a place for the bool operand. */
7416 26472 : if (ptarget_bool == NULL
7417 26472 : || (target_bool = *ptarget_bool) == NULL
7418 7244 : || GET_MODE (target_bool) != bool_mode)
7419 19483 : target_bool = gen_reg_rtx (bool_mode);
7420 :
7421 : /* Emit the compare_and_swap. */
7422 26727 : create_output_operand (&ops[0], target_bool, bool_mode);
7423 26727 : create_output_operand (&ops[1], target_oval, mode);
7424 26727 : create_fixed_operand (&ops[2], mem);
7425 26727 : create_input_operand (&ops[3], expected, mode);
7426 26727 : create_input_operand (&ops[4], desired, mode);
7427 26727 : create_integer_operand (&ops[5], is_weak);
7428 26727 : create_integer_operand (&ops[6], succ_model);
7429 26727 : create_integer_operand (&ops[7], fail_model);
7430 26727 : if (maybe_expand_insn (icode, 8, ops))
7431 : {
7432 : /* Return success/failure. */
7433 26727 : target_bool = ops[0].value;
7434 26727 : target_oval = ops[1].value;
7435 26727 : goto success;
7436 : }
7437 : }
7438 :
7439 : /* Otherwise fall back to the original __sync_val_compare_and_swap
7440 : which is always seq-cst. */
7441 2 : icode = optab_handler (sync_compare_and_swap_optab, mode);
7442 2 : if (icode != CODE_FOR_nothing)
7443 : {
7444 0 : rtx cc_reg;
7445 :
7446 0 : create_output_operand (&ops[0], target_oval, mode);
7447 0 : create_fixed_operand (&ops[1], mem);
7448 0 : create_input_operand (&ops[2], expected, mode);
7449 0 : create_input_operand (&ops[3], desired, mode);
7450 0 : if (!maybe_expand_insn (icode, 4, ops))
7451 0 : return false;
7452 :
7453 0 : target_oval = ops[0].value;
7454 :
7455 : /* If the caller isn't interested in the boolean return value,
7456 : skip the computation of it. */
7457 0 : if (ptarget_bool == NULL)
7458 0 : goto success;
7459 :
7460 : /* Otherwise, work out if the compare-and-swap succeeded. */
7461 0 : cc_reg = NULL_RTX;
7462 0 : if (have_insn_for (COMPARE, CCmode))
7463 0 : note_stores (get_last_insn (), find_cc_set, &cc_reg);
7464 0 : if (cc_reg)
7465 : {
7466 0 : target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7467 : const0_rtx, VOIDmode, 0, 1);
7468 0 : goto success;
7469 : }
7470 0 : goto success_bool_from_val;
7471 : }
7472 :
7473 : /* Also check for library support for __sync_val_compare_and_swap. */
7474 2 : libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7475 2 : if (libfunc != NULL)
7476 : {
7477 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7478 0 : rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7479 : mode, addr, ptr_mode,
7480 : expected, mode, desired, mode);
7481 0 : emit_move_insn (target_oval, target);
7482 :
7483 : /* Compute the boolean return value only if requested. */
7484 0 : if (ptarget_bool)
7485 0 : goto success_bool_from_val;
7486 : else
7487 0 : goto success;
7488 : }
7489 :
7490 : /* Failure. */
7491 : return false;
7492 :
7493 0 : success_bool_from_val:
7494 0 : target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7495 : expected, VOIDmode, 1, 1);
7496 26727 : success:
7497 : /* Make sure that the oval output winds up where the caller asked. */
7498 26727 : if (ptarget_oval)
7499 26503 : *ptarget_oval = target_oval;
7500 26727 : if (ptarget_bool)
7501 26472 : *ptarget_bool = target_bool;
7502 : return true;
7503 : }
7504 :
7505 : /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7506 :
7507 : static void
7508 0 : expand_asm_memory_blockage (void)
7509 : {
7510 0 : rtx asm_op, clob;
7511 :
7512 0 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7513 : rtvec_alloc (0), rtvec_alloc (0),
7514 : rtvec_alloc (0), UNKNOWN_LOCATION);
7515 0 : MEM_VOLATILE_P (asm_op) = 1;
7516 :
7517 0 : clob = gen_rtx_SCRATCH (VOIDmode);
7518 0 : clob = gen_rtx_MEM (BLKmode, clob);
7519 0 : clob = gen_rtx_CLOBBER (VOIDmode, clob);
7520 :
7521 0 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7522 0 : }
7523 :
7524 : /* Do not propagate memory accesses across this point. */
7525 :
7526 : static void
7527 115713 : expand_memory_blockage (void)
7528 : {
7529 115713 : if (targetm.have_memory_blockage ())
7530 115713 : emit_insn (targetm.gen_memory_blockage ());
7531 : else
7532 0 : expand_asm_memory_blockage ();
7533 115713 : }
7534 :
7535 : /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7536 : same time clobbering the register set specified by REGS. */
7537 :
7538 : void
7539 131 : expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs)
7540 : {
7541 131 : rtx asm_op, clob_mem;
7542 :
7543 131 : unsigned int num_of_regs = 0;
7544 12183 : for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7545 12052 : if (TEST_HARD_REG_BIT (regs, i))
7546 1119 : num_of_regs++;
7547 :
7548 131 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7549 : rtvec_alloc (0), rtvec_alloc (0),
7550 : rtvec_alloc (0), UNKNOWN_LOCATION);
7551 131 : MEM_VOLATILE_P (asm_op) = 1;
7552 :
7553 131 : rtvec v = rtvec_alloc (num_of_regs + 2);
7554 :
7555 131 : clob_mem = gen_rtx_SCRATCH (VOIDmode);
7556 131 : clob_mem = gen_rtx_MEM (BLKmode, clob_mem);
7557 131 : clob_mem = gen_rtx_CLOBBER (VOIDmode, clob_mem);
7558 :
7559 131 : RTVEC_ELT (v, 0) = asm_op;
7560 131 : RTVEC_ELT (v, 1) = clob_mem;
7561 :
7562 131 : if (num_of_regs > 0)
7563 : {
7564 : unsigned int j = 2;
7565 12183 : for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7566 12052 : if (TEST_HARD_REG_BIT (regs, i))
7567 : {
7568 1119 : RTVEC_ELT (v, j) = gen_rtx_CLOBBER (VOIDmode, regno_reg_rtx[i]);
7569 1119 : j++;
7570 : }
7571 131 : gcc_assert (j == (num_of_regs + 2));
7572 : }
7573 :
7574 131 : emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
7575 131 : }
7576 :
7577 : /* This routine will either emit the mem_thread_fence pattern or issue a
7578 : sync_synchronize to generate a fence for memory model MEMMODEL. */
7579 :
7580 : void
7581 966 : expand_mem_thread_fence (enum memmodel model)
7582 : {
7583 966 : if (is_mm_relaxed (model))
7584 : return;
7585 957 : if (targetm.have_mem_thread_fence ())
7586 : {
7587 957 : emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
7588 957 : expand_memory_blockage ();
7589 : }
7590 0 : else if (targetm.have_memory_barrier ())
7591 0 : emit_insn (targetm.gen_memory_barrier ());
7592 0 : else if (synchronize_libfunc != NULL_RTX)
7593 0 : emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
7594 : else
7595 0 : expand_memory_blockage ();
7596 : }
7597 :
7598 : /* Emit a signal fence with given memory model. */
7599 :
7600 : void
7601 60 : expand_mem_signal_fence (enum memmodel model)
7602 : {
7603 : /* No machine barrier is required to implement a signal fence, but
7604 : a compiler memory barrier must be issued, except for relaxed MM. */
7605 60 : if (!is_mm_relaxed (model))
7606 51 : expand_memory_blockage ();
7607 60 : }
7608 :
7609 : /* This function expands the atomic load operation:
7610 : return the atomically loaded value in MEM.
7611 :
7612 : MEMMODEL is the memory model variant to use.
7613 : TARGET is an option place to stick the return value. */
7614 :
7615 : rtx
7616 70081 : expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7617 : {
7618 70081 : machine_mode mode = GET_MODE (mem);
7619 70081 : enum insn_code icode;
7620 :
7621 : /* If the target supports the load directly, great. */
7622 70081 : icode = direct_optab_handler (atomic_load_optab, mode);
7623 70081 : if (icode != CODE_FOR_nothing)
7624 : {
7625 66253 : class expand_operand ops[3];
7626 66253 : rtx_insn *last = get_last_insn ();
7627 66253 : if (is_mm_seq_cst (model))
7628 38213 : expand_memory_blockage ();
7629 :
7630 66253 : create_output_operand (&ops[0], target, mode);
7631 66253 : create_fixed_operand (&ops[1], mem);
7632 66253 : create_integer_operand (&ops[2], model);
7633 66253 : if (maybe_expand_insn (icode, 3, ops))
7634 : {
7635 66253 : if (!is_mm_relaxed (model))
7636 51718 : expand_memory_blockage ();
7637 66253 : return ops[0].value;
7638 : }
7639 0 : delete_insns_since (last);
7640 : }
7641 :
7642 : /* If the size of the object is greater than word size on this target,
7643 : then we assume that a load will not be atomic. We could try to
7644 : emulate a load with a compare-and-swap operation, but the store that
7645 : doing this could result in would be incorrect if this is a volatile
7646 : atomic load or targetting read-only-mapped memory. */
7647 3828 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7648 : /* If there is no atomic load, leave the library call. */
7649 : return NULL_RTX;
7650 :
7651 : /* Otherwise assume loads are atomic, and emit the proper barriers. */
7652 0 : if (!target || target == const0_rtx)
7653 0 : target = gen_reg_rtx (mode);
7654 :
7655 : /* For SEQ_CST, emit a barrier before the load. */
7656 0 : if (is_mm_seq_cst (model))
7657 0 : expand_mem_thread_fence (model);
7658 :
7659 0 : emit_move_insn (target, mem);
7660 :
7661 : /* Emit the appropriate barrier after the load. */
7662 0 : expand_mem_thread_fence (model);
7663 :
7664 0 : return target;
7665 : }
7666 :
7667 : /* This function expands the atomic store operation:
7668 : Atomically store VAL in MEM.
7669 : MEMMODEL is the memory model variant to use.
7670 : USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7671 : function returns const0_rtx if a pattern was emitted. */
7672 :
7673 : rtx
7674 17588 : expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7675 : {
7676 17588 : machine_mode mode = GET_MODE (mem);
7677 17588 : enum insn_code icode;
7678 17588 : class expand_operand ops[3];
7679 :
7680 : /* If the target supports the store directly, great. */
7681 17588 : icode = direct_optab_handler (atomic_store_optab, mode);
7682 17588 : if (icode != CODE_FOR_nothing)
7683 : {
7684 15990 : rtx_insn *last = get_last_insn ();
7685 15990 : if (!is_mm_relaxed (model))
7686 13240 : expand_memory_blockage ();
7687 15990 : create_fixed_operand (&ops[0], mem);
7688 15990 : create_input_operand (&ops[1], val, mode);
7689 15990 : create_integer_operand (&ops[2], model);
7690 15990 : if (maybe_expand_insn (icode, 3, ops))
7691 : {
7692 15990 : if (is_mm_seq_cst (model))
7693 11534 : expand_memory_blockage ();
7694 15990 : return const0_rtx;
7695 : }
7696 0 : delete_insns_since (last);
7697 : }
7698 :
7699 : /* If using __sync_lock_release is a viable alternative, try it.
7700 : Note that this will not be set to true if we are expanding a generic
7701 : __atomic_store_n. */
7702 1598 : if (use_release)
7703 : {
7704 2 : icode = direct_optab_handler (sync_lock_release_optab, mode);
7705 2 : if (icode != CODE_FOR_nothing)
7706 : {
7707 0 : create_fixed_operand (&ops[0], mem);
7708 0 : create_input_operand (&ops[1], const0_rtx, mode);
7709 0 : if (maybe_expand_insn (icode, 2, ops))
7710 : {
7711 : /* lock_release is only a release barrier. */
7712 0 : if (is_mm_seq_cst (model))
7713 0 : expand_mem_thread_fence (model);
7714 0 : return const0_rtx;
7715 : }
7716 : }
7717 : }
7718 :
7719 : /* If the size of the object is greater than word size on this target,
7720 : a default store will not be atomic. */
7721 1598 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7722 : {
7723 : /* If loads are atomic or we are called to provide a __sync builtin,
7724 : we can try a atomic_exchange and throw away the result. Otherwise,
7725 : don't do anything so that we do not create an inconsistency between
7726 : loads and stores. */
7727 1598 : if (can_atomic_load_p (mode) || is_mm_sync (model))
7728 : {
7729 2 : rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7730 2 : if (!target)
7731 2 : target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
7732 : val);
7733 2 : if (target)
7734 0 : return const0_rtx;
7735 : }
7736 1598 : return NULL_RTX;
7737 : }
7738 :
7739 : /* Otherwise assume stores are atomic, and emit the proper barriers. */
7740 0 : expand_mem_thread_fence (model);
7741 :
7742 0 : emit_move_insn (mem, val);
7743 :
7744 : /* For SEQ_CST, also emit a barrier after the store. */
7745 0 : if (is_mm_seq_cst (model))
7746 0 : expand_mem_thread_fence (model);
7747 :
7748 0 : return const0_rtx;
7749 : }
7750 :
7751 :
7752 : /* Structure containing the pointers and values required to process the
7753 : various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7754 :
7755 : struct atomic_op_functions
7756 : {
7757 : direct_optab mem_fetch_before;
7758 : direct_optab mem_fetch_after;
7759 : direct_optab mem_no_result;
7760 : optab fetch_before;
7761 : optab fetch_after;
7762 : direct_optab no_result;
7763 : enum rtx_code reverse_code;
7764 : };
7765 :
7766 :
7767 : /* Fill in structure pointed to by OP with the various optab entries for an
7768 : operation of type CODE. */
7769 :
7770 : static void
7771 34789 : get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7772 : {
7773 34789 : gcc_assert (op!= NULL);
7774 :
7775 : /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7776 : in the source code during compilation, and the optab entries are not
7777 : computable until runtime. Fill in the values at runtime. */
7778 34789 : switch (code)
7779 : {
7780 22519 : case PLUS:
7781 22519 : op->mem_fetch_before = atomic_fetch_add_optab;
7782 22519 : op->mem_fetch_after = atomic_add_fetch_optab;
7783 22519 : op->mem_no_result = atomic_add_optab;
7784 22519 : op->fetch_before = sync_old_add_optab;
7785 22519 : op->fetch_after = sync_new_add_optab;
7786 22519 : op->no_result = sync_add_optab;
7787 22519 : op->reverse_code = MINUS;
7788 22519 : break;
7789 4809 : case MINUS:
7790 4809 : op->mem_fetch_before = atomic_fetch_sub_optab;
7791 4809 : op->mem_fetch_after = atomic_sub_fetch_optab;
7792 4809 : op->mem_no_result = atomic_sub_optab;
7793 4809 : op->fetch_before = sync_old_sub_optab;
7794 4809 : op->fetch_after = sync_new_sub_optab;
7795 4809 : op->no_result = sync_sub_optab;
7796 4809 : op->reverse_code = PLUS;
7797 4809 : break;
7798 2194 : case XOR:
7799 2194 : op->mem_fetch_before = atomic_fetch_xor_optab;
7800 2194 : op->mem_fetch_after = atomic_xor_fetch_optab;
7801 2194 : op->mem_no_result = atomic_xor_optab;
7802 2194 : op->fetch_before = sync_old_xor_optab;
7803 2194 : op->fetch_after = sync_new_xor_optab;
7804 2194 : op->no_result = sync_xor_optab;
7805 2194 : op->reverse_code = XOR;
7806 2194 : break;
7807 2068 : case AND:
7808 2068 : op->mem_fetch_before = atomic_fetch_and_optab;
7809 2068 : op->mem_fetch_after = atomic_and_fetch_optab;
7810 2068 : op->mem_no_result = atomic_and_optab;
7811 2068 : op->fetch_before = sync_old_and_optab;
7812 2068 : op->fetch_after = sync_new_and_optab;
7813 2068 : op->no_result = sync_and_optab;
7814 2068 : op->reverse_code = UNKNOWN;
7815 2068 : break;
7816 2496 : case IOR:
7817 2496 : op->mem_fetch_before = atomic_fetch_or_optab;
7818 2496 : op->mem_fetch_after = atomic_or_fetch_optab;
7819 2496 : op->mem_no_result = atomic_or_optab;
7820 2496 : op->fetch_before = sync_old_ior_optab;
7821 2496 : op->fetch_after = sync_new_ior_optab;
7822 2496 : op->no_result = sync_ior_optab;
7823 2496 : op->reverse_code = UNKNOWN;
7824 2496 : break;
7825 703 : case NOT:
7826 703 : op->mem_fetch_before = atomic_fetch_nand_optab;
7827 703 : op->mem_fetch_after = atomic_nand_fetch_optab;
7828 703 : op->mem_no_result = atomic_nand_optab;
7829 703 : op->fetch_before = sync_old_nand_optab;
7830 703 : op->fetch_after = sync_new_nand_optab;
7831 703 : op->no_result = sync_nand_optab;
7832 703 : op->reverse_code = UNKNOWN;
7833 703 : break;
7834 0 : default:
7835 0 : gcc_unreachable ();
7836 : }
7837 34789 : }
7838 :
7839 : /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7840 : using memory order MODEL. If AFTER is true the operation needs to return
7841 : the value of *MEM after the operation, otherwise the previous value.
7842 : TARGET is an optional place to place the result. The result is unused if
7843 : it is const0_rtx.
7844 : Return the result if there is a better sequence, otherwise NULL_RTX. */
7845 :
7846 : static rtx
7847 34787 : maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7848 : enum memmodel model, bool after)
7849 : {
7850 : /* If the value is prefetched, or not used, it may be possible to replace
7851 : the sequence with a native exchange operation. */
7852 34787 : if (!after || target == const0_rtx)
7853 : {
7854 : /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
7855 26686 : if (code == AND && val == const0_rtx)
7856 : {
7857 327 : if (target == const0_rtx)
7858 150 : target = gen_reg_rtx (GET_MODE (mem));
7859 327 : return maybe_emit_atomic_exchange (target, mem, val, model);
7860 : }
7861 :
7862 : /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
7863 26359 : if (code == IOR && val == constm1_rtx)
7864 : {
7865 165 : if (target == const0_rtx)
7866 164 : target = gen_reg_rtx (GET_MODE (mem));
7867 165 : return maybe_emit_atomic_exchange (target, mem, val, model);
7868 : }
7869 : }
7870 :
7871 : return NULL_RTX;
7872 : }
7873 :
7874 : /* Try to emit an instruction for a specific operation varaition.
7875 : OPTAB contains the OP functions.
7876 : TARGET is an optional place to return the result. const0_rtx means unused.
7877 : MEM is the memory location to operate on.
7878 : VAL is the value to use in the operation.
7879 : USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
7880 : MODEL is the memory model, if used.
7881 : AFTER is true if the returned result is the value after the operation. */
7882 :
7883 : static rtx
7884 64245 : maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
7885 : rtx val, bool use_memmodel, enum memmodel model, bool after)
7886 : {
7887 64245 : machine_mode mode = GET_MODE (mem);
7888 64245 : class expand_operand ops[4];
7889 64245 : enum insn_code icode;
7890 64245 : int op_counter = 0;
7891 64245 : int num_ops;
7892 :
7893 : /* Check to see if there is a result returned. */
7894 64245 : if (target == const0_rtx)
7895 : {
7896 14640 : if (use_memmodel)
7897 : {
7898 14281 : icode = direct_optab_handler (optab->mem_no_result, mode);
7899 14281 : create_integer_operand (&ops[2], model);
7900 14281 : num_ops = 3;
7901 : }
7902 : else
7903 : {
7904 359 : icode = direct_optab_handler (optab->no_result, mode);
7905 359 : num_ops = 2;
7906 : }
7907 : }
7908 : /* Otherwise, we need to generate a result. */
7909 : else
7910 : {
7911 49605 : if (use_memmodel)
7912 : {
7913 30964 : icode = direct_optab_handler (after ? optab->mem_fetch_after
7914 : : optab->mem_fetch_before, mode);
7915 30964 : create_integer_operand (&ops[3], model);
7916 30964 : num_ops = 4;
7917 : }
7918 : else
7919 : {
7920 18641 : icode = optab_handler (after ? optab->fetch_after
7921 : : optab->fetch_before, mode);
7922 18641 : num_ops = 3;
7923 : }
7924 49605 : create_output_operand (&ops[op_counter++], target, mode);
7925 : }
7926 64245 : if (icode == CODE_FOR_nothing)
7927 : return NULL_RTX;
7928 :
7929 26245 : create_fixed_operand (&ops[op_counter++], mem);
7930 : /* VAL may have been promoted to a wider mode. Shrink it if so. */
7931 26245 : create_convert_operand_to (&ops[op_counter++], val, mode, true);
7932 :
7933 26245 : if (maybe_expand_insn (icode, num_ops, ops))
7934 26245 : return (target == const0_rtx ? const0_rtx : ops[0].value);
7935 :
7936 : return NULL_RTX;
7937 : }
7938 :
7939 :
7940 : /* This function expands an atomic fetch_OP or OP_fetch operation:
7941 : TARGET is an option place to stick the return value. const0_rtx indicates
7942 : the result is unused.
7943 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
7944 : CODE is the operation being performed (OP)
7945 : MEMMODEL is the memory model variant to use.
7946 : AFTER is true to return the result of the operation (OP_fetch).
7947 : AFTER is false to return the value before the operation (fetch_OP).
7948 :
7949 : This function will *only* generate instructions if there is a direct
7950 : optab. No compare and swap loops or libcalls will be generated. */
7951 :
7952 : static rtx
7953 34787 : expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
7954 : enum rtx_code code, enum memmodel model,
7955 : bool after)
7956 : {
7957 34787 : machine_mode mode = GET_MODE (mem);
7958 34787 : struct atomic_op_functions optab;
7959 34787 : rtx result;
7960 34787 : bool unused_result = (target == const0_rtx);
7961 :
7962 34787 : get_atomic_op_for_code (&optab, code);
7963 :
7964 : /* Check to see if there are any better instructions. */
7965 34787 : result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
7966 34787 : if (result)
7967 : return result;
7968 :
7969 : /* Check for the case where the result isn't used and try those patterns. */
7970 34295 : if (unused_result)
7971 : {
7972 : /* Try the memory model variant first. */
7973 14281 : result = maybe_emit_op (&optab, target, mem, val, true, model, true);
7974 14281 : if (result)
7975 : return result;
7976 :
7977 : /* Next try the old style withuot a memory model. */
7978 359 : result = maybe_emit_op (&optab, target, mem, val, false, model, true);
7979 359 : if (result)
7980 : return result;
7981 :
7982 : /* There is no no-result pattern, so try patterns with a result. */
7983 : target = NULL_RTX;
7984 : }
7985 :
7986 : /* Try the __atomic version. */
7987 20373 : result = maybe_emit_op (&optab, target, mem, val, true, model, after);
7988 20373 : if (result)
7989 : return result;
7990 :
7991 : /* Try the older __sync version. */
7992 11874 : result = maybe_emit_op (&optab, target, mem, val, false, model, after);
7993 11874 : if (result)
7994 : return result;
7995 :
7996 : /* If the fetch value can be calculated from the other variation of fetch,
7997 : try that operation. */
7998 11874 : if (after || unused_result || optab.reverse_code != UNKNOWN)
7999 : {
8000 : /* Try the __atomic version, then the older __sync version. */
8001 10591 : result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
8002 10591 : if (!result)
8003 6767 : result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
8004 :
8005 6767 : if (result)
8006 : {
8007 : /* If the result isn't used, no need to do compensation code. */
8008 3824 : if (unused_result)
8009 : return result;
8010 :
8011 : /* Issue compensation code. Fetch_after == fetch_before OP val.
8012 : Fetch_before == after REVERSE_OP val. */
8013 3824 : if (!after)
8014 0 : code = optab.reverse_code;
8015 3824 : if (code == NOT)
8016 : {
8017 0 : result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
8018 : true, OPTAB_LIB_WIDEN);
8019 0 : result = expand_simple_unop (mode, NOT, result, target, true);
8020 : }
8021 : else
8022 3824 : result = expand_simple_binop (mode, code, result, val, target,
8023 : true, OPTAB_LIB_WIDEN);
8024 3824 : return result;
8025 : }
8026 : }
8027 :
8028 : /* No direct opcode can be generated. */
8029 : return NULL_RTX;
8030 : }
8031 :
8032 :
8033 :
8034 : /* This function expands an atomic fetch_OP or OP_fetch operation:
8035 : TARGET is an option place to stick the return value. const0_rtx indicates
8036 : the result is unused.
8037 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
8038 : CODE is the operation being performed (OP)
8039 : MEMMODEL is the memory model variant to use.
8040 : AFTER is true to return the result of the operation (OP_fetch).
8041 : AFTER is false to return the value before the operation (fetch_OP). */
8042 : rtx
8043 32395 : expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
8044 : enum memmodel model, bool after)
8045 : {
8046 32395 : machine_mode mode = GET_MODE (mem);
8047 32395 : rtx result;
8048 32395 : bool unused_result = (target == const0_rtx);
8049 :
8050 : /* If loads are not atomic for the required size and we are not called to
8051 : provide a __sync builtin, do not do anything so that we stay consistent
8052 : with atomic loads of the same size. */
8053 32395 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
8054 : return NULL_RTX;
8055 :
8056 31796 : result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
8057 : after);
8058 :
8059 31796 : if (result)
8060 : return result;
8061 :
8062 : /* Add/sub can be implemented by doing the reverse operation with -(val). */
8063 8011 : if (code == PLUS || code == MINUS)
8064 : {
8065 2991 : rtx tmp;
8066 2991 : enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
8067 :
8068 2991 : start_sequence ();
8069 2991 : tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
8070 2991 : result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
8071 : model, after);
8072 2991 : if (result)
8073 : {
8074 : /* PLUS worked so emit the insns and return. */
8075 2952 : tmp = end_sequence ();
8076 2952 : emit_insn (tmp);
8077 2952 : return result;
8078 : }
8079 :
8080 : /* PLUS did not work, so throw away the negation code and continue. */
8081 39 : end_sequence ();
8082 : }
8083 :
8084 : /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
8085 5059 : if (!can_compare_and_swap_p (mode, false))
8086 : {
8087 2 : rtx libfunc;
8088 2 : bool fixup = false;
8089 2 : enum rtx_code orig_code = code;
8090 2 : struct atomic_op_functions optab;
8091 :
8092 2 : get_atomic_op_for_code (&optab, code);
8093 2 : libfunc = optab_libfunc (after ? optab.fetch_after
8094 : : optab.fetch_before, mode);
8095 2 : if (libfunc == NULL
8096 2 : && (after || unused_result || optab.reverse_code != UNKNOWN))
8097 : {
8098 2 : fixup = true;
8099 2 : if (!after)
8100 1 : code = optab.reverse_code;
8101 2 : libfunc = optab_libfunc (after ? optab.fetch_before
8102 : : optab.fetch_after, mode);
8103 : }
8104 2 : if (libfunc != NULL)
8105 : {
8106 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
8107 0 : result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
8108 : addr, ptr_mode, val, mode);
8109 :
8110 0 : if (!unused_result && fixup)
8111 0 : result = expand_simple_binop (mode, code, result, val, target,
8112 : true, OPTAB_LIB_WIDEN);
8113 0 : return result;
8114 : }
8115 :
8116 : /* We need the original code for any further attempts. */
8117 2 : code = orig_code;
8118 : }
8119 :
8120 : /* If nothing else has succeeded, default to a compare and swap loop. */
8121 5059 : if (can_compare_and_swap_p (mode, true))
8122 : {
8123 5057 : rtx_insn *insn;
8124 5057 : rtx t0 = gen_reg_rtx (mode), t1;
8125 :
8126 5057 : start_sequence ();
8127 :
8128 : /* If the result is used, get a register for it. */
8129 5057 : if (!unused_result)
8130 : {
8131 4721 : if (!target || !register_operand (target, mode))
8132 12 : target = gen_reg_rtx (mode);
8133 : /* If fetch_before, copy the value now. */
8134 4721 : if (!after)
8135 2068 : emit_move_insn (target, t0);
8136 : }
8137 : else
8138 336 : target = const0_rtx;
8139 :
8140 5057 : t1 = t0;
8141 5057 : if (code == NOT)
8142 : {
8143 687 : t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
8144 : true, OPTAB_LIB_WIDEN);
8145 687 : t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
8146 : }
8147 : else
8148 4370 : t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
8149 : OPTAB_LIB_WIDEN);
8150 :
8151 : /* For after, copy the value now. */
8152 5057 : if (!unused_result && after)
8153 2653 : emit_move_insn (target, t1);
8154 5057 : insn = end_sequence ();
8155 :
8156 5057 : if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
8157 : return target;
8158 : }
8159 :
8160 : return NULL_RTX;
8161 : }
8162 :
8163 : /* Return true if OPERAND is suitable for operand number OPNO of
8164 : instruction ICODE. */
8165 :
8166 : bool
8167 151484860 : insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
8168 : {
8169 151484860 : return (!insn_data[(int) icode].operand[opno].predicate
8170 301416432 : || (insn_data[(int) icode].operand[opno].predicate
8171 149931572 : (operand, insn_data[(int) icode].operand[opno].mode)));
8172 : }
8173 :
8174 : /* TARGET is a target of a multiword operation that we are going to
8175 : implement as a series of word-mode operations. Return true if
8176 : TARGET is suitable for this purpose. */
8177 :
8178 : bool
8179 188 : valid_multiword_target_p (rtx target)
8180 : {
8181 188 : machine_mode mode;
8182 188 : int i, size;
8183 :
8184 188 : mode = GET_MODE (target);
8185 376 : if (!GET_MODE_SIZE (mode).is_constant (&size))
8186 : return false;
8187 860 : for (i = 0; i < size; i += UNITS_PER_WORD)
8188 376 : if (!validate_subreg (word_mode, mode, target, i))
8189 : return false;
8190 : return true;
8191 : }
8192 :
8193 : /* Make OP describe an input operand that has value INTVAL and that has
8194 : no inherent mode. This function should only be used for operands that
8195 : are always expand-time constants. The backend may request that INTVAL
8196 : be copied into a different kind of rtx, but it must specify the mode
8197 : of that rtx if so. */
8198 :
8199 : void
8200 1974726 : create_integer_operand (class expand_operand *op, poly_int64 intval)
8201 : {
8202 1974726 : create_expand_operand (op, EXPAND_INTEGER,
8203 : gen_int_mode (intval, MAX_MODE_INT),
8204 : VOIDmode, false, intval);
8205 1974726 : }
8206 :
8207 : /* Like maybe_legitimize_operand, but do not change the code of the
8208 : current rtx value. */
8209 :
8210 : static bool
8211 50459197 : maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
8212 : class expand_operand *op)
8213 : {
8214 : /* See if the operand matches in its current form. */
8215 50459197 : if (insn_operand_matches (icode, opno, op->value))
8216 : return true;
8217 :
8218 : /* If the operand is a memory whose address has no side effects,
8219 : try forcing the address into a non-virtual pseudo register.
8220 : The check for side effects is important because copy_to_mode_reg
8221 : cannot handle things like auto-modified addresses. */
8222 1510537 : if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
8223 : {
8224 13122 : rtx addr, mem;
8225 :
8226 13122 : mem = op->value;
8227 13122 : addr = XEXP (mem, 0);
8228 984 : if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
8229 13928 : && !side_effects_p (addr))
8230 : {
8231 8487 : rtx_insn *last;
8232 8487 : machine_mode mode;
8233 :
8234 8487 : last = get_last_insn ();
8235 8487 : mode = get_address_mode (mem);
8236 8487 : mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
8237 8487 : if (insn_operand_matches (icode, opno, mem))
8238 : {
8239 0 : op->value = mem;
8240 0 : return true;
8241 : }
8242 8487 : delete_insns_since (last);
8243 : }
8244 : }
8245 :
8246 : return false;
8247 : }
8248 :
8249 : /* Try to make OP match operand OPNO of instruction ICODE. Return true
8250 : on success, storing the new operand value back in OP. */
8251 :
8252 : static bool
8253 57978881 : maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
8254 : class expand_operand *op)
8255 : {
8256 57978881 : machine_mode mode, imode, tmode;
8257 :
8258 57978881 : mode = op->mode;
8259 57978881 : switch (op->type)
8260 : {
8261 4207959 : case EXPAND_FIXED:
8262 4207959 : {
8263 4207959 : temporary_volatile_ok v (true);
8264 4207959 : return maybe_legitimize_operand_same_code (icode, opno, op);
8265 4207959 : }
8266 :
8267 18704918 : case EXPAND_OUTPUT:
8268 18704918 : gcc_assert (mode != VOIDmode);
8269 18704918 : if (op->value
8270 11583859 : && op->value != const0_rtx
8271 11583439 : && GET_MODE (op->value) == mode
8272 30253508 : && maybe_legitimize_operand_same_code (icode, opno, op))
8273 : return true;
8274 :
8275 7277891 : op->value = gen_reg_rtx (mode);
8276 7277891 : op->target = 0;
8277 7277891 : break;
8278 :
8279 34702648 : case EXPAND_INPUT:
8280 34702648 : input:
8281 34702648 : gcc_assert (mode != VOIDmode);
8282 34702648 : gcc_assert (GET_MODE (op->value) == VOIDmode
8283 : || GET_MODE (op->value) == mode);
8284 34702648 : if (maybe_legitimize_operand_same_code (icode, opno, op))
8285 : return true;
8286 :
8287 1244586 : op->value = copy_to_mode_reg (mode, op->value);
8288 1244586 : break;
8289 :
8290 198703 : case EXPAND_CONVERT_TO:
8291 198703 : gcc_assert (mode != VOIDmode);
8292 198703 : op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8293 198703 : goto input;
8294 :
8295 604128 : case EXPAND_CONVERT_FROM:
8296 604128 : if (GET_MODE (op->value) != VOIDmode)
8297 397940 : mode = GET_MODE (op->value);
8298 : else
8299 : /* The caller must tell us what mode this value has. */
8300 206188 : gcc_assert (mode != VOIDmode);
8301 :
8302 604128 : imode = insn_data[(int) icode].operand[opno].mode;
8303 604128 : tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
8304 604128 : ? GET_MODE_INNER (imode) : imode);
8305 604128 : if (tmode != VOIDmode && tmode != mode)
8306 : {
8307 1501 : op->value = convert_modes (tmode, mode, op->value, op->unsigned_p);
8308 1501 : mode = tmode;
8309 : }
8310 604128 : if (imode != VOIDmode && imode != mode)
8311 : {
8312 58 : gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
8313 58 : op->value = expand_vector_broadcast (imode, op->value);
8314 58 : mode = imode;
8315 : }
8316 604128 : goto input;
8317 :
8318 3347 : case EXPAND_ADDRESS:
8319 3347 : op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
8320 : op->value);
8321 3347 : goto input;
8322 :
8323 1618808 : case EXPAND_INTEGER:
8324 1618808 : mode = insn_data[(int) icode].operand[opno].mode;
8325 1618808 : if (mode != VOIDmode
8326 1618808 : && known_eq (trunc_int_for_mode (op->int_value, mode),
8327 : op->int_value))
8328 : {
8329 1255452 : op->value = gen_int_mode (op->int_value, mode);
8330 1255452 : goto input;
8331 : }
8332 : break;
8333 :
8334 240 : case EXPAND_UNDEFINED_INPUT:
8335 : /* See if the predicate accepts a SCRATCH rtx, which in this context
8336 : indicates an undefined value. Use an uninitialized register if not. */
8337 240 : if (!insn_operand_matches (icode, opno, op->value))
8338 : {
8339 240 : op->value = gen_reg_rtx (op->mode);
8340 240 : goto input;
8341 : }
8342 : return true;
8343 : }
8344 8885833 : return insn_operand_matches (icode, opno, op->value);
8345 : }
8346 :
8347 : /* Make OP describe an input operand that should have the same value
8348 : as VALUE, after any mode conversion that the target might request.
8349 : TYPE is the type of VALUE. */
8350 :
8351 : void
8352 2614 : create_convert_operand_from_type (class expand_operand *op,
8353 : rtx value, tree type)
8354 : {
8355 2614 : create_convert_operand_from (op, value, TYPE_MODE (type),
8356 2614 : TYPE_UNSIGNED (type));
8357 2614 : }
8358 :
8359 : /* Return true if the requirements on operands OP1 and OP2 of instruction
8360 : ICODE are similar enough for the result of legitimizing OP1 to be
8361 : reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
8362 : with OP1 and OP2 respectively. */
8363 :
8364 : static inline bool
8365 64303301 : can_reuse_operands_p (enum insn_code icode,
8366 : unsigned int opno1, unsigned int opno2,
8367 : const class expand_operand *op1,
8368 : const class expand_operand *op2)
8369 : {
8370 : /* Check requirements that are common to all types. */
8371 64303301 : if (op1->type != op2->type
8372 19967252 : || op1->mode != op2->mode
8373 18492326 : || (insn_data[(int) icode].operand[opno1].mode
8374 18492326 : != insn_data[(int) icode].operand[opno2].mode))
8375 : return false;
8376 :
8377 : /* Check the requirements for specific types. */
8378 16715661 : switch (op1->type)
8379 : {
8380 : case EXPAND_OUTPUT:
8381 : case EXPAND_UNDEFINED_INPUT:
8382 : /* Outputs and undefined intputs must remain distinct. */
8383 : return false;
8384 :
8385 : case EXPAND_FIXED:
8386 : case EXPAND_INPUT:
8387 : case EXPAND_ADDRESS:
8388 : case EXPAND_INTEGER:
8389 : return true;
8390 :
8391 99790 : case EXPAND_CONVERT_TO:
8392 99790 : case EXPAND_CONVERT_FROM:
8393 99790 : return op1->unsigned_p == op2->unsigned_p;
8394 : }
8395 0 : gcc_unreachable ();
8396 : }
8397 :
8398 : /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8399 : of instruction ICODE. Return true on success, leaving the new operand
8400 : values in the OPS themselves. Emit no code on failure. */
8401 :
8402 : bool
8403 19427744 : maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8404 : unsigned int nops, class expand_operand *ops)
8405 : {
8406 19427744 : rtx_insn *last = get_last_insn ();
8407 19427744 : rtx *orig_values = XALLOCAVEC (rtx, nops);
8408 77629257 : for (unsigned int i = 0; i < nops; i++)
8409 : {
8410 58353644 : orig_values[i] = ops[i].value;
8411 :
8412 : /* First try reusing the result of an earlier legitimization.
8413 : This avoids duplicate rtl and ensures that tied operands
8414 : remain tied.
8415 :
8416 : This search is linear, but NOPS is bounded at compile time
8417 : to a small number (current a single digit). */
8418 58353644 : unsigned int j = 0;
8419 122282182 : for (; j < i; ++j)
8420 64303301 : if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
8421 16695376 : && rtx_equal_p (orig_values[j], orig_values[i])
8422 385920 : && ops[j].value
8423 64678068 : && insn_operand_matches (icode, opno + i, ops[j].value))
8424 : {
8425 374763 : ops[i].value = copy_rtx (ops[j].value);
8426 374763 : break;
8427 : }
8428 :
8429 : /* Otherwise try legitimizing the operand on its own. */
8430 58353644 : if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
8431 : {
8432 152131 : delete_insns_since (last);
8433 152131 : return false;
8434 : }
8435 : }
8436 : return true;
8437 : }
8438 :
8439 : /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8440 : as its operands. Return the instruction pattern on success,
8441 : and emit any necessary set-up code. Return null and emit no
8442 : code on failure. */
8443 :
8444 : rtx_insn *
8445 19403677 : maybe_gen_insn (enum insn_code icode, unsigned int nops,
8446 : class expand_operand *ops)
8447 : {
8448 19403677 : gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8449 19403677 : if (!maybe_legitimize_operands (icode, 0, nops, ops))
8450 : return NULL;
8451 :
8452 19251546 : switch (nops)
8453 : {
8454 0 : case 0:
8455 0 : return GEN_FCN (icode) ();
8456 1311 : case 1:
8457 1311 : return GEN_FCN (icode) (ops[0].value);
8458 2837631 : case 2:
8459 2837631 : return GEN_FCN (icode) (ops[0].value, ops[1].value);
8460 14459963 : case 3:
8461 14459963 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8462 1571478 : case 4:
8463 1571478 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8464 1571478 : ops[3].value);
8465 189891 : case 5:
8466 189891 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8467 189891 : ops[3].value, ops[4].value);
8468 262 : case 6:
8469 262 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8470 262 : ops[3].value, ops[4].value, ops[5].value);
8471 0 : case 7:
8472 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8473 0 : ops[3].value, ops[4].value, ops[5].value,
8474 0 : ops[6].value);
8475 26727 : case 8:
8476 26727 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8477 26727 : ops[3].value, ops[4].value, ops[5].value,
8478 26727 : ops[6].value, ops[7].value);
8479 164283 : case 9:
8480 164283 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8481 164283 : ops[3].value, ops[4].value, ops[5].value,
8482 164283 : ops[6].value, ops[7].value, ops[8].value);
8483 0 : case 10:
8484 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8485 0 : ops[3].value, ops[4].value, ops[5].value,
8486 0 : ops[6].value, ops[7].value, ops[8].value,
8487 0 : ops[9].value);
8488 0 : case 11:
8489 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8490 0 : ops[3].value, ops[4].value, ops[5].value,
8491 0 : ops[6].value, ops[7].value, ops[8].value,
8492 0 : ops[9].value, ops[10].value);
8493 : }
8494 0 : gcc_unreachable ();
8495 : }
8496 :
8497 : /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8498 : as its operands. Return true on success and emit no code on failure. */
8499 :
8500 : bool
8501 2858374 : maybe_expand_insn (enum insn_code icode, unsigned int nops,
8502 : class expand_operand *ops)
8503 : {
8504 2858374 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8505 2858374 : if (pat)
8506 : {
8507 2070293 : emit_insn (pat);
8508 2070293 : return true;
8509 : }
8510 : return false;
8511 : }
8512 :
8513 : /* Like maybe_expand_insn, but for jumps. */
8514 :
8515 : bool
8516 1308 : maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8517 : class expand_operand *ops)
8518 : {
8519 1308 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8520 1308 : if (pat)
8521 : {
8522 1308 : emit_jump_insn (pat);
8523 1308 : return true;
8524 : }
8525 : return false;
8526 : }
8527 :
8528 : /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8529 : as its operands. */
8530 :
8531 : void
8532 631898 : expand_insn (enum insn_code icode, unsigned int nops,
8533 : class expand_operand *ops)
8534 : {
8535 631898 : if (!maybe_expand_insn (icode, nops, ops))
8536 0 : gcc_unreachable ();
8537 631898 : }
8538 :
8539 : /* Like expand_insn, but for jumps. */
8540 :
8541 : void
8542 1308 : expand_jump_insn (enum insn_code icode, unsigned int nops,
8543 : class expand_operand *ops)
8544 : {
8545 1308 : if (!maybe_expand_jump_insn (icode, nops, ops))
8546 0 : gcc_unreachable ();
8547 1308 : }
|
