Line data Source code
1 : /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "predict.h"
30 : #include "tm_p.h"
31 : #include "optabs.h"
32 : #include "expmed.h"
33 : #include "emit-rtl.h"
34 : #include "recog.h"
35 : #include "diagnostic-core.h"
36 : #include "rtx-vector-builder.h"
37 :
38 : /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 : is properly defined. */
40 : #include "stor-layout.h"
41 : #include "except.h"
42 : #include "dojump.h"
43 : #include "explow.h"
44 : #include "expr.h"
45 : #include "optabs-tree.h"
46 : #include "libfuncs.h"
47 : #include "internal-fn.h"
48 : #include "langhooks.h"
49 : #include "gimple.h"
50 : #include "ssa.h"
51 : #include "tree-ssa-live.h"
52 : #include "tree-outof-ssa.h"
53 :
54 : static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
55 : machine_mode *);
56 : static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
57 : static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
58 :
59 : static rtx emit_conditional_move_1 (rtx, rtx, rtx, rtx, machine_mode);
60 :
61 : /* Debug facility for use in GDB. */
62 : void debug_optab_libfuncs (void);
63 :
64 : /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
65 : the result of operation CODE applied to OP0 (and OP1 if it is a binary
66 : operation). OP0_MODE is OP0's mode.
67 :
68 : If the last insn does not set TARGET, don't do anything, but return true.
69 :
70 : If the last insn or a previous insn sets TARGET and TARGET is one of OP0
71 : or OP1, don't add the REG_EQUAL note but return false. Our caller can then
72 : try again, ensuring that TARGET is not one of the operands. */
73 :
74 : static bool
75 1275596 : add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
76 : rtx op1, machine_mode op0_mode)
77 : {
78 1275596 : rtx_insn *last_insn;
79 1275596 : rtx set;
80 1275596 : rtx note;
81 :
82 2551192 : gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
83 :
84 1275596 : if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
85 : && GET_RTX_CLASS (code) != RTX_BIN_ARITH
86 : && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
87 : && GET_RTX_CLASS (code) != RTX_COMPARE
88 : && GET_RTX_CLASS (code) != RTX_UNARY)
89 : return true;
90 :
91 1268006 : if (GET_CODE (target) == ZERO_EXTRACT)
92 : return true;
93 :
94 : for (last_insn = insns;
95 2807438 : NEXT_INSN (last_insn) != NULL_RTX;
96 : last_insn = NEXT_INSN (last_insn))
97 : ;
98 :
99 : /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
100 : a value changing in the insn, so the note would be invalid for CSE. */
101 1268006 : if (reg_overlap_mentioned_p (target, op0)
102 1268006 : || (op1 && reg_overlap_mentioned_p (target, op1)))
103 : {
104 58240 : if (MEM_P (target)
105 58240 : && (rtx_equal_p (target, op0)
106 43300 : || (op1 && rtx_equal_p (target, op1))))
107 : {
108 : /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
109 : over expanding it as temp = MEM op X, MEM = temp. If the target
110 : supports MEM = MEM op X instructions, it is sometimes too hard
111 : to reconstruct that form later, especially if X is also a memory,
112 : and due to multiple occurrences of addresses the address might
113 : be forced into register unnecessarily.
114 : Note that not emitting the REG_EQUIV note might inhibit
115 : CSE in some cases. */
116 3572 : set = single_set (last_insn);
117 3572 : if (set
118 3572 : && GET_CODE (SET_SRC (set)) == code
119 3545 : && MEM_P (SET_DEST (set))
120 7117 : && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
121 0 : || (op1 && rtx_equal_p (SET_DEST (set),
122 0 : XEXP (SET_SRC (set), 1)))))
123 : return true;
124 : }
125 54695 : return false;
126 : }
127 :
128 1209766 : set = set_for_reg_notes (last_insn);
129 1209766 : if (set == NULL_RTX)
130 : return true;
131 :
132 736251 : if (! rtx_equal_p (SET_DEST (set), target)
133 : /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
134 736251 : && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
135 0 : || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
136 : return true;
137 :
138 735477 : if (GET_RTX_CLASS (code) == RTX_UNARY)
139 43512 : switch (code)
140 : {
141 3319 : case FFS:
142 3319 : case CLZ:
143 3319 : case CTZ:
144 3319 : case CLRSB:
145 3319 : case POPCOUNT:
146 3319 : case PARITY:
147 3319 : case BSWAP:
148 3319 : if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
149 : {
150 251 : note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
151 251 : if (GET_MODE_UNIT_SIZE (op0_mode)
152 502 : > GET_MODE_UNIT_SIZE (GET_MODE (target)))
153 251 : note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
154 : note, op0_mode);
155 : else
156 0 : note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
157 : note, op0_mode);
158 : break;
159 : }
160 : /* FALLTHRU */
161 43261 : default:
162 43261 : note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
163 43261 : break;
164 : }
165 : else
166 691965 : note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
167 :
168 735477 : set_unique_reg_note (last_insn, REG_EQUAL, note);
169 :
170 735477 : return true;
171 : }
172 :
173 : /* Given two input operands, OP0 and OP1, determine what the correct from_mode
174 : for a widening operation would be. In most cases this would be OP0, but if
175 : that's a constant it'll be VOIDmode, which isn't useful. */
176 :
177 : static machine_mode
178 38679 : widened_mode (machine_mode to_mode, rtx op0, rtx op1)
179 : {
180 38679 : machine_mode m0 = GET_MODE (op0);
181 38679 : machine_mode m1 = GET_MODE (op1);
182 38679 : machine_mode result;
183 :
184 38679 : if (m0 == VOIDmode && m1 == VOIDmode)
185 : return to_mode;
186 77358 : else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
187 : result = m1;
188 : else
189 : result = m0;
190 :
191 116037 : if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
192 0 : return to_mode;
193 :
194 : return result;
195 : }
196 :
197 : /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
198 : says whether OP is signed or unsigned. NO_EXTEND is true if we need
199 : not actually do a sign-extend or zero-extend, but can leave the
200 : higher-order bits of the result rtx undefined, for example, in the case
201 : of logical operations, but not right shifts. */
202 :
203 : static rtx
204 588 : widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
205 : int unsignedp, bool no_extend)
206 : {
207 588 : rtx result;
208 588 : scalar_int_mode int_mode;
209 :
210 : /* If we don't have to extend and this is a constant, return it. */
211 588 : if (no_extend && GET_MODE (op) == VOIDmode)
212 : return op;
213 :
214 : /* If we must extend do so. If OP is a SUBREG for a promoted object, also
215 : extend since it will be more efficient to do so unless the signedness of
216 : a promoted object differs from our extension. */
217 7 : if (! no_extend
218 588 : || !is_a <scalar_int_mode> (mode, &int_mode)
219 7 : || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
220 0 : && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
221 581 : return convert_modes (mode, oldmode, op, unsignedp);
222 :
223 : /* If MODE is no wider than a single word, we return a lowpart or paradoxical
224 : SUBREG. */
225 14 : if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
226 7 : return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
227 :
228 : /* Otherwise, get an object of MODE, clobber it, and set the low-order
229 : part to OP. */
230 :
231 0 : result = gen_reg_rtx (int_mode);
232 0 : emit_clobber (result);
233 0 : emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
234 0 : return result;
235 : }
236 :
237 : /* Expand vector widening operations.
238 :
239 : There are two different classes of operations handled here:
240 : 1) Operations whose result is wider than all the arguments to the operation.
241 : Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
242 : In this case OP0 and optionally OP1 would be initialized,
243 : but WIDE_OP wouldn't (not relevant for this case).
244 : 2) Operations whose result is of the same size as the last argument to the
245 : operation, but wider than all the other arguments to the operation.
246 : Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
247 : In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
248 :
249 : E.g, when called to expand the following operations, this is how
250 : the arguments will be initialized:
251 : nops OP0 OP1 WIDE_OP
252 : widening-sum 2 oprnd0 - oprnd1
253 : widening-dot-product 3 oprnd0 oprnd1 oprnd2
254 : widening-mult 2 oprnd0 oprnd1 -
255 : type-promotion (vec-unpack) 1 oprnd0 - - */
256 :
257 : rtx
258 21410 : expand_widen_pattern_expr (const_sepops ops, rtx op0, rtx op1, rtx wide_op,
259 : rtx target, int unsignedp)
260 : {
261 21410 : class expand_operand eops[4];
262 21410 : tree oprnd0, oprnd1, oprnd2;
263 21410 : machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
264 21410 : optab widen_pattern_optab;
265 21410 : enum insn_code icode;
266 21410 : int nops = TREE_CODE_LENGTH (ops->code);
267 21410 : int op;
268 21410 : bool sbool = false;
269 :
270 21410 : oprnd0 = ops->op0;
271 21410 : oprnd1 = nops >= 2 ? ops->op1 : NULL_TREE;
272 1619 : oprnd2 = nops >= 3 ? ops->op2 : NULL_TREE;
273 :
274 21410 : tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
275 21410 : if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
276 21410 : || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
277 : /* The sign is from the result type rather than operand's type
278 : for these ops. */
279 34 : widen_pattern_optab
280 34 : = optab_for_tree_code (ops->code, ops->type, optab_default);
281 21376 : else if ((ops->code == VEC_UNPACK_HI_EXPR
282 21376 : || ops->code == VEC_UNPACK_LO_EXPR)
283 17921 : && VECTOR_BOOLEAN_TYPE_P (ops->type)
284 4362 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
285 4362 : && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
286 21876 : && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
287 : {
288 : /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
289 : the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
290 : vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
291 : the pattern number of elements in the wider vector. */
292 500 : widen_pattern_optab
293 247 : = (ops->code == VEC_UNPACK_HI_EXPR
294 500 : ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
295 : sbool = true;
296 : }
297 20876 : else if (ops->code == DOT_PROD_EXPR)
298 : {
299 441 : enum optab_subtype subtype = optab_default;
300 441 : signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0));
301 441 : signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1));
302 441 : if (sign1 == sign2)
303 : ;
304 82 : else if (sign1 == SIGNED && sign2 == UNSIGNED)
305 : {
306 : subtype = optab_vector_mixed_sign;
307 : /* Same as optab_vector_mixed_sign but flip the operands. */
308 : std::swap (op0, op1);
309 : }
310 60 : else if (sign1 == UNSIGNED && sign2 == SIGNED)
311 : subtype = optab_vector_mixed_sign;
312 : else
313 0 : gcc_unreachable ();
314 :
315 441 : widen_pattern_optab
316 441 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype);
317 : }
318 : else
319 20435 : widen_pattern_optab
320 20435 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
321 21410 : if (ops->code == WIDEN_MULT_PLUS_EXPR
322 21410 : || ops->code == WIDEN_MULT_MINUS_EXPR
323 21410 : || ops->code == DOT_PROD_EXPR
324 20969 : || ops->code == SAD_EXPR)
325 560 : icode = find_widening_optab_handler (widen_pattern_optab,
326 : TYPE_MODE (TREE_TYPE (ops->op2)),
327 : tmode0);
328 20850 : else if (ops->code == WIDEN_SUM_EXPR)
329 0 : icode = find_widening_optab_handler (widen_pattern_optab,
330 : TYPE_MODE (TREE_TYPE (ops->op1)),
331 : tmode0);
332 : else
333 20850 : icode = optab_handler (widen_pattern_optab, tmode0);
334 21410 : gcc_assert (icode != CODE_FOR_nothing);
335 :
336 21410 : if (nops >= 2)
337 1619 : tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
338 19791 : else if (sbool)
339 : {
340 500 : nops = 2;
341 500 : op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
342 500 : tmode1 = tmode0;
343 : }
344 :
345 : /* The last operand is of a wider mode than the rest of the operands. */
346 21410 : if (nops == 2)
347 : wmode = tmode1;
348 19851 : else if (nops == 3)
349 : {
350 560 : gcc_assert (tmode1 == tmode0);
351 560 : gcc_assert (op1);
352 560 : wmode = TYPE_MODE (TREE_TYPE (oprnd2));
353 : }
354 :
355 21410 : op = 0;
356 21410 : create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
357 21410 : create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
358 21410 : if (op1)
359 2119 : create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
360 21410 : if (wide_op)
361 560 : create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
362 21410 : expand_insn (icode, op, eops);
363 21410 : return eops[0].value;
364 : }
365 :
366 : /* Generate code to perform an operation specified by TERNARY_OPTAB
367 : on operands OP0, OP1 and OP2, with result having machine-mode MODE.
368 :
369 : UNSIGNEDP is for the case where we have to widen the operands
370 : to perform the operation. It says to use zero-extension.
371 :
372 : If TARGET is nonzero, the value
373 : is generated there, if it is convenient to do so.
374 : In all cases an rtx is returned for the locus of the value;
375 : this may or may not be TARGET. */
376 :
377 : rtx
378 0 : expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
379 : rtx op1, rtx op2, rtx target, int unsignedp)
380 : {
381 0 : class expand_operand ops[4];
382 0 : enum insn_code icode = optab_handler (ternary_optab, mode);
383 :
384 0 : gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
385 :
386 0 : create_output_operand (&ops[0], target, mode);
387 0 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
388 0 : create_convert_operand_from (&ops[2], op1, mode, unsignedp);
389 0 : create_convert_operand_from (&ops[3], op2, mode, unsignedp);
390 0 : expand_insn (icode, 4, ops);
391 0 : return ops[0].value;
392 : }
393 :
394 :
395 : /* Like expand_binop, but return a constant rtx if the result can be
396 : calculated at compile time. The arguments and return value are
397 : otherwise the same as for expand_binop. */
398 :
399 : rtx
400 1645 : simplify_expand_binop (machine_mode mode, optab binoptab,
401 : rtx op0, rtx op1, rtx target, int unsignedp,
402 : enum optab_methods methods)
403 : {
404 1645 : if (CONSTANT_P (op0) && CONSTANT_P (op1))
405 : {
406 184 : rtx x = simplify_binary_operation (optab_to_code (binoptab),
407 : mode, op0, op1);
408 184 : if (x)
409 : return x;
410 : }
411 :
412 1461 : return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
413 : }
414 :
415 : /* Like simplify_expand_binop, but always put the result in TARGET.
416 : Return true if the expansion succeeded. */
417 :
418 : bool
419 1277 : force_expand_binop (machine_mode mode, optab binoptab,
420 : rtx op0, rtx op1, rtx target, int unsignedp,
421 : enum optab_methods methods)
422 : {
423 1277 : rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
424 : target, unsignedp, methods);
425 1277 : if (x == 0)
426 : return false;
427 1277 : if (x != target)
428 0 : emit_move_insn (target, x);
429 : return true;
430 : }
431 :
432 : /* Create a new vector value in VMODE with all elements set to OP. If OP
433 : is not a constant, the mode of it must be the element mode of VMODE
434 : (if the element is BImode, additionally OP is allowed to be in QImode).
435 : If OP is a constant, then the return value will be a constant. */
436 :
437 : rtx
438 46 : expand_vector_broadcast (machine_mode vmode, rtx op)
439 : {
440 46 : int n;
441 46 : rtvec vec;
442 :
443 46 : gcc_checking_assert (VECTOR_MODE_P (vmode));
444 46 : gcc_checking_assert (CONST_INT_P (op)
445 : || GET_MODE_INNER (vmode) == GET_MODE (op)
446 : || (GET_MODE_INNER (vmode) == BImode
447 : && GET_MODE (op) == QImode));
448 :
449 46 : if (valid_for_const_vector_p (vmode, op))
450 46 : return gen_const_vec_duplicate (vmode, op);
451 :
452 0 : insn_code icode = optab_handler (vec_duplicate_optab, vmode);
453 0 : if (icode != CODE_FOR_nothing)
454 : {
455 0 : class expand_operand ops[2];
456 0 : create_output_operand (&ops[0], NULL_RTX, vmode);
457 0 : create_input_operand (&ops[1], op, GET_MODE (op));
458 0 : expand_insn (icode, 2, ops);
459 0 : return ops[0].value;
460 : }
461 :
462 0 : if (!GET_MODE_NUNITS (vmode).is_constant (&n))
463 : return NULL;
464 :
465 : /* ??? If the target doesn't have a vec_init, then we have no easy way
466 : of performing this operation. Most of this sort of generic support
467 : is hidden away in the vector lowering support in gimple. */
468 0 : icode = convert_optab_handler (vec_init_optab, vmode,
469 0 : GET_MODE_INNER (vmode));
470 0 : if (icode == CODE_FOR_nothing)
471 : return NULL;
472 :
473 0 : vec = rtvec_alloc (n);
474 0 : for (int i = 0; i < n; ++i)
475 0 : RTVEC_ELT (vec, i) = op;
476 0 : rtx ret = gen_reg_rtx (vmode);
477 0 : emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
478 :
479 0 : return ret;
480 : }
481 :
482 : /* This subroutine of expand_doubleword_shift handles the cases in which
483 : the effective shift value is >= BITS_PER_WORD. The arguments and return
484 : value are the same as for the parent routine, except that SUPERWORD_OP1
485 : is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
486 : INTO_TARGET may be null if the caller has decided to calculate it. */
487 :
488 : static bool
489 0 : expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
490 : rtx outof_target, rtx into_target,
491 : int unsignedp, enum optab_methods methods)
492 : {
493 0 : if (into_target != 0)
494 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
495 : into_target, unsignedp, methods))
496 : return false;
497 :
498 0 : if (outof_target != 0)
499 : {
500 : /* For a signed right shift, we must fill OUTOF_TARGET with copies
501 : of the sign bit, otherwise we must fill it with zeros. */
502 0 : if (binoptab != ashr_optab)
503 0 : emit_move_insn (outof_target, CONST0_RTX (word_mode));
504 : else
505 0 : if (!force_expand_binop (word_mode, binoptab, outof_input,
506 : gen_int_shift_amount (word_mode,
507 0 : BITS_PER_WORD - 1),
508 : outof_target, unsignedp, methods))
509 : return false;
510 : }
511 : return true;
512 : }
513 :
514 : /* This subroutine of expand_doubleword_shift handles the cases in which
515 : the effective shift value is < BITS_PER_WORD. The arguments and return
516 : value are the same as for the parent routine. */
517 :
518 : static bool
519 0 : expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
520 : rtx outof_input, rtx into_input, rtx op1,
521 : rtx outof_target, rtx into_target,
522 : int unsignedp, enum optab_methods methods,
523 : unsigned HOST_WIDE_INT shift_mask)
524 : {
525 0 : optab reverse_unsigned_shift, unsigned_shift;
526 0 : rtx tmp, carries;
527 :
528 0 : reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
529 0 : unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
530 :
531 : /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
532 : We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
533 : the opposite direction to BINOPTAB. */
534 0 : if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
535 : {
536 0 : carries = outof_input;
537 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
538 : op1_mode), op1_mode);
539 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
540 : 0, true, methods);
541 : }
542 : else
543 : {
544 : /* We must avoid shifting by BITS_PER_WORD bits since that is either
545 : the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
546 : has unknown behavior. Do a single shift first, then shift by the
547 : remainder. It's OK to use ~OP1 as the remainder if shift counts
548 : are truncated to the mode size. */
549 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
550 : outof_input, const1_rtx, 0,
551 : unsignedp, methods);
552 0 : if (carries == const0_rtx)
553 : tmp = const0_rtx;
554 0 : else if (shift_mask == BITS_PER_WORD - 1)
555 0 : tmp = expand_unop (op1_mode, one_cmpl_optab, op1, 0, true);
556 : else
557 : {
558 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
559 : op1_mode), op1_mode);
560 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
561 : 0, true, methods);
562 : }
563 : }
564 0 : if (tmp == 0 || carries == 0)
565 : return false;
566 0 : if (carries != const0_rtx && tmp != const0_rtx)
567 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
568 : carries, tmp, 0, unsignedp, methods);
569 0 : if (carries == 0)
570 : return false;
571 :
572 0 : if (into_input != const0_rtx)
573 : {
574 : /* Shift INTO_INPUT logically by OP1. This is the last use of
575 : INTO_INPUT so the result can go directly into INTO_TARGET if
576 : convenient. */
577 0 : tmp = simplify_expand_binop (word_mode, unsigned_shift, into_input,
578 : op1, into_target, unsignedp, methods);
579 0 : if (tmp == 0)
580 : return false;
581 :
582 : /* Now OR/PLUS in the bits carried over from OUTOF_INPUT. */
583 0 : if (!force_expand_binop (word_mode, add_optab, tmp, carries,
584 : into_target, unsignedp, methods))
585 : return false;
586 : }
587 : else
588 0 : emit_move_insn (into_target, carries);
589 :
590 : /* Use a standard word_mode shift for the out-of half. */
591 0 : if (outof_target != 0)
592 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
593 : outof_target, unsignedp, methods))
594 : return false;
595 :
596 : return true;
597 : }
598 :
599 :
600 : /* Try implementing expand_doubleword_shift using conditional moves.
601 : The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
602 : otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
603 : are the shift counts to use in the former and latter case. All other
604 : arguments are the same as the parent routine. */
605 :
606 : static bool
607 0 : expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
608 : enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
609 : rtx outof_input, rtx into_input,
610 : rtx subword_op1, rtx superword_op1,
611 : rtx outof_target, rtx into_target,
612 : int unsignedp, enum optab_methods methods,
613 : unsigned HOST_WIDE_INT shift_mask)
614 : {
615 0 : rtx outof_superword, into_superword;
616 :
617 : /* Put the superword version of the output into OUTOF_SUPERWORD and
618 : INTO_SUPERWORD. */
619 0 : outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
620 0 : if (outof_target != 0 && subword_op1 == superword_op1)
621 : {
622 : /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
623 : OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
624 0 : into_superword = outof_target;
625 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
626 : outof_superword, 0, unsignedp, methods))
627 : return false;
628 : }
629 : else
630 : {
631 0 : into_superword = gen_reg_rtx (word_mode);
632 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
633 : outof_superword, into_superword,
634 : unsignedp, methods))
635 : return false;
636 : }
637 :
638 : /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
639 0 : if (!expand_subword_shift (op1_mode, binoptab,
640 : outof_input, into_input, subword_op1,
641 : outof_target, into_target,
642 : unsignedp, methods, shift_mask))
643 : return false;
644 :
645 : /* Select between them. Do the INTO half first because INTO_SUPERWORD
646 : might be the current value of OUTOF_TARGET. */
647 0 : if (!emit_conditional_move (into_target, { cmp_code, cmp1, cmp2, op1_mode },
648 : into_target, into_superword, word_mode, false))
649 : return false;
650 :
651 0 : if (outof_target != 0)
652 0 : if (!emit_conditional_move (outof_target,
653 0 : { cmp_code, cmp1, cmp2, op1_mode },
654 : outof_target, outof_superword,
655 : word_mode, false))
656 0 : return false;
657 :
658 : return true;
659 : }
660 :
661 : /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
662 : OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
663 : input operand; the shift moves bits in the direction OUTOF_INPUT->
664 : INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
665 : of the target. OP1 is the shift count and OP1_MODE is its mode.
666 : If OP1 is constant, it will have been truncated as appropriate
667 : and is known to be nonzero.
668 :
669 : If SHIFT_MASK is zero, the result of word shifts is undefined when the
670 : shift count is outside the range [0, BITS_PER_WORD). This routine must
671 : avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
672 :
673 : If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
674 : masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
675 : fill with zeros or sign bits as appropriate.
676 :
677 : If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
678 : a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
679 : Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
680 : In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
681 : are undefined.
682 :
683 : BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
684 : may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
685 : OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
686 : function wants to calculate it itself.
687 :
688 : Return true if the shift could be successfully synthesized. */
689 :
690 : static bool
691 0 : expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
692 : rtx outof_input, rtx into_input, rtx op1,
693 : rtx outof_target, rtx into_target,
694 : int unsignedp, enum optab_methods methods,
695 : unsigned HOST_WIDE_INT shift_mask)
696 : {
697 0 : rtx superword_op1, tmp, cmp1, cmp2;
698 0 : enum rtx_code cmp_code;
699 :
700 : /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
701 : fill the result with sign or zero bits as appropriate. If so, the value
702 : of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
703 : this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
704 : and INTO_INPUT), then emit code to set up OUTOF_TARGET.
705 :
706 : This isn't worthwhile for constant shifts since the optimizers will
707 : cope better with in-range shift counts. */
708 0 : if (shift_mask >= BITS_PER_WORD
709 0 : && outof_target != 0
710 0 : && !CONSTANT_P (op1))
711 : {
712 0 : if (!expand_doubleword_shift (op1_mode, binoptab,
713 : outof_input, into_input, op1,
714 : 0, into_target,
715 : unsignedp, methods, shift_mask))
716 : return false;
717 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
718 : outof_target, unsignedp, methods))
719 : return false;
720 : return true;
721 : }
722 :
723 : /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
724 : is true when the effective shift value is less than BITS_PER_WORD.
725 : Set SUPERWORD_OP1 to the shift count that should be used to shift
726 : OUTOF_INPUT into INTO_TARGET when the condition is false. */
727 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
728 0 : if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
729 : {
730 : /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
731 : is a subword shift count. */
732 0 : cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
733 : 0, true, methods);
734 0 : cmp2 = CONST0_RTX (op1_mode);
735 0 : cmp_code = EQ;
736 0 : superword_op1 = op1;
737 : }
738 : else
739 : {
740 : /* Set CMP1 to OP1 - BITS_PER_WORD. */
741 0 : cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
742 : 0, true, methods);
743 0 : cmp2 = CONST0_RTX (op1_mode);
744 0 : cmp_code = LT;
745 0 : superword_op1 = cmp1;
746 : }
747 0 : if (cmp1 == 0)
748 : return false;
749 :
750 : /* If we can compute the condition at compile time, pick the
751 : appropriate subroutine. */
752 0 : tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
753 0 : if (tmp != 0 && CONST_INT_P (tmp))
754 : {
755 0 : if (tmp == const0_rtx)
756 0 : return expand_superword_shift (binoptab, outof_input, superword_op1,
757 : outof_target, into_target,
758 0 : unsignedp, methods);
759 : else
760 0 : return expand_subword_shift (op1_mode, binoptab,
761 : outof_input, into_input, op1,
762 : outof_target, into_target,
763 0 : unsignedp, methods, shift_mask);
764 : }
765 :
766 : /* Try using conditional moves to generate straight-line code. */
767 0 : if (HAVE_conditional_move)
768 : {
769 0 : rtx_insn *start = get_last_insn ();
770 0 : if (expand_doubleword_shift_condmove (op1_mode, binoptab,
771 : cmp_code, cmp1, cmp2,
772 : outof_input, into_input,
773 : op1, superword_op1,
774 : outof_target, into_target,
775 : unsignedp, methods, shift_mask))
776 : return true;
777 0 : delete_insns_since (start);
778 : }
779 :
780 : /* As a last resort, use branches to select the correct alternative. */
781 0 : rtx_code_label *subword_label = gen_label_rtx ();
782 0 : rtx_code_label *done_label = gen_label_rtx ();
783 :
784 0 : NO_DEFER_POP;
785 0 : do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
786 : 0, 0, subword_label,
787 : profile_probability::uninitialized ());
788 0 : OK_DEFER_POP;
789 :
790 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
791 : outof_target, into_target,
792 : unsignedp, methods))
793 : return false;
794 :
795 0 : emit_jump_insn (targetm.gen_jump (done_label));
796 0 : emit_barrier ();
797 0 : emit_label (subword_label);
798 :
799 0 : if (!expand_subword_shift (op1_mode, binoptab,
800 : outof_input, into_input, op1,
801 : outof_target, into_target,
802 : unsignedp, methods, shift_mask))
803 : return false;
804 :
805 0 : emit_label (done_label);
806 0 : return true;
807 : }
808 :
809 : /* Subroutine of expand_binop. Perform a double word multiplication of
810 : operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
811 : as the target's word_mode. This function return NULL_RTX if anything
812 : goes wrong, in which case it may have already emitted instructions
813 : which need to be deleted.
814 :
815 : If we want to multiply two two-word values and have normal and widening
816 : multiplies of single-word values, we can do this with three smaller
817 : multiplications.
818 :
819 : The multiplication proceeds as follows:
820 : _______________________
821 : [__op0_high_|__op0_low__]
822 : _______________________
823 : * [__op1_high_|__op1_low__]
824 : _______________________________________________
825 : _______________________
826 : (1) [__op0_low__*__op1_low__]
827 : _______________________
828 : (2a) [__op0_low__*__op1_high_]
829 : _______________________
830 : (2b) [__op0_high_*__op1_low__]
831 : _______________________
832 : (3) [__op0_high_*__op1_high_]
833 :
834 :
835 : This gives a 4-word result. Since we are only interested in the
836 : lower 2 words, partial result (3) and the upper words of (2a) and
837 : (2b) don't need to be calculated. Hence (2a) and (2b) can be
838 : calculated using non-widening multiplication.
839 :
840 : (1), however, needs to be calculated with an unsigned widening
841 : multiplication. If this operation is not directly supported we
842 : try using a signed widening multiplication and adjust the result.
843 : This adjustment works as follows:
844 :
845 : If both operands are positive then no adjustment is needed.
846 :
847 : If the operands have different signs, for example op0_low < 0 and
848 : op1_low >= 0, the instruction treats the most significant bit of
849 : op0_low as a sign bit instead of a bit with significance
850 : 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
851 : with 2**BITS_PER_WORD - op0_low, and two's complements the
852 : result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
853 : the result.
854 :
855 : Similarly, if both operands are negative, we need to add
856 : (op0_low + op1_low) * 2**BITS_PER_WORD.
857 :
858 : We use a trick to adjust quickly. We logically shift op0_low right
859 : (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
860 : op0_high (op1_high) before it is used to calculate 2b (2a). If no
861 : logical shift exists, we do an arithmetic right shift and subtract
862 : the 0 or -1. */
863 :
864 : static rtx
865 15648 : expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
866 : bool umulp, enum optab_methods methods)
867 : {
868 15648 : int low = (WORDS_BIG_ENDIAN ? 1 : 0);
869 15648 : int high = (WORDS_BIG_ENDIAN ? 0 : 1);
870 15648 : rtx wordm1 = (umulp ? NULL_RTX
871 0 : : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
872 15648 : rtx product, adjust, product_high, temp;
873 :
874 15648 : rtx op0_high = operand_subword_force (op0, high, mode);
875 15648 : rtx op0_low = operand_subword_force (op0, low, mode);
876 15648 : rtx op1_high = operand_subword_force (op1, high, mode);
877 15648 : rtx op1_low = operand_subword_force (op1, low, mode);
878 :
879 : /* If we're using an unsigned multiply to directly compute the product
880 : of the low-order words of the operands and perform any required
881 : adjustments of the operands, we begin by trying two more multiplications
882 : and then computing the appropriate sum.
883 :
884 : We have checked above that the required addition is provided.
885 : Full-word addition will normally always succeed, especially if
886 : it is provided at all, so we don't worry about its failure. The
887 : multiplication may well fail, however, so we do handle that. */
888 :
889 15648 : if (!umulp)
890 : {
891 : /* ??? This could be done with emit_store_flag where available. */
892 0 : temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
893 : NULL_RTX, 1, methods);
894 0 : if (temp)
895 0 : op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
896 : NULL_RTX, 0, OPTAB_DIRECT);
897 : else
898 : {
899 0 : temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
900 : NULL_RTX, 0, methods);
901 0 : if (!temp)
902 : return NULL_RTX;
903 0 : op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
904 : NULL_RTX, 0, OPTAB_DIRECT);
905 : }
906 :
907 0 : if (!op0_high)
908 : return NULL_RTX;
909 : }
910 :
911 15648 : if (op1_low == const1_rtx)
912 : adjust = op0_high;
913 15608 : else if (op1_low == const0_rtx)
914 : adjust = const0_rtx;
915 15533 : else if (op1_low == const2_rtx)
916 0 : adjust = expand_binop (word_mode, add_optab, op0_high, op0_high,
917 : NULL_RTX, 0, OPTAB_DIRECT);
918 : else
919 15533 : adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
920 : NULL_RTX, 0, OPTAB_DIRECT);
921 15648 : if (!adjust)
922 : return NULL_RTX;
923 :
924 : /* OP0_HIGH should now be dead. */
925 :
926 15648 : if (!umulp)
927 : {
928 : /* ??? This could be done with emit_store_flag where available. */
929 0 : temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
930 : NULL_RTX, 1, methods);
931 0 : if (temp)
932 0 : op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
933 : NULL_RTX, 0, OPTAB_DIRECT);
934 : else
935 : {
936 0 : temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
937 : NULL_RTX, 0, methods);
938 0 : if (!temp)
939 : return NULL_RTX;
940 0 : op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
941 : NULL_RTX, 0, OPTAB_DIRECT);
942 : }
943 :
944 0 : if (!op1_high)
945 : return NULL_RTX;
946 : }
947 :
948 15648 : if (op1_high == const1_rtx)
949 : temp = op0_low;
950 15647 : else if (op1_high == const0_rtx)
951 : temp = const0_rtx;
952 14831 : else if (op1_high == const2_rtx)
953 0 : temp = expand_binop (word_mode, add_optab, op0_low, op0_low,
954 : NULL_RTX, 0, OPTAB_DIRECT);
955 : else
956 14831 : temp = expand_binop (word_mode, smul_optab, op0_low, op1_high,
957 : NULL_RTX, 0, OPTAB_DIRECT);
958 15648 : if (!temp)
959 : return NULL_RTX;
960 :
961 : /* OP1_HIGH should now be dead. */
962 :
963 15648 : adjust = expand_binop (word_mode, add_optab, adjust, temp,
964 : NULL_RTX, 0, OPTAB_DIRECT);
965 :
966 15648 : if (target && !REG_P (target))
967 9185 : target = NULL_RTX;
968 :
969 : /* *_widen_optab needs to determine operand mode, make sure at least
970 : one operand has non-VOID mode. */
971 15648 : if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
972 0 : op0_low = force_reg (word_mode, op0_low);
973 :
974 15648 : if (op1_low == const1_rtx)
975 40 : product = convert_modes (mode, word_mode, op0_low, umulp);
976 15608 : else if (umulp)
977 15608 : product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
978 : target, 1, OPTAB_DIRECT);
979 : else
980 0 : product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
981 : target, 1, OPTAB_DIRECT);
982 :
983 15648 : if (!product)
984 : return NULL_RTX;
985 :
986 15648 : product_high = operand_subword (product, high, 1, mode);
987 15648 : adjust = expand_binop (word_mode, add_optab, product_high, adjust,
988 : NULL_RTX, 0, OPTAB_DIRECT);
989 15648 : emit_move_insn (product_high, adjust);
990 15648 : return product;
991 : }
992 :
993 : /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
994 : constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
995 : (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
996 : computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
997 : + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
998 : depends on the bit value, if 2, then carry from the addition needs to be
999 : added too, i.e. like:
1000 : sum += __builtin_add_overflow (low, high, &sum)
1001 :
1002 : Optimize signed double-word OP0 % OP1 similarly, just apply some correction
1003 : factor to the sum before doing unsigned remainder, in the form of
1004 : sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
1005 : then perform unsigned
1006 : remainder = sum % OP1;
1007 : and finally
1008 : remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
1009 :
1010 : static rtx
1011 21092 : expand_doubleword_mod (machine_mode mode, rtx op0, rtx op1, bool unsignedp)
1012 : {
1013 21092 : if (INTVAL (op1) <= 1 || (INTVAL (op1) & 1) == 0)
1014 : return NULL_RTX;
1015 :
1016 20696 : rtx_insn *last = get_last_insn ();
1017 682487 : for (int bit = BITS_PER_WORD; bit >= BITS_PER_WORD / 2; bit--)
1018 : {
1019 661041 : wide_int w = wi::shifted_mask (bit, 1, false, 2 * BITS_PER_WORD);
1020 629400 : if (wi::ne_p (wi::umod_trunc (w, INTVAL (op1)), 1))
1021 628082 : continue;
1022 1318 : rtx sum = NULL_RTX, mask = NULL_RTX;
1023 1802 : if (bit == BITS_PER_WORD)
1024 : {
1025 : /* For signed modulo we need to add correction to the sum
1026 : and that might again overflow. */
1027 356 : if (!unsignedp)
1028 143 : continue;
1029 213 : if (optab_handler (uaddv4_optab, word_mode) == CODE_FOR_nothing)
1030 0 : continue;
1031 213 : tree wtype = lang_hooks.types.type_for_mode (word_mode, 1);
1032 213 : if (wtype == NULL_TREE)
1033 0 : continue;
1034 213 : tree ctype = build_complex_type (wtype);
1035 213 : if (TYPE_MODE (ctype) != GET_MODE_COMPLEX_MODE (word_mode))
1036 0 : continue;
1037 213 : machine_mode cmode = TYPE_MODE (ctype);
1038 213 : rtx op00 = operand_subword_force (op0, 0, mode);
1039 213 : rtx op01 = operand_subword_force (op0, 1, mode);
1040 213 : rtx cres = gen_rtx_CONCAT (cmode, gen_reg_rtx (word_mode),
1041 : gen_reg_rtx (word_mode));
1042 213 : tree lhs = make_tree (ctype, cres);
1043 213 : tree arg0 = make_tree (wtype, op00);
1044 213 : tree arg1 = make_tree (wtype, op01);
1045 213 : expand_addsub_overflow (UNKNOWN_LOCATION, PLUS_EXPR, lhs, arg0,
1046 : arg1, true, true, true, false, NULL);
1047 213 : sum = expand_simple_binop (word_mode, PLUS, XEXP (cres, 0),
1048 : XEXP (cres, 1), NULL_RTX, 1,
1049 : OPTAB_DIRECT);
1050 213 : if (sum == NULL_RTX)
1051 : return NULL_RTX;
1052 : }
1053 : else
1054 : {
1055 : /* Code below uses GEN_INT, so we need the masks to be representable
1056 : in HOST_WIDE_INTs. */
1057 962 : if (bit >= HOST_BITS_PER_WIDE_INT)
1058 0 : continue;
1059 : /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1060 : overflow. Consider 64-bit -1ULL for word size 32, if we add
1061 : 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1062 1214 : if (bit == BITS_PER_WORD - 1)
1063 150 : continue;
1064 :
1065 812 : int count = (2 * BITS_PER_WORD + bit - 1) / bit;
1066 812 : rtx sum_corr = NULL_RTX;
1067 :
1068 812 : if (!unsignedp)
1069 : {
1070 : /* For signed modulo, compute it as unsigned modulo of
1071 : sum with a correction added to it if OP0 is negative,
1072 : such that the result can be computed as unsigned
1073 : remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1074 537 : w = wi::min_value (2 * BITS_PER_WORD, SIGNED);
1075 427 : wide_int wmod1 = wi::umod_trunc (w, INTVAL (op1));
1076 427 : wide_int wmod2 = wi::smod_trunc (w, INTVAL (op1));
1077 : /* wmod2 == -wmod1. */
1078 427 : wmod2 = wmod2 + (INTVAL (op1) - 1);
1079 427 : if (wi::ne_p (wmod1, wmod2))
1080 : {
1081 427 : wide_int wcorr = wmod2 - wmod1;
1082 427 : if (wi::neg_p (w))
1083 427 : wcorr = wcorr + INTVAL (op1);
1084 : /* Now verify if the count sums can't overflow, and punt
1085 : if they could. */
1086 537 : w = wi::mask (bit, false, 2 * BITS_PER_WORD);
1087 427 : w = w * (count - 1);
1088 964 : w = w + wi::mask (2 * BITS_PER_WORD - (count - 1) * bit,
1089 854 : false, 2 * BITS_PER_WORD);
1090 427 : w = w + wcorr;
1091 537 : w = wi::lrshift (w, BITS_PER_WORD);
1092 427 : if (wi::ne_p (w, 0))
1093 0 : continue;
1094 :
1095 427 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1096 : mode);
1097 427 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1098 427 : GEN_INT (BITS_PER_WORD - 1),
1099 : NULL_RTX, 0, OPTAB_DIRECT);
1100 427 : if (mask == NULL_RTX)
1101 : return NULL_RTX;
1102 427 : sum_corr = immed_wide_int_const (wcorr, word_mode);
1103 427 : sum_corr = expand_simple_binop (word_mode, AND, mask,
1104 : sum_corr, NULL_RTX, 1,
1105 : OPTAB_DIRECT);
1106 427 : if (sum_corr == NULL_RTX)
1107 : return NULL_RTX;
1108 427 : }
1109 427 : }
1110 :
1111 3441 : for (int i = 0; i < count; i++)
1112 : {
1113 2629 : rtx v = op0;
1114 2629 : if (i)
1115 1817 : v = expand_simple_binop (mode, LSHIFTRT, v, GEN_INT (i * bit),
1116 : NULL_RTX, 1, OPTAB_DIRECT);
1117 2629 : if (v == NULL_RTX)
1118 : return NULL_RTX;
1119 2629 : v = force_lowpart_subreg (word_mode, v, mode);
1120 2629 : if (v == NULL_RTX)
1121 : return NULL_RTX;
1122 2629 : if (i != count - 1)
1123 1817 : v = expand_simple_binop (word_mode, AND, v,
1124 1817 : GEN_INT ((HOST_WIDE_INT_1U << bit)
1125 : - 1), NULL_RTX, 1,
1126 : OPTAB_DIRECT);
1127 2629 : if (v == NULL_RTX)
1128 : return NULL_RTX;
1129 2629 : if (sum == NULL_RTX)
1130 : sum = v;
1131 : else
1132 1817 : sum = expand_simple_binop (word_mode, PLUS, sum, v, NULL_RTX,
1133 : 1, OPTAB_DIRECT);
1134 2629 : if (sum == NULL_RTX)
1135 : return NULL_RTX;
1136 : }
1137 812 : if (sum_corr)
1138 : {
1139 427 : sum = expand_simple_binop (word_mode, PLUS, sum, sum_corr,
1140 : NULL_RTX, 1, OPTAB_DIRECT);
1141 427 : if (sum == NULL_RTX)
1142 : return NULL_RTX;
1143 : }
1144 : }
1145 1025 : rtx remainder = expand_divmod (1, TRUNC_MOD_EXPR, word_mode, sum,
1146 1025 : gen_int_mode (INTVAL (op1), word_mode),
1147 : NULL_RTX, 1, OPTAB_DIRECT);
1148 1025 : if (remainder == NULL_RTX)
1149 : return NULL_RTX;
1150 :
1151 1025 : if (!unsignedp)
1152 : {
1153 427 : if (mask == NULL_RTX)
1154 : {
1155 0 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1156 : mode);
1157 0 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1158 0 : GEN_INT (BITS_PER_WORD - 1),
1159 : NULL_RTX, 0, OPTAB_DIRECT);
1160 0 : if (mask == NULL_RTX)
1161 : return NULL_RTX;
1162 : }
1163 427 : mask = expand_simple_binop (word_mode, AND, mask,
1164 427 : gen_int_mode (1 - INTVAL (op1),
1165 : word_mode),
1166 : NULL_RTX, 1, OPTAB_DIRECT);
1167 427 : if (mask == NULL_RTX)
1168 : return NULL_RTX;
1169 427 : remainder = expand_simple_binop (word_mode, PLUS, remainder,
1170 : mask, NULL_RTX, 1, OPTAB_DIRECT);
1171 427 : if (remainder == NULL_RTX)
1172 : return NULL_RTX;
1173 : }
1174 :
1175 1025 : remainder = convert_modes (mode, word_mode, remainder, unsignedp);
1176 : /* Punt if we need any library calls. */
1177 1025 : if (last)
1178 893 : last = NEXT_INSN (last);
1179 : else
1180 132 : last = get_insns ();
1181 21123 : for (; last; last = NEXT_INSN (last))
1182 20098 : if (CALL_P (last))
1183 : return NULL_RTX;
1184 : return remainder;
1185 629400 : }
1186 : return NULL_RTX;
1187 : }
1188 :
1189 : /* Similarly to the above function, but compute both quotient and remainder.
1190 : Quotient can be computed from the remainder as:
1191 : rem = op0 % op1; // Handled using expand_doubleword_mod
1192 : quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1193 : // 2 * BITS_PER_WORD
1194 :
1195 : We can also handle cases where op1 is a multiple of power of two constant
1196 : and constant handled by expand_doubleword_mod.
1197 : op11 = 1 << __builtin_ctz (op1);
1198 : op12 = op1 / op11;
1199 : rem1 = op0 % op12; // Handled using expand_doubleword_mod
1200 : quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1201 : // 2 * BITS_PER_WORD
1202 : rem = (quot1 % op11) * op12 + rem1;
1203 : quot = quot1 / op11; */
1204 :
1205 : rtx
1206 20745 : expand_doubleword_divmod (machine_mode mode, rtx op0, rtx op1, rtx *rem,
1207 : bool unsignedp)
1208 : {
1209 20745 : *rem = NULL_RTX;
1210 :
1211 : /* Negative dividend should have been optimized into positive,
1212 : similarly modulo by 1 and modulo by power of two is optimized
1213 : differently too. */
1214 20745 : if (INTVAL (op1) <= 1 || pow2p_hwi (INTVAL (op1)))
1215 : return NULL_RTX;
1216 :
1217 20696 : rtx op11 = const1_rtx;
1218 20696 : rtx op12 = op1;
1219 20696 : if ((INTVAL (op1) & 1) == 0)
1220 : {
1221 3700 : int bit = ctz_hwi (INTVAL (op1));
1222 3700 : op11 = GEN_INT (HOST_WIDE_INT_1 << bit);
1223 3700 : op12 = GEN_INT (INTVAL (op1) >> bit);
1224 : }
1225 :
1226 20696 : rtx rem1 = expand_doubleword_mod (mode, op0, op12, unsignedp);
1227 20696 : if (rem1 == NULL_RTX)
1228 : return NULL_RTX;
1229 :
1230 1025 : int prec = 2 * BITS_PER_WORD;
1231 1025 : wide_int a = wide_int::from (INTVAL (op12), prec + 1, UNSIGNED);
1232 1025 : wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
1233 1025 : wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
1234 1025 : rtx inv = immed_wide_int_const (m, mode);
1235 :
1236 1025 : rtx_insn *last = get_last_insn ();
1237 1025 : rtx quot1 = expand_simple_binop (mode, MINUS, op0, rem1,
1238 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1239 1025 : if (quot1 == NULL_RTX)
1240 : return NULL_RTX;
1241 :
1242 1025 : quot1 = expand_simple_binop (mode, MULT, quot1, inv,
1243 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1244 1025 : if (quot1 == NULL_RTX)
1245 : return NULL_RTX;
1246 :
1247 1025 : if (op11 != const1_rtx)
1248 : {
1249 384 : rtx rem2 = expand_divmod (1, TRUNC_MOD_EXPR, mode, quot1, op11,
1250 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1251 384 : if (rem2 == NULL_RTX)
1252 : return NULL_RTX;
1253 :
1254 384 : rem2 = expand_simple_binop (mode, MULT, rem2, op12, NULL_RTX,
1255 : unsignedp, OPTAB_DIRECT);
1256 384 : if (rem2 == NULL_RTX)
1257 : return NULL_RTX;
1258 :
1259 384 : rem2 = expand_simple_binop (mode, PLUS, rem2, rem1, NULL_RTX,
1260 : unsignedp, OPTAB_DIRECT);
1261 384 : if (rem2 == NULL_RTX)
1262 : return NULL_RTX;
1263 :
1264 384 : rtx quot2 = expand_divmod (0, TRUNC_DIV_EXPR, mode, quot1, op11,
1265 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1266 384 : if (quot2 == NULL_RTX)
1267 : return NULL_RTX;
1268 :
1269 : rem1 = rem2;
1270 : quot1 = quot2;
1271 : }
1272 :
1273 : /* Punt if we need any library calls. */
1274 1025 : if (last)
1275 1025 : last = NEXT_INSN (last);
1276 : else
1277 0 : last = get_insns ();
1278 16316 : for (; last; last = NEXT_INSN (last))
1279 15291 : if (CALL_P (last))
1280 : return NULL_RTX;
1281 :
1282 1025 : *rem = rem1;
1283 1025 : return quot1;
1284 1025 : }
1285 :
1286 : /* Wrapper around expand_binop which takes an rtx code to specify
1287 : the operation to perform, not an optab pointer. All other
1288 : arguments are the same. */
1289 : rtx
1290 4158229 : expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
1291 : rtx op1, rtx target, int unsignedp,
1292 : enum optab_methods methods)
1293 : {
1294 4158229 : optab binop = code_to_optab (code);
1295 4158229 : gcc_assert (binop);
1296 :
1297 4158229 : return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1298 : }
1299 :
1300 : /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1301 : binop. Order them according to commutative_operand_precedence and, if
1302 : possible, try to put TARGET or a pseudo first. */
1303 : static bool
1304 12527696 : swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1305 : {
1306 12527696 : int op0_prec = commutative_operand_precedence (op0);
1307 12527696 : int op1_prec = commutative_operand_precedence (op1);
1308 :
1309 12527696 : if (op0_prec < op1_prec)
1310 : return true;
1311 :
1312 12330426 : if (op0_prec > op1_prec)
1313 : return false;
1314 :
1315 : /* With equal precedence, both orders are ok, but it is better if the
1316 : first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1317 2323803 : if (target == 0 || REG_P (target))
1318 2232274 : return (REG_P (op1) && !REG_P (op0)) || target == op1;
1319 : else
1320 91529 : return rtx_equal_p (op1, target);
1321 : }
1322 :
1323 : /* Return true if BINOPTAB implements a shift operation. */
1324 :
1325 : static bool
1326 24778379 : shift_optab_p (optab binoptab)
1327 : {
1328 9959705 : switch (optab_to_code (binoptab))
1329 : {
1330 : case ASHIFT:
1331 : case SS_ASHIFT:
1332 : case US_ASHIFT:
1333 : case ASHIFTRT:
1334 : case LSHIFTRT:
1335 : case ROTATE:
1336 : case ROTATERT:
1337 : return true;
1338 :
1339 13200656 : default:
1340 0 : return false;
1341 : }
1342 : }
1343 :
1344 : /* Return true if BINOPTAB implements a commutative binary operation. */
1345 :
1346 : static bool
1347 14998600 : commutative_optab_p (optab binoptab)
1348 : {
1349 14998600 : return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1350 : || binoptab == smul_widen_optab
1351 2562738 : || binoptab == umul_widen_optab
1352 : || binoptab == smul_highpart_optab
1353 2524059 : || binoptab == umul_highpart_optab
1354 2470904 : || binoptab == vec_widen_sadd_optab
1355 2470904 : || binoptab == vec_widen_uadd_optab
1356 : || binoptab == vec_widen_sadd_hi_optab
1357 2470904 : || binoptab == vec_widen_sadd_lo_optab
1358 : || binoptab == vec_widen_uadd_hi_optab
1359 2470904 : || binoptab == vec_widen_uadd_lo_optab
1360 : || binoptab == vec_widen_sadd_even_optab
1361 2470904 : || binoptab == vec_widen_sadd_odd_optab
1362 : || binoptab == vec_widen_uadd_even_optab
1363 17469504 : || binoptab == vec_widen_uadd_odd_optab);
1364 : }
1365 :
1366 : /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1367 : optimizing, and if the operand is a constant that costs more than
1368 : 1 instruction, force the constant into a register and return that
1369 : register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1370 :
1371 : static rtx
1372 28221223 : avoid_expensive_constant (machine_mode mode, optab binoptab,
1373 : int opn, rtx x, bool unsignedp)
1374 : {
1375 28221223 : bool speed = optimize_insn_for_speed_p ();
1376 :
1377 28221223 : if (mode != VOIDmode
1378 28221223 : && optimize
1379 21380950 : && CONSTANT_P (x)
1380 35569366 : && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1381 7348143 : > set_src_cost (x, mode, speed)))
1382 : {
1383 0 : if (CONST_INT_P (x))
1384 : {
1385 0 : HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1386 0 : if (intval != INTVAL (x))
1387 0 : x = GEN_INT (intval);
1388 : }
1389 : else
1390 0 : x = convert_modes (mode, VOIDmode, x, unsignedp);
1391 0 : x = force_reg (mode, x);
1392 : }
1393 28221223 : return x;
1394 : }
1395 :
1396 : /* Helper function for expand_binop: handle the case where there
1397 : is an insn ICODE that directly implements the indicated operation.
1398 : Returns null if this is not possible. */
1399 : static rtx
1400 14788020 : expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1401 : rtx op0, rtx op1,
1402 : rtx target, int unsignedp, enum optab_methods methods)
1403 : {
1404 14788020 : machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1405 14788020 : machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1406 14788020 : machine_mode mode0, mode1, tmp_mode;
1407 14788020 : class expand_operand ops[3];
1408 14788020 : bool commutative_p;
1409 14788020 : rtx_insn *pat;
1410 14788020 : rtx xop0 = op0, xop1 = op1;
1411 14788020 : bool canonicalize_op1 = false;
1412 14788020 : rtx_insn *last = get_last_insn ();
1413 :
1414 : /* If it is a commutative operator and the modes would match
1415 : if we would swap the operands, we can save the conversions. */
1416 14788020 : commutative_p = commutative_optab_p (binoptab);
1417 14788020 : if (commutative_p
1418 12466309 : && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1419 13697 : && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0)
1420 : std::swap (xop0, xop1);
1421 :
1422 : /* If we are optimizing, force expensive constants into a register. */
1423 14788020 : xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1424 14788020 : if (!shift_optab_p (binoptab))
1425 13200656 : xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1426 : else
1427 : /* Shifts and rotates often use a different mode for op1 from op0;
1428 : for VOIDmode constants we don't know the mode, so force it
1429 : to be canonicalized using convert_modes. */
1430 : canonicalize_op1 = true;
1431 :
1432 : /* In case the insn wants input operands in modes different from
1433 : those of the actual operands, convert the operands. It would
1434 : seem that we don't need to convert CONST_INTs, but we do, so
1435 : that they're properly zero-extended, sign-extended or truncated
1436 : for their mode. */
1437 :
1438 14788020 : mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1439 14788020 : if (xmode0 != VOIDmode && xmode0 != mode0)
1440 : {
1441 3370 : xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1442 3370 : mode0 = xmode0;
1443 : }
1444 :
1445 14788020 : mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1446 14788020 : ? GET_MODE (xop1) : mode);
1447 14788020 : if (xmode1 != VOIDmode && xmode1 != mode1)
1448 : {
1449 1589166 : xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1450 1589166 : mode1 = xmode1;
1451 : }
1452 :
1453 : /* If operation is commutative,
1454 : try to make the first operand a register.
1455 : Even better, try to make it the same as the target.
1456 : Also try to make the last operand a constant. */
1457 14788020 : if (commutative_p
1458 14788020 : && swap_commutative_operands_with_target (target, xop0, xop1))
1459 : std::swap (xop0, xop1);
1460 :
1461 : /* Now, if insn's predicates don't allow our operands, put them into
1462 : pseudo regs. */
1463 :
1464 14788020 : if (binoptab == vec_pack_trunc_optab
1465 14788020 : || binoptab == vec_pack_usat_optab
1466 14777369 : || binoptab == vec_pack_ssat_optab
1467 14777369 : || binoptab == vec_pack_ufix_trunc_optab
1468 14777211 : || binoptab == vec_pack_sfix_trunc_optab
1469 14777211 : || binoptab == vec_packu_float_optab
1470 14776991 : || binoptab == vec_packs_float_optab)
1471 : {
1472 : /* The mode of the result is different then the mode of the
1473 : arguments. */
1474 11049 : tmp_mode = insn_data[(int) icode].operand[0].mode;
1475 2692 : if (VECTOR_MODE_P (mode)
1476 31553 : && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1477 : return NULL_RTX;
1478 : }
1479 : else
1480 : tmp_mode = mode;
1481 :
1482 14788020 : create_output_operand (&ops[0], target, tmp_mode);
1483 14788020 : create_input_operand (&ops[1], xop0, mode0);
1484 14788020 : create_input_operand (&ops[2], xop1, mode1);
1485 14788020 : pat = maybe_gen_insn (icode, 3, ops);
1486 14788020 : if (pat)
1487 : {
1488 : /* If PAT is composed of more than one insn, try to add an appropriate
1489 : REG_EQUAL note to it. If we can't because TEMP conflicts with an
1490 : operand, call expand_binop again, this time without a target. */
1491 13607886 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1492 15999403 : && ! add_equal_note (pat, ops[0].value,
1493 : optab_to_code (binoptab),
1494 : ops[1].value, ops[2].value, mode0))
1495 : {
1496 53577 : delete_insns_since (last);
1497 53577 : return expand_binop_directly (icode, mode, binoptab, op0, op1,
1498 53577 : NULL_RTX, unsignedp, methods);
1499 : }
1500 :
1501 14734437 : emit_insn (pat);
1502 14734437 : return ops[0].value;
1503 : }
1504 :
1505 : return NULL_RTX;
1506 : }
1507 :
1508 : /* Generate code to perform an operation specified by BINOPTAB
1509 : on operands OP0 and OP1, with result having machine-mode MODE.
1510 :
1511 : UNSIGNEDP is for the case where we have to widen the operands
1512 : to perform the operation. It says to use zero-extension.
1513 :
1514 : If TARGET is nonzero, the value
1515 : is generated there, if it is convenient to do so.
1516 : In all cases an rtx is returned for the locus of the value;
1517 : this may or may not be TARGET. */
1518 :
1519 : rtx
1520 14945183 : expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1521 : rtx target, int unsignedp, enum optab_methods methods)
1522 : {
1523 12923702 : enum optab_methods next_methods
1524 15054607 : = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1525 15054607 : ? OPTAB_WIDEN : methods);
1526 15054607 : enum mode_class mclass;
1527 15054607 : enum insn_code icode;
1528 15054607 : machine_mode wider_mode;
1529 15054607 : scalar_int_mode int_mode;
1530 15054607 : rtx libfunc;
1531 15054607 : rtx temp;
1532 15054607 : rtx_insn *entry_last = get_last_insn ();
1533 15054607 : rtx_insn *last;
1534 :
1535 15054607 : mclass = GET_MODE_CLASS (mode);
1536 :
1537 : /* If subtracting an integer constant, convert this into an addition of
1538 : the negated constant. */
1539 :
1540 15054607 : if (binoptab == sub_optab && CONST_INT_P (op1))
1541 : {
1542 1037757 : op1 = negate_rtx (mode, op1);
1543 1037757 : binoptab = add_optab;
1544 : }
1545 : /* For shifts, constant invalid op1 might be expanded from different
1546 : mode than MODE. As those are invalid, force them to a register
1547 : to avoid further problems during expansion. */
1548 14016850 : else if (CONST_INT_P (op1)
1549 16580223 : && shift_optab_p (binoptab)
1550 18593698 : && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1551 : {
1552 160 : op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1553 160 : op1 = force_reg (GET_MODE_INNER (mode), op1);
1554 : }
1555 :
1556 : /* Record where to delete back to if we backtrack. */
1557 15054607 : last = get_last_insn ();
1558 :
1559 : /* If we can do it with a three-operand insn, do so. */
1560 :
1561 15054607 : if (methods != OPTAB_MUST_WIDEN)
1562 : {
1563 15054607 : if (convert_optab_p (binoptab))
1564 : {
1565 38679 : machine_mode from_mode = widened_mode (mode, op0, op1);
1566 38679 : icode = find_widening_optab_handler (binoptab, mode, from_mode);
1567 : }
1568 : else
1569 15015928 : icode = optab_handler (binoptab, mode);
1570 15054607 : if (icode != CODE_FOR_nothing)
1571 : {
1572 14734442 : temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1573 : target, unsignedp, methods);
1574 14734442 : if (temp)
1575 : return temp;
1576 5 : delete_insns_since (last);
1577 : }
1578 : }
1579 :
1580 : /* If we were trying to rotate, and that didn't work, try rotating
1581 : the other direction before falling back to shifts and bitwise-or. */
1582 320170 : if (((binoptab == rotl_optab
1583 25 : && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1584 320167 : || (binoptab == rotr_optab
1585 13 : && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1586 320175 : && is_int_mode (mode, &int_mode))
1587 : {
1588 1 : optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1589 1 : rtx newop1;
1590 1 : unsigned int bits = GET_MODE_PRECISION (int_mode);
1591 :
1592 1 : if (CONST_INT_P (op1))
1593 0 : newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1594 1 : else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1595 0 : newop1 = negate_rtx (GET_MODE (op1), op1);
1596 : else
1597 1 : newop1 = expand_binop (GET_MODE (op1), sub_optab,
1598 1 : gen_int_mode (bits, GET_MODE (op1)), op1,
1599 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1600 :
1601 1 : temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1602 : target, unsignedp, methods);
1603 1 : if (temp)
1604 14945183 : return temp;
1605 1 : delete_insns_since (last);
1606 : }
1607 :
1608 : /* If this is a multiply, see if we can do a widening operation that
1609 : takes operands of this mode and makes a wider mode. */
1610 :
1611 320170 : if (binoptab == smul_optab
1612 26453 : && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1613 339832 : && (convert_optab_handler ((unsignedp
1614 : ? umul_widen_optab
1615 : : smul_widen_optab),
1616 : wider_mode, mode) != CODE_FOR_nothing))
1617 : {
1618 : /* *_widen_optab needs to determine operand mode, make sure at least
1619 : one operand has non-VOID mode. */
1620 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1621 0 : op0 = force_reg (mode, op0);
1622 0 : temp = expand_binop (wider_mode,
1623 : unsignedp ? umul_widen_optab : smul_widen_optab,
1624 : op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1625 :
1626 0 : if (temp != 0)
1627 : {
1628 0 : if (GET_MODE_CLASS (mode) == MODE_INT
1629 0 : && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1630 0 : return gen_lowpart (mode, temp);
1631 : else
1632 0 : return convert_to_mode (mode, temp, unsignedp);
1633 : }
1634 : }
1635 :
1636 : /* If backend's machine description doesn't specify an any_or_plus
1637 : (AOP) preference, choose for it. */
1638 320170 : if (binoptab == aop_optab)
1639 : {
1640 109424 : binoptab = (mode == word_mode || mode == SImode) ? add_optab
1641 : : ior_optab;
1642 109424 : return expand_binop (mode, binoptab, op0, op1,
1643 109424 : target, unsignedp, methods);
1644 : }
1645 :
1646 : /* If this is a vector shift by a scalar, see if we can do a vector
1647 : shift by a vector. If so, broadcast the scalar into a vector. */
1648 210746 : if (mclass == MODE_VECTOR_INT)
1649 : {
1650 37 : optab otheroptab = unknown_optab;
1651 :
1652 37 : if (binoptab == ashl_optab)
1653 : otheroptab = vashl_optab;
1654 : else if (binoptab == ashr_optab)
1655 : otheroptab = vashr_optab;
1656 : else if (binoptab == lshr_optab)
1657 : otheroptab = vlshr_optab;
1658 : else if (binoptab == rotl_optab)
1659 : otheroptab = vrotl_optab;
1660 : else if (binoptab == rotr_optab)
1661 : otheroptab = vrotr_optab;
1662 :
1663 : if (otheroptab
1664 37 : && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1665 : {
1666 : /* The scalar may be wider or narrower than the vector element.
1667 : Truncate or extend it to the proper size to fit in the
1668 : broadcast vector. */
1669 0 : scalar_mode inner_mode = GET_MODE_INNER (mode);
1670 0 : if (!CONST_INT_P (op1))
1671 : {
1672 0 : auto mode1 = as_a <scalar_int_mode> (GET_MODE (op1));
1673 0 : int size1 = GET_MODE_BITSIZE (mode1);
1674 0 : int inner_size = GET_MODE_BITSIZE (inner_mode);
1675 :
1676 0 : if (size1 != inner_size)
1677 : {
1678 0 : auto unary = size1 > inner_size ? TRUNCATE : ZERO_EXTEND;
1679 0 : op1 = force_reg (inner_mode,
1680 : simplify_gen_unary (unary, inner_mode,
1681 : op1, mode1));
1682 : }
1683 : }
1684 :
1685 0 : rtx vop1 = expand_vector_broadcast (mode, op1);
1686 0 : if (vop1)
1687 : {
1688 0 : temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1689 : target, unsignedp, methods);
1690 0 : if (temp)
1691 14945183 : return temp;
1692 0 : delete_insns_since (last);
1693 : }
1694 : }
1695 : }
1696 :
1697 : /* Look for a wider mode of the same class for which we think we
1698 : can open-code the operation. Check for a widening multiply at the
1699 : wider mode as well. */
1700 :
1701 210746 : if (CLASS_HAS_WIDER_MODES_P (mclass)
1702 210709 : && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1703 117960 : FOR_EACH_WIDER_MODE (wider_mode, mode)
1704 : {
1705 63719 : machine_mode next_mode;
1706 63719 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1707 63719 : || (binoptab == smul_optab
1708 80838 : && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1709 20871 : && (find_widening_optab_handler ((unsignedp
1710 : ? umul_widen_optab
1711 : : smul_widen_optab),
1712 : next_mode, mode)
1713 : != CODE_FOR_nothing)))
1714 : {
1715 166 : rtx xop0 = op0, xop1 = op1;
1716 166 : bool no_extend = false;
1717 :
1718 : /* For certain integer operations, we need not actually extend
1719 : the narrow operands, as long as we will truncate
1720 : the results to the same narrowness. */
1721 :
1722 166 : if ((binoptab == ior_optab || binoptab == and_optab
1723 166 : || binoptab == xor_optab
1724 166 : || binoptab == add_optab || binoptab == sub_optab
1725 : || binoptab == smul_optab || binoptab == ashl_optab)
1726 91 : && mclass == MODE_INT)
1727 : {
1728 7 : no_extend = true;
1729 7 : xop0 = avoid_expensive_constant (mode, binoptab, 0,
1730 : xop0, unsignedp);
1731 7 : if (binoptab != ashl_optab)
1732 0 : xop1 = avoid_expensive_constant (mode, binoptab, 1,
1733 : xop1, unsignedp);
1734 : }
1735 :
1736 166 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1737 :
1738 : /* The second operand of a shift must always be extended. */
1739 332 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1740 166 : no_extend && binoptab != ashl_optab);
1741 :
1742 166 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1743 : unsignedp, OPTAB_DIRECT);
1744 166 : if (temp)
1745 : {
1746 166 : if (mclass != MODE_INT
1747 166 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1748 : {
1749 159 : if (target == 0)
1750 51 : target = gen_reg_rtx (mode);
1751 159 : convert_move (target, temp, 0);
1752 159 : return target;
1753 : }
1754 : else
1755 7 : return gen_lowpart (mode, temp);
1756 : }
1757 : else
1758 0 : delete_insns_since (last);
1759 : }
1760 : }
1761 :
1762 : /* If operation is commutative,
1763 : try to make the first operand a register.
1764 : Even better, try to make it the same as the target.
1765 : Also try to make the last operand a constant. */
1766 210580 : if (commutative_optab_p (binoptab)
1767 210580 : && swap_commutative_operands_with_target (target, op0, op1))
1768 : std::swap (op0, op1);
1769 :
1770 : /* These can be done a word at a time. */
1771 210580 : if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1772 8 : && is_int_mode (mode, &int_mode)
1773 16 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1774 210588 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1775 : {
1776 8 : int i;
1777 8 : rtx_insn *insns;
1778 :
1779 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1780 : won't be accurate, so use a new target. */
1781 8 : if (target == 0
1782 8 : || target == op0
1783 8 : || target == op1
1784 8 : || reg_overlap_mentioned_p (target, op0)
1785 8 : || reg_overlap_mentioned_p (target, op1)
1786 16 : || !valid_multiword_target_p (target))
1787 0 : target = gen_reg_rtx (int_mode);
1788 :
1789 8 : start_sequence ();
1790 :
1791 : /* Do the actual arithmetic. */
1792 8 : machine_mode op0_mode = GET_MODE (op0);
1793 8 : machine_mode op1_mode = GET_MODE (op1);
1794 8 : if (op0_mode == VOIDmode)
1795 0 : op0_mode = int_mode;
1796 8 : if (op1_mode == VOIDmode)
1797 8 : op1_mode = int_mode;
1798 120 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1799 : {
1800 32 : rtx target_piece = operand_subword (target, i, 1, int_mode);
1801 32 : rtx x = expand_binop (word_mode, binoptab,
1802 32 : operand_subword_force (op0, i, op0_mode),
1803 32 : operand_subword_force (op1, i, op1_mode),
1804 : target_piece, unsignedp, next_methods);
1805 :
1806 32 : if (x == 0)
1807 : break;
1808 :
1809 32 : if (target_piece != x)
1810 0 : emit_move_insn (target_piece, x);
1811 : }
1812 :
1813 8 : insns = end_sequence ();
1814 :
1815 24 : if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1816 : {
1817 8 : emit_insn (insns);
1818 8 : return target;
1819 : }
1820 : }
1821 :
1822 : /* Synthesize double word shifts from single word shifts. */
1823 210572 : if ((binoptab == lshr_optab || binoptab == ashl_optab
1824 210572 : || binoptab == ashr_optab)
1825 0 : && is_int_mode (mode, &int_mode)
1826 0 : && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1827 0 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1828 0 : && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1829 0 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1830 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1831 210572 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1832 : {
1833 0 : unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1834 0 : scalar_int_mode op1_mode;
1835 :
1836 0 : double_shift_mask = targetm.shift_truncation_mask (int_mode);
1837 0 : shift_mask = targetm.shift_truncation_mask (word_mode);
1838 0 : op1_mode = (GET_MODE (op1) != VOIDmode
1839 0 : ? as_a <scalar_int_mode> (GET_MODE (op1))
1840 : : word_mode);
1841 :
1842 : /* Apply the truncation to constant shifts. */
1843 0 : if (double_shift_mask > 0 && CONST_INT_P (op1))
1844 0 : op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1845 :
1846 0 : if (op1 == CONST0_RTX (op1_mode))
1847 14945183 : return op0;
1848 :
1849 : /* Make sure that this is a combination that expand_doubleword_shift
1850 : can handle. See the comments there for details. */
1851 0 : if (double_shift_mask == 0
1852 0 : || (shift_mask == BITS_PER_WORD - 1
1853 0 : && double_shift_mask == BITS_PER_WORD * 2 - 1))
1854 : {
1855 0 : rtx_insn *insns;
1856 0 : rtx into_target, outof_target;
1857 0 : rtx into_input, outof_input;
1858 0 : int left_shift, outof_word;
1859 :
1860 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1861 : won't be accurate, so use a new target. */
1862 0 : if (target == 0
1863 0 : || target == op0
1864 0 : || target == op1
1865 0 : || reg_overlap_mentioned_p (target, op0)
1866 0 : || reg_overlap_mentioned_p (target, op1)
1867 0 : || !valid_multiword_target_p (target))
1868 0 : target = gen_reg_rtx (int_mode);
1869 :
1870 0 : start_sequence ();
1871 :
1872 : /* OUTOF_* is the word we are shifting bits away from, and
1873 : INTO_* is the word that we are shifting bits towards, thus
1874 : they differ depending on the direction of the shift and
1875 : WORDS_BIG_ENDIAN. */
1876 :
1877 0 : left_shift = binoptab == ashl_optab;
1878 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1879 :
1880 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1881 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1882 :
1883 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1884 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1885 :
1886 0 : if (expand_doubleword_shift (op1_mode, binoptab,
1887 : outof_input, into_input, op1,
1888 : outof_target, into_target,
1889 : unsignedp, next_methods, shift_mask))
1890 : {
1891 0 : insns = end_sequence ();
1892 :
1893 0 : emit_insn (insns);
1894 0 : return target;
1895 : }
1896 0 : end_sequence ();
1897 : }
1898 : }
1899 :
1900 : /* Synthesize double word rotates from single word shifts. */
1901 210572 : if ((binoptab == rotl_optab || binoptab == rotr_optab)
1902 38 : && is_int_mode (mode, &int_mode)
1903 1 : && CONST_INT_P (op1)
1904 0 : && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1905 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1906 210572 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1907 : {
1908 0 : rtx_insn *insns;
1909 0 : rtx into_target, outof_target;
1910 0 : rtx into_input, outof_input;
1911 0 : rtx inter;
1912 0 : int shift_count, left_shift, outof_word;
1913 :
1914 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1915 : won't be accurate, so use a new target. Do this also if target is not
1916 : a REG, first because having a register instead may open optimization
1917 : opportunities, and second because if target and op0 happen to be MEMs
1918 : designating the same location, we would risk clobbering it too early
1919 : in the code sequence we generate below. */
1920 0 : if (target == 0
1921 0 : || target == op0
1922 0 : || target == op1
1923 0 : || !REG_P (target)
1924 0 : || reg_overlap_mentioned_p (target, op0)
1925 0 : || reg_overlap_mentioned_p (target, op1)
1926 0 : || !valid_multiword_target_p (target))
1927 0 : target = gen_reg_rtx (int_mode);
1928 :
1929 0 : start_sequence ();
1930 :
1931 0 : shift_count = INTVAL (op1);
1932 :
1933 : /* OUTOF_* is the word we are shifting bits away from, and
1934 : INTO_* is the word that we are shifting bits towards, thus
1935 : they differ depending on the direction of the shift and
1936 : WORDS_BIG_ENDIAN. */
1937 :
1938 0 : left_shift = (binoptab == rotl_optab);
1939 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1940 :
1941 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1942 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1943 :
1944 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1945 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1946 :
1947 0 : if (shift_count == BITS_PER_WORD)
1948 : {
1949 : /* This is just a word swap. */
1950 0 : emit_move_insn (outof_target, into_input);
1951 0 : emit_move_insn (into_target, outof_input);
1952 0 : inter = const0_rtx;
1953 : }
1954 : else
1955 : {
1956 0 : rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1957 0 : HOST_WIDE_INT first_shift_count, second_shift_count;
1958 0 : optab reverse_unsigned_shift, unsigned_shift;
1959 :
1960 0 : reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1961 0 : ? lshr_optab : ashl_optab);
1962 :
1963 0 : unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1964 0 : ? ashl_optab : lshr_optab);
1965 :
1966 0 : if (shift_count > BITS_PER_WORD)
1967 : {
1968 0 : first_shift_count = shift_count - BITS_PER_WORD;
1969 0 : second_shift_count = 2 * BITS_PER_WORD - shift_count;
1970 : }
1971 : else
1972 : {
1973 0 : first_shift_count = BITS_PER_WORD - shift_count;
1974 0 : second_shift_count = shift_count;
1975 : }
1976 0 : rtx first_shift_count_rtx
1977 0 : = gen_int_shift_amount (word_mode, first_shift_count);
1978 0 : rtx second_shift_count_rtx
1979 0 : = gen_int_shift_amount (word_mode, second_shift_count);
1980 :
1981 0 : into_temp1 = expand_binop (word_mode, unsigned_shift,
1982 : outof_input, first_shift_count_rtx,
1983 : NULL_RTX, unsignedp, next_methods);
1984 0 : into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1985 : into_input, second_shift_count_rtx,
1986 : NULL_RTX, unsignedp, next_methods);
1987 :
1988 0 : if (into_temp1 != 0 && into_temp2 != 0)
1989 0 : inter = expand_binop (word_mode, aop_optab, into_temp1, into_temp2,
1990 : into_target, unsignedp, next_methods);
1991 : else
1992 : inter = 0;
1993 :
1994 0 : if (inter != 0 && inter != into_target)
1995 0 : emit_move_insn (into_target, inter);
1996 :
1997 0 : outof_temp1 = expand_binop (word_mode, unsigned_shift,
1998 : into_input, first_shift_count_rtx,
1999 : NULL_RTX, unsignedp, next_methods);
2000 0 : outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
2001 : outof_input, second_shift_count_rtx,
2002 : NULL_RTX, unsignedp, next_methods);
2003 :
2004 0 : if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
2005 0 : inter = expand_binop (word_mode, aop_optab,
2006 : outof_temp1, outof_temp2,
2007 : outof_target, unsignedp, next_methods);
2008 :
2009 0 : if (inter != 0 && inter != outof_target)
2010 0 : emit_move_insn (outof_target, inter);
2011 : }
2012 :
2013 0 : insns = end_sequence ();
2014 :
2015 0 : if (inter != 0)
2016 : {
2017 0 : emit_insn (insns);
2018 0 : return target;
2019 : }
2020 : }
2021 :
2022 : /* These can be done a word at a time by propagating carries. */
2023 210572 : if ((binoptab == add_optab || binoptab == sub_optab)
2024 13679 : && is_int_mode (mode, &int_mode)
2025 0 : && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
2026 210572 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
2027 : {
2028 0 : unsigned int i;
2029 0 : optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
2030 0 : const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
2031 0 : rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
2032 0 : rtx xop0, xop1, xtarget;
2033 :
2034 : /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
2035 : value is one of those, use it. Otherwise, use 1 since it is the
2036 : one easiest to get. */
2037 : #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
2038 0 : int normalizep = STORE_FLAG_VALUE;
2039 : #else
2040 : int normalizep = 1;
2041 : #endif
2042 :
2043 : /* Prepare the operands. */
2044 0 : xop0 = force_reg (int_mode, op0);
2045 0 : xop1 = force_reg (int_mode, op1);
2046 :
2047 0 : xtarget = gen_reg_rtx (int_mode);
2048 :
2049 0 : if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
2050 : target = xtarget;
2051 :
2052 : /* Indicate for flow that the entire target reg is being set. */
2053 0 : if (REG_P (target))
2054 0 : emit_clobber (xtarget);
2055 :
2056 : /* Do the actual arithmetic. */
2057 0 : for (i = 0; i < nwords; i++)
2058 : {
2059 0 : int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
2060 0 : rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
2061 0 : rtx op0_piece = operand_subword_force (xop0, index, int_mode);
2062 0 : rtx op1_piece = operand_subword_force (xop1, index, int_mode);
2063 0 : rtx x;
2064 :
2065 : /* Main add/subtract of the input operands. */
2066 0 : x = expand_binop (word_mode, binoptab,
2067 : op0_piece, op1_piece,
2068 : target_piece, unsignedp, next_methods);
2069 0 : if (x == 0)
2070 : break;
2071 :
2072 0 : if (i + 1 < nwords)
2073 : {
2074 : /* Store carry from main add/subtract. */
2075 0 : carry_out = gen_reg_rtx (word_mode);
2076 0 : carry_out = emit_store_flag_force (carry_out,
2077 : (binoptab == add_optab
2078 : ? LT : GT),
2079 : x, op0_piece,
2080 : word_mode, 1, normalizep);
2081 : }
2082 :
2083 0 : if (i > 0)
2084 : {
2085 0 : rtx newx;
2086 :
2087 : /* Add/subtract previous carry to main result. */
2088 0 : newx = expand_binop (word_mode,
2089 : normalizep == 1 ? binoptab : otheroptab,
2090 : x, carry_in,
2091 : NULL_RTX, 1, next_methods);
2092 :
2093 0 : if (i + 1 < nwords)
2094 : {
2095 : /* Get out carry from adding/subtracting carry in. */
2096 0 : rtx carry_tmp = gen_reg_rtx (word_mode);
2097 0 : carry_tmp = emit_store_flag_force (carry_tmp,
2098 : (binoptab == add_optab
2099 : ? LT : GT),
2100 : newx, x,
2101 : word_mode, 1, normalizep);
2102 :
2103 : /* Logical-ior the two poss. carry together. */
2104 0 : carry_out = expand_binop (word_mode, ior_optab,
2105 : carry_out, carry_tmp,
2106 : carry_out, 0, next_methods);
2107 0 : if (carry_out == 0)
2108 : break;
2109 : }
2110 0 : emit_move_insn (target_piece, newx);
2111 : }
2112 : else
2113 : {
2114 0 : if (x != target_piece)
2115 0 : emit_move_insn (target_piece, x);
2116 : }
2117 :
2118 0 : carry_in = carry_out;
2119 : }
2120 :
2121 0 : if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
2122 : {
2123 0 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
2124 0 : || ! rtx_equal_p (target, xtarget))
2125 : {
2126 0 : rtx_insn *temp = emit_move_insn (target, xtarget);
2127 :
2128 0 : set_dst_reg_note (temp, REG_EQUAL,
2129 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2130 : int_mode, copy_rtx (xop0),
2131 : copy_rtx (xop1)),
2132 : target);
2133 : }
2134 : else
2135 : target = xtarget;
2136 :
2137 : return target;
2138 : }
2139 :
2140 : else
2141 0 : delete_insns_since (last);
2142 : }
2143 :
2144 : /* Attempt to synthesize double word multiplies using a sequence of word
2145 : mode multiplications. We first attempt to generate a sequence using a
2146 : more efficient unsigned widening multiply, and if that fails we then
2147 : try using a signed widening multiply. */
2148 :
2149 210572 : if (binoptab == smul_optab
2150 26431 : && is_int_mode (mode, &int_mode)
2151 19008 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2152 15648 : && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2153 226220 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2154 : {
2155 15648 : rtx product = NULL_RTX;
2156 15648 : if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
2157 : != CODE_FOR_nothing)
2158 : {
2159 15648 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2160 : true, methods);
2161 15648 : if (!product)
2162 0 : delete_insns_since (last);
2163 : }
2164 :
2165 0 : if (product == NULL_RTX
2166 0 : && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
2167 : != CODE_FOR_nothing))
2168 : {
2169 0 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2170 : false, methods);
2171 0 : if (!product)
2172 0 : delete_insns_since (last);
2173 : }
2174 :
2175 15648 : if (product != NULL_RTX)
2176 : {
2177 15648 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2178 : {
2179 24744 : rtx_insn *move = emit_move_insn (target ? target : product,
2180 : product);
2181 15648 : set_dst_reg_note (move,
2182 : REG_EQUAL,
2183 : gen_rtx_fmt_ee (MULT, int_mode,
2184 : copy_rtx (op0),
2185 : copy_rtx (op1)),
2186 : target ? target : product);
2187 : }
2188 : return product;
2189 : }
2190 : }
2191 :
2192 : /* Attempt to synthesize double word modulo by constant divisor. */
2193 194924 : if ((binoptab == umod_optab
2194 194924 : || binoptab == smod_optab
2195 194924 : || binoptab == udiv_optab
2196 : || binoptab == sdiv_optab)
2197 132317 : && optimize
2198 106806 : && CONST_INT_P (op1)
2199 22905 : && is_int_mode (mode, &int_mode)
2200 25346 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2201 31482 : && optab_handler ((binoptab == umod_optab || binoptab == udiv_optab)
2202 : ? udivmod_optab : sdivmod_optab,
2203 : int_mode) == CODE_FOR_nothing
2204 20161 : && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
2205 20161 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
2206 20161 : && optimize_insn_for_speed_p ())
2207 : {
2208 19963 : rtx res = NULL_RTX;
2209 19963 : if ((binoptab == umod_optab || binoptab == smod_optab)
2210 5196 : && (INTVAL (op1) & 1) == 0)
2211 396 : res = expand_doubleword_mod (int_mode, op0, op1,
2212 : binoptab == umod_optab);
2213 : else
2214 : {
2215 19567 : rtx quot = expand_doubleword_divmod (int_mode, op0, op1, &res,
2216 : binoptab == umod_optab
2217 : || binoptab == udiv_optab);
2218 19567 : if (quot == NULL_RTX)
2219 18886 : res = NULL_RTX;
2220 681 : else if (binoptab == udiv_optab || binoptab == sdiv_optab)
2221 474 : res = quot;
2222 : }
2223 19963 : if (res != NULL_RTX)
2224 : {
2225 681 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2226 : {
2227 848 : rtx_insn *move = emit_move_insn (target ? target : res,
2228 : res);
2229 681 : set_dst_reg_note (move, REG_EQUAL,
2230 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2231 : int_mode, copy_rtx (op0), op1),
2232 : target ? target : res);
2233 : }
2234 681 : return res;
2235 : }
2236 : else
2237 19282 : delete_insns_since (last);
2238 : }
2239 :
2240 : /* It can't be open-coded in this mode.
2241 : Use a library call if one is available and caller says that's ok. */
2242 :
2243 194243 : libfunc = optab_libfunc (binoptab, mode);
2244 194243 : if (libfunc
2245 129107 : && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2246 : {
2247 30654 : rtx_insn *insns;
2248 30654 : rtx op1x = op1;
2249 30654 : machine_mode op1_mode = mode;
2250 30654 : rtx value;
2251 :
2252 30654 : start_sequence ();
2253 :
2254 30654 : if (shift_optab_p (binoptab))
2255 : {
2256 0 : op1_mode = targetm.libgcc_shift_count_mode ();
2257 : /* Specify unsigned here,
2258 : since negative shift counts are meaningless. */
2259 0 : op1x = convert_to_mode (op1_mode, op1, 1);
2260 : }
2261 :
2262 30654 : if (GET_MODE (op0) != VOIDmode
2263 30602 : && GET_MODE (op0) != mode)
2264 0 : op0 = convert_to_mode (mode, op0, unsignedp);
2265 :
2266 : /* Pass 1 for NO_QUEUE so we don't lose any increments
2267 : if the libcall is cse'd or moved. */
2268 30654 : value = emit_library_call_value (libfunc,
2269 : NULL_RTX, LCT_CONST, mode,
2270 : op0, mode, op1x, op1_mode);
2271 :
2272 30654 : insns = end_sequence ();
2273 :
2274 30654 : bool trapv = trapv_binoptab_p (binoptab);
2275 30654 : target = gen_reg_rtx (mode);
2276 30654 : emit_libcall_block_1 (insns, target, value,
2277 : trapv ? NULL_RTX
2278 30164 : : gen_rtx_fmt_ee (optab_to_code (binoptab),
2279 : mode, op0, op1), trapv);
2280 :
2281 30654 : return target;
2282 : }
2283 :
2284 163589 : delete_insns_since (last);
2285 :
2286 : /* It can't be done in this mode. Can we do it in a wider mode? */
2287 :
2288 163589 : if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2289 : || methods == OPTAB_MUST_WIDEN))
2290 : {
2291 : /* Caller says, don't even try. */
2292 150726 : delete_insns_since (entry_last);
2293 150726 : return 0;
2294 : }
2295 :
2296 : /* Compute the value of METHODS to pass to recursive calls.
2297 : Don't allow widening to be tried recursively. */
2298 :
2299 12863 : methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2300 :
2301 : /* Look for a wider mode of the same class for which it appears we can do
2302 : the operation. */
2303 :
2304 12863 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2305 : {
2306 : /* This code doesn't make sense for conversion optabs, since we
2307 : wouldn't then want to extend the operands to be the same size
2308 : as the result. */
2309 12863 : gcc_assert (!convert_optab_p (binoptab));
2310 40513 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2311 : {
2312 27651 : if (optab_handler (binoptab, wider_mode)
2313 27651 : || (methods == OPTAB_LIB
2314 2 : && optab_libfunc (binoptab, wider_mode)))
2315 : {
2316 1 : rtx xop0 = op0, xop1 = op1;
2317 1 : bool no_extend = false;
2318 :
2319 : /* For certain integer operations, we need not actually extend
2320 : the narrow operands, as long as we will truncate
2321 : the results to the same narrowness. */
2322 :
2323 1 : if ((binoptab == ior_optab || binoptab == and_optab
2324 1 : || binoptab == xor_optab
2325 1 : || binoptab == add_optab || binoptab == sub_optab
2326 : || binoptab == smul_optab || binoptab == ashl_optab)
2327 0 : && mclass == MODE_INT)
2328 1 : no_extend = true;
2329 :
2330 1 : xop0 = widen_operand (xop0, wider_mode, mode,
2331 : unsignedp, no_extend);
2332 :
2333 : /* The second operand of a shift must always be extended. */
2334 2 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2335 1 : no_extend && binoptab != ashl_optab);
2336 :
2337 1 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2338 : unsignedp, methods);
2339 1 : if (temp)
2340 : {
2341 1 : if (mclass != MODE_INT
2342 1 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2343 : {
2344 0 : if (target == 0)
2345 0 : target = gen_reg_rtx (mode);
2346 0 : convert_move (target, temp, 0);
2347 0 : return target;
2348 : }
2349 : else
2350 1 : return gen_lowpart (mode, temp);
2351 : }
2352 : else
2353 0 : delete_insns_since (last);
2354 : }
2355 : }
2356 : }
2357 :
2358 12862 : delete_insns_since (entry_last);
2359 12862 : return 0;
2360 : }
2361 :
2362 : /* Expand a binary operator which has both signed and unsigned forms.
2363 : UOPTAB is the optab for unsigned operations, and SOPTAB is for
2364 : signed operations.
2365 :
2366 : If we widen unsigned operands, we may use a signed wider operation instead
2367 : of an unsigned wider operation, since the result would be the same. */
2368 :
2369 : rtx
2370 113861 : sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
2371 : rtx op0, rtx op1, rtx target, int unsignedp,
2372 : enum optab_methods methods)
2373 : {
2374 113861 : rtx temp;
2375 113861 : optab direct_optab = unsignedp ? uoptab : soptab;
2376 113861 : bool save_enable;
2377 :
2378 : /* Do it without widening, if possible. */
2379 113861 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2380 : unsignedp, OPTAB_DIRECT);
2381 113861 : if (temp || methods == OPTAB_DIRECT)
2382 : return temp;
2383 :
2384 : /* Try widening to a signed int. Disable any direct use of any
2385 : signed insn in the current mode. */
2386 8408 : save_enable = swap_optab_enable (soptab, mode, false);
2387 :
2388 8408 : temp = expand_binop (mode, soptab, op0, op1, target,
2389 : unsignedp, OPTAB_WIDEN);
2390 :
2391 : /* For unsigned operands, try widening to an unsigned int. */
2392 8408 : if (!temp && unsignedp)
2393 4307 : temp = expand_binop (mode, uoptab, op0, op1, target,
2394 : unsignedp, OPTAB_WIDEN);
2395 8408 : if (temp || methods == OPTAB_WIDEN)
2396 4847 : goto egress;
2397 :
2398 : /* Use the right width libcall if that exists. */
2399 3561 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2400 : unsignedp, OPTAB_LIB);
2401 3561 : if (temp || methods == OPTAB_LIB)
2402 3561 : goto egress;
2403 :
2404 : /* Must widen and use a libcall, use either signed or unsigned. */
2405 0 : temp = expand_binop (mode, soptab, op0, op1, target,
2406 : unsignedp, methods);
2407 0 : if (!temp && unsignedp)
2408 0 : temp = expand_binop (mode, uoptab, op0, op1, target,
2409 : unsignedp, methods);
2410 :
2411 0 : egress:
2412 : /* Undo the fiddling above. */
2413 8408 : if (save_enable)
2414 0 : swap_optab_enable (soptab, mode, true);
2415 : return temp;
2416 : }
2417 :
2418 : /* Generate code to perform an operation specified by UNOPPTAB
2419 : on operand OP0, with two results to TARG0 and TARG1.
2420 : We assume that the order of the operands for the instruction
2421 : is TARG0, TARG1, OP0.
2422 :
2423 : Either TARG0 or TARG1 may be zero, but what that means is that
2424 : the result is not actually wanted. We will generate it into
2425 : a dummy pseudo-reg and discard it. They may not both be zero.
2426 :
2427 : Returns true if this operation can be performed; false if not. */
2428 :
2429 : bool
2430 6 : expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2431 : int unsignedp)
2432 : {
2433 6 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2434 6 : enum mode_class mclass;
2435 6 : machine_mode wider_mode;
2436 6 : rtx_insn *entry_last = get_last_insn ();
2437 6 : rtx_insn *last;
2438 :
2439 6 : mclass = GET_MODE_CLASS (mode);
2440 :
2441 6 : if (!targ0)
2442 0 : targ0 = gen_reg_rtx (mode);
2443 6 : if (!targ1)
2444 0 : targ1 = gen_reg_rtx (mode);
2445 :
2446 : /* Record where to go back to if we fail. */
2447 6 : last = get_last_insn ();
2448 :
2449 6 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2450 : {
2451 6 : class expand_operand ops[3];
2452 6 : enum insn_code icode = optab_handler (unoptab, mode);
2453 :
2454 6 : create_fixed_operand (&ops[0], targ0);
2455 6 : create_fixed_operand (&ops[1], targ1);
2456 6 : create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2457 6 : if (maybe_expand_insn (icode, 3, ops))
2458 6 : return true;
2459 : }
2460 :
2461 : /* It can't be done in this mode. Can we do it in a wider mode? */
2462 :
2463 0 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2464 : {
2465 0 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2466 : {
2467 0 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2468 : {
2469 0 : rtx t0 = gen_reg_rtx (wider_mode);
2470 0 : rtx t1 = gen_reg_rtx (wider_mode);
2471 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2472 :
2473 0 : if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2474 : {
2475 0 : convert_move (targ0, t0, unsignedp);
2476 0 : convert_move (targ1, t1, unsignedp);
2477 0 : return true;
2478 : }
2479 : else
2480 0 : delete_insns_since (last);
2481 : }
2482 : }
2483 : }
2484 :
2485 0 : delete_insns_since (entry_last);
2486 0 : return false;
2487 : }
2488 :
2489 : /* Generate code to perform an operation specified by BINOPTAB
2490 : on operands OP0 and OP1, with two results to TARG1 and TARG2.
2491 : We assume that the order of the operands for the instruction
2492 : is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2493 : [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2494 :
2495 : Either TARG0 or TARG1 may be zero, but what that means is that
2496 : the result is not actually wanted. We will generate it into
2497 : a dummy pseudo-reg and discard it. They may not both be zero.
2498 :
2499 : Returns true if this operation can be performed; false if not. */
2500 :
2501 : bool
2502 121153 : expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2503 : int unsignedp)
2504 : {
2505 121153 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2506 121153 : enum mode_class mclass;
2507 121153 : machine_mode wider_mode;
2508 121153 : rtx_insn *entry_last = get_last_insn ();
2509 121153 : rtx_insn *last;
2510 :
2511 121153 : mclass = GET_MODE_CLASS (mode);
2512 :
2513 121153 : if (!targ0)
2514 41729 : targ0 = gen_reg_rtx (mode);
2515 121153 : if (!targ1)
2516 67929 : targ1 = gen_reg_rtx (mode);
2517 :
2518 : /* Record where to go back to if we fail. */
2519 121153 : last = get_last_insn ();
2520 :
2521 121153 : if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2522 : {
2523 116270 : class expand_operand ops[4];
2524 116270 : enum insn_code icode = optab_handler (binoptab, mode);
2525 116270 : machine_mode mode0 = insn_data[icode].operand[1].mode;
2526 116270 : machine_mode mode1 = insn_data[icode].operand[2].mode;
2527 116270 : rtx xop0 = op0, xop1 = op1;
2528 :
2529 : /* If we are optimizing, force expensive constants into a register. */
2530 116270 : xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2531 116270 : xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2532 :
2533 116270 : create_fixed_operand (&ops[0], targ0);
2534 116270 : create_convert_operand_from (&ops[1], xop0, mode, unsignedp);
2535 116270 : create_convert_operand_from (&ops[2], xop1, mode, unsignedp);
2536 116270 : create_fixed_operand (&ops[3], targ1);
2537 116270 : if (maybe_expand_insn (icode, 4, ops))
2538 116270 : return true;
2539 0 : delete_insns_since (last);
2540 : }
2541 :
2542 : /* It can't be done in this mode. Can we do it in a wider mode? */
2543 :
2544 4883 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2545 : {
2546 15510 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2547 : {
2548 10627 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2549 : {
2550 0 : rtx t0 = gen_reg_rtx (wider_mode);
2551 0 : rtx t1 = gen_reg_rtx (wider_mode);
2552 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2553 0 : rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2554 :
2555 0 : if (expand_twoval_binop (binoptab, cop0, cop1,
2556 : t0, t1, unsignedp))
2557 : {
2558 0 : convert_move (targ0, t0, unsignedp);
2559 0 : convert_move (targ1, t1, unsignedp);
2560 0 : return true;
2561 : }
2562 : else
2563 0 : delete_insns_since (last);
2564 : }
2565 : }
2566 : }
2567 :
2568 4883 : delete_insns_since (entry_last);
2569 4883 : return false;
2570 : }
2571 :
2572 : /* Expand the two-valued library call indicated by BINOPTAB, but
2573 : preserve only one of the values. If TARG0 is non-NULL, the first
2574 : value is placed into TARG0; otherwise the second value is placed
2575 : into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2576 : value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2577 : This routine assumes that the value returned by the library call is
2578 : as if the return value was of an integral mode twice as wide as the
2579 : mode of OP0. Returns 1 if the call was successful. */
2580 :
2581 : bool
2582 0 : expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2583 : rtx targ0, rtx targ1, enum rtx_code code)
2584 : {
2585 0 : machine_mode mode;
2586 0 : machine_mode libval_mode;
2587 0 : rtx libval;
2588 0 : rtx_insn *insns;
2589 0 : rtx libfunc;
2590 :
2591 : /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2592 0 : gcc_assert (!targ0 != !targ1);
2593 :
2594 0 : mode = GET_MODE (op0);
2595 0 : libfunc = optab_libfunc (binoptab, mode);
2596 0 : if (!libfunc)
2597 : return false;
2598 :
2599 : /* The value returned by the library function will have twice as
2600 : many bits as the nominal MODE. */
2601 0 : auto return_size = 2 * GET_MODE_BITSIZE (mode);
2602 0 : if (!smallest_int_mode_for_size (return_size).exists (&libval_mode))
2603 0 : return false;
2604 :
2605 0 : start_sequence ();
2606 0 : libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2607 : libval_mode,
2608 : op0, mode,
2609 : op1, mode);
2610 : /* Get the part of VAL containing the value that we want. */
2611 0 : libval = simplify_gen_subreg (mode, libval, libval_mode,
2612 0 : targ0 ? 0 : GET_MODE_SIZE (mode));
2613 0 : insns = end_sequence ();
2614 : /* Move the into the desired location. */
2615 0 : emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2616 : gen_rtx_fmt_ee (code, mode, op0, op1));
2617 :
2618 0 : return true;
2619 : }
2620 :
2621 :
2622 : /* Wrapper around expand_unop which takes an rtx code to specify
2623 : the operation to perform, not an optab pointer. All other
2624 : arguments are the same. */
2625 : rtx
2626 20005 : expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2627 : rtx target, int unsignedp)
2628 : {
2629 20005 : optab unop = code_to_optab (code);
2630 20005 : gcc_assert (unop);
2631 :
2632 20005 : return expand_unop (mode, unop, op0, target, unsignedp);
2633 : }
2634 :
2635 : /* Try calculating
2636 : (clz:narrow x)
2637 : as
2638 : (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2639 :
2640 : A similar operation can be used for clrsb. UNOPTAB says which operation
2641 : we are trying to expand. */
2642 : static rtx
2643 298 : widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2644 : {
2645 298 : opt_scalar_int_mode wider_mode_iter;
2646 1223 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2647 : {
2648 925 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2649 925 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2650 : {
2651 0 : rtx xop0, temp;
2652 0 : rtx_insn *last;
2653 :
2654 0 : last = get_last_insn ();
2655 :
2656 0 : if (target == 0)
2657 0 : target = gen_reg_rtx (mode);
2658 0 : xop0 = widen_operand (op0, wider_mode, mode,
2659 : unoptab != clrsb_optab, false);
2660 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2661 : unoptab != clrsb_optab);
2662 0 : if (temp != 0)
2663 0 : temp = expand_binop
2664 0 : (wider_mode, sub_optab, temp,
2665 0 : gen_int_mode (GET_MODE_PRECISION (wider_mode)
2666 0 : - GET_MODE_PRECISION (mode),
2667 : wider_mode),
2668 : target, true, OPTAB_DIRECT);
2669 0 : if (temp == 0)
2670 0 : delete_insns_since (last);
2671 :
2672 298 : return temp;
2673 : }
2674 : }
2675 : return 0;
2676 : }
2677 :
2678 : /* Attempt to emit (clrsb:mode op0) as
2679 : (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2680 : (const_int -1))
2681 : if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2682 : or as
2683 : (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2684 : (ashr:mode op0 (const_int prec-1)))
2685 : (const_int 1)))
2686 : otherwise. */
2687 :
2688 : static rtx
2689 92 : expand_clrsb_using_clz (scalar_int_mode mode, rtx op0, rtx target)
2690 : {
2691 92 : if (optimize_insn_for_size_p ()
2692 92 : || optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2693 : return NULL_RTX;
2694 :
2695 88 : start_sequence ();
2696 88 : HOST_WIDE_INT val = 0;
2697 88 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) != 2
2698 88 : || val != GET_MODE_PRECISION (mode))
2699 : val = 0;
2700 : else
2701 : val = 1;
2702 :
2703 86 : rtx temp2 = op0;
2704 86 : if (!val)
2705 : {
2706 86 : temp2 = expand_binop (mode, ashl_optab, op0, const1_rtx,
2707 : NULL_RTX, 0, OPTAB_DIRECT);
2708 86 : if (!temp2)
2709 : {
2710 0 : fail:
2711 0 : end_sequence ();
2712 0 : return NULL_RTX;
2713 : }
2714 : }
2715 :
2716 88 : rtx temp = expand_binop (mode, ashr_optab, op0,
2717 88 : GEN_INT (GET_MODE_PRECISION (mode) - 1),
2718 : NULL_RTX, 0, OPTAB_DIRECT);
2719 88 : if (!temp)
2720 0 : goto fail;
2721 :
2722 88 : temp = expand_binop (mode, xor_optab, temp2, temp, NULL_RTX, 0,
2723 : OPTAB_DIRECT);
2724 88 : if (!temp)
2725 0 : goto fail;
2726 :
2727 88 : if (!val)
2728 : {
2729 86 : temp = expand_binop (mode, ior_optab, temp, const1_rtx,
2730 : NULL_RTX, 0, OPTAB_DIRECT);
2731 86 : if (!temp)
2732 0 : goto fail;
2733 : }
2734 88 : temp = expand_unop_direct (mode, clz_optab, temp, val ? NULL_RTX : target,
2735 : true);
2736 88 : if (!temp)
2737 0 : goto fail;
2738 88 : if (val)
2739 : {
2740 2 : temp = expand_binop (mode, add_optab, temp, constm1_rtx,
2741 : target, 0, OPTAB_DIRECT);
2742 2 : if (!temp)
2743 0 : goto fail;
2744 : }
2745 :
2746 88 : rtx_insn *seq = end_sequence ();
2747 :
2748 88 : add_equal_note (seq, temp, CLRSB, op0, NULL_RTX, mode);
2749 88 : emit_insn (seq);
2750 88 : return temp;
2751 : }
2752 :
2753 : static rtx expand_ffs (scalar_int_mode, rtx, rtx);
2754 :
2755 : /* Try calculating clz, ctz or ffs of a double-word quantity as two clz, ctz or
2756 : ffs operations on word-sized quantities, choosing which based on whether the
2757 : high (for clz) or low (for ctz and ffs) word is nonzero. */
2758 : static rtx
2759 251 : expand_doubleword_clz_ctz_ffs (scalar_int_mode mode, rtx op0, rtx target,
2760 : optab unoptab)
2761 : {
2762 251 : rtx xop0 = force_reg (mode, op0);
2763 251 : rtx subhi = gen_highpart (word_mode, xop0);
2764 251 : rtx sublo = gen_lowpart (word_mode, xop0);
2765 251 : rtx_code_label *hi0_label = gen_label_rtx ();
2766 251 : rtx_code_label *after_label = gen_label_rtx ();
2767 251 : rtx_insn *seq;
2768 251 : rtx temp, result;
2769 251 : int addend = 0;
2770 :
2771 : /* If we were not given a target, use a word_mode register, not a
2772 : 'mode' register. The result will fit, and nobody is expecting
2773 : anything bigger (the return type of __builtin_clz* is int). */
2774 251 : if (!target)
2775 0 : target = gen_reg_rtx (word_mode);
2776 :
2777 : /* In any case, write to a word_mode scratch in both branches of the
2778 : conditional, so we can ensure there is a single move insn setting
2779 : 'target' to tag a REG_EQUAL note on. */
2780 251 : result = gen_reg_rtx (word_mode);
2781 :
2782 251 : if (unoptab != clz_optab)
2783 45 : std::swap (subhi, sublo);
2784 :
2785 251 : start_sequence ();
2786 :
2787 : /* If the high word is not equal to zero,
2788 : then clz of the full value is clz of the high word. */
2789 251 : emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2790 : word_mode, true, hi0_label);
2791 :
2792 251 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2793 251 : temp = expand_unop_direct (word_mode, unoptab, subhi, result, true);
2794 : else
2795 : {
2796 0 : gcc_assert (unoptab == ffs_optab);
2797 0 : temp = expand_ffs (word_mode, subhi, result);
2798 : }
2799 251 : if (!temp)
2800 0 : goto fail;
2801 :
2802 251 : if (temp != result)
2803 0 : convert_move (result, temp, true);
2804 :
2805 251 : emit_jump_insn (targetm.gen_jump (after_label));
2806 251 : emit_barrier ();
2807 :
2808 : /* Else clz of the full value is clz of the low word plus the number
2809 : of bits in the high word. Similarly for ctz/ffs of the high word,
2810 : except that ffs should be 0 when both words are zero. */
2811 251 : emit_label (hi0_label);
2812 :
2813 251 : if (unoptab == ffs_optab)
2814 : {
2815 0 : convert_move (result, const0_rtx, true);
2816 0 : emit_cmp_and_jump_insns (sublo, CONST0_RTX (word_mode), EQ, 0,
2817 : word_mode, true, after_label);
2818 : }
2819 :
2820 251 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2821 251 : temp = expand_unop_direct (word_mode, unoptab, sublo, NULL_RTX, true);
2822 : else
2823 : {
2824 0 : gcc_assert (unoptab == ffs_optab);
2825 0 : temp = expand_unop_direct (word_mode, ctz_optab, sublo, NULL_RTX, true);
2826 0 : addend = 1;
2827 : }
2828 :
2829 251 : if (!temp)
2830 0 : goto fail;
2831 :
2832 502 : temp = expand_binop (word_mode, add_optab, temp,
2833 251 : gen_int_mode (GET_MODE_BITSIZE (word_mode) + addend,
2834 : word_mode),
2835 : result, true, OPTAB_DIRECT);
2836 251 : if (!temp)
2837 0 : goto fail;
2838 251 : if (temp != result)
2839 0 : convert_move (result, temp, true);
2840 :
2841 251 : emit_label (after_label);
2842 251 : convert_move (target, result, true);
2843 :
2844 251 : seq = end_sequence ();
2845 :
2846 251 : add_equal_note (seq, target, optab_to_code (unoptab), xop0, NULL_RTX, mode);
2847 251 : emit_insn (seq);
2848 251 : return target;
2849 :
2850 0 : fail:
2851 0 : end_sequence ();
2852 0 : return 0;
2853 : }
2854 :
2855 : /* Try calculating popcount of a double-word quantity as two popcount's of
2856 : word-sized quantities and summing up the results. */
2857 : static rtx
2858 0 : expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2859 : {
2860 0 : rtx t0, t1, t;
2861 0 : rtx_insn *seq;
2862 :
2863 0 : start_sequence ();
2864 :
2865 0 : t0 = expand_unop_direct (word_mode, popcount_optab,
2866 : operand_subword_force (op0, 0, mode), NULL_RTX,
2867 : true);
2868 0 : t1 = expand_unop_direct (word_mode, popcount_optab,
2869 : operand_subword_force (op0, 1, mode), NULL_RTX,
2870 : true);
2871 0 : if (!t0 || !t1)
2872 : {
2873 0 : end_sequence ();
2874 0 : return NULL_RTX;
2875 : }
2876 :
2877 : /* If we were not given a target, use a word_mode register, not a
2878 : 'mode' register. The result will fit, and nobody is expecting
2879 : anything bigger (the return type of __builtin_popcount* is int). */
2880 0 : if (!target)
2881 0 : target = gen_reg_rtx (word_mode);
2882 :
2883 0 : t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2884 :
2885 0 : seq = end_sequence ();
2886 :
2887 0 : add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2888 0 : emit_insn (seq);
2889 0 : return t;
2890 : }
2891 :
2892 : /* Try calculating
2893 : (parity:wide x)
2894 : as
2895 : (parity:narrow (low (x) ^ high (x))) */
2896 : static rtx
2897 0 : expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2898 : {
2899 0 : rtx t = expand_binop (word_mode, xor_optab,
2900 : operand_subword_force (op0, 0, mode),
2901 : operand_subword_force (op0, 1, mode),
2902 : NULL_RTX, 0, OPTAB_DIRECT);
2903 0 : return expand_unop (word_mode, parity_optab, t, target, true);
2904 : }
2905 :
2906 : /* Try calculating
2907 : (bswap:narrow x)
2908 : as
2909 : (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow)))
2910 : or similarly for bitreverse. */
2911 : static rtx
2912 272 : widen_bswap_or_bitreverse (scalar_int_mode mode, rtx op0, rtx target,
2913 : optab unoptab)
2914 : {
2915 272 : rtx x;
2916 272 : rtx_insn *last;
2917 272 : opt_scalar_int_mode wider_mode_iter;
2918 :
2919 1103 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2920 831 : if (optab_handler (unoptab, wider_mode_iter.require ())
2921 : != CODE_FOR_nothing)
2922 : break;
2923 :
2924 272 : if (!wider_mode_iter.exists ())
2925 : return NULL_RTX;
2926 :
2927 0 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2928 0 : last = get_last_insn ();
2929 :
2930 0 : x = widen_operand (op0, wider_mode, mode, true, true);
2931 0 : x = expand_unop (wider_mode, unoptab, x, NULL_RTX, true);
2932 :
2933 0 : gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2934 : && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2935 0 : if (x != 0)
2936 0 : x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2937 0 : GET_MODE_BITSIZE (wider_mode)
2938 0 : - GET_MODE_BITSIZE (mode),
2939 : NULL_RTX, true);
2940 :
2941 0 : if (x != 0)
2942 : {
2943 0 : if (target == 0)
2944 0 : target = gen_reg_rtx (mode);
2945 0 : emit_move_insn (target, gen_lowpart (mode, x));
2946 : }
2947 : else
2948 0 : delete_insns_since (last);
2949 :
2950 : return target;
2951 : }
2952 :
2953 : /* Try calculating bswap as two bswaps of two word-sized operands.
2954 : Similarly for bitreverse. */
2955 :
2956 : static rtx
2957 163 : expand_doubleword_bswap_or_bitreverse (machine_mode mode, rtx op, rtx target,
2958 : optab unoptab)
2959 : {
2960 163 : rtx t0, t1;
2961 :
2962 163 : t1 = expand_unop (word_mode, unoptab,
2963 : operand_subword_force (op, 0, mode), NULL_RTX, true);
2964 163 : t0 = expand_unop (word_mode, unoptab,
2965 : operand_subword_force (op, 1, mode), NULL_RTX, true);
2966 :
2967 163 : if (target == 0 || !valid_multiword_target_p (target))
2968 6 : target = gen_reg_rtx (mode);
2969 163 : if (REG_P (target))
2970 163 : emit_clobber (target);
2971 163 : emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2972 163 : emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2973 :
2974 163 : return target;
2975 : }
2976 :
2977 : /* Try calculating (bitreverse x) using masks and shifts. */
2978 :
2979 : static rtx
2980 109 : expand_bitreverse (scalar_int_mode mode, rtx op0, rtx target)
2981 : {
2982 109 : unsigned int precision = GET_MODE_BITSIZE (mode);
2983 109 : rtx_insn *last;
2984 :
2985 : /* Operation requires at least 4 bits (one nibble swap makes no sense below
2986 : that). */
2987 109 : if (precision < 4)
2988 : return NULL_RTX;
2989 :
2990 109 : if (rtx temp = widen_bswap_or_bitreverse (mode, op0, target,
2991 : bitreverse_optab))
2992 : return temp;
2993 :
2994 218 : if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
2995 109 : && optab_handler (bitreverse_optab, word_mode) != CODE_FOR_nothing)
2996 0 : if (rtx temp = expand_doubleword_bswap_or_bitreverse (mode, op0, target,
2997 : bitreverse_optab))
2998 0 : return temp;
2999 :
3000 109 : if (target == NULL_RTX
3001 109 : || target == op0
3002 109 : || reg_overlap_mentioned_p (target, op0))
3003 0 : target = gen_reg_rtx (mode);
3004 :
3005 109 : last = get_last_insn ();
3006 :
3007 109 : rtx x, lo, hi;
3008 :
3009 : /* Step 1: byte-swap (only meaningful for >= 16 bits). */
3010 109 : if (precision >= 16)
3011 : {
3012 104 : x = expand_unop (mode, bswap_optab, op0, NULL_RTX, true);
3013 104 : if (x == NULL_RTX)
3014 0 : goto fail;
3015 : }
3016 : else
3017 : x = op0;
3018 :
3019 : /* Step 2: swap nibbles within each byte (shift=4, only for >= 8 bits). */
3020 109 : if (precision >= 8)
3021 : {
3022 109 : wide_int mask = wi::zero (precision);
3023 912 : for (unsigned int start = 0; start < precision; start += 8)
3024 1606 : mask = wi::bit_or (mask, wi::shifted_mask (start, 4, false,
3025 803 : precision));
3026 :
3027 109 : rtx mask_rtx = immed_wide_int_const (mask, mode);
3028 :
3029 109 : hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (4),
3030 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3031 109 : if (hi == NULL_RTX) goto fail;
3032 109 : hi = expand_binop (mode, and_optab, hi, mask_rtx,
3033 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3034 109 : if (hi == NULL_RTX) goto fail;
3035 :
3036 109 : lo = expand_binop (mode, and_optab, x, mask_rtx,
3037 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3038 109 : if (lo == NULL_RTX) goto fail;
3039 109 : lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (4),
3040 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3041 109 : if (lo == NULL_RTX) goto fail;
3042 :
3043 109 : x = expand_binop (mode, aop_optab, hi, lo,
3044 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3045 109 : if (x == NULL_RTX) goto fail;
3046 0 : }
3047 :
3048 : /* Step 3: swap pairs of bits within each nibble (shift=2). */
3049 109 : {
3050 109 : wide_int mask = wi::zero (precision);
3051 1715 : for (unsigned int start = 0; start < precision; start += 4)
3052 1606 : mask = wi::bit_or (mask, wi::shifted_mask (start, 2, false, precision));
3053 :
3054 109 : rtx mask_rtx = immed_wide_int_const (mask, mode);
3055 :
3056 109 : hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (2),
3057 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3058 109 : if (hi == NULL_RTX) goto fail;
3059 109 : hi = expand_binop (mode, and_optab, hi, mask_rtx,
3060 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3061 109 : if (hi == NULL_RTX) goto fail;
3062 :
3063 109 : lo = expand_binop (mode, and_optab, x, mask_rtx,
3064 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3065 109 : if (lo == NULL_RTX) goto fail;
3066 109 : lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (2),
3067 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3068 109 : if (lo == NULL_RTX) goto fail;
3069 :
3070 109 : x = expand_binop (mode, aop_optab, hi, lo,
3071 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3072 109 : if (x == NULL_RTX) goto fail;
3073 0 : }
3074 :
3075 : /* Step 4: swap adjacent bits (shift=1). */
3076 109 : {
3077 109 : wide_int mask = wi::zero (precision);
3078 3321 : for (unsigned int start = 0; start < precision; start += 2)
3079 6424 : mask = wi::bit_or (mask, wi::shifted_mask (start, 1, false,
3080 3212 : precision));
3081 :
3082 109 : rtx mask_rtx = immed_wide_int_const (mask, mode);
3083 :
3084 109 : hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (1),
3085 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3086 109 : if (hi == NULL_RTX) goto fail;
3087 109 : hi = expand_binop (mode, and_optab, hi, mask_rtx,
3088 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3089 109 : if (hi == NULL_RTX) goto fail;
3090 :
3091 109 : lo = expand_binop (mode, and_optab, x, mask_rtx,
3092 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3093 109 : if (lo == NULL_RTX) goto fail;
3094 109 : lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (1),
3095 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3096 109 : if (lo == NULL_RTX) goto fail;
3097 :
3098 109 : x = expand_binop (mode, aop_optab, hi, lo,
3099 : target, true, OPTAB_LIB_WIDEN);
3100 109 : if (x == NULL_RTX) goto fail;
3101 0 : }
3102 :
3103 109 : if (x != target)
3104 0 : emit_move_insn (target, x);
3105 :
3106 : return target;
3107 :
3108 0 : fail:
3109 0 : delete_insns_since (last);
3110 0 : return NULL_RTX;
3111 : }
3112 :
3113 : /* Try calculating (parity x) as (and (popcount x) 1), where
3114 : popcount can also be done in a wider mode. */
3115 : static rtx
3116 11 : expand_parity (scalar_int_mode mode, rtx op0, rtx target)
3117 : {
3118 11 : enum mode_class mclass = GET_MODE_CLASS (mode);
3119 11 : opt_scalar_int_mode wider_mode_iter;
3120 11 : FOR_EACH_MODE_FROM (wider_mode_iter, mode)
3121 : {
3122 11 : scalar_int_mode wider_mode = wider_mode_iter.require ();
3123 11 : if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
3124 : {
3125 11 : rtx xop0, temp;
3126 11 : rtx_insn *last;
3127 :
3128 11 : last = get_last_insn ();
3129 :
3130 11 : if (target == 0 || GET_MODE (target) != wider_mode)
3131 6 : target = gen_reg_rtx (wider_mode);
3132 :
3133 11 : xop0 = widen_operand (op0, wider_mode, mode, true, false);
3134 11 : temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
3135 : true);
3136 11 : if (temp != 0)
3137 11 : temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
3138 : target, true, OPTAB_DIRECT);
3139 :
3140 11 : if (temp)
3141 : {
3142 11 : if (mclass != MODE_INT
3143 11 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3144 0 : return convert_to_mode (mode, temp, 0);
3145 : else
3146 11 : return gen_lowpart (mode, temp);
3147 : }
3148 : else
3149 0 : delete_insns_since (last);
3150 : }
3151 : }
3152 : return 0;
3153 : }
3154 :
3155 : /* Try calculating ctz(x) as K - clz(x & -x) ,
3156 : where K is GET_MODE_PRECISION(mode) - 1.
3157 :
3158 : Both __builtin_ctz and __builtin_clz are undefined at zero, so we
3159 : don't have to worry about what the hardware does in that case. (If
3160 : the clz instruction produces the usual value at 0, which is K, the
3161 : result of this code sequence will be -1; expand_ffs, below, relies
3162 : on this. It might be nice to have it be K instead, for consistency
3163 : with the (very few) processors that provide a ctz with a defined
3164 : value, but that would take one more instruction, and it would be
3165 : less convenient for expand_ffs anyway. */
3166 :
3167 : static rtx
3168 48 : expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
3169 : {
3170 48 : rtx_insn *seq;
3171 48 : rtx temp;
3172 :
3173 48 : if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
3174 : return 0;
3175 :
3176 0 : start_sequence ();
3177 :
3178 0 : temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
3179 0 : if (temp)
3180 0 : temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
3181 : true, OPTAB_DIRECT);
3182 0 : if (temp)
3183 0 : temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
3184 0 : if (temp)
3185 0 : temp = expand_binop (mode, sub_optab,
3186 0 : gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
3187 : temp, target,
3188 : true, OPTAB_DIRECT);
3189 0 : if (temp == 0)
3190 : {
3191 0 : end_sequence ();
3192 0 : return 0;
3193 : }
3194 :
3195 0 : seq = end_sequence ();
3196 :
3197 0 : add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
3198 0 : emit_insn (seq);
3199 0 : return temp;
3200 : }
3201 :
3202 :
3203 : /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
3204 : else with the sequence used by expand_clz.
3205 :
3206 : The ffs builtin promises to return zero for a zero value and ctz/clz
3207 : may have an undefined value in that case. If they do not give us a
3208 : convenient value, we have to generate a test and branch. */
3209 : static rtx
3210 0 : expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
3211 : {
3212 0 : HOST_WIDE_INT val = 0;
3213 0 : bool defined_at_zero = false;
3214 0 : rtx temp;
3215 0 : rtx_insn *seq;
3216 :
3217 0 : if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
3218 : {
3219 0 : start_sequence ();
3220 :
3221 0 : temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
3222 0 : if (!temp)
3223 0 : goto fail;
3224 :
3225 0 : defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
3226 : }
3227 0 : else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
3228 : {
3229 0 : start_sequence ();
3230 0 : temp = expand_ctz (mode, op0, 0);
3231 0 : if (!temp)
3232 0 : goto fail;
3233 :
3234 0 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
3235 : {
3236 0 : defined_at_zero = true;
3237 0 : val = (GET_MODE_PRECISION (mode) - 1) - val;
3238 : }
3239 : }
3240 : else
3241 : return 0;
3242 :
3243 0 : if (defined_at_zero && val == -1)
3244 : /* No correction needed at zero. */;
3245 : else
3246 : {
3247 : /* We don't try to do anything clever with the situation found
3248 : on some processors (eg Alpha) where ctz(0:mode) ==
3249 : bitsize(mode). If someone can think of a way to send N to -1
3250 : and leave alone all values in the range 0..N-1 (where N is a
3251 : power of two), cheaper than this test-and-branch, please add it.
3252 :
3253 : The test-and-branch is done after the operation itself, in case
3254 : the operation sets condition codes that can be recycled for this.
3255 : (This is true on i386, for instance.) */
3256 :
3257 0 : rtx_code_label *nonzero_label = gen_label_rtx ();
3258 0 : emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
3259 : mode, true, nonzero_label);
3260 :
3261 0 : convert_move (temp, GEN_INT (-1), false);
3262 0 : emit_label (nonzero_label);
3263 : }
3264 :
3265 : /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3266 : to produce a value in the range 0..bitsize. */
3267 0 : temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
3268 : target, false, OPTAB_DIRECT);
3269 0 : if (!temp)
3270 0 : goto fail;
3271 :
3272 0 : seq = end_sequence ();
3273 :
3274 0 : add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
3275 0 : emit_insn (seq);
3276 0 : return temp;
3277 :
3278 0 : fail:
3279 0 : end_sequence ();
3280 0 : return 0;
3281 : }
3282 :
3283 : /* Expand a floating point absolute value or negation operation via a
3284 : logical operation on the sign bit. MODE is the mode of the operands
3285 : and FMODE is the scalar inner mode. */
3286 :
3287 : static rtx
3288 2400 : expand_absneg_bit (rtx_code code, machine_mode mode,
3289 : scalar_float_mode fmode, rtx op0, rtx target)
3290 : {
3291 2400 : int bitpos, word, nwords, i;
3292 2400 : machine_mode new_mode;
3293 2400 : scalar_int_mode imode;
3294 2400 : rtx temp;
3295 2400 : rtx_insn *insns;
3296 :
3297 2400 : auto op = code == NEG ? neg_optab : abs_optab;
3298 2400 : if (!get_absneg_bit_mode (op, mode, fmode, &bitpos).exists (&new_mode))
3299 0 : return NULL_RTX;
3300 :
3301 4800 : imode = as_a<scalar_int_mode> (GET_MODE_INNER (new_mode));
3302 4868 : if (VECTOR_MODE_P (mode) || GET_MODE_SIZE (fmode) <= UNITS_PER_WORD)
3303 : {
3304 : word = 0;
3305 : nwords = 1;
3306 : }
3307 : else
3308 : {
3309 17 : if (FLOAT_WORDS_BIG_ENDIAN)
3310 : word = (GET_MODE_BITSIZE (fmode) - bitpos) / BITS_PER_WORD;
3311 : else
3312 17 : word = bitpos / BITS_PER_WORD;
3313 17 : bitpos = bitpos % BITS_PER_WORD;
3314 34 : nwords = (GET_MODE_BITSIZE (fmode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3315 : }
3316 :
3317 2400 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3318 2400 : if (code == ABS)
3319 2110 : mask = ~mask;
3320 :
3321 2400 : if (target == 0
3322 2400 : || target == op0
3323 684 : || reg_overlap_mentioned_p (target, op0)
3324 3080 : || (nwords > 1 && !valid_multiword_target_p (target)))
3325 1720 : target = gen_reg_rtx (mode);
3326 :
3327 2400 : if (nwords > 1)
3328 : {
3329 17 : start_sequence ();
3330 :
3331 68 : for (i = 0; i < nwords; ++i)
3332 : {
3333 34 : rtx targ_piece = operand_subword (target, i, 1, mode);
3334 34 : rtx op0_piece = operand_subword_force (op0, i, mode);
3335 :
3336 34 : if (i == word)
3337 : {
3338 46 : temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
3339 : op0_piece,
3340 17 : immed_wide_int_const (mask, imode),
3341 : targ_piece, 1, OPTAB_LIB_WIDEN);
3342 17 : if (temp != targ_piece)
3343 0 : emit_move_insn (targ_piece, temp);
3344 : }
3345 : else
3346 17 : emit_move_insn (targ_piece, op0_piece);
3347 : }
3348 :
3349 17 : insns = end_sequence ();
3350 :
3351 17 : emit_insn (insns);
3352 : }
3353 : else
3354 : {
3355 2383 : rtx mask_rtx = immed_wide_int_const (mask, imode);
3356 2383 : if (VECTOR_MODE_P (new_mode))
3357 0 : mask_rtx = gen_const_vec_duplicate (new_mode, mask_rtx);
3358 2661 : temp = expand_binop (new_mode, code == ABS ? and_optab : xor_optab,
3359 2383 : gen_lowpart (new_mode, op0), mask_rtx,
3360 2383 : gen_lowpart (new_mode, target), 1, OPTAB_LIB_WIDEN);
3361 2383 : target = force_lowpart_subreg (mode, temp, new_mode);
3362 :
3363 2383 : set_dst_reg_note (get_last_insn (), REG_EQUAL,
3364 : gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
3365 : target);
3366 : }
3367 :
3368 2400 : return target;
3369 2400 : }
3370 :
3371 : /* As expand_unop, but will fail rather than attempt the operation in a
3372 : different mode or with a libcall. */
3373 : static rtx
3374 186528 : expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
3375 : int unsignedp)
3376 : {
3377 186528 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
3378 : {
3379 182585 : class expand_operand ops[2];
3380 182585 : enum insn_code icode = optab_handler (unoptab, mode);
3381 182585 : rtx_insn *last = get_last_insn ();
3382 182585 : rtx_insn *pat;
3383 :
3384 182585 : create_output_operand (&ops[0], target, mode);
3385 182585 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
3386 182585 : pat = maybe_gen_insn (icode, 2, ops);
3387 182585 : if (pat)
3388 : {
3389 181965 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3390 233787 : && ! add_equal_note (pat, ops[0].value,
3391 : optab_to_code (unoptab),
3392 : ops[1].value, NULL_RTX, mode))
3393 : {
3394 1089 : delete_insns_since (last);
3395 182585 : return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3396 : }
3397 :
3398 181496 : emit_insn (pat);
3399 :
3400 181496 : return ops[0].value;
3401 : }
3402 : }
3403 : return 0;
3404 : }
3405 :
3406 : /* Generate code to perform an operation specified by UNOPTAB
3407 : on operand OP0, with result having machine-mode MODE.
3408 :
3409 : UNSIGNEDP is for the case where we have to widen the operands
3410 : to perform the operation. It says to use zero-extension.
3411 :
3412 : If TARGET is nonzero, the value
3413 : is generated there, if it is convenient to do so.
3414 : In all cases an rtx is returned for the locus of the value;
3415 : this may or may not be TARGET. */
3416 :
3417 : rtx
3418 185938 : expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
3419 : int unsignedp)
3420 : {
3421 185938 : enum mode_class mclass = GET_MODE_CLASS (mode);
3422 185938 : machine_mode wider_mode;
3423 185938 : scalar_int_mode int_mode;
3424 185938 : scalar_float_mode float_mode;
3425 185938 : rtx temp;
3426 185938 : rtx libfunc;
3427 :
3428 185938 : temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3429 185938 : if (temp)
3430 : return temp;
3431 :
3432 : /* It can't be done in this mode. Can we open-code it in a wider mode? */
3433 :
3434 : /* Widening (or narrowing) clz needs special treatment. */
3435 3943 : if (unoptab == clz_optab)
3436 : {
3437 206 : if (is_a <scalar_int_mode> (mode, &int_mode))
3438 : {
3439 206 : temp = widen_leading (int_mode, op0, target, unoptab);
3440 206 : if (temp)
3441 : return temp;
3442 :
3443 412 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3444 206 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3445 : {
3446 206 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target,
3447 : unoptab);
3448 206 : if (temp)
3449 : return temp;
3450 : }
3451 : }
3452 :
3453 0 : goto try_libcall;
3454 : }
3455 :
3456 3737 : if (unoptab == clrsb_optab)
3457 : {
3458 92 : if (is_a <scalar_int_mode> (mode, &int_mode))
3459 : {
3460 92 : temp = widen_leading (int_mode, op0, target, unoptab);
3461 92 : if (temp)
3462 : return temp;
3463 92 : temp = expand_clrsb_using_clz (int_mode, op0, target);
3464 92 : if (temp)
3465 : return temp;
3466 : }
3467 4 : goto try_libcall;
3468 : }
3469 :
3470 3645 : if (unoptab == popcount_optab
3471 733 : && is_a <scalar_int_mode> (mode, &int_mode)
3472 765 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3473 2 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3474 3645 : && optimize_insn_for_speed_p ())
3475 : {
3476 0 : temp = expand_doubleword_popcount (int_mode, op0, target);
3477 0 : if (temp)
3478 : return temp;
3479 : }
3480 :
3481 3645 : if (unoptab == parity_optab
3482 11 : && is_a <scalar_int_mode> (mode, &int_mode)
3483 11 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3484 0 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3485 0 : || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
3486 3645 : && optimize_insn_for_speed_p ())
3487 : {
3488 0 : temp = expand_doubleword_parity (int_mode, op0, target);
3489 0 : if (temp)
3490 : return temp;
3491 : }
3492 :
3493 : /* Widening (or narrowing) bswap needs special treatment. */
3494 3645 : if (unoptab == bswap_optab)
3495 : {
3496 : /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3497 : or ROTATERT. First try these directly; if this fails, then try the
3498 : obvious pair of shifts with allowed widening, as this will probably
3499 : be always more efficient than the other fallback methods. */
3500 163 : if (mode == HImode)
3501 : {
3502 0 : rtx_insn *last;
3503 0 : rtx temp1, temp2;
3504 :
3505 0 : if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3506 : {
3507 0 : temp = expand_binop (mode, rotl_optab, op0,
3508 : gen_int_shift_amount (mode, 8),
3509 : target, unsignedp, OPTAB_DIRECT);
3510 0 : if (temp)
3511 : return temp;
3512 : }
3513 :
3514 0 : if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3515 : {
3516 0 : temp = expand_binop (mode, rotr_optab, op0,
3517 : gen_int_shift_amount (mode, 8),
3518 : target, unsignedp, OPTAB_DIRECT);
3519 0 : if (temp)
3520 : return temp;
3521 : }
3522 :
3523 0 : last = get_last_insn ();
3524 :
3525 0 : temp1 = expand_binop (mode, ashl_optab, op0,
3526 : gen_int_shift_amount (mode, 8), NULL_RTX,
3527 : unsignedp, OPTAB_WIDEN);
3528 0 : temp2 = expand_binop (mode, lshr_optab, op0,
3529 : gen_int_shift_amount (mode, 8), NULL_RTX,
3530 : unsignedp, OPTAB_WIDEN);
3531 0 : if (temp1 && temp2)
3532 : {
3533 0 : temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3534 : unsignedp, OPTAB_WIDEN);
3535 0 : if (temp)
3536 : return temp;
3537 : }
3538 :
3539 0 : delete_insns_since (last);
3540 : }
3541 :
3542 163 : if (is_a <scalar_int_mode> (mode, &int_mode))
3543 : {
3544 163 : temp = widen_bswap_or_bitreverse (int_mode, op0, target, unoptab);
3545 163 : if (temp)
3546 : return temp;
3547 :
3548 : /* We do not provide a 128-bit bswap in libgcc so force the use of
3549 : a double bswap for 64-bit targets. */
3550 326 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3551 163 : && (UNITS_PER_WORD == 8
3552 147 : || optab_handler (unoptab, word_mode) != CODE_FOR_nothing))
3553 : {
3554 163 : temp = expand_doubleword_bswap_or_bitreverse (mode, op0, target,
3555 : unoptab);
3556 163 : if (temp)
3557 : return temp;
3558 : }
3559 : }
3560 :
3561 0 : goto try_libcall;
3562 : }
3563 :
3564 3482 : if (unoptab == bitreverse_optab && is_a <scalar_int_mode> (mode, &int_mode))
3565 109 : if (rtx tem = expand_bitreverse (int_mode, op0, target))
3566 : return tem;
3567 :
3568 : /* Neg should be tried via expand_absneg_bit before widening. */
3569 3373 : if (optab_to_code (unoptab) == NEG)
3570 : {
3571 : /* Try negating floating point values by flipping the sign bit. */
3572 766 : if (is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3573 : {
3574 290 : temp = expand_absneg_bit (NEG, mode, float_mode, op0, target);
3575 290 : if (temp)
3576 : return temp;
3577 : }
3578 :
3579 : /* If there is no negation pattern, and we have no negative zero,
3580 : try subtracting from zero. */
3581 93 : if (!HONOR_SIGNED_ZEROS (mode))
3582 : {
3583 93 : temp = expand_binop (mode, (unoptab == negv_optab
3584 : ? subv_optab : sub_optab),
3585 : CONST0_RTX (mode), op0, target,
3586 : unsignedp, OPTAB_DIRECT);
3587 93 : if (temp)
3588 : return temp;
3589 : }
3590 : }
3591 :
3592 : /* ABS also needs to be handled similarly. */
3593 3083 : if (optab_to_code (unoptab) == ABS
3594 5281 : && is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3595 : {
3596 2110 : temp = expand_absneg_bit (ABS, mode, float_mode, op0, target);
3597 2110 : if (temp)
3598 : return temp;
3599 : }
3600 :
3601 973 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3602 4346 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3603 : {
3604 3373 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3605 : {
3606 0 : rtx xop0 = op0;
3607 0 : rtx_insn *last = get_last_insn ();
3608 :
3609 : /* For certain operations, we need not actually extend
3610 : the narrow operand, as long as we will truncate the
3611 : results to the same narrowness. */
3612 :
3613 0 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3614 0 : (unoptab == neg_optab
3615 0 : || unoptab == one_cmpl_optab)
3616 : && mclass == MODE_INT);
3617 :
3618 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3619 : unsignedp);
3620 :
3621 0 : if (temp)
3622 : {
3623 0 : if (mclass != MODE_INT
3624 0 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3625 : {
3626 0 : if (target == 0)
3627 0 : target = gen_reg_rtx (mode);
3628 0 : convert_move (target, temp, 0);
3629 0 : return target;
3630 : }
3631 : else
3632 0 : return gen_lowpart (mode, temp);
3633 : }
3634 : else
3635 0 : delete_insns_since (last);
3636 : }
3637 : }
3638 :
3639 : /* These can be done a word at a time. */
3640 973 : if (unoptab == one_cmpl_optab
3641 0 : && is_int_mode (mode, &int_mode)
3642 0 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
3643 973 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3644 : {
3645 0 : int i;
3646 0 : rtx_insn *insns;
3647 :
3648 0 : if (target == 0
3649 0 : || target == op0
3650 0 : || reg_overlap_mentioned_p (target, op0)
3651 0 : || !valid_multiword_target_p (target))
3652 0 : target = gen_reg_rtx (int_mode);
3653 :
3654 0 : start_sequence ();
3655 :
3656 : /* Do the actual arithmetic. */
3657 0 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
3658 : {
3659 0 : rtx target_piece = operand_subword (target, i, 1, int_mode);
3660 0 : rtx x = expand_unop (word_mode, unoptab,
3661 0 : operand_subword_force (op0, i, int_mode),
3662 : target_piece, unsignedp);
3663 :
3664 0 : if (target_piece != x)
3665 0 : emit_move_insn (target_piece, x);
3666 : }
3667 :
3668 0 : insns = end_sequence ();
3669 :
3670 0 : emit_insn (insns);
3671 0 : return target;
3672 : }
3673 :
3674 : /* Emit ~op0 as op0 ^ -1. */
3675 973 : if (unoptab == one_cmpl_optab
3676 0 : && (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
3677 973 : && optab_handler (xor_optab, mode) != CODE_FOR_nothing)
3678 : {
3679 0 : temp = expand_binop (mode, xor_optab, op0, CONSTM1_RTX (mode),
3680 : target, unsignedp, OPTAB_DIRECT);
3681 0 : if (temp)
3682 : return temp;
3683 : }
3684 :
3685 : /* Try calculating parity (x) as popcount (x) % 2. */
3686 973 : if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
3687 : {
3688 11 : temp = expand_parity (int_mode, op0, target);
3689 11 : if (temp)
3690 : return temp;
3691 : }
3692 :
3693 : /* Try implementing ffs (x) in terms of clz (x). */
3694 962 : if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
3695 : {
3696 0 : temp = expand_ffs (int_mode, op0, target);
3697 0 : if (temp)
3698 : return temp;
3699 : }
3700 :
3701 : /* Try implementing ctz (x) in terms of clz (x). */
3702 962 : if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
3703 : {
3704 48 : temp = expand_ctz (int_mode, op0, target);
3705 48 : if (temp)
3706 : return temp;
3707 : }
3708 :
3709 962 : if ((unoptab == ctz_optab || unoptab == ffs_optab)
3710 48 : && optimize_insn_for_speed_p ()
3711 45 : && is_a <scalar_int_mode> (mode, &int_mode)
3712 90 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3713 1007 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3714 0 : || optab_handler (ctz_optab, word_mode) != CODE_FOR_nothing))
3715 : {
3716 45 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target, unoptab);
3717 45 : if (temp)
3718 : return temp;
3719 : }
3720 :
3721 921 : try_libcall:
3722 : /* Now try a library call in this mode. */
3723 921 : libfunc = optab_libfunc (unoptab, mode);
3724 921 : if (libfunc)
3725 : {
3726 590 : rtx_insn *insns;
3727 590 : rtx value;
3728 590 : rtx eq_value;
3729 590 : machine_mode outmode = mode;
3730 :
3731 : /* All of these functions return small values. Thus we choose to
3732 : have them return something that isn't a double-word. */
3733 590 : if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3734 : || unoptab == clrsb_optab || unoptab == popcount_optab
3735 590 : || unoptab == parity_optab)
3736 497 : outmode
3737 497 : = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3738 : optab_libfunc (unoptab, mode)));
3739 :
3740 590 : start_sequence ();
3741 :
3742 : /* Pass 1 for NO_QUEUE so we don't lose any increments
3743 : if the libcall is cse'd or moved. */
3744 590 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3745 : op0, mode);
3746 590 : insns = end_sequence ();
3747 :
3748 590 : target = gen_reg_rtx (outmode);
3749 590 : bool trapv = trapv_unoptab_p (unoptab);
3750 590 : if (trapv)
3751 : eq_value = NULL_RTX;
3752 : else
3753 : {
3754 497 : eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3755 1491 : if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3756 467 : eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3757 90 : else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3758 0 : eq_value = simplify_gen_unary (ZERO_EXTEND,
3759 : outmode, eq_value, mode);
3760 : }
3761 590 : emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3762 :
3763 590 : return target;
3764 : }
3765 :
3766 : /* It can't be done in this mode. Can we do it in a wider mode? */
3767 :
3768 331 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3769 : {
3770 682 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3771 : {
3772 594 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3773 594 : || optab_libfunc (unoptab, wider_mode))
3774 : {
3775 243 : rtx xop0 = op0;
3776 243 : rtx_insn *last = get_last_insn ();
3777 :
3778 : /* For certain operations, we need not actually extend
3779 : the narrow operand, as long as we will truncate the
3780 : results to the same narrowness. */
3781 243 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3782 243 : (unoptab == neg_optab
3783 243 : || unoptab == one_cmpl_optab
3784 243 : || unoptab == bswap_optab)
3785 : && mclass == MODE_INT);
3786 :
3787 243 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3788 : unsignedp);
3789 :
3790 : /* If we are generating clz using wider mode, adjust the
3791 : result. Similarly for clrsb. */
3792 243 : if ((unoptab == clz_optab || unoptab == clrsb_optab)
3793 1 : && temp != 0)
3794 : {
3795 1 : scalar_int_mode wider_int_mode
3796 1 : = as_a <scalar_int_mode> (wider_mode);
3797 1 : int_mode = as_a <scalar_int_mode> (mode);
3798 1 : temp = expand_binop
3799 1 : (wider_mode, sub_optab, temp,
3800 1 : gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3801 2 : - GET_MODE_PRECISION (int_mode),
3802 : wider_int_mode),
3803 : target, true, OPTAB_DIRECT);
3804 : }
3805 :
3806 : /* Likewise for bswap. */
3807 243 : if (unoptab == bswap_optab && temp != 0)
3808 : {
3809 0 : scalar_int_mode wider_int_mode
3810 0 : = as_a <scalar_int_mode> (wider_mode);
3811 0 : int_mode = as_a <scalar_int_mode> (mode);
3812 0 : gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3813 : == GET_MODE_BITSIZE (wider_int_mode)
3814 : && GET_MODE_PRECISION (int_mode)
3815 : == GET_MODE_BITSIZE (int_mode));
3816 :
3817 0 : temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3818 0 : GET_MODE_BITSIZE (wider_int_mode)
3819 0 : - GET_MODE_BITSIZE (int_mode),
3820 : NULL_RTX, true);
3821 : }
3822 :
3823 243 : if (temp)
3824 : {
3825 243 : if (mclass != MODE_INT)
3826 : {
3827 0 : if (target == 0)
3828 0 : target = gen_reg_rtx (mode);
3829 0 : convert_move (target, temp, 0);
3830 0 : return target;
3831 : }
3832 : else
3833 243 : return gen_lowpart (mode, temp);
3834 : }
3835 : else
3836 0 : delete_insns_since (last);
3837 : }
3838 : }
3839 : }
3840 :
3841 : /* One final attempt at implementing negation via subtraction,
3842 : this time allowing widening of the operand. */
3843 88 : if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3844 : {
3845 0 : rtx temp;
3846 0 : temp = expand_binop (mode,
3847 : unoptab == negv_optab ? subv_optab : sub_optab,
3848 : CONST0_RTX (mode), op0,
3849 : target, unsignedp, OPTAB_LIB_WIDEN);
3850 0 : if (temp)
3851 : return temp;
3852 : }
3853 :
3854 : return 0;
3855 : }
3856 :
3857 : /* Emit code to compute the absolute value of OP0, with result to
3858 : TARGET if convenient. (TARGET may be 0.) The return value says
3859 : where the result actually is to be found.
3860 :
3861 : MODE is the mode of the operand; the mode of the result is
3862 : different but can be deduced from MODE.
3863 :
3864 : */
3865 :
3866 : rtx
3867 28460 : expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3868 : int result_unsignedp)
3869 : {
3870 28460 : rtx temp;
3871 :
3872 28460 : if (GET_MODE_CLASS (mode) != MODE_INT
3873 5100 : || ! flag_trapv)
3874 : result_unsignedp = 1;
3875 :
3876 : /* First try to do it with a special abs instruction. */
3877 28548 : temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3878 : op0, target, 0);
3879 28460 : if (temp != 0)
3880 : return temp;
3881 :
3882 : /* If we have a MAX insn, we can do this as MAX (x, -x). */
3883 88 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3884 88 : && !HONOR_SIGNED_ZEROS (mode))
3885 : {
3886 88 : rtx_insn *last = get_last_insn ();
3887 :
3888 176 : temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3889 : op0, NULL_RTX, 0);
3890 88 : if (temp != 0)
3891 88 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3892 : OPTAB_WIDEN);
3893 :
3894 88 : if (temp != 0)
3895 : return temp;
3896 :
3897 0 : delete_insns_since (last);
3898 : }
3899 :
3900 : /* If this machine has expensive jumps, we can do integer absolute
3901 : value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3902 : where W is the width of MODE. */
3903 :
3904 0 : scalar_int_mode int_mode;
3905 0 : if (is_int_mode (mode, &int_mode)
3906 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
3907 : false) >= 2)
3908 : {
3909 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3910 0 : GET_MODE_PRECISION (int_mode) - 1,
3911 : NULL_RTX, 0);
3912 :
3913 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3914 : OPTAB_LIB_WIDEN);
3915 0 : if (temp != 0)
3916 0 : temp = expand_binop (int_mode,
3917 : result_unsignedp ? sub_optab : subv_optab,
3918 : temp, extended, target, 0, OPTAB_LIB_WIDEN);
3919 :
3920 0 : if (temp != 0)
3921 : return temp;
3922 : }
3923 :
3924 : return NULL_RTX;
3925 : }
3926 :
3927 : rtx
3928 28436 : expand_abs (machine_mode mode, rtx op0, rtx target,
3929 : int result_unsignedp, int safe)
3930 : {
3931 28436 : rtx temp;
3932 28436 : rtx_code_label *op1;
3933 :
3934 28436 : if (GET_MODE_CLASS (mode) != MODE_INT
3935 5076 : || ! flag_trapv)
3936 28348 : result_unsignedp = 1;
3937 :
3938 28436 : temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3939 28436 : if (temp != 0)
3940 : return temp;
3941 :
3942 : /* If that does not win, use conditional jump and negate. */
3943 :
3944 : /* It is safe to use the target if it is the same
3945 : as the source if this is also a pseudo register */
3946 0 : if (op0 == target && REG_P (op0)
3947 0 : && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3948 : safe = 1;
3949 :
3950 0 : op1 = gen_label_rtx ();
3951 0 : if (target == 0 || ! safe
3952 0 : || GET_MODE (target) != mode
3953 0 : || (MEM_P (target) && MEM_VOLATILE_P (target))
3954 0 : || (REG_P (target)
3955 0 : && REGNO (target) < FIRST_PSEUDO_REGISTER))
3956 0 : target = gen_reg_rtx (mode);
3957 :
3958 0 : emit_move_insn (target, op0);
3959 0 : NO_DEFER_POP;
3960 :
3961 0 : do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3962 : NULL_RTX, NULL, op1,
3963 : profile_probability::uninitialized ());
3964 :
3965 0 : op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3966 : target, target, 0);
3967 0 : if (op0 != target)
3968 0 : emit_move_insn (target, op0);
3969 0 : emit_label (op1);
3970 0 : OK_DEFER_POP;
3971 0 : return target;
3972 : }
3973 :
3974 : /* Emit code to compute the one's complement absolute value of OP0
3975 : (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3976 : (TARGET may be NULL_RTX.) The return value says where the result
3977 : actually is to be found.
3978 :
3979 : MODE is the mode of the operand; the mode of the result is
3980 : different but can be deduced from MODE. */
3981 :
3982 : rtx
3983 0 : expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3984 : {
3985 0 : rtx temp;
3986 :
3987 : /* Not applicable for floating point modes. */
3988 0 : if (FLOAT_MODE_P (mode))
3989 : return NULL_RTX;
3990 :
3991 : /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3992 0 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3993 : {
3994 0 : rtx_insn *last = get_last_insn ();
3995 :
3996 0 : temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3997 0 : if (temp != 0)
3998 0 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3999 : OPTAB_WIDEN);
4000 :
4001 0 : if (temp != 0)
4002 : return temp;
4003 :
4004 0 : delete_insns_since (last);
4005 : }
4006 :
4007 : /* If this machine has expensive jumps, we can do one's complement
4008 : absolute value of X as (((signed) x >> (W-1)) ^ x). */
4009 :
4010 0 : scalar_int_mode int_mode;
4011 0 : if (is_int_mode (mode, &int_mode)
4012 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
4013 : false) >= 2)
4014 : {
4015 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
4016 0 : GET_MODE_PRECISION (int_mode) - 1,
4017 : NULL_RTX, 0);
4018 :
4019 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
4020 : OPTAB_LIB_WIDEN);
4021 :
4022 0 : if (temp != 0)
4023 : return temp;
4024 : }
4025 :
4026 : return NULL_RTX;
4027 : }
4028 :
4029 : /* A subroutine of expand_copysign, perform the copysign operation using the
4030 : abs and neg primitives advertised to exist on the target. The assumption
4031 : is that we have a split register file, and leaving op0 in fp registers,
4032 : and not playing with subregs so much, will help the register allocator. */
4033 :
4034 : static rtx
4035 11608 : expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
4036 : int bitpos, bool op0_is_abs)
4037 : {
4038 11608 : scalar_int_mode imode;
4039 11608 : enum insn_code icode;
4040 11608 : rtx sign;
4041 11608 : rtx_code_label *label;
4042 :
4043 11608 : if (target == op1)
4044 132 : target = NULL_RTX;
4045 :
4046 : /* Check if the back end provides an insn that handles signbit for the
4047 : argument's mode. */
4048 11608 : icode = optab_handler (signbit_optab, mode);
4049 11608 : if (icode != CODE_FOR_nothing)
4050 : {
4051 11568 : imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
4052 11568 : sign = gen_reg_rtx (imode);
4053 11568 : emit_unop_insn (icode, sign, op1, UNKNOWN);
4054 : }
4055 : else
4056 : {
4057 100 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
4058 : {
4059 40 : if (!int_mode_for_mode (mode).exists (&imode))
4060 0 : return NULL_RTX;
4061 40 : op1 = gen_lowpart (imode, op1);
4062 : }
4063 : else
4064 : {
4065 0 : int word;
4066 :
4067 0 : imode = word_mode;
4068 0 : if (FLOAT_WORDS_BIG_ENDIAN)
4069 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
4070 : else
4071 0 : word = bitpos / BITS_PER_WORD;
4072 0 : bitpos = bitpos % BITS_PER_WORD;
4073 0 : op1 = operand_subword_force (op1, word, mode);
4074 : }
4075 :
4076 40 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
4077 80 : sign = expand_binop (imode, and_optab, op1,
4078 80 : immed_wide_int_const (mask, imode),
4079 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4080 40 : }
4081 :
4082 11608 : if (!op0_is_abs)
4083 : {
4084 154 : op0 = expand_unop (mode, abs_optab, op0, target, 0);
4085 154 : if (op0 == NULL)
4086 : return NULL_RTX;
4087 : target = op0;
4088 : }
4089 : else
4090 : {
4091 11454 : if (target == NULL_RTX)
4092 84 : target = copy_to_reg (op0);
4093 : else
4094 11370 : emit_move_insn (target, op0);
4095 : }
4096 :
4097 11608 : label = gen_label_rtx ();
4098 11608 : emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
4099 :
4100 11608 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
4101 11454 : op0 = simplify_unary_operation (NEG, mode, op0, mode);
4102 : else
4103 154 : op0 = expand_unop (mode, neg_optab, op0, target, 0);
4104 11608 : if (op0 != target)
4105 11454 : emit_move_insn (target, op0);
4106 :
4107 11608 : emit_label (label);
4108 :
4109 11608 : return target;
4110 : }
4111 :
4112 :
4113 : /* A subroutine of expand_copysign, perform the entire copysign operation
4114 : with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
4115 : is true if op0 is known to have its sign bit clear. */
4116 :
4117 : static rtx
4118 68 : expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
4119 : int bitpos, bool op0_is_abs)
4120 : {
4121 68 : scalar_int_mode imode;
4122 68 : int word, nwords, i;
4123 68 : rtx temp;
4124 68 : rtx_insn *insns;
4125 :
4126 156 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
4127 : {
4128 68 : if (!int_mode_for_mode (mode).exists (&imode))
4129 0 : return NULL_RTX;
4130 : word = 0;
4131 : nwords = 1;
4132 : }
4133 : else
4134 : {
4135 0 : imode = word_mode;
4136 :
4137 0 : if (FLOAT_WORDS_BIG_ENDIAN)
4138 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
4139 : else
4140 0 : word = bitpos / BITS_PER_WORD;
4141 0 : bitpos = bitpos % BITS_PER_WORD;
4142 0 : nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
4143 : }
4144 :
4145 68 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
4146 :
4147 68 : if (target == 0
4148 68 : || target == op0
4149 68 : || target == op1
4150 64 : || reg_overlap_mentioned_p (target, op0)
4151 64 : || reg_overlap_mentioned_p (target, op1)
4152 132 : || (nwords > 1 && !valid_multiword_target_p (target)))
4153 4 : target = gen_reg_rtx (mode);
4154 :
4155 68 : if (nwords > 1)
4156 : {
4157 0 : start_sequence ();
4158 :
4159 0 : for (i = 0; i < nwords; ++i)
4160 : {
4161 0 : rtx targ_piece = operand_subword (target, i, 1, mode);
4162 0 : rtx op0_piece = operand_subword_force (op0, i, mode);
4163 :
4164 0 : if (i == word)
4165 : {
4166 0 : if (!op0_is_abs)
4167 0 : op0_piece
4168 0 : = expand_binop (imode, and_optab, op0_piece,
4169 0 : immed_wide_int_const (~mask, imode),
4170 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4171 0 : op1 = expand_binop (imode, and_optab,
4172 0 : operand_subword_force (op1, i, mode),
4173 0 : immed_wide_int_const (mask, imode),
4174 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4175 :
4176 0 : temp = expand_binop (imode, ior_optab, op0_piece, op1,
4177 : targ_piece, 1, OPTAB_LIB_WIDEN);
4178 0 : if (temp != targ_piece)
4179 0 : emit_move_insn (targ_piece, temp);
4180 : }
4181 : else
4182 0 : emit_move_insn (targ_piece, op0_piece);
4183 : }
4184 :
4185 0 : insns = end_sequence ();
4186 :
4187 0 : emit_insn (insns);
4188 : }
4189 : else
4190 : {
4191 136 : op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
4192 68 : immed_wide_int_const (mask, imode),
4193 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4194 :
4195 68 : op0 = gen_lowpart (imode, op0);
4196 68 : if (!op0_is_abs)
4197 136 : op0 = expand_binop (imode, and_optab, op0,
4198 136 : immed_wide_int_const (~mask, imode),
4199 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4200 :
4201 68 : temp = expand_binop (imode, ior_optab, op0, op1,
4202 68 : gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
4203 68 : target = force_lowpart_subreg (mode, temp, imode);
4204 : }
4205 :
4206 68 : return target;
4207 68 : }
4208 :
4209 : /* Expand the C99 copysign operation. OP0 and OP1 must be the same
4210 : scalar floating point mode. Return NULL if we do not know how to
4211 : expand the operation inline. */
4212 :
4213 : rtx
4214 11676 : expand_copysign (rtx op0, rtx op1, rtx target)
4215 : {
4216 11676 : scalar_float_mode mode;
4217 11676 : const struct real_format *fmt;
4218 11676 : bool op0_is_abs;
4219 11676 : rtx temp;
4220 :
4221 11676 : mode = as_a <scalar_float_mode> (GET_MODE (op0));
4222 11676 : gcc_assert (GET_MODE (op1) == mode);
4223 :
4224 : /* First try to do it with a special instruction. */
4225 11676 : temp = expand_binop (mode, copysign_optab, op0, op1,
4226 : target, 0, OPTAB_DIRECT);
4227 11676 : if (temp)
4228 : return temp;
4229 :
4230 11676 : fmt = REAL_MODE_FORMAT (mode);
4231 11676 : if (fmt == NULL || !fmt->has_signed_zero)
4232 : return NULL_RTX;
4233 :
4234 11676 : op0_is_abs = false;
4235 11676 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
4236 : {
4237 11454 : if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
4238 0 : op0 = simplify_unary_operation (ABS, mode, op0, mode);
4239 : op0_is_abs = true;
4240 : }
4241 :
4242 11676 : if (fmt->signbit_ro >= 0
4243 11676 : && (CONST_DOUBLE_AS_FLOAT_P (op0)
4244 222 : || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
4245 154 : && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
4246 : {
4247 23216 : temp = expand_copysign_absneg (mode, op0, op1, target,
4248 11608 : fmt->signbit_ro, op0_is_abs);
4249 11608 : if (temp)
4250 : return temp;
4251 : }
4252 :
4253 68 : if (fmt->signbit_rw < 0)
4254 : return NULL_RTX;
4255 68 : return expand_copysign_bit (mode, op0, op1, target,
4256 68 : fmt->signbit_rw, op0_is_abs);
4257 : }
4258 :
4259 : /* Generate an instruction whose insn-code is INSN_CODE,
4260 : with two operands: an output TARGET and an input OP0.
4261 : TARGET *must* be nonzero, and the output is always stored there.
4262 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4263 : the value that is stored into TARGET.
4264 :
4265 : Return false if expansion failed. */
4266 :
4267 : bool
4268 2168195 : maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
4269 : enum rtx_code code)
4270 : {
4271 2168195 : class expand_operand ops[2];
4272 2168195 : rtx_insn *pat;
4273 :
4274 2168195 : create_output_operand (&ops[0], target, GET_MODE (target));
4275 2168195 : create_input_operand (&ops[1], op0, GET_MODE (op0));
4276 2168195 : pat = maybe_gen_insn (icode, 2, ops);
4277 2168195 : if (!pat)
4278 : return false;
4279 :
4280 1226839 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
4281 2192581 : && code != UNKNOWN)
4282 12666 : add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
4283 12666 : GET_MODE (op0));
4284 :
4285 2168194 : emit_insn (pat);
4286 :
4287 2168194 : if (ops[0].value != target)
4288 39178 : emit_move_insn (target, ops[0].value);
4289 : return true;
4290 : }
4291 : /* Generate an instruction whose insn-code is INSN_CODE,
4292 : with two operands: an output TARGET and an input OP0.
4293 : TARGET *must* be nonzero, and the output is always stored there.
4294 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4295 : the value that is stored into TARGET. */
4296 :
4297 : void
4298 2121160 : emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
4299 : {
4300 2121160 : bool ok = maybe_emit_unop_insn (icode, target, op0, code);
4301 2121160 : gcc_assert (ok);
4302 2121160 : }
4303 :
4304 : struct no_conflict_data
4305 : {
4306 : rtx target;
4307 : rtx_insn *first, *insn;
4308 : bool must_stay;
4309 : };
4310 :
4311 : /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4312 : the currently examined clobber / store has to stay in the list of
4313 : insns that constitute the actual libcall block. */
4314 : static void
4315 57305 : no_conflict_move_test (rtx dest, const_rtx set, void *p0)
4316 : {
4317 57305 : struct no_conflict_data *p= (struct no_conflict_data *) p0;
4318 :
4319 : /* If this inns directly contributes to setting the target, it must stay. */
4320 57305 : if (reg_overlap_mentioned_p (p->target, dest))
4321 0 : p->must_stay = true;
4322 : /* If we haven't committed to keeping any other insns in the list yet,
4323 : there is nothing more to check. */
4324 57305 : else if (p->insn == p->first)
4325 : return;
4326 : /* If this insn sets / clobbers a register that feeds one of the insns
4327 : already in the list, this insn has to stay too. */
4328 26963 : else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
4329 26963 : || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
4330 26963 : || reg_used_between_p (dest, p->first, p->insn)
4331 : /* Likewise if this insn depends on a register set by a previous
4332 : insn in the list, or if it sets a result (presumably a hard
4333 : register) that is set or clobbered by a previous insn.
4334 : N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4335 : SET_DEST perform the former check on the address, and the latter
4336 : check on the MEM. */
4337 53926 : || (GET_CODE (set) == SET
4338 26963 : && (modified_in_p (SET_SRC (set), p->first)
4339 26961 : || modified_in_p (SET_DEST (set), p->first)
4340 26961 : || modified_between_p (SET_SRC (set), p->first, p->insn)
4341 26961 : || modified_between_p (SET_DEST (set), p->first, p->insn))))
4342 2 : p->must_stay = true;
4343 : }
4344 :
4345 :
4346 : /* Emit code to make a call to a constant function or a library call.
4347 :
4348 : INSNS is a list containing all insns emitted in the call.
4349 : These insns leave the result in RESULT. Our block is to copy RESULT
4350 : to TARGET, which is logically equivalent to EQUIV.
4351 :
4352 : We first emit any insns that set a pseudo on the assumption that these are
4353 : loading constants into registers; doing so allows them to be safely cse'ed
4354 : between blocks. Then we emit all the other insns in the block, followed by
4355 : an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4356 : note with an operand of EQUIV. */
4357 :
4358 : static void
4359 123059 : emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
4360 : bool equiv_may_trap)
4361 : {
4362 123059 : rtx final_dest = target;
4363 123059 : rtx_insn *next, *last, *insn;
4364 :
4365 : /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4366 : into a MEM later. Protect the libcall block from this change. */
4367 123059 : if (! REG_P (target) || REG_USERVAR_P (target))
4368 1810 : target = gen_reg_rtx (GET_MODE (target));
4369 :
4370 : /* If we're using non-call exceptions, a libcall corresponding to an
4371 : operation that may trap may also trap. */
4372 : /* ??? See the comment in front of make_reg_eh_region_note. */
4373 123059 : if (cfun->can_throw_non_call_exceptions
4374 123059 : && (equiv_may_trap || may_trap_p (equiv)))
4375 : {
4376 1032 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4377 829 : if (CALL_P (insn))
4378 : {
4379 203 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
4380 203 : if (note)
4381 : {
4382 203 : int lp_nr = INTVAL (XEXP (note, 0));
4383 203 : if (lp_nr == 0 || lp_nr == INT_MIN)
4384 203 : remove_note (insn, note);
4385 : }
4386 : }
4387 : }
4388 : else
4389 : {
4390 : /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4391 : reg note to indicate that this call cannot throw or execute a nonlocal
4392 : goto (unless there is already a REG_EH_REGION note, in which case
4393 : we update it). */
4394 535423 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4395 412567 : if (CALL_P (insn))
4396 122856 : make_reg_eh_region_note_nothrow_nononlocal (insn);
4397 : }
4398 :
4399 : /* First emit all insns that set pseudos. Remove them from the list as
4400 : we go. Avoid insns that set pseudos which were referenced in previous
4401 : insns. These can be generated by move_by_pieces, for example,
4402 : to update an address. Similarly, avoid insns that reference things
4403 : set in previous insns. */
4404 :
4405 536455 : for (insn = insns; insn; insn = next)
4406 : {
4407 413396 : rtx set = single_set (insn);
4408 :
4409 413396 : next = NEXT_INSN (insn);
4410 :
4411 398880 : if (set != 0 && REG_P (SET_DEST (set))
4412 774972 : && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
4413 : {
4414 47214 : struct no_conflict_data data;
4415 :
4416 47214 : data.target = const0_rtx;
4417 47214 : data.first = insns;
4418 47214 : data.insn = insn;
4419 47214 : data.must_stay = 0;
4420 47214 : note_stores (insn, no_conflict_move_test, &data);
4421 47214 : if (! data.must_stay)
4422 : {
4423 47212 : if (PREV_INSN (insn))
4424 26961 : SET_NEXT_INSN (PREV_INSN (insn)) = next;
4425 : else
4426 : insns = next;
4427 :
4428 47212 : if (next)
4429 47212 : SET_PREV_INSN (next) = PREV_INSN (insn);
4430 :
4431 47212 : add_insn (insn);
4432 : }
4433 : }
4434 :
4435 : /* Some ports use a loop to copy large arguments onto the stack.
4436 : Don't move anything outside such a loop. */
4437 413396 : if (LABEL_P (insn))
4438 : break;
4439 : }
4440 :
4441 : /* Write the remaining insns followed by the final copy. */
4442 489243 : for (insn = insns; insn; insn = next)
4443 : {
4444 366184 : next = NEXT_INSN (insn);
4445 :
4446 366184 : add_insn (insn);
4447 : }
4448 :
4449 123059 : last = emit_move_insn (target, result);
4450 123059 : if (equiv)
4451 122476 : set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
4452 :
4453 123059 : if (final_dest != target)
4454 1810 : emit_move_insn (final_dest, target);
4455 123059 : }
4456 :
4457 : void
4458 91815 : emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
4459 : {
4460 91815 : emit_libcall_block_1 (insns, target, result, equiv, false);
4461 91815 : }
4462 :
4463 : /* True if we can perform a comparison of mode MODE straightforwardly.
4464 : PURPOSE describes how this comparison will be used. CODE is the rtx
4465 : comparison code we will be using.
4466 :
4467 : ??? Actually, CODE is slightly weaker than that. A target is still
4468 : required to implement all of the normal bcc operations, but not
4469 : required to implement all (or any) of the unordered bcc operations. */
4470 :
4471 : bool
4472 18269227 : can_compare_p (enum rtx_code code, machine_mode mode,
4473 : enum can_compare_purpose purpose)
4474 : {
4475 18269227 : rtx test;
4476 18269227 : test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
4477 20357539 : do
4478 : {
4479 20357539 : enum insn_code icode;
4480 :
4481 20357539 : if (purpose == ccp_jump
4482 20154766 : && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
4483 39152151 : && insn_operand_matches (icode, 0, test))
4484 : return true;
4485 3384900 : if (purpose == ccp_store_flag
4486 202773 : && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
4487 3518754 : && insn_operand_matches (icode, 1, test))
4488 : return true;
4489 :
4490 3380309 : mode = GET_MODE_WIDER_MODE (mode).else_void ();
4491 3380309 : PUT_MODE (test, mode);
4492 : }
4493 3380309 : while (mode != VOIDmode);
4494 :
4495 : return false;
4496 : }
4497 :
4498 : /* Return whether RTL code CODE corresponds to an unsigned optab. */
4499 :
4500 : static bool
4501 1467484 : unsigned_optab_p (enum rtx_code code)
4502 : {
4503 1467484 : return code == LTU || code == LEU || code == GTU || code == GEU;
4504 : }
4505 :
4506 : /* Return whether the backend-emitted comparison for code CODE, comparing
4507 : operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4508 : operand OPNO of pattern ICODE. */
4509 :
4510 : static bool
4511 1057242 : insn_predicate_matches_p (enum insn_code icode, unsigned int opno,
4512 : enum rtx_code code, machine_mode mask_mode,
4513 : machine_mode value_mode)
4514 : {
4515 1057242 : rtx reg1 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 1);
4516 1057242 : rtx reg2 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 2);
4517 1057242 : rtx test = alloca_rtx_fmt_ee (code, mask_mode, reg1, reg2);
4518 1057242 : return insn_operand_matches (icode, opno, test);
4519 : }
4520 :
4521 : /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4522 : for code CODE, comparing operands of mode VALUE_MODE and producing a result
4523 : with MASK_MODE. */
4524 :
4525 : bool
4526 1467484 : can_vec_cmp_compare_p (enum rtx_code code, machine_mode value_mode,
4527 : machine_mode mask_mode)
4528 : {
4529 1467484 : enum insn_code icode
4530 1467484 : = get_vec_cmp_icode (value_mode, mask_mode, unsigned_optab_p (code));
4531 1467484 : if (icode == CODE_FOR_nothing)
4532 : return false;
4533 :
4534 1057242 : return insn_predicate_matches_p (icode, 1, code, mask_mode, value_mode);
4535 : }
4536 :
4537 : /* Return whether the backend can emit vector set instructions for inserting
4538 : element into vector at variable index position. */
4539 :
4540 : bool
4541 207 : can_vec_set_var_idx_p (machine_mode vec_mode)
4542 : {
4543 207 : if (!VECTOR_MODE_P (vec_mode))
4544 : return false;
4545 :
4546 205 : machine_mode inner_mode = GET_MODE_INNER (vec_mode);
4547 :
4548 205 : rtx reg1 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 1);
4549 205 : rtx reg2 = alloca_raw_REG (inner_mode, LAST_VIRTUAL_REGISTER + 2);
4550 :
4551 205 : enum insn_code icode = optab_handler (vec_set_optab, vec_mode);
4552 :
4553 205 : const struct insn_data_d *data = &insn_data[icode];
4554 205 : machine_mode idx_mode = data->operand[2].mode;
4555 :
4556 205 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4557 :
4558 204 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4559 204 : && insn_operand_matches (icode, 1, reg2)
4560 409 : && insn_operand_matches (icode, 2, reg3);
4561 : }
4562 :
4563 : /* Return whether the backend can emit a vec_extract instruction with
4564 : a non-constant index. */
4565 : bool
4566 22085 : can_vec_extract_var_idx_p (machine_mode vec_mode, machine_mode extr_mode)
4567 : {
4568 22085 : if (!VECTOR_MODE_P (vec_mode))
4569 : return false;
4570 :
4571 17464 : rtx reg1 = alloca_raw_REG (extr_mode, LAST_VIRTUAL_REGISTER + 1);
4572 17464 : rtx reg2 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 2);
4573 :
4574 17464 : enum insn_code icode = convert_optab_handler (vec_extract_optab,
4575 : vec_mode, extr_mode);
4576 :
4577 17464 : const struct insn_data_d *data = &insn_data[icode];
4578 17464 : machine_mode idx_mode = data->operand[2].mode;
4579 :
4580 17464 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4581 :
4582 16598 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4583 16598 : && insn_operand_matches (icode, 1, reg2)
4584 34062 : && insn_operand_matches (icode, 2, reg3);
4585 : }
4586 :
4587 : /* This function is called when we are going to emit a compare instruction that
4588 : compares the values found in X and Y, using the rtl operator COMPARISON.
4589 :
4590 : If they have mode BLKmode, then SIZE specifies the size of both operands.
4591 :
4592 : UNSIGNEDP nonzero says that the operands are unsigned;
4593 : this matters if they need to be widened (as given by METHODS).
4594 :
4595 : *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4596 : if we failed to produce one.
4597 :
4598 : *PMODE is the mode of the inputs (in case they are const_int).
4599 :
4600 : *OPTAB is the optab to check for OPTAB_DIRECT support. Defaults to
4601 : cbranch_optab.
4602 :
4603 : This function performs all the setup necessary so that the caller only has
4604 : to emit a single comparison insn. This setup can involve doing a BLKmode
4605 : comparison or emitting a library call to perform the comparison if no insn
4606 : is available to handle it.
4607 : The values which are passed in through pointers can be modified; the caller
4608 : should perform the comparison on the modified values. Constant
4609 : comparisons must have already been folded. */
4610 :
4611 : static void
4612 6963705 : prepare_cmp_insn (rtx x, rtx y, rtx *mask, enum rtx_code comparison, rtx size,
4613 : int unsignedp, enum optab_methods methods,
4614 : rtx *ptest, machine_mode *pmode, optab optab)
4615 : {
4616 6963705 : machine_mode mode = *pmode;
4617 6963705 : rtx libfunc, test;
4618 6963705 : machine_mode cmp_mode;
4619 :
4620 : /* The other methods are not needed. */
4621 6963705 : gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4622 : || methods == OPTAB_LIB_WIDEN);
4623 :
4624 6963705 : if (CONST_SCALAR_INT_P (y))
4625 4506053 : canonicalize_comparison (mode, &comparison, &y);
4626 :
4627 : /* If we are optimizing, force expensive constants into a register. */
4628 1 : if (CONSTANT_P (x) && optimize
4629 1 : && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
4630 : > COSTS_N_INSNS (1))
4631 6963705 : && can_create_pseudo_p ())
4632 0 : x = force_reg (mode, x);
4633 :
4634 5023807 : if (CONSTANT_P (y) && optimize
4635 3973711 : && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
4636 : > COSTS_N_INSNS (1))
4637 7132593 : && can_create_pseudo_p ())
4638 168888 : y = force_reg (mode, y);
4639 :
4640 : /* Don't let both operands fail to indicate the mode. */
4641 6963705 : if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4642 0 : x = force_reg (mode, x);
4643 6963705 : if (mode == VOIDmode)
4644 5019 : mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4645 :
4646 : /* Handle all BLKmode compares. */
4647 :
4648 6963705 : if (mode == BLKmode)
4649 : {
4650 0 : machine_mode result_mode;
4651 0 : enum insn_code cmp_code;
4652 0 : rtx result;
4653 0 : rtx opalign
4654 0 : = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4655 :
4656 0 : gcc_assert (size);
4657 :
4658 : /* Try to use a memory block compare insn - either cmpstr
4659 : or cmpmem will do. */
4660 0 : opt_scalar_int_mode cmp_mode_iter;
4661 0 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
4662 : {
4663 0 : scalar_int_mode cmp_mode = cmp_mode_iter.require ();
4664 0 : cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4665 0 : if (cmp_code == CODE_FOR_nothing)
4666 0 : cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4667 0 : if (cmp_code == CODE_FOR_nothing)
4668 0 : cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4669 0 : if (cmp_code == CODE_FOR_nothing)
4670 0 : continue;
4671 :
4672 : /* Must make sure the size fits the insn's mode. */
4673 0 : if (CONST_INT_P (size)
4674 0 : ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
4675 0 : : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
4676 0 : > GET_MODE_BITSIZE (cmp_mode)))
4677 0 : continue;
4678 :
4679 0 : result_mode = insn_data[cmp_code].operand[0].mode;
4680 0 : result = gen_reg_rtx (result_mode);
4681 0 : size = convert_to_mode (cmp_mode, size, 1);
4682 0 : emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4683 :
4684 0 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4685 0 : *pmode = result_mode;
4686 0 : return;
4687 : }
4688 :
4689 0 : if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4690 0 : goto fail;
4691 :
4692 : /* Otherwise call a library function. */
4693 0 : result = emit_block_comp_via_libcall (x, y, size);
4694 :
4695 0 : x = result;
4696 0 : y = const0_rtx;
4697 0 : mode = TYPE_MODE (integer_type_node);
4698 0 : methods = OPTAB_LIB_WIDEN;
4699 0 : unsignedp = false;
4700 : }
4701 :
4702 : /* Don't allow operands to the compare to trap, as that can put the
4703 : compare and branch in different basic blocks. */
4704 6963705 : if (cfun->can_throw_non_call_exceptions)
4705 : {
4706 1133186 : if (!can_create_pseudo_p () && (may_trap_p (x) || may_trap_p (y)))
4707 0 : goto fail;
4708 1133186 : if (may_trap_p (x))
4709 24748 : x = copy_to_reg (x);
4710 1133186 : if (may_trap_p (y))
4711 1534 : y = copy_to_reg (y);
4712 : }
4713 :
4714 6963705 : if (GET_MODE_CLASS (mode) == MODE_CC)
4715 : {
4716 438 : enum insn_code icode = optab_handler (cbranch_optab, CCmode);
4717 438 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4718 438 : if (icode != CODE_FOR_nothing
4719 438 : && insn_operand_matches (icode, 0, test))
4720 : {
4721 438 : *ptest = test;
4722 438 : return;
4723 : }
4724 : else
4725 0 : goto fail;
4726 : }
4727 :
4728 6963267 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4729 7051355 : FOR_EACH_WIDER_MODE_FROM (cmp_mode, mode)
4730 : {
4731 7003448 : enum insn_code icode;
4732 7003448 : icode = optab_handler (optab, cmp_mode);
4733 7003448 : if (icode != CODE_FOR_nothing
4734 7003448 : && insn_operand_matches (icode, 0, test))
4735 : {
4736 6915360 : rtx_insn *last = get_last_insn ();
4737 6915360 : rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4738 6915360 : rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4739 6915360 : if (op0 && op1
4740 6915360 : && insn_operand_matches (icode, 1, op0)
4741 13830720 : && insn_operand_matches (icode, 2, op1))
4742 : {
4743 6915360 : XEXP (test, 0) = op0;
4744 6915360 : XEXP (test, 1) = op1;
4745 6915360 : *ptest = test;
4746 6915360 : *pmode = cmp_mode;
4747 6915360 : return;
4748 : }
4749 0 : delete_insns_since (last);
4750 : }
4751 :
4752 88088 : if (methods == OPTAB_DIRECT)
4753 : break;
4754 : }
4755 :
4756 47907 : if (methods != OPTAB_LIB_WIDEN)
4757 2929 : goto fail;
4758 :
4759 44978 : if (SCALAR_FLOAT_MODE_P (mode))
4760 : {
4761 : /* Small trick if UNORDERED isn't implemented by the hardware. */
4762 44978 : if (comparison == UNORDERED && rtx_equal_p (x, y))
4763 : {
4764 773 : prepare_cmp_insn (x, y, mask, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
4765 : ptest, pmode, optab);
4766 773 : if (*ptest)
4767 : return;
4768 : }
4769 :
4770 44978 : prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4771 : }
4772 : else
4773 : {
4774 0 : rtx result;
4775 0 : machine_mode ret_mode;
4776 :
4777 : /* Handle a libcall just for the mode we are using. */
4778 0 : libfunc = optab_libfunc (cmp_optab, mode);
4779 0 : gcc_assert (libfunc);
4780 :
4781 : /* If we want unsigned, and this mode has a distinct unsigned
4782 : comparison routine, use that. */
4783 0 : if (unsignedp)
4784 : {
4785 0 : rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4786 0 : if (ulibfunc)
4787 0 : libfunc = ulibfunc;
4788 : }
4789 :
4790 0 : ret_mode = targetm.libgcc_cmp_return_mode ();
4791 0 : result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4792 : ret_mode, x, mode, y, mode);
4793 :
4794 : /* There are two kinds of comparison routines. Biased routines
4795 : return 0/1/2, and unbiased routines return -1/0/1. Other parts
4796 : of gcc expect that the comparison operation is equivalent
4797 : to the modified comparison. For signed comparisons compare the
4798 : result against 1 in the biased case, and zero in the unbiased
4799 : case. For unsigned comparisons always compare against 1 after
4800 : biasing the unbiased result by adding 1. This gives us a way to
4801 : represent LTU.
4802 : The comparisons in the fixed-point helper library are always
4803 : biased. */
4804 0 : x = result;
4805 0 : y = const1_rtx;
4806 :
4807 0 : if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4808 : {
4809 : if (unsignedp)
4810 : x = plus_constant (ret_mode, result, 1);
4811 : else
4812 : y = const0_rtx;
4813 : }
4814 :
4815 0 : *pmode = ret_mode;
4816 0 : prepare_cmp_insn (x, y, mask, comparison, NULL_RTX, unsignedp, methods,
4817 : ptest, pmode, optab);
4818 : }
4819 :
4820 : return;
4821 :
4822 2929 : fail:
4823 2929 : *ptest = NULL_RTX;
4824 : }
4825 :
4826 : /* Before emitting an insn with code ICODE, make sure that X, which is going
4827 : to be used for operand OPNUM of the insn, is converted from mode MODE to
4828 : WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4829 : that it is accepted by the operand predicate. Return the new value. */
4830 :
4831 : rtx
4832 15338486 : prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4833 : machine_mode wider_mode, int unsignedp)
4834 : {
4835 15338486 : if (mode != wider_mode)
4836 10852 : x = convert_modes (wider_mode, mode, x, unsignedp);
4837 :
4838 15338486 : if (!insn_operand_matches (icode, opnum, x))
4839 : {
4840 64344 : machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4841 64344 : if (reload_completed)
4842 : return NULL_RTX;
4843 64344 : if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4844 : return NULL_RTX;
4845 64060 : x = copy_to_mode_reg (op_mode, x);
4846 : }
4847 :
4848 : return x;
4849 : }
4850 :
4851 : /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4852 : we can do the branch. */
4853 :
4854 : static void
4855 6456332 : emit_cmp_and_jump_insn_1 (rtx test, rtx cond, rtx len, rtx bias,
4856 : machine_mode mode, rtx label, direct_optab cmp_optab,
4857 : profile_probability prob, bool test_branch)
4858 : {
4859 6456332 : machine_mode optab_mode;
4860 6456332 : enum mode_class mclass;
4861 6456332 : enum insn_code icode;
4862 6456332 : rtx_insn *insn;
4863 :
4864 6456332 : mclass = GET_MODE_CLASS (mode);
4865 6456332 : optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4866 6456332 : icode = optab_handler (cmp_optab, optab_mode);
4867 :
4868 6456332 : gcc_assert (icode != CODE_FOR_nothing);
4869 6456332 : gcc_assert (test_branch || insn_operand_matches (icode, 0, test));
4870 6456332 : gcc_assert (cond == NULL_RTX || (cond != NULL_RTX && !test_branch));
4871 6456314 : if (test_branch)
4872 0 : insn = emit_jump_insn (GEN_FCN (icode) (XEXP (test, 0),
4873 0 : XEXP (test, 1), label));
4874 6456332 : else if (len)
4875 : {
4876 0 : gcc_assert (cond);
4877 0 : gcc_assert (bias);
4878 0 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4879 0 : XEXP (test, 1), len, bias,
4880 : label));
4881 : }
4882 6456332 : else if (cond)
4883 18 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4884 18 : XEXP (test, 1), label));
4885 : else
4886 6456314 : insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4887 6456314 : XEXP (test, 1), label));
4888 :
4889 6456332 : if (prob.initialized_p ()
4890 4899275 : && profile_status_for_fn (cfun) != PROFILE_ABSENT
4891 4791957 : && insn
4892 4791957 : && JUMP_P (insn)
4893 4791068 : && any_condjump_p (insn)
4894 11247400 : && !find_reg_note (insn, REG_BR_PROB, 0))
4895 4789590 : add_reg_br_prob_note (insn, prob);
4896 6456332 : }
4897 :
4898 : /* PTEST points to a comparison that compares its first operand with zero.
4899 : Check to see if it can be performed as a bit-test-and-branch instead.
4900 : On success, return the instruction that performs the bit-test-and-branch
4901 : and replace the second operand of *PTEST with the bit number to test.
4902 : On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
4903 :
4904 : Note that the comparison described by *PTEST should not be taken
4905 : literally after a successful return. *PTEST is just a convenient
4906 : place to store the two operands of the bit-and-test.
4907 :
4908 : VAL must contain the original tree expression for the first operand
4909 : of *PTEST. */
4910 :
4911 : static enum insn_code
4912 2646129 : validate_test_and_branch (tree val, rtx *ptest, machine_mode *pmode, optab *res)
4913 : {
4914 2646129 : if (!val || TREE_CODE (val) != SSA_NAME)
4915 : return CODE_FOR_nothing;
4916 :
4917 2541931 : machine_mode mode = TYPE_MODE (TREE_TYPE (val));
4918 2541931 : rtx test = *ptest;
4919 2541931 : direct_optab optab;
4920 :
4921 2541931 : if (GET_CODE (test) == EQ)
4922 : optab = tbranch_eq_optab;
4923 1325471 : else if (GET_CODE (test) == NE)
4924 : optab = tbranch_ne_optab;
4925 : else
4926 : return CODE_FOR_nothing;
4927 :
4928 2308175 : *res = optab;
4929 :
4930 : /* If the target supports the testbit comparison directly, great. */
4931 2308175 : auto icode = direct_optab_handler (optab, mode);
4932 2308175 : if (icode == CODE_FOR_nothing)
4933 : return icode;
4934 :
4935 0 : if (tree_zero_one_valued_p (val))
4936 : {
4937 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 : 0;
4938 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4939 0 : *ptest = test;
4940 0 : *pmode = mode;
4941 0 : return icode;
4942 : }
4943 :
4944 0 : wide_int wcst = get_nonzero_bits (val);
4945 0 : if (wcst == -1)
4946 : return CODE_FOR_nothing;
4947 :
4948 0 : int bitpos;
4949 :
4950 0 : if ((bitpos = wi::exact_log2 (wcst)) == -1)
4951 : return CODE_FOR_nothing;
4952 :
4953 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 - bitpos : bitpos;
4954 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4955 0 : *ptest = test;
4956 0 : *pmode = mode;
4957 0 : return icode;
4958 0 : }
4959 :
4960 : /* Generate code to compare X with Y so that the condition codes are
4961 : set and to jump to LABEL if the condition is true. If X is a
4962 : constant and Y is not a constant, then the comparison is swapped to
4963 : ensure that the comparison RTL has the canonical form.
4964 :
4965 : UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4966 : need to be widened. UNSIGNEDP is also used to select the proper
4967 : branch condition code.
4968 :
4969 : If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4970 :
4971 : MODE is the mode of the inputs (in case they are const_int).
4972 :
4973 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4974 : It will be potentially converted into an unsigned variant based on
4975 : UNSIGNEDP to select a proper jump instruction.
4976 :
4977 : PROB is the probability of jumping to LABEL. If the comparison is against
4978 : zero then VAL contains the expression from which the non-zero RTL is
4979 : derived. */
4980 :
4981 : void
4982 6456332 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4983 : machine_mode mode, int unsignedp, tree val, rtx label,
4984 : profile_probability prob)
4985 : {
4986 6456332 : rtx op0 = x, op1 = y;
4987 6456332 : rtx test;
4988 :
4989 : /* Swap operands and condition to ensure canonical RTL. */
4990 6456332 : if (swap_commutative_operands_p (x, y)
4991 6456332 : && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4992 : {
4993 0 : op0 = y, op1 = x;
4994 0 : comparison = swap_condition (comparison);
4995 : }
4996 :
4997 : /* If OP0 is still a constant, then both X and Y must be constants
4998 : or the opposite comparison is not supported. Force X into a register
4999 : to create canonical RTL. */
5000 6456332 : if (CONSTANT_P (op0))
5001 20250 : op0 = force_reg (mode, op0);
5002 :
5003 6456332 : if (unsignedp)
5004 3842006 : comparison = unsigned_condition (comparison);
5005 :
5006 : /* cbranch is no longer preferred for vectors, so when using a vector mode
5007 : check vec_cbranch variants instead. */
5008 6456332 : if (!VECTOR_MODE_P (GET_MODE (op0)))
5009 6436910 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5010 : OPTAB_LIB_WIDEN, &test, &mode, cbranch_optab);
5011 :
5012 : /* Check if we're comparing a truth type with 0, and if so check if
5013 : the target supports tbranch. */
5014 6456332 : machine_mode tmode = mode;
5015 6456332 : direct_optab optab;
5016 6456332 : if (op1 == CONST0_RTX (GET_MODE (op1)))
5017 : {
5018 2647754 : if (!VECTOR_MODE_P (GET_MODE (op1))
5019 2646129 : && validate_test_and_branch (val, &test, &tmode,
5020 : &optab) != CODE_FOR_nothing)
5021 : {
5022 0 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, tmode,
5023 : label, optab, prob, true);
5024 18 : return;
5025 : }
5026 :
5027 : /* If we are comparing equality with 0, check if VAL is another equality
5028 : comparison and if the target supports it directly. */
5029 2647754 : gimple *def_stmt = NULL;
5030 2544390 : if (val && TREE_CODE (val) == SSA_NAME
5031 2543525 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (val))
5032 2524 : && (comparison == NE || comparison == EQ)
5033 2650278 : && (def_stmt = get_gimple_for_ssa_name (val)))
5034 : {
5035 1783 : tree masked_op = NULL_TREE;
5036 1783 : tree len_op = NULL_TREE;
5037 1783 : tree len_bias = NULL_TREE;
5038 : /* First determine if the operation should be masked or unmasked. */
5039 1783 : if (is_gimple_assign (def_stmt)
5040 1783 : && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
5041 : {
5042 : /* See if one side if a comparison, if so use the other side as
5043 : the mask. */
5044 20 : gimple *mask_def = NULL;
5045 20 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
5046 20 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
5047 20 : if (TREE_CODE (rhs1) == SSA_NAME
5048 20 : && (mask_def = get_gimple_for_ssa_name (rhs1))
5049 20 : && is_gimple_assign (mask_def)
5050 40 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
5051 : masked_op = rhs2;
5052 0 : else if (TREE_CODE (rhs2) == SSA_NAME
5053 0 : && (mask_def = get_gimple_for_ssa_name (rhs2))
5054 0 : && is_gimple_assign (mask_def)
5055 0 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
5056 : masked_op = rhs1;
5057 :
5058 20 : if (masked_op)
5059 1783 : def_stmt = mask_def;
5060 : }
5061 : /* Else check to see if we're a LEN target. */
5062 1763 : else if (is_gimple_call (def_stmt)
5063 0 : && gimple_call_internal_p (def_stmt)
5064 1763 : && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
5065 : {
5066 : /* Example to consume:
5067 :
5068 : a = _59 != vect__4.17_75;
5069 : vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
5070 : if (vcmp != { 0, ... })
5071 :
5072 : and transform into
5073 :
5074 : if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)). */
5075 0 : gcall *call = dyn_cast <gcall *> (def_stmt);
5076 0 : tree true_branch = gimple_call_arg (call, 1);
5077 0 : tree false_branch = gimple_call_arg (call, 2);
5078 0 : if (integer_minus_onep (true_branch)
5079 0 : && integer_zerop (false_branch))
5080 : {
5081 0 : len_op = gimple_call_arg (call, 3);
5082 0 : len_bias = gimple_call_arg (call, 4);
5083 0 : tree arg0 = gimple_call_arg (call, 0);
5084 :
5085 0 : if (TREE_CODE (arg0) == SSA_NAME)
5086 0 : def_stmt = get_gimple_for_ssa_name (arg0);
5087 : }
5088 : }
5089 :
5090 1783 : enum insn_code icode;
5091 1783 : if (def_stmt
5092 1783 : && is_gimple_assign (def_stmt)
5093 3566 : && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
5094 : == tcc_comparison)
5095 : {
5096 1595 : class expand_operand ops[5];
5097 1595 : rtx_insn *tmp = NULL;
5098 1595 : start_sequence ();
5099 1595 : tree t_op0 = gimple_assign_rhs1 (def_stmt);
5100 1595 : tree t_op1 = gimple_assign_rhs2 (def_stmt);
5101 1595 : rtx op0c = expand_normal (t_op0);
5102 1595 : rtx op1c = expand_normal (t_op1);
5103 1595 : machine_mode mode2 = GET_MODE (op0c);
5104 :
5105 1595 : int nops = masked_op ? 3 : (len_op ? 5 : 2);
5106 1595 : int offset = masked_op || len_op ? 1 : 0;
5107 1595 : create_input_operand (&ops[offset + 0], op0c, mode2);
5108 1595 : create_input_operand (&ops[offset + 1], op1c, mode2);
5109 1595 : if (masked_op)
5110 : {
5111 18 : auto mask_mode = TYPE_MODE (TREE_TYPE (masked_op));
5112 18 : rtx mask_op = expand_normal (masked_op);
5113 18 : create_input_operand (&ops[0], mask_op, mask_mode);
5114 : }
5115 1577 : else if (len_op)
5116 : {
5117 0 : rtx len_rtx = expand_normal (len_op);
5118 0 : rtx len_bias_rtx = expand_normal (len_bias);
5119 0 : tree lhs = gimple_get_lhs (def_stmt);
5120 0 : auto mask_mode = TYPE_MODE (TREE_TYPE (lhs));
5121 : /* ??? We could use something like internal_fn's
5122 : add_mask_else_and_len_args here. Currently it
5123 : only supports a fixed, consecutive order of
5124 : mask and len, though. */
5125 0 : create_input_operand (&ops[0], CONSTM1_RTX (mask_mode),
5126 : mask_mode);
5127 0 : create_convert_operand_from
5128 0 : (&ops[3], len_rtx, TYPE_MODE (TREE_TYPE (len_op)),
5129 0 : TYPE_UNSIGNED (TREE_TYPE (len_op)));
5130 0 : create_input_operand (&ops[4], len_bias_rtx, QImode);
5131 : }
5132 :
5133 1595 : int unsignedp2 = TYPE_UNSIGNED (TREE_TYPE (t_op0));
5134 1595 : auto inner_code = gimple_assign_rhs_code (def_stmt);
5135 1595 : rtx test2 = NULL_RTX;
5136 :
5137 1595 : enum rtx_code comparison2 = get_rtx_code (inner_code, unsignedp2);
5138 1595 : if (unsignedp2)
5139 526 : comparison2 = unsigned_condition (comparison2);
5140 1595 : if (comparison == NE)
5141 1815 : optab = masked_op ? cond_vec_cbranch_any_optab
5142 902 : : len_op ? cond_len_vec_cbranch_any_optab
5143 : : vec_cbranch_any_optab;
5144 : else
5145 1357 : optab = masked_op ? cond_vec_cbranch_all_optab
5146 675 : : len_op ? cond_len_vec_cbranch_all_optab
5147 : : vec_cbranch_all_optab;
5148 :
5149 1595 : if ((icode = optab_handler (optab, mode2))
5150 : != CODE_FOR_nothing
5151 1595 : && maybe_legitimize_operands (icode, 1, nops, ops))
5152 : {
5153 18 : test2 = gen_rtx_fmt_ee (comparison2, VOIDmode,
5154 : ops[offset + 0].value,
5155 : ops[offset + 1].value);
5156 18 : if (insn_operand_matches (icode, 0, test2))
5157 : {
5158 36 : rtx mask
5159 18 : = (masked_op || len_op) ? ops[0].value : NULL_RTX;
5160 18 : rtx len = len_op ? ops[3].value : NULL_RTX;
5161 0 : rtx bias = len_op ? ops[4].value : NULL_RTX;
5162 18 : emit_cmp_and_jump_insn_1 (test2, mask, len, bias, mode2,
5163 : label, optab, prob, false);
5164 18 : tmp = get_insns ();
5165 : }
5166 : }
5167 :
5168 18 : end_sequence ();
5169 1595 : if (tmp)
5170 : {
5171 18 : emit_insn (tmp);
5172 18 : return;
5173 : }
5174 : }
5175 : }
5176 : }
5177 :
5178 : /* cbranch should only be used for VECTOR_BOOLEAN_TYPE_P values. */
5179 6456314 : direct_optab base_optab = cbranch_optab;
5180 6456314 : if (VECTOR_MODE_P (GET_MODE (op0)))
5181 : {
5182 : /* If cbranch is provided, use it. If we get here it means we have an
5183 : instruction in between what created the boolean value and the gcond
5184 : that is not a masking operation. This can happen for instance during
5185 : unrolling of early-break where we have an OR-reduction to reduce the
5186 : masks. In this case knowing we have a mask can let us generate better
5187 : code. If it's not there there then check the vector specific
5188 : optabs. */
5189 19422 : if (optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
5190 : {
5191 0 : if (comparison == NE)
5192 : base_optab = vec_cbranch_any_optab;
5193 : else
5194 0 : base_optab = vec_cbranch_all_optab;
5195 :
5196 0 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5197 : OPTAB_DIRECT, &test, &mode, base_optab);
5198 :
5199 0 : enum insn_code icode = optab_handler (base_optab, mode);
5200 :
5201 : /* If the new cbranch isn't supported, degrade back to old one. */
5202 0 : if (icode == CODE_FOR_nothing
5203 0 : || !test
5204 0 : || !insn_operand_matches (icode, 0, test))
5205 : base_optab = cbranch_optab;
5206 : }
5207 :
5208 19422 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5209 : OPTAB_LIB_WIDEN, &test, &mode, base_optab);
5210 : }
5211 :
5212 6456314 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, mode, label,
5213 : base_optab, prob, false);
5214 : }
5215 :
5216 : /* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown. */
5217 :
5218 : void
5219 116074 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
5220 : machine_mode mode, int unsignedp, rtx label,
5221 : profile_probability prob)
5222 : {
5223 116074 : emit_cmp_and_jump_insns (x, y, comparison, size, mode, unsignedp, NULL,
5224 : label, prob);
5225 116074 : }
5226 :
5227 :
5228 : /* Emit a library call comparison between floating point X and Y.
5229 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
5230 :
5231 : static void
5232 44978 : prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
5233 : rtx *ptest, machine_mode *pmode)
5234 : {
5235 44978 : enum rtx_code swapped = swap_condition (comparison);
5236 44978 : enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
5237 44978 : machine_mode orig_mode = GET_MODE (x);
5238 44978 : machine_mode mode;
5239 44978 : rtx true_rtx, false_rtx;
5240 44978 : rtx value, target, equiv;
5241 44978 : rtx_insn *insns;
5242 44978 : rtx libfunc = 0;
5243 44978 : bool reversed_p = false;
5244 44978 : scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
5245 :
5246 44978 : FOR_EACH_WIDER_MODE_FROM (mode, orig_mode)
5247 : {
5248 44978 : if (code_to_optab (comparison)
5249 44978 : && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
5250 : break;
5251 :
5252 0 : if (code_to_optab (swapped)
5253 0 : && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
5254 : {
5255 : std::swap (x, y);
5256 : comparison = swapped;
5257 : break;
5258 : }
5259 :
5260 0 : if (code_to_optab (reversed)
5261 0 : && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
5262 : {
5263 : comparison = reversed;
5264 : reversed_p = true;
5265 : break;
5266 : }
5267 : }
5268 :
5269 44978 : gcc_assert (mode != VOIDmode);
5270 :
5271 44978 : if (mode != orig_mode)
5272 : {
5273 0 : x = convert_to_mode (mode, x, 0);
5274 0 : y = convert_to_mode (mode, y, 0);
5275 : }
5276 :
5277 : /* Attach a REG_EQUAL note describing the semantics of the libcall to
5278 : the RTL. The allows the RTL optimizers to delete the libcall if the
5279 : condition can be determined at compile-time. */
5280 44978 : if (comparison == UNORDERED
5281 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5282 : {
5283 5809 : true_rtx = const_true_rtx;
5284 5809 : false_rtx = const0_rtx;
5285 : }
5286 : else
5287 : {
5288 39169 : switch (comparison)
5289 : {
5290 3515 : case EQ:
5291 3515 : true_rtx = const0_rtx;
5292 3515 : false_rtx = const_true_rtx;
5293 3515 : break;
5294 :
5295 23583 : case NE:
5296 23583 : true_rtx = const_true_rtx;
5297 23583 : false_rtx = const0_rtx;
5298 23583 : break;
5299 :
5300 3030 : case GT:
5301 3030 : true_rtx = const1_rtx;
5302 3030 : false_rtx = const0_rtx;
5303 3030 : break;
5304 :
5305 3025 : case GE:
5306 3025 : true_rtx = const0_rtx;
5307 3025 : false_rtx = constm1_rtx;
5308 3025 : break;
5309 :
5310 2487 : case LT:
5311 2487 : true_rtx = constm1_rtx;
5312 2487 : false_rtx = const0_rtx;
5313 2487 : break;
5314 :
5315 3529 : case LE:
5316 3529 : true_rtx = const0_rtx;
5317 3529 : false_rtx = const1_rtx;
5318 3529 : break;
5319 :
5320 0 : default:
5321 0 : gcc_unreachable ();
5322 : }
5323 : }
5324 :
5325 44978 : if (comparison == UNORDERED)
5326 : {
5327 5809 : rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
5328 5809 : equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
5329 5809 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5330 : temp, const_true_rtx, equiv);
5331 : }
5332 : else
5333 : {
5334 39169 : equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
5335 39169 : if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5336 39169 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5337 : equiv, true_rtx, false_rtx);
5338 : }
5339 :
5340 44978 : start_sequence ();
5341 44978 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5342 : cmp_mode, x, mode, y, mode);
5343 44978 : insns = end_sequence ();
5344 :
5345 44978 : target = gen_reg_rtx (cmp_mode);
5346 44978 : emit_libcall_block (insns, target, value, equiv);
5347 :
5348 44978 : if (comparison == UNORDERED
5349 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
5350 44978 : || reversed_p)
5351 11618 : *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
5352 : else
5353 39169 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
5354 :
5355 44978 : *pmode = cmp_mode;
5356 44978 : }
5357 :
5358 : /* Generate code to indirectly jump to a location given in the rtx LOC. */
5359 :
5360 : void
5361 1338 : emit_indirect_jump (rtx loc)
5362 : {
5363 1338 : if (!targetm.have_indirect_jump ())
5364 0 : sorry ("indirect jumps are not available on this target");
5365 : else
5366 : {
5367 1338 : class expand_operand ops[1];
5368 1338 : create_address_operand (&ops[0], loc);
5369 1338 : expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
5370 1338 : emit_barrier ();
5371 : }
5372 1338 : }
5373 :
5374 :
5375 : /* Emit a conditional move instruction if the machine supports one for that
5376 : condition and machine mode.
5377 :
5378 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5379 : the mode to use should they be constants. If it is VOIDmode, they cannot
5380 : both be constants.
5381 :
5382 : OP2 should be stored in TARGET if the comparison is true, otherwise OP3
5383 : should be stored there. MODE is the mode to use should they be constants.
5384 : If it is VOIDmode, they cannot both be constants.
5385 :
5386 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5387 : is not supported. */
5388 :
5389 : rtx
5390 476120 : emit_conditional_move (rtx target, struct rtx_comparison comp,
5391 : rtx op2, rtx op3,
5392 : machine_mode mode, int unsignedp)
5393 : {
5394 476120 : rtx comparison;
5395 476120 : rtx_insn *last;
5396 476120 : enum insn_code icode;
5397 476120 : enum rtx_code reversed;
5398 :
5399 : /* If the two source operands are identical, that's just a move. */
5400 :
5401 476120 : if (rtx_equal_p (op2, op3))
5402 : {
5403 2262 : if (!target)
5404 0 : target = gen_reg_rtx (mode);
5405 :
5406 2262 : emit_move_insn (target, op3);
5407 2262 : return target;
5408 : }
5409 :
5410 : /* If one operand is constant, make it the second one. Only do this
5411 : if the other operand is not constant as well. */
5412 :
5413 473858 : if (swap_commutative_operands_p (comp.op0, comp.op1))
5414 : {
5415 18096 : std::swap (comp.op0, comp.op1);
5416 18096 : comp.code = swap_condition (comp.code);
5417 : }
5418 :
5419 : /* get_condition will prefer to generate LT and GT even if the old
5420 : comparison was against zero, so undo that canonicalization here since
5421 : comparisons against zero are cheaper. */
5422 :
5423 473858 : if (comp.code == LT && comp.op1 == const1_rtx)
5424 3358 : comp.code = LE, comp.op1 = const0_rtx;
5425 470500 : else if (comp.code == GT && comp.op1 == constm1_rtx)
5426 7233 : comp.code = GE, comp.op1 = const0_rtx;
5427 :
5428 473858 : if (comp.mode == VOIDmode)
5429 387575 : comp.mode = GET_MODE (comp.op0);
5430 :
5431 473858 : enum rtx_code orig_code = comp.code;
5432 473858 : bool swapped = false;
5433 473858 : if (swap_commutative_operands_p (op2, op3)
5434 473858 : && ((reversed =
5435 123089 : reversed_comparison_code_parts (comp.code, comp.op0, comp.op1, NULL))
5436 : != UNKNOWN))
5437 : {
5438 : std::swap (op2, op3);
5439 : comp.code = reversed;
5440 : swapped = true;
5441 : }
5442 :
5443 473858 : if (mode == VOIDmode)
5444 78 : mode = GET_MODE (op2);
5445 :
5446 473858 : icode = direct_optab_handler (movcc_optab, mode);
5447 :
5448 473858 : if (icode == CODE_FOR_nothing)
5449 : return NULL_RTX;
5450 :
5451 465997 : if (!target)
5452 32 : target = gen_reg_rtx (mode);
5453 :
5454 465997 : for (int pass = 0; ; pass++)
5455 : {
5456 497131 : comp.code = unsignedp ? unsigned_condition (comp.code) : comp.code;
5457 994262 : comparison =
5458 497131 : simplify_gen_relational (comp.code, VOIDmode,
5459 : comp.mode, comp.op0, comp.op1);
5460 :
5461 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5462 : punt and let the caller figure out how best to deal with this
5463 : situation. */
5464 497131 : if (COMPARISON_P (comparison))
5465 : {
5466 496291 : saved_pending_stack_adjust save;
5467 496291 : save_pending_stack_adjust (&save);
5468 496291 : last = get_last_insn ();
5469 496291 : do_pending_stack_adjust ();
5470 496291 : machine_mode cmpmode = comp.mode;
5471 496291 : rtx orig_op0 = XEXP (comparison, 0);
5472 496291 : rtx orig_op1 = XEXP (comparison, 1);
5473 496291 : rtx op2p = op2;
5474 496291 : rtx op3p = op3;
5475 : /* If we are optimizing, force expensive constants into a register
5476 : but preserve an eventual equality with op2/op3. */
5477 1 : if (CONSTANT_P (orig_op0) && optimize
5478 1 : && cmpmode == mode
5479 1 : && (rtx_cost (orig_op0, mode, COMPARE, 0,
5480 : optimize_insn_for_speed_p ())
5481 : > COSTS_N_INSNS (1))
5482 496291 : && can_create_pseudo_p ())
5483 : {
5484 0 : if (rtx_equal_p (orig_op0, op2))
5485 0 : op2p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5486 0 : else if (rtx_equal_p (orig_op0, op3))
5487 0 : op3p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5488 : }
5489 276580 : if (CONSTANT_P (orig_op1) && optimize
5490 255489 : && cmpmode == mode
5491 89932 : && (rtx_cost (orig_op1, mode, COMPARE, 0,
5492 : optimize_insn_for_speed_p ())
5493 : > COSTS_N_INSNS (1))
5494 496468 : && can_create_pseudo_p ())
5495 : {
5496 177 : if (rtx_equal_p (orig_op1, op2))
5497 110 : op2p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5498 67 : else if (rtx_equal_p (orig_op1, op3))
5499 45 : op3p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5500 : }
5501 496291 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5502 496291 : GET_CODE (comparison), NULL_RTX, unsignedp,
5503 : OPTAB_WIDEN, &comparison, &cmpmode, cbranch_optab);
5504 496291 : if (comparison)
5505 : {
5506 494135 : rtx res = emit_conditional_move_1 (target, comparison,
5507 : op2p, op3p, mode);
5508 494135 : if (res != NULL_RTX)
5509 432253 : return res;
5510 : }
5511 64038 : delete_insns_since (last);
5512 64038 : restore_pending_stack_adjust (&save);
5513 : }
5514 :
5515 64878 : if (pass == 1)
5516 : return NULL_RTX;
5517 :
5518 : /* If the preferred op2/op3 order is not usable, retry with other
5519 : operand order, perhaps it will expand successfully. */
5520 33802 : if (swapped)
5521 : comp.code = orig_code;
5522 66850 : else if ((reversed =
5523 33425 : reversed_comparison_code_parts (orig_code, comp.op0, comp.op1,
5524 : NULL))
5525 : != UNKNOWN)
5526 : comp.code = reversed;
5527 : else
5528 : return NULL_RTX;
5529 31134 : std::swap (op2, op3);
5530 31134 : }
5531 : }
5532 :
5533 : /* Helper function that, in addition to COMPARISON, also tries
5534 : the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
5535 : to when we pass the specific constituents of a comparison, no
5536 : additional insns are emitted for it. It might still be necessary
5537 : to emit more than one insn for the final conditional move, though. */
5538 :
5539 : rtx
5540 106556 : emit_conditional_move (rtx target, rtx comparison, rtx rev_comparison,
5541 : rtx op2, rtx op3, machine_mode mode)
5542 : {
5543 106556 : rtx res = emit_conditional_move_1 (target, comparison, op2, op3, mode);
5544 :
5545 106556 : if (res != NULL_RTX)
5546 : return res;
5547 :
5548 5002 : return emit_conditional_move_1 (target, rev_comparison, op3, op2, mode);
5549 : }
5550 :
5551 : /* Helper for emitting a conditional move. */
5552 :
5553 : static rtx
5554 605693 : emit_conditional_move_1 (rtx target, rtx comparison,
5555 : rtx op2, rtx op3, machine_mode mode)
5556 : {
5557 605693 : enum insn_code icode;
5558 :
5559 605693 : if (comparison == NULL_RTX || !COMPARISON_P (comparison))
5560 : return NULL_RTX;
5561 :
5562 : /* If the two source operands are identical, that's just a move.
5563 : As the comparison comes in non-canonicalized, we must make
5564 : sure not to discard any possible side effects. If there are
5565 : side effects, just let the target handle it. */
5566 605693 : if (!side_effects_p (comparison) && rtx_equal_p (op2, op3))
5567 : {
5568 0 : if (!target)
5569 0 : target = gen_reg_rtx (mode);
5570 :
5571 0 : emit_move_insn (target, op3);
5572 0 : return target;
5573 : }
5574 :
5575 605693 : if (mode == VOIDmode)
5576 0 : mode = GET_MODE (op2);
5577 :
5578 605693 : icode = direct_optab_handler (movcc_optab, mode);
5579 :
5580 605693 : if (icode == CODE_FOR_nothing)
5581 : return NULL_RTX;
5582 :
5583 605693 : if (!target)
5584 0 : target = gen_reg_rtx (mode);
5585 :
5586 605693 : class expand_operand ops[4];
5587 :
5588 605693 : create_output_operand (&ops[0], target, mode);
5589 605693 : create_fixed_operand (&ops[1], comparison);
5590 605693 : create_input_operand (&ops[2], op2, mode);
5591 605693 : create_input_operand (&ops[3], op3, mode);
5592 :
5593 605693 : if (maybe_expand_insn (icode, 4, ops))
5594 : {
5595 533807 : if (ops[0].value != target)
5596 57 : convert_move (target, ops[0].value, false);
5597 : return target;
5598 : }
5599 :
5600 : return NULL_RTX;
5601 : }
5602 :
5603 :
5604 : /* Emit a conditional negate or bitwise complement using the
5605 : negcc or notcc optabs if available. Return NULL_RTX if such operations
5606 : are not available. Otherwise return the RTX holding the result.
5607 : TARGET is the desired destination of the result. COMP is the comparison
5608 : on which to negate. If COND is true move into TARGET the negation
5609 : or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5610 : CODE is either NEG or NOT. MODE is the machine mode in which the
5611 : operation is performed. */
5612 :
5613 : rtx
5614 1378 : emit_conditional_neg_or_complement (rtx target, rtx_code code,
5615 : machine_mode mode, rtx cond, rtx op1,
5616 : rtx op2)
5617 : {
5618 1378 : optab op = unknown_optab;
5619 1378 : if (code == NEG)
5620 : op = negcc_optab;
5621 68 : else if (code == NOT)
5622 : op = notcc_optab;
5623 : else
5624 0 : gcc_unreachable ();
5625 :
5626 1378 : insn_code icode = direct_optab_handler (op, mode);
5627 :
5628 1378 : if (icode == CODE_FOR_nothing)
5629 : return NULL_RTX;
5630 :
5631 0 : if (!target)
5632 0 : target = gen_reg_rtx (mode);
5633 :
5634 0 : rtx_insn *last = get_last_insn ();
5635 0 : class expand_operand ops[4];
5636 :
5637 0 : create_output_operand (&ops[0], target, mode);
5638 0 : create_fixed_operand (&ops[1], cond);
5639 0 : create_input_operand (&ops[2], op1, mode);
5640 0 : create_input_operand (&ops[3], op2, mode);
5641 :
5642 0 : if (maybe_expand_insn (icode, 4, ops))
5643 : {
5644 0 : if (ops[0].value != target)
5645 0 : convert_move (target, ops[0].value, false);
5646 :
5647 : return target;
5648 : }
5649 0 : delete_insns_since (last);
5650 0 : return NULL_RTX;
5651 : }
5652 :
5653 : /* Emit a conditional addition instruction if the machine supports one for that
5654 : condition and machine mode.
5655 :
5656 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5657 : the mode to use should they be constants. If it is VOIDmode, they cannot
5658 : both be constants.
5659 :
5660 : OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5661 : should be stored there. MODE is the mode to use should they be constants.
5662 : If it is VOIDmode, they cannot both be constants.
5663 :
5664 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5665 : is not supported. */
5666 :
5667 : rtx
5668 11029 : emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
5669 : machine_mode cmode, rtx op2, rtx op3,
5670 : machine_mode mode, int unsignedp)
5671 : {
5672 11029 : rtx comparison;
5673 11029 : rtx_insn *last;
5674 11029 : enum insn_code icode;
5675 :
5676 : /* If one operand is constant, make it the second one. Only do this
5677 : if the other operand is not constant as well. */
5678 :
5679 11029 : if (swap_commutative_operands_p (op0, op1))
5680 : {
5681 2 : std::swap (op0, op1);
5682 2 : code = swap_condition (code);
5683 : }
5684 :
5685 : /* get_condition will prefer to generate LT and GT even if the old
5686 : comparison was against zero, so undo that canonicalization here since
5687 : comparisons against zero are cheaper. */
5688 11029 : if (code == LT && op1 == const1_rtx)
5689 20 : code = LE, op1 = const0_rtx;
5690 11009 : else if (code == GT && op1 == constm1_rtx)
5691 67 : code = GE, op1 = const0_rtx;
5692 :
5693 11029 : if (cmode == VOIDmode)
5694 11029 : cmode = GET_MODE (op0);
5695 :
5696 11029 : if (mode == VOIDmode)
5697 0 : mode = GET_MODE (op2);
5698 :
5699 11029 : icode = optab_handler (addcc_optab, mode);
5700 :
5701 11029 : if (icode == CODE_FOR_nothing)
5702 : return 0;
5703 :
5704 10309 : if (!target)
5705 0 : target = gen_reg_rtx (mode);
5706 :
5707 10309 : code = unsignedp ? unsigned_condition (code) : code;
5708 10309 : comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
5709 :
5710 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5711 : return NULL and let the caller figure out how best to deal with this
5712 : situation. */
5713 10309 : if (!COMPARISON_P (comparison))
5714 : return NULL_RTX;
5715 :
5716 10309 : do_pending_stack_adjust ();
5717 10309 : last = get_last_insn ();
5718 10309 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5719 10309 : GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
5720 : &comparison, &cmode, cbranch_optab);
5721 10309 : if (comparison)
5722 : {
5723 10309 : class expand_operand ops[4];
5724 :
5725 10309 : create_output_operand (&ops[0], target, mode);
5726 10309 : create_fixed_operand (&ops[1], comparison);
5727 10309 : create_input_operand (&ops[2], op2, mode);
5728 10309 : create_input_operand (&ops[3], op3, mode);
5729 10309 : if (maybe_expand_insn (icode, 4, ops))
5730 : {
5731 342 : if (ops[0].value != target)
5732 0 : convert_move (target, ops[0].value, false);
5733 342 : return target;
5734 : }
5735 : }
5736 9967 : delete_insns_since (last);
5737 9967 : return NULL_RTX;
5738 : }
5739 :
5740 : /* These functions attempt to generate an insn body, rather than
5741 : emitting the insn, but if the gen function already emits them, we
5742 : make no attempt to turn them back into naked patterns. */
5743 :
5744 : /* Generate and return an insn body to add Y to X. */
5745 :
5746 : rtx_insn *
5747 250 : gen_add2_insn (rtx x, rtx y)
5748 : {
5749 250 : enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
5750 :
5751 250 : gcc_assert (insn_operand_matches (icode, 0, x));
5752 250 : gcc_assert (insn_operand_matches (icode, 1, x));
5753 250 : gcc_assert (insn_operand_matches (icode, 2, y));
5754 :
5755 250 : return GEN_FCN (icode) (x, x, y);
5756 : }
5757 :
5758 : /* Generate and return an insn body to add r1 and c,
5759 : storing the result in r0. */
5760 :
5761 : rtx_insn *
5762 0 : gen_add3_insn (rtx r0, rtx r1, rtx c)
5763 : {
5764 0 : enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
5765 :
5766 0 : if (icode == CODE_FOR_nothing
5767 0 : || !insn_operand_matches (icode, 0, r0)
5768 0 : || !insn_operand_matches (icode, 1, r1)
5769 0 : || !insn_operand_matches (icode, 2, c))
5770 : return NULL;
5771 :
5772 0 : return GEN_FCN (icode) (r0, r1, c);
5773 : }
5774 :
5775 : bool
5776 4301 : have_add2_insn (rtx x, rtx y)
5777 : {
5778 4301 : enum insn_code icode;
5779 :
5780 4301 : gcc_assert (GET_MODE (x) != VOIDmode);
5781 :
5782 4301 : icode = optab_handler (add_optab, GET_MODE (x));
5783 :
5784 4301 : if (icode == CODE_FOR_nothing)
5785 : return false;
5786 :
5787 4301 : if (!insn_operand_matches (icode, 0, x)
5788 4301 : || !insn_operand_matches (icode, 1, x)
5789 8602 : || !insn_operand_matches (icode, 2, y))
5790 212 : return false;
5791 :
5792 : return true;
5793 : }
5794 :
5795 : /* Generate and return an insn body to add Y to X. */
5796 :
5797 : rtx_insn *
5798 0 : gen_addptr3_insn (rtx x, rtx y, rtx z)
5799 : {
5800 0 : enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
5801 :
5802 0 : gcc_assert (insn_operand_matches (icode, 0, x));
5803 0 : gcc_assert (insn_operand_matches (icode, 1, y));
5804 0 : gcc_assert (insn_operand_matches (icode, 2, z));
5805 :
5806 0 : return GEN_FCN (icode) (x, y, z);
5807 : }
5808 :
5809 : /* Return true if the target implements an addptr pattern and X, Y,
5810 : and Z are valid for the pattern predicates. */
5811 :
5812 : bool
5813 692940 : have_addptr3_insn (rtx x, rtx y, rtx z)
5814 : {
5815 692940 : enum insn_code icode;
5816 :
5817 692940 : gcc_assert (GET_MODE (x) != VOIDmode);
5818 :
5819 692940 : icode = optab_handler (addptr3_optab, GET_MODE (x));
5820 :
5821 692940 : if (icode == CODE_FOR_nothing)
5822 : return false;
5823 :
5824 0 : if (!insn_operand_matches (icode, 0, x)
5825 0 : || !insn_operand_matches (icode, 1, y)
5826 0 : || !insn_operand_matches (icode, 2, z))
5827 0 : return false;
5828 :
5829 : return true;
5830 : }
5831 :
5832 : /* Generate and return an insn body to subtract Y from X. */
5833 :
5834 : rtx_insn *
5835 11 : gen_sub2_insn (rtx x, rtx y)
5836 : {
5837 11 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
5838 :
5839 11 : gcc_assert (insn_operand_matches (icode, 0, x));
5840 11 : gcc_assert (insn_operand_matches (icode, 1, x));
5841 11 : gcc_assert (insn_operand_matches (icode, 2, y));
5842 :
5843 11 : return GEN_FCN (icode) (x, x, y);
5844 : }
5845 :
5846 : /* Generate and return an insn body to subtract r1 and c,
5847 : storing the result in r0. */
5848 :
5849 : rtx_insn *
5850 1832 : gen_sub3_insn (rtx r0, rtx r1, rtx c)
5851 : {
5852 1832 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
5853 :
5854 1832 : if (icode == CODE_FOR_nothing
5855 1832 : || !insn_operand_matches (icode, 0, r0)
5856 1832 : || !insn_operand_matches (icode, 1, r1)
5857 3664 : || !insn_operand_matches (icode, 2, c))
5858 : return NULL;
5859 :
5860 1832 : return GEN_FCN (icode) (r0, r1, c);
5861 : }
5862 :
5863 : bool
5864 0 : have_sub2_insn (rtx x, rtx y)
5865 : {
5866 0 : enum insn_code icode;
5867 :
5868 0 : gcc_assert (GET_MODE (x) != VOIDmode);
5869 :
5870 0 : icode = optab_handler (sub_optab, GET_MODE (x));
5871 :
5872 0 : if (icode == CODE_FOR_nothing)
5873 : return false;
5874 :
5875 0 : if (!insn_operand_matches (icode, 0, x)
5876 0 : || !insn_operand_matches (icode, 1, x)
5877 0 : || !insn_operand_matches (icode, 2, y))
5878 0 : return false;
5879 :
5880 : return true;
5881 : }
5882 :
5883 : /* Generate the body of an insn to extend Y (with mode MFROM)
5884 : into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5885 :
5886 : rtx_insn *
5887 30441 : gen_extend_insn (rtx x, rtx y, machine_mode mto,
5888 : machine_mode mfrom, int unsignedp)
5889 : {
5890 30441 : enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
5891 30441 : return GEN_FCN (icode) (x, y);
5892 : }
5893 :
5894 : /* Generate code to convert FROM to floating point
5895 : and store in TO. FROM must be fixed point and not VOIDmode.
5896 : UNSIGNEDP nonzero means regard FROM as unsigned.
5897 : Normally this is done by correcting the final value
5898 : if it is negative. */
5899 :
5900 : void
5901 147722 : expand_float (rtx to, rtx from, int unsignedp)
5902 : {
5903 147722 : enum insn_code icode;
5904 147722 : rtx target = to;
5905 147722 : scalar_mode from_mode, to_mode;
5906 147722 : machine_mode fmode, imode;
5907 147722 : bool can_do_signed = false;
5908 :
5909 : /* Crash now, because we won't be able to decide which mode to use. */
5910 147722 : gcc_assert (GET_MODE (from) != VOIDmode);
5911 :
5912 : /* Look for an insn to do the conversion. Do it in the specified
5913 : modes if possible; otherwise convert either input, output or both to
5914 : wider mode. If the integer mode is wider than the mode of FROM,
5915 : we can do the conversion signed even if the input is unsigned. */
5916 :
5917 169667 : FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
5918 263967 : FOR_EACH_MODE_FROM (imode, GET_MODE (from))
5919 : {
5920 242022 : int doing_unsigned = unsignedp;
5921 :
5922 272280 : if (fmode != GET_MODE (to)
5923 242022 : && (significand_size (fmode)
5924 94428 : < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
5925 30258 : continue;
5926 :
5927 211764 : icode = can_float_p (fmode, imode, unsignedp);
5928 211764 : if (icode == CODE_FOR_nothing && unsignedp)
5929 : {
5930 35466 : enum insn_code scode = can_float_p (fmode, imode, 0);
5931 35466 : if (scode != CODE_FOR_nothing)
5932 6411 : can_do_signed = true;
5933 35466 : if (imode != GET_MODE (from))
5934 24411 : icode = scode, doing_unsigned = 0;
5935 : }
5936 :
5937 211764 : if (icode != CODE_FOR_nothing)
5938 : {
5939 139218 : if (imode != GET_MODE (from))
5940 10330 : from = convert_to_mode (imode, from, unsignedp);
5941 :
5942 139218 : if (fmode != GET_MODE (to))
5943 1645 : target = gen_reg_rtx (fmode);
5944 :
5945 272728 : emit_unop_insn (icode, target, from,
5946 : doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5947 :
5948 139218 : if (target != to)
5949 1645 : convert_move (to, target, 0);
5950 139218 : return;
5951 : }
5952 : }
5953 :
5954 : /* Unsigned integer, and no way to convert directly. Convert as signed,
5955 : then unconditionally adjust the result. */
5956 8504 : if (unsignedp
5957 8504 : && can_do_signed
5958 1104 : && is_a <scalar_mode> (GET_MODE (to), &to_mode)
5959 9608 : && is_a <scalar_mode> (GET_MODE (from), &from_mode))
5960 : {
5961 1104 : opt_scalar_mode fmode_iter;
5962 1104 : rtx_code_label *label = gen_label_rtx ();
5963 1104 : rtx temp;
5964 1104 : REAL_VALUE_TYPE offset;
5965 :
5966 : /* Look for a usable floating mode FMODE wider than the source and at
5967 : least as wide as the target. Using FMODE will avoid rounding woes
5968 : with unsigned values greater than the signed maximum value. */
5969 :
5970 2125 : FOR_EACH_MODE_FROM (fmode_iter, to_mode)
5971 : {
5972 2125 : scalar_mode fmode = fmode_iter.require ();
5973 3146 : if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
5974 2125 : && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
5975 : break;
5976 : }
5977 :
5978 1104 : if (!fmode_iter.exists (&fmode))
5979 : {
5980 : /* There is no such mode. Pretend the target is wide enough. */
5981 0 : fmode = to_mode;
5982 :
5983 : /* Avoid double-rounding when TO is narrower than FROM. */
5984 0 : if ((significand_size (fmode) + 1)
5985 0 : < GET_MODE_PRECISION (from_mode))
5986 : {
5987 0 : rtx temp1;
5988 0 : rtx_code_label *neglabel = gen_label_rtx ();
5989 :
5990 : /* Don't use TARGET if it isn't a register, is a hard register,
5991 : or is the wrong mode. */
5992 0 : if (!REG_P (target)
5993 0 : || REGNO (target) < FIRST_PSEUDO_REGISTER
5994 0 : || GET_MODE (target) != fmode)
5995 0 : target = gen_reg_rtx (fmode);
5996 :
5997 0 : imode = from_mode;
5998 0 : do_pending_stack_adjust ();
5999 :
6000 : /* Test whether the sign bit is set. */
6001 0 : emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
6002 : 0, neglabel);
6003 :
6004 : /* The sign bit is not set. Convert as signed. */
6005 0 : expand_float (target, from, 0);
6006 0 : emit_jump_insn (targetm.gen_jump (label));
6007 0 : emit_barrier ();
6008 :
6009 : /* The sign bit is set.
6010 : Convert to a usable (positive signed) value by shifting right
6011 : one bit, while remembering if a nonzero bit was shifted
6012 : out; i.e., compute (from & 1) | (from >> 1). */
6013 :
6014 0 : emit_label (neglabel);
6015 0 : temp = expand_binop (imode, and_optab, from, const1_rtx,
6016 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
6017 0 : temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
6018 0 : temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
6019 : OPTAB_LIB_WIDEN);
6020 0 : expand_float (target, temp, 0);
6021 :
6022 : /* Multiply by 2 to undo the shift above. */
6023 0 : temp = expand_binop (fmode, add_optab, target, target,
6024 : target, 0, OPTAB_LIB_WIDEN);
6025 0 : if (temp != target)
6026 0 : emit_move_insn (target, temp);
6027 :
6028 0 : do_pending_stack_adjust ();
6029 0 : emit_label (label);
6030 0 : goto done;
6031 : }
6032 : }
6033 :
6034 : /* If we are about to do some arithmetic to correct for an
6035 : unsigned operand, do it in a pseudo-register. */
6036 :
6037 1104 : if (to_mode != fmode
6038 1104 : || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
6039 403 : target = gen_reg_rtx (fmode);
6040 :
6041 : /* Convert as signed integer to floating. */
6042 1104 : expand_float (target, from, 0);
6043 :
6044 : /* If FROM is negative (and therefore TO is negative),
6045 : correct its value by 2**bitwidth. */
6046 :
6047 1104 : do_pending_stack_adjust ();
6048 1104 : emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
6049 : 0, label);
6050 :
6051 :
6052 1104 : real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
6053 1104 : temp = expand_binop (fmode, add_optab, target,
6054 : const_double_from_real_value (offset, fmode),
6055 : target, 0, OPTAB_LIB_WIDEN);
6056 1104 : if (temp != target)
6057 0 : emit_move_insn (target, temp);
6058 :
6059 1104 : do_pending_stack_adjust ();
6060 1104 : emit_label (label);
6061 1104 : goto done;
6062 : }
6063 :
6064 : /* No hardware instruction available; call a library routine. */
6065 7400 : {
6066 7400 : rtx libfunc;
6067 7400 : rtx_insn *insns;
6068 7400 : rtx value;
6069 7400 : convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
6070 :
6071 7400 : if (is_narrower_int_mode (GET_MODE (from), SImode))
6072 978 : from = convert_to_mode (SImode, from, unsignedp);
6073 :
6074 7400 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
6075 7400 : gcc_assert (libfunc);
6076 :
6077 7400 : start_sequence ();
6078 :
6079 14800 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
6080 7400 : GET_MODE (to), from, GET_MODE (from));
6081 7400 : insns = end_sequence ();
6082 :
6083 7400 : emit_libcall_block (insns, target, value,
6084 7400 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
6085 : GET_MODE (to), from));
6086 : }
6087 :
6088 8504 : done:
6089 :
6090 : /* Copy result to requested destination
6091 : if we have been computing in a temp location. */
6092 :
6093 8504 : if (target != to)
6094 : {
6095 403 : if (GET_MODE (target) == GET_MODE (to))
6096 4 : emit_move_insn (to, target);
6097 : else
6098 399 : convert_move (to, target, 0);
6099 : }
6100 : }
6101 :
6102 : /* Generate code to convert FROM to fixed point and store in TO. FROM
6103 : must be floating point. */
6104 :
6105 : void
6106 59926 : expand_fix (rtx to, rtx from, int unsignedp)
6107 : {
6108 59926 : enum insn_code icode;
6109 59926 : rtx target = to;
6110 59926 : machine_mode fmode, imode;
6111 59926 : opt_scalar_mode fmode_iter;
6112 59926 : bool must_trunc = false;
6113 :
6114 : /* We first try to find a pair of modes, one real and one integer, at
6115 : least as wide as FROM and TO, respectively, in which we can open-code
6116 : this conversion. If the integer mode is wider than the mode of TO,
6117 : we can do the conversion either signed or unsigned. */
6118 :
6119 90537 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
6120 216311 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
6121 : {
6122 185700 : int doing_unsigned = unsignedp;
6123 :
6124 185700 : icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
6125 185700 : if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
6126 62664 : icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
6127 :
6128 142052 : if (icode != CODE_FOR_nothing)
6129 : {
6130 46829 : rtx_insn *last = get_last_insn ();
6131 46829 : rtx from1 = from;
6132 46829 : if (fmode != GET_MODE (from))
6133 : {
6134 1114 : if (REAL_MODE_FORMAT (GET_MODE (from))
6135 : == &arm_bfloat_half_format
6136 1114 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
6137 : /* The BF -> SF conversions can be just a shift, doesn't
6138 : need to handle sNANs. */
6139 : {
6140 46 : int save_flag_finite_math_only = flag_finite_math_only;
6141 46 : flag_finite_math_only = true;
6142 46 : from1 = convert_to_mode (fmode, from, 0);
6143 46 : flag_finite_math_only = save_flag_finite_math_only;
6144 : }
6145 : else
6146 1068 : from1 = convert_to_mode (fmode, from, 0);
6147 : }
6148 :
6149 46829 : if (must_trunc)
6150 : {
6151 0 : rtx temp = gen_reg_rtx (GET_MODE (from1));
6152 0 : from1 = expand_unop (GET_MODE (from1), ftrunc_optab, from1,
6153 : temp, 0);
6154 : }
6155 :
6156 46829 : if (imode != GET_MODE (to))
6157 10792 : target = gen_reg_rtx (imode);
6158 :
6159 89398 : if (maybe_emit_unop_insn (icode, target, from1,
6160 : doing_unsigned ? UNSIGNED_FIX : FIX))
6161 : {
6162 46829 : if (target != to)
6163 10792 : convert_move (to, target, unsignedp);
6164 50922 : return;
6165 : }
6166 0 : delete_insns_since (last);
6167 : }
6168 : }
6169 :
6170 : /* For an unsigned conversion, there is one more way to do it.
6171 : If we have a signed conversion, we generate code that compares
6172 : the real value to the largest representable positive number. If if
6173 : is smaller, the conversion is done normally. Otherwise, subtract
6174 : one plus the highest signed number, convert, and add it back.
6175 :
6176 : We only need to check all real modes, since we know we didn't find
6177 : anything with a wider integer mode.
6178 :
6179 : This code used to extend FP value into mode wider than the destination.
6180 : This is needed for decimal float modes which cannot accurately
6181 : represent one plus the highest signed number of the same size, but
6182 : not for binary modes. Consider, for instance conversion from SFmode
6183 : into DImode.
6184 :
6185 : The hot path through the code is dealing with inputs smaller than 2^63
6186 : and doing just the conversion, so there is no bits to lose.
6187 :
6188 : In the other path we know the value is positive in the range 2^63..2^64-1
6189 : inclusive. (as for other input overflow happens and result is undefined)
6190 : So we know that the most important bit set in mantissa corresponds to
6191 : 2^63. The subtraction of 2^63 should not generate any rounding as it
6192 : simply clears out that bit. The rest is trivial. */
6193 :
6194 13097 : scalar_int_mode to_mode;
6195 13097 : if (unsignedp
6196 7469 : && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
6197 20566 : && HWI_COMPUTABLE_MODE_P (to_mode))
6198 9095 : FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
6199 : {
6200 6890 : scalar_mode fmode = fmode_iter.require ();
6201 6890 : if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
6202 : 0, &must_trunc)
6203 6890 : && (!DECIMAL_FLOAT_MODE_P (fmode)
6204 0 : || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
6205 : {
6206 4087 : int bitsize;
6207 4087 : REAL_VALUE_TYPE offset;
6208 4087 : rtx limit;
6209 4087 : rtx_code_label *lab1, *lab2;
6210 4087 : rtx_insn *insn;
6211 :
6212 4087 : bitsize = GET_MODE_PRECISION (to_mode);
6213 4087 : real_2expN (&offset, bitsize - 1, fmode);
6214 4087 : limit = const_double_from_real_value (offset, fmode);
6215 4087 : lab1 = gen_label_rtx ();
6216 4087 : lab2 = gen_label_rtx ();
6217 :
6218 4087 : if (fmode != GET_MODE (from))
6219 : {
6220 188 : if (REAL_MODE_FORMAT (GET_MODE (from))
6221 : == &arm_bfloat_half_format
6222 188 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
6223 : /* The BF -> SF conversions can be just a shift, doesn't
6224 : need to handle sNANs. */
6225 : {
6226 6 : int save_flag_finite_math_only = flag_finite_math_only;
6227 6 : flag_finite_math_only = true;
6228 6 : from = convert_to_mode (fmode, from, 0);
6229 6 : flag_finite_math_only = save_flag_finite_math_only;
6230 : }
6231 : else
6232 182 : from = convert_to_mode (fmode, from, 0);
6233 : }
6234 :
6235 : /* See if we need to do the subtraction. */
6236 4087 : do_pending_stack_adjust ();
6237 4087 : emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
6238 4087 : GET_MODE (from), 0, lab1);
6239 :
6240 : /* If not, do the signed "fix" and branch around fixup code. */
6241 4087 : expand_fix (to, from, 0);
6242 4087 : emit_jump_insn (targetm.gen_jump (lab2));
6243 4087 : emit_barrier ();
6244 :
6245 : /* Otherwise, subtract 2**(N-1), convert to signed number,
6246 : then add 2**(N-1). Do the addition using XOR since this
6247 : will often generate better code. */
6248 4087 : emit_label (lab1);
6249 4087 : target = expand_binop (GET_MODE (from), sub_optab, from, limit,
6250 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
6251 4087 : expand_fix (to, target, 0);
6252 4087 : target = expand_binop (to_mode, xor_optab, to,
6253 : gen_int_mode
6254 4087 : (HOST_WIDE_INT_1 << (bitsize - 1),
6255 : to_mode),
6256 : to, 1, OPTAB_LIB_WIDEN);
6257 :
6258 4087 : if (target != to)
6259 0 : emit_move_insn (to, target);
6260 :
6261 4087 : emit_label (lab2);
6262 :
6263 4087 : if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
6264 : {
6265 : /* Make a place for a REG_NOTE and add it. */
6266 4087 : insn = emit_move_insn (to, to);
6267 4087 : set_dst_reg_note (insn, REG_EQUAL,
6268 : gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
6269 : copy_rtx (from)),
6270 : to);
6271 : }
6272 :
6273 4087 : return;
6274 : }
6275 : }
6276 :
6277 : #ifdef HAVE_SFmode
6278 9010 : if (REAL_MODE_FORMAT (GET_MODE (from)) == &arm_bfloat_half_format
6279 9010 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
6280 : /* We don't have BF -> TI library functions, use BF -> SF -> TI
6281 : instead but the BF -> SF conversion can be just a shift, doesn't
6282 : need to handle sNANs. */
6283 : {
6284 6 : int save_flag_finite_math_only = flag_finite_math_only;
6285 6 : flag_finite_math_only = true;
6286 6 : from = convert_to_mode (SFmode, from, 0);
6287 6 : flag_finite_math_only = save_flag_finite_math_only;
6288 6 : expand_fix (to, from, unsignedp);
6289 6 : return;
6290 : }
6291 : #endif
6292 :
6293 : /* We can't do it with an insn, so use a library call. But first ensure
6294 : that the mode of TO is at least as wide as SImode, since those are the
6295 : only library calls we know about. */
6296 :
6297 9004 : if (is_narrower_int_mode (GET_MODE (to), SImode))
6298 : {
6299 1958 : target = gen_reg_rtx (SImode);
6300 :
6301 1958 : expand_fix (target, from, unsignedp);
6302 : }
6303 : else
6304 : {
6305 7046 : rtx_insn *insns;
6306 7046 : rtx value;
6307 7046 : rtx libfunc;
6308 :
6309 7046 : convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
6310 7046 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
6311 7046 : gcc_assert (libfunc);
6312 :
6313 7046 : start_sequence ();
6314 :
6315 14092 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
6316 7046 : GET_MODE (to), from, GET_MODE (from));
6317 7046 : insns = end_sequence ();
6318 :
6319 7046 : emit_libcall_block (insns, target, value,
6320 7046 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
6321 : GET_MODE (to), from));
6322 : }
6323 :
6324 9004 : if (target != to)
6325 : {
6326 1958 : if (GET_MODE (to) == GET_MODE (target))
6327 0 : emit_move_insn (to, target);
6328 : else
6329 1958 : convert_move (to, target, 0);
6330 : }
6331 : }
6332 :
6333 :
6334 : /* Promote integer arguments for a libcall if necessary.
6335 : emit_library_call_value cannot do the promotion because it does not
6336 : know if it should do a signed or unsigned promotion. This is because
6337 : there are no tree types defined for libcalls. */
6338 :
6339 : static rtx
6340 0 : prepare_libcall_arg (rtx arg, int uintp)
6341 : {
6342 0 : scalar_int_mode mode;
6343 0 : machine_mode arg_mode;
6344 0 : if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
6345 : {
6346 : /* If we need to promote the integer function argument we need to do
6347 : it here instead of inside emit_library_call_value because in
6348 : emit_library_call_value we don't know if we should do a signed or
6349 : unsigned promotion. */
6350 :
6351 0 : int unsigned_p = 0;
6352 0 : arg_mode = promote_function_mode (NULL_TREE, mode,
6353 : &unsigned_p, NULL_TREE, 0);
6354 0 : if (arg_mode != mode)
6355 0 : return convert_to_mode (arg_mode, arg, uintp);
6356 : }
6357 : return arg;
6358 : }
6359 :
6360 : /* Generate code to convert FROM or TO a fixed-point.
6361 : If UINTP is true, either TO or FROM is an unsigned integer.
6362 : If SATP is true, we need to saturate the result. */
6363 :
6364 : void
6365 0 : expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
6366 : {
6367 0 : machine_mode to_mode = GET_MODE (to);
6368 0 : machine_mode from_mode = GET_MODE (from);
6369 0 : convert_optab tab;
6370 0 : enum rtx_code this_code;
6371 0 : enum insn_code code;
6372 0 : rtx_insn *insns;
6373 0 : rtx value;
6374 0 : rtx libfunc;
6375 :
6376 0 : if (to_mode == from_mode)
6377 : {
6378 0 : emit_move_insn (to, from);
6379 0 : return;
6380 : }
6381 :
6382 0 : if (uintp)
6383 : {
6384 0 : tab = satp ? satfractuns_optab : fractuns_optab;
6385 0 : this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
6386 : }
6387 : else
6388 : {
6389 0 : tab = satp ? satfract_optab : fract_optab;
6390 0 : this_code = satp ? SAT_FRACT : FRACT_CONVERT;
6391 : }
6392 0 : code = convert_optab_handler (tab, to_mode, from_mode);
6393 0 : if (code != CODE_FOR_nothing)
6394 : {
6395 0 : emit_unop_insn (code, to, from, this_code);
6396 0 : return;
6397 : }
6398 :
6399 0 : libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
6400 0 : gcc_assert (libfunc);
6401 :
6402 0 : from = prepare_libcall_arg (from, uintp);
6403 0 : from_mode = GET_MODE (from);
6404 :
6405 0 : start_sequence ();
6406 0 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
6407 : from, from_mode);
6408 0 : insns = end_sequence ();
6409 :
6410 0 : emit_libcall_block (insns, to, value,
6411 : gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
6412 : }
6413 :
6414 : /* Generate code to convert FROM to fixed point and store in TO. FROM
6415 : must be floating point, TO must be signed. Use the conversion optab
6416 : TAB to do the conversion. */
6417 :
6418 : bool
6419 473 : expand_sfix_optab (rtx to, rtx from, convert_optab tab)
6420 : {
6421 473 : enum insn_code icode;
6422 473 : rtx target = to;
6423 473 : machine_mode fmode, imode;
6424 :
6425 : /* We first try to find a pair of modes, one real and one integer, at
6426 : least as wide as FROM and TO, respectively, in which we can open-code
6427 : this conversion. If the integer mode is wider than the mode of TO,
6428 : we can do the conversion either signed or unsigned. */
6429 :
6430 1973 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
6431 8450 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
6432 : {
6433 6950 : icode = convert_optab_handler (tab, imode, fmode,
6434 : insn_optimization_type ());
6435 6950 : if (icode != CODE_FOR_nothing)
6436 : {
6437 52 : rtx_insn *last = get_last_insn ();
6438 52 : if (fmode != GET_MODE (from))
6439 52 : from = convert_to_mode (fmode, from, 0);
6440 :
6441 52 : if (imode != GET_MODE (to))
6442 0 : target = gen_reg_rtx (imode);
6443 :
6444 52 : if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
6445 : {
6446 0 : delete_insns_since (last);
6447 0 : continue;
6448 : }
6449 52 : if (target != to)
6450 0 : convert_move (to, target, 0);
6451 : return true;
6452 : }
6453 : }
6454 :
6455 : return false;
6456 : }
6457 :
6458 : /* Report whether we have an instruction to perform the operation
6459 : specified by CODE on operands of mode MODE. */
6460 : bool
6461 98517420 : have_insn_for (enum rtx_code code, machine_mode mode)
6462 : {
6463 98517420 : return (code_to_optab (code)
6464 98517420 : && (optab_handler (code_to_optab (code), mode)
6465 98517420 : != CODE_FOR_nothing));
6466 : }
6467 :
6468 : /* Print information about the current contents of the optabs on
6469 : STDERR. */
6470 :
6471 : DEBUG_FUNCTION void
6472 0 : debug_optab_libfuncs (void)
6473 : {
6474 0 : int i, j, k;
6475 :
6476 : /* Dump the arithmetic optabs. */
6477 0 : for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6478 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6479 : {
6480 0 : rtx l = optab_libfunc ((optab) i, (machine_mode) j);
6481 0 : if (l)
6482 : {
6483 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6484 0 : fprintf (stderr, "%s\t%s:\t%s\n",
6485 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6486 0 : GET_MODE_NAME (j),
6487 : XSTR (l, 0));
6488 : }
6489 : }
6490 :
6491 : /* Dump the conversion optabs. */
6492 0 : for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6493 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6494 0 : for (k = 0; k < NUM_MACHINE_MODES; ++k)
6495 : {
6496 0 : rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
6497 : (machine_mode) k);
6498 0 : if (l)
6499 : {
6500 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6501 0 : fprintf (stderr, "%s\t%s\t%s:\t%s\n",
6502 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6503 0 : GET_MODE_NAME (j),
6504 0 : GET_MODE_NAME (k),
6505 : XSTR (l, 0));
6506 : }
6507 : }
6508 0 : }
6509 :
6510 : /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6511 : CODE. Return 0 on failure. */
6512 :
6513 : rtx_insn *
6514 0 : gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6515 : {
6516 0 : machine_mode mode = GET_MODE (op1);
6517 0 : enum insn_code icode;
6518 0 : rtx_insn *insn;
6519 0 : rtx trap_rtx;
6520 :
6521 0 : if (mode == VOIDmode)
6522 : return 0;
6523 :
6524 0 : icode = optab_handler (ctrap_optab, mode);
6525 0 : if (icode == CODE_FOR_nothing)
6526 : return 0;
6527 :
6528 : /* Some targets only accept a zero trap code. */
6529 0 : if (!insn_operand_matches (icode, 3, tcode))
6530 : return 0;
6531 :
6532 0 : do_pending_stack_adjust ();
6533 0 : start_sequence ();
6534 0 : prepare_cmp_insn (op1, op2, NULL, code, NULL_RTX, false, OPTAB_DIRECT,
6535 : &trap_rtx, &mode, cbranch_optab);
6536 0 : if (!trap_rtx)
6537 : insn = NULL;
6538 : else
6539 0 : insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6540 : tcode);
6541 :
6542 : /* If that failed, then give up. */
6543 0 : if (insn == 0)
6544 : {
6545 0 : end_sequence ();
6546 0 : return 0;
6547 : }
6548 :
6549 0 : emit_insn (insn);
6550 0 : insn = end_sequence ();
6551 0 : return insn;
6552 : }
6553 :
6554 : /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
6555 : or unsigned operation code. */
6556 :
6557 : enum rtx_code
6558 1626743 : get_rtx_code_1 (enum tree_code tcode, bool unsignedp)
6559 : {
6560 1626743 : enum rtx_code code;
6561 1626743 : switch (tcode)
6562 : {
6563 : case EQ_EXPR:
6564 : code = EQ;
6565 : break;
6566 818539 : case NE_EXPR:
6567 818539 : code = NE;
6568 818539 : break;
6569 149482 : case LT_EXPR:
6570 149482 : code = unsignedp ? LTU : LT;
6571 : break;
6572 74538 : case LE_EXPR:
6573 74538 : code = unsignedp ? LEU : LE;
6574 : break;
6575 114628 : case GT_EXPR:
6576 114628 : code = unsignedp ? GTU : GT;
6577 : break;
6578 81257 : case GE_EXPR:
6579 81257 : code = unsignedp ? GEU : GE;
6580 : break;
6581 :
6582 1994 : case UNORDERED_EXPR:
6583 1994 : code = UNORDERED;
6584 1994 : break;
6585 1393 : case ORDERED_EXPR:
6586 1393 : code = ORDERED;
6587 1393 : break;
6588 730 : case UNLT_EXPR:
6589 730 : code = UNLT;
6590 730 : break;
6591 4944 : case UNLE_EXPR:
6592 4944 : code = UNLE;
6593 4944 : break;
6594 726 : case UNGT_EXPR:
6595 726 : code = UNGT;
6596 726 : break;
6597 3754 : case UNGE_EXPR:
6598 3754 : code = UNGE;
6599 3754 : break;
6600 543 : case UNEQ_EXPR:
6601 543 : code = UNEQ;
6602 543 : break;
6603 89 : case LTGT_EXPR:
6604 89 : code = LTGT;
6605 89 : break;
6606 :
6607 32792 : case BIT_AND_EXPR:
6608 32792 : code = AND;
6609 32792 : break;
6610 :
6611 22 : case BIT_IOR_EXPR:
6612 22 : code = IOR;
6613 22 : break;
6614 :
6615 0 : default:
6616 0 : code = UNKNOWN;
6617 0 : break;
6618 : }
6619 1626743 : return code;
6620 : }
6621 :
6622 : /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6623 : or unsigned operation code. */
6624 :
6625 : enum rtx_code
6626 159259 : get_rtx_code (enum tree_code tcode, bool unsignedp)
6627 : {
6628 159259 : enum rtx_code code = get_rtx_code_1 (tcode, unsignedp);
6629 159259 : gcc_assert (code != UNKNOWN);
6630 159259 : return code;
6631 : }
6632 :
6633 : /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6634 : select signed or unsigned operators. OPNO holds the index of the
6635 : first comparison operand for insn ICODE. Do not generate the
6636 : compare instruction itself. */
6637 :
6638 : rtx
6639 26454 : vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
6640 : tree t_op0, tree t_op1, bool unsignedp,
6641 : enum insn_code icode, unsigned int opno)
6642 : {
6643 26454 : class expand_operand ops[2];
6644 26454 : rtx rtx_op0, rtx_op1;
6645 26454 : machine_mode m0, m1;
6646 26454 : enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6647 :
6648 26454 : gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6649 :
6650 : /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6651 : has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6652 : cases, use the original mode. */
6653 26454 : rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6654 : EXPAND_STACK_PARM);
6655 26454 : m0 = GET_MODE (rtx_op0);
6656 26454 : if (m0 == VOIDmode)
6657 0 : m0 = TYPE_MODE (TREE_TYPE (t_op0));
6658 :
6659 26454 : rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6660 : EXPAND_STACK_PARM);
6661 26454 : m1 = GET_MODE (rtx_op1);
6662 26454 : if (m1 == VOIDmode)
6663 0 : m1 = TYPE_MODE (TREE_TYPE (t_op1));
6664 :
6665 26454 : create_input_operand (&ops[0], rtx_op0, m0);
6666 26454 : create_input_operand (&ops[1], rtx_op1, m1);
6667 26454 : if (!maybe_legitimize_operands (icode, opno, 2, ops))
6668 0 : gcc_unreachable ();
6669 26454 : return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
6670 : }
6671 :
6672 : /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6673 : the first vec_perm operand, assuming the second operand (for left shift
6674 : first operand) is a constant vector of zeros. Return the shift distance
6675 : in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6676 : mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6677 : shift or vec_shl_optab for left shift. */
6678 : static rtx
6679 2996 : shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
6680 : optab shift_optab)
6681 : {
6682 2996 : unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
6683 2996 : poly_int64 first = sel[0];
6684 5992 : if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
6685 : return NULL_RTX;
6686 :
6687 2995 : if (shift_optab == vec_shl_optab)
6688 : {
6689 877 : unsigned int nelt;
6690 1754 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6691 2996 : return NULL_RTX;
6692 877 : unsigned firstidx = 0;
6693 5778 : for (unsigned int i = 0; i < nelt; i++)
6694 : {
6695 5020 : if (known_eq (sel[i], nelt))
6696 : {
6697 794 : if (i == 0 || firstidx)
6698 : return NULL_RTX;
6699 : firstidx = i;
6700 : }
6701 8452 : else if (firstidx
6702 5868 : ? maybe_ne (sel[i], nelt + i - firstidx)
6703 4226 : : maybe_ge (sel[i], nelt))
6704 : return NULL_RTX;
6705 : }
6706 :
6707 758 : if (firstidx == 0)
6708 : return NULL_RTX;
6709 758 : first = firstidx;
6710 : }
6711 2118 : else if (!sel.series_p (0, 1, first, 1))
6712 : {
6713 376 : unsigned int nelt;
6714 752 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6715 2996 : return NULL_RTX;
6716 2166 : for (unsigned int i = 1; i < nelt; i++)
6717 : {
6718 1964 : poly_int64 expected = i + first;
6719 : /* Indices into the second vector are all equivalent. */
6720 1964 : if (maybe_lt (sel[i], nelt)
6721 2770 : ? maybe_ne (sel[i], expected)
6722 806 : : maybe_lt (expected, nelt))
6723 2996 : return NULL_RTX;
6724 : }
6725 : }
6726 :
6727 2702 : return gen_int_shift_amount (mode, first * bitsize);
6728 : }
6729 :
6730 : /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6731 :
6732 : static rtx
6733 18 : expand_vec_perm_1 (enum insn_code icode, rtx target,
6734 : rtx v0, rtx v1, rtx sel)
6735 : {
6736 18 : machine_mode tmode = GET_MODE (target);
6737 18 : machine_mode smode = GET_MODE (sel);
6738 18 : class expand_operand ops[4];
6739 :
6740 18 : gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
6741 : || related_int_vector_mode (tmode).require () == smode);
6742 18 : create_output_operand (&ops[0], target, tmode);
6743 18 : create_input_operand (&ops[3], sel, smode);
6744 :
6745 : /* Make an effort to preserve v0 == v1. The target expander is able to
6746 : rely on this to determine if we're permuting a single input operand. */
6747 18 : if (rtx_equal_p (v0, v1))
6748 : {
6749 14 : if (!insn_operand_matches (icode, 1, v0))
6750 0 : v0 = force_reg (tmode, v0);
6751 14 : gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6752 14 : gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6753 :
6754 14 : create_fixed_operand (&ops[1], v0);
6755 14 : create_fixed_operand (&ops[2], v0);
6756 : }
6757 : else
6758 : {
6759 4 : create_input_operand (&ops[1], v0, tmode);
6760 4 : create_input_operand (&ops[2], v1, tmode);
6761 : }
6762 :
6763 18 : if (maybe_expand_insn (icode, 4, ops))
6764 18 : return ops[0].value;
6765 : return NULL_RTX;
6766 : }
6767 :
6768 : /* Check if vec_perm mask SEL is a constant equivalent to an and operation of
6769 : the non-zero vec_perm operand with some mask consisting of 0xffs and 0x00s,
6770 : assuming the other vec_perm operand is a constant vector of zeros. Return
6771 : the mask for the equivalent and operation, or NULL_RTX if the vec_perm can
6772 : not be modeled as an and. MODE is the mode of the value being anded.
6773 : ZERO_OP0_P is true if the first operand of the vec_perm is a constant vector
6774 : of zeros or false if the second operand of the vec_perm is a constant vector
6775 : of zeros. */
6776 : rtx
6777 0 : vec_perm_and_mask (machine_mode mode, const vec_perm_indices &sel,
6778 : bool zero_op0_p)
6779 : {
6780 0 : unsigned int nelt;
6781 0 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6782 : return NULL_RTX;
6783 :
6784 0 : rtx_vector_builder builder (mode, nelt, 1);
6785 0 : machine_mode emode = GET_MODE_INNER (mode);
6786 :
6787 0 : for (unsigned int i = 0; i < nelt; i++)
6788 : {
6789 0 : if (zero_op0_p)
6790 : {
6791 0 : if (known_eq (sel[i], nelt + i))
6792 0 : builder.quick_push (CONSTM1_RTX (emode));
6793 0 : else if (known_lt (sel[i], nelt))
6794 0 : builder.quick_push (CONST0_RTX (emode));
6795 : else
6796 0 : return NULL_RTX;
6797 : }
6798 : else
6799 : {
6800 0 : if (known_eq (sel[i], i))
6801 0 : builder.quick_push (CONSTM1_RTX (emode));
6802 0 : else if (known_ge (sel[i], nelt))
6803 0 : builder.quick_push (CONST0_RTX (emode));
6804 : else
6805 : return NULL_RTX;
6806 : }
6807 : }
6808 :
6809 0 : return builder.build ();
6810 0 : }
6811 :
6812 : /* Implement a permutation of vectors v0 and v1 using the permutation
6813 : vector in SEL and return the result. Use TARGET to hold the result
6814 : if nonnull and convenient.
6815 :
6816 : MODE is the mode of the result. V0 and V1 have the same mode, which
6817 : has the same element mode as MODE but may have a different number of
6818 : elements. SEL_MODE is the TYPE_MODE associated with SEL, or BLKmode
6819 : if SEL isn't known to have a particular mode. */
6820 :
6821 : rtx
6822 76327 : expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
6823 : const vec_perm_builder &sel, machine_mode sel_mode,
6824 : rtx target)
6825 : {
6826 76327 : if (!target || !register_operand (target, mode))
6827 61566 : target = gen_reg_rtx (mode);
6828 :
6829 : /* Set QIMODE to a different vector mode with byte elements.
6830 : If no such mode, or if MODE already has byte elements, use VOIDmode. */
6831 76327 : machine_mode qimode;
6832 76327 : if (!qimode_for_vec_perm (mode).exists (&qimode))
6833 76327 : qimode = VOIDmode;
6834 :
6835 76327 : rtx_insn *last = get_last_insn ();
6836 :
6837 76327 : bool single_arg_p = rtx_equal_p (v0, v1);
6838 : /* The selector indexes the concatenation of the inputs, which may have
6839 : a different number of elements than the result. */
6840 76327 : machine_mode op_mode = GET_MODE (v0);
6841 76327 : gcc_checking_assert (op_mode == GET_MODE (v1));
6842 : /* Always specify two input vectors here and leave the target to handle
6843 : cases in which the inputs are equal. Not all backends can cope with
6844 : the single-input representation when testing for a double-input
6845 : target instruction. */
6846 152654 : vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (op_mode));
6847 :
6848 : /* See if this can be handled with a vec_shr or vec_shl. We only do this
6849 : if the second (for vec_shr) or first (for vec_shl) vector is all
6850 : zeroes. */
6851 76327 : insn_code shift_code = CODE_FOR_nothing;
6852 76327 : insn_code shift_code_qi = CODE_FOR_nothing;
6853 76327 : optab shift_optab = unknown_optab;
6854 76327 : rtx v2 = v0;
6855 76327 : if (v1 == CONST0_RTX (op_mode))
6856 : shift_optab = vec_shr_optab;
6857 74145 : else if (v0 == CONST0_RTX (op_mode))
6858 : {
6859 1201 : shift_optab = vec_shl_optab;
6860 1201 : v2 = v1;
6861 : }
6862 76327 : if (mode == op_mode
6863 76327 : && GET_MODE_NUNITS (mode).is_constant ()
6864 76327 : && shift_optab != unknown_optab)
6865 : {
6866 3383 : shift_code = optab_handler (shift_optab, mode);
6867 5970 : shift_code_qi = ((qimode != VOIDmode && qimode != mode)
6868 6368 : ? optab_handler (shift_optab, qimode)
6869 : : CODE_FOR_nothing);
6870 : }
6871 3383 : if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
6872 : {
6873 2996 : rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices, shift_optab);
6874 2996 : if (shift_amt)
6875 : {
6876 2702 : class expand_operand ops[3];
6877 2702 : if (shift_amt == const0_rtx)
6878 2702 : return v2;
6879 2702 : if (shift_code != CODE_FOR_nothing)
6880 : {
6881 2702 : create_output_operand (&ops[0], target, mode);
6882 2702 : create_input_operand (&ops[1], v2, mode);
6883 2702 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6884 2702 : if (maybe_expand_insn (shift_code, 3, ops))
6885 2702 : return ops[0].value;
6886 : }
6887 0 : if (shift_code_qi != CODE_FOR_nothing)
6888 : {
6889 0 : rtx tmp = gen_reg_rtx (qimode);
6890 0 : create_output_operand (&ops[0], tmp, qimode);
6891 0 : create_input_operand (&ops[1], gen_lowpart (qimode, v2), qimode);
6892 0 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6893 0 : if (maybe_expand_insn (shift_code_qi, 3, ops))
6894 0 : return gen_lowpart (mode, ops[0].value);
6895 : }
6896 : }
6897 : }
6898 :
6899 73625 : if (targetm.vectorize.vec_perm_const != NULL)
6900 : {
6901 73625 : if (single_arg_p)
6902 18073 : v1 = v0;
6903 :
6904 73625 : if (targetm.vectorize.vec_perm_const (mode, op_mode, target, v0, v1,
6905 : indices))
6906 : return target;
6907 : }
6908 :
6909 : /* The fallbacks below need the inputs to have the result mode. */
6910 0 : if (mode != op_mode)
6911 : {
6912 0 : delete_insns_since (last);
6913 0 : return NULL_RTX;
6914 : }
6915 :
6916 : /* Fall back to a constant byte-based permutation. */
6917 0 : vec_perm_indices qimode_indices;
6918 0 : rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
6919 0 : if (qimode != VOIDmode)
6920 : {
6921 0 : qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
6922 0 : target_qi = gen_reg_rtx (qimode);
6923 0 : v0_qi = gen_lowpart (qimode, v0);
6924 0 : v1_qi = gen_lowpart (qimode, v1);
6925 0 : if (targetm.vectorize.vec_perm_const != NULL
6926 0 : && targetm.vectorize.vec_perm_const (qimode, qimode, target_qi, v0_qi,
6927 : v1_qi, qimode_indices))
6928 0 : return gen_lowpart (mode, target_qi);
6929 : }
6930 :
6931 0 : v0 = force_reg (mode, v0);
6932 0 : if (single_arg_p)
6933 0 : v1 = v0;
6934 0 : v1 = force_reg (mode, v1);
6935 :
6936 : /* Otherwise expand as a fully variable permutation. */
6937 :
6938 : /* The optabs are only defined for selectors with the same width
6939 : as the values being permuted. */
6940 0 : machine_mode required_sel_mode;
6941 0 : if (!related_int_vector_mode (mode).exists (&required_sel_mode))
6942 : {
6943 0 : delete_insns_since (last);
6944 0 : return NULL_RTX;
6945 : }
6946 :
6947 : /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6948 : If that isn't the mode we want then we need to prove that using
6949 : REQUIRED_SEL_MODE is OK. */
6950 0 : if (sel_mode != required_sel_mode)
6951 : {
6952 0 : if (!selector_fits_mode_p (required_sel_mode, indices))
6953 : {
6954 0 : delete_insns_since (last);
6955 0 : return NULL_RTX;
6956 : }
6957 : sel_mode = required_sel_mode;
6958 : }
6959 :
6960 0 : insn_code icode = direct_optab_handler (vec_perm_optab, mode);
6961 0 : if (icode != CODE_FOR_nothing)
6962 : {
6963 0 : rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
6964 0 : rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
6965 0 : if (tmp)
6966 : return tmp;
6967 : }
6968 :
6969 0 : if (qimode != VOIDmode
6970 0 : && selector_fits_mode_p (qimode, qimode_indices))
6971 : {
6972 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
6973 0 : if (icode != CODE_FOR_nothing)
6974 : {
6975 0 : rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
6976 0 : rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
6977 0 : if (tmp)
6978 0 : return gen_lowpart (mode, tmp);
6979 : }
6980 : }
6981 :
6982 0 : delete_insns_since (last);
6983 0 : return NULL_RTX;
6984 0 : }
6985 :
6986 : /* Implement a permutation of vectors v0 and v1 using the permutation
6987 : vector in SEL and return the result. Use TARGET to hold the result
6988 : if nonnull and convenient.
6989 :
6990 : MODE is the mode of the vectors being permuted (V0 and V1).
6991 : SEL must have the integer equivalent of MODE and is known to be
6992 : unsuitable for permutes with a constant permutation vector. */
6993 :
6994 : rtx
6995 18 : expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6996 : {
6997 18 : enum insn_code icode;
6998 18 : unsigned int i, u;
6999 18 : rtx tmp, sel_qi;
7000 :
7001 18 : u = GET_MODE_UNIT_SIZE (mode);
7002 :
7003 18 : if (!target || GET_MODE (target) != mode)
7004 0 : target = gen_reg_rtx (mode);
7005 :
7006 18 : icode = direct_optab_handler (vec_perm_optab, mode);
7007 18 : if (icode != CODE_FOR_nothing)
7008 : {
7009 18 : tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
7010 18 : if (tmp)
7011 : return tmp;
7012 : }
7013 :
7014 : /* As a special case to aid several targets, lower the element-based
7015 : permutation to a byte-based permutation and try again. */
7016 0 : machine_mode qimode;
7017 0 : if (!qimode_for_vec_perm (mode).exists (&qimode)
7018 0 : || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
7019 : return NULL_RTX;
7020 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
7021 0 : if (icode == CODE_FOR_nothing)
7022 : return NULL_RTX;
7023 :
7024 : /* Multiply each element by its byte size. */
7025 0 : machine_mode selmode = GET_MODE (sel);
7026 0 : if (u == 2)
7027 0 : sel = expand_simple_binop (selmode, PLUS, sel, sel,
7028 : NULL, 0, OPTAB_DIRECT);
7029 : else
7030 0 : sel = expand_simple_binop (selmode, ASHIFT, sel,
7031 0 : gen_int_shift_amount (selmode, exact_log2 (u)),
7032 : NULL, 0, OPTAB_DIRECT);
7033 0 : gcc_assert (sel != NULL);
7034 :
7035 : /* Broadcast the low byte each element into each of its bytes.
7036 : The encoding has U interleaved stepped patterns, one for each
7037 : byte of an element. */
7038 0 : vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
7039 0 : unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
7040 0 : for (i = 0; i < 3; ++i)
7041 0 : for (unsigned int j = 0; j < u; ++j)
7042 0 : const_sel.quick_push (i * u + low_byte_in_u);
7043 0 : sel = gen_lowpart (qimode, sel);
7044 0 : sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
7045 0 : gcc_assert (sel != NULL);
7046 :
7047 : /* Add the byte offset to each byte element. */
7048 : /* Note that the definition of the indices here is memory ordering,
7049 : so there should be no difference between big and little endian. */
7050 0 : rtx_vector_builder byte_indices (qimode, u, 1);
7051 0 : for (i = 0; i < u; ++i)
7052 0 : byte_indices.quick_push (GEN_INT (i));
7053 0 : tmp = byte_indices.build ();
7054 0 : sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
7055 : sel, 0, OPTAB_DIRECT);
7056 0 : gcc_assert (sel_qi != NULL);
7057 :
7058 0 : tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
7059 0 : tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
7060 0 : gen_lowpart (qimode, v1), sel_qi);
7061 0 : if (tmp)
7062 0 : tmp = gen_lowpart (mode, tmp);
7063 0 : return tmp;
7064 0 : }
7065 :
7066 : /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
7067 : Use TARGET for the result if nonnull and convenient. */
7068 :
7069 : rtx
7070 0 : expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
7071 : {
7072 0 : class expand_operand ops[3];
7073 0 : enum insn_code icode;
7074 0 : machine_mode emode = GET_MODE_INNER (vmode);
7075 :
7076 0 : icode = direct_optab_handler (vec_series_optab, vmode);
7077 0 : gcc_assert (icode != CODE_FOR_nothing);
7078 :
7079 0 : create_output_operand (&ops[0], target, vmode);
7080 0 : create_input_operand (&ops[1], op0, emode);
7081 0 : create_input_operand (&ops[2], op1, emode);
7082 :
7083 0 : expand_insn (icode, 3, ops);
7084 0 : return ops[0].value;
7085 : }
7086 :
7087 : /* Generate insns for a vector comparison into a mask. */
7088 :
7089 : rtx
7090 26454 : expand_vec_cmp_expr (tree type, tree exp, rtx target)
7091 : {
7092 26454 : class expand_operand ops[4];
7093 26454 : enum insn_code icode;
7094 26454 : rtx comparison;
7095 26454 : machine_mode mask_mode = TYPE_MODE (type);
7096 26454 : machine_mode vmode;
7097 26454 : bool unsignedp;
7098 26454 : tree op0a, op0b;
7099 26454 : enum tree_code tcode;
7100 :
7101 26454 : op0a = TREE_OPERAND (exp, 0);
7102 26454 : op0b = TREE_OPERAND (exp, 1);
7103 26454 : tcode = TREE_CODE (exp);
7104 :
7105 26454 : unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
7106 26454 : vmode = TYPE_MODE (TREE_TYPE (op0a));
7107 :
7108 26454 : icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
7109 26454 : if (icode == CODE_FOR_nothing)
7110 : {
7111 983 : if (tcode == EQ_EXPR || tcode == NE_EXPR)
7112 983 : icode = get_vec_cmp_eq_icode (vmode, mask_mode);
7113 983 : if (icode == CODE_FOR_nothing)
7114 : return 0;
7115 : }
7116 :
7117 26454 : comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
7118 : unsignedp, icode, 2);
7119 26454 : create_output_operand (&ops[0], target, mask_mode);
7120 26454 : create_fixed_operand (&ops[1], comparison);
7121 26454 : create_fixed_operand (&ops[2], XEXP (comparison, 0));
7122 26454 : create_fixed_operand (&ops[3], XEXP (comparison, 1));
7123 26454 : expand_insn (icode, 4, ops);
7124 26454 : return ops[0].value;
7125 : }
7126 :
7127 : /* Expand a highpart multiply. */
7128 :
7129 : rtx
7130 3528 : expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
7131 : rtx target, bool uns_p)
7132 : {
7133 3528 : class expand_operand eops[3];
7134 3528 : enum insn_code icode;
7135 3528 : int method, i;
7136 3528 : machine_mode wmode;
7137 3528 : rtx m1, m2;
7138 3528 : optab tab1, tab2;
7139 :
7140 3528 : method = can_mult_highpart_p (mode, uns_p);
7141 3528 : switch (method)
7142 : {
7143 : case 0:
7144 : return NULL_RTX;
7145 2795 : case 1:
7146 2795 : tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
7147 2795 : return expand_binop (mode, tab1, op0, op1, target, uns_p,
7148 2795 : OPTAB_LIB_WIDEN);
7149 0 : case 2:
7150 0 : return expmed_mult_highpart_optab (as_a <scalar_int_mode> (mode),
7151 0 : op0, op1, target, uns_p, INT_MAX);
7152 724 : case 3:
7153 724 : tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
7154 525 : tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
7155 : break;
7156 9 : case 4:
7157 9 : tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
7158 9 : tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
7159 : if (BYTES_BIG_ENDIAN)
7160 : std::swap (tab1, tab2);
7161 : break;
7162 0 : default:
7163 0 : gcc_unreachable ();
7164 : }
7165 :
7166 733 : icode = optab_handler (tab1, mode);
7167 733 : wmode = insn_data[icode].operand[0].mode;
7168 2199 : gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
7169 : GET_MODE_NUNITS (mode)));
7170 2199 : gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
7171 :
7172 733 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7173 733 : create_input_operand (&eops[1], op0, mode);
7174 733 : create_input_operand (&eops[2], op1, mode);
7175 733 : expand_insn (icode, 3, eops);
7176 733 : m1 = gen_lowpart (mode, eops[0].value);
7177 :
7178 733 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7179 733 : create_input_operand (&eops[1], op0, mode);
7180 733 : create_input_operand (&eops[2], op1, mode);
7181 733 : expand_insn (optab_handler (tab2, mode), 3, eops);
7182 733 : m2 = gen_lowpart (mode, eops[0].value);
7183 :
7184 1466 : vec_perm_builder sel;
7185 733 : if (method == 3)
7186 : {
7187 : /* The encoding has 2 interleaved stepped patterns. */
7188 1448 : sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
7189 5068 : for (i = 0; i < 6; ++i)
7190 13032 : sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
7191 6516 : + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
7192 : }
7193 : else
7194 : {
7195 : /* The encoding has a single interleaved stepped pattern. */
7196 18 : sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
7197 36 : for (i = 0; i < 3; ++i)
7198 27 : sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
7199 : }
7200 :
7201 733 : return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
7202 : }
7203 :
7204 : /* Helper function to find the MODE_CC set in a sync_compare_and_swap
7205 : pattern. */
7206 :
7207 : static void
7208 0 : find_cc_set (rtx x, const_rtx pat, void *data)
7209 : {
7210 0 : if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
7211 0 : && GET_CODE (pat) == SET)
7212 : {
7213 0 : rtx *p_cc_reg = (rtx *) data;
7214 0 : gcc_assert (!*p_cc_reg);
7215 0 : *p_cc_reg = x;
7216 : }
7217 0 : }
7218 :
7219 : /* This is a helper function for the other atomic operations. This function
7220 : emits a loop that contains SEQ that iterates until a compare-and-swap
7221 : operation at the end succeeds. MEM is the memory to be modified. SEQ is
7222 : a set of instructions that takes a value from OLD_REG as an input and
7223 : produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
7224 : set to the current contents of MEM. After SEQ, a compare-and-swap will
7225 : attempt to update MEM with NEW_REG. The function returns true when the
7226 : loop was generated successfully. */
7227 :
7228 : static bool
7229 5258 : expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx_insn *seq)
7230 : {
7231 5258 : machine_mode mode = GET_MODE (mem);
7232 5258 : rtx_code_label *label;
7233 5258 : rtx cmp_reg, success, oldval;
7234 :
7235 : /* The loop we want to generate looks like
7236 :
7237 : cmp_reg = mem;
7238 : label:
7239 : old_reg = cmp_reg;
7240 : seq;
7241 : (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
7242 : if (success)
7243 : goto label;
7244 :
7245 : Note that we only do the plain load from memory once. Subsequent
7246 : iterations use the value loaded by the compare-and-swap pattern. */
7247 :
7248 5258 : label = gen_label_rtx ();
7249 5258 : cmp_reg = gen_reg_rtx (mode);
7250 :
7251 5258 : emit_move_insn (cmp_reg, mem);
7252 5258 : emit_label (label);
7253 5258 : emit_move_insn (old_reg, cmp_reg);
7254 5258 : if (seq)
7255 5245 : emit_insn (seq);
7256 :
7257 5258 : success = NULL_RTX;
7258 5258 : oldval = cmp_reg;
7259 5258 : if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
7260 : new_reg, false, MEMMODEL_SYNC_SEQ_CST,
7261 : MEMMODEL_RELAXED))
7262 : return false;
7263 :
7264 5258 : if (oldval != cmp_reg)
7265 0 : emit_move_insn (cmp_reg, oldval);
7266 :
7267 : /* Mark this jump predicted not taken. */
7268 5258 : emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
7269 5258 : GET_MODE (success), 1, label,
7270 : profile_probability::guessed_never ());
7271 5258 : return true;
7272 : }
7273 :
7274 :
7275 : /* This function tries to emit an atomic_exchange instruction. VAL is written
7276 : to *MEM using memory model MODEL. The previous contents of *MEM are returned,
7277 : using TARGET if possible. */
7278 :
7279 : static rtx
7280 3941 : maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7281 : {
7282 3941 : machine_mode mode = GET_MODE (mem);
7283 3941 : enum insn_code icode;
7284 :
7285 : /* If the target supports the exchange directly, great. */
7286 3941 : icode = direct_optab_handler (atomic_exchange_optab, mode);
7287 3941 : if (icode != CODE_FOR_nothing)
7288 : {
7289 3924 : class expand_operand ops[4];
7290 :
7291 3924 : create_output_operand (&ops[0], target, mode);
7292 3924 : create_fixed_operand (&ops[1], mem);
7293 3924 : create_input_operand (&ops[2], val, mode);
7294 3924 : create_integer_operand (&ops[3], model);
7295 3924 : if (maybe_expand_insn (icode, 4, ops))
7296 3924 : return ops[0].value;
7297 : }
7298 :
7299 : return NULL_RTX;
7300 : }
7301 :
7302 : /* This function tries to implement an atomic exchange operation using
7303 : __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
7304 : The previous contents of *MEM are returned, using TARGET if possible.
7305 : Since this instructionn is an acquire barrier only, stronger memory
7306 : models may require additional barriers to be emitted. */
7307 :
7308 : static rtx
7309 2 : maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
7310 : enum memmodel model)
7311 : {
7312 2 : machine_mode mode = GET_MODE (mem);
7313 2 : enum insn_code icode;
7314 2 : rtx_insn *last_insn = get_last_insn ();
7315 :
7316 2 : icode = optab_handler (sync_lock_test_and_set_optab, mode);
7317 :
7318 : /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7319 : exists, and the memory model is stronger than acquire, add a release
7320 : barrier before the instruction. */
7321 :
7322 2 : if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
7323 0 : expand_mem_thread_fence (model);
7324 :
7325 2 : if (icode != CODE_FOR_nothing)
7326 : {
7327 0 : class expand_operand ops[3];
7328 0 : create_output_operand (&ops[0], target, mode);
7329 0 : create_fixed_operand (&ops[1], mem);
7330 0 : create_input_operand (&ops[2], val, mode);
7331 0 : if (maybe_expand_insn (icode, 3, ops))
7332 0 : return ops[0].value;
7333 : }
7334 :
7335 : /* If an external test-and-set libcall is provided, use that instead of
7336 : any external compare-and-swap that we might get from the compare-and-
7337 : swap-loop expansion later. */
7338 2 : if (!can_compare_and_swap_p (mode, false))
7339 : {
7340 2 : rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
7341 2 : if (libfunc != NULL)
7342 : {
7343 0 : rtx addr;
7344 :
7345 0 : addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7346 0 : return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7347 : mode, addr, ptr_mode,
7348 0 : val, mode);
7349 : }
7350 : }
7351 :
7352 : /* If the test_and_set can't be emitted, eliminate any barrier that might
7353 : have been emitted. */
7354 2 : delete_insns_since (last_insn);
7355 2 : return NULL_RTX;
7356 : }
7357 :
7358 : /* This function tries to implement an atomic exchange operation using a
7359 : compare_and_swap loop. VAL is written to *MEM. The previous contents of
7360 : *MEM are returned, using TARGET if possible. No memory model is required
7361 : since a compare_and_swap loop is seq-cst. */
7362 :
7363 : static rtx
7364 17 : maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
7365 : {
7366 17 : machine_mode mode = GET_MODE (mem);
7367 :
7368 17 : if (can_compare_and_swap_p (mode, true))
7369 : {
7370 13 : rtx_insn *start = get_last_insn ();
7371 : /* Force val into a register if it could change value when the
7372 : atomic insn updates mem. */
7373 13 : if (reg_overlap_mentioned_p (mem, val))
7374 2 : val = force_reg (mode, val);
7375 13 : if (!target || !register_operand (target, mode))
7376 1 : target = gen_reg_rtx (mode);
7377 13 : if (expand_compare_and_swap_loop (mem, target, val, nullptr))
7378 : return target;
7379 : else
7380 0 : delete_insns_since (start);
7381 : }
7382 :
7383 : return NULL_RTX;
7384 : }
7385 :
7386 : /* This function tries to implement an atomic test-and-set operation
7387 : using the atomic_test_and_set instruction pattern. A boolean value
7388 : is returned from the operation, using TARGET if possible. */
7389 :
7390 : static rtx
7391 268 : maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7392 : {
7393 268 : machine_mode pat_bool_mode;
7394 268 : class expand_operand ops[3];
7395 :
7396 268 : if (!targetm.have_atomic_test_and_set ())
7397 : return NULL_RTX;
7398 :
7399 : /* While we always get QImode from __atomic_test_and_set, we get
7400 : other memory modes from __sync_lock_test_and_set. Note that we
7401 : use no endian adjustment here. This matches the 4.6 behavior
7402 : in the Sparc backend. */
7403 0 : enum insn_code icode = targetm.code_for_atomic_test_and_set;
7404 0 : gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
7405 0 : if (GET_MODE (mem) != QImode)
7406 0 : mem = adjust_address_nv (mem, QImode, 0);
7407 :
7408 0 : pat_bool_mode = insn_data[icode].operand[0].mode;
7409 0 : create_output_operand (&ops[0], target, pat_bool_mode);
7410 0 : create_fixed_operand (&ops[1], mem);
7411 0 : create_integer_operand (&ops[2], model);
7412 :
7413 0 : if (maybe_expand_insn (icode, 3, ops))
7414 0 : return ops[0].value;
7415 : return NULL_RTX;
7416 : }
7417 :
7418 : /* This function expands the legacy _sync_lock test_and_set operation which is
7419 : generally an atomic exchange. Some limited targets only allow the
7420 : constant 1 to be stored. This is an ACQUIRE operation.
7421 :
7422 : TARGET is an optional place to stick the return value.
7423 : MEM is where VAL is stored. */
7424 :
7425 : rtx
7426 326 : expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7427 : {
7428 326 : rtx ret;
7429 :
7430 : /* Try an atomic_exchange first. */
7431 326 : ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
7432 326 : if (ret)
7433 : return ret;
7434 :
7435 2 : ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
7436 : MEMMODEL_SYNC_ACQUIRE);
7437 2 : if (ret)
7438 : return ret;
7439 :
7440 2 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7441 2 : if (ret)
7442 : return ret;
7443 :
7444 : /* If there are no other options, try atomic_test_and_set if the value
7445 : being stored is 1. */
7446 2 : if (val == const1_rtx)
7447 2 : ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
7448 :
7449 : return ret;
7450 : }
7451 :
7452 : /* This function expands the atomic test_and_set operation:
7453 : atomically store a boolean TRUE into MEM and return the previous value.
7454 :
7455 : MEMMODEL is the memory model variant to use.
7456 : TARGET is an optional place to stick the return value. */
7457 :
7458 : rtx
7459 266 : expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7460 : {
7461 266 : machine_mode mode = GET_MODE (mem);
7462 266 : rtx ret, trueval, subtarget;
7463 :
7464 266 : ret = maybe_emit_atomic_test_and_set (target, mem, model);
7465 266 : if (ret)
7466 : return ret;
7467 :
7468 : /* Be binary compatible with non-default settings of trueval, and different
7469 : cpu revisions. E.g. one revision may have atomic-test-and-set, but
7470 : another only has atomic-exchange. */
7471 266 : if (targetm.atomic_test_and_set_trueval == 1)
7472 : {
7473 266 : trueval = const1_rtx;
7474 266 : subtarget = target ? target : gen_reg_rtx (mode);
7475 : }
7476 : else
7477 : {
7478 0 : trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7479 0 : subtarget = gen_reg_rtx (mode);
7480 : }
7481 :
7482 : /* Try the atomic-exchange optab... */
7483 266 : ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
7484 :
7485 : /* ... then an atomic-compare-and-swap loop ... */
7486 266 : if (!ret)
7487 0 : ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
7488 :
7489 : /* ... before trying the vaguely defined legacy lock_test_and_set. */
7490 0 : if (!ret)
7491 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
7492 :
7493 : /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7494 : things with the value 1. Thus we try again without trueval. */
7495 266 : if (!ret && targetm.atomic_test_and_set_trueval != 1)
7496 : {
7497 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
7498 :
7499 0 : if (ret)
7500 : {
7501 : /* Rectify the not-one trueval. */
7502 0 : ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7503 0 : gcc_assert (ret);
7504 : }
7505 : }
7506 :
7507 : return ret;
7508 : }
7509 :
7510 : /* This function expands the atomic exchange operation:
7511 : atomically store VAL in MEM and return the previous value in MEM.
7512 :
7513 : MEMMODEL is the memory model variant to use.
7514 : TARGET is an optional place to stick the return value. */
7515 :
7516 : rtx
7517 2886 : expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7518 : {
7519 2886 : machine_mode mode = GET_MODE (mem);
7520 2886 : rtx ret;
7521 :
7522 : /* If loads are not atomic for the required size and we are not called to
7523 : provide a __sync builtin, do not do anything so that we stay consistent
7524 : with atomic loads of the same size. */
7525 2886 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7526 : return NULL_RTX;
7527 :
7528 2855 : ret = maybe_emit_atomic_exchange (target, mem, val, model);
7529 :
7530 : /* Next try a compare-and-swap loop for the exchange. */
7531 2855 : if (!ret)
7532 13 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7533 :
7534 : return ret;
7535 : }
7536 :
7537 : /* This function expands the atomic compare exchange operation:
7538 :
7539 : *PTARGET_BOOL is an optional place to store the boolean success/failure.
7540 : *PTARGET_OVAL is an optional place to store the old value from memory.
7541 : Both target parameters may be NULL or const0_rtx to indicate that we do
7542 : not care about that return value. Both target parameters are updated on
7543 : success to the actual location of the corresponding result.
7544 :
7545 : MEMMODEL is the memory model variant to use.
7546 :
7547 : The return value of the function is true for success. */
7548 :
7549 : bool
7550 29106 : expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7551 : rtx mem, rtx expected, rtx desired,
7552 : bool is_weak, enum memmodel succ_model,
7553 : enum memmodel fail_model)
7554 : {
7555 29106 : machine_mode mode = GET_MODE (mem);
7556 29106 : class expand_operand ops[8];
7557 29106 : enum insn_code icode;
7558 29106 : rtx target_oval, target_bool = NULL_RTX;
7559 29106 : rtx libfunc;
7560 :
7561 : /* If loads are not atomic for the required size and we are not called to
7562 : provide a __sync builtin, do not do anything so that we stay consistent
7563 : with atomic loads of the same size. */
7564 29106 : if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
7565 : return false;
7566 :
7567 : /* Load expected into a register for the compare and swap. */
7568 27357 : if (MEM_P (expected))
7569 7629 : expected = copy_to_reg (expected);
7570 :
7571 : /* Make sure we always have some place to put the return oldval.
7572 : Further, make sure that place is distinct from the input expected,
7573 : just in case we need that path down below. */
7574 27357 : if (ptarget_oval && *ptarget_oval == const0_rtx)
7575 : ptarget_oval = NULL;
7576 :
7577 27127 : if (ptarget_oval == NULL
7578 27127 : || (target_oval = *ptarget_oval) == NULL
7579 5488 : || reg_overlap_mentioned_p (expected, target_oval))
7580 21947 : target_oval = gen_reg_rtx (mode);
7581 :
7582 27357 : icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7583 27357 : if (icode != CODE_FOR_nothing)
7584 : {
7585 27355 : machine_mode bool_mode = insn_data[icode].operand[0].mode;
7586 :
7587 27355 : if (ptarget_bool && *ptarget_bool == const0_rtx)
7588 : ptarget_bool = NULL;
7589 :
7590 : /* Make sure we always have a place for the bool operand. */
7591 27100 : if (ptarget_bool == NULL
7592 27100 : || (target_bool = *ptarget_bool) == NULL
7593 7516 : || GET_MODE (target_bool) != bool_mode)
7594 19839 : target_bool = gen_reg_rtx (bool_mode);
7595 :
7596 : /* Emit the compare_and_swap. */
7597 27355 : create_output_operand (&ops[0], target_bool, bool_mode);
7598 27355 : create_output_operand (&ops[1], target_oval, mode);
7599 27355 : create_fixed_operand (&ops[2], mem);
7600 27355 : create_input_operand (&ops[3], expected, mode);
7601 27355 : create_input_operand (&ops[4], desired, mode);
7602 27355 : create_integer_operand (&ops[5], is_weak);
7603 27355 : create_integer_operand (&ops[6], succ_model);
7604 27355 : create_integer_operand (&ops[7], fail_model);
7605 27355 : if (maybe_expand_insn (icode, 8, ops))
7606 : {
7607 : /* Return success/failure. */
7608 27355 : target_bool = ops[0].value;
7609 27355 : target_oval = ops[1].value;
7610 27355 : goto success;
7611 : }
7612 : }
7613 :
7614 : /* Otherwise fall back to the original __sync_val_compare_and_swap
7615 : which is always seq-cst. */
7616 2 : icode = optab_handler (sync_compare_and_swap_optab, mode);
7617 2 : if (icode != CODE_FOR_nothing)
7618 : {
7619 0 : rtx cc_reg;
7620 :
7621 0 : create_output_operand (&ops[0], target_oval, mode);
7622 0 : create_fixed_operand (&ops[1], mem);
7623 0 : create_input_operand (&ops[2], expected, mode);
7624 0 : create_input_operand (&ops[3], desired, mode);
7625 0 : if (!maybe_expand_insn (icode, 4, ops))
7626 0 : return false;
7627 :
7628 0 : target_oval = ops[0].value;
7629 :
7630 : /* If the caller isn't interested in the boolean return value,
7631 : skip the computation of it. */
7632 0 : if (ptarget_bool == NULL)
7633 0 : goto success;
7634 :
7635 : /* Otherwise, work out if the compare-and-swap succeeded. */
7636 0 : cc_reg = NULL_RTX;
7637 0 : if (have_insn_for (COMPARE, CCmode))
7638 0 : note_stores (get_last_insn (), find_cc_set, &cc_reg);
7639 0 : if (cc_reg)
7640 : {
7641 0 : target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7642 : const0_rtx, VOIDmode, 0, 1);
7643 0 : goto success;
7644 : }
7645 0 : goto success_bool_from_val;
7646 : }
7647 :
7648 : /* Also check for library support for __sync_val_compare_and_swap. */
7649 2 : libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7650 2 : if (libfunc != NULL)
7651 : {
7652 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7653 0 : rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7654 : mode, addr, ptr_mode,
7655 : expected, mode, desired, mode);
7656 0 : emit_move_insn (target_oval, target);
7657 :
7658 : /* Compute the boolean return value only if requested. */
7659 0 : if (ptarget_bool)
7660 0 : goto success_bool_from_val;
7661 : else
7662 0 : goto success;
7663 : }
7664 :
7665 : /* Failure. */
7666 : return false;
7667 :
7668 0 : success_bool_from_val:
7669 0 : target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7670 : expected, VOIDmode, 1, 1);
7671 27355 : success:
7672 : /* Make sure that the oval output winds up where the caller asked. */
7673 27355 : if (ptarget_oval)
7674 27127 : *ptarget_oval = target_oval;
7675 27355 : if (ptarget_bool)
7676 27100 : *ptarget_bool = target_bool;
7677 : return true;
7678 : }
7679 :
7680 : /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7681 :
7682 : static void
7683 0 : expand_asm_memory_blockage (void)
7684 : {
7685 0 : rtx asm_op, clob;
7686 :
7687 0 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7688 : rtvec_alloc (0), rtvec_alloc (0),
7689 : rtvec_alloc (0), UNKNOWN_LOCATION);
7690 0 : MEM_VOLATILE_P (asm_op) = 1;
7691 :
7692 0 : clob = gen_rtx_SCRATCH (VOIDmode);
7693 0 : clob = gen_rtx_MEM (BLKmode, clob);
7694 0 : clob = gen_rtx_CLOBBER (VOIDmode, clob);
7695 :
7696 0 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7697 0 : }
7698 :
7699 : /* Do not propagate memory accesses across this point. */
7700 :
7701 : static void
7702 116989 : expand_memory_blockage (void)
7703 : {
7704 116989 : if (targetm.have_memory_blockage ())
7705 116989 : emit_insn (targetm.gen_memory_blockage ());
7706 : else
7707 0 : expand_asm_memory_blockage ();
7708 116989 : }
7709 :
7710 : /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7711 : same time clobbering the register set specified by REGS. */
7712 :
7713 : void
7714 131 : expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs)
7715 : {
7716 131 : rtx asm_op, clob_mem;
7717 :
7718 131 : unsigned int num_of_regs = 0;
7719 131 : unsigned int i;
7720 :
7721 131 : num_of_regs = hard_reg_set_popcount (regs);
7722 :
7723 131 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7724 : rtvec_alloc (0), rtvec_alloc (0),
7725 : rtvec_alloc (0), UNKNOWN_LOCATION);
7726 131 : MEM_VOLATILE_P (asm_op) = 1;
7727 :
7728 131 : rtvec v = rtvec_alloc (num_of_regs + 2);
7729 :
7730 131 : clob_mem = gen_rtx_SCRATCH (VOIDmode);
7731 131 : clob_mem = gen_rtx_MEM (BLKmode, clob_mem);
7732 131 : clob_mem = gen_rtx_CLOBBER (VOIDmode, clob_mem);
7733 :
7734 131 : RTVEC_ELT (v, 0) = asm_op;
7735 131 : RTVEC_ELT (v, 1) = clob_mem;
7736 :
7737 131 : if (num_of_regs > 0)
7738 : {
7739 131 : unsigned int j = 2;
7740 131 : hard_reg_set_iterator hrsi2;
7741 131 : i = 0;
7742 1250 : EXECUTE_IF_SET_IN_HARD_REG_SET (regs, 0, i, hrsi2)
7743 : {
7744 1119 : RTVEC_ELT (v, j) = gen_rtx_CLOBBER (VOIDmode, regno_reg_rtx[i]);
7745 1119 : j++;
7746 : }
7747 131 : gcc_assert (j == (num_of_regs + 2));
7748 : }
7749 :
7750 131 : emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
7751 131 : }
7752 :
7753 : /* This routine will either emit the mem_thread_fence pattern or issue a
7754 : sync_synchronize to generate a fence for memory model MEMMODEL. */
7755 :
7756 : void
7757 994 : expand_mem_thread_fence (enum memmodel model)
7758 : {
7759 994 : if (is_mm_relaxed (model))
7760 : return;
7761 985 : if (targetm.have_mem_thread_fence ())
7762 : {
7763 985 : emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
7764 985 : expand_memory_blockage ();
7765 : }
7766 0 : else if (targetm.have_memory_barrier ())
7767 0 : emit_insn (targetm.gen_memory_barrier ());
7768 0 : else if (synchronize_libfunc != NULL_RTX)
7769 0 : emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
7770 : else
7771 0 : expand_memory_blockage ();
7772 : }
7773 :
7774 : /* Emit a signal fence with given memory model. */
7775 :
7776 : void
7777 60 : expand_mem_signal_fence (enum memmodel model)
7778 : {
7779 : /* No machine barrier is required to implement a signal fence, but
7780 : a compiler memory barrier must be issued, except for relaxed MM. */
7781 60 : if (!is_mm_relaxed (model))
7782 51 : expand_memory_blockage ();
7783 60 : }
7784 :
7785 : /* This function expands the atomic load operation:
7786 : return the atomically loaded value in MEM.
7787 :
7788 : MEMMODEL is the memory model variant to use.
7789 : TARGET is an option place to stick the return value. */
7790 :
7791 : rtx
7792 71523 : expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7793 : {
7794 71523 : machine_mode mode = GET_MODE (mem);
7795 71523 : enum insn_code icode;
7796 :
7797 : /* If the target supports the load directly, great. */
7798 71523 : icode = direct_optab_handler (atomic_load_optab, mode);
7799 71523 : if (icode != CODE_FOR_nothing)
7800 : {
7801 67695 : class expand_operand ops[3];
7802 67695 : rtx_insn *last = get_last_insn ();
7803 67695 : if (is_mm_seq_cst (model))
7804 38347 : expand_memory_blockage ();
7805 :
7806 67695 : create_output_operand (&ops[0], target, mode);
7807 67695 : create_fixed_operand (&ops[1], mem);
7808 67695 : create_integer_operand (&ops[2], model);
7809 67695 : if (maybe_expand_insn (icode, 3, ops))
7810 : {
7811 67695 : if (!is_mm_relaxed (model))
7812 52694 : expand_memory_blockage ();
7813 67695 : return ops[0].value;
7814 : }
7815 0 : delete_insns_since (last);
7816 : }
7817 :
7818 : /* If the size of the object is greater than word size on this target,
7819 : then we assume that a load will not be atomic. We could try to
7820 : emulate a load with a compare-and-swap operation, but the store that
7821 : doing this could result in would be incorrect if this is a volatile
7822 : atomic load or targeting read-only-mapped memory. */
7823 3828 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7824 : /* If there is no atomic load, leave the library call. */
7825 : return NULL_RTX;
7826 :
7827 : /* Otherwise assume loads are atomic, and emit the proper barriers. */
7828 0 : if (!target || target == const0_rtx)
7829 0 : target = gen_reg_rtx (mode);
7830 :
7831 : /* For SEQ_CST, emit a barrier before the load. */
7832 0 : if (is_mm_seq_cst (model))
7833 0 : expand_mem_thread_fence (model);
7834 :
7835 0 : emit_move_insn (target, mem);
7836 :
7837 : /* Emit the appropriate barrier after the load. */
7838 0 : expand_mem_thread_fence (model);
7839 :
7840 0 : return target;
7841 : }
7842 :
7843 : /* This function expands the atomic store operation:
7844 : Atomically store VAL in MEM.
7845 : MEMMODEL is the memory model variant to use.
7846 : USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7847 : function returns const0_rtx if a pattern was emitted. */
7848 :
7849 : rtx
7850 17776 : expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7851 : {
7852 17776 : machine_mode mode = GET_MODE (mem);
7853 17776 : enum insn_code icode;
7854 17776 : class expand_operand ops[3];
7855 :
7856 : /* If the target supports the store directly, great. */
7857 17776 : icode = direct_optab_handler (atomic_store_optab, mode);
7858 17776 : if (icode != CODE_FOR_nothing)
7859 : {
7860 16173 : rtx_insn *last = get_last_insn ();
7861 16173 : if (!is_mm_relaxed (model))
7862 13350 : expand_memory_blockage ();
7863 16173 : create_fixed_operand (&ops[0], mem);
7864 16173 : create_input_operand (&ops[1], val, mode);
7865 16173 : create_integer_operand (&ops[2], model);
7866 16173 : if (maybe_expand_insn (icode, 3, ops))
7867 : {
7868 16173 : if (is_mm_seq_cst (model))
7869 11562 : expand_memory_blockage ();
7870 16173 : return const0_rtx;
7871 : }
7872 0 : delete_insns_since (last);
7873 : }
7874 :
7875 : /* If using __sync_lock_release is a viable alternative, try it.
7876 : Note that this will not be set to true if we are expanding a generic
7877 : __atomic_store_n. */
7878 1603 : if (use_release)
7879 : {
7880 2 : icode = direct_optab_handler (sync_lock_release_optab, mode);
7881 2 : if (icode != CODE_FOR_nothing)
7882 : {
7883 0 : create_fixed_operand (&ops[0], mem);
7884 0 : create_input_operand (&ops[1], const0_rtx, mode);
7885 0 : if (maybe_expand_insn (icode, 2, ops))
7886 : {
7887 : /* lock_release is only a release barrier. */
7888 0 : if (is_mm_seq_cst (model))
7889 0 : expand_mem_thread_fence (model);
7890 0 : return const0_rtx;
7891 : }
7892 : }
7893 : }
7894 :
7895 : /* If the size of the object is greater than word size on this target,
7896 : a default store will not be atomic. */
7897 1603 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7898 : {
7899 : /* If loads are atomic or we are called to provide a __sync builtin,
7900 : we can try a atomic_exchange and throw away the result. Otherwise,
7901 : don't do anything so that we do not create an inconsistency between
7902 : loads and stores. */
7903 1603 : if (can_atomic_load_p (mode) || is_mm_sync (model))
7904 : {
7905 2 : rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7906 2 : if (!target)
7907 2 : target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
7908 : val);
7909 2 : if (target)
7910 0 : return const0_rtx;
7911 : }
7912 : return NULL_RTX;
7913 : }
7914 :
7915 : /* Otherwise assume stores are atomic, and emit the proper barriers. */
7916 0 : expand_mem_thread_fence (model);
7917 :
7918 0 : emit_move_insn (mem, val);
7919 :
7920 : /* For SEQ_CST, also emit a barrier after the store. */
7921 0 : if (is_mm_seq_cst (model))
7922 0 : expand_mem_thread_fence (model);
7923 :
7924 0 : return const0_rtx;
7925 : }
7926 :
7927 :
7928 : /* Structure containing the pointers and values required to process the
7929 : various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7930 :
7931 : struct atomic_op_functions
7932 : {
7933 : direct_optab mem_fetch_before;
7934 : direct_optab mem_fetch_after;
7935 : direct_optab mem_no_result;
7936 : optab fetch_before;
7937 : optab fetch_after;
7938 : direct_optab no_result;
7939 : enum rtx_code reverse_code;
7940 : };
7941 :
7942 :
7943 : /* Fill in structure pointed to by OP with the various optab entries for an
7944 : operation of type CODE. */
7945 :
7946 : static void
7947 35339 : get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7948 : {
7949 35339 : gcc_assert (op!= NULL);
7950 :
7951 : /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7952 : in the source code during compilation, and the optab entries are not
7953 : computable until runtime. Fill in the values at runtime. */
7954 35339 : switch (code)
7955 : {
7956 22773 : case PLUS:
7957 22773 : op->mem_fetch_before = atomic_fetch_add_optab;
7958 22773 : op->mem_fetch_after = atomic_add_fetch_optab;
7959 22773 : op->mem_no_result = atomic_add_optab;
7960 22773 : op->fetch_before = sync_old_add_optab;
7961 22773 : op->fetch_after = sync_new_add_optab;
7962 22773 : op->no_result = sync_add_optab;
7963 22773 : op->reverse_code = MINUS;
7964 22773 : break;
7965 4854 : case MINUS:
7966 4854 : op->mem_fetch_before = atomic_fetch_sub_optab;
7967 4854 : op->mem_fetch_after = atomic_sub_fetch_optab;
7968 4854 : op->mem_no_result = atomic_sub_optab;
7969 4854 : op->fetch_before = sync_old_sub_optab;
7970 4854 : op->fetch_after = sync_new_sub_optab;
7971 4854 : op->no_result = sync_sub_optab;
7972 4854 : op->reverse_code = PLUS;
7973 4854 : break;
7974 2244 : case XOR:
7975 2244 : op->mem_fetch_before = atomic_fetch_xor_optab;
7976 2244 : op->mem_fetch_after = atomic_xor_fetch_optab;
7977 2244 : op->mem_no_result = atomic_xor_optab;
7978 2244 : op->fetch_before = sync_old_xor_optab;
7979 2244 : op->fetch_after = sync_new_xor_optab;
7980 2244 : op->no_result = sync_xor_optab;
7981 2244 : op->reverse_code = XOR;
7982 2244 : break;
7983 2122 : case AND:
7984 2122 : op->mem_fetch_before = atomic_fetch_and_optab;
7985 2122 : op->mem_fetch_after = atomic_and_fetch_optab;
7986 2122 : op->mem_no_result = atomic_and_optab;
7987 2122 : op->fetch_before = sync_old_and_optab;
7988 2122 : op->fetch_after = sync_new_and_optab;
7989 2122 : op->no_result = sync_and_optab;
7990 2122 : op->reverse_code = UNKNOWN;
7991 2122 : break;
7992 2593 : case IOR:
7993 2593 : op->mem_fetch_before = atomic_fetch_or_optab;
7994 2593 : op->mem_fetch_after = atomic_or_fetch_optab;
7995 2593 : op->mem_no_result = atomic_or_optab;
7996 2593 : op->fetch_before = sync_old_ior_optab;
7997 2593 : op->fetch_after = sync_new_ior_optab;
7998 2593 : op->no_result = sync_ior_optab;
7999 2593 : op->reverse_code = UNKNOWN;
8000 2593 : break;
8001 753 : case NOT:
8002 753 : op->mem_fetch_before = atomic_fetch_nand_optab;
8003 753 : op->mem_fetch_after = atomic_nand_fetch_optab;
8004 753 : op->mem_no_result = atomic_nand_optab;
8005 753 : op->fetch_before = sync_old_nand_optab;
8006 753 : op->fetch_after = sync_new_nand_optab;
8007 753 : op->no_result = sync_nand_optab;
8008 753 : op->reverse_code = UNKNOWN;
8009 753 : break;
8010 0 : default:
8011 0 : gcc_unreachable ();
8012 : }
8013 35339 : }
8014 :
8015 : /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
8016 : using memory order MODEL. If AFTER is true the operation needs to return
8017 : the value of *MEM after the operation, otherwise the previous value.
8018 : TARGET is an optional place to place the result. The result is unused if
8019 : it is const0_rtx.
8020 : Return the result if there is a better sequence, otherwise NULL_RTX. */
8021 :
8022 : static rtx
8023 35337 : maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
8024 : enum memmodel model, bool after)
8025 : {
8026 : /* If the value is prefetched, or not used, it may be possible to replace
8027 : the sequence with a native exchange operation. */
8028 35337 : if (!after || target == const0_rtx)
8029 : {
8030 : /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
8031 27179 : if (code == AND && val == const0_rtx)
8032 : {
8033 327 : if (target == const0_rtx)
8034 150 : target = gen_reg_rtx (GET_MODE (mem));
8035 327 : return maybe_emit_atomic_exchange (target, mem, val, model);
8036 : }
8037 :
8038 : /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
8039 26852 : if (code == IOR && val == constm1_rtx)
8040 : {
8041 165 : if (target == const0_rtx)
8042 164 : target = gen_reg_rtx (GET_MODE (mem));
8043 165 : return maybe_emit_atomic_exchange (target, mem, val, model);
8044 : }
8045 : }
8046 :
8047 : return NULL_RTX;
8048 : }
8049 :
8050 : /* Try to emit an instruction for a specific operation variation.
8051 : OPTAB contains the OP functions.
8052 : TARGET is an optional place to return the result. const0_rtx means unused.
8053 : MEM is the memory location to operate on.
8054 : VAL is the value to use in the operation.
8055 : USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
8056 : MODEL is the memory model, if used.
8057 : AFTER is true if the returned result is the value after the operation. */
8058 :
8059 : static rtx
8060 65290 : maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
8061 : rtx val, bool use_memmodel, enum memmodel model, bool after)
8062 : {
8063 65290 : machine_mode mode = GET_MODE (mem);
8064 65290 : class expand_operand ops[4];
8065 65290 : enum insn_code icode;
8066 65290 : int op_counter = 0;
8067 65290 : int num_ops;
8068 :
8069 : /* Check to see if there is a result returned. */
8070 65290 : if (target == const0_rtx)
8071 : {
8072 14812 : if (use_memmodel)
8073 : {
8074 14453 : icode = direct_optab_handler (optab->mem_no_result, mode);
8075 14453 : create_integer_operand (&ops[2], model);
8076 14453 : num_ops = 3;
8077 : }
8078 : else
8079 : {
8080 359 : icode = direct_optab_handler (optab->no_result, mode);
8081 359 : num_ops = 2;
8082 : }
8083 : }
8084 : /* Otherwise, we need to generate a result. */
8085 : else
8086 : {
8087 50478 : if (use_memmodel)
8088 : {
8089 31470 : icode = direct_optab_handler (after ? optab->mem_fetch_after
8090 : : optab->mem_fetch_before, mode);
8091 31470 : create_integer_operand (&ops[3], model);
8092 31470 : num_ops = 4;
8093 : }
8094 : else
8095 : {
8096 19008 : icode = optab_handler (after ? optab->fetch_after
8097 : : optab->fetch_before, mode);
8098 19008 : num_ops = 3;
8099 : }
8100 50478 : create_output_operand (&ops[op_counter++], target, mode);
8101 : }
8102 65290 : if (icode == CODE_FOR_nothing)
8103 : return NULL_RTX;
8104 :
8105 26556 : create_fixed_operand (&ops[op_counter++], mem);
8106 : /* VAL may have been promoted to a wider mode. Shrink it if so. */
8107 26556 : create_convert_operand_to (&ops[op_counter++], val, mode, true);
8108 :
8109 26556 : if (maybe_expand_insn (icode, num_ops, ops))
8110 26556 : return (target == const0_rtx ? const0_rtx : ops[0].value);
8111 :
8112 : return NULL_RTX;
8113 : }
8114 :
8115 :
8116 : /* This function expands an atomic fetch_OP or OP_fetch operation:
8117 : TARGET is an option place to stick the return value. const0_rtx indicates
8118 : the result is unused.
8119 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
8120 : CODE is the operation being performed (OP)
8121 : MEMMODEL is the memory model variant to use.
8122 : AFTER is true to return the result of the operation (OP_fetch).
8123 : AFTER is false to return the value before the operation (fetch_OP).
8124 :
8125 : This function will *only* generate instructions if there is a direct
8126 : optab. No compare and swap loops or libcalls will be generated. */
8127 :
8128 : static rtx
8129 35337 : expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
8130 : enum rtx_code code, enum memmodel model,
8131 : bool after)
8132 : {
8133 35337 : machine_mode mode = GET_MODE (mem);
8134 35337 : struct atomic_op_functions optab;
8135 35337 : rtx result;
8136 35337 : bool unused_result = (target == const0_rtx);
8137 :
8138 35337 : get_atomic_op_for_code (&optab, code);
8139 :
8140 : /* Check to see if there are any better instructions. */
8141 35337 : result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
8142 35337 : if (result)
8143 : return result;
8144 :
8145 : /* Check for the case where the result isn't used and try those patterns. */
8146 34845 : if (unused_result)
8147 : {
8148 : /* Try the memory model variant first. */
8149 14453 : result = maybe_emit_op (&optab, target, mem, val, true, model, true);
8150 14453 : if (result)
8151 : return result;
8152 :
8153 : /* Next try the old style withuot a memory model. */
8154 359 : result = maybe_emit_op (&optab, target, mem, val, false, model, true);
8155 359 : if (result)
8156 : return result;
8157 :
8158 : /* There is no no-result pattern, so try patterns with a result. */
8159 : target = NULL_RTX;
8160 : }
8161 :
8162 : /* Try the __atomic version. */
8163 20751 : result = maybe_emit_op (&optab, target, mem, val, true, model, after);
8164 20751 : if (result)
8165 : return result;
8166 :
8167 : /* Try the older __sync version. */
8168 12133 : result = maybe_emit_op (&optab, target, mem, val, false, model, after);
8169 12133 : if (result)
8170 : return result;
8171 :
8172 : /* If the fetch value can be calculated from the other variation of fetch,
8173 : try that operation. */
8174 12133 : if (after || unused_result || optab.reverse_code != UNKNOWN)
8175 : {
8176 10719 : rtx_insn *start = get_last_insn ();
8177 : /* Force val into a register if it could change value when the
8178 : atomic insn updates mem. */
8179 10719 : if (!unused_result && reg_overlap_mentioned_p (mem, val))
8180 114 : val = force_reg (mode, val);
8181 :
8182 : /* Try the __atomic version, then the older __sync version. */
8183 10719 : result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
8184 10719 : if (!result)
8185 6875 : result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
8186 :
8187 6875 : if (result)
8188 : {
8189 : /* If the result isn't used, no need to do compensation code. */
8190 3844 : if (unused_result)
8191 : return result;
8192 :
8193 : /* Issue compensation code. Fetch_after == fetch_before OP val.
8194 : Fetch_before == after REVERSE_OP val. */
8195 3844 : if (!after)
8196 0 : code = optab.reverse_code;
8197 3844 : if (code == NOT)
8198 : {
8199 0 : result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
8200 : true, OPTAB_LIB_WIDEN);
8201 0 : result = expand_simple_unop (mode, NOT, result, target, true);
8202 : }
8203 : else
8204 3844 : result = expand_simple_binop (mode, code, result, val, target,
8205 : true, OPTAB_LIB_WIDEN);
8206 : return result;
8207 : }
8208 : else
8209 6875 : delete_insns_since (start);
8210 : }
8211 :
8212 : /* No direct opcode can be generated. */
8213 : return NULL_RTX;
8214 : }
8215 :
8216 :
8217 :
8218 : /* This function expands an atomic fetch_OP or OP_fetch operation:
8219 : TARGET is an option place to stick the return value. const0_rtx indicates
8220 : the result is unused.
8221 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
8222 : CODE is the operation being performed (OP)
8223 : MEMMODEL is the memory model variant to use.
8224 : AFTER is true to return the result of the operation (OP_fetch).
8225 : AFTER is false to return the value before the operation (fetch_OP). */
8226 : rtx
8227 32911 : expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
8228 : enum memmodel model, bool after)
8229 : {
8230 32911 : machine_mode mode = GET_MODE (mem);
8231 32911 : rtx result;
8232 32911 : bool unused_result = (target == const0_rtx);
8233 :
8234 : /* If loads are not atomic for the required size and we are not called to
8235 : provide a __sync builtin, do not do anything so that we stay consistent
8236 : with atomic loads of the same size. */
8237 32911 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
8238 : return NULL_RTX;
8239 :
8240 32295 : result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
8241 : after);
8242 :
8243 32295 : if (result)
8244 : return result;
8245 :
8246 : /* Add/sub can be implemented by doing the reverse operation with -(val). */
8247 8248 : if (code == PLUS || code == MINUS)
8248 : {
8249 3042 : rtx tmp;
8250 3042 : enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
8251 :
8252 3042 : start_sequence ();
8253 3042 : tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
8254 3042 : result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
8255 : model, after);
8256 3042 : if (result)
8257 : {
8258 : /* PLUS worked so emit the insns and return. */
8259 3001 : tmp = end_sequence ();
8260 3001 : emit_insn (tmp);
8261 3001 : return result;
8262 : }
8263 :
8264 : /* PLUS did not work, so throw away the negation code and continue. */
8265 41 : end_sequence ();
8266 : }
8267 :
8268 : /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
8269 5247 : if (!can_compare_and_swap_p (mode, false))
8270 : {
8271 2 : rtx libfunc;
8272 2 : bool fixup = false;
8273 2 : enum rtx_code orig_code = code;
8274 2 : struct atomic_op_functions optab;
8275 :
8276 2 : get_atomic_op_for_code (&optab, code);
8277 2 : libfunc = optab_libfunc (after ? optab.fetch_after
8278 : : optab.fetch_before, mode);
8279 2 : if (libfunc == NULL
8280 2 : && (after || unused_result || optab.reverse_code != UNKNOWN))
8281 : {
8282 2 : fixup = true;
8283 2 : if (!after)
8284 1 : code = optab.reverse_code;
8285 2 : libfunc = optab_libfunc (after ? optab.fetch_before
8286 : : optab.fetch_after, mode);
8287 : }
8288 2 : if (libfunc != NULL)
8289 : {
8290 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
8291 0 : result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
8292 : addr, ptr_mode, val, mode);
8293 :
8294 0 : if (!unused_result && fixup)
8295 0 : result = expand_simple_binop (mode, code, result, val, target,
8296 : true, OPTAB_LIB_WIDEN);
8297 0 : return result;
8298 : }
8299 :
8300 : /* We need the original code for any further attempts. */
8301 2 : code = orig_code;
8302 : }
8303 :
8304 : /* If nothing else has succeeded, default to a compare and swap loop. */
8305 5247 : if (can_compare_and_swap_p (mode, true))
8306 : {
8307 5245 : rtx_insn *insn;
8308 5245 : rtx t0 = gen_reg_rtx (mode), t1;
8309 5245 : rtx_insn *start = get_last_insn ();
8310 : /* Force val into a register if it could change value when the
8311 : atomic insn updates mem. */
8312 5245 : if (reg_overlap_mentioned_p (mem, val))
8313 128 : val = force_reg (mode, val);
8314 :
8315 5245 : start_sequence ();
8316 :
8317 : /* If the result is used, get a register for it. */
8318 5245 : if (!unused_result)
8319 : {
8320 4909 : if (!target || !register_operand (target, mode))
8321 12 : target = gen_reg_rtx (mode);
8322 : /* If fetch_before, copy the value now. */
8323 4909 : if (!after)
8324 2236 : emit_move_insn (target, t0);
8325 : }
8326 : else
8327 336 : target = const0_rtx;
8328 :
8329 5245 : t1 = t0;
8330 5245 : if (code == NOT)
8331 : {
8332 733 : t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
8333 : true, OPTAB_LIB_WIDEN);
8334 733 : t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
8335 : }
8336 : else
8337 4512 : t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
8338 : OPTAB_LIB_WIDEN);
8339 :
8340 : /* For after, copy the value now. */
8341 5245 : if (!unused_result && after)
8342 2673 : emit_move_insn (target, t1);
8343 5245 : insn = end_sequence ();
8344 :
8345 5245 : if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
8346 : return target;
8347 : else
8348 0 : delete_insns_since (start);
8349 : }
8350 :
8351 : return NULL_RTX;
8352 : }
8353 :
8354 : /* Return true if OPERAND is suitable for operand number OPNO of
8355 : instruction ICODE. */
8356 :
8357 : bool
8358 156973558 : insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
8359 : {
8360 156973558 : return (!insn_data[(int) icode].operand[opno].predicate
8361 312410141 : || (insn_data[(int) icode].operand[opno].predicate
8362 155436583 : (operand, insn_data[(int) icode].operand[opno].mode)));
8363 : }
8364 :
8365 : /* TARGET is a target of a multiword operation that we are going to
8366 : implement as a series of word-mode operations. Return true if
8367 : TARGET is suitable for this purpose. */
8368 :
8369 : bool
8370 201 : valid_multiword_target_p (rtx target)
8371 : {
8372 201 : machine_mode mode;
8373 201 : int i, size;
8374 :
8375 201 : mode = GET_MODE (target);
8376 402 : if (!GET_MODE_SIZE (mode).is_constant (&size))
8377 : return false;
8378 945 : for (i = 0; i < size; i += UNITS_PER_WORD)
8379 418 : if (!validate_subreg (word_mode, mode, target, i))
8380 : return false;
8381 : return true;
8382 : }
8383 :
8384 : /* Make OP describe an input operand that has value INTVAL and that has
8385 : no inherent mode. This function should only be used for operands that
8386 : are always expand-time constants. The backend may request that INTVAL
8387 : be copied into a different kind of rtx, but it must specify the mode
8388 : of that rtx if so. */
8389 :
8390 : void
8391 2035768 : create_integer_operand (class expand_operand *op, poly_int64 intval)
8392 : {
8393 2035768 : create_expand_operand (op, EXPAND_INTEGER,
8394 : gen_int_mode (intval, MAX_MODE_INT),
8395 : VOIDmode, false, intval);
8396 2035768 : }
8397 :
8398 : /* Like maybe_legitimize_operand, but do not change the code of the
8399 : current rtx value. */
8400 :
8401 : static bool
8402 52659540 : maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
8403 : class expand_operand *op)
8404 : {
8405 : /* See if the operand matches in its current form. */
8406 52659540 : if (insn_operand_matches (icode, opno, op->value))
8407 : return true;
8408 :
8409 : /* If the operand is a memory whose address has no side effects,
8410 : try forcing the address into a non-virtual pseudo register.
8411 : The check for side effects is important because copy_to_mode_reg
8412 : cannot handle things like auto-modified addresses. */
8413 1623520 : if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
8414 : {
8415 13134 : rtx addr, mem;
8416 :
8417 13134 : mem = op->value;
8418 13134 : addr = XEXP (mem, 0);
8419 1007 : if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
8420 13940 : && !side_effects_p (addr))
8421 : {
8422 8410 : rtx_insn *last;
8423 8410 : machine_mode mode;
8424 :
8425 8410 : last = get_last_insn ();
8426 8410 : mode = get_address_mode (mem);
8427 8410 : mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
8428 8410 : if (insn_operand_matches (icode, opno, mem))
8429 : {
8430 0 : op->value = mem;
8431 0 : return true;
8432 : }
8433 8410 : delete_insns_since (last);
8434 : }
8435 : }
8436 :
8437 : return false;
8438 : }
8439 :
8440 : /* Try to make OP match operand OPNO of instruction ICODE. Return true
8441 : on success, storing the new operand value back in OP. */
8442 :
8443 : static bool
8444 60406411 : maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
8445 : class expand_operand *op)
8446 : {
8447 60406411 : machine_mode mode, imode, tmode;
8448 :
8449 60406411 : mode = op->mode;
8450 60406411 : switch (op->type)
8451 : {
8452 4521335 : case EXPAND_FIXED:
8453 4521335 : {
8454 4521335 : temporary_volatile_ok v (true);
8455 4521335 : return maybe_legitimize_operand_same_code (icode, opno, op);
8456 4521335 : }
8457 :
8458 19433646 : case EXPAND_OUTPUT:
8459 19433646 : gcc_assert (mode != VOIDmode);
8460 19433646 : if (op->value
8461 12076238 : && op->value != const0_rtx
8462 12075809 : && GET_MODE (op->value) == mode
8463 31473606 : && maybe_legitimize_operand_same_code (icode, opno, op))
8464 : return true;
8465 :
8466 7518361 : op->value = gen_reg_rtx (mode);
8467 7518361 : op->target = 0;
8468 7518361 : break;
8469 :
8470 36098245 : case EXPAND_INPUT:
8471 36098245 : input:
8472 36098245 : gcc_assert (mode != VOIDmode);
8473 36098245 : gcc_assert (GET_MODE (op->value) == VOIDmode
8474 : || GET_MODE (op->value) == mode);
8475 36098245 : if (maybe_legitimize_operand_same_code (icode, opno, op))
8476 : return true;
8477 :
8478 1352645 : op->value = copy_to_mode_reg (mode, op->value);
8479 1352645 : break;
8480 :
8481 204849 : case EXPAND_CONVERT_TO:
8482 204849 : gcc_assert (mode != VOIDmode);
8483 204849 : op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8484 204849 : goto input;
8485 :
8486 670797 : case EXPAND_CONVERT_FROM:
8487 670797 : if (GET_MODE (op->value) != VOIDmode)
8488 460267 : mode = GET_MODE (op->value);
8489 : else
8490 : /* The caller must tell us what mode this value has. */
8491 210530 : gcc_assert (mode != VOIDmode);
8492 :
8493 670797 : imode = insn_data[(int) icode].operand[opno].mode;
8494 670797 : tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
8495 670797 : ? GET_MODE_INNER (imode) : imode);
8496 670797 : if (tmode != VOIDmode && tmode != mode)
8497 : {
8498 1752 : op->value = convert_modes (tmode, mode, op->value, op->unsigned_p);
8499 1752 : mode = tmode;
8500 : }
8501 670797 : if (imode != VOIDmode && imode != mode)
8502 : {
8503 46 : gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
8504 46 : op->value = expand_vector_broadcast (imode, op->value);
8505 46 : mode = imode;
8506 : }
8507 670797 : goto input;
8508 :
8509 3399 : case EXPAND_ADDRESS:
8510 3399 : op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
8511 : op->value);
8512 3399 : goto input;
8513 :
8514 1659851 : case EXPAND_INTEGER:
8515 1659851 : mode = insn_data[(int) icode].operand[opno].mode;
8516 1659851 : if (mode != VOIDmode
8517 1659851 : && known_eq (trunc_int_for_mode (op->int_value, mode),
8518 : op->int_value))
8519 : {
8520 1306666 : op->value = gen_int_mode (op->int_value, mode);
8521 1306666 : goto input;
8522 : }
8523 : break;
8524 :
8525 249 : case EXPAND_UNDEFINED_INPUT:
8526 : /* See if the predicate accepts a SCRATCH rtx, which in this context
8527 : indicates an undefined value. Use an uninitialized register if not. */
8528 249 : if (!insn_operand_matches (icode, opno, op->value))
8529 : {
8530 249 : op->value = gen_reg_rtx (op->mode);
8531 249 : goto input;
8532 : }
8533 : return true;
8534 : }
8535 9224191 : return insn_operand_matches (icode, opno, op->value);
8536 : }
8537 :
8538 : /* Make OP describe an input operand that should have the same value
8539 : as VALUE, after any mode conversion that the target might request.
8540 : TYPE is the type of VALUE. */
8541 :
8542 : void
8543 2702 : create_convert_operand_from_type (class expand_operand *op,
8544 : rtx value, tree type)
8545 : {
8546 2702 : create_convert_operand_from (op, value, TYPE_MODE (type),
8547 2702 : TYPE_UNSIGNED (type));
8548 2702 : }
8549 :
8550 : /* Return true if the requirements on operands OP1 and OP2 of instruction
8551 : ICODE are similar enough for the result of legitimizing OP1 to be
8552 : reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
8553 : with OP1 and OP2 respectively. */
8554 :
8555 : static inline bool
8556 67237437 : can_reuse_operands_p (enum insn_code icode,
8557 : unsigned int opno1, unsigned int opno2,
8558 : const class expand_operand *op1,
8559 : const class expand_operand *op2)
8560 : {
8561 : /* Check requirements that are common to all types. */
8562 67237437 : if (op1->type != op2->type
8563 20881684 : || op1->mode != op2->mode
8564 19324903 : || (insn_data[(int) icode].operand[opno1].mode
8565 19324903 : != insn_data[(int) icode].operand[opno2].mode))
8566 : return false;
8567 :
8568 : /* Check the requirements for specific types. */
8569 17397165 : switch (op1->type)
8570 : {
8571 : case EXPAND_OUTPUT:
8572 : case EXPAND_UNDEFINED_INPUT:
8573 : /* Outputs and undefined inputs must remain distinct. */
8574 : return false;
8575 :
8576 : case EXPAND_FIXED:
8577 : case EXPAND_INPUT:
8578 : case EXPAND_ADDRESS:
8579 : case EXPAND_INTEGER:
8580 : return true;
8581 :
8582 119497 : case EXPAND_CONVERT_TO:
8583 119497 : case EXPAND_CONVERT_FROM:
8584 119497 : return op1->unsigned_p == op2->unsigned_p;
8585 : }
8586 0 : gcc_unreachable ();
8587 : }
8588 :
8589 : /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8590 : of instruction ICODE. Return true on success, leaving the new operand
8591 : values in the OPS themselves. Emit no code on failure. */
8592 :
8593 : bool
8594 20190550 : maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8595 : unsigned int nops, class expand_operand *ops)
8596 : {
8597 20190550 : rtx_insn *last = get_last_insn ();
8598 20190550 : rtx *orig_values = XALLOCAVEC (rtx, nops);
8599 80838252 : for (unsigned int i = 0; i < nops; i++)
8600 : {
8601 60801430 : orig_values[i] = ops[i].value;
8602 :
8603 : /* First try reusing the result of an earlier legitimization.
8604 : This avoids duplicate rtl and ensures that tied operands
8605 : remain tied.
8606 :
8607 : This search is linear, but NOPS is bounded at compile time
8608 : to a small number (current a single digit). */
8609 60801430 : unsigned int j = 0;
8610 127643848 : for (; j < i; ++j)
8611 67237437 : if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
8612 17376041 : && rtx_equal_p (orig_values[j], orig_values[i])
8613 413483 : && ops[j].value
8614 67632456 : && insn_operand_matches (icode, opno + i, ops[j].value))
8615 : {
8616 395019 : ops[i].value = copy_rtx (ops[j].value);
8617 395019 : break;
8618 : }
8619 :
8620 : /* Otherwise try legitimizing the operand on its own. */
8621 60801430 : if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
8622 : {
8623 153728 : delete_insns_since (last);
8624 153728 : return false;
8625 : }
8626 : }
8627 : return true;
8628 : }
8629 :
8630 : /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8631 : as its operands. Return the instruction pattern on success,
8632 : and emit any necessary set-up code. Return null and emit no
8633 : code on failure. */
8634 :
8635 : rtx_insn *
8636 20164076 : maybe_gen_insn (enum insn_code icode, unsigned int nops,
8637 : class expand_operand *ops)
8638 : {
8639 20164076 : gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8640 20164076 : if (!maybe_legitimize_operands (icode, 0, nops, ops))
8641 : return NULL;
8642 :
8643 20010348 : switch (nops)
8644 : {
8645 0 : case 0:
8646 0 : return GEN_FCN (icode) ();
8647 1341 : case 1:
8648 1341 : return GEN_FCN (icode) (ops[0].value);
8649 2889187 : case 2:
8650 2889187 : return GEN_FCN (icode) (ops[0].value, ops[1].value);
8651 14991118 : case 3:
8652 14991118 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8653 1738204 : case 4:
8654 1738204 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8655 1738204 : ops[3].value);
8656 191523 : case 5:
8657 191523 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8658 191523 : ops[3].value, ops[4].value);
8659 322 : case 6:
8660 322 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8661 322 : ops[3].value, ops[4].value, ops[5].value);
8662 0 : case 7:
8663 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8664 0 : ops[3].value, ops[4].value, ops[5].value,
8665 0 : ops[6].value);
8666 27355 : case 8:
8667 27355 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8668 27355 : ops[3].value, ops[4].value, ops[5].value,
8669 27355 : ops[6].value, ops[7].value);
8670 171298 : case 9:
8671 171298 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8672 171298 : ops[3].value, ops[4].value, ops[5].value,
8673 171298 : ops[6].value, ops[7].value, ops[8].value);
8674 0 : case 10:
8675 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8676 0 : ops[3].value, ops[4].value, ops[5].value,
8677 0 : ops[6].value, ops[7].value, ops[8].value,
8678 0 : ops[9].value);
8679 0 : case 11:
8680 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8681 0 : ops[3].value, ops[4].value, ops[5].value,
8682 0 : ops[6].value, ops[7].value, ops[8].value,
8683 0 : ops[9].value, ops[10].value);
8684 : }
8685 0 : gcc_unreachable ();
8686 : }
8687 :
8688 : /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8689 : as its operands. Return true on success and emit no code on failure. */
8690 :
8691 : bool
8692 3023938 : maybe_expand_insn (enum insn_code icode, unsigned int nops,
8693 : class expand_operand *ops)
8694 : {
8695 3023938 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8696 3023938 : if (pat)
8697 : {
8698 2203834 : emit_insn (pat);
8699 2203834 : return true;
8700 : }
8701 : return false;
8702 : }
8703 :
8704 : /* Like maybe_expand_insn, but for jumps. */
8705 :
8706 : bool
8707 1338 : maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8708 : class expand_operand *ops)
8709 : {
8710 1338 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8711 1338 : if (pat)
8712 : {
8713 1338 : emit_jump_insn (pat);
8714 1338 : return true;
8715 : }
8716 : return false;
8717 : }
8718 :
8719 : /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8720 : as its operands. */
8721 :
8722 : void
8723 637132 : expand_insn (enum insn_code icode, unsigned int nops,
8724 : class expand_operand *ops)
8725 : {
8726 637132 : if (!maybe_expand_insn (icode, nops, ops))
8727 0 : gcc_unreachable ();
8728 637132 : }
8729 :
8730 : /* Like expand_insn, but for jumps. */
8731 :
8732 : void
8733 1338 : expand_jump_insn (enum insn_code icode, unsigned int nops,
8734 : class expand_operand *ops)
8735 : {
8736 1338 : if (!maybe_expand_jump_insn (icode, nops, ops))
8737 0 : gcc_unreachable ();
8738 1338 : }
|