Line data Source code
1 : /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "predict.h"
30 : #include "tm_p.h"
31 : #include "optabs.h"
32 : #include "expmed.h"
33 : #include "emit-rtl.h"
34 : #include "recog.h"
35 : #include "diagnostic-core.h"
36 : #include "rtx-vector-builder.h"
37 :
38 : /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 : is properly defined. */
40 : #include "stor-layout.h"
41 : #include "except.h"
42 : #include "dojump.h"
43 : #include "explow.h"
44 : #include "expr.h"
45 : #include "optabs-tree.h"
46 : #include "libfuncs.h"
47 : #include "internal-fn.h"
48 : #include "langhooks.h"
49 : #include "gimple.h"
50 : #include "ssa.h"
51 : #include "tree-ssa-live.h"
52 : #include "tree-outof-ssa.h"
53 :
54 : static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
55 : machine_mode *);
56 : static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
57 : static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
58 :
59 : static rtx emit_conditional_move_1 (rtx, rtx, rtx, rtx, machine_mode);
60 :
61 : /* Debug facility for use in GDB. */
62 : void debug_optab_libfuncs (void);
63 :
64 : /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
65 : the result of operation CODE applied to OP0 (and OP1 if it is a binary
66 : operation). OP0_MODE is OP0's mode.
67 :
68 : If the last insn does not set TARGET, don't do anything, but return true.
69 :
70 : If the last insn or a previous insn sets TARGET and TARGET is one of OP0
71 : or OP1, don't add the REG_EQUAL note but return false. Our caller can then
72 : try again, ensuring that TARGET is not one of the operands. */
73 :
74 : static bool
75 1269919 : add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
76 : rtx op1, machine_mode op0_mode)
77 : {
78 1269919 : rtx_insn *last_insn;
79 1269919 : rtx set;
80 1269919 : rtx note;
81 :
82 2539838 : gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
83 :
84 1269919 : if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
85 : && GET_RTX_CLASS (code) != RTX_BIN_ARITH
86 : && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
87 : && GET_RTX_CLASS (code) != RTX_COMPARE
88 : && GET_RTX_CLASS (code) != RTX_UNARY)
89 : return true;
90 :
91 1262627 : if (GET_CODE (target) == ZERO_EXTRACT)
92 : return true;
93 :
94 : for (last_insn = insns;
95 2782406 : NEXT_INSN (last_insn) != NULL_RTX;
96 : last_insn = NEXT_INSN (last_insn))
97 : ;
98 :
99 : /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
100 : a value changing in the insn, so the note would be invalid for CSE. */
101 1262627 : if (reg_overlap_mentioned_p (target, op0)
102 1262627 : || (op1 && reg_overlap_mentioned_p (target, op1)))
103 : {
104 57155 : if (MEM_P (target)
105 57155 : && (rtx_equal_p (target, op0)
106 42762 : || (op1 && rtx_equal_p (target, op1))))
107 : {
108 : /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
109 : over expanding it as temp = MEM op X, MEM = temp. If the target
110 : supports MEM = MEM op X instructions, it is sometimes too hard
111 : to reconstruct that form later, especially if X is also a memory,
112 : and due to multiple occurrences of addresses the address might
113 : be forced into register unnecessarily.
114 : Note that not emitting the REG_EQUIV note might inhibit
115 : CSE in some cases. */
116 3507 : set = single_set (last_insn);
117 3507 : if (set
118 3507 : && GET_CODE (SET_SRC (set)) == code
119 3483 : && MEM_P (SET_DEST (set))
120 6990 : && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
121 0 : || (op1 && rtx_equal_p (SET_DEST (set),
122 0 : XEXP (SET_SRC (set), 1)))))
123 : return true;
124 : }
125 53672 : return false;
126 : }
127 :
128 1205472 : set = set_for_reg_notes (last_insn);
129 1205472 : if (set == NULL_RTX)
130 : return true;
131 :
132 734431 : if (! rtx_equal_p (SET_DEST (set), target)
133 : /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
134 734431 : && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
135 0 : || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
136 : return true;
137 :
138 733598 : if (GET_RTX_CLASS (code) == RTX_UNARY)
139 43202 : switch (code)
140 : {
141 3113 : case FFS:
142 3113 : case CLZ:
143 3113 : case CTZ:
144 3113 : case CLRSB:
145 3113 : case POPCOUNT:
146 3113 : case PARITY:
147 3113 : case BSWAP:
148 3113 : if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
149 : {
150 250 : note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
151 250 : if (GET_MODE_UNIT_SIZE (op0_mode)
152 500 : > GET_MODE_UNIT_SIZE (GET_MODE (target)))
153 250 : note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
154 : note, op0_mode);
155 : else
156 0 : note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
157 : note, op0_mode);
158 : break;
159 : }
160 : /* FALLTHRU */
161 42952 : default:
162 42952 : note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
163 42952 : break;
164 : }
165 : else
166 690396 : note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
167 :
168 733598 : set_unique_reg_note (last_insn, REG_EQUAL, note);
169 :
170 733598 : return true;
171 : }
172 :
173 : /* Given two input operands, OP0 and OP1, determine what the correct from_mode
174 : for a widening operation would be. In most cases this would be OP0, but if
175 : that's a constant it'll be VOIDmode, which isn't useful. */
176 :
177 : static machine_mode
178 37350 : widened_mode (machine_mode to_mode, rtx op0, rtx op1)
179 : {
180 37350 : machine_mode m0 = GET_MODE (op0);
181 37350 : machine_mode m1 = GET_MODE (op1);
182 37350 : machine_mode result;
183 :
184 37350 : if (m0 == VOIDmode && m1 == VOIDmode)
185 : return to_mode;
186 74700 : else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
187 : result = m1;
188 : else
189 : result = m0;
190 :
191 112050 : if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
192 0 : return to_mode;
193 :
194 : return result;
195 : }
196 :
197 : /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
198 : says whether OP is signed or unsigned. NO_EXTEND is true if we need
199 : not actually do a sign-extend or zero-extend, but can leave the
200 : higher-order bits of the result rtx undefined, for example, in the case
201 : of logical operations, but not right shifts. */
202 :
203 : static rtx
204 555 : widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
205 : int unsignedp, bool no_extend)
206 : {
207 555 : rtx result;
208 555 : scalar_int_mode int_mode;
209 :
210 : /* If we don't have to extend and this is a constant, return it. */
211 555 : if (no_extend && GET_MODE (op) == VOIDmode)
212 : return op;
213 :
214 : /* If we must extend do so. If OP is a SUBREG for a promoted object, also
215 : extend since it will be more efficient to do so unless the signedness of
216 : a promoted object differs from our extension. */
217 7 : if (! no_extend
218 555 : || !is_a <scalar_int_mode> (mode, &int_mode)
219 7 : || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
220 0 : && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
221 548 : return convert_modes (mode, oldmode, op, unsignedp);
222 :
223 : /* If MODE is no wider than a single word, we return a lowpart or paradoxical
224 : SUBREG. */
225 14 : if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
226 7 : return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
227 :
228 : /* Otherwise, get an object of MODE, clobber it, and set the low-order
229 : part to OP. */
230 :
231 0 : result = gen_reg_rtx (int_mode);
232 0 : emit_clobber (result);
233 0 : emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
234 0 : return result;
235 : }
236 :
237 : /* Expand vector widening operations.
238 :
239 : There are two different classes of operations handled here:
240 : 1) Operations whose result is wider than all the arguments to the operation.
241 : Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
242 : In this case OP0 and optionally OP1 would be initialized,
243 : but WIDE_OP wouldn't (not relevant for this case).
244 : 2) Operations whose result is of the same size as the last argument to the
245 : operation, but wider than all the other arguments to the operation.
246 : Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
247 : In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
248 :
249 : E.g, when called to expand the following operations, this is how
250 : the arguments will be initialized:
251 : nops OP0 OP1 WIDE_OP
252 : widening-sum 2 oprnd0 - oprnd1
253 : widening-dot-product 3 oprnd0 oprnd1 oprnd2
254 : widening-mult 2 oprnd0 oprnd1 -
255 : type-promotion (vec-unpack) 1 oprnd0 - - */
256 :
257 : rtx
258 21290 : expand_widen_pattern_expr (const_sepops ops, rtx op0, rtx op1, rtx wide_op,
259 : rtx target, int unsignedp)
260 : {
261 21290 : class expand_operand eops[4];
262 21290 : tree oprnd0, oprnd1, oprnd2;
263 21290 : machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
264 21290 : optab widen_pattern_optab;
265 21290 : enum insn_code icode;
266 21290 : int nops = TREE_CODE_LENGTH (ops->code);
267 21290 : int op;
268 21290 : bool sbool = false;
269 :
270 21290 : oprnd0 = ops->op0;
271 21290 : oprnd1 = nops >= 2 ? ops->op1 : NULL_TREE;
272 1555 : oprnd2 = nops >= 3 ? ops->op2 : NULL_TREE;
273 :
274 21290 : tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
275 21290 : if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
276 21290 : || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
277 : /* The sign is from the result type rather than operand's type
278 : for these ops. */
279 34 : widen_pattern_optab
280 34 : = optab_for_tree_code (ops->code, ops->type, optab_default);
281 21256 : else if ((ops->code == VEC_UNPACK_HI_EXPR
282 21256 : || ops->code == VEC_UNPACK_LO_EXPR)
283 17941 : && VECTOR_BOOLEAN_TYPE_P (ops->type)
284 4576 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
285 4576 : && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
286 21756 : && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
287 : {
288 : /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
289 : the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
290 : vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
291 : the pattern number of elements in the wider vector. */
292 500 : widen_pattern_optab
293 247 : = (ops->code == VEC_UNPACK_HI_EXPR
294 500 : ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
295 : sbool = true;
296 : }
297 20756 : else if (ops->code == DOT_PROD_EXPR)
298 : {
299 441 : enum optab_subtype subtype = optab_default;
300 441 : signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0));
301 441 : signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1));
302 441 : if (sign1 == sign2)
303 : ;
304 82 : else if (sign1 == SIGNED && sign2 == UNSIGNED)
305 : {
306 : subtype = optab_vector_mixed_sign;
307 : /* Same as optab_vector_mixed_sign but flip the operands. */
308 : std::swap (op0, op1);
309 : }
310 60 : else if (sign1 == UNSIGNED && sign2 == SIGNED)
311 : subtype = optab_vector_mixed_sign;
312 : else
313 0 : gcc_unreachable ();
314 :
315 441 : widen_pattern_optab
316 441 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype);
317 : }
318 : else
319 20315 : widen_pattern_optab
320 20315 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
321 21290 : if (ops->code == WIDEN_MULT_PLUS_EXPR
322 21290 : || ops->code == WIDEN_MULT_MINUS_EXPR
323 21290 : || ops->code == DOT_PROD_EXPR)
324 441 : icode = find_widening_optab_handler (widen_pattern_optab,
325 : TYPE_MODE (TREE_TYPE (ops->op2)),
326 : tmode0);
327 20849 : else if (ops->code == WIDEN_SUM_EXPR)
328 0 : icode = find_widening_optab_handler (widen_pattern_optab,
329 : TYPE_MODE (TREE_TYPE (ops->op1)),
330 : tmode0);
331 : else
332 20849 : icode = optab_handler (widen_pattern_optab, tmode0);
333 21290 : gcc_assert (icode != CODE_FOR_nothing);
334 :
335 21290 : if (nops >= 2)
336 1555 : tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
337 19735 : else if (sbool)
338 : {
339 500 : nops = 2;
340 500 : op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
341 500 : tmode1 = tmode0;
342 : }
343 :
344 : /* The last operand is of a wider mode than the rest of the operands. */
345 21290 : if (nops == 2)
346 : wmode = tmode1;
347 19795 : else if (nops == 3)
348 : {
349 560 : gcc_assert (tmode1 == tmode0);
350 560 : gcc_assert (op1);
351 560 : wmode = TYPE_MODE (TREE_TYPE (oprnd2));
352 : }
353 :
354 21290 : op = 0;
355 21290 : create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
356 21290 : create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
357 21290 : if (op1)
358 2055 : create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
359 21290 : if (wide_op)
360 560 : create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
361 21290 : expand_insn (icode, op, eops);
362 21290 : return eops[0].value;
363 : }
364 :
365 : /* Generate code to perform an operation specified by TERNARY_OPTAB
366 : on operands OP0, OP1 and OP2, with result having machine-mode MODE.
367 :
368 : UNSIGNEDP is for the case where we have to widen the operands
369 : to perform the operation. It says to use zero-extension.
370 :
371 : If TARGET is nonzero, the value
372 : is generated there, if it is convenient to do so.
373 : In all cases an rtx is returned for the locus of the value;
374 : this may or may not be TARGET. */
375 :
376 : rtx
377 0 : expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
378 : rtx op1, rtx op2, rtx target, int unsignedp)
379 : {
380 0 : class expand_operand ops[4];
381 0 : enum insn_code icode = optab_handler (ternary_optab, mode);
382 :
383 0 : gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
384 :
385 0 : create_output_operand (&ops[0], target, mode);
386 0 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
387 0 : create_convert_operand_from (&ops[2], op1, mode, unsignedp);
388 0 : create_convert_operand_from (&ops[3], op2, mode, unsignedp);
389 0 : expand_insn (icode, 4, ops);
390 0 : return ops[0].value;
391 : }
392 :
393 :
394 : /* Like expand_binop, but return a constant rtx if the result can be
395 : calculated at compile time. The arguments and return value are
396 : otherwise the same as for expand_binop. */
397 :
398 : rtx
399 1568 : simplify_expand_binop (machine_mode mode, optab binoptab,
400 : rtx op0, rtx op1, rtx target, int unsignedp,
401 : enum optab_methods methods)
402 : {
403 1568 : if (CONSTANT_P (op0) && CONSTANT_P (op1))
404 : {
405 184 : rtx x = simplify_binary_operation (optab_to_code (binoptab),
406 : mode, op0, op1);
407 184 : if (x)
408 : return x;
409 : }
410 :
411 1384 : return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
412 : }
413 :
414 : /* Like simplify_expand_binop, but always put the result in TARGET.
415 : Return true if the expansion succeeded. */
416 :
417 : bool
418 1200 : force_expand_binop (machine_mode mode, optab binoptab,
419 : rtx op0, rtx op1, rtx target, int unsignedp,
420 : enum optab_methods methods)
421 : {
422 1200 : rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
423 : target, unsignedp, methods);
424 1200 : if (x == 0)
425 : return false;
426 1200 : if (x != target)
427 0 : emit_move_insn (target, x);
428 : return true;
429 : }
430 :
431 : /* Create a new vector value in VMODE with all elements set to OP. If OP
432 : is not a constant, the mode of it must be the element mode of VMODE
433 : (if the element is BImode, additionally OP is allowed to be in QImode).
434 : If OP is a constant, then the return value will be a constant. */
435 :
436 : rtx
437 46 : expand_vector_broadcast (machine_mode vmode, rtx op)
438 : {
439 46 : int n;
440 46 : rtvec vec;
441 :
442 46 : gcc_checking_assert (VECTOR_MODE_P (vmode));
443 46 : gcc_checking_assert (CONST_INT_P (op)
444 : || GET_MODE_INNER (vmode) == GET_MODE (op)
445 : || (GET_MODE_INNER (vmode) == BImode
446 : && GET_MODE (op) == QImode));
447 :
448 46 : if (valid_for_const_vector_p (vmode, op))
449 46 : return gen_const_vec_duplicate (vmode, op);
450 :
451 0 : insn_code icode = optab_handler (vec_duplicate_optab, vmode);
452 0 : if (icode != CODE_FOR_nothing)
453 : {
454 0 : class expand_operand ops[2];
455 0 : create_output_operand (&ops[0], NULL_RTX, vmode);
456 0 : create_input_operand (&ops[1], op, GET_MODE (op));
457 0 : expand_insn (icode, 2, ops);
458 0 : return ops[0].value;
459 : }
460 :
461 0 : if (!GET_MODE_NUNITS (vmode).is_constant (&n))
462 : return NULL;
463 :
464 : /* ??? If the target doesn't have a vec_init, then we have no easy way
465 : of performing this operation. Most of this sort of generic support
466 : is hidden away in the vector lowering support in gimple. */
467 0 : icode = convert_optab_handler (vec_init_optab, vmode,
468 0 : GET_MODE_INNER (vmode));
469 0 : if (icode == CODE_FOR_nothing)
470 : return NULL;
471 :
472 0 : vec = rtvec_alloc (n);
473 0 : for (int i = 0; i < n; ++i)
474 0 : RTVEC_ELT (vec, i) = op;
475 0 : rtx ret = gen_reg_rtx (vmode);
476 0 : emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
477 :
478 0 : return ret;
479 : }
480 :
481 : /* This subroutine of expand_doubleword_shift handles the cases in which
482 : the effective shift value is >= BITS_PER_WORD. The arguments and return
483 : value are the same as for the parent routine, except that SUPERWORD_OP1
484 : is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
485 : INTO_TARGET may be null if the caller has decided to calculate it. */
486 :
487 : static bool
488 0 : expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
489 : rtx outof_target, rtx into_target,
490 : int unsignedp, enum optab_methods methods)
491 : {
492 0 : if (into_target != 0)
493 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
494 : into_target, unsignedp, methods))
495 : return false;
496 :
497 0 : if (outof_target != 0)
498 : {
499 : /* For a signed right shift, we must fill OUTOF_TARGET with copies
500 : of the sign bit, otherwise we must fill it with zeros. */
501 0 : if (binoptab != ashr_optab)
502 0 : emit_move_insn (outof_target, CONST0_RTX (word_mode));
503 : else
504 0 : if (!force_expand_binop (word_mode, binoptab, outof_input,
505 : gen_int_shift_amount (word_mode,
506 0 : BITS_PER_WORD - 1),
507 : outof_target, unsignedp, methods))
508 : return false;
509 : }
510 : return true;
511 : }
512 :
513 : /* This subroutine of expand_doubleword_shift handles the cases in which
514 : the effective shift value is < BITS_PER_WORD. The arguments and return
515 : value are the same as for the parent routine. */
516 :
517 : static bool
518 0 : expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
519 : rtx outof_input, rtx into_input, rtx op1,
520 : rtx outof_target, rtx into_target,
521 : int unsignedp, enum optab_methods methods,
522 : unsigned HOST_WIDE_INT shift_mask)
523 : {
524 0 : optab reverse_unsigned_shift, unsigned_shift;
525 0 : rtx tmp, carries;
526 :
527 0 : reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
528 0 : unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
529 :
530 : /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
531 : We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
532 : the opposite direction to BINOPTAB. */
533 0 : if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
534 : {
535 0 : carries = outof_input;
536 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
537 : op1_mode), op1_mode);
538 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
539 : 0, true, methods);
540 : }
541 : else
542 : {
543 : /* We must avoid shifting by BITS_PER_WORD bits since that is either
544 : the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
545 : has unknown behavior. Do a single shift first, then shift by the
546 : remainder. It's OK to use ~OP1 as the remainder if shift counts
547 : are truncated to the mode size. */
548 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
549 : outof_input, const1_rtx, 0,
550 : unsignedp, methods);
551 0 : if (carries == const0_rtx)
552 : tmp = const0_rtx;
553 0 : else if (shift_mask == BITS_PER_WORD - 1)
554 0 : tmp = expand_unop (op1_mode, one_cmpl_optab, op1, 0, true);
555 : else
556 : {
557 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
558 : op1_mode), op1_mode);
559 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
560 : 0, true, methods);
561 : }
562 : }
563 0 : if (tmp == 0 || carries == 0)
564 : return false;
565 0 : if (carries != const0_rtx && tmp != const0_rtx)
566 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
567 : carries, tmp, 0, unsignedp, methods);
568 0 : if (carries == 0)
569 : return false;
570 :
571 0 : if (into_input != const0_rtx)
572 : {
573 : /* Shift INTO_INPUT logically by OP1. This is the last use of
574 : INTO_INPUT so the result can go directly into INTO_TARGET if
575 : convenient. */
576 0 : tmp = simplify_expand_binop (word_mode, unsigned_shift, into_input,
577 : op1, into_target, unsignedp, methods);
578 0 : if (tmp == 0)
579 : return false;
580 :
581 : /* Now OR/PLUS in the bits carried over from OUTOF_INPUT. */
582 0 : if (!force_expand_binop (word_mode, add_optab, tmp, carries,
583 : into_target, unsignedp, methods))
584 : return false;
585 : }
586 : else
587 0 : emit_move_insn (into_target, carries);
588 :
589 : /* Use a standard word_mode shift for the out-of half. */
590 0 : if (outof_target != 0)
591 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
592 : outof_target, unsignedp, methods))
593 : return false;
594 :
595 : return true;
596 : }
597 :
598 :
599 : /* Try implementing expand_doubleword_shift using conditional moves.
600 : The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
601 : otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
602 : are the shift counts to use in the former and latter case. All other
603 : arguments are the same as the parent routine. */
604 :
605 : static bool
606 0 : expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
607 : enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
608 : rtx outof_input, rtx into_input,
609 : rtx subword_op1, rtx superword_op1,
610 : rtx outof_target, rtx into_target,
611 : int unsignedp, enum optab_methods methods,
612 : unsigned HOST_WIDE_INT shift_mask)
613 : {
614 0 : rtx outof_superword, into_superword;
615 :
616 : /* Put the superword version of the output into OUTOF_SUPERWORD and
617 : INTO_SUPERWORD. */
618 0 : outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
619 0 : if (outof_target != 0 && subword_op1 == superword_op1)
620 : {
621 : /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
622 : OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
623 0 : into_superword = outof_target;
624 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
625 : outof_superword, 0, unsignedp, methods))
626 : return false;
627 : }
628 : else
629 : {
630 0 : into_superword = gen_reg_rtx (word_mode);
631 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
632 : outof_superword, into_superword,
633 : unsignedp, methods))
634 : return false;
635 : }
636 :
637 : /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
638 0 : if (!expand_subword_shift (op1_mode, binoptab,
639 : outof_input, into_input, subword_op1,
640 : outof_target, into_target,
641 : unsignedp, methods, shift_mask))
642 : return false;
643 :
644 : /* Select between them. Do the INTO half first because INTO_SUPERWORD
645 : might be the current value of OUTOF_TARGET. */
646 0 : if (!emit_conditional_move (into_target, { cmp_code, cmp1, cmp2, op1_mode },
647 : into_target, into_superword, word_mode, false))
648 : return false;
649 :
650 0 : if (outof_target != 0)
651 0 : if (!emit_conditional_move (outof_target,
652 0 : { cmp_code, cmp1, cmp2, op1_mode },
653 : outof_target, outof_superword,
654 : word_mode, false))
655 0 : return false;
656 :
657 : return true;
658 : }
659 :
660 : /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
661 : OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
662 : input operand; the shift moves bits in the direction OUTOF_INPUT->
663 : INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
664 : of the target. OP1 is the shift count and OP1_MODE is its mode.
665 : If OP1 is constant, it will have been truncated as appropriate
666 : and is known to be nonzero.
667 :
668 : If SHIFT_MASK is zero, the result of word shifts is undefined when the
669 : shift count is outside the range [0, BITS_PER_WORD). This routine must
670 : avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
671 :
672 : If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
673 : masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
674 : fill with zeros or sign bits as appropriate.
675 :
676 : If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
677 : a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
678 : Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
679 : In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
680 : are undefined.
681 :
682 : BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
683 : may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
684 : OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
685 : function wants to calculate it itself.
686 :
687 : Return true if the shift could be successfully synthesized. */
688 :
689 : static bool
690 0 : expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
691 : rtx outof_input, rtx into_input, rtx op1,
692 : rtx outof_target, rtx into_target,
693 : int unsignedp, enum optab_methods methods,
694 : unsigned HOST_WIDE_INT shift_mask)
695 : {
696 0 : rtx superword_op1, tmp, cmp1, cmp2;
697 0 : enum rtx_code cmp_code;
698 :
699 : /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
700 : fill the result with sign or zero bits as appropriate. If so, the value
701 : of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
702 : this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
703 : and INTO_INPUT), then emit code to set up OUTOF_TARGET.
704 :
705 : This isn't worthwhile for constant shifts since the optimizers will
706 : cope better with in-range shift counts. */
707 0 : if (shift_mask >= BITS_PER_WORD
708 0 : && outof_target != 0
709 0 : && !CONSTANT_P (op1))
710 : {
711 0 : if (!expand_doubleword_shift (op1_mode, binoptab,
712 : outof_input, into_input, op1,
713 : 0, into_target,
714 : unsignedp, methods, shift_mask))
715 : return false;
716 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
717 : outof_target, unsignedp, methods))
718 : return false;
719 : return true;
720 : }
721 :
722 : /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
723 : is true when the effective shift value is less than BITS_PER_WORD.
724 : Set SUPERWORD_OP1 to the shift count that should be used to shift
725 : OUTOF_INPUT into INTO_TARGET when the condition is false. */
726 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
727 0 : if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
728 : {
729 : /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
730 : is a subword shift count. */
731 0 : cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
732 : 0, true, methods);
733 0 : cmp2 = CONST0_RTX (op1_mode);
734 0 : cmp_code = EQ;
735 0 : superword_op1 = op1;
736 : }
737 : else
738 : {
739 : /* Set CMP1 to OP1 - BITS_PER_WORD. */
740 0 : cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
741 : 0, true, methods);
742 0 : cmp2 = CONST0_RTX (op1_mode);
743 0 : cmp_code = LT;
744 0 : superword_op1 = cmp1;
745 : }
746 0 : if (cmp1 == 0)
747 : return false;
748 :
749 : /* If we can compute the condition at compile time, pick the
750 : appropriate subroutine. */
751 0 : tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
752 0 : if (tmp != 0 && CONST_INT_P (tmp))
753 : {
754 0 : if (tmp == const0_rtx)
755 0 : return expand_superword_shift (binoptab, outof_input, superword_op1,
756 : outof_target, into_target,
757 0 : unsignedp, methods);
758 : else
759 0 : return expand_subword_shift (op1_mode, binoptab,
760 : outof_input, into_input, op1,
761 : outof_target, into_target,
762 0 : unsignedp, methods, shift_mask);
763 : }
764 :
765 : /* Try using conditional moves to generate straight-line code. */
766 0 : if (HAVE_conditional_move)
767 : {
768 0 : rtx_insn *start = get_last_insn ();
769 0 : if (expand_doubleword_shift_condmove (op1_mode, binoptab,
770 : cmp_code, cmp1, cmp2,
771 : outof_input, into_input,
772 : op1, superword_op1,
773 : outof_target, into_target,
774 : unsignedp, methods, shift_mask))
775 : return true;
776 0 : delete_insns_since (start);
777 : }
778 :
779 : /* As a last resort, use branches to select the correct alternative. */
780 0 : rtx_code_label *subword_label = gen_label_rtx ();
781 0 : rtx_code_label *done_label = gen_label_rtx ();
782 :
783 0 : NO_DEFER_POP;
784 0 : do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
785 : 0, 0, subword_label,
786 : profile_probability::uninitialized ());
787 0 : OK_DEFER_POP;
788 :
789 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
790 : outof_target, into_target,
791 : unsignedp, methods))
792 : return false;
793 :
794 0 : emit_jump_insn (targetm.gen_jump (done_label));
795 0 : emit_barrier ();
796 0 : emit_label (subword_label);
797 :
798 0 : if (!expand_subword_shift (op1_mode, binoptab,
799 : outof_input, into_input, op1,
800 : outof_target, into_target,
801 : unsignedp, methods, shift_mask))
802 : return false;
803 :
804 0 : emit_label (done_label);
805 0 : return true;
806 : }
807 :
808 : /* Subroutine of expand_binop. Perform a double word multiplication of
809 : operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
810 : as the target's word_mode. This function return NULL_RTX if anything
811 : goes wrong, in which case it may have already emitted instructions
812 : which need to be deleted.
813 :
814 : If we want to multiply two two-word values and have normal and widening
815 : multiplies of single-word values, we can do this with three smaller
816 : multiplications.
817 :
818 : The multiplication proceeds as follows:
819 : _______________________
820 : [__op0_high_|__op0_low__]
821 : _______________________
822 : * [__op1_high_|__op1_low__]
823 : _______________________________________________
824 : _______________________
825 : (1) [__op0_low__*__op1_low__]
826 : _______________________
827 : (2a) [__op0_low__*__op1_high_]
828 : _______________________
829 : (2b) [__op0_high_*__op1_low__]
830 : _______________________
831 : (3) [__op0_high_*__op1_high_]
832 :
833 :
834 : This gives a 4-word result. Since we are only interested in the
835 : lower 2 words, partial result (3) and the upper words of (2a) and
836 : (2b) don't need to be calculated. Hence (2a) and (2b) can be
837 : calculated using non-widening multiplication.
838 :
839 : (1), however, needs to be calculated with an unsigned widening
840 : multiplication. If this operation is not directly supported we
841 : try using a signed widening multiplication and adjust the result.
842 : This adjustment works as follows:
843 :
844 : If both operands are positive then no adjustment is needed.
845 :
846 : If the operands have different signs, for example op0_low < 0 and
847 : op1_low >= 0, the instruction treats the most significant bit of
848 : op0_low as a sign bit instead of a bit with significance
849 : 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
850 : with 2**BITS_PER_WORD - op0_low, and two's complements the
851 : result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
852 : the result.
853 :
854 : Similarly, if both operands are negative, we need to add
855 : (op0_low + op1_low) * 2**BITS_PER_WORD.
856 :
857 : We use a trick to adjust quickly. We logically shift op0_low right
858 : (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
859 : op0_high (op1_high) before it is used to calculate 2b (2a). If no
860 : logical shift exists, we do an arithmetic right shift and subtract
861 : the 0 or -1. */
862 :
863 : static rtx
864 14948 : expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
865 : bool umulp, enum optab_methods methods)
866 : {
867 14948 : int low = (WORDS_BIG_ENDIAN ? 1 : 0);
868 14948 : int high = (WORDS_BIG_ENDIAN ? 0 : 1);
869 14948 : rtx wordm1 = (umulp ? NULL_RTX
870 0 : : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
871 14948 : rtx product, adjust, product_high, temp;
872 :
873 14948 : rtx op0_high = operand_subword_force (op0, high, mode);
874 14948 : rtx op0_low = operand_subword_force (op0, low, mode);
875 14948 : rtx op1_high = operand_subword_force (op1, high, mode);
876 14948 : rtx op1_low = operand_subword_force (op1, low, mode);
877 :
878 : /* If we're using an unsigned multiply to directly compute the product
879 : of the low-order words of the operands and perform any required
880 : adjustments of the operands, we begin by trying two more multiplications
881 : and then computing the appropriate sum.
882 :
883 : We have checked above that the required addition is provided.
884 : Full-word addition will normally always succeed, especially if
885 : it is provided at all, so we don't worry about its failure. The
886 : multiplication may well fail, however, so we do handle that. */
887 :
888 14948 : if (!umulp)
889 : {
890 : /* ??? This could be done with emit_store_flag where available. */
891 0 : temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
892 : NULL_RTX, 1, methods);
893 0 : if (temp)
894 0 : op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
895 : NULL_RTX, 0, OPTAB_DIRECT);
896 : else
897 : {
898 0 : temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
899 : NULL_RTX, 0, methods);
900 0 : if (!temp)
901 : return NULL_RTX;
902 0 : op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
903 : NULL_RTX, 0, OPTAB_DIRECT);
904 : }
905 :
906 0 : if (!op0_high)
907 : return NULL_RTX;
908 : }
909 :
910 14948 : if (op1_low == const1_rtx)
911 : adjust = op0_high;
912 14908 : else if (op1_low == const0_rtx)
913 : adjust = const0_rtx;
914 14833 : else if (op1_low == const2_rtx)
915 0 : adjust = expand_binop (word_mode, add_optab, op0_high, op0_high,
916 : NULL_RTX, 0, OPTAB_DIRECT);
917 : else
918 14833 : adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
919 : NULL_RTX, 0, OPTAB_DIRECT);
920 14948 : if (!adjust)
921 : return NULL_RTX;
922 :
923 : /* OP0_HIGH should now be dead. */
924 :
925 14948 : if (!umulp)
926 : {
927 : /* ??? This could be done with emit_store_flag where available. */
928 0 : temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
929 : NULL_RTX, 1, methods);
930 0 : if (temp)
931 0 : op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
932 : NULL_RTX, 0, OPTAB_DIRECT);
933 : else
934 : {
935 0 : temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
936 : NULL_RTX, 0, methods);
937 0 : if (!temp)
938 : return NULL_RTX;
939 0 : op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
940 : NULL_RTX, 0, OPTAB_DIRECT);
941 : }
942 :
943 0 : if (!op1_high)
944 : return NULL_RTX;
945 : }
946 :
947 14948 : if (op1_high == const1_rtx)
948 : temp = op0_low;
949 14947 : else if (op1_high == const0_rtx)
950 : temp = const0_rtx;
951 14135 : else if (op1_high == const2_rtx)
952 0 : temp = expand_binop (word_mode, add_optab, op0_low, op0_low,
953 : NULL_RTX, 0, OPTAB_DIRECT);
954 : else
955 14135 : temp = expand_binop (word_mode, smul_optab, op0_low, op1_high,
956 : NULL_RTX, 0, OPTAB_DIRECT);
957 14948 : if (!temp)
958 : return NULL_RTX;
959 :
960 : /* OP1_HIGH should now be dead. */
961 :
962 14948 : adjust = expand_binop (word_mode, add_optab, adjust, temp,
963 : NULL_RTX, 0, OPTAB_DIRECT);
964 :
965 14948 : if (target && !REG_P (target))
966 8491 : target = NULL_RTX;
967 :
968 : /* *_widen_optab needs to determine operand mode, make sure at least
969 : one operand has non-VOID mode. */
970 14948 : if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
971 0 : op0_low = force_reg (word_mode, op0_low);
972 :
973 14948 : if (op1_low == const1_rtx)
974 40 : product = convert_modes (mode, word_mode, op0_low, umulp);
975 14908 : else if (umulp)
976 14908 : product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
977 : target, 1, OPTAB_DIRECT);
978 : else
979 0 : product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
980 : target, 1, OPTAB_DIRECT);
981 :
982 14948 : if (!product)
983 : return NULL_RTX;
984 :
985 14948 : product_high = operand_subword (product, high, 1, mode);
986 14948 : adjust = expand_binop (word_mode, add_optab, product_high, adjust,
987 : NULL_RTX, 0, OPTAB_DIRECT);
988 14948 : emit_move_insn (product_high, adjust);
989 14948 : return product;
990 : }
991 :
992 : /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
993 : constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
994 : (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
995 : computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
996 : + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
997 : depends on the bit value, if 2, then carry from the addition needs to be
998 : added too, i.e. like:
999 : sum += __builtin_add_overflow (low, high, &sum)
1000 :
1001 : Optimize signed double-word OP0 % OP1 similarly, just apply some correction
1002 : factor to the sum before doing unsigned remainder, in the form of
1003 : sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
1004 : then perform unsigned
1005 : remainder = sum % OP1;
1006 : and finally
1007 : remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
1008 :
1009 : static rtx
1010 21055 : expand_doubleword_mod (machine_mode mode, rtx op0, rtx op1, bool unsignedp)
1011 : {
1012 21055 : if (INTVAL (op1) <= 1 || (INTVAL (op1) & 1) == 0)
1013 : return NULL_RTX;
1014 :
1015 20659 : rtx_insn *last = get_last_insn ();
1016 680596 : for (int bit = BITS_PER_WORD; bit >= BITS_PER_WORD / 2; bit--)
1017 : {
1018 659206 : wide_int w = wi::shifted_mask (bit, 1, false, 2 * BITS_PER_WORD);
1019 627551 : if (wi::ne_p (wi::umod_trunc (w, INTVAL (op1)), 1))
1020 626217 : continue;
1021 1334 : rtx sum = NULL_RTX, mask = NULL_RTX;
1022 1838 : if (bit == BITS_PER_WORD)
1023 : {
1024 : /* For signed modulo we need to add correction to the sum
1025 : and that might again overflow. */
1026 377 : if (!unsignedp)
1027 140 : continue;
1028 237 : if (optab_handler (uaddv4_optab, word_mode) == CODE_FOR_nothing)
1029 0 : continue;
1030 237 : tree wtype = lang_hooks.types.type_for_mode (word_mode, 1);
1031 237 : if (wtype == NULL_TREE)
1032 0 : continue;
1033 237 : tree ctype = build_complex_type (wtype);
1034 237 : if (TYPE_MODE (ctype) != GET_MODE_COMPLEX_MODE (word_mode))
1035 0 : continue;
1036 237 : machine_mode cmode = TYPE_MODE (ctype);
1037 237 : rtx op00 = operand_subword_force (op0, 0, mode);
1038 237 : rtx op01 = operand_subword_force (op0, 1, mode);
1039 237 : rtx cres = gen_rtx_CONCAT (cmode, gen_reg_rtx (word_mode),
1040 : gen_reg_rtx (word_mode));
1041 237 : tree lhs = make_tree (ctype, cres);
1042 237 : tree arg0 = make_tree (wtype, op00);
1043 237 : tree arg1 = make_tree (wtype, op01);
1044 237 : expand_addsub_overflow (UNKNOWN_LOCATION, PLUS_EXPR, lhs, arg0,
1045 : arg1, true, true, true, false, NULL);
1046 237 : sum = expand_simple_binop (word_mode, PLUS, XEXP (cres, 0),
1047 : XEXP (cres, 1), NULL_RTX, 1,
1048 : OPTAB_DIRECT);
1049 237 : if (sum == NULL_RTX)
1050 : return NULL_RTX;
1051 : }
1052 : else
1053 : {
1054 : /* Code below uses GEN_INT, so we need the masks to be representable
1055 : in HOST_WIDE_INTs. */
1056 957 : if (bit >= HOST_BITS_PER_WIDE_INT)
1057 0 : continue;
1058 : /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1059 : overflow. Consider 64-bit -1ULL for word size 32, if we add
1060 : 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1061 1207 : if (bit == BITS_PER_WORD - 1)
1062 150 : continue;
1063 :
1064 807 : int count = (2 * BITS_PER_WORD + bit - 1) / bit;
1065 807 : rtx sum_corr = NULL_RTX;
1066 :
1067 807 : if (!unsignedp)
1068 : {
1069 : /* For signed modulo, compute it as unsigned modulo of
1070 : sum with a correction added to it if OP0 is negative,
1071 : such that the result can be computed as unsigned
1072 : remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1073 532 : w = wi::min_value (2 * BITS_PER_WORD, SIGNED);
1074 424 : wide_int wmod1 = wi::umod_trunc (w, INTVAL (op1));
1075 424 : wide_int wmod2 = wi::smod_trunc (w, INTVAL (op1));
1076 : /* wmod2 == -wmod1. */
1077 424 : wmod2 = wmod2 + (INTVAL (op1) - 1);
1078 424 : if (wi::ne_p (wmod1, wmod2))
1079 : {
1080 424 : wide_int wcorr = wmod2 - wmod1;
1081 424 : if (wi::neg_p (w))
1082 424 : wcorr = wcorr + INTVAL (op1);
1083 : /* Now verify if the count sums can't overflow, and punt
1084 : if they could. */
1085 532 : w = wi::mask (bit, false, 2 * BITS_PER_WORD);
1086 424 : w = w * (count - 1);
1087 956 : w = w + wi::mask (2 * BITS_PER_WORD - (count - 1) * bit,
1088 848 : false, 2 * BITS_PER_WORD);
1089 424 : w = w + wcorr;
1090 532 : w = wi::lrshift (w, BITS_PER_WORD);
1091 424 : if (wi::ne_p (w, 0))
1092 0 : continue;
1093 :
1094 424 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1095 : mode);
1096 424 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1097 424 : GEN_INT (BITS_PER_WORD - 1),
1098 : NULL_RTX, 0, OPTAB_DIRECT);
1099 424 : if (mask == NULL_RTX)
1100 : return NULL_RTX;
1101 424 : sum_corr = immed_wide_int_const (wcorr, word_mode);
1102 424 : sum_corr = expand_simple_binop (word_mode, AND, mask,
1103 : sum_corr, NULL_RTX, 1,
1104 : OPTAB_DIRECT);
1105 424 : if (sum_corr == NULL_RTX)
1106 : return NULL_RTX;
1107 424 : }
1108 424 : }
1109 :
1110 3421 : for (int i = 0; i < count; i++)
1111 : {
1112 2614 : rtx v = op0;
1113 2614 : if (i)
1114 1807 : v = expand_simple_binop (mode, LSHIFTRT, v, GEN_INT (i * bit),
1115 : NULL_RTX, 1, OPTAB_DIRECT);
1116 2614 : if (v == NULL_RTX)
1117 : return NULL_RTX;
1118 2614 : v = force_lowpart_subreg (word_mode, v, mode);
1119 2614 : if (v == NULL_RTX)
1120 : return NULL_RTX;
1121 2614 : if (i != count - 1)
1122 1807 : v = expand_simple_binop (word_mode, AND, v,
1123 1807 : GEN_INT ((HOST_WIDE_INT_1U << bit)
1124 : - 1), NULL_RTX, 1,
1125 : OPTAB_DIRECT);
1126 2614 : if (v == NULL_RTX)
1127 : return NULL_RTX;
1128 2614 : if (sum == NULL_RTX)
1129 : sum = v;
1130 : else
1131 1807 : sum = expand_simple_binop (word_mode, PLUS, sum, v, NULL_RTX,
1132 : 1, OPTAB_DIRECT);
1133 2614 : if (sum == NULL_RTX)
1134 : return NULL_RTX;
1135 : }
1136 807 : if (sum_corr)
1137 : {
1138 424 : sum = expand_simple_binop (word_mode, PLUS, sum, sum_corr,
1139 : NULL_RTX, 1, OPTAB_DIRECT);
1140 424 : if (sum == NULL_RTX)
1141 : return NULL_RTX;
1142 : }
1143 : }
1144 1044 : rtx remainder = expand_divmod (1, TRUNC_MOD_EXPR, word_mode, sum,
1145 1044 : gen_int_mode (INTVAL (op1), word_mode),
1146 : NULL_RTX, 1, OPTAB_DIRECT);
1147 1044 : if (remainder == NULL_RTX)
1148 : return NULL_RTX;
1149 :
1150 1044 : if (!unsignedp)
1151 : {
1152 424 : if (mask == NULL_RTX)
1153 : {
1154 0 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1155 : mode);
1156 0 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1157 0 : GEN_INT (BITS_PER_WORD - 1),
1158 : NULL_RTX, 0, OPTAB_DIRECT);
1159 0 : if (mask == NULL_RTX)
1160 : return NULL_RTX;
1161 : }
1162 424 : mask = expand_simple_binop (word_mode, AND, mask,
1163 424 : gen_int_mode (1 - INTVAL (op1),
1164 : word_mode),
1165 : NULL_RTX, 1, OPTAB_DIRECT);
1166 424 : if (mask == NULL_RTX)
1167 : return NULL_RTX;
1168 424 : remainder = expand_simple_binop (word_mode, PLUS, remainder,
1169 : mask, NULL_RTX, 1, OPTAB_DIRECT);
1170 424 : if (remainder == NULL_RTX)
1171 : return NULL_RTX;
1172 : }
1173 :
1174 1044 : remainder = convert_modes (mode, word_mode, remainder, unsignedp);
1175 : /* Punt if we need any library calls. */
1176 1044 : if (last)
1177 916 : last = NEXT_INSN (last);
1178 : else
1179 128 : last = get_insns ();
1180 21473 : for (; last; last = NEXT_INSN (last))
1181 20429 : if (CALL_P (last))
1182 : return NULL_RTX;
1183 : return remainder;
1184 627551 : }
1185 : return NULL_RTX;
1186 : }
1187 :
1188 : /* Similarly to the above function, but compute both quotient and remainder.
1189 : Quotient can be computed from the remainder as:
1190 : rem = op0 % op1; // Handled using expand_doubleword_mod
1191 : quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1192 : // 2 * BITS_PER_WORD
1193 :
1194 : We can also handle cases where op1 is a multiple of power of two constant
1195 : and constant handled by expand_doubleword_mod.
1196 : op11 = 1 << __builtin_ctz (op1);
1197 : op12 = op1 / op11;
1198 : rem1 = op0 % op12; // Handled using expand_doubleword_mod
1199 : quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1200 : // 2 * BITS_PER_WORD
1201 : rem = (quot1 % op11) * op12 + rem1;
1202 : quot = quot1 / op11; */
1203 :
1204 : rtx
1205 20708 : expand_doubleword_divmod (machine_mode mode, rtx op0, rtx op1, rtx *rem,
1206 : bool unsignedp)
1207 : {
1208 20708 : *rem = NULL_RTX;
1209 :
1210 : /* Negative dividend should have been optimized into positive,
1211 : similarly modulo by 1 and modulo by power of two is optimized
1212 : differently too. */
1213 20708 : if (INTVAL (op1) <= 1 || pow2p_hwi (INTVAL (op1)))
1214 : return NULL_RTX;
1215 :
1216 20659 : rtx op11 = const1_rtx;
1217 20659 : rtx op12 = op1;
1218 20659 : if ((INTVAL (op1) & 1) == 0)
1219 : {
1220 3639 : int bit = ctz_hwi (INTVAL (op1));
1221 3639 : op11 = GEN_INT (HOST_WIDE_INT_1 << bit);
1222 3639 : op12 = GEN_INT (INTVAL (op1) >> bit);
1223 : }
1224 :
1225 20659 : rtx rem1 = expand_doubleword_mod (mode, op0, op12, unsignedp);
1226 20659 : if (rem1 == NULL_RTX)
1227 : return NULL_RTX;
1228 :
1229 1044 : int prec = 2 * BITS_PER_WORD;
1230 1044 : wide_int a = wide_int::from (INTVAL (op12), prec + 1, UNSIGNED);
1231 1044 : wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
1232 1044 : wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
1233 1044 : rtx inv = immed_wide_int_const (m, mode);
1234 :
1235 1044 : rtx_insn *last = get_last_insn ();
1236 1044 : rtx quot1 = expand_simple_binop (mode, MINUS, op0, rem1,
1237 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1238 1044 : if (quot1 == NULL_RTX)
1239 : return NULL_RTX;
1240 :
1241 1044 : quot1 = expand_simple_binop (mode, MULT, quot1, inv,
1242 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1243 1044 : if (quot1 == NULL_RTX)
1244 : return NULL_RTX;
1245 :
1246 1044 : if (op11 != const1_rtx)
1247 : {
1248 379 : rtx rem2 = expand_divmod (1, TRUNC_MOD_EXPR, mode, quot1, op11,
1249 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1250 379 : if (rem2 == NULL_RTX)
1251 : return NULL_RTX;
1252 :
1253 379 : rem2 = expand_simple_binop (mode, MULT, rem2, op12, NULL_RTX,
1254 : unsignedp, OPTAB_DIRECT);
1255 379 : if (rem2 == NULL_RTX)
1256 : return NULL_RTX;
1257 :
1258 379 : rem2 = expand_simple_binop (mode, PLUS, rem2, rem1, NULL_RTX,
1259 : unsignedp, OPTAB_DIRECT);
1260 379 : if (rem2 == NULL_RTX)
1261 : return NULL_RTX;
1262 :
1263 379 : rtx quot2 = expand_divmod (0, TRUNC_DIV_EXPR, mode, quot1, op11,
1264 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1265 379 : if (quot2 == NULL_RTX)
1266 : return NULL_RTX;
1267 :
1268 : rem1 = rem2;
1269 : quot1 = quot2;
1270 : }
1271 :
1272 : /* Punt if we need any library calls. */
1273 1044 : if (last)
1274 1044 : last = NEXT_INSN (last);
1275 : else
1276 0 : last = get_insns ();
1277 16429 : for (; last; last = NEXT_INSN (last))
1278 15385 : if (CALL_P (last))
1279 : return NULL_RTX;
1280 :
1281 1044 : *rem = rem1;
1282 1044 : return quot1;
1283 1044 : }
1284 :
1285 : /* Wrapper around expand_binop which takes an rtx code to specify
1286 : the operation to perform, not an optab pointer. All other
1287 : arguments are the same. */
1288 : rtx
1289 4154187 : expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
1290 : rtx op1, rtx target, int unsignedp,
1291 : enum optab_methods methods)
1292 : {
1293 4154187 : optab binop = code_to_optab (code);
1294 4154187 : gcc_assert (binop);
1295 :
1296 4154187 : return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1297 : }
1298 :
1299 : /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1300 : binop. Order them according to commutative_operand_precedence and, if
1301 : possible, try to put TARGET or a pseudo first. */
1302 : static bool
1303 12476836 : swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1304 : {
1305 12476836 : int op0_prec = commutative_operand_precedence (op0);
1306 12476836 : int op1_prec = commutative_operand_precedence (op1);
1307 :
1308 12476836 : if (op0_prec < op1_prec)
1309 : return true;
1310 :
1311 12282137 : if (op0_prec > op1_prec)
1312 : return false;
1313 :
1314 : /* With equal precedence, both orders are ok, but it is better if the
1315 : first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1316 2296260 : if (target == 0 || REG_P (target))
1317 2205978 : return (REG_P (op1) && !REG_P (op0)) || target == op1;
1318 : else
1319 90282 : return rtx_equal_p (op1, target);
1320 : }
1321 :
1322 : /* Return true if BINOPTAB implements a shift operation. */
1323 :
1324 : static bool
1325 24657853 : shift_optab_p (optab binoptab)
1326 : {
1327 9915715 : switch (optab_to_code (binoptab))
1328 : {
1329 : case ASHIFT:
1330 : case SS_ASHIFT:
1331 : case US_ASHIFT:
1332 : case ASHIFTRT:
1333 : case LSHIFTRT:
1334 : case ROTATE:
1335 : case ROTATERT:
1336 : return true;
1337 :
1338 13146595 : default:
1339 0 : return false;
1340 : }
1341 : }
1342 :
1343 : /* Return true if BINOPTAB implements a commutative binary operation. */
1344 :
1345 : static bool
1346 14919080 : commutative_optab_p (optab binoptab)
1347 : {
1348 14919080 : return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1349 : || binoptab == smul_widen_optab
1350 2531640 : || binoptab == umul_widen_optab
1351 : || binoptab == smul_highpart_optab
1352 2494290 : || binoptab == umul_highpart_optab
1353 2442244 : || binoptab == vec_widen_sadd_optab
1354 2442244 : || binoptab == vec_widen_uadd_optab
1355 : || binoptab == vec_widen_sadd_hi_optab
1356 2442244 : || binoptab == vec_widen_sadd_lo_optab
1357 : || binoptab == vec_widen_uadd_hi_optab
1358 2442244 : || binoptab == vec_widen_uadd_lo_optab
1359 : || binoptab == vec_widen_sadd_even_optab
1360 2442244 : || binoptab == vec_widen_sadd_odd_optab
1361 : || binoptab == vec_widen_uadd_even_optab
1362 17361324 : || binoptab == vec_widen_uadd_odd_optab);
1363 : }
1364 :
1365 : /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1366 : optimizing, and if the operand is a constant that costs more than
1367 : 1 instruction, force the constant into a register and return that
1368 : register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1369 :
1370 : static rtx
1371 28088128 : avoid_expensive_constant (machine_mode mode, optab binoptab,
1372 : int opn, rtx x, bool unsignedp)
1373 : {
1374 28088128 : bool speed = optimize_insn_for_speed_p ();
1375 :
1376 28088128 : if (mode != VOIDmode
1377 28088128 : && optimize
1378 21287341 : && CONSTANT_P (x)
1379 35428782 : && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1380 7340654 : > set_src_cost (x, mode, speed)))
1381 : {
1382 0 : if (CONST_INT_P (x))
1383 : {
1384 0 : HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1385 0 : if (intval != INTVAL (x))
1386 0 : x = GEN_INT (intval);
1387 : }
1388 : else
1389 0 : x = convert_modes (mode, VOIDmode, x, unsignedp);
1390 0 : x = force_reg (mode, x);
1391 : }
1392 28088128 : return x;
1393 : }
1394 :
1395 : /* Helper function for expand_binop: handle the case where there
1396 : is an insn ICODE that directly implements the indicated operation.
1397 : Returns null if this is not possible. */
1398 : static rtx
1399 14711504 : expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1400 : rtx op0, rtx op1,
1401 : rtx target, int unsignedp, enum optab_methods methods)
1402 : {
1403 14711504 : machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1404 14711504 : machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1405 14711504 : machine_mode mode0, mode1, tmp_mode;
1406 14711504 : class expand_operand ops[3];
1407 14711504 : bool commutative_p;
1408 14711504 : rtx_insn *pat;
1409 14711504 : rtx xop0 = op0, xop1 = op1;
1410 14711504 : bool canonicalize_op1 = false;
1411 14711504 : rtx_insn *last = get_last_insn ();
1412 :
1413 : /* If it is a commutative operator and the modes would match
1414 : if we would swap the operands, we can save the conversions. */
1415 14711504 : commutative_p = commutative_optab_p (binoptab);
1416 14711504 : if (commutative_p
1417 12417177 : && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1418 13580 : && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0)
1419 : std::swap (xop0, xop1);
1420 :
1421 : /* If we are optimizing, force expensive constants into a register. */
1422 14711504 : xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1423 14711504 : if (!shift_optab_p (binoptab))
1424 13146595 : xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1425 : else
1426 : /* Shifts and rotates often use a different mode for op1 from op0;
1427 : for VOIDmode constants we don't know the mode, so force it
1428 : to be canonicalized using convert_modes. */
1429 : canonicalize_op1 = true;
1430 :
1431 : /* In case the insn wants input operands in modes different from
1432 : those of the actual operands, convert the operands. It would
1433 : seem that we don't need to convert CONST_INTs, but we do, so
1434 : that they're properly zero-extended, sign-extended or truncated
1435 : for their mode. */
1436 :
1437 14711504 : mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1438 14711504 : if (xmode0 != VOIDmode && xmode0 != mode0)
1439 : {
1440 3320 : xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1441 3320 : mode0 = xmode0;
1442 : }
1443 :
1444 14711504 : mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1445 14711504 : ? GET_MODE (xop1) : mode);
1446 14711504 : if (xmode1 != VOIDmode && xmode1 != mode1)
1447 : {
1448 1566640 : xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1449 1566640 : mode1 = xmode1;
1450 : }
1451 :
1452 : /* If operation is commutative,
1453 : try to make the first operand a register.
1454 : Even better, try to make it the same as the target.
1455 : Also try to make the last operand a constant. */
1456 14711504 : if (commutative_p
1457 14711504 : && swap_commutative_operands_with_target (target, xop0, xop1))
1458 : std::swap (xop0, xop1);
1459 :
1460 : /* Now, if insn's predicates don't allow our operands, put them into
1461 : pseudo regs. */
1462 :
1463 14711504 : if (binoptab == vec_pack_trunc_optab
1464 14711504 : || binoptab == vec_pack_usat_optab
1465 14700726 : || binoptab == vec_pack_ssat_optab
1466 14700726 : || binoptab == vec_pack_ufix_trunc_optab
1467 14700569 : || binoptab == vec_pack_sfix_trunc_optab
1468 14700569 : || binoptab == vec_packu_float_optab
1469 14700349 : || binoptab == vec_packs_float_optab)
1470 : {
1471 : /* The mode of the result is different then the mode of the
1472 : arguments. */
1473 11175 : tmp_mode = insn_data[(int) icode].operand[0].mode;
1474 2645 : if (VECTOR_MODE_P (mode)
1475 31943 : && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1476 : return NULL_RTX;
1477 : }
1478 : else
1479 : tmp_mode = mode;
1480 :
1481 14711504 : create_output_operand (&ops[0], target, tmp_mode);
1482 14711504 : create_input_operand (&ops[1], xop0, mode0);
1483 14711504 : create_input_operand (&ops[2], xop1, mode1);
1484 14711504 : pat = maybe_gen_insn (icode, 3, ops);
1485 14711504 : if (pat)
1486 : {
1487 : /* If PAT is composed of more than one insn, try to add an appropriate
1488 : REG_EQUAL note to it. If we can't because TEMP conflicts with an
1489 : operand, call expand_binop again, this time without a target. */
1490 13542815 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1491 15918769 : && ! add_equal_note (pat, ops[0].value,
1492 : optab_to_code (binoptab),
1493 : ops[1].value, ops[2].value, mode0))
1494 : {
1495 52555 : delete_insns_since (last);
1496 52555 : return expand_binop_directly (icode, mode, binoptab, op0, op1,
1497 52555 : NULL_RTX, unsignedp, methods);
1498 : }
1499 :
1500 14658947 : emit_insn (pat);
1501 14658947 : return ops[0].value;
1502 : }
1503 :
1504 : return NULL_RTX;
1505 : }
1506 :
1507 : /* Generate code to perform an operation specified by BINOPTAB
1508 : on operands OP0 and OP1, with result having machine-mode MODE.
1509 :
1510 : UNSIGNEDP is for the case where we have to widen the operands
1511 : to perform the operation. It says to use zero-extension.
1512 :
1513 : If TARGET is nonzero, the value
1514 : is generated there, if it is convenient to do so.
1515 : In all cases an rtx is returned for the locus of the value;
1516 : this may or may not be TARGET. */
1517 :
1518 : rtx
1519 14866687 : expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1520 : rtx target, int unsignedp, enum optab_methods methods)
1521 : {
1522 12857585 : enum optab_methods next_methods
1523 14956395 : = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1524 14956395 : ? OPTAB_WIDEN : methods);
1525 14956395 : enum mode_class mclass;
1526 14956395 : enum insn_code icode;
1527 14956395 : machine_mode wider_mode;
1528 14956395 : scalar_int_mode int_mode;
1529 14956395 : rtx libfunc;
1530 14956395 : rtx temp;
1531 14956395 : rtx_insn *entry_last = get_last_insn ();
1532 14956395 : rtx_insn *last;
1533 :
1534 14956395 : mclass = GET_MODE_CLASS (mode);
1535 :
1536 : /* If subtracting an integer constant, convert this into an addition of
1537 : the negated constant. */
1538 :
1539 14956395 : if (binoptab == sub_optab && CONST_INT_P (op1))
1540 : {
1541 1036966 : op1 = negate_rtx (mode, op1);
1542 1036966 : binoptab = add_optab;
1543 : }
1544 : /* For shifts, constant invalid op1 might be expanded from different
1545 : mode than MODE. As those are invalid, force them to a register
1546 : to avoid further problems during expansion. */
1547 13919429 : else if (CONST_INT_P (op1)
1548 16460159 : && shift_optab_p (binoptab)
1549 18430721 : && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1550 : {
1551 160 : op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1552 160 : op1 = force_reg (GET_MODE_INNER (mode), op1);
1553 : }
1554 :
1555 : /* Record where to delete back to if we backtrack. */
1556 14956395 : last = get_last_insn ();
1557 :
1558 : /* If we can do it with a three-operand insn, do so. */
1559 :
1560 14956395 : if (methods != OPTAB_MUST_WIDEN)
1561 : {
1562 14956395 : if (convert_optab_p (binoptab))
1563 : {
1564 37350 : machine_mode from_mode = widened_mode (mode, op0, op1);
1565 37350 : icode = find_widening_optab_handler (binoptab, mode, from_mode);
1566 : }
1567 : else
1568 14919045 : icode = optab_handler (binoptab, mode);
1569 14956395 : if (icode != CODE_FOR_nothing)
1570 : {
1571 14658948 : temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1572 : target, unsignedp, methods);
1573 14658948 : if (temp)
1574 : return temp;
1575 1 : delete_insns_since (last);
1576 : }
1577 : }
1578 :
1579 : /* If we were trying to rotate, and that didn't work, try rotating
1580 : the other direction before falling back to shifts and bitwise-or. */
1581 297448 : if (((binoptab == rotl_optab
1582 23 : && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1583 297447 : || (binoptab == rotr_optab
1584 11 : && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1585 297449 : && is_int_mode (mode, &int_mode))
1586 : {
1587 1 : optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1588 1 : rtx newop1;
1589 1 : unsigned int bits = GET_MODE_PRECISION (int_mode);
1590 :
1591 1 : if (CONST_INT_P (op1))
1592 0 : newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1593 1 : else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1594 0 : newop1 = negate_rtx (GET_MODE (op1), op1);
1595 : else
1596 1 : newop1 = expand_binop (GET_MODE (op1), sub_optab,
1597 1 : gen_int_mode (bits, GET_MODE (op1)), op1,
1598 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1599 :
1600 1 : temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1601 : target, unsignedp, methods);
1602 1 : if (temp)
1603 14866687 : return temp;
1604 1 : delete_insns_since (last);
1605 : }
1606 :
1607 : /* If this is a multiply, see if we can do a widening operation that
1608 : takes operands of this mode and makes a wider mode. */
1609 :
1610 297448 : if (binoptab == smul_optab
1611 25751 : && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1612 316378 : && (convert_optab_handler ((unsignedp
1613 : ? umul_widen_optab
1614 : : smul_widen_optab),
1615 : wider_mode, mode) != CODE_FOR_nothing))
1616 : {
1617 : /* *_widen_optab needs to determine operand mode, make sure at least
1618 : one operand has non-VOID mode. */
1619 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1620 0 : op0 = force_reg (mode, op0);
1621 0 : temp = expand_binop (wider_mode,
1622 : unsignedp ? umul_widen_optab : smul_widen_optab,
1623 : op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1624 :
1625 0 : if (temp != 0)
1626 : {
1627 0 : if (GET_MODE_CLASS (mode) == MODE_INT
1628 0 : && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1629 0 : return gen_lowpart (mode, temp);
1630 : else
1631 0 : return convert_to_mode (mode, temp, unsignedp);
1632 : }
1633 : }
1634 :
1635 : /* If backend's machine description doesn't specify an any_or_plus
1636 : (AOP) preference, choose for it. */
1637 297448 : if (binoptab == aop_optab)
1638 : {
1639 89708 : binoptab = (mode == word_mode || mode == SImode) ? add_optab
1640 : : ior_optab;
1641 89708 : return expand_binop (mode, binoptab, op0, op1,
1642 89708 : target, unsignedp, methods);
1643 : }
1644 :
1645 : /* If this is a vector shift by a scalar, see if we can do a vector
1646 : shift by a vector. If so, broadcast the scalar into a vector. */
1647 207740 : if (mclass == MODE_VECTOR_INT)
1648 : {
1649 33 : optab otheroptab = unknown_optab;
1650 :
1651 33 : if (binoptab == ashl_optab)
1652 : otheroptab = vashl_optab;
1653 : else if (binoptab == ashr_optab)
1654 : otheroptab = vashr_optab;
1655 : else if (binoptab == lshr_optab)
1656 : otheroptab = vlshr_optab;
1657 : else if (binoptab == rotl_optab)
1658 : otheroptab = vrotl_optab;
1659 : else if (binoptab == rotr_optab)
1660 : otheroptab = vrotr_optab;
1661 :
1662 : if (otheroptab
1663 33 : && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1664 : {
1665 : /* The scalar may be wider or narrower than the vector element.
1666 : Truncate or extend it to the proper size to fit in the
1667 : broadcast vector. */
1668 0 : scalar_mode inner_mode = GET_MODE_INNER (mode);
1669 0 : if (!CONST_INT_P (op1))
1670 : {
1671 0 : auto mode1 = as_a <scalar_int_mode> (GET_MODE (op1));
1672 0 : int size1 = GET_MODE_BITSIZE (mode1);
1673 0 : int inner_size = GET_MODE_BITSIZE (inner_mode);
1674 :
1675 0 : if (size1 != inner_size)
1676 : {
1677 0 : auto unary = size1 > inner_size ? TRUNCATE : ZERO_EXTEND;
1678 0 : op1 = force_reg (inner_mode,
1679 : simplify_gen_unary (unary, inner_mode,
1680 : op1, mode1));
1681 : }
1682 : }
1683 :
1684 0 : rtx vop1 = expand_vector_broadcast (mode, op1);
1685 0 : if (vop1)
1686 : {
1687 0 : temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1688 : target, unsignedp, methods);
1689 0 : if (temp)
1690 14866687 : return temp;
1691 0 : delete_insns_since (last);
1692 : }
1693 : }
1694 : }
1695 :
1696 : /* Look for a wider mode of the same class for which we think we
1697 : can open-code the operation. Check for a widening multiply at the
1698 : wider mode as well. */
1699 :
1700 207740 : if (CLASS_HAS_WIDER_MODES_P (mclass)
1701 207707 : && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1702 115608 : FOR_EACH_WIDER_MODE (wider_mode, mode)
1703 : {
1704 62144 : machine_mode next_mode;
1705 62144 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1706 62144 : || (binoptab == smul_optab
1707 78502 : && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1708 20086 : && (find_widening_optab_handler ((unsignedp
1709 : ? umul_widen_optab
1710 : : smul_widen_optab),
1711 : next_mode, mode)
1712 : != CODE_FOR_nothing)))
1713 : {
1714 164 : rtx xop0 = op0, xop1 = op1;
1715 164 : bool no_extend = false;
1716 :
1717 : /* For certain integer operations, we need not actually extend
1718 : the narrow operands, as long as we will truncate
1719 : the results to the same narrowness. */
1720 :
1721 164 : if ((binoptab == ior_optab || binoptab == and_optab
1722 164 : || binoptab == xor_optab
1723 164 : || binoptab == add_optab || binoptab == sub_optab
1724 : || binoptab == smul_optab || binoptab == ashl_optab)
1725 89 : && mclass == MODE_INT)
1726 : {
1727 7 : no_extend = true;
1728 7 : xop0 = avoid_expensive_constant (mode, binoptab, 0,
1729 : xop0, unsignedp);
1730 7 : if (binoptab != ashl_optab)
1731 0 : xop1 = avoid_expensive_constant (mode, binoptab, 1,
1732 : xop1, unsignedp);
1733 : }
1734 :
1735 164 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1736 :
1737 : /* The second operand of a shift must always be extended. */
1738 328 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1739 164 : no_extend && binoptab != ashl_optab);
1740 :
1741 164 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1742 : unsignedp, OPTAB_DIRECT);
1743 164 : if (temp)
1744 : {
1745 164 : if (mclass != MODE_INT
1746 164 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1747 : {
1748 157 : if (target == 0)
1749 49 : target = gen_reg_rtx (mode);
1750 157 : convert_move (target, temp, 0);
1751 157 : return target;
1752 : }
1753 : else
1754 7 : return gen_lowpart (mode, temp);
1755 : }
1756 : else
1757 0 : delete_insns_since (last);
1758 : }
1759 : }
1760 :
1761 : /* If operation is commutative,
1762 : try to make the first operand a register.
1763 : Even better, try to make it the same as the target.
1764 : Also try to make the last operand a constant. */
1765 207576 : if (commutative_optab_p (binoptab)
1766 207576 : && swap_commutative_operands_with_target (target, op0, op1))
1767 : std::swap (op0, op1);
1768 :
1769 : /* These can be done a word at a time. */
1770 207576 : if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1771 8 : && is_int_mode (mode, &int_mode)
1772 16 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1773 207584 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1774 : {
1775 8 : int i;
1776 8 : rtx_insn *insns;
1777 :
1778 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1779 : won't be accurate, so use a new target. */
1780 8 : if (target == 0
1781 8 : || target == op0
1782 8 : || target == op1
1783 8 : || reg_overlap_mentioned_p (target, op0)
1784 8 : || reg_overlap_mentioned_p (target, op1)
1785 16 : || !valid_multiword_target_p (target))
1786 0 : target = gen_reg_rtx (int_mode);
1787 :
1788 8 : start_sequence ();
1789 :
1790 : /* Do the actual arithmetic. */
1791 8 : machine_mode op0_mode = GET_MODE (op0);
1792 8 : machine_mode op1_mode = GET_MODE (op1);
1793 8 : if (op0_mode == VOIDmode)
1794 0 : op0_mode = int_mode;
1795 8 : if (op1_mode == VOIDmode)
1796 8 : op1_mode = int_mode;
1797 120 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1798 : {
1799 32 : rtx target_piece = operand_subword (target, i, 1, int_mode);
1800 32 : rtx x = expand_binop (word_mode, binoptab,
1801 32 : operand_subword_force (op0, i, op0_mode),
1802 32 : operand_subword_force (op1, i, op1_mode),
1803 : target_piece, unsignedp, next_methods);
1804 :
1805 32 : if (x == 0)
1806 : break;
1807 :
1808 32 : if (target_piece != x)
1809 0 : emit_move_insn (target_piece, x);
1810 : }
1811 :
1812 8 : insns = end_sequence ();
1813 :
1814 24 : if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1815 : {
1816 8 : emit_insn (insns);
1817 8 : return target;
1818 : }
1819 : }
1820 :
1821 : /* Synthesize double word shifts from single word shifts. */
1822 207568 : if ((binoptab == lshr_optab || binoptab == ashl_optab
1823 207568 : || binoptab == ashr_optab)
1824 0 : && is_int_mode (mode, &int_mode)
1825 0 : && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1826 0 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1827 0 : && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1828 0 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1829 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1830 207568 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1831 : {
1832 0 : unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1833 0 : scalar_int_mode op1_mode;
1834 :
1835 0 : double_shift_mask = targetm.shift_truncation_mask (int_mode);
1836 0 : shift_mask = targetm.shift_truncation_mask (word_mode);
1837 0 : op1_mode = (GET_MODE (op1) != VOIDmode
1838 0 : ? as_a <scalar_int_mode> (GET_MODE (op1))
1839 : : word_mode);
1840 :
1841 : /* Apply the truncation to constant shifts. */
1842 0 : if (double_shift_mask > 0 && CONST_INT_P (op1))
1843 0 : op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1844 :
1845 0 : if (op1 == CONST0_RTX (op1_mode))
1846 14866687 : return op0;
1847 :
1848 : /* Make sure that this is a combination that expand_doubleword_shift
1849 : can handle. See the comments there for details. */
1850 0 : if (double_shift_mask == 0
1851 0 : || (shift_mask == BITS_PER_WORD - 1
1852 0 : && double_shift_mask == BITS_PER_WORD * 2 - 1))
1853 : {
1854 0 : rtx_insn *insns;
1855 0 : rtx into_target, outof_target;
1856 0 : rtx into_input, outof_input;
1857 0 : int left_shift, outof_word;
1858 :
1859 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1860 : won't be accurate, so use a new target. */
1861 0 : if (target == 0
1862 0 : || target == op0
1863 0 : || target == op1
1864 0 : || reg_overlap_mentioned_p (target, op0)
1865 0 : || reg_overlap_mentioned_p (target, op1)
1866 0 : || !valid_multiword_target_p (target))
1867 0 : target = gen_reg_rtx (int_mode);
1868 :
1869 0 : start_sequence ();
1870 :
1871 : /* OUTOF_* is the word we are shifting bits away from, and
1872 : INTO_* is the word that we are shifting bits towards, thus
1873 : they differ depending on the direction of the shift and
1874 : WORDS_BIG_ENDIAN. */
1875 :
1876 0 : left_shift = binoptab == ashl_optab;
1877 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1878 :
1879 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1880 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1881 :
1882 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1883 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1884 :
1885 0 : if (expand_doubleword_shift (op1_mode, binoptab,
1886 : outof_input, into_input, op1,
1887 : outof_target, into_target,
1888 : unsignedp, next_methods, shift_mask))
1889 : {
1890 0 : insns = end_sequence ();
1891 :
1892 0 : emit_insn (insns);
1893 0 : return target;
1894 : }
1895 0 : end_sequence ();
1896 : }
1897 : }
1898 :
1899 : /* Synthesize double word rotates from single word shifts. */
1900 207568 : if ((binoptab == rotl_optab || binoptab == rotr_optab)
1901 34 : && is_int_mode (mode, &int_mode)
1902 1 : && CONST_INT_P (op1)
1903 0 : && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1904 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1905 207568 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1906 : {
1907 0 : rtx_insn *insns;
1908 0 : rtx into_target, outof_target;
1909 0 : rtx into_input, outof_input;
1910 0 : rtx inter;
1911 0 : int shift_count, left_shift, outof_word;
1912 :
1913 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1914 : won't be accurate, so use a new target. Do this also if target is not
1915 : a REG, first because having a register instead may open optimization
1916 : opportunities, and second because if target and op0 happen to be MEMs
1917 : designating the same location, we would risk clobbering it too early
1918 : in the code sequence we generate below. */
1919 0 : if (target == 0
1920 0 : || target == op0
1921 0 : || target == op1
1922 0 : || !REG_P (target)
1923 0 : || reg_overlap_mentioned_p (target, op0)
1924 0 : || reg_overlap_mentioned_p (target, op1)
1925 0 : || !valid_multiword_target_p (target))
1926 0 : target = gen_reg_rtx (int_mode);
1927 :
1928 0 : start_sequence ();
1929 :
1930 0 : shift_count = INTVAL (op1);
1931 :
1932 : /* OUTOF_* is the word we are shifting bits away from, and
1933 : INTO_* is the word that we are shifting bits towards, thus
1934 : they differ depending on the direction of the shift and
1935 : WORDS_BIG_ENDIAN. */
1936 :
1937 0 : left_shift = (binoptab == rotl_optab);
1938 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1939 :
1940 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1941 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1942 :
1943 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1944 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1945 :
1946 0 : if (shift_count == BITS_PER_WORD)
1947 : {
1948 : /* This is just a word swap. */
1949 0 : emit_move_insn (outof_target, into_input);
1950 0 : emit_move_insn (into_target, outof_input);
1951 0 : inter = const0_rtx;
1952 : }
1953 : else
1954 : {
1955 0 : rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1956 0 : HOST_WIDE_INT first_shift_count, second_shift_count;
1957 0 : optab reverse_unsigned_shift, unsigned_shift;
1958 :
1959 0 : reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1960 0 : ? lshr_optab : ashl_optab);
1961 :
1962 0 : unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1963 0 : ? ashl_optab : lshr_optab);
1964 :
1965 0 : if (shift_count > BITS_PER_WORD)
1966 : {
1967 0 : first_shift_count = shift_count - BITS_PER_WORD;
1968 0 : second_shift_count = 2 * BITS_PER_WORD - shift_count;
1969 : }
1970 : else
1971 : {
1972 0 : first_shift_count = BITS_PER_WORD - shift_count;
1973 0 : second_shift_count = shift_count;
1974 : }
1975 0 : rtx first_shift_count_rtx
1976 0 : = gen_int_shift_amount (word_mode, first_shift_count);
1977 0 : rtx second_shift_count_rtx
1978 0 : = gen_int_shift_amount (word_mode, second_shift_count);
1979 :
1980 0 : into_temp1 = expand_binop (word_mode, unsigned_shift,
1981 : outof_input, first_shift_count_rtx,
1982 : NULL_RTX, unsignedp, next_methods);
1983 0 : into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1984 : into_input, second_shift_count_rtx,
1985 : NULL_RTX, unsignedp, next_methods);
1986 :
1987 0 : if (into_temp1 != 0 && into_temp2 != 0)
1988 0 : inter = expand_binop (word_mode, aop_optab, into_temp1, into_temp2,
1989 : into_target, unsignedp, next_methods);
1990 : else
1991 : inter = 0;
1992 :
1993 0 : if (inter != 0 && inter != into_target)
1994 0 : emit_move_insn (into_target, inter);
1995 :
1996 0 : outof_temp1 = expand_binop (word_mode, unsigned_shift,
1997 : into_input, first_shift_count_rtx,
1998 : NULL_RTX, unsignedp, next_methods);
1999 0 : outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
2000 : outof_input, second_shift_count_rtx,
2001 : NULL_RTX, unsignedp, next_methods);
2002 :
2003 0 : if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
2004 0 : inter = expand_binop (word_mode, aop_optab,
2005 : outof_temp1, outof_temp2,
2006 : outof_target, unsignedp, next_methods);
2007 :
2008 0 : if (inter != 0 && inter != outof_target)
2009 0 : emit_move_insn (outof_target, inter);
2010 : }
2011 :
2012 0 : insns = end_sequence ();
2013 :
2014 0 : if (inter != 0)
2015 : {
2016 0 : emit_insn (insns);
2017 0 : return target;
2018 : }
2019 : }
2020 :
2021 : /* These can be done a word at a time by propagating carries. */
2022 207568 : if ((binoptab == add_optab || binoptab == sub_optab)
2023 13676 : && is_int_mode (mode, &int_mode)
2024 0 : && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
2025 207568 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
2026 : {
2027 0 : unsigned int i;
2028 0 : optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
2029 0 : const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
2030 0 : rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
2031 0 : rtx xop0, xop1, xtarget;
2032 :
2033 : /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
2034 : value is one of those, use it. Otherwise, use 1 since it is the
2035 : one easiest to get. */
2036 : #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
2037 0 : int normalizep = STORE_FLAG_VALUE;
2038 : #else
2039 : int normalizep = 1;
2040 : #endif
2041 :
2042 : /* Prepare the operands. */
2043 0 : xop0 = force_reg (int_mode, op0);
2044 0 : xop1 = force_reg (int_mode, op1);
2045 :
2046 0 : xtarget = gen_reg_rtx (int_mode);
2047 :
2048 0 : if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
2049 : target = xtarget;
2050 :
2051 : /* Indicate for flow that the entire target reg is being set. */
2052 0 : if (REG_P (target))
2053 0 : emit_clobber (xtarget);
2054 :
2055 : /* Do the actual arithmetic. */
2056 0 : for (i = 0; i < nwords; i++)
2057 : {
2058 0 : int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
2059 0 : rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
2060 0 : rtx op0_piece = operand_subword_force (xop0, index, int_mode);
2061 0 : rtx op1_piece = operand_subword_force (xop1, index, int_mode);
2062 0 : rtx x;
2063 :
2064 : /* Main add/subtract of the input operands. */
2065 0 : x = expand_binop (word_mode, binoptab,
2066 : op0_piece, op1_piece,
2067 : target_piece, unsignedp, next_methods);
2068 0 : if (x == 0)
2069 : break;
2070 :
2071 0 : if (i + 1 < nwords)
2072 : {
2073 : /* Store carry from main add/subtract. */
2074 0 : carry_out = gen_reg_rtx (word_mode);
2075 0 : carry_out = emit_store_flag_force (carry_out,
2076 : (binoptab == add_optab
2077 : ? LT : GT),
2078 : x, op0_piece,
2079 : word_mode, 1, normalizep);
2080 : }
2081 :
2082 0 : if (i > 0)
2083 : {
2084 0 : rtx newx;
2085 :
2086 : /* Add/subtract previous carry to main result. */
2087 0 : newx = expand_binop (word_mode,
2088 : normalizep == 1 ? binoptab : otheroptab,
2089 : x, carry_in,
2090 : NULL_RTX, 1, next_methods);
2091 :
2092 0 : if (i + 1 < nwords)
2093 : {
2094 : /* Get out carry from adding/subtracting carry in. */
2095 0 : rtx carry_tmp = gen_reg_rtx (word_mode);
2096 0 : carry_tmp = emit_store_flag_force (carry_tmp,
2097 : (binoptab == add_optab
2098 : ? LT : GT),
2099 : newx, x,
2100 : word_mode, 1, normalizep);
2101 :
2102 : /* Logical-ior the two poss. carry together. */
2103 0 : carry_out = expand_binop (word_mode, ior_optab,
2104 : carry_out, carry_tmp,
2105 : carry_out, 0, next_methods);
2106 0 : if (carry_out == 0)
2107 : break;
2108 : }
2109 0 : emit_move_insn (target_piece, newx);
2110 : }
2111 : else
2112 : {
2113 0 : if (x != target_piece)
2114 0 : emit_move_insn (target_piece, x);
2115 : }
2116 :
2117 0 : carry_in = carry_out;
2118 : }
2119 :
2120 0 : if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
2121 : {
2122 0 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
2123 0 : || ! rtx_equal_p (target, xtarget))
2124 : {
2125 0 : rtx_insn *temp = emit_move_insn (target, xtarget);
2126 :
2127 0 : set_dst_reg_note (temp, REG_EQUAL,
2128 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2129 : int_mode, copy_rtx (xop0),
2130 : copy_rtx (xop1)),
2131 : target);
2132 : }
2133 : else
2134 : target = xtarget;
2135 :
2136 : return target;
2137 : }
2138 :
2139 : else
2140 0 : delete_insns_since (last);
2141 : }
2142 :
2143 : /* Attempt to synthesize double word multiplies using a sequence of word
2144 : mode multiplications. We first attempt to generate a sequence using a
2145 : more efficient unsigned widening multiply, and if that fails we then
2146 : try using a signed widening multiply. */
2147 :
2148 207568 : if (binoptab == smul_optab
2149 25731 : && is_int_mode (mode, &int_mode)
2150 18279 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2151 14948 : && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2152 222516 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2153 : {
2154 14948 : rtx product = NULL_RTX;
2155 14948 : if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
2156 : != CODE_FOR_nothing)
2157 : {
2158 14948 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2159 : true, methods);
2160 14948 : if (!product)
2161 0 : delete_insns_since (last);
2162 : }
2163 :
2164 0 : if (product == NULL_RTX
2165 0 : && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
2166 : != CODE_FOR_nothing))
2167 : {
2168 0 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2169 : false, methods);
2170 0 : if (!product)
2171 0 : delete_insns_since (last);
2172 : }
2173 :
2174 14948 : if (product != NULL_RTX)
2175 : {
2176 14948 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2177 : {
2178 23350 : rtx_insn *move = emit_move_insn (target ? target : product,
2179 : product);
2180 14948 : set_dst_reg_note (move,
2181 : REG_EQUAL,
2182 : gen_rtx_fmt_ee (MULT, int_mode,
2183 : copy_rtx (op0),
2184 : copy_rtx (op1)),
2185 : target ? target : product);
2186 : }
2187 : return product;
2188 : }
2189 : }
2190 :
2191 : /* Attempt to synthesize double word modulo by constant divisor. */
2192 192620 : if ((binoptab == umod_optab
2193 192620 : || binoptab == smod_optab
2194 192620 : || binoptab == udiv_optab
2195 : || binoptab == sdiv_optab)
2196 131044 : && optimize
2197 105639 : && CONST_INT_P (op1)
2198 22840 : && is_int_mode (mode, &int_mode)
2199 25291 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2200 31420 : && optab_handler ((binoptab == umod_optab || binoptab == udiv_optab)
2201 : ? udivmod_optab : sdivmod_optab,
2202 : int_mode) == CODE_FOR_nothing
2203 20114 : && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
2204 20114 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
2205 20114 : && optimize_insn_for_speed_p ())
2206 : {
2207 19902 : rtx res = NULL_RTX;
2208 19902 : if ((binoptab == umod_optab || binoptab == smod_optab)
2209 5196 : && (INTVAL (op1) & 1) == 0)
2210 396 : res = expand_doubleword_mod (int_mode, op0, op1,
2211 : binoptab == umod_optab);
2212 : else
2213 : {
2214 19506 : rtx quot = expand_doubleword_divmod (int_mode, op0, op1, &res,
2215 : binoptab == umod_optab
2216 : || binoptab == udiv_optab);
2217 19506 : if (quot == NULL_RTX)
2218 18830 : res = NULL_RTX;
2219 676 : else if (binoptab == udiv_optab || binoptab == sdiv_optab)
2220 469 : res = quot;
2221 : }
2222 19902 : if (res != NULL_RTX)
2223 : {
2224 676 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2225 : {
2226 831 : rtx_insn *move = emit_move_insn (target ? target : res,
2227 : res);
2228 676 : set_dst_reg_note (move, REG_EQUAL,
2229 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2230 : int_mode, copy_rtx (op0), op1),
2231 : target ? target : res);
2232 : }
2233 676 : return res;
2234 : }
2235 : else
2236 19226 : delete_insns_since (last);
2237 : }
2238 :
2239 : /* It can't be open-coded in this mode.
2240 : Use a library call if one is available and caller says that's ok. */
2241 :
2242 191944 : libfunc = optab_libfunc (binoptab, mode);
2243 191944 : if (libfunc
2244 127909 : && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2245 : {
2246 30634 : rtx_insn *insns;
2247 30634 : rtx op1x = op1;
2248 30634 : machine_mode op1_mode = mode;
2249 30634 : rtx value;
2250 :
2251 30634 : start_sequence ();
2252 :
2253 30634 : if (shift_optab_p (binoptab))
2254 : {
2255 0 : op1_mode = targetm.libgcc_shift_count_mode ();
2256 : /* Specify unsigned here,
2257 : since negative shift counts are meaningless. */
2258 0 : op1x = convert_to_mode (op1_mode, op1, 1);
2259 : }
2260 :
2261 30634 : if (GET_MODE (op0) != VOIDmode
2262 30582 : && GET_MODE (op0) != mode)
2263 0 : op0 = convert_to_mode (mode, op0, unsignedp);
2264 :
2265 : /* Pass 1 for NO_QUEUE so we don't lose any increments
2266 : if the libcall is cse'd or moved. */
2267 30634 : value = emit_library_call_value (libfunc,
2268 : NULL_RTX, LCT_CONST, mode,
2269 : op0, mode, op1x, op1_mode);
2270 :
2271 30634 : insns = end_sequence ();
2272 :
2273 30634 : bool trapv = trapv_binoptab_p (binoptab);
2274 30634 : target = gen_reg_rtx (mode);
2275 30634 : emit_libcall_block_1 (insns, target, value,
2276 : trapv ? NULL_RTX
2277 30154 : : gen_rtx_fmt_ee (optab_to_code (binoptab),
2278 : mode, op0, op1), trapv);
2279 :
2280 30634 : return target;
2281 : }
2282 :
2283 161310 : delete_insns_since (last);
2284 :
2285 : /* It can't be done in this mode. Can we do it in a wider mode? */
2286 :
2287 161310 : if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2288 : || methods == OPTAB_MUST_WIDEN))
2289 : {
2290 : /* Caller says, don't even try. */
2291 148496 : delete_insns_since (entry_last);
2292 148496 : return 0;
2293 : }
2294 :
2295 : /* Compute the value of METHODS to pass to recursive calls.
2296 : Don't allow widening to be tried recursively. */
2297 :
2298 12814 : methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2299 :
2300 : /* Look for a wider mode of the same class for which it appears we can do
2301 : the operation. */
2302 :
2303 12814 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2304 : {
2305 : /* This code doesn't make sense for conversion optabs, since we
2306 : wouldn't then want to extend the operands to be the same size
2307 : as the result. */
2308 12814 : gcc_assert (!convert_optab_p (binoptab));
2309 40414 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2310 : {
2311 27601 : if (optab_handler (binoptab, wider_mode)
2312 27601 : || (methods == OPTAB_LIB
2313 2 : && optab_libfunc (binoptab, wider_mode)))
2314 : {
2315 1 : rtx xop0 = op0, xop1 = op1;
2316 1 : bool no_extend = false;
2317 :
2318 : /* For certain integer operations, we need not actually extend
2319 : the narrow operands, as long as we will truncate
2320 : the results to the same narrowness. */
2321 :
2322 1 : if ((binoptab == ior_optab || binoptab == and_optab
2323 1 : || binoptab == xor_optab
2324 1 : || binoptab == add_optab || binoptab == sub_optab
2325 : || binoptab == smul_optab || binoptab == ashl_optab)
2326 0 : && mclass == MODE_INT)
2327 1 : no_extend = true;
2328 :
2329 1 : xop0 = widen_operand (xop0, wider_mode, mode,
2330 : unsignedp, no_extend);
2331 :
2332 : /* The second operand of a shift must always be extended. */
2333 2 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2334 1 : no_extend && binoptab != ashl_optab);
2335 :
2336 1 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2337 : unsignedp, methods);
2338 1 : if (temp)
2339 : {
2340 1 : if (mclass != MODE_INT
2341 1 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2342 : {
2343 0 : if (target == 0)
2344 0 : target = gen_reg_rtx (mode);
2345 0 : convert_move (target, temp, 0);
2346 0 : return target;
2347 : }
2348 : else
2349 1 : return gen_lowpart (mode, temp);
2350 : }
2351 : else
2352 0 : delete_insns_since (last);
2353 : }
2354 : }
2355 : }
2356 :
2357 12813 : delete_insns_since (entry_last);
2358 12813 : return 0;
2359 : }
2360 :
2361 : /* Expand a binary operator which has both signed and unsigned forms.
2362 : UOPTAB is the optab for unsigned operations, and SOPTAB is for
2363 : signed operations.
2364 :
2365 : If we widen unsigned operands, we may use a signed wider operation instead
2366 : of an unsigned wider operation, since the result would be the same. */
2367 :
2368 : rtx
2369 112619 : sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
2370 : rtx op0, rtx op1, rtx target, int unsignedp,
2371 : enum optab_methods methods)
2372 : {
2373 112619 : rtx temp;
2374 112619 : optab direct_optab = unsignedp ? uoptab : soptab;
2375 112619 : bool save_enable;
2376 :
2377 : /* Do it without widening, if possible. */
2378 112619 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2379 : unsignedp, OPTAB_DIRECT);
2380 112619 : if (temp || methods == OPTAB_DIRECT)
2381 : return temp;
2382 :
2383 : /* Try widening to a signed int. Disable any direct use of any
2384 : signed insn in the current mode. */
2385 8396 : save_enable = swap_optab_enable (soptab, mode, false);
2386 :
2387 8396 : temp = expand_binop (mode, soptab, op0, op1, target,
2388 : unsignedp, OPTAB_WIDEN);
2389 :
2390 : /* For unsigned operands, try widening to an unsigned int. */
2391 8396 : if (!temp && unsignedp)
2392 4295 : temp = expand_binop (mode, uoptab, op0, op1, target,
2393 : unsignedp, OPTAB_WIDEN);
2394 8396 : if (temp || methods == OPTAB_WIDEN)
2395 4841 : goto egress;
2396 :
2397 : /* Use the right width libcall if that exists. */
2398 3555 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2399 : unsignedp, OPTAB_LIB);
2400 3555 : if (temp || methods == OPTAB_LIB)
2401 3555 : goto egress;
2402 :
2403 : /* Must widen and use a libcall, use either signed or unsigned. */
2404 0 : temp = expand_binop (mode, soptab, op0, op1, target,
2405 : unsignedp, methods);
2406 0 : if (!temp && unsignedp)
2407 0 : temp = expand_binop (mode, uoptab, op0, op1, target,
2408 : unsignedp, methods);
2409 :
2410 0 : egress:
2411 : /* Undo the fiddling above. */
2412 8396 : if (save_enable)
2413 0 : swap_optab_enable (soptab, mode, true);
2414 : return temp;
2415 : }
2416 :
2417 : /* Generate code to perform an operation specified by UNOPPTAB
2418 : on operand OP0, with two results to TARG0 and TARG1.
2419 : We assume that the order of the operands for the instruction
2420 : is TARG0, TARG1, OP0.
2421 :
2422 : Either TARG0 or TARG1 may be zero, but what that means is that
2423 : the result is not actually wanted. We will generate it into
2424 : a dummy pseudo-reg and discard it. They may not both be zero.
2425 :
2426 : Returns true if this operation can be performed; false if not. */
2427 :
2428 : bool
2429 6 : expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2430 : int unsignedp)
2431 : {
2432 6 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2433 6 : enum mode_class mclass;
2434 6 : machine_mode wider_mode;
2435 6 : rtx_insn *entry_last = get_last_insn ();
2436 6 : rtx_insn *last;
2437 :
2438 6 : mclass = GET_MODE_CLASS (mode);
2439 :
2440 6 : if (!targ0)
2441 0 : targ0 = gen_reg_rtx (mode);
2442 6 : if (!targ1)
2443 0 : targ1 = gen_reg_rtx (mode);
2444 :
2445 : /* Record where to go back to if we fail. */
2446 6 : last = get_last_insn ();
2447 :
2448 6 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2449 : {
2450 6 : class expand_operand ops[3];
2451 6 : enum insn_code icode = optab_handler (unoptab, mode);
2452 :
2453 6 : create_fixed_operand (&ops[0], targ0);
2454 6 : create_fixed_operand (&ops[1], targ1);
2455 6 : create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2456 6 : if (maybe_expand_insn (icode, 3, ops))
2457 6 : return true;
2458 : }
2459 :
2460 : /* It can't be done in this mode. Can we do it in a wider mode? */
2461 :
2462 0 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2463 : {
2464 0 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2465 : {
2466 0 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2467 : {
2468 0 : rtx t0 = gen_reg_rtx (wider_mode);
2469 0 : rtx t1 = gen_reg_rtx (wider_mode);
2470 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2471 :
2472 0 : if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2473 : {
2474 0 : convert_move (targ0, t0, unsignedp);
2475 0 : convert_move (targ1, t1, unsignedp);
2476 0 : return true;
2477 : }
2478 : else
2479 0 : delete_insns_since (last);
2480 : }
2481 : }
2482 : }
2483 :
2484 0 : delete_insns_since (entry_last);
2485 0 : return false;
2486 : }
2487 :
2488 : /* Generate code to perform an operation specified by BINOPTAB
2489 : on operands OP0 and OP1, with two results to TARG1 and TARG2.
2490 : We assume that the order of the operands for the instruction
2491 : is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2492 : [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2493 :
2494 : Either TARG0 or TARG1 may be zero, but what that means is that
2495 : the result is not actually wanted. We will generate it into
2496 : a dummy pseudo-reg and discard it. They may not both be zero.
2497 :
2498 : Returns true if this operation can be performed; false if not. */
2499 :
2500 : bool
2501 119888 : expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2502 : int unsignedp)
2503 : {
2504 119888 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2505 119888 : enum mode_class mclass;
2506 119888 : machine_mode wider_mode;
2507 119888 : rtx_insn *entry_last = get_last_insn ();
2508 119888 : rtx_insn *last;
2509 :
2510 119888 : mclass = GET_MODE_CLASS (mode);
2511 :
2512 119888 : if (!targ0)
2513 41189 : targ0 = gen_reg_rtx (mode);
2514 119888 : if (!targ1)
2515 67238 : targ1 = gen_reg_rtx (mode);
2516 :
2517 : /* Record where to go back to if we fail. */
2518 119888 : last = get_last_insn ();
2519 :
2520 119888 : if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2521 : {
2522 115011 : class expand_operand ops[4];
2523 115011 : enum insn_code icode = optab_handler (binoptab, mode);
2524 115011 : machine_mode mode0 = insn_data[icode].operand[1].mode;
2525 115011 : machine_mode mode1 = insn_data[icode].operand[2].mode;
2526 115011 : rtx xop0 = op0, xop1 = op1;
2527 :
2528 : /* If we are optimizing, force expensive constants into a register. */
2529 115011 : xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2530 115011 : xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2531 :
2532 115011 : create_fixed_operand (&ops[0], targ0);
2533 115011 : create_convert_operand_from (&ops[1], xop0, mode, unsignedp);
2534 115011 : create_convert_operand_from (&ops[2], xop1, mode, unsignedp);
2535 115011 : create_fixed_operand (&ops[3], targ1);
2536 115011 : if (maybe_expand_insn (icode, 4, ops))
2537 115011 : return true;
2538 0 : delete_insns_since (last);
2539 : }
2540 :
2541 : /* It can't be done in this mode. Can we do it in a wider mode? */
2542 :
2543 4877 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2544 : {
2545 15494 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2546 : {
2547 10617 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2548 : {
2549 0 : rtx t0 = gen_reg_rtx (wider_mode);
2550 0 : rtx t1 = gen_reg_rtx (wider_mode);
2551 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2552 0 : rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2553 :
2554 0 : if (expand_twoval_binop (binoptab, cop0, cop1,
2555 : t0, t1, unsignedp))
2556 : {
2557 0 : convert_move (targ0, t0, unsignedp);
2558 0 : convert_move (targ1, t1, unsignedp);
2559 0 : return true;
2560 : }
2561 : else
2562 0 : delete_insns_since (last);
2563 : }
2564 : }
2565 : }
2566 :
2567 4877 : delete_insns_since (entry_last);
2568 4877 : return false;
2569 : }
2570 :
2571 : /* Expand the two-valued library call indicated by BINOPTAB, but
2572 : preserve only one of the values. If TARG0 is non-NULL, the first
2573 : value is placed into TARG0; otherwise the second value is placed
2574 : into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2575 : value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2576 : This routine assumes that the value returned by the library call is
2577 : as if the return value was of an integral mode twice as wide as the
2578 : mode of OP0. Returns 1 if the call was successful. */
2579 :
2580 : bool
2581 0 : expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2582 : rtx targ0, rtx targ1, enum rtx_code code)
2583 : {
2584 0 : machine_mode mode;
2585 0 : machine_mode libval_mode;
2586 0 : rtx libval;
2587 0 : rtx_insn *insns;
2588 0 : rtx libfunc;
2589 :
2590 : /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2591 0 : gcc_assert (!targ0 != !targ1);
2592 :
2593 0 : mode = GET_MODE (op0);
2594 0 : libfunc = optab_libfunc (binoptab, mode);
2595 0 : if (!libfunc)
2596 : return false;
2597 :
2598 : /* The value returned by the library function will have twice as
2599 : many bits as the nominal MODE. */
2600 0 : auto return_size = 2 * GET_MODE_BITSIZE (mode);
2601 0 : if (!smallest_int_mode_for_size (return_size).exists (&libval_mode))
2602 0 : return false;
2603 :
2604 0 : start_sequence ();
2605 0 : libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2606 : libval_mode,
2607 : op0, mode,
2608 : op1, mode);
2609 : /* Get the part of VAL containing the value that we want. */
2610 0 : libval = simplify_gen_subreg (mode, libval, libval_mode,
2611 0 : targ0 ? 0 : GET_MODE_SIZE (mode));
2612 0 : insns = end_sequence ();
2613 : /* Move the into the desired location. */
2614 0 : emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2615 : gen_rtx_fmt_ee (code, mode, op0, op1));
2616 :
2617 0 : return true;
2618 : }
2619 :
2620 :
2621 : /* Wrapper around expand_unop which takes an rtx code to specify
2622 : the operation to perform, not an optab pointer. All other
2623 : arguments are the same. */
2624 : rtx
2625 19495 : expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2626 : rtx target, int unsignedp)
2627 : {
2628 19495 : optab unop = code_to_optab (code);
2629 19495 : gcc_assert (unop);
2630 :
2631 19495 : return expand_unop (mode, unop, op0, target, unsignedp);
2632 : }
2633 :
2634 : /* Try calculating
2635 : (clz:narrow x)
2636 : as
2637 : (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2638 :
2639 : A similar operation can be used for clrsb. UNOPTAB says which operation
2640 : we are trying to expand. */
2641 : static rtx
2642 297 : widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2643 : {
2644 297 : opt_scalar_int_mode wider_mode_iter;
2645 1219 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2646 : {
2647 922 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2648 922 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2649 : {
2650 0 : rtx xop0, temp;
2651 0 : rtx_insn *last;
2652 :
2653 0 : last = get_last_insn ();
2654 :
2655 0 : if (target == 0)
2656 0 : target = gen_reg_rtx (mode);
2657 0 : xop0 = widen_operand (op0, wider_mode, mode,
2658 : unoptab != clrsb_optab, false);
2659 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2660 : unoptab != clrsb_optab);
2661 0 : if (temp != 0)
2662 0 : temp = expand_binop
2663 0 : (wider_mode, sub_optab, temp,
2664 0 : gen_int_mode (GET_MODE_PRECISION (wider_mode)
2665 0 : - GET_MODE_PRECISION (mode),
2666 : wider_mode),
2667 : target, true, OPTAB_DIRECT);
2668 0 : if (temp == 0)
2669 0 : delete_insns_since (last);
2670 :
2671 297 : return temp;
2672 : }
2673 : }
2674 : return 0;
2675 : }
2676 :
2677 : /* Attempt to emit (clrsb:mode op0) as
2678 : (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2679 : (const_int -1))
2680 : if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2681 : or as
2682 : (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2683 : (ashr:mode op0 (const_int prec-1)))
2684 : (const_int 1)))
2685 : otherwise. */
2686 :
2687 : static rtx
2688 92 : expand_clrsb_using_clz (scalar_int_mode mode, rtx op0, rtx target)
2689 : {
2690 92 : if (optimize_insn_for_size_p ()
2691 92 : || optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2692 : return NULL_RTX;
2693 :
2694 88 : start_sequence ();
2695 88 : HOST_WIDE_INT val = 0;
2696 88 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) != 2
2697 88 : || val != GET_MODE_PRECISION (mode))
2698 : val = 0;
2699 : else
2700 : val = 1;
2701 :
2702 86 : rtx temp2 = op0;
2703 86 : if (!val)
2704 : {
2705 86 : temp2 = expand_binop (mode, ashl_optab, op0, const1_rtx,
2706 : NULL_RTX, 0, OPTAB_DIRECT);
2707 86 : if (!temp2)
2708 : {
2709 0 : fail:
2710 0 : end_sequence ();
2711 0 : return NULL_RTX;
2712 : }
2713 : }
2714 :
2715 88 : rtx temp = expand_binop (mode, ashr_optab, op0,
2716 88 : GEN_INT (GET_MODE_PRECISION (mode) - 1),
2717 : NULL_RTX, 0, OPTAB_DIRECT);
2718 88 : if (!temp)
2719 0 : goto fail;
2720 :
2721 88 : temp = expand_binop (mode, xor_optab, temp2, temp, NULL_RTX, 0,
2722 : OPTAB_DIRECT);
2723 88 : if (!temp)
2724 0 : goto fail;
2725 :
2726 88 : if (!val)
2727 : {
2728 86 : temp = expand_binop (mode, ior_optab, temp, const1_rtx,
2729 : NULL_RTX, 0, OPTAB_DIRECT);
2730 86 : if (!temp)
2731 0 : goto fail;
2732 : }
2733 88 : temp = expand_unop_direct (mode, clz_optab, temp, val ? NULL_RTX : target,
2734 : true);
2735 88 : if (!temp)
2736 0 : goto fail;
2737 88 : if (val)
2738 : {
2739 2 : temp = expand_binop (mode, add_optab, temp, constm1_rtx,
2740 : target, 0, OPTAB_DIRECT);
2741 2 : if (!temp)
2742 0 : goto fail;
2743 : }
2744 :
2745 88 : rtx_insn *seq = end_sequence ();
2746 :
2747 88 : add_equal_note (seq, temp, CLRSB, op0, NULL_RTX, mode);
2748 88 : emit_insn (seq);
2749 88 : return temp;
2750 : }
2751 :
2752 : static rtx expand_ffs (scalar_int_mode, rtx, rtx);
2753 :
2754 : /* Try calculating clz, ctz or ffs of a double-word quantity as two clz, ctz or
2755 : ffs operations on word-sized quantities, choosing which based on whether the
2756 : high (for clz) or low (for ctz and ffs) word is nonzero. */
2757 : static rtx
2758 250 : expand_doubleword_clz_ctz_ffs (scalar_int_mode mode, rtx op0, rtx target,
2759 : optab unoptab)
2760 : {
2761 250 : rtx xop0 = force_reg (mode, op0);
2762 250 : rtx subhi = gen_highpart (word_mode, xop0);
2763 250 : rtx sublo = gen_lowpart (word_mode, xop0);
2764 250 : rtx_code_label *hi0_label = gen_label_rtx ();
2765 250 : rtx_code_label *after_label = gen_label_rtx ();
2766 250 : rtx_insn *seq;
2767 250 : rtx temp, result;
2768 250 : int addend = 0;
2769 :
2770 : /* If we were not given a target, use a word_mode register, not a
2771 : 'mode' register. The result will fit, and nobody is expecting
2772 : anything bigger (the return type of __builtin_clz* is int). */
2773 250 : if (!target)
2774 0 : target = gen_reg_rtx (word_mode);
2775 :
2776 : /* In any case, write to a word_mode scratch in both branches of the
2777 : conditional, so we can ensure there is a single move insn setting
2778 : 'target' to tag a REG_EQUAL note on. */
2779 250 : result = gen_reg_rtx (word_mode);
2780 :
2781 250 : if (unoptab != clz_optab)
2782 45 : std::swap (subhi, sublo);
2783 :
2784 250 : start_sequence ();
2785 :
2786 : /* If the high word is not equal to zero,
2787 : then clz of the full value is clz of the high word. */
2788 250 : emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2789 : word_mode, true, hi0_label);
2790 :
2791 250 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2792 250 : temp = expand_unop_direct (word_mode, unoptab, subhi, result, true);
2793 : else
2794 : {
2795 0 : gcc_assert (unoptab == ffs_optab);
2796 0 : temp = expand_ffs (word_mode, subhi, result);
2797 : }
2798 250 : if (!temp)
2799 0 : goto fail;
2800 :
2801 250 : if (temp != result)
2802 0 : convert_move (result, temp, true);
2803 :
2804 250 : emit_jump_insn (targetm.gen_jump (after_label));
2805 250 : emit_barrier ();
2806 :
2807 : /* Else clz of the full value is clz of the low word plus the number
2808 : of bits in the high word. Similarly for ctz/ffs of the high word,
2809 : except that ffs should be 0 when both words are zero. */
2810 250 : emit_label (hi0_label);
2811 :
2812 250 : if (unoptab == ffs_optab)
2813 : {
2814 0 : convert_move (result, const0_rtx, true);
2815 0 : emit_cmp_and_jump_insns (sublo, CONST0_RTX (word_mode), EQ, 0,
2816 : word_mode, true, after_label);
2817 : }
2818 :
2819 250 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2820 250 : temp = expand_unop_direct (word_mode, unoptab, sublo, NULL_RTX, true);
2821 : else
2822 : {
2823 0 : gcc_assert (unoptab == ffs_optab);
2824 0 : temp = expand_unop_direct (word_mode, ctz_optab, sublo, NULL_RTX, true);
2825 0 : addend = 1;
2826 : }
2827 :
2828 250 : if (!temp)
2829 0 : goto fail;
2830 :
2831 500 : temp = expand_binop (word_mode, add_optab, temp,
2832 250 : gen_int_mode (GET_MODE_BITSIZE (word_mode) + addend,
2833 : word_mode),
2834 : result, true, OPTAB_DIRECT);
2835 250 : if (!temp)
2836 0 : goto fail;
2837 250 : if (temp != result)
2838 0 : convert_move (result, temp, true);
2839 :
2840 250 : emit_label (after_label);
2841 250 : convert_move (target, result, true);
2842 :
2843 250 : seq = end_sequence ();
2844 :
2845 250 : add_equal_note (seq, target, optab_to_code (unoptab), xop0, NULL_RTX, mode);
2846 250 : emit_insn (seq);
2847 250 : return target;
2848 :
2849 0 : fail:
2850 0 : end_sequence ();
2851 0 : return 0;
2852 : }
2853 :
2854 : /* Try calculating popcount of a double-word quantity as two popcount's of
2855 : word-sized quantities and summing up the results. */
2856 : static rtx
2857 0 : expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2858 : {
2859 0 : rtx t0, t1, t;
2860 0 : rtx_insn *seq;
2861 :
2862 0 : start_sequence ();
2863 :
2864 0 : t0 = expand_unop_direct (word_mode, popcount_optab,
2865 : operand_subword_force (op0, 0, mode), NULL_RTX,
2866 : true);
2867 0 : t1 = expand_unop_direct (word_mode, popcount_optab,
2868 : operand_subword_force (op0, 1, mode), NULL_RTX,
2869 : true);
2870 0 : if (!t0 || !t1)
2871 : {
2872 0 : end_sequence ();
2873 0 : return NULL_RTX;
2874 : }
2875 :
2876 : /* If we were not given a target, use a word_mode register, not a
2877 : 'mode' register. The result will fit, and nobody is expecting
2878 : anything bigger (the return type of __builtin_popcount* is int). */
2879 0 : if (!target)
2880 0 : target = gen_reg_rtx (word_mode);
2881 :
2882 0 : t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2883 :
2884 0 : seq = end_sequence ();
2885 :
2886 0 : add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2887 0 : emit_insn (seq);
2888 0 : return t;
2889 : }
2890 :
2891 : /* Try calculating
2892 : (parity:wide x)
2893 : as
2894 : (parity:narrow (low (x) ^ high (x))) */
2895 : static rtx
2896 0 : expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2897 : {
2898 0 : rtx t = expand_binop (word_mode, xor_optab,
2899 : operand_subword_force (op0, 0, mode),
2900 : operand_subword_force (op0, 1, mode),
2901 : NULL_RTX, 0, OPTAB_DIRECT);
2902 0 : return expand_unop (word_mode, parity_optab, t, target, true);
2903 : }
2904 :
2905 : /* Try calculating
2906 : (bswap:narrow x)
2907 : as
2908 : (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow)))
2909 : or similarly for bitreverse. */
2910 : static rtx
2911 210 : widen_bswap_or_bitreverse (scalar_int_mode mode, rtx op0, rtx target,
2912 : optab unoptab)
2913 : {
2914 210 : rtx x;
2915 210 : rtx_insn *last;
2916 210 : opt_scalar_int_mode wider_mode_iter;
2917 :
2918 855 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2919 645 : if (optab_handler (unoptab, wider_mode_iter.require ())
2920 : != CODE_FOR_nothing)
2921 : break;
2922 :
2923 210 : if (!wider_mode_iter.exists ())
2924 : return NULL_RTX;
2925 :
2926 0 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2927 0 : last = get_last_insn ();
2928 :
2929 0 : x = widen_operand (op0, wider_mode, mode, true, true);
2930 0 : x = expand_unop (wider_mode, unoptab, x, NULL_RTX, true);
2931 :
2932 0 : gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2933 : && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2934 0 : if (x != 0)
2935 0 : x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2936 0 : GET_MODE_BITSIZE (wider_mode)
2937 0 : - GET_MODE_BITSIZE (mode),
2938 : NULL_RTX, true);
2939 :
2940 0 : if (x != 0)
2941 : {
2942 0 : if (target == 0)
2943 0 : target = gen_reg_rtx (mode);
2944 0 : emit_move_insn (target, gen_lowpart (mode, x));
2945 : }
2946 : else
2947 0 : delete_insns_since (last);
2948 :
2949 : return target;
2950 : }
2951 :
2952 : /* Try calculating bswap as two bswaps of two word-sized operands.
2953 : Similarly for bitreverse. */
2954 :
2955 : static rtx
2956 163 : expand_doubleword_bswap_or_bitreverse (machine_mode mode, rtx op, rtx target,
2957 : optab unoptab)
2958 : {
2959 163 : rtx t0, t1;
2960 :
2961 163 : t1 = expand_unop (word_mode, unoptab,
2962 : operand_subword_force (op, 0, mode), NULL_RTX, true);
2963 163 : t0 = expand_unop (word_mode, unoptab,
2964 : operand_subword_force (op, 1, mode), NULL_RTX, true);
2965 :
2966 163 : if (target == 0 || !valid_multiword_target_p (target))
2967 6 : target = gen_reg_rtx (mode);
2968 163 : if (REG_P (target))
2969 163 : emit_clobber (target);
2970 163 : emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2971 163 : emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2972 :
2973 163 : return target;
2974 : }
2975 :
2976 : /* Try calculating (bitreverse x) using masks and shifts. */
2977 :
2978 : static rtx
2979 47 : expand_bitreverse (scalar_int_mode mode, rtx op0, rtx target)
2980 : {
2981 47 : unsigned int precision = GET_MODE_BITSIZE (mode);
2982 47 : rtx_insn *last;
2983 :
2984 : /* Operation requires at least 4 bits (one nibble swap makes no sense below
2985 : that). */
2986 47 : if (precision < 4)
2987 : return NULL_RTX;
2988 :
2989 47 : if (rtx temp = widen_bswap_or_bitreverse (mode, op0, target,
2990 : bitreverse_optab))
2991 : return temp;
2992 :
2993 94 : if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
2994 47 : && optab_handler (bitreverse_optab, word_mode) != CODE_FOR_nothing)
2995 0 : if (rtx temp = expand_doubleword_bswap_or_bitreverse (mode, op0, target,
2996 : bitreverse_optab))
2997 0 : return temp;
2998 :
2999 47 : if (target == NULL_RTX
3000 47 : || target == op0
3001 47 : || reg_overlap_mentioned_p (target, op0))
3002 0 : target = gen_reg_rtx (mode);
3003 :
3004 47 : last = get_last_insn ();
3005 :
3006 47 : rtx x, lo, hi;
3007 :
3008 : /* Step 1: byte-swap (only meaningful for >= 16 bits). */
3009 47 : if (precision >= 16)
3010 : {
3011 42 : x = expand_unop (mode, bswap_optab, op0, NULL_RTX, true);
3012 42 : if (x == NULL_RTX)
3013 0 : goto fail;
3014 : }
3015 : else
3016 : x = op0;
3017 :
3018 : /* Step 2: swap nibbles within each byte (shift=4, only for >= 8 bits). */
3019 47 : if (precision >= 8)
3020 : {
3021 47 : wide_int mask = wi::zero (precision);
3022 354 : for (unsigned int start = 0; start < precision; start += 8)
3023 614 : mask = wi::bit_or (mask, wi::shifted_mask (start, 4, false,
3024 307 : precision));
3025 :
3026 47 : rtx mask_rtx = immed_wide_int_const (mask, mode);
3027 :
3028 47 : hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (4),
3029 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3030 47 : if (hi == NULL_RTX) goto fail;
3031 47 : hi = expand_binop (mode, and_optab, hi, mask_rtx,
3032 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3033 47 : if (hi == NULL_RTX) goto fail;
3034 :
3035 47 : lo = expand_binop (mode, and_optab, x, mask_rtx,
3036 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3037 47 : if (lo == NULL_RTX) goto fail;
3038 47 : lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (4),
3039 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3040 47 : if (lo == NULL_RTX) goto fail;
3041 :
3042 47 : x = expand_binop (mode, aop_optab, hi, lo,
3043 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3044 47 : if (x == NULL_RTX) goto fail;
3045 0 : }
3046 :
3047 : /* Step 3: swap pairs of bits within each nibble (shift=2). */
3048 47 : {
3049 47 : wide_int mask = wi::zero (precision);
3050 661 : for (unsigned int start = 0; start < precision; start += 4)
3051 614 : mask = wi::bit_or (mask, wi::shifted_mask (start, 2, false, precision));
3052 :
3053 47 : rtx mask_rtx = immed_wide_int_const (mask, mode);
3054 :
3055 47 : hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (2),
3056 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3057 47 : if (hi == NULL_RTX) goto fail;
3058 47 : hi = expand_binop (mode, and_optab, hi, mask_rtx,
3059 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3060 47 : if (hi == NULL_RTX) goto fail;
3061 :
3062 47 : lo = expand_binop (mode, and_optab, x, mask_rtx,
3063 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3064 47 : if (lo == NULL_RTX) goto fail;
3065 47 : lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (2),
3066 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3067 47 : if (lo == NULL_RTX) goto fail;
3068 :
3069 47 : x = expand_binop (mode, aop_optab, hi, lo,
3070 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3071 47 : if (x == NULL_RTX) goto fail;
3072 0 : }
3073 :
3074 : /* Step 4: swap adjacent bits (shift=1). */
3075 47 : {
3076 47 : wide_int mask = wi::zero (precision);
3077 1275 : for (unsigned int start = 0; start < precision; start += 2)
3078 2456 : mask = wi::bit_or (mask, wi::shifted_mask (start, 1, false,
3079 1228 : precision));
3080 :
3081 47 : rtx mask_rtx = immed_wide_int_const (mask, mode);
3082 :
3083 47 : hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (1),
3084 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3085 47 : if (hi == NULL_RTX) goto fail;
3086 47 : hi = expand_binop (mode, and_optab, hi, mask_rtx,
3087 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3088 47 : if (hi == NULL_RTX) goto fail;
3089 :
3090 47 : lo = expand_binop (mode, and_optab, x, mask_rtx,
3091 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3092 47 : if (lo == NULL_RTX) goto fail;
3093 47 : lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (1),
3094 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3095 47 : if (lo == NULL_RTX) goto fail;
3096 :
3097 47 : x = expand_binop (mode, aop_optab, hi, lo,
3098 : target, true, OPTAB_LIB_WIDEN);
3099 47 : if (x == NULL_RTX) goto fail;
3100 0 : }
3101 :
3102 47 : if (x != target)
3103 0 : emit_move_insn (target, x);
3104 :
3105 : return target;
3106 :
3107 0 : fail:
3108 0 : delete_insns_since (last);
3109 0 : return NULL_RTX;
3110 : }
3111 :
3112 : /* Try calculating (parity x) as (and (popcount x) 1), where
3113 : popcount can also be done in a wider mode. */
3114 : static rtx
3115 11 : expand_parity (scalar_int_mode mode, rtx op0, rtx target)
3116 : {
3117 11 : enum mode_class mclass = GET_MODE_CLASS (mode);
3118 11 : opt_scalar_int_mode wider_mode_iter;
3119 11 : FOR_EACH_MODE_FROM (wider_mode_iter, mode)
3120 : {
3121 11 : scalar_int_mode wider_mode = wider_mode_iter.require ();
3122 11 : if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
3123 : {
3124 11 : rtx xop0, temp;
3125 11 : rtx_insn *last;
3126 :
3127 11 : last = get_last_insn ();
3128 :
3129 11 : if (target == 0 || GET_MODE (target) != wider_mode)
3130 6 : target = gen_reg_rtx (wider_mode);
3131 :
3132 11 : xop0 = widen_operand (op0, wider_mode, mode, true, false);
3133 11 : temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
3134 : true);
3135 11 : if (temp != 0)
3136 11 : temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
3137 : target, true, OPTAB_DIRECT);
3138 :
3139 11 : if (temp)
3140 : {
3141 11 : if (mclass != MODE_INT
3142 11 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3143 0 : return convert_to_mode (mode, temp, 0);
3144 : else
3145 11 : return gen_lowpart (mode, temp);
3146 : }
3147 : else
3148 0 : delete_insns_since (last);
3149 : }
3150 : }
3151 : return 0;
3152 : }
3153 :
3154 : /* Try calculating ctz(x) as K - clz(x & -x) ,
3155 : where K is GET_MODE_PRECISION(mode) - 1.
3156 :
3157 : Both __builtin_ctz and __builtin_clz are undefined at zero, so we
3158 : don't have to worry about what the hardware does in that case. (If
3159 : the clz instruction produces the usual value at 0, which is K, the
3160 : result of this code sequence will be -1; expand_ffs, below, relies
3161 : on this. It might be nice to have it be K instead, for consistency
3162 : with the (very few) processors that provide a ctz with a defined
3163 : value, but that would take one more instruction, and it would be
3164 : less convenient for expand_ffs anyway. */
3165 :
3166 : static rtx
3167 48 : expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
3168 : {
3169 48 : rtx_insn *seq;
3170 48 : rtx temp;
3171 :
3172 48 : if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
3173 : return 0;
3174 :
3175 0 : start_sequence ();
3176 :
3177 0 : temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
3178 0 : if (temp)
3179 0 : temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
3180 : true, OPTAB_DIRECT);
3181 0 : if (temp)
3182 0 : temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
3183 0 : if (temp)
3184 0 : temp = expand_binop (mode, sub_optab,
3185 0 : gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
3186 : temp, target,
3187 : true, OPTAB_DIRECT);
3188 0 : if (temp == 0)
3189 : {
3190 0 : end_sequence ();
3191 0 : return 0;
3192 : }
3193 :
3194 0 : seq = end_sequence ();
3195 :
3196 0 : add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
3197 0 : emit_insn (seq);
3198 0 : return temp;
3199 : }
3200 :
3201 :
3202 : /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
3203 : else with the sequence used by expand_clz.
3204 :
3205 : The ffs builtin promises to return zero for a zero value and ctz/clz
3206 : may have an undefined value in that case. If they do not give us a
3207 : convenient value, we have to generate a test and branch. */
3208 : static rtx
3209 0 : expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
3210 : {
3211 0 : HOST_WIDE_INT val = 0;
3212 0 : bool defined_at_zero = false;
3213 0 : rtx temp;
3214 0 : rtx_insn *seq;
3215 :
3216 0 : if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
3217 : {
3218 0 : start_sequence ();
3219 :
3220 0 : temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
3221 0 : if (!temp)
3222 0 : goto fail;
3223 :
3224 0 : defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
3225 : }
3226 0 : else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
3227 : {
3228 0 : start_sequence ();
3229 0 : temp = expand_ctz (mode, op0, 0);
3230 0 : if (!temp)
3231 0 : goto fail;
3232 :
3233 0 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
3234 : {
3235 0 : defined_at_zero = true;
3236 0 : val = (GET_MODE_PRECISION (mode) - 1) - val;
3237 : }
3238 : }
3239 : else
3240 : return 0;
3241 :
3242 0 : if (defined_at_zero && val == -1)
3243 : /* No correction needed at zero. */;
3244 : else
3245 : {
3246 : /* We don't try to do anything clever with the situation found
3247 : on some processors (eg Alpha) where ctz(0:mode) ==
3248 : bitsize(mode). If someone can think of a way to send N to -1
3249 : and leave alone all values in the range 0..N-1 (where N is a
3250 : power of two), cheaper than this test-and-branch, please add it.
3251 :
3252 : The test-and-branch is done after the operation itself, in case
3253 : the operation sets condition codes that can be recycled for this.
3254 : (This is true on i386, for instance.) */
3255 :
3256 0 : rtx_code_label *nonzero_label = gen_label_rtx ();
3257 0 : emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
3258 : mode, true, nonzero_label);
3259 :
3260 0 : convert_move (temp, GEN_INT (-1), false);
3261 0 : emit_label (nonzero_label);
3262 : }
3263 :
3264 : /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3265 : to produce a value in the range 0..bitsize. */
3266 0 : temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
3267 : target, false, OPTAB_DIRECT);
3268 0 : if (!temp)
3269 0 : goto fail;
3270 :
3271 0 : seq = end_sequence ();
3272 :
3273 0 : add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
3274 0 : emit_insn (seq);
3275 0 : return temp;
3276 :
3277 0 : fail:
3278 0 : end_sequence ();
3279 0 : return 0;
3280 : }
3281 :
3282 : /* Expand a floating point absolute value or negation operation via a
3283 : logical operation on the sign bit. MODE is the mode of the operands
3284 : and FMODE is the scalar inner mode. */
3285 :
3286 : static rtx
3287 2366 : expand_absneg_bit (rtx_code code, machine_mode mode,
3288 : scalar_float_mode fmode, rtx op0, rtx target)
3289 : {
3290 2366 : int bitpos, word, nwords, i;
3291 2366 : machine_mode new_mode;
3292 2366 : scalar_int_mode imode;
3293 2366 : rtx temp;
3294 2366 : rtx_insn *insns;
3295 :
3296 2366 : auto op = code == NEG ? neg_optab : abs_optab;
3297 2366 : if (!get_absneg_bit_mode (op, mode, fmode, &bitpos).exists (&new_mode))
3298 0 : return NULL_RTX;
3299 :
3300 4732 : imode = as_a<scalar_int_mode> (GET_MODE_INNER (new_mode));
3301 4800 : if (VECTOR_MODE_P (mode) || GET_MODE_SIZE (fmode) <= UNITS_PER_WORD)
3302 : {
3303 : word = 0;
3304 : nwords = 1;
3305 : }
3306 : else
3307 : {
3308 17 : if (FLOAT_WORDS_BIG_ENDIAN)
3309 : word = (GET_MODE_BITSIZE (fmode) - bitpos) / BITS_PER_WORD;
3310 : else
3311 17 : word = bitpos / BITS_PER_WORD;
3312 17 : bitpos = bitpos % BITS_PER_WORD;
3313 34 : nwords = (GET_MODE_BITSIZE (fmode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3314 : }
3315 :
3316 2366 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3317 2366 : if (code == ABS)
3318 2078 : mask = ~mask;
3319 :
3320 2366 : if (target == 0
3321 2366 : || target == op0
3322 682 : || reg_overlap_mentioned_p (target, op0)
3323 3044 : || (nwords > 1 && !valid_multiword_target_p (target)))
3324 1688 : target = gen_reg_rtx (mode);
3325 :
3326 2366 : if (nwords > 1)
3327 : {
3328 17 : start_sequence ();
3329 :
3330 68 : for (i = 0; i < nwords; ++i)
3331 : {
3332 34 : rtx targ_piece = operand_subword (target, i, 1, mode);
3333 34 : rtx op0_piece = operand_subword_force (op0, i, mode);
3334 :
3335 34 : if (i == word)
3336 : {
3337 46 : temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
3338 : op0_piece,
3339 17 : immed_wide_int_const (mask, imode),
3340 : targ_piece, 1, OPTAB_LIB_WIDEN);
3341 17 : if (temp != targ_piece)
3342 0 : emit_move_insn (targ_piece, temp);
3343 : }
3344 : else
3345 17 : emit_move_insn (targ_piece, op0_piece);
3346 : }
3347 :
3348 17 : insns = end_sequence ();
3349 :
3350 17 : emit_insn (insns);
3351 : }
3352 : else
3353 : {
3354 2349 : rtx mask_rtx = immed_wide_int_const (mask, imode);
3355 2349 : if (VECTOR_MODE_P (new_mode))
3356 0 : mask_rtx = gen_const_vec_duplicate (new_mode, mask_rtx);
3357 2625 : temp = expand_binop (new_mode, code == ABS ? and_optab : xor_optab,
3358 2349 : gen_lowpart (new_mode, op0), mask_rtx,
3359 2349 : gen_lowpart (new_mode, target), 1, OPTAB_LIB_WIDEN);
3360 2349 : target = force_lowpart_subreg (mode, temp, new_mode);
3361 :
3362 2349 : set_dst_reg_note (get_last_insn (), REG_EQUAL,
3363 : gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
3364 : target);
3365 : }
3366 :
3367 2366 : return target;
3368 2366 : }
3369 :
3370 : /* As expand_unop, but will fail rather than attempt the operation in a
3371 : different mode or with a libcall. */
3372 : static rtx
3373 184000 : expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
3374 : int unsignedp)
3375 : {
3376 184000 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
3377 : {
3378 180227 : class expand_operand ops[2];
3379 180227 : enum insn_code icode = optab_handler (unoptab, mode);
3380 180227 : rtx_insn *last = get_last_insn ();
3381 180227 : rtx_insn *pat;
3382 :
3383 180227 : create_output_operand (&ops[0], target, mode);
3384 180227 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
3385 180227 : pat = maybe_gen_insn (icode, 2, ops);
3386 180227 : if (pat)
3387 : {
3388 179916 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3389 230709 : && ! add_equal_note (pat, ops[0].value,
3390 : optab_to_code (unoptab),
3391 : ops[1].value, NULL_RTX, mode))
3392 : {
3393 1088 : delete_insns_since (last);
3394 180227 : return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3395 : }
3396 :
3397 179139 : emit_insn (pat);
3398 :
3399 179139 : return ops[0].value;
3400 : }
3401 : }
3402 : return 0;
3403 : }
3404 :
3405 : /* Generate code to perform an operation specified by UNOPTAB
3406 : on operand OP0, with result having machine-mode MODE.
3407 :
3408 : UNSIGNEDP is for the case where we have to widen the operands
3409 : to perform the operation. It says to use zero-extension.
3410 :
3411 : If TARGET is nonzero, the value
3412 : is generated there, if it is convenient to do so.
3413 : In all cases an rtx is returned for the locus of the value;
3414 : this may or may not be TARGET. */
3415 :
3416 : rtx
3417 183412 : expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
3418 : int unsignedp)
3419 : {
3420 183412 : enum mode_class mclass = GET_MODE_CLASS (mode);
3421 183412 : machine_mode wider_mode;
3422 183412 : scalar_int_mode int_mode;
3423 183412 : scalar_float_mode float_mode;
3424 183412 : rtx temp;
3425 183412 : rtx libfunc;
3426 :
3427 183412 : temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3428 183412 : if (temp)
3429 : return temp;
3430 :
3431 : /* It can't be done in this mode. Can we open-code it in a wider mode? */
3432 :
3433 : /* Widening (or narrowing) clz needs special treatment. */
3434 3773 : if (unoptab == clz_optab)
3435 : {
3436 205 : if (is_a <scalar_int_mode> (mode, &int_mode))
3437 : {
3438 205 : temp = widen_leading (int_mode, op0, target, unoptab);
3439 205 : if (temp)
3440 : return temp;
3441 :
3442 410 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3443 205 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3444 : {
3445 205 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target,
3446 : unoptab);
3447 205 : if (temp)
3448 : return temp;
3449 : }
3450 : }
3451 :
3452 0 : goto try_libcall;
3453 : }
3454 :
3455 3568 : if (unoptab == clrsb_optab)
3456 : {
3457 92 : if (is_a <scalar_int_mode> (mode, &int_mode))
3458 : {
3459 92 : temp = widen_leading (int_mode, op0, target, unoptab);
3460 92 : if (temp)
3461 : return temp;
3462 92 : temp = expand_clrsb_using_clz (int_mode, op0, target);
3463 92 : if (temp)
3464 : return temp;
3465 : }
3466 4 : goto try_libcall;
3467 : }
3468 :
3469 3476 : if (unoptab == popcount_optab
3470 660 : && is_a <scalar_int_mode> (mode, &int_mode)
3471 692 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3472 2 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3473 3476 : && optimize_insn_for_speed_p ())
3474 : {
3475 0 : temp = expand_doubleword_popcount (int_mode, op0, target);
3476 0 : if (temp)
3477 : return temp;
3478 : }
3479 :
3480 3476 : if (unoptab == parity_optab
3481 11 : && is_a <scalar_int_mode> (mode, &int_mode)
3482 11 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3483 0 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3484 0 : || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
3485 3476 : && optimize_insn_for_speed_p ())
3486 : {
3487 0 : temp = expand_doubleword_parity (int_mode, op0, target);
3488 0 : if (temp)
3489 : return temp;
3490 : }
3491 :
3492 : /* Widening (or narrowing) bswap needs special treatment. */
3493 3476 : if (unoptab == bswap_optab)
3494 : {
3495 : /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3496 : or ROTATERT. First try these directly; if this fails, then try the
3497 : obvious pair of shifts with allowed widening, as this will probably
3498 : be always more efficient than the other fallback methods. */
3499 163 : if (mode == HImode)
3500 : {
3501 0 : rtx_insn *last;
3502 0 : rtx temp1, temp2;
3503 :
3504 0 : if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3505 : {
3506 0 : temp = expand_binop (mode, rotl_optab, op0,
3507 : gen_int_shift_amount (mode, 8),
3508 : target, unsignedp, OPTAB_DIRECT);
3509 0 : if (temp)
3510 : return temp;
3511 : }
3512 :
3513 0 : if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3514 : {
3515 0 : temp = expand_binop (mode, rotr_optab, op0,
3516 : gen_int_shift_amount (mode, 8),
3517 : target, unsignedp, OPTAB_DIRECT);
3518 0 : if (temp)
3519 : return temp;
3520 : }
3521 :
3522 0 : last = get_last_insn ();
3523 :
3524 0 : temp1 = expand_binop (mode, ashl_optab, op0,
3525 : gen_int_shift_amount (mode, 8), NULL_RTX,
3526 : unsignedp, OPTAB_WIDEN);
3527 0 : temp2 = expand_binop (mode, lshr_optab, op0,
3528 : gen_int_shift_amount (mode, 8), NULL_RTX,
3529 : unsignedp, OPTAB_WIDEN);
3530 0 : if (temp1 && temp2)
3531 : {
3532 0 : temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3533 : unsignedp, OPTAB_WIDEN);
3534 0 : if (temp)
3535 : return temp;
3536 : }
3537 :
3538 0 : delete_insns_since (last);
3539 : }
3540 :
3541 163 : if (is_a <scalar_int_mode> (mode, &int_mode))
3542 : {
3543 163 : temp = widen_bswap_or_bitreverse (int_mode, op0, target, unoptab);
3544 163 : if (temp)
3545 : return temp;
3546 :
3547 : /* We do not provide a 128-bit bswap in libgcc so force the use of
3548 : a double bswap for 64-bit targets. */
3549 326 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3550 163 : && (UNITS_PER_WORD == 8
3551 147 : || optab_handler (unoptab, word_mode) != CODE_FOR_nothing))
3552 : {
3553 163 : temp = expand_doubleword_bswap_or_bitreverse (mode, op0, target,
3554 : unoptab);
3555 163 : if (temp)
3556 : return temp;
3557 : }
3558 : }
3559 :
3560 0 : goto try_libcall;
3561 : }
3562 :
3563 3313 : if (unoptab == bitreverse_optab && is_a <scalar_int_mode> (mode, &int_mode))
3564 47 : if (rtx tem = expand_bitreverse (int_mode, op0, target))
3565 : return tem;
3566 :
3567 : /* Neg should be tried via expand_absneg_bit before widening. */
3568 3266 : if (optab_to_code (unoptab) == NEG)
3569 : {
3570 : /* Try negating floating point values by flipping the sign bit. */
3571 762 : if (is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3572 : {
3573 288 : temp = expand_absneg_bit (NEG, mode, float_mode, op0, target);
3574 288 : if (temp)
3575 : return temp;
3576 : }
3577 :
3578 : /* If there is no negation pattern, and we have no negative zero,
3579 : try subtracting from zero. */
3580 93 : if (!HONOR_SIGNED_ZEROS (mode))
3581 : {
3582 93 : temp = expand_binop (mode, (unoptab == negv_optab
3583 : ? subv_optab : sub_optab),
3584 : CONST0_RTX (mode), op0, target,
3585 : unsignedp, OPTAB_DIRECT);
3586 93 : if (temp)
3587 : return temp;
3588 : }
3589 : }
3590 :
3591 : /* ABS also needs to be handled similarly. */
3592 2978 : if (optab_to_code (unoptab) == ABS
3593 5144 : && is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3594 : {
3595 2078 : temp = expand_absneg_bit (ABS, mode, float_mode, op0, target);
3596 2078 : if (temp)
3597 : return temp;
3598 : }
3599 :
3600 900 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3601 4025 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3602 : {
3603 3125 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3604 : {
3605 0 : rtx xop0 = op0;
3606 0 : rtx_insn *last = get_last_insn ();
3607 :
3608 : /* For certain operations, we need not actually extend
3609 : the narrow operand, as long as we will truncate the
3610 : results to the same narrowness. */
3611 :
3612 0 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3613 0 : (unoptab == neg_optab
3614 0 : || unoptab == one_cmpl_optab)
3615 : && mclass == MODE_INT);
3616 :
3617 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3618 : unsignedp);
3619 :
3620 0 : if (temp)
3621 : {
3622 0 : if (mclass != MODE_INT
3623 0 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3624 : {
3625 0 : if (target == 0)
3626 0 : target = gen_reg_rtx (mode);
3627 0 : convert_move (target, temp, 0);
3628 0 : return target;
3629 : }
3630 : else
3631 0 : return gen_lowpart (mode, temp);
3632 : }
3633 : else
3634 0 : delete_insns_since (last);
3635 : }
3636 : }
3637 :
3638 : /* These can be done a word at a time. */
3639 900 : if (unoptab == one_cmpl_optab
3640 0 : && is_int_mode (mode, &int_mode)
3641 0 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
3642 900 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3643 : {
3644 0 : int i;
3645 0 : rtx_insn *insns;
3646 :
3647 0 : if (target == 0
3648 0 : || target == op0
3649 0 : || reg_overlap_mentioned_p (target, op0)
3650 0 : || !valid_multiword_target_p (target))
3651 0 : target = gen_reg_rtx (int_mode);
3652 :
3653 0 : start_sequence ();
3654 :
3655 : /* Do the actual arithmetic. */
3656 0 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
3657 : {
3658 0 : rtx target_piece = operand_subword (target, i, 1, int_mode);
3659 0 : rtx x = expand_unop (word_mode, unoptab,
3660 0 : operand_subword_force (op0, i, int_mode),
3661 : target_piece, unsignedp);
3662 :
3663 0 : if (target_piece != x)
3664 0 : emit_move_insn (target_piece, x);
3665 : }
3666 :
3667 0 : insns = end_sequence ();
3668 :
3669 0 : emit_insn (insns);
3670 0 : return target;
3671 : }
3672 :
3673 : /* Emit ~op0 as op0 ^ -1. */
3674 900 : if (unoptab == one_cmpl_optab
3675 0 : && (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
3676 900 : && optab_handler (xor_optab, mode) != CODE_FOR_nothing)
3677 : {
3678 0 : temp = expand_binop (mode, xor_optab, op0, CONSTM1_RTX (mode),
3679 : target, unsignedp, OPTAB_DIRECT);
3680 0 : if (temp)
3681 : return temp;
3682 : }
3683 :
3684 : /* Try calculating parity (x) as popcount (x) % 2. */
3685 900 : if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
3686 : {
3687 11 : temp = expand_parity (int_mode, op0, target);
3688 11 : if (temp)
3689 : return temp;
3690 : }
3691 :
3692 : /* Try implementing ffs (x) in terms of clz (x). */
3693 889 : if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
3694 : {
3695 0 : temp = expand_ffs (int_mode, op0, target);
3696 0 : if (temp)
3697 : return temp;
3698 : }
3699 :
3700 : /* Try implementing ctz (x) in terms of clz (x). */
3701 889 : if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
3702 : {
3703 48 : temp = expand_ctz (int_mode, op0, target);
3704 48 : if (temp)
3705 : return temp;
3706 : }
3707 :
3708 889 : if ((unoptab == ctz_optab || unoptab == ffs_optab)
3709 48 : && optimize_insn_for_speed_p ()
3710 45 : && is_a <scalar_int_mode> (mode, &int_mode)
3711 90 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3712 934 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3713 0 : || optab_handler (ctz_optab, word_mode) != CODE_FOR_nothing))
3714 : {
3715 45 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target, unoptab);
3716 45 : if (temp)
3717 : return temp;
3718 : }
3719 :
3720 848 : try_libcall:
3721 : /* Now try a library call in this mode. */
3722 848 : libfunc = optab_libfunc (unoptab, mode);
3723 848 : if (libfunc)
3724 : {
3725 546 : rtx_insn *insns;
3726 546 : rtx value;
3727 546 : rtx eq_value;
3728 546 : machine_mode outmode = mode;
3729 :
3730 : /* All of these functions return small values. Thus we choose to
3731 : have them return something that isn't a double-word. */
3732 546 : if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3733 : || unoptab == clrsb_optab || unoptab == popcount_optab
3734 546 : || unoptab == parity_optab)
3735 453 : outmode
3736 453 : = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3737 : optab_libfunc (unoptab, mode)));
3738 :
3739 546 : start_sequence ();
3740 :
3741 : /* Pass 1 for NO_QUEUE so we don't lose any increments
3742 : if the libcall is cse'd or moved. */
3743 546 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3744 : op0, mode);
3745 546 : insns = end_sequence ();
3746 :
3747 546 : target = gen_reg_rtx (outmode);
3748 546 : bool trapv = trapv_unoptab_p (unoptab);
3749 546 : if (trapv)
3750 : eq_value = NULL_RTX;
3751 : else
3752 : {
3753 453 : eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3754 1359 : if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3755 423 : eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3756 90 : else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3757 0 : eq_value = simplify_gen_unary (ZERO_EXTEND,
3758 : outmode, eq_value, mode);
3759 : }
3760 546 : emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3761 :
3762 546 : return target;
3763 : }
3764 :
3765 : /* It can't be done in this mode. Can we do it in a wider mode? */
3766 :
3767 302 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3768 : {
3769 653 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3770 : {
3771 565 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3772 565 : || optab_libfunc (unoptab, wider_mode))
3773 : {
3774 214 : rtx xop0 = op0;
3775 214 : rtx_insn *last = get_last_insn ();
3776 :
3777 : /* For certain operations, we need not actually extend
3778 : the narrow operand, as long as we will truncate the
3779 : results to the same narrowness. */
3780 214 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3781 214 : (unoptab == neg_optab
3782 214 : || unoptab == one_cmpl_optab
3783 214 : || unoptab == bswap_optab)
3784 : && mclass == MODE_INT);
3785 :
3786 214 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3787 : unsignedp);
3788 :
3789 : /* If we are generating clz using wider mode, adjust the
3790 : result. Similarly for clrsb. */
3791 214 : if ((unoptab == clz_optab || unoptab == clrsb_optab)
3792 1 : && temp != 0)
3793 : {
3794 1 : scalar_int_mode wider_int_mode
3795 1 : = as_a <scalar_int_mode> (wider_mode);
3796 1 : int_mode = as_a <scalar_int_mode> (mode);
3797 1 : temp = expand_binop
3798 1 : (wider_mode, sub_optab, temp,
3799 1 : gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3800 2 : - GET_MODE_PRECISION (int_mode),
3801 : wider_int_mode),
3802 : target, true, OPTAB_DIRECT);
3803 : }
3804 :
3805 : /* Likewise for bswap. */
3806 214 : if (unoptab == bswap_optab && temp != 0)
3807 : {
3808 0 : scalar_int_mode wider_int_mode
3809 0 : = as_a <scalar_int_mode> (wider_mode);
3810 0 : int_mode = as_a <scalar_int_mode> (mode);
3811 0 : gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3812 : == GET_MODE_BITSIZE (wider_int_mode)
3813 : && GET_MODE_PRECISION (int_mode)
3814 : == GET_MODE_BITSIZE (int_mode));
3815 :
3816 0 : temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3817 0 : GET_MODE_BITSIZE (wider_int_mode)
3818 0 : - GET_MODE_BITSIZE (int_mode),
3819 : NULL_RTX, true);
3820 : }
3821 :
3822 214 : if (temp)
3823 : {
3824 214 : if (mclass != MODE_INT)
3825 : {
3826 0 : if (target == 0)
3827 0 : target = gen_reg_rtx (mode);
3828 0 : convert_move (target, temp, 0);
3829 0 : return target;
3830 : }
3831 : else
3832 214 : return gen_lowpart (mode, temp);
3833 : }
3834 : else
3835 0 : delete_insns_since (last);
3836 : }
3837 : }
3838 : }
3839 :
3840 : /* One final attempt at implementing negation via subtraction,
3841 : this time allowing widening of the operand. */
3842 88 : if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3843 : {
3844 0 : rtx temp;
3845 0 : temp = expand_binop (mode,
3846 : unoptab == negv_optab ? subv_optab : sub_optab,
3847 : CONST0_RTX (mode), op0,
3848 : target, unsignedp, OPTAB_LIB_WIDEN);
3849 0 : if (temp)
3850 : return temp;
3851 : }
3852 :
3853 : return 0;
3854 : }
3855 :
3856 : /* Emit code to compute the absolute value of OP0, with result to
3857 : TARGET if convenient. (TARGET may be 0.) The return value says
3858 : where the result actually is to be found.
3859 :
3860 : MODE is the mode of the operand; the mode of the result is
3861 : different but can be deduced from MODE.
3862 :
3863 : */
3864 :
3865 : rtx
3866 28321 : expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3867 : int result_unsignedp)
3868 : {
3869 28321 : rtx temp;
3870 :
3871 28321 : if (GET_MODE_CLASS (mode) != MODE_INT
3872 5091 : || ! flag_trapv)
3873 : result_unsignedp = 1;
3874 :
3875 : /* First try to do it with a special abs instruction. */
3876 28409 : temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3877 : op0, target, 0);
3878 28321 : if (temp != 0)
3879 : return temp;
3880 :
3881 : /* If we have a MAX insn, we can do this as MAX (x, -x). */
3882 88 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3883 88 : && !HONOR_SIGNED_ZEROS (mode))
3884 : {
3885 88 : rtx_insn *last = get_last_insn ();
3886 :
3887 176 : temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3888 : op0, NULL_RTX, 0);
3889 88 : if (temp != 0)
3890 88 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3891 : OPTAB_WIDEN);
3892 :
3893 88 : if (temp != 0)
3894 : return temp;
3895 :
3896 0 : delete_insns_since (last);
3897 : }
3898 :
3899 : /* If this machine has expensive jumps, we can do integer absolute
3900 : value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3901 : where W is the width of MODE. */
3902 :
3903 0 : scalar_int_mode int_mode;
3904 0 : if (is_int_mode (mode, &int_mode)
3905 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
3906 : false) >= 2)
3907 : {
3908 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3909 0 : GET_MODE_PRECISION (int_mode) - 1,
3910 : NULL_RTX, 0);
3911 :
3912 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3913 : OPTAB_LIB_WIDEN);
3914 0 : if (temp != 0)
3915 0 : temp = expand_binop (int_mode,
3916 : result_unsignedp ? sub_optab : subv_optab,
3917 : temp, extended, target, 0, OPTAB_LIB_WIDEN);
3918 :
3919 0 : if (temp != 0)
3920 : return temp;
3921 : }
3922 :
3923 : return NULL_RTX;
3924 : }
3925 :
3926 : rtx
3927 28297 : expand_abs (machine_mode mode, rtx op0, rtx target,
3928 : int result_unsignedp, int safe)
3929 : {
3930 28297 : rtx temp;
3931 28297 : rtx_code_label *op1;
3932 :
3933 28297 : if (GET_MODE_CLASS (mode) != MODE_INT
3934 5067 : || ! flag_trapv)
3935 28209 : result_unsignedp = 1;
3936 :
3937 28297 : temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3938 28297 : if (temp != 0)
3939 : return temp;
3940 :
3941 : /* If that does not win, use conditional jump and negate. */
3942 :
3943 : /* It is safe to use the target if it is the same
3944 : as the source if this is also a pseudo register */
3945 0 : if (op0 == target && REG_P (op0)
3946 0 : && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3947 : safe = 1;
3948 :
3949 0 : op1 = gen_label_rtx ();
3950 0 : if (target == 0 || ! safe
3951 0 : || GET_MODE (target) != mode
3952 0 : || (MEM_P (target) && MEM_VOLATILE_P (target))
3953 0 : || (REG_P (target)
3954 0 : && REGNO (target) < FIRST_PSEUDO_REGISTER))
3955 0 : target = gen_reg_rtx (mode);
3956 :
3957 0 : emit_move_insn (target, op0);
3958 0 : NO_DEFER_POP;
3959 :
3960 0 : do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3961 : NULL_RTX, NULL, op1,
3962 : profile_probability::uninitialized ());
3963 :
3964 0 : op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3965 : target, target, 0);
3966 0 : if (op0 != target)
3967 0 : emit_move_insn (target, op0);
3968 0 : emit_label (op1);
3969 0 : OK_DEFER_POP;
3970 0 : return target;
3971 : }
3972 :
3973 : /* Emit code to compute the one's complement absolute value of OP0
3974 : (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3975 : (TARGET may be NULL_RTX.) The return value says where the result
3976 : actually is to be found.
3977 :
3978 : MODE is the mode of the operand; the mode of the result is
3979 : different but can be deduced from MODE. */
3980 :
3981 : rtx
3982 0 : expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3983 : {
3984 0 : rtx temp;
3985 :
3986 : /* Not applicable for floating point modes. */
3987 0 : if (FLOAT_MODE_P (mode))
3988 : return NULL_RTX;
3989 :
3990 : /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3991 0 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3992 : {
3993 0 : rtx_insn *last = get_last_insn ();
3994 :
3995 0 : temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3996 0 : if (temp != 0)
3997 0 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3998 : OPTAB_WIDEN);
3999 :
4000 0 : if (temp != 0)
4001 : return temp;
4002 :
4003 0 : delete_insns_since (last);
4004 : }
4005 :
4006 : /* If this machine has expensive jumps, we can do one's complement
4007 : absolute value of X as (((signed) x >> (W-1)) ^ x). */
4008 :
4009 0 : scalar_int_mode int_mode;
4010 0 : if (is_int_mode (mode, &int_mode)
4011 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
4012 : false) >= 2)
4013 : {
4014 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
4015 0 : GET_MODE_PRECISION (int_mode) - 1,
4016 : NULL_RTX, 0);
4017 :
4018 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
4019 : OPTAB_LIB_WIDEN);
4020 :
4021 0 : if (temp != 0)
4022 : return temp;
4023 : }
4024 :
4025 : return NULL_RTX;
4026 : }
4027 :
4028 : /* A subroutine of expand_copysign, perform the copysign operation using the
4029 : abs and neg primitives advertised to exist on the target. The assumption
4030 : is that we have a split register file, and leaving op0 in fp registers,
4031 : and not playing with subregs so much, will help the register allocator. */
4032 :
4033 : static rtx
4034 11608 : expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
4035 : int bitpos, bool op0_is_abs)
4036 : {
4037 11608 : scalar_int_mode imode;
4038 11608 : enum insn_code icode;
4039 11608 : rtx sign;
4040 11608 : rtx_code_label *label;
4041 :
4042 11608 : if (target == op1)
4043 132 : target = NULL_RTX;
4044 :
4045 : /* Check if the back end provides an insn that handles signbit for the
4046 : argument's mode. */
4047 11608 : icode = optab_handler (signbit_optab, mode);
4048 11608 : if (icode != CODE_FOR_nothing)
4049 : {
4050 11568 : imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
4051 11568 : sign = gen_reg_rtx (imode);
4052 11568 : emit_unop_insn (icode, sign, op1, UNKNOWN);
4053 : }
4054 : else
4055 : {
4056 100 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
4057 : {
4058 40 : if (!int_mode_for_mode (mode).exists (&imode))
4059 0 : return NULL_RTX;
4060 40 : op1 = gen_lowpart (imode, op1);
4061 : }
4062 : else
4063 : {
4064 0 : int word;
4065 :
4066 0 : imode = word_mode;
4067 0 : if (FLOAT_WORDS_BIG_ENDIAN)
4068 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
4069 : else
4070 0 : word = bitpos / BITS_PER_WORD;
4071 0 : bitpos = bitpos % BITS_PER_WORD;
4072 0 : op1 = operand_subword_force (op1, word, mode);
4073 : }
4074 :
4075 40 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
4076 80 : sign = expand_binop (imode, and_optab, op1,
4077 80 : immed_wide_int_const (mask, imode),
4078 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4079 40 : }
4080 :
4081 11608 : if (!op0_is_abs)
4082 : {
4083 154 : op0 = expand_unop (mode, abs_optab, op0, target, 0);
4084 154 : if (op0 == NULL)
4085 : return NULL_RTX;
4086 : target = op0;
4087 : }
4088 : else
4089 : {
4090 11454 : if (target == NULL_RTX)
4091 84 : target = copy_to_reg (op0);
4092 : else
4093 11370 : emit_move_insn (target, op0);
4094 : }
4095 :
4096 11608 : label = gen_label_rtx ();
4097 11608 : emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
4098 :
4099 11608 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
4100 11454 : op0 = simplify_unary_operation (NEG, mode, op0, mode);
4101 : else
4102 154 : op0 = expand_unop (mode, neg_optab, op0, target, 0);
4103 11608 : if (op0 != target)
4104 11454 : emit_move_insn (target, op0);
4105 :
4106 11608 : emit_label (label);
4107 :
4108 11608 : return target;
4109 : }
4110 :
4111 :
4112 : /* A subroutine of expand_copysign, perform the entire copysign operation
4113 : with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
4114 : is true if op0 is known to have its sign bit clear. */
4115 :
4116 : static rtx
4117 68 : expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
4118 : int bitpos, bool op0_is_abs)
4119 : {
4120 68 : scalar_int_mode imode;
4121 68 : int word, nwords, i;
4122 68 : rtx temp;
4123 68 : rtx_insn *insns;
4124 :
4125 156 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
4126 : {
4127 68 : if (!int_mode_for_mode (mode).exists (&imode))
4128 0 : return NULL_RTX;
4129 : word = 0;
4130 : nwords = 1;
4131 : }
4132 : else
4133 : {
4134 0 : imode = word_mode;
4135 :
4136 0 : if (FLOAT_WORDS_BIG_ENDIAN)
4137 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
4138 : else
4139 0 : word = bitpos / BITS_PER_WORD;
4140 0 : bitpos = bitpos % BITS_PER_WORD;
4141 0 : nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
4142 : }
4143 :
4144 68 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
4145 :
4146 68 : if (target == 0
4147 68 : || target == op0
4148 68 : || target == op1
4149 64 : || reg_overlap_mentioned_p (target, op0)
4150 64 : || reg_overlap_mentioned_p (target, op1)
4151 132 : || (nwords > 1 && !valid_multiword_target_p (target)))
4152 4 : target = gen_reg_rtx (mode);
4153 :
4154 68 : if (nwords > 1)
4155 : {
4156 0 : start_sequence ();
4157 :
4158 0 : for (i = 0; i < nwords; ++i)
4159 : {
4160 0 : rtx targ_piece = operand_subword (target, i, 1, mode);
4161 0 : rtx op0_piece = operand_subword_force (op0, i, mode);
4162 :
4163 0 : if (i == word)
4164 : {
4165 0 : if (!op0_is_abs)
4166 0 : op0_piece
4167 0 : = expand_binop (imode, and_optab, op0_piece,
4168 0 : immed_wide_int_const (~mask, imode),
4169 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4170 0 : op1 = expand_binop (imode, and_optab,
4171 0 : operand_subword_force (op1, i, mode),
4172 0 : immed_wide_int_const (mask, imode),
4173 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4174 :
4175 0 : temp = expand_binop (imode, ior_optab, op0_piece, op1,
4176 : targ_piece, 1, OPTAB_LIB_WIDEN);
4177 0 : if (temp != targ_piece)
4178 0 : emit_move_insn (targ_piece, temp);
4179 : }
4180 : else
4181 0 : emit_move_insn (targ_piece, op0_piece);
4182 : }
4183 :
4184 0 : insns = end_sequence ();
4185 :
4186 0 : emit_insn (insns);
4187 : }
4188 : else
4189 : {
4190 136 : op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
4191 68 : immed_wide_int_const (mask, imode),
4192 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4193 :
4194 68 : op0 = gen_lowpart (imode, op0);
4195 68 : if (!op0_is_abs)
4196 136 : op0 = expand_binop (imode, and_optab, op0,
4197 136 : immed_wide_int_const (~mask, imode),
4198 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4199 :
4200 68 : temp = expand_binop (imode, ior_optab, op0, op1,
4201 68 : gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
4202 68 : target = force_lowpart_subreg (mode, temp, imode);
4203 : }
4204 :
4205 68 : return target;
4206 68 : }
4207 :
4208 : /* Expand the C99 copysign operation. OP0 and OP1 must be the same
4209 : scalar floating point mode. Return NULL if we do not know how to
4210 : expand the operation inline. */
4211 :
4212 : rtx
4213 11676 : expand_copysign (rtx op0, rtx op1, rtx target)
4214 : {
4215 11676 : scalar_float_mode mode;
4216 11676 : const struct real_format *fmt;
4217 11676 : bool op0_is_abs;
4218 11676 : rtx temp;
4219 :
4220 11676 : mode = as_a <scalar_float_mode> (GET_MODE (op0));
4221 11676 : gcc_assert (GET_MODE (op1) == mode);
4222 :
4223 : /* First try to do it with a special instruction. */
4224 11676 : temp = expand_binop (mode, copysign_optab, op0, op1,
4225 : target, 0, OPTAB_DIRECT);
4226 11676 : if (temp)
4227 : return temp;
4228 :
4229 11676 : fmt = REAL_MODE_FORMAT (mode);
4230 11676 : if (fmt == NULL || !fmt->has_signed_zero)
4231 : return NULL_RTX;
4232 :
4233 11676 : op0_is_abs = false;
4234 11676 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
4235 : {
4236 11454 : if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
4237 0 : op0 = simplify_unary_operation (ABS, mode, op0, mode);
4238 : op0_is_abs = true;
4239 : }
4240 :
4241 11676 : if (fmt->signbit_ro >= 0
4242 11676 : && (CONST_DOUBLE_AS_FLOAT_P (op0)
4243 222 : || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
4244 154 : && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
4245 : {
4246 23216 : temp = expand_copysign_absneg (mode, op0, op1, target,
4247 11608 : fmt->signbit_ro, op0_is_abs);
4248 11608 : if (temp)
4249 : return temp;
4250 : }
4251 :
4252 68 : if (fmt->signbit_rw < 0)
4253 : return NULL_RTX;
4254 68 : return expand_copysign_bit (mode, op0, op1, target,
4255 68 : fmt->signbit_rw, op0_is_abs);
4256 : }
4257 :
4258 : /* Generate an instruction whose insn-code is INSN_CODE,
4259 : with two operands: an output TARGET and an input OP0.
4260 : TARGET *must* be nonzero, and the output is always stored there.
4261 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4262 : the value that is stored into TARGET.
4263 :
4264 : Return false if expansion failed. */
4265 :
4266 : bool
4267 2147953 : maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
4268 : enum rtx_code code)
4269 : {
4270 2147953 : class expand_operand ops[2];
4271 2147953 : rtx_insn *pat;
4272 :
4273 2147953 : create_output_operand (&ops[0], target, GET_MODE (target));
4274 2147953 : create_input_operand (&ops[1], op0, GET_MODE (op0));
4275 2147953 : pat = maybe_gen_insn (icode, 2, ops);
4276 2147953 : if (!pat)
4277 : return false;
4278 :
4279 1218796 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
4280 2171505 : && code != UNKNOWN)
4281 11832 : add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
4282 11832 : GET_MODE (op0));
4283 :
4284 2147952 : emit_insn (pat);
4285 :
4286 2147952 : if (ops[0].value != target)
4287 38723 : emit_move_insn (target, ops[0].value);
4288 : return true;
4289 : }
4290 : /* Generate an instruction whose insn-code is INSN_CODE,
4291 : with two operands: an output TARGET and an input OP0.
4292 : TARGET *must* be nonzero, and the output is always stored there.
4293 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4294 : the value that is stored into TARGET. */
4295 :
4296 : void
4297 2101270 : emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
4298 : {
4299 2101270 : bool ok = maybe_emit_unop_insn (icode, target, op0, code);
4300 2101270 : gcc_assert (ok);
4301 2101270 : }
4302 :
4303 : struct no_conflict_data
4304 : {
4305 : rtx target;
4306 : rtx_insn *first, *insn;
4307 : bool must_stay;
4308 : };
4309 :
4310 : /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4311 : the currently examined clobber / store has to stay in the list of
4312 : insns that constitute the actual libcall block. */
4313 : static void
4314 57282 : no_conflict_move_test (rtx dest, const_rtx set, void *p0)
4315 : {
4316 57282 : struct no_conflict_data *p= (struct no_conflict_data *) p0;
4317 :
4318 : /* If this inns directly contributes to setting the target, it must stay. */
4319 57282 : if (reg_overlap_mentioned_p (p->target, dest))
4320 0 : p->must_stay = true;
4321 : /* If we haven't committed to keeping any other insns in the list yet,
4322 : there is nothing more to check. */
4323 57282 : else if (p->insn == p->first)
4324 : return;
4325 : /* If this insn sets / clobbers a register that feeds one of the insns
4326 : already in the list, this insn has to stay too. */
4327 26942 : else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
4328 26942 : || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
4329 26942 : || reg_used_between_p (dest, p->first, p->insn)
4330 : /* Likewise if this insn depends on a register set by a previous
4331 : insn in the list, or if it sets a result (presumably a hard
4332 : register) that is set or clobbered by a previous insn.
4333 : N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4334 : SET_DEST perform the former check on the address, and the latter
4335 : check on the MEM. */
4336 53884 : || (GET_CODE (set) == SET
4337 26942 : && (modified_in_p (SET_SRC (set), p->first)
4338 26940 : || modified_in_p (SET_DEST (set), p->first)
4339 26940 : || modified_between_p (SET_SRC (set), p->first, p->insn)
4340 26940 : || modified_between_p (SET_DEST (set), p->first, p->insn))))
4341 2 : p->must_stay = true;
4342 : }
4343 :
4344 :
4345 : /* Emit code to make a call to a constant function or a library call.
4346 :
4347 : INSNS is a list containing all insns emitted in the call.
4348 : These insns leave the result in RESULT. Our block is to copy RESULT
4349 : to TARGET, which is logically equivalent to EQUIV.
4350 :
4351 : We first emit any insns that set a pseudo on the assumption that these are
4352 : loading constants into registers; doing so allows them to be safely cse'ed
4353 : between blocks. Then we emit all the other insns in the block, followed by
4354 : an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4355 : note with an operand of EQUIV. */
4356 :
4357 : static void
4358 122690 : emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
4359 : bool equiv_may_trap)
4360 : {
4361 122690 : rtx final_dest = target;
4362 122690 : rtx_insn *next, *last, *insn;
4363 :
4364 : /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4365 : into a MEM later. Protect the libcall block from this change. */
4366 122690 : if (! REG_P (target) || REG_USERVAR_P (target))
4367 1806 : target = gen_reg_rtx (GET_MODE (target));
4368 :
4369 : /* If we're using non-call exceptions, a libcall corresponding to an
4370 : operation that may trap may also trap. */
4371 : /* ??? See the comment in front of make_reg_eh_region_note. */
4372 122690 : if (cfun->can_throw_non_call_exceptions
4373 122690 : && (equiv_may_trap || may_trap_p (equiv)))
4374 : {
4375 1036 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4376 832 : if (CALL_P (insn))
4377 : {
4378 204 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
4379 204 : if (note)
4380 : {
4381 204 : int lp_nr = INTVAL (XEXP (note, 0));
4382 204 : if (lp_nr == 0 || lp_nr == INT_MIN)
4383 204 : remove_note (insn, note);
4384 : }
4385 : }
4386 : }
4387 : else
4388 : {
4389 : /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4390 : reg note to indicate that this call cannot throw or execute a nonlocal
4391 : goto (unless there is already a REG_EH_REGION note, in which case
4392 : we update it). */
4393 534154 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4394 411668 : if (CALL_P (insn))
4395 122486 : make_reg_eh_region_note_nothrow_nononlocal (insn);
4396 : }
4397 :
4398 : /* First emit all insns that set pseudos. Remove them from the list as
4399 : we go. Avoid insns that set pseudos which were referenced in previous
4400 : insns. These can be generated by move_by_pieces, for example,
4401 : to update an address. Similarly, avoid insns that reference things
4402 : set in previous insns. */
4403 :
4404 535190 : for (insn = insns; insn; insn = next)
4405 : {
4406 412500 : rtx set = single_set (insn);
4407 :
4408 412500 : next = NEXT_INSN (insn);
4409 :
4410 397996 : if (set != 0 && REG_P (SET_DEST (set))
4411 773188 : && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
4412 : {
4413 47191 : struct no_conflict_data data;
4414 :
4415 47191 : data.target = const0_rtx;
4416 47191 : data.first = insns;
4417 47191 : data.insn = insn;
4418 47191 : data.must_stay = 0;
4419 47191 : note_stores (insn, no_conflict_move_test, &data);
4420 47191 : if (! data.must_stay)
4421 : {
4422 47189 : if (PREV_INSN (insn))
4423 26940 : SET_NEXT_INSN (PREV_INSN (insn)) = next;
4424 : else
4425 : insns = next;
4426 :
4427 47189 : if (next)
4428 47189 : SET_PREV_INSN (next) = PREV_INSN (insn);
4429 :
4430 47189 : add_insn (insn);
4431 : }
4432 : }
4433 :
4434 : /* Some ports use a loop to copy large arguments onto the stack.
4435 : Don't move anything outside such a loop. */
4436 412500 : if (LABEL_P (insn))
4437 : break;
4438 : }
4439 :
4440 : /* Write the remaining insns followed by the final copy. */
4441 488001 : for (insn = insns; insn; insn = next)
4442 : {
4443 365311 : next = NEXT_INSN (insn);
4444 :
4445 365311 : add_insn (insn);
4446 : }
4447 :
4448 122690 : last = emit_move_insn (target, result);
4449 122690 : if (equiv)
4450 122117 : set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
4451 :
4452 122690 : if (final_dest != target)
4453 1806 : emit_move_insn (final_dest, target);
4454 122690 : }
4455 :
4456 : void
4457 91510 : emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
4458 : {
4459 91510 : emit_libcall_block_1 (insns, target, result, equiv, false);
4460 91510 : }
4461 :
4462 : /* True if we can perform a comparison of mode MODE straightforwardly.
4463 : PURPOSE describes how this comparison will be used. CODE is the rtx
4464 : comparison code we will be using.
4465 :
4466 : ??? Actually, CODE is slightly weaker than that. A target is still
4467 : required to implement all of the normal bcc operations, but not
4468 : required to implement all (or any) of the unordered bcc operations. */
4469 :
4470 : bool
4471 18269995 : can_compare_p (enum rtx_code code, machine_mode mode,
4472 : enum can_compare_purpose purpose)
4473 : {
4474 18269995 : rtx test;
4475 18269995 : test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
4476 20350707 : do
4477 : {
4478 20350707 : enum insn_code icode;
4479 :
4480 20350707 : if (purpose == ccp_jump
4481 20148255 : && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
4482 39136516 : && insn_operand_matches (icode, 0, test))
4483 : return true;
4484 3379806 : if (purpose == ccp_store_flag
4485 202452 : && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
4486 3513500 : && insn_operand_matches (icode, 1, test))
4487 : return true;
4488 :
4489 3374998 : mode = GET_MODE_WIDER_MODE (mode).else_void ();
4490 3374998 : PUT_MODE (test, mode);
4491 : }
4492 3374998 : while (mode != VOIDmode);
4493 :
4494 : return false;
4495 : }
4496 :
4497 : /* Return whether RTL code CODE corresponds to an unsigned optab. */
4498 :
4499 : static bool
4500 1468323 : unsigned_optab_p (enum rtx_code code)
4501 : {
4502 1468323 : return code == LTU || code == LEU || code == GTU || code == GEU;
4503 : }
4504 :
4505 : /* Return whether the backend-emitted comparison for code CODE, comparing
4506 : operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4507 : operand OPNO of pattern ICODE. */
4508 :
4509 : static bool
4510 1056464 : insn_predicate_matches_p (enum insn_code icode, unsigned int opno,
4511 : enum rtx_code code, machine_mode mask_mode,
4512 : machine_mode value_mode)
4513 : {
4514 1056464 : rtx reg1 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 1);
4515 1056464 : rtx reg2 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 2);
4516 1056464 : rtx test = alloca_rtx_fmt_ee (code, mask_mode, reg1, reg2);
4517 1056464 : return insn_operand_matches (icode, opno, test);
4518 : }
4519 :
4520 : /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4521 : for code CODE, comparing operands of mode VALUE_MODE and producing a result
4522 : with MASK_MODE. */
4523 :
4524 : bool
4525 1468323 : can_vec_cmp_compare_p (enum rtx_code code, machine_mode value_mode,
4526 : machine_mode mask_mode)
4527 : {
4528 1468323 : enum insn_code icode
4529 1468323 : = get_vec_cmp_icode (value_mode, mask_mode, unsigned_optab_p (code));
4530 1468323 : if (icode == CODE_FOR_nothing)
4531 : return false;
4532 :
4533 1056464 : return insn_predicate_matches_p (icode, 1, code, mask_mode, value_mode);
4534 : }
4535 :
4536 : /* Return whether the backend can emit vector set instructions for inserting
4537 : element into vector at variable index position. */
4538 :
4539 : bool
4540 207 : can_vec_set_var_idx_p (machine_mode vec_mode)
4541 : {
4542 207 : if (!VECTOR_MODE_P (vec_mode))
4543 : return false;
4544 :
4545 205 : machine_mode inner_mode = GET_MODE_INNER (vec_mode);
4546 :
4547 205 : rtx reg1 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 1);
4548 205 : rtx reg2 = alloca_raw_REG (inner_mode, LAST_VIRTUAL_REGISTER + 2);
4549 :
4550 205 : enum insn_code icode = optab_handler (vec_set_optab, vec_mode);
4551 :
4552 205 : const struct insn_data_d *data = &insn_data[icode];
4553 205 : machine_mode idx_mode = data->operand[2].mode;
4554 :
4555 205 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4556 :
4557 204 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4558 204 : && insn_operand_matches (icode, 1, reg2)
4559 409 : && insn_operand_matches (icode, 2, reg3);
4560 : }
4561 :
4562 : /* Return whether the backend can emit a vec_extract instruction with
4563 : a non-constant index. */
4564 : bool
4565 18780 : can_vec_extract_var_idx_p (machine_mode vec_mode, machine_mode extr_mode)
4566 : {
4567 18780 : if (!VECTOR_MODE_P (vec_mode))
4568 : return false;
4569 :
4570 16207 : rtx reg1 = alloca_raw_REG (extr_mode, LAST_VIRTUAL_REGISTER + 1);
4571 16207 : rtx reg2 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 2);
4572 :
4573 16207 : enum insn_code icode = convert_optab_handler (vec_extract_optab,
4574 : vec_mode, extr_mode);
4575 :
4576 16207 : const struct insn_data_d *data = &insn_data[icode];
4577 16207 : machine_mode idx_mode = data->operand[2].mode;
4578 :
4579 16207 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4580 :
4581 15345 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4582 15345 : && insn_operand_matches (icode, 1, reg2)
4583 31552 : && insn_operand_matches (icode, 2, reg3);
4584 : }
4585 :
4586 : /* This function is called when we are going to emit a compare instruction that
4587 : compares the values found in X and Y, using the rtl operator COMPARISON.
4588 :
4589 : If they have mode BLKmode, then SIZE specifies the size of both operands.
4590 :
4591 : UNSIGNEDP nonzero says that the operands are unsigned;
4592 : this matters if they need to be widened (as given by METHODS).
4593 :
4594 : *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4595 : if we failed to produce one.
4596 :
4597 : *PMODE is the mode of the inputs (in case they are const_int).
4598 :
4599 : *OPTAB is the optab to check for OPTAB_DIRECT support. Defaults to
4600 : cbranch_optab.
4601 :
4602 : This function performs all the setup necessary so that the caller only has
4603 : to emit a single comparison insn. This setup can involve doing a BLKmode
4604 : comparison or emitting a library call to perform the comparison if no insn
4605 : is available to handle it.
4606 : The values which are passed in through pointers can be modified; the caller
4607 : should perform the comparison on the modified values. Constant
4608 : comparisons must have already been folded. */
4609 :
4610 : static void
4611 6949995 : prepare_cmp_insn (rtx x, rtx y, rtx *mask, enum rtx_code comparison, rtx size,
4612 : int unsignedp, enum optab_methods methods,
4613 : rtx *ptest, machine_mode *pmode, optab optab)
4614 : {
4615 6949995 : machine_mode mode = *pmode;
4616 6949995 : rtx libfunc, test;
4617 6949995 : machine_mode cmp_mode;
4618 :
4619 : /* The other methods are not needed. */
4620 6949995 : gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4621 : || methods == OPTAB_LIB_WIDEN);
4622 :
4623 6949995 : if (CONST_SCALAR_INT_P (y))
4624 4486145 : canonicalize_comparison (mode, &comparison, &y);
4625 :
4626 : /* If we are optimizing, force expensive constants into a register. */
4627 1 : if (CONSTANT_P (x) && optimize
4628 1 : && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
4629 : > COSTS_N_INSNS (1))
4630 6949995 : && can_create_pseudo_p ())
4631 0 : x = force_reg (mode, x);
4632 :
4633 5004571 : if (CONSTANT_P (y) && optimize
4634 3960102 : && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
4635 : > COSTS_N_INSNS (1))
4636 7120004 : && can_create_pseudo_p ())
4637 170009 : y = force_reg (mode, y);
4638 :
4639 : /* Don't let both operands fail to indicate the mode. */
4640 6949995 : if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4641 0 : x = force_reg (mode, x);
4642 6949995 : if (mode == VOIDmode)
4643 4971 : mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4644 :
4645 : /* Handle all BLKmode compares. */
4646 :
4647 6949995 : if (mode == BLKmode)
4648 : {
4649 0 : machine_mode result_mode;
4650 0 : enum insn_code cmp_code;
4651 0 : rtx result;
4652 0 : rtx opalign
4653 0 : = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4654 :
4655 0 : gcc_assert (size);
4656 :
4657 : /* Try to use a memory block compare insn - either cmpstr
4658 : or cmpmem will do. */
4659 0 : opt_scalar_int_mode cmp_mode_iter;
4660 0 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
4661 : {
4662 0 : scalar_int_mode cmp_mode = cmp_mode_iter.require ();
4663 0 : cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4664 0 : if (cmp_code == CODE_FOR_nothing)
4665 0 : cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4666 0 : if (cmp_code == CODE_FOR_nothing)
4667 0 : cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4668 0 : if (cmp_code == CODE_FOR_nothing)
4669 0 : continue;
4670 :
4671 : /* Must make sure the size fits the insn's mode. */
4672 0 : if (CONST_INT_P (size)
4673 0 : ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
4674 0 : : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
4675 0 : > GET_MODE_BITSIZE (cmp_mode)))
4676 0 : continue;
4677 :
4678 0 : result_mode = insn_data[cmp_code].operand[0].mode;
4679 0 : result = gen_reg_rtx (result_mode);
4680 0 : size = convert_to_mode (cmp_mode, size, 1);
4681 0 : emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4682 :
4683 0 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4684 0 : *pmode = result_mode;
4685 0 : return;
4686 : }
4687 :
4688 0 : if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4689 0 : goto fail;
4690 :
4691 : /* Otherwise call a library function. */
4692 0 : result = emit_block_comp_via_libcall (x, y, size);
4693 :
4694 0 : x = result;
4695 0 : y = const0_rtx;
4696 0 : mode = TYPE_MODE (integer_type_node);
4697 0 : methods = OPTAB_LIB_WIDEN;
4698 0 : unsignedp = false;
4699 : }
4700 :
4701 : /* Don't allow operands to the compare to trap, as that can put the
4702 : compare and branch in different basic blocks. */
4703 6949995 : if (cfun->can_throw_non_call_exceptions)
4704 : {
4705 1133294 : if (!can_create_pseudo_p () && (may_trap_p (x) || may_trap_p (y)))
4706 0 : goto fail;
4707 1133294 : if (may_trap_p (x))
4708 24911 : x = copy_to_reg (x);
4709 1133294 : if (may_trap_p (y))
4710 1534 : y = copy_to_reg (y);
4711 : }
4712 :
4713 6949995 : if (GET_MODE_CLASS (mode) == MODE_CC)
4714 : {
4715 427 : enum insn_code icode = optab_handler (cbranch_optab, CCmode);
4716 427 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4717 427 : if (icode != CODE_FOR_nothing
4718 427 : && insn_operand_matches (icode, 0, test))
4719 : {
4720 427 : *ptest = test;
4721 427 : return;
4722 : }
4723 : else
4724 0 : goto fail;
4725 : }
4726 :
4727 6949568 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4728 7037460 : FOR_EACH_WIDER_MODE_FROM (cmp_mode, mode)
4729 : {
4730 6989686 : enum insn_code icode;
4731 6989686 : icode = optab_handler (optab, cmp_mode);
4732 6989686 : if (icode != CODE_FOR_nothing
4733 6989686 : && insn_operand_matches (icode, 0, test))
4734 : {
4735 6901794 : rtx_insn *last = get_last_insn ();
4736 6901794 : rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4737 6901794 : rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4738 6901794 : if (op0 && op1
4739 6901794 : && insn_operand_matches (icode, 1, op0)
4740 13803588 : && insn_operand_matches (icode, 2, op1))
4741 : {
4742 6901794 : XEXP (test, 0) = op0;
4743 6901794 : XEXP (test, 1) = op1;
4744 6901794 : *ptest = test;
4745 6901794 : *pmode = cmp_mode;
4746 6901794 : return;
4747 : }
4748 0 : delete_insns_since (last);
4749 : }
4750 :
4751 87892 : if (methods == OPTAB_DIRECT)
4752 : break;
4753 : }
4754 :
4755 47774 : if (methods != OPTAB_LIB_WIDEN)
4756 2919 : goto fail;
4757 :
4758 44855 : if (SCALAR_FLOAT_MODE_P (mode))
4759 : {
4760 : /* Small trick if UNORDERED isn't implemented by the hardware. */
4761 44855 : if (comparison == UNORDERED && rtx_equal_p (x, y))
4762 : {
4763 773 : prepare_cmp_insn (x, y, mask, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
4764 : ptest, pmode, optab);
4765 773 : if (*ptest)
4766 : return;
4767 : }
4768 :
4769 44855 : prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4770 : }
4771 : else
4772 : {
4773 0 : rtx result;
4774 0 : machine_mode ret_mode;
4775 :
4776 : /* Handle a libcall just for the mode we are using. */
4777 0 : libfunc = optab_libfunc (cmp_optab, mode);
4778 0 : gcc_assert (libfunc);
4779 :
4780 : /* If we want unsigned, and this mode has a distinct unsigned
4781 : comparison routine, use that. */
4782 0 : if (unsignedp)
4783 : {
4784 0 : rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4785 0 : if (ulibfunc)
4786 0 : libfunc = ulibfunc;
4787 : }
4788 :
4789 0 : ret_mode = targetm.libgcc_cmp_return_mode ();
4790 0 : result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4791 : ret_mode, x, mode, y, mode);
4792 :
4793 : /* There are two kinds of comparison routines. Biased routines
4794 : return 0/1/2, and unbiased routines return -1/0/1. Other parts
4795 : of gcc expect that the comparison operation is equivalent
4796 : to the modified comparison. For signed comparisons compare the
4797 : result against 1 in the biased case, and zero in the unbiased
4798 : case. For unsigned comparisons always compare against 1 after
4799 : biasing the unbiased result by adding 1. This gives us a way to
4800 : represent LTU.
4801 : The comparisons in the fixed-point helper library are always
4802 : biased. */
4803 0 : x = result;
4804 0 : y = const1_rtx;
4805 :
4806 0 : if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4807 : {
4808 : if (unsignedp)
4809 : x = plus_constant (ret_mode, result, 1);
4810 : else
4811 : y = const0_rtx;
4812 : }
4813 :
4814 0 : *pmode = ret_mode;
4815 0 : prepare_cmp_insn (x, y, mask, comparison, NULL_RTX, unsignedp, methods,
4816 : ptest, pmode, optab);
4817 : }
4818 :
4819 : return;
4820 :
4821 2919 : fail:
4822 2919 : *ptest = NULL_RTX;
4823 : }
4824 :
4825 : /* Before emitting an insn with code ICODE, make sure that X, which is going
4826 : to be used for operand OPNUM of the insn, is converted from mode MODE to
4827 : WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4828 : that it is accepted by the operand predicate. Return the new value. */
4829 :
4830 : rtx
4831 15322944 : prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4832 : machine_mode wider_mode, int unsignedp)
4833 : {
4834 15322944 : if (mode != wider_mode)
4835 10808 : x = convert_modes (wider_mode, mode, x, unsignedp);
4836 :
4837 15322944 : if (!insn_operand_matches (icode, opnum, x))
4838 : {
4839 64194 : machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4840 64194 : if (reload_completed)
4841 : return NULL_RTX;
4842 64194 : if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4843 : return NULL_RTX;
4844 63910 : x = copy_to_mode_reg (op_mode, x);
4845 : }
4846 :
4847 : return x;
4848 : }
4849 :
4850 : /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4851 : we can do the branch. */
4852 :
4853 : static void
4854 6444379 : emit_cmp_and_jump_insn_1 (rtx test, rtx cond, rtx len, rtx bias,
4855 : machine_mode mode, rtx label, direct_optab cmp_optab,
4856 : profile_probability prob, bool test_branch)
4857 : {
4858 6444379 : machine_mode optab_mode;
4859 6444379 : enum mode_class mclass;
4860 6444379 : enum insn_code icode;
4861 6444379 : rtx_insn *insn;
4862 :
4863 6444379 : mclass = GET_MODE_CLASS (mode);
4864 6444379 : optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4865 6444379 : icode = optab_handler (cmp_optab, optab_mode);
4866 :
4867 6444379 : gcc_assert (icode != CODE_FOR_nothing);
4868 6444379 : gcc_assert (test_branch || insn_operand_matches (icode, 0, test));
4869 6444379 : gcc_assert (cond == NULL_RTX || (cond != NULL_RTX && !test_branch));
4870 6444361 : if (test_branch)
4871 0 : insn = emit_jump_insn (GEN_FCN (icode) (XEXP (test, 0),
4872 0 : XEXP (test, 1), label));
4873 6444379 : else if (len)
4874 : {
4875 0 : gcc_assert (cond);
4876 0 : gcc_assert (bias);
4877 0 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4878 0 : XEXP (test, 1), len, bias,
4879 : label));
4880 : }
4881 6444379 : else if (cond)
4882 18 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4883 18 : XEXP (test, 1), label));
4884 : else
4885 6444361 : insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4886 6444361 : XEXP (test, 1), label));
4887 :
4888 6444379 : if (prob.initialized_p ()
4889 4894051 : && profile_status_for_fn (cfun) != PROFILE_ABSENT
4890 4787309 : && insn
4891 4787309 : && JUMP_P (insn)
4892 4786663 : && any_condjump_p (insn)
4893 11231042 : && !find_reg_note (insn, REG_BR_PROB, 0))
4894 4785201 : add_reg_br_prob_note (insn, prob);
4895 6444379 : }
4896 :
4897 : /* PTEST points to a comparison that compares its first operand with zero.
4898 : Check to see if it can be performed as a bit-test-and-branch instead.
4899 : On success, return the instruction that performs the bit-test-and-branch
4900 : and replace the second operand of *PTEST with the bit number to test.
4901 : On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
4902 :
4903 : Note that the comparison described by *PTEST should not be taken
4904 : literally after a successful return. *PTEST is just a convenient
4905 : place to store the two operands of the bit-and-test.
4906 :
4907 : VAL must contain the original tree expression for the first operand
4908 : of *PTEST. */
4909 :
4910 : static enum insn_code
4911 2636616 : validate_test_and_branch (tree val, rtx *ptest, machine_mode *pmode, optab *res)
4912 : {
4913 2636616 : if (!val || TREE_CODE (val) != SSA_NAME)
4914 : return CODE_FOR_nothing;
4915 :
4916 2534038 : machine_mode mode = TYPE_MODE (TREE_TYPE (val));
4917 2534038 : rtx test = *ptest;
4918 2534038 : direct_optab optab;
4919 :
4920 2534038 : if (GET_CODE (test) == EQ)
4921 : optab = tbranch_eq_optab;
4922 1321364 : else if (GET_CODE (test) == NE)
4923 : optab = tbranch_ne_optab;
4924 : else
4925 : return CODE_FOR_nothing;
4926 :
4927 2302937 : *res = optab;
4928 :
4929 : /* If the target supports the testbit comparison directly, great. */
4930 2302937 : auto icode = direct_optab_handler (optab, mode);
4931 2302937 : if (icode == CODE_FOR_nothing)
4932 : return icode;
4933 :
4934 0 : if (tree_zero_one_valued_p (val))
4935 : {
4936 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 : 0;
4937 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4938 0 : *ptest = test;
4939 0 : *pmode = mode;
4940 0 : return icode;
4941 : }
4942 :
4943 0 : wide_int wcst = get_nonzero_bits (val);
4944 0 : if (wcst == -1)
4945 : return CODE_FOR_nothing;
4946 :
4947 0 : int bitpos;
4948 :
4949 0 : if ((bitpos = wi::exact_log2 (wcst)) == -1)
4950 : return CODE_FOR_nothing;
4951 :
4952 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 - bitpos : bitpos;
4953 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4954 0 : *ptest = test;
4955 0 : *pmode = mode;
4956 0 : return icode;
4957 0 : }
4958 :
4959 : /* Generate code to compare X with Y so that the condition codes are
4960 : set and to jump to LABEL if the condition is true. If X is a
4961 : constant and Y is not a constant, then the comparison is swapped to
4962 : ensure that the comparison RTL has the canonical form.
4963 :
4964 : UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4965 : need to be widened. UNSIGNEDP is also used to select the proper
4966 : branch condition code.
4967 :
4968 : If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4969 :
4970 : MODE is the mode of the inputs (in case they are const_int).
4971 :
4972 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4973 : It will be potentially converted into an unsigned variant based on
4974 : UNSIGNEDP to select a proper jump instruction.
4975 :
4976 : PROB is the probability of jumping to LABEL. If the comparison is against
4977 : zero then VAL contains the expression from which the non-zero RTL is
4978 : derived. */
4979 :
4980 : void
4981 6444379 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4982 : machine_mode mode, int unsignedp, tree val, rtx label,
4983 : profile_probability prob)
4984 : {
4985 6444379 : rtx op0 = x, op1 = y;
4986 6444379 : rtx test;
4987 :
4988 : /* Swap operands and condition to ensure canonical RTL. */
4989 6444379 : if (swap_commutative_operands_p (x, y)
4990 6444379 : && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4991 : {
4992 0 : op0 = y, op1 = x;
4993 0 : comparison = swap_condition (comparison);
4994 : }
4995 :
4996 : /* If OP0 is still a constant, then both X and Y must be constants
4997 : or the opposite comparison is not supported. Force X into a register
4998 : to create canonical RTL. */
4999 6444379 : if (CONSTANT_P (op0))
5000 20180 : op0 = force_reg (mode, op0);
5001 :
5002 6444379 : if (unsignedp)
5003 3838985 : comparison = unsigned_condition (comparison);
5004 :
5005 : /* cbranch is no longer preferred for vectors, so when using a vector mode
5006 : check vec_cbranch variants instead. */
5007 6444379 : if (!VECTOR_MODE_P (GET_MODE (op0)))
5008 6424959 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5009 : OPTAB_LIB_WIDEN, &test, &mode, cbranch_optab);
5010 :
5011 : /* Check if we're comparing a truth type with 0, and if so check if
5012 : the target supports tbranch. */
5013 6444379 : machine_mode tmode = mode;
5014 6444379 : direct_optab optab;
5015 6444379 : if (op1 == CONST0_RTX (GET_MODE (op1)))
5016 : {
5017 2638259 : if (!VECTOR_MODE_P (GET_MODE (op1))
5018 2636616 : && validate_test_and_branch (val, &test, &tmode,
5019 : &optab) != CODE_FOR_nothing)
5020 : {
5021 0 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, tmode,
5022 : label, optab, prob, true);
5023 18 : return;
5024 : }
5025 :
5026 : /* If we are comparing equality with 0, check if VAL is another equality
5027 : comparison and if the target supports it directly. */
5028 2638259 : gimple *def_stmt = NULL;
5029 2536504 : if (val && TREE_CODE (val) == SSA_NAME
5030 2535641 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (val))
5031 2526 : && (comparison == NE || comparison == EQ)
5032 2640785 : && (def_stmt = get_gimple_for_ssa_name (val)))
5033 : {
5034 1802 : tree masked_op = NULL_TREE;
5035 1802 : tree len_op = NULL_TREE;
5036 1802 : tree len_bias = NULL_TREE;
5037 : /* First determine if the operation should be masked or unmasked. */
5038 1802 : if (is_gimple_assign (def_stmt)
5039 1802 : && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
5040 : {
5041 : /* See if one side if a comparison, if so use the other side as
5042 : the mask. */
5043 20 : gimple *mask_def = NULL;
5044 20 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
5045 20 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
5046 20 : if (TREE_CODE (rhs1) == SSA_NAME
5047 20 : && (mask_def = get_gimple_for_ssa_name (rhs1))
5048 20 : && is_gimple_assign (mask_def)
5049 40 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
5050 : masked_op = rhs2;
5051 0 : else if (TREE_CODE (rhs2) == SSA_NAME
5052 0 : && (mask_def = get_gimple_for_ssa_name (rhs2))
5053 0 : && is_gimple_assign (mask_def)
5054 0 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
5055 : masked_op = rhs1;
5056 :
5057 20 : if (masked_op)
5058 1802 : def_stmt = mask_def;
5059 : }
5060 : /* Else check to see if we're a LEN target. */
5061 1782 : else if (is_gimple_call (def_stmt)
5062 0 : && gimple_call_internal_p (def_stmt)
5063 1782 : && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
5064 : {
5065 : /* Example to consume:
5066 :
5067 : a = _59 != vect__4.17_75;
5068 : vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
5069 : if (vcmp != { 0, ... })
5070 :
5071 : and transform into
5072 :
5073 : if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)). */
5074 0 : gcall *call = dyn_cast <gcall *> (def_stmt);
5075 0 : tree true_branch = gimple_call_arg (call, 1);
5076 0 : tree false_branch = gimple_call_arg (call, 2);
5077 0 : if (integer_minus_onep (true_branch)
5078 0 : && integer_zerop (false_branch))
5079 : {
5080 0 : len_op = gimple_call_arg (call, 3);
5081 0 : len_bias = gimple_call_arg (call, 4);
5082 0 : tree arg0 = gimple_call_arg (call, 0);
5083 :
5084 0 : if (TREE_CODE (arg0) == SSA_NAME)
5085 0 : def_stmt = get_gimple_for_ssa_name (arg0);
5086 : }
5087 : }
5088 :
5089 1802 : enum insn_code icode;
5090 1802 : if (def_stmt
5091 1802 : && is_gimple_assign (def_stmt)
5092 3604 : && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
5093 : == tcc_comparison)
5094 : {
5095 1618 : class expand_operand ops[5];
5096 1618 : rtx_insn *tmp = NULL;
5097 1618 : start_sequence ();
5098 1618 : tree t_op0 = gimple_assign_rhs1 (def_stmt);
5099 1618 : tree t_op1 = gimple_assign_rhs2 (def_stmt);
5100 1618 : rtx op0c = expand_normal (t_op0);
5101 1618 : rtx op1c = expand_normal (t_op1);
5102 1618 : machine_mode mode2 = GET_MODE (op0c);
5103 :
5104 1618 : int nops = masked_op ? 3 : (len_op ? 5 : 2);
5105 1618 : int offset = masked_op || len_op ? 1 : 0;
5106 1618 : create_input_operand (&ops[offset + 0], op0c, mode2);
5107 1618 : create_input_operand (&ops[offset + 1], op1c, mode2);
5108 1618 : if (masked_op)
5109 : {
5110 18 : auto mask_mode = TYPE_MODE (TREE_TYPE (masked_op));
5111 18 : rtx mask_op = expand_normal (masked_op);
5112 18 : create_input_operand (&ops[0], mask_op, mask_mode);
5113 : }
5114 1600 : else if (len_op)
5115 : {
5116 0 : rtx len_rtx = expand_normal (len_op);
5117 0 : rtx len_bias_rtx = expand_normal (len_bias);
5118 0 : tree lhs = gimple_get_lhs (def_stmt);
5119 0 : auto mask_mode = TYPE_MODE (TREE_TYPE (lhs));
5120 : /* ??? We could use something like internal_fn's
5121 : add_mask_else_and_len_args here. Currently it
5122 : only supports a fixed, consecutive order of
5123 : mask and len, though. */
5124 0 : create_input_operand (&ops[0], CONSTM1_RTX (mask_mode),
5125 : mask_mode);
5126 0 : create_convert_operand_from
5127 0 : (&ops[3], len_rtx, TYPE_MODE (TREE_TYPE (len_op)),
5128 0 : TYPE_UNSIGNED (TREE_TYPE (len_op)));
5129 0 : create_input_operand (&ops[4], len_bias_rtx, QImode);
5130 : }
5131 :
5132 1618 : int unsignedp2 = TYPE_UNSIGNED (TREE_TYPE (t_op0));
5133 1618 : auto inner_code = gimple_assign_rhs_code (def_stmt);
5134 1618 : rtx test2 = NULL_RTX;
5135 :
5136 1618 : enum rtx_code comparison2 = get_rtx_code (inner_code, unsignedp2);
5137 1618 : if (unsignedp2)
5138 585 : comparison2 = unsigned_condition (comparison2);
5139 1618 : if (comparison == NE)
5140 1833 : optab = masked_op ? cond_vec_cbranch_any_optab
5141 911 : : len_op ? cond_len_vec_cbranch_any_optab
5142 : : vec_cbranch_any_optab;
5143 : else
5144 1385 : optab = masked_op ? cond_vec_cbranch_all_optab
5145 689 : : len_op ? cond_len_vec_cbranch_all_optab
5146 : : vec_cbranch_all_optab;
5147 :
5148 1618 : if ((icode = optab_handler (optab, mode2))
5149 : != CODE_FOR_nothing
5150 1618 : && maybe_legitimize_operands (icode, 1, nops, ops))
5151 : {
5152 18 : test2 = gen_rtx_fmt_ee (comparison2, VOIDmode,
5153 : ops[offset + 0].value,
5154 : ops[offset + 1].value);
5155 18 : if (insn_operand_matches (icode, 0, test2))
5156 : {
5157 36 : rtx mask
5158 18 : = (masked_op || len_op) ? ops[0].value : NULL_RTX;
5159 18 : rtx len = len_op ? ops[3].value : NULL_RTX;
5160 0 : rtx bias = len_op ? ops[4].value : NULL_RTX;
5161 18 : emit_cmp_and_jump_insn_1 (test2, mask, len, bias, mode2,
5162 : label, optab, prob, false);
5163 18 : tmp = get_insns ();
5164 : }
5165 : }
5166 :
5167 18 : end_sequence ();
5168 1618 : if (tmp)
5169 : {
5170 18 : emit_insn (tmp);
5171 18 : return;
5172 : }
5173 : }
5174 : }
5175 : }
5176 :
5177 : /* cbranch should only be used for VECTOR_BOOLEAN_TYPE_P values. */
5178 6444361 : direct_optab base_optab = cbranch_optab;
5179 6444361 : if (VECTOR_MODE_P (GET_MODE (op0)))
5180 : {
5181 : /* If cbranch is provided, use it. If we get here it means we have an
5182 : instruction in between what created the boolean value and the gcond
5183 : that is not a masking operation. This can happen for instance during
5184 : unrolling of early-break where we have an OR-reduction to reduce the
5185 : masks. In this case knowing we have a mask can let us generate better
5186 : code. If it's not there there then check the vector specific
5187 : optabs. */
5188 19420 : if (optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
5189 : {
5190 0 : if (comparison == NE)
5191 : base_optab = vec_cbranch_any_optab;
5192 : else
5193 0 : base_optab = vec_cbranch_all_optab;
5194 :
5195 0 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5196 : OPTAB_DIRECT, &test, &mode, base_optab);
5197 :
5198 0 : enum insn_code icode = optab_handler (base_optab, mode);
5199 :
5200 : /* If the new cbranch isn't supported, degrade back to old one. */
5201 0 : if (icode == CODE_FOR_nothing
5202 0 : || !test
5203 0 : || !insn_operand_matches (icode, 0, test))
5204 : base_optab = cbranch_optab;
5205 : }
5206 :
5207 19420 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5208 : OPTAB_LIB_WIDEN, &test, &mode, base_optab);
5209 : }
5210 :
5211 6444361 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, mode, label,
5212 : base_optab, prob, false);
5213 : }
5214 :
5215 : /* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown. */
5216 :
5217 : void
5218 113406 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
5219 : machine_mode mode, int unsignedp, rtx label,
5220 : profile_probability prob)
5221 : {
5222 113406 : emit_cmp_and_jump_insns (x, y, comparison, size, mode, unsignedp, NULL,
5223 : label, prob);
5224 113406 : }
5225 :
5226 :
5227 : /* Emit a library call comparison between floating point X and Y.
5228 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
5229 :
5230 : static void
5231 44855 : prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
5232 : rtx *ptest, machine_mode *pmode)
5233 : {
5234 44855 : enum rtx_code swapped = swap_condition (comparison);
5235 44855 : enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
5236 44855 : machine_mode orig_mode = GET_MODE (x);
5237 44855 : machine_mode mode;
5238 44855 : rtx true_rtx, false_rtx;
5239 44855 : rtx value, target, equiv;
5240 44855 : rtx_insn *insns;
5241 44855 : rtx libfunc = 0;
5242 44855 : bool reversed_p = false;
5243 44855 : scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
5244 :
5245 44855 : FOR_EACH_WIDER_MODE_FROM (mode, orig_mode)
5246 : {
5247 44855 : if (code_to_optab (comparison)
5248 44855 : && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
5249 : break;
5250 :
5251 0 : if (code_to_optab (swapped)
5252 0 : && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
5253 : {
5254 : std::swap (x, y);
5255 : comparison = swapped;
5256 : break;
5257 : }
5258 :
5259 0 : if (code_to_optab (reversed)
5260 0 : && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
5261 : {
5262 : comparison = reversed;
5263 : reversed_p = true;
5264 : break;
5265 : }
5266 : }
5267 :
5268 44855 : gcc_assert (mode != VOIDmode);
5269 :
5270 44855 : if (mode != orig_mode)
5271 : {
5272 0 : x = convert_to_mode (mode, x, 0);
5273 0 : y = convert_to_mode (mode, y, 0);
5274 : }
5275 :
5276 : /* Attach a REG_EQUAL note describing the semantics of the libcall to
5277 : the RTL. The allows the RTL optimizers to delete the libcall if the
5278 : condition can be determined at compile-time. */
5279 44855 : if (comparison == UNORDERED
5280 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5281 : {
5282 5769 : true_rtx = const_true_rtx;
5283 5769 : false_rtx = const0_rtx;
5284 : }
5285 : else
5286 : {
5287 39086 : switch (comparison)
5288 : {
5289 3514 : case EQ:
5290 3514 : true_rtx = const0_rtx;
5291 3514 : false_rtx = const_true_rtx;
5292 3514 : break;
5293 :
5294 23567 : case NE:
5295 23567 : true_rtx = const_true_rtx;
5296 23567 : false_rtx = const0_rtx;
5297 23567 : break;
5298 :
5299 3008 : case GT:
5300 3008 : true_rtx = const1_rtx;
5301 3008 : false_rtx = const0_rtx;
5302 3008 : break;
5303 :
5304 3002 : case GE:
5305 3002 : true_rtx = const0_rtx;
5306 3002 : false_rtx = constm1_rtx;
5307 3002 : break;
5308 :
5309 2493 : case LT:
5310 2493 : true_rtx = constm1_rtx;
5311 2493 : false_rtx = const0_rtx;
5312 2493 : break;
5313 :
5314 3502 : case LE:
5315 3502 : true_rtx = const0_rtx;
5316 3502 : false_rtx = const1_rtx;
5317 3502 : break;
5318 :
5319 0 : default:
5320 0 : gcc_unreachable ();
5321 : }
5322 : }
5323 :
5324 44855 : if (comparison == UNORDERED)
5325 : {
5326 5769 : rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
5327 5769 : equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
5328 5769 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5329 : temp, const_true_rtx, equiv);
5330 : }
5331 : else
5332 : {
5333 39086 : equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
5334 39086 : if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5335 39086 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5336 : equiv, true_rtx, false_rtx);
5337 : }
5338 :
5339 44855 : start_sequence ();
5340 44855 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5341 : cmp_mode, x, mode, y, mode);
5342 44855 : insns = end_sequence ();
5343 :
5344 44855 : target = gen_reg_rtx (cmp_mode);
5345 44855 : emit_libcall_block (insns, target, value, equiv);
5346 :
5347 44855 : if (comparison == UNORDERED
5348 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
5349 44855 : || reversed_p)
5350 11538 : *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
5351 : else
5352 39086 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
5353 :
5354 44855 : *pmode = cmp_mode;
5355 44855 : }
5356 :
5357 : /* Generate code to indirectly jump to a location given in the rtx LOC. */
5358 :
5359 : void
5360 1313 : emit_indirect_jump (rtx loc)
5361 : {
5362 1313 : if (!targetm.have_indirect_jump ())
5363 0 : sorry ("indirect jumps are not available on this target");
5364 : else
5365 : {
5366 1313 : class expand_operand ops[1];
5367 1313 : create_address_operand (&ops[0], loc);
5368 1313 : expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
5369 1313 : emit_barrier ();
5370 : }
5371 1313 : }
5372 :
5373 :
5374 : /* Emit a conditional move instruction if the machine supports one for that
5375 : condition and machine mode.
5376 :
5377 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5378 : the mode to use should they be constants. If it is VOIDmode, they cannot
5379 : both be constants.
5380 :
5381 : OP2 should be stored in TARGET if the comparison is true, otherwise OP3
5382 : should be stored there. MODE is the mode to use should they be constants.
5383 : If it is VOIDmode, they cannot both be constants.
5384 :
5385 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5386 : is not supported. */
5387 :
5388 : rtx
5389 474602 : emit_conditional_move (rtx target, struct rtx_comparison comp,
5390 : rtx op2, rtx op3,
5391 : machine_mode mode, int unsignedp)
5392 : {
5393 474602 : rtx comparison;
5394 474602 : rtx_insn *last;
5395 474602 : enum insn_code icode;
5396 474602 : enum rtx_code reversed;
5397 :
5398 : /* If the two source operands are identical, that's just a move. */
5399 :
5400 474602 : if (rtx_equal_p (op2, op3))
5401 : {
5402 2256 : if (!target)
5403 0 : target = gen_reg_rtx (mode);
5404 :
5405 2256 : emit_move_insn (target, op3);
5406 2256 : return target;
5407 : }
5408 :
5409 : /* If one operand is constant, make it the second one. Only do this
5410 : if the other operand is not constant as well. */
5411 :
5412 472346 : if (swap_commutative_operands_p (comp.op0, comp.op1))
5413 : {
5414 18039 : std::swap (comp.op0, comp.op1);
5415 18039 : comp.code = swap_condition (comp.code);
5416 : }
5417 :
5418 : /* get_condition will prefer to generate LT and GT even if the old
5419 : comparison was against zero, so undo that canonicalization here since
5420 : comparisons against zero are cheaper. */
5421 :
5422 472346 : if (comp.code == LT && comp.op1 == const1_rtx)
5423 3665 : comp.code = LE, comp.op1 = const0_rtx;
5424 468681 : else if (comp.code == GT && comp.op1 == constm1_rtx)
5425 7398 : comp.code = GE, comp.op1 = const0_rtx;
5426 :
5427 472346 : if (comp.mode == VOIDmode)
5428 386410 : comp.mode = GET_MODE (comp.op0);
5429 :
5430 472346 : enum rtx_code orig_code = comp.code;
5431 472346 : bool swapped = false;
5432 472346 : if (swap_commutative_operands_p (op2, op3)
5433 472346 : && ((reversed =
5434 185191 : reversed_comparison_code_parts (comp.code, comp.op0, comp.op1, NULL))
5435 : != UNKNOWN))
5436 : {
5437 : std::swap (op2, op3);
5438 : comp.code = reversed;
5439 : swapped = true;
5440 : }
5441 :
5442 472346 : if (mode == VOIDmode)
5443 78 : mode = GET_MODE (op2);
5444 :
5445 472346 : icode = direct_optab_handler (movcc_optab, mode);
5446 :
5447 472346 : if (icode == CODE_FOR_nothing)
5448 : return NULL_RTX;
5449 :
5450 464418 : if (!target)
5451 32 : target = gen_reg_rtx (mode);
5452 :
5453 464418 : for (int pass = 0; ; pass++)
5454 : {
5455 495909 : comp.code = unsignedp ? unsigned_condition (comp.code) : comp.code;
5456 991818 : comparison =
5457 495909 : simplify_gen_relational (comp.code, VOIDmode,
5458 : comp.mode, comp.op0, comp.op1);
5459 :
5460 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5461 : punt and let the caller figure out how best to deal with this
5462 : situation. */
5463 495909 : if (COMPARISON_P (comparison))
5464 : {
5465 494625 : saved_pending_stack_adjust save;
5466 494625 : save_pending_stack_adjust (&save);
5467 494625 : last = get_last_insn ();
5468 494625 : do_pending_stack_adjust ();
5469 494625 : machine_mode cmpmode = comp.mode;
5470 494625 : rtx orig_op0 = XEXP (comparison, 0);
5471 494625 : rtx orig_op1 = XEXP (comparison, 1);
5472 494625 : rtx op2p = op2;
5473 494625 : rtx op3p = op3;
5474 : /* If we are optimizing, force expensive constants into a register
5475 : but preserve an eventual equality with op2/op3. */
5476 1 : if (CONSTANT_P (orig_op0) && optimize
5477 1 : && cmpmode == mode
5478 1 : && (rtx_cost (orig_op0, mode, COMPARE, 0,
5479 : optimize_insn_for_speed_p ())
5480 : > COSTS_N_INSNS (1))
5481 494625 : && can_create_pseudo_p ())
5482 : {
5483 0 : if (rtx_equal_p (orig_op0, op2))
5484 0 : op2p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5485 0 : else if (rtx_equal_p (orig_op0, op3))
5486 0 : op3p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5487 : }
5488 274924 : if (CONSTANT_P (orig_op1) && optimize
5489 253866 : && cmpmode == mode
5490 89176 : && (rtx_cost (orig_op1, mode, COMPARE, 0,
5491 : optimize_insn_for_speed_p ())
5492 : > COSTS_N_INSNS (1))
5493 494802 : && can_create_pseudo_p ())
5494 : {
5495 177 : if (rtx_equal_p (orig_op1, op2))
5496 110 : op2p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5497 67 : else if (rtx_equal_p (orig_op1, op3))
5498 45 : op3p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5499 : }
5500 494625 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5501 494625 : GET_CODE (comparison), NULL_RTX, unsignedp,
5502 : OPTAB_WIDEN, &comparison, &cmpmode, cbranch_optab);
5503 494625 : if (comparison)
5504 : {
5505 492479 : rtx res = emit_conditional_move_1 (target, comparison,
5506 : op2p, op3p, mode);
5507 492479 : if (res != NULL_RTX)
5508 430871 : return res;
5509 : }
5510 63754 : delete_insns_since (last);
5511 63754 : restore_pending_stack_adjust (&save);
5512 : }
5513 :
5514 65038 : if (pass == 1)
5515 : return NULL_RTX;
5516 :
5517 : /* If the preferred op2/op3 order is not usable, retry with other
5518 : operand order, perhaps it will expand successfully. */
5519 33609 : if (swapped)
5520 : comp.code = orig_code;
5521 65442 : else if ((reversed =
5522 32721 : reversed_comparison_code_parts (orig_code, comp.op0, comp.op1,
5523 : NULL))
5524 : != UNKNOWN)
5525 : comp.code = reversed;
5526 : else
5527 : return NULL_RTX;
5528 31491 : std::swap (op2, op3);
5529 31491 : }
5530 : }
5531 :
5532 : /* Helper function that, in addition to COMPARISON, also tries
5533 : the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
5534 : to when we pass the specific constituents of a comparison, no
5535 : additional insns are emitted for it. It might still be necessary
5536 : to emit more than one insn for the final conditional move, though. */
5537 :
5538 : rtx
5539 106350 : emit_conditional_move (rtx target, rtx comparison, rtx rev_comparison,
5540 : rtx op2, rtx op3, machine_mode mode)
5541 : {
5542 106350 : rtx res = emit_conditional_move_1 (target, comparison, op2, op3, mode);
5543 :
5544 106350 : if (res != NULL_RTX)
5545 : return res;
5546 :
5547 5040 : return emit_conditional_move_1 (target, rev_comparison, op3, op2, mode);
5548 : }
5549 :
5550 : /* Helper for emitting a conditional move. */
5551 :
5552 : static rtx
5553 603869 : emit_conditional_move_1 (rtx target, rtx comparison,
5554 : rtx op2, rtx op3, machine_mode mode)
5555 : {
5556 603869 : enum insn_code icode;
5557 :
5558 603869 : if (comparison == NULL_RTX || !COMPARISON_P (comparison))
5559 : return NULL_RTX;
5560 :
5561 : /* If the two source operands are identical, that's just a move.
5562 : As the comparison comes in non-canonicalized, we must make
5563 : sure not to discard any possible side effects. If there are
5564 : side effects, just let the target handle it. */
5565 603869 : if (!side_effects_p (comparison) && rtx_equal_p (op2, op3))
5566 : {
5567 0 : if (!target)
5568 0 : target = gen_reg_rtx (mode);
5569 :
5570 0 : emit_move_insn (target, op3);
5571 0 : return target;
5572 : }
5573 :
5574 603869 : if (mode == VOIDmode)
5575 0 : mode = GET_MODE (op2);
5576 :
5577 603869 : icode = direct_optab_handler (movcc_optab, mode);
5578 :
5579 603869 : if (icode == CODE_FOR_nothing)
5580 : return NULL_RTX;
5581 :
5582 603869 : if (!target)
5583 0 : target = gen_reg_rtx (mode);
5584 :
5585 603869 : class expand_operand ops[4];
5586 :
5587 603869 : create_output_operand (&ops[0], target, mode);
5588 603869 : create_fixed_operand (&ops[1], comparison);
5589 603869 : create_input_operand (&ops[2], op2, mode);
5590 603869 : create_input_operand (&ops[3], op3, mode);
5591 :
5592 603869 : if (maybe_expand_insn (icode, 4, ops))
5593 : {
5594 532181 : if (ops[0].value != target)
5595 57 : convert_move (target, ops[0].value, false);
5596 : return target;
5597 : }
5598 :
5599 : return NULL_RTX;
5600 : }
5601 :
5602 :
5603 : /* Emit a conditional negate or bitwise complement using the
5604 : negcc or notcc optabs if available. Return NULL_RTX if such operations
5605 : are not available. Otherwise return the RTX holding the result.
5606 : TARGET is the desired destination of the result. COMP is the comparison
5607 : on which to negate. If COND is true move into TARGET the negation
5608 : or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5609 : CODE is either NEG or NOT. MODE is the machine mode in which the
5610 : operation is performed. */
5611 :
5612 : rtx
5613 1359 : emit_conditional_neg_or_complement (rtx target, rtx_code code,
5614 : machine_mode mode, rtx cond, rtx op1,
5615 : rtx op2)
5616 : {
5617 1359 : optab op = unknown_optab;
5618 1359 : if (code == NEG)
5619 : op = negcc_optab;
5620 68 : else if (code == NOT)
5621 : op = notcc_optab;
5622 : else
5623 0 : gcc_unreachable ();
5624 :
5625 1359 : insn_code icode = direct_optab_handler (op, mode);
5626 :
5627 1359 : if (icode == CODE_FOR_nothing)
5628 : return NULL_RTX;
5629 :
5630 0 : if (!target)
5631 0 : target = gen_reg_rtx (mode);
5632 :
5633 0 : rtx_insn *last = get_last_insn ();
5634 0 : class expand_operand ops[4];
5635 :
5636 0 : create_output_operand (&ops[0], target, mode);
5637 0 : create_fixed_operand (&ops[1], cond);
5638 0 : create_input_operand (&ops[2], op1, mode);
5639 0 : create_input_operand (&ops[3], op2, mode);
5640 :
5641 0 : if (maybe_expand_insn (icode, 4, ops))
5642 : {
5643 0 : if (ops[0].value != target)
5644 0 : convert_move (target, ops[0].value, false);
5645 :
5646 : return target;
5647 : }
5648 0 : delete_insns_since (last);
5649 0 : return NULL_RTX;
5650 : }
5651 :
5652 : /* Emit a conditional addition instruction if the machine supports one for that
5653 : condition and machine mode.
5654 :
5655 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5656 : the mode to use should they be constants. If it is VOIDmode, they cannot
5657 : both be constants.
5658 :
5659 : OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5660 : should be stored there. MODE is the mode to use should they be constants.
5661 : If it is VOIDmode, they cannot both be constants.
5662 :
5663 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5664 : is not supported. */
5665 :
5666 : rtx
5667 10924 : emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
5668 : machine_mode cmode, rtx op2, rtx op3,
5669 : machine_mode mode, int unsignedp)
5670 : {
5671 10924 : rtx comparison;
5672 10924 : rtx_insn *last;
5673 10924 : enum insn_code icode;
5674 :
5675 : /* If one operand is constant, make it the second one. Only do this
5676 : if the other operand is not constant as well. */
5677 :
5678 10924 : if (swap_commutative_operands_p (op0, op1))
5679 : {
5680 2 : std::swap (op0, op1);
5681 2 : code = swap_condition (code);
5682 : }
5683 :
5684 : /* get_condition will prefer to generate LT and GT even if the old
5685 : comparison was against zero, so undo that canonicalization here since
5686 : comparisons against zero are cheaper. */
5687 10924 : if (code == LT && op1 == const1_rtx)
5688 20 : code = LE, op1 = const0_rtx;
5689 10904 : else if (code == GT && op1 == constm1_rtx)
5690 54 : code = GE, op1 = const0_rtx;
5691 :
5692 10924 : if (cmode == VOIDmode)
5693 10924 : cmode = GET_MODE (op0);
5694 :
5695 10924 : if (mode == VOIDmode)
5696 0 : mode = GET_MODE (op2);
5697 :
5698 10924 : icode = optab_handler (addcc_optab, mode);
5699 :
5700 10924 : if (icode == CODE_FOR_nothing)
5701 : return 0;
5702 :
5703 10218 : if (!target)
5704 0 : target = gen_reg_rtx (mode);
5705 :
5706 10218 : code = unsignedp ? unsigned_condition (code) : code;
5707 10218 : comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
5708 :
5709 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5710 : return NULL and let the caller figure out how best to deal with this
5711 : situation. */
5712 10218 : if (!COMPARISON_P (comparison))
5713 : return NULL_RTX;
5714 :
5715 10218 : do_pending_stack_adjust ();
5716 10218 : last = get_last_insn ();
5717 10218 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5718 10218 : GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
5719 : &comparison, &cmode, cbranch_optab);
5720 10218 : if (comparison)
5721 : {
5722 10218 : class expand_operand ops[4];
5723 :
5724 10218 : create_output_operand (&ops[0], target, mode);
5725 10218 : create_fixed_operand (&ops[1], comparison);
5726 10218 : create_input_operand (&ops[2], op2, mode);
5727 10218 : create_input_operand (&ops[3], op3, mode);
5728 10218 : if (maybe_expand_insn (icode, 4, ops))
5729 : {
5730 332 : if (ops[0].value != target)
5731 0 : convert_move (target, ops[0].value, false);
5732 332 : return target;
5733 : }
5734 : }
5735 9886 : delete_insns_since (last);
5736 9886 : return NULL_RTX;
5737 : }
5738 :
5739 : /* These functions attempt to generate an insn body, rather than
5740 : emitting the insn, but if the gen function already emits them, we
5741 : make no attempt to turn them back into naked patterns. */
5742 :
5743 : /* Generate and return an insn body to add Y to X. */
5744 :
5745 : rtx_insn *
5746 253 : gen_add2_insn (rtx x, rtx y)
5747 : {
5748 253 : enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
5749 :
5750 253 : gcc_assert (insn_operand_matches (icode, 0, x));
5751 253 : gcc_assert (insn_operand_matches (icode, 1, x));
5752 253 : gcc_assert (insn_operand_matches (icode, 2, y));
5753 :
5754 253 : return GEN_FCN (icode) (x, x, y);
5755 : }
5756 :
5757 : /* Generate and return an insn body to add r1 and c,
5758 : storing the result in r0. */
5759 :
5760 : rtx_insn *
5761 0 : gen_add3_insn (rtx r0, rtx r1, rtx c)
5762 : {
5763 0 : enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
5764 :
5765 0 : if (icode == CODE_FOR_nothing
5766 0 : || !insn_operand_matches (icode, 0, r0)
5767 0 : || !insn_operand_matches (icode, 1, r1)
5768 0 : || !insn_operand_matches (icode, 2, c))
5769 : return NULL;
5770 :
5771 0 : return GEN_FCN (icode) (r0, r1, c);
5772 : }
5773 :
5774 : bool
5775 4273 : have_add2_insn (rtx x, rtx y)
5776 : {
5777 4273 : enum insn_code icode;
5778 :
5779 4273 : gcc_assert (GET_MODE (x) != VOIDmode);
5780 :
5781 4273 : icode = optab_handler (add_optab, GET_MODE (x));
5782 :
5783 4273 : if (icode == CODE_FOR_nothing)
5784 : return false;
5785 :
5786 4273 : if (!insn_operand_matches (icode, 0, x)
5787 4273 : || !insn_operand_matches (icode, 1, x)
5788 8546 : || !insn_operand_matches (icode, 2, y))
5789 203 : return false;
5790 :
5791 : return true;
5792 : }
5793 :
5794 : /* Generate and return an insn body to add Y to X. */
5795 :
5796 : rtx_insn *
5797 0 : gen_addptr3_insn (rtx x, rtx y, rtx z)
5798 : {
5799 0 : enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
5800 :
5801 0 : gcc_assert (insn_operand_matches (icode, 0, x));
5802 0 : gcc_assert (insn_operand_matches (icode, 1, y));
5803 0 : gcc_assert (insn_operand_matches (icode, 2, z));
5804 :
5805 0 : return GEN_FCN (icode) (x, y, z);
5806 : }
5807 :
5808 : /* Return true if the target implements an addptr pattern and X, Y,
5809 : and Z are valid for the pattern predicates. */
5810 :
5811 : bool
5812 689944 : have_addptr3_insn (rtx x, rtx y, rtx z)
5813 : {
5814 689944 : enum insn_code icode;
5815 :
5816 689944 : gcc_assert (GET_MODE (x) != VOIDmode);
5817 :
5818 689944 : icode = optab_handler (addptr3_optab, GET_MODE (x));
5819 :
5820 689944 : if (icode == CODE_FOR_nothing)
5821 : return false;
5822 :
5823 0 : if (!insn_operand_matches (icode, 0, x)
5824 0 : || !insn_operand_matches (icode, 1, y)
5825 0 : || !insn_operand_matches (icode, 2, z))
5826 0 : return false;
5827 :
5828 : return true;
5829 : }
5830 :
5831 : /* Generate and return an insn body to subtract Y from X. */
5832 :
5833 : rtx_insn *
5834 11 : gen_sub2_insn (rtx x, rtx y)
5835 : {
5836 11 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
5837 :
5838 11 : gcc_assert (insn_operand_matches (icode, 0, x));
5839 11 : gcc_assert (insn_operand_matches (icode, 1, x));
5840 11 : gcc_assert (insn_operand_matches (icode, 2, y));
5841 :
5842 11 : return GEN_FCN (icode) (x, x, y);
5843 : }
5844 :
5845 : /* Generate and return an insn body to subtract r1 and c,
5846 : storing the result in r0. */
5847 :
5848 : rtx_insn *
5849 1760 : gen_sub3_insn (rtx r0, rtx r1, rtx c)
5850 : {
5851 1760 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
5852 :
5853 1760 : if (icode == CODE_FOR_nothing
5854 1760 : || !insn_operand_matches (icode, 0, r0)
5855 1760 : || !insn_operand_matches (icode, 1, r1)
5856 3520 : || !insn_operand_matches (icode, 2, c))
5857 : return NULL;
5858 :
5859 1760 : return GEN_FCN (icode) (r0, r1, c);
5860 : }
5861 :
5862 : bool
5863 0 : have_sub2_insn (rtx x, rtx y)
5864 : {
5865 0 : enum insn_code icode;
5866 :
5867 0 : gcc_assert (GET_MODE (x) != VOIDmode);
5868 :
5869 0 : icode = optab_handler (sub_optab, GET_MODE (x));
5870 :
5871 0 : if (icode == CODE_FOR_nothing)
5872 : return false;
5873 :
5874 0 : if (!insn_operand_matches (icode, 0, x)
5875 0 : || !insn_operand_matches (icode, 1, x)
5876 0 : || !insn_operand_matches (icode, 2, y))
5877 0 : return false;
5878 :
5879 : return true;
5880 : }
5881 :
5882 : /* Generate the body of an insn to extend Y (with mode MFROM)
5883 : into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5884 :
5885 : rtx_insn *
5886 29358 : gen_extend_insn (rtx x, rtx y, machine_mode mto,
5887 : machine_mode mfrom, int unsignedp)
5888 : {
5889 29358 : enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
5890 29358 : return GEN_FCN (icode) (x, y);
5891 : }
5892 :
5893 : /* Generate code to convert FROM to floating point
5894 : and store in TO. FROM must be fixed point and not VOIDmode.
5895 : UNSIGNEDP nonzero means regard FROM as unsigned.
5896 : Normally this is done by correcting the final value
5897 : if it is negative. */
5898 :
5899 : void
5900 144718 : expand_float (rtx to, rtx from, int unsignedp)
5901 : {
5902 144718 : enum insn_code icode;
5903 144718 : rtx target = to;
5904 144718 : scalar_mode from_mode, to_mode;
5905 144718 : machine_mode fmode, imode;
5906 144718 : bool can_do_signed = false;
5907 :
5908 : /* Crash now, because we won't be able to decide which mode to use. */
5909 144718 : gcc_assert (GET_MODE (from) != VOIDmode);
5910 :
5911 : /* Look for an insn to do the conversion. Do it in the specified
5912 : modes if possible; otherwise convert either input, output or both to
5913 : wider mode. If the integer mode is wider than the mode of FROM,
5914 : we can do the conversion signed even if the input is unsigned. */
5915 :
5916 166659 : FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
5917 260930 : FOR_EACH_MODE_FROM (imode, GET_MODE (from))
5918 : {
5919 238989 : int doing_unsigned = unsignedp;
5920 :
5921 269247 : if (fmode != GET_MODE (to)
5922 238989 : && (significand_size (fmode)
5923 94412 : < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
5924 30258 : continue;
5925 :
5926 208731 : icode = can_float_p (fmode, imode, unsignedp);
5927 208731 : if (icode == CODE_FOR_nothing && unsignedp)
5928 : {
5929 35463 : enum insn_code scode = can_float_p (fmode, imode, 0);
5930 35463 : if (scode != CODE_FOR_nothing)
5931 6422 : can_do_signed = true;
5932 35463 : if (imode != GET_MODE (from))
5933 24395 : icode = scode, doing_unsigned = 0;
5934 : }
5935 :
5936 208731 : if (icode != CODE_FOR_nothing)
5937 : {
5938 136216 : if (imode != GET_MODE (from))
5939 10323 : from = convert_to_mode (imode, from, unsignedp);
5940 :
5941 136216 : if (fmode != GET_MODE (to))
5942 1645 : target = gen_reg_rtx (fmode);
5943 :
5944 267625 : emit_unop_insn (icode, target, from,
5945 : doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5946 :
5947 136216 : if (target != to)
5948 1645 : convert_move (to, target, 0);
5949 136216 : return;
5950 : }
5951 : }
5952 :
5953 : /* Unsigned integer, and no way to convert directly. Convert as signed,
5954 : then unconditionally adjust the result. */
5955 8502 : if (unsignedp
5956 8502 : && can_do_signed
5957 1102 : && is_a <scalar_mode> (GET_MODE (to), &to_mode)
5958 9604 : && is_a <scalar_mode> (GET_MODE (from), &from_mode))
5959 : {
5960 1102 : opt_scalar_mode fmode_iter;
5961 1102 : rtx_code_label *label = gen_label_rtx ();
5962 1102 : rtx temp;
5963 1102 : REAL_VALUE_TYPE offset;
5964 :
5965 : /* Look for a usable floating mode FMODE wider than the source and at
5966 : least as wide as the target. Using FMODE will avoid rounding woes
5967 : with unsigned values greater than the signed maximum value. */
5968 :
5969 2123 : FOR_EACH_MODE_FROM (fmode_iter, to_mode)
5970 : {
5971 2123 : scalar_mode fmode = fmode_iter.require ();
5972 3144 : if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
5973 2123 : && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
5974 : break;
5975 : }
5976 :
5977 1102 : if (!fmode_iter.exists (&fmode))
5978 : {
5979 : /* There is no such mode. Pretend the target is wide enough. */
5980 0 : fmode = to_mode;
5981 :
5982 : /* Avoid double-rounding when TO is narrower than FROM. */
5983 0 : if ((significand_size (fmode) + 1)
5984 0 : < GET_MODE_PRECISION (from_mode))
5985 : {
5986 0 : rtx temp1;
5987 0 : rtx_code_label *neglabel = gen_label_rtx ();
5988 :
5989 : /* Don't use TARGET if it isn't a register, is a hard register,
5990 : or is the wrong mode. */
5991 0 : if (!REG_P (target)
5992 0 : || REGNO (target) < FIRST_PSEUDO_REGISTER
5993 0 : || GET_MODE (target) != fmode)
5994 0 : target = gen_reg_rtx (fmode);
5995 :
5996 0 : imode = from_mode;
5997 0 : do_pending_stack_adjust ();
5998 :
5999 : /* Test whether the sign bit is set. */
6000 0 : emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
6001 : 0, neglabel);
6002 :
6003 : /* The sign bit is not set. Convert as signed. */
6004 0 : expand_float (target, from, 0);
6005 0 : emit_jump_insn (targetm.gen_jump (label));
6006 0 : emit_barrier ();
6007 :
6008 : /* The sign bit is set.
6009 : Convert to a usable (positive signed) value by shifting right
6010 : one bit, while remembering if a nonzero bit was shifted
6011 : out; i.e., compute (from & 1) | (from >> 1). */
6012 :
6013 0 : emit_label (neglabel);
6014 0 : temp = expand_binop (imode, and_optab, from, const1_rtx,
6015 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
6016 0 : temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
6017 0 : temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
6018 : OPTAB_LIB_WIDEN);
6019 0 : expand_float (target, temp, 0);
6020 :
6021 : /* Multiply by 2 to undo the shift above. */
6022 0 : temp = expand_binop (fmode, add_optab, target, target,
6023 : target, 0, OPTAB_LIB_WIDEN);
6024 0 : if (temp != target)
6025 0 : emit_move_insn (target, temp);
6026 :
6027 0 : do_pending_stack_adjust ();
6028 0 : emit_label (label);
6029 0 : goto done;
6030 : }
6031 : }
6032 :
6033 : /* If we are about to do some arithmetic to correct for an
6034 : unsigned operand, do it in a pseudo-register. */
6035 :
6036 1102 : if (to_mode != fmode
6037 1102 : || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
6038 403 : target = gen_reg_rtx (fmode);
6039 :
6040 : /* Convert as signed integer to floating. */
6041 1102 : expand_float (target, from, 0);
6042 :
6043 : /* If FROM is negative (and therefore TO is negative),
6044 : correct its value by 2**bitwidth. */
6045 :
6046 1102 : do_pending_stack_adjust ();
6047 1102 : emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
6048 : 0, label);
6049 :
6050 :
6051 1102 : real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
6052 1102 : temp = expand_binop (fmode, add_optab, target,
6053 : const_double_from_real_value (offset, fmode),
6054 : target, 0, OPTAB_LIB_WIDEN);
6055 1102 : if (temp != target)
6056 0 : emit_move_insn (target, temp);
6057 :
6058 1102 : do_pending_stack_adjust ();
6059 1102 : emit_label (label);
6060 1102 : goto done;
6061 : }
6062 :
6063 : /* No hardware instruction available; call a library routine. */
6064 7400 : {
6065 7400 : rtx libfunc;
6066 7400 : rtx_insn *insns;
6067 7400 : rtx value;
6068 7400 : convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
6069 :
6070 7400 : if (is_narrower_int_mode (GET_MODE (from), SImode))
6071 978 : from = convert_to_mode (SImode, from, unsignedp);
6072 :
6073 7400 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
6074 7400 : gcc_assert (libfunc);
6075 :
6076 7400 : start_sequence ();
6077 :
6078 14800 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
6079 7400 : GET_MODE (to), from, GET_MODE (from));
6080 7400 : insns = end_sequence ();
6081 :
6082 7400 : emit_libcall_block (insns, target, value,
6083 7400 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
6084 : GET_MODE (to), from));
6085 : }
6086 :
6087 8502 : done:
6088 :
6089 : /* Copy result to requested destination
6090 : if we have been computing in a temp location. */
6091 :
6092 8502 : if (target != to)
6093 : {
6094 403 : if (GET_MODE (target) == GET_MODE (to))
6095 4 : emit_move_insn (to, target);
6096 : else
6097 399 : convert_move (to, target, 0);
6098 : }
6099 : }
6100 :
6101 : /* Generate code to convert FROM to fixed point and store in TO. FROM
6102 : must be floating point. */
6103 :
6104 : void
6105 59536 : expand_fix (rtx to, rtx from, int unsignedp)
6106 : {
6107 59536 : enum insn_code icode;
6108 59536 : rtx target = to;
6109 59536 : machine_mode fmode, imode;
6110 59536 : opt_scalar_mode fmode_iter;
6111 59536 : bool must_trunc = false;
6112 :
6113 : /* We first try to find a pair of modes, one real and one integer, at
6114 : least as wide as FROM and TO, respectively, in which we can open-code
6115 : this conversion. If the integer mode is wider than the mode of TO,
6116 : we can do the conversion either signed or unsigned. */
6117 :
6118 90042 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
6119 215425 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
6120 : {
6121 184919 : int doing_unsigned = unsignedp;
6122 :
6123 184919 : icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
6124 184919 : if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
6125 62350 : icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
6126 :
6127 141624 : if (icode != CODE_FOR_nothing)
6128 : {
6129 46477 : rtx_insn *last = get_last_insn ();
6130 46477 : rtx from1 = from;
6131 46477 : if (fmode != GET_MODE (from))
6132 : {
6133 1114 : if (REAL_MODE_FORMAT (GET_MODE (from))
6134 : == &arm_bfloat_half_format
6135 1114 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
6136 : /* The BF -> SF conversions can be just a shift, doesn't
6137 : need to handle sNANs. */
6138 : {
6139 46 : int save_flag_finite_math_only = flag_finite_math_only;
6140 46 : flag_finite_math_only = true;
6141 46 : from1 = convert_to_mode (fmode, from, 0);
6142 46 : flag_finite_math_only = save_flag_finite_math_only;
6143 : }
6144 : else
6145 1068 : from1 = convert_to_mode (fmode, from, 0);
6146 : }
6147 :
6148 46477 : if (must_trunc)
6149 : {
6150 0 : rtx temp = gen_reg_rtx (GET_MODE (from1));
6151 0 : from1 = expand_unop (GET_MODE (from1), ftrunc_optab, from1,
6152 : temp, 0);
6153 : }
6154 :
6155 46477 : if (imode != GET_MODE (to))
6156 10787 : target = gen_reg_rtx (imode);
6157 :
6158 88694 : if (maybe_emit_unop_insn (icode, target, from1,
6159 : doing_unsigned ? UNSIGNED_FIX : FIX))
6160 : {
6161 46477 : if (target != to)
6162 10787 : convert_move (to, target, unsignedp);
6163 50532 : return;
6164 : }
6165 0 : delete_insns_since (last);
6166 : }
6167 : }
6168 :
6169 : /* For an unsigned conversion, there is one more way to do it.
6170 : If we have a signed conversion, we generate code that compares
6171 : the real value to the largest representable positive number. If if
6172 : is smaller, the conversion is done normally. Otherwise, subtract
6173 : one plus the highest signed number, convert, and add it back.
6174 :
6175 : We only need to check all real modes, since we know we didn't find
6176 : anything with a wider integer mode.
6177 :
6178 : This code used to extend FP value into mode wider than the destination.
6179 : This is needed for decimal float modes which cannot accurately
6180 : represent one plus the highest signed number of the same size, but
6181 : not for binary modes. Consider, for instance conversion from SFmode
6182 : into DImode.
6183 :
6184 : The hot path through the code is dealing with inputs smaller than 2^63
6185 : and doing just the conversion, so there is no bits to lose.
6186 :
6187 : In the other path we know the value is positive in the range 2^63..2^64-1
6188 : inclusive. (as for other input overflow happens and result is undefined)
6189 : So we know that the most important bit set in mantissa corresponds to
6190 : 2^63. The subtraction of 2^63 should not generate any rounding as it
6191 : simply clears out that bit. The rest is trivial. */
6192 :
6193 13059 : scalar_int_mode to_mode;
6194 13059 : if (unsignedp
6195 7431 : && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
6196 20490 : && HWI_COMPUTABLE_MODE_P (to_mode))
6197 9057 : FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
6198 : {
6199 6852 : scalar_mode fmode = fmode_iter.require ();
6200 6852 : if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
6201 : 0, &must_trunc)
6202 6852 : && (!DECIMAL_FLOAT_MODE_P (fmode)
6203 0 : || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
6204 : {
6205 4049 : int bitsize;
6206 4049 : REAL_VALUE_TYPE offset;
6207 4049 : rtx limit;
6208 4049 : rtx_code_label *lab1, *lab2;
6209 4049 : rtx_insn *insn;
6210 :
6211 4049 : bitsize = GET_MODE_PRECISION (to_mode);
6212 4049 : real_2expN (&offset, bitsize - 1, fmode);
6213 4049 : limit = const_double_from_real_value (offset, fmode);
6214 4049 : lab1 = gen_label_rtx ();
6215 4049 : lab2 = gen_label_rtx ();
6216 :
6217 4049 : if (fmode != GET_MODE (from))
6218 : {
6219 188 : if (REAL_MODE_FORMAT (GET_MODE (from))
6220 : == &arm_bfloat_half_format
6221 188 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
6222 : /* The BF -> SF conversions can be just a shift, doesn't
6223 : need to handle sNANs. */
6224 : {
6225 6 : int save_flag_finite_math_only = flag_finite_math_only;
6226 6 : flag_finite_math_only = true;
6227 6 : from = convert_to_mode (fmode, from, 0);
6228 6 : flag_finite_math_only = save_flag_finite_math_only;
6229 : }
6230 : else
6231 182 : from = convert_to_mode (fmode, from, 0);
6232 : }
6233 :
6234 : /* See if we need to do the subtraction. */
6235 4049 : do_pending_stack_adjust ();
6236 4049 : emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
6237 4049 : GET_MODE (from), 0, lab1);
6238 :
6239 : /* If not, do the signed "fix" and branch around fixup code. */
6240 4049 : expand_fix (to, from, 0);
6241 4049 : emit_jump_insn (targetm.gen_jump (lab2));
6242 4049 : emit_barrier ();
6243 :
6244 : /* Otherwise, subtract 2**(N-1), convert to signed number,
6245 : then add 2**(N-1). Do the addition using XOR since this
6246 : will often generate better code. */
6247 4049 : emit_label (lab1);
6248 4049 : target = expand_binop (GET_MODE (from), sub_optab, from, limit,
6249 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
6250 4049 : expand_fix (to, target, 0);
6251 4049 : target = expand_binop (to_mode, xor_optab, to,
6252 : gen_int_mode
6253 4049 : (HOST_WIDE_INT_1 << (bitsize - 1),
6254 : to_mode),
6255 : to, 1, OPTAB_LIB_WIDEN);
6256 :
6257 4049 : if (target != to)
6258 0 : emit_move_insn (to, target);
6259 :
6260 4049 : emit_label (lab2);
6261 :
6262 4049 : if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
6263 : {
6264 : /* Make a place for a REG_NOTE and add it. */
6265 4049 : insn = emit_move_insn (to, to);
6266 4049 : set_dst_reg_note (insn, REG_EQUAL,
6267 : gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
6268 : copy_rtx (from)),
6269 : to);
6270 : }
6271 :
6272 4049 : return;
6273 : }
6274 : }
6275 :
6276 : #ifdef HAVE_SFmode
6277 9010 : if (REAL_MODE_FORMAT (GET_MODE (from)) == &arm_bfloat_half_format
6278 9010 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
6279 : /* We don't have BF -> TI library functions, use BF -> SF -> TI
6280 : instead but the BF -> SF conversion can be just a shift, doesn't
6281 : need to handle sNANs. */
6282 : {
6283 6 : int save_flag_finite_math_only = flag_finite_math_only;
6284 6 : flag_finite_math_only = true;
6285 6 : from = convert_to_mode (SFmode, from, 0);
6286 6 : flag_finite_math_only = save_flag_finite_math_only;
6287 6 : expand_fix (to, from, unsignedp);
6288 6 : return;
6289 : }
6290 : #endif
6291 :
6292 : /* We can't do it with an insn, so use a library call. But first ensure
6293 : that the mode of TO is at least as wide as SImode, since those are the
6294 : only library calls we know about. */
6295 :
6296 9004 : if (is_narrower_int_mode (GET_MODE (to), SImode))
6297 : {
6298 1958 : target = gen_reg_rtx (SImode);
6299 :
6300 1958 : expand_fix (target, from, unsignedp);
6301 : }
6302 : else
6303 : {
6304 7046 : rtx_insn *insns;
6305 7046 : rtx value;
6306 7046 : rtx libfunc;
6307 :
6308 7046 : convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
6309 7046 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
6310 7046 : gcc_assert (libfunc);
6311 :
6312 7046 : start_sequence ();
6313 :
6314 14092 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
6315 7046 : GET_MODE (to), from, GET_MODE (from));
6316 7046 : insns = end_sequence ();
6317 :
6318 7046 : emit_libcall_block (insns, target, value,
6319 7046 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
6320 : GET_MODE (to), from));
6321 : }
6322 :
6323 9004 : if (target != to)
6324 : {
6325 1958 : if (GET_MODE (to) == GET_MODE (target))
6326 0 : emit_move_insn (to, target);
6327 : else
6328 1958 : convert_move (to, target, 0);
6329 : }
6330 : }
6331 :
6332 :
6333 : /* Promote integer arguments for a libcall if necessary.
6334 : emit_library_call_value cannot do the promotion because it does not
6335 : know if it should do a signed or unsigned promotion. This is because
6336 : there are no tree types defined for libcalls. */
6337 :
6338 : static rtx
6339 0 : prepare_libcall_arg (rtx arg, int uintp)
6340 : {
6341 0 : scalar_int_mode mode;
6342 0 : machine_mode arg_mode;
6343 0 : if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
6344 : {
6345 : /* If we need to promote the integer function argument we need to do
6346 : it here instead of inside emit_library_call_value because in
6347 : emit_library_call_value we don't know if we should do a signed or
6348 : unsigned promotion. */
6349 :
6350 0 : int unsigned_p = 0;
6351 0 : arg_mode = promote_function_mode (NULL_TREE, mode,
6352 : &unsigned_p, NULL_TREE, 0);
6353 0 : if (arg_mode != mode)
6354 0 : return convert_to_mode (arg_mode, arg, uintp);
6355 : }
6356 : return arg;
6357 : }
6358 :
6359 : /* Generate code to convert FROM or TO a fixed-point.
6360 : If UINTP is true, either TO or FROM is an unsigned integer.
6361 : If SATP is true, we need to saturate the result. */
6362 :
6363 : void
6364 0 : expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
6365 : {
6366 0 : machine_mode to_mode = GET_MODE (to);
6367 0 : machine_mode from_mode = GET_MODE (from);
6368 0 : convert_optab tab;
6369 0 : enum rtx_code this_code;
6370 0 : enum insn_code code;
6371 0 : rtx_insn *insns;
6372 0 : rtx value;
6373 0 : rtx libfunc;
6374 :
6375 0 : if (to_mode == from_mode)
6376 : {
6377 0 : emit_move_insn (to, from);
6378 0 : return;
6379 : }
6380 :
6381 0 : if (uintp)
6382 : {
6383 0 : tab = satp ? satfractuns_optab : fractuns_optab;
6384 0 : this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
6385 : }
6386 : else
6387 : {
6388 0 : tab = satp ? satfract_optab : fract_optab;
6389 0 : this_code = satp ? SAT_FRACT : FRACT_CONVERT;
6390 : }
6391 0 : code = convert_optab_handler (tab, to_mode, from_mode);
6392 0 : if (code != CODE_FOR_nothing)
6393 : {
6394 0 : emit_unop_insn (code, to, from, this_code);
6395 0 : return;
6396 : }
6397 :
6398 0 : libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
6399 0 : gcc_assert (libfunc);
6400 :
6401 0 : from = prepare_libcall_arg (from, uintp);
6402 0 : from_mode = GET_MODE (from);
6403 :
6404 0 : start_sequence ();
6405 0 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
6406 : from, from_mode);
6407 0 : insns = end_sequence ();
6408 :
6409 0 : emit_libcall_block (insns, to, value,
6410 : gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
6411 : }
6412 :
6413 : /* Generate code to convert FROM to fixed point and store in TO. FROM
6414 : must be floating point, TO must be signed. Use the conversion optab
6415 : TAB to do the conversion. */
6416 :
6417 : bool
6418 480 : expand_sfix_optab (rtx to, rtx from, convert_optab tab)
6419 : {
6420 480 : enum insn_code icode;
6421 480 : rtx target = to;
6422 480 : machine_mode fmode, imode;
6423 :
6424 : /* We first try to find a pair of modes, one real and one integer, at
6425 : least as wide as FROM and TO, respectively, in which we can open-code
6426 : this conversion. If the integer mode is wider than the mode of TO,
6427 : we can do the conversion either signed or unsigned. */
6428 :
6429 2008 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
6430 8618 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
6431 : {
6432 7090 : icode = convert_optab_handler (tab, imode, fmode,
6433 : insn_optimization_type ());
6434 7090 : if (icode != CODE_FOR_nothing)
6435 : {
6436 52 : rtx_insn *last = get_last_insn ();
6437 52 : if (fmode != GET_MODE (from))
6438 52 : from = convert_to_mode (fmode, from, 0);
6439 :
6440 52 : if (imode != GET_MODE (to))
6441 0 : target = gen_reg_rtx (imode);
6442 :
6443 52 : if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
6444 : {
6445 0 : delete_insns_since (last);
6446 0 : continue;
6447 : }
6448 52 : if (target != to)
6449 0 : convert_move (to, target, 0);
6450 : return true;
6451 : }
6452 : }
6453 :
6454 : return false;
6455 : }
6456 :
6457 : /* Report whether we have an instruction to perform the operation
6458 : specified by CODE on operands of mode MODE. */
6459 : bool
6460 96699487 : have_insn_for (enum rtx_code code, machine_mode mode)
6461 : {
6462 96699487 : return (code_to_optab (code)
6463 96699487 : && (optab_handler (code_to_optab (code), mode)
6464 96699487 : != CODE_FOR_nothing));
6465 : }
6466 :
6467 : /* Print information about the current contents of the optabs on
6468 : STDERR. */
6469 :
6470 : DEBUG_FUNCTION void
6471 0 : debug_optab_libfuncs (void)
6472 : {
6473 0 : int i, j, k;
6474 :
6475 : /* Dump the arithmetic optabs. */
6476 0 : for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6477 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6478 : {
6479 0 : rtx l = optab_libfunc ((optab) i, (machine_mode) j);
6480 0 : if (l)
6481 : {
6482 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6483 0 : fprintf (stderr, "%s\t%s:\t%s\n",
6484 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6485 0 : GET_MODE_NAME (j),
6486 : XSTR (l, 0));
6487 : }
6488 : }
6489 :
6490 : /* Dump the conversion optabs. */
6491 0 : for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6492 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6493 0 : for (k = 0; k < NUM_MACHINE_MODES; ++k)
6494 : {
6495 0 : rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
6496 : (machine_mode) k);
6497 0 : if (l)
6498 : {
6499 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6500 0 : fprintf (stderr, "%s\t%s\t%s:\t%s\n",
6501 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6502 0 : GET_MODE_NAME (j),
6503 0 : GET_MODE_NAME (k),
6504 : XSTR (l, 0));
6505 : }
6506 : }
6507 0 : }
6508 :
6509 : /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6510 : CODE. Return 0 on failure. */
6511 :
6512 : rtx_insn *
6513 0 : gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6514 : {
6515 0 : machine_mode mode = GET_MODE (op1);
6516 0 : enum insn_code icode;
6517 0 : rtx_insn *insn;
6518 0 : rtx trap_rtx;
6519 :
6520 0 : if (mode == VOIDmode)
6521 : return 0;
6522 :
6523 0 : icode = optab_handler (ctrap_optab, mode);
6524 0 : if (icode == CODE_FOR_nothing)
6525 : return 0;
6526 :
6527 : /* Some targets only accept a zero trap code. */
6528 0 : if (!insn_operand_matches (icode, 3, tcode))
6529 : return 0;
6530 :
6531 0 : do_pending_stack_adjust ();
6532 0 : start_sequence ();
6533 0 : prepare_cmp_insn (op1, op2, NULL, code, NULL_RTX, false, OPTAB_DIRECT,
6534 : &trap_rtx, &mode, cbranch_optab);
6535 0 : if (!trap_rtx)
6536 : insn = NULL;
6537 : else
6538 0 : insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6539 : tcode);
6540 :
6541 : /* If that failed, then give up. */
6542 0 : if (insn == 0)
6543 : {
6544 0 : end_sequence ();
6545 0 : return 0;
6546 : }
6547 :
6548 0 : emit_insn (insn);
6549 0 : insn = end_sequence ();
6550 0 : return insn;
6551 : }
6552 :
6553 : /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
6554 : or unsigned operation code. */
6555 :
6556 : enum rtx_code
6557 1627354 : get_rtx_code_1 (enum tree_code tcode, bool unsignedp)
6558 : {
6559 1627354 : enum rtx_code code;
6560 1627354 : switch (tcode)
6561 : {
6562 : case EQ_EXPR:
6563 : code = EQ;
6564 : break;
6565 806361 : case NE_EXPR:
6566 806361 : code = NE;
6567 806361 : break;
6568 163974 : case LT_EXPR:
6569 163974 : code = unsignedp ? LTU : LT;
6570 : break;
6571 74368 : case LE_EXPR:
6572 74368 : code = unsignedp ? LEU : LE;
6573 : break;
6574 117861 : case GT_EXPR:
6575 117861 : code = unsignedp ? GTU : GT;
6576 : break;
6577 79945 : case GE_EXPR:
6578 79945 : code = unsignedp ? GEU : GE;
6579 : break;
6580 :
6581 1908 : case UNORDERED_EXPR:
6582 1908 : code = UNORDERED;
6583 1908 : break;
6584 1059 : case ORDERED_EXPR:
6585 1059 : code = ORDERED;
6586 1059 : break;
6587 691 : case UNLT_EXPR:
6588 691 : code = UNLT;
6589 691 : break;
6590 4935 : case UNLE_EXPR:
6591 4935 : code = UNLE;
6592 4935 : break;
6593 717 : case UNGT_EXPR:
6594 717 : code = UNGT;
6595 717 : break;
6596 3739 : case UNGE_EXPR:
6597 3739 : code = UNGE;
6598 3739 : break;
6599 279 : case UNEQ_EXPR:
6600 279 : code = UNEQ;
6601 279 : break;
6602 503 : case LTGT_EXPR:
6603 503 : code = LTGT;
6604 503 : break;
6605 :
6606 32792 : case BIT_AND_EXPR:
6607 32792 : code = AND;
6608 32792 : break;
6609 :
6610 22 : case BIT_IOR_EXPR:
6611 22 : code = IOR;
6612 22 : break;
6613 :
6614 0 : default:
6615 0 : code = UNKNOWN;
6616 0 : break;
6617 : }
6618 1627354 : return code;
6619 : }
6620 :
6621 : /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6622 : or unsigned operation code. */
6623 :
6624 : enum rtx_code
6625 159031 : get_rtx_code (enum tree_code tcode, bool unsignedp)
6626 : {
6627 159031 : enum rtx_code code = get_rtx_code_1 (tcode, unsignedp);
6628 159031 : gcc_assert (code != UNKNOWN);
6629 159031 : return code;
6630 : }
6631 :
6632 : /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6633 : select signed or unsigned operators. OPNO holds the index of the
6634 : first comparison operand for insn ICODE. Do not generate the
6635 : compare instruction itself. */
6636 :
6637 : rtx
6638 26203 : vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
6639 : tree t_op0, tree t_op1, bool unsignedp,
6640 : enum insn_code icode, unsigned int opno)
6641 : {
6642 26203 : class expand_operand ops[2];
6643 26203 : rtx rtx_op0, rtx_op1;
6644 26203 : machine_mode m0, m1;
6645 26203 : enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6646 :
6647 26203 : gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6648 :
6649 : /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6650 : has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6651 : cases, use the original mode. */
6652 26203 : rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6653 : EXPAND_STACK_PARM);
6654 26203 : m0 = GET_MODE (rtx_op0);
6655 26203 : if (m0 == VOIDmode)
6656 0 : m0 = TYPE_MODE (TREE_TYPE (t_op0));
6657 :
6658 26203 : rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6659 : EXPAND_STACK_PARM);
6660 26203 : m1 = GET_MODE (rtx_op1);
6661 26203 : if (m1 == VOIDmode)
6662 0 : m1 = TYPE_MODE (TREE_TYPE (t_op1));
6663 :
6664 26203 : create_input_operand (&ops[0], rtx_op0, m0);
6665 26203 : create_input_operand (&ops[1], rtx_op1, m1);
6666 26203 : if (!maybe_legitimize_operands (icode, opno, 2, ops))
6667 0 : gcc_unreachable ();
6668 26203 : return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
6669 : }
6670 :
6671 : /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6672 : the first vec_perm operand, assuming the second operand (for left shift
6673 : first operand) is a constant vector of zeros. Return the shift distance
6674 : in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6675 : mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6676 : shift or vec_shl_optab for left shift. */
6677 : static rtx
6678 3194 : shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
6679 : optab shift_optab)
6680 : {
6681 3194 : unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
6682 3194 : poly_int64 first = sel[0];
6683 6388 : if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
6684 : return NULL_RTX;
6685 :
6686 3193 : if (shift_optab == vec_shl_optab)
6687 : {
6688 888 : unsigned int nelt;
6689 1776 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6690 3194 : return NULL_RTX;
6691 888 : unsigned firstidx = 0;
6692 5811 : for (unsigned int i = 0; i < nelt; i++)
6693 : {
6694 5044 : if (known_eq (sel[i], nelt))
6695 : {
6696 803 : if (i == 0 || firstidx)
6697 : return NULL_RTX;
6698 : firstidx = i;
6699 : }
6700 8482 : else if (firstidx
6701 5898 : ? maybe_ne (sel[i], nelt + i - firstidx)
6702 4241 : : maybe_ge (sel[i], nelt))
6703 : return NULL_RTX;
6704 : }
6705 :
6706 767 : if (firstidx == 0)
6707 : return NULL_RTX;
6708 767 : first = firstidx;
6709 : }
6710 2305 : else if (!sel.series_p (0, 1, first, 1))
6711 : {
6712 387 : unsigned int nelt;
6713 774 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6714 3194 : return NULL_RTX;
6715 2214 : for (unsigned int i = 1; i < nelt; i++)
6716 : {
6717 2012 : poly_int64 expected = i + first;
6718 : /* Indices into the second vector are all equivalent. */
6719 2012 : if (maybe_lt (sel[i], nelt)
6720 2824 : ? maybe_ne (sel[i], expected)
6721 812 : : maybe_lt (expected, nelt))
6722 3194 : return NULL_RTX;
6723 : }
6724 : }
6725 :
6726 2887 : return gen_int_shift_amount (mode, first * bitsize);
6727 : }
6728 :
6729 : /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6730 :
6731 : static rtx
6732 18 : expand_vec_perm_1 (enum insn_code icode, rtx target,
6733 : rtx v0, rtx v1, rtx sel)
6734 : {
6735 18 : machine_mode tmode = GET_MODE (target);
6736 18 : machine_mode smode = GET_MODE (sel);
6737 18 : class expand_operand ops[4];
6738 :
6739 18 : gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
6740 : || related_int_vector_mode (tmode).require () == smode);
6741 18 : create_output_operand (&ops[0], target, tmode);
6742 18 : create_input_operand (&ops[3], sel, smode);
6743 :
6744 : /* Make an effort to preserve v0 == v1. The target expander is able to
6745 : rely on this to determine if we're permuting a single input operand. */
6746 18 : if (rtx_equal_p (v0, v1))
6747 : {
6748 14 : if (!insn_operand_matches (icode, 1, v0))
6749 0 : v0 = force_reg (tmode, v0);
6750 14 : gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6751 14 : gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6752 :
6753 14 : create_fixed_operand (&ops[1], v0);
6754 14 : create_fixed_operand (&ops[2], v0);
6755 : }
6756 : else
6757 : {
6758 4 : create_input_operand (&ops[1], v0, tmode);
6759 4 : create_input_operand (&ops[2], v1, tmode);
6760 : }
6761 :
6762 18 : if (maybe_expand_insn (icode, 4, ops))
6763 18 : return ops[0].value;
6764 : return NULL_RTX;
6765 : }
6766 :
6767 : /* Check if vec_perm mask SEL is a constant equivalent to an and operation of
6768 : the non-zero vec_perm operand with some mask consisting of 0xffs and 0x00s,
6769 : assuming the other vec_perm operand is a constant vector of zeros. Return
6770 : the mask for the equivalent and operation, or NULL_RTX if the vec_perm can
6771 : not be modeled as an and. MODE is the mode of the value being anded.
6772 : ZERO_OP0_P is true if the first operand of the vec_perm is a constant vector
6773 : of zeros or false if the second operand of the vec_perm is a constant vector
6774 : of zeros. */
6775 : rtx
6776 0 : vec_perm_and_mask (machine_mode mode, const vec_perm_indices &sel,
6777 : bool zero_op0_p)
6778 : {
6779 0 : unsigned int nelt;
6780 0 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6781 : return NULL_RTX;
6782 :
6783 0 : rtx_vector_builder builder (mode, nelt, 1);
6784 0 : machine_mode emode = GET_MODE_INNER (mode);
6785 :
6786 0 : for (unsigned int i = 0; i < nelt; i++)
6787 : {
6788 0 : if (zero_op0_p)
6789 : {
6790 0 : if (known_eq (sel[i], nelt + i))
6791 0 : builder.quick_push (CONSTM1_RTX (emode));
6792 0 : else if (known_lt (sel[i], nelt))
6793 0 : builder.quick_push (CONST0_RTX (emode));
6794 : else
6795 0 : return NULL_RTX;
6796 : }
6797 : else
6798 : {
6799 0 : if (known_eq (sel[i], i))
6800 0 : builder.quick_push (CONSTM1_RTX (emode));
6801 0 : else if (known_ge (sel[i], nelt))
6802 0 : builder.quick_push (CONST0_RTX (emode));
6803 : else
6804 : return NULL_RTX;
6805 : }
6806 : }
6807 :
6808 0 : return builder.build ();
6809 0 : }
6810 :
6811 : /* Implement a permutation of vectors v0 and v1 using the permutation
6812 : vector in SEL and return the result. Use TARGET to hold the result
6813 : if nonnull and convenient.
6814 :
6815 : MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6816 : is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6817 : to have a particular mode. */
6818 :
6819 : rtx
6820 77491 : expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
6821 : const vec_perm_builder &sel, machine_mode sel_mode,
6822 : rtx target)
6823 : {
6824 77491 : if (!target || !register_operand (target, mode))
6825 62519 : target = gen_reg_rtx (mode);
6826 :
6827 : /* Set QIMODE to a different vector mode with byte elements.
6828 : If no such mode, or if MODE already has byte elements, use VOIDmode. */
6829 77491 : machine_mode qimode;
6830 77491 : if (!qimode_for_vec_perm (mode).exists (&qimode))
6831 77491 : qimode = VOIDmode;
6832 :
6833 77491 : rtx_insn *last = get_last_insn ();
6834 :
6835 77491 : bool single_arg_p = rtx_equal_p (v0, v1);
6836 : /* Always specify two input vectors here and leave the target to handle
6837 : cases in which the inputs are equal. Not all backends can cope with
6838 : the single-input representation when testing for a double-input
6839 : target instruction. */
6840 154982 : vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
6841 :
6842 : /* See if this can be handled with a vec_shr or vec_shl. We only do this
6843 : if the second (for vec_shr) or first (for vec_shl) vector is all
6844 : zeroes. */
6845 77491 : insn_code shift_code = CODE_FOR_nothing;
6846 77491 : insn_code shift_code_qi = CODE_FOR_nothing;
6847 77491 : optab shift_optab = unknown_optab;
6848 77491 : rtx v2 = v0;
6849 77491 : if (v1 == CONST0_RTX (GET_MODE (v1)))
6850 : shift_optab = vec_shr_optab;
6851 75121 : else if (v0 == CONST0_RTX (GET_MODE (v0)))
6852 : {
6853 : shift_optab = vec_shl_optab;
6854 : v2 = v1;
6855 : }
6856 : if (shift_optab != unknown_optab)
6857 : {
6858 3582 : shift_code = optab_handler (shift_optab, mode);
6859 6360 : shift_code_qi = ((qimode != VOIDmode && qimode != mode)
6860 6762 : ? optab_handler (shift_optab, qimode)
6861 : : CODE_FOR_nothing);
6862 : }
6863 3582 : if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
6864 : {
6865 3194 : rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices, shift_optab);
6866 3194 : if (shift_amt)
6867 : {
6868 2887 : class expand_operand ops[3];
6869 2887 : if (shift_amt == const0_rtx)
6870 2887 : return v2;
6871 2887 : if (shift_code != CODE_FOR_nothing)
6872 : {
6873 2887 : create_output_operand (&ops[0], target, mode);
6874 2887 : create_input_operand (&ops[1], v2, mode);
6875 2887 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6876 2887 : if (maybe_expand_insn (shift_code, 3, ops))
6877 2887 : return ops[0].value;
6878 : }
6879 0 : if (shift_code_qi != CODE_FOR_nothing)
6880 : {
6881 0 : rtx tmp = gen_reg_rtx (qimode);
6882 0 : create_output_operand (&ops[0], tmp, qimode);
6883 0 : create_input_operand (&ops[1], gen_lowpart (qimode, v2), qimode);
6884 0 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6885 0 : if (maybe_expand_insn (shift_code_qi, 3, ops))
6886 0 : return gen_lowpart (mode, ops[0].value);
6887 : }
6888 : }
6889 : }
6890 :
6891 74604 : if (targetm.vectorize.vec_perm_const != NULL)
6892 : {
6893 74604 : if (single_arg_p)
6894 18871 : v1 = v0;
6895 :
6896 74604 : gcc_checking_assert (GET_MODE (v0) == GET_MODE (v1));
6897 74604 : machine_mode op_mode = GET_MODE (v0);
6898 74604 : if (targetm.vectorize.vec_perm_const (mode, op_mode, target, v0, v1,
6899 : indices))
6900 : return target;
6901 : }
6902 :
6903 : /* Fall back to a constant byte-based permutation. */
6904 0 : vec_perm_indices qimode_indices;
6905 0 : rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
6906 0 : if (qimode != VOIDmode)
6907 : {
6908 0 : qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
6909 0 : target_qi = gen_reg_rtx (qimode);
6910 0 : v0_qi = gen_lowpart (qimode, v0);
6911 0 : v1_qi = gen_lowpart (qimode, v1);
6912 0 : if (targetm.vectorize.vec_perm_const != NULL
6913 0 : && targetm.vectorize.vec_perm_const (qimode, qimode, target_qi, v0_qi,
6914 : v1_qi, qimode_indices))
6915 0 : return gen_lowpart (mode, target_qi);
6916 : }
6917 :
6918 0 : v0 = force_reg (mode, v0);
6919 0 : if (single_arg_p)
6920 0 : v1 = v0;
6921 0 : v1 = force_reg (mode, v1);
6922 :
6923 : /* Otherwise expand as a fully variable permutation. */
6924 :
6925 : /* The optabs are only defined for selectors with the same width
6926 : as the values being permuted. */
6927 0 : machine_mode required_sel_mode;
6928 0 : if (!related_int_vector_mode (mode).exists (&required_sel_mode))
6929 : {
6930 0 : delete_insns_since (last);
6931 0 : return NULL_RTX;
6932 : }
6933 :
6934 : /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6935 : If that isn't the mode we want then we need to prove that using
6936 : REQUIRED_SEL_MODE is OK. */
6937 0 : if (sel_mode != required_sel_mode)
6938 : {
6939 0 : if (!selector_fits_mode_p (required_sel_mode, indices))
6940 : {
6941 0 : delete_insns_since (last);
6942 0 : return NULL_RTX;
6943 : }
6944 : sel_mode = required_sel_mode;
6945 : }
6946 :
6947 0 : insn_code icode = direct_optab_handler (vec_perm_optab, mode);
6948 0 : if (icode != CODE_FOR_nothing)
6949 : {
6950 0 : rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
6951 0 : rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
6952 0 : if (tmp)
6953 : return tmp;
6954 : }
6955 :
6956 0 : if (qimode != VOIDmode
6957 0 : && selector_fits_mode_p (qimode, qimode_indices))
6958 : {
6959 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
6960 0 : if (icode != CODE_FOR_nothing)
6961 : {
6962 0 : rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
6963 0 : rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
6964 0 : if (tmp)
6965 0 : return gen_lowpart (mode, tmp);
6966 : }
6967 : }
6968 :
6969 0 : delete_insns_since (last);
6970 0 : return NULL_RTX;
6971 0 : }
6972 :
6973 : /* Implement a permutation of vectors v0 and v1 using the permutation
6974 : vector in SEL and return the result. Use TARGET to hold the result
6975 : if nonnull and convenient.
6976 :
6977 : MODE is the mode of the vectors being permuted (V0 and V1).
6978 : SEL must have the integer equivalent of MODE and is known to be
6979 : unsuitable for permutes with a constant permutation vector. */
6980 :
6981 : rtx
6982 18 : expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6983 : {
6984 18 : enum insn_code icode;
6985 18 : unsigned int i, u;
6986 18 : rtx tmp, sel_qi;
6987 :
6988 18 : u = GET_MODE_UNIT_SIZE (mode);
6989 :
6990 18 : if (!target || GET_MODE (target) != mode)
6991 0 : target = gen_reg_rtx (mode);
6992 :
6993 18 : icode = direct_optab_handler (vec_perm_optab, mode);
6994 18 : if (icode != CODE_FOR_nothing)
6995 : {
6996 18 : tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6997 18 : if (tmp)
6998 : return tmp;
6999 : }
7000 :
7001 : /* As a special case to aid several targets, lower the element-based
7002 : permutation to a byte-based permutation and try again. */
7003 0 : machine_mode qimode;
7004 0 : if (!qimode_for_vec_perm (mode).exists (&qimode)
7005 0 : || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
7006 : return NULL_RTX;
7007 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
7008 0 : if (icode == CODE_FOR_nothing)
7009 : return NULL_RTX;
7010 :
7011 : /* Multiply each element by its byte size. */
7012 0 : machine_mode selmode = GET_MODE (sel);
7013 0 : if (u == 2)
7014 0 : sel = expand_simple_binop (selmode, PLUS, sel, sel,
7015 : NULL, 0, OPTAB_DIRECT);
7016 : else
7017 0 : sel = expand_simple_binop (selmode, ASHIFT, sel,
7018 0 : gen_int_shift_amount (selmode, exact_log2 (u)),
7019 : NULL, 0, OPTAB_DIRECT);
7020 0 : gcc_assert (sel != NULL);
7021 :
7022 : /* Broadcast the low byte each element into each of its bytes.
7023 : The encoding has U interleaved stepped patterns, one for each
7024 : byte of an element. */
7025 0 : vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
7026 0 : unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
7027 0 : for (i = 0; i < 3; ++i)
7028 0 : for (unsigned int j = 0; j < u; ++j)
7029 0 : const_sel.quick_push (i * u + low_byte_in_u);
7030 0 : sel = gen_lowpart (qimode, sel);
7031 0 : sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
7032 0 : gcc_assert (sel != NULL);
7033 :
7034 : /* Add the byte offset to each byte element. */
7035 : /* Note that the definition of the indices here is memory ordering,
7036 : so there should be no difference between big and little endian. */
7037 0 : rtx_vector_builder byte_indices (qimode, u, 1);
7038 0 : for (i = 0; i < u; ++i)
7039 0 : byte_indices.quick_push (GEN_INT (i));
7040 0 : tmp = byte_indices.build ();
7041 0 : sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
7042 : sel, 0, OPTAB_DIRECT);
7043 0 : gcc_assert (sel_qi != NULL);
7044 :
7045 0 : tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
7046 0 : tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
7047 0 : gen_lowpart (qimode, v1), sel_qi);
7048 0 : if (tmp)
7049 0 : tmp = gen_lowpart (mode, tmp);
7050 0 : return tmp;
7051 0 : }
7052 :
7053 : /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
7054 : Use TARGET for the result if nonnull and convenient. */
7055 :
7056 : rtx
7057 0 : expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
7058 : {
7059 0 : class expand_operand ops[3];
7060 0 : enum insn_code icode;
7061 0 : machine_mode emode = GET_MODE_INNER (vmode);
7062 :
7063 0 : icode = direct_optab_handler (vec_series_optab, vmode);
7064 0 : gcc_assert (icode != CODE_FOR_nothing);
7065 :
7066 0 : create_output_operand (&ops[0], target, vmode);
7067 0 : create_input_operand (&ops[1], op0, emode);
7068 0 : create_input_operand (&ops[2], op1, emode);
7069 :
7070 0 : expand_insn (icode, 3, ops);
7071 0 : return ops[0].value;
7072 : }
7073 :
7074 : /* Generate insns for a vector comparison into a mask. */
7075 :
7076 : rtx
7077 26203 : expand_vec_cmp_expr (tree type, tree exp, rtx target)
7078 : {
7079 26203 : class expand_operand ops[4];
7080 26203 : enum insn_code icode;
7081 26203 : rtx comparison;
7082 26203 : machine_mode mask_mode = TYPE_MODE (type);
7083 26203 : machine_mode vmode;
7084 26203 : bool unsignedp;
7085 26203 : tree op0a, op0b;
7086 26203 : enum tree_code tcode;
7087 :
7088 26203 : op0a = TREE_OPERAND (exp, 0);
7089 26203 : op0b = TREE_OPERAND (exp, 1);
7090 26203 : tcode = TREE_CODE (exp);
7091 :
7092 26203 : unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
7093 26203 : vmode = TYPE_MODE (TREE_TYPE (op0a));
7094 :
7095 26203 : icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
7096 26203 : if (icode == CODE_FOR_nothing)
7097 : {
7098 915 : if (tcode == EQ_EXPR || tcode == NE_EXPR)
7099 915 : icode = get_vec_cmp_eq_icode (vmode, mask_mode);
7100 915 : if (icode == CODE_FOR_nothing)
7101 : return 0;
7102 : }
7103 :
7104 26203 : comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
7105 : unsignedp, icode, 2);
7106 26203 : create_output_operand (&ops[0], target, mask_mode);
7107 26203 : create_fixed_operand (&ops[1], comparison);
7108 26203 : create_fixed_operand (&ops[2], XEXP (comparison, 0));
7109 26203 : create_fixed_operand (&ops[3], XEXP (comparison, 1));
7110 26203 : expand_insn (icode, 4, ops);
7111 26203 : return ops[0].value;
7112 : }
7113 :
7114 : /* Expand a highpart multiply. */
7115 :
7116 : rtx
7117 3507 : expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
7118 : rtx target, bool uns_p)
7119 : {
7120 3507 : class expand_operand eops[3];
7121 3507 : enum insn_code icode;
7122 3507 : int method, i;
7123 3507 : machine_mode wmode;
7124 3507 : rtx m1, m2;
7125 3507 : optab tab1, tab2;
7126 :
7127 3507 : method = can_mult_highpart_p (mode, uns_p);
7128 3507 : switch (method)
7129 : {
7130 : case 0:
7131 : return NULL_RTX;
7132 2760 : case 1:
7133 2760 : tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
7134 2760 : return expand_binop (mode, tab1, op0, op1, target, uns_p,
7135 2760 : OPTAB_LIB_WIDEN);
7136 0 : case 2:
7137 0 : return expmed_mult_highpart_optab (as_a <scalar_int_mode> (mode),
7138 0 : op0, op1, target, uns_p, INT_MAX);
7139 738 : case 3:
7140 738 : tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
7141 531 : tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
7142 : break;
7143 9 : case 4:
7144 9 : tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
7145 9 : tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
7146 : if (BYTES_BIG_ENDIAN)
7147 : std::swap (tab1, tab2);
7148 : break;
7149 0 : default:
7150 0 : gcc_unreachable ();
7151 : }
7152 :
7153 747 : icode = optab_handler (tab1, mode);
7154 747 : wmode = insn_data[icode].operand[0].mode;
7155 2241 : gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
7156 : GET_MODE_NUNITS (mode)));
7157 2241 : gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
7158 :
7159 747 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7160 747 : create_input_operand (&eops[1], op0, mode);
7161 747 : create_input_operand (&eops[2], op1, mode);
7162 747 : expand_insn (icode, 3, eops);
7163 747 : m1 = gen_lowpart (mode, eops[0].value);
7164 :
7165 747 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7166 747 : create_input_operand (&eops[1], op0, mode);
7167 747 : create_input_operand (&eops[2], op1, mode);
7168 747 : expand_insn (optab_handler (tab2, mode), 3, eops);
7169 747 : m2 = gen_lowpart (mode, eops[0].value);
7170 :
7171 1494 : vec_perm_builder sel;
7172 747 : if (method == 3)
7173 : {
7174 : /* The encoding has 2 interleaved stepped patterns. */
7175 1476 : sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
7176 5166 : for (i = 0; i < 6; ++i)
7177 13284 : sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
7178 6642 : + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
7179 : }
7180 : else
7181 : {
7182 : /* The encoding has a single interleaved stepped pattern. */
7183 18 : sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
7184 36 : for (i = 0; i < 3; ++i)
7185 27 : sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
7186 : }
7187 :
7188 747 : return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
7189 : }
7190 :
7191 : /* Helper function to find the MODE_CC set in a sync_compare_and_swap
7192 : pattern. */
7193 :
7194 : static void
7195 0 : find_cc_set (rtx x, const_rtx pat, void *data)
7196 : {
7197 0 : if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
7198 0 : && GET_CODE (pat) == SET)
7199 : {
7200 0 : rtx *p_cc_reg = (rtx *) data;
7201 0 : gcc_assert (!*p_cc_reg);
7202 0 : *p_cc_reg = x;
7203 : }
7204 0 : }
7205 :
7206 : /* This is a helper function for the other atomic operations. This function
7207 : emits a loop that contains SEQ that iterates until a compare-and-swap
7208 : operation at the end succeeds. MEM is the memory to be modified. SEQ is
7209 : a set of instructions that takes a value from OLD_REG as an input and
7210 : produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
7211 : set to the current contents of MEM. After SEQ, a compare-and-swap will
7212 : attempt to update MEM with NEW_REG. The function returns true when the
7213 : loop was generated successfully. */
7214 :
7215 : static bool
7216 5094 : expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
7217 : {
7218 5094 : machine_mode mode = GET_MODE (mem);
7219 5094 : rtx_code_label *label;
7220 5094 : rtx cmp_reg, success, oldval;
7221 :
7222 : /* The loop we want to generate looks like
7223 :
7224 : cmp_reg = mem;
7225 : label:
7226 : old_reg = cmp_reg;
7227 : seq;
7228 : (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
7229 : if (success)
7230 : goto label;
7231 :
7232 : Note that we only do the plain load from memory once. Subsequent
7233 : iterations use the value loaded by the compare-and-swap pattern. */
7234 :
7235 5094 : label = gen_label_rtx ();
7236 5094 : cmp_reg = gen_reg_rtx (mode);
7237 :
7238 5094 : emit_move_insn (cmp_reg, mem);
7239 5094 : emit_label (label);
7240 5094 : emit_move_insn (old_reg, cmp_reg);
7241 5094 : if (seq)
7242 5081 : emit_insn (seq);
7243 :
7244 5094 : success = NULL_RTX;
7245 5094 : oldval = cmp_reg;
7246 5094 : if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
7247 : new_reg, false, MEMMODEL_SYNC_SEQ_CST,
7248 : MEMMODEL_RELAXED))
7249 : return false;
7250 :
7251 5094 : if (oldval != cmp_reg)
7252 0 : emit_move_insn (cmp_reg, oldval);
7253 :
7254 : /* Mark this jump predicted not taken. */
7255 5094 : emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
7256 5094 : GET_MODE (success), 1, label,
7257 : profile_probability::guessed_never ());
7258 5094 : return true;
7259 : }
7260 :
7261 :
7262 : /* This function tries to emit an atomic_exchange instruction. VAL is written
7263 : to *MEM using memory model MODEL. The previous contents of *MEM are returned,
7264 : using TARGET if possible. */
7265 :
7266 : static rtx
7267 3941 : maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7268 : {
7269 3941 : machine_mode mode = GET_MODE (mem);
7270 3941 : enum insn_code icode;
7271 :
7272 : /* If the target supports the exchange directly, great. */
7273 3941 : icode = direct_optab_handler (atomic_exchange_optab, mode);
7274 3941 : if (icode != CODE_FOR_nothing)
7275 : {
7276 3924 : class expand_operand ops[4];
7277 :
7278 3924 : create_output_operand (&ops[0], target, mode);
7279 3924 : create_fixed_operand (&ops[1], mem);
7280 3924 : create_input_operand (&ops[2], val, mode);
7281 3924 : create_integer_operand (&ops[3], model);
7282 3924 : if (maybe_expand_insn (icode, 4, ops))
7283 3924 : return ops[0].value;
7284 : }
7285 :
7286 : return NULL_RTX;
7287 : }
7288 :
7289 : /* This function tries to implement an atomic exchange operation using
7290 : __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
7291 : The previous contents of *MEM are returned, using TARGET if possible.
7292 : Since this instructionn is an acquire barrier only, stronger memory
7293 : models may require additional barriers to be emitted. */
7294 :
7295 : static rtx
7296 2 : maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
7297 : enum memmodel model)
7298 : {
7299 2 : machine_mode mode = GET_MODE (mem);
7300 2 : enum insn_code icode;
7301 2 : rtx_insn *last_insn = get_last_insn ();
7302 :
7303 2 : icode = optab_handler (sync_lock_test_and_set_optab, mode);
7304 :
7305 : /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7306 : exists, and the memory model is stronger than acquire, add a release
7307 : barrier before the instruction. */
7308 :
7309 2 : if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
7310 0 : expand_mem_thread_fence (model);
7311 :
7312 2 : if (icode != CODE_FOR_nothing)
7313 : {
7314 0 : class expand_operand ops[3];
7315 0 : create_output_operand (&ops[0], target, mode);
7316 0 : create_fixed_operand (&ops[1], mem);
7317 0 : create_input_operand (&ops[2], val, mode);
7318 0 : if (maybe_expand_insn (icode, 3, ops))
7319 0 : return ops[0].value;
7320 : }
7321 :
7322 : /* If an external test-and-set libcall is provided, use that instead of
7323 : any external compare-and-swap that we might get from the compare-and-
7324 : swap-loop expansion later. */
7325 2 : if (!can_compare_and_swap_p (mode, false))
7326 : {
7327 2 : rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
7328 2 : if (libfunc != NULL)
7329 : {
7330 0 : rtx addr;
7331 :
7332 0 : addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7333 0 : return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7334 : mode, addr, ptr_mode,
7335 0 : val, mode);
7336 : }
7337 : }
7338 :
7339 : /* If the test_and_set can't be emitted, eliminate any barrier that might
7340 : have been emitted. */
7341 2 : delete_insns_since (last_insn);
7342 2 : return NULL_RTX;
7343 : }
7344 :
7345 : /* This function tries to implement an atomic exchange operation using a
7346 : compare_and_swap loop. VAL is written to *MEM. The previous contents of
7347 : *MEM are returned, using TARGET if possible. No memory model is required
7348 : since a compare_and_swap loop is seq-cst. */
7349 :
7350 : static rtx
7351 17 : maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
7352 : {
7353 17 : machine_mode mode = GET_MODE (mem);
7354 :
7355 17 : if (can_compare_and_swap_p (mode, true))
7356 : {
7357 13 : rtx_insn *start = get_last_insn ();
7358 : /* Force val into a register if it could change value when the
7359 : atomic insn updates mem. */
7360 13 : if (reg_overlap_mentioned_p (mem, val))
7361 2 : val = force_reg (mode, val);
7362 13 : if (!target || !register_operand (target, mode))
7363 1 : target = gen_reg_rtx (mode);
7364 13 : if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
7365 : return target;
7366 : else
7367 0 : delete_insns_since (start);
7368 : }
7369 :
7370 : return NULL_RTX;
7371 : }
7372 :
7373 : /* This function tries to implement an atomic test-and-set operation
7374 : using the atomic_test_and_set instruction pattern. A boolean value
7375 : is returned from the operation, using TARGET if possible. */
7376 :
7377 : static rtx
7378 268 : maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7379 : {
7380 268 : machine_mode pat_bool_mode;
7381 268 : class expand_operand ops[3];
7382 :
7383 268 : if (!targetm.have_atomic_test_and_set ())
7384 : return NULL_RTX;
7385 :
7386 : /* While we always get QImode from __atomic_test_and_set, we get
7387 : other memory modes from __sync_lock_test_and_set. Note that we
7388 : use no endian adjustment here. This matches the 4.6 behavior
7389 : in the Sparc backend. */
7390 0 : enum insn_code icode = targetm.code_for_atomic_test_and_set;
7391 0 : gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
7392 0 : if (GET_MODE (mem) != QImode)
7393 0 : mem = adjust_address_nv (mem, QImode, 0);
7394 :
7395 0 : pat_bool_mode = insn_data[icode].operand[0].mode;
7396 0 : create_output_operand (&ops[0], target, pat_bool_mode);
7397 0 : create_fixed_operand (&ops[1], mem);
7398 0 : create_integer_operand (&ops[2], model);
7399 :
7400 0 : if (maybe_expand_insn (icode, 3, ops))
7401 0 : return ops[0].value;
7402 : return NULL_RTX;
7403 : }
7404 :
7405 : /* This function expands the legacy _sync_lock test_and_set operation which is
7406 : generally an atomic exchange. Some limited targets only allow the
7407 : constant 1 to be stored. This is an ACQUIRE operation.
7408 :
7409 : TARGET is an optional place to stick the return value.
7410 : MEM is where VAL is stored. */
7411 :
7412 : rtx
7413 326 : expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7414 : {
7415 326 : rtx ret;
7416 :
7417 : /* Try an atomic_exchange first. */
7418 326 : ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
7419 326 : if (ret)
7420 : return ret;
7421 :
7422 2 : ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
7423 : MEMMODEL_SYNC_ACQUIRE);
7424 2 : if (ret)
7425 : return ret;
7426 :
7427 2 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7428 2 : if (ret)
7429 : return ret;
7430 :
7431 : /* If there are no other options, try atomic_test_and_set if the value
7432 : being stored is 1. */
7433 2 : if (val == const1_rtx)
7434 2 : ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
7435 :
7436 : return ret;
7437 : }
7438 :
7439 : /* This function expands the atomic test_and_set operation:
7440 : atomically store a boolean TRUE into MEM and return the previous value.
7441 :
7442 : MEMMODEL is the memory model variant to use.
7443 : TARGET is an optional place to stick the return value. */
7444 :
7445 : rtx
7446 266 : expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7447 : {
7448 266 : machine_mode mode = GET_MODE (mem);
7449 266 : rtx ret, trueval, subtarget;
7450 :
7451 266 : ret = maybe_emit_atomic_test_and_set (target, mem, model);
7452 266 : if (ret)
7453 : return ret;
7454 :
7455 : /* Be binary compatible with non-default settings of trueval, and different
7456 : cpu revisions. E.g. one revision may have atomic-test-and-set, but
7457 : another only has atomic-exchange. */
7458 266 : if (targetm.atomic_test_and_set_trueval == 1)
7459 : {
7460 266 : trueval = const1_rtx;
7461 266 : subtarget = target ? target : gen_reg_rtx (mode);
7462 : }
7463 : else
7464 : {
7465 0 : trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7466 0 : subtarget = gen_reg_rtx (mode);
7467 : }
7468 :
7469 : /* Try the atomic-exchange optab... */
7470 266 : ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
7471 :
7472 : /* ... then an atomic-compare-and-swap loop ... */
7473 266 : if (!ret)
7474 0 : ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
7475 :
7476 : /* ... before trying the vaguely defined legacy lock_test_and_set. */
7477 0 : if (!ret)
7478 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
7479 :
7480 : /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7481 : things with the value 1. Thus we try again without trueval. */
7482 266 : if (!ret && targetm.atomic_test_and_set_trueval != 1)
7483 : {
7484 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
7485 :
7486 0 : if (ret)
7487 : {
7488 : /* Rectify the not-one trueval. */
7489 0 : ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7490 0 : gcc_assert (ret);
7491 : }
7492 : }
7493 :
7494 : return ret;
7495 : }
7496 :
7497 : /* This function expands the atomic exchange operation:
7498 : atomically store VAL in MEM and return the previous value in MEM.
7499 :
7500 : MEMMODEL is the memory model variant to use.
7501 : TARGET is an optional place to stick the return value. */
7502 :
7503 : rtx
7504 2886 : expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7505 : {
7506 2886 : machine_mode mode = GET_MODE (mem);
7507 2886 : rtx ret;
7508 :
7509 : /* If loads are not atomic for the required size and we are not called to
7510 : provide a __sync builtin, do not do anything so that we stay consistent
7511 : with atomic loads of the same size. */
7512 2886 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7513 : return NULL_RTX;
7514 :
7515 2855 : ret = maybe_emit_atomic_exchange (target, mem, val, model);
7516 :
7517 : /* Next try a compare-and-swap loop for the exchange. */
7518 2855 : if (!ret)
7519 13 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7520 :
7521 : return ret;
7522 : }
7523 :
7524 : /* This function expands the atomic compare exchange operation:
7525 :
7526 : *PTARGET_BOOL is an optional place to store the boolean success/failure.
7527 : *PTARGET_OVAL is an optional place to store the old value from memory.
7528 : Both target parameters may be NULL or const0_rtx to indicate that we do
7529 : not care about that return value. Both target parameters are updated on
7530 : success to the actual location of the corresponding result.
7531 :
7532 : MEMMODEL is the memory model variant to use.
7533 :
7534 : The return value of the function is true for success. */
7535 :
7536 : bool
7537 28578 : expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7538 : rtx mem, rtx expected, rtx desired,
7539 : bool is_weak, enum memmodel succ_model,
7540 : enum memmodel fail_model)
7541 : {
7542 28578 : machine_mode mode = GET_MODE (mem);
7543 28578 : class expand_operand ops[8];
7544 28578 : enum insn_code icode;
7545 28578 : rtx target_oval, target_bool = NULL_RTX;
7546 28578 : rtx libfunc;
7547 :
7548 : /* If loads are not atomic for the required size and we are not called to
7549 : provide a __sync builtin, do not do anything so that we stay consistent
7550 : with atomic loads of the same size. */
7551 28578 : if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
7552 : return false;
7553 :
7554 : /* Load expected into a register for the compare and swap. */
7555 26829 : if (MEM_P (expected))
7556 7360 : expected = copy_to_reg (expected);
7557 :
7558 : /* Make sure we always have some place to put the return oldval.
7559 : Further, make sure that place is distinct from the input expected,
7560 : just in case we need that path down below. */
7561 26829 : if (ptarget_oval && *ptarget_oval == const0_rtx)
7562 : ptarget_oval = NULL;
7563 :
7564 26599 : if (ptarget_oval == NULL
7565 26599 : || (target_oval = *ptarget_oval) == NULL
7566 5324 : || reg_overlap_mentioned_p (expected, target_oval))
7567 21583 : target_oval = gen_reg_rtx (mode);
7568 :
7569 26829 : icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7570 26829 : if (icode != CODE_FOR_nothing)
7571 : {
7572 26827 : machine_mode bool_mode = insn_data[icode].operand[0].mode;
7573 :
7574 26827 : if (ptarget_bool && *ptarget_bool == const0_rtx)
7575 : ptarget_bool = NULL;
7576 :
7577 : /* Make sure we always have a place for the bool operand. */
7578 26572 : if (ptarget_bool == NULL
7579 26572 : || (target_bool = *ptarget_bool) == NULL
7580 7248 : || GET_MODE (target_bool) != bool_mode)
7581 19579 : target_bool = gen_reg_rtx (bool_mode);
7582 :
7583 : /* Emit the compare_and_swap. */
7584 26827 : create_output_operand (&ops[0], target_bool, bool_mode);
7585 26827 : create_output_operand (&ops[1], target_oval, mode);
7586 26827 : create_fixed_operand (&ops[2], mem);
7587 26827 : create_input_operand (&ops[3], expected, mode);
7588 26827 : create_input_operand (&ops[4], desired, mode);
7589 26827 : create_integer_operand (&ops[5], is_weak);
7590 26827 : create_integer_operand (&ops[6], succ_model);
7591 26827 : create_integer_operand (&ops[7], fail_model);
7592 26827 : if (maybe_expand_insn (icode, 8, ops))
7593 : {
7594 : /* Return success/failure. */
7595 26827 : target_bool = ops[0].value;
7596 26827 : target_oval = ops[1].value;
7597 26827 : goto success;
7598 : }
7599 : }
7600 :
7601 : /* Otherwise fall back to the original __sync_val_compare_and_swap
7602 : which is always seq-cst. */
7603 2 : icode = optab_handler (sync_compare_and_swap_optab, mode);
7604 2 : if (icode != CODE_FOR_nothing)
7605 : {
7606 0 : rtx cc_reg;
7607 :
7608 0 : create_output_operand (&ops[0], target_oval, mode);
7609 0 : create_fixed_operand (&ops[1], mem);
7610 0 : create_input_operand (&ops[2], expected, mode);
7611 0 : create_input_operand (&ops[3], desired, mode);
7612 0 : if (!maybe_expand_insn (icode, 4, ops))
7613 0 : return false;
7614 :
7615 0 : target_oval = ops[0].value;
7616 :
7617 : /* If the caller isn't interested in the boolean return value,
7618 : skip the computation of it. */
7619 0 : if (ptarget_bool == NULL)
7620 0 : goto success;
7621 :
7622 : /* Otherwise, work out if the compare-and-swap succeeded. */
7623 0 : cc_reg = NULL_RTX;
7624 0 : if (have_insn_for (COMPARE, CCmode))
7625 0 : note_stores (get_last_insn (), find_cc_set, &cc_reg);
7626 0 : if (cc_reg)
7627 : {
7628 0 : target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7629 : const0_rtx, VOIDmode, 0, 1);
7630 0 : goto success;
7631 : }
7632 0 : goto success_bool_from_val;
7633 : }
7634 :
7635 : /* Also check for library support for __sync_val_compare_and_swap. */
7636 2 : libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7637 2 : if (libfunc != NULL)
7638 : {
7639 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7640 0 : rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7641 : mode, addr, ptr_mode,
7642 : expected, mode, desired, mode);
7643 0 : emit_move_insn (target_oval, target);
7644 :
7645 : /* Compute the boolean return value only if requested. */
7646 0 : if (ptarget_bool)
7647 0 : goto success_bool_from_val;
7648 : else
7649 0 : goto success;
7650 : }
7651 :
7652 : /* Failure. */
7653 : return false;
7654 :
7655 0 : success_bool_from_val:
7656 0 : target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7657 : expected, VOIDmode, 1, 1);
7658 26827 : success:
7659 : /* Make sure that the oval output winds up where the caller asked. */
7660 26827 : if (ptarget_oval)
7661 26599 : *ptarget_oval = target_oval;
7662 26827 : if (ptarget_bool)
7663 26572 : *ptarget_bool = target_bool;
7664 : return true;
7665 : }
7666 :
7667 : /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7668 :
7669 : static void
7670 0 : expand_asm_memory_blockage (void)
7671 : {
7672 0 : rtx asm_op, clob;
7673 :
7674 0 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7675 : rtvec_alloc (0), rtvec_alloc (0),
7676 : rtvec_alloc (0), UNKNOWN_LOCATION);
7677 0 : MEM_VOLATILE_P (asm_op) = 1;
7678 :
7679 0 : clob = gen_rtx_SCRATCH (VOIDmode);
7680 0 : clob = gen_rtx_MEM (BLKmode, clob);
7681 0 : clob = gen_rtx_CLOBBER (VOIDmode, clob);
7682 :
7683 0 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7684 0 : }
7685 :
7686 : /* Do not propagate memory accesses across this point. */
7687 :
7688 : static void
7689 116332 : expand_memory_blockage (void)
7690 : {
7691 116332 : if (targetm.have_memory_blockage ())
7692 116332 : emit_insn (targetm.gen_memory_blockage ());
7693 : else
7694 0 : expand_asm_memory_blockage ();
7695 116332 : }
7696 :
7697 : /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7698 : same time clobbering the register set specified by REGS. */
7699 :
7700 : void
7701 131 : expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs)
7702 : {
7703 131 : rtx asm_op, clob_mem;
7704 :
7705 131 : unsigned int num_of_regs = 0;
7706 131 : unsigned int i;
7707 :
7708 131 : num_of_regs = hard_reg_set_popcount (regs);
7709 :
7710 131 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7711 : rtvec_alloc (0), rtvec_alloc (0),
7712 : rtvec_alloc (0), UNKNOWN_LOCATION);
7713 131 : MEM_VOLATILE_P (asm_op) = 1;
7714 :
7715 131 : rtvec v = rtvec_alloc (num_of_regs + 2);
7716 :
7717 131 : clob_mem = gen_rtx_SCRATCH (VOIDmode);
7718 131 : clob_mem = gen_rtx_MEM (BLKmode, clob_mem);
7719 131 : clob_mem = gen_rtx_CLOBBER (VOIDmode, clob_mem);
7720 :
7721 131 : RTVEC_ELT (v, 0) = asm_op;
7722 131 : RTVEC_ELT (v, 1) = clob_mem;
7723 :
7724 131 : if (num_of_regs > 0)
7725 : {
7726 131 : unsigned int j = 2;
7727 131 : hard_reg_set_iterator hrsi2;
7728 131 : i = 0;
7729 1250 : EXECUTE_IF_SET_IN_HARD_REG_SET (regs, 0, i, hrsi2)
7730 : {
7731 1119 : RTVEC_ELT (v, j) = gen_rtx_CLOBBER (VOIDmode, regno_reg_rtx[i]);
7732 1119 : j++;
7733 : }
7734 131 : gcc_assert (j == (num_of_regs + 2));
7735 : }
7736 :
7737 131 : emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
7738 131 : }
7739 :
7740 : /* This routine will either emit the mem_thread_fence pattern or issue a
7741 : sync_synchronize to generate a fence for memory model MEMMODEL. */
7742 :
7743 : void
7744 966 : expand_mem_thread_fence (enum memmodel model)
7745 : {
7746 966 : if (is_mm_relaxed (model))
7747 : return;
7748 957 : if (targetm.have_mem_thread_fence ())
7749 : {
7750 957 : emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
7751 957 : expand_memory_blockage ();
7752 : }
7753 0 : else if (targetm.have_memory_barrier ())
7754 0 : emit_insn (targetm.gen_memory_barrier ());
7755 0 : else if (synchronize_libfunc != NULL_RTX)
7756 0 : emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
7757 : else
7758 0 : expand_memory_blockage ();
7759 : }
7760 :
7761 : /* Emit a signal fence with given memory model. */
7762 :
7763 : void
7764 60 : expand_mem_signal_fence (enum memmodel model)
7765 : {
7766 : /* No machine barrier is required to implement a signal fence, but
7767 : a compiler memory barrier must be issued, except for relaxed MM. */
7768 60 : if (!is_mm_relaxed (model))
7769 51 : expand_memory_blockage ();
7770 60 : }
7771 :
7772 : /* This function expands the atomic load operation:
7773 : return the atomically loaded value in MEM.
7774 :
7775 : MEMMODEL is the memory model variant to use.
7776 : TARGET is an option place to stick the return value. */
7777 :
7778 : rtx
7779 70646 : expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7780 : {
7781 70646 : machine_mode mode = GET_MODE (mem);
7782 70646 : enum insn_code icode;
7783 :
7784 : /* If the target supports the load directly, great. */
7785 70646 : icode = direct_optab_handler (atomic_load_optab, mode);
7786 70646 : if (icode != CODE_FOR_nothing)
7787 : {
7788 66818 : class expand_operand ops[3];
7789 66818 : rtx_insn *last = get_last_insn ();
7790 66818 : if (is_mm_seq_cst (model))
7791 38270 : expand_memory_blockage ();
7792 :
7793 66818 : create_output_operand (&ops[0], target, mode);
7794 66818 : create_fixed_operand (&ops[1], mem);
7795 66818 : create_integer_operand (&ops[2], model);
7796 66818 : if (maybe_expand_insn (icode, 3, ops))
7797 : {
7798 66818 : if (!is_mm_relaxed (model))
7799 52222 : expand_memory_blockage ();
7800 66818 : return ops[0].value;
7801 : }
7802 0 : delete_insns_since (last);
7803 : }
7804 :
7805 : /* If the size of the object is greater than word size on this target,
7806 : then we assume that a load will not be atomic. We could try to
7807 : emulate a load with a compare-and-swap operation, but the store that
7808 : doing this could result in would be incorrect if this is a volatile
7809 : atomic load or targeting read-only-mapped memory. */
7810 3828 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7811 : /* If there is no atomic load, leave the library call. */
7812 : return NULL_RTX;
7813 :
7814 : /* Otherwise assume loads are atomic, and emit the proper barriers. */
7815 0 : if (!target || target == const0_rtx)
7816 0 : target = gen_reg_rtx (mode);
7817 :
7818 : /* For SEQ_CST, emit a barrier before the load. */
7819 0 : if (is_mm_seq_cst (model))
7820 0 : expand_mem_thread_fence (model);
7821 :
7822 0 : emit_move_insn (target, mem);
7823 :
7824 : /* Emit the appropriate barrier after the load. */
7825 0 : expand_mem_thread_fence (model);
7826 :
7827 0 : return target;
7828 : }
7829 :
7830 : /* This function expands the atomic store operation:
7831 : Atomically store VAL in MEM.
7832 : MEMMODEL is the memory model variant to use.
7833 : USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7834 : function returns const0_rtx if a pattern was emitted. */
7835 :
7836 : rtx
7837 17633 : expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7838 : {
7839 17633 : machine_mode mode = GET_MODE (mem);
7840 17633 : enum insn_code icode;
7841 17633 : class expand_operand ops[3];
7842 :
7843 : /* If the target supports the store directly, great. */
7844 17633 : icode = direct_optab_handler (atomic_store_optab, mode);
7845 17633 : if (icode != CODE_FOR_nothing)
7846 : {
7847 16030 : rtx_insn *last = get_last_insn ();
7848 16030 : if (!is_mm_relaxed (model))
7849 13270 : expand_memory_blockage ();
7850 16030 : create_fixed_operand (&ops[0], mem);
7851 16030 : create_input_operand (&ops[1], val, mode);
7852 16030 : create_integer_operand (&ops[2], model);
7853 16030 : if (maybe_expand_insn (icode, 3, ops))
7854 : {
7855 16030 : if (is_mm_seq_cst (model))
7856 11562 : expand_memory_blockage ();
7857 16030 : return const0_rtx;
7858 : }
7859 0 : delete_insns_since (last);
7860 : }
7861 :
7862 : /* If using __sync_lock_release is a viable alternative, try it.
7863 : Note that this will not be set to true if we are expanding a generic
7864 : __atomic_store_n. */
7865 1603 : if (use_release)
7866 : {
7867 2 : icode = direct_optab_handler (sync_lock_release_optab, mode);
7868 2 : if (icode != CODE_FOR_nothing)
7869 : {
7870 0 : create_fixed_operand (&ops[0], mem);
7871 0 : create_input_operand (&ops[1], const0_rtx, mode);
7872 0 : if (maybe_expand_insn (icode, 2, ops))
7873 : {
7874 : /* lock_release is only a release barrier. */
7875 0 : if (is_mm_seq_cst (model))
7876 0 : expand_mem_thread_fence (model);
7877 0 : return const0_rtx;
7878 : }
7879 : }
7880 : }
7881 :
7882 : /* If the size of the object is greater than word size on this target,
7883 : a default store will not be atomic. */
7884 1603 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7885 : {
7886 : /* If loads are atomic or we are called to provide a __sync builtin,
7887 : we can try a atomic_exchange and throw away the result. Otherwise,
7888 : don't do anything so that we do not create an inconsistency between
7889 : loads and stores. */
7890 1603 : if (can_atomic_load_p (mode) || is_mm_sync (model))
7891 : {
7892 2 : rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7893 2 : if (!target)
7894 2 : target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
7895 : val);
7896 2 : if (target)
7897 0 : return const0_rtx;
7898 : }
7899 : return NULL_RTX;
7900 : }
7901 :
7902 : /* Otherwise assume stores are atomic, and emit the proper barriers. */
7903 0 : expand_mem_thread_fence (model);
7904 :
7905 0 : emit_move_insn (mem, val);
7906 :
7907 : /* For SEQ_CST, also emit a barrier after the store. */
7908 0 : if (is_mm_seq_cst (model))
7909 0 : expand_mem_thread_fence (model);
7910 :
7911 0 : return const0_rtx;
7912 : }
7913 :
7914 :
7915 : /* Structure containing the pointers and values required to process the
7916 : various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7917 :
7918 : struct atomic_op_functions
7919 : {
7920 : direct_optab mem_fetch_before;
7921 : direct_optab mem_fetch_after;
7922 : direct_optab mem_no_result;
7923 : optab fetch_before;
7924 : optab fetch_after;
7925 : direct_optab no_result;
7926 : enum rtx_code reverse_code;
7927 : };
7928 :
7929 :
7930 : /* Fill in structure pointed to by OP with the various optab entries for an
7931 : operation of type CODE. */
7932 :
7933 : static void
7934 34954 : get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7935 : {
7936 34954 : gcc_assert (op!= NULL);
7937 :
7938 : /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7939 : in the source code during compilation, and the optab entries are not
7940 : computable until runtime. Fill in the values at runtime. */
7941 34954 : switch (code)
7942 : {
7943 22615 : case PLUS:
7944 22615 : op->mem_fetch_before = atomic_fetch_add_optab;
7945 22615 : op->mem_fetch_after = atomic_add_fetch_optab;
7946 22615 : op->mem_no_result = atomic_add_optab;
7947 22615 : op->fetch_before = sync_old_add_optab;
7948 22615 : op->fetch_after = sync_new_add_optab;
7949 22615 : op->no_result = sync_add_optab;
7950 22615 : op->reverse_code = MINUS;
7951 22615 : break;
7952 4825 : case MINUS:
7953 4825 : op->mem_fetch_before = atomic_fetch_sub_optab;
7954 4825 : op->mem_fetch_after = atomic_sub_fetch_optab;
7955 4825 : op->mem_no_result = atomic_sub_optab;
7956 4825 : op->fetch_before = sync_old_sub_optab;
7957 4825 : op->fetch_after = sync_new_sub_optab;
7958 4825 : op->no_result = sync_sub_optab;
7959 4825 : op->reverse_code = PLUS;
7960 4825 : break;
7961 2204 : case XOR:
7962 2204 : op->mem_fetch_before = atomic_fetch_xor_optab;
7963 2204 : op->mem_fetch_after = atomic_xor_fetch_optab;
7964 2204 : op->mem_no_result = atomic_xor_optab;
7965 2204 : op->fetch_before = sync_old_xor_optab;
7966 2204 : op->fetch_after = sync_new_xor_optab;
7967 2204 : op->no_result = sync_xor_optab;
7968 2204 : op->reverse_code = XOR;
7969 2204 : break;
7970 2078 : case AND:
7971 2078 : op->mem_fetch_before = atomic_fetch_and_optab;
7972 2078 : op->mem_fetch_after = atomic_and_fetch_optab;
7973 2078 : op->mem_no_result = atomic_and_optab;
7974 2078 : op->fetch_before = sync_old_and_optab;
7975 2078 : op->fetch_after = sync_new_and_optab;
7976 2078 : op->no_result = sync_and_optab;
7977 2078 : op->reverse_code = UNKNOWN;
7978 2078 : break;
7979 2519 : case IOR:
7980 2519 : op->mem_fetch_before = atomic_fetch_or_optab;
7981 2519 : op->mem_fetch_after = atomic_or_fetch_optab;
7982 2519 : op->mem_no_result = atomic_or_optab;
7983 2519 : op->fetch_before = sync_old_ior_optab;
7984 2519 : op->fetch_after = sync_new_ior_optab;
7985 2519 : op->no_result = sync_ior_optab;
7986 2519 : op->reverse_code = UNKNOWN;
7987 2519 : break;
7988 713 : case NOT:
7989 713 : op->mem_fetch_before = atomic_fetch_nand_optab;
7990 713 : op->mem_fetch_after = atomic_nand_fetch_optab;
7991 713 : op->mem_no_result = atomic_nand_optab;
7992 713 : op->fetch_before = sync_old_nand_optab;
7993 713 : op->fetch_after = sync_new_nand_optab;
7994 713 : op->no_result = sync_nand_optab;
7995 713 : op->reverse_code = UNKNOWN;
7996 713 : break;
7997 0 : default:
7998 0 : gcc_unreachable ();
7999 : }
8000 34954 : }
8001 :
8002 : /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
8003 : using memory order MODEL. If AFTER is true the operation needs to return
8004 : the value of *MEM after the operation, otherwise the previous value.
8005 : TARGET is an optional place to place the result. The result is unused if
8006 : it is const0_rtx.
8007 : Return the result if there is a better sequence, otherwise NULL_RTX. */
8008 :
8009 : static rtx
8010 34952 : maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
8011 : enum memmodel model, bool after)
8012 : {
8013 : /* If the value is prefetched, or not used, it may be possible to replace
8014 : the sequence with a native exchange operation. */
8015 34952 : if (!after || target == const0_rtx)
8016 : {
8017 : /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
8018 26803 : if (code == AND && val == const0_rtx)
8019 : {
8020 327 : if (target == const0_rtx)
8021 150 : target = gen_reg_rtx (GET_MODE (mem));
8022 327 : return maybe_emit_atomic_exchange (target, mem, val, model);
8023 : }
8024 :
8025 : /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
8026 26476 : if (code == IOR && val == constm1_rtx)
8027 : {
8028 165 : if (target == const0_rtx)
8029 164 : target = gen_reg_rtx (GET_MODE (mem));
8030 165 : return maybe_emit_atomic_exchange (target, mem, val, model);
8031 : }
8032 : }
8033 :
8034 : return NULL_RTX;
8035 : }
8036 :
8037 : /* Try to emit an instruction for a specific operation variation.
8038 : OPTAB contains the OP functions.
8039 : TARGET is an optional place to return the result. const0_rtx means unused.
8040 : MEM is the memory location to operate on.
8041 : VAL is the value to use in the operation.
8042 : USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
8043 : MODEL is the memory model, if used.
8044 : AFTER is true if the returned result is the value after the operation. */
8045 :
8046 : static rtx
8047 64552 : maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
8048 : rtx val, bool use_memmodel, enum memmodel model, bool after)
8049 : {
8050 64552 : machine_mode mode = GET_MODE (mem);
8051 64552 : class expand_operand ops[4];
8052 64552 : enum insn_code icode;
8053 64552 : int op_counter = 0;
8054 64552 : int num_ops;
8055 :
8056 : /* Check to see if there is a result returned. */
8057 64552 : if (target == const0_rtx)
8058 : {
8059 14676 : if (use_memmodel)
8060 : {
8061 14317 : icode = direct_optab_handler (optab->mem_no_result, mode);
8062 14317 : create_integer_operand (&ops[2], model);
8063 14317 : num_ops = 3;
8064 : }
8065 : else
8066 : {
8067 359 : icode = direct_optab_handler (optab->no_result, mode);
8068 359 : num_ops = 2;
8069 : }
8070 : }
8071 : /* Otherwise, we need to generate a result. */
8072 : else
8073 : {
8074 49876 : if (use_memmodel)
8075 : {
8076 31144 : icode = direct_optab_handler (after ? optab->mem_fetch_after
8077 : : optab->mem_fetch_before, mode);
8078 31144 : create_integer_operand (&ops[3], model);
8079 31144 : num_ops = 4;
8080 : }
8081 : else
8082 : {
8083 18732 : icode = optab_handler (after ? optab->fetch_after
8084 : : optab->fetch_before, mode);
8085 18732 : num_ops = 3;
8086 : }
8087 49876 : create_output_operand (&ops[op_counter++], target, mode);
8088 : }
8089 64552 : if (icode == CODE_FOR_nothing)
8090 : return NULL_RTX;
8091 :
8092 26370 : create_fixed_operand (&ops[op_counter++], mem);
8093 : /* VAL may have been promoted to a wider mode. Shrink it if so. */
8094 26370 : create_convert_operand_to (&ops[op_counter++], val, mode, true);
8095 :
8096 26370 : if (maybe_expand_insn (icode, num_ops, ops))
8097 26370 : return (target == const0_rtx ? const0_rtx : ops[0].value);
8098 :
8099 : return NULL_RTX;
8100 : }
8101 :
8102 :
8103 : /* This function expands an atomic fetch_OP or OP_fetch operation:
8104 : TARGET is an option place to stick the return value. const0_rtx indicates
8105 : the result is unused.
8106 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
8107 : CODE is the operation being performed (OP)
8108 : MEMMODEL is the memory model variant to use.
8109 : AFTER is true to return the result of the operation (OP_fetch).
8110 : AFTER is false to return the value before the operation (fetch_OP).
8111 :
8112 : This function will *only* generate instructions if there is a direct
8113 : optab. No compare and swap loops or libcalls will be generated. */
8114 :
8115 : static rtx
8116 34952 : expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
8117 : enum rtx_code code, enum memmodel model,
8118 : bool after)
8119 : {
8120 34952 : machine_mode mode = GET_MODE (mem);
8121 34952 : struct atomic_op_functions optab;
8122 34952 : rtx result;
8123 34952 : bool unused_result = (target == const0_rtx);
8124 :
8125 34952 : get_atomic_op_for_code (&optab, code);
8126 :
8127 : /* Check to see if there are any better instructions. */
8128 34952 : result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
8129 34952 : if (result)
8130 : return result;
8131 :
8132 : /* Check for the case where the result isn't used and try those patterns. */
8133 34460 : if (unused_result)
8134 : {
8135 : /* Try the memory model variant first. */
8136 14317 : result = maybe_emit_op (&optab, target, mem, val, true, model, true);
8137 14317 : if (result)
8138 : return result;
8139 :
8140 : /* Next try the old style withuot a memory model. */
8141 359 : result = maybe_emit_op (&optab, target, mem, val, false, model, true);
8142 359 : if (result)
8143 : return result;
8144 :
8145 : /* There is no no-result pattern, so try patterns with a result. */
8146 : target = NULL_RTX;
8147 : }
8148 :
8149 : /* Try the __atomic version. */
8150 20502 : result = maybe_emit_op (&optab, target, mem, val, true, model, after);
8151 20502 : if (result)
8152 : return result;
8153 :
8154 : /* Try the older __sync version. */
8155 11934 : result = maybe_emit_op (&optab, target, mem, val, false, model, after);
8156 11934 : if (result)
8157 : return result;
8158 :
8159 : /* If the fetch value can be calculated from the other variation of fetch,
8160 : try that operation. */
8161 11934 : if (after || unused_result || optab.reverse_code != UNKNOWN)
8162 : {
8163 10642 : rtx_insn *start = get_last_insn ();
8164 : /* Force val into a register if it could change value when the
8165 : atomic insn updates mem. */
8166 10642 : if (!unused_result && reg_overlap_mentioned_p (mem, val))
8167 103 : val = force_reg (mode, val);
8168 :
8169 : /* Try the __atomic version, then the older __sync version. */
8170 10642 : result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
8171 10642 : if (!result)
8172 6798 : result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
8173 :
8174 6798 : if (result)
8175 : {
8176 : /* If the result isn't used, no need to do compensation code. */
8177 3844 : if (unused_result)
8178 : return result;
8179 :
8180 : /* Issue compensation code. Fetch_after == fetch_before OP val.
8181 : Fetch_before == after REVERSE_OP val. */
8182 3844 : if (!after)
8183 0 : code = optab.reverse_code;
8184 3844 : if (code == NOT)
8185 : {
8186 0 : result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
8187 : true, OPTAB_LIB_WIDEN);
8188 0 : result = expand_simple_unop (mode, NOT, result, target, true);
8189 : }
8190 : else
8191 3844 : result = expand_simple_binop (mode, code, result, val, target,
8192 : true, OPTAB_LIB_WIDEN);
8193 : return result;
8194 : }
8195 : else
8196 6798 : delete_insns_since (start);
8197 : }
8198 :
8199 : /* No direct opcode can be generated. */
8200 : return NULL_RTX;
8201 : }
8202 :
8203 :
8204 :
8205 : /* This function expands an atomic fetch_OP or OP_fetch operation:
8206 : TARGET is an option place to stick the return value. const0_rtx indicates
8207 : the result is unused.
8208 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
8209 : CODE is the operation being performed (OP)
8210 : MEMMODEL is the memory model variant to use.
8211 : AFTER is true to return the result of the operation (OP_fetch).
8212 : AFTER is false to return the value before the operation (fetch_OP). */
8213 : rtx
8214 32560 : expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
8215 : enum memmodel model, bool after)
8216 : {
8217 32560 : machine_mode mode = GET_MODE (mem);
8218 32560 : rtx result;
8219 32560 : bool unused_result = (target == const0_rtx);
8220 :
8221 : /* If loads are not atomic for the required size and we are not called to
8222 : provide a __sync builtin, do not do anything so that we stay consistent
8223 : with atomic loads of the same size. */
8224 32560 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
8225 : return NULL_RTX;
8226 :
8227 31945 : result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
8228 : after);
8229 :
8230 31945 : if (result)
8231 : return result;
8232 :
8233 : /* Add/sub can be implemented by doing the reverse operation with -(val). */
8234 8051 : if (code == PLUS || code == MINUS)
8235 : {
8236 3007 : rtx tmp;
8237 3007 : enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
8238 :
8239 3007 : start_sequence ();
8240 3007 : tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
8241 3007 : result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
8242 : model, after);
8243 3007 : if (result)
8244 : {
8245 : /* PLUS worked so emit the insns and return. */
8246 2968 : tmp = end_sequence ();
8247 2968 : emit_insn (tmp);
8248 2968 : return result;
8249 : }
8250 :
8251 : /* PLUS did not work, so throw away the negation code and continue. */
8252 39 : end_sequence ();
8253 : }
8254 :
8255 : /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
8256 5083 : if (!can_compare_and_swap_p (mode, false))
8257 : {
8258 2 : rtx libfunc;
8259 2 : bool fixup = false;
8260 2 : enum rtx_code orig_code = code;
8261 2 : struct atomic_op_functions optab;
8262 :
8263 2 : get_atomic_op_for_code (&optab, code);
8264 2 : libfunc = optab_libfunc (after ? optab.fetch_after
8265 : : optab.fetch_before, mode);
8266 2 : if (libfunc == NULL
8267 2 : && (after || unused_result || optab.reverse_code != UNKNOWN))
8268 : {
8269 2 : fixup = true;
8270 2 : if (!after)
8271 1 : code = optab.reverse_code;
8272 2 : libfunc = optab_libfunc (after ? optab.fetch_before
8273 : : optab.fetch_after, mode);
8274 : }
8275 2 : if (libfunc != NULL)
8276 : {
8277 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
8278 0 : result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
8279 : addr, ptr_mode, val, mode);
8280 :
8281 0 : if (!unused_result && fixup)
8282 0 : result = expand_simple_binop (mode, code, result, val, target,
8283 : true, OPTAB_LIB_WIDEN);
8284 0 : return result;
8285 : }
8286 :
8287 : /* We need the original code for any further attempts. */
8288 2 : code = orig_code;
8289 : }
8290 :
8291 : /* If nothing else has succeeded, default to a compare and swap loop. */
8292 5083 : if (can_compare_and_swap_p (mode, true))
8293 : {
8294 5081 : rtx_insn *insn;
8295 5081 : rtx t0 = gen_reg_rtx (mode), t1;
8296 5081 : rtx_insn *start = get_last_insn ();
8297 : /* Force val into a register if it could change value when the
8298 : atomic insn updates mem. */
8299 5081 : if (reg_overlap_mentioned_p (mem, val))
8300 108 : val = force_reg (mode, val);
8301 :
8302 5081 : start_sequence ();
8303 :
8304 : /* If the result is used, get a register for it. */
8305 5081 : if (!unused_result)
8306 : {
8307 4745 : if (!target || !register_operand (target, mode))
8308 12 : target = gen_reg_rtx (mode);
8309 : /* If fetch_before, copy the value now. */
8310 4745 : if (!after)
8311 2080 : emit_move_insn (target, t0);
8312 : }
8313 : else
8314 336 : target = const0_rtx;
8315 :
8316 5081 : t1 = t0;
8317 5081 : if (code == NOT)
8318 : {
8319 693 : t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
8320 : true, OPTAB_LIB_WIDEN);
8321 693 : t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
8322 : }
8323 : else
8324 4388 : t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
8325 : OPTAB_LIB_WIDEN);
8326 :
8327 : /* For after, copy the value now. */
8328 5081 : if (!unused_result && after)
8329 2665 : emit_move_insn (target, t1);
8330 5081 : insn = end_sequence ();
8331 :
8332 5081 : if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
8333 : return target;
8334 : else
8335 0 : delete_insns_since (start);
8336 : }
8337 :
8338 : return NULL_RTX;
8339 : }
8340 :
8341 : /* Return true if OPERAND is suitable for operand number OPNO of
8342 : instruction ICODE. */
8343 :
8344 : bool
8345 156573004 : insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
8346 : {
8347 156573004 : return (!insn_data[(int) icode].operand[opno].predicate
8348 311608574 : || (insn_data[(int) icode].operand[opno].predicate
8349 155035570 : (operand, insn_data[(int) icode].operand[opno].mode)));
8350 : }
8351 :
8352 : /* TARGET is a target of a multiword operation that we are going to
8353 : implement as a series of word-mode operations. Return true if
8354 : TARGET is suitable for this purpose. */
8355 :
8356 : bool
8357 201 : valid_multiword_target_p (rtx target)
8358 : {
8359 201 : machine_mode mode;
8360 201 : int i, size;
8361 :
8362 201 : mode = GET_MODE (target);
8363 402 : if (!GET_MODE_SIZE (mode).is_constant (&size))
8364 : return false;
8365 945 : for (i = 0; i < size; i += UNITS_PER_WORD)
8366 418 : if (!validate_subreg (word_mode, mode, target, i))
8367 : return false;
8368 : return true;
8369 : }
8370 :
8371 : /* Make OP describe an input operand that has value INTVAL and that has
8372 : no inherent mode. This function should only be used for operands that
8373 : are always expand-time constants. The backend may request that INTVAL
8374 : be copied into a different kind of rtx, but it must specify the mode
8375 : of that rtx if so. */
8376 :
8377 : void
8378 2066468 : create_integer_operand (class expand_operand *op, poly_int64 intval)
8379 : {
8380 2066468 : create_expand_operand (op, EXPAND_INTEGER,
8381 : gen_int_mode (intval, MAX_MODE_INT),
8382 : VOIDmode, false, intval);
8383 2066468 : }
8384 :
8385 : /* Like maybe_legitimize_operand, but do not change the code of the
8386 : current rtx value. */
8387 :
8388 : static bool
8389 52485898 : maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
8390 : class expand_operand *op)
8391 : {
8392 : /* See if the operand matches in its current form. */
8393 52485898 : if (insn_operand_matches (icode, opno, op->value))
8394 : return true;
8395 :
8396 : /* If the operand is a memory whose address has no side effects,
8397 : try forcing the address into a non-virtual pseudo register.
8398 : The check for side effects is important because copy_to_mode_reg
8399 : cannot handle things like auto-modified addresses. */
8400 1614978 : if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
8401 : {
8402 13098 : rtx addr, mem;
8403 :
8404 13098 : mem = op->value;
8405 13098 : addr = XEXP (mem, 0);
8406 1005 : if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
8407 13904 : && !side_effects_p (addr))
8408 : {
8409 8408 : rtx_insn *last;
8410 8408 : machine_mode mode;
8411 :
8412 8408 : last = get_last_insn ();
8413 8408 : mode = get_address_mode (mem);
8414 8408 : mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
8415 8408 : if (insn_operand_matches (icode, opno, mem))
8416 : {
8417 0 : op->value = mem;
8418 0 : return true;
8419 : }
8420 8408 : delete_insns_since (last);
8421 : }
8422 : }
8423 :
8424 : return false;
8425 : }
8426 :
8427 : /* Try to make OP match operand OPNO of instruction ICODE. Return true
8428 : on success, storing the new operand value back in OP. */
8429 :
8430 : static bool
8431 60266591 : maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
8432 : class expand_operand *op)
8433 : {
8434 60266591 : machine_mode mode, imode, tmode;
8435 :
8436 60266591 : mode = op->mode;
8437 60266591 : switch (op->type)
8438 : {
8439 4547344 : case EXPAND_FIXED:
8440 4547344 : {
8441 4547344 : temporary_volatile_ok v (true);
8442 4547344 : return maybe_legitimize_operand_same_code (icode, opno, op);
8443 4547344 : }
8444 :
8445 19383427 : case EXPAND_OUTPUT:
8446 19383427 : gcc_assert (mode != VOIDmode);
8447 19383427 : if (op->value
8448 12026098 : && op->value != const0_rtx
8449 12025678 : && GET_MODE (op->value) == mode
8450 31373542 : && maybe_legitimize_operand_same_code (icode, opno, op))
8451 : return true;
8452 :
8453 7517539 : op->value = gen_reg_rtx (mode);
8454 7517539 : op->target = 0;
8455 7517539 : break;
8456 :
8457 35948439 : case EXPAND_INPUT:
8458 35948439 : input:
8459 35948439 : gcc_assert (mode != VOIDmode);
8460 35948439 : gcc_assert (GET_MODE (op->value) == VOIDmode
8461 : || GET_MODE (op->value) == mode);
8462 35948439 : if (maybe_legitimize_operand_same_code (icode, opno, op))
8463 : return true;
8464 :
8465 1344826 : op->value = copy_to_mode_reg (mode, op->value);
8466 1344826 : break;
8467 :
8468 204957 : case EXPAND_CONVERT_TO:
8469 204957 : gcc_assert (mode != VOIDmode);
8470 204957 : op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8471 204957 : goto input;
8472 :
8473 665190 : case EXPAND_CONVERT_FROM:
8474 665190 : if (GET_MODE (op->value) != VOIDmode)
8475 455114 : mode = GET_MODE (op->value);
8476 : else
8477 : /* The caller must tell us what mode this value has. */
8478 210076 : gcc_assert (mode != VOIDmode);
8479 :
8480 665190 : imode = insn_data[(int) icode].operand[opno].mode;
8481 665190 : tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
8482 665190 : ? GET_MODE_INNER (imode) : imode);
8483 665190 : if (tmode != VOIDmode && tmode != mode)
8484 : {
8485 1760 : op->value = convert_modes (tmode, mode, op->value, op->unsigned_p);
8486 1760 : mode = tmode;
8487 : }
8488 665190 : if (imode != VOIDmode && imode != mode)
8489 : {
8490 46 : gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
8491 46 : op->value = expand_vector_broadcast (imode, op->value);
8492 46 : mode = imode;
8493 : }
8494 665190 : goto input;
8495 :
8496 3371 : case EXPAND_ADDRESS:
8497 3371 : op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
8498 : op->value);
8499 3371 : goto input;
8500 :
8501 1692516 : case EXPAND_INTEGER:
8502 1692516 : mode = insn_data[(int) icode].operand[opno].mode;
8503 1692516 : if (mode != VOIDmode
8504 1692516 : && known_eq (trunc_int_for_mode (op->int_value, mode),
8505 : op->int_value))
8506 : {
8507 1305135 : op->value = gen_int_mode (op->int_value, mode);
8508 1305135 : goto input;
8509 : }
8510 : break;
8511 :
8512 249 : case EXPAND_UNDEFINED_INPUT:
8513 : /* See if the predicate accepts a SCRATCH rtx, which in this context
8514 : indicates an undefined value. Use an uninitialized register if not. */
8515 249 : if (!insn_operand_matches (icode, opno, op->value))
8516 : {
8517 249 : op->value = gen_reg_rtx (op->mode);
8518 249 : goto input;
8519 : }
8520 : return true;
8521 : }
8522 9249746 : return insn_operand_matches (icode, opno, op->value);
8523 : }
8524 :
8525 : /* Make OP describe an input operand that should have the same value
8526 : as VALUE, after any mode conversion that the target might request.
8527 : TYPE is the type of VALUE. */
8528 :
8529 : void
8530 2887 : create_convert_operand_from_type (class expand_operand *op,
8531 : rtx value, tree type)
8532 : {
8533 2887 : create_convert_operand_from (op, value, TYPE_MODE (type),
8534 2887 : TYPE_UNSIGNED (type));
8535 2887 : }
8536 :
8537 : /* Return true if the requirements on operands OP1 and OP2 of instruction
8538 : ICODE are similar enough for the result of legitimizing OP1 to be
8539 : reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
8540 : with OP1 and OP2 respectively. */
8541 :
8542 : static inline bool
8543 67108211 : can_reuse_operands_p (enum insn_code icode,
8544 : unsigned int opno1, unsigned int opno2,
8545 : const class expand_operand *op1,
8546 : const class expand_operand *op2)
8547 : {
8548 : /* Check requirements that are common to all types. */
8549 67108211 : if (op1->type != op2->type
8550 20821684 : || op1->mode != op2->mode
8551 19286879 : || (insn_data[(int) icode].operand[opno1].mode
8552 19286879 : != insn_data[(int) icode].operand[opno2].mode))
8553 : return false;
8554 :
8555 : /* Check the requirements for specific types. */
8556 17342488 : switch (op1->type)
8557 : {
8558 : case EXPAND_OUTPUT:
8559 : case EXPAND_UNDEFINED_INPUT:
8560 : /* Outputs and undefined inputs must remain distinct. */
8561 : return false;
8562 :
8563 : case EXPAND_FIXED:
8564 : case EXPAND_INPUT:
8565 : case EXPAND_ADDRESS:
8566 : case EXPAND_INTEGER:
8567 : return true;
8568 :
8569 118179 : case EXPAND_CONVERT_TO:
8570 118179 : case EXPAND_CONVERT_FROM:
8571 118179 : return op1->unsigned_p == op2->unsigned_p;
8572 : }
8573 0 : gcc_unreachable ();
8574 : }
8575 :
8576 : /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8577 : of instruction ICODE. Return true on success, leaving the new operand
8578 : values in the OPS themselves. Emit no code on failure. */
8579 :
8580 : bool
8581 20141379 : maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8582 : unsigned int nops, class expand_operand *ops)
8583 : {
8584 20141379 : rtx_insn *last = get_last_insn ();
8585 20141379 : rtx *orig_values = XALLOCAVEC (rtx, nops);
8586 80648741 : for (unsigned int i = 0; i < nops; i++)
8587 : {
8588 60660805 : orig_values[i] = ops[i].value;
8589 :
8590 : /* First try reusing the result of an earlier legitimization.
8591 : This avoids duplicate rtl and ensures that tied operands
8592 : remain tied.
8593 :
8594 : This search is linear, but NOPS is bounded at compile time
8595 : to a small number (current a single digit). */
8596 60660805 : unsigned int j = 0;
8597 127374802 : for (; j < i; ++j)
8598 67108211 : if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
8599 17321938 : && rtx_equal_p (orig_values[j], orig_values[i])
8600 413025 : && ops[j].value
8601 67502425 : && insn_operand_matches (icode, opno + i, ops[j].value))
8602 : {
8603 394214 : ops[i].value = copy_rtx (ops[j].value);
8604 394214 : break;
8605 : }
8606 :
8607 : /* Otherwise try legitimizing the operand on its own. */
8608 60660805 : if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
8609 : {
8610 153443 : delete_insns_since (last);
8611 153443 : return false;
8612 : }
8613 : }
8614 : return true;
8615 : }
8616 :
8617 : /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8618 : as its operands. Return the instruction pattern on success,
8619 : and emit any necessary set-up code. Return null and emit no
8620 : code on failure. */
8621 :
8622 : rtx_insn *
8623 20115156 : maybe_gen_insn (enum insn_code icode, unsigned int nops,
8624 : class expand_operand *ops)
8625 : {
8626 20115156 : gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8627 20115156 : if (!maybe_legitimize_operands (icode, 0, nops, ops))
8628 : return NULL;
8629 :
8630 19961713 : switch (nops)
8631 : {
8632 0 : case 0:
8633 0 : return GEN_FCN (icode) ();
8634 1316 : case 1:
8635 1316 : return GEN_FCN (icode) (ops[0].value);
8636 2883917 : case 2:
8637 2883917 : return GEN_FCN (icode) (ops[0].value, ops[1].value);
8638 14946197 : case 3:
8639 14946197 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8640 1740154 : case 4:
8641 1740154 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8642 1740154 : ops[3].value);
8643 191371 : case 5:
8644 191371 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8645 191371 : ops[3].value, ops[4].value);
8646 262 : case 6:
8647 262 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8648 262 : ops[3].value, ops[4].value, ops[5].value);
8649 0 : case 7:
8650 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8651 0 : ops[3].value, ops[4].value, ops[5].value,
8652 0 : ops[6].value);
8653 26827 : case 8:
8654 26827 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8655 26827 : ops[3].value, ops[4].value, ops[5].value,
8656 26827 : ops[6].value, ops[7].value);
8657 171669 : case 9:
8658 171669 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8659 171669 : ops[3].value, ops[4].value, ops[5].value,
8660 171669 : ops[6].value, ops[7].value, ops[8].value);
8661 0 : case 10:
8662 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8663 0 : ops[3].value, ops[4].value, ops[5].value,
8664 0 : ops[6].value, ops[7].value, ops[8].value,
8665 0 : ops[9].value);
8666 0 : case 11:
8667 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8668 0 : ops[3].value, ops[4].value, ops[5].value,
8669 0 : ops[6].value, ops[7].value, ops[8].value,
8670 0 : ops[9].value, ops[10].value);
8671 : }
8672 0 : gcc_unreachable ();
8673 : }
8674 :
8675 : /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8676 : as its operands. Return true on success and emit no code on failure. */
8677 :
8678 : bool
8679 3074159 : maybe_expand_insn (enum insn_code icode, unsigned int nops,
8680 : class expand_operand *ops)
8681 : {
8682 3074159 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8683 3074159 : if (pat)
8684 : {
8685 2253543 : emit_insn (pat);
8686 2253543 : return true;
8687 : }
8688 : return false;
8689 : }
8690 :
8691 : /* Like maybe_expand_insn, but for jumps. */
8692 :
8693 : bool
8694 1313 : maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8695 : class expand_operand *ops)
8696 : {
8697 1313 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8698 1313 : if (pat)
8699 : {
8700 1313 : emit_jump_insn (pat);
8701 1313 : return true;
8702 : }
8703 : return false;
8704 : }
8705 :
8706 : /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8707 : as its operands. */
8708 :
8709 : void
8710 653783 : expand_insn (enum insn_code icode, unsigned int nops,
8711 : class expand_operand *ops)
8712 : {
8713 653783 : if (!maybe_expand_insn (icode, nops, ops))
8714 0 : gcc_unreachable ();
8715 653783 : }
8716 :
8717 : /* Like expand_insn, but for jumps. */
8718 :
8719 : void
8720 1313 : expand_jump_insn (enum insn_code icode, unsigned int nops,
8721 : class expand_operand *ops)
8722 : {
8723 1313 : if (!maybe_expand_jump_insn (icode, nops, ops))
8724 0 : gcc_unreachable ();
8725 1313 : }
|
