Line data Source code
1 : /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
2 : Copyright (C) 1987-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify it under
7 : the terms of the GNU General Public License as published by the Free
8 : Software Foundation; either version 3, or (at your option) any later
9 : version.
10 :
11 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
12 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 : for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 :
21 : #include "config.h"
22 : #include "system.h"
23 : #include "coretypes.h"
24 : #include "backend.h"
25 : #include "target.h"
26 : #include "rtl.h"
27 : #include "tree.h"
28 : #include "memmodel.h"
29 : #include "predict.h"
30 : #include "tm_p.h"
31 : #include "optabs.h"
32 : #include "expmed.h"
33 : #include "emit-rtl.h"
34 : #include "recog.h"
35 : #include "diagnostic-core.h"
36 : #include "rtx-vector-builder.h"
37 :
38 : /* Include insn-config.h before expr.h so that HAVE_conditional_move
39 : is properly defined. */
40 : #include "stor-layout.h"
41 : #include "except.h"
42 : #include "dojump.h"
43 : #include "explow.h"
44 : #include "expr.h"
45 : #include "optabs-tree.h"
46 : #include "libfuncs.h"
47 : #include "internal-fn.h"
48 : #include "langhooks.h"
49 : #include "gimple.h"
50 : #include "ssa.h"
51 : #include "tree-ssa-live.h"
52 : #include "tree-outof-ssa.h"
53 :
54 : static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
55 : machine_mode *);
56 : static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
57 : static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
58 :
59 : static rtx emit_conditional_move_1 (rtx, rtx, rtx, rtx, machine_mode);
60 :
61 : /* Debug facility for use in GDB. */
62 : void debug_optab_libfuncs (void);
63 :
64 : /* Add a REG_EQUAL note to the last insn in INSNS. TARGET is being set to
65 : the result of operation CODE applied to OP0 (and OP1 if it is a binary
66 : operation). OP0_MODE is OP0's mode.
67 :
68 : If the last insn does not set TARGET, don't do anything, but return true.
69 :
70 : If the last insn or a previous insn sets TARGET and TARGET is one of OP0
71 : or OP1, don't add the REG_EQUAL note but return false. Our caller can then
72 : try again, ensuring that TARGET is not one of the operands. */
73 :
74 : static bool
75 1218734 : add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
76 : rtx op1, machine_mode op0_mode)
77 : {
78 1218734 : rtx_insn *last_insn;
79 1218734 : rtx set;
80 1218734 : rtx note;
81 :
82 2437468 : gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
83 :
84 1218734 : if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
85 : && GET_RTX_CLASS (code) != RTX_BIN_ARITH
86 : && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
87 : && GET_RTX_CLASS (code) != RTX_COMPARE
88 : && GET_RTX_CLASS (code) != RTX_UNARY)
89 : return true;
90 :
91 1211579 : if (GET_CODE (target) == ZERO_EXTRACT)
92 : return true;
93 :
94 : for (last_insn = insns;
95 2670129 : NEXT_INSN (last_insn) != NULL_RTX;
96 : last_insn = NEXT_INSN (last_insn))
97 : ;
98 :
99 : /* If TARGET is in OP0 or OP1, punt. We'd end up with a note referencing
100 : a value changing in the insn, so the note would be invalid for CSE. */
101 1211579 : if (reg_overlap_mentioned_p (target, op0)
102 1211579 : || (op1 && reg_overlap_mentioned_p (target, op1)))
103 : {
104 56283 : if (MEM_P (target)
105 56283 : && (rtx_equal_p (target, op0)
106 42057 : || (op1 && rtx_equal_p (target, op1))))
107 : {
108 : /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
109 : over expanding it as temp = MEM op X, MEM = temp. If the target
110 : supports MEM = MEM op X instructions, it is sometimes too hard
111 : to reconstruct that form later, especially if X is also a memory,
112 : and due to multiple occurrences of addresses the address might
113 : be forced into register unnecessarily.
114 : Note that not emitting the REG_EQUIV note might inhibit
115 : CSE in some cases. */
116 3435 : set = single_set (last_insn);
117 3435 : if (set
118 3435 : && GET_CODE (SET_SRC (set)) == code
119 3424 : && MEM_P (SET_DEST (set))
120 6859 : && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
121 0 : || (op1 && rtx_equal_p (SET_DEST (set),
122 0 : XEXP (SET_SRC (set), 1)))))
123 3424 : return true;
124 : }
125 52859 : return false;
126 : }
127 :
128 1155296 : set = set_for_reg_notes (last_insn);
129 1155296 : if (set == NULL_RTX)
130 : return true;
131 :
132 703079 : if (! rtx_equal_p (SET_DEST (set), target)
133 : /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it. */
134 703079 : && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
135 0 : || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
136 375 : return true;
137 :
138 702704 : if (GET_RTX_CLASS (code) == RTX_UNARY)
139 40613 : switch (code)
140 : {
141 3054 : case FFS:
142 3054 : case CLZ:
143 3054 : case CTZ:
144 3054 : case CLRSB:
145 3054 : case POPCOUNT:
146 3054 : case PARITY:
147 3054 : case BSWAP:
148 3054 : if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
149 : {
150 252 : note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
151 252 : if (GET_MODE_UNIT_SIZE (op0_mode)
152 504 : > GET_MODE_UNIT_SIZE (GET_MODE (target)))
153 252 : note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
154 : note, op0_mode);
155 : else
156 0 : note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
157 : note, op0_mode);
158 : break;
159 : }
160 : /* FALLTHRU */
161 40361 : default:
162 40361 : note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
163 40361 : break;
164 : }
165 : else
166 662091 : note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
167 :
168 702704 : set_unique_reg_note (last_insn, REG_EQUAL, note);
169 :
170 702704 : return true;
171 : }
172 :
173 : /* Given two input operands, OP0 and OP1, determine what the correct from_mode
174 : for a widening operation would be. In most cases this would be OP0, but if
175 : that's a constant it'll be VOIDmode, which isn't useful. */
176 :
177 : static machine_mode
178 31560 : widened_mode (machine_mode to_mode, rtx op0, rtx op1)
179 : {
180 31560 : machine_mode m0 = GET_MODE (op0);
181 31560 : machine_mode m1 = GET_MODE (op1);
182 31560 : machine_mode result;
183 :
184 31560 : if (m0 == VOIDmode && m1 == VOIDmode)
185 : return to_mode;
186 63120 : else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
187 : result = m1;
188 : else
189 : result = m0;
190 :
191 94680 : if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
192 0 : return to_mode;
193 :
194 : return result;
195 : }
196 :
197 : /* Widen OP to MODE and return the rtx for the widened operand. UNSIGNEDP
198 : says whether OP is signed or unsigned. NO_EXTEND is true if we need
199 : not actually do a sign-extend or zero-extend, but can leave the
200 : higher-order bits of the result rtx undefined, for example, in the case
201 : of logical operations, but not right shifts. */
202 :
203 : static rtx
204 539 : widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
205 : int unsignedp, bool no_extend)
206 : {
207 539 : rtx result;
208 539 : scalar_int_mode int_mode;
209 :
210 : /* If we don't have to extend and this is a constant, return it. */
211 539 : if (no_extend && GET_MODE (op) == VOIDmode)
212 : return op;
213 :
214 : /* If we must extend do so. If OP is a SUBREG for a promoted object, also
215 : extend since it will be more efficient to do so unless the signedness of
216 : a promoted object differs from our extension. */
217 7 : if (! no_extend
218 539 : || !is_a <scalar_int_mode> (mode, &int_mode)
219 7 : || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
220 0 : && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
221 532 : return convert_modes (mode, oldmode, op, unsignedp);
222 :
223 : /* If MODE is no wider than a single word, we return a lowpart or paradoxical
224 : SUBREG. */
225 14 : if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
226 7 : return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
227 :
228 : /* Otherwise, get an object of MODE, clobber it, and set the low-order
229 : part to OP. */
230 :
231 0 : result = gen_reg_rtx (int_mode);
232 0 : emit_clobber (result);
233 0 : emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
234 0 : return result;
235 : }
236 :
237 : /* Expand vector widening operations.
238 :
239 : There are two different classes of operations handled here:
240 : 1) Operations whose result is wider than all the arguments to the operation.
241 : Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
242 : In this case OP0 and optionally OP1 would be initialized,
243 : but WIDE_OP wouldn't (not relevant for this case).
244 : 2) Operations whose result is of the same size as the last argument to the
245 : operation, but wider than all the other arguments to the operation.
246 : Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
247 : In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
248 :
249 : E.g, when called to expand the following operations, this is how
250 : the arguments will be initialized:
251 : nops OP0 OP1 WIDE_OP
252 : widening-sum 2 oprnd0 - oprnd1
253 : widening-dot-product 3 oprnd0 oprnd1 oprnd2
254 : widening-mult 2 oprnd0 oprnd1 -
255 : type-promotion (vec-unpack) 1 oprnd0 - - */
256 :
257 : rtx
258 20955 : expand_widen_pattern_expr (const_sepops ops, rtx op0, rtx op1, rtx wide_op,
259 : rtx target, int unsignedp)
260 : {
261 20955 : class expand_operand eops[4];
262 20955 : tree oprnd0, oprnd1, oprnd2;
263 20955 : machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
264 20955 : optab widen_pattern_optab;
265 20955 : enum insn_code icode;
266 20955 : int nops = TREE_CODE_LENGTH (ops->code);
267 20955 : int op;
268 20955 : bool sbool = false;
269 :
270 20955 : oprnd0 = ops->op0;
271 20955 : oprnd1 = nops >= 2 ? ops->op1 : NULL_TREE;
272 1525 : oprnd2 = nops >= 3 ? ops->op2 : NULL_TREE;
273 :
274 20955 : tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
275 20955 : if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
276 20955 : || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
277 : /* The sign is from the result type rather than operand's type
278 : for these ops. */
279 34 : widen_pattern_optab
280 34 : = optab_for_tree_code (ops->code, ops->type, optab_default);
281 20921 : else if ((ops->code == VEC_UNPACK_HI_EXPR
282 20921 : || ops->code == VEC_UNPACK_LO_EXPR)
283 17668 : && VECTOR_BOOLEAN_TYPE_P (ops->type)
284 4534 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
285 4534 : && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
286 21405 : && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
287 : {
288 : /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
289 : the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
290 : vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
291 : the pattern number of elements in the wider vector. */
292 484 : widen_pattern_optab
293 239 : = (ops->code == VEC_UNPACK_HI_EXPR
294 484 : ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
295 : sbool = true;
296 : }
297 20437 : else if (ops->code == DOT_PROD_EXPR)
298 : {
299 419 : enum optab_subtype subtype = optab_default;
300 419 : signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0));
301 419 : signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1));
302 419 : if (sign1 == sign2)
303 : ;
304 79 : else if (sign1 == SIGNED && sign2 == UNSIGNED)
305 : {
306 : subtype = optab_vector_mixed_sign;
307 : /* Same as optab_vector_mixed_sign but flip the operands. */
308 : std::swap (op0, op1);
309 : }
310 60 : else if (sign1 == UNSIGNED && sign2 == SIGNED)
311 : subtype = optab_vector_mixed_sign;
312 : else
313 0 : gcc_unreachable ();
314 :
315 419 : widen_pattern_optab
316 419 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype);
317 : }
318 : else
319 20018 : widen_pattern_optab
320 20018 : = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
321 20955 : if (ops->code == WIDEN_MULT_PLUS_EXPR
322 20955 : || ops->code == WIDEN_MULT_MINUS_EXPR
323 20955 : || ops->code == DOT_PROD_EXPR)
324 419 : icode = find_widening_optab_handler (widen_pattern_optab,
325 : TYPE_MODE (TREE_TYPE (ops->op2)),
326 : tmode0);
327 20536 : else if (ops->code == WIDEN_SUM_EXPR)
328 0 : icode = find_widening_optab_handler (widen_pattern_optab,
329 : TYPE_MODE (TREE_TYPE (ops->op1)),
330 : tmode0);
331 : else
332 20536 : icode = optab_handler (widen_pattern_optab, tmode0);
333 20955 : gcc_assert (icode != CODE_FOR_nothing);
334 :
335 20955 : if (nops >= 2)
336 1525 : tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
337 19430 : else if (sbool)
338 : {
339 484 : nops = 2;
340 484 : op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
341 484 : tmode1 = tmode0;
342 : }
343 :
344 : /* The last operand is of a wider mode than the rest of the operands. */
345 20955 : if (nops == 2)
346 : wmode = tmode1;
347 19478 : else if (nops == 3)
348 : {
349 532 : gcc_assert (tmode1 == tmode0);
350 532 : gcc_assert (op1);
351 532 : wmode = TYPE_MODE (TREE_TYPE (oprnd2));
352 : }
353 :
354 20955 : op = 0;
355 20955 : create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
356 20955 : create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
357 20955 : if (op1)
358 2009 : create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
359 20955 : if (wide_op)
360 532 : create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
361 20955 : expand_insn (icode, op, eops);
362 20955 : return eops[0].value;
363 : }
364 :
365 : /* Generate code to perform an operation specified by TERNARY_OPTAB
366 : on operands OP0, OP1 and OP2, with result having machine-mode MODE.
367 :
368 : UNSIGNEDP is for the case where we have to widen the operands
369 : to perform the operation. It says to use zero-extension.
370 :
371 : If TARGET is nonzero, the value
372 : is generated there, if it is convenient to do so.
373 : In all cases an rtx is returned for the locus of the value;
374 : this may or may not be TARGET. */
375 :
376 : rtx
377 0 : expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
378 : rtx op1, rtx op2, rtx target, int unsignedp)
379 : {
380 0 : class expand_operand ops[4];
381 0 : enum insn_code icode = optab_handler (ternary_optab, mode);
382 :
383 0 : gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
384 :
385 0 : create_output_operand (&ops[0], target, mode);
386 0 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
387 0 : create_convert_operand_from (&ops[2], op1, mode, unsignedp);
388 0 : create_convert_operand_from (&ops[3], op2, mode, unsignedp);
389 0 : expand_insn (icode, 4, ops);
390 0 : return ops[0].value;
391 : }
392 :
393 :
394 : /* Like expand_binop, but return a constant rtx if the result can be
395 : calculated at compile time. The arguments and return value are
396 : otherwise the same as for expand_binop. */
397 :
398 : rtx
399 1113 : simplify_expand_binop (machine_mode mode, optab binoptab,
400 : rtx op0, rtx op1, rtx target, int unsignedp,
401 : enum optab_methods methods)
402 : {
403 1113 : if (CONSTANT_P (op0) && CONSTANT_P (op1))
404 : {
405 0 : rtx x = simplify_binary_operation (optab_to_code (binoptab),
406 : mode, op0, op1);
407 0 : if (x)
408 : return x;
409 : }
410 :
411 1113 : return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
412 : }
413 :
414 : /* Like simplify_expand_binop, but always put the result in TARGET.
415 : Return true if the expansion succeeded. */
416 :
417 : bool
418 1113 : force_expand_binop (machine_mode mode, optab binoptab,
419 : rtx op0, rtx op1, rtx target, int unsignedp,
420 : enum optab_methods methods)
421 : {
422 1113 : rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
423 : target, unsignedp, methods);
424 1113 : if (x == 0)
425 : return false;
426 1113 : if (x != target)
427 0 : emit_move_insn (target, x);
428 : return true;
429 : }
430 :
431 : /* Create a new vector value in VMODE with all elements set to OP. If OP
432 : is not a constant, the mode of it must be the element mode of VMODE
433 : (if the element is BImode, additionally OP is allowed to be in QImode).
434 : If OP is a constant, then the return value will be a constant. */
435 :
436 : rtx
437 58 : expand_vector_broadcast (machine_mode vmode, rtx op)
438 : {
439 58 : int n;
440 58 : rtvec vec;
441 :
442 58 : gcc_checking_assert (VECTOR_MODE_P (vmode));
443 58 : gcc_checking_assert (CONST_INT_P (op)
444 : || GET_MODE_INNER (vmode) == GET_MODE (op)
445 : || (GET_MODE_INNER (vmode) == BImode
446 : && GET_MODE (op) == QImode));
447 :
448 58 : if (valid_for_const_vector_p (vmode, op))
449 58 : return gen_const_vec_duplicate (vmode, op);
450 :
451 0 : insn_code icode = optab_handler (vec_duplicate_optab, vmode);
452 0 : if (icode != CODE_FOR_nothing)
453 : {
454 0 : class expand_operand ops[2];
455 0 : create_output_operand (&ops[0], NULL_RTX, vmode);
456 0 : create_input_operand (&ops[1], op, GET_MODE (op));
457 0 : expand_insn (icode, 2, ops);
458 0 : return ops[0].value;
459 : }
460 :
461 0 : if (!GET_MODE_NUNITS (vmode).is_constant (&n))
462 : return NULL;
463 :
464 : /* ??? If the target doesn't have a vec_init, then we have no easy way
465 : of performing this operation. Most of this sort of generic support
466 : is hidden away in the vector lowering support in gimple. */
467 0 : icode = convert_optab_handler (vec_init_optab, vmode,
468 0 : GET_MODE_INNER (vmode));
469 0 : if (icode == CODE_FOR_nothing)
470 : return NULL;
471 :
472 0 : vec = rtvec_alloc (n);
473 0 : for (int i = 0; i < n; ++i)
474 0 : RTVEC_ELT (vec, i) = op;
475 0 : rtx ret = gen_reg_rtx (vmode);
476 0 : emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
477 :
478 0 : return ret;
479 : }
480 :
481 : /* This subroutine of expand_doubleword_shift handles the cases in which
482 : the effective shift value is >= BITS_PER_WORD. The arguments and return
483 : value are the same as for the parent routine, except that SUPERWORD_OP1
484 : is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
485 : INTO_TARGET may be null if the caller has decided to calculate it. */
486 :
487 : static bool
488 0 : expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
489 : rtx outof_target, rtx into_target,
490 : int unsignedp, enum optab_methods methods)
491 : {
492 0 : if (into_target != 0)
493 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
494 : into_target, unsignedp, methods))
495 : return false;
496 :
497 0 : if (outof_target != 0)
498 : {
499 : /* For a signed right shift, we must fill OUTOF_TARGET with copies
500 : of the sign bit, otherwise we must fill it with zeros. */
501 0 : if (binoptab != ashr_optab)
502 0 : emit_move_insn (outof_target, CONST0_RTX (word_mode));
503 : else
504 0 : if (!force_expand_binop (word_mode, binoptab, outof_input,
505 : gen_int_shift_amount (word_mode,
506 0 : BITS_PER_WORD - 1),
507 : outof_target, unsignedp, methods))
508 : return false;
509 : }
510 : return true;
511 : }
512 :
513 : /* This subroutine of expand_doubleword_shift handles the cases in which
514 : the effective shift value is < BITS_PER_WORD. The arguments and return
515 : value are the same as for the parent routine. */
516 :
517 : static bool
518 0 : expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
519 : rtx outof_input, rtx into_input, rtx op1,
520 : rtx outof_target, rtx into_target,
521 : int unsignedp, enum optab_methods methods,
522 : unsigned HOST_WIDE_INT shift_mask)
523 : {
524 0 : optab reverse_unsigned_shift, unsigned_shift;
525 0 : rtx tmp, carries;
526 :
527 0 : reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
528 0 : unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
529 :
530 : /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
531 : We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
532 : the opposite direction to BINOPTAB. */
533 0 : if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
534 : {
535 0 : carries = outof_input;
536 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
537 : op1_mode), op1_mode);
538 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
539 : 0, true, methods);
540 : }
541 : else
542 : {
543 : /* We must avoid shifting by BITS_PER_WORD bits since that is either
544 : the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
545 : has unknown behavior. Do a single shift first, then shift by the
546 : remainder. It's OK to use ~OP1 as the remainder if shift counts
547 : are truncated to the mode size. */
548 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
549 : outof_input, const1_rtx, 0,
550 : unsignedp, methods);
551 0 : if (carries == const0_rtx)
552 : tmp = const0_rtx;
553 0 : else if (shift_mask == BITS_PER_WORD - 1)
554 0 : tmp = expand_unop (op1_mode, one_cmpl_optab, op1, 0, true);
555 : else
556 : {
557 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
558 : op1_mode), op1_mode);
559 0 : tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
560 : 0, true, methods);
561 : }
562 : }
563 0 : if (tmp == 0 || carries == 0)
564 : return false;
565 0 : if (carries != const0_rtx && tmp != const0_rtx)
566 0 : carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
567 : carries, tmp, 0, unsignedp, methods);
568 0 : if (carries == 0)
569 : return false;
570 :
571 0 : if (into_input != const0_rtx)
572 : {
573 : /* Shift INTO_INPUT logically by OP1. This is the last use of
574 : INTO_INPUT so the result can go directly into INTO_TARGET if
575 : convenient. */
576 0 : tmp = simplify_expand_binop (word_mode, unsigned_shift, into_input,
577 : op1, into_target, unsignedp, methods);
578 0 : if (tmp == 0)
579 : return false;
580 :
581 : /* Now OR/PLUS in the bits carried over from OUTOF_INPUT. */
582 0 : if (!force_expand_binop (word_mode, add_optab, tmp, carries,
583 : into_target, unsignedp, methods))
584 : return false;
585 : }
586 : else
587 0 : emit_move_insn (into_target, carries);
588 :
589 : /* Use a standard word_mode shift for the out-of half. */
590 0 : if (outof_target != 0)
591 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
592 : outof_target, unsignedp, methods))
593 : return false;
594 :
595 : return true;
596 : }
597 :
598 :
599 : /* Try implementing expand_doubleword_shift using conditional moves.
600 : The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
601 : otherwise it is by >= BITS_PER_WORD. SUBWORD_OP1 and SUPERWORD_OP1
602 : are the shift counts to use in the former and latter case. All other
603 : arguments are the same as the parent routine. */
604 :
605 : static bool
606 0 : expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
607 : enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
608 : rtx outof_input, rtx into_input,
609 : rtx subword_op1, rtx superword_op1,
610 : rtx outof_target, rtx into_target,
611 : int unsignedp, enum optab_methods methods,
612 : unsigned HOST_WIDE_INT shift_mask)
613 : {
614 0 : rtx outof_superword, into_superword;
615 :
616 : /* Put the superword version of the output into OUTOF_SUPERWORD and
617 : INTO_SUPERWORD. */
618 0 : outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
619 0 : if (outof_target != 0 && subword_op1 == superword_op1)
620 : {
621 : /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
622 : OUTOF_TARGET, is the same as the value of INTO_SUPERWORD. */
623 0 : into_superword = outof_target;
624 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
625 : outof_superword, 0, unsignedp, methods))
626 : return false;
627 : }
628 : else
629 : {
630 0 : into_superword = gen_reg_rtx (word_mode);
631 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
632 : outof_superword, into_superword,
633 : unsignedp, methods))
634 : return false;
635 : }
636 :
637 : /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET. */
638 0 : if (!expand_subword_shift (op1_mode, binoptab,
639 : outof_input, into_input, subword_op1,
640 : outof_target, into_target,
641 : unsignedp, methods, shift_mask))
642 : return false;
643 :
644 : /* Select between them. Do the INTO half first because INTO_SUPERWORD
645 : might be the current value of OUTOF_TARGET. */
646 0 : if (!emit_conditional_move (into_target, { cmp_code, cmp1, cmp2, op1_mode },
647 : into_target, into_superword, word_mode, false))
648 : return false;
649 :
650 0 : if (outof_target != 0)
651 0 : if (!emit_conditional_move (outof_target,
652 0 : { cmp_code, cmp1, cmp2, op1_mode },
653 : outof_target, outof_superword,
654 : word_mode, false))
655 : return false;
656 :
657 : return true;
658 : }
659 :
660 : /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
661 : OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
662 : input operand; the shift moves bits in the direction OUTOF_INPUT->
663 : INTO_TARGET. OUTOF_TARGET and INTO_TARGET are the equivalent words
664 : of the target. OP1 is the shift count and OP1_MODE is its mode.
665 : If OP1 is constant, it will have been truncated as appropriate
666 : and is known to be nonzero.
667 :
668 : If SHIFT_MASK is zero, the result of word shifts is undefined when the
669 : shift count is outside the range [0, BITS_PER_WORD). This routine must
670 : avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
671 :
672 : If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
673 : masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
674 : fill with zeros or sign bits as appropriate.
675 :
676 : If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
677 : a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
678 : Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
679 : In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
680 : are undefined.
681 :
682 : BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop. This function
683 : may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
684 : OUTOF_INPUT and OUTOF_TARGET. OUTOF_TARGET can be null if the parent
685 : function wants to calculate it itself.
686 :
687 : Return true if the shift could be successfully synthesized. */
688 :
689 : static bool
690 0 : expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
691 : rtx outof_input, rtx into_input, rtx op1,
692 : rtx outof_target, rtx into_target,
693 : int unsignedp, enum optab_methods methods,
694 : unsigned HOST_WIDE_INT shift_mask)
695 : {
696 0 : rtx superword_op1, tmp, cmp1, cmp2;
697 0 : enum rtx_code cmp_code;
698 :
699 : /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
700 : fill the result with sign or zero bits as appropriate. If so, the value
701 : of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1). Recursively call
702 : this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
703 : and INTO_INPUT), then emit code to set up OUTOF_TARGET.
704 :
705 : This isn't worthwhile for constant shifts since the optimizers will
706 : cope better with in-range shift counts. */
707 0 : if (shift_mask >= BITS_PER_WORD
708 0 : && outof_target != 0
709 0 : && !CONSTANT_P (op1))
710 : {
711 0 : if (!expand_doubleword_shift (op1_mode, binoptab,
712 : outof_input, into_input, op1,
713 : 0, into_target,
714 : unsignedp, methods, shift_mask))
715 : return false;
716 0 : if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
717 : outof_target, unsignedp, methods))
718 : return false;
719 : return true;
720 : }
721 :
722 : /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
723 : is true when the effective shift value is less than BITS_PER_WORD.
724 : Set SUPERWORD_OP1 to the shift count that should be used to shift
725 : OUTOF_INPUT into INTO_TARGET when the condition is false. */
726 0 : tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
727 0 : if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
728 : {
729 : /* Set CMP1 to OP1 & BITS_PER_WORD. The result is zero iff OP1
730 : is a subword shift count. */
731 0 : cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
732 : 0, true, methods);
733 0 : cmp2 = CONST0_RTX (op1_mode);
734 0 : cmp_code = EQ;
735 0 : superword_op1 = op1;
736 : }
737 : else
738 : {
739 : /* Set CMP1 to OP1 - BITS_PER_WORD. */
740 0 : cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
741 : 0, true, methods);
742 0 : cmp2 = CONST0_RTX (op1_mode);
743 0 : cmp_code = LT;
744 0 : superword_op1 = cmp1;
745 : }
746 0 : if (cmp1 == 0)
747 : return false;
748 :
749 : /* If we can compute the condition at compile time, pick the
750 : appropriate subroutine. */
751 0 : tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
752 0 : if (tmp != 0 && CONST_INT_P (tmp))
753 : {
754 0 : if (tmp == const0_rtx)
755 0 : return expand_superword_shift (binoptab, outof_input, superword_op1,
756 : outof_target, into_target,
757 0 : unsignedp, methods);
758 : else
759 0 : return expand_subword_shift (op1_mode, binoptab,
760 : outof_input, into_input, op1,
761 : outof_target, into_target,
762 0 : unsignedp, methods, shift_mask);
763 : }
764 :
765 : /* Try using conditional moves to generate straight-line code. */
766 0 : if (HAVE_conditional_move)
767 : {
768 0 : rtx_insn *start = get_last_insn ();
769 0 : if (expand_doubleword_shift_condmove (op1_mode, binoptab,
770 : cmp_code, cmp1, cmp2,
771 : outof_input, into_input,
772 : op1, superword_op1,
773 : outof_target, into_target,
774 : unsignedp, methods, shift_mask))
775 : return true;
776 0 : delete_insns_since (start);
777 : }
778 :
779 : /* As a last resort, use branches to select the correct alternative. */
780 0 : rtx_code_label *subword_label = gen_label_rtx ();
781 0 : rtx_code_label *done_label = gen_label_rtx ();
782 :
783 0 : NO_DEFER_POP;
784 0 : do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
785 : 0, 0, subword_label,
786 : profile_probability::uninitialized ());
787 0 : OK_DEFER_POP;
788 :
789 0 : if (!expand_superword_shift (binoptab, outof_input, superword_op1,
790 : outof_target, into_target,
791 : unsignedp, methods))
792 : return false;
793 :
794 0 : emit_jump_insn (targetm.gen_jump (done_label));
795 0 : emit_barrier ();
796 0 : emit_label (subword_label);
797 :
798 0 : if (!expand_subword_shift (op1_mode, binoptab,
799 : outof_input, into_input, op1,
800 : outof_target, into_target,
801 : unsignedp, methods, shift_mask))
802 : return false;
803 :
804 0 : emit_label (done_label);
805 0 : return true;
806 : }
807 :
808 : /* Subroutine of expand_binop. Perform a double word multiplication of
809 : operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
810 : as the target's word_mode. This function return NULL_RTX if anything
811 : goes wrong, in which case it may have already emitted instructions
812 : which need to be deleted.
813 :
814 : If we want to multiply two two-word values and have normal and widening
815 : multiplies of single-word values, we can do this with three smaller
816 : multiplications.
817 :
818 : The multiplication proceeds as follows:
819 : _______________________
820 : [__op0_high_|__op0_low__]
821 : _______________________
822 : * [__op1_high_|__op1_low__]
823 : _______________________________________________
824 : _______________________
825 : (1) [__op0_low__*__op1_low__]
826 : _______________________
827 : (2a) [__op0_low__*__op1_high_]
828 : _______________________
829 : (2b) [__op0_high_*__op1_low__]
830 : _______________________
831 : (3) [__op0_high_*__op1_high_]
832 :
833 :
834 : This gives a 4-word result. Since we are only interested in the
835 : lower 2 words, partial result (3) and the upper words of (2a) and
836 : (2b) don't need to be calculated. Hence (2a) and (2b) can be
837 : calculated using non-widening multiplication.
838 :
839 : (1), however, needs to be calculated with an unsigned widening
840 : multiplication. If this operation is not directly supported we
841 : try using a signed widening multiplication and adjust the result.
842 : This adjustment works as follows:
843 :
844 : If both operands are positive then no adjustment is needed.
845 :
846 : If the operands have different signs, for example op0_low < 0 and
847 : op1_low >= 0, the instruction treats the most significant bit of
848 : op0_low as a sign bit instead of a bit with significance
849 : 2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
850 : with 2**BITS_PER_WORD - op0_low, and two's complements the
851 : result. Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
852 : the result.
853 :
854 : Similarly, if both operands are negative, we need to add
855 : (op0_low + op1_low) * 2**BITS_PER_WORD.
856 :
857 : We use a trick to adjust quickly. We logically shift op0_low right
858 : (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
859 : op0_high (op1_high) before it is used to calculate 2b (2a). If no
860 : logical shift exists, we do an arithmetic right shift and subtract
861 : the 0 or -1. */
862 :
863 : static rtx
864 14221 : expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
865 : bool umulp, enum optab_methods methods)
866 : {
867 14221 : int low = (WORDS_BIG_ENDIAN ? 1 : 0);
868 14221 : int high = (WORDS_BIG_ENDIAN ? 0 : 1);
869 14221 : rtx wordm1 = (umulp ? NULL_RTX
870 0 : : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
871 14221 : rtx product, adjust, product_high, temp;
872 :
873 14221 : rtx op0_high = operand_subword_force (op0, high, mode);
874 14221 : rtx op0_low = operand_subword_force (op0, low, mode);
875 14221 : rtx op1_high = operand_subword_force (op1, high, mode);
876 14221 : rtx op1_low = operand_subword_force (op1, low, mode);
877 :
878 : /* If we're using an unsigned multiply to directly compute the product
879 : of the low-order words of the operands and perform any required
880 : adjustments of the operands, we begin by trying two more multiplications
881 : and then computing the appropriate sum.
882 :
883 : We have checked above that the required addition is provided.
884 : Full-word addition will normally always succeed, especially if
885 : it is provided at all, so we don't worry about its failure. The
886 : multiplication may well fail, however, so we do handle that. */
887 :
888 14221 : if (!umulp)
889 : {
890 : /* ??? This could be done with emit_store_flag where available. */
891 0 : temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
892 : NULL_RTX, 1, methods);
893 0 : if (temp)
894 0 : op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
895 : NULL_RTX, 0, OPTAB_DIRECT);
896 : else
897 : {
898 0 : temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
899 : NULL_RTX, 0, methods);
900 0 : if (!temp)
901 : return NULL_RTX;
902 0 : op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
903 : NULL_RTX, 0, OPTAB_DIRECT);
904 : }
905 :
906 0 : if (!op0_high)
907 : return NULL_RTX;
908 : }
909 :
910 14221 : if (op1_low == const1_rtx)
911 : adjust = op0_high;
912 14181 : else if (op1_low == const0_rtx)
913 : adjust = const0_rtx;
914 14106 : else if (op1_low == const2_rtx)
915 0 : adjust = expand_binop (word_mode, add_optab, op0_high, op0_high,
916 : NULL_RTX, 0, OPTAB_DIRECT);
917 : else
918 14106 : adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
919 : NULL_RTX, 0, OPTAB_DIRECT);
920 14221 : if (!adjust)
921 : return NULL_RTX;
922 :
923 : /* OP0_HIGH should now be dead. */
924 :
925 14221 : if (!umulp)
926 : {
927 : /* ??? This could be done with emit_store_flag where available. */
928 0 : temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
929 : NULL_RTX, 1, methods);
930 0 : if (temp)
931 0 : op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
932 : NULL_RTX, 0, OPTAB_DIRECT);
933 : else
934 : {
935 0 : temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
936 : NULL_RTX, 0, methods);
937 0 : if (!temp)
938 : return NULL_RTX;
939 0 : op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
940 : NULL_RTX, 0, OPTAB_DIRECT);
941 : }
942 :
943 0 : if (!op1_high)
944 : return NULL_RTX;
945 : }
946 :
947 14221 : if (op1_high == const1_rtx)
948 : temp = op0_low;
949 14220 : else if (op1_high == const0_rtx)
950 : temp = const0_rtx;
951 13447 : else if (op1_high == const2_rtx)
952 0 : temp = expand_binop (word_mode, add_optab, op0_low, op0_low,
953 : NULL_RTX, 0, OPTAB_DIRECT);
954 : else
955 13447 : temp = expand_binop (word_mode, smul_optab, op0_low, op1_high,
956 : NULL_RTX, 0, OPTAB_DIRECT);
957 14221 : if (!temp)
958 : return NULL_RTX;
959 :
960 : /* OP1_HIGH should now be dead. */
961 :
962 14221 : adjust = expand_binop (word_mode, add_optab, adjust, temp,
963 : NULL_RTX, 0, OPTAB_DIRECT);
964 :
965 14221 : if (target && !REG_P (target))
966 8136 : target = NULL_RTX;
967 :
968 : /* *_widen_optab needs to determine operand mode, make sure at least
969 : one operand has non-VOID mode. */
970 14221 : if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
971 0 : op0_low = force_reg (word_mode, op0_low);
972 :
973 14221 : if (op1_low == const1_rtx)
974 40 : product = convert_modes (mode, word_mode, op0_low, umulp);
975 14181 : else if (umulp)
976 14181 : product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
977 : target, 1, OPTAB_DIRECT);
978 : else
979 0 : product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
980 : target, 1, OPTAB_DIRECT);
981 :
982 14221 : if (!product)
983 : return NULL_RTX;
984 :
985 14221 : product_high = operand_subword (product, high, 1, mode);
986 14221 : adjust = expand_binop (word_mode, add_optab, product_high, adjust,
987 : NULL_RTX, 0, OPTAB_DIRECT);
988 14221 : emit_move_insn (product_high, adjust);
989 14221 : return product;
990 : }
991 :
992 : /* Subroutine of expand_binop. Optimize unsigned double-word OP0 % OP1 for
993 : constant OP1. If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
994 : (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
995 : computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
996 : + (OP0 >> (2 * bit))) % OP1. Whether we need to sum 2, 3 or 4 values
997 : depends on the bit value, if 2, then carry from the addition needs to be
998 : added too, i.e. like:
999 : sum += __builtin_add_overflow (low, high, &sum)
1000 :
1001 : Optimize signed double-word OP0 % OP1 similarly, just apply some correction
1002 : factor to the sum before doing unsigned remainder, in the form of
1003 : sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
1004 : then perform unsigned
1005 : remainder = sum % OP1;
1006 : and finally
1007 : remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1); */
1008 :
1009 : static rtx
1010 20745 : expand_doubleword_mod (machine_mode mode, rtx op0, rtx op1, bool unsignedp)
1011 : {
1012 20745 : if (INTVAL (op1) <= 1 || (INTVAL (op1) & 1) == 0)
1013 : return NULL_RTX;
1014 :
1015 20424 : rtx_insn *last = get_last_insn ();
1016 693676 : for (int bit = BITS_PER_WORD; bit >= BITS_PER_WORD / 2; bit--)
1017 : {
1018 652147 : wide_int w = wi::shifted_mask (bit, 1, false, 2 * BITS_PER_WORD);
1019 622346 : if (wi::ne_p (wi::umod_trunc (w, INTVAL (op1)), 1))
1020 621061 : continue;
1021 1285 : rtx sum = NULL_RTX, mask = NULL_RTX;
1022 1753 : if (bit == BITS_PER_WORD)
1023 : {
1024 : /* For signed modulo we need to add correction to the sum
1025 : and that might again overflow. */
1026 363 : if (!unsignedp)
1027 137 : continue;
1028 226 : if (optab_handler (uaddv4_optab, word_mode) == CODE_FOR_nothing)
1029 0 : continue;
1030 226 : tree wtype = lang_hooks.types.type_for_mode (word_mode, 1);
1031 226 : if (wtype == NULL_TREE)
1032 0 : continue;
1033 226 : tree ctype = build_complex_type (wtype);
1034 226 : if (TYPE_MODE (ctype) != GET_MODE_COMPLEX_MODE (word_mode))
1035 0 : continue;
1036 226 : machine_mode cmode = TYPE_MODE (ctype);
1037 226 : rtx op00 = operand_subword_force (op0, 0, mode);
1038 226 : rtx op01 = operand_subword_force (op0, 1, mode);
1039 226 : rtx cres = gen_rtx_CONCAT (cmode, gen_reg_rtx (word_mode),
1040 : gen_reg_rtx (word_mode));
1041 226 : tree lhs = make_tree (ctype, cres);
1042 226 : tree arg0 = make_tree (wtype, op00);
1043 226 : tree arg1 = make_tree (wtype, op01);
1044 226 : expand_addsub_overflow (UNKNOWN_LOCATION, PLUS_EXPR, lhs, arg0,
1045 : arg1, true, true, true, false, NULL);
1046 226 : sum = expand_simple_binop (word_mode, PLUS, XEXP (cres, 0),
1047 : XEXP (cres, 1), NULL_RTX, 1,
1048 : OPTAB_DIRECT);
1049 226 : if (sum == NULL_RTX)
1050 : return NULL_RTX;
1051 : }
1052 : else
1053 : {
1054 : /* Code below uses GEN_INT, so we need the masks to be representable
1055 : in HOST_WIDE_INTs. */
1056 922 : if (bit >= HOST_BITS_PER_WIDE_INT)
1057 0 : continue;
1058 : /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
1059 : overflow. Consider 64-bit -1ULL for word size 32, if we add
1060 : 0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1. */
1061 1150 : if (bit == BITS_PER_WORD - 1)
1062 147 : continue;
1063 :
1064 775 : int count = (2 * BITS_PER_WORD + bit - 1) / bit;
1065 775 : rtx sum_corr = NULL_RTX;
1066 :
1067 775 : if (!unsignedp)
1068 : {
1069 : /* For signed modulo, compute it as unsigned modulo of
1070 : sum with a correction added to it if OP0 is negative,
1071 : such that the result can be computed as unsigned
1072 : remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1). */
1073 498 : w = wi::min_value (2 * BITS_PER_WORD, SIGNED);
1074 407 : wide_int wmod1 = wi::umod_trunc (w, INTVAL (op1));
1075 407 : wide_int wmod2 = wi::smod_trunc (w, INTVAL (op1));
1076 : /* wmod2 == -wmod1. */
1077 407 : wmod2 = wmod2 + (INTVAL (op1) - 1);
1078 407 : if (wi::ne_p (wmod1, wmod2))
1079 : {
1080 407 : wide_int wcorr = wmod2 - wmod1;
1081 407 : if (wi::neg_p (w))
1082 407 : wcorr = wcorr + INTVAL (op1);
1083 : /* Now verify if the count sums can't overflow, and punt
1084 : if they could. */
1085 498 : w = wi::mask (bit, false, 2 * BITS_PER_WORD);
1086 407 : w = w * (count - 1);
1087 905 : w = w + wi::mask (2 * BITS_PER_WORD - (count - 1) * bit,
1088 814 : false, 2 * BITS_PER_WORD);
1089 407 : w = w + wcorr;
1090 498 : w = wi::lrshift (w, BITS_PER_WORD);
1091 407 : if (wi::ne_p (w, 0))
1092 0 : continue;
1093 :
1094 407 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1095 : mode);
1096 407 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1097 407 : GEN_INT (BITS_PER_WORD - 1),
1098 : NULL_RTX, 0, OPTAB_DIRECT);
1099 407 : if (mask == NULL_RTX)
1100 : return NULL_RTX;
1101 407 : sum_corr = immed_wide_int_const (wcorr, word_mode);
1102 407 : sum_corr = expand_simple_binop (word_mode, AND, mask,
1103 : sum_corr, NULL_RTX, 1,
1104 : OPTAB_DIRECT);
1105 407 : if (sum_corr == NULL_RTX)
1106 : return NULL_RTX;
1107 407 : }
1108 407 : }
1109 :
1110 3274 : for (int i = 0; i < count; i++)
1111 : {
1112 2499 : rtx v = op0;
1113 2499 : if (i)
1114 1724 : v = expand_simple_binop (mode, LSHIFTRT, v, GEN_INT (i * bit),
1115 : NULL_RTX, 1, OPTAB_DIRECT);
1116 2499 : if (v == NULL_RTX)
1117 : return NULL_RTX;
1118 2499 : v = force_lowpart_subreg (word_mode, v, mode);
1119 2499 : if (v == NULL_RTX)
1120 : return NULL_RTX;
1121 2499 : if (i != count - 1)
1122 1724 : v = expand_simple_binop (word_mode, AND, v,
1123 1724 : GEN_INT ((HOST_WIDE_INT_1U << bit)
1124 : - 1), NULL_RTX, 1,
1125 : OPTAB_DIRECT);
1126 2499 : if (v == NULL_RTX)
1127 : return NULL_RTX;
1128 2499 : if (sum == NULL_RTX)
1129 : sum = v;
1130 : else
1131 1724 : sum = expand_simple_binop (word_mode, PLUS, sum, v, NULL_RTX,
1132 : 1, OPTAB_DIRECT);
1133 2499 : if (sum == NULL_RTX)
1134 : return NULL_RTX;
1135 : }
1136 775 : if (sum_corr)
1137 : {
1138 407 : sum = expand_simple_binop (word_mode, PLUS, sum, sum_corr,
1139 : NULL_RTX, 1, OPTAB_DIRECT);
1140 407 : if (sum == NULL_RTX)
1141 : return NULL_RTX;
1142 : }
1143 : }
1144 1001 : rtx remainder = expand_divmod (1, TRUNC_MOD_EXPR, word_mode, sum,
1145 1001 : gen_int_mode (INTVAL (op1), word_mode),
1146 : NULL_RTX, 1, OPTAB_DIRECT);
1147 1001 : if (remainder == NULL_RTX)
1148 : return NULL_RTX;
1149 :
1150 1001 : if (!unsignedp)
1151 : {
1152 407 : if (mask == NULL_RTX)
1153 : {
1154 0 : mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
1155 : mode);
1156 0 : mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
1157 0 : GEN_INT (BITS_PER_WORD - 1),
1158 : NULL_RTX, 0, OPTAB_DIRECT);
1159 0 : if (mask == NULL_RTX)
1160 : return NULL_RTX;
1161 : }
1162 407 : mask = expand_simple_binop (word_mode, AND, mask,
1163 407 : gen_int_mode (1 - INTVAL (op1),
1164 : word_mode),
1165 : NULL_RTX, 1, OPTAB_DIRECT);
1166 407 : if (mask == NULL_RTX)
1167 : return NULL_RTX;
1168 407 : remainder = expand_simple_binop (word_mode, PLUS, remainder,
1169 : mask, NULL_RTX, 1, OPTAB_DIRECT);
1170 407 : if (remainder == NULL_RTX)
1171 : return NULL_RTX;
1172 : }
1173 :
1174 1001 : remainder = convert_modes (mode, word_mode, remainder, unsignedp);
1175 : /* Punt if we need any library calls. */
1176 1001 : if (last)
1177 905 : last = NEXT_INSN (last);
1178 : else
1179 96 : last = get_insns ();
1180 20545 : for (; last; last = NEXT_INSN (last))
1181 19544 : if (CALL_P (last))
1182 : return NULL_RTX;
1183 : return remainder;
1184 622346 : }
1185 : return NULL_RTX;
1186 : }
1187 :
1188 : /* Similarly to the above function, but compute both quotient and remainder.
1189 : Quotient can be computed from the remainder as:
1190 : rem = op0 % op1; // Handled using expand_doubleword_mod
1191 : quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
1192 : // 2 * BITS_PER_WORD
1193 :
1194 : We can also handle cases where op1 is a multiple of power of two constant
1195 : and constant handled by expand_doubleword_mod.
1196 : op11 = 1 << __builtin_ctz (op1);
1197 : op12 = op1 / op11;
1198 : rem1 = op0 % op12; // Handled using expand_doubleword_mod
1199 : quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
1200 : // 2 * BITS_PER_WORD
1201 : rem = (quot1 % op11) * op12 + rem1;
1202 : quot = quot1 / op11; */
1203 :
1204 : rtx
1205 20473 : expand_doubleword_divmod (machine_mode mode, rtx op0, rtx op1, rtx *rem,
1206 : bool unsignedp)
1207 : {
1208 20473 : *rem = NULL_RTX;
1209 :
1210 : /* Negative dividend should have been optimized into positive,
1211 : similarly modulo by 1 and modulo by power of two is optimized
1212 : differently too. */
1213 39896 : if (INTVAL (op1) <= 1 || pow2p_hwi (INTVAL (op1)))
1214 : return NULL_RTX;
1215 :
1216 20424 : rtx op11 = const1_rtx;
1217 20424 : rtx op12 = op1;
1218 20424 : if ((INTVAL (op1) & 1) == 0)
1219 : {
1220 3408 : int bit = ctz_hwi (INTVAL (op1));
1221 3408 : op11 = GEN_INT (HOST_WIDE_INT_1 << bit);
1222 3408 : op12 = GEN_INT (INTVAL (op1) >> bit);
1223 : }
1224 :
1225 20424 : rtx rem1 = expand_doubleword_mod (mode, op0, op12, unsignedp);
1226 20424 : if (rem1 == NULL_RTX)
1227 : return NULL_RTX;
1228 :
1229 1001 : int prec = 2 * BITS_PER_WORD;
1230 1001 : wide_int a = wide_int::from (INTVAL (op12), prec + 1, UNSIGNED);
1231 1001 : wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
1232 1001 : wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
1233 1001 : rtx inv = immed_wide_int_const (m, mode);
1234 :
1235 1001 : rtx_insn *last = get_last_insn ();
1236 1001 : rtx quot1 = expand_simple_binop (mode, MINUS, op0, rem1,
1237 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1238 1001 : if (quot1 == NULL_RTX)
1239 : return NULL_RTX;
1240 :
1241 1001 : quot1 = expand_simple_binop (mode, MULT, quot1, inv,
1242 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1243 1001 : if (quot1 == NULL_RTX)
1244 : return NULL_RTX;
1245 :
1246 1001 : if (op11 != const1_rtx)
1247 : {
1248 340 : rtx rem2 = expand_divmod (1, TRUNC_MOD_EXPR, mode, quot1, op11,
1249 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1250 340 : if (rem2 == NULL_RTX)
1251 : return NULL_RTX;
1252 :
1253 340 : rem2 = expand_simple_binop (mode, MULT, rem2, op12, NULL_RTX,
1254 : unsignedp, OPTAB_DIRECT);
1255 340 : if (rem2 == NULL_RTX)
1256 : return NULL_RTX;
1257 :
1258 340 : rem2 = expand_simple_binop (mode, PLUS, rem2, rem1, NULL_RTX,
1259 : unsignedp, OPTAB_DIRECT);
1260 340 : if (rem2 == NULL_RTX)
1261 : return NULL_RTX;
1262 :
1263 340 : rtx quot2 = expand_divmod (0, TRUNC_DIV_EXPR, mode, quot1, op11,
1264 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1265 340 : if (quot2 == NULL_RTX)
1266 : return NULL_RTX;
1267 :
1268 : rem1 = rem2;
1269 : quot1 = quot2;
1270 : }
1271 :
1272 : /* Punt if we need any library calls. */
1273 1001 : if (last)
1274 1001 : last = NEXT_INSN (last);
1275 : else
1276 0 : last = get_insns ();
1277 15453 : for (; last; last = NEXT_INSN (last))
1278 14452 : if (CALL_P (last))
1279 : return NULL_RTX;
1280 :
1281 1001 : *rem = rem1;
1282 1001 : return quot1;
1283 1001 : }
1284 :
1285 : /* Wrapper around expand_binop which takes an rtx code to specify
1286 : the operation to perform, not an optab pointer. All other
1287 : arguments are the same. */
1288 : rtx
1289 4033188 : expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
1290 : rtx op1, rtx target, int unsignedp,
1291 : enum optab_methods methods)
1292 : {
1293 4033188 : optab binop = code_to_optab (code);
1294 4033188 : gcc_assert (binop);
1295 :
1296 4033188 : return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
1297 : }
1298 :
1299 : /* Return whether OP0 and OP1 should be swapped when expanding a commutative
1300 : binop. Order them according to commutative_operand_precedence and, if
1301 : possible, try to put TARGET or a pseudo first. */
1302 : static bool
1303 12195399 : swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
1304 : {
1305 12195399 : int op0_prec = commutative_operand_precedence (op0);
1306 12195399 : int op1_prec = commutative_operand_precedence (op1);
1307 :
1308 12195399 : if (op0_prec < op1_prec)
1309 : return true;
1310 :
1311 12006831 : if (op0_prec > op1_prec)
1312 : return false;
1313 :
1314 : /* With equal precedence, both orders are ok, but it is better if the
1315 : first operand is TARGET, or if both TARGET and OP0 are pseudos. */
1316 2231830 : if (target == 0 || REG_P (target))
1317 2494515 : return (REG_P (op1) && !REG_P (op0)) || target == op1;
1318 : else
1319 87541 : return rtx_equal_p (op1, target);
1320 : }
1321 :
1322 : /* Return true if BINOPTAB implements a shift operation. */
1323 :
1324 : static bool
1325 23959598 : shift_optab_p (optab binoptab)
1326 : {
1327 9593481 : switch (optab_to_code (binoptab))
1328 : {
1329 : case ASHIFT:
1330 : case SS_ASHIFT:
1331 : case US_ASHIFT:
1332 : case ASHIFTRT:
1333 : case LSHIFTRT:
1334 : case ROTATE:
1335 : case ROTATERT:
1336 : return true;
1337 :
1338 12817765 : default:
1339 0 : return false;
1340 : }
1341 : }
1342 :
1343 : /* Return true if BINOPTAB implements a commutative binary operation. */
1344 :
1345 : static bool
1346 14524884 : commutative_optab_p (optab binoptab)
1347 : {
1348 14524884 : return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
1349 : || binoptab == smul_widen_optab
1350 2411418 : || binoptab == umul_widen_optab
1351 : || binoptab == smul_highpart_optab
1352 2379858 : || binoptab == umul_highpart_optab
1353 2329485 : || binoptab == vec_widen_sadd_optab
1354 2329485 : || binoptab == vec_widen_uadd_optab
1355 : || binoptab == vec_widen_sadd_hi_optab
1356 2329485 : || binoptab == vec_widen_sadd_lo_optab
1357 : || binoptab == vec_widen_uadd_hi_optab
1358 2329485 : || binoptab == vec_widen_uadd_lo_optab
1359 : || binoptab == vec_widen_sadd_even_optab
1360 2329485 : || binoptab == vec_widen_sadd_odd_optab
1361 : || binoptab == vec_widen_uadd_even_optab
1362 16854369 : || binoptab == vec_widen_uadd_odd_optab);
1363 : }
1364 :
1365 : /* X is to be used in mode MODE as operand OPN to BINOPTAB. If we're
1366 : optimizing, and if the operand is a constant that costs more than
1367 : 1 instruction, force the constant into a register and return that
1368 : register. Return X otherwise. UNSIGNEDP says whether X is unsigned. */
1369 :
1370 : static rtx
1371 27347158 : avoid_expensive_constant (machine_mode mode, optab binoptab,
1372 : int opn, rtx x, bool unsignedp)
1373 : {
1374 27347158 : bool speed = optimize_insn_for_speed_p ();
1375 :
1376 27347158 : if (mode != VOIDmode
1377 27347158 : && optimize
1378 20564984 : && CONSTANT_P (x)
1379 34454755 : && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
1380 7107597 : > set_src_cost (x, mode, speed)))
1381 : {
1382 0 : if (CONST_INT_P (x))
1383 : {
1384 0 : HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
1385 0 : if (intval != INTVAL (x))
1386 0 : x = GEN_INT (intval);
1387 : }
1388 : else
1389 0 : x = convert_modes (mode, VOIDmode, x, unsignedp);
1390 0 : x = force_reg (mode, x);
1391 : }
1392 27347158 : return x;
1393 : }
1394 :
1395 : /* Helper function for expand_binop: handle the case where there
1396 : is an insn ICODE that directly implements the indicated operation.
1397 : Returns null if this is not possible. */
1398 : static rtx
1399 14335476 : expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
1400 : rtx op0, rtx op1,
1401 : rtx target, int unsignedp, enum optab_methods methods)
1402 : {
1403 14335476 : machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
1404 14335476 : machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
1405 14335476 : machine_mode mode0, mode1, tmp_mode;
1406 14335476 : class expand_operand ops[3];
1407 14335476 : bool commutative_p;
1408 14335476 : rtx_insn *pat;
1409 14335476 : rtx xop0 = op0, xop1 = op1;
1410 14335476 : bool canonicalize_op1 = false;
1411 14335476 : rtx_insn *last = get_last_insn ();
1412 :
1413 : /* If it is a commutative operator and the modes would match
1414 : if we would swap the operands, we can save the conversions. */
1415 14335476 : commutative_p = commutative_optab_p (binoptab);
1416 14335476 : if (commutative_p
1417 12135428 : && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
1418 13238 : && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0)
1419 : std::swap (xop0, xop1);
1420 :
1421 : /* If we are optimizing, force expensive constants into a register. */
1422 14335476 : xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
1423 14335476 : if (!shift_optab_p (binoptab))
1424 12817765 : xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
1425 : else
1426 : /* Shifts and rotates often use a different mode for op1 from op0;
1427 : for VOIDmode constants we don't know the mode, so force it
1428 : to be canonicalized using convert_modes. */
1429 : canonicalize_op1 = true;
1430 :
1431 : /* In case the insn wants input operands in modes different from
1432 : those of the actual operands, convert the operands. It would
1433 : seem that we don't need to convert CONST_INTs, but we do, so
1434 : that they're properly zero-extended, sign-extended or truncated
1435 : for their mode. */
1436 :
1437 14335476 : mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
1438 14335476 : if (xmode0 != VOIDmode && xmode0 != mode0)
1439 : {
1440 3011 : xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
1441 3011 : mode0 = xmode0;
1442 : }
1443 :
1444 14335476 : mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
1445 14335476 : ? GET_MODE (xop1) : mode);
1446 14335476 : if (xmode1 != VOIDmode && xmode1 != mode1)
1447 : {
1448 1569423 : xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
1449 1569423 : mode1 = xmode1;
1450 : }
1451 :
1452 : /* If operation is commutative,
1453 : try to make the first operand a register.
1454 : Even better, try to make it the same as the target.
1455 : Also try to make the last operand a constant. */
1456 14335476 : if (commutative_p
1457 14335476 : && swap_commutative_operands_with_target (target, xop0, xop1))
1458 : std::swap (xop0, xop1);
1459 :
1460 : /* Now, if insn's predicates don't allow our operands, put them into
1461 : pseudo regs. */
1462 :
1463 14335476 : if (binoptab == vec_pack_trunc_optab
1464 14335476 : || binoptab == vec_pack_usat_optab
1465 14325219 : || binoptab == vec_pack_ssat_optab
1466 14325219 : || binoptab == vec_pack_ufix_trunc_optab
1467 14325081 : || binoptab == vec_pack_sfix_trunc_optab
1468 14325081 : || binoptab == vec_packu_float_optab
1469 14324861 : || binoptab == vec_packs_float_optab)
1470 : {
1471 : /* The mode of the result is different then the mode of the
1472 : arguments. */
1473 10635 : tmp_mode = insn_data[(int) icode].operand[0].mode;
1474 2598 : if (VECTOR_MODE_P (mode)
1475 30321 : && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
1476 : return NULL_RTX;
1477 : }
1478 : else
1479 : tmp_mode = mode;
1480 :
1481 14335476 : create_output_operand (&ops[0], target, tmp_mode);
1482 14335476 : create_input_operand (&ops[1], xop0, mode0);
1483 14335476 : create_input_operand (&ops[2], xop1, mode1);
1484 14335476 : pat = maybe_gen_insn (icode, 3, ops);
1485 14335476 : if (pat)
1486 : {
1487 : /* If PAT is composed of more than one insn, try to add an appropriate
1488 : REG_EQUAL note to it. If we can't because TEMP conflicts with an
1489 : operand, call expand_binop again, this time without a target. */
1490 13227108 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
1491 15503252 : && ! add_equal_note (pat, ops[0].value,
1492 : optab_to_code (binoptab),
1493 : ops[1].value, ops[2].value, mode0))
1494 : {
1495 51761 : delete_insns_since (last);
1496 51761 : return expand_binop_directly (icode, mode, binoptab, op0, op1,
1497 51761 : NULL_RTX, unsignedp, methods);
1498 : }
1499 :
1500 14283713 : emit_insn (pat);
1501 14283713 : return ops[0].value;
1502 : }
1503 :
1504 : return NULL_RTX;
1505 : }
1506 :
1507 : /* Generate code to perform an operation specified by BINOPTAB
1508 : on operands OP0 and OP1, with result having machine-mode MODE.
1509 :
1510 : UNSIGNEDP is for the case where we have to widen the operands
1511 : to perform the operation. It says to use zero-extension.
1512 :
1513 : If TARGET is nonzero, the value
1514 : is generated there, if it is convenient to do so.
1515 : In all cases an rtx is returned for the locus of the value;
1516 : this may or may not be TARGET. */
1517 :
1518 : rtx
1519 14473285 : expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
1520 : rtx target, int unsignedp, enum optab_methods methods)
1521 : {
1522 12466301 : enum optab_methods next_methods
1523 14473285 : = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
1524 14473285 : ? OPTAB_WIDEN : methods);
1525 14473285 : enum mode_class mclass;
1526 14473285 : enum insn_code icode;
1527 14473285 : machine_mode wider_mode;
1528 14473285 : scalar_int_mode int_mode;
1529 14473285 : rtx libfunc;
1530 14473285 : rtx temp;
1531 14473285 : rtx_insn *entry_last = get_last_insn ();
1532 14473285 : rtx_insn *last;
1533 :
1534 14473285 : mclass = GET_MODE_CLASS (mode);
1535 :
1536 : /* If subtracting an integer constant, convert this into an addition of
1537 : the negated constant. */
1538 :
1539 14473285 : if (binoptab == sub_optab && CONST_INT_P (op1))
1540 : {
1541 1035191 : op1 = negate_rtx (mode, op1);
1542 1035191 : binoptab = add_optab;
1543 : }
1544 : /* For shifts, constant invalid op1 might be expanded from different
1545 : mode than MODE. As those are invalid, force them to a register
1546 : to avoid further problems during expansion. */
1547 13438094 : else if (CONST_INT_P (op1)
1548 15932635 : && shift_optab_p (binoptab)
1549 17816144 : && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
1550 : {
1551 138 : op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
1552 138 : op1 = force_reg (GET_MODE_INNER (mode), op1);
1553 : }
1554 :
1555 : /* Record where to delete back to if we backtrack. */
1556 14473285 : last = get_last_insn ();
1557 :
1558 : /* If we can do it with a three-operand insn, do so. */
1559 :
1560 14473285 : if (methods != OPTAB_MUST_WIDEN)
1561 : {
1562 14473285 : if (convert_optab_p (binoptab))
1563 : {
1564 31560 : machine_mode from_mode = widened_mode (mode, op0, op1);
1565 31560 : icode = find_widening_optab_handler (binoptab, mode, from_mode);
1566 : }
1567 : else
1568 14441725 : icode = optab_handler (binoptab, mode);
1569 14473285 : if (icode != CODE_FOR_nothing)
1570 : {
1571 14283714 : temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
1572 : target, unsignedp, methods);
1573 14283714 : if (temp)
1574 : return temp;
1575 1 : delete_insns_since (last);
1576 : }
1577 : }
1578 :
1579 : /* If we were trying to rotate, and that didn't work, try rotating
1580 : the other direction before falling back to shifts and bitwise-or. */
1581 189572 : if (((binoptab == rotl_optab
1582 23 : && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
1583 189571 : || (binoptab == rotr_optab
1584 11 : && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
1585 189573 : && is_int_mode (mode, &int_mode))
1586 : {
1587 1 : optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
1588 1 : rtx newop1;
1589 1 : unsigned int bits = GET_MODE_PRECISION (int_mode);
1590 :
1591 1 : if (CONST_INT_P (op1))
1592 0 : newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
1593 1 : else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
1594 0 : newop1 = negate_rtx (GET_MODE (op1), op1);
1595 : else
1596 1 : newop1 = expand_binop (GET_MODE (op1), sub_optab,
1597 1 : gen_int_mode (bits, GET_MODE (op1)), op1,
1598 : NULL_RTX, unsignedp, OPTAB_DIRECT);
1599 :
1600 1 : temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
1601 : target, unsignedp, methods);
1602 1 : if (temp)
1603 14473285 : return temp;
1604 1 : delete_insns_since (last);
1605 : }
1606 :
1607 : /* If this is a multiply, see if we can do a widening operation that
1608 : takes operands of this mode and makes a wider mode. */
1609 :
1610 189572 : if (binoptab == smul_optab
1611 25004 : && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
1612 207679 : && (convert_optab_handler ((unsignedp
1613 : ? umul_widen_optab
1614 : : smul_widen_optab),
1615 : wider_mode, mode) != CODE_FOR_nothing))
1616 : {
1617 : /* *_widen_optab needs to determine operand mode, make sure at least
1618 : one operand has non-VOID mode. */
1619 0 : if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
1620 0 : op0 = force_reg (mode, op0);
1621 0 : temp = expand_binop (wider_mode,
1622 : unsignedp ? umul_widen_optab : smul_widen_optab,
1623 : op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
1624 :
1625 0 : if (temp != 0)
1626 : {
1627 0 : if (GET_MODE_CLASS (mode) == MODE_INT
1628 0 : && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
1629 0 : return gen_lowpart (mode, temp);
1630 : else
1631 0 : return convert_to_mode (mode, temp, unsignedp);
1632 : }
1633 : }
1634 :
1635 : /* If this is a vector shift by a scalar, see if we can do a vector
1636 : shift by a vector. If so, broadcast the scalar into a vector. */
1637 189572 : if (mclass == MODE_VECTOR_INT)
1638 : {
1639 33 : optab otheroptab = unknown_optab;
1640 :
1641 33 : if (binoptab == ashl_optab)
1642 : otheroptab = vashl_optab;
1643 : else if (binoptab == ashr_optab)
1644 : otheroptab = vashr_optab;
1645 : else if (binoptab == lshr_optab)
1646 : otheroptab = vlshr_optab;
1647 : else if (binoptab == rotl_optab)
1648 : otheroptab = vrotl_optab;
1649 : else if (binoptab == rotr_optab)
1650 : otheroptab = vrotr_optab;
1651 :
1652 : if (otheroptab
1653 33 : && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
1654 : {
1655 : /* The scalar may be wider or narrower than the vector element.
1656 : Truncate or extend it to the proper size to fit in the
1657 : broadcast vector. */
1658 0 : scalar_mode inner_mode = GET_MODE_INNER (mode);
1659 0 : if (!CONST_INT_P (op1))
1660 : {
1661 0 : auto mode1 = as_a <scalar_int_mode> (GET_MODE (op1));
1662 0 : int size1 = GET_MODE_BITSIZE (mode1);
1663 0 : int inner_size = GET_MODE_BITSIZE (inner_mode);
1664 :
1665 0 : if (size1 != inner_size)
1666 : {
1667 0 : auto unary = size1 > inner_size ? TRUNCATE : ZERO_EXTEND;
1668 0 : op1 = force_reg (inner_mode,
1669 : simplify_gen_unary (unary, inner_mode,
1670 : op1, mode1));
1671 : }
1672 : }
1673 :
1674 0 : rtx vop1 = expand_vector_broadcast (mode, op1);
1675 0 : if (vop1)
1676 : {
1677 0 : temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
1678 : target, unsignedp, methods);
1679 0 : if (temp)
1680 14473285 : return temp;
1681 0 : delete_insns_since (last);
1682 : }
1683 : }
1684 : }
1685 :
1686 : /* Look for a wider mode of the same class for which we think we
1687 : can open-code the operation. Check for a widening multiply at the
1688 : wider mode as well. */
1689 :
1690 189572 : if (CLASS_HAS_WIDER_MODES_P (mclass)
1691 189539 : && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
1692 112711 : FOR_EACH_WIDER_MODE (wider_mode, mode)
1693 : {
1694 60087 : machine_mode next_mode;
1695 60087 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
1696 60087 : || (binoptab == smul_optab
1697 75443 : && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
1698 18995 : && (find_widening_optab_handler ((unsignedp
1699 : ? umul_widen_optab
1700 : : smul_widen_optab),
1701 : next_mode, mode)
1702 : != CODE_FOR_nothing)))
1703 : {
1704 164 : rtx xop0 = op0, xop1 = op1;
1705 164 : bool no_extend = false;
1706 :
1707 : /* For certain integer operations, we need not actually extend
1708 : the narrow operands, as long as we will truncate
1709 : the results to the same narrowness. */
1710 :
1711 164 : if ((binoptab == ior_optab || binoptab == and_optab
1712 164 : || binoptab == xor_optab
1713 164 : || binoptab == add_optab || binoptab == sub_optab
1714 : || binoptab == smul_optab || binoptab == ashl_optab)
1715 89 : && mclass == MODE_INT)
1716 : {
1717 7 : no_extend = true;
1718 7 : xop0 = avoid_expensive_constant (mode, binoptab, 0,
1719 : xop0, unsignedp);
1720 7 : if (binoptab != ashl_optab)
1721 0 : xop1 = avoid_expensive_constant (mode, binoptab, 1,
1722 : xop1, unsignedp);
1723 : }
1724 :
1725 164 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
1726 :
1727 : /* The second operand of a shift must always be extended. */
1728 328 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
1729 164 : no_extend && binoptab != ashl_optab);
1730 :
1731 164 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
1732 : unsignedp, OPTAB_DIRECT);
1733 164 : if (temp)
1734 : {
1735 164 : if (mclass != MODE_INT
1736 164 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
1737 : {
1738 157 : if (target == 0)
1739 49 : target = gen_reg_rtx (mode);
1740 157 : convert_move (target, temp, 0);
1741 157 : return target;
1742 : }
1743 : else
1744 7 : return gen_lowpart (mode, temp);
1745 : }
1746 : else
1747 0 : delete_insns_since (last);
1748 : }
1749 : }
1750 :
1751 : /* If operation is commutative,
1752 : try to make the first operand a register.
1753 : Even better, try to make it the same as the target.
1754 : Also try to make the last operand a constant. */
1755 189408 : if (commutative_optab_p (binoptab)
1756 189408 : && swap_commutative_operands_with_target (target, op0, op1))
1757 : std::swap (op0, op1);
1758 :
1759 : /* These can be done a word at a time. */
1760 189408 : if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
1761 0 : && is_int_mode (mode, &int_mode)
1762 0 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
1763 189408 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
1764 : {
1765 0 : int i;
1766 0 : rtx_insn *insns;
1767 :
1768 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1769 : won't be accurate, so use a new target. */
1770 0 : if (target == 0
1771 0 : || target == op0
1772 0 : || target == op1
1773 0 : || reg_overlap_mentioned_p (target, op0)
1774 0 : || reg_overlap_mentioned_p (target, op1)
1775 0 : || !valid_multiword_target_p (target))
1776 0 : target = gen_reg_rtx (int_mode);
1777 :
1778 0 : start_sequence ();
1779 :
1780 : /* Do the actual arithmetic. */
1781 0 : machine_mode op0_mode = GET_MODE (op0);
1782 0 : machine_mode op1_mode = GET_MODE (op1);
1783 0 : if (op0_mode == VOIDmode)
1784 0 : op0_mode = int_mode;
1785 0 : if (op1_mode == VOIDmode)
1786 0 : op1_mode = int_mode;
1787 0 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
1788 : {
1789 0 : rtx target_piece = operand_subword (target, i, 1, int_mode);
1790 0 : rtx x = expand_binop (word_mode, binoptab,
1791 0 : operand_subword_force (op0, i, op0_mode),
1792 0 : operand_subword_force (op1, i, op1_mode),
1793 : target_piece, unsignedp, next_methods);
1794 :
1795 0 : if (x == 0)
1796 : break;
1797 :
1798 0 : if (target_piece != x)
1799 0 : emit_move_insn (target_piece, x);
1800 : }
1801 :
1802 0 : insns = end_sequence ();
1803 :
1804 0 : if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
1805 : {
1806 0 : emit_insn (insns);
1807 0 : return target;
1808 : }
1809 : }
1810 :
1811 : /* Synthesize double word shifts from single word shifts. */
1812 189408 : if ((binoptab == lshr_optab || binoptab == ashl_optab
1813 189408 : || binoptab == ashr_optab)
1814 0 : && is_int_mode (mode, &int_mode)
1815 0 : && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
1816 0 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
1817 0 : && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
1818 0 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
1819 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1820 189408 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1821 : {
1822 0 : unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
1823 0 : scalar_int_mode op1_mode;
1824 :
1825 0 : double_shift_mask = targetm.shift_truncation_mask (int_mode);
1826 0 : shift_mask = targetm.shift_truncation_mask (word_mode);
1827 0 : op1_mode = (GET_MODE (op1) != VOIDmode
1828 0 : ? as_a <scalar_int_mode> (GET_MODE (op1))
1829 : : word_mode);
1830 :
1831 : /* Apply the truncation to constant shifts. */
1832 0 : if (double_shift_mask > 0 && CONST_INT_P (op1))
1833 0 : op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
1834 :
1835 0 : if (op1 == CONST0_RTX (op1_mode))
1836 14473285 : return op0;
1837 :
1838 : /* Make sure that this is a combination that expand_doubleword_shift
1839 : can handle. See the comments there for details. */
1840 0 : if (double_shift_mask == 0
1841 0 : || (shift_mask == BITS_PER_WORD - 1
1842 0 : && double_shift_mask == BITS_PER_WORD * 2 - 1))
1843 : {
1844 0 : rtx_insn *insns;
1845 0 : rtx into_target, outof_target;
1846 0 : rtx into_input, outof_input;
1847 0 : int left_shift, outof_word;
1848 :
1849 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1850 : won't be accurate, so use a new target. */
1851 0 : if (target == 0
1852 0 : || target == op0
1853 0 : || target == op1
1854 0 : || reg_overlap_mentioned_p (target, op0)
1855 0 : || reg_overlap_mentioned_p (target, op1)
1856 0 : || !valid_multiword_target_p (target))
1857 0 : target = gen_reg_rtx (int_mode);
1858 :
1859 0 : start_sequence ();
1860 :
1861 : /* OUTOF_* is the word we are shifting bits away from, and
1862 : INTO_* is the word that we are shifting bits towards, thus
1863 : they differ depending on the direction of the shift and
1864 : WORDS_BIG_ENDIAN. */
1865 :
1866 0 : left_shift = binoptab == ashl_optab;
1867 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1868 :
1869 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1870 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1871 :
1872 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1873 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1874 :
1875 0 : if (expand_doubleword_shift (op1_mode, binoptab,
1876 : outof_input, into_input, op1,
1877 : outof_target, into_target,
1878 : unsignedp, next_methods, shift_mask))
1879 : {
1880 0 : insns = end_sequence ();
1881 :
1882 0 : emit_insn (insns);
1883 0 : return target;
1884 : }
1885 0 : end_sequence ();
1886 : }
1887 : }
1888 :
1889 : /* Synthesize double word rotates from single word shifts. */
1890 189408 : if ((binoptab == rotl_optab || binoptab == rotr_optab)
1891 34 : && is_int_mode (mode, &int_mode)
1892 1 : && CONST_INT_P (op1)
1893 0 : && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
1894 0 : && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
1895 189408 : && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
1896 : {
1897 0 : rtx_insn *insns;
1898 0 : rtx into_target, outof_target;
1899 0 : rtx into_input, outof_input;
1900 0 : rtx inter;
1901 0 : int shift_count, left_shift, outof_word;
1902 :
1903 : /* If TARGET is the same as one of the operands, the REG_EQUAL note
1904 : won't be accurate, so use a new target. Do this also if target is not
1905 : a REG, first because having a register instead may open optimization
1906 : opportunities, and second because if target and op0 happen to be MEMs
1907 : designating the same location, we would risk clobbering it too early
1908 : in the code sequence we generate below. */
1909 0 : if (target == 0
1910 0 : || target == op0
1911 0 : || target == op1
1912 0 : || !REG_P (target)
1913 0 : || reg_overlap_mentioned_p (target, op0)
1914 0 : || reg_overlap_mentioned_p (target, op1)
1915 0 : || !valid_multiword_target_p (target))
1916 0 : target = gen_reg_rtx (int_mode);
1917 :
1918 0 : start_sequence ();
1919 :
1920 0 : shift_count = INTVAL (op1);
1921 :
1922 : /* OUTOF_* is the word we are shifting bits away from, and
1923 : INTO_* is the word that we are shifting bits towards, thus
1924 : they differ depending on the direction of the shift and
1925 : WORDS_BIG_ENDIAN. */
1926 :
1927 0 : left_shift = (binoptab == rotl_optab);
1928 0 : outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
1929 :
1930 0 : outof_target = operand_subword (target, outof_word, 1, int_mode);
1931 0 : into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
1932 :
1933 0 : outof_input = operand_subword_force (op0, outof_word, int_mode);
1934 0 : into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
1935 :
1936 0 : if (shift_count == BITS_PER_WORD)
1937 : {
1938 : /* This is just a word swap. */
1939 0 : emit_move_insn (outof_target, into_input);
1940 0 : emit_move_insn (into_target, outof_input);
1941 0 : inter = const0_rtx;
1942 : }
1943 : else
1944 : {
1945 0 : rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
1946 0 : HOST_WIDE_INT first_shift_count, second_shift_count;
1947 0 : optab reverse_unsigned_shift, unsigned_shift;
1948 :
1949 0 : reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1950 0 : ? lshr_optab : ashl_optab);
1951 :
1952 0 : unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
1953 0 : ? ashl_optab : lshr_optab);
1954 :
1955 0 : if (shift_count > BITS_PER_WORD)
1956 : {
1957 0 : first_shift_count = shift_count - BITS_PER_WORD;
1958 0 : second_shift_count = 2 * BITS_PER_WORD - shift_count;
1959 : }
1960 : else
1961 : {
1962 0 : first_shift_count = BITS_PER_WORD - shift_count;
1963 0 : second_shift_count = shift_count;
1964 : }
1965 0 : rtx first_shift_count_rtx
1966 0 : = gen_int_shift_amount (word_mode, first_shift_count);
1967 0 : rtx second_shift_count_rtx
1968 0 : = gen_int_shift_amount (word_mode, second_shift_count);
1969 :
1970 0 : into_temp1 = expand_binop (word_mode, unsigned_shift,
1971 : outof_input, first_shift_count_rtx,
1972 : NULL_RTX, unsignedp, next_methods);
1973 0 : into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1974 : into_input, second_shift_count_rtx,
1975 : NULL_RTX, unsignedp, next_methods);
1976 :
1977 0 : if (into_temp1 != 0 && into_temp2 != 0)
1978 0 : inter = expand_binop (word_mode, add_optab, into_temp1, into_temp2,
1979 : into_target, unsignedp, next_methods);
1980 : else
1981 : inter = 0;
1982 :
1983 0 : if (inter != 0 && inter != into_target)
1984 0 : emit_move_insn (into_target, inter);
1985 :
1986 0 : outof_temp1 = expand_binop (word_mode, unsigned_shift,
1987 : into_input, first_shift_count_rtx,
1988 : NULL_RTX, unsignedp, next_methods);
1989 0 : outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
1990 : outof_input, second_shift_count_rtx,
1991 : NULL_RTX, unsignedp, next_methods);
1992 :
1993 0 : if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
1994 0 : inter = expand_binop (word_mode, add_optab,
1995 : outof_temp1, outof_temp2,
1996 : outof_target, unsignedp, next_methods);
1997 :
1998 0 : if (inter != 0 && inter != outof_target)
1999 0 : emit_move_insn (outof_target, inter);
2000 : }
2001 :
2002 0 : insns = end_sequence ();
2003 :
2004 0 : if (inter != 0)
2005 : {
2006 0 : emit_insn (insns);
2007 0 : return target;
2008 : }
2009 : }
2010 :
2011 : /* These can be done a word at a time by propagating carries. */
2012 189408 : if ((binoptab == add_optab || binoptab == sub_optab)
2013 13703 : && is_int_mode (mode, &int_mode)
2014 0 : && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
2015 189408 : && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
2016 : {
2017 0 : unsigned int i;
2018 0 : optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
2019 0 : const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
2020 0 : rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
2021 0 : rtx xop0, xop1, xtarget;
2022 :
2023 : /* We can handle either a 1 or -1 value for the carry. If STORE_FLAG
2024 : value is one of those, use it. Otherwise, use 1 since it is the
2025 : one easiest to get. */
2026 : #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
2027 0 : int normalizep = STORE_FLAG_VALUE;
2028 : #else
2029 : int normalizep = 1;
2030 : #endif
2031 :
2032 : /* Prepare the operands. */
2033 0 : xop0 = force_reg (int_mode, op0);
2034 0 : xop1 = force_reg (int_mode, op1);
2035 :
2036 0 : xtarget = gen_reg_rtx (int_mode);
2037 :
2038 0 : if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
2039 : target = xtarget;
2040 :
2041 : /* Indicate for flow that the entire target reg is being set. */
2042 0 : if (REG_P (target))
2043 0 : emit_clobber (xtarget);
2044 :
2045 : /* Do the actual arithmetic. */
2046 0 : for (i = 0; i < nwords; i++)
2047 : {
2048 0 : int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
2049 0 : rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
2050 0 : rtx op0_piece = operand_subword_force (xop0, index, int_mode);
2051 0 : rtx op1_piece = operand_subword_force (xop1, index, int_mode);
2052 0 : rtx x;
2053 :
2054 : /* Main add/subtract of the input operands. */
2055 0 : x = expand_binop (word_mode, binoptab,
2056 : op0_piece, op1_piece,
2057 : target_piece, unsignedp, next_methods);
2058 0 : if (x == 0)
2059 : break;
2060 :
2061 0 : if (i + 1 < nwords)
2062 : {
2063 : /* Store carry from main add/subtract. */
2064 0 : carry_out = gen_reg_rtx (word_mode);
2065 0 : carry_out = emit_store_flag_force (carry_out,
2066 : (binoptab == add_optab
2067 : ? LT : GT),
2068 : x, op0_piece,
2069 : word_mode, 1, normalizep);
2070 : }
2071 :
2072 0 : if (i > 0)
2073 : {
2074 0 : rtx newx;
2075 :
2076 : /* Add/subtract previous carry to main result. */
2077 0 : newx = expand_binop (word_mode,
2078 : normalizep == 1 ? binoptab : otheroptab,
2079 : x, carry_in,
2080 : NULL_RTX, 1, next_methods);
2081 :
2082 0 : if (i + 1 < nwords)
2083 : {
2084 : /* Get out carry from adding/subtracting carry in. */
2085 0 : rtx carry_tmp = gen_reg_rtx (word_mode);
2086 0 : carry_tmp = emit_store_flag_force (carry_tmp,
2087 : (binoptab == add_optab
2088 : ? LT : GT),
2089 : newx, x,
2090 : word_mode, 1, normalizep);
2091 :
2092 : /* Logical-ior the two poss. carry together. */
2093 0 : carry_out = expand_binop (word_mode, ior_optab,
2094 : carry_out, carry_tmp,
2095 : carry_out, 0, next_methods);
2096 0 : if (carry_out == 0)
2097 : break;
2098 : }
2099 0 : emit_move_insn (target_piece, newx);
2100 : }
2101 : else
2102 : {
2103 0 : if (x != target_piece)
2104 0 : emit_move_insn (target_piece, x);
2105 : }
2106 :
2107 : carry_in = carry_out;
2108 : }
2109 :
2110 0 : if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
2111 : {
2112 0 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
2113 0 : || ! rtx_equal_p (target, xtarget))
2114 : {
2115 0 : rtx_insn *temp = emit_move_insn (target, xtarget);
2116 :
2117 0 : set_dst_reg_note (temp, REG_EQUAL,
2118 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2119 : int_mode, copy_rtx (xop0),
2120 : copy_rtx (xop1)),
2121 : target);
2122 : }
2123 : else
2124 : target = xtarget;
2125 :
2126 0 : return target;
2127 : }
2128 :
2129 : else
2130 0 : delete_insns_since (last);
2131 : }
2132 :
2133 : /* Attempt to synthesize double word multiplies using a sequence of word
2134 : mode multiplications. We first attempt to generate a sequence using a
2135 : more efficient unsigned widening multiply, and if that fails we then
2136 : try using a signed widening multiply. */
2137 :
2138 189408 : if (binoptab == smul_optab
2139 24984 : && is_int_mode (mode, &int_mode)
2140 17120 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2141 14221 : && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
2142 203629 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
2143 : {
2144 14221 : rtx product = NULL_RTX;
2145 14221 : if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
2146 : != CODE_FOR_nothing)
2147 : {
2148 14221 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2149 : true, methods);
2150 14221 : if (!product)
2151 0 : delete_insns_since (last);
2152 : }
2153 :
2154 0 : if (product == NULL_RTX
2155 0 : && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
2156 : != CODE_FOR_nothing))
2157 : {
2158 0 : product = expand_doubleword_mult (int_mode, op0, op1, target,
2159 : false, methods);
2160 0 : if (!product)
2161 0 : delete_insns_since (last);
2162 : }
2163 :
2164 14221 : if (product != NULL_RTX)
2165 : {
2166 14221 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2167 : {
2168 22282 : rtx_insn *move = emit_move_insn (target ? target : product,
2169 : product);
2170 14221 : set_dst_reg_note (move,
2171 : REG_EQUAL,
2172 : gen_rtx_fmt_ee (MULT, int_mode,
2173 : copy_rtx (op0),
2174 : copy_rtx (op1)),
2175 : target ? target : product);
2176 : }
2177 14221 : return product;
2178 : }
2179 : }
2180 :
2181 : /* Attempt to synthetize double word modulo by constant divisor. */
2182 175187 : if ((binoptab == umod_optab
2183 175187 : || binoptab == smod_optab
2184 175187 : || binoptab == udiv_optab
2185 : || binoptab == sdiv_optab)
2186 112558 : && optimize
2187 87797 : && CONST_INT_P (op1)
2188 22526 : && is_int_mode (mode, &int_mode)
2189 24807 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
2190 30992 : && optab_handler ((binoptab == umod_optab || binoptab == udiv_optab)
2191 : ? udivmod_optab : sdivmod_optab,
2192 : int_mode) == CODE_FOR_nothing
2193 19799 : && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
2194 19799 : && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
2195 19799 : && optimize_insn_for_speed_p ())
2196 : {
2197 19592 : rtx res = NULL_RTX;
2198 19592 : if ((binoptab == umod_optab || binoptab == smod_optab)
2199 5121 : && (INTVAL (op1) & 1) == 0)
2200 321 : res = expand_doubleword_mod (int_mode, op0, op1,
2201 : binoptab == umod_optab);
2202 : else
2203 : {
2204 19271 : rtx quot = expand_doubleword_divmod (int_mode, op0, op1, &res,
2205 : binoptab == umod_optab
2206 : || binoptab == udiv_optab);
2207 19271 : if (quot == NULL_RTX)
2208 18637 : res = NULL_RTX;
2209 634 : else if (binoptab == udiv_optab || binoptab == sdiv_optab)
2210 427 : res = quot;
2211 : }
2212 19592 : if (res != NULL_RTX)
2213 : {
2214 634 : if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
2215 : {
2216 751 : rtx_insn *move = emit_move_insn (target ? target : res,
2217 : res);
2218 634 : set_dst_reg_note (move, REG_EQUAL,
2219 : gen_rtx_fmt_ee (optab_to_code (binoptab),
2220 : int_mode, copy_rtx (op0), op1),
2221 : target ? target : res);
2222 : }
2223 634 : return res;
2224 : }
2225 : else
2226 18958 : delete_insns_since (last);
2227 : }
2228 :
2229 : /* It can't be open-coded in this mode.
2230 : Use a library call if one is available and caller says that's ok. */
2231 :
2232 174553 : libfunc = optab_libfunc (binoptab, mode);
2233 174553 : if (libfunc
2234 110321 : && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
2235 : {
2236 30641 : rtx_insn *insns;
2237 30641 : rtx op1x = op1;
2238 30641 : machine_mode op1_mode = mode;
2239 30641 : rtx value;
2240 :
2241 30641 : start_sequence ();
2242 :
2243 30641 : if (shift_optab_p (binoptab))
2244 : {
2245 0 : op1_mode = targetm.libgcc_shift_count_mode ();
2246 : /* Specify unsigned here,
2247 : since negative shift counts are meaningless. */
2248 0 : op1x = convert_to_mode (op1_mode, op1, 1);
2249 : }
2250 :
2251 30641 : if (GET_MODE (op0) != VOIDmode
2252 30589 : && GET_MODE (op0) != mode)
2253 0 : op0 = convert_to_mode (mode, op0, unsignedp);
2254 :
2255 : /* Pass 1 for NO_QUEUE so we don't lose any increments
2256 : if the libcall is cse'd or moved. */
2257 30641 : value = emit_library_call_value (libfunc,
2258 : NULL_RTX, LCT_CONST, mode,
2259 : op0, mode, op1x, op1_mode);
2260 :
2261 30641 : insns = end_sequence ();
2262 :
2263 30641 : bool trapv = trapv_binoptab_p (binoptab);
2264 30641 : target = gen_reg_rtx (mode);
2265 30641 : emit_libcall_block_1 (insns, target, value,
2266 : trapv ? NULL_RTX
2267 30125 : : gen_rtx_fmt_ee (optab_to_code (binoptab),
2268 : mode, op0, op1), trapv);
2269 :
2270 30641 : return target;
2271 : }
2272 :
2273 143912 : delete_insns_since (last);
2274 :
2275 : /* It can't be done in this mode. Can we do it in a wider mode? */
2276 :
2277 143912 : if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
2278 : || methods == OPTAB_MUST_WIDEN))
2279 : {
2280 : /* Caller says, don't even try. */
2281 131342 : delete_insns_since (entry_last);
2282 131342 : return 0;
2283 : }
2284 :
2285 : /* Compute the value of METHODS to pass to recursive calls.
2286 : Don't allow widening to be tried recursively. */
2287 :
2288 12570 : methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
2289 :
2290 : /* Look for a wider mode of the same class for which it appears we can do
2291 : the operation. */
2292 :
2293 12570 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2294 : {
2295 : /* This code doesn't make sense for conversion optabs, since we
2296 : wouldn't then want to extend the operands to be the same size
2297 : as the result. */
2298 12570 : gcc_assert (!convert_optab_p (binoptab));
2299 39608 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2300 : {
2301 27039 : if (optab_handler (binoptab, wider_mode)
2302 27039 : || (methods == OPTAB_LIB
2303 2 : && optab_libfunc (binoptab, wider_mode)))
2304 : {
2305 1 : rtx xop0 = op0, xop1 = op1;
2306 1 : bool no_extend = false;
2307 :
2308 : /* For certain integer operations, we need not actually extend
2309 : the narrow operands, as long as we will truncate
2310 : the results to the same narrowness. */
2311 :
2312 1 : if ((binoptab == ior_optab || binoptab == and_optab
2313 1 : || binoptab == xor_optab
2314 1 : || binoptab == add_optab || binoptab == sub_optab
2315 : || binoptab == smul_optab || binoptab == ashl_optab)
2316 0 : && mclass == MODE_INT)
2317 1 : no_extend = true;
2318 :
2319 1 : xop0 = widen_operand (xop0, wider_mode, mode,
2320 : unsignedp, no_extend);
2321 :
2322 : /* The second operand of a shift must always be extended. */
2323 2 : xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
2324 1 : no_extend && binoptab != ashl_optab);
2325 :
2326 1 : temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
2327 : unsignedp, methods);
2328 1 : if (temp)
2329 : {
2330 1 : if (mclass != MODE_INT
2331 1 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
2332 : {
2333 0 : if (target == 0)
2334 0 : target = gen_reg_rtx (mode);
2335 0 : convert_move (target, temp, 0);
2336 0 : return target;
2337 : }
2338 : else
2339 1 : return gen_lowpart (mode, temp);
2340 : }
2341 : else
2342 0 : delete_insns_since (last);
2343 : }
2344 : }
2345 : }
2346 :
2347 12569 : delete_insns_since (entry_last);
2348 12569 : return 0;
2349 : }
2350 :
2351 : /* Expand a binary operator which has both signed and unsigned forms.
2352 : UOPTAB is the optab for unsigned operations, and SOPTAB is for
2353 : signed operations.
2354 :
2355 : If we widen unsigned operands, we may use a signed wider operation instead
2356 : of an unsigned wider operation, since the result would be the same. */
2357 :
2358 : rtx
2359 94385 : sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
2360 : rtx op0, rtx op1, rtx target, int unsignedp,
2361 : enum optab_methods methods)
2362 : {
2363 94385 : rtx temp;
2364 94385 : optab direct_optab = unsignedp ? uoptab : soptab;
2365 94385 : bool save_enable;
2366 :
2367 : /* Do it without widening, if possible. */
2368 94385 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2369 : unsignedp, OPTAB_DIRECT);
2370 94385 : if (temp || methods == OPTAB_DIRECT)
2371 : return temp;
2372 :
2373 : /* Try widening to a signed int. Disable any direct use of any
2374 : signed insn in the current mode. */
2375 8267 : save_enable = swap_optab_enable (soptab, mode, false);
2376 :
2377 8267 : temp = expand_binop (mode, soptab, op0, op1, target,
2378 : unsignedp, OPTAB_WIDEN);
2379 :
2380 : /* For unsigned operands, try widening to an unsigned int. */
2381 8267 : if (!temp && unsignedp)
2382 4180 : temp = expand_binop (mode, uoptab, op0, op1, target,
2383 : unsignedp, OPTAB_WIDEN);
2384 8267 : if (temp || methods == OPTAB_WIDEN)
2385 4762 : goto egress;
2386 :
2387 : /* Use the right width libcall if that exists. */
2388 3505 : temp = expand_binop (mode, direct_optab, op0, op1, target,
2389 : unsignedp, OPTAB_LIB);
2390 3505 : if (temp || methods == OPTAB_LIB)
2391 3505 : goto egress;
2392 :
2393 : /* Must widen and use a libcall, use either signed or unsigned. */
2394 0 : temp = expand_binop (mode, soptab, op0, op1, target,
2395 : unsignedp, methods);
2396 0 : if (!temp && unsignedp)
2397 0 : temp = expand_binop (mode, uoptab, op0, op1, target,
2398 : unsignedp, methods);
2399 :
2400 0 : egress:
2401 : /* Undo the fiddling above. */
2402 8267 : if (save_enable)
2403 0 : swap_optab_enable (soptab, mode, true);
2404 : return temp;
2405 : }
2406 :
2407 : /* Generate code to perform an operation specified by UNOPPTAB
2408 : on operand OP0, with two results to TARG0 and TARG1.
2409 : We assume that the order of the operands for the instruction
2410 : is TARG0, TARG1, OP0.
2411 :
2412 : Either TARG0 or TARG1 may be zero, but what that means is that
2413 : the result is not actually wanted. We will generate it into
2414 : a dummy pseudo-reg and discard it. They may not both be zero.
2415 :
2416 : Returns true if this operation can be performed; false if not. */
2417 :
2418 : bool
2419 6 : expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
2420 : int unsignedp)
2421 : {
2422 6 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2423 6 : enum mode_class mclass;
2424 6 : machine_mode wider_mode;
2425 6 : rtx_insn *entry_last = get_last_insn ();
2426 6 : rtx_insn *last;
2427 :
2428 6 : mclass = GET_MODE_CLASS (mode);
2429 :
2430 6 : if (!targ0)
2431 0 : targ0 = gen_reg_rtx (mode);
2432 6 : if (!targ1)
2433 0 : targ1 = gen_reg_rtx (mode);
2434 :
2435 : /* Record where to go back to if we fail. */
2436 6 : last = get_last_insn ();
2437 :
2438 6 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
2439 : {
2440 6 : class expand_operand ops[3];
2441 6 : enum insn_code icode = optab_handler (unoptab, mode);
2442 :
2443 6 : create_fixed_operand (&ops[0], targ0);
2444 6 : create_fixed_operand (&ops[1], targ1);
2445 6 : create_convert_operand_from (&ops[2], op0, mode, unsignedp);
2446 6 : if (maybe_expand_insn (icode, 3, ops))
2447 6 : return true;
2448 : }
2449 :
2450 : /* It can't be done in this mode. Can we do it in a wider mode? */
2451 :
2452 0 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2453 : {
2454 0 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2455 : {
2456 0 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2457 : {
2458 0 : rtx t0 = gen_reg_rtx (wider_mode);
2459 0 : rtx t1 = gen_reg_rtx (wider_mode);
2460 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2461 :
2462 0 : if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
2463 : {
2464 0 : convert_move (targ0, t0, unsignedp);
2465 0 : convert_move (targ1, t1, unsignedp);
2466 0 : return true;
2467 : }
2468 : else
2469 0 : delete_insns_since (last);
2470 : }
2471 : }
2472 : }
2473 :
2474 0 : delete_insns_since (entry_last);
2475 0 : return false;
2476 : }
2477 :
2478 : /* Generate code to perform an operation specified by BINOPTAB
2479 : on operands OP0 and OP1, with two results to TARG1 and TARG2.
2480 : We assume that the order of the operands for the instruction
2481 : is TARG0, OP0, OP1, TARG1, which would fit a pattern like
2482 : [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
2483 :
2484 : Either TARG0 or TARG1 may be zero, but what that means is that
2485 : the result is not actually wanted. We will generate it into
2486 : a dummy pseudo-reg and discard it. They may not both be zero.
2487 :
2488 : Returns true if this operation can be performed; false if not. */
2489 :
2490 : bool
2491 101753 : expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
2492 : int unsignedp)
2493 : {
2494 101753 : machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
2495 101753 : enum mode_class mclass;
2496 101753 : machine_mode wider_mode;
2497 101753 : rtx_insn *entry_last = get_last_insn ();
2498 101753 : rtx_insn *last;
2499 :
2500 101753 : mclass = GET_MODE_CLASS (mode);
2501 :
2502 101753 : if (!targ0)
2503 31521 : targ0 = gen_reg_rtx (mode);
2504 101753 : if (!targ1)
2505 58764 : targ1 = gen_reg_rtx (mode);
2506 :
2507 : /* Record where to go back to if we fail. */
2508 101753 : last = get_last_insn ();
2509 :
2510 101753 : if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
2511 : {
2512 96955 : class expand_operand ops[4];
2513 96955 : enum insn_code icode = optab_handler (binoptab, mode);
2514 96955 : machine_mode mode0 = insn_data[icode].operand[1].mode;
2515 96955 : machine_mode mode1 = insn_data[icode].operand[2].mode;
2516 96955 : rtx xop0 = op0, xop1 = op1;
2517 :
2518 : /* If we are optimizing, force expensive constants into a register. */
2519 96955 : xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
2520 96955 : xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
2521 :
2522 96955 : create_fixed_operand (&ops[0], targ0);
2523 96955 : create_convert_operand_from (&ops[1], xop0, mode, unsignedp);
2524 96955 : create_convert_operand_from (&ops[2], xop1, mode, unsignedp);
2525 96955 : create_fixed_operand (&ops[3], targ1);
2526 96955 : if (maybe_expand_insn (icode, 4, ops))
2527 96955 : return true;
2528 0 : delete_insns_since (last);
2529 : }
2530 :
2531 : /* It can't be done in this mode. Can we do it in a wider mode? */
2532 :
2533 4798 : if (CLASS_HAS_WIDER_MODES_P (mclass))
2534 : {
2535 15228 : FOR_EACH_WIDER_MODE (wider_mode, mode)
2536 : {
2537 10430 : if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
2538 : {
2539 0 : rtx t0 = gen_reg_rtx (wider_mode);
2540 0 : rtx t1 = gen_reg_rtx (wider_mode);
2541 0 : rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
2542 0 : rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
2543 :
2544 0 : if (expand_twoval_binop (binoptab, cop0, cop1,
2545 : t0, t1, unsignedp))
2546 : {
2547 0 : convert_move (targ0, t0, unsignedp);
2548 0 : convert_move (targ1, t1, unsignedp);
2549 0 : return true;
2550 : }
2551 : else
2552 0 : delete_insns_since (last);
2553 : }
2554 : }
2555 : }
2556 :
2557 4798 : delete_insns_since (entry_last);
2558 4798 : return false;
2559 : }
2560 :
2561 : /* Expand the two-valued library call indicated by BINOPTAB, but
2562 : preserve only one of the values. If TARG0 is non-NULL, the first
2563 : value is placed into TARG0; otherwise the second value is placed
2564 : into TARG1. Exactly one of TARG0 and TARG1 must be non-NULL. The
2565 : value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
2566 : This routine assumes that the value returned by the library call is
2567 : as if the return value was of an integral mode twice as wide as the
2568 : mode of OP0. Returns 1 if the call was successful. */
2569 :
2570 : bool
2571 0 : expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
2572 : rtx targ0, rtx targ1, enum rtx_code code)
2573 : {
2574 0 : machine_mode mode;
2575 0 : machine_mode libval_mode;
2576 0 : rtx libval;
2577 0 : rtx_insn *insns;
2578 0 : rtx libfunc;
2579 :
2580 : /* Exactly one of TARG0 or TARG1 should be non-NULL. */
2581 0 : gcc_assert (!targ0 != !targ1);
2582 :
2583 0 : mode = GET_MODE (op0);
2584 0 : libfunc = optab_libfunc (binoptab, mode);
2585 0 : if (!libfunc)
2586 : return false;
2587 :
2588 : /* The value returned by the library function will have twice as
2589 : many bits as the nominal MODE. */
2590 0 : auto return_size = 2 * GET_MODE_BITSIZE (mode);
2591 0 : if (!smallest_int_mode_for_size (return_size).exists (&libval_mode))
2592 0 : return false;
2593 :
2594 0 : start_sequence ();
2595 0 : libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
2596 : libval_mode,
2597 : op0, mode,
2598 : op1, mode);
2599 : /* Get the part of VAL containing the value that we want. */
2600 0 : libval = simplify_gen_subreg (mode, libval, libval_mode,
2601 0 : targ0 ? 0 : GET_MODE_SIZE (mode));
2602 0 : insns = end_sequence ();
2603 : /* Move the into the desired location. */
2604 0 : emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
2605 : gen_rtx_fmt_ee (code, mode, op0, op1));
2606 :
2607 0 : return true;
2608 : }
2609 :
2610 :
2611 : /* Wrapper around expand_unop which takes an rtx code to specify
2612 : the operation to perform, not an optab pointer. All other
2613 : arguments are the same. */
2614 : rtx
2615 16948 : expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
2616 : rtx target, int unsignedp)
2617 : {
2618 16948 : optab unop = code_to_optab (code);
2619 16948 : gcc_assert (unop);
2620 :
2621 16948 : return expand_unop (mode, unop, op0, target, unsignedp);
2622 : }
2623 :
2624 : /* Try calculating
2625 : (clz:narrow x)
2626 : as
2627 : (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
2628 :
2629 : A similar operation can be used for clrsb. UNOPTAB says which operation
2630 : we are trying to expand. */
2631 : static rtx
2632 296 : widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
2633 : {
2634 296 : opt_scalar_int_mode wider_mode_iter;
2635 1215 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2636 : {
2637 919 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2638 919 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
2639 : {
2640 0 : rtx xop0, temp;
2641 0 : rtx_insn *last;
2642 :
2643 0 : last = get_last_insn ();
2644 :
2645 0 : if (target == 0)
2646 0 : target = gen_reg_rtx (mode);
2647 0 : xop0 = widen_operand (op0, wider_mode, mode,
2648 : unoptab != clrsb_optab, false);
2649 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
2650 : unoptab != clrsb_optab);
2651 0 : if (temp != 0)
2652 0 : temp = expand_binop
2653 0 : (wider_mode, sub_optab, temp,
2654 0 : gen_int_mode (GET_MODE_PRECISION (wider_mode)
2655 0 : - GET_MODE_PRECISION (mode),
2656 : wider_mode),
2657 : target, true, OPTAB_DIRECT);
2658 0 : if (temp == 0)
2659 0 : delete_insns_since (last);
2660 :
2661 0 : return temp;
2662 : }
2663 : }
2664 : return 0;
2665 : }
2666 :
2667 : /* Attempt to emit (clrsb:mode op0) as
2668 : (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
2669 : (const_int -1))
2670 : if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
2671 : or as
2672 : (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
2673 : (ashr:mode op0 (const_int prec-1)))
2674 : (const_int 1)))
2675 : otherwise. */
2676 :
2677 : static rtx
2678 91 : expand_clrsb_using_clz (scalar_int_mode mode, rtx op0, rtx target)
2679 : {
2680 91 : if (optimize_insn_for_size_p ()
2681 91 : || optab_handler (clz_optab, mode) == CODE_FOR_nothing)
2682 4 : return NULL_RTX;
2683 :
2684 87 : start_sequence ();
2685 87 : HOST_WIDE_INT val = 0;
2686 87 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) != 2
2687 87 : || val != GET_MODE_PRECISION (mode))
2688 : val = 0;
2689 : else
2690 : val = 1;
2691 :
2692 85 : rtx temp2 = op0;
2693 85 : if (!val)
2694 : {
2695 85 : temp2 = expand_binop (mode, ashl_optab, op0, const1_rtx,
2696 : NULL_RTX, 0, OPTAB_DIRECT);
2697 85 : if (!temp2)
2698 : {
2699 0 : fail:
2700 0 : end_sequence ();
2701 0 : return NULL_RTX;
2702 : }
2703 : }
2704 :
2705 87 : rtx temp = expand_binop (mode, ashr_optab, op0,
2706 87 : GEN_INT (GET_MODE_PRECISION (mode) - 1),
2707 : NULL_RTX, 0, OPTAB_DIRECT);
2708 87 : if (!temp)
2709 0 : goto fail;
2710 :
2711 87 : temp = expand_binop (mode, xor_optab, temp2, temp, NULL_RTX, 0,
2712 : OPTAB_DIRECT);
2713 87 : if (!temp)
2714 0 : goto fail;
2715 :
2716 87 : if (!val)
2717 : {
2718 85 : temp = expand_binop (mode, ior_optab, temp, const1_rtx,
2719 : NULL_RTX, 0, OPTAB_DIRECT);
2720 85 : if (!temp)
2721 0 : goto fail;
2722 : }
2723 87 : temp = expand_unop_direct (mode, clz_optab, temp, val ? NULL_RTX : target,
2724 : true);
2725 87 : if (!temp)
2726 0 : goto fail;
2727 87 : if (val)
2728 : {
2729 2 : temp = expand_binop (mode, add_optab, temp, constm1_rtx,
2730 : target, 0, OPTAB_DIRECT);
2731 2 : if (!temp)
2732 0 : goto fail;
2733 : }
2734 :
2735 87 : rtx_insn *seq = end_sequence ();
2736 :
2737 87 : add_equal_note (seq, temp, CLRSB, op0, NULL_RTX, mode);
2738 87 : emit_insn (seq);
2739 87 : return temp;
2740 : }
2741 :
2742 : static rtx expand_ffs (scalar_int_mode, rtx, rtx);
2743 :
2744 : /* Try calculating clz, ctz or ffs of a double-word quantity as two clz, ctz or
2745 : ffs operations on word-sized quantities, choosing which based on whether the
2746 : high (for clz) or low (for ctz and ffs) word is nonzero. */
2747 : static rtx
2748 252 : expand_doubleword_clz_ctz_ffs (scalar_int_mode mode, rtx op0, rtx target,
2749 : optab unoptab)
2750 : {
2751 252 : rtx xop0 = force_reg (mode, op0);
2752 252 : rtx subhi = gen_highpart (word_mode, xop0);
2753 252 : rtx sublo = gen_lowpart (word_mode, xop0);
2754 252 : rtx_code_label *hi0_label = gen_label_rtx ();
2755 252 : rtx_code_label *after_label = gen_label_rtx ();
2756 252 : rtx_insn *seq;
2757 252 : rtx temp, result;
2758 252 : int addend = 0;
2759 :
2760 : /* If we were not given a target, use a word_mode register, not a
2761 : 'mode' register. The result will fit, and nobody is expecting
2762 : anything bigger (the return type of __builtin_clz* is int). */
2763 252 : if (!target)
2764 0 : target = gen_reg_rtx (word_mode);
2765 :
2766 : /* In any case, write to a word_mode scratch in both branches of the
2767 : conditional, so we can ensure there is a single move insn setting
2768 : 'target' to tag a REG_EQUAL note on. */
2769 252 : result = gen_reg_rtx (word_mode);
2770 :
2771 252 : if (unoptab != clz_optab)
2772 47 : std::swap (subhi, sublo);
2773 :
2774 252 : start_sequence ();
2775 :
2776 : /* If the high word is not equal to zero,
2777 : then clz of the full value is clz of the high word. */
2778 252 : emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
2779 : word_mode, true, hi0_label);
2780 :
2781 252 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2782 252 : temp = expand_unop_direct (word_mode, unoptab, subhi, result, true);
2783 : else
2784 : {
2785 0 : gcc_assert (unoptab == ffs_optab);
2786 0 : temp = expand_ffs (word_mode, subhi, result);
2787 : }
2788 252 : if (!temp)
2789 0 : goto fail;
2790 :
2791 252 : if (temp != result)
2792 0 : convert_move (result, temp, true);
2793 :
2794 252 : emit_jump_insn (targetm.gen_jump (after_label));
2795 252 : emit_barrier ();
2796 :
2797 : /* Else clz of the full value is clz of the low word plus the number
2798 : of bits in the high word. Similarly for ctz/ffs of the high word,
2799 : except that ffs should be 0 when both words are zero. */
2800 252 : emit_label (hi0_label);
2801 :
2802 252 : if (unoptab == ffs_optab)
2803 : {
2804 0 : convert_move (result, const0_rtx, true);
2805 0 : emit_cmp_and_jump_insns (sublo, CONST0_RTX (word_mode), EQ, 0,
2806 : word_mode, true, after_label);
2807 : }
2808 :
2809 252 : if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
2810 252 : temp = expand_unop_direct (word_mode, unoptab, sublo, NULL_RTX, true);
2811 : else
2812 : {
2813 0 : gcc_assert (unoptab == ffs_optab);
2814 0 : temp = expand_unop_direct (word_mode, ctz_optab, sublo, NULL_RTX, true);
2815 0 : addend = 1;
2816 : }
2817 :
2818 252 : if (!temp)
2819 0 : goto fail;
2820 :
2821 504 : temp = expand_binop (word_mode, add_optab, temp,
2822 252 : gen_int_mode (GET_MODE_BITSIZE (word_mode) + addend,
2823 : word_mode),
2824 : result, true, OPTAB_DIRECT);
2825 252 : if (!temp)
2826 0 : goto fail;
2827 252 : if (temp != result)
2828 0 : convert_move (result, temp, true);
2829 :
2830 252 : emit_label (after_label);
2831 252 : convert_move (target, result, true);
2832 :
2833 252 : seq = end_sequence ();
2834 :
2835 252 : add_equal_note (seq, target, optab_to_code (unoptab), xop0, NULL_RTX, mode);
2836 252 : emit_insn (seq);
2837 252 : return target;
2838 :
2839 0 : fail:
2840 0 : end_sequence ();
2841 0 : return 0;
2842 : }
2843 :
2844 : /* Try calculating popcount of a double-word quantity as two popcount's of
2845 : word-sized quantities and summing up the results. */
2846 : static rtx
2847 0 : expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
2848 : {
2849 0 : rtx t0, t1, t;
2850 0 : rtx_insn *seq;
2851 :
2852 0 : start_sequence ();
2853 :
2854 0 : t0 = expand_unop_direct (word_mode, popcount_optab,
2855 : operand_subword_force (op0, 0, mode), NULL_RTX,
2856 : true);
2857 0 : t1 = expand_unop_direct (word_mode, popcount_optab,
2858 : operand_subword_force (op0, 1, mode), NULL_RTX,
2859 : true);
2860 0 : if (!t0 || !t1)
2861 : {
2862 0 : end_sequence ();
2863 0 : return NULL_RTX;
2864 : }
2865 :
2866 : /* If we were not given a target, use a word_mode register, not a
2867 : 'mode' register. The result will fit, and nobody is expecting
2868 : anything bigger (the return type of __builtin_popcount* is int). */
2869 0 : if (!target)
2870 0 : target = gen_reg_rtx (word_mode);
2871 :
2872 0 : t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
2873 :
2874 0 : seq = end_sequence ();
2875 :
2876 0 : add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
2877 0 : emit_insn (seq);
2878 0 : return t;
2879 : }
2880 :
2881 : /* Try calculating
2882 : (parity:wide x)
2883 : as
2884 : (parity:narrow (low (x) ^ high (x))) */
2885 : static rtx
2886 0 : expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
2887 : {
2888 0 : rtx t = expand_binop (word_mode, xor_optab,
2889 : operand_subword_force (op0, 0, mode),
2890 : operand_subword_force (op0, 1, mode),
2891 : NULL_RTX, 0, OPTAB_DIRECT);
2892 0 : return expand_unop (word_mode, parity_optab, t, target, true);
2893 : }
2894 :
2895 : /* Try calculating
2896 : (bswap:narrow x)
2897 : as
2898 : (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow)))
2899 : or similarly for bitreverse. */
2900 : static rtx
2901 161 : widen_bswap_or_bitreverse (scalar_int_mode mode, rtx op0, rtx target,
2902 : optab unoptab)
2903 : {
2904 161 : rtx x;
2905 161 : rtx_insn *last;
2906 161 : opt_scalar_int_mode wider_mode_iter;
2907 :
2908 641 : FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
2909 480 : if (optab_handler (unoptab, wider_mode_iter.require ())
2910 : != CODE_FOR_nothing)
2911 : break;
2912 :
2913 161 : if (!wider_mode_iter.exists ())
2914 : return NULL_RTX;
2915 :
2916 0 : scalar_int_mode wider_mode = wider_mode_iter.require ();
2917 0 : last = get_last_insn ();
2918 :
2919 0 : x = widen_operand (op0, wider_mode, mode, true, true);
2920 0 : x = expand_unop (wider_mode, unoptab, x, NULL_RTX, true);
2921 :
2922 0 : gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
2923 : && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
2924 0 : if (x != 0)
2925 0 : x = expand_shift (RSHIFT_EXPR, wider_mode, x,
2926 0 : GET_MODE_BITSIZE (wider_mode)
2927 0 : - GET_MODE_BITSIZE (mode),
2928 : NULL_RTX, true);
2929 :
2930 0 : if (x != 0)
2931 : {
2932 0 : if (target == 0)
2933 0 : target = gen_reg_rtx (mode);
2934 0 : emit_move_insn (target, gen_lowpart (mode, x));
2935 : }
2936 : else
2937 0 : delete_insns_since (last);
2938 :
2939 : return target;
2940 : }
2941 :
2942 : /* Try calculating bswap as two bswaps of two word-sized operands.
2943 : Similarly for bitreverse. */
2944 :
2945 : static rtx
2946 156 : expand_doubleword_bswap_or_bitreverse (machine_mode mode, rtx op, rtx target,
2947 : optab unoptab)
2948 : {
2949 156 : rtx t0, t1;
2950 :
2951 156 : t1 = expand_unop (word_mode, unoptab,
2952 : operand_subword_force (op, 0, mode), NULL_RTX, true);
2953 156 : t0 = expand_unop (word_mode, unoptab,
2954 : operand_subword_force (op, 1, mode), NULL_RTX, true);
2955 :
2956 156 : if (target == 0 || !valid_multiword_target_p (target))
2957 2 : target = gen_reg_rtx (mode);
2958 156 : if (REG_P (target))
2959 156 : emit_clobber (target);
2960 156 : emit_move_insn (operand_subword (target, 0, 1, mode), t0);
2961 156 : emit_move_insn (operand_subword (target, 1, 1, mode), t1);
2962 :
2963 156 : return target;
2964 : }
2965 :
2966 : /* Try calculating (bitreverse x) using masks and shifts. */
2967 :
2968 : static rtx
2969 5 : expand_bitreverse (scalar_int_mode mode, rtx op0, rtx target)
2970 : {
2971 5 : unsigned int precision = GET_MODE_BITSIZE (mode);
2972 5 : rtx_insn *last;
2973 :
2974 : /* Operation requires at least 4 bits (one nibble swap makes no sense below
2975 : that). */
2976 5 : if (precision < 4)
2977 : return NULL_RTX;
2978 :
2979 5 : if (rtx temp = widen_bswap_or_bitreverse (mode, op0, target,
2980 : bitreverse_optab))
2981 : return temp;
2982 :
2983 10 : if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
2984 5 : && optab_handler (bitreverse_optab, word_mode) != CODE_FOR_nothing)
2985 0 : if (rtx temp = expand_doubleword_bswap_or_bitreverse (mode, op0, target,
2986 : bitreverse_optab))
2987 0 : return temp;
2988 :
2989 5 : if (target == NULL_RTX
2990 5 : || target == op0
2991 5 : || reg_overlap_mentioned_p (target, op0))
2992 0 : target = gen_reg_rtx (mode);
2993 :
2994 5 : last = get_last_insn ();
2995 :
2996 5 : rtx x, lo, hi;
2997 :
2998 : /* Step 1: byte-swap (only meaningful for >= 16 bits). */
2999 5 : if (precision >= 16)
3000 : {
3001 4 : x = expand_unop (mode, bswap_optab, op0, NULL_RTX, true);
3002 4 : if (x == NULL_RTX)
3003 0 : goto fail;
3004 : }
3005 : else
3006 : x = op0;
3007 :
3008 : /* Step 2: swap nibbles within each byte (shift=4, only for >= 8 bits). */
3009 5 : if (precision >= 8)
3010 : {
3011 5 : wide_int mask = wi::zero (precision);
3012 36 : for (unsigned int start = 0; start < precision; start += 8)
3013 62 : mask = wi::bit_or (mask, wi::shifted_mask (start, 4, false,
3014 31 : precision));
3015 :
3016 5 : rtx mask_rtx = immed_wide_int_const (mask, mode);
3017 :
3018 5 : hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (4),
3019 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3020 5 : if (hi == NULL_RTX) goto fail;
3021 5 : hi = expand_binop (mode, and_optab, hi, mask_rtx,
3022 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3023 5 : if (hi == NULL_RTX) goto fail;
3024 :
3025 5 : lo = expand_binop (mode, and_optab, x, mask_rtx,
3026 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3027 5 : if (lo == NULL_RTX) goto fail;
3028 5 : lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (4),
3029 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3030 5 : if (lo == NULL_RTX) goto fail;
3031 :
3032 5 : x = expand_binop (mode, ior_optab, hi, lo,
3033 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3034 5 : if (x == NULL_RTX) goto fail;
3035 5 : }
3036 :
3037 : /* Step 3: swap pairs of bits within each nibble (shift=2). */
3038 5 : {
3039 5 : wide_int mask = wi::zero (precision);
3040 67 : for (unsigned int start = 0; start < precision; start += 4)
3041 62 : mask = wi::bit_or (mask, wi::shifted_mask (start, 2, false, precision));
3042 :
3043 5 : rtx mask_rtx = immed_wide_int_const (mask, mode);
3044 :
3045 5 : hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (2),
3046 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3047 5 : if (hi == NULL_RTX) goto fail;
3048 5 : hi = expand_binop (mode, and_optab, hi, mask_rtx,
3049 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3050 5 : if (hi == NULL_RTX) goto fail;
3051 :
3052 5 : lo = expand_binop (mode, and_optab, x, mask_rtx,
3053 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3054 5 : if (lo == NULL_RTX) goto fail;
3055 5 : lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (2),
3056 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3057 5 : if (lo == NULL_RTX) goto fail;
3058 :
3059 5 : x = expand_binop (mode, ior_optab, hi, lo,
3060 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3061 5 : if (x == NULL_RTX) goto fail;
3062 0 : }
3063 :
3064 : /* Step 4: swap adjacent bits (shift=1). */
3065 5 : {
3066 5 : wide_int mask = wi::zero (precision);
3067 129 : for (unsigned int start = 0; start < precision; start += 2)
3068 248 : mask = wi::bit_or (mask, wi::shifted_mask (start, 1, false,
3069 124 : precision));
3070 :
3071 5 : rtx mask_rtx = immed_wide_int_const (mask, mode);
3072 :
3073 5 : hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (1),
3074 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3075 5 : if (hi == NULL_RTX) goto fail;
3076 5 : hi = expand_binop (mode, and_optab, hi, mask_rtx,
3077 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3078 5 : if (hi == NULL_RTX) goto fail;
3079 :
3080 5 : lo = expand_binop (mode, and_optab, x, mask_rtx,
3081 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3082 5 : if (lo == NULL_RTX) goto fail;
3083 5 : lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (1),
3084 : NULL_RTX, true, OPTAB_LIB_WIDEN);
3085 5 : if (lo == NULL_RTX) goto fail;
3086 :
3087 5 : x = expand_binop (mode, ior_optab, hi, lo,
3088 : target, true, OPTAB_LIB_WIDEN);
3089 5 : if (x == NULL_RTX) goto fail;
3090 0 : }
3091 :
3092 5 : if (x != target)
3093 0 : emit_move_insn (target, x);
3094 :
3095 : return target;
3096 :
3097 0 : fail:
3098 0 : delete_insns_since (last);
3099 0 : return NULL_RTX;
3100 : }
3101 :
3102 : /* Try calculating (parity x) as (and (popcount x) 1), where
3103 : popcount can also be done in a wider mode. */
3104 : static rtx
3105 11 : expand_parity (scalar_int_mode mode, rtx op0, rtx target)
3106 : {
3107 11 : enum mode_class mclass = GET_MODE_CLASS (mode);
3108 11 : opt_scalar_int_mode wider_mode_iter;
3109 11 : FOR_EACH_MODE_FROM (wider_mode_iter, mode)
3110 : {
3111 11 : scalar_int_mode wider_mode = wider_mode_iter.require ();
3112 11 : if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
3113 : {
3114 11 : rtx xop0, temp;
3115 11 : rtx_insn *last;
3116 :
3117 11 : last = get_last_insn ();
3118 :
3119 11 : if (target == 0 || GET_MODE (target) != wider_mode)
3120 6 : target = gen_reg_rtx (wider_mode);
3121 :
3122 11 : xop0 = widen_operand (op0, wider_mode, mode, true, false);
3123 11 : temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
3124 : true);
3125 11 : if (temp != 0)
3126 11 : temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
3127 : target, true, OPTAB_DIRECT);
3128 :
3129 11 : if (temp)
3130 : {
3131 11 : if (mclass != MODE_INT
3132 11 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3133 0 : return convert_to_mode (mode, temp, 0);
3134 : else
3135 11 : return gen_lowpart (mode, temp);
3136 : }
3137 : else
3138 0 : delete_insns_since (last);
3139 : }
3140 : }
3141 : return 0;
3142 : }
3143 :
3144 : /* Try calculating ctz(x) as K - clz(x & -x) ,
3145 : where K is GET_MODE_PRECISION(mode) - 1.
3146 :
3147 : Both __builtin_ctz and __builtin_clz are undefined at zero, so we
3148 : don't have to worry about what the hardware does in that case. (If
3149 : the clz instruction produces the usual value at 0, which is K, the
3150 : result of this code sequence will be -1; expand_ffs, below, relies
3151 : on this. It might be nice to have it be K instead, for consistency
3152 : with the (very few) processors that provide a ctz with a defined
3153 : value, but that would take one more instruction, and it would be
3154 : less convenient for expand_ffs anyway. */
3155 :
3156 : static rtx
3157 50 : expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
3158 : {
3159 50 : rtx_insn *seq;
3160 50 : rtx temp;
3161 :
3162 50 : if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
3163 : return 0;
3164 :
3165 0 : start_sequence ();
3166 :
3167 0 : temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
3168 0 : if (temp)
3169 0 : temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
3170 : true, OPTAB_DIRECT);
3171 0 : if (temp)
3172 0 : temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
3173 0 : if (temp)
3174 0 : temp = expand_binop (mode, sub_optab,
3175 0 : gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
3176 : temp, target,
3177 : true, OPTAB_DIRECT);
3178 0 : if (temp == 0)
3179 : {
3180 0 : end_sequence ();
3181 0 : return 0;
3182 : }
3183 :
3184 0 : seq = end_sequence ();
3185 :
3186 0 : add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
3187 0 : emit_insn (seq);
3188 0 : return temp;
3189 : }
3190 :
3191 :
3192 : /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
3193 : else with the sequence used by expand_clz.
3194 :
3195 : The ffs builtin promises to return zero for a zero value and ctz/clz
3196 : may have an undefined value in that case. If they do not give us a
3197 : convenient value, we have to generate a test and branch. */
3198 : static rtx
3199 0 : expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
3200 : {
3201 0 : HOST_WIDE_INT val = 0;
3202 0 : bool defined_at_zero = false;
3203 0 : rtx temp;
3204 0 : rtx_insn *seq;
3205 :
3206 0 : if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
3207 : {
3208 0 : start_sequence ();
3209 :
3210 0 : temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
3211 0 : if (!temp)
3212 0 : goto fail;
3213 :
3214 0 : defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
3215 : }
3216 0 : else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
3217 : {
3218 0 : start_sequence ();
3219 0 : temp = expand_ctz (mode, op0, 0);
3220 0 : if (!temp)
3221 0 : goto fail;
3222 :
3223 0 : if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
3224 : {
3225 0 : defined_at_zero = true;
3226 0 : val = (GET_MODE_PRECISION (mode) - 1) - val;
3227 : }
3228 : }
3229 : else
3230 : return 0;
3231 :
3232 0 : if (defined_at_zero && val == -1)
3233 : /* No correction needed at zero. */;
3234 : else
3235 : {
3236 : /* We don't try to do anything clever with the situation found
3237 : on some processors (eg Alpha) where ctz(0:mode) ==
3238 : bitsize(mode). If someone can think of a way to send N to -1
3239 : and leave alone all values in the range 0..N-1 (where N is a
3240 : power of two), cheaper than this test-and-branch, please add it.
3241 :
3242 : The test-and-branch is done after the operation itself, in case
3243 : the operation sets condition codes that can be recycled for this.
3244 : (This is true on i386, for instance.) */
3245 :
3246 0 : rtx_code_label *nonzero_label = gen_label_rtx ();
3247 0 : emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
3248 : mode, true, nonzero_label);
3249 :
3250 0 : convert_move (temp, GEN_INT (-1), false);
3251 0 : emit_label (nonzero_label);
3252 : }
3253 :
3254 : /* temp now has a value in the range -1..bitsize-1. ffs is supposed
3255 : to produce a value in the range 0..bitsize. */
3256 0 : temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
3257 : target, false, OPTAB_DIRECT);
3258 0 : if (!temp)
3259 0 : goto fail;
3260 :
3261 0 : seq = end_sequence ();
3262 :
3263 0 : add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
3264 0 : emit_insn (seq);
3265 0 : return temp;
3266 :
3267 0 : fail:
3268 0 : end_sequence ();
3269 0 : return 0;
3270 : }
3271 :
3272 : /* Expand a floating point absolute value or negation operation via a
3273 : logical operation on the sign bit. MODE is the mode of the operands
3274 : and FMODE is the scalar inner mode. */
3275 :
3276 : static rtx
3277 1101 : expand_absneg_bit (rtx_code code, machine_mode mode,
3278 : scalar_float_mode fmode, rtx op0, rtx target)
3279 : {
3280 1101 : int bitpos, word, nwords, i;
3281 1101 : machine_mode new_mode;
3282 1101 : scalar_int_mode imode;
3283 1101 : rtx temp;
3284 1101 : rtx_insn *insns;
3285 :
3286 1101 : auto op = code == NEG ? neg_optab : abs_optab;
3287 1101 : if (!get_absneg_bit_mode (op, mode, fmode, &bitpos).exists (&new_mode))
3288 0 : return NULL_RTX;
3289 :
3290 2202 : imode = as_a<scalar_int_mode> (GET_MODE_INNER (new_mode));
3291 2252 : if (VECTOR_MODE_P (mode) || GET_MODE_SIZE (fmode) <= UNITS_PER_WORD)
3292 : {
3293 : word = 0;
3294 : nwords = 1;
3295 : }
3296 : else
3297 : {
3298 17 : if (FLOAT_WORDS_BIG_ENDIAN)
3299 : word = (GET_MODE_BITSIZE (fmode) - bitpos) / BITS_PER_WORD;
3300 : else
3301 17 : word = bitpos / BITS_PER_WORD;
3302 17 : bitpos = bitpos % BITS_PER_WORD;
3303 34 : nwords = (GET_MODE_BITSIZE (fmode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
3304 : }
3305 :
3306 1101 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
3307 1101 : if (code == ABS)
3308 840 : mask = ~mask;
3309 :
3310 1101 : if (target == 0
3311 1101 : || target == op0
3312 633 : || reg_overlap_mentioned_p (target, op0)
3313 1730 : || (nwords > 1 && !valid_multiword_target_p (target)))
3314 472 : target = gen_reg_rtx (mode);
3315 :
3316 1101 : if (nwords > 1)
3317 : {
3318 17 : start_sequence ();
3319 :
3320 51 : for (i = 0; i < nwords; ++i)
3321 : {
3322 34 : rtx targ_piece = operand_subword (target, i, 1, mode);
3323 34 : rtx op0_piece = operand_subword_force (op0, i, mode);
3324 :
3325 34 : if (i == word)
3326 : {
3327 46 : temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
3328 : op0_piece,
3329 17 : immed_wide_int_const (mask, imode),
3330 : targ_piece, 1, OPTAB_LIB_WIDEN);
3331 17 : if (temp != targ_piece)
3332 0 : emit_move_insn (targ_piece, temp);
3333 : }
3334 : else
3335 17 : emit_move_insn (targ_piece, op0_piece);
3336 : }
3337 :
3338 17 : insns = end_sequence ();
3339 :
3340 17 : emit_insn (insns);
3341 : }
3342 : else
3343 : {
3344 1084 : rtx mask_rtx = immed_wide_int_const (mask, imode);
3345 1084 : if (VECTOR_MODE_P (new_mode))
3346 0 : mask_rtx = gen_const_vec_duplicate (new_mode, mask_rtx);
3347 1333 : temp = expand_binop (new_mode, code == ABS ? and_optab : xor_optab,
3348 1084 : gen_lowpart (new_mode, op0), mask_rtx,
3349 1084 : gen_lowpart (new_mode, target), 1, OPTAB_LIB_WIDEN);
3350 1084 : target = force_lowpart_subreg (mode, temp, new_mode);
3351 :
3352 1084 : set_dst_reg_note (get_last_insn (), REG_EQUAL,
3353 : gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
3354 : target);
3355 : }
3356 :
3357 1101 : return target;
3358 1101 : }
3359 :
3360 : /* As expand_unop, but will fail rather than attempt the operation in a
3361 : different mode or with a libcall. */
3362 : static rtx
3363 166655 : expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
3364 : int unsignedp)
3365 : {
3366 166655 : if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
3367 : {
3368 164242 : class expand_operand ops[2];
3369 164242 : enum insn_code icode = optab_handler (unoptab, mode);
3370 164242 : rtx_insn *last = get_last_insn ();
3371 164242 : rtx_insn *pat;
3372 :
3373 164242 : create_output_operand (&ops[0], target, mode);
3374 164242 : create_convert_operand_from (&ops[1], op0, mode, unsignedp);
3375 164242 : pat = maybe_gen_insn (icode, 2, ops);
3376 164242 : if (pat)
3377 : {
3378 163956 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
3379 203812 : && ! add_equal_note (pat, ops[0].value,
3380 : optab_to_code (unoptab),
3381 : ops[1].value, NULL_RTX, mode))
3382 : {
3383 1069 : delete_insns_since (last);
3384 164242 : return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
3385 : }
3386 :
3387 163173 : emit_insn (pat);
3388 :
3389 163173 : return ops[0].value;
3390 : }
3391 : }
3392 : return 0;
3393 : }
3394 :
3395 : /* Generate code to perform an operation specified by UNOPTAB
3396 : on operand OP0, with result having machine-mode MODE.
3397 :
3398 : UNSIGNEDP is for the case where we have to widen the operands
3399 : to perform the operation. It says to use zero-extension.
3400 :
3401 : If TARGET is nonzero, the value
3402 : is generated there, if it is convenient to do so.
3403 : In all cases an rtx is returned for the locus of the value;
3404 : this may or may not be TARGET. */
3405 :
3406 : rtx
3407 166064 : expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
3408 : int unsignedp)
3409 : {
3410 166064 : enum mode_class mclass = GET_MODE_CLASS (mode);
3411 166064 : machine_mode wider_mode;
3412 166064 : scalar_int_mode int_mode;
3413 166064 : scalar_float_mode float_mode;
3414 166064 : rtx temp;
3415 166064 : rtx libfunc;
3416 :
3417 166064 : temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
3418 166064 : if (temp)
3419 : return temp;
3420 :
3421 : /* It can't be done in this mode. Can we open-code it in a wider mode? */
3422 :
3423 : /* Widening (or narrowing) clz needs special treatment. */
3424 2413 : if (unoptab == clz_optab)
3425 : {
3426 205 : if (is_a <scalar_int_mode> (mode, &int_mode))
3427 : {
3428 205 : temp = widen_leading (int_mode, op0, target, unoptab);
3429 205 : if (temp)
3430 : return temp;
3431 :
3432 410 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3433 205 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3434 : {
3435 205 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target,
3436 : unoptab);
3437 205 : if (temp)
3438 : return temp;
3439 : }
3440 : }
3441 :
3442 0 : goto try_libcall;
3443 : }
3444 :
3445 2208 : if (unoptab == clrsb_optab)
3446 : {
3447 91 : if (is_a <scalar_int_mode> (mode, &int_mode))
3448 : {
3449 91 : temp = widen_leading (int_mode, op0, target, unoptab);
3450 91 : if (temp)
3451 : return temp;
3452 91 : temp = expand_clrsb_using_clz (int_mode, op0, target);
3453 91 : if (temp)
3454 : return temp;
3455 : }
3456 4 : goto try_libcall;
3457 : }
3458 :
3459 2117 : if (unoptab == popcount_optab
3460 618 : && is_a <scalar_int_mode> (mode, &int_mode)
3461 650 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3462 2 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3463 2117 : && optimize_insn_for_speed_p ())
3464 : {
3465 0 : temp = expand_doubleword_popcount (int_mode, op0, target);
3466 0 : if (temp)
3467 : return temp;
3468 : }
3469 :
3470 2117 : if (unoptab == parity_optab
3471 11 : && is_a <scalar_int_mode> (mode, &int_mode)
3472 11 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3473 0 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3474 0 : || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
3475 2117 : && optimize_insn_for_speed_p ())
3476 : {
3477 0 : temp = expand_doubleword_parity (int_mode, op0, target);
3478 0 : if (temp)
3479 : return temp;
3480 : }
3481 :
3482 : /* Widening (or narrowing) bswap needs special treatment. */
3483 2117 : if (unoptab == bswap_optab)
3484 : {
3485 : /* HImode is special because in this mode BSWAP is equivalent to ROTATE
3486 : or ROTATERT. First try these directly; if this fails, then try the
3487 : obvious pair of shifts with allowed widening, as this will probably
3488 : be always more efficient than the other fallback methods. */
3489 156 : if (mode == HImode)
3490 : {
3491 0 : rtx_insn *last;
3492 0 : rtx temp1, temp2;
3493 :
3494 0 : if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
3495 : {
3496 0 : temp = expand_binop (mode, rotl_optab, op0,
3497 : gen_int_shift_amount (mode, 8),
3498 : target, unsignedp, OPTAB_DIRECT);
3499 0 : if (temp)
3500 : return temp;
3501 : }
3502 :
3503 0 : if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
3504 : {
3505 0 : temp = expand_binop (mode, rotr_optab, op0,
3506 : gen_int_shift_amount (mode, 8),
3507 : target, unsignedp, OPTAB_DIRECT);
3508 0 : if (temp)
3509 : return temp;
3510 : }
3511 :
3512 0 : last = get_last_insn ();
3513 :
3514 0 : temp1 = expand_binop (mode, ashl_optab, op0,
3515 : gen_int_shift_amount (mode, 8), NULL_RTX,
3516 : unsignedp, OPTAB_WIDEN);
3517 0 : temp2 = expand_binop (mode, lshr_optab, op0,
3518 : gen_int_shift_amount (mode, 8), NULL_RTX,
3519 : unsignedp, OPTAB_WIDEN);
3520 0 : if (temp1 && temp2)
3521 : {
3522 0 : temp = expand_binop (mode, ior_optab, temp1, temp2, target,
3523 : unsignedp, OPTAB_WIDEN);
3524 0 : if (temp)
3525 : return temp;
3526 : }
3527 :
3528 0 : delete_insns_since (last);
3529 : }
3530 :
3531 156 : if (is_a <scalar_int_mode> (mode, &int_mode))
3532 : {
3533 156 : temp = widen_bswap_or_bitreverse (int_mode, op0, target, unoptab);
3534 156 : if (temp)
3535 : return temp;
3536 :
3537 : /* We do not provide a 128-bit bswap in libgcc so force the use of
3538 : a double bswap for 64-bit targets. */
3539 312 : if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3540 156 : && (UNITS_PER_WORD == 8
3541 148 : || optab_handler (unoptab, word_mode) != CODE_FOR_nothing))
3542 : {
3543 156 : temp = expand_doubleword_bswap_or_bitreverse (mode, op0, target,
3544 : unoptab);
3545 156 : if (temp)
3546 : return temp;
3547 : }
3548 : }
3549 :
3550 0 : goto try_libcall;
3551 : }
3552 :
3553 1961 : if (unoptab == bitreverse_optab && is_a <scalar_int_mode> (mode, &int_mode))
3554 5 : if (rtx tem = expand_bitreverse (int_mode, op0, target))
3555 : return tem;
3556 :
3557 : /* Neg should be tried via expand_absneg_bit before widening. */
3558 1956 : if (optab_to_code (unoptab) == NEG)
3559 : {
3560 : /* Try negating floating point values by flipping the sign bit. */
3561 702 : if (is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3562 : {
3563 261 : temp = expand_absneg_bit (NEG, mode, float_mode, op0, target);
3564 261 : if (temp)
3565 : return temp;
3566 : }
3567 :
3568 : /* If there is no negation pattern, and we have no negative zero,
3569 : try subtracting from zero. */
3570 90 : if (!HONOR_SIGNED_ZEROS (mode))
3571 : {
3572 90 : temp = expand_binop (mode, (unoptab == negv_optab
3573 : ? subv_optab : sub_optab),
3574 : CONST0_RTX (mode), op0, target,
3575 : unsignedp, OPTAB_DIRECT);
3576 90 : if (temp)
3577 : return temp;
3578 : }
3579 : }
3580 :
3581 : /* ABS also needs to be handled similarly. */
3582 1695 : if (optab_to_code (unoptab) == ABS
3583 2621 : && is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
3584 : {
3585 840 : temp = expand_absneg_bit (ABS, mode, float_mode, op0, target);
3586 840 : if (temp)
3587 : return temp;
3588 : }
3589 :
3590 855 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3591 3824 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3592 : {
3593 2969 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
3594 : {
3595 0 : rtx xop0 = op0;
3596 0 : rtx_insn *last = get_last_insn ();
3597 :
3598 : /* For certain operations, we need not actually extend
3599 : the narrow operand, as long as we will truncate the
3600 : results to the same narrowness. */
3601 :
3602 0 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3603 0 : (unoptab == neg_optab
3604 0 : || unoptab == one_cmpl_optab)
3605 0 : && mclass == MODE_INT);
3606 :
3607 0 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3608 : unsignedp);
3609 :
3610 0 : if (temp)
3611 : {
3612 0 : if (mclass != MODE_INT
3613 0 : || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
3614 : {
3615 0 : if (target == 0)
3616 0 : target = gen_reg_rtx (mode);
3617 0 : convert_move (target, temp, 0);
3618 0 : return target;
3619 : }
3620 : else
3621 0 : return gen_lowpart (mode, temp);
3622 : }
3623 : else
3624 0 : delete_insns_since (last);
3625 : }
3626 : }
3627 :
3628 : /* These can be done a word at a time. */
3629 855 : if (unoptab == one_cmpl_optab
3630 0 : && is_int_mode (mode, &int_mode)
3631 0 : && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
3632 855 : && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
3633 : {
3634 0 : int i;
3635 0 : rtx_insn *insns;
3636 :
3637 0 : if (target == 0
3638 0 : || target == op0
3639 0 : || reg_overlap_mentioned_p (target, op0)
3640 0 : || !valid_multiword_target_p (target))
3641 0 : target = gen_reg_rtx (int_mode);
3642 :
3643 0 : start_sequence ();
3644 :
3645 : /* Do the actual arithmetic. */
3646 0 : for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
3647 : {
3648 0 : rtx target_piece = operand_subword (target, i, 1, int_mode);
3649 0 : rtx x = expand_unop (word_mode, unoptab,
3650 0 : operand_subword_force (op0, i, int_mode),
3651 : target_piece, unsignedp);
3652 :
3653 0 : if (target_piece != x)
3654 0 : emit_move_insn (target_piece, x);
3655 : }
3656 :
3657 0 : insns = end_sequence ();
3658 :
3659 0 : emit_insn (insns);
3660 0 : return target;
3661 : }
3662 :
3663 : /* Emit ~op0 as op0 ^ -1. */
3664 855 : if (unoptab == one_cmpl_optab
3665 0 : && (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
3666 855 : && optab_handler (xor_optab, mode) != CODE_FOR_nothing)
3667 : {
3668 0 : temp = expand_binop (mode, xor_optab, op0, CONSTM1_RTX (mode),
3669 : target, unsignedp, OPTAB_DIRECT);
3670 0 : if (temp)
3671 : return temp;
3672 : }
3673 :
3674 : /* Try calculating parity (x) as popcount (x) % 2. */
3675 855 : if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
3676 : {
3677 11 : temp = expand_parity (int_mode, op0, target);
3678 11 : if (temp)
3679 : return temp;
3680 : }
3681 :
3682 : /* Try implementing ffs (x) in terms of clz (x). */
3683 844 : if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
3684 : {
3685 0 : temp = expand_ffs (int_mode, op0, target);
3686 0 : if (temp)
3687 : return temp;
3688 : }
3689 :
3690 : /* Try implementing ctz (x) in terms of clz (x). */
3691 844 : if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
3692 : {
3693 50 : temp = expand_ctz (int_mode, op0, target);
3694 50 : if (temp)
3695 : return temp;
3696 : }
3697 :
3698 844 : if ((unoptab == ctz_optab || unoptab == ffs_optab)
3699 50 : && optimize_insn_for_speed_p ()
3700 47 : && is_a <scalar_int_mode> (mode, &int_mode)
3701 94 : && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
3702 891 : && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
3703 0 : || optab_handler (ctz_optab, word_mode) != CODE_FOR_nothing))
3704 : {
3705 47 : temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target, unoptab);
3706 47 : if (temp)
3707 : return temp;
3708 : }
3709 :
3710 801 : try_libcall:
3711 : /* Now try a library call in this mode. */
3712 801 : libfunc = optab_libfunc (unoptab, mode);
3713 801 : if (libfunc)
3714 : {
3715 517 : rtx_insn *insns;
3716 517 : rtx value;
3717 517 : rtx eq_value;
3718 517 : machine_mode outmode = mode;
3719 :
3720 : /* All of these functions return small values. Thus we choose to
3721 : have them return something that isn't a double-word. */
3722 517 : if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
3723 : || unoptab == clrsb_optab || unoptab == popcount_optab
3724 517 : || unoptab == parity_optab)
3725 427 : outmode
3726 427 : = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
3727 : optab_libfunc (unoptab, mode)));
3728 :
3729 517 : start_sequence ();
3730 :
3731 : /* Pass 1 for NO_QUEUE so we don't lose any increments
3732 : if the libcall is cse'd or moved. */
3733 517 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
3734 : op0, mode);
3735 517 : insns = end_sequence ();
3736 :
3737 517 : target = gen_reg_rtx (outmode);
3738 517 : bool trapv = trapv_unoptab_p (unoptab);
3739 517 : if (trapv)
3740 : eq_value = NULL_RTX;
3741 : else
3742 : {
3743 427 : eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
3744 1281 : if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
3745 397 : eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
3746 90 : else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
3747 0 : eq_value = simplify_gen_unary (ZERO_EXTEND,
3748 : outmode, eq_value, mode);
3749 : }
3750 517 : emit_libcall_block_1 (insns, target, value, eq_value, trapv);
3751 :
3752 517 : return target;
3753 : }
3754 :
3755 : /* It can't be done in this mode. Can we do it in a wider mode? */
3756 :
3757 284 : if (CLASS_HAS_WIDER_MODES_P (mclass))
3758 : {
3759 627 : FOR_EACH_WIDER_MODE (wider_mode, mode)
3760 : {
3761 541 : if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
3762 541 : || optab_libfunc (unoptab, wider_mode))
3763 : {
3764 198 : rtx xop0 = op0;
3765 198 : rtx_insn *last = get_last_insn ();
3766 :
3767 : /* For certain operations, we need not actually extend
3768 : the narrow operand, as long as we will truncate the
3769 : results to the same narrowness. */
3770 198 : xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
3771 198 : (unoptab == neg_optab
3772 198 : || unoptab == one_cmpl_optab
3773 198 : || unoptab == bswap_optab)
3774 198 : && mclass == MODE_INT);
3775 :
3776 198 : temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
3777 : unsignedp);
3778 :
3779 : /* If we are generating clz using wider mode, adjust the
3780 : result. Similarly for clrsb. */
3781 198 : if ((unoptab == clz_optab || unoptab == clrsb_optab)
3782 1 : && temp != 0)
3783 : {
3784 1 : scalar_int_mode wider_int_mode
3785 1 : = as_a <scalar_int_mode> (wider_mode);
3786 1 : int_mode = as_a <scalar_int_mode> (mode);
3787 1 : temp = expand_binop
3788 1 : (wider_mode, sub_optab, temp,
3789 1 : gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
3790 2 : - GET_MODE_PRECISION (int_mode),
3791 : wider_int_mode),
3792 : target, true, OPTAB_DIRECT);
3793 : }
3794 :
3795 : /* Likewise for bswap. */
3796 198 : if (unoptab == bswap_optab && temp != 0)
3797 : {
3798 0 : scalar_int_mode wider_int_mode
3799 0 : = as_a <scalar_int_mode> (wider_mode);
3800 0 : int_mode = as_a <scalar_int_mode> (mode);
3801 0 : gcc_assert (GET_MODE_PRECISION (wider_int_mode)
3802 : == GET_MODE_BITSIZE (wider_int_mode)
3803 : && GET_MODE_PRECISION (int_mode)
3804 : == GET_MODE_BITSIZE (int_mode));
3805 :
3806 0 : temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
3807 0 : GET_MODE_BITSIZE (wider_int_mode)
3808 0 : - GET_MODE_BITSIZE (int_mode),
3809 : NULL_RTX, true);
3810 : }
3811 :
3812 198 : if (temp)
3813 : {
3814 198 : if (mclass != MODE_INT)
3815 : {
3816 0 : if (target == 0)
3817 0 : target = gen_reg_rtx (mode);
3818 0 : convert_move (target, temp, 0);
3819 0 : return target;
3820 : }
3821 : else
3822 198 : return gen_lowpart (mode, temp);
3823 : }
3824 : else
3825 0 : delete_insns_since (last);
3826 : }
3827 : }
3828 : }
3829 :
3830 : /* One final attempt at implementing negation via subtraction,
3831 : this time allowing widening of the operand. */
3832 86 : if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
3833 : {
3834 0 : rtx temp;
3835 0 : temp = expand_binop (mode,
3836 : unoptab == negv_optab ? subv_optab : sub_optab,
3837 : CONST0_RTX (mode), op0,
3838 : target, unsignedp, OPTAB_LIB_WIDEN);
3839 0 : if (temp)
3840 : return temp;
3841 : }
3842 :
3843 : return 0;
3844 : }
3845 :
3846 : /* Emit code to compute the absolute value of OP0, with result to
3847 : TARGET if convenient. (TARGET may be 0.) The return value says
3848 : where the result actually is to be found.
3849 :
3850 : MODE is the mode of the operand; the mode of the result is
3851 : different but can be deduced from MODE.
3852 :
3853 : */
3854 :
3855 : rtx
3856 25033 : expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
3857 : int result_unsignedp)
3858 : {
3859 25033 : rtx temp;
3860 :
3861 25033 : if (GET_MODE_CLASS (mode) != MODE_INT
3862 5076 : || ! flag_trapv)
3863 : result_unsignedp = 1;
3864 :
3865 : /* First try to do it with a special abs instruction. */
3866 25119 : temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
3867 : op0, target, 0);
3868 25033 : if (temp != 0)
3869 : return temp;
3870 :
3871 : /* If we have a MAX insn, we can do this as MAX (x, -x). */
3872 86 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
3873 86 : && !HONOR_SIGNED_ZEROS (mode))
3874 : {
3875 86 : rtx_insn *last = get_last_insn ();
3876 :
3877 172 : temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3878 : op0, NULL_RTX, 0);
3879 86 : if (temp != 0)
3880 86 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3881 : OPTAB_WIDEN);
3882 :
3883 86 : if (temp != 0)
3884 : return temp;
3885 :
3886 0 : delete_insns_since (last);
3887 : }
3888 :
3889 : /* If this machine has expensive jumps, we can do integer absolute
3890 : value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
3891 : where W is the width of MODE. */
3892 :
3893 0 : scalar_int_mode int_mode;
3894 0 : if (is_int_mode (mode, &int_mode)
3895 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
3896 : false) >= 2)
3897 : {
3898 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
3899 0 : GET_MODE_PRECISION (int_mode) - 1,
3900 : NULL_RTX, 0);
3901 :
3902 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
3903 : OPTAB_LIB_WIDEN);
3904 0 : if (temp != 0)
3905 0 : temp = expand_binop (int_mode,
3906 : result_unsignedp ? sub_optab : subv_optab,
3907 : temp, extended, target, 0, OPTAB_LIB_WIDEN);
3908 :
3909 0 : if (temp != 0)
3910 : return temp;
3911 : }
3912 :
3913 : return NULL_RTX;
3914 : }
3915 :
3916 : rtx
3917 25009 : expand_abs (machine_mode mode, rtx op0, rtx target,
3918 : int result_unsignedp, int safe)
3919 : {
3920 25009 : rtx temp;
3921 25009 : rtx_code_label *op1;
3922 :
3923 25009 : if (GET_MODE_CLASS (mode) != MODE_INT
3924 5052 : || ! flag_trapv)
3925 24923 : result_unsignedp = 1;
3926 :
3927 25009 : temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
3928 25009 : if (temp != 0)
3929 : return temp;
3930 :
3931 : /* If that does not win, use conditional jump and negate. */
3932 :
3933 : /* It is safe to use the target if it is the same
3934 : as the source if this is also a pseudo register */
3935 0 : if (op0 == target && REG_P (op0)
3936 0 : && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
3937 : safe = 1;
3938 :
3939 0 : op1 = gen_label_rtx ();
3940 0 : if (target == 0 || ! safe
3941 0 : || GET_MODE (target) != mode
3942 0 : || (MEM_P (target) && MEM_VOLATILE_P (target))
3943 0 : || (REG_P (target)
3944 0 : && REGNO (target) < FIRST_PSEUDO_REGISTER))
3945 0 : target = gen_reg_rtx (mode);
3946 :
3947 0 : emit_move_insn (target, op0);
3948 0 : NO_DEFER_POP;
3949 :
3950 0 : do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
3951 : NULL_RTX, NULL, op1,
3952 : profile_probability::uninitialized ());
3953 :
3954 0 : op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
3955 : target, target, 0);
3956 0 : if (op0 != target)
3957 0 : emit_move_insn (target, op0);
3958 0 : emit_label (op1);
3959 0 : OK_DEFER_POP;
3960 0 : return target;
3961 : }
3962 :
3963 : /* Emit code to compute the one's complement absolute value of OP0
3964 : (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
3965 : (TARGET may be NULL_RTX.) The return value says where the result
3966 : actually is to be found.
3967 :
3968 : MODE is the mode of the operand; the mode of the result is
3969 : different but can be deduced from MODE. */
3970 :
3971 : rtx
3972 0 : expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
3973 : {
3974 0 : rtx temp;
3975 :
3976 : /* Not applicable for floating point modes. */
3977 0 : if (FLOAT_MODE_P (mode))
3978 : return NULL_RTX;
3979 :
3980 : /* If we have a MAX insn, we can do this as MAX (x, ~x). */
3981 0 : if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
3982 : {
3983 0 : rtx_insn *last = get_last_insn ();
3984 :
3985 0 : temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
3986 0 : if (temp != 0)
3987 0 : temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
3988 : OPTAB_WIDEN);
3989 :
3990 0 : if (temp != 0)
3991 : return temp;
3992 :
3993 0 : delete_insns_since (last);
3994 : }
3995 :
3996 : /* If this machine has expensive jumps, we can do one's complement
3997 : absolute value of X as (((signed) x >> (W-1)) ^ x). */
3998 :
3999 0 : scalar_int_mode int_mode;
4000 0 : if (is_int_mode (mode, &int_mode)
4001 0 : && BRANCH_COST (optimize_insn_for_speed_p (),
4002 : false) >= 2)
4003 : {
4004 0 : rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
4005 0 : GET_MODE_PRECISION (int_mode) - 1,
4006 : NULL_RTX, 0);
4007 :
4008 0 : temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
4009 : OPTAB_LIB_WIDEN);
4010 :
4011 0 : if (temp != 0)
4012 : return temp;
4013 : }
4014 :
4015 : return NULL_RTX;
4016 : }
4017 :
4018 : /* A subroutine of expand_copysign, perform the copysign operation using the
4019 : abs and neg primitives advertised to exist on the target. The assumption
4020 : is that we have a split register file, and leaving op0 in fp registers,
4021 : and not playing with subregs so much, will help the register allocator. */
4022 :
4023 : static rtx
4024 11604 : expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
4025 : int bitpos, bool op0_is_abs)
4026 : {
4027 11604 : scalar_int_mode imode;
4028 11604 : enum insn_code icode;
4029 11604 : rtx sign;
4030 11604 : rtx_code_label *label;
4031 :
4032 11604 : if (target == op1)
4033 132 : target = NULL_RTX;
4034 :
4035 : /* Check if the back end provides an insn that handles signbit for the
4036 : argument's mode. */
4037 11604 : icode = optab_handler (signbit_optab, mode);
4038 11604 : if (icode != CODE_FOR_nothing)
4039 : {
4040 11564 : imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
4041 11564 : sign = gen_reg_rtx (imode);
4042 11564 : emit_unop_insn (icode, sign, op1, UNKNOWN);
4043 : }
4044 : else
4045 : {
4046 100 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
4047 : {
4048 40 : if (!int_mode_for_mode (mode).exists (&imode))
4049 0 : return NULL_RTX;
4050 40 : op1 = gen_lowpart (imode, op1);
4051 : }
4052 : else
4053 : {
4054 0 : int word;
4055 :
4056 0 : imode = word_mode;
4057 0 : if (FLOAT_WORDS_BIG_ENDIAN)
4058 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
4059 : else
4060 0 : word = bitpos / BITS_PER_WORD;
4061 0 : bitpos = bitpos % BITS_PER_WORD;
4062 0 : op1 = operand_subword_force (op1, word, mode);
4063 : }
4064 :
4065 40 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
4066 80 : sign = expand_binop (imode, and_optab, op1,
4067 80 : immed_wide_int_const (mask, imode),
4068 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4069 40 : }
4070 :
4071 11604 : if (!op0_is_abs)
4072 : {
4073 150 : op0 = expand_unop (mode, abs_optab, op0, target, 0);
4074 150 : if (op0 == NULL)
4075 : return NULL_RTX;
4076 : target = op0;
4077 : }
4078 : else
4079 : {
4080 11454 : if (target == NULL_RTX)
4081 84 : target = copy_to_reg (op0);
4082 : else
4083 11370 : emit_move_insn (target, op0);
4084 : }
4085 :
4086 11604 : label = gen_label_rtx ();
4087 11604 : emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
4088 :
4089 11604 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
4090 11454 : op0 = simplify_unary_operation (NEG, mode, op0, mode);
4091 : else
4092 150 : op0 = expand_unop (mode, neg_optab, op0, target, 0);
4093 11604 : if (op0 != target)
4094 11454 : emit_move_insn (target, op0);
4095 :
4096 11604 : emit_label (label);
4097 :
4098 11604 : return target;
4099 : }
4100 :
4101 :
4102 : /* A subroutine of expand_copysign, perform the entire copysign operation
4103 : with integer bitmasks. BITPOS is the position of the sign bit; OP0_IS_ABS
4104 : is true if op0 is known to have its sign bit clear. */
4105 :
4106 : static rtx
4107 68 : expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
4108 : int bitpos, bool op0_is_abs)
4109 : {
4110 68 : scalar_int_mode imode;
4111 68 : int word, nwords, i;
4112 68 : rtx temp;
4113 68 : rtx_insn *insns;
4114 :
4115 156 : if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
4116 : {
4117 68 : if (!int_mode_for_mode (mode).exists (&imode))
4118 0 : return NULL_RTX;
4119 : word = 0;
4120 : nwords = 1;
4121 : }
4122 : else
4123 : {
4124 0 : imode = word_mode;
4125 :
4126 0 : if (FLOAT_WORDS_BIG_ENDIAN)
4127 : word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
4128 : else
4129 0 : word = bitpos / BITS_PER_WORD;
4130 0 : bitpos = bitpos % BITS_PER_WORD;
4131 0 : nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
4132 : }
4133 :
4134 68 : wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
4135 :
4136 68 : if (target == 0
4137 68 : || target == op0
4138 68 : || target == op1
4139 64 : || reg_overlap_mentioned_p (target, op0)
4140 64 : || reg_overlap_mentioned_p (target, op1)
4141 132 : || (nwords > 1 && !valid_multiword_target_p (target)))
4142 4 : target = gen_reg_rtx (mode);
4143 :
4144 68 : if (nwords > 1)
4145 : {
4146 0 : start_sequence ();
4147 :
4148 0 : for (i = 0; i < nwords; ++i)
4149 : {
4150 0 : rtx targ_piece = operand_subword (target, i, 1, mode);
4151 0 : rtx op0_piece = operand_subword_force (op0, i, mode);
4152 :
4153 0 : if (i == word)
4154 : {
4155 0 : if (!op0_is_abs)
4156 0 : op0_piece
4157 0 : = expand_binop (imode, and_optab, op0_piece,
4158 0 : immed_wide_int_const (~mask, imode),
4159 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4160 0 : op1 = expand_binop (imode, and_optab,
4161 0 : operand_subword_force (op1, i, mode),
4162 0 : immed_wide_int_const (mask, imode),
4163 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4164 :
4165 0 : temp = expand_binop (imode, ior_optab, op0_piece, op1,
4166 : targ_piece, 1, OPTAB_LIB_WIDEN);
4167 0 : if (temp != targ_piece)
4168 0 : emit_move_insn (targ_piece, temp);
4169 : }
4170 : else
4171 0 : emit_move_insn (targ_piece, op0_piece);
4172 : }
4173 :
4174 0 : insns = end_sequence ();
4175 :
4176 0 : emit_insn (insns);
4177 : }
4178 : else
4179 : {
4180 136 : op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
4181 68 : immed_wide_int_const (mask, imode),
4182 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4183 :
4184 68 : op0 = gen_lowpart (imode, op0);
4185 68 : if (!op0_is_abs)
4186 136 : op0 = expand_binop (imode, and_optab, op0,
4187 136 : immed_wide_int_const (~mask, imode),
4188 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
4189 :
4190 68 : temp = expand_binop (imode, ior_optab, op0, op1,
4191 68 : gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
4192 68 : target = force_lowpart_subreg (mode, temp, imode);
4193 : }
4194 :
4195 68 : return target;
4196 68 : }
4197 :
4198 : /* Expand the C99 copysign operation. OP0 and OP1 must be the same
4199 : scalar floating point mode. Return NULL if we do not know how to
4200 : expand the operation inline. */
4201 :
4202 : rtx
4203 11672 : expand_copysign (rtx op0, rtx op1, rtx target)
4204 : {
4205 11672 : scalar_float_mode mode;
4206 11672 : const struct real_format *fmt;
4207 11672 : bool op0_is_abs;
4208 11672 : rtx temp;
4209 :
4210 11672 : mode = as_a <scalar_float_mode> (GET_MODE (op0));
4211 11672 : gcc_assert (GET_MODE (op1) == mode);
4212 :
4213 : /* First try to do it with a special instruction. */
4214 11672 : temp = expand_binop (mode, copysign_optab, op0, op1,
4215 : target, 0, OPTAB_DIRECT);
4216 11672 : if (temp)
4217 : return temp;
4218 :
4219 11672 : fmt = REAL_MODE_FORMAT (mode);
4220 11672 : if (fmt == NULL || !fmt->has_signed_zero)
4221 : return NULL_RTX;
4222 :
4223 11672 : op0_is_abs = false;
4224 11672 : if (CONST_DOUBLE_AS_FLOAT_P (op0))
4225 : {
4226 11454 : if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
4227 0 : op0 = simplify_unary_operation (ABS, mode, op0, mode);
4228 : op0_is_abs = true;
4229 : }
4230 :
4231 11672 : if (fmt->signbit_ro >= 0
4232 11672 : && (CONST_DOUBLE_AS_FLOAT_P (op0)
4233 218 : || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
4234 150 : && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
4235 : {
4236 23208 : temp = expand_copysign_absneg (mode, op0, op1, target,
4237 11604 : fmt->signbit_ro, op0_is_abs);
4238 11604 : if (temp)
4239 : return temp;
4240 : }
4241 :
4242 68 : if (fmt->signbit_rw < 0)
4243 : return NULL_RTX;
4244 68 : return expand_copysign_bit (mode, op0, op1, target,
4245 68 : fmt->signbit_rw, op0_is_abs);
4246 : }
4247 :
4248 : /* Generate an instruction whose insn-code is INSN_CODE,
4249 : with two operands: an output TARGET and an input OP0.
4250 : TARGET *must* be nonzero, and the output is always stored there.
4251 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4252 : the value that is stored into TARGET.
4253 :
4254 : Return false if expansion failed. */
4255 :
4256 : bool
4257 2133958 : maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
4258 : enum rtx_code code)
4259 : {
4260 2133958 : class expand_operand ops[2];
4261 2133958 : rtx_insn *pat;
4262 :
4263 2133958 : create_output_operand (&ops[0], target, GET_MODE (target));
4264 2133958 : create_input_operand (&ops[1], op0, GET_MODE (op0));
4265 2133958 : pat = maybe_gen_insn (icode, 2, ops);
4266 2133958 : if (!pat)
4267 : return false;
4268 :
4269 1198243 : if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
4270 2156721 : && code != UNKNOWN)
4271 11047 : add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
4272 11047 : GET_MODE (op0));
4273 :
4274 2133957 : emit_insn (pat);
4275 :
4276 2133957 : if (ops[0].value != target)
4277 38400 : emit_move_insn (target, ops[0].value);
4278 : return true;
4279 : }
4280 : /* Generate an instruction whose insn-code is INSN_CODE,
4281 : with two operands: an output TARGET and an input OP0.
4282 : TARGET *must* be nonzero, and the output is always stored there.
4283 : CODE is an rtx code such that (CODE OP0) is an rtx that describes
4284 : the value that is stored into TARGET. */
4285 :
4286 : void
4287 2087626 : emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
4288 : {
4289 2087626 : bool ok = maybe_emit_unop_insn (icode, target, op0, code);
4290 2087626 : gcc_assert (ok);
4291 2087626 : }
4292 :
4293 : struct no_conflict_data
4294 : {
4295 : rtx target;
4296 : rtx_insn *first, *insn;
4297 : bool must_stay;
4298 : };
4299 :
4300 : /* Called via note_stores by emit_libcall_block. Set P->must_stay if
4301 : the currently examined clobber / store has to stay in the list of
4302 : insns that constitute the actual libcall block. */
4303 : static void
4304 56756 : no_conflict_move_test (rtx dest, const_rtx set, void *p0)
4305 : {
4306 56756 : struct no_conflict_data *p= (struct no_conflict_data *) p0;
4307 :
4308 : /* If this inns directly contributes to setting the target, it must stay. */
4309 56756 : if (reg_overlap_mentioned_p (p->target, dest))
4310 0 : p->must_stay = true;
4311 : /* If we haven't committed to keeping any other insns in the list yet,
4312 : there is nothing more to check. */
4313 56756 : else if (p->insn == p->first)
4314 : return;
4315 : /* If this insn sets / clobbers a register that feeds one of the insns
4316 : already in the list, this insn has to stay too. */
4317 26958 : else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
4318 26958 : || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
4319 26958 : || reg_used_between_p (dest, p->first, p->insn)
4320 : /* Likewise if this insn depends on a register set by a previous
4321 : insn in the list, or if it sets a result (presumably a hard
4322 : register) that is set or clobbered by a previous insn.
4323 : N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
4324 : SET_DEST perform the former check on the address, and the latter
4325 : check on the MEM. */
4326 53916 : || (GET_CODE (set) == SET
4327 26958 : && (modified_in_p (SET_SRC (set), p->first)
4328 26956 : || modified_in_p (SET_DEST (set), p->first)
4329 26956 : || modified_between_p (SET_SRC (set), p->first, p->insn)
4330 26956 : || modified_between_p (SET_DEST (set), p->first, p->insn))))
4331 2 : p->must_stay = true;
4332 : }
4333 :
4334 :
4335 : /* Emit code to make a call to a constant function or a library call.
4336 :
4337 : INSNS is a list containing all insns emitted in the call.
4338 : These insns leave the result in RESULT. Our block is to copy RESULT
4339 : to TARGET, which is logically equivalent to EQUIV.
4340 :
4341 : We first emit any insns that set a pseudo on the assumption that these are
4342 : loading constants into registers; doing so allows them to be safely cse'ed
4343 : between blocks. Then we emit all the other insns in the block, followed by
4344 : an insn to move RESULT to TARGET. This last insn will have a REQ_EQUAL
4345 : note with an operand of EQUIV. */
4346 :
4347 : static void
4348 116124 : emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
4349 : bool equiv_may_trap)
4350 : {
4351 116124 : rtx final_dest = target;
4352 116124 : rtx_insn *next, *last, *insn;
4353 :
4354 : /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
4355 : into a MEM later. Protect the libcall block from this change. */
4356 116124 : if (! REG_P (target) || REG_USERVAR_P (target))
4357 1822 : target = gen_reg_rtx (GET_MODE (target));
4358 :
4359 : /* If we're using non-call exceptions, a libcall corresponding to an
4360 : operation that may trap may also trap. */
4361 : /* ??? See the comment in front of make_reg_eh_region_note. */
4362 116124 : if (cfun->can_throw_non_call_exceptions
4363 116124 : && (equiv_may_trap || may_trap_p (equiv)))
4364 : {
4365 1036 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4366 832 : if (CALL_P (insn))
4367 : {
4368 204 : rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
4369 204 : if (note)
4370 : {
4371 204 : int lp_nr = INTVAL (XEXP (note, 0));
4372 204 : if (lp_nr == 0 || lp_nr == INT_MIN)
4373 204 : remove_note (insn, note);
4374 : }
4375 : }
4376 : }
4377 : else
4378 : {
4379 : /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
4380 : reg note to indicate that this call cannot throw or execute a nonlocal
4381 : goto (unless there is already a REG_EH_REGION note, in which case
4382 : we update it). */
4383 512916 : for (insn = insns; insn; insn = NEXT_INSN (insn))
4384 396996 : if (CALL_P (insn))
4385 115920 : make_reg_eh_region_note_nothrow_nononlocal (insn);
4386 : }
4387 :
4388 : /* First emit all insns that set pseudos. Remove them from the list as
4389 : we go. Avoid insns that set pseudos which were referenced in previous
4390 : insns. These can be generated by move_by_pieces, for example,
4391 : to update an address. Similarly, avoid insns that reference things
4392 : set in previous insns. */
4393 :
4394 513952 : for (insn = insns; insn; insn = next)
4395 : {
4396 397828 : rtx set = single_set (insn);
4397 :
4398 397828 : next = NEXT_INSN (insn);
4399 :
4400 383326 : if (set != 0 && REG_P (SET_DEST (set))
4401 743900 : && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
4402 : {
4403 46655 : struct no_conflict_data data;
4404 :
4405 46655 : data.target = const0_rtx;
4406 46655 : data.first = insns;
4407 46655 : data.insn = insn;
4408 46655 : data.must_stay = 0;
4409 46655 : note_stores (insn, no_conflict_move_test, &data);
4410 46655 : if (! data.must_stay)
4411 : {
4412 46653 : if (PREV_INSN (insn))
4413 26956 : SET_NEXT_INSN (PREV_INSN (insn)) = next;
4414 : else
4415 : insns = next;
4416 :
4417 46653 : if (next)
4418 46653 : SET_PREV_INSN (next) = PREV_INSN (insn);
4419 :
4420 46653 : add_insn (insn);
4421 : }
4422 : }
4423 :
4424 : /* Some ports use a loop to copy large arguments onto the stack.
4425 : Don't move anything outside such a loop. */
4426 397828 : if (LABEL_P (insn))
4427 : break;
4428 : }
4429 :
4430 : /* Write the remaining insns followed by the final copy. */
4431 467299 : for (insn = insns; insn; insn = next)
4432 : {
4433 351175 : next = NEXT_INSN (insn);
4434 :
4435 351175 : add_insn (insn);
4436 : }
4437 :
4438 116124 : last = emit_move_insn (target, result);
4439 116124 : if (equiv)
4440 115518 : set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
4441 :
4442 116124 : if (final_dest != target)
4443 1822 : emit_move_insn (final_dest, target);
4444 116124 : }
4445 :
4446 : void
4447 84966 : emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
4448 : {
4449 84966 : emit_libcall_block_1 (insns, target, result, equiv, false);
4450 84966 : }
4451 :
4452 : /* True if we can perform a comparison of mode MODE straightforwardly.
4453 : PURPOSE describes how this comparison will be used. CODE is the rtx
4454 : comparison code we will be using.
4455 :
4456 : ??? Actually, CODE is slightly weaker than that. A target is still
4457 : required to implement all of the normal bcc operations, but not
4458 : required to implement all (or any) of the unordered bcc operations. */
4459 :
4460 : bool
4461 17851704 : can_compare_p (enum rtx_code code, machine_mode mode,
4462 : enum can_compare_purpose purpose)
4463 : {
4464 17851704 : rtx test;
4465 17851704 : test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
4466 19892850 : do
4467 : {
4468 19892850 : enum insn_code icode;
4469 :
4470 19892850 : if (purpose == ccp_jump
4471 19696411 : && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
4472 38247827 : && insn_operand_matches (icode, 0, test))
4473 : return true;
4474 3318899 : if (purpose == ccp_store_flag
4475 196439 : && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
4476 3447853 : && insn_operand_matches (icode, 1, test))
4477 : return true;
4478 :
4479 3316462 : mode = GET_MODE_WIDER_MODE (mode).else_void ();
4480 3316462 : PUT_MODE (test, mode);
4481 : }
4482 3316462 : while (mode != VOIDmode);
4483 :
4484 : return false;
4485 : }
4486 :
4487 : /* Return whether RTL code CODE corresponds to an unsigned optab. */
4488 :
4489 : static bool
4490 1367304 : unsigned_optab_p (enum rtx_code code)
4491 : {
4492 1367304 : return code == LTU || code == LEU || code == GTU || code == GEU;
4493 : }
4494 :
4495 : /* Return whether the backend-emitted comparison for code CODE, comparing
4496 : operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
4497 : operand OPNO of pattern ICODE. */
4498 :
4499 : static bool
4500 996650 : insn_predicate_matches_p (enum insn_code icode, unsigned int opno,
4501 : enum rtx_code code, machine_mode mask_mode,
4502 : machine_mode value_mode)
4503 : {
4504 996650 : rtx reg1 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 1);
4505 996650 : rtx reg2 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 2);
4506 996650 : rtx test = alloca_rtx_fmt_ee (code, mask_mode, reg1, reg2);
4507 996650 : return insn_operand_matches (icode, opno, test);
4508 : }
4509 :
4510 : /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
4511 : for code CODE, comparing operands of mode VALUE_MODE and producing a result
4512 : with MASK_MODE. */
4513 :
4514 : bool
4515 1367304 : can_vec_cmp_compare_p (enum rtx_code code, machine_mode value_mode,
4516 : machine_mode mask_mode)
4517 : {
4518 1367304 : enum insn_code icode
4519 1367304 : = get_vec_cmp_icode (value_mode, mask_mode, unsigned_optab_p (code));
4520 1367304 : if (icode == CODE_FOR_nothing)
4521 : return false;
4522 :
4523 996650 : return insn_predicate_matches_p (icode, 1, code, mask_mode, value_mode);
4524 : }
4525 :
4526 : /* Return whether the backend can emit vector set instructions for inserting
4527 : element into vector at variable index position. */
4528 :
4529 : bool
4530 166 : can_vec_set_var_idx_p (machine_mode vec_mode)
4531 : {
4532 166 : if (!VECTOR_MODE_P (vec_mode))
4533 : return false;
4534 :
4535 164 : machine_mode inner_mode = GET_MODE_INNER (vec_mode);
4536 :
4537 164 : rtx reg1 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 1);
4538 164 : rtx reg2 = alloca_raw_REG (inner_mode, LAST_VIRTUAL_REGISTER + 2);
4539 :
4540 164 : enum insn_code icode = optab_handler (vec_set_optab, vec_mode);
4541 :
4542 164 : const struct insn_data_d *data = &insn_data[icode];
4543 164 : machine_mode idx_mode = data->operand[2].mode;
4544 :
4545 164 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4546 :
4547 163 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4548 163 : && insn_operand_matches (icode, 1, reg2)
4549 327 : && insn_operand_matches (icode, 2, reg3);
4550 : }
4551 :
4552 : /* Return whether the backend can emit a vec_extract instruction with
4553 : a non-constant index. */
4554 : bool
4555 18503 : can_vec_extract_var_idx_p (machine_mode vec_mode, machine_mode extr_mode)
4556 : {
4557 18503 : if (!VECTOR_MODE_P (vec_mode))
4558 : return false;
4559 :
4560 15927 : rtx reg1 = alloca_raw_REG (extr_mode, LAST_VIRTUAL_REGISTER + 1);
4561 15927 : rtx reg2 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 2);
4562 :
4563 15927 : enum insn_code icode = convert_optab_handler (vec_extract_optab,
4564 : vec_mode, extr_mode);
4565 :
4566 15927 : const struct insn_data_d *data = &insn_data[icode];
4567 15927 : machine_mode idx_mode = data->operand[2].mode;
4568 :
4569 15927 : rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
4570 :
4571 15068 : return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
4572 15068 : && insn_operand_matches (icode, 1, reg2)
4573 30995 : && insn_operand_matches (icode, 2, reg3);
4574 : }
4575 :
4576 : /* This function is called when we are going to emit a compare instruction that
4577 : compares the values found in X and Y, using the rtl operator COMPARISON.
4578 :
4579 : If they have mode BLKmode, then SIZE specifies the size of both operands.
4580 :
4581 : UNSIGNEDP nonzero says that the operands are unsigned;
4582 : this matters if they need to be widened (as given by METHODS).
4583 :
4584 : *PTEST is where the resulting comparison RTX is returned or NULL_RTX
4585 : if we failed to produce one.
4586 :
4587 : *PMODE is the mode of the inputs (in case they are const_int).
4588 :
4589 : *OPTAB is the optab to check for OPTAB_DIRECT support. Defaults to
4590 : cbranch_optab.
4591 :
4592 : This function performs all the setup necessary so that the caller only has
4593 : to emit a single comparison insn. This setup can involve doing a BLKmode
4594 : comparison or emitting a library call to perform the comparison if no insn
4595 : is available to handle it.
4596 : The values which are passed in through pointers can be modified; the caller
4597 : should perform the comparison on the modified values. Constant
4598 : comparisons must have already been folded. */
4599 :
4600 : static void
4601 6769552 : prepare_cmp_insn (rtx x, rtx y, rtx *mask, enum rtx_code comparison, rtx size,
4602 : int unsignedp, enum optab_methods methods,
4603 : rtx *ptest, machine_mode *pmode, optab optab)
4604 : {
4605 6769552 : machine_mode mode = *pmode;
4606 6769552 : rtx libfunc, test;
4607 6769552 : machine_mode cmp_mode;
4608 :
4609 : /* The other methods are not needed. */
4610 6769552 : gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
4611 : || methods == OPTAB_LIB_WIDEN);
4612 :
4613 6769552 : if (CONST_SCALAR_INT_P (y))
4614 4373041 : canonicalize_comparison (mode, &comparison, &y);
4615 :
4616 : /* If we are optimizing, force expensive constants into a register. */
4617 1 : if (CONSTANT_P (x) && optimize
4618 1 : && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
4619 : > COSTS_N_INSNS (1))
4620 6769552 : && can_create_pseudo_p ())
4621 0 : x = force_reg (mode, x);
4622 :
4623 4886498 : if (CONSTANT_P (y) && optimize
4624 3856599 : && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
4625 : > COSTS_N_INSNS (1))
4626 6941980 : && can_create_pseudo_p ())
4627 172428 : y = force_reg (mode, y);
4628 :
4629 : /* Don't let both operands fail to indicate the mode. */
4630 6769552 : if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
4631 0 : x = force_reg (mode, x);
4632 6769552 : if (mode == VOIDmode)
4633 4755 : mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
4634 :
4635 : /* Handle all BLKmode compares. */
4636 :
4637 6769552 : if (mode == BLKmode)
4638 : {
4639 0 : machine_mode result_mode;
4640 0 : enum insn_code cmp_code;
4641 0 : rtx result;
4642 0 : rtx opalign
4643 0 : = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
4644 :
4645 0 : gcc_assert (size);
4646 :
4647 : /* Try to use a memory block compare insn - either cmpstr
4648 : or cmpmem will do. */
4649 0 : opt_scalar_int_mode cmp_mode_iter;
4650 0 : FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
4651 : {
4652 0 : scalar_int_mode cmp_mode = cmp_mode_iter.require ();
4653 0 : cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
4654 0 : if (cmp_code == CODE_FOR_nothing)
4655 0 : cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
4656 0 : if (cmp_code == CODE_FOR_nothing)
4657 0 : cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
4658 0 : if (cmp_code == CODE_FOR_nothing)
4659 0 : continue;
4660 :
4661 : /* Must make sure the size fits the insn's mode. */
4662 0 : if (CONST_INT_P (size)
4663 0 : ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
4664 0 : : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
4665 0 : > GET_MODE_BITSIZE (cmp_mode)))
4666 0 : continue;
4667 :
4668 0 : result_mode = insn_data[cmp_code].operand[0].mode;
4669 0 : result = gen_reg_rtx (result_mode);
4670 0 : size = convert_to_mode (cmp_mode, size, 1);
4671 0 : emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
4672 :
4673 0 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
4674 0 : *pmode = result_mode;
4675 0 : return;
4676 : }
4677 :
4678 0 : if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
4679 0 : goto fail;
4680 :
4681 : /* Otherwise call a library function. */
4682 0 : result = emit_block_comp_via_libcall (x, y, size);
4683 :
4684 0 : x = result;
4685 0 : y = const0_rtx;
4686 0 : mode = TYPE_MODE (integer_type_node);
4687 0 : methods = OPTAB_LIB_WIDEN;
4688 0 : unsignedp = false;
4689 : }
4690 :
4691 : /* Don't allow operands to the compare to trap, as that can put the
4692 : compare and branch in different basic blocks. */
4693 6769552 : if (cfun->can_throw_non_call_exceptions)
4694 : {
4695 1134927 : if (!can_create_pseudo_p () && (may_trap_p (x) || may_trap_p (y)))
4696 0 : goto fail;
4697 1134927 : if (may_trap_p (x))
4698 25098 : x = copy_to_reg (x);
4699 1134927 : if (may_trap_p (y))
4700 1522 : y = copy_to_reg (y);
4701 : }
4702 :
4703 6769552 : if (GET_MODE_CLASS (mode) == MODE_CC)
4704 : {
4705 436 : enum insn_code icode = optab_handler (cbranch_optab, CCmode);
4706 436 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4707 436 : if (icode != CODE_FOR_nothing
4708 436 : && insn_operand_matches (icode, 0, test))
4709 : {
4710 436 : *ptest = test;
4711 436 : return;
4712 : }
4713 : else
4714 0 : goto fail;
4715 : }
4716 :
4717 6769116 : test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
4718 6854226 : FOR_EACH_WIDER_MODE_FROM (cmp_mode, mode)
4719 : {
4720 6807406 : enum insn_code icode;
4721 6807406 : icode = optab_handler (optab, cmp_mode);
4722 6807406 : if (icode != CODE_FOR_nothing
4723 6807406 : && insn_operand_matches (icode, 0, test))
4724 : {
4725 6722296 : rtx_insn *last = get_last_insn ();
4726 6722296 : rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
4727 6722296 : rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
4728 6722296 : if (op0 && op1
4729 6722296 : && insn_operand_matches (icode, 1, op0)
4730 13444592 : && insn_operand_matches (icode, 2, op1))
4731 : {
4732 6722296 : XEXP (test, 0) = op0;
4733 6722296 : XEXP (test, 1) = op1;
4734 6722296 : *ptest = test;
4735 6722296 : *pmode = cmp_mode;
4736 6722296 : return;
4737 : }
4738 0 : delete_insns_since (last);
4739 : }
4740 :
4741 85110 : if (methods == OPTAB_DIRECT)
4742 : break;
4743 : }
4744 :
4745 46820 : if (methods != OPTAB_LIB_WIDEN)
4746 2901 : goto fail;
4747 :
4748 43919 : if (SCALAR_FLOAT_MODE_P (mode))
4749 : {
4750 : /* Small trick if UNORDERED isn't implemented by the hardware. */
4751 43919 : if (comparison == UNORDERED && rtx_equal_p (x, y))
4752 : {
4753 765 : prepare_cmp_insn (x, y, mask, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
4754 : ptest, pmode, optab);
4755 765 : if (*ptest)
4756 : return;
4757 : }
4758 :
4759 43919 : prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
4760 : }
4761 : else
4762 : {
4763 0 : rtx result;
4764 0 : machine_mode ret_mode;
4765 :
4766 : /* Handle a libcall just for the mode we are using. */
4767 0 : libfunc = optab_libfunc (cmp_optab, mode);
4768 0 : gcc_assert (libfunc);
4769 :
4770 : /* If we want unsigned, and this mode has a distinct unsigned
4771 : comparison routine, use that. */
4772 0 : if (unsignedp)
4773 : {
4774 0 : rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
4775 0 : if (ulibfunc)
4776 0 : libfunc = ulibfunc;
4777 : }
4778 :
4779 0 : ret_mode = targetm.libgcc_cmp_return_mode ();
4780 0 : result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
4781 : ret_mode, x, mode, y, mode);
4782 :
4783 : /* There are two kinds of comparison routines. Biased routines
4784 : return 0/1/2, and unbiased routines return -1/0/1. Other parts
4785 : of gcc expect that the comparison operation is equivalent
4786 : to the modified comparison. For signed comparisons compare the
4787 : result against 1 in the biased case, and zero in the unbiased
4788 : case. For unsigned comparisons always compare against 1 after
4789 : biasing the unbiased result by adding 1. This gives us a way to
4790 : represent LTU.
4791 : The comparisons in the fixed-point helper library are always
4792 : biased. */
4793 0 : x = result;
4794 0 : y = const1_rtx;
4795 :
4796 0 : if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
4797 : {
4798 : if (unsignedp)
4799 : x = plus_constant (ret_mode, result, 1);
4800 : else
4801 : y = const0_rtx;
4802 : }
4803 :
4804 0 : *pmode = ret_mode;
4805 0 : prepare_cmp_insn (x, y, mask, comparison, NULL_RTX, unsignedp, methods,
4806 : ptest, pmode, optab);
4807 : }
4808 :
4809 : return;
4810 :
4811 2901 : fail:
4812 2901 : *ptest = NULL_RTX;
4813 : }
4814 :
4815 : /* Before emitting an insn with code ICODE, make sure that X, which is going
4816 : to be used for operand OPNUM of the insn, is converted from mode MODE to
4817 : WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
4818 : that it is accepted by the operand predicate. Return the new value. */
4819 :
4820 : rtx
4821 14858406 : prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
4822 : machine_mode wider_mode, int unsignedp)
4823 : {
4824 14858406 : if (mode != wider_mode)
4825 7266 : x = convert_modes (wider_mode, mode, x, unsignedp);
4826 :
4827 14858406 : if (!insn_operand_matches (icode, opnum, x))
4828 : {
4829 63024 : machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
4830 63024 : if (reload_completed)
4831 : return NULL_RTX;
4832 63024 : if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
4833 : return NULL_RTX;
4834 62740 : x = copy_to_mode_reg (op_mode, x);
4835 : }
4836 :
4837 : return x;
4838 : }
4839 :
4840 : /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
4841 : we can do the branch. */
4842 :
4843 : static void
4844 6314409 : emit_cmp_and_jump_insn_1 (rtx test, rtx cond, rtx len, rtx bias,
4845 : machine_mode mode, rtx label, direct_optab cmp_optab,
4846 : profile_probability prob, bool test_branch)
4847 : {
4848 6314409 : machine_mode optab_mode;
4849 6314409 : enum mode_class mclass;
4850 6314409 : enum insn_code icode;
4851 6314409 : rtx_insn *insn;
4852 :
4853 6314409 : mclass = GET_MODE_CLASS (mode);
4854 6314409 : optab_mode = (mclass == MODE_CC) ? CCmode : mode;
4855 6314409 : icode = optab_handler (cmp_optab, optab_mode);
4856 :
4857 6314409 : gcc_assert (icode != CODE_FOR_nothing);
4858 6314409 : gcc_assert (test_branch || insn_operand_matches (icode, 0, test));
4859 6314409 : gcc_assert (cond == NULL_RTX || (cond != NULL_RTX && !test_branch));
4860 6314409 : if (test_branch)
4861 0 : insn = emit_jump_insn (GEN_FCN (icode) (XEXP (test, 0),
4862 0 : XEXP (test, 1), label));
4863 6314409 : else if (len)
4864 : {
4865 0 : gcc_assert (cond);
4866 0 : gcc_assert (bias);
4867 0 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4868 0 : XEXP (test, 1), len, bias,
4869 : label));
4870 : }
4871 6314409 : else if (cond)
4872 0 : insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
4873 0 : XEXP (test, 1), label));
4874 : else
4875 6314409 : insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
4876 6314409 : XEXP (test, 1), label));
4877 :
4878 6314409 : if (prob.initialized_p ()
4879 4746391 : && profile_status_for_fn (cfun) != PROFILE_ABSENT
4880 4646114 : && insn
4881 4646114 : && JUMP_P (insn)
4882 4645456 : && any_condjump_p (insn)
4883 10959865 : && !find_reg_note (insn, REG_BR_PROB, 0))
4884 4644535 : add_reg_br_prob_note (insn, prob);
4885 6314409 : }
4886 :
4887 : /* PTEST points to a comparison that compares its first operand with zero.
4888 : Check to see if it can be performed as a bit-test-and-branch instead.
4889 : On success, return the instruction that performs the bit-test-and-branch
4890 : and replace the second operand of *PTEST with the bit number to test.
4891 : On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
4892 :
4893 : Note that the comparison described by *PTEST should not be taken
4894 : literally after a successful return. *PTEST is just a convenient
4895 : place to store the two operands of the bit-and-test.
4896 :
4897 : VAL must contain the original tree expression for the first operand
4898 : of *PTEST. */
4899 :
4900 : static enum insn_code
4901 2592147 : validate_test_and_branch (tree val, rtx *ptest, machine_mode *pmode, optab *res)
4902 : {
4903 2592147 : if (!val || TREE_CODE (val) != SSA_NAME)
4904 : return CODE_FOR_nothing;
4905 :
4906 2449825 : machine_mode mode = TYPE_MODE (TREE_TYPE (val));
4907 2449825 : rtx test = *ptest;
4908 2449825 : direct_optab optab;
4909 :
4910 2449825 : if (GET_CODE (test) == EQ)
4911 : optab = tbranch_eq_optab;
4912 1293301 : else if (GET_CODE (test) == NE)
4913 : optab = tbranch_ne_optab;
4914 : else
4915 : return CODE_FOR_nothing;
4916 :
4917 2220053 : *res = optab;
4918 :
4919 : /* If the target supports the testbit comparison directly, great. */
4920 2220053 : auto icode = direct_optab_handler (optab, mode);
4921 2220053 : if (icode == CODE_FOR_nothing)
4922 : return icode;
4923 :
4924 0 : if (tree_zero_one_valued_p (val))
4925 : {
4926 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 : 0;
4927 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4928 0 : *ptest = test;
4929 0 : *pmode = mode;
4930 0 : return icode;
4931 : }
4932 :
4933 0 : wide_int wcst = get_nonzero_bits (val);
4934 0 : if (wcst == -1)
4935 : return CODE_FOR_nothing;
4936 :
4937 0 : int bitpos;
4938 :
4939 0 : if ((bitpos = wi::exact_log2 (wcst)) == -1)
4940 : return CODE_FOR_nothing;
4941 :
4942 0 : auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 - bitpos : bitpos;
4943 0 : XEXP (test, 1) = gen_int_mode (pos, mode);
4944 0 : *ptest = test;
4945 0 : *pmode = mode;
4946 0 : return icode;
4947 0 : }
4948 :
4949 : /* Generate code to compare X with Y so that the condition codes are
4950 : set and to jump to LABEL if the condition is true. If X is a
4951 : constant and Y is not a constant, then the comparison is swapped to
4952 : ensure that the comparison RTL has the canonical form.
4953 :
4954 : UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
4955 : need to be widened. UNSIGNEDP is also used to select the proper
4956 : branch condition code.
4957 :
4958 : If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
4959 :
4960 : MODE is the mode of the inputs (in case they are const_int).
4961 :
4962 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
4963 : It will be potentially converted into an unsigned variant based on
4964 : UNSIGNEDP to select a proper jump instruction.
4965 :
4966 : PROB is the probability of jumping to LABEL. If the comparison is against
4967 : zero then VAL contains the expression from which the non-zero RTL is
4968 : derived. */
4969 :
4970 : void
4971 6314409 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
4972 : machine_mode mode, int unsignedp, tree val, rtx label,
4973 : profile_probability prob)
4974 : {
4975 6314409 : rtx op0 = x, op1 = y;
4976 6314409 : rtx test;
4977 :
4978 : /* Swap operands and condition to ensure canonical RTL. */
4979 6314409 : if (swap_commutative_operands_p (x, y)
4980 6314409 : && can_compare_p (swap_condition (comparison), mode, ccp_jump))
4981 : {
4982 0 : op0 = y, op1 = x;
4983 0 : comparison = swap_condition (comparison);
4984 : }
4985 :
4986 : /* If OP0 is still a constant, then both X and Y must be constants
4987 : or the opposite comparison is not supported. Force X into a register
4988 : to create canonical RTL. */
4989 6314409 : if (CONSTANT_P (op0))
4990 17741 : op0 = force_reg (mode, op0);
4991 :
4992 6314409 : if (unsignedp)
4993 3761179 : comparison = unsigned_condition (comparison);
4994 :
4995 : /* cbranch is no longer preferred for vectors, so when using a vector mode
4996 : check vec_cbranch variants instead. */
4997 6314409 : if (!VECTOR_MODE_P (GET_MODE (op0)))
4998 6295027 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
4999 : OPTAB_LIB_WIDEN, &test, &mode, cbranch_optab);
5000 :
5001 : /* Check if we're comparing a truth type with 0, and if so check if
5002 : the target supports tbranch. */
5003 6314409 : machine_mode tmode = mode;
5004 6314409 : direct_optab optab;
5005 6314409 : if (op1 == CONST0_RTX (GET_MODE (op1)))
5006 : {
5007 2593742 : if (!VECTOR_MODE_P (GET_MODE (op1))
5008 2592147 : && validate_test_and_branch (val, &test, &tmode,
5009 : &optab) != CODE_FOR_nothing)
5010 : {
5011 0 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, tmode,
5012 : label, optab, prob, true);
5013 0 : return;
5014 : }
5015 :
5016 : /* If we are comparing equality with 0, check if VAL is another equality
5017 : comparison and if the target supports it directly. */
5018 2593742 : gimple *def_stmt = NULL;
5019 2452301 : if (val && TREE_CODE (val) == SSA_NAME
5020 2451378 : && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (val))
5021 2292 : && (comparison == NE || comparison == EQ)
5022 2596034 : && (def_stmt = get_gimple_for_ssa_name (val)))
5023 : {
5024 1610 : tree masked_op = NULL_TREE;
5025 1610 : tree len_op = NULL_TREE;
5026 1610 : tree len_bias = NULL_TREE;
5027 : /* First determine if the operation should be masked or unmasked. */
5028 1610 : if (is_gimple_assign (def_stmt)
5029 1610 : && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
5030 : {
5031 : /* See if one side if a comparison, if so use the other side as
5032 : the mask. */
5033 2 : gimple *mask_def = NULL;
5034 2 : tree rhs1 = gimple_assign_rhs1 (def_stmt);
5035 2 : tree rhs2 = gimple_assign_rhs2 (def_stmt);
5036 2 : if (TREE_CODE (rhs1) == SSA_NAME
5037 2 : && (mask_def = get_gimple_for_ssa_name (rhs1))
5038 2 : && is_gimple_assign (mask_def)
5039 4 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
5040 : masked_op = rhs2;
5041 0 : else if (TREE_CODE (rhs2) == SSA_NAME
5042 0 : && (mask_def = get_gimple_for_ssa_name (rhs2))
5043 0 : && is_gimple_assign (mask_def)
5044 0 : && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
5045 : masked_op = rhs1;
5046 :
5047 2 : if (masked_op)
5048 1610 : def_stmt = mask_def;
5049 : }
5050 : /* Else check to see if we're a LEN target. */
5051 1608 : else if (is_gimple_call (def_stmt)
5052 0 : && gimple_call_internal_p (def_stmt)
5053 1608 : && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
5054 : {
5055 : /* Example to consume:
5056 :
5057 : a = _59 != vect__4.17_75;
5058 : vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
5059 : if (vcmp != { 0, ... })
5060 :
5061 : and transform into
5062 :
5063 : if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)). */
5064 0 : gcall *call = dyn_cast <gcall *> (def_stmt);
5065 0 : tree true_branch = gimple_call_arg (call, 1);
5066 0 : tree false_branch = gimple_call_arg (call, 2);
5067 0 : if (integer_minus_onep (true_branch)
5068 0 : && integer_zerop (false_branch))
5069 : {
5070 0 : len_op = gimple_call_arg (call, 3);
5071 0 : len_bias = gimple_call_arg (call, 4);
5072 0 : tree arg0 = gimple_call_arg (call, 0);
5073 :
5074 0 : if (TREE_CODE (arg0) == SSA_NAME)
5075 0 : def_stmt = get_gimple_for_ssa_name (arg0);
5076 : }
5077 : }
5078 :
5079 1610 : enum insn_code icode;
5080 1610 : if (def_stmt
5081 1610 : && is_gimple_assign (def_stmt)
5082 3220 : && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
5083 : == tcc_comparison)
5084 : {
5085 1427 : class expand_operand ops[5];
5086 1427 : rtx_insn *tmp = NULL;
5087 1427 : start_sequence ();
5088 1427 : tree t_op0 = gimple_assign_rhs1 (def_stmt);
5089 1427 : tree t_op1 = gimple_assign_rhs2 (def_stmt);
5090 1427 : rtx op0c = expand_normal (t_op0);
5091 1427 : rtx op1c = expand_normal (t_op1);
5092 1427 : machine_mode mode2 = GET_MODE (op0c);
5093 :
5094 1427 : int nops = masked_op ? 3 : (len_op ? 5 : 2);
5095 1427 : int offset = masked_op || len_op ? 1 : 0;
5096 1427 : create_input_operand (&ops[offset + 0], op0c, mode2);
5097 1427 : create_input_operand (&ops[offset + 1], op1c, mode2);
5098 1427 : if (masked_op)
5099 : {
5100 0 : rtx mask_op = expand_normal (masked_op);
5101 0 : auto mask_mode = GET_MODE (mask_op);
5102 0 : create_input_operand (&ops[0], mask_op, mask_mode);
5103 : }
5104 1427 : else if (len_op)
5105 : {
5106 0 : rtx len_rtx = expand_normal (len_op);
5107 0 : rtx len_bias_rtx = expand_normal (len_bias);
5108 0 : tree lhs = gimple_get_lhs (def_stmt);
5109 0 : auto mask_mode = TYPE_MODE (TREE_TYPE (lhs));
5110 : /* ??? We could use something like internal_fn's
5111 : add_mask_else_and_len_args here. Currently it
5112 : only supports a fixed, consecutive order of
5113 : mask and len, though. */
5114 0 : create_input_operand (&ops[0], CONSTM1_RTX (mask_mode),
5115 : mask_mode);
5116 0 : create_convert_operand_from
5117 0 : (&ops[3], len_rtx, TYPE_MODE (TREE_TYPE (len_op)),
5118 0 : TYPE_UNSIGNED (TREE_TYPE (len_op)));
5119 0 : create_input_operand (&ops[4], len_bias_rtx, QImode);
5120 : }
5121 :
5122 1427 : int unsignedp2 = TYPE_UNSIGNED (TREE_TYPE (t_op0));
5123 1427 : auto inner_code = gimple_assign_rhs_code (def_stmt);
5124 1427 : rtx test2 = NULL_RTX;
5125 :
5126 1427 : enum rtx_code comparison2 = get_rtx_code (inner_code, unsignedp2);
5127 1427 : if (unsignedp2)
5128 487 : comparison2 = unsigned_condition (comparison2);
5129 1427 : if (comparison == NE)
5130 1618 : optab = masked_op ? cond_vec_cbranch_any_optab
5131 809 : : len_op ? cond_len_vec_cbranch_any_optab
5132 : : vec_cbranch_any_optab;
5133 : else
5134 1236 : optab = masked_op ? cond_vec_cbranch_all_optab
5135 618 : : len_op ? cond_len_vec_cbranch_all_optab
5136 : : vec_cbranch_all_optab;
5137 :
5138 1427 : if ((icode = optab_handler (optab, mode2))
5139 : != CODE_FOR_nothing
5140 1427 : && maybe_legitimize_operands (icode, 1, nops, ops))
5141 : {
5142 0 : test2 = gen_rtx_fmt_ee (comparison2, VOIDmode,
5143 : ops[offset + 0].value,
5144 : ops[offset + 1].value);
5145 0 : if (insn_operand_matches (icode, 0, test2))
5146 : {
5147 0 : rtx mask
5148 0 : = (masked_op || len_op) ? ops[0].value : NULL_RTX;
5149 0 : rtx len = len_op ? ops[3].value : NULL_RTX;
5150 0 : rtx bias = len_op ? ops[4].value : NULL_RTX;
5151 0 : emit_cmp_and_jump_insn_1 (test2, mask, len, bias, mode2,
5152 : label, optab, prob, false);
5153 0 : tmp = get_insns ();
5154 : }
5155 : }
5156 :
5157 0 : end_sequence ();
5158 1427 : if (tmp)
5159 : {
5160 0 : emit_insn (tmp);
5161 0 : return;
5162 : }
5163 : }
5164 : }
5165 : }
5166 :
5167 : /* cbranch should only be used for VECTOR_BOOLEAN_TYPE_P values. */
5168 6314409 : direct_optab base_optab = cbranch_optab;
5169 6314409 : if (VECTOR_MODE_P (GET_MODE (op0)))
5170 : {
5171 : /* If cbranch is provided, use it. If we get here it means we have an
5172 : instruction in between what created the boolean value and the gcond
5173 : that is not a masking operation. This can happen for instance during
5174 : unrolling of early-break where we have an OR-reduction to reduce the
5175 : masks. In this case knowing we have a mask can let us generate better
5176 : code. If it's not there there then check the vector specific
5177 : optabs. */
5178 19382 : if (optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
5179 : {
5180 0 : if (comparison == NE)
5181 : base_optab = vec_cbranch_any_optab;
5182 : else
5183 0 : base_optab = vec_cbranch_all_optab;
5184 :
5185 0 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5186 : OPTAB_DIRECT, &test, &mode, base_optab);
5187 :
5188 0 : enum insn_code icode = optab_handler (base_optab, mode);
5189 :
5190 : /* If the new cbranch isn't supported, degrade back to old one. */
5191 0 : if (icode == CODE_FOR_nothing
5192 0 : || !test
5193 0 : || !insn_operand_matches (icode, 0, test))
5194 : base_optab = cbranch_optab;
5195 : }
5196 :
5197 19382 : prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
5198 : OPTAB_LIB_WIDEN, &test, &mode, base_optab);
5199 : }
5200 :
5201 6314409 : emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, mode, label,
5202 : base_optab, prob, false);
5203 : }
5204 :
5205 : /* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown. */
5206 :
5207 : void
5208 145179 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
5209 : machine_mode mode, int unsignedp, rtx label,
5210 : profile_probability prob)
5211 : {
5212 145179 : emit_cmp_and_jump_insns (x, y, comparison, size, mode, unsignedp, NULL,
5213 : label, prob);
5214 145179 : }
5215 :
5216 :
5217 : /* Emit a library call comparison between floating point X and Y.
5218 : COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.). */
5219 :
5220 : static void
5221 43919 : prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
5222 : rtx *ptest, machine_mode *pmode)
5223 : {
5224 43919 : enum rtx_code swapped = swap_condition (comparison);
5225 43919 : enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
5226 43919 : machine_mode orig_mode = GET_MODE (x);
5227 43919 : machine_mode mode;
5228 43919 : rtx true_rtx, false_rtx;
5229 43919 : rtx value, target, equiv;
5230 43919 : rtx_insn *insns;
5231 43919 : rtx libfunc = 0;
5232 43919 : bool reversed_p = false;
5233 43919 : scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
5234 :
5235 43919 : FOR_EACH_WIDER_MODE_FROM (mode, orig_mode)
5236 : {
5237 43919 : if (code_to_optab (comparison)
5238 43919 : && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
5239 : break;
5240 :
5241 0 : if (code_to_optab (swapped)
5242 0 : && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
5243 : {
5244 : std::swap (x, y);
5245 : comparison = swapped;
5246 : break;
5247 : }
5248 :
5249 0 : if (code_to_optab (reversed)
5250 0 : && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
5251 : {
5252 : comparison = reversed;
5253 : reversed_p = true;
5254 : break;
5255 : }
5256 : }
5257 :
5258 43919 : gcc_assert (mode != VOIDmode);
5259 :
5260 43919 : if (mode != orig_mode)
5261 : {
5262 0 : x = convert_to_mode (mode, x, 0);
5263 0 : y = convert_to_mode (mode, y, 0);
5264 : }
5265 :
5266 : /* Attach a REG_EQUAL note describing the semantics of the libcall to
5267 : the RTL. The allows the RTL optimizers to delete the libcall if the
5268 : condition can be determined at compile-time. */
5269 43919 : if (comparison == UNORDERED
5270 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5271 : {
5272 5426 : true_rtx = const_true_rtx;
5273 5426 : false_rtx = const0_rtx;
5274 : }
5275 : else
5276 : {
5277 38493 : switch (comparison)
5278 : {
5279 3368 : case EQ:
5280 3368 : true_rtx = const0_rtx;
5281 3368 : false_rtx = const_true_rtx;
5282 3368 : break;
5283 :
5284 23465 : case NE:
5285 23465 : true_rtx = const_true_rtx;
5286 23465 : false_rtx = const0_rtx;
5287 23465 : break;
5288 :
5289 2562 : case GT:
5290 2562 : true_rtx = const1_rtx;
5291 2562 : false_rtx = const0_rtx;
5292 2562 : break;
5293 :
5294 3076 : case GE:
5295 3076 : true_rtx = const0_rtx;
5296 3076 : false_rtx = constm1_rtx;
5297 3076 : break;
5298 :
5299 2437 : case LT:
5300 2437 : true_rtx = constm1_rtx;
5301 2437 : false_rtx = const0_rtx;
5302 2437 : break;
5303 :
5304 3585 : case LE:
5305 3585 : true_rtx = const0_rtx;
5306 3585 : false_rtx = const1_rtx;
5307 3585 : break;
5308 :
5309 0 : default:
5310 0 : gcc_unreachable ();
5311 : }
5312 : }
5313 :
5314 43919 : if (comparison == UNORDERED)
5315 : {
5316 5426 : rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
5317 5426 : equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
5318 5426 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5319 : temp, const_true_rtx, equiv);
5320 : }
5321 : else
5322 : {
5323 38493 : equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
5324 38493 : if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
5325 38493 : equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
5326 : equiv, true_rtx, false_rtx);
5327 : }
5328 :
5329 43919 : start_sequence ();
5330 43919 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
5331 : cmp_mode, x, mode, y, mode);
5332 43919 : insns = end_sequence ();
5333 :
5334 43919 : target = gen_reg_rtx (cmp_mode);
5335 43919 : emit_libcall_block (insns, target, value, equiv);
5336 :
5337 43919 : if (comparison == UNORDERED
5338 : || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
5339 43919 : || reversed_p)
5340 10852 : *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
5341 : else
5342 38493 : *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
5343 :
5344 43919 : *pmode = cmp_mode;
5345 43919 : }
5346 :
5347 : /* Generate code to indirectly jump to a location given in the rtx LOC. */
5348 :
5349 : void
5350 1308 : emit_indirect_jump (rtx loc)
5351 : {
5352 1308 : if (!targetm.have_indirect_jump ())
5353 0 : sorry ("indirect jumps are not available on this target");
5354 : else
5355 : {
5356 1308 : class expand_operand ops[1];
5357 1308 : create_address_operand (&ops[0], loc);
5358 1308 : expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
5359 1308 : emit_barrier ();
5360 : }
5361 1308 : }
5362 :
5363 :
5364 : /* Emit a conditional move instruction if the machine supports one for that
5365 : condition and machine mode.
5366 :
5367 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5368 : the mode to use should they be constants. If it is VOIDmode, they cannot
5369 : both be constants.
5370 :
5371 : OP2 should be stored in TARGET if the comparison is true, otherwise OP3
5372 : should be stored there. MODE is the mode to use should they be constants.
5373 : If it is VOIDmode, they cannot both be constants.
5374 :
5375 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5376 : is not supported. */
5377 :
5378 : rtx
5379 427444 : emit_conditional_move (rtx target, struct rtx_comparison comp,
5380 : rtx op2, rtx op3,
5381 : machine_mode mode, int unsignedp)
5382 : {
5383 427444 : rtx comparison;
5384 427444 : rtx_insn *last;
5385 427444 : enum insn_code icode;
5386 427444 : enum rtx_code reversed;
5387 :
5388 : /* If the two source operands are identical, that's just a move. */
5389 :
5390 427444 : if (rtx_equal_p (op2, op3))
5391 : {
5392 11763 : if (!target)
5393 0 : target = gen_reg_rtx (mode);
5394 :
5395 11763 : emit_move_insn (target, op3);
5396 11763 : return target;
5397 : }
5398 :
5399 : /* If one operand is constant, make it the second one. Only do this
5400 : if the other operand is not constant as well. */
5401 :
5402 415681 : if (swap_commutative_operands_p (comp.op0, comp.op1))
5403 : {
5404 403 : std::swap (comp.op0, comp.op1);
5405 403 : comp.code = swap_condition (comp.code);
5406 : }
5407 :
5408 : /* get_condition will prefer to generate LT and GT even if the old
5409 : comparison was against zero, so undo that canonicalization here since
5410 : comparisons against zero are cheaper. */
5411 :
5412 415681 : if (comp.code == LT && comp.op1 == const1_rtx)
5413 1980 : comp.code = LE, comp.op1 = const0_rtx;
5414 413701 : else if (comp.code == GT && comp.op1 == constm1_rtx)
5415 7248 : comp.code = GE, comp.op1 = const0_rtx;
5416 :
5417 415681 : if (comp.mode == VOIDmode)
5418 331337 : comp.mode = GET_MODE (comp.op0);
5419 :
5420 415681 : enum rtx_code orig_code = comp.code;
5421 415681 : bool swapped = false;
5422 415681 : if (swap_commutative_operands_p (op2, op3)
5423 415681 : && ((reversed =
5424 127207 : reversed_comparison_code_parts (comp.code, comp.op0, comp.op1, NULL))
5425 : != UNKNOWN))
5426 : {
5427 : std::swap (op2, op3);
5428 : comp.code = reversed;
5429 : swapped = true;
5430 : }
5431 :
5432 415681 : if (mode == VOIDmode)
5433 0 : mode = GET_MODE (op2);
5434 :
5435 415681 : icode = direct_optab_handler (movcc_optab, mode);
5436 :
5437 415681 : if (icode == CODE_FOR_nothing)
5438 : return NULL_RTX;
5439 :
5440 409093 : if (!target)
5441 0 : target = gen_reg_rtx (mode);
5442 :
5443 409093 : for (int pass = 0; ; pass++)
5444 : {
5445 440647 : comp.code = unsignedp ? unsigned_condition (comp.code) : comp.code;
5446 881294 : comparison =
5447 440647 : simplify_gen_relational (comp.code, VOIDmode,
5448 : comp.mode, comp.op0, comp.op1);
5449 :
5450 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5451 : punt and let the caller figure out how best to deal with this
5452 : situation. */
5453 440647 : if (COMPARISON_P (comparison))
5454 : {
5455 439781 : saved_pending_stack_adjust save;
5456 439781 : save_pending_stack_adjust (&save);
5457 439781 : last = get_last_insn ();
5458 439781 : do_pending_stack_adjust ();
5459 439781 : machine_mode cmpmode = comp.mode;
5460 439781 : rtx orig_op0 = XEXP (comparison, 0);
5461 439781 : rtx orig_op1 = XEXP (comparison, 1);
5462 439781 : rtx op2p = op2;
5463 439781 : rtx op3p = op3;
5464 : /* If we are optimizing, force expensive constants into a register
5465 : but preserve an eventual equality with op2/op3. */
5466 1 : if (CONSTANT_P (orig_op0) && optimize
5467 1 : && cmpmode == mode
5468 1 : && (rtx_cost (orig_op0, mode, COMPARE, 0,
5469 1 : optimize_insn_for_speed_p ())
5470 : > COSTS_N_INSNS (1))
5471 439781 : && can_create_pseudo_p ())
5472 : {
5473 0 : if (rtx_equal_p (orig_op0, op2))
5474 0 : op2p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5475 0 : else if (rtx_equal_p (orig_op0, op3))
5476 0 : op3p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
5477 : }
5478 264360 : if (CONSTANT_P (orig_op1) && optimize
5479 243417 : && cmpmode == mode
5480 73433 : && (rtx_cost (orig_op1, mode, COMPARE, 0,
5481 73433 : optimize_insn_for_speed_p ())
5482 : > COSTS_N_INSNS (1))
5483 439949 : && can_create_pseudo_p ())
5484 : {
5485 168 : if (rtx_equal_p (orig_op1, op2))
5486 102 : op2p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5487 66 : else if (rtx_equal_p (orig_op1, op3))
5488 42 : op3p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
5489 : }
5490 439781 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5491 439781 : GET_CODE (comparison), NULL_RTX, unsignedp,
5492 : OPTAB_WIDEN, &comparison, &cmpmode, cbranch_optab);
5493 439781 : if (comparison)
5494 : {
5495 437645 : rtx res = emit_conditional_move_1 (target, comparison,
5496 : op2p, op3p, mode);
5497 437645 : if (res != NULL_RTX)
5498 375435 : return res;
5499 : }
5500 64346 : delete_insns_since (last);
5501 64346 : restore_pending_stack_adjust (&save);
5502 : }
5503 :
5504 65212 : if (pass == 1)
5505 : return NULL_RTX;
5506 :
5507 : /* If the preferred op2/op3 order is not usable, retry with other
5508 : operand order, perhaps it will expand successfully. */
5509 33702 : if (swapped)
5510 : comp.code = orig_code;
5511 65876 : else if ((reversed =
5512 32938 : reversed_comparison_code_parts (orig_code, comp.op0, comp.op1,
5513 : NULL))
5514 : != UNKNOWN)
5515 : comp.code = reversed;
5516 : else
5517 : return NULL_RTX;
5518 : std::swap (op2, op3);
5519 : }
5520 : }
5521 :
5522 : /* Helper function that, in addition to COMPARISON, also tries
5523 : the reversed REV_COMPARISON with swapped OP2 and OP3. As opposed
5524 : to when we pass the specific constituents of a comparison, no
5525 : additional insns are emitted for it. It might still be necessary
5526 : to emit more than one insn for the final conditional move, though. */
5527 :
5528 : rtx
5529 88821 : emit_conditional_move (rtx target, rtx comparison, rtx rev_comparison,
5530 : rtx op2, rtx op3, machine_mode mode)
5531 : {
5532 88821 : rtx res = emit_conditional_move_1 (target, comparison, op2, op3, mode);
5533 :
5534 88821 : if (res != NULL_RTX)
5535 : return res;
5536 :
5537 4977 : return emit_conditional_move_1 (target, rev_comparison, op3, op2, mode);
5538 : }
5539 :
5540 : /* Helper for emitting a conditional move. */
5541 :
5542 : static rtx
5543 531443 : emit_conditional_move_1 (rtx target, rtx comparison,
5544 : rtx op2, rtx op3, machine_mode mode)
5545 : {
5546 531443 : enum insn_code icode;
5547 :
5548 531443 : if (comparison == NULL_RTX || !COMPARISON_P (comparison))
5549 : return NULL_RTX;
5550 :
5551 : /* If the two source operands are identical, that's just a move.
5552 : As the comparison comes in non-canonicalized, we must make
5553 : sure not to discard any possible side effects. If there are
5554 : side effects, just let the target handle it. */
5555 531443 : if (!side_effects_p (comparison) && rtx_equal_p (op2, op3))
5556 : {
5557 0 : if (!target)
5558 0 : target = gen_reg_rtx (mode);
5559 :
5560 0 : emit_move_insn (target, op3);
5561 0 : return target;
5562 : }
5563 :
5564 531443 : if (mode == VOIDmode)
5565 0 : mode = GET_MODE (op2);
5566 :
5567 531443 : icode = direct_optab_handler (movcc_optab, mode);
5568 :
5569 531443 : if (icode == CODE_FOR_nothing)
5570 : return NULL_RTX;
5571 :
5572 531443 : if (!target)
5573 0 : target = gen_reg_rtx (mode);
5574 :
5575 531443 : class expand_operand ops[4];
5576 :
5577 531443 : create_output_operand (&ops[0], target, mode);
5578 531443 : create_fixed_operand (&ops[1], comparison);
5579 531443 : create_input_operand (&ops[2], op2, mode);
5580 531443 : create_input_operand (&ops[3], op3, mode);
5581 :
5582 531443 : if (maybe_expand_insn (icode, 4, ops))
5583 : {
5584 459279 : if (ops[0].value != target)
5585 55 : convert_move (target, ops[0].value, false);
5586 459279 : return target;
5587 : }
5588 :
5589 : return NULL_RTX;
5590 : }
5591 :
5592 :
5593 : /* Emit a conditional negate or bitwise complement using the
5594 : negcc or notcc optabs if available. Return NULL_RTX if such operations
5595 : are not available. Otherwise return the RTX holding the result.
5596 : TARGET is the desired destination of the result. COMP is the comparison
5597 : on which to negate. If COND is true move into TARGET the negation
5598 : or bitwise complement of OP1. Otherwise move OP2 into TARGET.
5599 : CODE is either NEG or NOT. MODE is the machine mode in which the
5600 : operation is performed. */
5601 :
5602 : rtx
5603 1230 : emit_conditional_neg_or_complement (rtx target, rtx_code code,
5604 : machine_mode mode, rtx cond, rtx op1,
5605 : rtx op2)
5606 : {
5607 1230 : optab op = unknown_optab;
5608 1230 : if (code == NEG)
5609 : op = negcc_optab;
5610 68 : else if (code == NOT)
5611 : op = notcc_optab;
5612 : else
5613 0 : gcc_unreachable ();
5614 :
5615 1230 : insn_code icode = direct_optab_handler (op, mode);
5616 :
5617 1230 : if (icode == CODE_FOR_nothing)
5618 : return NULL_RTX;
5619 :
5620 0 : if (!target)
5621 0 : target = gen_reg_rtx (mode);
5622 :
5623 0 : rtx_insn *last = get_last_insn ();
5624 0 : class expand_operand ops[4];
5625 :
5626 0 : create_output_operand (&ops[0], target, mode);
5627 0 : create_fixed_operand (&ops[1], cond);
5628 0 : create_input_operand (&ops[2], op1, mode);
5629 0 : create_input_operand (&ops[3], op2, mode);
5630 :
5631 0 : if (maybe_expand_insn (icode, 4, ops))
5632 : {
5633 0 : if (ops[0].value != target)
5634 0 : convert_move (target, ops[0].value, false);
5635 :
5636 0 : return target;
5637 : }
5638 0 : delete_insns_since (last);
5639 0 : return NULL_RTX;
5640 : }
5641 :
5642 : /* Emit a conditional addition instruction if the machine supports one for that
5643 : condition and machine mode.
5644 :
5645 : OP0 and OP1 are the operands that should be compared using CODE. CMODE is
5646 : the mode to use should they be constants. If it is VOIDmode, they cannot
5647 : both be constants.
5648 :
5649 : OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
5650 : should be stored there. MODE is the mode to use should they be constants.
5651 : If it is VOIDmode, they cannot both be constants.
5652 :
5653 : The result is either TARGET (perhaps modified) or NULL_RTX if the operation
5654 : is not supported. */
5655 :
5656 : rtx
5657 15172 : emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
5658 : machine_mode cmode, rtx op2, rtx op3,
5659 : machine_mode mode, int unsignedp)
5660 : {
5661 15172 : rtx comparison;
5662 15172 : rtx_insn *last;
5663 15172 : enum insn_code icode;
5664 :
5665 : /* If one operand is constant, make it the second one. Only do this
5666 : if the other operand is not constant as well. */
5667 :
5668 15172 : if (swap_commutative_operands_p (op0, op1))
5669 : {
5670 0 : std::swap (op0, op1);
5671 0 : code = swap_condition (code);
5672 : }
5673 :
5674 : /* get_condition will prefer to generate LT and GT even if the old
5675 : comparison was against zero, so undo that canonicalization here since
5676 : comparisons against zero are cheaper. */
5677 15172 : if (code == LT && op1 == const1_rtx)
5678 36 : code = LE, op1 = const0_rtx;
5679 15136 : else if (code == GT && op1 == constm1_rtx)
5680 89 : code = GE, op1 = const0_rtx;
5681 :
5682 15172 : if (cmode == VOIDmode)
5683 15172 : cmode = GET_MODE (op0);
5684 :
5685 15172 : if (mode == VOIDmode)
5686 0 : mode = GET_MODE (op2);
5687 :
5688 15172 : icode = optab_handler (addcc_optab, mode);
5689 :
5690 15172 : if (icode == CODE_FOR_nothing)
5691 : return 0;
5692 :
5693 14597 : if (!target)
5694 0 : target = gen_reg_rtx (mode);
5695 :
5696 14597 : code = unsignedp ? unsigned_condition (code) : code;
5697 14597 : comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
5698 :
5699 : /* We can get const0_rtx or const_true_rtx in some circumstances. Just
5700 : return NULL and let the caller figure out how best to deal with this
5701 : situation. */
5702 14597 : if (!COMPARISON_P (comparison))
5703 : return NULL_RTX;
5704 :
5705 14597 : do_pending_stack_adjust ();
5706 14597 : last = get_last_insn ();
5707 14597 : prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
5708 14597 : GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
5709 : &comparison, &cmode, cbranch_optab);
5710 14597 : if (comparison)
5711 : {
5712 14597 : class expand_operand ops[4];
5713 :
5714 14597 : create_output_operand (&ops[0], target, mode);
5715 14597 : create_fixed_operand (&ops[1], comparison);
5716 14597 : create_input_operand (&ops[2], op2, mode);
5717 14597 : create_input_operand (&ops[3], op3, mode);
5718 14597 : if (maybe_expand_insn (icode, 4, ops))
5719 : {
5720 1270 : if (ops[0].value != target)
5721 0 : convert_move (target, ops[0].value, false);
5722 1270 : return target;
5723 : }
5724 : }
5725 13327 : delete_insns_since (last);
5726 13327 : return NULL_RTX;
5727 : }
5728 :
5729 : /* These functions attempt to generate an insn body, rather than
5730 : emitting the insn, but if the gen function already emits them, we
5731 : make no attempt to turn them back into naked patterns. */
5732 :
5733 : /* Generate and return an insn body to add Y to X. */
5734 :
5735 : rtx_insn *
5736 251 : gen_add2_insn (rtx x, rtx y)
5737 : {
5738 251 : enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
5739 :
5740 251 : gcc_assert (insn_operand_matches (icode, 0, x));
5741 251 : gcc_assert (insn_operand_matches (icode, 1, x));
5742 251 : gcc_assert (insn_operand_matches (icode, 2, y));
5743 :
5744 251 : return GEN_FCN (icode) (x, x, y);
5745 : }
5746 :
5747 : /* Generate and return an insn body to add r1 and c,
5748 : storing the result in r0. */
5749 :
5750 : rtx_insn *
5751 0 : gen_add3_insn (rtx r0, rtx r1, rtx c)
5752 : {
5753 0 : enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
5754 :
5755 0 : if (icode == CODE_FOR_nothing
5756 0 : || !insn_operand_matches (icode, 0, r0)
5757 0 : || !insn_operand_matches (icode, 1, r1)
5758 0 : || !insn_operand_matches (icode, 2, c))
5759 0 : return NULL;
5760 :
5761 0 : return GEN_FCN (icode) (r0, r1, c);
5762 : }
5763 :
5764 : bool
5765 4219 : have_add2_insn (rtx x, rtx y)
5766 : {
5767 4219 : enum insn_code icode;
5768 :
5769 4219 : gcc_assert (GET_MODE (x) != VOIDmode);
5770 :
5771 4219 : icode = optab_handler (add_optab, GET_MODE (x));
5772 :
5773 4219 : if (icode == CODE_FOR_nothing)
5774 : return false;
5775 :
5776 4219 : if (!insn_operand_matches (icode, 0, x)
5777 4219 : || !insn_operand_matches (icode, 1, x)
5778 8438 : || !insn_operand_matches (icode, 2, y))
5779 201 : return false;
5780 :
5781 : return true;
5782 : }
5783 :
5784 : /* Generate and return an insn body to add Y to X. */
5785 :
5786 : rtx_insn *
5787 0 : gen_addptr3_insn (rtx x, rtx y, rtx z)
5788 : {
5789 0 : enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
5790 :
5791 0 : gcc_assert (insn_operand_matches (icode, 0, x));
5792 0 : gcc_assert (insn_operand_matches (icode, 1, y));
5793 0 : gcc_assert (insn_operand_matches (icode, 2, z));
5794 :
5795 0 : return GEN_FCN (icode) (x, y, z);
5796 : }
5797 :
5798 : /* Return true if the target implements an addptr pattern and X, Y,
5799 : and Z are valid for the pattern predicates. */
5800 :
5801 : bool
5802 628445 : have_addptr3_insn (rtx x, rtx y, rtx z)
5803 : {
5804 628445 : enum insn_code icode;
5805 :
5806 628445 : gcc_assert (GET_MODE (x) != VOIDmode);
5807 :
5808 628445 : icode = optab_handler (addptr3_optab, GET_MODE (x));
5809 :
5810 628445 : if (icode == CODE_FOR_nothing)
5811 : return false;
5812 :
5813 0 : if (!insn_operand_matches (icode, 0, x)
5814 0 : || !insn_operand_matches (icode, 1, y)
5815 0 : || !insn_operand_matches (icode, 2, z))
5816 0 : return false;
5817 :
5818 : return true;
5819 : }
5820 :
5821 : /* Generate and return an insn body to subtract Y from X. */
5822 :
5823 : rtx_insn *
5824 11 : gen_sub2_insn (rtx x, rtx y)
5825 : {
5826 11 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
5827 :
5828 11 : gcc_assert (insn_operand_matches (icode, 0, x));
5829 11 : gcc_assert (insn_operand_matches (icode, 1, x));
5830 11 : gcc_assert (insn_operand_matches (icode, 2, y));
5831 :
5832 11 : return GEN_FCN (icode) (x, x, y);
5833 : }
5834 :
5835 : /* Generate and return an insn body to subtract r1 and c,
5836 : storing the result in r0. */
5837 :
5838 : rtx_insn *
5839 1532 : gen_sub3_insn (rtx r0, rtx r1, rtx c)
5840 : {
5841 1532 : enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
5842 :
5843 1532 : if (icode == CODE_FOR_nothing
5844 1532 : || !insn_operand_matches (icode, 0, r0)
5845 1532 : || !insn_operand_matches (icode, 1, r1)
5846 3064 : || !insn_operand_matches (icode, 2, c))
5847 0 : return NULL;
5848 :
5849 1532 : return GEN_FCN (icode) (r0, r1, c);
5850 : }
5851 :
5852 : bool
5853 0 : have_sub2_insn (rtx x, rtx y)
5854 : {
5855 0 : enum insn_code icode;
5856 :
5857 0 : gcc_assert (GET_MODE (x) != VOIDmode);
5858 :
5859 0 : icode = optab_handler (sub_optab, GET_MODE (x));
5860 :
5861 0 : if (icode == CODE_FOR_nothing)
5862 : return false;
5863 :
5864 0 : if (!insn_operand_matches (icode, 0, x)
5865 0 : || !insn_operand_matches (icode, 1, x)
5866 0 : || !insn_operand_matches (icode, 2, y))
5867 0 : return false;
5868 :
5869 : return true;
5870 : }
5871 :
5872 : /* Generate the body of an insn to extend Y (with mode MFROM)
5873 : into X (with mode MTO). Do zero-extension if UNSIGNEDP is nonzero. */
5874 :
5875 : rtx_insn *
5876 29976 : gen_extend_insn (rtx x, rtx y, machine_mode mto,
5877 : machine_mode mfrom, int unsignedp)
5878 : {
5879 29976 : enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
5880 29976 : return GEN_FCN (icode) (x, y);
5881 : }
5882 :
5883 : /* Generate code to convert FROM to floating point
5884 : and store in TO. FROM must be fixed point and not VOIDmode.
5885 : UNSIGNEDP nonzero means regard FROM as unsigned.
5886 : Normally this is done by correcting the final value
5887 : if it is negative. */
5888 :
5889 : void
5890 144046 : expand_float (rtx to, rtx from, int unsignedp)
5891 : {
5892 144046 : enum insn_code icode;
5893 144046 : rtx target = to;
5894 144046 : scalar_mode from_mode, to_mode;
5895 144046 : machine_mode fmode, imode;
5896 144046 : bool can_do_signed = false;
5897 :
5898 : /* Crash now, because we won't be able to decide which mode to use. */
5899 144046 : gcc_assert (GET_MODE (from) != VOIDmode);
5900 :
5901 : /* Look for an insn to do the conversion. Do it in the specified
5902 : modes if possible; otherwise convert either input, output or both to
5903 : wider mode. If the integer mode is wider than the mode of FROM,
5904 : we can do the conversion signed even if the input is unsigned. */
5905 :
5906 166086 : FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
5907 260157 : FOR_EACH_MODE_FROM (imode, GET_MODE (from))
5908 : {
5909 238117 : int doing_unsigned = unsignedp;
5910 :
5911 268207 : if (fmode != GET_MODE (to)
5912 238117 : && (significand_size (fmode)
5913 94442 : < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
5914 30090 : continue;
5915 :
5916 208027 : icode = can_float_p (fmode, imode, unsignedp);
5917 208027 : if (icode == CODE_FOR_nothing && unsignedp)
5918 : {
5919 35690 : enum insn_code scode = can_float_p (fmode, imode, 0);
5920 35690 : if (scode != CODE_FOR_nothing)
5921 6337 : can_do_signed = true;
5922 35690 : if (imode != GET_MODE (from))
5923 24599 : icode = scode, doing_unsigned = 0;
5924 : }
5925 :
5926 208027 : if (icode != CODE_FOR_nothing)
5927 : {
5928 135454 : if (imode != GET_MODE (from))
5929 10169 : from = convert_to_mode (imode, from, unsignedp);
5930 :
5931 135454 : if (fmode != GET_MODE (to))
5932 1590 : target = gen_reg_rtx (fmode);
5933 :
5934 266198 : emit_unop_insn (icode, target, from,
5935 : doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
5936 :
5937 135454 : if (target != to)
5938 1590 : convert_move (to, target, 0);
5939 135454 : return;
5940 : }
5941 : }
5942 :
5943 : /* Unsigned integer, and no way to convert directly. Convert as signed,
5944 : then unconditionally adjust the result. */
5945 8592 : if (unsignedp
5946 8592 : && can_do_signed
5947 1098 : && is_a <scalar_mode> (GET_MODE (to), &to_mode)
5948 9690 : && is_a <scalar_mode> (GET_MODE (from), &from_mode))
5949 : {
5950 1098 : opt_scalar_mode fmode_iter;
5951 1098 : rtx_code_label *label = gen_label_rtx ();
5952 1098 : rtx temp;
5953 1098 : REAL_VALUE_TYPE offset;
5954 :
5955 : /* Look for a usable floating mode FMODE wider than the source and at
5956 : least as wide as the target. Using FMODE will avoid rounding woes
5957 : with unsigned values greater than the signed maximum value. */
5958 :
5959 2105 : FOR_EACH_MODE_FROM (fmode_iter, to_mode)
5960 : {
5961 2105 : scalar_mode fmode = fmode_iter.require ();
5962 3112 : if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
5963 2105 : && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
5964 : break;
5965 : }
5966 :
5967 1098 : if (!fmode_iter.exists (&fmode))
5968 : {
5969 : /* There is no such mode. Pretend the target is wide enough. */
5970 0 : fmode = to_mode;
5971 :
5972 : /* Avoid double-rounding when TO is narrower than FROM. */
5973 0 : if ((significand_size (fmode) + 1)
5974 0 : < GET_MODE_PRECISION (from_mode))
5975 : {
5976 0 : rtx temp1;
5977 0 : rtx_code_label *neglabel = gen_label_rtx ();
5978 :
5979 : /* Don't use TARGET if it isn't a register, is a hard register,
5980 : or is the wrong mode. */
5981 0 : if (!REG_P (target)
5982 0 : || REGNO (target) < FIRST_PSEUDO_REGISTER
5983 0 : || GET_MODE (target) != fmode)
5984 0 : target = gen_reg_rtx (fmode);
5985 :
5986 0 : imode = from_mode;
5987 0 : do_pending_stack_adjust ();
5988 :
5989 : /* Test whether the sign bit is set. */
5990 0 : emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
5991 : 0, neglabel);
5992 :
5993 : /* The sign bit is not set. Convert as signed. */
5994 0 : expand_float (target, from, 0);
5995 0 : emit_jump_insn (targetm.gen_jump (label));
5996 0 : emit_barrier ();
5997 :
5998 : /* The sign bit is set.
5999 : Convert to a usable (positive signed) value by shifting right
6000 : one bit, while remembering if a nonzero bit was shifted
6001 : out; i.e., compute (from & 1) | (from >> 1). */
6002 :
6003 0 : emit_label (neglabel);
6004 0 : temp = expand_binop (imode, and_optab, from, const1_rtx,
6005 : NULL_RTX, 1, OPTAB_LIB_WIDEN);
6006 0 : temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
6007 0 : temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
6008 : OPTAB_LIB_WIDEN);
6009 0 : expand_float (target, temp, 0);
6010 :
6011 : /* Multiply by 2 to undo the shift above. */
6012 0 : temp = expand_binop (fmode, add_optab, target, target,
6013 : target, 0, OPTAB_LIB_WIDEN);
6014 0 : if (temp != target)
6015 0 : emit_move_insn (target, temp);
6016 :
6017 0 : do_pending_stack_adjust ();
6018 0 : emit_label (label);
6019 0 : goto done;
6020 : }
6021 : }
6022 :
6023 : /* If we are about to do some arithmetic to correct for an
6024 : unsigned operand, do it in a pseudo-register. */
6025 :
6026 1098 : if (to_mode != fmode
6027 1098 : || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
6028 399 : target = gen_reg_rtx (fmode);
6029 :
6030 : /* Convert as signed integer to floating. */
6031 1098 : expand_float (target, from, 0);
6032 :
6033 : /* If FROM is negative (and therefore TO is negative),
6034 : correct its value by 2**bitwidth. */
6035 :
6036 1098 : do_pending_stack_adjust ();
6037 1098 : emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
6038 : 0, label);
6039 :
6040 :
6041 1098 : real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
6042 1098 : temp = expand_binop (fmode, add_optab, target,
6043 : const_double_from_real_value (offset, fmode),
6044 : target, 0, OPTAB_LIB_WIDEN);
6045 1098 : if (temp != target)
6046 0 : emit_move_insn (target, temp);
6047 :
6048 1098 : do_pending_stack_adjust ();
6049 1098 : emit_label (label);
6050 1098 : goto done;
6051 : }
6052 :
6053 : /* No hardware instruction available; call a library routine. */
6054 7494 : {
6055 7494 : rtx libfunc;
6056 7494 : rtx_insn *insns;
6057 7494 : rtx value;
6058 7494 : convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
6059 :
6060 7494 : if (is_narrower_int_mode (GET_MODE (from), SImode))
6061 974 : from = convert_to_mode (SImode, from, unsignedp);
6062 :
6063 7494 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
6064 7494 : gcc_assert (libfunc);
6065 :
6066 7494 : start_sequence ();
6067 :
6068 14988 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
6069 7494 : GET_MODE (to), from, GET_MODE (from));
6070 7494 : insns = end_sequence ();
6071 :
6072 7494 : emit_libcall_block (insns, target, value,
6073 7494 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
6074 : GET_MODE (to), from));
6075 : }
6076 :
6077 8592 : done:
6078 :
6079 : /* Copy result to requested destination
6080 : if we have been computing in a temp location. */
6081 :
6082 8592 : if (target != to)
6083 : {
6084 399 : if (GET_MODE (target) == GET_MODE (to))
6085 4 : emit_move_insn (to, target);
6086 : else
6087 395 : convert_move (to, target, 0);
6088 : }
6089 : }
6090 :
6091 : /* Generate code to convert FROM to fixed point and store in TO. FROM
6092 : must be floating point. */
6093 :
6094 : void
6095 59113 : expand_fix (rtx to, rtx from, int unsignedp)
6096 : {
6097 59113 : enum insn_code icode;
6098 59113 : rtx target = to;
6099 59113 : machine_mode fmode, imode;
6100 59113 : opt_scalar_mode fmode_iter;
6101 59113 : bool must_trunc = false;
6102 :
6103 : /* We first try to find a pair of modes, one real and one integer, at
6104 : least as wide as FROM and TO, respectively, in which we can open-code
6105 : this conversion. If the integer mode is wider than the mode of TO,
6106 : we can do the conversion either signed or unsigned. */
6107 :
6108 89334 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
6109 213688 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
6110 : {
6111 183467 : int doing_unsigned = unsignedp;
6112 :
6113 183467 : icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
6114 183467 : if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
6115 61686 : icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
6116 :
6117 140555 : if (icode != CODE_FOR_nothing)
6118 : {
6119 46126 : rtx_insn *last = get_last_insn ();
6120 46126 : rtx from1 = from;
6121 46126 : if (fmode != GET_MODE (from))
6122 : {
6123 1095 : if (REAL_MODE_FORMAT (GET_MODE (from))
6124 : == &arm_bfloat_half_format
6125 1095 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
6126 : /* The BF -> SF conversions can be just a shift, doesn't
6127 : need to handle sNANs. */
6128 : {
6129 38 : int save_flag_finite_math_only = flag_finite_math_only;
6130 38 : flag_finite_math_only = true;
6131 38 : from1 = convert_to_mode (fmode, from, 0);
6132 38 : flag_finite_math_only = save_flag_finite_math_only;
6133 : }
6134 : else
6135 1057 : from1 = convert_to_mode (fmode, from, 0);
6136 : }
6137 :
6138 46126 : if (must_trunc)
6139 : {
6140 0 : rtx temp = gen_reg_rtx (GET_MODE (from1));
6141 0 : from1 = expand_unop (GET_MODE (from1), ftrunc_optab, from1,
6142 : temp, 0);
6143 : }
6144 :
6145 46126 : if (imode != GET_MODE (to))
6146 10833 : target = gen_reg_rtx (imode);
6147 :
6148 87986 : if (maybe_emit_unop_insn (icode, target, from1,
6149 : doing_unsigned ? UNSIGNED_FIX : FIX))
6150 : {
6151 46126 : if (target != to)
6152 10833 : convert_move (to, target, unsignedp);
6153 50136 : return;
6154 : }
6155 0 : delete_insns_since (last);
6156 : }
6157 : }
6158 :
6159 : /* For an unsigned conversion, there is one more way to do it.
6160 : If we have a signed conversion, we generate code that compares
6161 : the real value to the largest representable positive number. If if
6162 : is smaller, the conversion is done normally. Otherwise, subtract
6163 : one plus the highest signed number, convert, and add it back.
6164 :
6165 : We only need to check all real modes, since we know we didn't find
6166 : anything with a wider integer mode.
6167 :
6168 : This code used to extend FP value into mode wider than the destination.
6169 : This is needed for decimal float modes which cannot accurately
6170 : represent one plus the highest signed number of the same size, but
6171 : not for binary modes. Consider, for instance conversion from SFmode
6172 : into DImode.
6173 :
6174 : The hot path through the code is dealing with inputs smaller than 2^63
6175 : and doing just the conversion, so there is no bits to lose.
6176 :
6177 : In the other path we know the value is positive in the range 2^63..2^64-1
6178 : inclusive. (as for other input overflow happens and result is undefined)
6179 : So we know that the most important bit set in mantissa corresponds to
6180 : 2^63. The subtraction of 2^63 should not generate any rounding as it
6181 : simply clears out that bit. The rest is trivial. */
6182 :
6183 12987 : scalar_int_mode to_mode;
6184 12987 : if (unsignedp
6185 7375 : && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
6186 20362 : && HWI_COMPUTABLE_MODE_P (to_mode))
6187 8994 : FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
6188 : {
6189 6796 : scalar_mode fmode = fmode_iter.require ();
6190 6796 : if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
6191 : 0, &must_trunc)
6192 6796 : && (!DECIMAL_FLOAT_MODE_P (fmode)
6193 0 : || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
6194 : {
6195 4006 : int bitsize;
6196 4006 : REAL_VALUE_TYPE offset;
6197 4006 : rtx limit;
6198 4006 : rtx_code_label *lab1, *lab2;
6199 4006 : rtx_insn *insn;
6200 :
6201 4006 : bitsize = GET_MODE_PRECISION (to_mode);
6202 4006 : real_2expN (&offset, bitsize - 1, fmode);
6203 4006 : limit = const_double_from_real_value (offset, fmode);
6204 4006 : lab1 = gen_label_rtx ();
6205 4006 : lab2 = gen_label_rtx ();
6206 :
6207 4006 : if (fmode != GET_MODE (from))
6208 : {
6209 184 : if (REAL_MODE_FORMAT (GET_MODE (from))
6210 : == &arm_bfloat_half_format
6211 184 : && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
6212 : /* The BF -> SF conversions can be just a shift, doesn't
6213 : need to handle sNANs. */
6214 : {
6215 4 : int save_flag_finite_math_only = flag_finite_math_only;
6216 4 : flag_finite_math_only = true;
6217 4 : from = convert_to_mode (fmode, from, 0);
6218 4 : flag_finite_math_only = save_flag_finite_math_only;
6219 : }
6220 : else
6221 180 : from = convert_to_mode (fmode, from, 0);
6222 : }
6223 :
6224 : /* See if we need to do the subtraction. */
6225 4006 : do_pending_stack_adjust ();
6226 4006 : emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
6227 4006 : GET_MODE (from), 0, lab1);
6228 :
6229 : /* If not, do the signed "fix" and branch around fixup code. */
6230 4006 : expand_fix (to, from, 0);
6231 4006 : emit_jump_insn (targetm.gen_jump (lab2));
6232 4006 : emit_barrier ();
6233 :
6234 : /* Otherwise, subtract 2**(N-1), convert to signed number,
6235 : then add 2**(N-1). Do the addition using XOR since this
6236 : will often generate better code. */
6237 4006 : emit_label (lab1);
6238 4006 : target = expand_binop (GET_MODE (from), sub_optab, from, limit,
6239 : NULL_RTX, 0, OPTAB_LIB_WIDEN);
6240 4006 : expand_fix (to, target, 0);
6241 4006 : target = expand_binop (to_mode, xor_optab, to,
6242 : gen_int_mode
6243 4006 : (HOST_WIDE_INT_1 << (bitsize - 1),
6244 : to_mode),
6245 : to, 1, OPTAB_LIB_WIDEN);
6246 :
6247 4006 : if (target != to)
6248 0 : emit_move_insn (to, target);
6249 :
6250 4006 : emit_label (lab2);
6251 :
6252 4006 : if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
6253 : {
6254 : /* Make a place for a REG_NOTE and add it. */
6255 4006 : insn = emit_move_insn (to, to);
6256 4006 : set_dst_reg_note (insn, REG_EQUAL,
6257 : gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
6258 : copy_rtx (from)),
6259 : to);
6260 : }
6261 :
6262 4006 : return;
6263 : }
6264 : }
6265 :
6266 : #ifdef HAVE_SFmode
6267 8981 : if (REAL_MODE_FORMAT (GET_MODE (from)) == &arm_bfloat_half_format
6268 8981 : && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
6269 : /* We don't have BF -> TI library functions, use BF -> SF -> TI
6270 : instead but the BF -> SF conversion can be just a shift, doesn't
6271 : need to handle sNANs. */
6272 : {
6273 4 : int save_flag_finite_math_only = flag_finite_math_only;
6274 4 : flag_finite_math_only = true;
6275 4 : from = convert_to_mode (SFmode, from, 0);
6276 4 : flag_finite_math_only = save_flag_finite_math_only;
6277 4 : expand_fix (to, from, unsignedp);
6278 4 : return;
6279 : }
6280 : #endif
6281 :
6282 : /* We can't do it with an insn, so use a library call. But first ensure
6283 : that the mode of TO is at least as wide as SImode, since those are the
6284 : only library calls we know about. */
6285 :
6286 8977 : if (is_narrower_int_mode (GET_MODE (to), SImode))
6287 : {
6288 1954 : target = gen_reg_rtx (SImode);
6289 :
6290 1954 : expand_fix (target, from, unsignedp);
6291 : }
6292 : else
6293 : {
6294 7023 : rtx_insn *insns;
6295 7023 : rtx value;
6296 7023 : rtx libfunc;
6297 :
6298 7023 : convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
6299 7023 : libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
6300 7023 : gcc_assert (libfunc);
6301 :
6302 7023 : start_sequence ();
6303 :
6304 14046 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
6305 7023 : GET_MODE (to), from, GET_MODE (from));
6306 7023 : insns = end_sequence ();
6307 :
6308 7023 : emit_libcall_block (insns, target, value,
6309 7023 : gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
6310 : GET_MODE (to), from));
6311 : }
6312 :
6313 8977 : if (target != to)
6314 : {
6315 1954 : if (GET_MODE (to) == GET_MODE (target))
6316 0 : emit_move_insn (to, target);
6317 : else
6318 1954 : convert_move (to, target, 0);
6319 : }
6320 : }
6321 :
6322 :
6323 : /* Promote integer arguments for a libcall if necessary.
6324 : emit_library_call_value cannot do the promotion because it does not
6325 : know if it should do a signed or unsigned promotion. This is because
6326 : there are no tree types defined for libcalls. */
6327 :
6328 : static rtx
6329 0 : prepare_libcall_arg (rtx arg, int uintp)
6330 : {
6331 0 : scalar_int_mode mode;
6332 0 : machine_mode arg_mode;
6333 0 : if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
6334 : {
6335 : /* If we need to promote the integer function argument we need to do
6336 : it here instead of inside emit_library_call_value because in
6337 : emit_library_call_value we don't know if we should do a signed or
6338 : unsigned promotion. */
6339 :
6340 0 : int unsigned_p = 0;
6341 0 : arg_mode = promote_function_mode (NULL_TREE, mode,
6342 : &unsigned_p, NULL_TREE, 0);
6343 0 : if (arg_mode != mode)
6344 0 : return convert_to_mode (arg_mode, arg, uintp);
6345 : }
6346 : return arg;
6347 : }
6348 :
6349 : /* Generate code to convert FROM or TO a fixed-point.
6350 : If UINTP is true, either TO or FROM is an unsigned integer.
6351 : If SATP is true, we need to saturate the result. */
6352 :
6353 : void
6354 0 : expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
6355 : {
6356 0 : machine_mode to_mode = GET_MODE (to);
6357 0 : machine_mode from_mode = GET_MODE (from);
6358 0 : convert_optab tab;
6359 0 : enum rtx_code this_code;
6360 0 : enum insn_code code;
6361 0 : rtx_insn *insns;
6362 0 : rtx value;
6363 0 : rtx libfunc;
6364 :
6365 0 : if (to_mode == from_mode)
6366 : {
6367 0 : emit_move_insn (to, from);
6368 0 : return;
6369 : }
6370 :
6371 0 : if (uintp)
6372 : {
6373 0 : tab = satp ? satfractuns_optab : fractuns_optab;
6374 0 : this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
6375 : }
6376 : else
6377 : {
6378 0 : tab = satp ? satfract_optab : fract_optab;
6379 0 : this_code = satp ? SAT_FRACT : FRACT_CONVERT;
6380 : }
6381 0 : code = convert_optab_handler (tab, to_mode, from_mode);
6382 0 : if (code != CODE_FOR_nothing)
6383 : {
6384 0 : emit_unop_insn (code, to, from, this_code);
6385 0 : return;
6386 : }
6387 :
6388 0 : libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
6389 0 : gcc_assert (libfunc);
6390 :
6391 0 : from = prepare_libcall_arg (from, uintp);
6392 0 : from_mode = GET_MODE (from);
6393 :
6394 0 : start_sequence ();
6395 0 : value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
6396 : from, from_mode);
6397 0 : insns = end_sequence ();
6398 :
6399 0 : emit_libcall_block (insns, to, value,
6400 : gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
6401 : }
6402 :
6403 : /* Generate code to convert FROM to fixed point and store in TO. FROM
6404 : must be floating point, TO must be signed. Use the conversion optab
6405 : TAB to do the conversion. */
6406 :
6407 : bool
6408 480 : expand_sfix_optab (rtx to, rtx from, convert_optab tab)
6409 : {
6410 480 : enum insn_code icode;
6411 480 : rtx target = to;
6412 480 : machine_mode fmode, imode;
6413 :
6414 : /* We first try to find a pair of modes, one real and one integer, at
6415 : least as wide as FROM and TO, respectively, in which we can open-code
6416 : this conversion. If the integer mode is wider than the mode of TO,
6417 : we can do the conversion either signed or unsigned. */
6418 :
6419 2008 : FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
6420 8618 : FOR_EACH_MODE_FROM (imode, GET_MODE (to))
6421 : {
6422 7090 : icode = convert_optab_handler (tab, imode, fmode,
6423 : insn_optimization_type ());
6424 7090 : if (icode != CODE_FOR_nothing)
6425 : {
6426 52 : rtx_insn *last = get_last_insn ();
6427 52 : if (fmode != GET_MODE (from))
6428 52 : from = convert_to_mode (fmode, from, 0);
6429 :
6430 52 : if (imode != GET_MODE (to))
6431 0 : target = gen_reg_rtx (imode);
6432 :
6433 52 : if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
6434 : {
6435 0 : delete_insns_since (last);
6436 0 : continue;
6437 : }
6438 52 : if (target != to)
6439 0 : convert_move (to, target, 0);
6440 52 : return true;
6441 : }
6442 : }
6443 :
6444 : return false;
6445 : }
6446 :
6447 : /* Report whether we have an instruction to perform the operation
6448 : specified by CODE on operands of mode MODE. */
6449 : bool
6450 93306328 : have_insn_for (enum rtx_code code, machine_mode mode)
6451 : {
6452 93306328 : return (code_to_optab (code)
6453 93306328 : && (optab_handler (code_to_optab (code), mode)
6454 93306328 : != CODE_FOR_nothing));
6455 : }
6456 :
6457 : /* Print information about the current contents of the optabs on
6458 : STDERR. */
6459 :
6460 : DEBUG_FUNCTION void
6461 0 : debug_optab_libfuncs (void)
6462 : {
6463 0 : int i, j, k;
6464 :
6465 : /* Dump the arithmetic optabs. */
6466 0 : for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
6467 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6468 : {
6469 0 : rtx l = optab_libfunc ((optab) i, (machine_mode) j);
6470 0 : if (l)
6471 : {
6472 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6473 0 : fprintf (stderr, "%s\t%s:\t%s\n",
6474 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6475 0 : GET_MODE_NAME (j),
6476 : XSTR (l, 0));
6477 : }
6478 : }
6479 :
6480 : /* Dump the conversion optabs. */
6481 0 : for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
6482 0 : for (j = 0; j < NUM_MACHINE_MODES; ++j)
6483 0 : for (k = 0; k < NUM_MACHINE_MODES; ++k)
6484 : {
6485 0 : rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
6486 : (machine_mode) k);
6487 0 : if (l)
6488 : {
6489 0 : gcc_assert (GET_CODE (l) == SYMBOL_REF);
6490 0 : fprintf (stderr, "%s\t%s\t%s:\t%s\n",
6491 0 : GET_RTX_NAME (optab_to_code ((optab) i)),
6492 0 : GET_MODE_NAME (j),
6493 0 : GET_MODE_NAME (k),
6494 : XSTR (l, 0));
6495 : }
6496 : }
6497 0 : }
6498 :
6499 : /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
6500 : CODE. Return 0 on failure. */
6501 :
6502 : rtx_insn *
6503 0 : gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
6504 : {
6505 0 : machine_mode mode = GET_MODE (op1);
6506 0 : enum insn_code icode;
6507 0 : rtx_insn *insn;
6508 0 : rtx trap_rtx;
6509 :
6510 0 : if (mode == VOIDmode)
6511 : return 0;
6512 :
6513 0 : icode = optab_handler (ctrap_optab, mode);
6514 0 : if (icode == CODE_FOR_nothing)
6515 : return 0;
6516 :
6517 : /* Some targets only accept a zero trap code. */
6518 0 : if (!insn_operand_matches (icode, 3, tcode))
6519 : return 0;
6520 :
6521 0 : do_pending_stack_adjust ();
6522 0 : start_sequence ();
6523 0 : prepare_cmp_insn (op1, op2, NULL, code, NULL_RTX, false, OPTAB_DIRECT,
6524 : &trap_rtx, &mode, cbranch_optab);
6525 0 : if (!trap_rtx)
6526 : insn = NULL;
6527 : else
6528 0 : insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
6529 : tcode);
6530 :
6531 : /* If that failed, then give up. */
6532 0 : if (insn == 0)
6533 : {
6534 0 : end_sequence ();
6535 0 : return 0;
6536 : }
6537 :
6538 0 : emit_insn (insn);
6539 0 : insn = end_sequence ();
6540 0 : return insn;
6541 : }
6542 :
6543 : /* Return rtx code for TCODE or UNKNOWN. Use UNSIGNEDP to select signed
6544 : or unsigned operation code. */
6545 :
6546 : enum rtx_code
6547 1524030 : get_rtx_code_1 (enum tree_code tcode, bool unsignedp)
6548 : {
6549 1524030 : enum rtx_code code;
6550 1524030 : switch (tcode)
6551 : {
6552 : case EQ_EXPR:
6553 : code = EQ;
6554 : break;
6555 767344 : case NE_EXPR:
6556 767344 : code = NE;
6557 767344 : break;
6558 157833 : case LT_EXPR:
6559 157833 : code = unsignedp ? LTU : LT;
6560 : break;
6561 72836 : case LE_EXPR:
6562 72836 : code = unsignedp ? LEU : LE;
6563 : break;
6564 118643 : case GT_EXPR:
6565 118643 : code = unsignedp ? GTU : GT;
6566 : break;
6567 78542 : case GE_EXPR:
6568 78542 : code = unsignedp ? GEU : GE;
6569 : break;
6570 :
6571 1235 : case UNORDERED_EXPR:
6572 1235 : code = UNORDERED;
6573 1235 : break;
6574 1034 : case ORDERED_EXPR:
6575 1034 : code = ORDERED;
6576 1034 : break;
6577 452 : case UNLT_EXPR:
6578 452 : code = UNLT;
6579 452 : break;
6580 4341 : case UNLE_EXPR:
6581 4341 : code = UNLE;
6582 4341 : break;
6583 457 : case UNGT_EXPR:
6584 457 : code = UNGT;
6585 457 : break;
6586 3438 : case UNGE_EXPR:
6587 3438 : code = UNGE;
6588 3438 : break;
6589 222 : case UNEQ_EXPR:
6590 222 : code = UNEQ;
6591 222 : break;
6592 497 : case LTGT_EXPR:
6593 497 : code = LTGT;
6594 497 : break;
6595 :
6596 32788 : case BIT_AND_EXPR:
6597 32788 : code = AND;
6598 32788 : break;
6599 :
6600 21 : case BIT_IOR_EXPR:
6601 21 : code = IOR;
6602 21 : break;
6603 :
6604 0 : default:
6605 0 : code = UNKNOWN;
6606 0 : break;
6607 : }
6608 1524030 : return code;
6609 : }
6610 :
6611 : /* Return rtx code for TCODE. Use UNSIGNEDP to select signed
6612 : or unsigned operation code. */
6613 :
6614 : enum rtx_code
6615 156726 : get_rtx_code (enum tree_code tcode, bool unsignedp)
6616 : {
6617 156726 : enum rtx_code code = get_rtx_code_1 (tcode, unsignedp);
6618 156726 : gcc_assert (code != UNKNOWN);
6619 156726 : return code;
6620 : }
6621 :
6622 : /* Return a comparison rtx of mode CMP_MODE for COND. Use UNSIGNEDP to
6623 : select signed or unsigned operators. OPNO holds the index of the
6624 : first comparison operand for insn ICODE. Do not generate the
6625 : compare instruction itself. */
6626 :
6627 : rtx
6628 24105 : vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
6629 : tree t_op0, tree t_op1, bool unsignedp,
6630 : enum insn_code icode, unsigned int opno)
6631 : {
6632 24105 : class expand_operand ops[2];
6633 24105 : rtx rtx_op0, rtx_op1;
6634 24105 : machine_mode m0, m1;
6635 24105 : enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
6636 :
6637 24105 : gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
6638 :
6639 : /* Expand operands. For vector types with scalar modes, e.g. where int64x1_t
6640 : has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
6641 : cases, use the original mode. */
6642 24105 : rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
6643 : EXPAND_STACK_PARM);
6644 24105 : m0 = GET_MODE (rtx_op0);
6645 24105 : if (m0 == VOIDmode)
6646 0 : m0 = TYPE_MODE (TREE_TYPE (t_op0));
6647 :
6648 24105 : rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
6649 : EXPAND_STACK_PARM);
6650 24105 : m1 = GET_MODE (rtx_op1);
6651 24105 : if (m1 == VOIDmode)
6652 0 : m1 = TYPE_MODE (TREE_TYPE (t_op1));
6653 :
6654 24105 : create_input_operand (&ops[0], rtx_op0, m0);
6655 24105 : create_input_operand (&ops[1], rtx_op1, m1);
6656 24105 : if (!maybe_legitimize_operands (icode, opno, 2, ops))
6657 0 : gcc_unreachable ();
6658 24105 : return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
6659 : }
6660 :
6661 : /* Check if vec_perm mask SEL is a constant equivalent to a shift of
6662 : the first vec_perm operand, assuming the second operand (for left shift
6663 : first operand) is a constant vector of zeros. Return the shift distance
6664 : in bits if so, or NULL_RTX if the vec_perm is not a shift. MODE is the
6665 : mode of the value being shifted. SHIFT_OPTAB is vec_shr_optab for right
6666 : shift or vec_shl_optab for left shift. */
6667 : static rtx
6668 2801 : shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
6669 : optab shift_optab)
6670 : {
6671 2801 : unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
6672 2801 : poly_int64 first = sel[0];
6673 5602 : if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
6674 : return NULL_RTX;
6675 :
6676 2800 : if (shift_optab == vec_shl_optab)
6677 : {
6678 807 : unsigned int nelt;
6679 1614 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6680 2801 : return NULL_RTX;
6681 807 : unsigned firstidx = 0;
6682 5547 : for (unsigned int i = 0; i < nelt; i++)
6683 : {
6684 4794 : if (known_eq (sel[i], nelt))
6685 : {
6686 777 : if (i == 0 || firstidx)
6687 : return NULL_RTX;
6688 : firstidx = i;
6689 : }
6690 8034 : else if (firstidx
6691 5464 : ? maybe_ne (sel[i], nelt + i - firstidx)
6692 4017 : : maybe_ge (sel[i], nelt))
6693 : return NULL_RTX;
6694 : }
6695 :
6696 753 : if (firstidx == 0)
6697 : return NULL_RTX;
6698 753 : first = firstidx;
6699 : }
6700 1993 : else if (!sel.series_p (0, 1, first, 1))
6701 : {
6702 162 : unsigned int nelt;
6703 324 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6704 187 : return NULL_RTX;
6705 306 : for (unsigned int i = 1; i < nelt; i++)
6706 : {
6707 276 : poly_int64 expected = i + first;
6708 : /* Indices into the second vector are all equivalent. */
6709 276 : if (maybe_lt (sel[i], nelt)
6710 427 : ? maybe_ne (sel[i], expected)
6711 151 : : maybe_lt (expected, nelt))
6712 187 : return NULL_RTX;
6713 : }
6714 : }
6715 :
6716 2614 : return gen_int_shift_amount (mode, first * bitsize);
6717 : }
6718 :
6719 : /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn. */
6720 :
6721 : static rtx
6722 18 : expand_vec_perm_1 (enum insn_code icode, rtx target,
6723 : rtx v0, rtx v1, rtx sel)
6724 : {
6725 18 : machine_mode tmode = GET_MODE (target);
6726 18 : machine_mode smode = GET_MODE (sel);
6727 18 : class expand_operand ops[4];
6728 :
6729 18 : gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
6730 : || related_int_vector_mode (tmode).require () == smode);
6731 18 : create_output_operand (&ops[0], target, tmode);
6732 18 : create_input_operand (&ops[3], sel, smode);
6733 :
6734 : /* Make an effort to preserve v0 == v1. The target expander is able to
6735 : rely on this to determine if we're permuting a single input operand. */
6736 18 : if (rtx_equal_p (v0, v1))
6737 : {
6738 14 : if (!insn_operand_matches (icode, 1, v0))
6739 0 : v0 = force_reg (tmode, v0);
6740 14 : gcc_checking_assert (insn_operand_matches (icode, 1, v0));
6741 14 : gcc_checking_assert (insn_operand_matches (icode, 2, v0));
6742 :
6743 14 : create_fixed_operand (&ops[1], v0);
6744 14 : create_fixed_operand (&ops[2], v0);
6745 : }
6746 : else
6747 : {
6748 4 : create_input_operand (&ops[1], v0, tmode);
6749 4 : create_input_operand (&ops[2], v1, tmode);
6750 : }
6751 :
6752 18 : if (maybe_expand_insn (icode, 4, ops))
6753 18 : return ops[0].value;
6754 : return NULL_RTX;
6755 : }
6756 :
6757 : /* Check if vec_perm mask SEL is a constant equivalent to an and operation of
6758 : the non-zero vec_perm operand with some mask consisting of 0xffs and 0x00s,
6759 : assuming the other vec_perm operand is a constant vector of zeros. Return
6760 : the mask for the equivalent and operation, or NULL_RTX if the vec_perm can
6761 : not be modeled as an and. MODE is the mode of the value being anded.
6762 : ZERO_OP0_P is true if the first operand of the vec_perm is a constant vector
6763 : of zeros or false if the second operand of the vec_perm is a constant vector
6764 : of zeros. */
6765 : rtx
6766 0 : vec_perm_and_mask (machine_mode mode, const vec_perm_indices &sel,
6767 : bool zero_op0_p)
6768 : {
6769 0 : unsigned int nelt;
6770 0 : if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
6771 : return NULL_RTX;
6772 :
6773 0 : rtx_vector_builder builder (mode, nelt, 1);
6774 0 : machine_mode emode = GET_MODE_INNER (mode);
6775 :
6776 0 : for (unsigned int i = 0; i < nelt; i++)
6777 : {
6778 0 : if (zero_op0_p)
6779 : {
6780 0 : if (known_eq (sel[i], nelt + i))
6781 0 : builder.quick_push (CONSTM1_RTX (emode));
6782 0 : else if (known_lt (sel[i], nelt))
6783 0 : builder.quick_push (CONST0_RTX (emode));
6784 : else
6785 0 : return NULL_RTX;
6786 : }
6787 : else
6788 : {
6789 0 : if (known_eq (sel[i], i))
6790 0 : builder.quick_push (CONSTM1_RTX (emode));
6791 0 : else if (known_ge (sel[i], nelt))
6792 0 : builder.quick_push (CONST0_RTX (emode));
6793 : else
6794 : return NULL_RTX;
6795 : }
6796 : }
6797 :
6798 0 : return builder.build ();
6799 0 : }
6800 :
6801 : /* Implement a permutation of vectors v0 and v1 using the permutation
6802 : vector in SEL and return the result. Use TARGET to hold the result
6803 : if nonnull and convenient.
6804 :
6805 : MODE is the mode of the vectors being permuted (V0 and V1). SEL_MODE
6806 : is the TYPE_MODE associated with SEL, or BLKmode if SEL isn't known
6807 : to have a particular mode. */
6808 :
6809 : rtx
6810 75159 : expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
6811 : const vec_perm_builder &sel, machine_mode sel_mode,
6812 : rtx target)
6813 : {
6814 75159 : if (!target || !register_operand (target, mode))
6815 60914 : target = gen_reg_rtx (mode);
6816 :
6817 : /* Set QIMODE to a different vector mode with byte elements.
6818 : If no such mode, or if MODE already has byte elements, use VOIDmode. */
6819 75159 : machine_mode qimode;
6820 75159 : if (!qimode_for_vec_perm (mode).exists (&qimode))
6821 7085 : qimode = VOIDmode;
6822 :
6823 75159 : rtx_insn *last = get_last_insn ();
6824 :
6825 75159 : bool single_arg_p = rtx_equal_p (v0, v1);
6826 : /* Always specify two input vectors here and leave the target to handle
6827 : cases in which the inputs are equal. Not all backends can cope with
6828 : the single-input representation when testing for a double-input
6829 : target instruction. */
6830 150318 : vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (mode));
6831 :
6832 : /* See if this can be handled with a vec_shr or vec_shl. We only do this
6833 : if the second (for vec_shr) or first (for vec_shl) vector is all
6834 : zeroes. */
6835 75159 : insn_code shift_code = CODE_FOR_nothing;
6836 75159 : insn_code shift_code_qi = CODE_FOR_nothing;
6837 75159 : optab shift_optab = unknown_optab;
6838 75159 : rtx v2 = v0;
6839 75159 : if (v1 == CONST0_RTX (GET_MODE (v1)))
6840 : shift_optab = vec_shr_optab;
6841 73101 : else if (v0 == CONST0_RTX (GET_MODE (v0)))
6842 : {
6843 : shift_optab = vec_shl_optab;
6844 : v2 = v1;
6845 : }
6846 : if (shift_optab != unknown_optab)
6847 : {
6848 3189 : shift_code = optab_handler (shift_optab, mode);
6849 5778 : shift_code_qi = ((qimode != VOIDmode && qimode != mode)
6850 6078 : ? optab_handler (shift_optab, qimode)
6851 : : CODE_FOR_nothing);
6852 : }
6853 3189 : if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
6854 : {
6855 2801 : rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices, shift_optab);
6856 2801 : if (shift_amt)
6857 : {
6858 2614 : class expand_operand ops[3];
6859 2614 : if (shift_amt == const0_rtx)
6860 2614 : return v2;
6861 2614 : if (shift_code != CODE_FOR_nothing)
6862 : {
6863 2614 : create_output_operand (&ops[0], target, mode);
6864 2614 : create_input_operand (&ops[1], v2, mode);
6865 2614 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6866 2614 : if (maybe_expand_insn (shift_code, 3, ops))
6867 2614 : return ops[0].value;
6868 : }
6869 0 : if (shift_code_qi != CODE_FOR_nothing)
6870 : {
6871 0 : rtx tmp = gen_reg_rtx (qimode);
6872 0 : create_output_operand (&ops[0], tmp, qimode);
6873 0 : create_input_operand (&ops[1], gen_lowpart (qimode, v2), qimode);
6874 0 : create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
6875 0 : if (maybe_expand_insn (shift_code_qi, 3, ops))
6876 0 : return gen_lowpart (mode, ops[0].value);
6877 : }
6878 : }
6879 : }
6880 :
6881 72545 : if (targetm.vectorize.vec_perm_const != NULL)
6882 : {
6883 72545 : if (single_arg_p)
6884 18554 : v1 = v0;
6885 :
6886 72545 : gcc_checking_assert (GET_MODE (v0) == GET_MODE (v1));
6887 72545 : machine_mode op_mode = GET_MODE (v0);
6888 72545 : if (targetm.vectorize.vec_perm_const (mode, op_mode, target, v0, v1,
6889 : indices))
6890 : return target;
6891 : }
6892 :
6893 : /* Fall back to a constant byte-based permutation. */
6894 0 : vec_perm_indices qimode_indices;
6895 0 : rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
6896 0 : if (qimode != VOIDmode)
6897 : {
6898 0 : qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
6899 0 : target_qi = gen_reg_rtx (qimode);
6900 0 : v0_qi = gen_lowpart (qimode, v0);
6901 0 : v1_qi = gen_lowpart (qimode, v1);
6902 0 : if (targetm.vectorize.vec_perm_const != NULL
6903 0 : && targetm.vectorize.vec_perm_const (qimode, qimode, target_qi, v0_qi,
6904 : v1_qi, qimode_indices))
6905 0 : return gen_lowpart (mode, target_qi);
6906 : }
6907 :
6908 0 : v0 = force_reg (mode, v0);
6909 0 : if (single_arg_p)
6910 0 : v1 = v0;
6911 0 : v1 = force_reg (mode, v1);
6912 :
6913 : /* Otherwise expand as a fully variable permutation. */
6914 :
6915 : /* The optabs are only defined for selectors with the same width
6916 : as the values being permuted. */
6917 0 : machine_mode required_sel_mode;
6918 0 : if (!related_int_vector_mode (mode).exists (&required_sel_mode))
6919 : {
6920 0 : delete_insns_since (last);
6921 0 : return NULL_RTX;
6922 : }
6923 :
6924 : /* We know that it is semantically valid to treat SEL as having SEL_MODE.
6925 : If that isn't the mode we want then we need to prove that using
6926 : REQUIRED_SEL_MODE is OK. */
6927 0 : if (sel_mode != required_sel_mode)
6928 : {
6929 0 : if (!selector_fits_mode_p (required_sel_mode, indices))
6930 : {
6931 0 : delete_insns_since (last);
6932 0 : return NULL_RTX;
6933 : }
6934 : sel_mode = required_sel_mode;
6935 : }
6936 :
6937 0 : insn_code icode = direct_optab_handler (vec_perm_optab, mode);
6938 0 : if (icode != CODE_FOR_nothing)
6939 : {
6940 0 : rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
6941 0 : rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
6942 0 : if (tmp)
6943 : return tmp;
6944 : }
6945 :
6946 0 : if (qimode != VOIDmode
6947 0 : && selector_fits_mode_p (qimode, qimode_indices))
6948 : {
6949 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
6950 0 : if (icode != CODE_FOR_nothing)
6951 : {
6952 0 : rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
6953 0 : rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
6954 0 : if (tmp)
6955 0 : return gen_lowpart (mode, tmp);
6956 : }
6957 : }
6958 :
6959 0 : delete_insns_since (last);
6960 0 : return NULL_RTX;
6961 0 : }
6962 :
6963 : /* Implement a permutation of vectors v0 and v1 using the permutation
6964 : vector in SEL and return the result. Use TARGET to hold the result
6965 : if nonnull and convenient.
6966 :
6967 : MODE is the mode of the vectors being permuted (V0 and V1).
6968 : SEL must have the integer equivalent of MODE and is known to be
6969 : unsuitable for permutes with a constant permutation vector. */
6970 :
6971 : rtx
6972 18 : expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
6973 : {
6974 18 : enum insn_code icode;
6975 18 : unsigned int i, u;
6976 18 : rtx tmp, sel_qi;
6977 :
6978 18 : u = GET_MODE_UNIT_SIZE (mode);
6979 :
6980 18 : if (!target || GET_MODE (target) != mode)
6981 0 : target = gen_reg_rtx (mode);
6982 :
6983 18 : icode = direct_optab_handler (vec_perm_optab, mode);
6984 18 : if (icode != CODE_FOR_nothing)
6985 : {
6986 18 : tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
6987 18 : if (tmp)
6988 : return tmp;
6989 : }
6990 :
6991 : /* As a special case to aid several targets, lower the element-based
6992 : permutation to a byte-based permutation and try again. */
6993 0 : machine_mode qimode;
6994 0 : if (!qimode_for_vec_perm (mode).exists (&qimode)
6995 0 : || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
6996 0 : return NULL_RTX;
6997 0 : icode = direct_optab_handler (vec_perm_optab, qimode);
6998 0 : if (icode == CODE_FOR_nothing)
6999 : return NULL_RTX;
7000 :
7001 : /* Multiply each element by its byte size. */
7002 0 : machine_mode selmode = GET_MODE (sel);
7003 0 : if (u == 2)
7004 0 : sel = expand_simple_binop (selmode, PLUS, sel, sel,
7005 : NULL, 0, OPTAB_DIRECT);
7006 : else
7007 0 : sel = expand_simple_binop (selmode, ASHIFT, sel,
7008 0 : gen_int_shift_amount (selmode, exact_log2 (u)),
7009 : NULL, 0, OPTAB_DIRECT);
7010 0 : gcc_assert (sel != NULL);
7011 :
7012 : /* Broadcast the low byte each element into each of its bytes.
7013 : The encoding has U interleaved stepped patterns, one for each
7014 : byte of an element. */
7015 0 : vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
7016 0 : unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
7017 0 : for (i = 0; i < 3; ++i)
7018 0 : for (unsigned int j = 0; j < u; ++j)
7019 0 : const_sel.quick_push (i * u + low_byte_in_u);
7020 0 : sel = gen_lowpart (qimode, sel);
7021 0 : sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
7022 0 : gcc_assert (sel != NULL);
7023 :
7024 : /* Add the byte offset to each byte element. */
7025 : /* Note that the definition of the indicies here is memory ordering,
7026 : so there should be no difference between big and little endian. */
7027 0 : rtx_vector_builder byte_indices (qimode, u, 1);
7028 0 : for (i = 0; i < u; ++i)
7029 0 : byte_indices.quick_push (GEN_INT (i));
7030 0 : tmp = byte_indices.build ();
7031 0 : sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
7032 : sel, 0, OPTAB_DIRECT);
7033 0 : gcc_assert (sel_qi != NULL);
7034 :
7035 0 : tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
7036 0 : tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
7037 0 : gen_lowpart (qimode, v1), sel_qi);
7038 0 : if (tmp)
7039 0 : tmp = gen_lowpart (mode, tmp);
7040 0 : return tmp;
7041 0 : }
7042 :
7043 : /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
7044 : Use TARGET for the result if nonnull and convenient. */
7045 :
7046 : rtx
7047 0 : expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
7048 : {
7049 0 : class expand_operand ops[3];
7050 0 : enum insn_code icode;
7051 0 : machine_mode emode = GET_MODE_INNER (vmode);
7052 :
7053 0 : icode = direct_optab_handler (vec_series_optab, vmode);
7054 0 : gcc_assert (icode != CODE_FOR_nothing);
7055 :
7056 0 : create_output_operand (&ops[0], target, vmode);
7057 0 : create_input_operand (&ops[1], op0, emode);
7058 0 : create_input_operand (&ops[2], op1, emode);
7059 :
7060 0 : expand_insn (icode, 3, ops);
7061 0 : return ops[0].value;
7062 : }
7063 :
7064 : /* Generate insns for a vector comparison into a mask. */
7065 :
7066 : rtx
7067 24105 : expand_vec_cmp_expr (tree type, tree exp, rtx target)
7068 : {
7069 24105 : class expand_operand ops[4];
7070 24105 : enum insn_code icode;
7071 24105 : rtx comparison;
7072 24105 : machine_mode mask_mode = TYPE_MODE (type);
7073 24105 : machine_mode vmode;
7074 24105 : bool unsignedp;
7075 24105 : tree op0a, op0b;
7076 24105 : enum tree_code tcode;
7077 :
7078 24105 : op0a = TREE_OPERAND (exp, 0);
7079 24105 : op0b = TREE_OPERAND (exp, 1);
7080 24105 : tcode = TREE_CODE (exp);
7081 :
7082 24105 : unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
7083 24105 : vmode = TYPE_MODE (TREE_TYPE (op0a));
7084 :
7085 24105 : icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
7086 24105 : if (icode == CODE_FOR_nothing)
7087 : {
7088 527 : if (tcode == EQ_EXPR || tcode == NE_EXPR)
7089 527 : icode = get_vec_cmp_eq_icode (vmode, mask_mode);
7090 527 : if (icode == CODE_FOR_nothing)
7091 : return 0;
7092 : }
7093 :
7094 24105 : comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
7095 : unsignedp, icode, 2);
7096 24105 : create_output_operand (&ops[0], target, mask_mode);
7097 24105 : create_fixed_operand (&ops[1], comparison);
7098 24105 : create_fixed_operand (&ops[2], XEXP (comparison, 0));
7099 24105 : create_fixed_operand (&ops[3], XEXP (comparison, 1));
7100 24105 : expand_insn (icode, 4, ops);
7101 24105 : return ops[0].value;
7102 : }
7103 :
7104 : /* Expand a highpart multiply. */
7105 :
7106 : rtx
7107 1736 : expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
7108 : rtx target, bool uns_p)
7109 : {
7110 1736 : class expand_operand eops[3];
7111 1736 : enum insn_code icode;
7112 1736 : int method, i;
7113 1736 : machine_mode wmode;
7114 1736 : rtx m1, m2;
7115 1736 : optab tab1, tab2;
7116 :
7117 1736 : method = can_mult_highpart_p (mode, uns_p);
7118 1736 : switch (method)
7119 : {
7120 : case 0:
7121 : return NULL_RTX;
7122 1024 : case 1:
7123 1024 : tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
7124 1024 : return expand_binop (mode, tab1, op0, op1, target, uns_p,
7125 1024 : OPTAB_LIB_WIDEN);
7126 0 : case 2:
7127 0 : return expmed_mult_highpart_optab (as_a <scalar_int_mode> (mode),
7128 0 : op0, op1, target, uns_p, INT_MAX);
7129 704 : case 3:
7130 704 : tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
7131 502 : tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
7132 : break;
7133 8 : case 4:
7134 8 : tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
7135 8 : tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
7136 : if (BYTES_BIG_ENDIAN)
7137 : std::swap (tab1, tab2);
7138 : break;
7139 0 : default:
7140 0 : gcc_unreachable ();
7141 : }
7142 :
7143 712 : icode = optab_handler (tab1, mode);
7144 712 : wmode = insn_data[icode].operand[0].mode;
7145 2136 : gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
7146 : GET_MODE_NUNITS (mode)));
7147 2136 : gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
7148 :
7149 712 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7150 712 : create_input_operand (&eops[1], op0, mode);
7151 712 : create_input_operand (&eops[2], op1, mode);
7152 712 : expand_insn (icode, 3, eops);
7153 712 : m1 = gen_lowpart (mode, eops[0].value);
7154 :
7155 712 : create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
7156 712 : create_input_operand (&eops[1], op0, mode);
7157 712 : create_input_operand (&eops[2], op1, mode);
7158 712 : expand_insn (optab_handler (tab2, mode), 3, eops);
7159 712 : m2 = gen_lowpart (mode, eops[0].value);
7160 :
7161 1424 : vec_perm_builder sel;
7162 712 : if (method == 3)
7163 : {
7164 : /* The encoding has 2 interleaved stepped patterns. */
7165 1408 : sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
7166 4928 : for (i = 0; i < 6; ++i)
7167 12672 : sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
7168 6336 : + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
7169 : }
7170 : else
7171 : {
7172 : /* The encoding has a single interleaved stepped pattern. */
7173 16 : sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
7174 32 : for (i = 0; i < 3; ++i)
7175 24 : sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
7176 : }
7177 :
7178 712 : return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
7179 : }
7180 :
7181 : /* Helper function to find the MODE_CC set in a sync_compare_and_swap
7182 : pattern. */
7183 :
7184 : static void
7185 0 : find_cc_set (rtx x, const_rtx pat, void *data)
7186 : {
7187 0 : if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
7188 0 : && GET_CODE (pat) == SET)
7189 : {
7190 0 : rtx *p_cc_reg = (rtx *) data;
7191 0 : gcc_assert (!*p_cc_reg);
7192 0 : *p_cc_reg = x;
7193 : }
7194 0 : }
7195 :
7196 : /* This is a helper function for the other atomic operations. This function
7197 : emits a loop that contains SEQ that iterates until a compare-and-swap
7198 : operation at the end succeeds. MEM is the memory to be modified. SEQ is
7199 : a set of instructions that takes a value from OLD_REG as an input and
7200 : produces a value in NEW_REG as an output. Before SEQ, OLD_REG will be
7201 : set to the current contents of MEM. After SEQ, a compare-and-swap will
7202 : attempt to update MEM with NEW_REG. The function returns true when the
7203 : loop was generated successfully. */
7204 :
7205 : static bool
7206 5070 : expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx seq)
7207 : {
7208 5070 : machine_mode mode = GET_MODE (mem);
7209 5070 : rtx_code_label *label;
7210 5070 : rtx cmp_reg, success, oldval;
7211 :
7212 : /* The loop we want to generate looks like
7213 :
7214 : cmp_reg = mem;
7215 : label:
7216 : old_reg = cmp_reg;
7217 : seq;
7218 : (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
7219 : if (success)
7220 : goto label;
7221 :
7222 : Note that we only do the plain load from memory once. Subsequent
7223 : iterations use the value loaded by the compare-and-swap pattern. */
7224 :
7225 5070 : label = gen_label_rtx ();
7226 5070 : cmp_reg = gen_reg_rtx (mode);
7227 :
7228 5070 : emit_move_insn (cmp_reg, mem);
7229 5070 : emit_label (label);
7230 5070 : emit_move_insn (old_reg, cmp_reg);
7231 5070 : if (seq)
7232 5057 : emit_insn (seq);
7233 :
7234 5070 : success = NULL_RTX;
7235 5070 : oldval = cmp_reg;
7236 5070 : if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
7237 : new_reg, false, MEMMODEL_SYNC_SEQ_CST,
7238 : MEMMODEL_RELAXED))
7239 : return false;
7240 :
7241 5070 : if (oldval != cmp_reg)
7242 0 : emit_move_insn (cmp_reg, oldval);
7243 :
7244 : /* Mark this jump predicted not taken. */
7245 5070 : emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
7246 5070 : GET_MODE (success), 1, label,
7247 : profile_probability::guessed_never ());
7248 5070 : return true;
7249 : }
7250 :
7251 :
7252 : /* This function tries to emit an atomic_exchange intruction. VAL is written
7253 : to *MEM using memory model MODEL. The previous contents of *MEM are returned,
7254 : using TARGET if possible. */
7255 :
7256 : static rtx
7257 3885 : maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7258 : {
7259 3885 : machine_mode mode = GET_MODE (mem);
7260 3885 : enum insn_code icode;
7261 :
7262 : /* If the target supports the exchange directly, great. */
7263 3885 : icode = direct_optab_handler (atomic_exchange_optab, mode);
7264 3885 : if (icode != CODE_FOR_nothing)
7265 : {
7266 3868 : class expand_operand ops[4];
7267 :
7268 3868 : create_output_operand (&ops[0], target, mode);
7269 3868 : create_fixed_operand (&ops[1], mem);
7270 3868 : create_input_operand (&ops[2], val, mode);
7271 3868 : create_integer_operand (&ops[3], model);
7272 3868 : if (maybe_expand_insn (icode, 4, ops))
7273 3868 : return ops[0].value;
7274 : }
7275 :
7276 : return NULL_RTX;
7277 : }
7278 :
7279 : /* This function tries to implement an atomic exchange operation using
7280 : __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
7281 : The previous contents of *MEM are returned, using TARGET if possible.
7282 : Since this instructionn is an acquire barrier only, stronger memory
7283 : models may require additional barriers to be emitted. */
7284 :
7285 : static rtx
7286 2 : maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
7287 : enum memmodel model)
7288 : {
7289 2 : machine_mode mode = GET_MODE (mem);
7290 2 : enum insn_code icode;
7291 2 : rtx_insn *last_insn = get_last_insn ();
7292 :
7293 2 : icode = optab_handler (sync_lock_test_and_set_optab, mode);
7294 :
7295 : /* Legacy sync_lock_test_and_set is an acquire barrier. If the pattern
7296 : exists, and the memory model is stronger than acquire, add a release
7297 : barrier before the instruction. */
7298 :
7299 2 : if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
7300 0 : expand_mem_thread_fence (model);
7301 :
7302 2 : if (icode != CODE_FOR_nothing)
7303 : {
7304 0 : class expand_operand ops[3];
7305 0 : create_output_operand (&ops[0], target, mode);
7306 0 : create_fixed_operand (&ops[1], mem);
7307 0 : create_input_operand (&ops[2], val, mode);
7308 0 : if (maybe_expand_insn (icode, 3, ops))
7309 0 : return ops[0].value;
7310 : }
7311 :
7312 : /* If an external test-and-set libcall is provided, use that instead of
7313 : any external compare-and-swap that we might get from the compare-and-
7314 : swap-loop expansion later. */
7315 2 : if (!can_compare_and_swap_p (mode, false))
7316 : {
7317 2 : rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
7318 2 : if (libfunc != NULL)
7319 : {
7320 0 : rtx addr;
7321 :
7322 0 : addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7323 0 : return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7324 : mode, addr, ptr_mode,
7325 0 : val, mode);
7326 : }
7327 : }
7328 :
7329 : /* If the test_and_set can't be emitted, eliminate any barrier that might
7330 : have been emitted. */
7331 2 : delete_insns_since (last_insn);
7332 2 : return NULL_RTX;
7333 : }
7334 :
7335 : /* This function tries to implement an atomic exchange operation using a
7336 : compare_and_swap loop. VAL is written to *MEM. The previous contents of
7337 : *MEM are returned, using TARGET if possible. No memory model is required
7338 : since a compare_and_swap loop is seq-cst. */
7339 :
7340 : static rtx
7341 17 : maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
7342 : {
7343 17 : machine_mode mode = GET_MODE (mem);
7344 :
7345 17 : if (can_compare_and_swap_p (mode, true))
7346 : {
7347 13 : if (!target || !register_operand (target, mode))
7348 1 : target = gen_reg_rtx (mode);
7349 13 : if (expand_compare_and_swap_loop (mem, target, val, NULL_RTX))
7350 : return target;
7351 : }
7352 :
7353 : return NULL_RTX;
7354 : }
7355 :
7356 : /* This function tries to implement an atomic test-and-set operation
7357 : using the atomic_test_and_set instruction pattern. A boolean value
7358 : is returned from the operation, using TARGET if possible. */
7359 :
7360 : static rtx
7361 268 : maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7362 : {
7363 268 : machine_mode pat_bool_mode;
7364 268 : class expand_operand ops[3];
7365 :
7366 268 : if (!targetm.have_atomic_test_and_set ())
7367 : return NULL_RTX;
7368 :
7369 : /* While we always get QImode from __atomic_test_and_set, we get
7370 : other memory modes from __sync_lock_test_and_set. Note that we
7371 : use no endian adjustment here. This matches the 4.6 behavior
7372 : in the Sparc backend. */
7373 0 : enum insn_code icode = targetm.code_for_atomic_test_and_set;
7374 0 : gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
7375 0 : if (GET_MODE (mem) != QImode)
7376 0 : mem = adjust_address_nv (mem, QImode, 0);
7377 :
7378 0 : pat_bool_mode = insn_data[icode].operand[0].mode;
7379 0 : create_output_operand (&ops[0], target, pat_bool_mode);
7380 0 : create_fixed_operand (&ops[1], mem);
7381 0 : create_integer_operand (&ops[2], model);
7382 :
7383 0 : if (maybe_expand_insn (icode, 3, ops))
7384 0 : return ops[0].value;
7385 : return NULL_RTX;
7386 : }
7387 :
7388 : /* This function expands the legacy _sync_lock test_and_set operation which is
7389 : generally an atomic exchange. Some limited targets only allow the
7390 : constant 1 to be stored. This is an ACQUIRE operation.
7391 :
7392 : TARGET is an optional place to stick the return value.
7393 : MEM is where VAL is stored. */
7394 :
7395 : rtx
7396 326 : expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
7397 : {
7398 326 : rtx ret;
7399 :
7400 : /* Try an atomic_exchange first. */
7401 326 : ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
7402 326 : if (ret)
7403 : return ret;
7404 :
7405 2 : ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
7406 : MEMMODEL_SYNC_ACQUIRE);
7407 2 : if (ret)
7408 : return ret;
7409 :
7410 2 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7411 2 : if (ret)
7412 : return ret;
7413 :
7414 : /* If there are no other options, try atomic_test_and_set if the value
7415 : being stored is 1. */
7416 2 : if (val == const1_rtx)
7417 2 : ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
7418 :
7419 : return ret;
7420 : }
7421 :
7422 : /* This function expands the atomic test_and_set operation:
7423 : atomically store a boolean TRUE into MEM and return the previous value.
7424 :
7425 : MEMMODEL is the memory model variant to use.
7426 : TARGET is an optional place to stick the return value. */
7427 :
7428 : rtx
7429 266 : expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
7430 : {
7431 266 : machine_mode mode = GET_MODE (mem);
7432 266 : rtx ret, trueval, subtarget;
7433 :
7434 266 : ret = maybe_emit_atomic_test_and_set (target, mem, model);
7435 266 : if (ret)
7436 : return ret;
7437 :
7438 : /* Be binary compatible with non-default settings of trueval, and different
7439 : cpu revisions. E.g. one revision may have atomic-test-and-set, but
7440 : another only has atomic-exchange. */
7441 266 : if (targetm.atomic_test_and_set_trueval == 1)
7442 : {
7443 266 : trueval = const1_rtx;
7444 266 : subtarget = target ? target : gen_reg_rtx (mode);
7445 : }
7446 : else
7447 : {
7448 0 : trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
7449 0 : subtarget = gen_reg_rtx (mode);
7450 : }
7451 :
7452 : /* Try the atomic-exchange optab... */
7453 266 : ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
7454 :
7455 : /* ... then an atomic-compare-and-swap loop ... */
7456 266 : if (!ret)
7457 0 : ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
7458 :
7459 : /* ... before trying the vaguely defined legacy lock_test_and_set. */
7460 0 : if (!ret)
7461 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
7462 :
7463 : /* Recall that the legacy lock_test_and_set optab was allowed to do magic
7464 : things with the value 1. Thus we try again without trueval. */
7465 266 : if (!ret && targetm.atomic_test_and_set_trueval != 1)
7466 : {
7467 0 : ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
7468 :
7469 0 : if (ret)
7470 : {
7471 : /* Rectify the not-one trueval. */
7472 0 : ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
7473 0 : gcc_assert (ret);
7474 : }
7475 : }
7476 :
7477 : return ret;
7478 : }
7479 :
7480 : /* This function expands the atomic exchange operation:
7481 : atomically store VAL in MEM and return the previous value in MEM.
7482 :
7483 : MEMMODEL is the memory model variant to use.
7484 : TARGET is an optional place to stick the return value. */
7485 :
7486 : rtx
7487 2830 : expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
7488 : {
7489 2830 : machine_mode mode = GET_MODE (mem);
7490 2830 : rtx ret;
7491 :
7492 : /* If loads are not atomic for the required size and we are not called to
7493 : provide a __sync builtin, do not do anything so that we stay consistent
7494 : with atomic loads of the same size. */
7495 2830 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
7496 : return NULL_RTX;
7497 :
7498 2799 : ret = maybe_emit_atomic_exchange (target, mem, val, model);
7499 :
7500 : /* Next try a compare-and-swap loop for the exchange. */
7501 2799 : if (!ret)
7502 13 : ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
7503 :
7504 : return ret;
7505 : }
7506 :
7507 : /* This function expands the atomic compare exchange operation:
7508 :
7509 : *PTARGET_BOOL is an optional place to store the boolean success/failure.
7510 : *PTARGET_OVAL is an optional place to store the old value from memory.
7511 : Both target parameters may be NULL or const0_rtx to indicate that we do
7512 : not care about that return value. Both target parameters are updated on
7513 : success to the actual location of the corresponding result.
7514 :
7515 : MEMMODEL is the memory model variant to use.
7516 :
7517 : The return value of the function is true for success. */
7518 :
7519 : bool
7520 28477 : expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
7521 : rtx mem, rtx expected, rtx desired,
7522 : bool is_weak, enum memmodel succ_model,
7523 : enum memmodel fail_model)
7524 : {
7525 28477 : machine_mode mode = GET_MODE (mem);
7526 28477 : class expand_operand ops[8];
7527 28477 : enum insn_code icode;
7528 28477 : rtx target_oval, target_bool = NULL_RTX;
7529 28477 : rtx libfunc;
7530 :
7531 : /* If loads are not atomic for the required size and we are not called to
7532 : provide a __sync builtin, do not do anything so that we stay consistent
7533 : with atomic loads of the same size. */
7534 28477 : if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
7535 : return false;
7536 :
7537 : /* Load expected into a register for the compare and swap. */
7538 26726 : if (MEM_P (expected))
7539 7362 : expected = copy_to_reg (expected);
7540 :
7541 : /* Make sure we always have some place to put the return oldval.
7542 : Further, make sure that place is distinct from the input expected,
7543 : just in case we need that path down below. */
7544 26726 : if (ptarget_oval && *ptarget_oval == const0_rtx)
7545 : ptarget_oval = NULL;
7546 :
7547 26500 : if (ptarget_oval == NULL
7548 26500 : || (target_oval = *ptarget_oval) == NULL
7549 5300 : || reg_overlap_mentioned_p (expected, target_oval))
7550 21504 : target_oval = gen_reg_rtx (mode);
7551 :
7552 26726 : icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
7553 26726 : if (icode != CODE_FOR_nothing)
7554 : {
7555 26724 : machine_mode bool_mode = insn_data[icode].operand[0].mode;
7556 :
7557 26724 : if (ptarget_bool && *ptarget_bool == const0_rtx)
7558 : ptarget_bool = NULL;
7559 :
7560 : /* Make sure we always have a place for the bool operand. */
7561 26469 : if (ptarget_bool == NULL
7562 26469 : || (target_bool = *ptarget_bool) == NULL
7563 7244 : || GET_MODE (target_bool) != bool_mode)
7564 19480 : target_bool = gen_reg_rtx (bool_mode);
7565 :
7566 : /* Emit the compare_and_swap. */
7567 26724 : create_output_operand (&ops[0], target_bool, bool_mode);
7568 26724 : create_output_operand (&ops[1], target_oval, mode);
7569 26724 : create_fixed_operand (&ops[2], mem);
7570 26724 : create_input_operand (&ops[3], expected, mode);
7571 26724 : create_input_operand (&ops[4], desired, mode);
7572 26724 : create_integer_operand (&ops[5], is_weak);
7573 26724 : create_integer_operand (&ops[6], succ_model);
7574 26724 : create_integer_operand (&ops[7], fail_model);
7575 26724 : if (maybe_expand_insn (icode, 8, ops))
7576 : {
7577 : /* Return success/failure. */
7578 26724 : target_bool = ops[0].value;
7579 26724 : target_oval = ops[1].value;
7580 26724 : goto success;
7581 : }
7582 : }
7583 :
7584 : /* Otherwise fall back to the original __sync_val_compare_and_swap
7585 : which is always seq-cst. */
7586 2 : icode = optab_handler (sync_compare_and_swap_optab, mode);
7587 2 : if (icode != CODE_FOR_nothing)
7588 : {
7589 0 : rtx cc_reg;
7590 :
7591 0 : create_output_operand (&ops[0], target_oval, mode);
7592 0 : create_fixed_operand (&ops[1], mem);
7593 0 : create_input_operand (&ops[2], expected, mode);
7594 0 : create_input_operand (&ops[3], desired, mode);
7595 0 : if (!maybe_expand_insn (icode, 4, ops))
7596 0 : return false;
7597 :
7598 0 : target_oval = ops[0].value;
7599 :
7600 : /* If the caller isn't interested in the boolean return value,
7601 : skip the computation of it. */
7602 0 : if (ptarget_bool == NULL)
7603 0 : goto success;
7604 :
7605 : /* Otherwise, work out if the compare-and-swap succeeded. */
7606 0 : cc_reg = NULL_RTX;
7607 0 : if (have_insn_for (COMPARE, CCmode))
7608 0 : note_stores (get_last_insn (), find_cc_set, &cc_reg);
7609 0 : if (cc_reg)
7610 : {
7611 0 : target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
7612 : const0_rtx, VOIDmode, 0, 1);
7613 0 : goto success;
7614 : }
7615 0 : goto success_bool_from_val;
7616 : }
7617 :
7618 : /* Also check for library support for __sync_val_compare_and_swap. */
7619 2 : libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
7620 2 : if (libfunc != NULL)
7621 : {
7622 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
7623 0 : rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
7624 : mode, addr, ptr_mode,
7625 : expected, mode, desired, mode);
7626 0 : emit_move_insn (target_oval, target);
7627 :
7628 : /* Compute the boolean return value only if requested. */
7629 0 : if (ptarget_bool)
7630 0 : goto success_bool_from_val;
7631 : else
7632 0 : goto success;
7633 : }
7634 :
7635 : /* Failure. */
7636 : return false;
7637 :
7638 0 : success_bool_from_val:
7639 0 : target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
7640 : expected, VOIDmode, 1, 1);
7641 26724 : success:
7642 : /* Make sure that the oval output winds up where the caller asked. */
7643 26724 : if (ptarget_oval)
7644 26500 : *ptarget_oval = target_oval;
7645 26724 : if (ptarget_bool)
7646 26469 : *ptarget_bool = target_bool;
7647 : return true;
7648 : }
7649 :
7650 : /* Generate asm volatile("" : : : "memory") as the memory blockage. */
7651 :
7652 : static void
7653 0 : expand_asm_memory_blockage (void)
7654 : {
7655 0 : rtx asm_op, clob;
7656 :
7657 0 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7658 : rtvec_alloc (0), rtvec_alloc (0),
7659 : rtvec_alloc (0), UNKNOWN_LOCATION);
7660 0 : MEM_VOLATILE_P (asm_op) = 1;
7661 :
7662 0 : clob = gen_rtx_SCRATCH (VOIDmode);
7663 0 : clob = gen_rtx_MEM (BLKmode, clob);
7664 0 : clob = gen_rtx_CLOBBER (VOIDmode, clob);
7665 :
7666 0 : emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
7667 0 : }
7668 :
7669 : /* Do not propagate memory accesses across this point. */
7670 :
7671 : static void
7672 115601 : expand_memory_blockage (void)
7673 : {
7674 115601 : if (targetm.have_memory_blockage ())
7675 115601 : emit_insn (targetm.gen_memory_blockage ());
7676 : else
7677 0 : expand_asm_memory_blockage ();
7678 115601 : }
7679 :
7680 : /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
7681 : same time clobbering the register set specified by REGS. */
7682 :
7683 : void
7684 131 : expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs)
7685 : {
7686 131 : rtx asm_op, clob_mem;
7687 :
7688 131 : unsigned int num_of_regs = 0;
7689 12183 : for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7690 12052 : if (TEST_HARD_REG_BIT (regs, i))
7691 1119 : num_of_regs++;
7692 :
7693 131 : asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
7694 : rtvec_alloc (0), rtvec_alloc (0),
7695 : rtvec_alloc (0), UNKNOWN_LOCATION);
7696 131 : MEM_VOLATILE_P (asm_op) = 1;
7697 :
7698 131 : rtvec v = rtvec_alloc (num_of_regs + 2);
7699 :
7700 131 : clob_mem = gen_rtx_SCRATCH (VOIDmode);
7701 131 : clob_mem = gen_rtx_MEM (BLKmode, clob_mem);
7702 131 : clob_mem = gen_rtx_CLOBBER (VOIDmode, clob_mem);
7703 :
7704 131 : RTVEC_ELT (v, 0) = asm_op;
7705 131 : RTVEC_ELT (v, 1) = clob_mem;
7706 :
7707 131 : if (num_of_regs > 0)
7708 : {
7709 : unsigned int j = 2;
7710 12183 : for (unsigned int i = 0; i < FIRST_PSEUDO_REGISTER; i++)
7711 12052 : if (TEST_HARD_REG_BIT (regs, i))
7712 : {
7713 1119 : RTVEC_ELT (v, j) = gen_rtx_CLOBBER (VOIDmode, regno_reg_rtx[i]);
7714 1119 : j++;
7715 : }
7716 131 : gcc_assert (j == (num_of_regs + 2));
7717 : }
7718 :
7719 131 : emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
7720 131 : }
7721 :
7722 : /* This routine will either emit the mem_thread_fence pattern or issue a
7723 : sync_synchronize to generate a fence for memory model MEMMODEL. */
7724 :
7725 : void
7726 966 : expand_mem_thread_fence (enum memmodel model)
7727 : {
7728 966 : if (is_mm_relaxed (model))
7729 : return;
7730 957 : if (targetm.have_mem_thread_fence ())
7731 : {
7732 957 : emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
7733 957 : expand_memory_blockage ();
7734 : }
7735 0 : else if (targetm.have_memory_barrier ())
7736 0 : emit_insn (targetm.gen_memory_barrier ());
7737 0 : else if (synchronize_libfunc != NULL_RTX)
7738 0 : emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
7739 : else
7740 0 : expand_memory_blockage ();
7741 : }
7742 :
7743 : /* Emit a signal fence with given memory model. */
7744 :
7745 : void
7746 60 : expand_mem_signal_fence (enum memmodel model)
7747 : {
7748 : /* No machine barrier is required to implement a signal fence, but
7749 : a compiler memory barrier must be issued, except for relaxed MM. */
7750 60 : if (!is_mm_relaxed (model))
7751 51 : expand_memory_blockage ();
7752 60 : }
7753 :
7754 : /* This function expands the atomic load operation:
7755 : return the atomically loaded value in MEM.
7756 :
7757 : MEMMODEL is the memory model variant to use.
7758 : TARGET is an option place to stick the return value. */
7759 :
7760 : rtx
7761 69965 : expand_atomic_load (rtx target, rtx mem, enum memmodel model)
7762 : {
7763 69965 : machine_mode mode = GET_MODE (mem);
7764 69965 : enum insn_code icode;
7765 :
7766 : /* If the target supports the load directly, great. */
7767 69965 : icode = direct_optab_handler (atomic_load_optab, mode);
7768 69965 : if (icode != CODE_FOR_nothing)
7769 : {
7770 66137 : class expand_operand ops[3];
7771 66137 : rtx_insn *last = get_last_insn ();
7772 66137 : if (is_mm_seq_cst (model))
7773 38213 : expand_memory_blockage ();
7774 :
7775 66137 : create_output_operand (&ops[0], target, mode);
7776 66137 : create_fixed_operand (&ops[1], mem);
7777 66137 : create_integer_operand (&ops[2], model);
7778 66137 : if (maybe_expand_insn (icode, 3, ops))
7779 : {
7780 66137 : if (!is_mm_relaxed (model))
7781 51604 : expand_memory_blockage ();
7782 66137 : return ops[0].value;
7783 : }
7784 0 : delete_insns_since (last);
7785 : }
7786 :
7787 : /* If the size of the object is greater than word size on this target,
7788 : then we assume that a load will not be atomic. We could try to
7789 : emulate a load with a compare-and-swap operation, but the store that
7790 : doing this could result in would be incorrect if this is a volatile
7791 : atomic load or targetting read-only-mapped memory. */
7792 3828 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7793 : /* If there is no atomic load, leave the library call. */
7794 : return NULL_RTX;
7795 :
7796 : /* Otherwise assume loads are atomic, and emit the proper barriers. */
7797 0 : if (!target || target == const0_rtx)
7798 0 : target = gen_reg_rtx (mode);
7799 :
7800 : /* For SEQ_CST, emit a barrier before the load. */
7801 0 : if (is_mm_seq_cst (model))
7802 0 : expand_mem_thread_fence (model);
7803 :
7804 0 : emit_move_insn (target, mem);
7805 :
7806 : /* Emit the appropriate barrier after the load. */
7807 0 : expand_mem_thread_fence (model);
7808 :
7809 0 : return target;
7810 : }
7811 :
7812 : /* This function expands the atomic store operation:
7813 : Atomically store VAL in MEM.
7814 : MEMMODEL is the memory model variant to use.
7815 : USE_RELEASE is true if __sync_lock_release can be used as a fall back.
7816 : function returns const0_rtx if a pattern was emitted. */
7817 :
7818 : rtx
7819 17590 : expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
7820 : {
7821 17590 : machine_mode mode = GET_MODE (mem);
7822 17590 : enum insn_code icode;
7823 17590 : class expand_operand ops[3];
7824 :
7825 : /* If the target supports the store directly, great. */
7826 17590 : icode = direct_optab_handler (atomic_store_optab, mode);
7827 17590 : if (icode != CODE_FOR_nothing)
7828 : {
7829 15992 : rtx_insn *last = get_last_insn ();
7830 15992 : if (!is_mm_relaxed (model))
7831 13242 : expand_memory_blockage ();
7832 15992 : create_fixed_operand (&ops[0], mem);
7833 15992 : create_input_operand (&ops[1], val, mode);
7834 15992 : create_integer_operand (&ops[2], model);
7835 15992 : if (maybe_expand_insn (icode, 3, ops))
7836 : {
7837 15992 : if (is_mm_seq_cst (model))
7838 11534 : expand_memory_blockage ();
7839 15992 : return const0_rtx;
7840 : }
7841 0 : delete_insns_since (last);
7842 : }
7843 :
7844 : /* If using __sync_lock_release is a viable alternative, try it.
7845 : Note that this will not be set to true if we are expanding a generic
7846 : __atomic_store_n. */
7847 1598 : if (use_release)
7848 : {
7849 2 : icode = direct_optab_handler (sync_lock_release_optab, mode);
7850 2 : if (icode != CODE_FOR_nothing)
7851 : {
7852 0 : create_fixed_operand (&ops[0], mem);
7853 0 : create_input_operand (&ops[1], const0_rtx, mode);
7854 0 : if (maybe_expand_insn (icode, 2, ops))
7855 : {
7856 : /* lock_release is only a release barrier. */
7857 0 : if (is_mm_seq_cst (model))
7858 0 : expand_mem_thread_fence (model);
7859 0 : return const0_rtx;
7860 : }
7861 : }
7862 : }
7863 :
7864 : /* If the size of the object is greater than word size on this target,
7865 : a default store will not be atomic. */
7866 1598 : if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
7867 : {
7868 : /* If loads are atomic or we are called to provide a __sync builtin,
7869 : we can try a atomic_exchange and throw away the result. Otherwise,
7870 : don't do anything so that we do not create an inconsistency between
7871 : loads and stores. */
7872 1598 : if (can_atomic_load_p (mode) || is_mm_sync (model))
7873 : {
7874 2 : rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
7875 2 : if (!target)
7876 2 : target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
7877 : val);
7878 2 : if (target)
7879 0 : return const0_rtx;
7880 : }
7881 1598 : return NULL_RTX;
7882 : }
7883 :
7884 : /* Otherwise assume stores are atomic, and emit the proper barriers. */
7885 0 : expand_mem_thread_fence (model);
7886 :
7887 0 : emit_move_insn (mem, val);
7888 :
7889 : /* For SEQ_CST, also emit a barrier after the store. */
7890 0 : if (is_mm_seq_cst (model))
7891 0 : expand_mem_thread_fence (model);
7892 :
7893 0 : return const0_rtx;
7894 : }
7895 :
7896 :
7897 : /* Structure containing the pointers and values required to process the
7898 : various forms of the atomic_fetch_op and atomic_op_fetch builtins. */
7899 :
7900 : struct atomic_op_functions
7901 : {
7902 : direct_optab mem_fetch_before;
7903 : direct_optab mem_fetch_after;
7904 : direct_optab mem_no_result;
7905 : optab fetch_before;
7906 : optab fetch_after;
7907 : direct_optab no_result;
7908 : enum rtx_code reverse_code;
7909 : };
7910 :
7911 :
7912 : /* Fill in structure pointed to by OP with the various optab entries for an
7913 : operation of type CODE. */
7914 :
7915 : static void
7916 34746 : get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
7917 : {
7918 34746 : gcc_assert (op!= NULL);
7919 :
7920 : /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
7921 : in the source code during compilation, and the optab entries are not
7922 : computable until runtime. Fill in the values at runtime. */
7923 34746 : switch (code)
7924 : {
7925 22476 : case PLUS:
7926 22476 : op->mem_fetch_before = atomic_fetch_add_optab;
7927 22476 : op->mem_fetch_after = atomic_add_fetch_optab;
7928 22476 : op->mem_no_result = atomic_add_optab;
7929 22476 : op->fetch_before = sync_old_add_optab;
7930 22476 : op->fetch_after = sync_new_add_optab;
7931 22476 : op->no_result = sync_add_optab;
7932 22476 : op->reverse_code = MINUS;
7933 22476 : break;
7934 4809 : case MINUS:
7935 4809 : op->mem_fetch_before = atomic_fetch_sub_optab;
7936 4809 : op->mem_fetch_after = atomic_sub_fetch_optab;
7937 4809 : op->mem_no_result = atomic_sub_optab;
7938 4809 : op->fetch_before = sync_old_sub_optab;
7939 4809 : op->fetch_after = sync_new_sub_optab;
7940 4809 : op->no_result = sync_sub_optab;
7941 4809 : op->reverse_code = PLUS;
7942 4809 : break;
7943 2194 : case XOR:
7944 2194 : op->mem_fetch_before = atomic_fetch_xor_optab;
7945 2194 : op->mem_fetch_after = atomic_xor_fetch_optab;
7946 2194 : op->mem_no_result = atomic_xor_optab;
7947 2194 : op->fetch_before = sync_old_xor_optab;
7948 2194 : op->fetch_after = sync_new_xor_optab;
7949 2194 : op->no_result = sync_xor_optab;
7950 2194 : op->reverse_code = XOR;
7951 2194 : break;
7952 2068 : case AND:
7953 2068 : op->mem_fetch_before = atomic_fetch_and_optab;
7954 2068 : op->mem_fetch_after = atomic_and_fetch_optab;
7955 2068 : op->mem_no_result = atomic_and_optab;
7956 2068 : op->fetch_before = sync_old_and_optab;
7957 2068 : op->fetch_after = sync_new_and_optab;
7958 2068 : op->no_result = sync_and_optab;
7959 2068 : op->reverse_code = UNKNOWN;
7960 2068 : break;
7961 2496 : case IOR:
7962 2496 : op->mem_fetch_before = atomic_fetch_or_optab;
7963 2496 : op->mem_fetch_after = atomic_or_fetch_optab;
7964 2496 : op->mem_no_result = atomic_or_optab;
7965 2496 : op->fetch_before = sync_old_ior_optab;
7966 2496 : op->fetch_after = sync_new_ior_optab;
7967 2496 : op->no_result = sync_ior_optab;
7968 2496 : op->reverse_code = UNKNOWN;
7969 2496 : break;
7970 703 : case NOT:
7971 703 : op->mem_fetch_before = atomic_fetch_nand_optab;
7972 703 : op->mem_fetch_after = atomic_nand_fetch_optab;
7973 703 : op->mem_no_result = atomic_nand_optab;
7974 703 : op->fetch_before = sync_old_nand_optab;
7975 703 : op->fetch_after = sync_new_nand_optab;
7976 703 : op->no_result = sync_nand_optab;
7977 703 : op->reverse_code = UNKNOWN;
7978 703 : break;
7979 0 : default:
7980 0 : gcc_unreachable ();
7981 : }
7982 34746 : }
7983 :
7984 : /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
7985 : using memory order MODEL. If AFTER is true the operation needs to return
7986 : the value of *MEM after the operation, otherwise the previous value.
7987 : TARGET is an optional place to place the result. The result is unused if
7988 : it is const0_rtx.
7989 : Return the result if there is a better sequence, otherwise NULL_RTX. */
7990 :
7991 : static rtx
7992 34744 : maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
7993 : enum memmodel model, bool after)
7994 : {
7995 : /* If the value is prefetched, or not used, it may be possible to replace
7996 : the sequence with a native exchange operation. */
7997 34744 : if (!after || target == const0_rtx)
7998 : {
7999 : /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m). */
8000 26643 : if (code == AND && val == const0_rtx)
8001 : {
8002 327 : if (target == const0_rtx)
8003 150 : target = gen_reg_rtx (GET_MODE (mem));
8004 327 : return maybe_emit_atomic_exchange (target, mem, val, model);
8005 : }
8006 :
8007 : /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m). */
8008 26316 : if (code == IOR && val == constm1_rtx)
8009 : {
8010 165 : if (target == const0_rtx)
8011 164 : target = gen_reg_rtx (GET_MODE (mem));
8012 165 : return maybe_emit_atomic_exchange (target, mem, val, model);
8013 : }
8014 : }
8015 :
8016 : return NULL_RTX;
8017 : }
8018 :
8019 : /* Try to emit an instruction for a specific operation varaition.
8020 : OPTAB contains the OP functions.
8021 : TARGET is an optional place to return the result. const0_rtx means unused.
8022 : MEM is the memory location to operate on.
8023 : VAL is the value to use in the operation.
8024 : USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
8025 : MODEL is the memory model, if used.
8026 : AFTER is true if the returned result is the value after the operation. */
8027 :
8028 : static rtx
8029 64202 : maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
8030 : rtx val, bool use_memmodel, enum memmodel model, bool after)
8031 : {
8032 64202 : machine_mode mode = GET_MODE (mem);
8033 64202 : class expand_operand ops[4];
8034 64202 : enum insn_code icode;
8035 64202 : int op_counter = 0;
8036 64202 : int num_ops;
8037 :
8038 : /* Check to see if there is a result returned. */
8039 64202 : if (target == const0_rtx)
8040 : {
8041 14640 : if (use_memmodel)
8042 : {
8043 14281 : icode = direct_optab_handler (optab->mem_no_result, mode);
8044 14281 : create_integer_operand (&ops[2], model);
8045 14281 : num_ops = 3;
8046 : }
8047 : else
8048 : {
8049 359 : icode = direct_optab_handler (optab->no_result, mode);
8050 359 : num_ops = 2;
8051 : }
8052 : }
8053 : /* Otherwise, we need to generate a result. */
8054 : else
8055 : {
8056 49562 : if (use_memmodel)
8057 : {
8058 30921 : icode = direct_optab_handler (after ? optab->mem_fetch_after
8059 : : optab->mem_fetch_before, mode);
8060 30921 : create_integer_operand (&ops[3], model);
8061 30921 : num_ops = 4;
8062 : }
8063 : else
8064 : {
8065 18641 : icode = optab_handler (after ? optab->fetch_after
8066 : : optab->fetch_before, mode);
8067 18641 : num_ops = 3;
8068 : }
8069 49562 : create_output_operand (&ops[op_counter++], target, mode);
8070 : }
8071 64202 : if (icode == CODE_FOR_nothing)
8072 : return NULL_RTX;
8073 :
8074 26202 : create_fixed_operand (&ops[op_counter++], mem);
8075 : /* VAL may have been promoted to a wider mode. Shrink it if so. */
8076 26202 : create_convert_operand_to (&ops[op_counter++], val, mode, true);
8077 :
8078 26202 : if (maybe_expand_insn (icode, num_ops, ops))
8079 26202 : return (target == const0_rtx ? const0_rtx : ops[0].value);
8080 :
8081 : return NULL_RTX;
8082 : }
8083 :
8084 :
8085 : /* This function expands an atomic fetch_OP or OP_fetch operation:
8086 : TARGET is an option place to stick the return value. const0_rtx indicates
8087 : the result is unused.
8088 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
8089 : CODE is the operation being performed (OP)
8090 : MEMMODEL is the memory model variant to use.
8091 : AFTER is true to return the result of the operation (OP_fetch).
8092 : AFTER is false to return the value before the operation (fetch_OP).
8093 :
8094 : This function will *only* generate instructions if there is a direct
8095 : optab. No compare and swap loops or libcalls will be generated. */
8096 :
8097 : static rtx
8098 34744 : expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
8099 : enum rtx_code code, enum memmodel model,
8100 : bool after)
8101 : {
8102 34744 : machine_mode mode = GET_MODE (mem);
8103 34744 : struct atomic_op_functions optab;
8104 34744 : rtx result;
8105 34744 : bool unused_result = (target == const0_rtx);
8106 :
8107 34744 : get_atomic_op_for_code (&optab, code);
8108 :
8109 : /* Check to see if there are any better instructions. */
8110 34744 : result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
8111 34744 : if (result)
8112 : return result;
8113 :
8114 : /* Check for the case where the result isn't used and try those patterns. */
8115 34252 : if (unused_result)
8116 : {
8117 : /* Try the memory model variant first. */
8118 14281 : result = maybe_emit_op (&optab, target, mem, val, true, model, true);
8119 14281 : if (result)
8120 : return result;
8121 :
8122 : /* Next try the old style withuot a memory model. */
8123 359 : result = maybe_emit_op (&optab, target, mem, val, false, model, true);
8124 359 : if (result)
8125 : return result;
8126 :
8127 : /* There is no no-result pattern, so try patterns with a result. */
8128 : target = NULL_RTX;
8129 : }
8130 :
8131 : /* Try the __atomic version. */
8132 20330 : result = maybe_emit_op (&optab, target, mem, val, true, model, after);
8133 20330 : if (result)
8134 : return result;
8135 :
8136 : /* Try the older __sync version. */
8137 11874 : result = maybe_emit_op (&optab, target, mem, val, false, model, after);
8138 11874 : if (result)
8139 : return result;
8140 :
8141 : /* If the fetch value can be calculated from the other variation of fetch,
8142 : try that operation. */
8143 11874 : if (after || unused_result || optab.reverse_code != UNKNOWN)
8144 : {
8145 : /* Try the __atomic version, then the older __sync version. */
8146 10591 : result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
8147 10591 : if (!result)
8148 6767 : result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
8149 :
8150 6767 : if (result)
8151 : {
8152 : /* If the result isn't used, no need to do compensation code. */
8153 3824 : if (unused_result)
8154 : return result;
8155 :
8156 : /* Issue compensation code. Fetch_after == fetch_before OP val.
8157 : Fetch_before == after REVERSE_OP val. */
8158 3824 : if (!after)
8159 0 : code = optab.reverse_code;
8160 3824 : if (code == NOT)
8161 : {
8162 0 : result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
8163 : true, OPTAB_LIB_WIDEN);
8164 0 : result = expand_simple_unop (mode, NOT, result, target, true);
8165 : }
8166 : else
8167 3824 : result = expand_simple_binop (mode, code, result, val, target,
8168 : true, OPTAB_LIB_WIDEN);
8169 3824 : return result;
8170 : }
8171 : }
8172 :
8173 : /* No direct opcode can be generated. */
8174 : return NULL_RTX;
8175 : }
8176 :
8177 :
8178 :
8179 : /* This function expands an atomic fetch_OP or OP_fetch operation:
8180 : TARGET is an option place to stick the return value. const0_rtx indicates
8181 : the result is unused.
8182 : atomically fetch MEM, perform the operation with VAL and return it to MEM.
8183 : CODE is the operation being performed (OP)
8184 : MEMMODEL is the memory model variant to use.
8185 : AFTER is true to return the result of the operation (OP_fetch).
8186 : AFTER is false to return the value before the operation (fetch_OP). */
8187 : rtx
8188 32352 : expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
8189 : enum memmodel model, bool after)
8190 : {
8191 32352 : machine_mode mode = GET_MODE (mem);
8192 32352 : rtx result;
8193 32352 : bool unused_result = (target == const0_rtx);
8194 :
8195 : /* If loads are not atomic for the required size and we are not called to
8196 : provide a __sync builtin, do not do anything so that we stay consistent
8197 : with atomic loads of the same size. */
8198 32352 : if (!can_atomic_load_p (mode) && !is_mm_sync (model))
8199 : return NULL_RTX;
8200 :
8201 31753 : result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
8202 : after);
8203 :
8204 31753 : if (result)
8205 : return result;
8206 :
8207 : /* Add/sub can be implemented by doing the reverse operation with -(val). */
8208 8011 : if (code == PLUS || code == MINUS)
8209 : {
8210 2991 : rtx tmp;
8211 2991 : enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
8212 :
8213 2991 : start_sequence ();
8214 2991 : tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
8215 2991 : result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
8216 : model, after);
8217 2991 : if (result)
8218 : {
8219 : /* PLUS worked so emit the insns and return. */
8220 2952 : tmp = end_sequence ();
8221 2952 : emit_insn (tmp);
8222 2952 : return result;
8223 : }
8224 :
8225 : /* PLUS did not work, so throw away the negation code and continue. */
8226 39 : end_sequence ();
8227 : }
8228 :
8229 : /* Try the __sync libcalls only if we can't do compare-and-swap inline. */
8230 5059 : if (!can_compare_and_swap_p (mode, false))
8231 : {
8232 2 : rtx libfunc;
8233 2 : bool fixup = false;
8234 2 : enum rtx_code orig_code = code;
8235 2 : struct atomic_op_functions optab;
8236 :
8237 2 : get_atomic_op_for_code (&optab, code);
8238 2 : libfunc = optab_libfunc (after ? optab.fetch_after
8239 : : optab.fetch_before, mode);
8240 2 : if (libfunc == NULL
8241 2 : && (after || unused_result || optab.reverse_code != UNKNOWN))
8242 : {
8243 2 : fixup = true;
8244 2 : if (!after)
8245 1 : code = optab.reverse_code;
8246 2 : libfunc = optab_libfunc (after ? optab.fetch_before
8247 : : optab.fetch_after, mode);
8248 : }
8249 2 : if (libfunc != NULL)
8250 : {
8251 0 : rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
8252 0 : result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
8253 : addr, ptr_mode, val, mode);
8254 :
8255 0 : if (!unused_result && fixup)
8256 0 : result = expand_simple_binop (mode, code, result, val, target,
8257 : true, OPTAB_LIB_WIDEN);
8258 0 : return result;
8259 : }
8260 :
8261 : /* We need the original code for any further attempts. */
8262 2 : code = orig_code;
8263 : }
8264 :
8265 : /* If nothing else has succeeded, default to a compare and swap loop. */
8266 5059 : if (can_compare_and_swap_p (mode, true))
8267 : {
8268 5057 : rtx_insn *insn;
8269 5057 : rtx t0 = gen_reg_rtx (mode), t1;
8270 :
8271 5057 : start_sequence ();
8272 :
8273 : /* If the result is used, get a register for it. */
8274 5057 : if (!unused_result)
8275 : {
8276 4721 : if (!target || !register_operand (target, mode))
8277 12 : target = gen_reg_rtx (mode);
8278 : /* If fetch_before, copy the value now. */
8279 4721 : if (!after)
8280 2068 : emit_move_insn (target, t0);
8281 : }
8282 : else
8283 336 : target = const0_rtx;
8284 :
8285 5057 : t1 = t0;
8286 5057 : if (code == NOT)
8287 : {
8288 687 : t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
8289 : true, OPTAB_LIB_WIDEN);
8290 687 : t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
8291 : }
8292 : else
8293 4370 : t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
8294 : OPTAB_LIB_WIDEN);
8295 :
8296 : /* For after, copy the value now. */
8297 5057 : if (!unused_result && after)
8298 2653 : emit_move_insn (target, t1);
8299 5057 : insn = end_sequence ();
8300 :
8301 5057 : if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
8302 : return target;
8303 : }
8304 :
8305 : return NULL_RTX;
8306 : }
8307 :
8308 : /* Return true if OPERAND is suitable for operand number OPNO of
8309 : instruction ICODE. */
8310 :
8311 : bool
8312 152162284 : insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
8313 : {
8314 152162284 : return (!insn_data[(int) icode].operand[opno].predicate
8315 302764398 : || (insn_data[(int) icode].operand[opno].predicate
8316 150602114 : (operand, insn_data[(int) icode].operand[opno].mode)));
8317 : }
8318 :
8319 : /* TARGET is a target of a multiword operation that we are going to
8320 : implement as a series of word-mode operations. Return true if
8321 : TARGET is suitable for this purpose. */
8322 :
8323 : bool
8324 190 : valid_multiword_target_p (rtx target)
8325 : {
8326 190 : machine_mode mode;
8327 190 : int i, size;
8328 :
8329 190 : mode = GET_MODE (target);
8330 380 : if (!GET_MODE_SIZE (mode).is_constant (&size))
8331 : return false;
8332 866 : for (i = 0; i < size; i += UNITS_PER_WORD)
8333 380 : if (!validate_subreg (word_mode, mode, target, i))
8334 : return false;
8335 : return true;
8336 : }
8337 :
8338 : /* Make OP describe an input operand that has value INTVAL and that has
8339 : no inherent mode. This function should only be used for operands that
8340 : are always expand-time constants. The backend may request that INTVAL
8341 : be copied into a different kind of rtx, but it must specify the mode
8342 : of that rtx if so. */
8343 :
8344 : void
8345 1984422 : create_integer_operand (class expand_operand *op, poly_int64 intval)
8346 : {
8347 1984422 : create_expand_operand (op, EXPAND_INTEGER,
8348 : gen_int_mode (intval, MAX_MODE_INT),
8349 : VOIDmode, false, intval);
8350 1984422 : }
8351 :
8352 : /* Like maybe_legitimize_operand, but do not change the code of the
8353 : current rtx value. */
8354 :
8355 : static bool
8356 50720879 : maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
8357 : class expand_operand *op)
8358 : {
8359 : /* See if the operand matches in its current form. */
8360 50720879 : if (insn_operand_matches (icode, opno, op->value))
8361 : return true;
8362 :
8363 : /* If the operand is a memory whose address has no side effects,
8364 : try forcing the address into a non-virtual pseudo register.
8365 : The check for side effects is important because copy_to_mode_reg
8366 : cannot handle things like auto-modified addresses. */
8367 1500978 : if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
8368 : {
8369 13146 : rtx addr, mem;
8370 :
8371 13146 : mem = op->value;
8372 13146 : addr = XEXP (mem, 0);
8373 985 : if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
8374 13952 : && !side_effects_p (addr))
8375 : {
8376 8488 : rtx_insn *last;
8377 8488 : machine_mode mode;
8378 :
8379 8488 : last = get_last_insn ();
8380 8488 : mode = get_address_mode (mem);
8381 8488 : mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
8382 8488 : if (insn_operand_matches (icode, opno, mem))
8383 : {
8384 0 : op->value = mem;
8385 0 : return true;
8386 : }
8387 8488 : delete_insns_since (last);
8388 : }
8389 : }
8390 :
8391 : return false;
8392 : }
8393 :
8394 : /* Try to make OP match operand OPNO of instruction ICODE. Return true
8395 : on success, storing the new operand value back in OP. */
8396 :
8397 : static bool
8398 58300099 : maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
8399 : class expand_operand *op)
8400 : {
8401 58300099 : machine_mode mode, imode, tmode;
8402 :
8403 58300099 : mode = op->mode;
8404 58300099 : switch (op->type)
8405 : {
8406 4224919 : case EXPAND_FIXED:
8407 4224919 : {
8408 4224919 : temporary_volatile_ok v (true);
8409 4224919 : return maybe_legitimize_operand_same_code (icode, opno, op);
8410 4224919 : }
8411 :
8412 18803978 : case EXPAND_OUTPUT:
8413 18803978 : gcc_assert (mode != VOIDmode);
8414 18803978 : if (op->value
8415 11625251 : && op->value != const0_rtx
8416 11624829 : && GET_MODE (op->value) == mode
8417 30393887 : && maybe_legitimize_operand_same_code (icode, opno, op))
8418 : return true;
8419 :
8420 7335899 : op->value = gen_reg_rtx (mode);
8421 7335899 : op->target = 0;
8422 7335899 : break;
8423 :
8424 34906051 : case EXPAND_INPUT:
8425 34906051 : input:
8426 34906051 : gcc_assert (mode != VOIDmode);
8427 34906051 : gcc_assert (GET_MODE (op->value) == VOIDmode
8428 : || GET_MODE (op->value) == mode);
8429 34906051 : if (maybe_legitimize_operand_same_code (icode, opno, op))
8430 : return true;
8431 :
8432 1234696 : op->value = copy_to_mode_reg (mode, op->value);
8433 1234696 : break;
8434 :
8435 199512 : case EXPAND_CONVERT_TO:
8436 199512 : gcc_assert (mode != VOIDmode);
8437 199512 : op->value = convert_to_mode (mode, op->value, op->unsigned_p);
8438 199512 : goto input;
8439 :
8440 607123 : case EXPAND_CONVERT_FROM:
8441 607123 : if (GET_MODE (op->value) != VOIDmode)
8442 400673 : mode = GET_MODE (op->value);
8443 : else
8444 : /* The caller must tell us what mode this value has. */
8445 206450 : gcc_assert (mode != VOIDmode);
8446 :
8447 607123 : imode = insn_data[(int) icode].operand[opno].mode;
8448 607123 : tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
8449 607123 : ? GET_MODE_INNER (imode) : imode);
8450 607123 : if (tmode != VOIDmode && tmode != mode)
8451 : {
8452 1501 : op->value = convert_modes (tmode, mode, op->value, op->unsigned_p);
8453 1501 : mode = tmode;
8454 : }
8455 607123 : if (imode != VOIDmode && imode != mode)
8456 : {
8457 58 : gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
8458 58 : op->value = expand_vector_broadcast (imode, op->value);
8459 58 : mode = imode;
8460 : }
8461 607123 : goto input;
8462 :
8463 3349 : case EXPAND_ADDRESS:
8464 3349 : op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
8465 : op->value);
8466 3349 : goto input;
8467 :
8468 1626375 : case EXPAND_INTEGER:
8469 1626375 : mode = insn_data[(int) icode].operand[opno].mode;
8470 1626375 : if (mode != VOIDmode
8471 1626375 : && known_eq (trunc_int_for_mode (op->int_value, mode),
8472 : op->int_value))
8473 : {
8474 1261224 : op->value = gen_int_mode (op->int_value, mode);
8475 1261224 : goto input;
8476 : }
8477 : break;
8478 :
8479 240 : case EXPAND_UNDEFINED_INPUT:
8480 : /* See if the predicate accepts a SCRATCH rtx, which in this context
8481 : indicates an undefined value. Use an uninitialized register if not. */
8482 240 : if (!insn_operand_matches (icode, opno, op->value))
8483 : {
8484 240 : op->value = gen_reg_rtx (op->mode);
8485 240 : goto input;
8486 : }
8487 : return true;
8488 : }
8489 8935746 : return insn_operand_matches (icode, opno, op->value);
8490 : }
8491 :
8492 : /* Make OP describe an input operand that should have the same value
8493 : as VALUE, after any mode conversion that the target might request.
8494 : TYPE is the type of VALUE. */
8495 :
8496 : void
8497 2614 : create_convert_operand_from_type (class expand_operand *op,
8498 : rtx value, tree type)
8499 : {
8500 2614 : create_convert_operand_from (op, value, TYPE_MODE (type),
8501 2614 : TYPE_UNSIGNED (type));
8502 2614 : }
8503 :
8504 : /* Return true if the requirements on operands OP1 and OP2 of instruction
8505 : ICODE are similar enough for the result of legitimizing OP1 to be
8506 : reusable for OP2. OPNO1 and OPNO2 are the operand numbers associated
8507 : with OP1 and OP2 respectively. */
8508 :
8509 : static inline bool
8510 64666799 : can_reuse_operands_p (enum insn_code icode,
8511 : unsigned int opno1, unsigned int opno2,
8512 : const class expand_operand *op1,
8513 : const class expand_operand *op2)
8514 : {
8515 : /* Check requirements that are common to all types. */
8516 64666799 : if (op1->type != op2->type
8517 20082578 : || op1->mode != op2->mode
8518 18594661 : || (insn_data[(int) icode].operand[opno1].mode
8519 18594661 : != insn_data[(int) icode].operand[opno2].mode))
8520 : return false;
8521 :
8522 : /* Check the requirements for specific types. */
8523 16809511 : switch (op1->type)
8524 : {
8525 : case EXPAND_OUTPUT:
8526 : case EXPAND_UNDEFINED_INPUT:
8527 : /* Outputs and undefined intputs must remain distinct. */
8528 : return false;
8529 :
8530 : case EXPAND_FIXED:
8531 : case EXPAND_INPUT:
8532 : case EXPAND_ADDRESS:
8533 : case EXPAND_INTEGER:
8534 : return true;
8535 :
8536 99868 : case EXPAND_CONVERT_TO:
8537 99868 : case EXPAND_CONVERT_FROM:
8538 99868 : return op1->unsigned_p == op2->unsigned_p;
8539 : }
8540 0 : gcc_unreachable ();
8541 : }
8542 :
8543 : /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
8544 : of instruction ICODE. Return true on success, leaving the new operand
8545 : values in the OPS themselves. Emit no code on failure. */
8546 :
8547 : bool
8548 19530996 : maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
8549 : unsigned int nops, class expand_operand *ops)
8550 : {
8551 19530996 : rtx_insn *last = get_last_insn ();
8552 19530996 : rtx *orig_values = XALLOCAVEC (rtx, nops);
8553 78055765 : for (unsigned int i = 0; i < nops; i++)
8554 : {
8555 58676973 : orig_values[i] = ops[i].value;
8556 :
8557 : /* First try reusing the result of an earlier legitimization.
8558 : This avoids duplicate rtl and ensures that tied operands
8559 : remain tied.
8560 :
8561 : This search is linear, but NOPS is bounded at compile time
8562 : to a small number (current a single digit). */
8563 58676973 : unsigned int j = 0;
8564 122966898 : for (; j < i; ++j)
8565 64666799 : if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
8566 16789220 : && rtx_equal_p (orig_values[j], orig_values[i])
8567 388001 : && ops[j].value
8568 65043677 : && insn_operand_matches (icode, opno + i, ops[j].value))
8569 : {
8570 376874 : ops[i].value = copy_rtx (ops[j].value);
8571 376874 : break;
8572 : }
8573 :
8574 : /* Otherwise try legitimizing the operand on its own. */
8575 58676973 : if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
8576 : {
8577 152204 : delete_insns_since (last);
8578 152204 : return false;
8579 : }
8580 : }
8581 : return true;
8582 : }
8583 :
8584 : /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
8585 : as its operands. Return the instruction pattern on success,
8586 : and emit any necessary set-up code. Return null and emit no
8587 : code on failure. */
8588 :
8589 : rtx_insn *
8590 19506889 : maybe_gen_insn (enum insn_code icode, unsigned int nops,
8591 : class expand_operand *ops)
8592 : {
8593 19506889 : gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
8594 19506889 : if (!maybe_legitimize_operands (icode, 0, nops, ops))
8595 : return NULL;
8596 :
8597 19354685 : switch (nops)
8598 : {
8599 0 : case 0:
8600 0 : return GEN_FCN (icode) ();
8601 1311 : case 1:
8602 1311 : return GEN_FCN (icode) (ops[0].value);
8603 2839193 : case 2:
8604 2839193 : return GEN_FCN (icode) (ops[0].value, ops[1].value);
8605 14550910 : case 3:
8606 14550910 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
8607 1580911 : case 4:
8608 1580911 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8609 1580911 : ops[3].value);
8610 190238 : case 5:
8611 190238 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8612 190238 : ops[3].value, ops[4].value);
8613 262 : case 6:
8614 262 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8615 262 : ops[3].value, ops[4].value, ops[5].value);
8616 0 : case 7:
8617 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8618 0 : ops[3].value, ops[4].value, ops[5].value,
8619 0 : ops[6].value);
8620 26724 : case 8:
8621 26724 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8622 26724 : ops[3].value, ops[4].value, ops[5].value,
8623 26724 : ops[6].value, ops[7].value);
8624 165136 : case 9:
8625 165136 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8626 165136 : ops[3].value, ops[4].value, ops[5].value,
8627 165136 : ops[6].value, ops[7].value, ops[8].value);
8628 0 : case 10:
8629 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8630 0 : ops[3].value, ops[4].value, ops[5].value,
8631 0 : ops[6].value, ops[7].value, ops[8].value,
8632 0 : ops[9].value);
8633 0 : case 11:
8634 0 : return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
8635 0 : ops[3].value, ops[4].value, ops[5].value,
8636 0 : ops[6].value, ops[7].value, ops[8].value,
8637 0 : ops[9].value, ops[10].value);
8638 : }
8639 0 : gcc_unreachable ();
8640 : }
8641 :
8642 : /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
8643 : as its operands. Return true on success and emit no code on failure. */
8644 :
8645 : bool
8646 2871905 : maybe_expand_insn (enum insn_code icode, unsigned int nops,
8647 : class expand_operand *ops)
8648 : {
8649 2871905 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8650 2871905 : if (pat)
8651 : {
8652 2079564 : emit_insn (pat);
8653 2079564 : return true;
8654 : }
8655 : return false;
8656 : }
8657 :
8658 : /* Like maybe_expand_insn, but for jumps. */
8659 :
8660 : bool
8661 1308 : maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
8662 : class expand_operand *ops)
8663 : {
8664 1308 : rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
8665 1308 : if (pat)
8666 : {
8667 1308 : emit_jump_insn (pat);
8668 1308 : return true;
8669 : }
8670 : return false;
8671 : }
8672 :
8673 : /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
8674 : as its operands. */
8675 :
8676 : void
8677 634162 : expand_insn (enum insn_code icode, unsigned int nops,
8678 : class expand_operand *ops)
8679 : {
8680 634162 : if (!maybe_expand_insn (icode, nops, ops))
8681 0 : gcc_unreachable ();
8682 634162 : }
8683 :
8684 : /* Like expand_insn, but for jumps. */
8685 :
8686 : void
8687 1308 : expand_jump_insn (enum insn_code icode, unsigned int nops,
8688 : class expand_operand *ops)
8689 : {
8690 1308 : if (!maybe_expand_jump_insn (icode, nops, ops))
8691 0 : gcc_unreachable ();
8692 1308 : }
|
