LCOV - code coverage report
Current view: top level - gcc - optabs.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 68.1 % 3891 2650
Test Date: 2026-11-14 16:39:50 Functions: 84.3 % 134 113
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Expand the basic unary and binary arithmetic operations, for GNU compiler.
       2              :    Copyright (C) 1987-2026 Free Software Foundation, Inc.
       3              : 
       4              : This file is part of GCC.
       5              : 
       6              : GCC is free software; you can redistribute it and/or modify it under
       7              : the terms of the GNU General Public License as published by the Free
       8              : Software Foundation; either version 3, or (at your option) any later
       9              : version.
      10              : 
      11              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      12              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      13              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      14              : for more details.
      15              : 
      16              : You should have received a copy of the GNU General Public License
      17              : along with GCC; see the file COPYING3.  If not see
      18              : <http://www.gnu.org/licenses/>.  */
      19              : 
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "target.h"
      26              : #include "rtl.h"
      27              : #include "tree.h"
      28              : #include "memmodel.h"
      29              : #include "predict.h"
      30              : #include "tm_p.h"
      31              : #include "optabs.h"
      32              : #include "expmed.h"
      33              : #include "emit-rtl.h"
      34              : #include "recog.h"
      35              : #include "diagnostic-core.h"
      36              : #include "rtx-vector-builder.h"
      37              : 
      38              : /* Include insn-config.h before expr.h so that HAVE_conditional_move
      39              :    is properly defined.  */
      40              : #include "stor-layout.h"
      41              : #include "except.h"
      42              : #include "dojump.h"
      43              : #include "explow.h"
      44              : #include "expr.h"
      45              : #include "optabs-tree.h"
      46              : #include "libfuncs.h"
      47              : #include "internal-fn.h"
      48              : #include "langhooks.h"
      49              : #include "gimple.h"
      50              : #include "ssa.h"
      51              : #include "tree-ssa-live.h"
      52              : #include "tree-outof-ssa.h"
      53              : 
      54              : static void prepare_float_lib_cmp (rtx, rtx, enum rtx_code, rtx *,
      55              :                                    machine_mode *);
      56              : static rtx expand_unop_direct (machine_mode, optab, rtx, rtx, int);
      57              : static void emit_libcall_block_1 (rtx_insn *, rtx, rtx, rtx, bool);
      58              : 
      59              : static rtx emit_conditional_move_1 (rtx, rtx, rtx, rtx, machine_mode);
      60              : 
      61              : /* Debug facility for use in GDB.  */
      62              : void debug_optab_libfuncs (void);
      63              : 
      64              : /* Add a REG_EQUAL note to the last insn in INSNS.  TARGET is being set to
      65              :    the result of operation CODE applied to OP0 (and OP1 if it is a binary
      66              :    operation).  OP0_MODE is OP0's mode.
      67              : 
      68              :    If the last insn does not set TARGET, don't do anything, but return true.
      69              : 
      70              :    If the last insn or a previous insn sets TARGET and TARGET is one of OP0
      71              :    or OP1, don't add the REG_EQUAL note but return false.  Our caller can then
      72              :    try again, ensuring that TARGET is not one of the operands.  */
      73              : 
      74              : static bool
      75      1275596 : add_equal_note (rtx_insn *insns, rtx target, enum rtx_code code, rtx op0,
      76              :                 rtx op1, machine_mode op0_mode)
      77              : {
      78      1275596 :   rtx_insn *last_insn;
      79      1275596 :   rtx set;
      80      1275596 :   rtx note;
      81              : 
      82      2551192 :   gcc_assert (insns && INSN_P (insns) && NEXT_INSN (insns));
      83              : 
      84      1275596 :   if (GET_RTX_CLASS (code) != RTX_COMM_ARITH
      85              :       && GET_RTX_CLASS (code) != RTX_BIN_ARITH
      86              :       && GET_RTX_CLASS (code) != RTX_COMM_COMPARE
      87              :       && GET_RTX_CLASS (code) != RTX_COMPARE
      88              :       && GET_RTX_CLASS (code) != RTX_UNARY)
      89              :     return true;
      90              : 
      91      1268006 :   if (GET_CODE (target) == ZERO_EXTRACT)
      92              :     return true;
      93              : 
      94              :   for (last_insn = insns;
      95      2807438 :        NEXT_INSN (last_insn) != NULL_RTX;
      96              :        last_insn = NEXT_INSN (last_insn))
      97              :     ;
      98              : 
      99              :   /* If TARGET is in OP0 or OP1, punt.  We'd end up with a note referencing
     100              :      a value changing in the insn, so the note would be invalid for CSE.  */
     101      1268006 :   if (reg_overlap_mentioned_p (target, op0)
     102      1268006 :       || (op1 && reg_overlap_mentioned_p (target, op1)))
     103              :     {
     104        58240 :       if (MEM_P (target)
     105        58240 :           && (rtx_equal_p (target, op0)
     106        43300 :               || (op1 && rtx_equal_p (target, op1))))
     107              :         {
     108              :           /* For MEM target, with MEM = MEM op X, prefer no REG_EQUAL note
     109              :              over expanding it as temp = MEM op X, MEM = temp.  If the target
     110              :              supports MEM = MEM op X instructions, it is sometimes too hard
     111              :              to reconstruct that form later, especially if X is also a memory,
     112              :              and due to multiple occurrences of addresses the address might
     113              :              be forced into register unnecessarily.
     114              :              Note that not emitting the REG_EQUIV note might inhibit
     115              :              CSE in some cases.  */
     116         3572 :           set = single_set (last_insn);
     117         3572 :           if (set
     118         3572 :               && GET_CODE (SET_SRC (set)) == code
     119         3545 :               && MEM_P (SET_DEST (set))
     120         7117 :               && (rtx_equal_p (SET_DEST (set), XEXP (SET_SRC (set), 0))
     121            0 :                   || (op1 && rtx_equal_p (SET_DEST (set),
     122            0 :                                           XEXP (SET_SRC (set), 1)))))
     123              :             return true;
     124              :         }
     125        54695 :       return false;
     126              :     }
     127              : 
     128      1209766 :   set = set_for_reg_notes (last_insn);
     129      1209766 :   if (set == NULL_RTX)
     130              :     return true;
     131              : 
     132       736251 :   if (! rtx_equal_p (SET_DEST (set), target)
     133              :       /* For a STRICT_LOW_PART, the REG_NOTE applies to what is inside it.  */
     134       736251 :       && (GET_CODE (SET_DEST (set)) != STRICT_LOW_PART
     135            0 :           || ! rtx_equal_p (XEXP (SET_DEST (set), 0), target)))
     136              :     return true;
     137              : 
     138       735477 :   if (GET_RTX_CLASS (code) == RTX_UNARY)
     139        43512 :     switch (code)
     140              :       {
     141         3319 :       case FFS:
     142         3319 :       case CLZ:
     143         3319 :       case CTZ:
     144         3319 :       case CLRSB:
     145         3319 :       case POPCOUNT:
     146         3319 :       case PARITY:
     147         3319 :       case BSWAP:
     148         3319 :         if (op0_mode != VOIDmode && GET_MODE (target) != op0_mode)
     149              :           {
     150          251 :             note = gen_rtx_fmt_e (code, op0_mode, copy_rtx (op0));
     151          251 :             if (GET_MODE_UNIT_SIZE (op0_mode)
     152          502 :                 > GET_MODE_UNIT_SIZE (GET_MODE (target)))
     153          251 :               note = simplify_gen_unary (TRUNCATE, GET_MODE (target),
     154              :                                          note, op0_mode);
     155              :             else
     156            0 :               note = simplify_gen_unary (ZERO_EXTEND, GET_MODE (target),
     157              :                                          note, op0_mode);
     158              :             break;
     159              :           }
     160              :         /* FALLTHRU */
     161        43261 :       default:
     162        43261 :         note = gen_rtx_fmt_e (code, GET_MODE (target), copy_rtx (op0));
     163        43261 :         break;
     164              :       }
     165              :   else
     166       691965 :     note = gen_rtx_fmt_ee (code, GET_MODE (target), copy_rtx (op0), copy_rtx (op1));
     167              : 
     168       735477 :   set_unique_reg_note (last_insn, REG_EQUAL, note);
     169              : 
     170       735477 :   return true;
     171              : }
     172              : 
     173              : /* Given two input operands, OP0 and OP1, determine what the correct from_mode
     174              :    for a widening operation would be.  In most cases this would be OP0, but if
     175              :    that's a constant it'll be VOIDmode, which isn't useful.  */
     176              : 
     177              : static machine_mode
     178        38679 : widened_mode (machine_mode to_mode, rtx op0, rtx op1)
     179              : {
     180        38679 :   machine_mode m0 = GET_MODE (op0);
     181        38679 :   machine_mode m1 = GET_MODE (op1);
     182        38679 :   machine_mode result;
     183              : 
     184        38679 :   if (m0 == VOIDmode && m1 == VOIDmode)
     185              :     return to_mode;
     186        77358 :   else if (m0 == VOIDmode || GET_MODE_UNIT_SIZE (m0) < GET_MODE_UNIT_SIZE (m1))
     187              :     result = m1;
     188              :   else
     189              :     result = m0;
     190              : 
     191       116037 :   if (GET_MODE_UNIT_SIZE (result) > GET_MODE_UNIT_SIZE (to_mode))
     192            0 :     return to_mode;
     193              : 
     194              :   return result;
     195              : }
     196              : 
     197              : /* Widen OP to MODE and return the rtx for the widened operand.  UNSIGNEDP
     198              :    says whether OP is signed or unsigned.  NO_EXTEND is true if we need
     199              :    not actually do a sign-extend or zero-extend, but can leave the
     200              :    higher-order bits of the result rtx undefined, for example, in the case
     201              :    of logical operations, but not right shifts.  */
     202              : 
     203              : static rtx
     204          588 : widen_operand (rtx op, machine_mode mode, machine_mode oldmode,
     205              :                int unsignedp, bool no_extend)
     206              : {
     207          588 :   rtx result;
     208          588 :   scalar_int_mode int_mode;
     209              : 
     210              :   /* If we don't have to extend and this is a constant, return it.  */
     211          588 :   if (no_extend && GET_MODE (op) == VOIDmode)
     212              :     return op;
     213              : 
     214              :   /* If we must extend do so.  If OP is a SUBREG for a promoted object, also
     215              :      extend since it will be more efficient to do so unless the signedness of
     216              :      a promoted object differs from our extension.  */
     217            7 :   if (! no_extend
     218          588 :       || !is_a <scalar_int_mode> (mode, &int_mode)
     219            7 :       || (GET_CODE (op) == SUBREG && SUBREG_PROMOTED_VAR_P (op)
     220            0 :           && SUBREG_CHECK_PROMOTED_SIGN (op, unsignedp)))
     221          581 :     return convert_modes (mode, oldmode, op, unsignedp);
     222              : 
     223              :   /* If MODE is no wider than a single word, we return a lowpart or paradoxical
     224              :      SUBREG.  */
     225           14 :   if (GET_MODE_SIZE (int_mode) <= UNITS_PER_WORD)
     226            7 :     return gen_lowpart (int_mode, force_reg (GET_MODE (op), op));
     227              : 
     228              :   /* Otherwise, get an object of MODE, clobber it, and set the low-order
     229              :      part to OP.  */
     230              : 
     231            0 :   result = gen_reg_rtx (int_mode);
     232            0 :   emit_clobber (result);
     233            0 :   emit_move_insn (gen_lowpart (GET_MODE (op), result), op);
     234            0 :   return result;
     235              : }
     236              : 
     237              : /* Expand vector widening operations.
     238              : 
     239              :    There are two different classes of operations handled here:
     240              :    1) Operations whose result is wider than all the arguments to the operation.
     241              :       Examples: VEC_UNPACK_HI/LO_EXPR, VEC_WIDEN_MULT_HI/LO_EXPR
     242              :       In this case OP0 and optionally OP1 would be initialized,
     243              :       but WIDE_OP wouldn't (not relevant for this case).
     244              :    2) Operations whose result is of the same size as the last argument to the
     245              :       operation, but wider than all the other arguments to the operation.
     246              :       Examples: WIDEN_SUM_EXPR, VEC_DOT_PROD_EXPR.
     247              :       In the case WIDE_OP, OP0 and optionally OP1 would be initialized.
     248              : 
     249              :    E.g, when called to expand the following operations, this is how
     250              :    the arguments will be initialized:
     251              :                                 nops    OP0     OP1     WIDE_OP
     252              :    widening-sum                 2       oprnd0  -       oprnd1
     253              :    widening-dot-product         3       oprnd0  oprnd1  oprnd2
     254              :    widening-mult                2       oprnd0  oprnd1  -
     255              :    type-promotion (vec-unpack)  1       oprnd0  -       -  */
     256              : 
     257              : rtx
     258        21410 : expand_widen_pattern_expr (const_sepops ops, rtx op0, rtx op1, rtx wide_op,
     259              :                            rtx target, int unsignedp)
     260              : {
     261        21410 :   class expand_operand eops[4];
     262        21410 :   tree oprnd0, oprnd1, oprnd2;
     263        21410 :   machine_mode wmode = VOIDmode, tmode0, tmode1 = VOIDmode;
     264        21410 :   optab widen_pattern_optab;
     265        21410 :   enum insn_code icode;
     266        21410 :   int nops = TREE_CODE_LENGTH (ops->code);
     267        21410 :   int op;
     268        21410 :   bool sbool = false;
     269              : 
     270        21410 :   oprnd0 = ops->op0;
     271        21410 :   oprnd1 = nops >= 2 ? ops->op1 : NULL_TREE;
     272         1619 :   oprnd2 = nops >= 3 ? ops->op2 : NULL_TREE;
     273              : 
     274        21410 :   tmode0 = TYPE_MODE (TREE_TYPE (oprnd0));
     275        21410 :   if (ops->code == VEC_UNPACK_FIX_TRUNC_HI_EXPR
     276        21410 :       || ops->code == VEC_UNPACK_FIX_TRUNC_LO_EXPR)
     277              :     /* The sign is from the result type rather than operand's type
     278              :        for these ops.  */
     279           34 :     widen_pattern_optab
     280           34 :       = optab_for_tree_code (ops->code, ops->type, optab_default);
     281        21376 :   else if ((ops->code == VEC_UNPACK_HI_EXPR
     282        21376 :             || ops->code == VEC_UNPACK_LO_EXPR)
     283        17921 :            && VECTOR_BOOLEAN_TYPE_P (ops->type)
     284         4362 :            && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (oprnd0))
     285         4362 :            && TYPE_MODE (ops->type) == TYPE_MODE (TREE_TYPE (oprnd0))
     286        21876 :            && SCALAR_INT_MODE_P (TYPE_MODE (ops->type)))
     287              :     {
     288              :       /* For VEC_UNPACK_{LO,HI}_EXPR if the mode of op0 and result is
     289              :          the same scalar mode for VECTOR_BOOLEAN_TYPE_P vectors, use
     290              :          vec_unpacks_sbool_{lo,hi}_optab, so that we can pass in
     291              :          the pattern number of elements in the wider vector.  */
     292          500 :       widen_pattern_optab
     293          247 :         = (ops->code == VEC_UNPACK_HI_EXPR
     294          500 :            ? vec_unpacks_sbool_hi_optab : vec_unpacks_sbool_lo_optab);
     295              :       sbool = true;
     296              :     }
     297        20876 :   else if (ops->code == DOT_PROD_EXPR)
     298              :     {
     299          441 :       enum optab_subtype subtype = optab_default;
     300          441 :       signop sign1 = TYPE_SIGN (TREE_TYPE (oprnd0));
     301          441 :       signop sign2 = TYPE_SIGN (TREE_TYPE (oprnd1));
     302          441 :       if (sign1 == sign2)
     303              :         ;
     304           82 :       else if (sign1 == SIGNED && sign2 == UNSIGNED)
     305              :         {
     306              :           subtype = optab_vector_mixed_sign;
     307              :           /* Same as optab_vector_mixed_sign but flip the operands.  */
     308              :           std::swap (op0, op1);
     309              :         }
     310           60 :       else if (sign1 == UNSIGNED && sign2 == SIGNED)
     311              :         subtype = optab_vector_mixed_sign;
     312              :       else
     313            0 :         gcc_unreachable ();
     314              : 
     315          441 :       widen_pattern_optab
     316          441 :         = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), subtype);
     317              :     }
     318              :   else
     319        20435 :     widen_pattern_optab
     320        20435 :       = optab_for_tree_code (ops->code, TREE_TYPE (oprnd0), optab_default);
     321        21410 :   if (ops->code == WIDEN_MULT_PLUS_EXPR
     322        21410 :       || ops->code == WIDEN_MULT_MINUS_EXPR
     323        21410 :       || ops->code == DOT_PROD_EXPR
     324        20969 :       || ops->code == SAD_EXPR)
     325          560 :     icode = find_widening_optab_handler (widen_pattern_optab,
     326              :                                          TYPE_MODE (TREE_TYPE (ops->op2)),
     327              :                                          tmode0);
     328        20850 :   else if (ops->code == WIDEN_SUM_EXPR)
     329            0 :     icode = find_widening_optab_handler (widen_pattern_optab,
     330              :                                          TYPE_MODE (TREE_TYPE (ops->op1)),
     331              :                                          tmode0);
     332              :   else
     333        20850 :     icode = optab_handler (widen_pattern_optab, tmode0);
     334        21410 :   gcc_assert (icode != CODE_FOR_nothing);
     335              : 
     336        21410 :   if (nops >= 2)
     337         1619 :     tmode1 = TYPE_MODE (TREE_TYPE (oprnd1));
     338        19791 :   else if (sbool)
     339              :     {
     340          500 :       nops = 2;
     341          500 :       op1 = GEN_INT (TYPE_VECTOR_SUBPARTS (TREE_TYPE (oprnd0)).to_constant ());
     342          500 :       tmode1 = tmode0;
     343              :     }
     344              : 
     345              :   /* The last operand is of a wider mode than the rest of the operands.  */
     346        21410 :   if (nops == 2)
     347              :     wmode = tmode1;
     348        19851 :   else if (nops == 3)
     349              :     {
     350          560 :       gcc_assert (tmode1 == tmode0);
     351          560 :       gcc_assert (op1);
     352          560 :       wmode = TYPE_MODE (TREE_TYPE (oprnd2));
     353              :     }
     354              : 
     355        21410 :   op = 0;
     356        21410 :   create_output_operand (&eops[op++], target, TYPE_MODE (ops->type));
     357        21410 :   create_convert_operand_from (&eops[op++], op0, tmode0, unsignedp);
     358        21410 :   if (op1)
     359         2119 :     create_convert_operand_from (&eops[op++], op1, tmode1, unsignedp);
     360        21410 :   if (wide_op)
     361          560 :     create_convert_operand_from (&eops[op++], wide_op, wmode, unsignedp);
     362        21410 :   expand_insn (icode, op, eops);
     363        21410 :   return eops[0].value;
     364              : }
     365              : 
     366              : /* Generate code to perform an operation specified by TERNARY_OPTAB
     367              :    on operands OP0, OP1 and OP2, with result having machine-mode MODE.
     368              : 
     369              :    UNSIGNEDP is for the case where we have to widen the operands
     370              :    to perform the operation.  It says to use zero-extension.
     371              : 
     372              :    If TARGET is nonzero, the value
     373              :    is generated there, if it is convenient to do so.
     374              :    In all cases an rtx is returned for the locus of the value;
     375              :    this may or may not be TARGET.  */
     376              : 
     377              : rtx
     378            0 : expand_ternary_op (machine_mode mode, optab ternary_optab, rtx op0,
     379              :                    rtx op1, rtx op2, rtx target, int unsignedp)
     380              : {
     381            0 :   class expand_operand ops[4];
     382            0 :   enum insn_code icode = optab_handler (ternary_optab, mode);
     383              : 
     384            0 :   gcc_assert (optab_handler (ternary_optab, mode) != CODE_FOR_nothing);
     385              : 
     386            0 :   create_output_operand (&ops[0], target, mode);
     387            0 :   create_convert_operand_from (&ops[1], op0, mode, unsignedp);
     388            0 :   create_convert_operand_from (&ops[2], op1, mode, unsignedp);
     389            0 :   create_convert_operand_from (&ops[3], op2, mode, unsignedp);
     390            0 :   expand_insn (icode, 4, ops);
     391            0 :   return ops[0].value;
     392              : }
     393              : 
     394              : 
     395              : /* Like expand_binop, but return a constant rtx if the result can be
     396              :    calculated at compile time.  The arguments and return value are
     397              :    otherwise the same as for expand_binop.  */
     398              : 
     399              : rtx
     400         1645 : simplify_expand_binop (machine_mode mode, optab binoptab,
     401              :                        rtx op0, rtx op1, rtx target, int unsignedp,
     402              :                        enum optab_methods methods)
     403              : {
     404         1645 :   if (CONSTANT_P (op0) && CONSTANT_P (op1))
     405              :     {
     406          184 :       rtx x = simplify_binary_operation (optab_to_code (binoptab),
     407              :                                          mode, op0, op1);
     408          184 :       if (x)
     409              :         return x;
     410              :     }
     411              : 
     412         1461 :   return expand_binop (mode, binoptab, op0, op1, target, unsignedp, methods);
     413              : }
     414              : 
     415              : /* Like simplify_expand_binop, but always put the result in TARGET.
     416              :    Return true if the expansion succeeded.  */
     417              : 
     418              : bool
     419         1277 : force_expand_binop (machine_mode mode, optab binoptab,
     420              :                     rtx op0, rtx op1, rtx target, int unsignedp,
     421              :                     enum optab_methods methods)
     422              : {
     423         1277 :   rtx x = simplify_expand_binop (mode, binoptab, op0, op1,
     424              :                                  target, unsignedp, methods);
     425         1277 :   if (x == 0)
     426              :     return false;
     427         1277 :   if (x != target)
     428            0 :     emit_move_insn (target, x);
     429              :   return true;
     430              : }
     431              : 
     432              : /* Create a new vector value in VMODE with all elements set to OP.  If OP
     433              :    is not a constant, the mode of it must be the element mode of VMODE
     434              :    (if the element is BImode, additionally OP is allowed to be in QImode).
     435              :    If OP is a constant, then the return value will be a constant.  */
     436              : 
     437              : rtx
     438           46 : expand_vector_broadcast (machine_mode vmode, rtx op)
     439              : {
     440           46 :   int n;
     441           46 :   rtvec vec;
     442              : 
     443           46 :   gcc_checking_assert (VECTOR_MODE_P (vmode));
     444           46 :   gcc_checking_assert (CONST_INT_P (op)
     445              :                        || GET_MODE_INNER (vmode) == GET_MODE (op)
     446              :                        || (GET_MODE_INNER (vmode) == BImode
     447              :                            && GET_MODE (op) == QImode));
     448              : 
     449           46 :   if (valid_for_const_vector_p (vmode, op))
     450           46 :     return gen_const_vec_duplicate (vmode, op);
     451              : 
     452            0 :   insn_code icode = optab_handler (vec_duplicate_optab, vmode);
     453            0 :   if (icode != CODE_FOR_nothing)
     454              :     {
     455            0 :       class expand_operand ops[2];
     456            0 :       create_output_operand (&ops[0], NULL_RTX, vmode);
     457            0 :       create_input_operand (&ops[1], op, GET_MODE (op));
     458            0 :       expand_insn (icode, 2, ops);
     459            0 :       return ops[0].value;
     460              :     }
     461              : 
     462            0 :   if (!GET_MODE_NUNITS (vmode).is_constant (&n))
     463              :     return NULL;
     464              : 
     465              :   /* ??? If the target doesn't have a vec_init, then we have no easy way
     466              :      of performing this operation.  Most of this sort of generic support
     467              :      is hidden away in the vector lowering support in gimple.  */
     468            0 :   icode = convert_optab_handler (vec_init_optab, vmode,
     469            0 :                                  GET_MODE_INNER (vmode));
     470            0 :   if (icode == CODE_FOR_nothing)
     471              :     return NULL;
     472              : 
     473            0 :   vec = rtvec_alloc (n);
     474            0 :   for (int i = 0; i < n; ++i)
     475            0 :     RTVEC_ELT (vec, i) = op;
     476            0 :   rtx ret = gen_reg_rtx (vmode);
     477            0 :   emit_insn (GEN_FCN (icode) (ret, gen_rtx_PARALLEL (vmode, vec)));
     478              : 
     479            0 :   return ret;
     480              : }
     481              : 
     482              : /* This subroutine of expand_doubleword_shift handles the cases in which
     483              :    the effective shift value is >= BITS_PER_WORD.  The arguments and return
     484              :    value are the same as for the parent routine, except that SUPERWORD_OP1
     485              :    is the shift count to use when shifting OUTOF_INPUT into INTO_TARGET.
     486              :    INTO_TARGET may be null if the caller has decided to calculate it.  */
     487              : 
     488              : static bool
     489            0 : expand_superword_shift (optab binoptab, rtx outof_input, rtx superword_op1,
     490              :                         rtx outof_target, rtx into_target,
     491              :                         int unsignedp, enum optab_methods methods)
     492              : {
     493            0 :   if (into_target != 0)
     494            0 :     if (!force_expand_binop (word_mode, binoptab, outof_input, superword_op1,
     495              :                              into_target, unsignedp, methods))
     496              :       return false;
     497              : 
     498            0 :   if (outof_target != 0)
     499              :     {
     500              :       /* For a signed right shift, we must fill OUTOF_TARGET with copies
     501              :          of the sign bit, otherwise we must fill it with zeros.  */
     502            0 :       if (binoptab != ashr_optab)
     503            0 :         emit_move_insn (outof_target, CONST0_RTX (word_mode));
     504              :       else
     505            0 :         if (!force_expand_binop (word_mode, binoptab, outof_input,
     506              :                                  gen_int_shift_amount (word_mode,
     507            0 :                                                        BITS_PER_WORD - 1),
     508              :                                  outof_target, unsignedp, methods))
     509              :           return false;
     510              :     }
     511              :   return true;
     512              : }
     513              : 
     514              : /* This subroutine of expand_doubleword_shift handles the cases in which
     515              :    the effective shift value is < BITS_PER_WORD.  The arguments and return
     516              :    value are the same as for the parent routine.  */
     517              : 
     518              : static bool
     519            0 : expand_subword_shift (scalar_int_mode op1_mode, optab binoptab,
     520              :                       rtx outof_input, rtx into_input, rtx op1,
     521              :                       rtx outof_target, rtx into_target,
     522              :                       int unsignedp, enum optab_methods methods,
     523              :                       unsigned HOST_WIDE_INT shift_mask)
     524              : {
     525            0 :   optab reverse_unsigned_shift, unsigned_shift;
     526            0 :   rtx tmp, carries;
     527              : 
     528            0 :   reverse_unsigned_shift = (binoptab == ashl_optab ? lshr_optab : ashl_optab);
     529            0 :   unsigned_shift = (binoptab == ashl_optab ? ashl_optab : lshr_optab);
     530              : 
     531              :   /* The low OP1 bits of INTO_TARGET come from the high bits of OUTOF_INPUT.
     532              :      We therefore need to shift OUTOF_INPUT by (BITS_PER_WORD - OP1) bits in
     533              :      the opposite direction to BINOPTAB.  */
     534            0 :   if (CONSTANT_P (op1) || shift_mask >= BITS_PER_WORD)
     535              :     {
     536            0 :       carries = outof_input;
     537            0 :       tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD,
     538              :                                             op1_mode), op1_mode);
     539            0 :       tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
     540              :                                    0, true, methods);
     541              :     }
     542              :   else
     543              :     {
     544              :       /* We must avoid shifting by BITS_PER_WORD bits since that is either
     545              :          the same as a zero shift (if shift_mask == BITS_PER_WORD - 1) or
     546              :          has unknown behavior.  Do a single shift first, then shift by the
     547              :          remainder.  It's OK to use ~OP1 as the remainder if shift counts
     548              :          are truncated to the mode size.  */
     549            0 :       carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
     550              :                                        outof_input, const1_rtx, 0,
     551              :                                        unsignedp, methods);
     552            0 :       if (carries == const0_rtx)
     553              :         tmp = const0_rtx;
     554            0 :       else if (shift_mask == BITS_PER_WORD - 1)
     555            0 :         tmp = expand_unop (op1_mode, one_cmpl_optab, op1, 0, true);
     556              :       else
     557              :         {
     558            0 :           tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD - 1,
     559              :                                                 op1_mode), op1_mode);
     560            0 :           tmp = simplify_expand_binop (op1_mode, sub_optab, tmp, op1,
     561              :                                        0, true, methods);
     562              :         }
     563              :     }
     564            0 :   if (tmp == 0 || carries == 0)
     565              :     return false;
     566            0 :   if (carries != const0_rtx && tmp != const0_rtx)
     567            0 :     carries = simplify_expand_binop (word_mode, reverse_unsigned_shift,
     568              :                                      carries, tmp, 0, unsignedp, methods);
     569            0 :   if (carries == 0)
     570              :     return false;
     571              : 
     572            0 :   if (into_input != const0_rtx)
     573              :     {
     574              :       /* Shift INTO_INPUT logically by OP1.  This is the last use of
     575              :          INTO_INPUT so the result can go directly into INTO_TARGET if
     576              :          convenient.  */
     577            0 :       tmp = simplify_expand_binop (word_mode, unsigned_shift, into_input,
     578              :                                    op1, into_target, unsignedp, methods);
     579            0 :       if (tmp == 0)
     580              :         return false;
     581              : 
     582              :       /* Now OR/PLUS in the bits carried over from OUTOF_INPUT.  */
     583            0 :       if (!force_expand_binop (word_mode, add_optab, tmp, carries,
     584              :                                into_target, unsignedp, methods))
     585              :         return false;
     586              :     }
     587              :   else
     588            0 :     emit_move_insn (into_target, carries);
     589              : 
     590              :   /* Use a standard word_mode shift for the out-of half.  */
     591            0 :   if (outof_target != 0)
     592            0 :     if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
     593              :                              outof_target, unsignedp, methods))
     594              :       return false;
     595              : 
     596              :   return true;
     597              : }
     598              : 
     599              : 
     600              : /* Try implementing expand_doubleword_shift using conditional moves.
     601              :    The shift is by < BITS_PER_WORD if (CMP_CODE CMP1 CMP2) is true,
     602              :    otherwise it is by >= BITS_PER_WORD.  SUBWORD_OP1 and SUPERWORD_OP1
     603              :    are the shift counts to use in the former and latter case.  All other
     604              :    arguments are the same as the parent routine.  */
     605              : 
     606              : static bool
     607            0 : expand_doubleword_shift_condmove (scalar_int_mode op1_mode, optab binoptab,
     608              :                                   enum rtx_code cmp_code, rtx cmp1, rtx cmp2,
     609              :                                   rtx outof_input, rtx into_input,
     610              :                                   rtx subword_op1, rtx superword_op1,
     611              :                                   rtx outof_target, rtx into_target,
     612              :                                   int unsignedp, enum optab_methods methods,
     613              :                                   unsigned HOST_WIDE_INT shift_mask)
     614              : {
     615            0 :   rtx outof_superword, into_superword;
     616              : 
     617              :   /* Put the superword version of the output into OUTOF_SUPERWORD and
     618              :      INTO_SUPERWORD.  */
     619            0 :   outof_superword = outof_target != 0 ? gen_reg_rtx (word_mode) : 0;
     620            0 :   if (outof_target != 0 && subword_op1 == superword_op1)
     621              :     {
     622              :       /* The value INTO_TARGET >> SUBWORD_OP1, which we later store in
     623              :          OUTOF_TARGET, is the same as the value of INTO_SUPERWORD.  */
     624            0 :       into_superword = outof_target;
     625            0 :       if (!expand_superword_shift (binoptab, outof_input, superword_op1,
     626              :                                    outof_superword, 0, unsignedp, methods))
     627              :         return false;
     628              :     }
     629              :   else
     630              :     {
     631            0 :       into_superword = gen_reg_rtx (word_mode);
     632            0 :       if (!expand_superword_shift (binoptab, outof_input, superword_op1,
     633              :                                    outof_superword, into_superword,
     634              :                                    unsignedp, methods))
     635              :         return false;
     636              :     }
     637              : 
     638              :   /* Put the subword version directly in OUTOF_TARGET and INTO_TARGET.  */
     639            0 :   if (!expand_subword_shift (op1_mode, binoptab,
     640              :                              outof_input, into_input, subword_op1,
     641              :                              outof_target, into_target,
     642              :                              unsignedp, methods, shift_mask))
     643              :     return false;
     644              : 
     645              :   /* Select between them.  Do the INTO half first because INTO_SUPERWORD
     646              :      might be the current value of OUTOF_TARGET.  */
     647            0 :   if (!emit_conditional_move (into_target, { cmp_code, cmp1, cmp2, op1_mode },
     648              :                               into_target, into_superword, word_mode, false))
     649              :     return false;
     650              : 
     651            0 :   if (outof_target != 0)
     652            0 :     if (!emit_conditional_move (outof_target,
     653            0 :                                 { cmp_code, cmp1, cmp2, op1_mode },
     654              :                                 outof_target, outof_superword,
     655              :                                 word_mode, false))
     656            0 :       return false;
     657              : 
     658              :   return true;
     659              : }
     660              : 
     661              : /* Expand a doubleword shift (ashl, ashr or lshr) using word-mode shifts.
     662              :    OUTOF_INPUT and INTO_INPUT are the two word-sized halves of the first
     663              :    input operand; the shift moves bits in the direction OUTOF_INPUT->
     664              :    INTO_TARGET.  OUTOF_TARGET and INTO_TARGET are the equivalent words
     665              :    of the target.  OP1 is the shift count and OP1_MODE is its mode.
     666              :    If OP1 is constant, it will have been truncated as appropriate
     667              :    and is known to be nonzero.
     668              : 
     669              :    If SHIFT_MASK is zero, the result of word shifts is undefined when the
     670              :    shift count is outside the range [0, BITS_PER_WORD).  This routine must
     671              :    avoid generating such shifts for OP1s in the range [0, BITS_PER_WORD * 2).
     672              : 
     673              :    If SHIFT_MASK is nonzero, all word-mode shift counts are effectively
     674              :    masked by it and shifts in the range [BITS_PER_WORD, SHIFT_MASK) will
     675              :    fill with zeros or sign bits as appropriate.
     676              : 
     677              :    If SHIFT_MASK is BITS_PER_WORD - 1, this routine will synthesize
     678              :    a doubleword shift whose equivalent mask is BITS_PER_WORD * 2 - 1.
     679              :    Doing this preserves semantics required by SHIFT_COUNT_TRUNCATED.
     680              :    In all other cases, shifts by values outside [0, BITS_PER_UNIT * 2)
     681              :    are undefined.
     682              : 
     683              :    BINOPTAB, UNSIGNEDP and METHODS are as for expand_binop.  This function
     684              :    may not use INTO_INPUT after modifying INTO_TARGET, and similarly for
     685              :    OUTOF_INPUT and OUTOF_TARGET.  OUTOF_TARGET can be null if the parent
     686              :    function wants to calculate it itself.
     687              : 
     688              :    Return true if the shift could be successfully synthesized.  */
     689              : 
     690              : static bool
     691            0 : expand_doubleword_shift (scalar_int_mode op1_mode, optab binoptab,
     692              :                          rtx outof_input, rtx into_input, rtx op1,
     693              :                          rtx outof_target, rtx into_target,
     694              :                          int unsignedp, enum optab_methods methods,
     695              :                          unsigned HOST_WIDE_INT shift_mask)
     696              : {
     697            0 :   rtx superword_op1, tmp, cmp1, cmp2;
     698            0 :   enum rtx_code cmp_code;
     699              : 
     700              :   /* See if word-mode shifts by BITS_PER_WORD...BITS_PER_WORD * 2 - 1 will
     701              :      fill the result with sign or zero bits as appropriate.  If so, the value
     702              :      of OUTOF_TARGET will always be (SHIFT OUTOF_INPUT OP1).   Recursively call
     703              :      this routine to calculate INTO_TARGET (which depends on both OUTOF_INPUT
     704              :      and INTO_INPUT), then emit code to set up OUTOF_TARGET.
     705              : 
     706              :      This isn't worthwhile for constant shifts since the optimizers will
     707              :      cope better with in-range shift counts.  */
     708            0 :   if (shift_mask >= BITS_PER_WORD
     709            0 :       && outof_target != 0
     710            0 :       && !CONSTANT_P (op1))
     711              :     {
     712            0 :       if (!expand_doubleword_shift (op1_mode, binoptab,
     713              :                                     outof_input, into_input, op1,
     714              :                                     0, into_target,
     715              :                                     unsignedp, methods, shift_mask))
     716              :         return false;
     717            0 :       if (!force_expand_binop (word_mode, binoptab, outof_input, op1,
     718              :                                outof_target, unsignedp, methods))
     719              :         return false;
     720              :       return true;
     721              :     }
     722              : 
     723              :   /* Set CMP_CODE, CMP1 and CMP2 so that the rtx (CMP_CODE CMP1 CMP2)
     724              :      is true when the effective shift value is less than BITS_PER_WORD.
     725              :      Set SUPERWORD_OP1 to the shift count that should be used to shift
     726              :      OUTOF_INPUT into INTO_TARGET when the condition is false.  */
     727            0 :   tmp = immed_wide_int_const (wi::shwi (BITS_PER_WORD, op1_mode), op1_mode);
     728            0 :   if (!CONSTANT_P (op1) && shift_mask == BITS_PER_WORD - 1)
     729              :     {
     730              :       /* Set CMP1 to OP1 & BITS_PER_WORD.  The result is zero iff OP1
     731              :          is a subword shift count.  */
     732            0 :       cmp1 = simplify_expand_binop (op1_mode, and_optab, op1, tmp,
     733              :                                     0, true, methods);
     734            0 :       cmp2 = CONST0_RTX (op1_mode);
     735            0 :       cmp_code = EQ;
     736            0 :       superword_op1 = op1;
     737              :     }
     738              :   else
     739              :     {
     740              :       /* Set CMP1 to OP1 - BITS_PER_WORD.  */
     741            0 :       cmp1 = simplify_expand_binop (op1_mode, sub_optab, op1, tmp,
     742              :                                     0, true, methods);
     743            0 :       cmp2 = CONST0_RTX (op1_mode);
     744            0 :       cmp_code = LT;
     745            0 :       superword_op1 = cmp1;
     746              :     }
     747            0 :   if (cmp1 == 0)
     748              :     return false;
     749              : 
     750              :   /* If we can compute the condition at compile time, pick the
     751              :      appropriate subroutine.  */
     752            0 :   tmp = simplify_relational_operation (cmp_code, SImode, op1_mode, cmp1, cmp2);
     753            0 :   if (tmp != 0 && CONST_INT_P (tmp))
     754              :     {
     755            0 :       if (tmp == const0_rtx)
     756            0 :         return expand_superword_shift (binoptab, outof_input, superword_op1,
     757              :                                        outof_target, into_target,
     758            0 :                                        unsignedp, methods);
     759              :       else
     760            0 :         return expand_subword_shift (op1_mode, binoptab,
     761              :                                      outof_input, into_input, op1,
     762              :                                      outof_target, into_target,
     763            0 :                                      unsignedp, methods, shift_mask);
     764              :     }
     765              : 
     766              :   /* Try using conditional moves to generate straight-line code.  */
     767            0 :   if (HAVE_conditional_move)
     768              :     {
     769            0 :       rtx_insn *start = get_last_insn ();
     770            0 :       if (expand_doubleword_shift_condmove (op1_mode, binoptab,
     771              :                                             cmp_code, cmp1, cmp2,
     772              :                                             outof_input, into_input,
     773              :                                             op1, superword_op1,
     774              :                                             outof_target, into_target,
     775              :                                             unsignedp, methods, shift_mask))
     776              :         return true;
     777            0 :       delete_insns_since (start);
     778              :     }
     779              : 
     780              :   /* As a last resort, use branches to select the correct alternative.  */
     781            0 :   rtx_code_label *subword_label = gen_label_rtx ();
     782            0 :   rtx_code_label *done_label = gen_label_rtx ();
     783              : 
     784            0 :   NO_DEFER_POP;
     785            0 :   do_compare_rtx_and_jump (cmp1, cmp2, cmp_code, false, op1_mode,
     786              :                            0, 0, subword_label,
     787              :                            profile_probability::uninitialized ());
     788            0 :   OK_DEFER_POP;
     789              : 
     790            0 :   if (!expand_superword_shift (binoptab, outof_input, superword_op1,
     791              :                                outof_target, into_target,
     792              :                                unsignedp, methods))
     793              :     return false;
     794              : 
     795            0 :   emit_jump_insn (targetm.gen_jump (done_label));
     796            0 :   emit_barrier ();
     797            0 :   emit_label (subword_label);
     798              : 
     799            0 :   if (!expand_subword_shift (op1_mode, binoptab,
     800              :                              outof_input, into_input, op1,
     801              :                              outof_target, into_target,
     802              :                              unsignedp, methods, shift_mask))
     803              :     return false;
     804              : 
     805            0 :   emit_label (done_label);
     806            0 :   return true;
     807              : }
     808              : 
     809              : /* Subroutine of expand_binop.  Perform a double word multiplication of
     810              :    operands OP0 and OP1 both of mode MODE, which is exactly twice as wide
     811              :    as the target's word_mode.  This function return NULL_RTX if anything
     812              :    goes wrong, in which case it may have already emitted instructions
     813              :    which need to be deleted.
     814              : 
     815              :    If we want to multiply two two-word values and have normal and widening
     816              :    multiplies of single-word values, we can do this with three smaller
     817              :    multiplications.
     818              : 
     819              :    The multiplication proceeds as follows:
     820              :                                  _______________________
     821              :                                 [__op0_high_|__op0_low__]
     822              :                                  _______________________
     823              :         *                       [__op1_high_|__op1_low__]
     824              :         _______________________________________________
     825              :                                  _______________________
     826              :     (1)                         [__op0_low__*__op1_low__]
     827              :                      _______________________
     828              :     (2a)            [__op0_low__*__op1_high_]
     829              :                      _______________________
     830              :     (2b)            [__op0_high_*__op1_low__]
     831              :          _______________________
     832              :     (3) [__op0_high_*__op1_high_]
     833              : 
     834              : 
     835              :   This gives a 4-word result.  Since we are only interested in the
     836              :   lower 2 words, partial result (3) and the upper words of (2a) and
     837              :   (2b) don't need to be calculated.  Hence (2a) and (2b) can be
     838              :   calculated using non-widening multiplication.
     839              : 
     840              :   (1), however, needs to be calculated with an unsigned widening
     841              :   multiplication.  If this operation is not directly supported we
     842              :   try using a signed widening multiplication and adjust the result.
     843              :   This adjustment works as follows:
     844              : 
     845              :       If both operands are positive then no adjustment is needed.
     846              : 
     847              :       If the operands have different signs, for example op0_low < 0 and
     848              :       op1_low >= 0, the instruction treats the most significant bit of
     849              :       op0_low as a sign bit instead of a bit with significance
     850              :       2**(BITS_PER_WORD-1), i.e. the instruction multiplies op1_low
     851              :       with 2**BITS_PER_WORD - op0_low, and two's complements the
     852              :       result.  Conclusion: We need to add op1_low * 2**BITS_PER_WORD to
     853              :       the result.
     854              : 
     855              :       Similarly, if both operands are negative, we need to add
     856              :       (op0_low + op1_low) * 2**BITS_PER_WORD.
     857              : 
     858              :       We use a trick to adjust quickly.  We logically shift op0_low right
     859              :       (op1_low) BITS_PER_WORD-1 steps to get 0 or 1, and add this to
     860              :       op0_high (op1_high) before it is used to calculate 2b (2a).  If no
     861              :       logical shift exists, we do an arithmetic right shift and subtract
     862              :       the 0 or -1.  */
     863              : 
     864              : static rtx
     865        15648 : expand_doubleword_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
     866              :                        bool umulp, enum optab_methods methods)
     867              : {
     868        15648 :   int low = (WORDS_BIG_ENDIAN ? 1 : 0);
     869        15648 :   int high = (WORDS_BIG_ENDIAN ? 0 : 1);
     870        15648 :   rtx wordm1 = (umulp ? NULL_RTX
     871            0 :                 : gen_int_shift_amount (word_mode, BITS_PER_WORD - 1));
     872        15648 :   rtx product, adjust, product_high, temp;
     873              : 
     874        15648 :   rtx op0_high = operand_subword_force (op0, high, mode);
     875        15648 :   rtx op0_low = operand_subword_force (op0, low, mode);
     876        15648 :   rtx op1_high = operand_subword_force (op1, high, mode);
     877        15648 :   rtx op1_low = operand_subword_force (op1, low, mode);
     878              : 
     879              :   /* If we're using an unsigned multiply to directly compute the product
     880              :      of the low-order words of the operands and perform any required
     881              :      adjustments of the operands, we begin by trying two more multiplications
     882              :      and then computing the appropriate sum.
     883              : 
     884              :      We have checked above that the required addition is provided.
     885              :      Full-word addition will normally always succeed, especially if
     886              :      it is provided at all, so we don't worry about its failure.  The
     887              :      multiplication may well fail, however, so we do handle that.  */
     888              : 
     889        15648 :   if (!umulp)
     890              :     {
     891              :       /* ??? This could be done with emit_store_flag where available.  */
     892            0 :       temp = expand_binop (word_mode, lshr_optab, op0_low, wordm1,
     893              :                            NULL_RTX, 1, methods);
     894            0 :       if (temp)
     895            0 :         op0_high = expand_binop (word_mode, add_optab, op0_high, temp,
     896              :                                  NULL_RTX, 0, OPTAB_DIRECT);
     897              :       else
     898              :         {
     899            0 :           temp = expand_binop (word_mode, ashr_optab, op0_low, wordm1,
     900              :                                NULL_RTX, 0, methods);
     901            0 :           if (!temp)
     902              :             return NULL_RTX;
     903            0 :           op0_high = expand_binop (word_mode, sub_optab, op0_high, temp,
     904              :                                    NULL_RTX, 0, OPTAB_DIRECT);
     905              :         }
     906              : 
     907            0 :       if (!op0_high)
     908              :         return NULL_RTX;
     909              :     }
     910              : 
     911        15648 :   if (op1_low == const1_rtx)
     912              :     adjust = op0_high;
     913        15608 :   else if (op1_low == const0_rtx)
     914              :     adjust = const0_rtx;
     915        15533 :   else if (op1_low == const2_rtx)
     916            0 :     adjust = expand_binop (word_mode, add_optab, op0_high, op0_high,
     917              :                            NULL_RTX, 0, OPTAB_DIRECT);
     918              :   else
     919        15533 :     adjust = expand_binop (word_mode, smul_optab, op0_high, op1_low,
     920              :                            NULL_RTX, 0, OPTAB_DIRECT);
     921        15648 :   if (!adjust)
     922              :     return NULL_RTX;
     923              : 
     924              :   /* OP0_HIGH should now be dead.  */
     925              : 
     926        15648 :   if (!umulp)
     927              :     {
     928              :       /* ??? This could be done with emit_store_flag where available.  */
     929            0 :       temp = expand_binop (word_mode, lshr_optab, op1_low, wordm1,
     930              :                            NULL_RTX, 1, methods);
     931            0 :       if (temp)
     932            0 :         op1_high = expand_binop (word_mode, add_optab, op1_high, temp,
     933              :                                  NULL_RTX, 0, OPTAB_DIRECT);
     934              :       else
     935              :         {
     936            0 :           temp = expand_binop (word_mode, ashr_optab, op1_low, wordm1,
     937              :                                NULL_RTX, 0, methods);
     938            0 :           if (!temp)
     939              :             return NULL_RTX;
     940            0 :           op1_high = expand_binop (word_mode, sub_optab, op1_high, temp,
     941              :                                    NULL_RTX, 0, OPTAB_DIRECT);
     942              :         }
     943              : 
     944            0 :       if (!op1_high)
     945              :         return NULL_RTX;
     946              :     }
     947              : 
     948        15648 :   if (op1_high == const1_rtx)
     949              :     temp = op0_low;
     950        15647 :   else if (op1_high == const0_rtx)
     951              :     temp = const0_rtx;
     952        14831 :   else if (op1_high == const2_rtx)
     953            0 :     temp = expand_binop (word_mode, add_optab, op0_low, op0_low,
     954              :                          NULL_RTX, 0, OPTAB_DIRECT);
     955              :   else
     956        14831 :     temp = expand_binop (word_mode, smul_optab, op0_low, op1_high,
     957              :                          NULL_RTX, 0, OPTAB_DIRECT);
     958        15648 :   if (!temp)
     959              :     return NULL_RTX;
     960              : 
     961              :   /* OP1_HIGH should now be dead.  */
     962              : 
     963        15648 :   adjust = expand_binop (word_mode, add_optab, adjust, temp,
     964              :                          NULL_RTX, 0, OPTAB_DIRECT);
     965              : 
     966        15648 :   if (target && !REG_P (target))
     967         9185 :     target = NULL_RTX;
     968              : 
     969              :   /* *_widen_optab needs to determine operand mode, make sure at least
     970              :      one operand has non-VOID mode.  */
     971        15648 :   if (GET_MODE (op0_low) == VOIDmode && GET_MODE (op1_low) == VOIDmode)
     972            0 :     op0_low = force_reg (word_mode, op0_low);
     973              : 
     974        15648 :   if (op1_low == const1_rtx)
     975           40 :     product = convert_modes (mode, word_mode, op0_low, umulp);
     976        15608 :   else if (umulp)
     977        15608 :     product = expand_binop (mode, umul_widen_optab, op0_low, op1_low,
     978              :                             target, 1, OPTAB_DIRECT);
     979              :   else
     980            0 :     product = expand_binop (mode, smul_widen_optab, op0_low, op1_low,
     981              :                             target, 1, OPTAB_DIRECT);
     982              : 
     983        15648 :   if (!product)
     984              :     return NULL_RTX;
     985              : 
     986        15648 :   product_high = operand_subword (product, high, 1, mode);
     987        15648 :   adjust = expand_binop (word_mode, add_optab, product_high, adjust,
     988              :                          NULL_RTX, 0, OPTAB_DIRECT);
     989        15648 :   emit_move_insn (product_high, adjust);
     990        15648 :   return product;
     991              : }
     992              : 
     993              : /* Subroutine of expand_binop.  Optimize unsigned double-word OP0 % OP1 for
     994              :    constant OP1.  If for some bit in [BITS_PER_WORD / 2, BITS_PER_WORD] range
     995              :    (prefer higher bits) ((1w << bit) % OP1) == 1, then the modulo can be
     996              :    computed in word-mode as ((OP0 & (bit - 1)) + ((OP0 >> bit) & (bit - 1))
     997              :    + (OP0 >> (2 * bit))) % OP1.  Whether we need to sum 2, 3 or 4 values
     998              :    depends on the bit value, if 2, then carry from the addition needs to be
     999              :    added too, i.e. like:
    1000              :    sum += __builtin_add_overflow (low, high, &sum)
    1001              : 
    1002              :    Optimize signed double-word OP0 % OP1 similarly, just apply some correction
    1003              :    factor to the sum before doing unsigned remainder, in the form of
    1004              :    sum += (((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & const);
    1005              :    then perform unsigned
    1006              :    remainder = sum % OP1;
    1007              :    and finally
    1008              :    remainder += ((signed) OP0 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1);  */
    1009              : 
    1010              : static rtx
    1011        21092 : expand_doubleword_mod (machine_mode mode, rtx op0, rtx op1, bool unsignedp)
    1012              : {
    1013        21092 :   if (INTVAL (op1) <= 1 || (INTVAL (op1) & 1) == 0)
    1014              :     return NULL_RTX;
    1015              : 
    1016        20696 :   rtx_insn *last = get_last_insn ();
    1017       682487 :   for (int bit = BITS_PER_WORD; bit >= BITS_PER_WORD / 2; bit--)
    1018              :     {
    1019       661041 :       wide_int w = wi::shifted_mask (bit, 1, false, 2 * BITS_PER_WORD);
    1020       629400 :       if (wi::ne_p (wi::umod_trunc (w, INTVAL (op1)), 1))
    1021       628082 :         continue;
    1022         1318 :       rtx sum = NULL_RTX, mask = NULL_RTX;
    1023         1802 :       if (bit == BITS_PER_WORD)
    1024              :         {
    1025              :           /* For signed modulo we need to add correction to the sum
    1026              :              and that might again overflow.  */
    1027          356 :           if (!unsignedp)
    1028          143 :             continue;
    1029          213 :           if (optab_handler (uaddv4_optab, word_mode) == CODE_FOR_nothing)
    1030            0 :             continue;
    1031          213 :           tree wtype = lang_hooks.types.type_for_mode (word_mode, 1);
    1032          213 :           if (wtype == NULL_TREE)
    1033            0 :             continue;
    1034          213 :           tree ctype = build_complex_type (wtype);
    1035          213 :           if (TYPE_MODE (ctype) != GET_MODE_COMPLEX_MODE (word_mode))
    1036            0 :             continue;
    1037          213 :           machine_mode cmode = TYPE_MODE (ctype);
    1038          213 :           rtx op00 = operand_subword_force (op0, 0, mode);
    1039          213 :           rtx op01 = operand_subword_force (op0, 1, mode);
    1040          213 :           rtx cres = gen_rtx_CONCAT (cmode, gen_reg_rtx (word_mode),
    1041              :                                      gen_reg_rtx (word_mode));
    1042          213 :           tree lhs = make_tree (ctype, cres);
    1043          213 :           tree arg0 = make_tree (wtype, op00);
    1044          213 :           tree arg1 = make_tree (wtype, op01);
    1045          213 :           expand_addsub_overflow (UNKNOWN_LOCATION, PLUS_EXPR, lhs, arg0,
    1046              :                                   arg1, true, true, true, false, NULL);
    1047          213 :           sum = expand_simple_binop (word_mode, PLUS, XEXP (cres, 0),
    1048              :                                      XEXP (cres, 1), NULL_RTX, 1,
    1049              :                                      OPTAB_DIRECT);
    1050          213 :           if (sum == NULL_RTX)
    1051              :             return NULL_RTX;
    1052              :         }
    1053              :       else
    1054              :         {
    1055              :           /* Code below uses GEN_INT, so we need the masks to be representable
    1056              :              in HOST_WIDE_INTs.  */
    1057          962 :           if (bit >= HOST_BITS_PER_WIDE_INT)
    1058            0 :             continue;
    1059              :           /* If op0 is e.g. -1 or -2 unsigned, then the 2 additions might
    1060              :              overflow.  Consider 64-bit -1ULL for word size 32, if we add
    1061              :              0x7fffffffU + 0x7fffffffU + 3U, it wraps around to 1.  */
    1062         1214 :           if (bit == BITS_PER_WORD - 1)
    1063          150 :             continue;
    1064              : 
    1065          812 :           int count = (2 * BITS_PER_WORD + bit - 1) / bit;
    1066          812 :           rtx sum_corr = NULL_RTX;
    1067              : 
    1068          812 :           if (!unsignedp)
    1069              :             {
    1070              :               /* For signed modulo, compute it as unsigned modulo of
    1071              :                  sum with a correction added to it if OP0 is negative,
    1072              :                  such that the result can be computed as unsigned
    1073              :                  remainder + ((OP1 >> (2 * BITS_PER_WORD - 1)) & (1 - OP1).  */
    1074          537 :               w = wi::min_value (2 * BITS_PER_WORD, SIGNED);
    1075          427 :               wide_int wmod1 = wi::umod_trunc (w, INTVAL (op1));
    1076          427 :               wide_int wmod2 = wi::smod_trunc (w, INTVAL (op1));
    1077              :               /* wmod2 == -wmod1.  */
    1078          427 :               wmod2 = wmod2 + (INTVAL (op1) - 1);
    1079          427 :               if (wi::ne_p (wmod1, wmod2))
    1080              :                 {
    1081          427 :                   wide_int wcorr = wmod2 - wmod1;
    1082          427 :                   if (wi::neg_p (w))
    1083          427 :                     wcorr = wcorr + INTVAL (op1);
    1084              :                   /* Now verify if the count sums can't overflow, and punt
    1085              :                      if they could.  */
    1086          537 :                   w = wi::mask (bit, false, 2 * BITS_PER_WORD);
    1087          427 :                   w = w * (count - 1);
    1088          964 :                   w = w + wi::mask (2 * BITS_PER_WORD - (count - 1) * bit,
    1089          854 :                                     false, 2 * BITS_PER_WORD);
    1090          427 :                   w = w + wcorr;
    1091          537 :                   w = wi::lrshift (w, BITS_PER_WORD);
    1092          427 :                   if (wi::ne_p (w, 0))
    1093            0 :                     continue;
    1094              : 
    1095          427 :                   mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
    1096              :                                                 mode);
    1097          427 :                   mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
    1098          427 :                                               GEN_INT (BITS_PER_WORD - 1),
    1099              :                                               NULL_RTX, 0, OPTAB_DIRECT);
    1100          427 :                   if (mask == NULL_RTX)
    1101              :                     return NULL_RTX;
    1102          427 :                   sum_corr = immed_wide_int_const (wcorr, word_mode);
    1103          427 :                   sum_corr = expand_simple_binop (word_mode, AND, mask,
    1104              :                                                   sum_corr, NULL_RTX, 1,
    1105              :                                                   OPTAB_DIRECT);
    1106          427 :                   if (sum_corr == NULL_RTX)
    1107              :                     return NULL_RTX;
    1108          427 :                 }
    1109          427 :             }
    1110              : 
    1111         3441 :           for (int i = 0; i < count; i++)
    1112              :             {
    1113         2629 :               rtx v = op0;
    1114         2629 :               if (i)
    1115         1817 :                 v = expand_simple_binop (mode, LSHIFTRT, v, GEN_INT (i * bit),
    1116              :                                          NULL_RTX, 1, OPTAB_DIRECT);
    1117         2629 :               if (v == NULL_RTX)
    1118              :                 return NULL_RTX;
    1119         2629 :               v = force_lowpart_subreg (word_mode, v, mode);
    1120         2629 :               if (v == NULL_RTX)
    1121              :                 return NULL_RTX;
    1122         2629 :               if (i != count - 1)
    1123         1817 :                 v = expand_simple_binop (word_mode, AND, v,
    1124         1817 :                                          GEN_INT ((HOST_WIDE_INT_1U << bit)
    1125              :                                                   - 1), NULL_RTX, 1,
    1126              :                                          OPTAB_DIRECT);
    1127         2629 :               if (v == NULL_RTX)
    1128              :                 return NULL_RTX;
    1129         2629 :               if (sum == NULL_RTX)
    1130              :                 sum = v;
    1131              :               else
    1132         1817 :                 sum = expand_simple_binop (word_mode, PLUS, sum, v, NULL_RTX,
    1133              :                                            1, OPTAB_DIRECT);
    1134         2629 :               if (sum == NULL_RTX)
    1135              :                 return NULL_RTX;
    1136              :             }
    1137          812 :           if (sum_corr)
    1138              :             {
    1139          427 :               sum = expand_simple_binop (word_mode, PLUS, sum, sum_corr,
    1140              :                                          NULL_RTX, 1, OPTAB_DIRECT);
    1141          427 :               if (sum == NULL_RTX)
    1142              :                 return NULL_RTX;
    1143              :             }
    1144              :         }
    1145         1025 :       rtx remainder = expand_divmod (1, TRUNC_MOD_EXPR, word_mode, sum,
    1146         1025 :                                      gen_int_mode (INTVAL (op1), word_mode),
    1147              :                                      NULL_RTX, 1, OPTAB_DIRECT);
    1148         1025 :       if (remainder == NULL_RTX)
    1149              :         return NULL_RTX;
    1150              : 
    1151         1025 :       if (!unsignedp)
    1152              :         {
    1153          427 :           if (mask == NULL_RTX)
    1154              :             {
    1155            0 :               mask = operand_subword_force (op0, WORDS_BIG_ENDIAN ? 0 : 1,
    1156              :                                             mode);
    1157            0 :               mask = expand_simple_binop (word_mode, ASHIFTRT, mask,
    1158            0 :                                           GEN_INT (BITS_PER_WORD - 1),
    1159              :                                           NULL_RTX, 0, OPTAB_DIRECT);
    1160            0 :               if (mask == NULL_RTX)
    1161              :                 return NULL_RTX;
    1162              :             }
    1163          427 :           mask = expand_simple_binop (word_mode, AND, mask,
    1164          427 :                                       gen_int_mode (1 - INTVAL (op1),
    1165              :                                                     word_mode),
    1166              :                                       NULL_RTX, 1, OPTAB_DIRECT);
    1167          427 :           if (mask == NULL_RTX)
    1168              :             return NULL_RTX;
    1169          427 :           remainder = expand_simple_binop (word_mode, PLUS, remainder,
    1170              :                                            mask, NULL_RTX, 1, OPTAB_DIRECT);
    1171          427 :           if (remainder == NULL_RTX)
    1172              :             return NULL_RTX;
    1173              :         }
    1174              : 
    1175         1025 :       remainder = convert_modes (mode, word_mode, remainder, unsignedp);
    1176              :       /* Punt if we need any library calls.  */
    1177         1025 :       if (last)
    1178          893 :         last = NEXT_INSN (last);
    1179              :       else
    1180          132 :         last = get_insns ();
    1181        21123 :       for (; last; last = NEXT_INSN (last))
    1182        20098 :         if (CALL_P (last))
    1183              :           return NULL_RTX;
    1184              :       return remainder;
    1185       629400 :     }
    1186              :   return NULL_RTX;
    1187              : }
    1188              : 
    1189              : /* Similarly to the above function, but compute both quotient and remainder.
    1190              :    Quotient can be computed from the remainder as:
    1191              :    rem = op0 % op1;  // Handled using expand_doubleword_mod
    1192              :    quot = (op0 - rem) * inv; // inv is multiplicative inverse of op1 modulo
    1193              :                              // 2 * BITS_PER_WORD
    1194              : 
    1195              :    We can also handle cases where op1 is a multiple of power of two constant
    1196              :    and constant handled by expand_doubleword_mod.
    1197              :    op11 = 1 << __builtin_ctz (op1);
    1198              :    op12 = op1 / op11;
    1199              :    rem1 = op0 % op12;  // Handled using expand_doubleword_mod
    1200              :    quot1 = (op0 - rem1) * inv; // inv is multiplicative inverse of op12 modulo
    1201              :                                // 2 * BITS_PER_WORD
    1202              :    rem = (quot1 % op11) * op12 + rem1;
    1203              :    quot = quot1 / op11;  */
    1204              : 
    1205              : rtx
    1206        20745 : expand_doubleword_divmod (machine_mode mode, rtx op0, rtx op1, rtx *rem,
    1207              :                           bool unsignedp)
    1208              : {
    1209        20745 :   *rem = NULL_RTX;
    1210              : 
    1211              :   /* Negative dividend should have been optimized into positive,
    1212              :      similarly modulo by 1 and modulo by power of two is optimized
    1213              :      differently too.  */
    1214        20745 :   if (INTVAL (op1) <= 1 || pow2p_hwi (INTVAL (op1)))
    1215              :     return NULL_RTX;
    1216              : 
    1217        20696 :   rtx op11 = const1_rtx;
    1218        20696 :   rtx op12 = op1;
    1219        20696 :   if ((INTVAL (op1) & 1) == 0)
    1220              :     {
    1221         3700 :       int bit = ctz_hwi (INTVAL (op1));
    1222         3700 :       op11 = GEN_INT (HOST_WIDE_INT_1 << bit);
    1223         3700 :       op12 = GEN_INT (INTVAL (op1) >> bit);
    1224              :     }
    1225              : 
    1226        20696 :   rtx rem1 = expand_doubleword_mod (mode, op0, op12, unsignedp);
    1227        20696 :   if (rem1 == NULL_RTX)
    1228              :     return NULL_RTX;
    1229              : 
    1230         1025 :   int prec = 2 * BITS_PER_WORD;
    1231         1025 :   wide_int a = wide_int::from (INTVAL (op12), prec + 1, UNSIGNED);
    1232         1025 :   wide_int b = wi::shifted_mask (prec, 1, false, prec + 1);
    1233         1025 :   wide_int m = wide_int::from (wi::mod_inv (a, b), prec, UNSIGNED);
    1234         1025 :   rtx inv = immed_wide_int_const (m, mode);
    1235              : 
    1236         1025 :   rtx_insn *last = get_last_insn ();
    1237         1025 :   rtx quot1 = expand_simple_binop (mode, MINUS, op0, rem1,
    1238              :                                    NULL_RTX, unsignedp, OPTAB_DIRECT);
    1239         1025 :   if (quot1 == NULL_RTX)
    1240              :     return NULL_RTX;
    1241              : 
    1242         1025 :   quot1 = expand_simple_binop (mode, MULT, quot1, inv,
    1243              :                                NULL_RTX, unsignedp, OPTAB_DIRECT);
    1244         1025 :   if (quot1 == NULL_RTX)
    1245              :     return NULL_RTX;
    1246              : 
    1247         1025 :   if (op11 != const1_rtx)
    1248              :     {
    1249          384 :       rtx rem2 = expand_divmod (1, TRUNC_MOD_EXPR, mode, quot1, op11,
    1250              :                                 NULL_RTX, unsignedp, OPTAB_DIRECT);
    1251          384 :       if (rem2 == NULL_RTX)
    1252              :         return NULL_RTX;
    1253              : 
    1254          384 :       rem2 = expand_simple_binop (mode, MULT, rem2, op12, NULL_RTX,
    1255              :                                   unsignedp, OPTAB_DIRECT);
    1256          384 :       if (rem2 == NULL_RTX)
    1257              :         return NULL_RTX;
    1258              : 
    1259          384 :       rem2 = expand_simple_binop (mode, PLUS, rem2, rem1, NULL_RTX,
    1260              :                                   unsignedp, OPTAB_DIRECT);
    1261          384 :       if (rem2 == NULL_RTX)
    1262              :         return NULL_RTX;
    1263              : 
    1264          384 :       rtx quot2 = expand_divmod (0, TRUNC_DIV_EXPR, mode, quot1, op11,
    1265              :                                  NULL_RTX, unsignedp, OPTAB_DIRECT);
    1266          384 :       if (quot2 == NULL_RTX)
    1267              :         return NULL_RTX;
    1268              : 
    1269              :       rem1 = rem2;
    1270              :       quot1 = quot2;
    1271              :     }
    1272              : 
    1273              :   /* Punt if we need any library calls.  */
    1274         1025 :   if (last)
    1275         1025 :     last = NEXT_INSN (last);
    1276              :   else
    1277            0 :     last = get_insns ();
    1278        16316 :   for (; last; last = NEXT_INSN (last))
    1279        15291 :     if (CALL_P (last))
    1280              :       return NULL_RTX;
    1281              : 
    1282         1025 :   *rem = rem1;
    1283         1025 :   return quot1;
    1284         1025 : }
    1285              : 
    1286              : /* Wrapper around expand_binop which takes an rtx code to specify
    1287              :    the operation to perform, not an optab pointer.  All other
    1288              :    arguments are the same.  */
    1289              : rtx
    1290      4158229 : expand_simple_binop (machine_mode mode, enum rtx_code code, rtx op0,
    1291              :                      rtx op1, rtx target, int unsignedp,
    1292              :                      enum optab_methods methods)
    1293              : {
    1294      4158229 :   optab binop = code_to_optab (code);
    1295      4158229 :   gcc_assert (binop);
    1296              : 
    1297      4158229 :   return expand_binop (mode, binop, op0, op1, target, unsignedp, methods);
    1298              : }
    1299              : 
    1300              : /* Return whether OP0 and OP1 should be swapped when expanding a commutative
    1301              :    binop.  Order them according to commutative_operand_precedence and, if
    1302              :    possible, try to put TARGET or a pseudo first.  */
    1303              : static bool
    1304     12527696 : swap_commutative_operands_with_target (rtx target, rtx op0, rtx op1)
    1305              : {
    1306     12527696 :   int op0_prec = commutative_operand_precedence (op0);
    1307     12527696 :   int op1_prec = commutative_operand_precedence (op1);
    1308              : 
    1309     12527696 :   if (op0_prec < op1_prec)
    1310              :     return true;
    1311              : 
    1312     12330426 :   if (op0_prec > op1_prec)
    1313              :     return false;
    1314              : 
    1315              :   /* With equal precedence, both orders are ok, but it is better if the
    1316              :      first operand is TARGET, or if both TARGET and OP0 are pseudos.  */
    1317      2323803 :   if (target == 0 || REG_P (target))
    1318      2232274 :     return (REG_P (op1) && !REG_P (op0)) || target == op1;
    1319              :   else
    1320        91529 :     return rtx_equal_p (op1, target);
    1321              : }
    1322              : 
    1323              : /* Return true if BINOPTAB implements a shift operation.  */
    1324              : 
    1325              : static bool
    1326     24778379 : shift_optab_p (optab binoptab)
    1327              : {
    1328      9959705 :   switch (optab_to_code (binoptab))
    1329              :     {
    1330              :     case ASHIFT:
    1331              :     case SS_ASHIFT:
    1332              :     case US_ASHIFT:
    1333              :     case ASHIFTRT:
    1334              :     case LSHIFTRT:
    1335              :     case ROTATE:
    1336              :     case ROTATERT:
    1337              :       return true;
    1338              : 
    1339     13200656 :     default:
    1340            0 :       return false;
    1341              :     }
    1342              : }
    1343              : 
    1344              : /* Return true if BINOPTAB implements a commutative binary operation.  */
    1345              : 
    1346              : static bool
    1347     14998600 : commutative_optab_p (optab binoptab)
    1348              : {
    1349     14998600 :   return (GET_RTX_CLASS (optab_to_code (binoptab)) == RTX_COMM_ARITH
    1350              :           || binoptab == smul_widen_optab
    1351      2562738 :           || binoptab == umul_widen_optab
    1352              :           || binoptab == smul_highpart_optab
    1353      2524059 :           || binoptab == umul_highpart_optab
    1354      2470904 :           || binoptab == vec_widen_sadd_optab
    1355      2470904 :           || binoptab == vec_widen_uadd_optab
    1356              :           || binoptab == vec_widen_sadd_hi_optab
    1357      2470904 :           || binoptab == vec_widen_sadd_lo_optab
    1358              :           || binoptab == vec_widen_uadd_hi_optab
    1359      2470904 :           || binoptab == vec_widen_uadd_lo_optab
    1360              :           || binoptab == vec_widen_sadd_even_optab
    1361      2470904 :           || binoptab == vec_widen_sadd_odd_optab
    1362              :           || binoptab == vec_widen_uadd_even_optab
    1363     17469504 :           || binoptab == vec_widen_uadd_odd_optab);
    1364              : }
    1365              : 
    1366              : /* X is to be used in mode MODE as operand OPN to BINOPTAB.  If we're
    1367              :    optimizing, and if the operand is a constant that costs more than
    1368              :    1 instruction, force the constant into a register and return that
    1369              :    register.  Return X otherwise.  UNSIGNEDP says whether X is unsigned.  */
    1370              : 
    1371              : static rtx
    1372     28221223 : avoid_expensive_constant (machine_mode mode, optab binoptab,
    1373              :                           int opn, rtx x, bool unsignedp)
    1374              : {
    1375     28221223 :   bool speed = optimize_insn_for_speed_p ();
    1376              : 
    1377     28221223 :   if (mode != VOIDmode
    1378     28221223 :       && optimize
    1379     21380950 :       && CONSTANT_P (x)
    1380     35569366 :       && (rtx_cost (x, mode, optab_to_code (binoptab), opn, speed)
    1381      7348143 :           > set_src_cost (x, mode, speed)))
    1382              :     {
    1383            0 :       if (CONST_INT_P (x))
    1384              :         {
    1385            0 :           HOST_WIDE_INT intval = trunc_int_for_mode (INTVAL (x), mode);
    1386            0 :           if (intval != INTVAL (x))
    1387            0 :             x = GEN_INT (intval);
    1388              :         }
    1389              :       else
    1390            0 :         x = convert_modes (mode, VOIDmode, x, unsignedp);
    1391            0 :       x = force_reg (mode, x);
    1392              :     }
    1393     28221223 :   return x;
    1394              : }
    1395              : 
    1396              : /* Helper function for expand_binop: handle the case where there
    1397              :    is an insn ICODE that directly implements the indicated operation.
    1398              :    Returns null if this is not possible.  */
    1399              : static rtx
    1400     14788020 : expand_binop_directly (enum insn_code icode, machine_mode mode, optab binoptab,
    1401              :                        rtx op0, rtx op1,
    1402              :                        rtx target, int unsignedp, enum optab_methods methods)
    1403              : {
    1404     14788020 :   machine_mode xmode0 = insn_data[(int) icode].operand[1].mode;
    1405     14788020 :   machine_mode xmode1 = insn_data[(int) icode].operand[2].mode;
    1406     14788020 :   machine_mode mode0, mode1, tmp_mode;
    1407     14788020 :   class expand_operand ops[3];
    1408     14788020 :   bool commutative_p;
    1409     14788020 :   rtx_insn *pat;
    1410     14788020 :   rtx xop0 = op0, xop1 = op1;
    1411     14788020 :   bool canonicalize_op1 = false;
    1412     14788020 :   rtx_insn *last = get_last_insn ();
    1413              : 
    1414              :   /* If it is a commutative operator and the modes would match
    1415              :      if we would swap the operands, we can save the conversions.  */
    1416     14788020 :   commutative_p = commutative_optab_p (binoptab);
    1417     14788020 :   if (commutative_p
    1418     12466309 :       && GET_MODE (xop0) != xmode0 && GET_MODE (xop1) != xmode1
    1419        13697 :       && GET_MODE (xop0) == xmode1 && GET_MODE (xop1) == xmode0)
    1420              :     std::swap (xop0, xop1);
    1421              : 
    1422              :   /* If we are optimizing, force expensive constants into a register.  */
    1423     14788020 :   xop0 = avoid_expensive_constant (xmode0, binoptab, 0, xop0, unsignedp);
    1424     14788020 :   if (!shift_optab_p (binoptab))
    1425     13200656 :     xop1 = avoid_expensive_constant (xmode1, binoptab, 1, xop1, unsignedp);
    1426              :   else
    1427              :     /* Shifts and rotates often use a different mode for op1 from op0;
    1428              :        for VOIDmode constants we don't know the mode, so force it
    1429              :        to be canonicalized using convert_modes.  */
    1430              :     canonicalize_op1 = true;
    1431              : 
    1432              :   /* In case the insn wants input operands in modes different from
    1433              :      those of the actual operands, convert the operands.  It would
    1434              :      seem that we don't need to convert CONST_INTs, but we do, so
    1435              :      that they're properly zero-extended, sign-extended or truncated
    1436              :      for their mode.  */
    1437              : 
    1438     14788020 :   mode0 = GET_MODE (xop0) != VOIDmode ? GET_MODE (xop0) : mode;
    1439     14788020 :   if (xmode0 != VOIDmode && xmode0 != mode0)
    1440              :     {
    1441         3370 :       xop0 = convert_modes (xmode0, mode0, xop0, unsignedp);
    1442         3370 :       mode0 = xmode0;
    1443              :     }
    1444              : 
    1445     14788020 :   mode1 = ((GET_MODE (xop1) != VOIDmode || canonicalize_op1)
    1446     14788020 :            ? GET_MODE (xop1) : mode);
    1447     14788020 :   if (xmode1 != VOIDmode && xmode1 != mode1)
    1448              :     {
    1449      1589166 :       xop1 = convert_modes (xmode1, mode1, xop1, unsignedp);
    1450      1589166 :       mode1 = xmode1;
    1451              :     }
    1452              : 
    1453              :   /* If operation is commutative,
    1454              :      try to make the first operand a register.
    1455              :      Even better, try to make it the same as the target.
    1456              :      Also try to make the last operand a constant.  */
    1457     14788020 :   if (commutative_p
    1458     14788020 :       && swap_commutative_operands_with_target (target, xop0, xop1))
    1459              :     std::swap (xop0, xop1);
    1460              : 
    1461              :   /* Now, if insn's predicates don't allow our operands, put them into
    1462              :      pseudo regs.  */
    1463              : 
    1464     14788020 :   if (binoptab == vec_pack_trunc_optab
    1465     14788020 :       || binoptab == vec_pack_usat_optab
    1466     14777369 :       || binoptab == vec_pack_ssat_optab
    1467     14777369 :       || binoptab == vec_pack_ufix_trunc_optab
    1468     14777211 :       || binoptab == vec_pack_sfix_trunc_optab
    1469     14777211 :       || binoptab == vec_packu_float_optab
    1470     14776991 :       || binoptab == vec_packs_float_optab)
    1471              :     {
    1472              :       /* The mode of the result is different then the mode of the
    1473              :          arguments.  */
    1474        11049 :       tmp_mode = insn_data[(int) icode].operand[0].mode;
    1475         2692 :       if (VECTOR_MODE_P (mode)
    1476        31553 :           && maybe_ne (GET_MODE_NUNITS (tmp_mode), 2 * GET_MODE_NUNITS (mode)))
    1477              :         return NULL_RTX;
    1478              :     }
    1479              :   else
    1480              :     tmp_mode = mode;
    1481              : 
    1482     14788020 :   create_output_operand (&ops[0], target, tmp_mode);
    1483     14788020 :   create_input_operand (&ops[1], xop0, mode0);
    1484     14788020 :   create_input_operand (&ops[2], xop1, mode1);
    1485     14788020 :   pat = maybe_gen_insn (icode, 3, ops);
    1486     14788020 :   if (pat)
    1487              :     {
    1488              :       /* If PAT is composed of more than one insn, try to add an appropriate
    1489              :          REG_EQUAL note to it.  If we can't because TEMP conflicts with an
    1490              :          operand, call expand_binop again, this time without a target.  */
    1491     13607886 :       if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
    1492     15999403 :           && ! add_equal_note (pat, ops[0].value,
    1493              :                                optab_to_code (binoptab),
    1494              :                                ops[1].value, ops[2].value, mode0))
    1495              :         {
    1496        53577 :           delete_insns_since (last);
    1497        53577 :           return expand_binop_directly (icode, mode, binoptab, op0, op1,
    1498        53577 :                                         NULL_RTX, unsignedp, methods);
    1499              :         }
    1500              : 
    1501     14734437 :       emit_insn (pat);
    1502     14734437 :       return ops[0].value;
    1503              :     }
    1504              : 
    1505              :   return NULL_RTX;
    1506              : }
    1507              : 
    1508              : /* Generate code to perform an operation specified by BINOPTAB
    1509              :    on operands OP0 and OP1, with result having machine-mode MODE.
    1510              : 
    1511              :    UNSIGNEDP is for the case where we have to widen the operands
    1512              :    to perform the operation.  It says to use zero-extension.
    1513              : 
    1514              :    If TARGET is nonzero, the value
    1515              :    is generated there, if it is convenient to do so.
    1516              :    In all cases an rtx is returned for the locus of the value;
    1517              :    this may or may not be TARGET.  */
    1518              : 
    1519              : rtx
    1520     14945183 : expand_binop (machine_mode mode, optab binoptab, rtx op0, rtx op1,
    1521              :               rtx target, int unsignedp, enum optab_methods methods)
    1522              : {
    1523     12923702 :   enum optab_methods next_methods
    1524     15054607 :     = (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN
    1525     15054607 :        ? OPTAB_WIDEN : methods);
    1526     15054607 :   enum mode_class mclass;
    1527     15054607 :   enum insn_code icode;
    1528     15054607 :   machine_mode wider_mode;
    1529     15054607 :   scalar_int_mode int_mode;
    1530     15054607 :   rtx libfunc;
    1531     15054607 :   rtx temp;
    1532     15054607 :   rtx_insn *entry_last = get_last_insn ();
    1533     15054607 :   rtx_insn *last;
    1534              : 
    1535     15054607 :   mclass = GET_MODE_CLASS (mode);
    1536              : 
    1537              :   /* If subtracting an integer constant, convert this into an addition of
    1538              :      the negated constant.  */
    1539              : 
    1540     15054607 :   if (binoptab == sub_optab && CONST_INT_P (op1))
    1541              :     {
    1542      1037757 :       op1 = negate_rtx (mode, op1);
    1543      1037757 :       binoptab = add_optab;
    1544              :     }
    1545              :   /* For shifts, constant invalid op1 might be expanded from different
    1546              :      mode than MODE.  As those are invalid, force them to a register
    1547              :      to avoid further problems during expansion.  */
    1548     14016850 :   else if (CONST_INT_P (op1)
    1549     16580223 :            && shift_optab_p (binoptab)
    1550     18593698 :            && UINTVAL (op1) >= GET_MODE_BITSIZE (GET_MODE_INNER (mode)))
    1551              :     {
    1552          160 :       op1 = gen_int_mode (INTVAL (op1), GET_MODE_INNER (mode));
    1553          160 :       op1 = force_reg (GET_MODE_INNER (mode), op1);
    1554              :     }
    1555              : 
    1556              :   /* Record where to delete back to if we backtrack.  */
    1557     15054607 :   last = get_last_insn ();
    1558              : 
    1559              :   /* If we can do it with a three-operand insn, do so.  */
    1560              : 
    1561     15054607 :   if (methods != OPTAB_MUST_WIDEN)
    1562              :     {
    1563     15054607 :       if (convert_optab_p (binoptab))
    1564              :         {
    1565        38679 :           machine_mode from_mode = widened_mode (mode, op0, op1);
    1566        38679 :           icode = find_widening_optab_handler (binoptab, mode, from_mode);
    1567              :         }
    1568              :       else
    1569     15015928 :         icode = optab_handler (binoptab, mode);
    1570     15054607 :       if (icode != CODE_FOR_nothing)
    1571              :         {
    1572     14734442 :           temp = expand_binop_directly (icode, mode, binoptab, op0, op1,
    1573              :                                         target, unsignedp, methods);
    1574     14734442 :           if (temp)
    1575              :             return temp;
    1576            5 :           delete_insns_since (last);
    1577              :         }
    1578              :     }
    1579              : 
    1580              :   /* If we were trying to rotate, and that didn't work, try rotating
    1581              :      the other direction before falling back to shifts and bitwise-or.  */
    1582       320170 :   if (((binoptab == rotl_optab
    1583           25 :         && (icode = optab_handler (rotr_optab, mode)) != CODE_FOR_nothing)
    1584       320167 :        || (binoptab == rotr_optab
    1585           13 :            && (icode = optab_handler (rotl_optab, mode)) != CODE_FOR_nothing))
    1586       320175 :       && is_int_mode (mode, &int_mode))
    1587              :     {
    1588            1 :       optab otheroptab = (binoptab == rotl_optab ? rotr_optab : rotl_optab);
    1589            1 :       rtx newop1;
    1590            1 :       unsigned int bits = GET_MODE_PRECISION (int_mode);
    1591              : 
    1592            1 :       if (CONST_INT_P (op1))
    1593            0 :         newop1 = gen_int_shift_amount (int_mode, bits - INTVAL (op1));
    1594            1 :       else if (targetm.shift_truncation_mask (int_mode) == bits - 1)
    1595            0 :         newop1 = negate_rtx (GET_MODE (op1), op1);
    1596              :       else
    1597            1 :         newop1 = expand_binop (GET_MODE (op1), sub_optab,
    1598            1 :                                gen_int_mode (bits, GET_MODE (op1)), op1,
    1599              :                                NULL_RTX, unsignedp, OPTAB_DIRECT);
    1600              : 
    1601            1 :       temp = expand_binop_directly (icode, int_mode, otheroptab, op0, newop1,
    1602              :                                     target, unsignedp, methods);
    1603            1 :       if (temp)
    1604     14945183 :         return temp;
    1605            1 :       delete_insns_since (last);
    1606              :     }
    1607              : 
    1608              :   /* If this is a multiply, see if we can do a widening operation that
    1609              :      takes operands of this mode and makes a wider mode.  */
    1610              : 
    1611       320170 :   if (binoptab == smul_optab
    1612        26453 :       && GET_MODE_2XWIDER_MODE (mode).exists (&wider_mode)
    1613       339832 :       && (convert_optab_handler ((unsignedp
    1614              :                                   ? umul_widen_optab
    1615              :                                   : smul_widen_optab),
    1616              :                                  wider_mode, mode) != CODE_FOR_nothing))
    1617              :     {
    1618              :       /* *_widen_optab needs to determine operand mode, make sure at least
    1619              :          one operand has non-VOID mode.  */
    1620            0 :       if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
    1621            0 :         op0 = force_reg (mode, op0);
    1622            0 :       temp = expand_binop (wider_mode,
    1623              :                            unsignedp ? umul_widen_optab : smul_widen_optab,
    1624              :                            op0, op1, NULL_RTX, unsignedp, OPTAB_DIRECT);
    1625              : 
    1626            0 :       if (temp != 0)
    1627              :         {
    1628            0 :           if (GET_MODE_CLASS (mode) == MODE_INT
    1629            0 :               && TRULY_NOOP_TRUNCATION_MODES_P (mode, GET_MODE (temp)))
    1630            0 :             return gen_lowpart (mode, temp);
    1631              :           else
    1632            0 :             return convert_to_mode (mode, temp, unsignedp);
    1633              :         }
    1634              :     }
    1635              : 
    1636              :   /* If backend's machine description doesn't specify an any_or_plus
    1637              :      (AOP) preference, choose for it.  */
    1638       320170 :   if (binoptab == aop_optab)
    1639              :     {
    1640       109424 :       binoptab = (mode == word_mode || mode == SImode) ? add_optab
    1641              :                                                        : ior_optab;
    1642       109424 :       return expand_binop (mode, binoptab, op0, op1,
    1643       109424 :                            target, unsignedp, methods);
    1644              :     }
    1645              : 
    1646              :   /* If this is a vector shift by a scalar, see if we can do a vector
    1647              :      shift by a vector.  If so, broadcast the scalar into a vector.  */
    1648       210746 :   if (mclass == MODE_VECTOR_INT)
    1649              :     {
    1650           37 :       optab otheroptab = unknown_optab;
    1651              : 
    1652           37 :       if (binoptab == ashl_optab)
    1653              :         otheroptab = vashl_optab;
    1654              :       else if (binoptab == ashr_optab)
    1655              :         otheroptab = vashr_optab;
    1656              :       else if (binoptab == lshr_optab)
    1657              :         otheroptab = vlshr_optab;
    1658              :       else if (binoptab == rotl_optab)
    1659              :         otheroptab = vrotl_optab;
    1660              :       else if (binoptab == rotr_optab)
    1661              :         otheroptab = vrotr_optab;
    1662              : 
    1663              :       if (otheroptab
    1664           37 :           && (icode = optab_handler (otheroptab, mode)) != CODE_FOR_nothing)
    1665              :         {
    1666              :           /* The scalar may be wider or narrower than the vector element.
    1667              :              Truncate or extend it to the proper size to fit in the
    1668              :              broadcast vector.  */
    1669            0 :           scalar_mode inner_mode = GET_MODE_INNER (mode);
    1670            0 :           if (!CONST_INT_P (op1))
    1671              :             {
    1672            0 :               auto mode1 = as_a <scalar_int_mode> (GET_MODE (op1));
    1673            0 :               int size1 = GET_MODE_BITSIZE (mode1);
    1674            0 :               int inner_size = GET_MODE_BITSIZE (inner_mode);
    1675              : 
    1676            0 :               if (size1 != inner_size)
    1677              :                 {
    1678            0 :                   auto unary = size1 > inner_size ? TRUNCATE : ZERO_EXTEND;
    1679            0 :                   op1 = force_reg (inner_mode,
    1680              :                                    simplify_gen_unary (unary, inner_mode,
    1681              :                                                        op1, mode1));
    1682              :                 }
    1683              :             }
    1684              : 
    1685            0 :           rtx vop1 = expand_vector_broadcast (mode, op1);
    1686            0 :           if (vop1)
    1687              :             {
    1688            0 :               temp = expand_binop_directly (icode, mode, otheroptab, op0, vop1,
    1689              :                                             target, unsignedp, methods);
    1690            0 :               if (temp)
    1691     14945183 :                 return temp;
    1692            0 :               delete_insns_since (last);
    1693              :             }
    1694              :         }
    1695              :     }
    1696              : 
    1697              :   /* Look for a wider mode of the same class for which we think we
    1698              :      can open-code the operation.  Check for a widening multiply at the
    1699              :      wider mode as well.  */
    1700              : 
    1701       210746 :   if (CLASS_HAS_WIDER_MODES_P (mclass)
    1702       210709 :       && methods != OPTAB_DIRECT && methods != OPTAB_LIB)
    1703       117960 :     FOR_EACH_WIDER_MODE (wider_mode, mode)
    1704              :       {
    1705        63719 :         machine_mode next_mode;
    1706        63719 :         if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing
    1707        63719 :             || (binoptab == smul_optab
    1708        80838 :                 && GET_MODE_WIDER_MODE (wider_mode).exists (&next_mode)
    1709        20871 :                 && (find_widening_optab_handler ((unsignedp
    1710              :                                                   ? umul_widen_optab
    1711              :                                                   : smul_widen_optab),
    1712              :                                                  next_mode, mode)
    1713              :                     != CODE_FOR_nothing)))
    1714              :           {
    1715          166 :             rtx xop0 = op0, xop1 = op1;
    1716          166 :             bool no_extend = false;
    1717              : 
    1718              :             /* For certain integer operations, we need not actually extend
    1719              :                the narrow operands, as long as we will truncate
    1720              :                the results to the same narrowness.  */
    1721              : 
    1722          166 :             if ((binoptab == ior_optab || binoptab == and_optab
    1723          166 :                  || binoptab == xor_optab
    1724          166 :                  || binoptab == add_optab || binoptab == sub_optab
    1725              :                  || binoptab == smul_optab || binoptab == ashl_optab)
    1726           91 :                 && mclass == MODE_INT)
    1727              :               {
    1728            7 :                 no_extend = true;
    1729            7 :                 xop0 = avoid_expensive_constant (mode, binoptab, 0,
    1730              :                                                  xop0, unsignedp);
    1731            7 :                 if (binoptab != ashl_optab)
    1732            0 :                   xop1 = avoid_expensive_constant (mode, binoptab, 1,
    1733              :                                                    xop1, unsignedp);
    1734              :               }
    1735              : 
    1736          166 :             xop0 = widen_operand (xop0, wider_mode, mode, unsignedp, no_extend);
    1737              : 
    1738              :             /* The second operand of a shift must always be extended.  */
    1739          332 :             xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
    1740          166 :                                   no_extend && binoptab != ashl_optab);
    1741              : 
    1742          166 :             temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
    1743              :                                  unsignedp, OPTAB_DIRECT);
    1744          166 :             if (temp)
    1745              :               {
    1746          166 :                 if (mclass != MODE_INT
    1747          166 :                     || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
    1748              :                   {
    1749          159 :                     if (target == 0)
    1750           51 :                       target = gen_reg_rtx (mode);
    1751          159 :                     convert_move (target, temp, 0);
    1752          159 :                     return target;
    1753              :                   }
    1754              :                 else
    1755            7 :                   return gen_lowpart (mode, temp);
    1756              :               }
    1757              :             else
    1758            0 :               delete_insns_since (last);
    1759              :           }
    1760              :       }
    1761              : 
    1762              :   /* If operation is commutative,
    1763              :      try to make the first operand a register.
    1764              :      Even better, try to make it the same as the target.
    1765              :      Also try to make the last operand a constant.  */
    1766       210580 :   if (commutative_optab_p (binoptab)
    1767       210580 :       && swap_commutative_operands_with_target (target, op0, op1))
    1768              :     std::swap (op0, op1);
    1769              : 
    1770              :   /* These can be done a word at a time.  */
    1771       210580 :   if ((binoptab == and_optab || binoptab == ior_optab || binoptab == xor_optab)
    1772            8 :       && is_int_mode (mode, &int_mode)
    1773           16 :       && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
    1774       210588 :       && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
    1775              :     {
    1776            8 :       int i;
    1777            8 :       rtx_insn *insns;
    1778              : 
    1779              :       /* If TARGET is the same as one of the operands, the REG_EQUAL note
    1780              :          won't be accurate, so use a new target.  */
    1781            8 :       if (target == 0
    1782            8 :           || target == op0
    1783            8 :           || target == op1
    1784            8 :           || reg_overlap_mentioned_p (target, op0)
    1785            8 :           || reg_overlap_mentioned_p (target, op1)
    1786           16 :           || !valid_multiword_target_p (target))
    1787            0 :         target = gen_reg_rtx (int_mode);
    1788              : 
    1789            8 :       start_sequence ();
    1790              : 
    1791              :       /* Do the actual arithmetic.  */
    1792            8 :       machine_mode op0_mode = GET_MODE (op0);
    1793            8 :       machine_mode op1_mode = GET_MODE (op1);
    1794            8 :       if (op0_mode == VOIDmode)
    1795            0 :         op0_mode = int_mode;
    1796            8 :       if (op1_mode == VOIDmode)
    1797            8 :         op1_mode = int_mode;
    1798          120 :       for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
    1799              :         {
    1800           32 :           rtx target_piece = operand_subword (target, i, 1, int_mode);
    1801           32 :           rtx x = expand_binop (word_mode, binoptab,
    1802           32 :                                 operand_subword_force (op0, i, op0_mode),
    1803           32 :                                 operand_subword_force (op1, i, op1_mode),
    1804              :                                 target_piece, unsignedp, next_methods);
    1805              : 
    1806           32 :           if (x == 0)
    1807              :             break;
    1808              : 
    1809           32 :           if (target_piece != x)
    1810            0 :             emit_move_insn (target_piece, x);
    1811              :         }
    1812              : 
    1813            8 :       insns = end_sequence ();
    1814              : 
    1815           24 :       if (i == GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD)
    1816              :         {
    1817            8 :           emit_insn (insns);
    1818            8 :           return target;
    1819              :         }
    1820              :     }
    1821              : 
    1822              :   /* Synthesize double word shifts from single word shifts.  */
    1823       210572 :   if ((binoptab == lshr_optab || binoptab == ashl_optab
    1824       210572 :        || binoptab == ashr_optab)
    1825            0 :       && is_int_mode (mode, &int_mode)
    1826            0 :       && (CONST_INT_P (op1) || optimize_insn_for_speed_p ())
    1827            0 :       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
    1828            0 :       && GET_MODE_PRECISION (int_mode) == GET_MODE_BITSIZE (int_mode)
    1829            0 :       && optab_handler (binoptab, word_mode) != CODE_FOR_nothing
    1830            0 :       && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
    1831       210572 :       && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
    1832              :     {
    1833            0 :       unsigned HOST_WIDE_INT shift_mask, double_shift_mask;
    1834            0 :       scalar_int_mode op1_mode;
    1835              : 
    1836            0 :       double_shift_mask = targetm.shift_truncation_mask (int_mode);
    1837            0 :       shift_mask = targetm.shift_truncation_mask (word_mode);
    1838            0 :       op1_mode = (GET_MODE (op1) != VOIDmode
    1839            0 :                   ? as_a <scalar_int_mode> (GET_MODE (op1))
    1840              :                   : word_mode);
    1841              : 
    1842              :       /* Apply the truncation to constant shifts.  */
    1843            0 :       if (double_shift_mask > 0 && CONST_INT_P (op1))
    1844            0 :         op1 = gen_int_mode (INTVAL (op1) & double_shift_mask, op1_mode);
    1845              : 
    1846            0 :       if (op1 == CONST0_RTX (op1_mode))
    1847     14945183 :         return op0;
    1848              : 
    1849              :       /* Make sure that this is a combination that expand_doubleword_shift
    1850              :          can handle.  See the comments there for details.  */
    1851            0 :       if (double_shift_mask == 0
    1852            0 :           || (shift_mask == BITS_PER_WORD - 1
    1853            0 :               && double_shift_mask == BITS_PER_WORD * 2 - 1))
    1854              :         {
    1855            0 :           rtx_insn *insns;
    1856            0 :           rtx into_target, outof_target;
    1857            0 :           rtx into_input, outof_input;
    1858            0 :           int left_shift, outof_word;
    1859              : 
    1860              :           /* If TARGET is the same as one of the operands, the REG_EQUAL note
    1861              :              won't be accurate, so use a new target.  */
    1862            0 :           if (target == 0
    1863            0 :               || target == op0
    1864            0 :               || target == op1
    1865            0 :               || reg_overlap_mentioned_p (target, op0)
    1866            0 :               || reg_overlap_mentioned_p (target, op1)
    1867            0 :               || !valid_multiword_target_p (target))
    1868            0 :             target = gen_reg_rtx (int_mode);
    1869              : 
    1870            0 :           start_sequence ();
    1871              : 
    1872              :           /* OUTOF_* is the word we are shifting bits away from, and
    1873              :              INTO_* is the word that we are shifting bits towards, thus
    1874              :              they differ depending on the direction of the shift and
    1875              :              WORDS_BIG_ENDIAN.  */
    1876              : 
    1877            0 :           left_shift = binoptab == ashl_optab;
    1878            0 :           outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
    1879              : 
    1880            0 :           outof_target = operand_subword (target, outof_word, 1, int_mode);
    1881            0 :           into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
    1882              : 
    1883            0 :           outof_input = operand_subword_force (op0, outof_word, int_mode);
    1884            0 :           into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
    1885              : 
    1886            0 :           if (expand_doubleword_shift (op1_mode, binoptab,
    1887              :                                        outof_input, into_input, op1,
    1888              :                                        outof_target, into_target,
    1889              :                                        unsignedp, next_methods, shift_mask))
    1890              :             {
    1891            0 :               insns = end_sequence ();
    1892              : 
    1893            0 :               emit_insn (insns);
    1894            0 :               return target;
    1895              :             }
    1896            0 :           end_sequence ();
    1897              :         }
    1898              :     }
    1899              : 
    1900              :   /* Synthesize double word rotates from single word shifts.  */
    1901       210572 :   if ((binoptab == rotl_optab || binoptab == rotr_optab)
    1902           38 :       && is_int_mode (mode, &int_mode)
    1903            1 :       && CONST_INT_P (op1)
    1904            0 :       && GET_MODE_PRECISION (int_mode) == 2 * BITS_PER_WORD
    1905            0 :       && optab_handler (ashl_optab, word_mode) != CODE_FOR_nothing
    1906       210572 :       && optab_handler (lshr_optab, word_mode) != CODE_FOR_nothing)
    1907              :     {
    1908            0 :       rtx_insn *insns;
    1909            0 :       rtx into_target, outof_target;
    1910            0 :       rtx into_input, outof_input;
    1911            0 :       rtx inter;
    1912            0 :       int shift_count, left_shift, outof_word;
    1913              : 
    1914              :       /* If TARGET is the same as one of the operands, the REG_EQUAL note
    1915              :          won't be accurate, so use a new target. Do this also if target is not
    1916              :          a REG, first because having a register instead may open optimization
    1917              :          opportunities, and second because if target and op0 happen to be MEMs
    1918              :          designating the same location, we would risk clobbering it too early
    1919              :          in the code sequence we generate below.  */
    1920            0 :       if (target == 0
    1921            0 :           || target == op0
    1922            0 :           || target == op1
    1923            0 :           || !REG_P (target)
    1924            0 :           || reg_overlap_mentioned_p (target, op0)
    1925            0 :           || reg_overlap_mentioned_p (target, op1)
    1926            0 :           || !valid_multiword_target_p (target))
    1927            0 :         target = gen_reg_rtx (int_mode);
    1928              : 
    1929            0 :       start_sequence ();
    1930              : 
    1931            0 :       shift_count = INTVAL (op1);
    1932              : 
    1933              :       /* OUTOF_* is the word we are shifting bits away from, and
    1934              :          INTO_* is the word that we are shifting bits towards, thus
    1935              :          they differ depending on the direction of the shift and
    1936              :          WORDS_BIG_ENDIAN.  */
    1937              : 
    1938            0 :       left_shift = (binoptab == rotl_optab);
    1939            0 :       outof_word = left_shift ^ ! WORDS_BIG_ENDIAN;
    1940              : 
    1941            0 :       outof_target = operand_subword (target, outof_word, 1, int_mode);
    1942            0 :       into_target = operand_subword (target, 1 - outof_word, 1, int_mode);
    1943              : 
    1944            0 :       outof_input = operand_subword_force (op0, outof_word, int_mode);
    1945            0 :       into_input = operand_subword_force (op0, 1 - outof_word, int_mode);
    1946              : 
    1947            0 :       if (shift_count == BITS_PER_WORD)
    1948              :         {
    1949              :           /* This is just a word swap.  */
    1950            0 :           emit_move_insn (outof_target, into_input);
    1951            0 :           emit_move_insn (into_target, outof_input);
    1952            0 :           inter = const0_rtx;
    1953              :         }
    1954              :       else
    1955              :         {
    1956            0 :           rtx into_temp1, into_temp2, outof_temp1, outof_temp2;
    1957            0 :           HOST_WIDE_INT first_shift_count, second_shift_count;
    1958            0 :           optab reverse_unsigned_shift, unsigned_shift;
    1959              : 
    1960            0 :           reverse_unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
    1961            0 :                                     ? lshr_optab : ashl_optab);
    1962              : 
    1963            0 :           unsigned_shift = (left_shift ^ (shift_count < BITS_PER_WORD)
    1964            0 :                             ? ashl_optab : lshr_optab);
    1965              : 
    1966            0 :           if (shift_count > BITS_PER_WORD)
    1967              :             {
    1968            0 :               first_shift_count = shift_count - BITS_PER_WORD;
    1969            0 :               second_shift_count = 2 * BITS_PER_WORD - shift_count;
    1970              :             }
    1971              :           else
    1972              :             {
    1973            0 :               first_shift_count = BITS_PER_WORD - shift_count;
    1974            0 :               second_shift_count = shift_count;
    1975              :             }
    1976            0 :           rtx first_shift_count_rtx
    1977            0 :             = gen_int_shift_amount (word_mode, first_shift_count);
    1978            0 :           rtx second_shift_count_rtx
    1979            0 :             = gen_int_shift_amount (word_mode, second_shift_count);
    1980              : 
    1981            0 :           into_temp1 = expand_binop (word_mode, unsigned_shift,
    1982              :                                      outof_input, first_shift_count_rtx,
    1983              :                                      NULL_RTX, unsignedp, next_methods);
    1984            0 :           into_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
    1985              :                                      into_input, second_shift_count_rtx,
    1986              :                                      NULL_RTX, unsignedp, next_methods);
    1987              : 
    1988            0 :           if (into_temp1 != 0 && into_temp2 != 0)
    1989            0 :             inter = expand_binop (word_mode, aop_optab, into_temp1, into_temp2,
    1990              :                                   into_target, unsignedp, next_methods);
    1991              :           else
    1992              :             inter = 0;
    1993              : 
    1994            0 :           if (inter != 0 && inter != into_target)
    1995            0 :             emit_move_insn (into_target, inter);
    1996              : 
    1997            0 :           outof_temp1 = expand_binop (word_mode, unsigned_shift,
    1998              :                                       into_input, first_shift_count_rtx,
    1999              :                                       NULL_RTX, unsignedp, next_methods);
    2000            0 :           outof_temp2 = expand_binop (word_mode, reverse_unsigned_shift,
    2001              :                                       outof_input, second_shift_count_rtx,
    2002              :                                       NULL_RTX, unsignedp, next_methods);
    2003              : 
    2004            0 :           if (inter != 0 && outof_temp1 != 0 && outof_temp2 != 0)
    2005            0 :             inter = expand_binop (word_mode, aop_optab,
    2006              :                                   outof_temp1, outof_temp2,
    2007              :                                   outof_target, unsignedp, next_methods);
    2008              : 
    2009            0 :           if (inter != 0 && inter != outof_target)
    2010            0 :             emit_move_insn (outof_target, inter);
    2011              :         }
    2012              : 
    2013            0 :       insns = end_sequence ();
    2014              : 
    2015            0 :       if (inter != 0)
    2016              :         {
    2017            0 :           emit_insn (insns);
    2018            0 :           return target;
    2019              :         }
    2020              :     }
    2021              : 
    2022              :   /* These can be done a word at a time by propagating carries.  */
    2023       210572 :   if ((binoptab == add_optab || binoptab == sub_optab)
    2024        13679 :       && is_int_mode (mode, &int_mode)
    2025            0 :       && GET_MODE_SIZE (int_mode) >= 2 * UNITS_PER_WORD
    2026       210572 :       && optab_handler (binoptab, word_mode) != CODE_FOR_nothing)
    2027              :     {
    2028            0 :       unsigned int i;
    2029            0 :       optab otheroptab = binoptab == add_optab ? sub_optab : add_optab;
    2030            0 :       const unsigned int nwords = GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD;
    2031            0 :       rtx carry_in = NULL_RTX, carry_out = NULL_RTX;
    2032            0 :       rtx xop0, xop1, xtarget;
    2033              : 
    2034              :       /* We can handle either a 1 or -1 value for the carry.  If STORE_FLAG
    2035              :          value is one of those, use it.  Otherwise, use 1 since it is the
    2036              :          one easiest to get.  */
    2037              : #if STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1
    2038            0 :       int normalizep = STORE_FLAG_VALUE;
    2039              : #else
    2040              :       int normalizep = 1;
    2041              : #endif
    2042              : 
    2043              :       /* Prepare the operands.  */
    2044            0 :       xop0 = force_reg (int_mode, op0);
    2045            0 :       xop1 = force_reg (int_mode, op1);
    2046              : 
    2047            0 :       xtarget = gen_reg_rtx (int_mode);
    2048              : 
    2049            0 :       if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
    2050              :         target = xtarget;
    2051              : 
    2052              :       /* Indicate for flow that the entire target reg is being set.  */
    2053            0 :       if (REG_P (target))
    2054            0 :         emit_clobber (xtarget);
    2055              : 
    2056              :       /* Do the actual arithmetic.  */
    2057            0 :       for (i = 0; i < nwords; i++)
    2058              :         {
    2059            0 :           int index = (WORDS_BIG_ENDIAN ? nwords - i - 1 : i);
    2060            0 :           rtx target_piece = operand_subword (xtarget, index, 1, int_mode);
    2061            0 :           rtx op0_piece = operand_subword_force (xop0, index, int_mode);
    2062            0 :           rtx op1_piece = operand_subword_force (xop1, index, int_mode);
    2063            0 :           rtx x;
    2064              : 
    2065              :           /* Main add/subtract of the input operands.  */
    2066            0 :           x = expand_binop (word_mode, binoptab,
    2067              :                             op0_piece, op1_piece,
    2068              :                             target_piece, unsignedp, next_methods);
    2069            0 :           if (x == 0)
    2070              :             break;
    2071              : 
    2072            0 :           if (i + 1 < nwords)
    2073              :             {
    2074              :               /* Store carry from main add/subtract.  */
    2075            0 :               carry_out = gen_reg_rtx (word_mode);
    2076            0 :               carry_out = emit_store_flag_force (carry_out,
    2077              :                                                  (binoptab == add_optab
    2078              :                                                   ? LT : GT),
    2079              :                                                  x, op0_piece,
    2080              :                                                  word_mode, 1, normalizep);
    2081              :             }
    2082              : 
    2083            0 :           if (i > 0)
    2084              :             {
    2085            0 :               rtx newx;
    2086              : 
    2087              :               /* Add/subtract previous carry to main result.  */
    2088            0 :               newx = expand_binop (word_mode,
    2089              :                                    normalizep == 1 ? binoptab : otheroptab,
    2090              :                                    x, carry_in,
    2091              :                                    NULL_RTX, 1, next_methods);
    2092              : 
    2093            0 :               if (i + 1 < nwords)
    2094              :                 {
    2095              :                   /* Get out carry from adding/subtracting carry in.  */
    2096            0 :                   rtx carry_tmp = gen_reg_rtx (word_mode);
    2097            0 :                   carry_tmp = emit_store_flag_force (carry_tmp,
    2098              :                                                      (binoptab == add_optab
    2099              :                                                       ? LT : GT),
    2100              :                                                      newx, x,
    2101              :                                                      word_mode, 1, normalizep);
    2102              : 
    2103              :                   /* Logical-ior the two poss. carry together.  */
    2104            0 :                   carry_out = expand_binop (word_mode, ior_optab,
    2105              :                                             carry_out, carry_tmp,
    2106              :                                             carry_out, 0, next_methods);
    2107            0 :                   if (carry_out == 0)
    2108              :                     break;
    2109              :                 }
    2110            0 :               emit_move_insn (target_piece, newx);
    2111              :             }
    2112              :           else
    2113              :             {
    2114            0 :               if (x != target_piece)
    2115            0 :                 emit_move_insn (target_piece, x);
    2116              :             }
    2117              : 
    2118            0 :           carry_in = carry_out;
    2119              :         }
    2120              : 
    2121            0 :       if (i == GET_MODE_BITSIZE (int_mode) / (unsigned) BITS_PER_WORD)
    2122              :         {
    2123            0 :           if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing
    2124            0 :               || ! rtx_equal_p (target, xtarget))
    2125              :             {
    2126            0 :               rtx_insn *temp = emit_move_insn (target, xtarget);
    2127              : 
    2128            0 :               set_dst_reg_note (temp, REG_EQUAL,
    2129              :                                 gen_rtx_fmt_ee (optab_to_code (binoptab),
    2130              :                                                 int_mode, copy_rtx (xop0),
    2131              :                                                 copy_rtx (xop1)),
    2132              :                                 target);
    2133              :             }
    2134              :           else
    2135              :             target = xtarget;
    2136              : 
    2137              :           return target;
    2138              :         }
    2139              : 
    2140              :       else
    2141            0 :         delete_insns_since (last);
    2142              :     }
    2143              : 
    2144              :   /* Attempt to synthesize double word multiplies using a sequence of word
    2145              :      mode multiplications.  We first attempt to generate a sequence using a
    2146              :      more efficient unsigned widening multiply, and if that fails we then
    2147              :      try using a signed widening multiply.  */
    2148              : 
    2149       210572 :   if (binoptab == smul_optab
    2150        26431 :       && is_int_mode (mode, &int_mode)
    2151        19008 :       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
    2152        15648 :       && optab_handler (smul_optab, word_mode) != CODE_FOR_nothing
    2153       226220 :       && optab_handler (add_optab, word_mode) != CODE_FOR_nothing)
    2154              :     {
    2155        15648 :       rtx product = NULL_RTX;
    2156        15648 :       if (convert_optab_handler (umul_widen_optab, int_mode, word_mode)
    2157              :           != CODE_FOR_nothing)
    2158              :         {
    2159        15648 :           product = expand_doubleword_mult (int_mode, op0, op1, target,
    2160              :                                             true, methods);
    2161        15648 :           if (!product)
    2162            0 :             delete_insns_since (last);
    2163              :         }
    2164              : 
    2165            0 :       if (product == NULL_RTX
    2166            0 :           && (convert_optab_handler (smul_widen_optab, int_mode, word_mode)
    2167              :               != CODE_FOR_nothing))
    2168              :         {
    2169            0 :           product = expand_doubleword_mult (int_mode, op0, op1, target,
    2170              :                                             false, methods);
    2171            0 :           if (!product)
    2172            0 :             delete_insns_since (last);
    2173              :         }
    2174              : 
    2175        15648 :       if (product != NULL_RTX)
    2176              :         {
    2177        15648 :           if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
    2178              :             {
    2179        24744 :               rtx_insn *move = emit_move_insn (target ? target : product,
    2180              :                                                product);
    2181        15648 :               set_dst_reg_note (move,
    2182              :                                 REG_EQUAL,
    2183              :                                 gen_rtx_fmt_ee (MULT, int_mode,
    2184              :                                                 copy_rtx (op0),
    2185              :                                                 copy_rtx (op1)),
    2186              :                                 target ? target : product);
    2187              :             }
    2188              :           return product;
    2189              :         }
    2190              :     }
    2191              : 
    2192              :   /* Attempt to synthesize double word modulo by constant divisor.  */
    2193       194924 :   if ((binoptab == umod_optab
    2194       194924 :        || binoptab == smod_optab
    2195       194924 :        || binoptab == udiv_optab
    2196              :        || binoptab == sdiv_optab)
    2197       132317 :       && optimize
    2198       106806 :       && CONST_INT_P (op1)
    2199        22905 :       && is_int_mode (mode, &int_mode)
    2200        25346 :       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
    2201        31482 :       && optab_handler ((binoptab == umod_optab || binoptab == udiv_optab)
    2202              :                         ? udivmod_optab : sdivmod_optab,
    2203              :                         int_mode) == CODE_FOR_nothing
    2204        20161 :       && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
    2205        20161 :       && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
    2206        20161 :       && optimize_insn_for_speed_p ())
    2207              :     {
    2208        19963 :       rtx res = NULL_RTX;
    2209        19963 :       if ((binoptab == umod_optab || binoptab == smod_optab)
    2210         5196 :           && (INTVAL (op1) & 1) == 0)
    2211          396 :         res = expand_doubleword_mod (int_mode, op0, op1,
    2212              :                                      binoptab == umod_optab);
    2213              :       else
    2214              :         {
    2215        19567 :           rtx quot = expand_doubleword_divmod (int_mode, op0, op1, &res,
    2216              :                                                binoptab == umod_optab
    2217              :                                                || binoptab == udiv_optab);
    2218        19567 :           if (quot == NULL_RTX)
    2219        18886 :             res = NULL_RTX;
    2220          681 :           else if (binoptab == udiv_optab || binoptab == sdiv_optab)
    2221          474 :             res = quot;
    2222              :         }
    2223        19963 :       if (res != NULL_RTX)
    2224              :         {
    2225          681 :           if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
    2226              :             {
    2227          848 :               rtx_insn *move = emit_move_insn (target ? target : res,
    2228              :                                                res);
    2229          681 :               set_dst_reg_note (move, REG_EQUAL,
    2230              :                                 gen_rtx_fmt_ee (optab_to_code (binoptab),
    2231              :                                                 int_mode, copy_rtx (op0), op1),
    2232              :                                 target ? target : res);
    2233              :             }
    2234          681 :           return res;
    2235              :         }
    2236              :       else
    2237        19282 :         delete_insns_since (last);
    2238              :     }
    2239              : 
    2240              :   /* It can't be open-coded in this mode.
    2241              :      Use a library call if one is available and caller says that's ok.  */
    2242              : 
    2243       194243 :   libfunc = optab_libfunc (binoptab, mode);
    2244       194243 :   if (libfunc
    2245       129107 :       && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
    2246              :     {
    2247        30654 :       rtx_insn *insns;
    2248        30654 :       rtx op1x = op1;
    2249        30654 :       machine_mode op1_mode = mode;
    2250        30654 :       rtx value;
    2251              : 
    2252        30654 :       start_sequence ();
    2253              : 
    2254        30654 :       if (shift_optab_p (binoptab))
    2255              :         {
    2256            0 :           op1_mode = targetm.libgcc_shift_count_mode ();
    2257              :           /* Specify unsigned here,
    2258              :              since negative shift counts are meaningless.  */
    2259            0 :           op1x = convert_to_mode (op1_mode, op1, 1);
    2260              :         }
    2261              : 
    2262        30654 :       if (GET_MODE (op0) != VOIDmode
    2263        30602 :           && GET_MODE (op0) != mode)
    2264            0 :         op0 = convert_to_mode (mode, op0, unsignedp);
    2265              : 
    2266              :       /* Pass 1 for NO_QUEUE so we don't lose any increments
    2267              :          if the libcall is cse'd or moved.  */
    2268        30654 :       value = emit_library_call_value (libfunc,
    2269              :                                        NULL_RTX, LCT_CONST, mode,
    2270              :                                        op0, mode, op1x, op1_mode);
    2271              : 
    2272        30654 :       insns = end_sequence ();
    2273              : 
    2274        30654 :       bool trapv = trapv_binoptab_p (binoptab);
    2275        30654 :       target = gen_reg_rtx (mode);
    2276        30654 :       emit_libcall_block_1 (insns, target, value,
    2277              :                             trapv ? NULL_RTX
    2278        30164 :                             : gen_rtx_fmt_ee (optab_to_code (binoptab),
    2279              :                                               mode, op0, op1), trapv);
    2280              : 
    2281        30654 :       return target;
    2282              :     }
    2283              : 
    2284       163589 :   delete_insns_since (last);
    2285              : 
    2286              :   /* It can't be done in this mode.  Can we do it in a wider mode?  */
    2287              : 
    2288       163589 :   if (! (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN
    2289              :          || methods == OPTAB_MUST_WIDEN))
    2290              :     {
    2291              :       /* Caller says, don't even try.  */
    2292       150726 :       delete_insns_since (entry_last);
    2293       150726 :       return 0;
    2294              :     }
    2295              : 
    2296              :   /* Compute the value of METHODS to pass to recursive calls.
    2297              :      Don't allow widening to be tried recursively.  */
    2298              : 
    2299        12863 :   methods = (methods == OPTAB_LIB_WIDEN ? OPTAB_LIB : OPTAB_DIRECT);
    2300              : 
    2301              :   /* Look for a wider mode of the same class for which it appears we can do
    2302              :      the operation.  */
    2303              : 
    2304        12863 :   if (CLASS_HAS_WIDER_MODES_P (mclass))
    2305              :     {
    2306              :       /* This code doesn't make sense for conversion optabs, since we
    2307              :          wouldn't then want to extend the operands to be the same size
    2308              :          as the result.  */
    2309        12863 :       gcc_assert (!convert_optab_p (binoptab));
    2310        40513 :       FOR_EACH_WIDER_MODE (wider_mode, mode)
    2311              :         {
    2312        27651 :           if (optab_handler (binoptab, wider_mode)
    2313        27651 :               || (methods == OPTAB_LIB
    2314            2 :                   && optab_libfunc (binoptab, wider_mode)))
    2315              :             {
    2316            1 :               rtx xop0 = op0, xop1 = op1;
    2317            1 :               bool no_extend = false;
    2318              : 
    2319              :               /* For certain integer operations, we need not actually extend
    2320              :                  the narrow operands, as long as we will truncate
    2321              :                  the results to the same narrowness.  */
    2322              : 
    2323            1 :               if ((binoptab == ior_optab || binoptab == and_optab
    2324            1 :                    || binoptab == xor_optab
    2325            1 :                    || binoptab == add_optab || binoptab == sub_optab
    2326              :                    || binoptab == smul_optab || binoptab == ashl_optab)
    2327            0 :                   && mclass == MODE_INT)
    2328            1 :                 no_extend = true;
    2329              : 
    2330            1 :               xop0 = widen_operand (xop0, wider_mode, mode,
    2331              :                                     unsignedp, no_extend);
    2332              : 
    2333              :               /* The second operand of a shift must always be extended.  */
    2334            2 :               xop1 = widen_operand (xop1, wider_mode, mode, unsignedp,
    2335            1 :                                     no_extend && binoptab != ashl_optab);
    2336              : 
    2337            1 :               temp = expand_binop (wider_mode, binoptab, xop0, xop1, NULL_RTX,
    2338              :                                    unsignedp, methods);
    2339            1 :               if (temp)
    2340              :                 {
    2341            1 :                   if (mclass != MODE_INT
    2342            1 :                       || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
    2343              :                     {
    2344            0 :                       if (target == 0)
    2345            0 :                         target = gen_reg_rtx (mode);
    2346            0 :                       convert_move (target, temp, 0);
    2347            0 :                       return target;
    2348              :                     }
    2349              :                   else
    2350            1 :                     return gen_lowpart (mode, temp);
    2351              :                 }
    2352              :               else
    2353            0 :                 delete_insns_since (last);
    2354              :             }
    2355              :         }
    2356              :     }
    2357              : 
    2358        12862 :   delete_insns_since (entry_last);
    2359        12862 :   return 0;
    2360              : }
    2361              : 
    2362              : /* Expand a binary operator which has both signed and unsigned forms.
    2363              :    UOPTAB is the optab for unsigned operations, and SOPTAB is for
    2364              :    signed operations.
    2365              : 
    2366              :    If we widen unsigned operands, we may use a signed wider operation instead
    2367              :    of an unsigned wider operation, since the result would be the same.  */
    2368              : 
    2369              : rtx
    2370       113861 : sign_expand_binop (machine_mode mode, optab uoptab, optab soptab,
    2371              :                    rtx op0, rtx op1, rtx target, int unsignedp,
    2372              :                    enum optab_methods methods)
    2373              : {
    2374       113861 :   rtx temp;
    2375       113861 :   optab direct_optab = unsignedp ? uoptab : soptab;
    2376       113861 :   bool save_enable;
    2377              : 
    2378              :   /* Do it without widening, if possible.  */
    2379       113861 :   temp = expand_binop (mode, direct_optab, op0, op1, target,
    2380              :                        unsignedp, OPTAB_DIRECT);
    2381       113861 :   if (temp || methods == OPTAB_DIRECT)
    2382              :     return temp;
    2383              : 
    2384              :   /* Try widening to a signed int.  Disable any direct use of any
    2385              :      signed insn in the current mode.  */
    2386         8408 :   save_enable = swap_optab_enable (soptab, mode, false);
    2387              : 
    2388         8408 :   temp = expand_binop (mode, soptab, op0, op1, target,
    2389              :                        unsignedp, OPTAB_WIDEN);
    2390              : 
    2391              :   /* For unsigned operands, try widening to an unsigned int.  */
    2392         8408 :   if (!temp && unsignedp)
    2393         4307 :     temp = expand_binop (mode, uoptab, op0, op1, target,
    2394              :                          unsignedp, OPTAB_WIDEN);
    2395         8408 :   if (temp || methods == OPTAB_WIDEN)
    2396         4847 :     goto egress;
    2397              : 
    2398              :   /* Use the right width libcall if that exists.  */
    2399         3561 :   temp = expand_binop (mode, direct_optab, op0, op1, target,
    2400              :                        unsignedp, OPTAB_LIB);
    2401         3561 :   if (temp || methods == OPTAB_LIB)
    2402         3561 :     goto egress;
    2403              : 
    2404              :   /* Must widen and use a libcall, use either signed or unsigned.  */
    2405            0 :   temp = expand_binop (mode, soptab, op0, op1, target,
    2406              :                        unsignedp, methods);
    2407            0 :   if (!temp && unsignedp)
    2408            0 :     temp = expand_binop (mode, uoptab, op0, op1, target,
    2409              :                          unsignedp, methods);
    2410              : 
    2411            0 :  egress:
    2412              :   /* Undo the fiddling above.  */
    2413         8408 :   if (save_enable)
    2414            0 :     swap_optab_enable (soptab, mode, true);
    2415              :   return temp;
    2416              : }
    2417              : 
    2418              : /* Generate code to perform an operation specified by UNOPPTAB
    2419              :    on operand OP0, with two results to TARG0 and TARG1.
    2420              :    We assume that the order of the operands for the instruction
    2421              :    is TARG0, TARG1, OP0.
    2422              : 
    2423              :    Either TARG0 or TARG1 may be zero, but what that means is that
    2424              :    the result is not actually wanted.  We will generate it into
    2425              :    a dummy pseudo-reg and discard it.  They may not both be zero.
    2426              : 
    2427              :    Returns true if this operation can be performed; false if not.  */
    2428              : 
    2429              : bool
    2430            6 : expand_twoval_unop (optab unoptab, rtx op0, rtx targ0, rtx targ1,
    2431              :                     int unsignedp)
    2432              : {
    2433            6 :   machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
    2434            6 :   enum mode_class mclass;
    2435            6 :   machine_mode wider_mode;
    2436            6 :   rtx_insn *entry_last = get_last_insn ();
    2437            6 :   rtx_insn *last;
    2438              : 
    2439            6 :   mclass = GET_MODE_CLASS (mode);
    2440              : 
    2441            6 :   if (!targ0)
    2442            0 :     targ0 = gen_reg_rtx (mode);
    2443            6 :   if (!targ1)
    2444            0 :     targ1 = gen_reg_rtx (mode);
    2445              : 
    2446              :   /* Record where to go back to if we fail.  */
    2447            6 :   last = get_last_insn ();
    2448              : 
    2449            6 :   if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
    2450              :     {
    2451            6 :       class expand_operand ops[3];
    2452            6 :       enum insn_code icode = optab_handler (unoptab, mode);
    2453              : 
    2454            6 :       create_fixed_operand (&ops[0], targ0);
    2455            6 :       create_fixed_operand (&ops[1], targ1);
    2456            6 :       create_convert_operand_from (&ops[2], op0, mode, unsignedp);
    2457            6 :       if (maybe_expand_insn (icode, 3, ops))
    2458            6 :         return true;
    2459              :     }
    2460              : 
    2461              :   /* It can't be done in this mode.  Can we do it in a wider mode?  */
    2462              : 
    2463            0 :   if (CLASS_HAS_WIDER_MODES_P (mclass))
    2464              :     {
    2465            0 :       FOR_EACH_WIDER_MODE (wider_mode, mode)
    2466              :         {
    2467            0 :           if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
    2468              :             {
    2469            0 :               rtx t0 = gen_reg_rtx (wider_mode);
    2470            0 :               rtx t1 = gen_reg_rtx (wider_mode);
    2471            0 :               rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
    2472              : 
    2473            0 :               if (expand_twoval_unop (unoptab, cop0, t0, t1, unsignedp))
    2474              :                 {
    2475            0 :                   convert_move (targ0, t0, unsignedp);
    2476            0 :                   convert_move (targ1, t1, unsignedp);
    2477            0 :                   return true;
    2478              :                 }
    2479              :               else
    2480            0 :                 delete_insns_since (last);
    2481              :             }
    2482              :         }
    2483              :     }
    2484              : 
    2485            0 :   delete_insns_since (entry_last);
    2486            0 :   return false;
    2487              : }
    2488              : 
    2489              : /* Generate code to perform an operation specified by BINOPTAB
    2490              :    on operands OP0 and OP1, with two results to TARG1 and TARG2.
    2491              :    We assume that the order of the operands for the instruction
    2492              :    is TARG0, OP0, OP1, TARG1, which would fit a pattern like
    2493              :    [(set TARG0 (operate OP0 OP1)) (set TARG1 (operate ...))].
    2494              : 
    2495              :    Either TARG0 or TARG1 may be zero, but what that means is that
    2496              :    the result is not actually wanted.  We will generate it into
    2497              :    a dummy pseudo-reg and discard it.  They may not both be zero.
    2498              : 
    2499              :    Returns true if this operation can be performed; false if not.  */
    2500              : 
    2501              : bool
    2502       121153 : expand_twoval_binop (optab binoptab, rtx op0, rtx op1, rtx targ0, rtx targ1,
    2503              :                      int unsignedp)
    2504              : {
    2505       121153 :   machine_mode mode = GET_MODE (targ0 ? targ0 : targ1);
    2506       121153 :   enum mode_class mclass;
    2507       121153 :   machine_mode wider_mode;
    2508       121153 :   rtx_insn *entry_last = get_last_insn ();
    2509       121153 :   rtx_insn *last;
    2510              : 
    2511       121153 :   mclass = GET_MODE_CLASS (mode);
    2512              : 
    2513       121153 :   if (!targ0)
    2514        41729 :     targ0 = gen_reg_rtx (mode);
    2515       121153 :   if (!targ1)
    2516        67929 :     targ1 = gen_reg_rtx (mode);
    2517              : 
    2518              :   /* Record where to go back to if we fail.  */
    2519       121153 :   last = get_last_insn ();
    2520              : 
    2521       121153 :   if (optab_handler (binoptab, mode) != CODE_FOR_nothing)
    2522              :     {
    2523       116270 :       class expand_operand ops[4];
    2524       116270 :       enum insn_code icode = optab_handler (binoptab, mode);
    2525       116270 :       machine_mode mode0 = insn_data[icode].operand[1].mode;
    2526       116270 :       machine_mode mode1 = insn_data[icode].operand[2].mode;
    2527       116270 :       rtx xop0 = op0, xop1 = op1;
    2528              : 
    2529              :       /* If we are optimizing, force expensive constants into a register.  */
    2530       116270 :       xop0 = avoid_expensive_constant (mode0, binoptab, 0, xop0, unsignedp);
    2531       116270 :       xop1 = avoid_expensive_constant (mode1, binoptab, 1, xop1, unsignedp);
    2532              : 
    2533       116270 :       create_fixed_operand (&ops[0], targ0);
    2534       116270 :       create_convert_operand_from (&ops[1], xop0, mode, unsignedp);
    2535       116270 :       create_convert_operand_from (&ops[2], xop1, mode, unsignedp);
    2536       116270 :       create_fixed_operand (&ops[3], targ1);
    2537       116270 :       if (maybe_expand_insn (icode, 4, ops))
    2538       116270 :         return true;
    2539            0 :       delete_insns_since (last);
    2540              :     }
    2541              : 
    2542              :   /* It can't be done in this mode.  Can we do it in a wider mode?  */
    2543              : 
    2544         4883 :   if (CLASS_HAS_WIDER_MODES_P (mclass))
    2545              :     {
    2546        15510 :       FOR_EACH_WIDER_MODE (wider_mode, mode)
    2547              :         {
    2548        10627 :           if (optab_handler (binoptab, wider_mode) != CODE_FOR_nothing)
    2549              :             {
    2550            0 :               rtx t0 = gen_reg_rtx (wider_mode);
    2551            0 :               rtx t1 = gen_reg_rtx (wider_mode);
    2552            0 :               rtx cop0 = convert_modes (wider_mode, mode, op0, unsignedp);
    2553            0 :               rtx cop1 = convert_modes (wider_mode, mode, op1, unsignedp);
    2554              : 
    2555            0 :               if (expand_twoval_binop (binoptab, cop0, cop1,
    2556              :                                        t0, t1, unsignedp))
    2557              :                 {
    2558            0 :                   convert_move (targ0, t0, unsignedp);
    2559            0 :                   convert_move (targ1, t1, unsignedp);
    2560            0 :                   return true;
    2561              :                 }
    2562              :               else
    2563            0 :                 delete_insns_since (last);
    2564              :             }
    2565              :         }
    2566              :     }
    2567              : 
    2568         4883 :   delete_insns_since (entry_last);
    2569         4883 :   return false;
    2570              : }
    2571              : 
    2572              : /* Expand the two-valued library call indicated by BINOPTAB, but
    2573              :    preserve only one of the values.  If TARG0 is non-NULL, the first
    2574              :    value is placed into TARG0; otherwise the second value is placed
    2575              :    into TARG1.  Exactly one of TARG0 and TARG1 must be non-NULL.  The
    2576              :    value stored into TARG0 or TARG1 is equivalent to (CODE OP0 OP1).
    2577              :    This routine assumes that the value returned by the library call is
    2578              :    as if the return value was of an integral mode twice as wide as the
    2579              :    mode of OP0.  Returns 1 if the call was successful.  */
    2580              : 
    2581              : bool
    2582            0 : expand_twoval_binop_libfunc (optab binoptab, rtx op0, rtx op1,
    2583              :                              rtx targ0, rtx targ1, enum rtx_code code)
    2584              : {
    2585            0 :   machine_mode mode;
    2586            0 :   machine_mode libval_mode;
    2587            0 :   rtx libval;
    2588            0 :   rtx_insn *insns;
    2589            0 :   rtx libfunc;
    2590              : 
    2591              :   /* Exactly one of TARG0 or TARG1 should be non-NULL.  */
    2592            0 :   gcc_assert (!targ0 != !targ1);
    2593              : 
    2594            0 :   mode = GET_MODE (op0);
    2595            0 :   libfunc = optab_libfunc (binoptab, mode);
    2596            0 :   if (!libfunc)
    2597              :     return false;
    2598              : 
    2599              :   /* The value returned by the library function will have twice as
    2600              :      many bits as the nominal MODE.  */
    2601            0 :   auto return_size = 2 * GET_MODE_BITSIZE (mode);
    2602            0 :   if (!smallest_int_mode_for_size (return_size).exists (&libval_mode))
    2603            0 :     return false;
    2604              : 
    2605            0 :   start_sequence ();
    2606            0 :   libval = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
    2607              :                                     libval_mode,
    2608              :                                     op0, mode,
    2609              :                                     op1, mode);
    2610              :   /* Get the part of VAL containing the value that we want.  */
    2611            0 :   libval = simplify_gen_subreg (mode, libval, libval_mode,
    2612            0 :                                 targ0 ? 0 : GET_MODE_SIZE (mode));
    2613            0 :   insns = end_sequence ();
    2614              :   /* Move the into the desired location.  */
    2615            0 :   emit_libcall_block (insns, targ0 ? targ0 : targ1, libval,
    2616              :                       gen_rtx_fmt_ee (code, mode, op0, op1));
    2617              : 
    2618            0 :   return true;
    2619              : }
    2620              : 
    2621              : 
    2622              : /* Wrapper around expand_unop which takes an rtx code to specify
    2623              :    the operation to perform, not an optab pointer.  All other
    2624              :    arguments are the same.  */
    2625              : rtx
    2626        20005 : expand_simple_unop (machine_mode mode, enum rtx_code code, rtx op0,
    2627              :                     rtx target, int unsignedp)
    2628              : {
    2629        20005 :   optab unop = code_to_optab (code);
    2630        20005 :   gcc_assert (unop);
    2631              : 
    2632        20005 :   return expand_unop (mode, unop, op0, target, unsignedp);
    2633              : }
    2634              : 
    2635              : /* Try calculating
    2636              :         (clz:narrow x)
    2637              :    as
    2638              :         (clz:wide (zero_extend:wide x)) - ((width wide) - (width narrow)).
    2639              : 
    2640              :    A similar operation can be used for clrsb.  UNOPTAB says which operation
    2641              :    we are trying to expand.  */
    2642              : static rtx
    2643          298 : widen_leading (scalar_int_mode mode, rtx op0, rtx target, optab unoptab)
    2644              : {
    2645          298 :   opt_scalar_int_mode wider_mode_iter;
    2646         1223 :   FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
    2647              :     {
    2648          925 :       scalar_int_mode wider_mode = wider_mode_iter.require ();
    2649          925 :       if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
    2650              :         {
    2651            0 :           rtx xop0, temp;
    2652            0 :           rtx_insn *last;
    2653              : 
    2654            0 :           last = get_last_insn ();
    2655              : 
    2656            0 :           if (target == 0)
    2657            0 :             target = gen_reg_rtx (mode);
    2658            0 :           xop0 = widen_operand (op0, wider_mode, mode,
    2659              :                                 unoptab != clrsb_optab, false);
    2660            0 :           temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
    2661              :                               unoptab != clrsb_optab);
    2662            0 :           if (temp != 0)
    2663            0 :             temp = expand_binop
    2664            0 :               (wider_mode, sub_optab, temp,
    2665            0 :                gen_int_mode (GET_MODE_PRECISION (wider_mode)
    2666            0 :                              - GET_MODE_PRECISION (mode),
    2667              :                              wider_mode),
    2668              :                target, true, OPTAB_DIRECT);
    2669            0 :           if (temp == 0)
    2670            0 :             delete_insns_since (last);
    2671              : 
    2672          298 :           return temp;
    2673              :         }
    2674              :     }
    2675              :   return 0;
    2676              : }
    2677              : 
    2678              : /* Attempt to emit (clrsb:mode op0) as
    2679              :    (plus:mode (clz:mode (xor:mode op0 (ashr:mode op0 (const_int prec-1))))
    2680              :               (const_int -1))
    2681              :    if CLZ_DEFINED_VALUE_AT_ZERO (mode, val) is 2 and val is prec,
    2682              :    or as
    2683              :    (clz:mode (ior:mode (xor:mode (ashl:mode op0 (const_int 1))
    2684              :                                  (ashr:mode op0 (const_int prec-1)))
    2685              :                        (const_int 1)))
    2686              :    otherwise.  */
    2687              : 
    2688              : static rtx
    2689           92 : expand_clrsb_using_clz (scalar_int_mode mode, rtx op0, rtx target)
    2690              : {
    2691           92 :   if (optimize_insn_for_size_p ()
    2692           92 :       || optab_handler (clz_optab, mode) == CODE_FOR_nothing)
    2693              :     return NULL_RTX;
    2694              : 
    2695           88 :   start_sequence ();
    2696           88 :   HOST_WIDE_INT val = 0;
    2697           88 :   if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) != 2
    2698           88 :       || val != GET_MODE_PRECISION (mode))
    2699              :     val = 0;
    2700              :   else
    2701              :     val = 1;
    2702              : 
    2703           86 :   rtx temp2 = op0;
    2704           86 :   if (!val)
    2705              :     {
    2706           86 :       temp2 = expand_binop (mode, ashl_optab, op0, const1_rtx,
    2707              :                             NULL_RTX, 0, OPTAB_DIRECT);
    2708           86 :       if (!temp2)
    2709              :         {
    2710            0 :         fail:
    2711            0 :           end_sequence ();
    2712            0 :           return NULL_RTX;
    2713              :         }
    2714              :     }
    2715              : 
    2716           88 :   rtx temp = expand_binop (mode, ashr_optab, op0,
    2717           88 :                            GEN_INT (GET_MODE_PRECISION (mode) - 1),
    2718              :                            NULL_RTX, 0, OPTAB_DIRECT);
    2719           88 :   if (!temp)
    2720            0 :     goto fail;
    2721              : 
    2722           88 :   temp = expand_binop (mode, xor_optab, temp2, temp, NULL_RTX, 0,
    2723              :                        OPTAB_DIRECT);
    2724           88 :   if (!temp)
    2725            0 :     goto fail;
    2726              : 
    2727           88 :   if (!val)
    2728              :     {
    2729           86 :       temp = expand_binop (mode, ior_optab, temp, const1_rtx,
    2730              :                            NULL_RTX, 0, OPTAB_DIRECT);
    2731           86 :       if (!temp)
    2732            0 :         goto fail;
    2733              :     }
    2734           88 :   temp = expand_unop_direct (mode, clz_optab, temp, val ? NULL_RTX : target,
    2735              :                              true);
    2736           88 :   if (!temp)
    2737            0 :     goto fail;
    2738           88 :   if (val)
    2739              :     {
    2740            2 :       temp = expand_binop (mode, add_optab, temp, constm1_rtx,
    2741              :                            target, 0, OPTAB_DIRECT);
    2742            2 :       if (!temp)
    2743            0 :         goto fail;
    2744              :     }
    2745              : 
    2746           88 :   rtx_insn *seq = end_sequence ();
    2747              : 
    2748           88 :   add_equal_note (seq, temp, CLRSB, op0, NULL_RTX, mode);
    2749           88 :   emit_insn (seq);
    2750           88 :   return temp;
    2751              : }
    2752              : 
    2753              : static rtx expand_ffs (scalar_int_mode, rtx, rtx);
    2754              : 
    2755              : /* Try calculating clz, ctz or ffs of a double-word quantity as two clz, ctz or
    2756              :    ffs operations on word-sized quantities, choosing which based on whether the
    2757              :    high (for clz) or low (for ctz and ffs) word is nonzero.  */
    2758              : static rtx
    2759          251 : expand_doubleword_clz_ctz_ffs (scalar_int_mode mode, rtx op0, rtx target,
    2760              :                                optab unoptab)
    2761              : {
    2762          251 :   rtx xop0 = force_reg (mode, op0);
    2763          251 :   rtx subhi = gen_highpart (word_mode, xop0);
    2764          251 :   rtx sublo = gen_lowpart (word_mode, xop0);
    2765          251 :   rtx_code_label *hi0_label = gen_label_rtx ();
    2766          251 :   rtx_code_label *after_label = gen_label_rtx ();
    2767          251 :   rtx_insn *seq;
    2768          251 :   rtx temp, result;
    2769          251 :   int addend = 0;
    2770              : 
    2771              :   /* If we were not given a target, use a word_mode register, not a
    2772              :      'mode' register.  The result will fit, and nobody is expecting
    2773              :      anything bigger (the return type of __builtin_clz* is int).  */
    2774          251 :   if (!target)
    2775            0 :     target = gen_reg_rtx (word_mode);
    2776              : 
    2777              :   /* In any case, write to a word_mode scratch in both branches of the
    2778              :      conditional, so we can ensure there is a single move insn setting
    2779              :      'target' to tag a REG_EQUAL note on.  */
    2780          251 :   result = gen_reg_rtx (word_mode);
    2781              : 
    2782          251 :   if (unoptab != clz_optab)
    2783           45 :     std::swap (subhi, sublo);
    2784              : 
    2785          251 :   start_sequence ();
    2786              : 
    2787              :   /* If the high word is not equal to zero,
    2788              :      then clz of the full value is clz of the high word.  */
    2789          251 :   emit_cmp_and_jump_insns (subhi, CONST0_RTX (word_mode), EQ, 0,
    2790              :                            word_mode, true, hi0_label);
    2791              : 
    2792          251 :   if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
    2793          251 :     temp = expand_unop_direct (word_mode, unoptab, subhi, result, true);
    2794              :   else
    2795              :     {
    2796            0 :       gcc_assert (unoptab == ffs_optab);
    2797            0 :       temp = expand_ffs (word_mode, subhi, result);
    2798              :     }
    2799          251 :   if (!temp)
    2800            0 :     goto fail;
    2801              : 
    2802          251 :   if (temp != result)
    2803            0 :     convert_move (result, temp, true);
    2804              : 
    2805          251 :   emit_jump_insn (targetm.gen_jump (after_label));
    2806          251 :   emit_barrier ();
    2807              : 
    2808              :   /* Else clz of the full value is clz of the low word plus the number
    2809              :      of bits in the high word.  Similarly for ctz/ffs of the high word,
    2810              :      except that ffs should be 0 when both words are zero.  */
    2811          251 :   emit_label (hi0_label);
    2812              : 
    2813          251 :   if (unoptab == ffs_optab)
    2814              :     {
    2815            0 :       convert_move (result, const0_rtx, true);
    2816            0 :       emit_cmp_and_jump_insns (sublo, CONST0_RTX (word_mode), EQ, 0,
    2817              :                                word_mode, true, after_label);
    2818              :     }
    2819              : 
    2820          251 :   if (optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
    2821          251 :     temp = expand_unop_direct (word_mode, unoptab, sublo, NULL_RTX, true);
    2822              :   else
    2823              :     {
    2824            0 :       gcc_assert (unoptab == ffs_optab);
    2825            0 :       temp = expand_unop_direct (word_mode, ctz_optab, sublo, NULL_RTX, true);
    2826            0 :       addend = 1;
    2827              :     }
    2828              : 
    2829          251 :   if (!temp)
    2830            0 :     goto fail;
    2831              : 
    2832          502 :   temp = expand_binop (word_mode, add_optab, temp,
    2833          251 :                        gen_int_mode (GET_MODE_BITSIZE (word_mode) + addend,
    2834              :                                      word_mode),
    2835              :                        result, true, OPTAB_DIRECT);
    2836          251 :   if (!temp)
    2837            0 :     goto fail;
    2838          251 :   if (temp != result)
    2839            0 :     convert_move (result, temp, true);
    2840              : 
    2841          251 :   emit_label (after_label);
    2842          251 :   convert_move (target, result, true);
    2843              : 
    2844          251 :   seq = end_sequence ();
    2845              : 
    2846          251 :   add_equal_note (seq, target, optab_to_code (unoptab), xop0, NULL_RTX, mode);
    2847          251 :   emit_insn (seq);
    2848          251 :   return target;
    2849              : 
    2850            0 :  fail:
    2851            0 :   end_sequence ();
    2852            0 :   return 0;
    2853              : }
    2854              : 
    2855              : /* Try calculating popcount of a double-word quantity as two popcount's of
    2856              :    word-sized quantities and summing up the results.  */
    2857              : static rtx
    2858            0 : expand_doubleword_popcount (scalar_int_mode mode, rtx op0, rtx target)
    2859              : {
    2860            0 :   rtx t0, t1, t;
    2861            0 :   rtx_insn *seq;
    2862              : 
    2863            0 :   start_sequence ();
    2864              : 
    2865            0 :   t0 = expand_unop_direct (word_mode, popcount_optab,
    2866              :                            operand_subword_force (op0, 0, mode), NULL_RTX,
    2867              :                            true);
    2868            0 :   t1 = expand_unop_direct (word_mode, popcount_optab,
    2869              :                            operand_subword_force (op0, 1, mode), NULL_RTX,
    2870              :                            true);
    2871            0 :   if (!t0 || !t1)
    2872              :     {
    2873            0 :       end_sequence ();
    2874            0 :       return NULL_RTX;
    2875              :     }
    2876              : 
    2877              :   /* If we were not given a target, use a word_mode register, not a
    2878              :      'mode' register.  The result will fit, and nobody is expecting
    2879              :      anything bigger (the return type of __builtin_popcount* is int).  */
    2880            0 :   if (!target)
    2881            0 :     target = gen_reg_rtx (word_mode);
    2882              : 
    2883            0 :   t = expand_binop (word_mode, add_optab, t0, t1, target, 0, OPTAB_DIRECT);
    2884              : 
    2885            0 :   seq = end_sequence ();
    2886              : 
    2887            0 :   add_equal_note (seq, t, POPCOUNT, op0, NULL_RTX, mode);
    2888            0 :   emit_insn (seq);
    2889            0 :   return t;
    2890              : }
    2891              : 
    2892              : /* Try calculating
    2893              :         (parity:wide x)
    2894              :    as
    2895              :         (parity:narrow (low (x) ^ high (x))) */
    2896              : static rtx
    2897            0 : expand_doubleword_parity (scalar_int_mode mode, rtx op0, rtx target)
    2898              : {
    2899            0 :   rtx t = expand_binop (word_mode, xor_optab,
    2900              :                         operand_subword_force (op0, 0, mode),
    2901              :                         operand_subword_force (op0, 1, mode),
    2902              :                         NULL_RTX, 0, OPTAB_DIRECT);
    2903            0 :   return expand_unop (word_mode, parity_optab, t, target, true);
    2904              : }
    2905              : 
    2906              : /* Try calculating
    2907              :         (bswap:narrow x)
    2908              :    as
    2909              :         (lshiftrt:wide (bswap:wide x) ((width wide) - (width narrow)))
    2910              :    or similarly for bitreverse.  */
    2911              : static rtx
    2912          272 : widen_bswap_or_bitreverse (scalar_int_mode mode, rtx op0, rtx target,
    2913              :                            optab unoptab)
    2914              : {
    2915          272 :   rtx x;
    2916          272 :   rtx_insn *last;
    2917          272 :   opt_scalar_int_mode wider_mode_iter;
    2918              : 
    2919         1103 :   FOR_EACH_WIDER_MODE (wider_mode_iter, mode)
    2920          831 :     if (optab_handler (unoptab, wider_mode_iter.require ())
    2921              :         != CODE_FOR_nothing)
    2922              :       break;
    2923              : 
    2924          272 :   if (!wider_mode_iter.exists ())
    2925              :     return NULL_RTX;
    2926              : 
    2927            0 :   scalar_int_mode wider_mode = wider_mode_iter.require ();
    2928            0 :   last = get_last_insn ();
    2929              : 
    2930            0 :   x = widen_operand (op0, wider_mode, mode, true, true);
    2931            0 :   x = expand_unop (wider_mode, unoptab, x, NULL_RTX, true);
    2932              : 
    2933            0 :   gcc_assert (GET_MODE_PRECISION (wider_mode) == GET_MODE_BITSIZE (wider_mode)
    2934              :               && GET_MODE_PRECISION (mode) == GET_MODE_BITSIZE (mode));
    2935            0 :   if (x != 0)
    2936            0 :     x = expand_shift (RSHIFT_EXPR, wider_mode, x,
    2937            0 :                       GET_MODE_BITSIZE (wider_mode)
    2938            0 :                       - GET_MODE_BITSIZE (mode),
    2939              :                       NULL_RTX, true);
    2940              : 
    2941            0 :   if (x != 0)
    2942              :     {
    2943            0 :       if (target == 0)
    2944            0 :         target = gen_reg_rtx (mode);
    2945            0 :       emit_move_insn (target, gen_lowpart (mode, x));
    2946              :     }
    2947              :   else
    2948            0 :     delete_insns_since (last);
    2949              : 
    2950              :   return target;
    2951              : }
    2952              : 
    2953              : /* Try calculating bswap as two bswaps of two word-sized operands.
    2954              :    Similarly for bitreverse.  */
    2955              : 
    2956              : static rtx
    2957          163 : expand_doubleword_bswap_or_bitreverse (machine_mode mode, rtx op, rtx target,
    2958              :                                        optab unoptab)
    2959              : {
    2960          163 :   rtx t0, t1;
    2961              : 
    2962          163 :   t1 = expand_unop (word_mode, unoptab,
    2963              :                     operand_subword_force (op, 0, mode), NULL_RTX, true);
    2964          163 :   t0 = expand_unop (word_mode, unoptab,
    2965              :                     operand_subword_force (op, 1, mode), NULL_RTX, true);
    2966              : 
    2967          163 :   if (target == 0 || !valid_multiword_target_p (target))
    2968            6 :     target = gen_reg_rtx (mode);
    2969          163 :   if (REG_P (target))
    2970          163 :     emit_clobber (target);
    2971          163 :   emit_move_insn (operand_subword (target, 0, 1, mode), t0);
    2972          163 :   emit_move_insn (operand_subword (target, 1, 1, mode), t1);
    2973              : 
    2974          163 :   return target;
    2975              : }
    2976              : 
    2977              : /* Try calculating (bitreverse x) using masks and shifts.  */
    2978              : 
    2979              : static rtx
    2980          109 : expand_bitreverse (scalar_int_mode mode, rtx op0, rtx target)
    2981              : {
    2982          109 :   unsigned int precision = GET_MODE_BITSIZE (mode);
    2983          109 :   rtx_insn *last;
    2984              : 
    2985              :   /* Operation requires at least 4 bits (one nibble swap makes no sense below
    2986              :      that).  */
    2987          109 :   if (precision < 4)
    2988              :     return NULL_RTX;
    2989              : 
    2990          109 :   if (rtx temp = widen_bswap_or_bitreverse (mode, op0, target,
    2991              :                                             bitreverse_optab))
    2992              :     return temp;
    2993              : 
    2994          218 :   if (GET_MODE_SIZE (mode) == 2 * UNITS_PER_WORD
    2995          109 :       && optab_handler (bitreverse_optab, word_mode) != CODE_FOR_nothing)
    2996            0 :     if (rtx temp = expand_doubleword_bswap_or_bitreverse (mode, op0, target,
    2997              :                                                           bitreverse_optab))
    2998            0 :       return temp;
    2999              : 
    3000          109 :   if (target == NULL_RTX
    3001          109 :       || target == op0
    3002          109 :       || reg_overlap_mentioned_p (target, op0))
    3003            0 :     target = gen_reg_rtx (mode);
    3004              : 
    3005          109 :   last = get_last_insn ();
    3006              : 
    3007          109 :   rtx x, lo, hi;
    3008              : 
    3009              :   /* Step 1: byte-swap (only meaningful for >= 16 bits).  */
    3010          109 :   if (precision >= 16)
    3011              :     {
    3012          104 :       x = expand_unop (mode, bswap_optab, op0, NULL_RTX, true);
    3013          104 :       if (x == NULL_RTX)
    3014            0 :         goto fail;
    3015              :     }
    3016              :   else
    3017              :     x = op0;
    3018              : 
    3019              :   /* Step 2: swap nibbles within each byte (shift=4, only for >= 8 bits).  */
    3020          109 :   if (precision >= 8)
    3021              :     {
    3022          109 :       wide_int mask = wi::zero (precision);
    3023          912 :       for (unsigned int start = 0; start < precision; start += 8)
    3024         1606 :         mask = wi::bit_or (mask, wi::shifted_mask (start, 4, false,
    3025          803 :                                                    precision));
    3026              : 
    3027          109 :       rtx mask_rtx = immed_wide_int_const (mask, mode);
    3028              : 
    3029          109 :       hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (4),
    3030              :                                 NULL_RTX, true, OPTAB_LIB_WIDEN);
    3031          109 :       if (hi == NULL_RTX) goto fail;
    3032          109 :       hi = expand_binop (mode, and_optab, hi, mask_rtx,
    3033              :                          NULL_RTX, true, OPTAB_LIB_WIDEN);
    3034          109 :       if (hi == NULL_RTX) goto fail;
    3035              : 
    3036          109 :       lo = expand_binop (mode, and_optab, x, mask_rtx,
    3037              :                          NULL_RTX, true, OPTAB_LIB_WIDEN);
    3038          109 :       if (lo == NULL_RTX) goto fail;
    3039          109 :       lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (4),
    3040              :                                 NULL_RTX, true, OPTAB_LIB_WIDEN);
    3041          109 :       if (lo == NULL_RTX) goto fail;
    3042              : 
    3043          109 :       x = expand_binop (mode, aop_optab, hi, lo,
    3044              :                         NULL_RTX, true, OPTAB_LIB_WIDEN);
    3045          109 :       if (x == NULL_RTX) goto fail;
    3046            0 :     }
    3047              : 
    3048              :   /* Step 3: swap pairs of bits within each nibble (shift=2).  */
    3049          109 :   {
    3050          109 :     wide_int mask = wi::zero (precision);
    3051         1715 :     for (unsigned int start = 0; start < precision; start += 4)
    3052         1606 :       mask = wi::bit_or (mask, wi::shifted_mask (start, 2, false, precision));
    3053              : 
    3054          109 :     rtx mask_rtx = immed_wide_int_const (mask, mode);
    3055              : 
    3056          109 :     hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (2),
    3057              :                               NULL_RTX, true, OPTAB_LIB_WIDEN);
    3058          109 :     if (hi == NULL_RTX) goto fail;
    3059          109 :     hi = expand_binop (mode, and_optab, hi, mask_rtx,
    3060              :                        NULL_RTX, true, OPTAB_LIB_WIDEN);
    3061          109 :     if (hi == NULL_RTX) goto fail;
    3062              : 
    3063          109 :     lo = expand_binop (mode, and_optab, x, mask_rtx,
    3064              :                        NULL_RTX, true, OPTAB_LIB_WIDEN);
    3065          109 :     if (lo == NULL_RTX) goto fail;
    3066          109 :     lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (2),
    3067              :                               NULL_RTX, true, OPTAB_LIB_WIDEN);
    3068          109 :     if (lo == NULL_RTX) goto fail;
    3069              : 
    3070          109 :     x = expand_binop (mode, aop_optab, hi, lo,
    3071              :                       NULL_RTX, true, OPTAB_LIB_WIDEN);
    3072          109 :     if (x == NULL_RTX) goto fail;
    3073            0 :   }
    3074              : 
    3075              :   /* Step 4: swap adjacent bits (shift=1).  */
    3076          109 :   {
    3077          109 :     wide_int mask = wi::zero (precision);
    3078         3321 :     for (unsigned int start = 0; start < precision; start += 2)
    3079         6424 :       mask = wi::bit_or (mask, wi::shifted_mask (start, 1, false,
    3080         3212 :                                                  precision));
    3081              : 
    3082          109 :     rtx mask_rtx = immed_wide_int_const (mask, mode);
    3083              : 
    3084          109 :     hi = expand_simple_binop (mode, LSHIFTRT, x, GEN_INT (1),
    3085              :                               NULL_RTX, true, OPTAB_LIB_WIDEN);
    3086          109 :     if (hi == NULL_RTX) goto fail;
    3087          109 :     hi = expand_binop (mode, and_optab, hi, mask_rtx,
    3088              :                        NULL_RTX, true, OPTAB_LIB_WIDEN);
    3089          109 :     if (hi == NULL_RTX) goto fail;
    3090              : 
    3091          109 :     lo = expand_binop (mode, and_optab, x, mask_rtx,
    3092              :                        NULL_RTX, true, OPTAB_LIB_WIDEN);
    3093          109 :     if (lo == NULL_RTX) goto fail;
    3094          109 :     lo = expand_simple_binop (mode, ASHIFT, lo, GEN_INT (1),
    3095              :                               NULL_RTX, true, OPTAB_LIB_WIDEN);
    3096          109 :     if (lo == NULL_RTX) goto fail;
    3097              : 
    3098          109 :     x = expand_binop (mode, aop_optab, hi, lo,
    3099              :                       target, true, OPTAB_LIB_WIDEN);
    3100          109 :     if (x == NULL_RTX) goto fail;
    3101            0 :   }
    3102              : 
    3103          109 :   if (x != target)
    3104            0 :     emit_move_insn (target, x);
    3105              : 
    3106              :   return target;
    3107              : 
    3108            0 :  fail:
    3109            0 :   delete_insns_since (last);
    3110            0 :   return NULL_RTX;
    3111              : }
    3112              : 
    3113              : /* Try calculating (parity x) as (and (popcount x) 1), where
    3114              :    popcount can also be done in a wider mode.  */
    3115              : static rtx
    3116           11 : expand_parity (scalar_int_mode mode, rtx op0, rtx target)
    3117              : {
    3118           11 :   enum mode_class mclass = GET_MODE_CLASS (mode);
    3119           11 :   opt_scalar_int_mode wider_mode_iter;
    3120           11 :   FOR_EACH_MODE_FROM (wider_mode_iter, mode)
    3121              :     {
    3122           11 :       scalar_int_mode wider_mode = wider_mode_iter.require ();
    3123           11 :       if (optab_handler (popcount_optab, wider_mode) != CODE_FOR_nothing)
    3124              :         {
    3125           11 :           rtx xop0, temp;
    3126           11 :           rtx_insn *last;
    3127              : 
    3128           11 :           last = get_last_insn ();
    3129              : 
    3130           11 :           if (target == 0 || GET_MODE (target) != wider_mode)
    3131            6 :             target = gen_reg_rtx (wider_mode);
    3132              : 
    3133           11 :           xop0 = widen_operand (op0, wider_mode, mode, true, false);
    3134           11 :           temp = expand_unop (wider_mode, popcount_optab, xop0, NULL_RTX,
    3135              :                               true);
    3136           11 :           if (temp != 0)
    3137           11 :             temp = expand_binop (wider_mode, and_optab, temp, const1_rtx,
    3138              :                                  target, true, OPTAB_DIRECT);
    3139              : 
    3140           11 :           if (temp)
    3141              :             {
    3142           11 :               if (mclass != MODE_INT
    3143           11 :                   || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
    3144            0 :                 return convert_to_mode (mode, temp, 0);
    3145              :               else
    3146           11 :                 return gen_lowpart (mode, temp);
    3147              :             }
    3148              :           else
    3149            0 :             delete_insns_since (last);
    3150              :         }
    3151              :     }
    3152              :   return 0;
    3153              : }
    3154              : 
    3155              : /* Try calculating ctz(x) as K - clz(x & -x) ,
    3156              :    where K is GET_MODE_PRECISION(mode) - 1.
    3157              : 
    3158              :    Both __builtin_ctz and __builtin_clz are undefined at zero, so we
    3159              :    don't have to worry about what the hardware does in that case.  (If
    3160              :    the clz instruction produces the usual value at 0, which is K, the
    3161              :    result of this code sequence will be -1; expand_ffs, below, relies
    3162              :    on this.  It might be nice to have it be K instead, for consistency
    3163              :    with the (very few) processors that provide a ctz with a defined
    3164              :    value, but that would take one more instruction, and it would be
    3165              :    less convenient for expand_ffs anyway.  */
    3166              : 
    3167              : static rtx
    3168           48 : expand_ctz (scalar_int_mode mode, rtx op0, rtx target)
    3169              : {
    3170           48 :   rtx_insn *seq;
    3171           48 :   rtx temp;
    3172              : 
    3173           48 :   if (optab_handler (clz_optab, mode) == CODE_FOR_nothing)
    3174              :     return 0;
    3175              : 
    3176            0 :   start_sequence ();
    3177              : 
    3178            0 :   temp = expand_unop_direct (mode, neg_optab, op0, NULL_RTX, true);
    3179            0 :   if (temp)
    3180            0 :     temp = expand_binop (mode, and_optab, op0, temp, NULL_RTX,
    3181              :                          true, OPTAB_DIRECT);
    3182            0 :   if (temp)
    3183            0 :     temp = expand_unop_direct (mode, clz_optab, temp, NULL_RTX, true);
    3184            0 :   if (temp)
    3185            0 :     temp = expand_binop (mode, sub_optab,
    3186            0 :                          gen_int_mode (GET_MODE_PRECISION (mode) - 1, mode),
    3187              :                          temp, target,
    3188              :                          true, OPTAB_DIRECT);
    3189            0 :   if (temp == 0)
    3190              :     {
    3191            0 :       end_sequence ();
    3192            0 :       return 0;
    3193              :     }
    3194              : 
    3195            0 :   seq = end_sequence ();
    3196              : 
    3197            0 :   add_equal_note (seq, temp, CTZ, op0, NULL_RTX, mode);
    3198            0 :   emit_insn (seq);
    3199            0 :   return temp;
    3200              : }
    3201              : 
    3202              : 
    3203              : /* Try calculating ffs(x) using ctz(x) if we have that instruction, or
    3204              :    else with the sequence used by expand_clz.
    3205              : 
    3206              :    The ffs builtin promises to return zero for a zero value and ctz/clz
    3207              :    may have an undefined value in that case.  If they do not give us a
    3208              :    convenient value, we have to generate a test and branch.  */
    3209              : static rtx
    3210            0 : expand_ffs (scalar_int_mode mode, rtx op0, rtx target)
    3211              : {
    3212            0 :   HOST_WIDE_INT val = 0;
    3213            0 :   bool defined_at_zero = false;
    3214            0 :   rtx temp;
    3215            0 :   rtx_insn *seq;
    3216              : 
    3217            0 :   if (optab_handler (ctz_optab, mode) != CODE_FOR_nothing)
    3218              :     {
    3219            0 :       start_sequence ();
    3220              : 
    3221            0 :       temp = expand_unop_direct (mode, ctz_optab, op0, 0, true);
    3222            0 :       if (!temp)
    3223            0 :         goto fail;
    3224              : 
    3225            0 :       defined_at_zero = (CTZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2);
    3226              :     }
    3227            0 :   else if (optab_handler (clz_optab, mode) != CODE_FOR_nothing)
    3228              :     {
    3229            0 :       start_sequence ();
    3230            0 :       temp = expand_ctz (mode, op0, 0);
    3231            0 :       if (!temp)
    3232            0 :         goto fail;
    3233              : 
    3234            0 :       if (CLZ_DEFINED_VALUE_AT_ZERO (mode, val) == 2)
    3235              :         {
    3236            0 :           defined_at_zero = true;
    3237            0 :           val = (GET_MODE_PRECISION (mode) - 1) - val;
    3238              :         }
    3239              :     }
    3240              :   else
    3241              :     return 0;
    3242              : 
    3243            0 :   if (defined_at_zero && val == -1)
    3244              :     /* No correction needed at zero.  */;
    3245              :   else
    3246              :     {
    3247              :       /* We don't try to do anything clever with the situation found
    3248              :          on some processors (eg Alpha) where ctz(0:mode) ==
    3249              :          bitsize(mode).  If someone can think of a way to send N to -1
    3250              :          and leave alone all values in the range 0..N-1 (where N is a
    3251              :          power of two), cheaper than this test-and-branch, please add it.
    3252              : 
    3253              :          The test-and-branch is done after the operation itself, in case
    3254              :          the operation sets condition codes that can be recycled for this.
    3255              :          (This is true on i386, for instance.)  */
    3256              : 
    3257            0 :       rtx_code_label *nonzero_label = gen_label_rtx ();
    3258            0 :       emit_cmp_and_jump_insns (op0, CONST0_RTX (mode), NE, 0,
    3259              :                                mode, true, nonzero_label);
    3260              : 
    3261            0 :       convert_move (temp, GEN_INT (-1), false);
    3262            0 :       emit_label (nonzero_label);
    3263              :     }
    3264              : 
    3265              :   /* temp now has a value in the range -1..bitsize-1.  ffs is supposed
    3266              :      to produce a value in the range 0..bitsize.  */
    3267            0 :   temp = expand_binop (mode, add_optab, temp, gen_int_mode (1, mode),
    3268              :                        target, false, OPTAB_DIRECT);
    3269            0 :   if (!temp)
    3270            0 :     goto fail;
    3271              : 
    3272            0 :   seq = end_sequence ();
    3273              : 
    3274            0 :   add_equal_note (seq, temp, FFS, op0, NULL_RTX, mode);
    3275            0 :   emit_insn (seq);
    3276            0 :   return temp;
    3277              : 
    3278            0 :  fail:
    3279            0 :   end_sequence ();
    3280            0 :   return 0;
    3281              : }
    3282              : 
    3283              : /* Expand a floating point absolute value or negation operation via a
    3284              :    logical operation on the sign bit.  MODE is the mode of the operands
    3285              :    and FMODE is the scalar inner mode.  */
    3286              : 
    3287              : static rtx
    3288         2400 : expand_absneg_bit (rtx_code code, machine_mode mode,
    3289              :                    scalar_float_mode fmode, rtx op0, rtx target)
    3290              : {
    3291         2400 :   int bitpos, word, nwords, i;
    3292         2400 :   machine_mode new_mode;
    3293         2400 :   scalar_int_mode imode;
    3294         2400 :   rtx temp;
    3295         2400 :   rtx_insn *insns;
    3296              : 
    3297         2400 :   auto op = code == NEG ? neg_optab : abs_optab;
    3298         2400 :   if (!get_absneg_bit_mode (op, mode, fmode, &bitpos).exists (&new_mode))
    3299            0 :     return NULL_RTX;
    3300              : 
    3301         4800 :   imode = as_a<scalar_int_mode> (GET_MODE_INNER (new_mode));
    3302         4868 :   if (VECTOR_MODE_P (mode) || GET_MODE_SIZE (fmode) <= UNITS_PER_WORD)
    3303              :     {
    3304              :       word = 0;
    3305              :       nwords = 1;
    3306              :     }
    3307              :   else
    3308              :     {
    3309           17 :       if (FLOAT_WORDS_BIG_ENDIAN)
    3310              :         word = (GET_MODE_BITSIZE (fmode) - bitpos) / BITS_PER_WORD;
    3311              :       else
    3312           17 :         word = bitpos / BITS_PER_WORD;
    3313           17 :       bitpos = bitpos % BITS_PER_WORD;
    3314           34 :       nwords = (GET_MODE_BITSIZE (fmode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
    3315              :     }
    3316              : 
    3317         2400 :   wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
    3318         2400 :   if (code == ABS)
    3319         2110 :     mask = ~mask;
    3320              : 
    3321         2400 :   if (target == 0
    3322         2400 :       || target == op0
    3323          684 :       || reg_overlap_mentioned_p (target, op0)
    3324         3080 :       || (nwords > 1 && !valid_multiword_target_p (target)))
    3325         1720 :     target = gen_reg_rtx (mode);
    3326              : 
    3327         2400 :   if (nwords > 1)
    3328              :     {
    3329           17 :       start_sequence ();
    3330              : 
    3331           68 :       for (i = 0; i < nwords; ++i)
    3332              :         {
    3333           34 :           rtx targ_piece = operand_subword (target, i, 1, mode);
    3334           34 :           rtx op0_piece = operand_subword_force (op0, i, mode);
    3335              : 
    3336           34 :           if (i == word)
    3337              :             {
    3338           46 :               temp = expand_binop (imode, code == ABS ? and_optab : xor_optab,
    3339              :                                    op0_piece,
    3340           17 :                                    immed_wide_int_const (mask, imode),
    3341              :                                    targ_piece, 1, OPTAB_LIB_WIDEN);
    3342           17 :               if (temp != targ_piece)
    3343            0 :                 emit_move_insn (targ_piece, temp);
    3344              :             }
    3345              :           else
    3346           17 :             emit_move_insn (targ_piece, op0_piece);
    3347              :         }
    3348              : 
    3349           17 :       insns = end_sequence ();
    3350              : 
    3351           17 :       emit_insn (insns);
    3352              :     }
    3353              :   else
    3354              :     {
    3355         2383 :       rtx mask_rtx = immed_wide_int_const (mask, imode);
    3356         2383 :       if (VECTOR_MODE_P (new_mode))
    3357            0 :         mask_rtx = gen_const_vec_duplicate (new_mode, mask_rtx);
    3358         2661 :       temp = expand_binop (new_mode, code == ABS ? and_optab : xor_optab,
    3359         2383 :                            gen_lowpart (new_mode, op0), mask_rtx,
    3360         2383 :                            gen_lowpart (new_mode, target), 1, OPTAB_LIB_WIDEN);
    3361         2383 :       target = force_lowpart_subreg (mode, temp, new_mode);
    3362              : 
    3363         2383 :       set_dst_reg_note (get_last_insn (), REG_EQUAL,
    3364              :                         gen_rtx_fmt_e (code, mode, copy_rtx (op0)),
    3365              :                         target);
    3366              :     }
    3367              : 
    3368         2400 :   return target;
    3369         2400 : }
    3370              : 
    3371              : /* As expand_unop, but will fail rather than attempt the operation in a
    3372              :    different mode or with a libcall.  */
    3373              : static rtx
    3374       186528 : expand_unop_direct (machine_mode mode, optab unoptab, rtx op0, rtx target,
    3375              :                     int unsignedp)
    3376              : {
    3377       186528 :   if (optab_handler (unoptab, mode) != CODE_FOR_nothing)
    3378              :     {
    3379       182585 :       class expand_operand ops[2];
    3380       182585 :       enum insn_code icode = optab_handler (unoptab, mode);
    3381       182585 :       rtx_insn *last = get_last_insn ();
    3382       182585 :       rtx_insn *pat;
    3383              : 
    3384       182585 :       create_output_operand (&ops[0], target, mode);
    3385       182585 :       create_convert_operand_from (&ops[1], op0, mode, unsignedp);
    3386       182585 :       pat = maybe_gen_insn (icode, 2, ops);
    3387       182585 :       if (pat)
    3388              :         {
    3389       181965 :           if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
    3390       233787 :               && ! add_equal_note (pat, ops[0].value,
    3391              :                                    optab_to_code (unoptab),
    3392              :                                    ops[1].value, NULL_RTX, mode))
    3393              :             {
    3394         1089 :               delete_insns_since (last);
    3395       182585 :               return expand_unop (mode, unoptab, op0, NULL_RTX, unsignedp);
    3396              :             }
    3397              : 
    3398       181496 :           emit_insn (pat);
    3399              : 
    3400       181496 :           return ops[0].value;
    3401              :         }
    3402              :     }
    3403              :   return 0;
    3404              : }
    3405              : 
    3406              : /* Generate code to perform an operation specified by UNOPTAB
    3407              :    on operand OP0, with result having machine-mode MODE.
    3408              : 
    3409              :    UNSIGNEDP is for the case where we have to widen the operands
    3410              :    to perform the operation.  It says to use zero-extension.
    3411              : 
    3412              :    If TARGET is nonzero, the value
    3413              :    is generated there, if it is convenient to do so.
    3414              :    In all cases an rtx is returned for the locus of the value;
    3415              :    this may or may not be TARGET.  */
    3416              : 
    3417              : rtx
    3418       185938 : expand_unop (machine_mode mode, optab unoptab, rtx op0, rtx target,
    3419              :              int unsignedp)
    3420              : {
    3421       185938 :   enum mode_class mclass = GET_MODE_CLASS (mode);
    3422       185938 :   machine_mode wider_mode;
    3423       185938 :   scalar_int_mode int_mode;
    3424       185938 :   scalar_float_mode float_mode;
    3425       185938 :   rtx temp;
    3426       185938 :   rtx libfunc;
    3427              : 
    3428       185938 :   temp = expand_unop_direct (mode, unoptab, op0, target, unsignedp);
    3429       185938 :   if (temp)
    3430              :     return temp;
    3431              : 
    3432              :   /* It can't be done in this mode.  Can we open-code it in a wider mode?  */
    3433              : 
    3434              :   /* Widening (or narrowing) clz needs special treatment.  */
    3435         3943 :   if (unoptab == clz_optab)
    3436              :     {
    3437          206 :       if (is_a <scalar_int_mode> (mode, &int_mode))
    3438              :         {
    3439          206 :           temp = widen_leading (int_mode, op0, target, unoptab);
    3440          206 :           if (temp)
    3441              :             return temp;
    3442              : 
    3443          412 :           if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
    3444          206 :               && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
    3445              :             {
    3446          206 :               temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target,
    3447              :                                                     unoptab);
    3448          206 :               if (temp)
    3449              :                 return temp;
    3450              :             }
    3451              :         }
    3452              : 
    3453            0 :       goto try_libcall;
    3454              :     }
    3455              : 
    3456         3737 :   if (unoptab == clrsb_optab)
    3457              :     {
    3458           92 :       if (is_a <scalar_int_mode> (mode, &int_mode))
    3459              :         {
    3460           92 :           temp = widen_leading (int_mode, op0, target, unoptab);
    3461           92 :           if (temp)
    3462              :             return temp;
    3463           92 :           temp = expand_clrsb_using_clz (int_mode, op0, target);
    3464           92 :           if (temp)
    3465              :             return temp;
    3466              :         }
    3467            4 :       goto try_libcall;
    3468              :     }
    3469              : 
    3470         3645 :   if (unoptab == popcount_optab
    3471          733 :       && is_a <scalar_int_mode> (mode, &int_mode)
    3472          765 :       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
    3473            2 :       && optab_handler (unoptab, word_mode) != CODE_FOR_nothing
    3474         3645 :       && optimize_insn_for_speed_p ())
    3475              :     {
    3476            0 :       temp = expand_doubleword_popcount (int_mode, op0, target);
    3477            0 :       if (temp)
    3478              :         return temp;
    3479              :     }
    3480              : 
    3481         3645 :   if (unoptab == parity_optab
    3482           11 :       && is_a <scalar_int_mode> (mode, &int_mode)
    3483           11 :       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
    3484            0 :       && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
    3485            0 :           || optab_handler (popcount_optab, word_mode) != CODE_FOR_nothing)
    3486         3645 :       && optimize_insn_for_speed_p ())
    3487              :     {
    3488            0 :       temp = expand_doubleword_parity (int_mode, op0, target);
    3489            0 :       if (temp)
    3490              :         return temp;
    3491              :     }
    3492              : 
    3493              :   /* Widening (or narrowing) bswap needs special treatment.  */
    3494         3645 :   if (unoptab == bswap_optab)
    3495              :     {
    3496              :       /* HImode is special because in this mode BSWAP is equivalent to ROTATE
    3497              :          or ROTATERT.  First try these directly; if this fails, then try the
    3498              :          obvious pair of shifts with allowed widening, as this will probably
    3499              :          be always more efficient than the other fallback methods.  */
    3500          163 :       if (mode == HImode)
    3501              :         {
    3502            0 :           rtx_insn *last;
    3503            0 :           rtx temp1, temp2;
    3504              : 
    3505            0 :           if (optab_handler (rotl_optab, mode) != CODE_FOR_nothing)
    3506              :             {
    3507            0 :               temp = expand_binop (mode, rotl_optab, op0,
    3508              :                                    gen_int_shift_amount (mode, 8),
    3509              :                                    target, unsignedp, OPTAB_DIRECT);
    3510            0 :               if (temp)
    3511              :                 return temp;
    3512              :              }
    3513              : 
    3514            0 :           if (optab_handler (rotr_optab, mode) != CODE_FOR_nothing)
    3515              :             {
    3516            0 :               temp = expand_binop (mode, rotr_optab, op0,
    3517              :                                    gen_int_shift_amount (mode, 8),
    3518              :                                    target, unsignedp, OPTAB_DIRECT);
    3519            0 :               if (temp)
    3520              :                 return temp;
    3521              :             }
    3522              : 
    3523            0 :           last = get_last_insn ();
    3524              : 
    3525            0 :           temp1 = expand_binop (mode, ashl_optab, op0,
    3526              :                                 gen_int_shift_amount (mode, 8), NULL_RTX,
    3527              :                                 unsignedp, OPTAB_WIDEN);
    3528            0 :           temp2 = expand_binop (mode, lshr_optab, op0,
    3529              :                                 gen_int_shift_amount (mode, 8), NULL_RTX,
    3530              :                                 unsignedp, OPTAB_WIDEN);
    3531            0 :           if (temp1 && temp2)
    3532              :             {
    3533            0 :               temp = expand_binop (mode, ior_optab, temp1, temp2, target,
    3534              :                                    unsignedp, OPTAB_WIDEN);
    3535            0 :               if (temp)
    3536              :                 return temp;
    3537              :             }
    3538              : 
    3539            0 :           delete_insns_since (last);
    3540              :         }
    3541              : 
    3542          163 :       if (is_a <scalar_int_mode> (mode, &int_mode))
    3543              :         {
    3544          163 :           temp = widen_bswap_or_bitreverse (int_mode, op0, target, unoptab);
    3545          163 :           if (temp)
    3546              :             return temp;
    3547              : 
    3548              :           /* We do not provide a 128-bit bswap in libgcc so force the use of
    3549              :              a double bswap for 64-bit targets.  */
    3550          326 :           if (GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
    3551          163 :               && (UNITS_PER_WORD == 8
    3552          147 :                   || optab_handler (unoptab, word_mode) != CODE_FOR_nothing))
    3553              :             {
    3554          163 :               temp = expand_doubleword_bswap_or_bitreverse (mode, op0, target,
    3555              :                                                             unoptab);
    3556          163 :               if (temp)
    3557              :                 return temp;
    3558              :             }
    3559              :         }
    3560              : 
    3561            0 :       goto try_libcall;
    3562              :     }
    3563              : 
    3564         3482 :   if (unoptab == bitreverse_optab && is_a <scalar_int_mode> (mode, &int_mode))
    3565          109 :     if (rtx tem = expand_bitreverse (int_mode, op0, target))
    3566              :       return tem;
    3567              : 
    3568              :   /* Neg should be tried via expand_absneg_bit before widening.  */
    3569         3373 :   if (optab_to_code (unoptab) == NEG)
    3570              :     {
    3571              :       /* Try negating floating point values by flipping the sign bit.  */
    3572          766 :       if (is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
    3573              :         {
    3574          290 :           temp = expand_absneg_bit (NEG, mode, float_mode, op0, target);
    3575          290 :           if (temp)
    3576              :             return temp;
    3577              :         }
    3578              : 
    3579              :       /* If there is no negation pattern, and we have no negative zero,
    3580              :          try subtracting from zero.  */
    3581           93 :       if (!HONOR_SIGNED_ZEROS (mode))
    3582              :         {
    3583           93 :           temp = expand_binop (mode, (unoptab == negv_optab
    3584              :                                       ? subv_optab : sub_optab),
    3585              :                                CONST0_RTX (mode), op0, target,
    3586              :                                unsignedp, OPTAB_DIRECT);
    3587           93 :           if (temp)
    3588              :             return temp;
    3589              :         }
    3590              :     }
    3591              : 
    3592              :   /* ABS also needs to be handled similarly.  */
    3593         3083 :   if (optab_to_code (unoptab) == ABS
    3594         5281 :       && is_a <scalar_float_mode> (GET_MODE_INNER (mode), &float_mode))
    3595              :     {
    3596         2110 :       temp = expand_absneg_bit (ABS, mode, float_mode, op0, target);
    3597         2110 :       if (temp)
    3598              :         return temp;
    3599              :     }
    3600              : 
    3601          973 :   if (CLASS_HAS_WIDER_MODES_P (mclass))
    3602         4346 :     FOR_EACH_WIDER_MODE (wider_mode, mode)
    3603              :       {
    3604         3373 :         if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing)
    3605              :           {
    3606            0 :             rtx xop0 = op0;
    3607            0 :             rtx_insn *last = get_last_insn ();
    3608              : 
    3609              :             /* For certain operations, we need not actually extend
    3610              :                the narrow operand, as long as we will truncate the
    3611              :                results to the same narrowness.  */
    3612              : 
    3613            0 :             xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
    3614            0 :                                   (unoptab == neg_optab
    3615            0 :                                    || unoptab == one_cmpl_optab)
    3616              :                                   && mclass == MODE_INT);
    3617              : 
    3618            0 :             temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
    3619              :                                 unsignedp);
    3620              : 
    3621            0 :             if (temp)
    3622              :               {
    3623            0 :                 if (mclass != MODE_INT
    3624            0 :                     || !TRULY_NOOP_TRUNCATION_MODES_P (mode, wider_mode))
    3625              :                   {
    3626            0 :                     if (target == 0)
    3627            0 :                       target = gen_reg_rtx (mode);
    3628            0 :                     convert_move (target, temp, 0);
    3629            0 :                     return target;
    3630              :                   }
    3631              :                 else
    3632            0 :                   return gen_lowpart (mode, temp);
    3633              :               }
    3634              :             else
    3635            0 :               delete_insns_since (last);
    3636              :           }
    3637              :       }
    3638              : 
    3639              :   /* These can be done a word at a time.  */
    3640          973 :   if (unoptab == one_cmpl_optab
    3641            0 :       && is_int_mode (mode, &int_mode)
    3642            0 :       && GET_MODE_SIZE (int_mode) > UNITS_PER_WORD
    3643          973 :       && optab_handler (unoptab, word_mode) != CODE_FOR_nothing)
    3644              :     {
    3645            0 :       int i;
    3646            0 :       rtx_insn *insns;
    3647              : 
    3648            0 :       if (target == 0
    3649            0 :           || target == op0
    3650            0 :           || reg_overlap_mentioned_p (target, op0)
    3651            0 :           || !valid_multiword_target_p (target))
    3652            0 :         target = gen_reg_rtx (int_mode);
    3653              : 
    3654            0 :       start_sequence ();
    3655              : 
    3656              :       /* Do the actual arithmetic.  */
    3657            0 :       for (i = 0; i < GET_MODE_BITSIZE (int_mode) / BITS_PER_WORD; i++)
    3658              :         {
    3659            0 :           rtx target_piece = operand_subword (target, i, 1, int_mode);
    3660            0 :           rtx x = expand_unop (word_mode, unoptab,
    3661            0 :                                operand_subword_force (op0, i, int_mode),
    3662              :                                target_piece, unsignedp);
    3663              : 
    3664            0 :           if (target_piece != x)
    3665            0 :             emit_move_insn (target_piece, x);
    3666              :         }
    3667              : 
    3668            0 :       insns = end_sequence ();
    3669              : 
    3670            0 :       emit_insn (insns);
    3671            0 :       return target;
    3672              :     }
    3673              : 
    3674              :   /* Emit ~op0 as op0 ^ -1.  */
    3675          973 :   if (unoptab == one_cmpl_optab
    3676            0 :       && (SCALAR_INT_MODE_P (mode) || GET_MODE_CLASS (mode) == MODE_VECTOR_INT)
    3677          973 :       && optab_handler (xor_optab, mode) != CODE_FOR_nothing)
    3678              :     {
    3679            0 :       temp = expand_binop (mode, xor_optab, op0, CONSTM1_RTX (mode),
    3680              :                            target, unsignedp, OPTAB_DIRECT);
    3681            0 :       if (temp)
    3682              :         return temp;
    3683              :     }
    3684              : 
    3685              :   /* Try calculating parity (x) as popcount (x) % 2.  */
    3686          973 :   if (unoptab == parity_optab && is_a <scalar_int_mode> (mode, &int_mode))
    3687              :     {
    3688           11 :       temp = expand_parity (int_mode, op0, target);
    3689           11 :       if (temp)
    3690              :         return temp;
    3691              :     }
    3692              : 
    3693              :   /* Try implementing ffs (x) in terms of clz (x).  */
    3694          962 :   if (unoptab == ffs_optab && is_a <scalar_int_mode> (mode, &int_mode))
    3695              :     {
    3696            0 :       temp = expand_ffs (int_mode, op0, target);
    3697            0 :       if (temp)
    3698              :         return temp;
    3699              :     }
    3700              : 
    3701              :   /* Try implementing ctz (x) in terms of clz (x).  */
    3702          962 :   if (unoptab == ctz_optab && is_a <scalar_int_mode> (mode, &int_mode))
    3703              :     {
    3704           48 :       temp = expand_ctz (int_mode, op0, target);
    3705           48 :       if (temp)
    3706              :         return temp;
    3707              :     }
    3708              : 
    3709          962 :   if ((unoptab == ctz_optab || unoptab == ffs_optab)
    3710           48 :       && optimize_insn_for_speed_p ()
    3711           45 :       && is_a <scalar_int_mode> (mode, &int_mode)
    3712           90 :       && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
    3713         1007 :       && (optab_handler (unoptab, word_mode) != CODE_FOR_nothing
    3714            0 :           || optab_handler (ctz_optab, word_mode) != CODE_FOR_nothing))
    3715              :     {
    3716           45 :       temp = expand_doubleword_clz_ctz_ffs (int_mode, op0, target, unoptab);
    3717           45 :       if (temp)
    3718              :         return temp;
    3719              :     }
    3720              : 
    3721          921 :  try_libcall:
    3722              :   /* Now try a library call in this mode.  */
    3723          921 :   libfunc = optab_libfunc (unoptab, mode);
    3724          921 :   if (libfunc)
    3725              :     {
    3726          590 :       rtx_insn *insns;
    3727          590 :       rtx value;
    3728          590 :       rtx eq_value;
    3729          590 :       machine_mode outmode = mode;
    3730              : 
    3731              :       /* All of these functions return small values.  Thus we choose to
    3732              :          have them return something that isn't a double-word.  */
    3733          590 :       if (unoptab == ffs_optab || unoptab == clz_optab || unoptab == ctz_optab
    3734              :           || unoptab == clrsb_optab || unoptab == popcount_optab
    3735          590 :           || unoptab == parity_optab)
    3736          497 :         outmode
    3737          497 :           = GET_MODE (hard_libcall_value (TYPE_MODE (integer_type_node),
    3738              :                                           optab_libfunc (unoptab, mode)));
    3739              : 
    3740          590 :       start_sequence ();
    3741              : 
    3742              :       /* Pass 1 for NO_QUEUE so we don't lose any increments
    3743              :          if the libcall is cse'd or moved.  */
    3744          590 :       value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, outmode,
    3745              :                                        op0, mode);
    3746          590 :       insns = end_sequence ();
    3747              : 
    3748          590 :       target = gen_reg_rtx (outmode);
    3749          590 :       bool trapv = trapv_unoptab_p (unoptab);
    3750          590 :       if (trapv)
    3751              :         eq_value = NULL_RTX;
    3752              :       else
    3753              :         {
    3754          497 :           eq_value = gen_rtx_fmt_e (optab_to_code (unoptab), mode, op0);
    3755         1491 :           if (GET_MODE_UNIT_SIZE (outmode) < GET_MODE_UNIT_SIZE (mode))
    3756          467 :             eq_value = simplify_gen_unary (TRUNCATE, outmode, eq_value, mode);
    3757           90 :           else if (GET_MODE_UNIT_SIZE (outmode) > GET_MODE_UNIT_SIZE (mode))
    3758            0 :             eq_value = simplify_gen_unary (ZERO_EXTEND,
    3759              :                                            outmode, eq_value, mode);
    3760              :         }
    3761          590 :       emit_libcall_block_1 (insns, target, value, eq_value, trapv);
    3762              : 
    3763          590 :       return target;
    3764              :     }
    3765              : 
    3766              :   /* It can't be done in this mode.  Can we do it in a wider mode?  */
    3767              : 
    3768          331 :   if (CLASS_HAS_WIDER_MODES_P (mclass))
    3769              :     {
    3770          682 :       FOR_EACH_WIDER_MODE (wider_mode, mode)
    3771              :         {
    3772          594 :           if (optab_handler (unoptab, wider_mode) != CODE_FOR_nothing
    3773          594 :               || optab_libfunc (unoptab, wider_mode))
    3774              :             {
    3775          243 :               rtx xop0 = op0;
    3776          243 :               rtx_insn *last = get_last_insn ();
    3777              : 
    3778              :               /* For certain operations, we need not actually extend
    3779              :                  the narrow operand, as long as we will truncate the
    3780              :                  results to the same narrowness.  */
    3781          243 :               xop0 = widen_operand (xop0, wider_mode, mode, unsignedp,
    3782          243 :                                     (unoptab == neg_optab
    3783          243 :                                      || unoptab == one_cmpl_optab
    3784          243 :                                      || unoptab == bswap_optab)
    3785              :                                     && mclass == MODE_INT);
    3786              : 
    3787          243 :               temp = expand_unop (wider_mode, unoptab, xop0, NULL_RTX,
    3788              :                                   unsignedp);
    3789              : 
    3790              :               /* If we are generating clz using wider mode, adjust the
    3791              :                  result.  Similarly for clrsb.  */
    3792          243 :               if ((unoptab == clz_optab || unoptab == clrsb_optab)
    3793            1 :                   && temp != 0)
    3794              :                 {
    3795            1 :                   scalar_int_mode wider_int_mode
    3796            1 :                     = as_a <scalar_int_mode> (wider_mode);
    3797            1 :                   int_mode = as_a <scalar_int_mode> (mode);
    3798            1 :                   temp = expand_binop
    3799            1 :                     (wider_mode, sub_optab, temp,
    3800            1 :                      gen_int_mode (GET_MODE_PRECISION (wider_int_mode)
    3801            2 :                                    - GET_MODE_PRECISION (int_mode),
    3802              :                                    wider_int_mode),
    3803              :                      target, true, OPTAB_DIRECT);
    3804              :                 }
    3805              : 
    3806              :               /* Likewise for bswap.  */
    3807          243 :               if (unoptab == bswap_optab && temp != 0)
    3808              :                 {
    3809            0 :                   scalar_int_mode wider_int_mode
    3810            0 :                     = as_a <scalar_int_mode> (wider_mode);
    3811            0 :                   int_mode = as_a <scalar_int_mode> (mode);
    3812            0 :                   gcc_assert (GET_MODE_PRECISION (wider_int_mode)
    3813              :                               == GET_MODE_BITSIZE (wider_int_mode)
    3814              :                               && GET_MODE_PRECISION (int_mode)
    3815              :                                  == GET_MODE_BITSIZE (int_mode));
    3816              : 
    3817            0 :                   temp = expand_shift (RSHIFT_EXPR, wider_int_mode, temp,
    3818            0 :                                        GET_MODE_BITSIZE (wider_int_mode)
    3819            0 :                                        - GET_MODE_BITSIZE (int_mode),
    3820              :                                        NULL_RTX, true);
    3821              :                 }
    3822              : 
    3823          243 :               if (temp)
    3824              :                 {
    3825          243 :                   if (mclass != MODE_INT)
    3826              :                     {
    3827            0 :                       if (target == 0)
    3828            0 :                         target = gen_reg_rtx (mode);
    3829            0 :                       convert_move (target, temp, 0);
    3830            0 :                       return target;
    3831              :                     }
    3832              :                   else
    3833          243 :                     return gen_lowpart (mode, temp);
    3834              :                 }
    3835              :               else
    3836            0 :                 delete_insns_since (last);
    3837              :             }
    3838              :         }
    3839              :     }
    3840              : 
    3841              :   /* One final attempt at implementing negation via subtraction,
    3842              :      this time allowing widening of the operand.  */
    3843           88 :   if (optab_to_code (unoptab) == NEG && !HONOR_SIGNED_ZEROS (mode))
    3844              :     {
    3845            0 :       rtx temp;
    3846            0 :       temp = expand_binop (mode,
    3847              :                            unoptab == negv_optab ? subv_optab : sub_optab,
    3848              :                            CONST0_RTX (mode), op0,
    3849              :                            target, unsignedp, OPTAB_LIB_WIDEN);
    3850            0 :       if (temp)
    3851              :         return temp;
    3852              :     }
    3853              : 
    3854              :   return 0;
    3855              : }
    3856              : 
    3857              : /* Emit code to compute the absolute value of OP0, with result to
    3858              :    TARGET if convenient.  (TARGET may be 0.)  The return value says
    3859              :    where the result actually is to be found.
    3860              : 
    3861              :    MODE is the mode of the operand; the mode of the result is
    3862              :    different but can be deduced from MODE.
    3863              : 
    3864              :  */
    3865              : 
    3866              : rtx
    3867        28460 : expand_abs_nojump (machine_mode mode, rtx op0, rtx target,
    3868              :                    int result_unsignedp)
    3869              : {
    3870        28460 :   rtx temp;
    3871              : 
    3872        28460 :   if (GET_MODE_CLASS (mode) != MODE_INT
    3873         5100 :       || ! flag_trapv)
    3874              :     result_unsignedp = 1;
    3875              : 
    3876              :   /* First try to do it with a special abs instruction.  */
    3877        28548 :   temp = expand_unop (mode, result_unsignedp ? abs_optab : absv_optab,
    3878              :                       op0, target, 0);
    3879        28460 :   if (temp != 0)
    3880              :     return temp;
    3881              : 
    3882              :   /* If we have a MAX insn, we can do this as MAX (x, -x).  */
    3883           88 :   if (optab_handler (smax_optab, mode) != CODE_FOR_nothing
    3884           88 :       && !HONOR_SIGNED_ZEROS (mode))
    3885              :     {
    3886           88 :       rtx_insn *last = get_last_insn ();
    3887              : 
    3888          176 :       temp = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
    3889              :                           op0, NULL_RTX, 0);
    3890           88 :       if (temp != 0)
    3891           88 :         temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
    3892              :                              OPTAB_WIDEN);
    3893              : 
    3894           88 :       if (temp != 0)
    3895              :         return temp;
    3896              : 
    3897            0 :       delete_insns_since (last);
    3898              :     }
    3899              : 
    3900              :   /* If this machine has expensive jumps, we can do integer absolute
    3901              :      value of X as (((signed) x >> (W-1)) ^ x) - ((signed) x >> (W-1)),
    3902              :      where W is the width of MODE.  */
    3903              : 
    3904            0 :   scalar_int_mode int_mode;
    3905            0 :   if (is_int_mode (mode, &int_mode)
    3906            0 :       && BRANCH_COST (optimize_insn_for_speed_p (),
    3907              :                       false) >= 2)
    3908              :     {
    3909            0 :       rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
    3910            0 :                                    GET_MODE_PRECISION (int_mode) - 1,
    3911              :                                    NULL_RTX, 0);
    3912              : 
    3913            0 :       temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
    3914              :                            OPTAB_LIB_WIDEN);
    3915            0 :       if (temp != 0)
    3916            0 :         temp = expand_binop (int_mode,
    3917              :                              result_unsignedp ? sub_optab : subv_optab,
    3918              :                              temp, extended, target, 0, OPTAB_LIB_WIDEN);
    3919              : 
    3920            0 :       if (temp != 0)
    3921              :         return temp;
    3922              :     }
    3923              : 
    3924              :   return NULL_RTX;
    3925              : }
    3926              : 
    3927              : rtx
    3928        28436 : expand_abs (machine_mode mode, rtx op0, rtx target,
    3929              :             int result_unsignedp, int safe)
    3930              : {
    3931        28436 :   rtx temp;
    3932        28436 :   rtx_code_label *op1;
    3933              : 
    3934        28436 :   if (GET_MODE_CLASS (mode) != MODE_INT
    3935         5076 :       || ! flag_trapv)
    3936        28348 :     result_unsignedp = 1;
    3937              : 
    3938        28436 :   temp = expand_abs_nojump (mode, op0, target, result_unsignedp);
    3939        28436 :   if (temp != 0)
    3940              :     return temp;
    3941              : 
    3942              :   /* If that does not win, use conditional jump and negate.  */
    3943              : 
    3944              :   /* It is safe to use the target if it is the same
    3945              :      as the source if this is also a pseudo register */
    3946            0 :   if (op0 == target && REG_P (op0)
    3947            0 :       && REGNO (op0) >= FIRST_PSEUDO_REGISTER)
    3948              :     safe = 1;
    3949              : 
    3950            0 :   op1 = gen_label_rtx ();
    3951            0 :   if (target == 0 || ! safe
    3952            0 :       || GET_MODE (target) != mode
    3953            0 :       || (MEM_P (target) && MEM_VOLATILE_P (target))
    3954            0 :       || (REG_P (target)
    3955            0 :           && REGNO (target) < FIRST_PSEUDO_REGISTER))
    3956            0 :     target = gen_reg_rtx (mode);
    3957              : 
    3958            0 :   emit_move_insn (target, op0);
    3959            0 :   NO_DEFER_POP;
    3960              : 
    3961            0 :   do_compare_rtx_and_jump (target, CONST0_RTX (mode), GE, 0, mode,
    3962              :                            NULL_RTX, NULL, op1,
    3963              :                            profile_probability::uninitialized ());
    3964              : 
    3965            0 :   op0 = expand_unop (mode, result_unsignedp ? neg_optab : negv_optab,
    3966              :                      target, target, 0);
    3967            0 :   if (op0 != target)
    3968            0 :     emit_move_insn (target, op0);
    3969            0 :   emit_label (op1);
    3970            0 :   OK_DEFER_POP;
    3971            0 :   return target;
    3972              : }
    3973              : 
    3974              : /* Emit code to compute the one's complement absolute value of OP0
    3975              :    (if (OP0 < 0) OP0 = ~OP0), with result to TARGET if convenient.
    3976              :    (TARGET may be NULL_RTX.)  The return value says where the result
    3977              :    actually is to be found.
    3978              : 
    3979              :    MODE is the mode of the operand; the mode of the result is
    3980              :    different but can be deduced from MODE.  */
    3981              : 
    3982              : rtx
    3983            0 : expand_one_cmpl_abs_nojump (machine_mode mode, rtx op0, rtx target)
    3984              : {
    3985            0 :   rtx temp;
    3986              : 
    3987              :   /* Not applicable for floating point modes.  */
    3988            0 :   if (FLOAT_MODE_P (mode))
    3989              :     return NULL_RTX;
    3990              : 
    3991              :   /* If we have a MAX insn, we can do this as MAX (x, ~x).  */
    3992            0 :   if (optab_handler (smax_optab, mode) != CODE_FOR_nothing)
    3993              :     {
    3994            0 :       rtx_insn *last = get_last_insn ();
    3995              : 
    3996            0 :       temp = expand_unop (mode, one_cmpl_optab, op0, NULL_RTX, 0);
    3997            0 :       if (temp != 0)
    3998            0 :         temp = expand_binop (mode, smax_optab, op0, temp, target, 0,
    3999              :                              OPTAB_WIDEN);
    4000              : 
    4001            0 :       if (temp != 0)
    4002              :         return temp;
    4003              : 
    4004            0 :       delete_insns_since (last);
    4005              :     }
    4006              : 
    4007              :   /* If this machine has expensive jumps, we can do one's complement
    4008              :      absolute value of X as (((signed) x >> (W-1)) ^ x).  */
    4009              : 
    4010            0 :   scalar_int_mode int_mode;
    4011            0 :   if (is_int_mode (mode, &int_mode)
    4012            0 :       && BRANCH_COST (optimize_insn_for_speed_p (),
    4013              :                      false) >= 2)
    4014              :     {
    4015            0 :       rtx extended = expand_shift (RSHIFT_EXPR, int_mode, op0,
    4016            0 :                                    GET_MODE_PRECISION (int_mode) - 1,
    4017              :                                    NULL_RTX, 0);
    4018              : 
    4019            0 :       temp = expand_binop (int_mode, xor_optab, extended, op0, target, 0,
    4020              :                            OPTAB_LIB_WIDEN);
    4021              : 
    4022            0 :       if (temp != 0)
    4023              :         return temp;
    4024              :     }
    4025              : 
    4026              :   return NULL_RTX;
    4027              : }
    4028              : 
    4029              : /* A subroutine of expand_copysign, perform the copysign operation using the
    4030              :    abs and neg primitives advertised to exist on the target.  The assumption
    4031              :    is that we have a split register file, and leaving op0 in fp registers,
    4032              :    and not playing with subregs so much, will help the register allocator.  */
    4033              : 
    4034              : static rtx
    4035        11608 : expand_copysign_absneg (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
    4036              :                         int bitpos, bool op0_is_abs)
    4037              : {
    4038        11608 :   scalar_int_mode imode;
    4039        11608 :   enum insn_code icode;
    4040        11608 :   rtx sign;
    4041        11608 :   rtx_code_label *label;
    4042              : 
    4043        11608 :   if (target == op1)
    4044          132 :     target = NULL_RTX;
    4045              : 
    4046              :   /* Check if the back end provides an insn that handles signbit for the
    4047              :      argument's mode. */
    4048        11608 :   icode = optab_handler (signbit_optab, mode);
    4049        11608 :   if (icode != CODE_FOR_nothing)
    4050              :     {
    4051        11568 :       imode = as_a <scalar_int_mode> (insn_data[(int) icode].operand[0].mode);
    4052        11568 :       sign = gen_reg_rtx (imode);
    4053        11568 :       emit_unop_insn (icode, sign, op1, UNKNOWN);
    4054              :     }
    4055              :   else
    4056              :     {
    4057          100 :       if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
    4058              :         {
    4059           40 :           if (!int_mode_for_mode (mode).exists (&imode))
    4060            0 :             return NULL_RTX;
    4061           40 :           op1 = gen_lowpart (imode, op1);
    4062              :         }
    4063              :       else
    4064              :         {
    4065            0 :           int word;
    4066              : 
    4067            0 :           imode = word_mode;
    4068            0 :           if (FLOAT_WORDS_BIG_ENDIAN)
    4069              :             word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
    4070              :           else
    4071            0 :             word = bitpos / BITS_PER_WORD;
    4072            0 :           bitpos = bitpos % BITS_PER_WORD;
    4073            0 :           op1 = operand_subword_force (op1, word, mode);
    4074              :         }
    4075              : 
    4076           40 :       wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
    4077           80 :       sign = expand_binop (imode, and_optab, op1,
    4078           80 :                            immed_wide_int_const (mask, imode),
    4079              :                            NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4080           40 :     }
    4081              : 
    4082        11608 :   if (!op0_is_abs)
    4083              :     {
    4084          154 :       op0 = expand_unop (mode, abs_optab, op0, target, 0);
    4085          154 :       if (op0 == NULL)
    4086              :         return NULL_RTX;
    4087              :       target = op0;
    4088              :     }
    4089              :   else
    4090              :     {
    4091        11454 :       if (target == NULL_RTX)
    4092           84 :         target = copy_to_reg (op0);
    4093              :       else
    4094        11370 :         emit_move_insn (target, op0);
    4095              :     }
    4096              : 
    4097        11608 :   label = gen_label_rtx ();
    4098        11608 :   emit_cmp_and_jump_insns (sign, const0_rtx, EQ, NULL_RTX, imode, 1, label);
    4099              : 
    4100        11608 :   if (CONST_DOUBLE_AS_FLOAT_P (op0))
    4101        11454 :     op0 = simplify_unary_operation (NEG, mode, op0, mode);
    4102              :   else
    4103          154 :     op0 = expand_unop (mode, neg_optab, op0, target, 0);
    4104        11608 :   if (op0 != target)
    4105        11454 :     emit_move_insn (target, op0);
    4106              : 
    4107        11608 :   emit_label (label);
    4108              : 
    4109        11608 :   return target;
    4110              : }
    4111              : 
    4112              : 
    4113              : /* A subroutine of expand_copysign, perform the entire copysign operation
    4114              :    with integer bitmasks.  BITPOS is the position of the sign bit; OP0_IS_ABS
    4115              :    is true if op0 is known to have its sign bit clear.  */
    4116              : 
    4117              : static rtx
    4118           68 : expand_copysign_bit (scalar_float_mode mode, rtx op0, rtx op1, rtx target,
    4119              :                      int bitpos, bool op0_is_abs)
    4120              : {
    4121           68 :   scalar_int_mode imode;
    4122           68 :   int word, nwords, i;
    4123           68 :   rtx temp;
    4124           68 :   rtx_insn *insns;
    4125              : 
    4126          156 :   if (GET_MODE_SIZE (mode) <= UNITS_PER_WORD)
    4127              :     {
    4128           68 :       if (!int_mode_for_mode (mode).exists (&imode))
    4129            0 :         return NULL_RTX;
    4130              :       word = 0;
    4131              :       nwords = 1;
    4132              :     }
    4133              :   else
    4134              :     {
    4135            0 :       imode = word_mode;
    4136              : 
    4137            0 :       if (FLOAT_WORDS_BIG_ENDIAN)
    4138              :         word = (GET_MODE_BITSIZE (mode) - bitpos) / BITS_PER_WORD;
    4139              :       else
    4140            0 :         word = bitpos / BITS_PER_WORD;
    4141            0 :       bitpos = bitpos % BITS_PER_WORD;
    4142            0 :       nwords = (GET_MODE_BITSIZE (mode) + BITS_PER_WORD - 1) / BITS_PER_WORD;
    4143              :     }
    4144              : 
    4145           68 :   wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (imode));
    4146              : 
    4147           68 :   if (target == 0
    4148           68 :       || target == op0
    4149           68 :       || target == op1
    4150           64 :       || reg_overlap_mentioned_p (target, op0)
    4151           64 :       || reg_overlap_mentioned_p (target, op1)
    4152          132 :       || (nwords > 1 && !valid_multiword_target_p (target)))
    4153            4 :     target = gen_reg_rtx (mode);
    4154              : 
    4155           68 :   if (nwords > 1)
    4156              :     {
    4157            0 :       start_sequence ();
    4158              : 
    4159            0 :       for (i = 0; i < nwords; ++i)
    4160              :         {
    4161            0 :           rtx targ_piece = operand_subword (target, i, 1, mode);
    4162            0 :           rtx op0_piece = operand_subword_force (op0, i, mode);
    4163              : 
    4164            0 :           if (i == word)
    4165              :             {
    4166            0 :               if (!op0_is_abs)
    4167            0 :                 op0_piece
    4168            0 :                   = expand_binop (imode, and_optab, op0_piece,
    4169            0 :                                   immed_wide_int_const (~mask, imode),
    4170              :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4171            0 :               op1 = expand_binop (imode, and_optab,
    4172            0 :                                   operand_subword_force (op1, i, mode),
    4173            0 :                                   immed_wide_int_const (mask, imode),
    4174              :                                   NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4175              : 
    4176            0 :               temp = expand_binop (imode, ior_optab, op0_piece, op1,
    4177              :                                    targ_piece, 1, OPTAB_LIB_WIDEN);
    4178            0 :               if (temp != targ_piece)
    4179            0 :                 emit_move_insn (targ_piece, temp);
    4180              :             }
    4181              :           else
    4182            0 :             emit_move_insn (targ_piece, op0_piece);
    4183              :         }
    4184              : 
    4185            0 :       insns = end_sequence ();
    4186              : 
    4187            0 :       emit_insn (insns);
    4188              :     }
    4189              :   else
    4190              :     {
    4191          136 :       op1 = expand_binop (imode, and_optab, gen_lowpart (imode, op1),
    4192           68 :                           immed_wide_int_const (mask, imode),
    4193              :                           NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4194              : 
    4195           68 :       op0 = gen_lowpart (imode, op0);
    4196           68 :       if (!op0_is_abs)
    4197          136 :         op0 = expand_binop (imode, and_optab, op0,
    4198          136 :                             immed_wide_int_const (~mask, imode),
    4199              :                             NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4200              : 
    4201           68 :       temp = expand_binop (imode, ior_optab, op0, op1,
    4202           68 :                            gen_lowpart (imode, target), 1, OPTAB_LIB_WIDEN);
    4203           68 :       target = force_lowpart_subreg (mode, temp, imode);
    4204              :     }
    4205              : 
    4206           68 :   return target;
    4207           68 : }
    4208              : 
    4209              : /* Expand the C99 copysign operation.  OP0 and OP1 must be the same
    4210              :    scalar floating point mode.  Return NULL if we do not know how to
    4211              :    expand the operation inline.  */
    4212              : 
    4213              : rtx
    4214        11676 : expand_copysign (rtx op0, rtx op1, rtx target)
    4215              : {
    4216        11676 :   scalar_float_mode mode;
    4217        11676 :   const struct real_format *fmt;
    4218        11676 :   bool op0_is_abs;
    4219        11676 :   rtx temp;
    4220              : 
    4221        11676 :   mode = as_a <scalar_float_mode> (GET_MODE (op0));
    4222        11676 :   gcc_assert (GET_MODE (op1) == mode);
    4223              : 
    4224              :   /* First try to do it with a special instruction.  */
    4225        11676 :   temp = expand_binop (mode, copysign_optab, op0, op1,
    4226              :                        target, 0, OPTAB_DIRECT);
    4227        11676 :   if (temp)
    4228              :     return temp;
    4229              : 
    4230        11676 :   fmt = REAL_MODE_FORMAT (mode);
    4231        11676 :   if (fmt == NULL || !fmt->has_signed_zero)
    4232              :     return NULL_RTX;
    4233              : 
    4234        11676 :   op0_is_abs = false;
    4235        11676 :   if (CONST_DOUBLE_AS_FLOAT_P (op0))
    4236              :     {
    4237        11454 :       if (real_isneg (CONST_DOUBLE_REAL_VALUE (op0)))
    4238            0 :         op0 = simplify_unary_operation (ABS, mode, op0, mode);
    4239              :       op0_is_abs = true;
    4240              :     }
    4241              : 
    4242        11676 :   if (fmt->signbit_ro >= 0
    4243        11676 :       && (CONST_DOUBLE_AS_FLOAT_P (op0)
    4244          222 :           || (optab_handler (neg_optab, mode) != CODE_FOR_nothing
    4245          154 :               && optab_handler (abs_optab, mode) != CODE_FOR_nothing)))
    4246              :     {
    4247        23216 :       temp = expand_copysign_absneg (mode, op0, op1, target,
    4248        11608 :                                      fmt->signbit_ro, op0_is_abs);
    4249        11608 :       if (temp)
    4250              :         return temp;
    4251              :     }
    4252              : 
    4253           68 :   if (fmt->signbit_rw < 0)
    4254              :     return NULL_RTX;
    4255           68 :   return expand_copysign_bit (mode, op0, op1, target,
    4256           68 :                               fmt->signbit_rw, op0_is_abs);
    4257              : }
    4258              : 
    4259              : /* Generate an instruction whose insn-code is INSN_CODE,
    4260              :    with two operands: an output TARGET and an input OP0.
    4261              :    TARGET *must* be nonzero, and the output is always stored there.
    4262              :    CODE is an rtx code such that (CODE OP0) is an rtx that describes
    4263              :    the value that is stored into TARGET.
    4264              : 
    4265              :    Return false if expansion failed.  */
    4266              : 
    4267              : bool
    4268      2168195 : maybe_emit_unop_insn (enum insn_code icode, rtx target, rtx op0,
    4269              :                       enum rtx_code code)
    4270              : {
    4271      2168195 :   class expand_operand ops[2];
    4272      2168195 :   rtx_insn *pat;
    4273              : 
    4274      2168195 :   create_output_operand (&ops[0], target, GET_MODE (target));
    4275      2168195 :   create_input_operand (&ops[1], op0, GET_MODE (op0));
    4276      2168195 :   pat = maybe_gen_insn (icode, 2, ops);
    4277      2168195 :   if (!pat)
    4278              :     return false;
    4279              : 
    4280      1226839 :   if (INSN_P (pat) && NEXT_INSN (pat) != NULL_RTX
    4281      2192581 :       && code != UNKNOWN)
    4282        12666 :     add_equal_note (pat, ops[0].value, code, ops[1].value, NULL_RTX,
    4283        12666 :                     GET_MODE (op0));
    4284              : 
    4285      2168194 :   emit_insn (pat);
    4286              : 
    4287      2168194 :   if (ops[0].value != target)
    4288        39178 :     emit_move_insn (target, ops[0].value);
    4289              :   return true;
    4290              : }
    4291              : /* Generate an instruction whose insn-code is INSN_CODE,
    4292              :    with two operands: an output TARGET and an input OP0.
    4293              :    TARGET *must* be nonzero, and the output is always stored there.
    4294              :    CODE is an rtx code such that (CODE OP0) is an rtx that describes
    4295              :    the value that is stored into TARGET.  */
    4296              : 
    4297              : void
    4298      2121160 : emit_unop_insn (enum insn_code icode, rtx target, rtx op0, enum rtx_code code)
    4299              : {
    4300      2121160 :   bool ok = maybe_emit_unop_insn (icode, target, op0, code);
    4301      2121160 :   gcc_assert (ok);
    4302      2121160 : }
    4303              : 
    4304              : struct no_conflict_data
    4305              : {
    4306              :   rtx target;
    4307              :   rtx_insn *first, *insn;
    4308              :   bool must_stay;
    4309              : };
    4310              : 
    4311              : /* Called via note_stores by emit_libcall_block.  Set P->must_stay if
    4312              :    the currently examined clobber / store has to stay in the list of
    4313              :    insns that constitute the actual libcall block.  */
    4314              : static void
    4315        57305 : no_conflict_move_test (rtx dest, const_rtx set, void *p0)
    4316              : {
    4317        57305 :   struct no_conflict_data *p= (struct no_conflict_data *) p0;
    4318              : 
    4319              :   /* If this inns directly contributes to setting the target, it must stay.  */
    4320        57305 :   if (reg_overlap_mentioned_p (p->target, dest))
    4321            0 :     p->must_stay = true;
    4322              :   /* If we haven't committed to keeping any other insns in the list yet,
    4323              :      there is nothing more to check.  */
    4324        57305 :   else if (p->insn == p->first)
    4325              :     return;
    4326              :   /* If this insn sets / clobbers a register that feeds one of the insns
    4327              :      already in the list, this insn has to stay too.  */
    4328        26963 :   else if (reg_overlap_mentioned_p (dest, PATTERN (p->first))
    4329        26963 :            || (CALL_P (p->first) && (find_reg_fusage (p->first, USE, dest)))
    4330        26963 :            || reg_used_between_p (dest, p->first, p->insn)
    4331              :            /* Likewise if this insn depends on a register set by a previous
    4332              :               insn in the list, or if it sets a result (presumably a hard
    4333              :               register) that is set or clobbered by a previous insn.
    4334              :               N.B. the modified_*_p (SET_DEST...) tests applied to a MEM
    4335              :               SET_DEST perform the former check on the address, and the latter
    4336              :               check on the MEM.  */
    4337        53926 :            || (GET_CODE (set) == SET
    4338        26963 :                && (modified_in_p (SET_SRC (set), p->first)
    4339        26961 :                    || modified_in_p (SET_DEST (set), p->first)
    4340        26961 :                    || modified_between_p (SET_SRC (set), p->first, p->insn)
    4341        26961 :                    || modified_between_p (SET_DEST (set), p->first, p->insn))))
    4342            2 :     p->must_stay = true;
    4343              : }
    4344              : 
    4345              : 
    4346              : /* Emit code to make a call to a constant function or a library call.
    4347              : 
    4348              :    INSNS is a list containing all insns emitted in the call.
    4349              :    These insns leave the result in RESULT.  Our block is to copy RESULT
    4350              :    to TARGET, which is logically equivalent to EQUIV.
    4351              : 
    4352              :    We first emit any insns that set a pseudo on the assumption that these are
    4353              :    loading constants into registers; doing so allows them to be safely cse'ed
    4354              :    between blocks.  Then we emit all the other insns in the block, followed by
    4355              :    an insn to move RESULT to TARGET.  This last insn will have a REQ_EQUAL
    4356              :    note with an operand of EQUIV.  */
    4357              : 
    4358              : static void
    4359       123059 : emit_libcall_block_1 (rtx_insn *insns, rtx target, rtx result, rtx equiv,
    4360              :                       bool equiv_may_trap)
    4361              : {
    4362       123059 :   rtx final_dest = target;
    4363       123059 :   rtx_insn *next, *last, *insn;
    4364              : 
    4365              :   /* If this is a reg with REG_USERVAR_P set, then it could possibly turn
    4366              :      into a MEM later.  Protect the libcall block from this change.  */
    4367       123059 :   if (! REG_P (target) || REG_USERVAR_P (target))
    4368         1810 :     target = gen_reg_rtx (GET_MODE (target));
    4369              : 
    4370              :   /* If we're using non-call exceptions, a libcall corresponding to an
    4371              :      operation that may trap may also trap.  */
    4372              :   /* ??? See the comment in front of make_reg_eh_region_note.  */
    4373       123059 :   if (cfun->can_throw_non_call_exceptions
    4374       123059 :       && (equiv_may_trap || may_trap_p (equiv)))
    4375              :     {
    4376         1032 :       for (insn = insns; insn; insn = NEXT_INSN (insn))
    4377          829 :         if (CALL_P (insn))
    4378              :           {
    4379          203 :             rtx note = find_reg_note (insn, REG_EH_REGION, NULL_RTX);
    4380          203 :             if (note)
    4381              :               {
    4382          203 :                 int lp_nr = INTVAL (XEXP (note, 0));
    4383          203 :                 if (lp_nr == 0 || lp_nr == INT_MIN)
    4384          203 :                   remove_note (insn, note);
    4385              :               }
    4386              :           }
    4387              :     }
    4388              :   else
    4389              :     {
    4390              :       /* Look for any CALL_INSNs in this sequence, and attach a REG_EH_REGION
    4391              :          reg note to indicate that this call cannot throw or execute a nonlocal
    4392              :          goto (unless there is already a REG_EH_REGION note, in which case
    4393              :          we update it).  */
    4394       535423 :       for (insn = insns; insn; insn = NEXT_INSN (insn))
    4395       412567 :         if (CALL_P (insn))
    4396       122856 :           make_reg_eh_region_note_nothrow_nononlocal (insn);
    4397              :     }
    4398              : 
    4399              :   /* First emit all insns that set pseudos.  Remove them from the list as
    4400              :      we go.  Avoid insns that set pseudos which were referenced in previous
    4401              :      insns.  These can be generated by move_by_pieces, for example,
    4402              :      to update an address.  Similarly, avoid insns that reference things
    4403              :      set in previous insns.  */
    4404              : 
    4405       536455 :   for (insn = insns; insn; insn = next)
    4406              :     {
    4407       413396 :       rtx set = single_set (insn);
    4408              : 
    4409       413396 :       next = NEXT_INSN (insn);
    4410              : 
    4411       398880 :       if (set != 0 && REG_P (SET_DEST (set))
    4412       774972 :           && REGNO (SET_DEST (set)) >= FIRST_PSEUDO_REGISTER)
    4413              :         {
    4414        47214 :           struct no_conflict_data data;
    4415              : 
    4416        47214 :           data.target = const0_rtx;
    4417        47214 :           data.first = insns;
    4418        47214 :           data.insn = insn;
    4419        47214 :           data.must_stay = 0;
    4420        47214 :           note_stores (insn, no_conflict_move_test, &data);
    4421        47214 :           if (! data.must_stay)
    4422              :             {
    4423        47212 :               if (PREV_INSN (insn))
    4424        26961 :                 SET_NEXT_INSN (PREV_INSN (insn)) = next;
    4425              :               else
    4426              :                 insns = next;
    4427              : 
    4428        47212 :               if (next)
    4429        47212 :                 SET_PREV_INSN (next) = PREV_INSN (insn);
    4430              : 
    4431        47212 :               add_insn (insn);
    4432              :             }
    4433              :         }
    4434              : 
    4435              :       /* Some ports use a loop to copy large arguments onto the stack.
    4436              :          Don't move anything outside such a loop.  */
    4437       413396 :       if (LABEL_P (insn))
    4438              :         break;
    4439              :     }
    4440              : 
    4441              :   /* Write the remaining insns followed by the final copy.  */
    4442       489243 :   for (insn = insns; insn; insn = next)
    4443              :     {
    4444       366184 :       next = NEXT_INSN (insn);
    4445              : 
    4446       366184 :       add_insn (insn);
    4447              :     }
    4448              : 
    4449       123059 :   last = emit_move_insn (target, result);
    4450       123059 :   if (equiv)
    4451       122476 :     set_dst_reg_note (last, REG_EQUAL, copy_rtx (equiv), target);
    4452              : 
    4453       123059 :   if (final_dest != target)
    4454         1810 :     emit_move_insn (final_dest, target);
    4455       123059 : }
    4456              : 
    4457              : void
    4458        91815 : emit_libcall_block (rtx_insn *insns, rtx target, rtx result, rtx equiv)
    4459              : {
    4460        91815 :   emit_libcall_block_1 (insns, target, result, equiv, false);
    4461        91815 : }
    4462              : 
    4463              : /* True if we can perform a comparison of mode MODE straightforwardly.
    4464              :    PURPOSE describes how this comparison will be used.  CODE is the rtx
    4465              :    comparison code we will be using.
    4466              : 
    4467              :    ??? Actually, CODE is slightly weaker than that.  A target is still
    4468              :    required to implement all of the normal bcc operations, but not
    4469              :    required to implement all (or any) of the unordered bcc operations.  */
    4470              : 
    4471              : bool
    4472     18269227 : can_compare_p (enum rtx_code code, machine_mode mode,
    4473              :                enum can_compare_purpose purpose)
    4474              : {
    4475     18269227 :   rtx test;
    4476     18269227 :   test = gen_rtx_fmt_ee (code, mode, const0_rtx, const0_rtx);
    4477     20357539 :   do
    4478              :     {
    4479     20357539 :       enum insn_code icode;
    4480              : 
    4481     20357539 :       if (purpose == ccp_jump
    4482     20154766 :           && (icode = optab_handler (cbranch_optab, mode)) != CODE_FOR_nothing
    4483     39152151 :           && insn_operand_matches (icode, 0, test))
    4484              :         return true;
    4485      3384900 :       if (purpose == ccp_store_flag
    4486       202773 :           && (icode = optab_handler (cstore_optab, mode)) != CODE_FOR_nothing
    4487      3518754 :           && insn_operand_matches (icode, 1, test))
    4488              :         return true;
    4489              : 
    4490      3380309 :       mode = GET_MODE_WIDER_MODE (mode).else_void ();
    4491      3380309 :       PUT_MODE (test, mode);
    4492              :     }
    4493      3380309 :   while (mode != VOIDmode);
    4494              : 
    4495              :   return false;
    4496              : }
    4497              : 
    4498              : /* Return whether RTL code CODE corresponds to an unsigned optab.  */
    4499              : 
    4500              : static bool
    4501      1467484 : unsigned_optab_p (enum rtx_code code)
    4502              : {
    4503      1467484 :   return code == LTU || code == LEU || code == GTU || code == GEU;
    4504              : }
    4505              : 
    4506              : /* Return whether the backend-emitted comparison for code CODE, comparing
    4507              :    operands of mode VALUE_MODE and producing a result with MASK_MODE, matches
    4508              :    operand OPNO of pattern ICODE.  */
    4509              : 
    4510              : static bool
    4511      1057242 : insn_predicate_matches_p (enum insn_code icode, unsigned int opno,
    4512              :                           enum rtx_code code, machine_mode mask_mode,
    4513              :                           machine_mode value_mode)
    4514              : {
    4515      1057242 :   rtx reg1 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 1);
    4516      1057242 :   rtx reg2 = alloca_raw_REG (value_mode, LAST_VIRTUAL_REGISTER + 2);
    4517      1057242 :   rtx test = alloca_rtx_fmt_ee (code, mask_mode, reg1, reg2);
    4518      1057242 :   return insn_operand_matches (icode, opno, test);
    4519              : }
    4520              : 
    4521              : /* Return whether the backend can emit a vector comparison (vec_cmp/vec_cmpu)
    4522              :    for code CODE, comparing operands of mode VALUE_MODE and producing a result
    4523              :    with MASK_MODE.  */
    4524              : 
    4525              : bool
    4526      1467484 : can_vec_cmp_compare_p (enum rtx_code code, machine_mode value_mode,
    4527              :                        machine_mode mask_mode)
    4528              : {
    4529      1467484 :   enum insn_code icode
    4530      1467484 :       = get_vec_cmp_icode (value_mode, mask_mode, unsigned_optab_p (code));
    4531      1467484 :   if (icode == CODE_FOR_nothing)
    4532              :     return false;
    4533              : 
    4534      1057242 :   return insn_predicate_matches_p (icode, 1, code, mask_mode, value_mode);
    4535              : }
    4536              : 
    4537              : /* Return whether the backend can emit vector set instructions for inserting
    4538              :    element into vector at variable index position.  */
    4539              : 
    4540              : bool
    4541          207 : can_vec_set_var_idx_p (machine_mode vec_mode)
    4542              : {
    4543          207 :   if (!VECTOR_MODE_P (vec_mode))
    4544              :     return false;
    4545              : 
    4546          205 :   machine_mode inner_mode = GET_MODE_INNER (vec_mode);
    4547              : 
    4548          205 :   rtx reg1 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 1);
    4549          205 :   rtx reg2 = alloca_raw_REG (inner_mode, LAST_VIRTUAL_REGISTER + 2);
    4550              : 
    4551          205 :   enum insn_code icode = optab_handler (vec_set_optab, vec_mode);
    4552              : 
    4553          205 :   const struct insn_data_d *data = &insn_data[icode];
    4554          205 :   machine_mode idx_mode = data->operand[2].mode;
    4555              : 
    4556          205 :   rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
    4557              : 
    4558          204 :   return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
    4559          204 :          && insn_operand_matches (icode, 1, reg2)
    4560          409 :          && insn_operand_matches (icode, 2, reg3);
    4561              : }
    4562              : 
    4563              : /* Return whether the backend can emit a vec_extract instruction with
    4564              :    a non-constant index.  */
    4565              : bool
    4566        22085 : can_vec_extract_var_idx_p (machine_mode vec_mode, machine_mode extr_mode)
    4567              : {
    4568        22085 :   if (!VECTOR_MODE_P (vec_mode))
    4569              :     return false;
    4570              : 
    4571        17464 :   rtx reg1 = alloca_raw_REG (extr_mode, LAST_VIRTUAL_REGISTER + 1);
    4572        17464 :   rtx reg2 = alloca_raw_REG (vec_mode, LAST_VIRTUAL_REGISTER + 2);
    4573              : 
    4574        17464 :   enum insn_code icode = convert_optab_handler (vec_extract_optab,
    4575              :                                                 vec_mode, extr_mode);
    4576              : 
    4577        17464 :   const struct insn_data_d *data = &insn_data[icode];
    4578        17464 :   machine_mode idx_mode = data->operand[2].mode;
    4579              : 
    4580        17464 :   rtx reg3 = alloca_raw_REG (idx_mode, LAST_VIRTUAL_REGISTER + 3);
    4581              : 
    4582        16598 :   return icode != CODE_FOR_nothing && insn_operand_matches (icode, 0, reg1)
    4583        16598 :          && insn_operand_matches (icode, 1, reg2)
    4584        34062 :          && insn_operand_matches (icode, 2, reg3);
    4585              : }
    4586              : 
    4587              : /* This function is called when we are going to emit a compare instruction that
    4588              :    compares the values found in X and Y, using the rtl operator COMPARISON.
    4589              : 
    4590              :    If they have mode BLKmode, then SIZE specifies the size of both operands.
    4591              : 
    4592              :    UNSIGNEDP nonzero says that the operands are unsigned;
    4593              :    this matters if they need to be widened (as given by METHODS).
    4594              : 
    4595              :    *PTEST is where the resulting comparison RTX is returned or NULL_RTX
    4596              :    if we failed to produce one.
    4597              : 
    4598              :    *PMODE is the mode of the inputs (in case they are const_int).
    4599              : 
    4600              :    *OPTAB is the optab to check for OPTAB_DIRECT support.  Defaults to
    4601              :    cbranch_optab.
    4602              : 
    4603              :    This function performs all the setup necessary so that the caller only has
    4604              :    to emit a single comparison insn.  This setup can involve doing a BLKmode
    4605              :    comparison or emitting a library call to perform the comparison if no insn
    4606              :    is available to handle it.
    4607              :    The values which are passed in through pointers can be modified; the caller
    4608              :    should perform the comparison on the modified values.  Constant
    4609              :    comparisons must have already been folded.  */
    4610              : 
    4611              : static void
    4612      6963705 : prepare_cmp_insn (rtx x, rtx y, rtx *mask, enum rtx_code comparison, rtx size,
    4613              :                   int unsignedp, enum optab_methods methods,
    4614              :                   rtx *ptest, machine_mode *pmode, optab optab)
    4615              : {
    4616      6963705 :   machine_mode mode = *pmode;
    4617      6963705 :   rtx libfunc, test;
    4618      6963705 :   machine_mode cmp_mode;
    4619              : 
    4620              :   /* The other methods are not needed.  */
    4621      6963705 :   gcc_assert (methods == OPTAB_DIRECT || methods == OPTAB_WIDEN
    4622              :               || methods == OPTAB_LIB_WIDEN);
    4623              : 
    4624      6963705 :   if (CONST_SCALAR_INT_P (y))
    4625      4506053 :     canonicalize_comparison (mode, &comparison, &y);
    4626              : 
    4627              :   /* If we are optimizing, force expensive constants into a register.  */
    4628            1 :   if (CONSTANT_P (x) && optimize
    4629            1 :       && (rtx_cost (x, mode, COMPARE, 0, optimize_insn_for_speed_p ())
    4630              :           > COSTS_N_INSNS (1))
    4631      6963705 :       && can_create_pseudo_p ())
    4632            0 :     x = force_reg (mode, x);
    4633              : 
    4634      5023807 :   if (CONSTANT_P (y) && optimize
    4635      3973711 :       && (rtx_cost (y, mode, COMPARE, 1, optimize_insn_for_speed_p ())
    4636              :           > COSTS_N_INSNS (1))
    4637      7132593 :       && can_create_pseudo_p ())
    4638       168888 :     y = force_reg (mode, y);
    4639              : 
    4640              :   /* Don't let both operands fail to indicate the mode.  */
    4641      6963705 :   if (GET_MODE (x) == VOIDmode && GET_MODE (y) == VOIDmode)
    4642            0 :     x = force_reg (mode, x);
    4643      6963705 :   if (mode == VOIDmode)
    4644         5019 :     mode = GET_MODE (x) != VOIDmode ? GET_MODE (x) : GET_MODE (y);
    4645              : 
    4646              :   /* Handle all BLKmode compares.  */
    4647              : 
    4648      6963705 :   if (mode == BLKmode)
    4649              :     {
    4650            0 :       machine_mode result_mode;
    4651            0 :       enum insn_code cmp_code;
    4652            0 :       rtx result;
    4653            0 :       rtx opalign
    4654            0 :         = GEN_INT (MIN (MEM_ALIGN (x), MEM_ALIGN (y)) / BITS_PER_UNIT);
    4655              : 
    4656            0 :       gcc_assert (size);
    4657              : 
    4658              :       /* Try to use a memory block compare insn - either cmpstr
    4659              :          or cmpmem will do.  */
    4660            0 :       opt_scalar_int_mode cmp_mode_iter;
    4661            0 :       FOR_EACH_MODE_IN_CLASS (cmp_mode_iter, MODE_INT)
    4662              :         {
    4663            0 :           scalar_int_mode cmp_mode = cmp_mode_iter.require ();
    4664            0 :           cmp_code = direct_optab_handler (cmpmem_optab, cmp_mode);
    4665            0 :           if (cmp_code == CODE_FOR_nothing)
    4666            0 :             cmp_code = direct_optab_handler (cmpstr_optab, cmp_mode);
    4667            0 :           if (cmp_code == CODE_FOR_nothing)
    4668            0 :             cmp_code = direct_optab_handler (cmpstrn_optab, cmp_mode);
    4669            0 :           if (cmp_code == CODE_FOR_nothing)
    4670            0 :             continue;
    4671              : 
    4672              :           /* Must make sure the size fits the insn's mode.  */
    4673            0 :           if (CONST_INT_P (size)
    4674            0 :               ? UINTVAL (size) > GET_MODE_MASK (cmp_mode)
    4675            0 :               : (GET_MODE_BITSIZE (as_a <scalar_int_mode> (GET_MODE (size)))
    4676            0 :                  > GET_MODE_BITSIZE (cmp_mode)))
    4677            0 :             continue;
    4678              : 
    4679            0 :           result_mode = insn_data[cmp_code].operand[0].mode;
    4680            0 :           result = gen_reg_rtx (result_mode);
    4681            0 :           size = convert_to_mode (cmp_mode, size, 1);
    4682            0 :           emit_insn (GEN_FCN (cmp_code) (result, x, y, size, opalign));
    4683              : 
    4684            0 :           *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, result, const0_rtx);
    4685            0 :           *pmode = result_mode;
    4686            0 :           return;
    4687              :         }
    4688              : 
    4689            0 :       if (methods != OPTAB_LIB && methods != OPTAB_LIB_WIDEN)
    4690            0 :         goto fail;
    4691              : 
    4692              :       /* Otherwise call a library function.  */
    4693            0 :       result = emit_block_comp_via_libcall (x, y, size);
    4694              : 
    4695            0 :       x = result;
    4696            0 :       y = const0_rtx;
    4697            0 :       mode = TYPE_MODE (integer_type_node);
    4698            0 :       methods = OPTAB_LIB_WIDEN;
    4699            0 :       unsignedp = false;
    4700              :     }
    4701              : 
    4702              :   /* Don't allow operands to the compare to trap, as that can put the
    4703              :      compare and branch in different basic blocks.  */
    4704      6963705 :   if (cfun->can_throw_non_call_exceptions)
    4705              :     {
    4706      1133186 :       if (!can_create_pseudo_p () && (may_trap_p (x) || may_trap_p (y)))
    4707            0 :         goto fail;
    4708      1133186 :       if (may_trap_p (x))
    4709        24748 :         x = copy_to_reg (x);
    4710      1133186 :       if (may_trap_p (y))
    4711         1534 :         y = copy_to_reg (y);
    4712              :     }
    4713              : 
    4714      6963705 :   if (GET_MODE_CLASS (mode) == MODE_CC)
    4715              :     {
    4716          438 :       enum insn_code icode = optab_handler (cbranch_optab, CCmode);
    4717          438 :       test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
    4718          438 :       if (icode != CODE_FOR_nothing
    4719          438 :           && insn_operand_matches (icode, 0, test))
    4720              :         {
    4721          438 :           *ptest = test;
    4722          438 :           return;
    4723              :         }
    4724              :       else
    4725            0 :         goto fail;
    4726              :     }
    4727              : 
    4728      6963267 :   test = gen_rtx_fmt_ee (comparison, VOIDmode, x, y);
    4729      7051355 :   FOR_EACH_WIDER_MODE_FROM (cmp_mode, mode)
    4730              :     {
    4731      7003448 :       enum insn_code icode;
    4732      7003448 :       icode = optab_handler (optab, cmp_mode);
    4733      7003448 :       if (icode != CODE_FOR_nothing
    4734      7003448 :           && insn_operand_matches (icode, 0, test))
    4735              :         {
    4736      6915360 :           rtx_insn *last = get_last_insn ();
    4737      6915360 :           rtx op0 = prepare_operand (icode, x, 1, mode, cmp_mode, unsignedp);
    4738      6915360 :           rtx op1 = prepare_operand (icode, y, 2, mode, cmp_mode, unsignedp);
    4739      6915360 :           if (op0 && op1
    4740      6915360 :               && insn_operand_matches (icode, 1, op0)
    4741     13830720 :               && insn_operand_matches (icode, 2, op1))
    4742              :             {
    4743      6915360 :               XEXP (test, 0) = op0;
    4744      6915360 :               XEXP (test, 1) = op1;
    4745      6915360 :               *ptest = test;
    4746      6915360 :               *pmode = cmp_mode;
    4747      6915360 :               return;
    4748              :             }
    4749            0 :           delete_insns_since (last);
    4750              :         }
    4751              : 
    4752        88088 :       if (methods == OPTAB_DIRECT)
    4753              :         break;
    4754              :     }
    4755              : 
    4756        47907 :   if (methods != OPTAB_LIB_WIDEN)
    4757         2929 :     goto fail;
    4758              : 
    4759        44978 :   if (SCALAR_FLOAT_MODE_P (mode))
    4760              :     {
    4761              :       /* Small trick if UNORDERED isn't implemented by the hardware.  */
    4762        44978 :       if (comparison == UNORDERED && rtx_equal_p (x, y))
    4763              :         {
    4764          773 :           prepare_cmp_insn (x, y, mask, UNLT, NULL_RTX, unsignedp, OPTAB_WIDEN,
    4765              :                             ptest, pmode, optab);
    4766          773 :           if (*ptest)
    4767              :             return;
    4768              :         }
    4769              : 
    4770        44978 :       prepare_float_lib_cmp (x, y, comparison, ptest, pmode);
    4771              :     }
    4772              :   else
    4773              :     {
    4774            0 :       rtx result;
    4775            0 :       machine_mode ret_mode;
    4776              : 
    4777              :       /* Handle a libcall just for the mode we are using.  */
    4778            0 :       libfunc = optab_libfunc (cmp_optab, mode);
    4779            0 :       gcc_assert (libfunc);
    4780              : 
    4781              :       /* If we want unsigned, and this mode has a distinct unsigned
    4782              :          comparison routine, use that.  */
    4783            0 :       if (unsignedp)
    4784              :         {
    4785            0 :           rtx ulibfunc = optab_libfunc (ucmp_optab, mode);
    4786            0 :           if (ulibfunc)
    4787            0 :             libfunc = ulibfunc;
    4788              :         }
    4789              : 
    4790            0 :       ret_mode = targetm.libgcc_cmp_return_mode ();
    4791            0 :       result = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
    4792              :                                         ret_mode, x, mode, y, mode);
    4793              : 
    4794              :       /* There are two kinds of comparison routines. Biased routines
    4795              :          return 0/1/2, and unbiased routines return -1/0/1. Other parts
    4796              :          of gcc expect that the comparison operation is equivalent
    4797              :          to the modified comparison. For signed comparisons compare the
    4798              :          result against 1 in the biased case, and zero in the unbiased
    4799              :          case. For unsigned comparisons always compare against 1 after
    4800              :          biasing the unbiased result by adding 1. This gives us a way to
    4801              :          represent LTU.
    4802              :          The comparisons in the fixed-point helper library are always
    4803              :          biased.  */
    4804            0 :       x = result;
    4805            0 :       y = const1_rtx;
    4806              : 
    4807            0 :       if (!TARGET_LIB_INT_CMP_BIASED && !ALL_FIXED_POINT_MODE_P (mode))
    4808              :         {
    4809              :           if (unsignedp)
    4810              :             x = plus_constant (ret_mode, result, 1);
    4811              :           else
    4812              :             y = const0_rtx;
    4813              :         }
    4814              : 
    4815            0 :       *pmode = ret_mode;
    4816            0 :       prepare_cmp_insn (x, y, mask, comparison, NULL_RTX, unsignedp, methods,
    4817              :                         ptest, pmode, optab);
    4818              :     }
    4819              : 
    4820              :   return;
    4821              : 
    4822         2929 :  fail:
    4823         2929 :   *ptest = NULL_RTX;
    4824              : }
    4825              : 
    4826              : /* Before emitting an insn with code ICODE, make sure that X, which is going
    4827              :    to be used for operand OPNUM of the insn, is converted from mode MODE to
    4828              :    WIDER_MODE (UNSIGNEDP determines whether it is an unsigned conversion), and
    4829              :    that it is accepted by the operand predicate.  Return the new value.  */
    4830              : 
    4831              : rtx
    4832     15338486 : prepare_operand (enum insn_code icode, rtx x, int opnum, machine_mode mode,
    4833              :                  machine_mode wider_mode, int unsignedp)
    4834              : {
    4835     15338486 :   if (mode != wider_mode)
    4836        10852 :     x = convert_modes (wider_mode, mode, x, unsignedp);
    4837              : 
    4838     15338486 :   if (!insn_operand_matches (icode, opnum, x))
    4839              :     {
    4840        64344 :       machine_mode op_mode = insn_data[(int) icode].operand[opnum].mode;
    4841        64344 :       if (reload_completed)
    4842              :         return NULL_RTX;
    4843        64344 :       if (GET_MODE (x) != op_mode && GET_MODE (x) != VOIDmode)
    4844              :         return NULL_RTX;
    4845        64060 :       x = copy_to_mode_reg (op_mode, x);
    4846              :     }
    4847              : 
    4848              :   return x;
    4849              : }
    4850              : 
    4851              : /* Subroutine of emit_cmp_and_jump_insns; this function is called when we know
    4852              :    we can do the branch.  */
    4853              : 
    4854              : static void
    4855      6456332 : emit_cmp_and_jump_insn_1 (rtx test, rtx cond, rtx len, rtx bias,
    4856              :                           machine_mode mode, rtx label, direct_optab cmp_optab,
    4857              :                           profile_probability prob, bool test_branch)
    4858              : {
    4859      6456332 :   machine_mode optab_mode;
    4860      6456332 :   enum mode_class mclass;
    4861      6456332 :   enum insn_code icode;
    4862      6456332 :   rtx_insn *insn;
    4863              : 
    4864      6456332 :   mclass = GET_MODE_CLASS (mode);
    4865      6456332 :   optab_mode = (mclass == MODE_CC) ? CCmode : mode;
    4866      6456332 :   icode = optab_handler (cmp_optab, optab_mode);
    4867              : 
    4868      6456332 :   gcc_assert (icode != CODE_FOR_nothing);
    4869      6456332 :   gcc_assert (test_branch || insn_operand_matches (icode, 0, test));
    4870      6456332 :   gcc_assert (cond == NULL_RTX || (cond != NULL_RTX && !test_branch));
    4871      6456314 :   if (test_branch)
    4872            0 :     insn = emit_jump_insn (GEN_FCN (icode) (XEXP (test, 0),
    4873            0 :                                             XEXP (test, 1), label));
    4874      6456332 :   else if (len)
    4875              :     {
    4876            0 :       gcc_assert (cond);
    4877            0 :       gcc_assert (bias);
    4878            0 :       insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
    4879            0 :                                               XEXP (test, 1), len, bias,
    4880              :                                               label));
    4881              :     }
    4882      6456332 :   else if (cond)
    4883           18 :     insn = emit_jump_insn (GEN_FCN (icode) (test, cond, XEXP (test, 0),
    4884           18 :                                             XEXP (test, 1), label));
    4885              :   else
    4886      6456314 :     insn = emit_jump_insn (GEN_FCN (icode) (test, XEXP (test, 0),
    4887      6456314 :                                             XEXP (test, 1), label));
    4888              : 
    4889      6456332 :   if (prob.initialized_p ()
    4890      4899275 :       && profile_status_for_fn (cfun) != PROFILE_ABSENT
    4891      4791957 :       && insn
    4892      4791957 :       && JUMP_P (insn)
    4893      4791068 :       && any_condjump_p (insn)
    4894     11247400 :       && !find_reg_note (insn, REG_BR_PROB, 0))
    4895      4789590 :     add_reg_br_prob_note (insn, prob);
    4896      6456332 : }
    4897              : 
    4898              : /* PTEST points to a comparison that compares its first operand with zero.
    4899              :    Check to see if it can be performed as a bit-test-and-branch instead.
    4900              :    On success, return the instruction that performs the bit-test-and-branch
    4901              :    and replace the second operand of *PTEST with the bit number to test.
    4902              :    On failure, return CODE_FOR_nothing and leave *PTEST unchanged.
    4903              : 
    4904              :    Note that the comparison described by *PTEST should not be taken
    4905              :    literally after a successful return.  *PTEST is just a convenient
    4906              :    place to store the two operands of the bit-and-test.
    4907              : 
    4908              :    VAL must contain the original tree expression for the first operand
    4909              :    of *PTEST.  */
    4910              : 
    4911              : static enum insn_code
    4912      2646129 : validate_test_and_branch (tree val, rtx *ptest, machine_mode *pmode, optab *res)
    4913              : {
    4914      2646129 :   if (!val || TREE_CODE (val) != SSA_NAME)
    4915              :     return CODE_FOR_nothing;
    4916              : 
    4917      2541931 :   machine_mode mode = TYPE_MODE (TREE_TYPE (val));
    4918      2541931 :   rtx test = *ptest;
    4919      2541931 :   direct_optab optab;
    4920              : 
    4921      2541931 :   if (GET_CODE (test) == EQ)
    4922              :     optab = tbranch_eq_optab;
    4923      1325471 :   else if (GET_CODE (test) == NE)
    4924              :     optab = tbranch_ne_optab;
    4925              :   else
    4926              :     return CODE_FOR_nothing;
    4927              : 
    4928      2308175 :   *res = optab;
    4929              : 
    4930              :   /* If the target supports the testbit comparison directly, great.  */
    4931      2308175 :   auto icode = direct_optab_handler (optab, mode);
    4932      2308175 :   if (icode == CODE_FOR_nothing)
    4933              :     return icode;
    4934              : 
    4935            0 :   if (tree_zero_one_valued_p (val))
    4936              :     {
    4937            0 :       auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 : 0;
    4938            0 :       XEXP (test, 1) = gen_int_mode (pos, mode);
    4939            0 :       *ptest = test;
    4940            0 :       *pmode = mode;
    4941            0 :       return icode;
    4942              :     }
    4943              : 
    4944            0 :   wide_int wcst = get_nonzero_bits (val);
    4945            0 :   if (wcst == -1)
    4946              :     return CODE_FOR_nothing;
    4947              : 
    4948            0 :   int bitpos;
    4949              : 
    4950            0 :   if ((bitpos = wi::exact_log2 (wcst)) == -1)
    4951              :     return CODE_FOR_nothing;
    4952              : 
    4953            0 :   auto pos = BITS_BIG_ENDIAN ? GET_MODE_BITSIZE (mode) - 1 - bitpos : bitpos;
    4954            0 :   XEXP (test, 1) = gen_int_mode (pos, mode);
    4955            0 :   *ptest = test;
    4956            0 :   *pmode = mode;
    4957            0 :   return icode;
    4958            0 : }
    4959              : 
    4960              : /* Generate code to compare X with Y so that the condition codes are
    4961              :    set and to jump to LABEL if the condition is true.  If X is a
    4962              :    constant and Y is not a constant, then the comparison is swapped to
    4963              :    ensure that the comparison RTL has the canonical form.
    4964              : 
    4965              :    UNSIGNEDP nonzero says that X and Y are unsigned; this matters if they
    4966              :    need to be widened.  UNSIGNEDP is also used to select the proper
    4967              :    branch condition code.
    4968              : 
    4969              :    If X and Y have mode BLKmode, then SIZE specifies the size of both X and Y.
    4970              : 
    4971              :    MODE is the mode of the inputs (in case they are const_int).
    4972              : 
    4973              :    COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).
    4974              :    It will be potentially converted into an unsigned variant based on
    4975              :    UNSIGNEDP to select a proper jump instruction.
    4976              : 
    4977              :    PROB is the probability of jumping to LABEL.  If the comparison is against
    4978              :    zero then VAL contains the expression from which the non-zero RTL is
    4979              :    derived.  */
    4980              : 
    4981              : void
    4982      6456332 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
    4983              :                          machine_mode mode, int unsignedp, tree val, rtx label,
    4984              :                          profile_probability prob)
    4985              : {
    4986      6456332 :   rtx op0 = x, op1 = y;
    4987      6456332 :   rtx test;
    4988              : 
    4989              :   /* Swap operands and condition to ensure canonical RTL.  */
    4990      6456332 :   if (swap_commutative_operands_p (x, y)
    4991      6456332 :       && can_compare_p (swap_condition (comparison), mode, ccp_jump))
    4992              :     {
    4993            0 :       op0 = y, op1 = x;
    4994            0 :       comparison = swap_condition (comparison);
    4995              :     }
    4996              : 
    4997              :   /* If OP0 is still a constant, then both X and Y must be constants
    4998              :      or the opposite comparison is not supported.  Force X into a register
    4999              :      to create canonical RTL.  */
    5000      6456332 :   if (CONSTANT_P (op0))
    5001        20250 :     op0 = force_reg (mode, op0);
    5002              : 
    5003      6456332 :   if (unsignedp)
    5004      3842006 :     comparison = unsigned_condition (comparison);
    5005              : 
    5006              :   /* cbranch is no longer preferred for vectors, so when using a vector mode
    5007              :      check vec_cbranch variants instead.  */
    5008      6456332 :   if (!VECTOR_MODE_P (GET_MODE (op0)))
    5009      6436910 :     prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
    5010              :                       OPTAB_LIB_WIDEN, &test, &mode, cbranch_optab);
    5011              : 
    5012              :   /* Check if we're comparing a truth type with 0, and if so check if
    5013              :      the target supports tbranch.  */
    5014      6456332 :   machine_mode tmode = mode;
    5015      6456332 :   direct_optab optab;
    5016      6456332 :   if (op1 == CONST0_RTX (GET_MODE (op1)))
    5017              :     {
    5018      2647754 :       if (!VECTOR_MODE_P (GET_MODE (op1))
    5019      2646129 :           && validate_test_and_branch (val, &test, &tmode,
    5020              :                                        &optab) != CODE_FOR_nothing)
    5021              :         {
    5022            0 :           emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, tmode,
    5023              :                                     label, optab, prob, true);
    5024           18 :           return;
    5025              :         }
    5026              : 
    5027              :       /* If we are comparing equality with 0, check if VAL is another equality
    5028              :          comparison and if the target supports it directly.  */
    5029      2647754 :       gimple *def_stmt = NULL;
    5030      2544390 :       if (val && TREE_CODE (val) == SSA_NAME
    5031      2543525 :           && VECTOR_BOOLEAN_TYPE_P (TREE_TYPE (val))
    5032         2524 :           && (comparison == NE || comparison == EQ)
    5033      2650278 :           && (def_stmt = get_gimple_for_ssa_name (val)))
    5034              :         {
    5035         1783 :           tree masked_op = NULL_TREE;
    5036         1783 :           tree len_op = NULL_TREE;
    5037         1783 :           tree len_bias = NULL_TREE;
    5038              :           /* First determine if the operation should be masked or unmasked.  */
    5039         1783 :           if (is_gimple_assign (def_stmt)
    5040         1783 :               && gimple_assign_rhs_code (def_stmt) == BIT_AND_EXPR)
    5041              :             {
    5042              :               /* See if one side if a comparison, if so use the other side as
    5043              :                  the mask.  */
    5044           20 :               gimple *mask_def = NULL;
    5045           20 :               tree rhs1 = gimple_assign_rhs1 (def_stmt);
    5046           20 :               tree rhs2 = gimple_assign_rhs2 (def_stmt);
    5047           20 :               if (TREE_CODE (rhs1) == SSA_NAME
    5048           20 :                   && (mask_def = get_gimple_for_ssa_name (rhs1))
    5049           20 :                   && is_gimple_assign (mask_def)
    5050           40 :                   && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
    5051              :                 masked_op = rhs2;
    5052            0 :               else if (TREE_CODE (rhs2) == SSA_NAME
    5053            0 :                        && (mask_def = get_gimple_for_ssa_name (rhs2))
    5054            0 :                        && is_gimple_assign (mask_def)
    5055            0 :                        && TREE_CODE_CLASS (gimple_assign_rhs_code (mask_def)))
    5056              :                 masked_op = rhs1;
    5057              : 
    5058           20 :               if (masked_op)
    5059         1783 :                 def_stmt = mask_def;
    5060              :             }
    5061              :             /* Else check to see if we're a LEN target.  */
    5062         1763 :           else if (is_gimple_call (def_stmt)
    5063            0 :                    && gimple_call_internal_p (def_stmt)
    5064         1763 :                    && gimple_call_internal_fn (def_stmt) == IFN_VCOND_MASK_LEN)
    5065              :             {
    5066              :               /* Example to consume:
    5067              : 
    5068              :                    a = _59 != vect__4.17_75;
    5069              :                    vcmp = .VCOND_MASK_LEN (a, { -1, ... }, { 0, ... }, _90, 0);
    5070              :                    if (vcmp != { 0, ... })
    5071              : 
    5072              :                 and transform into
    5073              : 
    5074              :                    if (cond_len_vec_cbranch_any ({-1, ...}, a, _90, 0)).  */
    5075            0 :               gcall *call = dyn_cast <gcall *> (def_stmt);
    5076            0 :               tree true_branch = gimple_call_arg (call, 1);
    5077            0 :               tree false_branch = gimple_call_arg (call, 2);
    5078            0 :               if (integer_minus_onep (true_branch)
    5079            0 :                   && integer_zerop (false_branch))
    5080              :                 {
    5081            0 :                   len_op = gimple_call_arg (call, 3);
    5082            0 :                   len_bias = gimple_call_arg (call, 4);
    5083            0 :                   tree arg0 = gimple_call_arg (call, 0);
    5084              : 
    5085            0 :                   if (TREE_CODE (arg0) == SSA_NAME)
    5086            0 :                     def_stmt = get_gimple_for_ssa_name (arg0);
    5087              :                 }
    5088              :             }
    5089              : 
    5090         1783 :           enum insn_code icode;
    5091         1783 :           if (def_stmt
    5092         1783 :               && is_gimple_assign (def_stmt)
    5093         3566 :               && TREE_CODE_CLASS (gimple_assign_rhs_code (def_stmt))
    5094              :                    == tcc_comparison)
    5095              :             {
    5096         1595 :               class expand_operand ops[5];
    5097         1595 :               rtx_insn *tmp = NULL;
    5098         1595 :               start_sequence ();
    5099         1595 :               tree t_op0 = gimple_assign_rhs1 (def_stmt);
    5100         1595 :               tree t_op1 = gimple_assign_rhs2 (def_stmt);
    5101         1595 :               rtx op0c = expand_normal (t_op0);
    5102         1595 :               rtx op1c = expand_normal (t_op1);
    5103         1595 :               machine_mode mode2 = GET_MODE (op0c);
    5104              : 
    5105         1595 :               int nops = masked_op ? 3 : (len_op ? 5 : 2);
    5106         1595 :               int offset = masked_op || len_op ? 1 : 0;
    5107         1595 :               create_input_operand (&ops[offset + 0], op0c, mode2);
    5108         1595 :               create_input_operand (&ops[offset + 1], op1c, mode2);
    5109         1595 :               if (masked_op)
    5110              :                 {
    5111           18 :                   auto mask_mode = TYPE_MODE (TREE_TYPE (masked_op));
    5112           18 :                   rtx mask_op = expand_normal (masked_op);
    5113           18 :                   create_input_operand (&ops[0], mask_op, mask_mode);
    5114              :                 }
    5115         1577 :               else if (len_op)
    5116              :                 {
    5117            0 :                   rtx len_rtx = expand_normal (len_op);
    5118            0 :                   rtx len_bias_rtx = expand_normal (len_bias);
    5119            0 :                   tree lhs = gimple_get_lhs (def_stmt);
    5120            0 :                   auto mask_mode = TYPE_MODE (TREE_TYPE (lhs));
    5121              :                   /* ??? We could use something like internal_fn's
    5122              :                      add_mask_else_and_len_args here.  Currently it
    5123              :                      only supports a fixed, consecutive order of
    5124              :                      mask and len, though.  */
    5125            0 :                   create_input_operand (&ops[0], CONSTM1_RTX (mask_mode),
    5126              :                                         mask_mode);
    5127            0 :                   create_convert_operand_from
    5128            0 :                     (&ops[3], len_rtx, TYPE_MODE (TREE_TYPE (len_op)),
    5129            0 :                      TYPE_UNSIGNED (TREE_TYPE (len_op)));
    5130            0 :                   create_input_operand (&ops[4], len_bias_rtx, QImode);
    5131              :                 }
    5132              : 
    5133         1595 :               int unsignedp2 = TYPE_UNSIGNED (TREE_TYPE (t_op0));
    5134         1595 :               auto inner_code = gimple_assign_rhs_code (def_stmt);
    5135         1595 :               rtx test2 = NULL_RTX;
    5136              : 
    5137         1595 :               enum rtx_code comparison2 = get_rtx_code (inner_code, unsignedp2);
    5138         1595 :               if (unsignedp2)
    5139          526 :                 comparison2 = unsigned_condition (comparison2);
    5140         1595 :               if (comparison == NE)
    5141         1815 :                 optab = masked_op ? cond_vec_cbranch_any_optab
    5142          902 :                                   : len_op ? cond_len_vec_cbranch_any_optab
    5143              :                                            : vec_cbranch_any_optab;
    5144              :               else
    5145         1357 :                 optab = masked_op ? cond_vec_cbranch_all_optab
    5146          675 :                                   : len_op ? cond_len_vec_cbranch_all_optab
    5147              :                                            : vec_cbranch_all_optab;
    5148              : 
    5149         1595 :               if ((icode = optab_handler (optab, mode2))
    5150              :                   != CODE_FOR_nothing
    5151         1595 :                   && maybe_legitimize_operands (icode, 1, nops, ops))
    5152              :                 {
    5153           18 :                   test2 = gen_rtx_fmt_ee (comparison2, VOIDmode,
    5154              :                                           ops[offset + 0].value,
    5155              :                                           ops[offset + 1].value);
    5156           18 :                   if (insn_operand_matches (icode, 0, test2))
    5157              :                     {
    5158           36 :                       rtx mask
    5159           18 :                         = (masked_op || len_op) ? ops[0].value : NULL_RTX;
    5160           18 :                       rtx len = len_op ? ops[3].value : NULL_RTX;
    5161            0 :                       rtx bias = len_op ? ops[4].value : NULL_RTX;
    5162           18 :                       emit_cmp_and_jump_insn_1 (test2, mask, len, bias, mode2,
    5163              :                                                 label, optab, prob, false);
    5164           18 :                       tmp = get_insns ();
    5165              :                     }
    5166              :                 }
    5167              : 
    5168           18 :               end_sequence ();
    5169         1595 :               if (tmp)
    5170              :                 {
    5171           18 :                   emit_insn (tmp);
    5172           18 :                   return;
    5173              :                 }
    5174              :             }
    5175              :         }
    5176              :     }
    5177              : 
    5178              :   /*  cbranch should only be used for VECTOR_BOOLEAN_TYPE_P values.   */
    5179      6456314 :   direct_optab base_optab = cbranch_optab;
    5180      6456314 :   if (VECTOR_MODE_P (GET_MODE (op0)))
    5181              :     {
    5182              :       /* If cbranch is provided, use it.  If we get here it means we have an
    5183              :          instruction in between what created the boolean value and the gcond
    5184              :          that is not a masking operation.  This can happen for instance during
    5185              :          unrolling of early-break where we have an OR-reduction to reduce the
    5186              :          masks.  In this case knowing we have a mask can let us generate better
    5187              :          code.  If it's not there there then check the vector specific
    5188              :          optabs.  */
    5189        19422 :       if (optab_handler (cbranch_optab, mode) == CODE_FOR_nothing)
    5190              :         {
    5191            0 :           if (comparison == NE)
    5192              :             base_optab = vec_cbranch_any_optab;
    5193              :           else
    5194            0 :             base_optab = vec_cbranch_all_optab;
    5195              : 
    5196            0 :           prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
    5197              :                             OPTAB_DIRECT, &test, &mode, base_optab);
    5198              : 
    5199            0 :           enum insn_code icode = optab_handler (base_optab, mode);
    5200              : 
    5201              :           /* If the new cbranch isn't supported, degrade back to old one.  */
    5202            0 :           if (icode == CODE_FOR_nothing
    5203            0 :               || !test
    5204            0 :               || !insn_operand_matches (icode, 0, test))
    5205              :             base_optab = cbranch_optab;
    5206              :         }
    5207              : 
    5208        19422 :       prepare_cmp_insn (op0, op1, NULL, comparison, size, unsignedp,
    5209              :                         OPTAB_LIB_WIDEN, &test, &mode, base_optab);
    5210              :     }
    5211              : 
    5212      6456314 :   emit_cmp_and_jump_insn_1 (test, NULL_RTX, NULL_RTX, NULL_RTX, mode, label,
    5213              :                             base_optab, prob, false);
    5214              : }
    5215              : 
    5216              : /* Overloaded version of emit_cmp_and_jump_insns in which VAL is unknown.  */
    5217              : 
    5218              : void
    5219       116074 : emit_cmp_and_jump_insns (rtx x, rtx y, enum rtx_code comparison, rtx size,
    5220              :                          machine_mode mode, int unsignedp, rtx label,
    5221              :                          profile_probability prob)
    5222              : {
    5223       116074 :   emit_cmp_and_jump_insns (x, y, comparison, size, mode, unsignedp, NULL,
    5224              :                            label, prob);
    5225       116074 : }
    5226              : 
    5227              : 
    5228              : /* Emit a library call comparison between floating point X and Y.
    5229              :    COMPARISON is the rtl operator to compare with (EQ, NE, GT, etc.).  */
    5230              : 
    5231              : static void
    5232        44978 : prepare_float_lib_cmp (rtx x, rtx y, enum rtx_code comparison,
    5233              :                        rtx *ptest, machine_mode *pmode)
    5234              : {
    5235        44978 :   enum rtx_code swapped = swap_condition (comparison);
    5236        44978 :   enum rtx_code reversed = reverse_condition_maybe_unordered (comparison);
    5237        44978 :   machine_mode orig_mode = GET_MODE (x);
    5238        44978 :   machine_mode mode;
    5239        44978 :   rtx true_rtx, false_rtx;
    5240        44978 :   rtx value, target, equiv;
    5241        44978 :   rtx_insn *insns;
    5242        44978 :   rtx libfunc = 0;
    5243        44978 :   bool reversed_p = false;
    5244        44978 :   scalar_int_mode cmp_mode = targetm.libgcc_cmp_return_mode ();
    5245              : 
    5246        44978 :   FOR_EACH_WIDER_MODE_FROM (mode, orig_mode)
    5247              :     {
    5248        44978 :       if (code_to_optab (comparison)
    5249        44978 :           && (libfunc = optab_libfunc (code_to_optab (comparison), mode)))
    5250              :         break;
    5251              : 
    5252            0 :       if (code_to_optab (swapped)
    5253            0 :           && (libfunc = optab_libfunc (code_to_optab (swapped), mode)))
    5254              :         {
    5255              :           std::swap (x, y);
    5256              :           comparison = swapped;
    5257              :           break;
    5258              :         }
    5259              : 
    5260            0 :       if (code_to_optab (reversed)
    5261            0 :           && (libfunc = optab_libfunc (code_to_optab (reversed), mode)))
    5262              :         {
    5263              :           comparison = reversed;
    5264              :           reversed_p = true;
    5265              :           break;
    5266              :         }
    5267              :     }
    5268              : 
    5269        44978 :   gcc_assert (mode != VOIDmode);
    5270              : 
    5271        44978 :   if (mode != orig_mode)
    5272              :     {
    5273            0 :       x = convert_to_mode (mode, x, 0);
    5274            0 :       y = convert_to_mode (mode, y, 0);
    5275              :     }
    5276              : 
    5277              :   /* Attach a REG_EQUAL note describing the semantics of the libcall to
    5278              :      the RTL.  The allows the RTL optimizers to delete the libcall if the
    5279              :      condition can be determined at compile-time.  */
    5280        44978 :   if (comparison == UNORDERED
    5281              :       || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
    5282              :     {
    5283         5809 :       true_rtx = const_true_rtx;
    5284         5809 :       false_rtx = const0_rtx;
    5285              :     }
    5286              :   else
    5287              :     {
    5288        39169 :       switch (comparison)
    5289              :         {
    5290         3515 :         case EQ:
    5291         3515 :           true_rtx = const0_rtx;
    5292         3515 :           false_rtx = const_true_rtx;
    5293         3515 :           break;
    5294              : 
    5295        23583 :         case NE:
    5296        23583 :           true_rtx = const_true_rtx;
    5297        23583 :           false_rtx = const0_rtx;
    5298        23583 :           break;
    5299              : 
    5300         3030 :         case GT:
    5301         3030 :           true_rtx = const1_rtx;
    5302         3030 :           false_rtx = const0_rtx;
    5303         3030 :           break;
    5304              : 
    5305         3025 :         case GE:
    5306         3025 :           true_rtx = const0_rtx;
    5307         3025 :           false_rtx = constm1_rtx;
    5308         3025 :           break;
    5309              : 
    5310         2487 :         case LT:
    5311         2487 :           true_rtx = constm1_rtx;
    5312         2487 :           false_rtx = const0_rtx;
    5313         2487 :           break;
    5314              : 
    5315         3529 :         case LE:
    5316         3529 :           true_rtx = const0_rtx;
    5317         3529 :           false_rtx = const1_rtx;
    5318         3529 :           break;
    5319              : 
    5320            0 :         default:
    5321            0 :           gcc_unreachable ();
    5322              :         }
    5323              :     }
    5324              : 
    5325        44978 :   if (comparison == UNORDERED)
    5326              :     {
    5327         5809 :       rtx temp = simplify_gen_relational (NE, cmp_mode, mode, x, x);
    5328         5809 :       equiv = simplify_gen_relational (NE, cmp_mode, mode, y, y);
    5329         5809 :       equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
    5330              :                                     temp, const_true_rtx, equiv);
    5331              :     }
    5332              :   else
    5333              :     {
    5334        39169 :       equiv = simplify_gen_relational (comparison, cmp_mode, mode, x, y);
    5335        39169 :       if (! FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison))
    5336        39169 :         equiv = simplify_gen_ternary (IF_THEN_ELSE, cmp_mode, cmp_mode,
    5337              :                                       equiv, true_rtx, false_rtx);
    5338              :     }
    5339              : 
    5340        44978 :   start_sequence ();
    5341        44978 :   value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
    5342              :                                    cmp_mode, x, mode, y, mode);
    5343        44978 :   insns = end_sequence ();
    5344              : 
    5345        44978 :   target = gen_reg_rtx (cmp_mode);
    5346        44978 :   emit_libcall_block (insns, target, value, equiv);
    5347              : 
    5348        44978 :   if (comparison == UNORDERED
    5349              :       || FLOAT_LIB_COMPARE_RETURNS_BOOL (mode, comparison)
    5350        44978 :       || reversed_p)
    5351        11618 :     *ptest = gen_rtx_fmt_ee (reversed_p ? EQ : NE, VOIDmode, target, false_rtx);
    5352              :   else
    5353        39169 :     *ptest = gen_rtx_fmt_ee (comparison, VOIDmode, target, const0_rtx);
    5354              : 
    5355        44978 :   *pmode = cmp_mode;
    5356        44978 : }
    5357              : 
    5358              : /* Generate code to indirectly jump to a location given in the rtx LOC.  */
    5359              : 
    5360              : void
    5361         1338 : emit_indirect_jump (rtx loc)
    5362              : {
    5363         1338 :   if (!targetm.have_indirect_jump ())
    5364            0 :     sorry ("indirect jumps are not available on this target");
    5365              :   else
    5366              :     {
    5367         1338 :       class expand_operand ops[1];
    5368         1338 :       create_address_operand (&ops[0], loc);
    5369         1338 :       expand_jump_insn (targetm.code_for_indirect_jump, 1, ops);
    5370         1338 :       emit_barrier ();
    5371              :     }
    5372         1338 : }
    5373              : 
    5374              : 
    5375              : /* Emit a conditional move instruction if the machine supports one for that
    5376              :    condition and machine mode.
    5377              : 
    5378              :    OP0 and OP1 are the operands that should be compared using CODE.  CMODE is
    5379              :    the mode to use should they be constants.  If it is VOIDmode, they cannot
    5380              :    both be constants.
    5381              : 
    5382              :    OP2 should be stored in TARGET if the comparison is true, otherwise OP3
    5383              :    should be stored there.  MODE is the mode to use should they be constants.
    5384              :    If it is VOIDmode, they cannot both be constants.
    5385              : 
    5386              :    The result is either TARGET (perhaps modified) or NULL_RTX if the operation
    5387              :    is not supported.  */
    5388              : 
    5389              : rtx
    5390       476120 : emit_conditional_move (rtx target, struct rtx_comparison comp,
    5391              :                        rtx op2, rtx op3,
    5392              :                        machine_mode mode, int unsignedp)
    5393              : {
    5394       476120 :   rtx comparison;
    5395       476120 :   rtx_insn *last;
    5396       476120 :   enum insn_code icode;
    5397       476120 :   enum rtx_code reversed;
    5398              : 
    5399              :   /* If the two source operands are identical, that's just a move.  */
    5400              : 
    5401       476120 :   if (rtx_equal_p (op2, op3))
    5402              :     {
    5403         2262 :       if (!target)
    5404            0 :         target = gen_reg_rtx (mode);
    5405              : 
    5406         2262 :       emit_move_insn (target, op3);
    5407         2262 :       return target;
    5408              :     }
    5409              : 
    5410              :   /* If one operand is constant, make it the second one.  Only do this
    5411              :      if the other operand is not constant as well.  */
    5412              : 
    5413       473858 :   if (swap_commutative_operands_p (comp.op0, comp.op1))
    5414              :     {
    5415        18096 :       std::swap (comp.op0, comp.op1);
    5416        18096 :       comp.code = swap_condition (comp.code);
    5417              :     }
    5418              : 
    5419              :   /* get_condition will prefer to generate LT and GT even if the old
    5420              :      comparison was against zero, so undo that canonicalization here since
    5421              :      comparisons against zero are cheaper.  */
    5422              : 
    5423       473858 :   if (comp.code == LT && comp.op1 == const1_rtx)
    5424         3358 :     comp.code = LE, comp.op1 = const0_rtx;
    5425       470500 :   else if (comp.code == GT && comp.op1 == constm1_rtx)
    5426         7233 :     comp.code = GE, comp.op1 = const0_rtx;
    5427              : 
    5428       473858 :   if (comp.mode == VOIDmode)
    5429       387575 :     comp.mode = GET_MODE (comp.op0);
    5430              : 
    5431       473858 :   enum rtx_code orig_code = comp.code;
    5432       473858 :   bool swapped = false;
    5433       473858 :   if (swap_commutative_operands_p (op2, op3)
    5434       473858 :       && ((reversed =
    5435       123089 :            reversed_comparison_code_parts (comp.code, comp.op0, comp.op1, NULL))
    5436              :           != UNKNOWN))
    5437              :     {
    5438              :       std::swap (op2, op3);
    5439              :       comp.code = reversed;
    5440              :       swapped = true;
    5441              :     }
    5442              : 
    5443       473858 :   if (mode == VOIDmode)
    5444           78 :     mode = GET_MODE (op2);
    5445              : 
    5446       473858 :   icode = direct_optab_handler (movcc_optab, mode);
    5447              : 
    5448       473858 :   if (icode == CODE_FOR_nothing)
    5449              :     return NULL_RTX;
    5450              : 
    5451       465997 :   if (!target)
    5452           32 :     target = gen_reg_rtx (mode);
    5453              : 
    5454       465997 :   for (int pass = 0; ; pass++)
    5455              :     {
    5456       497131 :       comp.code = unsignedp ? unsigned_condition (comp.code) : comp.code;
    5457       994262 :       comparison =
    5458       497131 :         simplify_gen_relational (comp.code, VOIDmode,
    5459              :                                  comp.mode, comp.op0, comp.op1);
    5460              : 
    5461              :       /* We can get const0_rtx or const_true_rtx in some circumstances.  Just
    5462              :          punt and let the caller figure out how best to deal with this
    5463              :          situation.  */
    5464       497131 :       if (COMPARISON_P (comparison))
    5465              :         {
    5466       496291 :           saved_pending_stack_adjust save;
    5467       496291 :           save_pending_stack_adjust (&save);
    5468       496291 :           last = get_last_insn ();
    5469       496291 :           do_pending_stack_adjust ();
    5470       496291 :           machine_mode cmpmode = comp.mode;
    5471       496291 :           rtx orig_op0 = XEXP (comparison, 0);
    5472       496291 :           rtx orig_op1 = XEXP (comparison, 1);
    5473       496291 :           rtx op2p = op2;
    5474       496291 :           rtx op3p = op3;
    5475              :           /* If we are optimizing, force expensive constants into a register
    5476              :              but preserve an eventual equality with op2/op3.  */
    5477            1 :           if (CONSTANT_P (orig_op0) && optimize
    5478            1 :               && cmpmode == mode
    5479            1 :               && (rtx_cost (orig_op0, mode, COMPARE, 0,
    5480              :                             optimize_insn_for_speed_p ())
    5481              :                   > COSTS_N_INSNS (1))
    5482       496291 :               && can_create_pseudo_p ())
    5483              :             {
    5484            0 :               if (rtx_equal_p (orig_op0, op2))
    5485            0 :                 op2p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
    5486            0 :               else if (rtx_equal_p (orig_op0, op3))
    5487            0 :                 op3p = XEXP (comparison, 0) = force_reg (cmpmode, orig_op0);
    5488              :             }
    5489       276580 :           if (CONSTANT_P (orig_op1) && optimize
    5490       255489 :               && cmpmode == mode
    5491        89932 :               && (rtx_cost (orig_op1, mode, COMPARE, 0,
    5492              :                             optimize_insn_for_speed_p ())
    5493              :                   > COSTS_N_INSNS (1))
    5494       496468 :               && can_create_pseudo_p ())
    5495              :             {
    5496          177 :               if (rtx_equal_p (orig_op1, op2))
    5497          110 :                 op2p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
    5498           67 :               else if (rtx_equal_p (orig_op1, op3))
    5499           45 :                 op3p = XEXP (comparison, 1) = force_reg (cmpmode, orig_op1);
    5500              :             }
    5501       496291 :           prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
    5502       496291 :                             GET_CODE (comparison), NULL_RTX, unsignedp,
    5503              :                             OPTAB_WIDEN, &comparison, &cmpmode, cbranch_optab);
    5504       496291 :           if (comparison)
    5505              :             {
    5506       494135 :                rtx res = emit_conditional_move_1 (target, comparison,
    5507              :                                                   op2p, op3p, mode);
    5508       494135 :                if (res != NULL_RTX)
    5509       432253 :                  return res;
    5510              :             }
    5511        64038 :           delete_insns_since (last);
    5512        64038 :           restore_pending_stack_adjust (&save);
    5513              :         }
    5514              : 
    5515        64878 :       if (pass == 1)
    5516              :         return NULL_RTX;
    5517              : 
    5518              :       /* If the preferred op2/op3 order is not usable, retry with other
    5519              :          operand order, perhaps it will expand successfully.  */
    5520        33802 :       if (swapped)
    5521              :         comp.code = orig_code;
    5522        66850 :       else if ((reversed =
    5523        33425 :                 reversed_comparison_code_parts (orig_code, comp.op0, comp.op1,
    5524              :                                                            NULL))
    5525              :                != UNKNOWN)
    5526              :         comp.code = reversed;
    5527              :       else
    5528              :         return NULL_RTX;
    5529        31134 :       std::swap (op2, op3);
    5530        31134 :     }
    5531              : }
    5532              : 
    5533              : /* Helper function that, in addition to COMPARISON, also tries
    5534              :    the reversed REV_COMPARISON with swapped OP2 and OP3.  As opposed
    5535              :    to when we pass the specific constituents of a comparison, no
    5536              :    additional insns are emitted for it.  It might still be necessary
    5537              :    to emit more than one insn for the final conditional move, though.  */
    5538              : 
    5539              : rtx
    5540       106556 : emit_conditional_move (rtx target, rtx comparison, rtx rev_comparison,
    5541              :                        rtx op2, rtx op3, machine_mode mode)
    5542              : {
    5543       106556 :   rtx res = emit_conditional_move_1 (target, comparison, op2, op3, mode);
    5544              : 
    5545       106556 :   if (res != NULL_RTX)
    5546              :     return res;
    5547              : 
    5548         5002 :   return emit_conditional_move_1 (target, rev_comparison, op3, op2, mode);
    5549              : }
    5550              : 
    5551              : /* Helper for emitting a conditional move.  */
    5552              : 
    5553              : static rtx
    5554       605693 : emit_conditional_move_1 (rtx target, rtx comparison,
    5555              :                          rtx op2, rtx op3, machine_mode mode)
    5556              : {
    5557       605693 :   enum insn_code icode;
    5558              : 
    5559       605693 :   if (comparison == NULL_RTX || !COMPARISON_P (comparison))
    5560              :     return NULL_RTX;
    5561              : 
    5562              :   /* If the two source operands are identical, that's just a move.
    5563              :      As the comparison comes in non-canonicalized, we must make
    5564              :      sure not to discard any possible side effects.  If there are
    5565              :      side effects, just let the target handle it.  */
    5566       605693 :   if (!side_effects_p (comparison) && rtx_equal_p (op2, op3))
    5567              :     {
    5568            0 :       if (!target)
    5569            0 :         target = gen_reg_rtx (mode);
    5570              : 
    5571            0 :       emit_move_insn (target, op3);
    5572            0 :       return target;
    5573              :     }
    5574              : 
    5575       605693 :   if (mode == VOIDmode)
    5576            0 :     mode = GET_MODE (op2);
    5577              : 
    5578       605693 :   icode = direct_optab_handler (movcc_optab, mode);
    5579              : 
    5580       605693 :   if (icode == CODE_FOR_nothing)
    5581              :     return NULL_RTX;
    5582              : 
    5583       605693 :   if (!target)
    5584            0 :     target = gen_reg_rtx (mode);
    5585              : 
    5586       605693 :   class expand_operand ops[4];
    5587              : 
    5588       605693 :   create_output_operand (&ops[0], target, mode);
    5589       605693 :   create_fixed_operand (&ops[1], comparison);
    5590       605693 :   create_input_operand (&ops[2], op2, mode);
    5591       605693 :   create_input_operand (&ops[3], op3, mode);
    5592              : 
    5593       605693 :   if (maybe_expand_insn (icode, 4, ops))
    5594              :     {
    5595       533807 :       if (ops[0].value != target)
    5596           57 :         convert_move (target, ops[0].value, false);
    5597              :       return target;
    5598              :     }
    5599              : 
    5600              :   return NULL_RTX;
    5601              : }
    5602              : 
    5603              : 
    5604              : /* Emit a conditional negate or bitwise complement using the
    5605              :    negcc or notcc optabs if available.  Return NULL_RTX if such operations
    5606              :    are not available.  Otherwise return the RTX holding the result.
    5607              :    TARGET is the desired destination of the result.  COMP is the comparison
    5608              :    on which to negate.  If COND is true move into TARGET the negation
    5609              :    or bitwise complement of OP1.  Otherwise move OP2 into TARGET.
    5610              :    CODE is either NEG or NOT.  MODE is the machine mode in which the
    5611              :    operation is performed.  */
    5612              : 
    5613              : rtx
    5614         1378 : emit_conditional_neg_or_complement (rtx target, rtx_code code,
    5615              :                                      machine_mode mode, rtx cond, rtx op1,
    5616              :                                      rtx op2)
    5617              : {
    5618         1378 :   optab op = unknown_optab;
    5619         1378 :   if (code == NEG)
    5620              :     op = negcc_optab;
    5621           68 :   else if (code == NOT)
    5622              :     op = notcc_optab;
    5623              :   else
    5624            0 :     gcc_unreachable ();
    5625              : 
    5626         1378 :   insn_code icode = direct_optab_handler (op, mode);
    5627              : 
    5628         1378 :   if (icode == CODE_FOR_nothing)
    5629              :     return NULL_RTX;
    5630              : 
    5631            0 :   if (!target)
    5632            0 :     target = gen_reg_rtx (mode);
    5633              : 
    5634            0 :   rtx_insn *last = get_last_insn ();
    5635            0 :   class expand_operand ops[4];
    5636              : 
    5637            0 :   create_output_operand (&ops[0], target, mode);
    5638            0 :   create_fixed_operand (&ops[1], cond);
    5639            0 :   create_input_operand (&ops[2], op1, mode);
    5640            0 :   create_input_operand (&ops[3], op2, mode);
    5641              : 
    5642            0 :   if (maybe_expand_insn (icode, 4, ops))
    5643              :     {
    5644            0 :       if (ops[0].value != target)
    5645            0 :         convert_move (target, ops[0].value, false);
    5646              : 
    5647              :       return target;
    5648              :     }
    5649            0 :   delete_insns_since (last);
    5650            0 :   return NULL_RTX;
    5651              : }
    5652              : 
    5653              : /* Emit a conditional addition instruction if the machine supports one for that
    5654              :    condition and machine mode.
    5655              : 
    5656              :    OP0 and OP1 are the operands that should be compared using CODE.  CMODE is
    5657              :    the mode to use should they be constants.  If it is VOIDmode, they cannot
    5658              :    both be constants.
    5659              : 
    5660              :    OP2 should be stored in TARGET if the comparison is false, otherwise OP2+OP3
    5661              :    should be stored there.  MODE is the mode to use should they be constants.
    5662              :    If it is VOIDmode, they cannot both be constants.
    5663              : 
    5664              :    The result is either TARGET (perhaps modified) or NULL_RTX if the operation
    5665              :    is not supported.  */
    5666              : 
    5667              : rtx
    5668        11029 : emit_conditional_add (rtx target, enum rtx_code code, rtx op0, rtx op1,
    5669              :                       machine_mode cmode, rtx op2, rtx op3,
    5670              :                       machine_mode mode, int unsignedp)
    5671              : {
    5672        11029 :   rtx comparison;
    5673        11029 :   rtx_insn *last;
    5674        11029 :   enum insn_code icode;
    5675              : 
    5676              :   /* If one operand is constant, make it the second one.  Only do this
    5677              :      if the other operand is not constant as well.  */
    5678              : 
    5679        11029 :   if (swap_commutative_operands_p (op0, op1))
    5680              :     {
    5681            2 :       std::swap (op0, op1);
    5682            2 :       code = swap_condition (code);
    5683              :     }
    5684              : 
    5685              :   /* get_condition will prefer to generate LT and GT even if the old
    5686              :      comparison was against zero, so undo that canonicalization here since
    5687              :      comparisons against zero are cheaper.  */
    5688        11029 :   if (code == LT && op1 == const1_rtx)
    5689           20 :     code = LE, op1 = const0_rtx;
    5690        11009 :   else if (code == GT && op1 == constm1_rtx)
    5691           67 :     code = GE, op1 = const0_rtx;
    5692              : 
    5693        11029 :   if (cmode == VOIDmode)
    5694        11029 :     cmode = GET_MODE (op0);
    5695              : 
    5696        11029 :   if (mode == VOIDmode)
    5697            0 :     mode = GET_MODE (op2);
    5698              : 
    5699        11029 :   icode = optab_handler (addcc_optab, mode);
    5700              : 
    5701        11029 :   if (icode == CODE_FOR_nothing)
    5702              :     return 0;
    5703              : 
    5704        10309 :   if (!target)
    5705            0 :     target = gen_reg_rtx (mode);
    5706              : 
    5707        10309 :   code = unsignedp ? unsigned_condition (code) : code;
    5708        10309 :   comparison = simplify_gen_relational (code, VOIDmode, cmode, op0, op1);
    5709              : 
    5710              :   /* We can get const0_rtx or const_true_rtx in some circumstances.  Just
    5711              :      return NULL and let the caller figure out how best to deal with this
    5712              :      situation.  */
    5713        10309 :   if (!COMPARISON_P (comparison))
    5714              :     return NULL_RTX;
    5715              : 
    5716        10309 :   do_pending_stack_adjust ();
    5717        10309 :   last = get_last_insn ();
    5718        10309 :   prepare_cmp_insn (XEXP (comparison, 0), XEXP (comparison, 1), NULL,
    5719        10309 :                     GET_CODE (comparison), NULL_RTX, unsignedp, OPTAB_WIDEN,
    5720              :                     &comparison, &cmode, cbranch_optab);
    5721        10309 :   if (comparison)
    5722              :     {
    5723        10309 :       class expand_operand ops[4];
    5724              : 
    5725        10309 :       create_output_operand (&ops[0], target, mode);
    5726        10309 :       create_fixed_operand (&ops[1], comparison);
    5727        10309 :       create_input_operand (&ops[2], op2, mode);
    5728        10309 :       create_input_operand (&ops[3], op3, mode);
    5729        10309 :       if (maybe_expand_insn (icode, 4, ops))
    5730              :         {
    5731          342 :           if (ops[0].value != target)
    5732            0 :             convert_move (target, ops[0].value, false);
    5733          342 :           return target;
    5734              :         }
    5735              :     }
    5736         9967 :   delete_insns_since (last);
    5737         9967 :   return NULL_RTX;
    5738              : }
    5739              : 
    5740              : /* These functions attempt to generate an insn body, rather than
    5741              :    emitting the insn, but if the gen function already emits them, we
    5742              :    make no attempt to turn them back into naked patterns.  */
    5743              : 
    5744              : /* Generate and return an insn body to add Y to X.  */
    5745              : 
    5746              : rtx_insn *
    5747          250 : gen_add2_insn (rtx x, rtx y)
    5748              : {
    5749          250 :   enum insn_code icode = optab_handler (add_optab, GET_MODE (x));
    5750              : 
    5751          250 :   gcc_assert (insn_operand_matches (icode, 0, x));
    5752          250 :   gcc_assert (insn_operand_matches (icode, 1, x));
    5753          250 :   gcc_assert (insn_operand_matches (icode, 2, y));
    5754              : 
    5755          250 :   return GEN_FCN (icode) (x, x, y);
    5756              : }
    5757              : 
    5758              : /* Generate and return an insn body to add r1 and c,
    5759              :    storing the result in r0.  */
    5760              : 
    5761              : rtx_insn *
    5762            0 : gen_add3_insn (rtx r0, rtx r1, rtx c)
    5763              : {
    5764            0 :   enum insn_code icode = optab_handler (add_optab, GET_MODE (r0));
    5765              : 
    5766            0 :   if (icode == CODE_FOR_nothing
    5767            0 :       || !insn_operand_matches (icode, 0, r0)
    5768            0 :       || !insn_operand_matches (icode, 1, r1)
    5769            0 :       || !insn_operand_matches (icode, 2, c))
    5770              :     return NULL;
    5771              : 
    5772            0 :   return GEN_FCN (icode) (r0, r1, c);
    5773              : }
    5774              : 
    5775              : bool
    5776         4301 : have_add2_insn (rtx x, rtx y)
    5777              : {
    5778         4301 :   enum insn_code icode;
    5779              : 
    5780         4301 :   gcc_assert (GET_MODE (x) != VOIDmode);
    5781              : 
    5782         4301 :   icode = optab_handler (add_optab, GET_MODE (x));
    5783              : 
    5784         4301 :   if (icode == CODE_FOR_nothing)
    5785              :     return false;
    5786              : 
    5787         4301 :   if (!insn_operand_matches (icode, 0, x)
    5788         4301 :       || !insn_operand_matches (icode, 1, x)
    5789         8602 :       || !insn_operand_matches (icode, 2, y))
    5790          212 :     return false;
    5791              : 
    5792              :   return true;
    5793              : }
    5794              : 
    5795              : /* Generate and return an insn body to add Y to X.  */
    5796              : 
    5797              : rtx_insn *
    5798            0 : gen_addptr3_insn (rtx x, rtx y, rtx z)
    5799              : {
    5800            0 :   enum insn_code icode = optab_handler (addptr3_optab, GET_MODE (x));
    5801              : 
    5802            0 :   gcc_assert (insn_operand_matches (icode, 0, x));
    5803            0 :   gcc_assert (insn_operand_matches (icode, 1, y));
    5804            0 :   gcc_assert (insn_operand_matches (icode, 2, z));
    5805              : 
    5806            0 :   return GEN_FCN (icode) (x, y, z);
    5807              : }
    5808              : 
    5809              : /* Return true if the target implements an addptr pattern and X, Y,
    5810              :    and Z are valid for the pattern predicates.  */
    5811              : 
    5812              : bool
    5813       692940 : have_addptr3_insn (rtx x, rtx y, rtx z)
    5814              : {
    5815       692940 :   enum insn_code icode;
    5816              : 
    5817       692940 :   gcc_assert (GET_MODE (x) != VOIDmode);
    5818              : 
    5819       692940 :   icode = optab_handler (addptr3_optab, GET_MODE (x));
    5820              : 
    5821       692940 :   if (icode == CODE_FOR_nothing)
    5822              :     return false;
    5823              : 
    5824            0 :   if (!insn_operand_matches (icode, 0, x)
    5825            0 :       || !insn_operand_matches (icode, 1, y)
    5826            0 :       || !insn_operand_matches (icode, 2, z))
    5827            0 :     return false;
    5828              : 
    5829              :   return true;
    5830              : }
    5831              : 
    5832              : /* Generate and return an insn body to subtract Y from X.  */
    5833              : 
    5834              : rtx_insn *
    5835           11 : gen_sub2_insn (rtx x, rtx y)
    5836              : {
    5837           11 :   enum insn_code icode = optab_handler (sub_optab, GET_MODE (x));
    5838              : 
    5839           11 :   gcc_assert (insn_operand_matches (icode, 0, x));
    5840           11 :   gcc_assert (insn_operand_matches (icode, 1, x));
    5841           11 :   gcc_assert (insn_operand_matches (icode, 2, y));
    5842              : 
    5843           11 :   return GEN_FCN (icode) (x, x, y);
    5844              : }
    5845              : 
    5846              : /* Generate and return an insn body to subtract r1 and c,
    5847              :    storing the result in r0.  */
    5848              : 
    5849              : rtx_insn *
    5850         1832 : gen_sub3_insn (rtx r0, rtx r1, rtx c)
    5851              : {
    5852         1832 :   enum insn_code icode = optab_handler (sub_optab, GET_MODE (r0));
    5853              : 
    5854         1832 :   if (icode == CODE_FOR_nothing
    5855         1832 :       || !insn_operand_matches (icode, 0, r0)
    5856         1832 :       || !insn_operand_matches (icode, 1, r1)
    5857         3664 :       || !insn_operand_matches (icode, 2, c))
    5858              :     return NULL;
    5859              : 
    5860         1832 :   return GEN_FCN (icode) (r0, r1, c);
    5861              : }
    5862              : 
    5863              : bool
    5864            0 : have_sub2_insn (rtx x, rtx y)
    5865              : {
    5866            0 :   enum insn_code icode;
    5867              : 
    5868            0 :   gcc_assert (GET_MODE (x) != VOIDmode);
    5869              : 
    5870            0 :   icode = optab_handler (sub_optab, GET_MODE (x));
    5871              : 
    5872            0 :   if (icode == CODE_FOR_nothing)
    5873              :     return false;
    5874              : 
    5875            0 :   if (!insn_operand_matches (icode, 0, x)
    5876            0 :       || !insn_operand_matches (icode, 1, x)
    5877            0 :       || !insn_operand_matches (icode, 2, y))
    5878            0 :     return false;
    5879              : 
    5880              :   return true;
    5881              : }
    5882              : 
    5883              : /* Generate the body of an insn to extend Y (with mode MFROM)
    5884              :    into X (with mode MTO).  Do zero-extension if UNSIGNEDP is nonzero.  */
    5885              : 
    5886              : rtx_insn *
    5887        30441 : gen_extend_insn (rtx x, rtx y, machine_mode mto,
    5888              :                  machine_mode mfrom, int unsignedp)
    5889              : {
    5890        30441 :   enum insn_code icode = can_extend_p (mto, mfrom, unsignedp);
    5891        30441 :   return GEN_FCN (icode) (x, y);
    5892              : }
    5893              : 
    5894              : /* Generate code to convert FROM to floating point
    5895              :    and store in TO.  FROM must be fixed point and not VOIDmode.
    5896              :    UNSIGNEDP nonzero means regard FROM as unsigned.
    5897              :    Normally this is done by correcting the final value
    5898              :    if it is negative.  */
    5899              : 
    5900              : void
    5901       147722 : expand_float (rtx to, rtx from, int unsignedp)
    5902              : {
    5903       147722 :   enum insn_code icode;
    5904       147722 :   rtx target = to;
    5905       147722 :   scalar_mode from_mode, to_mode;
    5906       147722 :   machine_mode fmode, imode;
    5907       147722 :   bool can_do_signed = false;
    5908              : 
    5909              :   /* Crash now, because we won't be able to decide which mode to use.  */
    5910       147722 :   gcc_assert (GET_MODE (from) != VOIDmode);
    5911              : 
    5912              :   /* Look for an insn to do the conversion.  Do it in the specified
    5913              :      modes if possible; otherwise convert either input, output or both to
    5914              :      wider mode.  If the integer mode is wider than the mode of FROM,
    5915              :      we can do the conversion signed even if the input is unsigned.  */
    5916              : 
    5917       169667 :   FOR_EACH_MODE_FROM (fmode, GET_MODE (to))
    5918       263967 :     FOR_EACH_MODE_FROM (imode, GET_MODE (from))
    5919              :       {
    5920       242022 :         int doing_unsigned = unsignedp;
    5921              : 
    5922       272280 :         if (fmode != GET_MODE (to)
    5923       242022 :             && (significand_size (fmode)
    5924        94428 :                 < GET_MODE_UNIT_PRECISION (GET_MODE (from))))
    5925        30258 :           continue;
    5926              : 
    5927       211764 :         icode = can_float_p (fmode, imode, unsignedp);
    5928       211764 :         if (icode == CODE_FOR_nothing && unsignedp)
    5929              :           {
    5930        35466 :             enum insn_code scode = can_float_p (fmode, imode, 0);
    5931        35466 :             if (scode != CODE_FOR_nothing)
    5932         6411 :               can_do_signed = true;
    5933        35466 :             if (imode != GET_MODE (from))
    5934        24411 :               icode = scode, doing_unsigned = 0;
    5935              :           }
    5936              : 
    5937       211764 :         if (icode != CODE_FOR_nothing)
    5938              :           {
    5939       139218 :             if (imode != GET_MODE (from))
    5940        10330 :               from = convert_to_mode (imode, from, unsignedp);
    5941              : 
    5942       139218 :             if (fmode != GET_MODE (to))
    5943         1645 :               target = gen_reg_rtx (fmode);
    5944              : 
    5945       272728 :             emit_unop_insn (icode, target, from,
    5946              :                             doing_unsigned ? UNSIGNED_FLOAT : FLOAT);
    5947              : 
    5948       139218 :             if (target != to)
    5949         1645 :               convert_move (to, target, 0);
    5950       139218 :             return;
    5951              :           }
    5952              :       }
    5953              : 
    5954              :   /* Unsigned integer, and no way to convert directly.  Convert as signed,
    5955              :      then unconditionally adjust the result.  */
    5956         8504 :   if (unsignedp
    5957         8504 :       && can_do_signed
    5958         1104 :       && is_a <scalar_mode> (GET_MODE (to), &to_mode)
    5959         9608 :       && is_a <scalar_mode> (GET_MODE (from), &from_mode))
    5960              :     {
    5961         1104 :       opt_scalar_mode fmode_iter;
    5962         1104 :       rtx_code_label *label = gen_label_rtx ();
    5963         1104 :       rtx temp;
    5964         1104 :       REAL_VALUE_TYPE offset;
    5965              : 
    5966              :       /* Look for a usable floating mode FMODE wider than the source and at
    5967              :          least as wide as the target.  Using FMODE will avoid rounding woes
    5968              :          with unsigned values greater than the signed maximum value.  */
    5969              : 
    5970         2125 :       FOR_EACH_MODE_FROM (fmode_iter, to_mode)
    5971              :         {
    5972         2125 :           scalar_mode fmode = fmode_iter.require ();
    5973         3146 :           if (GET_MODE_PRECISION (from_mode) < GET_MODE_BITSIZE (fmode)
    5974         2125 :               && can_float_p (fmode, from_mode, 0) != CODE_FOR_nothing)
    5975              :             break;
    5976              :         }
    5977              : 
    5978         1104 :       if (!fmode_iter.exists (&fmode))
    5979              :         {
    5980              :           /* There is no such mode.  Pretend the target is wide enough.  */
    5981            0 :           fmode = to_mode;
    5982              : 
    5983              :           /* Avoid double-rounding when TO is narrower than FROM.  */
    5984            0 :           if ((significand_size (fmode) + 1)
    5985            0 :               < GET_MODE_PRECISION (from_mode))
    5986              :             {
    5987            0 :               rtx temp1;
    5988            0 :               rtx_code_label *neglabel = gen_label_rtx ();
    5989              : 
    5990              :               /* Don't use TARGET if it isn't a register, is a hard register,
    5991              :                  or is the wrong mode.  */
    5992            0 :               if (!REG_P (target)
    5993            0 :                   || REGNO (target) < FIRST_PSEUDO_REGISTER
    5994            0 :                   || GET_MODE (target) != fmode)
    5995            0 :                 target = gen_reg_rtx (fmode);
    5996              : 
    5997            0 :               imode = from_mode;
    5998            0 :               do_pending_stack_adjust ();
    5999              : 
    6000              :               /* Test whether the sign bit is set.  */
    6001            0 :               emit_cmp_and_jump_insns (from, const0_rtx, LT, NULL_RTX, imode,
    6002              :                                        0, neglabel);
    6003              : 
    6004              :               /* The sign bit is not set.  Convert as signed.  */
    6005            0 :               expand_float (target, from, 0);
    6006            0 :               emit_jump_insn (targetm.gen_jump (label));
    6007            0 :               emit_barrier ();
    6008              : 
    6009              :               /* The sign bit is set.
    6010              :                  Convert to a usable (positive signed) value by shifting right
    6011              :                  one bit, while remembering if a nonzero bit was shifted
    6012              :                  out; i.e., compute  (from & 1) | (from >> 1).  */
    6013              : 
    6014            0 :               emit_label (neglabel);
    6015            0 :               temp = expand_binop (imode, and_optab, from, const1_rtx,
    6016              :                                    NULL_RTX, 1, OPTAB_LIB_WIDEN);
    6017            0 :               temp1 = expand_shift (RSHIFT_EXPR, imode, from, 1, NULL_RTX, 1);
    6018            0 :               temp = expand_binop (imode, ior_optab, temp, temp1, temp, 1,
    6019              :                                    OPTAB_LIB_WIDEN);
    6020            0 :               expand_float (target, temp, 0);
    6021              : 
    6022              :               /* Multiply by 2 to undo the shift above.  */
    6023            0 :               temp = expand_binop (fmode, add_optab, target, target,
    6024              :                                    target, 0, OPTAB_LIB_WIDEN);
    6025            0 :               if (temp != target)
    6026            0 :                 emit_move_insn (target, temp);
    6027              : 
    6028            0 :               do_pending_stack_adjust ();
    6029            0 :               emit_label (label);
    6030            0 :               goto done;
    6031              :             }
    6032              :         }
    6033              : 
    6034              :       /* If we are about to do some arithmetic to correct for an
    6035              :          unsigned operand, do it in a pseudo-register.  */
    6036              : 
    6037         1104 :       if (to_mode != fmode
    6038         1104 :           || !REG_P (to) || REGNO (to) < FIRST_PSEUDO_REGISTER)
    6039          403 :         target = gen_reg_rtx (fmode);
    6040              : 
    6041              :       /* Convert as signed integer to floating.  */
    6042         1104 :       expand_float (target, from, 0);
    6043              : 
    6044              :       /* If FROM is negative (and therefore TO is negative),
    6045              :          correct its value by 2**bitwidth.  */
    6046              : 
    6047         1104 :       do_pending_stack_adjust ();
    6048         1104 :       emit_cmp_and_jump_insns (from, const0_rtx, GE, NULL_RTX, from_mode,
    6049              :                                0, label);
    6050              : 
    6051              : 
    6052         1104 :       real_2expN (&offset, GET_MODE_PRECISION (from_mode), fmode);
    6053         1104 :       temp = expand_binop (fmode, add_optab, target,
    6054              :                            const_double_from_real_value (offset, fmode),
    6055              :                            target, 0, OPTAB_LIB_WIDEN);
    6056         1104 :       if (temp != target)
    6057            0 :         emit_move_insn (target, temp);
    6058              : 
    6059         1104 :       do_pending_stack_adjust ();
    6060         1104 :       emit_label (label);
    6061         1104 :       goto done;
    6062              :     }
    6063              : 
    6064              :   /* No hardware instruction available; call a library routine.  */
    6065         7400 :     {
    6066         7400 :       rtx libfunc;
    6067         7400 :       rtx_insn *insns;
    6068         7400 :       rtx value;
    6069         7400 :       convert_optab tab = unsignedp ? ufloat_optab : sfloat_optab;
    6070              : 
    6071         7400 :       if (is_narrower_int_mode (GET_MODE (from), SImode))
    6072          978 :         from = convert_to_mode (SImode, from, unsignedp);
    6073              : 
    6074         7400 :       libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
    6075         7400 :       gcc_assert (libfunc);
    6076              : 
    6077         7400 :       start_sequence ();
    6078              : 
    6079        14800 :       value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
    6080         7400 :                                        GET_MODE (to), from, GET_MODE (from));
    6081         7400 :       insns = end_sequence ();
    6082              : 
    6083         7400 :       emit_libcall_block (insns, target, value,
    6084         7400 :                           gen_rtx_fmt_e (unsignedp ? UNSIGNED_FLOAT : FLOAT,
    6085              :                                          GET_MODE (to), from));
    6086              :     }
    6087              : 
    6088         8504 :  done:
    6089              : 
    6090              :   /* Copy result to requested destination
    6091              :      if we have been computing in a temp location.  */
    6092              : 
    6093         8504 :   if (target != to)
    6094              :     {
    6095          403 :       if (GET_MODE (target) == GET_MODE (to))
    6096            4 :         emit_move_insn (to, target);
    6097              :       else
    6098          399 :         convert_move (to, target, 0);
    6099              :     }
    6100              : }
    6101              : 
    6102              : /* Generate code to convert FROM to fixed point and store in TO.  FROM
    6103              :    must be floating point.  */
    6104              : 
    6105              : void
    6106        59926 : expand_fix (rtx to, rtx from, int unsignedp)
    6107              : {
    6108        59926 :   enum insn_code icode;
    6109        59926 :   rtx target = to;
    6110        59926 :   machine_mode fmode, imode;
    6111        59926 :   opt_scalar_mode fmode_iter;
    6112        59926 :   bool must_trunc = false;
    6113              : 
    6114              :   /* We first try to find a pair of modes, one real and one integer, at
    6115              :      least as wide as FROM and TO, respectively, in which we can open-code
    6116              :      this conversion.  If the integer mode is wider than the mode of TO,
    6117              :      we can do the conversion either signed or unsigned.  */
    6118              : 
    6119        90537 :   FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
    6120       216311 :     FOR_EACH_MODE_FROM (imode, GET_MODE (to))
    6121              :       {
    6122       185700 :         int doing_unsigned = unsignedp;
    6123              : 
    6124       185700 :         icode = can_fix_p (imode, fmode, unsignedp, &must_trunc);
    6125       185700 :         if (icode == CODE_FOR_nothing && imode != GET_MODE (to) && unsignedp)
    6126        62664 :           icode = can_fix_p (imode, fmode, 0, &must_trunc), doing_unsigned = 0;
    6127              : 
    6128       142052 :         if (icode != CODE_FOR_nothing)
    6129              :           {
    6130        46829 :             rtx_insn *last = get_last_insn ();
    6131        46829 :             rtx from1 = from;
    6132        46829 :             if (fmode != GET_MODE (from))
    6133              :               {
    6134         1114 :                 if (REAL_MODE_FORMAT (GET_MODE (from))
    6135              :                     == &arm_bfloat_half_format
    6136         1114 :                     && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
    6137              :                   /* The BF -> SF conversions can be just a shift, doesn't
    6138              :                      need to handle sNANs.  */
    6139              :                   {
    6140           46 :                     int save_flag_finite_math_only = flag_finite_math_only;
    6141           46 :                     flag_finite_math_only = true;
    6142           46 :                     from1 = convert_to_mode (fmode, from, 0);
    6143           46 :                     flag_finite_math_only = save_flag_finite_math_only;
    6144              :                   }
    6145              :                 else
    6146         1068 :                   from1 = convert_to_mode (fmode, from, 0);
    6147              :               }
    6148              : 
    6149        46829 :             if (must_trunc)
    6150              :               {
    6151            0 :                 rtx temp = gen_reg_rtx (GET_MODE (from1));
    6152            0 :                 from1 = expand_unop (GET_MODE (from1), ftrunc_optab, from1,
    6153              :                                      temp, 0);
    6154              :               }
    6155              : 
    6156        46829 :             if (imode != GET_MODE (to))
    6157        10792 :               target = gen_reg_rtx (imode);
    6158              : 
    6159        89398 :             if (maybe_emit_unop_insn (icode, target, from1,
    6160              :                                       doing_unsigned ? UNSIGNED_FIX : FIX))
    6161              :               {
    6162        46829 :                 if (target != to)
    6163        10792 :                   convert_move (to, target, unsignedp);
    6164        50922 :                 return;
    6165              :               }
    6166            0 :             delete_insns_since (last);
    6167              :           }
    6168              :       }
    6169              : 
    6170              :   /* For an unsigned conversion, there is one more way to do it.
    6171              :      If we have a signed conversion, we generate code that compares
    6172              :      the real value to the largest representable positive number.  If if
    6173              :      is smaller, the conversion is done normally.  Otherwise, subtract
    6174              :      one plus the highest signed number, convert, and add it back.
    6175              : 
    6176              :      We only need to check all real modes, since we know we didn't find
    6177              :      anything with a wider integer mode.
    6178              : 
    6179              :      This code used to extend FP value into mode wider than the destination.
    6180              :      This is needed for decimal float modes which cannot accurately
    6181              :      represent one plus the highest signed number of the same size, but
    6182              :      not for binary modes.  Consider, for instance conversion from SFmode
    6183              :      into DImode.
    6184              : 
    6185              :      The hot path through the code is dealing with inputs smaller than 2^63
    6186              :      and doing just the conversion, so there is no bits to lose.
    6187              : 
    6188              :      In the other path we know the value is positive in the range 2^63..2^64-1
    6189              :      inclusive.  (as for other input overflow happens and result is undefined)
    6190              :      So we know that the most important bit set in mantissa corresponds to
    6191              :      2^63.  The subtraction of 2^63 should not generate any rounding as it
    6192              :      simply clears out that bit.  The rest is trivial.  */
    6193              : 
    6194        13097 :   scalar_int_mode to_mode;
    6195        13097 :   if (unsignedp
    6196         7469 :       && is_a <scalar_int_mode> (GET_MODE (to), &to_mode)
    6197        20566 :       && HWI_COMPUTABLE_MODE_P (to_mode))
    6198         9095 :     FOR_EACH_MODE_FROM (fmode_iter, as_a <scalar_mode> (GET_MODE (from)))
    6199              :       {
    6200         6890 :         scalar_mode fmode = fmode_iter.require ();
    6201         6890 :         if (CODE_FOR_nothing != can_fix_p (to_mode, fmode,
    6202              :                                            0, &must_trunc)
    6203         6890 :             && (!DECIMAL_FLOAT_MODE_P (fmode)
    6204            0 :                 || (GET_MODE_BITSIZE (fmode) > GET_MODE_PRECISION (to_mode))))
    6205              :           {
    6206         4087 :             int bitsize;
    6207         4087 :             REAL_VALUE_TYPE offset;
    6208         4087 :             rtx limit;
    6209         4087 :             rtx_code_label *lab1, *lab2;
    6210         4087 :             rtx_insn *insn;
    6211              : 
    6212         4087 :             bitsize = GET_MODE_PRECISION (to_mode);
    6213         4087 :             real_2expN (&offset, bitsize - 1, fmode);
    6214         4087 :             limit = const_double_from_real_value (offset, fmode);
    6215         4087 :             lab1 = gen_label_rtx ();
    6216         4087 :             lab2 = gen_label_rtx ();
    6217              : 
    6218         4087 :             if (fmode != GET_MODE (from))
    6219              :               {
    6220          188 :                 if (REAL_MODE_FORMAT (GET_MODE (from))
    6221              :                     == &arm_bfloat_half_format
    6222          188 :                     && REAL_MODE_FORMAT (fmode) == &ieee_single_format)
    6223              :                   /* The BF -> SF conversions can be just a shift, doesn't
    6224              :                      need to handle sNANs.  */
    6225              :                   {
    6226            6 :                     int save_flag_finite_math_only = flag_finite_math_only;
    6227            6 :                     flag_finite_math_only = true;
    6228            6 :                     from = convert_to_mode (fmode, from, 0);
    6229            6 :                     flag_finite_math_only = save_flag_finite_math_only;
    6230              :                   }
    6231              :                 else
    6232          182 :                   from = convert_to_mode (fmode, from, 0);
    6233              :               }
    6234              : 
    6235              :             /* See if we need to do the subtraction.  */
    6236         4087 :             do_pending_stack_adjust ();
    6237         4087 :             emit_cmp_and_jump_insns (from, limit, GE, NULL_RTX,
    6238         4087 :                                      GET_MODE (from), 0, lab1);
    6239              : 
    6240              :             /* If not, do the signed "fix" and branch around fixup code.  */
    6241         4087 :             expand_fix (to, from, 0);
    6242         4087 :             emit_jump_insn (targetm.gen_jump (lab2));
    6243         4087 :             emit_barrier ();
    6244              : 
    6245              :             /* Otherwise, subtract 2**(N-1), convert to signed number,
    6246              :                then add 2**(N-1).  Do the addition using XOR since this
    6247              :                will often generate better code.  */
    6248         4087 :             emit_label (lab1);
    6249         4087 :             target = expand_binop (GET_MODE (from), sub_optab, from, limit,
    6250              :                                    NULL_RTX, 0, OPTAB_LIB_WIDEN);
    6251         4087 :             expand_fix (to, target, 0);
    6252         4087 :             target = expand_binop (to_mode, xor_optab, to,
    6253              :                                    gen_int_mode
    6254         4087 :                                    (HOST_WIDE_INT_1 << (bitsize - 1),
    6255              :                                     to_mode),
    6256              :                                    to, 1, OPTAB_LIB_WIDEN);
    6257              : 
    6258         4087 :             if (target != to)
    6259            0 :               emit_move_insn (to, target);
    6260              : 
    6261         4087 :             emit_label (lab2);
    6262              : 
    6263         4087 :             if (optab_handler (mov_optab, to_mode) != CODE_FOR_nothing)
    6264              :               {
    6265              :                 /* Make a place for a REG_NOTE and add it.  */
    6266         4087 :                 insn = emit_move_insn (to, to);
    6267         4087 :                 set_dst_reg_note (insn, REG_EQUAL,
    6268              :                                   gen_rtx_fmt_e (UNSIGNED_FIX, to_mode,
    6269              :                                                  copy_rtx (from)),
    6270              :                                   to);
    6271              :               }
    6272              : 
    6273         4087 :             return;
    6274              :           }
    6275              :       }
    6276              : 
    6277              : #ifdef HAVE_SFmode
    6278         9010 :   if (REAL_MODE_FORMAT (GET_MODE (from)) == &arm_bfloat_half_format
    6279         9010 :       && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
    6280              :     /* We don't have BF -> TI library functions, use BF -> SF -> TI
    6281              :        instead but the BF -> SF conversion can be just a shift, doesn't
    6282              :        need to handle sNANs.  */
    6283              :     {
    6284            6 :       int save_flag_finite_math_only = flag_finite_math_only;
    6285            6 :       flag_finite_math_only = true;
    6286            6 :       from = convert_to_mode (SFmode, from, 0);
    6287            6 :       flag_finite_math_only = save_flag_finite_math_only;
    6288            6 :       expand_fix (to, from, unsignedp);
    6289            6 :       return;
    6290              :     }
    6291              : #endif
    6292              : 
    6293              :   /* We can't do it with an insn, so use a library call.  But first ensure
    6294              :      that the mode of TO is at least as wide as SImode, since those are the
    6295              :      only library calls we know about.  */
    6296              : 
    6297         9004 :   if (is_narrower_int_mode (GET_MODE (to), SImode))
    6298              :     {
    6299         1958 :       target = gen_reg_rtx (SImode);
    6300              : 
    6301         1958 :       expand_fix (target, from, unsignedp);
    6302              :     }
    6303              :   else
    6304              :     {
    6305         7046 :       rtx_insn *insns;
    6306         7046 :       rtx value;
    6307         7046 :       rtx libfunc;
    6308              : 
    6309         7046 :       convert_optab tab = unsignedp ? ufix_optab : sfix_optab;
    6310         7046 :       libfunc = convert_optab_libfunc (tab, GET_MODE (to), GET_MODE (from));
    6311         7046 :       gcc_assert (libfunc);
    6312              : 
    6313         7046 :       start_sequence ();
    6314              : 
    6315        14092 :       value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST,
    6316         7046 :                                        GET_MODE (to), from, GET_MODE (from));
    6317         7046 :       insns = end_sequence ();
    6318              : 
    6319         7046 :       emit_libcall_block (insns, target, value,
    6320         7046 :                           gen_rtx_fmt_e (unsignedp ? UNSIGNED_FIX : FIX,
    6321              :                                          GET_MODE (to), from));
    6322              :     }
    6323              : 
    6324         9004 :   if (target != to)
    6325              :     {
    6326         1958 :       if (GET_MODE (to) == GET_MODE (target))
    6327            0 :         emit_move_insn (to, target);
    6328              :       else
    6329         1958 :         convert_move (to, target, 0);
    6330              :     }
    6331              : }
    6332              : 
    6333              : 
    6334              : /* Promote integer arguments for a libcall if necessary.
    6335              :    emit_library_call_value cannot do the promotion because it does not
    6336              :    know if it should do a signed or unsigned promotion.  This is because
    6337              :    there are no tree types defined for libcalls.  */
    6338              : 
    6339              : static rtx
    6340            0 : prepare_libcall_arg (rtx arg, int uintp)
    6341              : {
    6342            0 :   scalar_int_mode mode;
    6343            0 :   machine_mode arg_mode;
    6344            0 :   if (is_a <scalar_int_mode> (GET_MODE (arg), &mode))
    6345              :     {
    6346              :       /*  If we need to promote the integer function argument we need to do
    6347              :           it here instead of inside emit_library_call_value because in
    6348              :           emit_library_call_value we don't know if we should do a signed or
    6349              :           unsigned promotion.  */
    6350              : 
    6351            0 :       int unsigned_p = 0;
    6352            0 :       arg_mode = promote_function_mode (NULL_TREE, mode,
    6353              :                                         &unsigned_p, NULL_TREE, 0);
    6354            0 :       if (arg_mode != mode)
    6355            0 :         return convert_to_mode (arg_mode, arg, uintp);
    6356              :     }
    6357              :     return arg;
    6358              : }
    6359              : 
    6360              : /* Generate code to convert FROM or TO a fixed-point.
    6361              :    If UINTP is true, either TO or FROM is an unsigned integer.
    6362              :    If SATP is true, we need to saturate the result.  */
    6363              : 
    6364              : void
    6365            0 : expand_fixed_convert (rtx to, rtx from, int uintp, int satp)
    6366              : {
    6367            0 :   machine_mode to_mode = GET_MODE (to);
    6368            0 :   machine_mode from_mode = GET_MODE (from);
    6369            0 :   convert_optab tab;
    6370            0 :   enum rtx_code this_code;
    6371            0 :   enum insn_code code;
    6372            0 :   rtx_insn *insns;
    6373            0 :   rtx value;
    6374            0 :   rtx libfunc;
    6375              : 
    6376            0 :   if (to_mode == from_mode)
    6377              :     {
    6378            0 :       emit_move_insn (to, from);
    6379            0 :       return;
    6380              :     }
    6381              : 
    6382            0 :   if (uintp)
    6383              :     {
    6384            0 :       tab = satp ? satfractuns_optab : fractuns_optab;
    6385            0 :       this_code = satp ? UNSIGNED_SAT_FRACT : UNSIGNED_FRACT_CONVERT;
    6386              :     }
    6387              :   else
    6388              :     {
    6389            0 :       tab = satp ? satfract_optab : fract_optab;
    6390            0 :       this_code = satp ? SAT_FRACT : FRACT_CONVERT;
    6391              :     }
    6392            0 :   code = convert_optab_handler (tab, to_mode, from_mode);
    6393            0 :   if (code != CODE_FOR_nothing)
    6394              :     {
    6395            0 :       emit_unop_insn (code, to, from, this_code);
    6396            0 :       return;
    6397              :     }
    6398              : 
    6399            0 :   libfunc = convert_optab_libfunc (tab, to_mode, from_mode);
    6400            0 :   gcc_assert (libfunc);
    6401              : 
    6402            0 :   from = prepare_libcall_arg (from, uintp);
    6403            0 :   from_mode = GET_MODE (from);
    6404              : 
    6405            0 :   start_sequence ();
    6406            0 :   value = emit_library_call_value (libfunc, NULL_RTX, LCT_CONST, to_mode,
    6407              :                                    from, from_mode);
    6408            0 :   insns = end_sequence ();
    6409              : 
    6410            0 :   emit_libcall_block (insns, to, value,
    6411              :                       gen_rtx_fmt_e (optab_to_code (tab), to_mode, from));
    6412              : }
    6413              : 
    6414              : /* Generate code to convert FROM to fixed point and store in TO.  FROM
    6415              :    must be floating point, TO must be signed.  Use the conversion optab
    6416              :    TAB to do the conversion.  */
    6417              : 
    6418              : bool
    6419          473 : expand_sfix_optab (rtx to, rtx from, convert_optab tab)
    6420              : {
    6421          473 :   enum insn_code icode;
    6422          473 :   rtx target = to;
    6423          473 :   machine_mode fmode, imode;
    6424              : 
    6425              :   /* We first try to find a pair of modes, one real and one integer, at
    6426              :      least as wide as FROM and TO, respectively, in which we can open-code
    6427              :      this conversion.  If the integer mode is wider than the mode of TO,
    6428              :      we can do the conversion either signed or unsigned.  */
    6429              : 
    6430         1973 :   FOR_EACH_MODE_FROM (fmode, GET_MODE (from))
    6431         8450 :     FOR_EACH_MODE_FROM (imode, GET_MODE (to))
    6432              :       {
    6433         6950 :         icode = convert_optab_handler (tab, imode, fmode,
    6434              :                                        insn_optimization_type ());
    6435         6950 :         if (icode != CODE_FOR_nothing)
    6436              :           {
    6437           52 :             rtx_insn *last = get_last_insn ();
    6438           52 :             if (fmode != GET_MODE (from))
    6439           52 :               from = convert_to_mode (fmode, from, 0);
    6440              : 
    6441           52 :             if (imode != GET_MODE (to))
    6442            0 :               target = gen_reg_rtx (imode);
    6443              : 
    6444           52 :             if (!maybe_emit_unop_insn (icode, target, from, UNKNOWN))
    6445              :               {
    6446            0 :                 delete_insns_since (last);
    6447            0 :                 continue;
    6448              :               }
    6449           52 :             if (target != to)
    6450            0 :               convert_move (to, target, 0);
    6451              :             return true;
    6452              :           }
    6453              :       }
    6454              : 
    6455              :   return false;
    6456              : }
    6457              : 
    6458              : /* Report whether we have an instruction to perform the operation
    6459              :    specified by CODE on operands of mode MODE.  */
    6460              : bool
    6461     98517420 : have_insn_for (enum rtx_code code, machine_mode mode)
    6462              : {
    6463     98517420 :   return (code_to_optab (code)
    6464     98517420 :           && (optab_handler (code_to_optab (code), mode)
    6465     98517420 :               != CODE_FOR_nothing));
    6466              : }
    6467              : 
    6468              : /* Print information about the current contents of the optabs on
    6469              :    STDERR.  */
    6470              : 
    6471              : DEBUG_FUNCTION void
    6472            0 : debug_optab_libfuncs (void)
    6473              : {
    6474            0 :   int i, j, k;
    6475              : 
    6476              :   /* Dump the arithmetic optabs.  */
    6477            0 :   for (i = FIRST_NORM_OPTAB; i <= LAST_NORMLIB_OPTAB; ++i)
    6478            0 :     for (j = 0; j < NUM_MACHINE_MODES; ++j)
    6479              :       {
    6480            0 :         rtx l = optab_libfunc ((optab) i, (machine_mode) j);
    6481            0 :         if (l)
    6482              :           {
    6483            0 :             gcc_assert (GET_CODE (l) == SYMBOL_REF);
    6484            0 :             fprintf (stderr, "%s\t%s:\t%s\n",
    6485            0 :                      GET_RTX_NAME (optab_to_code ((optab) i)),
    6486            0 :                      GET_MODE_NAME (j),
    6487              :                      XSTR (l, 0));
    6488              :           }
    6489              :       }
    6490              : 
    6491              :   /* Dump the conversion optabs.  */
    6492            0 :   for (i = FIRST_CONV_OPTAB; i <= LAST_CONVLIB_OPTAB; ++i)
    6493            0 :     for (j = 0; j < NUM_MACHINE_MODES; ++j)
    6494            0 :       for (k = 0; k < NUM_MACHINE_MODES; ++k)
    6495              :         {
    6496            0 :           rtx l = convert_optab_libfunc ((optab) i, (machine_mode) j,
    6497              :                                          (machine_mode) k);
    6498            0 :           if (l)
    6499              :             {
    6500            0 :               gcc_assert (GET_CODE (l) == SYMBOL_REF);
    6501            0 :               fprintf (stderr, "%s\t%s\t%s:\t%s\n",
    6502            0 :                        GET_RTX_NAME (optab_to_code ((optab) i)),
    6503            0 :                        GET_MODE_NAME (j),
    6504            0 :                        GET_MODE_NAME (k),
    6505              :                        XSTR (l, 0));
    6506              :             }
    6507              :         }
    6508            0 : }
    6509              : 
    6510              : /* Generate insns to trap with code TCODE if OP1 and OP2 satisfy condition
    6511              :    CODE.  Return 0 on failure.  */
    6512              : 
    6513              : rtx_insn *
    6514            0 : gen_cond_trap (enum rtx_code code, rtx op1, rtx op2, rtx tcode)
    6515              : {
    6516            0 :   machine_mode mode = GET_MODE (op1);
    6517            0 :   enum insn_code icode;
    6518            0 :   rtx_insn *insn;
    6519            0 :   rtx trap_rtx;
    6520              : 
    6521            0 :   if (mode == VOIDmode)
    6522              :     return 0;
    6523              : 
    6524            0 :   icode = optab_handler (ctrap_optab, mode);
    6525            0 :   if (icode == CODE_FOR_nothing)
    6526              :     return 0;
    6527              : 
    6528              :   /* Some targets only accept a zero trap code.  */
    6529            0 :   if (!insn_operand_matches (icode, 3, tcode))
    6530              :     return 0;
    6531              : 
    6532            0 :   do_pending_stack_adjust ();
    6533            0 :   start_sequence ();
    6534            0 :   prepare_cmp_insn (op1, op2, NULL, code, NULL_RTX, false, OPTAB_DIRECT,
    6535              :                     &trap_rtx, &mode, cbranch_optab);
    6536            0 :   if (!trap_rtx)
    6537              :     insn = NULL;
    6538              :   else
    6539            0 :     insn = GEN_FCN (icode) (trap_rtx, XEXP (trap_rtx, 0), XEXP (trap_rtx, 1),
    6540              :                             tcode);
    6541              : 
    6542              :   /* If that failed, then give up.  */
    6543            0 :   if (insn == 0)
    6544              :     {
    6545            0 :       end_sequence ();
    6546            0 :       return 0;
    6547              :     }
    6548              : 
    6549            0 :   emit_insn (insn);
    6550            0 :   insn = end_sequence ();
    6551            0 :   return insn;
    6552              : }
    6553              : 
    6554              : /* Return rtx code for TCODE or UNKNOWN.  Use UNSIGNEDP to select signed
    6555              :    or unsigned operation code.  */
    6556              : 
    6557              : enum rtx_code
    6558      1626743 : get_rtx_code_1 (enum tree_code tcode, bool unsignedp)
    6559              : {
    6560      1626743 :   enum rtx_code code;
    6561      1626743 :   switch (tcode)
    6562              :     {
    6563              :     case EQ_EXPR:
    6564              :       code = EQ;
    6565              :       break;
    6566       818539 :     case NE_EXPR:
    6567       818539 :       code = NE;
    6568       818539 :       break;
    6569       149482 :     case LT_EXPR:
    6570       149482 :       code = unsignedp ? LTU : LT;
    6571              :       break;
    6572        74538 :     case LE_EXPR:
    6573        74538 :       code = unsignedp ? LEU : LE;
    6574              :       break;
    6575       114628 :     case GT_EXPR:
    6576       114628 :       code = unsignedp ? GTU : GT;
    6577              :       break;
    6578        81257 :     case GE_EXPR:
    6579        81257 :       code = unsignedp ? GEU : GE;
    6580              :       break;
    6581              : 
    6582         1994 :     case UNORDERED_EXPR:
    6583         1994 :       code = UNORDERED;
    6584         1994 :       break;
    6585         1393 :     case ORDERED_EXPR:
    6586         1393 :       code = ORDERED;
    6587         1393 :       break;
    6588          730 :     case UNLT_EXPR:
    6589          730 :       code = UNLT;
    6590          730 :       break;
    6591         4944 :     case UNLE_EXPR:
    6592         4944 :       code = UNLE;
    6593         4944 :       break;
    6594          726 :     case UNGT_EXPR:
    6595          726 :       code = UNGT;
    6596          726 :       break;
    6597         3754 :     case UNGE_EXPR:
    6598         3754 :       code = UNGE;
    6599         3754 :       break;
    6600          543 :     case UNEQ_EXPR:
    6601          543 :       code = UNEQ;
    6602          543 :       break;
    6603           89 :     case LTGT_EXPR:
    6604           89 :       code = LTGT;
    6605           89 :       break;
    6606              : 
    6607        32792 :     case BIT_AND_EXPR:
    6608        32792 :       code = AND;
    6609        32792 :       break;
    6610              : 
    6611           22 :     case BIT_IOR_EXPR:
    6612           22 :       code = IOR;
    6613           22 :       break;
    6614              : 
    6615            0 :     default:
    6616            0 :       code = UNKNOWN;
    6617            0 :       break;
    6618              :     }
    6619      1626743 :   return code;
    6620              : }
    6621              : 
    6622              : /* Return rtx code for TCODE.  Use UNSIGNEDP to select signed
    6623              :    or unsigned operation code.  */
    6624              : 
    6625              : enum rtx_code
    6626       159259 : get_rtx_code (enum tree_code tcode, bool unsignedp)
    6627              : {
    6628       159259 :   enum rtx_code code = get_rtx_code_1 (tcode, unsignedp);
    6629       159259 :   gcc_assert (code != UNKNOWN);
    6630       159259 :   return code;
    6631              : }
    6632              : 
    6633              : /* Return a comparison rtx of mode CMP_MODE for COND.  Use UNSIGNEDP to
    6634              :    select signed or unsigned operators.  OPNO holds the index of the
    6635              :    first comparison operand for insn ICODE.  Do not generate the
    6636              :    compare instruction itself.  */
    6637              : 
    6638              : rtx
    6639        26454 : vector_compare_rtx (machine_mode cmp_mode, enum tree_code tcode,
    6640              :                     tree t_op0, tree t_op1, bool unsignedp,
    6641              :                     enum insn_code icode, unsigned int opno)
    6642              : {
    6643        26454 :   class expand_operand ops[2];
    6644        26454 :   rtx rtx_op0, rtx_op1;
    6645        26454 :   machine_mode m0, m1;
    6646        26454 :   enum rtx_code rcode = get_rtx_code (tcode, unsignedp);
    6647              : 
    6648        26454 :   gcc_assert (TREE_CODE_CLASS (tcode) == tcc_comparison);
    6649              : 
    6650              :   /* Expand operands.  For vector types with scalar modes, e.g. where int64x1_t
    6651              :      has mode DImode, this can produce a constant RTX of mode VOIDmode; in such
    6652              :      cases, use the original mode.  */
    6653        26454 :   rtx_op0 = expand_expr (t_op0, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op0)),
    6654              :                          EXPAND_STACK_PARM);
    6655        26454 :   m0 = GET_MODE (rtx_op0);
    6656        26454 :   if (m0 == VOIDmode)
    6657            0 :     m0 = TYPE_MODE (TREE_TYPE (t_op0));
    6658              : 
    6659        26454 :   rtx_op1 = expand_expr (t_op1, NULL_RTX, TYPE_MODE (TREE_TYPE (t_op1)),
    6660              :                          EXPAND_STACK_PARM);
    6661        26454 :   m1 = GET_MODE (rtx_op1);
    6662        26454 :   if (m1 == VOIDmode)
    6663            0 :     m1 = TYPE_MODE (TREE_TYPE (t_op1));
    6664              : 
    6665        26454 :   create_input_operand (&ops[0], rtx_op0, m0);
    6666        26454 :   create_input_operand (&ops[1], rtx_op1, m1);
    6667        26454 :   if (!maybe_legitimize_operands (icode, opno, 2, ops))
    6668            0 :     gcc_unreachable ();
    6669        26454 :   return gen_rtx_fmt_ee (rcode, cmp_mode, ops[0].value, ops[1].value);
    6670              : }
    6671              : 
    6672              : /* Check if vec_perm mask SEL is a constant equivalent to a shift of
    6673              :    the first vec_perm operand, assuming the second operand (for left shift
    6674              :    first operand) is a constant vector of zeros.  Return the shift distance
    6675              :    in bits if so, or NULL_RTX if the vec_perm is not a shift.  MODE is the
    6676              :    mode of the value being shifted.  SHIFT_OPTAB is vec_shr_optab for right
    6677              :    shift or vec_shl_optab for left shift.  */
    6678              : static rtx
    6679         2996 : shift_amt_for_vec_perm_mask (machine_mode mode, const vec_perm_indices &sel,
    6680              :                              optab shift_optab)
    6681              : {
    6682         2996 :   unsigned int bitsize = GET_MODE_UNIT_BITSIZE (mode);
    6683         2996 :   poly_int64 first = sel[0];
    6684         5992 :   if (maybe_ge (sel[0], GET_MODE_NUNITS (mode)))
    6685              :     return NULL_RTX;
    6686              : 
    6687         2995 :   if (shift_optab == vec_shl_optab)
    6688              :     {
    6689          877 :       unsigned int nelt;
    6690         1754 :       if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
    6691         2996 :         return NULL_RTX;
    6692          877 :       unsigned firstidx = 0;
    6693         5778 :       for (unsigned int i = 0; i < nelt; i++)
    6694              :         {
    6695         5020 :           if (known_eq (sel[i], nelt))
    6696              :             {
    6697          794 :               if (i == 0 || firstidx)
    6698              :                 return NULL_RTX;
    6699              :               firstidx = i;
    6700              :             }
    6701         8452 :           else if (firstidx
    6702         5868 :                    ? maybe_ne (sel[i], nelt + i - firstidx)
    6703         4226 :                    : maybe_ge (sel[i], nelt))
    6704              :             return NULL_RTX;
    6705              :         }
    6706              : 
    6707          758 :       if (firstidx == 0)
    6708              :         return NULL_RTX;
    6709          758 :       first = firstidx;
    6710              :     }
    6711         2118 :   else if (!sel.series_p (0, 1, first, 1))
    6712              :     {
    6713          376 :       unsigned int nelt;
    6714          752 :       if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
    6715         2996 :         return NULL_RTX;
    6716         2166 :       for (unsigned int i = 1; i < nelt; i++)
    6717              :         {
    6718         1964 :           poly_int64 expected = i + first;
    6719              :           /* Indices into the second vector are all equivalent.  */
    6720         1964 :           if (maybe_lt (sel[i], nelt)
    6721         2770 :               ? maybe_ne (sel[i], expected)
    6722          806 :               : maybe_lt (expected, nelt))
    6723         2996 :             return NULL_RTX;
    6724              :         }
    6725              :     }
    6726              : 
    6727         2702 :   return gen_int_shift_amount (mode, first * bitsize);
    6728              : }
    6729              : 
    6730              : /* A subroutine of expand_vec_perm_var for expanding one vec_perm insn.  */
    6731              : 
    6732              : static rtx
    6733           18 : expand_vec_perm_1 (enum insn_code icode, rtx target,
    6734              :                    rtx v0, rtx v1, rtx sel)
    6735              : {
    6736           18 :   machine_mode tmode = GET_MODE (target);
    6737           18 :   machine_mode smode = GET_MODE (sel);
    6738           18 :   class expand_operand ops[4];
    6739              : 
    6740           18 :   gcc_assert (GET_MODE_CLASS (smode) == MODE_VECTOR_INT
    6741              :               || related_int_vector_mode (tmode).require () == smode);
    6742           18 :   create_output_operand (&ops[0], target, tmode);
    6743           18 :   create_input_operand (&ops[3], sel, smode);
    6744              : 
    6745              :   /* Make an effort to preserve v0 == v1.  The target expander is able to
    6746              :      rely on this to determine if we're permuting a single input operand.  */
    6747           18 :   if (rtx_equal_p (v0, v1))
    6748              :     {
    6749           14 :       if (!insn_operand_matches (icode, 1, v0))
    6750            0 :         v0 = force_reg (tmode, v0);
    6751           14 :       gcc_checking_assert (insn_operand_matches (icode, 1, v0));
    6752           14 :       gcc_checking_assert (insn_operand_matches (icode, 2, v0));
    6753              : 
    6754           14 :       create_fixed_operand (&ops[1], v0);
    6755           14 :       create_fixed_operand (&ops[2], v0);
    6756              :     }
    6757              :   else
    6758              :     {
    6759            4 :       create_input_operand (&ops[1], v0, tmode);
    6760            4 :       create_input_operand (&ops[2], v1, tmode);
    6761              :     }
    6762              : 
    6763           18 :   if (maybe_expand_insn (icode, 4, ops))
    6764           18 :     return ops[0].value;
    6765              :   return NULL_RTX;
    6766              : }
    6767              : 
    6768              : /* Check if vec_perm mask SEL is a constant equivalent to an and operation of
    6769              :    the non-zero vec_perm operand with some mask consisting of 0xffs and 0x00s,
    6770              :    assuming the other vec_perm operand is a constant vector of zeros.  Return
    6771              :    the mask for the equivalent and operation, or NULL_RTX if the vec_perm can
    6772              :    not be modeled as an and.  MODE is the mode of the value being anded.
    6773              :    ZERO_OP0_P is true if the first operand of the vec_perm is a constant vector
    6774              :    of zeros or false if the second operand of the vec_perm is a constant vector
    6775              :    of zeros.  */
    6776              : rtx
    6777            0 : vec_perm_and_mask (machine_mode mode, const vec_perm_indices &sel,
    6778              :                    bool zero_op0_p)
    6779              : {
    6780            0 :   unsigned int nelt;
    6781            0 :   if (!GET_MODE_NUNITS (mode).is_constant (&nelt))
    6782              :     return NULL_RTX;
    6783              : 
    6784            0 :   rtx_vector_builder builder (mode, nelt, 1);
    6785            0 :   machine_mode emode = GET_MODE_INNER (mode);
    6786              : 
    6787            0 :   for (unsigned int i = 0; i < nelt; i++)
    6788              :     {
    6789            0 :       if (zero_op0_p)
    6790              :         {
    6791            0 :           if (known_eq (sel[i], nelt + i))
    6792            0 :             builder.quick_push (CONSTM1_RTX (emode));
    6793            0 :           else if (known_lt (sel[i], nelt))
    6794            0 :             builder.quick_push (CONST0_RTX (emode));
    6795              :           else
    6796            0 :             return NULL_RTX;
    6797              :         }
    6798              :       else
    6799              :         {
    6800            0 :           if (known_eq (sel[i], i))
    6801            0 :             builder.quick_push (CONSTM1_RTX (emode));
    6802            0 :           else if (known_ge (sel[i], nelt))
    6803            0 :             builder.quick_push (CONST0_RTX (emode));
    6804              :           else
    6805              :             return NULL_RTX;
    6806              :         }
    6807              :     }
    6808              : 
    6809            0 :   return builder.build ();
    6810            0 : }
    6811              : 
    6812              : /* Implement a permutation of vectors v0 and v1 using the permutation
    6813              :    vector in SEL and return the result.  Use TARGET to hold the result
    6814              :    if nonnull and convenient.
    6815              : 
    6816              :    MODE is the mode of the result.  V0 and V1 have the same mode, which
    6817              :    has the same element mode as MODE but may have a different number of
    6818              :    elements.  SEL_MODE is the TYPE_MODE associated with SEL, or BLKmode
    6819              :    if SEL isn't known to have a particular mode.  */
    6820              : 
    6821              : rtx
    6822        76327 : expand_vec_perm_const (machine_mode mode, rtx v0, rtx v1,
    6823              :                        const vec_perm_builder &sel, machine_mode sel_mode,
    6824              :                        rtx target)
    6825              : {
    6826        76327 :   if (!target || !register_operand (target, mode))
    6827        61566 :     target = gen_reg_rtx (mode);
    6828              : 
    6829              :   /* Set QIMODE to a different vector mode with byte elements.
    6830              :      If no such mode, or if MODE already has byte elements, use VOIDmode.  */
    6831        76327 :   machine_mode qimode;
    6832        76327 :   if (!qimode_for_vec_perm (mode).exists (&qimode))
    6833        76327 :     qimode = VOIDmode;
    6834              : 
    6835        76327 :   rtx_insn *last = get_last_insn ();
    6836              : 
    6837        76327 :   bool single_arg_p = rtx_equal_p (v0, v1);
    6838              :   /* The selector indexes the concatenation of the inputs, which may have
    6839              :      a different number of elements than the result.  */
    6840        76327 :   machine_mode op_mode = GET_MODE (v0);
    6841        76327 :   gcc_checking_assert (op_mode == GET_MODE (v1));
    6842              :   /* Always specify two input vectors here and leave the target to handle
    6843              :      cases in which the inputs are equal.  Not all backends can cope with
    6844              :      the single-input representation when testing for a double-input
    6845              :      target instruction.  */
    6846       152654 :   vec_perm_indices indices (sel, 2, GET_MODE_NUNITS (op_mode));
    6847              : 
    6848              :   /* See if this can be handled with a vec_shr or vec_shl.  We only do this
    6849              :      if the second (for vec_shr) or first (for vec_shl) vector is all
    6850              :      zeroes.  */
    6851        76327 :   insn_code shift_code = CODE_FOR_nothing;
    6852        76327 :   insn_code shift_code_qi = CODE_FOR_nothing;
    6853        76327 :   optab shift_optab = unknown_optab;
    6854        76327 :   rtx v2 = v0;
    6855        76327 :   if (v1 == CONST0_RTX (op_mode))
    6856              :     shift_optab = vec_shr_optab;
    6857        74145 :   else if (v0 == CONST0_RTX (op_mode))
    6858              :     {
    6859         1201 :       shift_optab = vec_shl_optab;
    6860         1201 :       v2 = v1;
    6861              :     }
    6862        76327 :   if (mode == op_mode
    6863        76327 :       && GET_MODE_NUNITS (mode).is_constant ()
    6864        76327 :       && shift_optab != unknown_optab)
    6865              :     {
    6866         3383 :       shift_code = optab_handler (shift_optab, mode);
    6867         5970 :       shift_code_qi = ((qimode != VOIDmode && qimode != mode)
    6868         6368 :                        ? optab_handler (shift_optab, qimode)
    6869              :                        : CODE_FOR_nothing);
    6870              :     }
    6871         3383 :   if (shift_code != CODE_FOR_nothing || shift_code_qi != CODE_FOR_nothing)
    6872              :     {
    6873         2996 :       rtx shift_amt = shift_amt_for_vec_perm_mask (mode, indices, shift_optab);
    6874         2996 :       if (shift_amt)
    6875              :         {
    6876         2702 :           class expand_operand ops[3];
    6877         2702 :           if (shift_amt == const0_rtx)
    6878         2702 :             return v2;
    6879         2702 :           if (shift_code != CODE_FOR_nothing)
    6880              :             {
    6881         2702 :               create_output_operand (&ops[0], target, mode);
    6882         2702 :               create_input_operand (&ops[1], v2, mode);
    6883         2702 :               create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
    6884         2702 :               if (maybe_expand_insn (shift_code, 3, ops))
    6885         2702 :                 return ops[0].value;
    6886              :             }
    6887            0 :           if (shift_code_qi != CODE_FOR_nothing)
    6888              :             {
    6889            0 :               rtx tmp = gen_reg_rtx (qimode);
    6890            0 :               create_output_operand (&ops[0], tmp, qimode);
    6891            0 :               create_input_operand (&ops[1], gen_lowpart (qimode, v2), qimode);
    6892            0 :               create_convert_operand_from_type (&ops[2], shift_amt, sizetype);
    6893            0 :               if (maybe_expand_insn (shift_code_qi, 3, ops))
    6894            0 :                 return gen_lowpart (mode, ops[0].value);
    6895              :             }
    6896              :         }
    6897              :     }
    6898              : 
    6899        73625 :   if (targetm.vectorize.vec_perm_const != NULL)
    6900              :     {
    6901        73625 :       if (single_arg_p)
    6902        18073 :         v1 = v0;
    6903              : 
    6904        73625 :       if (targetm.vectorize.vec_perm_const (mode, op_mode, target, v0, v1,
    6905              :                                             indices))
    6906              :         return target;
    6907              :     }
    6908              : 
    6909              :   /* The fallbacks below need the inputs to have the result mode.  */
    6910            0 :   if (mode != op_mode)
    6911              :     {
    6912            0 :       delete_insns_since (last);
    6913            0 :       return NULL_RTX;
    6914              :     }
    6915              : 
    6916              :   /* Fall back to a constant byte-based permutation.  */
    6917            0 :   vec_perm_indices qimode_indices;
    6918            0 :   rtx target_qi = NULL_RTX, v0_qi = NULL_RTX, v1_qi = NULL_RTX;
    6919            0 :   if (qimode != VOIDmode)
    6920              :     {
    6921            0 :       qimode_indices.new_expanded_vector (indices, GET_MODE_UNIT_SIZE (mode));
    6922            0 :       target_qi = gen_reg_rtx (qimode);
    6923            0 :       v0_qi = gen_lowpart (qimode, v0);
    6924            0 :       v1_qi = gen_lowpart (qimode, v1);
    6925            0 :       if (targetm.vectorize.vec_perm_const != NULL
    6926            0 :           && targetm.vectorize.vec_perm_const (qimode, qimode, target_qi, v0_qi,
    6927              :                                                v1_qi, qimode_indices))
    6928            0 :         return gen_lowpart (mode, target_qi);
    6929              :     }
    6930              : 
    6931            0 :   v0 = force_reg (mode, v0);
    6932            0 :   if (single_arg_p)
    6933            0 :     v1 = v0;
    6934            0 :   v1 = force_reg (mode, v1);
    6935              : 
    6936              :   /* Otherwise expand as a fully variable permutation.  */
    6937              : 
    6938              :   /* The optabs are only defined for selectors with the same width
    6939              :      as the values being permuted.  */
    6940            0 :   machine_mode required_sel_mode;
    6941            0 :   if (!related_int_vector_mode (mode).exists (&required_sel_mode))
    6942              :     {
    6943            0 :       delete_insns_since (last);
    6944            0 :       return NULL_RTX;
    6945              :     }
    6946              : 
    6947              :   /* We know that it is semantically valid to treat SEL as having SEL_MODE.
    6948              :      If that isn't the mode we want then we need to prove that using
    6949              :      REQUIRED_SEL_MODE is OK.  */
    6950            0 :   if (sel_mode != required_sel_mode)
    6951              :     {
    6952            0 :       if (!selector_fits_mode_p (required_sel_mode, indices))
    6953              :         {
    6954            0 :           delete_insns_since (last);
    6955            0 :           return NULL_RTX;
    6956              :         }
    6957              :       sel_mode = required_sel_mode;
    6958              :     }
    6959              : 
    6960            0 :   insn_code icode = direct_optab_handler (vec_perm_optab, mode);
    6961            0 :   if (icode != CODE_FOR_nothing)
    6962              :     {
    6963            0 :       rtx sel_rtx = vec_perm_indices_to_rtx (sel_mode, indices);
    6964            0 :       rtx tmp = expand_vec_perm_1 (icode, target, v0, v1, sel_rtx);
    6965            0 :       if (tmp)
    6966              :         return tmp;
    6967              :     }
    6968              : 
    6969            0 :   if (qimode != VOIDmode
    6970            0 :       && selector_fits_mode_p (qimode, qimode_indices))
    6971              :     {
    6972            0 :       icode = direct_optab_handler (vec_perm_optab, qimode);
    6973            0 :       if (icode != CODE_FOR_nothing)
    6974              :         {
    6975            0 :           rtx sel_qi = vec_perm_indices_to_rtx (qimode, qimode_indices);
    6976            0 :           rtx tmp = expand_vec_perm_1 (icode, target_qi, v0_qi, v1_qi, sel_qi);
    6977            0 :           if (tmp)
    6978            0 :             return gen_lowpart (mode, tmp);
    6979              :         }
    6980              :     }
    6981              : 
    6982            0 :   delete_insns_since (last);
    6983            0 :   return NULL_RTX;
    6984            0 : }
    6985              : 
    6986              : /* Implement a permutation of vectors v0 and v1 using the permutation
    6987              :    vector in SEL and return the result.  Use TARGET to hold the result
    6988              :    if nonnull and convenient.
    6989              : 
    6990              :    MODE is the mode of the vectors being permuted (V0 and V1).
    6991              :    SEL must have the integer equivalent of MODE and is known to be
    6992              :    unsuitable for permutes with a constant permutation vector.  */
    6993              : 
    6994              : rtx
    6995           18 : expand_vec_perm_var (machine_mode mode, rtx v0, rtx v1, rtx sel, rtx target)
    6996              : {
    6997           18 :   enum insn_code icode;
    6998           18 :   unsigned int i, u;
    6999           18 :   rtx tmp, sel_qi;
    7000              : 
    7001           18 :   u = GET_MODE_UNIT_SIZE (mode);
    7002              : 
    7003           18 :   if (!target || GET_MODE (target) != mode)
    7004            0 :     target = gen_reg_rtx (mode);
    7005              : 
    7006           18 :   icode = direct_optab_handler (vec_perm_optab, mode);
    7007           18 :   if (icode != CODE_FOR_nothing)
    7008              :     {
    7009           18 :       tmp = expand_vec_perm_1 (icode, target, v0, v1, sel);
    7010           18 :       if (tmp)
    7011              :         return tmp;
    7012              :     }
    7013              : 
    7014              :   /* As a special case to aid several targets, lower the element-based
    7015              :      permutation to a byte-based permutation and try again.  */
    7016            0 :   machine_mode qimode;
    7017            0 :   if (!qimode_for_vec_perm (mode).exists (&qimode)
    7018            0 :       || maybe_gt (GET_MODE_NUNITS (qimode), GET_MODE_MASK (QImode) + 1))
    7019              :     return NULL_RTX;
    7020            0 :   icode = direct_optab_handler (vec_perm_optab, qimode);
    7021            0 :   if (icode == CODE_FOR_nothing)
    7022              :     return NULL_RTX;
    7023              : 
    7024              :   /* Multiply each element by its byte size.  */
    7025            0 :   machine_mode selmode = GET_MODE (sel);
    7026            0 :   if (u == 2)
    7027            0 :     sel = expand_simple_binop (selmode, PLUS, sel, sel,
    7028              :                                NULL, 0, OPTAB_DIRECT);
    7029              :   else
    7030            0 :     sel = expand_simple_binop (selmode, ASHIFT, sel,
    7031            0 :                                gen_int_shift_amount (selmode, exact_log2 (u)),
    7032              :                                NULL, 0, OPTAB_DIRECT);
    7033            0 :   gcc_assert (sel != NULL);
    7034              : 
    7035              :   /* Broadcast the low byte each element into each of its bytes.
    7036              :      The encoding has U interleaved stepped patterns, one for each
    7037              :      byte of an element.  */
    7038            0 :   vec_perm_builder const_sel (GET_MODE_SIZE (mode), u, 3);
    7039            0 :   unsigned int low_byte_in_u = BYTES_BIG_ENDIAN ? u - 1 : 0;
    7040            0 :   for (i = 0; i < 3; ++i)
    7041            0 :     for (unsigned int j = 0; j < u; ++j)
    7042            0 :       const_sel.quick_push (i * u + low_byte_in_u);
    7043            0 :   sel = gen_lowpart (qimode, sel);
    7044            0 :   sel = expand_vec_perm_const (qimode, sel, sel, const_sel, qimode, NULL);
    7045            0 :   gcc_assert (sel != NULL);
    7046              : 
    7047              :   /* Add the byte offset to each byte element.  */
    7048              :   /* Note that the definition of the indices here is memory ordering,
    7049              :      so there should be no difference between big and little endian.  */
    7050            0 :   rtx_vector_builder byte_indices (qimode, u, 1);
    7051            0 :   for (i = 0; i < u; ++i)
    7052            0 :     byte_indices.quick_push (GEN_INT (i));
    7053            0 :   tmp = byte_indices.build ();
    7054            0 :   sel_qi = expand_simple_binop (qimode, PLUS, sel, tmp,
    7055              :                                 sel, 0, OPTAB_DIRECT);
    7056            0 :   gcc_assert (sel_qi != NULL);
    7057              : 
    7058            0 :   tmp = mode != qimode ? gen_reg_rtx (qimode) : target;
    7059            0 :   tmp = expand_vec_perm_1 (icode, tmp, gen_lowpart (qimode, v0),
    7060            0 :                            gen_lowpart (qimode, v1), sel_qi);
    7061            0 :   if (tmp)
    7062            0 :     tmp = gen_lowpart (mode, tmp);
    7063            0 :   return tmp;
    7064            0 : }
    7065              : 
    7066              : /* Generate VEC_SERIES_EXPR <OP0, OP1>, returning a value of mode VMODE.
    7067              :    Use TARGET for the result if nonnull and convenient.  */
    7068              : 
    7069              : rtx
    7070            0 : expand_vec_series_expr (machine_mode vmode, rtx op0, rtx op1, rtx target)
    7071              : {
    7072            0 :   class expand_operand ops[3];
    7073            0 :   enum insn_code icode;
    7074            0 :   machine_mode emode = GET_MODE_INNER (vmode);
    7075              : 
    7076            0 :   icode = direct_optab_handler (vec_series_optab, vmode);
    7077            0 :   gcc_assert (icode != CODE_FOR_nothing);
    7078              : 
    7079            0 :   create_output_operand (&ops[0], target, vmode);
    7080            0 :   create_input_operand (&ops[1], op0, emode);
    7081            0 :   create_input_operand (&ops[2], op1, emode);
    7082              : 
    7083            0 :   expand_insn (icode, 3, ops);
    7084            0 :   return ops[0].value;
    7085              : }
    7086              : 
    7087              : /* Generate insns for a vector comparison into a mask.  */
    7088              : 
    7089              : rtx
    7090        26454 : expand_vec_cmp_expr (tree type, tree exp, rtx target)
    7091              : {
    7092        26454 :   class expand_operand ops[4];
    7093        26454 :   enum insn_code icode;
    7094        26454 :   rtx comparison;
    7095        26454 :   machine_mode mask_mode = TYPE_MODE (type);
    7096        26454 :   machine_mode vmode;
    7097        26454 :   bool unsignedp;
    7098        26454 :   tree op0a, op0b;
    7099        26454 :   enum tree_code tcode;
    7100              : 
    7101        26454 :   op0a = TREE_OPERAND (exp, 0);
    7102        26454 :   op0b = TREE_OPERAND (exp, 1);
    7103        26454 :   tcode = TREE_CODE (exp);
    7104              : 
    7105        26454 :   unsignedp = TYPE_UNSIGNED (TREE_TYPE (op0a));
    7106        26454 :   vmode = TYPE_MODE (TREE_TYPE (op0a));
    7107              : 
    7108        26454 :   icode = get_vec_cmp_icode (vmode, mask_mode, unsignedp);
    7109        26454 :   if (icode == CODE_FOR_nothing)
    7110              :     {
    7111          983 :       if (tcode == EQ_EXPR || tcode == NE_EXPR)
    7112          983 :         icode = get_vec_cmp_eq_icode (vmode, mask_mode);
    7113          983 :       if (icode == CODE_FOR_nothing)
    7114              :         return 0;
    7115              :     }
    7116              : 
    7117        26454 :   comparison = vector_compare_rtx (mask_mode, tcode, op0a, op0b,
    7118              :                                    unsignedp, icode, 2);
    7119        26454 :   create_output_operand (&ops[0], target, mask_mode);
    7120        26454 :   create_fixed_operand (&ops[1], comparison);
    7121        26454 :   create_fixed_operand (&ops[2], XEXP (comparison, 0));
    7122        26454 :   create_fixed_operand (&ops[3], XEXP (comparison, 1));
    7123        26454 :   expand_insn (icode, 4, ops);
    7124        26454 :   return ops[0].value;
    7125              : }
    7126              : 
    7127              : /* Expand a highpart multiply.  */
    7128              : 
    7129              : rtx
    7130         3528 : expand_mult_highpart (machine_mode mode, rtx op0, rtx op1,
    7131              :                       rtx target, bool uns_p)
    7132              : {
    7133         3528 :   class expand_operand eops[3];
    7134         3528 :   enum insn_code icode;
    7135         3528 :   int method, i;
    7136         3528 :   machine_mode wmode;
    7137         3528 :   rtx m1, m2;
    7138         3528 :   optab tab1, tab2;
    7139              : 
    7140         3528 :   method = can_mult_highpart_p (mode, uns_p);
    7141         3528 :   switch (method)
    7142              :     {
    7143              :     case 0:
    7144              :       return NULL_RTX;
    7145         2795 :     case 1:
    7146         2795 :       tab1 = uns_p ? umul_highpart_optab : smul_highpart_optab;
    7147         2795 :       return expand_binop (mode, tab1, op0, op1, target, uns_p,
    7148         2795 :                            OPTAB_LIB_WIDEN);
    7149            0 :     case 2:
    7150            0 :       return expmed_mult_highpart_optab (as_a <scalar_int_mode> (mode),
    7151            0 :                                          op0, op1, target, uns_p, INT_MAX);
    7152          724 :     case 3:
    7153          724 :       tab1 = uns_p ? vec_widen_umult_even_optab : vec_widen_smult_even_optab;
    7154          525 :       tab2 = uns_p ? vec_widen_umult_odd_optab : vec_widen_smult_odd_optab;
    7155              :       break;
    7156            9 :     case 4:
    7157            9 :       tab1 = uns_p ? vec_widen_umult_lo_optab : vec_widen_smult_lo_optab;
    7158            9 :       tab2 = uns_p ? vec_widen_umult_hi_optab : vec_widen_smult_hi_optab;
    7159              :       if (BYTES_BIG_ENDIAN)
    7160              :         std::swap (tab1, tab2);
    7161              :       break;
    7162            0 :     default:
    7163            0 :       gcc_unreachable ();
    7164              :     }
    7165              : 
    7166          733 :   icode = optab_handler (tab1, mode);
    7167          733 :   wmode = insn_data[icode].operand[0].mode;
    7168         2199 :   gcc_checking_assert (known_eq (2 * GET_MODE_NUNITS (wmode),
    7169              :                                  GET_MODE_NUNITS (mode)));
    7170         2199 :   gcc_checking_assert (known_eq (GET_MODE_SIZE (wmode), GET_MODE_SIZE (mode)));
    7171              : 
    7172          733 :   create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
    7173          733 :   create_input_operand (&eops[1], op0, mode);
    7174          733 :   create_input_operand (&eops[2], op1, mode);
    7175          733 :   expand_insn (icode, 3, eops);
    7176          733 :   m1 = gen_lowpart (mode, eops[0].value);
    7177              : 
    7178          733 :   create_output_operand (&eops[0], gen_reg_rtx (wmode), wmode);
    7179          733 :   create_input_operand (&eops[1], op0, mode);
    7180          733 :   create_input_operand (&eops[2], op1, mode);
    7181          733 :   expand_insn (optab_handler (tab2, mode), 3, eops);
    7182          733 :   m2 = gen_lowpart (mode, eops[0].value);
    7183              : 
    7184         1466 :   vec_perm_builder sel;
    7185          733 :   if (method == 3)
    7186              :     {
    7187              :       /* The encoding has 2 interleaved stepped patterns.  */
    7188         1448 :       sel.new_vector (GET_MODE_NUNITS (mode), 2, 3);
    7189         5068 :       for (i = 0; i < 6; ++i)
    7190        13032 :         sel.quick_push (!BYTES_BIG_ENDIAN + (i & ~1)
    7191         6516 :                         + ((i & 1) ? GET_MODE_NUNITS (mode) : 0));
    7192              :     }
    7193              :   else
    7194              :     {
    7195              :       /* The encoding has a single interleaved stepped pattern.  */
    7196           18 :       sel.new_vector (GET_MODE_NUNITS (mode), 1, 3);
    7197           36 :       for (i = 0; i < 3; ++i)
    7198           27 :         sel.quick_push (2 * i + (BYTES_BIG_ENDIAN ? 0 : 1));
    7199              :     }
    7200              : 
    7201          733 :   return expand_vec_perm_const (mode, m1, m2, sel, BLKmode, target);
    7202              : }
    7203              : 
    7204              : /* Helper function to find the MODE_CC set in a sync_compare_and_swap
    7205              :    pattern.  */
    7206              : 
    7207              : static void
    7208            0 : find_cc_set (rtx x, const_rtx pat, void *data)
    7209              : {
    7210            0 :   if (REG_P (x) && GET_MODE_CLASS (GET_MODE (x)) == MODE_CC
    7211            0 :       && GET_CODE (pat) == SET)
    7212              :     {
    7213            0 :       rtx *p_cc_reg = (rtx *) data;
    7214            0 :       gcc_assert (!*p_cc_reg);
    7215            0 :       *p_cc_reg = x;
    7216              :     }
    7217            0 : }
    7218              : 
    7219              : /* This is a helper function for the other atomic operations.  This function
    7220              :    emits a loop that contains SEQ that iterates until a compare-and-swap
    7221              :    operation at the end succeeds.  MEM is the memory to be modified.  SEQ is
    7222              :    a set of instructions that takes a value from OLD_REG as an input and
    7223              :    produces a value in NEW_REG as an output.  Before SEQ, OLD_REG will be
    7224              :    set to the current contents of MEM.  After SEQ, a compare-and-swap will
    7225              :    attempt to update MEM with NEW_REG.  The function returns true when the
    7226              :    loop was generated successfully.  */
    7227              : 
    7228              : static bool
    7229         5258 : expand_compare_and_swap_loop (rtx mem, rtx old_reg, rtx new_reg, rtx_insn *seq)
    7230              : {
    7231         5258 :   machine_mode mode = GET_MODE (mem);
    7232         5258 :   rtx_code_label *label;
    7233         5258 :   rtx cmp_reg, success, oldval;
    7234              : 
    7235              :   /* The loop we want to generate looks like
    7236              : 
    7237              :         cmp_reg = mem;
    7238              :       label:
    7239              :         old_reg = cmp_reg;
    7240              :         seq;
    7241              :         (success, cmp_reg) = compare-and-swap(mem, old_reg, new_reg)
    7242              :         if (success)
    7243              :           goto label;
    7244              : 
    7245              :      Note that we only do the plain load from memory once.  Subsequent
    7246              :      iterations use the value loaded by the compare-and-swap pattern.  */
    7247              : 
    7248         5258 :   label = gen_label_rtx ();
    7249         5258 :   cmp_reg = gen_reg_rtx (mode);
    7250              : 
    7251         5258 :   emit_move_insn (cmp_reg, mem);
    7252         5258 :   emit_label (label);
    7253         5258 :   emit_move_insn (old_reg, cmp_reg);
    7254         5258 :   if (seq)
    7255         5245 :     emit_insn (seq);
    7256              : 
    7257         5258 :   success = NULL_RTX;
    7258         5258 :   oldval = cmp_reg;
    7259         5258 :   if (!expand_atomic_compare_and_swap (&success, &oldval, mem, old_reg,
    7260              :                                        new_reg, false, MEMMODEL_SYNC_SEQ_CST,
    7261              :                                        MEMMODEL_RELAXED))
    7262              :     return false;
    7263              : 
    7264         5258 :   if (oldval != cmp_reg)
    7265            0 :     emit_move_insn (cmp_reg, oldval);
    7266              : 
    7267              :   /* Mark this jump predicted not taken.  */
    7268         5258 :   emit_cmp_and_jump_insns (success, const0_rtx, EQ, const0_rtx,
    7269         5258 :                            GET_MODE (success), 1, label,
    7270              :                            profile_probability::guessed_never ());
    7271         5258 :   return true;
    7272              : }
    7273              : 
    7274              : 
    7275              : /* This function tries to emit an atomic_exchange instruction.  VAL is written
    7276              :    to *MEM using memory model MODEL. The previous contents of *MEM are returned,
    7277              :    using TARGET if possible.  */
    7278              : 
    7279              : static rtx
    7280         3941 : maybe_emit_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
    7281              : {
    7282         3941 :   machine_mode mode = GET_MODE (mem);
    7283         3941 :   enum insn_code icode;
    7284              : 
    7285              :   /* If the target supports the exchange directly, great.  */
    7286         3941 :   icode = direct_optab_handler (atomic_exchange_optab, mode);
    7287         3941 :   if (icode != CODE_FOR_nothing)
    7288              :     {
    7289         3924 :       class expand_operand ops[4];
    7290              : 
    7291         3924 :       create_output_operand (&ops[0], target, mode);
    7292         3924 :       create_fixed_operand (&ops[1], mem);
    7293         3924 :       create_input_operand (&ops[2], val, mode);
    7294         3924 :       create_integer_operand (&ops[3], model);
    7295         3924 :       if (maybe_expand_insn (icode, 4, ops))
    7296         3924 :         return ops[0].value;
    7297              :     }
    7298              : 
    7299              :   return NULL_RTX;
    7300              : }
    7301              : 
    7302              : /* This function tries to implement an atomic exchange operation using
    7303              :    __sync_lock_test_and_set. VAL is written to *MEM using memory model MODEL.
    7304              :    The previous contents of *MEM are returned, using TARGET if possible.
    7305              :    Since this instructionn is an acquire barrier only, stronger memory
    7306              :    models may require additional barriers to be emitted.  */
    7307              : 
    7308              : static rtx
    7309            2 : maybe_emit_sync_lock_test_and_set (rtx target, rtx mem, rtx val,
    7310              :                                    enum memmodel model)
    7311              : {
    7312            2 :   machine_mode mode = GET_MODE (mem);
    7313            2 :   enum insn_code icode;
    7314            2 :   rtx_insn *last_insn = get_last_insn ();
    7315              : 
    7316            2 :   icode = optab_handler (sync_lock_test_and_set_optab, mode);
    7317              : 
    7318              :   /* Legacy sync_lock_test_and_set is an acquire barrier.  If the pattern
    7319              :      exists, and the memory model is stronger than acquire, add a release
    7320              :      barrier before the instruction.  */
    7321              : 
    7322            2 :   if (is_mm_seq_cst (model) || is_mm_release (model) || is_mm_acq_rel (model))
    7323            0 :     expand_mem_thread_fence (model);
    7324              : 
    7325            2 :   if (icode != CODE_FOR_nothing)
    7326              :     {
    7327            0 :       class expand_operand ops[3];
    7328            0 :       create_output_operand (&ops[0], target, mode);
    7329            0 :       create_fixed_operand (&ops[1], mem);
    7330            0 :       create_input_operand (&ops[2], val, mode);
    7331            0 :       if (maybe_expand_insn (icode, 3, ops))
    7332            0 :         return ops[0].value;
    7333              :     }
    7334              : 
    7335              :   /* If an external test-and-set libcall is provided, use that instead of
    7336              :      any external compare-and-swap that we might get from the compare-and-
    7337              :      swap-loop expansion later.  */
    7338            2 :   if (!can_compare_and_swap_p (mode, false))
    7339              :     {
    7340            2 :       rtx libfunc = optab_libfunc (sync_lock_test_and_set_optab, mode);
    7341            2 :       if (libfunc != NULL)
    7342              :         {
    7343            0 :           rtx addr;
    7344              : 
    7345            0 :           addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
    7346            0 :           return emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
    7347              :                                           mode, addr, ptr_mode,
    7348            0 :                                           val, mode);
    7349              :         }
    7350              :     }
    7351              : 
    7352              :   /* If the test_and_set can't be emitted, eliminate any barrier that might
    7353              :      have been emitted.  */
    7354            2 :   delete_insns_since (last_insn);
    7355            2 :   return NULL_RTX;
    7356              : }
    7357              : 
    7358              : /* This function tries to implement an atomic exchange operation using a
    7359              :    compare_and_swap loop. VAL is written to *MEM.  The previous contents of
    7360              :    *MEM are returned, using TARGET if possible.  No memory model is required
    7361              :    since a compare_and_swap loop is seq-cst.  */
    7362              : 
    7363              : static rtx
    7364           17 : maybe_emit_compare_and_swap_exchange_loop (rtx target, rtx mem, rtx val)
    7365              : {
    7366           17 :   machine_mode mode = GET_MODE (mem);
    7367              : 
    7368           17 :   if (can_compare_and_swap_p (mode, true))
    7369              :     {
    7370           13 :       rtx_insn *start = get_last_insn ();
    7371              :       /* Force val into a register if it could change value when the
    7372              :          atomic insn updates mem.  */
    7373           13 :       if (reg_overlap_mentioned_p (mem, val))
    7374            2 :         val = force_reg (mode, val);
    7375           13 :       if (!target || !register_operand (target, mode))
    7376            1 :         target = gen_reg_rtx (mode);
    7377           13 :       if (expand_compare_and_swap_loop (mem, target, val, nullptr))
    7378              :         return target;
    7379              :       else
    7380            0 :         delete_insns_since (start);
    7381              :     }
    7382              : 
    7383              :   return NULL_RTX;
    7384              : }
    7385              : 
    7386              : /* This function tries to implement an atomic test-and-set operation
    7387              :    using the atomic_test_and_set instruction pattern.  A boolean value
    7388              :    is returned from the operation, using TARGET if possible.  */
    7389              : 
    7390              : static rtx
    7391          268 : maybe_emit_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
    7392              : {
    7393          268 :   machine_mode pat_bool_mode;
    7394          268 :   class expand_operand ops[3];
    7395              : 
    7396          268 :   if (!targetm.have_atomic_test_and_set ())
    7397              :     return NULL_RTX;
    7398              : 
    7399              :   /* While we always get QImode from __atomic_test_and_set, we get
    7400              :      other memory modes from __sync_lock_test_and_set.  Note that we
    7401              :      use no endian adjustment here.  This matches the 4.6 behavior
    7402              :      in the Sparc backend.  */
    7403            0 :   enum insn_code icode = targetm.code_for_atomic_test_and_set;
    7404            0 :   gcc_checking_assert (insn_data[icode].operand[1].mode == QImode);
    7405            0 :   if (GET_MODE (mem) != QImode)
    7406            0 :     mem = adjust_address_nv (mem, QImode, 0);
    7407              : 
    7408            0 :   pat_bool_mode = insn_data[icode].operand[0].mode;
    7409            0 :   create_output_operand (&ops[0], target, pat_bool_mode);
    7410            0 :   create_fixed_operand (&ops[1], mem);
    7411            0 :   create_integer_operand (&ops[2], model);
    7412              : 
    7413            0 :   if (maybe_expand_insn (icode, 3, ops))
    7414            0 :     return ops[0].value;
    7415              :   return NULL_RTX;
    7416              : }
    7417              : 
    7418              : /* This function expands the legacy _sync_lock test_and_set operation which is
    7419              :    generally an atomic exchange.  Some limited targets only allow the
    7420              :    constant 1 to be stored.  This is an ACQUIRE operation.
    7421              : 
    7422              :    TARGET is an optional place to stick the return value.
    7423              :    MEM is where VAL is stored.  */
    7424              : 
    7425              : rtx
    7426          326 : expand_sync_lock_test_and_set (rtx target, rtx mem, rtx val)
    7427              : {
    7428          326 :   rtx ret;
    7429              : 
    7430              :   /* Try an atomic_exchange first.  */
    7431          326 :   ret = maybe_emit_atomic_exchange (target, mem, val, MEMMODEL_SYNC_ACQUIRE);
    7432          326 :   if (ret)
    7433              :     return ret;
    7434              : 
    7435            2 :   ret = maybe_emit_sync_lock_test_and_set (target, mem, val,
    7436              :                                            MEMMODEL_SYNC_ACQUIRE);
    7437            2 :   if (ret)
    7438              :     return ret;
    7439              : 
    7440            2 :   ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
    7441            2 :   if (ret)
    7442              :     return ret;
    7443              : 
    7444              :   /* If there are no other options, try atomic_test_and_set if the value
    7445              :      being stored is 1.  */
    7446            2 :   if (val == const1_rtx)
    7447            2 :     ret = maybe_emit_atomic_test_and_set (target, mem, MEMMODEL_SYNC_ACQUIRE);
    7448              : 
    7449              :   return ret;
    7450              : }
    7451              : 
    7452              : /* This function expands the atomic test_and_set operation:
    7453              :    atomically store a boolean TRUE into MEM and return the previous value.
    7454              : 
    7455              :    MEMMODEL is the memory model variant to use.
    7456              :    TARGET is an optional place to stick the return value.  */
    7457              : 
    7458              : rtx
    7459          266 : expand_atomic_test_and_set (rtx target, rtx mem, enum memmodel model)
    7460              : {
    7461          266 :   machine_mode mode = GET_MODE (mem);
    7462          266 :   rtx ret, trueval, subtarget;
    7463              : 
    7464          266 :   ret = maybe_emit_atomic_test_and_set (target, mem, model);
    7465          266 :   if (ret)
    7466              :     return ret;
    7467              : 
    7468              :   /* Be binary compatible with non-default settings of trueval, and different
    7469              :      cpu revisions.  E.g. one revision may have atomic-test-and-set, but
    7470              :      another only has atomic-exchange.  */
    7471          266 :   if (targetm.atomic_test_and_set_trueval == 1)
    7472              :     {
    7473          266 :       trueval = const1_rtx;
    7474          266 :       subtarget = target ? target : gen_reg_rtx (mode);
    7475              :     }
    7476              :   else
    7477              :     {
    7478            0 :       trueval = gen_int_mode (targetm.atomic_test_and_set_trueval, mode);
    7479            0 :       subtarget = gen_reg_rtx (mode);
    7480              :     }
    7481              : 
    7482              :   /* Try the atomic-exchange optab...  */
    7483          266 :   ret = maybe_emit_atomic_exchange (subtarget, mem, trueval, model);
    7484              : 
    7485              :   /* ... then an atomic-compare-and-swap loop ... */
    7486          266 :   if (!ret)
    7487            0 :     ret = maybe_emit_compare_and_swap_exchange_loop (subtarget, mem, trueval);
    7488              : 
    7489              :   /* ... before trying the vaguely defined legacy lock_test_and_set. */
    7490            0 :   if (!ret)
    7491            0 :     ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, trueval, model);
    7492              : 
    7493              :   /* Recall that the legacy lock_test_and_set optab was allowed to do magic
    7494              :      things with the value 1.  Thus we try again without trueval.  */
    7495          266 :   if (!ret && targetm.atomic_test_and_set_trueval != 1)
    7496              :     {
    7497            0 :       ret = maybe_emit_sync_lock_test_and_set (subtarget, mem, const1_rtx, model);
    7498              : 
    7499            0 :       if (ret)
    7500              :         {
    7501              :           /* Rectify the not-one trueval.  */
    7502            0 :           ret = emit_store_flag_force (target, NE, ret, const0_rtx, mode, 0, 1);
    7503            0 :           gcc_assert (ret);
    7504              :         }
    7505              :     }
    7506              : 
    7507              :   return ret;
    7508              : }
    7509              : 
    7510              : /* This function expands the atomic exchange operation:
    7511              :    atomically store VAL in MEM and return the previous value in MEM.
    7512              : 
    7513              :    MEMMODEL is the memory model variant to use.
    7514              :    TARGET is an optional place to stick the return value.  */
    7515              : 
    7516              : rtx
    7517         2886 : expand_atomic_exchange (rtx target, rtx mem, rtx val, enum memmodel model)
    7518              : {
    7519         2886 :   machine_mode mode = GET_MODE (mem);
    7520         2886 :   rtx ret;
    7521              : 
    7522              :   /* If loads are not atomic for the required size and we are not called to
    7523              :      provide a __sync builtin, do not do anything so that we stay consistent
    7524              :      with atomic loads of the same size.  */
    7525         2886 :   if (!can_atomic_load_p (mode) && !is_mm_sync (model))
    7526              :     return NULL_RTX;
    7527              : 
    7528         2855 :   ret = maybe_emit_atomic_exchange (target, mem, val, model);
    7529              : 
    7530              :   /* Next try a compare-and-swap loop for the exchange.  */
    7531         2855 :   if (!ret)
    7532           13 :     ret = maybe_emit_compare_and_swap_exchange_loop (target, mem, val);
    7533              : 
    7534              :   return ret;
    7535              : }
    7536              : 
    7537              : /* This function expands the atomic compare exchange operation:
    7538              : 
    7539              :    *PTARGET_BOOL is an optional place to store the boolean success/failure.
    7540              :    *PTARGET_OVAL is an optional place to store the old value from memory.
    7541              :    Both target parameters may be NULL or const0_rtx to indicate that we do
    7542              :    not care about that return value.  Both target parameters are updated on
    7543              :    success to the actual location of the corresponding result.
    7544              : 
    7545              :    MEMMODEL is the memory model variant to use.
    7546              : 
    7547              :    The return value of the function is true for success.  */
    7548              : 
    7549              : bool
    7550        29106 : expand_atomic_compare_and_swap (rtx *ptarget_bool, rtx *ptarget_oval,
    7551              :                                 rtx mem, rtx expected, rtx desired,
    7552              :                                 bool is_weak, enum memmodel succ_model,
    7553              :                                 enum memmodel fail_model)
    7554              : {
    7555        29106 :   machine_mode mode = GET_MODE (mem);
    7556        29106 :   class expand_operand ops[8];
    7557        29106 :   enum insn_code icode;
    7558        29106 :   rtx target_oval, target_bool = NULL_RTX;
    7559        29106 :   rtx libfunc;
    7560              : 
    7561              :   /* If loads are not atomic for the required size and we are not called to
    7562              :      provide a __sync builtin, do not do anything so that we stay consistent
    7563              :      with atomic loads of the same size.  */
    7564        29106 :   if (!can_atomic_load_p (mode) && !is_mm_sync (succ_model))
    7565              :     return false;
    7566              : 
    7567              :   /* Load expected into a register for the compare and swap.  */
    7568        27357 :   if (MEM_P (expected))
    7569         7629 :     expected = copy_to_reg (expected);
    7570              : 
    7571              :   /* Make sure we always have some place to put the return oldval.
    7572              :      Further, make sure that place is distinct from the input expected,
    7573              :      just in case we need that path down below.  */
    7574        27357 :   if (ptarget_oval && *ptarget_oval == const0_rtx)
    7575              :     ptarget_oval = NULL;
    7576              : 
    7577        27127 :   if (ptarget_oval == NULL
    7578        27127 :       || (target_oval = *ptarget_oval) == NULL
    7579         5488 :       || reg_overlap_mentioned_p (expected, target_oval))
    7580        21947 :     target_oval = gen_reg_rtx (mode);
    7581              : 
    7582        27357 :   icode = direct_optab_handler (atomic_compare_and_swap_optab, mode);
    7583        27357 :   if (icode != CODE_FOR_nothing)
    7584              :     {
    7585        27355 :       machine_mode bool_mode = insn_data[icode].operand[0].mode;
    7586              : 
    7587        27355 :       if (ptarget_bool && *ptarget_bool == const0_rtx)
    7588              :         ptarget_bool = NULL;
    7589              : 
    7590              :       /* Make sure we always have a place for the bool operand.  */
    7591        27100 :       if (ptarget_bool == NULL
    7592        27100 :           || (target_bool = *ptarget_bool) == NULL
    7593         7516 :           || GET_MODE (target_bool) != bool_mode)
    7594        19839 :         target_bool = gen_reg_rtx (bool_mode);
    7595              : 
    7596              :       /* Emit the compare_and_swap.  */
    7597        27355 :       create_output_operand (&ops[0], target_bool, bool_mode);
    7598        27355 :       create_output_operand (&ops[1], target_oval, mode);
    7599        27355 :       create_fixed_operand (&ops[2], mem);
    7600        27355 :       create_input_operand (&ops[3], expected, mode);
    7601        27355 :       create_input_operand (&ops[4], desired, mode);
    7602        27355 :       create_integer_operand (&ops[5], is_weak);
    7603        27355 :       create_integer_operand (&ops[6], succ_model);
    7604        27355 :       create_integer_operand (&ops[7], fail_model);
    7605        27355 :       if (maybe_expand_insn (icode, 8, ops))
    7606              :         {
    7607              :           /* Return success/failure.  */
    7608        27355 :           target_bool = ops[0].value;
    7609        27355 :           target_oval = ops[1].value;
    7610        27355 :           goto success;
    7611              :         }
    7612              :     }
    7613              : 
    7614              :   /* Otherwise fall back to the original __sync_val_compare_and_swap
    7615              :      which is always seq-cst.  */
    7616            2 :   icode = optab_handler (sync_compare_and_swap_optab, mode);
    7617            2 :   if (icode != CODE_FOR_nothing)
    7618              :     {
    7619            0 :       rtx cc_reg;
    7620              : 
    7621            0 :       create_output_operand (&ops[0], target_oval, mode);
    7622            0 :       create_fixed_operand (&ops[1], mem);
    7623            0 :       create_input_operand (&ops[2], expected, mode);
    7624            0 :       create_input_operand (&ops[3], desired, mode);
    7625            0 :       if (!maybe_expand_insn (icode, 4, ops))
    7626            0 :         return false;
    7627              : 
    7628            0 :       target_oval = ops[0].value;
    7629              : 
    7630              :       /* If the caller isn't interested in the boolean return value,
    7631              :          skip the computation of it.  */
    7632            0 :       if (ptarget_bool == NULL)
    7633            0 :         goto success;
    7634              : 
    7635              :       /* Otherwise, work out if the compare-and-swap succeeded.  */
    7636            0 :       cc_reg = NULL_RTX;
    7637            0 :       if (have_insn_for (COMPARE, CCmode))
    7638            0 :         note_stores (get_last_insn (), find_cc_set, &cc_reg);
    7639            0 :       if (cc_reg)
    7640              :         {
    7641            0 :           target_bool = emit_store_flag_force (target_bool, EQ, cc_reg,
    7642              :                                                const0_rtx, VOIDmode, 0, 1);
    7643            0 :           goto success;
    7644              :         }
    7645            0 :       goto success_bool_from_val;
    7646              :     }
    7647              : 
    7648              :   /* Also check for library support for __sync_val_compare_and_swap.  */
    7649            2 :   libfunc = optab_libfunc (sync_compare_and_swap_optab, mode);
    7650            2 :   if (libfunc != NULL)
    7651              :     {
    7652            0 :       rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
    7653            0 :       rtx target = emit_library_call_value (libfunc, NULL_RTX, LCT_NORMAL,
    7654              :                                             mode, addr, ptr_mode,
    7655              :                                             expected, mode, desired, mode);
    7656            0 :       emit_move_insn (target_oval, target);
    7657              : 
    7658              :       /* Compute the boolean return value only if requested.  */
    7659            0 :       if (ptarget_bool)
    7660            0 :         goto success_bool_from_val;
    7661              :       else
    7662            0 :         goto success;
    7663              :     }
    7664              : 
    7665              :   /* Failure.  */
    7666              :   return false;
    7667              : 
    7668            0 :  success_bool_from_val:
    7669            0 :    target_bool = emit_store_flag_force (target_bool, EQ, target_oval,
    7670              :                                         expected, VOIDmode, 1, 1);
    7671        27355 :  success:
    7672              :   /* Make sure that the oval output winds up where the caller asked.  */
    7673        27355 :   if (ptarget_oval)
    7674        27127 :     *ptarget_oval = target_oval;
    7675        27355 :   if (ptarget_bool)
    7676        27100 :     *ptarget_bool = target_bool;
    7677              :   return true;
    7678              : }
    7679              : 
    7680              : /* Generate asm volatile("" : : : "memory") as the memory blockage.  */
    7681              : 
    7682              : static void
    7683            0 : expand_asm_memory_blockage (void)
    7684              : {
    7685            0 :   rtx asm_op, clob;
    7686              : 
    7687            0 :   asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
    7688              :                                  rtvec_alloc (0), rtvec_alloc (0),
    7689              :                                  rtvec_alloc (0), UNKNOWN_LOCATION);
    7690            0 :   MEM_VOLATILE_P (asm_op) = 1;
    7691              : 
    7692            0 :   clob = gen_rtx_SCRATCH (VOIDmode);
    7693            0 :   clob = gen_rtx_MEM (BLKmode, clob);
    7694            0 :   clob = gen_rtx_CLOBBER (VOIDmode, clob);
    7695              : 
    7696            0 :   emit_insn (gen_rtx_PARALLEL (VOIDmode, gen_rtvec (2, asm_op, clob)));
    7697            0 : }
    7698              : 
    7699              : /* Do not propagate memory accesses across this point.  */
    7700              : 
    7701              : static void
    7702       116989 : expand_memory_blockage (void)
    7703              : {
    7704       116989 :   if (targetm.have_memory_blockage ())
    7705       116989 :     emit_insn (targetm.gen_memory_blockage ());
    7706              :   else
    7707            0 :     expand_asm_memory_blockage ();
    7708       116989 : }
    7709              : 
    7710              : /* Generate asm volatile("" : : : "memory") as a memory blockage, at the
    7711              :    same time clobbering the register set specified by REGS.  */
    7712              : 
    7713              : void
    7714          131 : expand_asm_reg_clobber_mem_blockage (HARD_REG_SET regs)
    7715              : {
    7716          131 :   rtx asm_op, clob_mem;
    7717              : 
    7718          131 :   unsigned int num_of_regs = 0;
    7719          131 :   unsigned int i;
    7720              : 
    7721          131 :   num_of_regs = hard_reg_set_popcount (regs);
    7722              : 
    7723          131 :   asm_op = gen_rtx_ASM_OPERANDS (VOIDmode, "", "", 0,
    7724              :                                  rtvec_alloc (0), rtvec_alloc (0),
    7725              :                                  rtvec_alloc (0), UNKNOWN_LOCATION);
    7726          131 :   MEM_VOLATILE_P (asm_op) = 1;
    7727              : 
    7728          131 :   rtvec v = rtvec_alloc (num_of_regs + 2);
    7729              : 
    7730          131 :   clob_mem = gen_rtx_SCRATCH (VOIDmode);
    7731          131 :   clob_mem = gen_rtx_MEM (BLKmode, clob_mem);
    7732          131 :   clob_mem = gen_rtx_CLOBBER (VOIDmode, clob_mem);
    7733              : 
    7734          131 :   RTVEC_ELT (v, 0) = asm_op;
    7735          131 :   RTVEC_ELT (v, 1) = clob_mem;
    7736              : 
    7737          131 :   if (num_of_regs > 0)
    7738              :     {
    7739          131 :       unsigned int j = 2;
    7740          131 :       hard_reg_set_iterator hrsi2;
    7741          131 :       i = 0;
    7742         1250 :       EXECUTE_IF_SET_IN_HARD_REG_SET (regs, 0, i, hrsi2)
    7743              :         {
    7744         1119 :           RTVEC_ELT (v, j) = gen_rtx_CLOBBER (VOIDmode, regno_reg_rtx[i]);
    7745         1119 :           j++;
    7746              :         }
    7747          131 :       gcc_assert (j == (num_of_regs + 2));
    7748              :     }
    7749              : 
    7750          131 :   emit_insn (gen_rtx_PARALLEL (VOIDmode, v));
    7751          131 : }
    7752              : 
    7753              : /* This routine will either emit the mem_thread_fence pattern or issue a
    7754              :    sync_synchronize to generate a fence for memory model MEMMODEL.  */
    7755              : 
    7756              : void
    7757          994 : expand_mem_thread_fence (enum memmodel model)
    7758              : {
    7759          994 :   if (is_mm_relaxed (model))
    7760              :     return;
    7761          985 :   if (targetm.have_mem_thread_fence ())
    7762              :     {
    7763          985 :       emit_insn (targetm.gen_mem_thread_fence (GEN_INT (model)));
    7764          985 :       expand_memory_blockage ();
    7765              :     }
    7766            0 :   else if (targetm.have_memory_barrier ())
    7767            0 :     emit_insn (targetm.gen_memory_barrier ());
    7768            0 :   else if (synchronize_libfunc != NULL_RTX)
    7769            0 :     emit_library_call (synchronize_libfunc, LCT_NORMAL, VOIDmode);
    7770              :   else
    7771            0 :     expand_memory_blockage ();
    7772              : }
    7773              : 
    7774              : /* Emit a signal fence with given memory model.  */
    7775              : 
    7776              : void
    7777           60 : expand_mem_signal_fence (enum memmodel model)
    7778              : {
    7779              :   /* No machine barrier is required to implement a signal fence, but
    7780              :      a compiler memory barrier must be issued, except for relaxed MM.  */
    7781           60 :   if (!is_mm_relaxed (model))
    7782           51 :     expand_memory_blockage ();
    7783           60 : }
    7784              : 
    7785              : /* This function expands the atomic load operation:
    7786              :    return the atomically loaded value in MEM.
    7787              : 
    7788              :    MEMMODEL is the memory model variant to use.
    7789              :    TARGET is an option place to stick the return value.  */
    7790              : 
    7791              : rtx
    7792        71523 : expand_atomic_load (rtx target, rtx mem, enum memmodel model)
    7793              : {
    7794        71523 :   machine_mode mode = GET_MODE (mem);
    7795        71523 :   enum insn_code icode;
    7796              : 
    7797              :   /* If the target supports the load directly, great.  */
    7798        71523 :   icode = direct_optab_handler (atomic_load_optab, mode);
    7799        71523 :   if (icode != CODE_FOR_nothing)
    7800              :     {
    7801        67695 :       class expand_operand ops[3];
    7802        67695 :       rtx_insn *last = get_last_insn ();
    7803        67695 :       if (is_mm_seq_cst (model))
    7804        38347 :         expand_memory_blockage ();
    7805              : 
    7806        67695 :       create_output_operand (&ops[0], target, mode);
    7807        67695 :       create_fixed_operand (&ops[1], mem);
    7808        67695 :       create_integer_operand (&ops[2], model);
    7809        67695 :       if (maybe_expand_insn (icode, 3, ops))
    7810              :         {
    7811        67695 :           if (!is_mm_relaxed (model))
    7812        52694 :             expand_memory_blockage ();
    7813        67695 :           return ops[0].value;
    7814              :         }
    7815            0 :       delete_insns_since (last);
    7816              :     }
    7817              : 
    7818              :   /* If the size of the object is greater than word size on this target,
    7819              :      then we assume that a load will not be atomic.  We could try to
    7820              :      emulate a load with a compare-and-swap operation, but the store that
    7821              :      doing this could result in would be incorrect if this is a volatile
    7822              :      atomic load or targeting read-only-mapped memory.  */
    7823         3828 :   if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
    7824              :     /* If there is no atomic load, leave the library call.  */
    7825              :     return NULL_RTX;
    7826              : 
    7827              :   /* Otherwise assume loads are atomic, and emit the proper barriers.  */
    7828            0 :   if (!target || target == const0_rtx)
    7829            0 :     target = gen_reg_rtx (mode);
    7830              : 
    7831              :   /* For SEQ_CST, emit a barrier before the load.  */
    7832            0 :   if (is_mm_seq_cst (model))
    7833            0 :     expand_mem_thread_fence (model);
    7834              : 
    7835            0 :   emit_move_insn (target, mem);
    7836              : 
    7837              :   /* Emit the appropriate barrier after the load.  */
    7838            0 :   expand_mem_thread_fence (model);
    7839              : 
    7840            0 :   return target;
    7841              : }
    7842              : 
    7843              : /* This function expands the atomic store operation:
    7844              :    Atomically store VAL in MEM.
    7845              :    MEMMODEL is the memory model variant to use.
    7846              :    USE_RELEASE is true if __sync_lock_release can be used as a fall back.
    7847              :    function returns const0_rtx if a pattern was emitted.  */
    7848              : 
    7849              : rtx
    7850        17776 : expand_atomic_store (rtx mem, rtx val, enum memmodel model, bool use_release)
    7851              : {
    7852        17776 :   machine_mode mode = GET_MODE (mem);
    7853        17776 :   enum insn_code icode;
    7854        17776 :   class expand_operand ops[3];
    7855              : 
    7856              :   /* If the target supports the store directly, great.  */
    7857        17776 :   icode = direct_optab_handler (atomic_store_optab, mode);
    7858        17776 :   if (icode != CODE_FOR_nothing)
    7859              :     {
    7860        16173 :       rtx_insn *last = get_last_insn ();
    7861        16173 :       if (!is_mm_relaxed (model))
    7862        13350 :         expand_memory_blockage ();
    7863        16173 :       create_fixed_operand (&ops[0], mem);
    7864        16173 :       create_input_operand (&ops[1], val, mode);
    7865        16173 :       create_integer_operand (&ops[2], model);
    7866        16173 :       if (maybe_expand_insn (icode, 3, ops))
    7867              :         {
    7868        16173 :           if (is_mm_seq_cst (model))
    7869        11562 :             expand_memory_blockage ();
    7870        16173 :           return const0_rtx;
    7871              :         }
    7872            0 :       delete_insns_since (last);
    7873              :     }
    7874              : 
    7875              :   /* If using __sync_lock_release is a viable alternative, try it.
    7876              :      Note that this will not be set to true if we are expanding a generic
    7877              :      __atomic_store_n.  */
    7878         1603 :   if (use_release)
    7879              :     {
    7880            2 :       icode = direct_optab_handler (sync_lock_release_optab, mode);
    7881            2 :       if (icode != CODE_FOR_nothing)
    7882              :         {
    7883            0 :           create_fixed_operand (&ops[0], mem);
    7884            0 :           create_input_operand (&ops[1], const0_rtx, mode);
    7885            0 :           if (maybe_expand_insn (icode, 2, ops))
    7886              :             {
    7887              :               /* lock_release is only a release barrier.  */
    7888            0 :               if (is_mm_seq_cst (model))
    7889            0 :                 expand_mem_thread_fence (model);
    7890            0 :               return const0_rtx;
    7891              :             }
    7892              :         }
    7893              :     }
    7894              : 
    7895              :   /* If the size of the object is greater than word size on this target,
    7896              :      a default store will not be atomic.  */
    7897         1603 :   if (maybe_gt (GET_MODE_PRECISION (mode), BITS_PER_WORD))
    7898              :     {
    7899              :       /* If loads are atomic or we are called to provide a __sync builtin,
    7900              :          we can try a atomic_exchange and throw away the result.  Otherwise,
    7901              :          don't do anything so that we do not create an inconsistency between
    7902              :          loads and stores.  */
    7903         1603 :       if (can_atomic_load_p (mode) || is_mm_sync (model))
    7904              :         {
    7905            2 :           rtx target = maybe_emit_atomic_exchange (NULL_RTX, mem, val, model);
    7906            2 :           if (!target)
    7907            2 :             target = maybe_emit_compare_and_swap_exchange_loop (NULL_RTX, mem,
    7908              :                                                                 val);
    7909            2 :           if (target)
    7910            0 :             return const0_rtx;
    7911              :         }
    7912              :         return NULL_RTX;
    7913              :     }
    7914              : 
    7915              :   /* Otherwise assume stores are atomic, and emit the proper barriers.  */
    7916            0 :   expand_mem_thread_fence (model);
    7917              : 
    7918            0 :   emit_move_insn (mem, val);
    7919              : 
    7920              :   /* For SEQ_CST, also emit a barrier after the store.  */
    7921            0 :   if (is_mm_seq_cst (model))
    7922            0 :     expand_mem_thread_fence (model);
    7923              : 
    7924            0 :   return const0_rtx;
    7925              : }
    7926              : 
    7927              : 
    7928              : /* Structure containing the pointers and values required to process the
    7929              :    various forms of the atomic_fetch_op and atomic_op_fetch builtins.  */
    7930              : 
    7931              : struct atomic_op_functions
    7932              : {
    7933              :   direct_optab mem_fetch_before;
    7934              :   direct_optab mem_fetch_after;
    7935              :   direct_optab mem_no_result;
    7936              :   optab fetch_before;
    7937              :   optab fetch_after;
    7938              :   direct_optab no_result;
    7939              :   enum rtx_code reverse_code;
    7940              : };
    7941              : 
    7942              : 
    7943              : /* Fill in structure pointed to by OP with the various optab entries for an
    7944              :    operation of type CODE.  */
    7945              : 
    7946              : static void
    7947        35339 : get_atomic_op_for_code (struct atomic_op_functions *op, enum rtx_code code)
    7948              : {
    7949        35339 :   gcc_assert (op!= NULL);
    7950              : 
    7951              :   /* If SWITCHABLE_TARGET is defined, then subtargets can be switched
    7952              :      in the source code during compilation, and the optab entries are not
    7953              :      computable until runtime.  Fill in the values at runtime.  */
    7954        35339 :   switch (code)
    7955              :     {
    7956        22773 :     case PLUS:
    7957        22773 :       op->mem_fetch_before = atomic_fetch_add_optab;
    7958        22773 :       op->mem_fetch_after = atomic_add_fetch_optab;
    7959        22773 :       op->mem_no_result = atomic_add_optab;
    7960        22773 :       op->fetch_before = sync_old_add_optab;
    7961        22773 :       op->fetch_after = sync_new_add_optab;
    7962        22773 :       op->no_result = sync_add_optab;
    7963        22773 :       op->reverse_code = MINUS;
    7964        22773 :       break;
    7965         4854 :     case MINUS:
    7966         4854 :       op->mem_fetch_before = atomic_fetch_sub_optab;
    7967         4854 :       op->mem_fetch_after = atomic_sub_fetch_optab;
    7968         4854 :       op->mem_no_result = atomic_sub_optab;
    7969         4854 :       op->fetch_before = sync_old_sub_optab;
    7970         4854 :       op->fetch_after = sync_new_sub_optab;
    7971         4854 :       op->no_result = sync_sub_optab;
    7972         4854 :       op->reverse_code = PLUS;
    7973         4854 :       break;
    7974         2244 :     case XOR:
    7975         2244 :       op->mem_fetch_before = atomic_fetch_xor_optab;
    7976         2244 :       op->mem_fetch_after = atomic_xor_fetch_optab;
    7977         2244 :       op->mem_no_result = atomic_xor_optab;
    7978         2244 :       op->fetch_before = sync_old_xor_optab;
    7979         2244 :       op->fetch_after = sync_new_xor_optab;
    7980         2244 :       op->no_result = sync_xor_optab;
    7981         2244 :       op->reverse_code = XOR;
    7982         2244 :       break;
    7983         2122 :     case AND:
    7984         2122 :       op->mem_fetch_before = atomic_fetch_and_optab;
    7985         2122 :       op->mem_fetch_after = atomic_and_fetch_optab;
    7986         2122 :       op->mem_no_result = atomic_and_optab;
    7987         2122 :       op->fetch_before = sync_old_and_optab;
    7988         2122 :       op->fetch_after = sync_new_and_optab;
    7989         2122 :       op->no_result = sync_and_optab;
    7990         2122 :       op->reverse_code = UNKNOWN;
    7991         2122 :       break;
    7992         2593 :     case IOR:
    7993         2593 :       op->mem_fetch_before = atomic_fetch_or_optab;
    7994         2593 :       op->mem_fetch_after = atomic_or_fetch_optab;
    7995         2593 :       op->mem_no_result = atomic_or_optab;
    7996         2593 :       op->fetch_before = sync_old_ior_optab;
    7997         2593 :       op->fetch_after = sync_new_ior_optab;
    7998         2593 :       op->no_result = sync_ior_optab;
    7999         2593 :       op->reverse_code = UNKNOWN;
    8000         2593 :       break;
    8001          753 :     case NOT:
    8002          753 :       op->mem_fetch_before = atomic_fetch_nand_optab;
    8003          753 :       op->mem_fetch_after = atomic_nand_fetch_optab;
    8004          753 :       op->mem_no_result = atomic_nand_optab;
    8005          753 :       op->fetch_before = sync_old_nand_optab;
    8006          753 :       op->fetch_after = sync_new_nand_optab;
    8007          753 :       op->no_result = sync_nand_optab;
    8008          753 :       op->reverse_code = UNKNOWN;
    8009          753 :       break;
    8010            0 :     default:
    8011            0 :       gcc_unreachable ();
    8012              :     }
    8013        35339 : }
    8014              : 
    8015              : /* See if there is a more optimal way to implement the operation "*MEM CODE VAL"
    8016              :    using memory order MODEL.  If AFTER is true the operation needs to return
    8017              :    the value of *MEM after the operation, otherwise the previous value.
    8018              :    TARGET is an optional place to place the result.  The result is unused if
    8019              :    it is const0_rtx.
    8020              :    Return the result if there is a better sequence, otherwise NULL_RTX.  */
    8021              : 
    8022              : static rtx
    8023        35337 : maybe_optimize_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
    8024              :                          enum memmodel model, bool after)
    8025              : {
    8026              :   /* If the value is prefetched, or not used, it may be possible to replace
    8027              :      the sequence with a native exchange operation.  */
    8028        35337 :   if (!after || target == const0_rtx)
    8029              :     {
    8030              :       /* fetch_and (&x, 0, m) can be replaced with exchange (&x, 0, m).  */
    8031        27179 :       if (code == AND && val == const0_rtx)
    8032              :         {
    8033          327 :           if (target == const0_rtx)
    8034          150 :             target = gen_reg_rtx (GET_MODE (mem));
    8035          327 :           return maybe_emit_atomic_exchange (target, mem, val, model);
    8036              :         }
    8037              : 
    8038              :       /* fetch_or (&x, -1, m) can be replaced with exchange (&x, -1, m).  */
    8039        26852 :       if (code == IOR && val == constm1_rtx)
    8040              :         {
    8041          165 :           if (target == const0_rtx)
    8042          164 :             target = gen_reg_rtx (GET_MODE (mem));
    8043          165 :           return maybe_emit_atomic_exchange (target, mem, val, model);
    8044              :         }
    8045              :     }
    8046              : 
    8047              :   return NULL_RTX;
    8048              : }
    8049              : 
    8050              : /* Try to emit an instruction for a specific operation variation.
    8051              :    OPTAB contains the OP functions.
    8052              :    TARGET is an optional place to return the result. const0_rtx means unused.
    8053              :    MEM is the memory location to operate on.
    8054              :    VAL is the value to use in the operation.
    8055              :    USE_MEMMODEL is TRUE if the variation with a memory model should be tried.
    8056              :    MODEL is the memory model, if used.
    8057              :    AFTER is true if the returned result is the value after the operation.  */
    8058              : 
    8059              : static rtx
    8060        65290 : maybe_emit_op (const struct atomic_op_functions *optab, rtx target, rtx mem,
    8061              :                rtx val, bool use_memmodel, enum memmodel model, bool after)
    8062              : {
    8063        65290 :   machine_mode mode = GET_MODE (mem);
    8064        65290 :   class expand_operand ops[4];
    8065        65290 :   enum insn_code icode;
    8066        65290 :   int op_counter = 0;
    8067        65290 :   int num_ops;
    8068              : 
    8069              :   /* Check to see if there is a result returned.  */
    8070        65290 :   if (target == const0_rtx)
    8071              :     {
    8072        14812 :       if (use_memmodel)
    8073              :         {
    8074        14453 :           icode = direct_optab_handler (optab->mem_no_result, mode);
    8075        14453 :           create_integer_operand (&ops[2], model);
    8076        14453 :           num_ops = 3;
    8077              :         }
    8078              :       else
    8079              :         {
    8080          359 :           icode = direct_optab_handler (optab->no_result, mode);
    8081          359 :           num_ops = 2;
    8082              :         }
    8083              :     }
    8084              :   /* Otherwise, we need to generate a result.  */
    8085              :   else
    8086              :     {
    8087        50478 :       if (use_memmodel)
    8088              :         {
    8089        31470 :           icode = direct_optab_handler (after ? optab->mem_fetch_after
    8090              :                                         : optab->mem_fetch_before, mode);
    8091        31470 :           create_integer_operand (&ops[3], model);
    8092        31470 :           num_ops = 4;
    8093              :         }
    8094              :       else
    8095              :         {
    8096        19008 :           icode = optab_handler (after ? optab->fetch_after
    8097              :                                  : optab->fetch_before, mode);
    8098        19008 :           num_ops = 3;
    8099              :         }
    8100        50478 :       create_output_operand (&ops[op_counter++], target, mode);
    8101              :     }
    8102        65290 :   if (icode == CODE_FOR_nothing)
    8103              :     return NULL_RTX;
    8104              : 
    8105        26556 :   create_fixed_operand (&ops[op_counter++], mem);
    8106              :   /* VAL may have been promoted to a wider mode.  Shrink it if so.  */
    8107        26556 :   create_convert_operand_to (&ops[op_counter++], val, mode, true);
    8108              : 
    8109        26556 :   if (maybe_expand_insn (icode, num_ops, ops))
    8110        26556 :     return (target == const0_rtx ? const0_rtx : ops[0].value);
    8111              : 
    8112              :   return NULL_RTX;
    8113              : }
    8114              : 
    8115              : 
    8116              : /* This function expands an atomic fetch_OP or OP_fetch operation:
    8117              :    TARGET is an option place to stick the return value.  const0_rtx indicates
    8118              :    the result is unused.
    8119              :    atomically fetch MEM, perform the operation with VAL and return it to MEM.
    8120              :    CODE is the operation being performed (OP)
    8121              :    MEMMODEL is the memory model variant to use.
    8122              :    AFTER is true to return the result of the operation (OP_fetch).
    8123              :    AFTER is false to return the value before the operation (fetch_OP).
    8124              : 
    8125              :    This function will *only* generate instructions if there is a direct
    8126              :    optab. No compare and swap loops or libcalls will be generated. */
    8127              : 
    8128              : static rtx
    8129        35337 : expand_atomic_fetch_op_no_fallback (rtx target, rtx mem, rtx val,
    8130              :                                     enum rtx_code code, enum memmodel model,
    8131              :                                     bool after)
    8132              : {
    8133        35337 :   machine_mode mode = GET_MODE (mem);
    8134        35337 :   struct atomic_op_functions optab;
    8135        35337 :   rtx result;
    8136        35337 :   bool unused_result = (target == const0_rtx);
    8137              : 
    8138        35337 :   get_atomic_op_for_code (&optab, code);
    8139              : 
    8140              :   /* Check to see if there are any better instructions.  */
    8141        35337 :   result = maybe_optimize_fetch_op (target, mem, val, code, model, after);
    8142        35337 :   if (result)
    8143              :     return result;
    8144              : 
    8145              :   /* Check for the case where the result isn't used and try those patterns.  */
    8146        34845 :   if (unused_result)
    8147              :     {
    8148              :       /* Try the memory model variant first.  */
    8149        14453 :       result = maybe_emit_op (&optab, target, mem, val, true, model, true);
    8150        14453 :       if (result)
    8151              :         return result;
    8152              : 
    8153              :       /* Next try the old style withuot a memory model.  */
    8154          359 :       result = maybe_emit_op (&optab, target, mem, val, false, model, true);
    8155          359 :       if (result)
    8156              :         return result;
    8157              : 
    8158              :       /* There is no no-result pattern, so try patterns with a result.  */
    8159              :       target = NULL_RTX;
    8160              :     }
    8161              : 
    8162              :   /* Try the __atomic version.  */
    8163        20751 :   result = maybe_emit_op (&optab, target, mem, val, true, model, after);
    8164        20751 :   if (result)
    8165              :     return result;
    8166              : 
    8167              :   /* Try the older __sync version.  */
    8168        12133 :   result = maybe_emit_op (&optab, target, mem, val, false, model, after);
    8169        12133 :   if (result)
    8170              :     return result;
    8171              : 
    8172              :   /* If the fetch value can be calculated from the other variation of fetch,
    8173              :      try that operation.  */
    8174        12133 :   if (after || unused_result || optab.reverse_code != UNKNOWN)
    8175              :     {
    8176        10719 :       rtx_insn *start = get_last_insn ();
    8177              :       /* Force val into a register if it could change value when the
    8178              :          atomic insn updates mem.  */
    8179        10719 :       if (!unused_result && reg_overlap_mentioned_p (mem, val))
    8180          114 :         val = force_reg (mode, val);
    8181              : 
    8182              :       /* Try the __atomic version, then the older __sync version.  */
    8183        10719 :       result = maybe_emit_op (&optab, target, mem, val, true, model, !after);
    8184        10719 :       if (!result)
    8185         6875 :         result = maybe_emit_op (&optab, target, mem, val, false, model, !after);
    8186              : 
    8187         6875 :       if (result)
    8188              :         {
    8189              :           /* If the result isn't used, no need to do compensation code.  */
    8190         3844 :           if (unused_result)
    8191              :             return result;
    8192              : 
    8193              :           /* Issue compensation code.  Fetch_after  == fetch_before OP val.
    8194              :              Fetch_before == after REVERSE_OP val.  */
    8195         3844 :           if (!after)
    8196            0 :             code = optab.reverse_code;
    8197         3844 :           if (code == NOT)
    8198              :             {
    8199            0 :               result = expand_simple_binop (mode, AND, result, val, NULL_RTX,
    8200              :                                             true, OPTAB_LIB_WIDEN);
    8201            0 :               result = expand_simple_unop (mode, NOT, result, target, true);
    8202              :             }
    8203              :           else
    8204         3844 :             result = expand_simple_binop (mode, code, result, val, target,
    8205              :                                           true, OPTAB_LIB_WIDEN);
    8206              :           return result;
    8207              :         }
    8208              :       else
    8209         6875 :         delete_insns_since (start);
    8210              :     }
    8211              : 
    8212              :   /* No direct opcode can be generated.  */
    8213              :   return NULL_RTX;
    8214              : }
    8215              : 
    8216              : 
    8217              : 
    8218              : /* This function expands an atomic fetch_OP or OP_fetch operation:
    8219              :    TARGET is an option place to stick the return value.  const0_rtx indicates
    8220              :    the result is unused.
    8221              :    atomically fetch MEM, perform the operation with VAL and return it to MEM.
    8222              :    CODE is the operation being performed (OP)
    8223              :    MEMMODEL is the memory model variant to use.
    8224              :    AFTER is true to return the result of the operation (OP_fetch).
    8225              :    AFTER is false to return the value before the operation (fetch_OP).  */
    8226              : rtx
    8227        32911 : expand_atomic_fetch_op (rtx target, rtx mem, rtx val, enum rtx_code code,
    8228              :                         enum memmodel model, bool after)
    8229              : {
    8230        32911 :   machine_mode mode = GET_MODE (mem);
    8231        32911 :   rtx result;
    8232        32911 :   bool unused_result = (target == const0_rtx);
    8233              : 
    8234              :   /* If loads are not atomic for the required size and we are not called to
    8235              :      provide a __sync builtin, do not do anything so that we stay consistent
    8236              :      with atomic loads of the same size.  */
    8237        32911 :   if (!can_atomic_load_p (mode) && !is_mm_sync (model))
    8238              :     return NULL_RTX;
    8239              : 
    8240        32295 :   result = expand_atomic_fetch_op_no_fallback (target, mem, val, code, model,
    8241              :                                                after);
    8242              : 
    8243        32295 :   if (result)
    8244              :     return result;
    8245              : 
    8246              :   /* Add/sub can be implemented by doing the reverse operation with -(val).  */
    8247         8248 :   if (code == PLUS || code == MINUS)
    8248              :     {
    8249         3042 :       rtx tmp;
    8250         3042 :       enum rtx_code reverse = (code == PLUS ? MINUS : PLUS);
    8251              : 
    8252         3042 :       start_sequence ();
    8253         3042 :       tmp = expand_simple_unop (mode, NEG, val, NULL_RTX, true);
    8254         3042 :       result = expand_atomic_fetch_op_no_fallback (target, mem, tmp, reverse,
    8255              :                                                    model, after);
    8256         3042 :       if (result)
    8257              :         {
    8258              :           /* PLUS worked so emit the insns and return.  */
    8259         3001 :           tmp = end_sequence ();
    8260         3001 :           emit_insn (tmp);
    8261         3001 :           return result;
    8262              :         }
    8263              : 
    8264              :       /* PLUS did not work, so throw away the negation code and continue.  */
    8265           41 :       end_sequence ();
    8266              :     }
    8267              : 
    8268              :   /* Try the __sync libcalls only if we can't do compare-and-swap inline.  */
    8269         5247 :   if (!can_compare_and_swap_p (mode, false))
    8270              :     {
    8271            2 :       rtx libfunc;
    8272            2 :       bool fixup = false;
    8273            2 :       enum rtx_code orig_code = code;
    8274            2 :       struct atomic_op_functions optab;
    8275              : 
    8276            2 :       get_atomic_op_for_code (&optab, code);
    8277            2 :       libfunc = optab_libfunc (after ? optab.fetch_after
    8278              :                                : optab.fetch_before, mode);
    8279            2 :       if (libfunc == NULL
    8280            2 :           && (after || unused_result || optab.reverse_code != UNKNOWN))
    8281              :         {
    8282            2 :           fixup = true;
    8283            2 :           if (!after)
    8284            1 :             code = optab.reverse_code;
    8285            2 :           libfunc = optab_libfunc (after ? optab.fetch_before
    8286              :                                    : optab.fetch_after, mode);
    8287              :         }
    8288            2 :       if (libfunc != NULL)
    8289              :         {
    8290            0 :           rtx addr = convert_memory_address (ptr_mode, XEXP (mem, 0));
    8291            0 :           result = emit_library_call_value (libfunc, NULL, LCT_NORMAL, mode,
    8292              :                                             addr, ptr_mode, val, mode);
    8293              : 
    8294            0 :           if (!unused_result && fixup)
    8295            0 :             result = expand_simple_binop (mode, code, result, val, target,
    8296              :                                           true, OPTAB_LIB_WIDEN);
    8297            0 :           return result;
    8298              :         }
    8299              : 
    8300              :       /* We need the original code for any further attempts.  */
    8301            2 :       code = orig_code;
    8302              :     }
    8303              : 
    8304              :   /* If nothing else has succeeded, default to a compare and swap loop.  */
    8305         5247 :   if (can_compare_and_swap_p (mode, true))
    8306              :     {
    8307         5245 :       rtx_insn *insn;
    8308         5245 :       rtx t0 = gen_reg_rtx (mode), t1;
    8309         5245 :       rtx_insn *start = get_last_insn ();
    8310              :       /* Force val into a register if it could change value when the
    8311              :          atomic insn updates mem.  */
    8312         5245 :       if (reg_overlap_mentioned_p (mem, val))
    8313          128 :         val = force_reg (mode, val);
    8314              : 
    8315         5245 :       start_sequence ();
    8316              : 
    8317              :       /* If the result is used, get a register for it.  */
    8318         5245 :       if (!unused_result)
    8319              :         {
    8320         4909 :           if (!target || !register_operand (target, mode))
    8321           12 :             target = gen_reg_rtx (mode);
    8322              :           /* If fetch_before, copy the value now.  */
    8323         4909 :           if (!after)
    8324         2236 :             emit_move_insn (target, t0);
    8325              :         }
    8326              :       else
    8327          336 :         target = const0_rtx;
    8328              : 
    8329         5245 :       t1 = t0;
    8330         5245 :       if (code == NOT)
    8331              :         {
    8332          733 :           t1 = expand_simple_binop (mode, AND, t1, val, NULL_RTX,
    8333              :                                     true, OPTAB_LIB_WIDEN);
    8334          733 :           t1 = expand_simple_unop (mode, code, t1, NULL_RTX, true);
    8335              :         }
    8336              :       else
    8337         4512 :         t1 = expand_simple_binop (mode, code, t1, val, NULL_RTX, true,
    8338              :                                   OPTAB_LIB_WIDEN);
    8339              : 
    8340              :       /* For after, copy the value now.  */
    8341         5245 :       if (!unused_result && after)
    8342         2673 :         emit_move_insn (target, t1);
    8343         5245 :       insn = end_sequence ();
    8344              : 
    8345         5245 :       if (t1 != NULL && expand_compare_and_swap_loop (mem, t0, t1, insn))
    8346              :         return target;
    8347              :       else
    8348            0 :         delete_insns_since (start);
    8349              :     }
    8350              : 
    8351              :   return NULL_RTX;
    8352              : }
    8353              : 
    8354              : /* Return true if OPERAND is suitable for operand number OPNO of
    8355              :    instruction ICODE.  */
    8356              : 
    8357              : bool
    8358    156973558 : insn_operand_matches (enum insn_code icode, unsigned int opno, rtx operand)
    8359              : {
    8360    156973558 :   return (!insn_data[(int) icode].operand[opno].predicate
    8361    312410141 :           || (insn_data[(int) icode].operand[opno].predicate
    8362    155436583 :               (operand, insn_data[(int) icode].operand[opno].mode)));
    8363              : }
    8364              : 
    8365              : /* TARGET is a target of a multiword operation that we are going to
    8366              :    implement as a series of word-mode operations.  Return true if
    8367              :    TARGET is suitable for this purpose.  */
    8368              : 
    8369              : bool
    8370          201 : valid_multiword_target_p (rtx target)
    8371              : {
    8372          201 :   machine_mode mode;
    8373          201 :   int i, size;
    8374              : 
    8375          201 :   mode = GET_MODE (target);
    8376          402 :   if (!GET_MODE_SIZE (mode).is_constant (&size))
    8377              :     return false;
    8378          945 :   for (i = 0; i < size; i += UNITS_PER_WORD)
    8379          418 :     if (!validate_subreg (word_mode, mode, target, i))
    8380              :       return false;
    8381              :   return true;
    8382              : }
    8383              : 
    8384              : /* Make OP describe an input operand that has value INTVAL and that has
    8385              :    no inherent mode.  This function should only be used for operands that
    8386              :    are always expand-time constants.  The backend may request that INTVAL
    8387              :    be copied into a different kind of rtx, but it must specify the mode
    8388              :    of that rtx if so.  */
    8389              : 
    8390              : void
    8391      2035768 : create_integer_operand (class expand_operand *op, poly_int64 intval)
    8392              : {
    8393      2035768 :   create_expand_operand (op, EXPAND_INTEGER,
    8394              :                          gen_int_mode (intval, MAX_MODE_INT),
    8395              :                          VOIDmode, false, intval);
    8396      2035768 : }
    8397              : 
    8398              : /* Like maybe_legitimize_operand, but do not change the code of the
    8399              :    current rtx value.  */
    8400              : 
    8401              : static bool
    8402     52659540 : maybe_legitimize_operand_same_code (enum insn_code icode, unsigned int opno,
    8403              :                                     class expand_operand *op)
    8404              : {
    8405              :   /* See if the operand matches in its current form.  */
    8406     52659540 :   if (insn_operand_matches (icode, opno, op->value))
    8407              :     return true;
    8408              : 
    8409              :   /* If the operand is a memory whose address has no side effects,
    8410              :      try forcing the address into a non-virtual pseudo register.
    8411              :      The check for side effects is important because copy_to_mode_reg
    8412              :      cannot handle things like auto-modified addresses.  */
    8413      1623520 :   if (insn_data[(int) icode].operand[opno].allows_mem && MEM_P (op->value))
    8414              :     {
    8415        13134 :       rtx addr, mem;
    8416              : 
    8417        13134 :       mem = op->value;
    8418        13134 :       addr = XEXP (mem, 0);
    8419         1007 :       if (!(REG_P (addr) && REGNO (addr) > LAST_VIRTUAL_REGISTER)
    8420        13940 :           && !side_effects_p (addr))
    8421              :         {
    8422         8410 :           rtx_insn *last;
    8423         8410 :           machine_mode mode;
    8424              : 
    8425         8410 :           last = get_last_insn ();
    8426         8410 :           mode = get_address_mode (mem);
    8427         8410 :           mem = replace_equiv_address (mem, copy_to_mode_reg (mode, addr));
    8428         8410 :           if (insn_operand_matches (icode, opno, mem))
    8429              :             {
    8430            0 :               op->value = mem;
    8431            0 :               return true;
    8432              :             }
    8433         8410 :           delete_insns_since (last);
    8434              :         }
    8435              :     }
    8436              : 
    8437              :   return false;
    8438              : }
    8439              : 
    8440              : /* Try to make OP match operand OPNO of instruction ICODE.  Return true
    8441              :    on success, storing the new operand value back in OP.  */
    8442              : 
    8443              : static bool
    8444     60406411 : maybe_legitimize_operand (enum insn_code icode, unsigned int opno,
    8445              :                           class expand_operand *op)
    8446              : {
    8447     60406411 :   machine_mode mode, imode, tmode;
    8448              : 
    8449     60406411 :   mode = op->mode;
    8450     60406411 :   switch (op->type)
    8451              :     {
    8452      4521335 :     case EXPAND_FIXED:
    8453      4521335 :       {
    8454      4521335 :         temporary_volatile_ok v (true);
    8455      4521335 :         return maybe_legitimize_operand_same_code (icode, opno, op);
    8456      4521335 :       }
    8457              : 
    8458     19433646 :     case EXPAND_OUTPUT:
    8459     19433646 :       gcc_assert (mode != VOIDmode);
    8460     19433646 :       if (op->value
    8461     12076238 :           && op->value != const0_rtx
    8462     12075809 :           && GET_MODE (op->value) == mode
    8463     31473606 :           && maybe_legitimize_operand_same_code (icode, opno, op))
    8464              :         return true;
    8465              : 
    8466      7518361 :       op->value = gen_reg_rtx (mode);
    8467      7518361 :       op->target = 0;
    8468      7518361 :       break;
    8469              : 
    8470     36098245 :     case EXPAND_INPUT:
    8471     36098245 :     input:
    8472     36098245 :       gcc_assert (mode != VOIDmode);
    8473     36098245 :       gcc_assert (GET_MODE (op->value) == VOIDmode
    8474              :                   || GET_MODE (op->value) == mode);
    8475     36098245 :       if (maybe_legitimize_operand_same_code (icode, opno, op))
    8476              :         return true;
    8477              : 
    8478      1352645 :       op->value = copy_to_mode_reg (mode, op->value);
    8479      1352645 :       break;
    8480              : 
    8481       204849 :     case EXPAND_CONVERT_TO:
    8482       204849 :       gcc_assert (mode != VOIDmode);
    8483       204849 :       op->value = convert_to_mode (mode, op->value, op->unsigned_p);
    8484       204849 :       goto input;
    8485              : 
    8486       670797 :     case EXPAND_CONVERT_FROM:
    8487       670797 :       if (GET_MODE (op->value) != VOIDmode)
    8488       460267 :         mode = GET_MODE (op->value);
    8489              :       else
    8490              :         /* The caller must tell us what mode this value has.  */
    8491       210530 :         gcc_assert (mode != VOIDmode);
    8492              : 
    8493       670797 :       imode = insn_data[(int) icode].operand[opno].mode;
    8494       670797 :       tmode = (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode)
    8495       670797 :                ? GET_MODE_INNER (imode) : imode);
    8496       670797 :       if (tmode != VOIDmode && tmode != mode)
    8497              :         {
    8498         1752 :           op->value = convert_modes (tmode, mode, op->value, op->unsigned_p);
    8499         1752 :           mode = tmode;
    8500              :         }
    8501       670797 :       if (imode != VOIDmode && imode != mode)
    8502              :         {
    8503           46 :           gcc_assert (VECTOR_MODE_P (imode) && !VECTOR_MODE_P (mode));
    8504           46 :           op->value = expand_vector_broadcast (imode, op->value);
    8505           46 :           mode = imode;
    8506              :         }
    8507       670797 :       goto input;
    8508              : 
    8509         3399 :     case EXPAND_ADDRESS:
    8510         3399 :       op->value = convert_memory_address (as_a <scalar_int_mode> (mode),
    8511              :                                           op->value);
    8512         3399 :       goto input;
    8513              : 
    8514      1659851 :     case EXPAND_INTEGER:
    8515      1659851 :       mode = insn_data[(int) icode].operand[opno].mode;
    8516      1659851 :       if (mode != VOIDmode
    8517      1659851 :           && known_eq (trunc_int_for_mode (op->int_value, mode),
    8518              :                        op->int_value))
    8519              :         {
    8520      1306666 :           op->value = gen_int_mode (op->int_value, mode);
    8521      1306666 :           goto input;
    8522              :         }
    8523              :       break;
    8524              : 
    8525          249 :     case EXPAND_UNDEFINED_INPUT:
    8526              :       /* See if the predicate accepts a SCRATCH rtx, which in this context
    8527              :          indicates an undefined value.  Use an uninitialized register if not. */
    8528          249 :       if (!insn_operand_matches (icode, opno, op->value))
    8529              :         {
    8530          249 :           op->value = gen_reg_rtx (op->mode);
    8531          249 :           goto input;
    8532              :         }
    8533              :       return true;
    8534              :     }
    8535      9224191 :   return insn_operand_matches (icode, opno, op->value);
    8536              : }
    8537              : 
    8538              : /* Make OP describe an input operand that should have the same value
    8539              :    as VALUE, after any mode conversion that the target might request.
    8540              :    TYPE is the type of VALUE.  */
    8541              : 
    8542              : void
    8543         2702 : create_convert_operand_from_type (class expand_operand *op,
    8544              :                                   rtx value, tree type)
    8545              : {
    8546         2702 :   create_convert_operand_from (op, value, TYPE_MODE (type),
    8547         2702 :                                TYPE_UNSIGNED (type));
    8548         2702 : }
    8549              : 
    8550              : /* Return true if the requirements on operands OP1 and OP2 of instruction
    8551              :    ICODE are similar enough for the result of legitimizing OP1 to be
    8552              :    reusable for OP2.  OPNO1 and OPNO2 are the operand numbers associated
    8553              :    with OP1 and OP2 respectively.  */
    8554              : 
    8555              : static inline bool
    8556     67237437 : can_reuse_operands_p (enum insn_code icode,
    8557              :                       unsigned int opno1, unsigned int opno2,
    8558              :                       const class expand_operand *op1,
    8559              :                       const class expand_operand *op2)
    8560              : {
    8561              :   /* Check requirements that are common to all types.  */
    8562     67237437 :   if (op1->type != op2->type
    8563     20881684 :       || op1->mode != op2->mode
    8564     19324903 :       || (insn_data[(int) icode].operand[opno1].mode
    8565     19324903 :           != insn_data[(int) icode].operand[opno2].mode))
    8566              :     return false;
    8567              : 
    8568              :   /* Check the requirements for specific types.  */
    8569     17397165 :   switch (op1->type)
    8570              :     {
    8571              :     case EXPAND_OUTPUT:
    8572              :     case EXPAND_UNDEFINED_INPUT:
    8573              :       /* Outputs and undefined inputs must remain distinct.  */
    8574              :       return false;
    8575              : 
    8576              :     case EXPAND_FIXED:
    8577              :     case EXPAND_INPUT:
    8578              :     case EXPAND_ADDRESS:
    8579              :     case EXPAND_INTEGER:
    8580              :       return true;
    8581              : 
    8582       119497 :     case EXPAND_CONVERT_TO:
    8583       119497 :     case EXPAND_CONVERT_FROM:
    8584       119497 :       return op1->unsigned_p == op2->unsigned_p;
    8585              :     }
    8586            0 :   gcc_unreachable ();
    8587              : }
    8588              : 
    8589              : /* Try to make operands [OPS, OPS + NOPS) match operands [OPNO, OPNO + NOPS)
    8590              :    of instruction ICODE.  Return true on success, leaving the new operand
    8591              :    values in the OPS themselves.  Emit no code on failure.  */
    8592              : 
    8593              : bool
    8594     20190550 : maybe_legitimize_operands (enum insn_code icode, unsigned int opno,
    8595              :                            unsigned int nops, class expand_operand *ops)
    8596              : {
    8597     20190550 :   rtx_insn *last = get_last_insn ();
    8598     20190550 :   rtx *orig_values = XALLOCAVEC (rtx, nops);
    8599     80838252 :   for (unsigned int i = 0; i < nops; i++)
    8600              :     {
    8601     60801430 :       orig_values[i] = ops[i].value;
    8602              : 
    8603              :       /* First try reusing the result of an earlier legitimization.
    8604              :          This avoids duplicate rtl and ensures that tied operands
    8605              :          remain tied.
    8606              : 
    8607              :          This search is linear, but NOPS is bounded at compile time
    8608              :          to a small number (current a single digit).  */
    8609     60801430 :       unsigned int j = 0;
    8610    127643848 :       for (; j < i; ++j)
    8611     67237437 :         if (can_reuse_operands_p (icode, opno + j, opno + i, &ops[j], &ops[i])
    8612     17376041 :             && rtx_equal_p (orig_values[j], orig_values[i])
    8613       413483 :             && ops[j].value
    8614     67632456 :             && insn_operand_matches (icode, opno + i, ops[j].value))
    8615              :           {
    8616       395019 :             ops[i].value = copy_rtx (ops[j].value);
    8617       395019 :             break;
    8618              :           }
    8619              : 
    8620              :       /* Otherwise try legitimizing the operand on its own.  */
    8621     60801430 :       if (j == i && !maybe_legitimize_operand (icode, opno + i, &ops[i]))
    8622              :         {
    8623       153728 :           delete_insns_since (last);
    8624       153728 :           return false;
    8625              :         }
    8626              :     }
    8627              :   return true;
    8628              : }
    8629              : 
    8630              : /* Try to generate instruction ICODE, using operands [OPS, OPS + NOPS)
    8631              :    as its operands.  Return the instruction pattern on success,
    8632              :    and emit any necessary set-up code.  Return null and emit no
    8633              :    code on failure.  */
    8634              : 
    8635              : rtx_insn *
    8636     20164076 : maybe_gen_insn (enum insn_code icode, unsigned int nops,
    8637              :                 class expand_operand *ops)
    8638              : {
    8639     20164076 :   gcc_assert (nops == (unsigned int) insn_data[(int) icode].n_generator_args);
    8640     20164076 :   if (!maybe_legitimize_operands (icode, 0, nops, ops))
    8641              :     return NULL;
    8642              : 
    8643     20010348 :   switch (nops)
    8644              :     {
    8645            0 :     case 0:
    8646            0 :       return GEN_FCN (icode) ();
    8647         1341 :     case 1:
    8648         1341 :       return GEN_FCN (icode) (ops[0].value);
    8649      2889187 :     case 2:
    8650      2889187 :       return GEN_FCN (icode) (ops[0].value, ops[1].value);
    8651     14991118 :     case 3:
    8652     14991118 :       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value);
    8653      1738204 :     case 4:
    8654      1738204 :       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
    8655      1738204 :                               ops[3].value);
    8656       191523 :     case 5:
    8657       191523 :       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
    8658       191523 :                               ops[3].value, ops[4].value);
    8659          322 :     case 6:
    8660          322 :       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
    8661          322 :                               ops[3].value, ops[4].value, ops[5].value);
    8662            0 :     case 7:
    8663            0 :       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
    8664            0 :                               ops[3].value, ops[4].value, ops[5].value,
    8665            0 :                               ops[6].value);
    8666        27355 :     case 8:
    8667        27355 :       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
    8668        27355 :                               ops[3].value, ops[4].value, ops[5].value,
    8669        27355 :                               ops[6].value, ops[7].value);
    8670       171298 :     case 9:
    8671       171298 :       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
    8672       171298 :                               ops[3].value, ops[4].value, ops[5].value,
    8673       171298 :                               ops[6].value, ops[7].value, ops[8].value);
    8674            0 :     case 10:
    8675            0 :       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
    8676            0 :                               ops[3].value, ops[4].value, ops[5].value,
    8677            0 :                               ops[6].value, ops[7].value, ops[8].value,
    8678            0 :                               ops[9].value);
    8679            0 :     case 11:
    8680            0 :       return GEN_FCN (icode) (ops[0].value, ops[1].value, ops[2].value,
    8681            0 :                               ops[3].value, ops[4].value, ops[5].value,
    8682            0 :                               ops[6].value, ops[7].value, ops[8].value,
    8683            0 :                               ops[9].value, ops[10].value);
    8684              :     }
    8685            0 :   gcc_unreachable ();
    8686              : }
    8687              : 
    8688              : /* Try to emit instruction ICODE, using operands [OPS, OPS + NOPS)
    8689              :    as its operands.  Return true on success and emit no code on failure.  */
    8690              : 
    8691              : bool
    8692      3023938 : maybe_expand_insn (enum insn_code icode, unsigned int nops,
    8693              :                    class expand_operand *ops)
    8694              : {
    8695      3023938 :   rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
    8696      3023938 :   if (pat)
    8697              :     {
    8698      2203834 :       emit_insn (pat);
    8699      2203834 :       return true;
    8700              :     }
    8701              :   return false;
    8702              : }
    8703              : 
    8704              : /* Like maybe_expand_insn, but for jumps.  */
    8705              : 
    8706              : bool
    8707         1338 : maybe_expand_jump_insn (enum insn_code icode, unsigned int nops,
    8708              :                         class expand_operand *ops)
    8709              : {
    8710         1338 :   rtx_insn *pat = maybe_gen_insn (icode, nops, ops);
    8711         1338 :   if (pat)
    8712              :     {
    8713         1338 :       emit_jump_insn (pat);
    8714         1338 :       return true;
    8715              :     }
    8716              :   return false;
    8717              : }
    8718              : 
    8719              : /* Emit instruction ICODE, using operands [OPS, OPS + NOPS)
    8720              :    as its operands.  */
    8721              : 
    8722              : void
    8723       637132 : expand_insn (enum insn_code icode, unsigned int nops,
    8724              :              class expand_operand *ops)
    8725              : {
    8726       637132 :   if (!maybe_expand_insn (icode, nops, ops))
    8727            0 :     gcc_unreachable ();
    8728       637132 : }
    8729              : 
    8730              : /* Like expand_insn, but for jumps.  */
    8731              : 
    8732              : void
    8733         1338 : expand_jump_insn (enum insn_code icode, unsigned int nops,
    8734              :                   class expand_operand *ops)
    8735              : {
    8736         1338 :   if (!maybe_expand_jump_insn (icode, nops, ops))
    8737            0 :     gcc_unreachable ();
    8738         1338 : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.