LCOV - code coverage report
Current view: top level - gcc - expmed.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 84.1 % 3013 2535
Test Date: 2026-11-14 16:39:50 Functions: 97.1 % 68 66
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Medium-level subroutines: convert bit-field store and extract
       2              :    and shifts, multiplies and divides to rtl instructions.
       3              :    Copyright (C) 1987-2026 Free Software Foundation, Inc.
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify it under
       8              : the terms of the GNU General Public License as published by the Free
       9              : Software Foundation; either version 3, or (at your option) any later
      10              : version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      13              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      14              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      15              : for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : /* Work around tree-optimization/91825.  */
      22              : #pragma GCC diagnostic warning "-Wmaybe-uninitialized"
      23              : 
      24              : #include "config.h"
      25              : #include "system.h"
      26              : #include "coretypes.h"
      27              : #include "backend.h"
      28              : #include "target.h"
      29              : #include "rtl.h"
      30              : #include "tree.h"
      31              : #include "predict.h"
      32              : #include "memmodel.h"
      33              : #include "tm_p.h"
      34              : #include "optabs.h"
      35              : #include "expmed.h"
      36              : #include "regs.h"
      37              : #include "emit-rtl.h"
      38              : #include "diagnostic-core.h"
      39              : #include "fold-const.h"
      40              : #include "stor-layout.h"
      41              : #include "dojump.h"
      42              : #include "explow.h"
      43              : #include "expr.h"
      44              : #include "langhooks.h"
      45              : #include "tree-vector-builder.h"
      46              : #include "recog.h"
      47              : 
      48              : struct target_expmed default_target_expmed;
      49              : #if SWITCHABLE_TARGET
      50              : struct target_expmed *this_target_expmed = &default_target_expmed;
      51              : #endif
      52              : 
      53              : static bool store_integral_bit_field (rtx, opt_scalar_int_mode,
      54              :                                       unsigned HOST_WIDE_INT,
      55              :                                       unsigned HOST_WIDE_INT,
      56              :                                       poly_uint64, poly_uint64,
      57              :                                       machine_mode, rtx, bool, bool);
      58              : /* The constant divisor of the last division expanded, so that a modulo by
      59              :    the same constant can reuse the quotient.  Reset for each function by
      60              :    prepare_function_start: carrying it across functions would make a
      61              :    function's code depend on what was expanded before it in the translation
      62              :    unit.  */
      63              : HOST_WIDE_INT last_div_const = 0;
      64              : 
      65              : static void store_fixed_bit_field (rtx, opt_scalar_int_mode,
      66              :                                    unsigned HOST_WIDE_INT,
      67              :                                    unsigned HOST_WIDE_INT,
      68              :                                    poly_uint64, poly_uint64,
      69              :                                    rtx, scalar_int_mode, bool);
      70              : static void store_fixed_bit_field_1 (rtx, scalar_int_mode,
      71              :                                      unsigned HOST_WIDE_INT,
      72              :                                      unsigned HOST_WIDE_INT,
      73              :                                      rtx, scalar_int_mode, bool);
      74              : static void store_split_bit_field (rtx, opt_scalar_int_mode,
      75              :                                    unsigned HOST_WIDE_INT,
      76              :                                    unsigned HOST_WIDE_INT,
      77              :                                    poly_uint64, poly_uint64,
      78              :                                    rtx, scalar_int_mode, bool);
      79              : static rtx extract_integral_bit_field (rtx, opt_scalar_int_mode,
      80              :                                        unsigned HOST_WIDE_INT,
      81              :                                        unsigned HOST_WIDE_INT, int, rtx,
      82              :                                        machine_mode, machine_mode, bool, bool);
      83              : static rtx extract_fixed_bit_field (machine_mode, rtx, opt_scalar_int_mode,
      84              :                                     unsigned HOST_WIDE_INT,
      85              :                                     unsigned HOST_WIDE_INT, rtx, int, bool);
      86              : static rtx extract_fixed_bit_field_1 (machine_mode, rtx, scalar_int_mode,
      87              :                                       unsigned HOST_WIDE_INT,
      88              :                                       unsigned HOST_WIDE_INT, rtx, int, bool);
      89              : static rtx lshift_value (machine_mode, unsigned HOST_WIDE_INT, int);
      90              : static rtx extract_split_bit_field (rtx, opt_scalar_int_mode,
      91              :                                     unsigned HOST_WIDE_INT,
      92              :                                     unsigned HOST_WIDE_INT, int, bool);
      93              : static void do_cmp_and_jump (rtx, rtx, enum rtx_code, machine_mode, rtx_code_label *);
      94              : static rtx expand_smod_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
      95              : static rtx expand_sdiv_pow2 (scalar_int_mode, rtx, HOST_WIDE_INT);
      96              : 
      97              : /* Return a constant integer mask value of mode MODE with BITSIZE ones
      98              :    followed by BITPOS zeros, or the complement of that if COMPLEMENT.
      99              :    The mask is truncated if necessary to the width of mode MODE.  The
     100              :    mask is zero-extended if BITSIZE+BITPOS is too small for MODE.  */
     101              : 
     102              : static inline rtx
     103       216627 : mask_rtx (scalar_int_mode mode, int bitpos, int bitsize, bool complement)
     104              : {
     105       216627 :   return immed_wide_int_const
     106       216627 :     (wi::shifted_mask (bitpos, bitsize, complement,
     107       216627 :                        GET_MODE_PRECISION (mode)), mode);
     108              : }
     109              : 
     110              : /* Test whether a value is zero of a power of two.  */
     111              : #define EXACT_POWER_OF_2_OR_ZERO_P(x) \
     112              :   (((x) & ((x) - HOST_WIDE_INT_1U)) == 0)
     113              : 
     114              : struct init_expmed_rtl
     115              : {
     116              :   rtx reg;
     117              :   rtx plus;
     118              :   rtx neg;
     119              :   rtx mult;
     120              :   rtx sdiv;
     121              :   rtx udiv;
     122              :   rtx sdiv_32;
     123              :   rtx smod_32;
     124              :   rtx wide_mult;
     125              :   rtx wide_lshr;
     126              :   rtx wide_trunc;
     127              :   rtx shift;
     128              :   rtx shift_mult;
     129              :   rtx shift_add;
     130              :   rtx shift_sub0;
     131              :   rtx shift_sub1;
     132              :   rtx zext;
     133              :   rtx trunc;
     134              : 
     135              :   rtx pow2[MAX_BITS_PER_WORD];
     136              :   rtx cint[MAX_BITS_PER_WORD];
     137              : };
     138              : 
     139              : static void
     140     31100440 : init_expmed_one_conv (struct init_expmed_rtl *all, scalar_int_mode to_mode,
     141              :                       scalar_int_mode from_mode, bool speed)
     142              : {
     143     31100440 :   int to_size, from_size;
     144     31100440 :   rtx which;
     145              : 
     146     31100440 :   to_size = GET_MODE_PRECISION (to_mode);
     147     31100440 :   from_size = GET_MODE_PRECISION (from_mode);
     148              : 
     149              :   /* Most partial integers have a precision less than the "full"
     150              :      integer it requires for storage.  In case one doesn't, for
     151              :      comparison purposes here, reduce the bit size by one in that
     152              :      case.  */
     153     31100440 :   if (GET_MODE_CLASS (to_mode) == MODE_PARTIAL_INT
     154     31100440 :       && pow2p_hwi (to_size))
     155      6220088 :     to_size --;
     156     31100440 :   if (GET_MODE_CLASS (from_mode) == MODE_PARTIAL_INT
     157     31100440 :       && pow2p_hwi (from_size))
     158            0 :     from_size --;
     159              : 
     160              :   /* Assume cost of zero-extend and sign-extend is the same.  */
     161     31100440 :   which = (to_size < from_size ? all->trunc : all->zext);
     162              : 
     163     31100440 :   PUT_MODE (all->reg, from_mode);
     164     31100440 :   set_convert_cost (to_mode, from_mode, speed,
     165              :                     set_src_cost (which, to_mode, speed));
     166              :   /* Restore all->reg's mode.  */
     167     31100440 :   PUT_MODE (all->reg, to_mode);
     168     31100440 : }
     169              : 
     170              : static void
     171     18215972 : init_expmed_one_mode (struct init_expmed_rtl *all,
     172              :                       machine_mode mode, int speed)
     173              : {
     174     18215972 :   int m, n, mode_bitsize;
     175     18215972 :   machine_mode mode_from;
     176              : 
     177     18215972 :   mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
     178              : 
     179     18215972 :   PUT_MODE (all->reg, mode);
     180     18215972 :   PUT_MODE (all->plus, mode);
     181     18215972 :   PUT_MODE (all->neg, mode);
     182     18215972 :   PUT_MODE (all->mult, mode);
     183     18215972 :   PUT_MODE (all->sdiv, mode);
     184     18215972 :   PUT_MODE (all->udiv, mode);
     185     18215972 :   PUT_MODE (all->sdiv_32, mode);
     186     18215972 :   PUT_MODE (all->smod_32, mode);
     187     18215972 :   PUT_MODE (all->wide_trunc, mode);
     188     18215972 :   PUT_MODE (all->shift, mode);
     189     18215972 :   PUT_MODE (all->shift_mult, mode);
     190     18215972 :   PUT_MODE (all->shift_add, mode);
     191     18215972 :   PUT_MODE (all->shift_sub0, mode);
     192     18215972 :   PUT_MODE (all->shift_sub1, mode);
     193     18215972 :   PUT_MODE (all->zext, mode);
     194     18215972 :   PUT_MODE (all->trunc, mode);
     195              : 
     196     18215972 :   set_add_cost (speed, mode, set_src_cost (all->plus, mode, speed));
     197     18215972 :   set_neg_cost (speed, mode, set_src_cost (all->neg, mode, speed));
     198     18215972 :   set_mul_cost (speed, mode, set_src_cost (all->mult, mode, speed));
     199     18215972 :   set_sdiv_cost (speed, mode, set_src_cost (all->sdiv, mode, speed));
     200     18215972 :   set_udiv_cost (speed, mode, set_src_cost (all->udiv, mode, speed));
     201              : 
     202     18215972 :   set_sdiv_pow2_cheap (speed, mode, (set_src_cost (all->sdiv_32, mode, speed)
     203     18215972 :                                      <= 2 * add_cost (speed, mode)));
     204     18215972 :   set_smod_pow2_cheap (speed, mode, (set_src_cost (all->smod_32, mode, speed)
     205     18215972 :                                      <= 4 * add_cost (speed, mode)));
     206              : 
     207     18215972 :   set_shift_cost (speed, mode, 0, 0);
     208     18215972 :   {
     209     18215972 :     int cost = add_cost (speed, mode);
     210     18215972 :     set_shiftadd_cost (speed, mode, 0, cost);
     211     18215972 :     set_shiftsub0_cost (speed, mode, 0, cost);
     212     18215972 :     set_shiftsub1_cost (speed, mode, 0, cost);
     213              :   }
     214              : 
     215     18215972 :   n = MIN (MAX_BITS_PER_WORD, mode_bitsize);
     216    611345792 :   for (m = 1; m < n; m++)
     217              :     {
     218    593129820 :       XEXP (all->shift, 1) = all->cint[m];
     219    593129820 :       XEXP (all->shift_mult, 1) = all->pow2[m];
     220              : 
     221    593129820 :       set_shift_cost (speed, mode, m, set_src_cost (all->shift, mode, speed));
     222    593129820 :       set_shiftadd_cost (speed, mode, m, set_src_cost (all->shift_add, mode,
     223              :                                                        speed));
     224    593129820 :       set_shiftsub0_cost (speed, mode, m, set_src_cost (all->shift_sub0, mode,
     225              :                                                         speed));
     226    593129820 :       set_shiftsub1_cost (speed, mode, m, set_src_cost (all->shift_sub1, mode,
     227              :                                                         speed));
     228              :     }
     229              : 
     230     18215972 :   scalar_int_mode int_mode_to;
     231     18215972 :   if (is_a <scalar_int_mode> (mode, &int_mode_to))
     232              :     {
     233     35543360 :       for (mode_from = MIN_MODE_INT; mode_from <= MAX_MODE_INT;
     234     31100440 :            mode_from = (machine_mode)(mode_from + 1))
     235     31100440 :         init_expmed_one_conv (all, int_mode_to,
     236              :                               as_a <scalar_int_mode> (mode_from), speed);
     237              : 
     238      4442920 :       scalar_int_mode wider_mode;
     239      4442920 :       if (GET_MODE_CLASS (int_mode_to) == MODE_INT
     240      4442920 :           && GET_MODE_WIDER_MODE (int_mode_to).exists (&wider_mode))
     241              :         {
     242      2665752 :           PUT_MODE (all->reg, mode);
     243      2665752 :           PUT_MODE (all->zext, wider_mode);
     244      2665752 :           PUT_MODE (all->wide_mult, wider_mode);
     245      2665752 :           PUT_MODE (all->wide_lshr, wider_mode);
     246      2665752 :           XEXP (all->wide_lshr, 1)
     247      2665752 :             = gen_int_shift_amount (wider_mode, mode_bitsize);
     248              : 
     249      2665752 :           set_mul_widen_cost (speed, wider_mode,
     250              :                               set_src_cost (all->wide_mult, wider_mode, speed));
     251      2665752 :           set_mul_highpart_cost (speed, int_mode_to,
     252              :                                  set_src_cost (all->wide_trunc,
     253              :                                                int_mode_to, speed));
     254              :         }
     255              :     }
     256     18215972 : }
     257              : 
     258              : void
     259       222146 : init_expmed (void)
     260              : {
     261       222146 :   struct init_expmed_rtl all;
     262       222146 :   machine_mode mode = QImode;
     263       222146 :   int m, speed;
     264              : 
     265       222146 :   memset (&all, 0, sizeof all);
     266     14217344 :   for (m = 1; m < MAX_BITS_PER_WORD; m++)
     267              :     {
     268     13995198 :       all.pow2[m] = GEN_INT (HOST_WIDE_INT_1 << m);
     269     13995198 :       all.cint[m] = GEN_INT (m);
     270              :     }
     271              : 
     272              :   /* Avoid using hard regs in ways which may be unsupported.  */
     273       222146 :   all.reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
     274       222146 :   all.plus = gen_rtx_PLUS (mode, all.reg, all.reg);
     275       222146 :   all.neg = gen_rtx_NEG (mode, all.reg);
     276       222146 :   all.mult = gen_rtx_MULT (mode, all.reg, all.reg);
     277       222146 :   all.sdiv = gen_rtx_DIV (mode, all.reg, all.reg);
     278       222146 :   all.udiv = gen_rtx_UDIV (mode, all.reg, all.reg);
     279       222146 :   all.sdiv_32 = gen_rtx_DIV (mode, all.reg, all.pow2[5]);
     280       222146 :   all.smod_32 = gen_rtx_MOD (mode, all.reg, all.pow2[5]);
     281       222146 :   all.zext = gen_rtx_ZERO_EXTEND (mode, all.reg);
     282       222146 :   all.wide_mult = gen_rtx_MULT (mode, all.zext, all.zext);
     283       222146 :   all.wide_lshr = gen_rtx_LSHIFTRT (mode, all.wide_mult, all.reg);
     284       222146 :   all.wide_trunc = gen_rtx_TRUNCATE (mode, all.wide_lshr);
     285       222146 :   all.shift = gen_rtx_ASHIFT (mode, all.reg, all.reg);
     286       222146 :   all.shift_mult = gen_rtx_MULT (mode, all.reg, all.reg);
     287       222146 :   all.shift_add = gen_rtx_PLUS (mode, all.shift_mult, all.reg);
     288       222146 :   all.shift_sub0 = gen_rtx_MINUS (mode, all.shift_mult, all.reg);
     289       222146 :   all.shift_sub1 = gen_rtx_MINUS (mode, all.reg, all.shift_mult);
     290       222146 :   all.trunc = gen_rtx_TRUNCATE (mode, all.reg);
     291              : 
     292       666438 :   for (speed = 0; speed < 2; speed++)
     293              :     {
     294       444292 :       crtl->maybe_hot_insn_p = speed;
     295       444292 :       set_zero_cost (speed, set_src_cost (const0_rtx, QImode, speed));
     296              : 
     297      3554336 :       for (mode = MIN_MODE_INT; mode <= MAX_MODE_INT;
     298      3110044 :            mode = (machine_mode)(mode + 1))
     299      3110044 :         init_expmed_one_mode (&all, mode, speed);
     300              : 
     301              :       if (MIN_MODE_PARTIAL_INT != VOIDmode)
     302      1777168 :         for (mode = MIN_MODE_PARTIAL_INT; mode <= MAX_MODE_PARTIAL_INT;
     303      1332876 :              mode = (machine_mode)(mode + 1))
     304      1332876 :           init_expmed_one_mode (&all, mode, speed);
     305              : 
     306              :       if (MIN_MODE_VECTOR_INT != VOIDmode)
     307     14217344 :         for (mode = MIN_MODE_VECTOR_INT; mode <= MAX_MODE_VECTOR_INT;
     308     13773052 :              mode = (machine_mode)(mode + 1))
     309     13773052 :           init_expmed_one_mode (&all, mode, speed);
     310              :     }
     311              : 
     312       222146 :   if (alg_hash_used_p ())
     313              :     {
     314         1057 :       struct alg_hash_entry *p = alg_hash_entry_ptr (0);
     315         1057 :       memset (p, 0, sizeof (*p) * NUM_ALG_HASH_ENTRIES);
     316              :     }
     317              :   else
     318       221089 :     set_alg_hash_used_p (true);
     319       222146 :   default_rtl_profile ();
     320              : 
     321       222146 :   ggc_free (all.trunc);
     322       222146 :   ggc_free (all.shift_sub1);
     323       222146 :   ggc_free (all.shift_sub0);
     324       222146 :   ggc_free (all.shift_add);
     325       222146 :   ggc_free (all.shift_mult);
     326       222146 :   ggc_free (all.shift);
     327       222146 :   ggc_free (all.wide_trunc);
     328       222146 :   ggc_free (all.wide_lshr);
     329       222146 :   ggc_free (all.wide_mult);
     330       222146 :   ggc_free (all.zext);
     331       222146 :   ggc_free (all.smod_32);
     332       222146 :   ggc_free (all.sdiv_32);
     333       222146 :   ggc_free (all.udiv);
     334       222146 :   ggc_free (all.sdiv);
     335       222146 :   ggc_free (all.mult);
     336       222146 :   ggc_free (all.neg);
     337       222146 :   ggc_free (all.plus);
     338       222146 :   ggc_free (all.reg);
     339       222146 : }
     340              : 
     341              : /* Return an rtx representing minus the value of X.
     342              :    MODE is the intended mode of the result,
     343              :    useful if X is a CONST_INT.  */
     344              : 
     345              : rtx
     346      1039149 : negate_rtx (machine_mode mode, rtx x)
     347              : {
     348      1039149 :   rtx result = simplify_unary_operation (NEG, mode, x, mode);
     349              : 
     350      1039149 :   if (result == 0)
     351         1392 :     result = expand_unop (mode, neg_optab, x, NULL_RTX, 0);
     352              : 
     353      1039149 :   return result;
     354              : }
     355              : 
     356              : /* Whether reverse storage order is supported on the target.  */
     357              : static int reverse_storage_order_supported = -1;
     358              : 
     359              : /* Check whether reverse storage order is supported on the target.  */
     360              : 
     361              : static void
     362          311 : check_reverse_storage_order_support (void)
     363              : {
     364          311 :   if (BYTES_BIG_ENDIAN != WORDS_BIG_ENDIAN)
     365              :     {
     366              :       reverse_storage_order_supported = 0;
     367              :       sorry ("reverse scalar storage order");
     368              :     }
     369              :   else
     370          311 :     reverse_storage_order_supported = 1;
     371            0 : }
     372              : 
     373              : /* Whether reverse FP storage order is supported on the target.  */
     374              : static int reverse_float_storage_order_supported = -1;
     375              : 
     376              : /* Check whether reverse FP storage order is supported on the target.  */
     377              : 
     378              : static void
     379           55 : check_reverse_float_storage_order_support (void)
     380              : {
     381           55 :   if (FLOAT_WORDS_BIG_ENDIAN != WORDS_BIG_ENDIAN)
     382              :     {
     383              :       reverse_float_storage_order_supported = 0;
     384              :       sorry ("reverse floating-point scalar storage order");
     385              :     }
     386              :   else
     387           55 :     reverse_float_storage_order_supported = 1;
     388            0 : }
     389              : 
     390              : /* Return an rtx representing value of X with reverse storage order.
     391              :    MODE is the intended mode of the result,
     392              :    useful if X is a CONST_INT.  */
     393              : 
     394              : rtx
     395         3382 : flip_storage_order (machine_mode mode, rtx x)
     396              : {
     397         3382 :   scalar_int_mode int_mode;
     398         3382 :   rtx result;
     399              : 
     400         3382 :   if (mode == QImode)
     401              :     return x;
     402              : 
     403         2495 :   if (COMPLEX_MODE_P (mode))
     404              :     {
     405           52 :       rtx real = read_complex_part (x, false);
     406           52 :       rtx imag = read_complex_part (x, true);
     407              : 
     408          104 :       real = flip_storage_order (GET_MODE_INNER (mode), real);
     409          104 :       imag = flip_storage_order (GET_MODE_INNER (mode), imag);
     410              : 
     411           52 :       return gen_rtx_CONCAT (mode, real, imag);
     412              :     }
     413              : 
     414         2443 :   if (UNLIKELY (reverse_storage_order_supported < 0))
     415          311 :     check_reverse_storage_order_support ();
     416              : 
     417         2443 :   if (!is_a <scalar_int_mode> (mode, &int_mode))
     418              :     {
     419          243 :       if (FLOAT_MODE_P (mode)
     420          243 :           && UNLIKELY (reverse_float_storage_order_supported < 0))
     421           55 :         check_reverse_float_storage_order_support ();
     422              : 
     423          243 :       if (!int_mode_for_size (GET_MODE_PRECISION (mode), 0).exists (&int_mode)
     424          243 :           || !targetm.scalar_mode_supported_p (int_mode))
     425              :         {
     426            0 :           sorry ("reverse storage order for %smode", GET_MODE_NAME (mode));
     427            0 :           return x;
     428              :         }
     429          243 :       x = gen_lowpart (int_mode, x);
     430              :     }
     431              : 
     432         2443 :   result = simplify_unary_operation (BSWAP, int_mode, x, int_mode);
     433         2443 :   if (result == 0)
     434         1099 :     result = expand_unop (int_mode, bswap_optab, x, NULL_RTX, 1);
     435              : 
     436         2443 :   if (int_mode != mode)
     437          243 :     result = gen_lowpart (mode, result);
     438              : 
     439              :   return result;
     440              : }
     441              : 
     442              : /* If MODE is set, adjust bitfield memory MEM so that it points to the
     443              :    first unit of mode MODE that contains a bitfield of size BITSIZE at
     444              :    bit position BITNUM.  If MODE is not set, return a BLKmode reference
     445              :    to every byte in the bitfield.  Set *NEW_BITNUM to the bit position
     446              :    of the field within the new memory.  */
     447              : 
     448              : static rtx
     449       425241 : narrow_bit_field_mem (rtx mem, opt_scalar_int_mode mode,
     450              :                       unsigned HOST_WIDE_INT bitsize,
     451              :                       unsigned HOST_WIDE_INT bitnum,
     452              :                       unsigned HOST_WIDE_INT *new_bitnum)
     453              : {
     454       425241 :   scalar_int_mode imode;
     455       425241 :   if (mode.exists (&imode))
     456              :     {
     457       425241 :       unsigned int unit = GET_MODE_BITSIZE (imode);
     458       425241 :       *new_bitnum = bitnum % unit;
     459       425241 :       HOST_WIDE_INT offset = (bitnum - *new_bitnum) / BITS_PER_UNIT;
     460       425241 :       return adjust_bitfield_address (mem, imode, offset);
     461              :     }
     462              :   else
     463              :     {
     464            0 :       *new_bitnum = bitnum % BITS_PER_UNIT;
     465            0 :       HOST_WIDE_INT offset = bitnum / BITS_PER_UNIT;
     466            0 :       HOST_WIDE_INT size = ((*new_bitnum + bitsize + BITS_PER_UNIT - 1)
     467            0 :                             / BITS_PER_UNIT);
     468            0 :       return adjust_bitfield_address_size (mem, BLKmode, offset, size);
     469              :     }
     470              : }
     471              : 
     472              : /* The caller wants to perform insertion or extraction PATTERN on a
     473              :    bitfield of size BITSIZE at BITNUM bits into memory operand OP0.
     474              :    BITREGION_START and BITREGION_END are as for store_bit_field
     475              :    and FIELDMODE is the natural mode of the field.
     476              : 
     477              :    Search for a mode that is compatible with the memory access
     478              :    restrictions and (where applicable) with a register insertion or
     479              :    extraction.  Return the new memory on success, storing the adjusted
     480              :    bit position in *NEW_BITNUM.  Return null otherwise.  */
     481              : 
     482              : static rtx
     483       205463 : adjust_bit_field_mem_for_reg (enum extraction_pattern pattern,
     484              :                               rtx op0, HOST_WIDE_INT bitsize,
     485              :                               HOST_WIDE_INT bitnum,
     486              :                               poly_uint64 bitregion_start,
     487              :                               poly_uint64 bitregion_end,
     488              :                               machine_mode fieldmode,
     489              :                               unsigned HOST_WIDE_INT *new_bitnum)
     490              : {
     491       410926 :   bit_field_mode_iterator iter (bitsize, bitnum, bitregion_start,
     492       410926 :                                 bitregion_end, MEM_ALIGN (op0),
     493       410926 :                                 MEM_VOLATILE_P (op0));
     494       205463 :   scalar_int_mode best_mode;
     495       205463 :   if (iter.next_mode (&best_mode))
     496              :     {
     497              :       /* We can use a memory in BEST_MODE.  See whether this is true for
     498              :          any wider modes.  All other things being equal, we prefer to
     499              :          use the widest mode possible because it tends to expose more
     500              :          CSE opportunities.  */
     501       198979 :       if (!iter.prefer_smaller_modes ())
     502              :         {
     503              :           /* Limit the search to the mode required by the corresponding
     504              :              register insertion or extraction instruction, if any.  */
     505          317 :           scalar_int_mode limit_mode = word_mode;
     506          317 :           extraction_insn insn;
     507          634 :           if (get_best_reg_extraction_insn (&insn, pattern,
     508          317 :                                             GET_MODE_BITSIZE (best_mode),
     509              :                                             fieldmode))
     510          317 :             limit_mode = insn.field_mode;
     511              : 
     512          317 :           scalar_int_mode wider_mode;
     513          317 :           while (iter.next_mode (&wider_mode)
     514         1052 :                  && GET_MODE_SIZE (wider_mode) <= GET_MODE_SIZE (limit_mode))
     515          111 :             best_mode = wider_mode;
     516              :         }
     517       198979 :       return narrow_bit_field_mem (op0, best_mode, bitsize, bitnum,
     518              :                                    new_bitnum);
     519              :     }
     520              :   return NULL_RTX;
     521              : }
     522              : 
     523              : /* Return true if a bitfield of size BITSIZE at bit number BITNUM within
     524              :    a structure of mode STRUCT_MODE represents a lowpart subreg.   The subreg
     525              :    offset is then BITNUM / BITS_PER_UNIT.  */
     526              : 
     527              : static bool
     528       836074 : lowpart_bit_field_p (poly_uint64 bitnum, poly_uint64 bitsize,
     529              :                      machine_mode struct_mode)
     530              : {
     531       836074 :   poly_uint64 regsize = REGMODE_NATURAL_SIZE (struct_mode);
     532       836074 :   if (BYTES_BIG_ENDIAN)
     533              :     return (multiple_p (bitnum, BITS_PER_UNIT)
     534              :             && (known_eq (bitnum + bitsize, GET_MODE_BITSIZE (struct_mode))
     535              :                 || multiple_p (bitnum + bitsize,
     536              :                                regsize * BITS_PER_UNIT)));
     537              :   else
     538       836074 :     return multiple_p (bitnum, regsize * BITS_PER_UNIT);
     539              : }
     540              : 
     541              : /* Return true if -fstrict-volatile-bitfields applies to an access of OP0
     542              :    containing BITSIZE bits starting at BITNUM, with field mode FIELDMODE.
     543              :    Return false if the access would touch memory outside the range
     544              :    BITREGION_START to BITREGION_END for conformance to the C++ memory
     545              :    model.  */
     546              : 
     547              : static bool
     548      1612937 : strict_volatile_bitfield_p (rtx op0, unsigned HOST_WIDE_INT bitsize,
     549              :                             unsigned HOST_WIDE_INT bitnum,
     550              :                             scalar_int_mode fieldmode,
     551              :                             poly_uint64 bitregion_start,
     552              :                             poly_uint64 bitregion_end)
     553              : {
     554      1612937 :   unsigned HOST_WIDE_INT modesize = GET_MODE_BITSIZE (fieldmode);
     555              : 
     556              :   /* -fstrict-volatile-bitfields must be enabled and we must have a
     557              :      volatile MEM.  */
     558      1612937 :   if (!MEM_P (op0)
     559       166375 :       || !MEM_VOLATILE_P (op0)
     560      1613149 :       || flag_strict_volatile_bitfields <= 0)
     561              :     return false;
     562              : 
     563              :   /* The bit size must not be larger than the field mode, and
     564              :      the field mode must not be larger than a word.  */
     565           14 :   if (bitsize > modesize || modesize > BITS_PER_WORD)
     566              :     return false;
     567              : 
     568              :   /* Check for cases of unaligned fields that must be split.  */
     569           14 :   if (bitnum % modesize + bitsize > modesize)
     570              :     return false;
     571              : 
     572              :   /* The memory must be sufficiently aligned for a MODESIZE access.
     573              :      This condition guarantees, that the memory access will not
     574              :      touch anything after the end of the structure.  */
     575           11 :   if (MEM_ALIGN (op0) < modesize)
     576              :     return false;
     577              : 
     578              :   /* Check for cases where the C++ memory model applies.  */
     579           11 :   if (maybe_ne (bitregion_end, 0U)
     580           11 :       && (maybe_lt (bitnum - bitnum % modesize, bitregion_start)
     581            4 :           || maybe_gt (bitnum - bitnum % modesize + modesize - 1,
     582              :                        bitregion_end)))
     583            0 :     return false;
     584              : 
     585              :   return true;
     586              : }
     587              : 
     588              : /* Return true if OP is a memory and if a bitfield of size BITSIZE at
     589              :    bit number BITNUM can be treated as a simple value of mode MODE.
     590              :    Store the byte offset in *BYTENUM if so.  */
     591              : 
     592              : static bool
     593       557396 : simple_mem_bitfield_p (rtx op0, poly_uint64 bitsize, poly_uint64 bitnum,
     594              :                        machine_mode mode, poly_uint64 *bytenum)
     595              : {
     596       557396 :   return (MEM_P (op0)
     597       260771 :           && multiple_p (bitnum, BITS_PER_UNIT, bytenum)
     598       208774 :           && known_eq (bitsize, GET_MODE_BITSIZE (mode))
     599       609808 :           && (!targetm.slow_unaligned_access (mode, MEM_ALIGN (op0))
     600            0 :               || (multiple_p (bitnum, GET_MODE_ALIGNMENT (mode))
     601            0 :                   && MEM_ALIGN (op0) >= GET_MODE_ALIGNMENT (mode))));
     602              : }
     603              : 
     604              : /* Try to use instruction INSV to store VALUE into a field of OP0.
     605              :    If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is a
     606              :    BLKmode MEM.  VALUE_MODE is the mode of VALUE.  BITSIZE and BITNUM
     607              :    are as for store_bit_field.  */
     608              : 
     609              : static bool
     610       110172 : store_bit_field_using_insv (const extraction_insn *insv, rtx op0,
     611              :                             opt_scalar_int_mode op0_mode,
     612              :                             unsigned HOST_WIDE_INT bitsize,
     613              :                             unsigned HOST_WIDE_INT bitnum,
     614              :                             rtx value, scalar_int_mode value_mode)
     615              : {
     616       110172 :   class expand_operand ops[4];
     617       110172 :   rtx value1;
     618       110172 :   rtx xop0 = op0;
     619       110172 :   rtx_insn *last = get_last_insn ();
     620       110172 :   bool copy_back = false;
     621              : 
     622       110172 :   scalar_int_mode op_mode = insv->field_mode;
     623       110172 :   unsigned int unit = GET_MODE_BITSIZE (op_mode);
     624       110172 :   if (bitsize == 0 || bitsize > unit)
     625              :     return false;
     626              : 
     627       110159 :   if (MEM_P (xop0))
     628              :     /* Get a reference to the first byte of the field.  */
     629            0 :     xop0 = narrow_bit_field_mem (xop0, insv->struct_mode, bitsize, bitnum,
     630              :                                  &bitnum);
     631              :   else
     632              :     {
     633              :       /* Convert from counting within OP0 to counting in OP_MODE.  */
     634       110159 :       if (BYTES_BIG_ENDIAN)
     635              :         bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
     636              : 
     637              :       /* If xop0 is a register, we need it in OP_MODE
     638              :          to make it acceptable to the format of insv.  */
     639       110159 :       if (GET_CODE (xop0) == SUBREG)
     640              :         {
     641              :           /* If such a SUBREG can't be created, give up.  */
     642        36060 :           if (!validate_subreg (op_mode, GET_MODE (SUBREG_REG (xop0)),
     643        36060 :                                 SUBREG_REG (xop0), SUBREG_BYTE (xop0)))
     644              :             return false;
     645              :           /* We can't just change the mode, because this might clobber op0,
     646              :              and we will need the original value of op0 if insv fails.  */
     647        36060 :           xop0 = gen_rtx_SUBREG (op_mode, SUBREG_REG (xop0),
     648        36060 :                                  SUBREG_BYTE (xop0));
     649              :         }
     650       110159 :       if (REG_P (xop0) && GET_MODE (xop0) != op_mode)
     651        27074 :         xop0 = gen_lowpart_SUBREG (op_mode, xop0);
     652              :     }
     653              : 
     654              :   /* If the destination is a paradoxical subreg such that we need a
     655              :      truncate to the inner mode, perform the insertion on a temporary and
     656              :      truncate the result to the original destination.  Note that we can't
     657              :      just truncate the paradoxical subreg as (truncate:N (subreg:W (reg:N
     658              :      X) 0)) is (reg:N X).  */
     659       110159 :   if (GET_CODE (xop0) == SUBREG
     660        63134 :       && REG_P (SUBREG_REG (xop0))
     661       173293 :       && !TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (SUBREG_REG (xop0)),
     662              :                                          op_mode))
     663              :     {
     664            0 :       rtx tem = gen_reg_rtx (op_mode);
     665            0 :       emit_move_insn (tem, xop0);
     666            0 :       xop0 = tem;
     667            0 :       copy_back = true;
     668              :     }
     669              : 
     670              :   /* There are similar overflow check at the start of store_bit_field_1,
     671              :      but that only check the situation where the field lies completely
     672              :      outside the register, while there do have situation where the field
     673              :      lies partially in the register, we need to adjust bitsize for this
     674              :      partial overflow situation.  Without this fix, pr48335-2.c on big-endian
     675              :      will broken on those arch support bit insert instruction, like arm, aarch64
     676              :      etc.  */
     677       110159 :   if (bitsize + bitnum > unit && bitnum < unit)
     678              :     {
     679            2 :       warning (OPT_Wextra, "write of %wu-bit data outside the bound of "
     680              :                "destination object, data truncated into %wu-bit",
     681              :                bitsize, unit - bitnum);
     682            2 :       bitsize = unit - bitnum;
     683              :     }
     684              : 
     685              :   /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
     686              :      "backwards" from the size of the unit we are inserting into.
     687              :      Otherwise, we count bits from the most significant on a
     688              :      BYTES/BITS_BIG_ENDIAN machine.  */
     689              : 
     690       110159 :   if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
     691              :     bitnum = unit - bitsize - bitnum;
     692              : 
     693              :   /* Convert VALUE to op_mode (which insv insn wants) in VALUE1.  */
     694       110159 :   value1 = value;
     695       110159 :   if (value_mode != op_mode)
     696              :     {
     697       142210 :       if (GET_MODE_BITSIZE (value_mode) >= bitsize)
     698              :         {
     699        71105 :           rtx tmp;
     700              :           /* Optimization: Don't bother really extending VALUE
     701              :              if it has all the bits we will actually use.  However,
     702              :              if we must narrow it, be sure we do it correctly.  */
     703              : 
     704       213315 :           if (GET_MODE_SIZE (value_mode) < GET_MODE_SIZE (op_mode))
     705              :             {
     706        33680 :               tmp = simplify_subreg (op_mode, value1, value_mode, 0);
     707        33680 :               if (! tmp)
     708        33093 :                 tmp = simplify_gen_subreg (op_mode,
     709              :                                            force_reg (value_mode, value1),
     710              :                                            value_mode, 0);
     711              :             }
     712              :           else
     713              :             {
     714        37425 :               if (targetm.mode_rep_extended (op_mode, value_mode) != UNKNOWN)
     715            0 :                 tmp = simplify_gen_unary (TRUNCATE, op_mode,
     716              :                                           value1, value_mode);
     717              :               else
     718              :                 {
     719        37425 :                   tmp = gen_lowpart_if_possible (op_mode, value1);
     720        37425 :                   if (! tmp)
     721            0 :                     tmp = gen_lowpart (op_mode, force_reg (value_mode, value1));
     722              :                 }
     723              :             }
     724              :           value1 = tmp;
     725              :         }
     726            0 :       else if (CONST_INT_P (value))
     727            0 :         value1 = gen_int_mode (INTVAL (value), op_mode);
     728              :       else
     729              :         /* Parse phase is supposed to make VALUE's data type
     730              :            match that of the component reference, which is a type
     731              :            at least as wide as the field; so VALUE should have
     732              :            a mode that corresponds to that type.  */
     733            0 :         gcc_assert (CONSTANT_P (value));
     734              :     }
     735              : 
     736       110159 :   create_fixed_operand (&ops[0], xop0);
     737       110159 :   create_integer_operand (&ops[1], bitsize);
     738       110159 :   create_integer_operand (&ops[2], bitnum);
     739       110159 :   create_input_operand (&ops[3], value1, op_mode);
     740       110159 :   if (maybe_expand_insn (insv->icode, 4, ops))
     741              :     {
     742         2120 :       if (copy_back)
     743            0 :         convert_move (op0, xop0, true);
     744              :       return true;
     745              :     }
     746       108039 :   delete_insns_since (last);
     747       108039 :   return false;
     748              : }
     749              : 
     750              : /* A subroutine of store_bit_field, with the same arguments.  Return true
     751              :    if the operation could be implemented.
     752              : 
     753              :    If FALLBACK_P is true, fall back to store_fixed_bit_field if we have
     754              :    no other way of implementing the operation.  If FALLBACK_P is false,
     755              :    return false instead.
     756              : 
     757              :    if UNDEFINED_P is true then STR_RTX is undefined and may be set using
     758              :    a subreg instead.  */
     759              : 
     760              : static bool
     761       906910 : store_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
     762              :                    poly_uint64 bitregion_start, poly_uint64 bitregion_end,
     763              :                    machine_mode fieldmode,
     764              :                    rtx value, bool reverse, bool fallback_p, bool undefined_p)
     765              : {
     766       906910 :   rtx op0 = str_rtx;
     767              : 
     768       906916 :   while (GET_CODE (op0) == SUBREG)
     769              :     {
     770            6 :       bitnum += subreg_memory_offset (op0) * BITS_PER_UNIT;
     771            6 :       op0 = SUBREG_REG (op0);
     772              :     }
     773              : 
     774              :   /* No action is needed if the target is a register and if the field
     775              :      lies completely outside that register.  This can occur if the source
     776              :      code contains an out-of-bounds access to a small array.  */
     777      1738505 :   if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
     778              :     return true;
     779              : 
     780              :   /* Use vec_set patterns for inserting parts of vectors whenever
     781              :      available.  */
     782       906907 :   machine_mode outermode = GET_MODE (op0);
     783       906907 :   scalar_mode innermode = GET_MODE_INNER (outermode);
     784       906907 :   poly_uint64 pos;
     785       904862 :   if (VECTOR_MODE_P (outermode)
     786         2460 :       && !MEM_P (op0)
     787         2448 :       && optab_handler (vec_set_optab, outermode) != CODE_FOR_nothing
     788         1264 :       && fieldmode == innermode
     789          959 :       && known_eq (bitsize, GET_MODE_PRECISION (innermode))
     790       907866 :       && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
     791              :     {
     792          959 :       class expand_operand ops[3];
     793          959 :       enum insn_code icode = optab_handler (vec_set_optab, outermode);
     794              : 
     795          959 :       create_fixed_operand (&ops[0], op0);
     796          959 :       create_input_operand (&ops[1], value, innermode);
     797          959 :       create_integer_operand (&ops[2], pos);
     798          959 :       if (maybe_expand_insn (icode, 3, ops))
     799          959 :         return true;
     800              :     }
     801              : 
     802              :   /* If the target is a register, overwriting the entire object, or storing
     803              :      a full-word or multi-word field can be done with just a SUBREG.  */
     804       905948 :   if (!MEM_P (op0)
     805      1736581 :       && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode)))
     806              :     {
     807              :       /* Use the subreg machinery either to narrow OP0 to the required
     808              :          words or to cope with mode punning between equal-sized modes.
     809              :          In the latter case, use subreg on the rhs side, not lhs.  */
     810       763530 :       rtx sub;
     811       763530 :       poly_uint64 bytenum;
     812       763530 :       poly_uint64 regsize = REGMODE_NATURAL_SIZE (GET_MODE (op0));
     813       763530 :       if (known_eq (bitnum, 0U)
     814      1149127 :           && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
     815              :         {
     816        61483 :           sub = force_subreg (GET_MODE (op0), value, fieldmode, 0);
     817        61483 :           if (sub)
     818              :             {
     819        61483 :               if (reverse)
     820            1 :                 sub = flip_storage_order (GET_MODE (op0), sub);
     821        61483 :               emit_move_insn (op0, sub);
     822        61483 :               return true;
     823              :             }
     824              :         }
     825       892794 :       else if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
     826       702040 :                && (undefined_p
     827       698386 :                    || (multiple_p (bitnum, regsize * BITS_PER_UNIT)
     828       686843 :                        && multiple_p (bitsize, regsize * BITS_PER_UNIT)))
     829      1307458 :                && known_ge (GET_MODE_BITSIZE (GET_MODE (op0)), bitsize))
     830              :         {
     831       653713 :           sub = simplify_gen_subreg (fieldmode, op0, GET_MODE (op0), bytenum);
     832       653713 :           if (sub)
     833              :             {
     834       653711 :               if (reverse)
     835            0 :                 value = flip_storage_order (fieldmode, value);
     836       653711 :               emit_move_insn (sub, value);
     837       653711 :               return true;
     838              :             }
     839              :         }
     840              :     }
     841              : 
     842              :   /* If the target is memory, storing any naturally aligned field can be
     843              :      done with a simple store.  For targets that support fast unaligned
     844              :      memory, any naturally sized, unit aligned field can be done directly.  */
     845       190754 :   poly_uint64 bytenum;
     846       190754 :   if (simple_mem_bitfield_p (op0, bitsize, bitnum, fieldmode, &bytenum))
     847              :     {
     848         7037 :       op0 = adjust_bitfield_address (op0, fieldmode, bytenum);
     849         7037 :       if (reverse)
     850            0 :         value = flip_storage_order (fieldmode, value);
     851         7037 :       emit_move_insn (op0, value);
     852         7037 :       return true;
     853              :     }
     854              : 
     855              :   /* It's possible we'll need to handle other cases here for
     856              :      polynomial bitnum and bitsize.  */
     857              : 
     858              :   /* From here on we need to be looking at a fixed-size insertion.  */
     859       183717 :   unsigned HOST_WIDE_INT ibitsize = bitsize.to_constant ();
     860       183717 :   unsigned HOST_WIDE_INT ibitnum = bitnum.to_constant ();
     861              : 
     862              :   /* Make sure we are playing with integral modes.  Pun with subregs
     863              :      if we aren't.  This must come after the entire register case above,
     864              :      since that case is valid for any mode.  The following cases are only
     865              :      valid for integral modes.  */
     866       183717 :   opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
     867       183717 :   scalar_int_mode imode;
     868       183717 :   bool need_stack_p = false;
     869       183717 :   if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
     870              :     {
     871        19916 :       if (MEM_P (op0))
     872        15257 :         op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
     873              :                                             0, MEM_SIZE (op0));
     874         4659 :       else if (!op0_mode.exists ()
     875         4659 :                || maybe_lt
     876         4799 :                ((unsigned) UNITS_PER_WORD,
     877         4659 :                 (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
     878              :         {
     879            0 :           if (ibitnum == 0
     880            0 :               && known_eq (ibitsize, GET_MODE_BITSIZE (GET_MODE (op0)))
     881            0 :               && MEM_P (value)
     882            0 :               && !reverse)
     883              :             {
     884            0 :               value = adjust_address (value, GET_MODE (op0), 0);
     885            0 :               emit_move_insn (op0, value);
     886            0 :               return true;
     887              :             }
     888            0 :           if (!fallback_p)
     889              :             return false;
     890              :           need_stack_p = true;
     891              :         }
     892              :       else
     893         4659 :         op0 = gen_lowpart (op0_mode.require (), op0);
     894              :     }
     895       163801 :   else if (!MEM_P (op0)
     896       275924 :            && maybe_lt ((unsigned) UNITS_PER_WORD,
     897       110780 :                         (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
     898              :     need_stack_p = true;
     899              : 
     900              :   /* With or without punning we might be faced with a mode that we cannot
     901              :      split into words.  If so, spill OP0 to the stack and recurse.
     902              :      This happens at most once.  */
     903        19916 :   if (need_stack_p)
     904              :     {
     905            0 :       rtx temp = assign_stack_temp (GET_MODE (op0),
     906            0 :                                     GET_MODE_SIZE (GET_MODE (op0)));
     907            0 :       emit_move_insn (temp, op0);
     908            0 :       store_bit_field_1 (temp, bitsize, bitnum, 0, 0, fieldmode, value,
     909              :                          reverse, fallback_p, undefined_p);
     910            0 :       emit_move_insn (op0, temp);
     911            0 :       return true;
     912              :     }
     913              : 
     914       183717 :   return store_integral_bit_field (op0, op0_mode, ibitsize, ibitnum,
     915              :                                    bitregion_start, bitregion_end,
     916       183717 :                                    fieldmode, value, reverse, fallback_p);
     917              : }
     918              : 
     919              : /* Subroutine of store_bit_field_1, with the same arguments, except
     920              :    that BITSIZE and BITNUM are constant.  Handle cases specific to
     921              :    integral modes.  If OP0_MODE is defined, it is the mode of OP0,
     922              :    otherwise OP0 is a BLKmode MEM.  */
     923              : 
     924              : static bool
     925       183717 : store_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
     926              :                           unsigned HOST_WIDE_INT bitsize,
     927              :                           unsigned HOST_WIDE_INT bitnum,
     928              :                           poly_uint64 bitregion_start,
     929              :                           poly_uint64 bitregion_end,
     930              :                           machine_mode fieldmode,
     931              :                           rtx value, bool reverse, bool fallback_p)
     932              : {
     933              :   /* Storing an lsb-aligned field in a register
     934              :      can be done with a movstrict instruction.  */
     935              : 
     936       183717 :   if (!MEM_P (op0)
     937       115439 :       && !reverse
     938       378088 :       && lowpart_bit_field_p (bitnum, bitsize, op0_mode.require ())
     939        83404 :       && known_eq (bitsize, GET_MODE_BITSIZE (fieldmode))
     940       220503 :       && optab_handler (movstrict_optab, fieldmode) != CODE_FOR_nothing)
     941              :     {
     942         4445 :       class expand_operand ops[2];
     943         4445 :       enum insn_code icode = optab_handler (movstrict_optab, fieldmode);
     944         4445 :       rtx arg0 = op0;
     945         4445 :       unsigned HOST_WIDE_INT subreg_off;
     946              : 
     947         4445 :       if (GET_CODE (arg0) == SUBREG)
     948              :         {
     949              :           /* Else we've got some float mode source being extracted into
     950              :              a different float mode destination -- this combination of
     951              :              subregs results in Severe Tire Damage.  */
     952          459 :           gcc_assert (GET_MODE (SUBREG_REG (arg0)) == fieldmode
     953              :                       || GET_MODE_CLASS (fieldmode) == MODE_INT
     954              :                       || GET_MODE_CLASS (fieldmode) == MODE_PARTIAL_INT);
     955              :           arg0 = SUBREG_REG (arg0);
     956              :         }
     957              : 
     958         4445 :       subreg_off = bitnum / BITS_PER_UNIT;
     959         4479 :       if (validate_subreg (fieldmode, GET_MODE (arg0), arg0, subreg_off)
     960              :           /* STRICT_LOW_PART must have a non-paradoxical subreg as
     961              :              operand.  */
     962         4445 :           && !paradoxical_subreg_p (fieldmode, GET_MODE (arg0)))
     963              :         {
     964         4411 :           arg0 = gen_rtx_SUBREG (fieldmode, arg0, subreg_off);
     965              : 
     966         4411 :           create_fixed_operand (&ops[0], arg0);
     967              :           /* Shrink the source operand to FIELDMODE.  */
     968         4411 :           create_convert_operand_to (&ops[1], value, fieldmode, false);
     969         4411 :           if (maybe_expand_insn (icode, 2, ops))
     970         4410 :             return true;
     971              :         }
     972              :     }
     973              : 
     974              :   /* Handle fields bigger than a word.  */
     975              : 
     976       180896 :   if (bitsize > BITS_PER_WORD)
     977              :     {
     978              :       /* Here we transfer the words of the field
     979              :          in the order least significant first.
     980              :          This is because the most significant word is the one which may
     981              :          be less than full.
     982              :          However, only do that if the value is not BLKmode.  */
     983              : 
     984          918 :       const bool backwards = WORDS_BIG_ENDIAN && fieldmode != BLKmode;
     985          918 :       const int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
     986          918 :       rtx_insn *last;
     987              : 
     988              :       /* This is the mode we must force value to, so that there will be enough
     989              :          subwords to extract.  Note that fieldmode will often (always?) be
     990              :          VOIDmode, because that is what store_field uses to indicate that this
     991              :          is a bit field, but passing VOIDmode to operand_subword_force
     992              :          is not allowed.
     993              : 
     994              :          The mode must be fixed-size, since insertions into variable-sized
     995              :          objects are meant to be handled before calling this function.  */
     996          918 :       fixed_size_mode value_mode = as_a <fixed_size_mode> (GET_MODE (value));
     997          918 :       if (value_mode == VOIDmode)
     998           24 :         value_mode
     999           24 :           = smallest_int_mode_for_size (nwords * BITS_PER_WORD).require ();
    1000              : 
    1001          918 :       last = get_last_insn ();
    1002         2749 :       for (int i = 0; i < nwords; i++)
    1003              :         {
    1004              :           /* Number of bits to be stored in this iteration, i.e. BITS_PER_WORD
    1005              :              except maybe for the last iteration.  */
    1006         3672 :           const unsigned HOST_WIDE_INT new_bitsize
    1007         1872 :             = MIN (BITS_PER_WORD, bitsize - i * BITS_PER_WORD);
    1008              :           /* Bit offset from the starting bit number in the target.  */
    1009         3684 :           const unsigned int bit_offset
    1010              :             = backwards ^ reverse
    1011         1836 :               ? MAX ((int) bitsize - (i + 1) * BITS_PER_WORD, 0)
    1012              :               : i * BITS_PER_WORD;
    1013              : 
    1014              :           /* No further action is needed if the target is a register and if
    1015              :              this field lies completely outside that register.  */
    1016         2110 :           if (REG_P (op0) && known_ge (bitnum + bit_offset,
    1017              :                                        GET_MODE_BITSIZE (GET_MODE (op0))))
    1018              :             {
    1019            5 :               if (backwards ^ reverse)
    1020            0 :                 continue;
    1021              :               /* For forward operation we are finished.  */
    1022       183717 :               return true;
    1023              :             }
    1024              : 
    1025              :           /* Starting word number in the value.  */
    1026         1831 :           const unsigned int wordnum
    1027              :             = backwards
    1028         1831 :               ? GET_MODE_SIZE (value_mode) / UNITS_PER_WORD - (i + 1)
    1029              :               : i;
    1030              :           /* The chunk of the value in word_mode.  We use bit-field extraction
    1031              :               in BLKmode to handle unaligned memory references and to shift the
    1032              :               last chunk right on big-endian machines if need be.  */
    1033         1831 :           rtx value_word
    1034              :             = (fieldmode == BLKmode
    1035         1580 :                || maybe_lt ((unsigned) UNITS_PER_WORD,
    1036         1580 :                             (poly_uint64) REGMODE_NATURAL_SIZE (value_mode)))
    1037         1867 :               ? extract_bit_field (value, new_bitsize, wordnum * BITS_PER_WORD,
    1038              :                                    1, NULL_RTX, word_mode, word_mode, false,
    1039              :                                    NULL)
    1040         1831 :               : operand_subword_force (value, wordnum, value_mode);
    1041              : 
    1042         1831 :           if (!store_bit_field_1 (op0, new_bitsize,
    1043         1831 :                                   bitnum + bit_offset,
    1044              :                                   bitregion_start, bitregion_end,
    1045              :                                   word_mode,
    1046              :                                   value_word, reverse, fallback_p, false))
    1047              :             {
    1048            0 :               delete_insns_since (last);
    1049            0 :               return false;
    1050              :             }
    1051              :         }
    1052              :       return true;
    1053              :     }
    1054              : 
    1055              :   /* If VALUE has a floating-point or complex mode, access it as an
    1056              :      integer of the corresponding size.  This can occur on a machine
    1057              :      with 64 bit registers that uses SFmode for float.  It can also
    1058              :      occur for unaligned float or complex fields.  */
    1059       178389 :   rtx orig_value = value;
    1060       178389 :   scalar_int_mode value_mode;
    1061       178389 :   if (GET_MODE (value) == VOIDmode)
    1062              :     /* By this point we've dealt with values that are bigger than a word,
    1063              :        so word_mode is a conservatively correct choice.  */
    1064       105884 :     value_mode = word_mode;
    1065        72505 :   else if (!is_a <scalar_int_mode> (GET_MODE (value), &value_mode))
    1066              :     {
    1067         1141 :       value_mode = int_mode_for_mode (GET_MODE (value)).require ();
    1068         1141 :       value = gen_reg_rtx (value_mode);
    1069         1141 :       emit_move_insn (gen_lowpart (GET_MODE (orig_value), value), orig_value);
    1070              :     }
    1071              : 
    1072              :   /* If OP0 is a multi-word register, narrow it to the affected word.
    1073              :      If the region spans two words, defer to store_split_bit_field.
    1074              :      Don't do this if op0 is a single hard register wider than word
    1075              :      such as a float or vector register.  */
    1076       178389 :   if (!MEM_P (op0)
    1077       222685 :       && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD
    1078       214510 :       && (!REG_P (op0)
    1079        36100 :           || !HARD_REGISTER_P (op0)
    1080       178389 :           || hard_regno_nregs (REGNO (op0), op0_mode.require ()) != 1))
    1081              :     {
    1082        36194 :       if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
    1083              :         {
    1084          698 :           if (!fallback_p)
    1085              :             return false;
    1086              : 
    1087           71 :           store_split_bit_field (op0, op0_mode, bitsize, bitnum,
    1088              :                                  bitregion_start, bitregion_end,
    1089              :                                  value, value_mode, reverse);
    1090           71 :           return true;
    1091              :         }
    1092        35421 :       rtx new_op0
    1093        35421 :         = simplify_gen_subreg (word_mode, op0, op0_mode.require (),
    1094        35496 :                                bitnum / BITS_PER_WORD * UNITS_PER_WORD);
    1095        35421 :       if (!new_op0)
    1096              :         {
    1097              :           /* No valid word-mode SUBREG of op0 at this offset.  Defer to
    1098              :              store_split_bit_field, which addresses op0 a word at a time.  */
    1099            0 :           if (!fallback_p)
    1100              :             return false;
    1101            0 :           store_split_bit_field (op0, op0_mode, bitsize, bitnum,
    1102              :                                  bitregion_start, bitregion_end,
    1103              :                                  value, value_mode, reverse);
    1104            0 :           return true;
    1105              :         }
    1106        35421 :       op0 = new_op0;
    1107        35421 :       op0_mode = word_mode;
    1108        35496 :       bitnum %= BITS_PER_WORD;
    1109              :     }
    1110              : 
    1111              :   /* From here on we can assume that the field to be stored in fits
    1112              :      within a word.  If the destination is a register, it too fits
    1113              :      in a word.  */
    1114              : 
    1115       177691 :   extraction_insn insv;
    1116       177691 :   if (!MEM_P (op0)
    1117       110191 :       && !reverse
    1118       110174 :       && get_best_reg_extraction_insn (&insv, EP_insv,
    1119       220348 :                                        GET_MODE_BITSIZE (op0_mode.require ()),
    1120              :                                        fieldmode)
    1121       287863 :       && store_bit_field_using_insv (&insv, op0, op0_mode,
    1122              :                                      bitsize, bitnum, value, value_mode))
    1123         2120 :     return true;
    1124              : 
    1125              :   /* If OP0 is a memory, try copying it to a register and seeing if a
    1126              :      cheap register alternative is available.  */
    1127       175571 :   if (MEM_P (op0) && !reverse)
    1128              :     {
    1129        67098 :       if (get_best_mem_extraction_insn (&insv, EP_insv, bitsize, bitnum,
    1130              :                                         fieldmode)
    1131        67098 :           && store_bit_field_using_insv (&insv, op0, op0_mode,
    1132              :                                          bitsize, bitnum, value, value_mode))
    1133            0 :         return true;
    1134              : 
    1135        67098 :       rtx_insn *last = get_last_insn ();
    1136              : 
    1137              :       /* Try loading part of OP0 into a register, inserting the bitfield
    1138              :          into that, and then copying the result back to OP0.  */
    1139        67098 :       unsigned HOST_WIDE_INT bitpos;
    1140        67098 :       rtx xop0 = adjust_bit_field_mem_for_reg (EP_insv, op0, bitsize, bitnum,
    1141              :                                                bitregion_start, bitregion_end,
    1142              :                                                fieldmode, &bitpos);
    1143        67098 :       if (xop0)
    1144              :         {
    1145        63261 :           rtx tempreg = copy_to_reg (xop0);
    1146        63261 :           if (store_bit_field_1 (tempreg, bitsize, bitpos,
    1147              :                                  bitregion_start, bitregion_end,
    1148              :                                  fieldmode, orig_value, reverse, false, false))
    1149              :             {
    1150            0 :               emit_move_insn (xop0, tempreg);
    1151            0 :               return true;
    1152              :             }
    1153        63261 :           delete_insns_since (last);
    1154              :         }
    1155              :     }
    1156              : 
    1157       175571 :   if (!fallback_p)
    1158              :     return false;
    1159              : 
    1160       112937 :   store_fixed_bit_field (op0, op0_mode, bitsize, bitnum, bitregion_start,
    1161              :                          bitregion_end, value, value_mode, reverse);
    1162       112937 :   return true;
    1163              : }
    1164              : 
    1165              : /* Generate code to store value from rtx VALUE
    1166              :    into a bit-field within structure STR_RTX
    1167              :    containing BITSIZE bits starting at bit BITNUM.
    1168              : 
    1169              :    BITREGION_START is bitpos of the first bitfield in this region.
    1170              :    BITREGION_END is the bitpos of the ending bitfield in this region.
    1171              :    These two fields are 0, if the C++ memory model does not apply,
    1172              :    or we are not interested in keeping track of bitfield regions.
    1173              : 
    1174              :    FIELDMODE is the machine-mode of the FIELD_DECL node for this field.
    1175              : 
    1176              :    If REVERSE is true, the store is to be done in reverse order.
    1177              : 
    1178              :    If UNDEFINED_P is true then STR_RTX is currently undefined.  */
    1179              : 
    1180              : void
    1181       841818 : store_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
    1182              :                  poly_uint64 bitregion_start, poly_uint64 bitregion_end,
    1183              :                  machine_mode fieldmode,
    1184              :                  rtx value, bool reverse, bool undefined_p)
    1185              : {
    1186              :   /* Handle -fstrict-volatile-bitfields in the cases where it applies.  */
    1187       841818 :   unsigned HOST_WIDE_INT ibitsize = 0, ibitnum = 0;
    1188       841818 :   scalar_int_mode int_mode;
    1189       841818 :   if (bitsize.is_constant (&ibitsize)
    1190       841818 :       && bitnum.is_constant (&ibitnum)
    1191      1589162 :       && is_a <scalar_int_mode> (fieldmode, &int_mode)
    1192       747348 :       && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum, int_mode,
    1193              :                                      bitregion_start, bitregion_end))
    1194              :     {
    1195              :       /* Storing of a full word can be done with a simple store.
    1196              :          We know here that the field can be accessed with one single
    1197              :          instruction.  For targets that support unaligned memory,
    1198              :          an unaligned access may be necessary.  */
    1199            8 :       if (ibitsize == GET_MODE_BITSIZE (int_mode))
    1200              :         {
    1201            0 :           str_rtx = adjust_bitfield_address (str_rtx, int_mode,
    1202              :                                              ibitnum / BITS_PER_UNIT);
    1203            0 :           if (reverse)
    1204            0 :             value = flip_storage_order (int_mode, value);
    1205            0 :           gcc_assert (ibitnum % BITS_PER_UNIT == 0);
    1206            0 :           emit_move_insn (str_rtx, value);
    1207              :         }
    1208              :       else
    1209              :         {
    1210            4 :           rtx temp;
    1211              : 
    1212            4 :           str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize,
    1213              :                                           ibitnum, &ibitnum);
    1214            8 :           gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
    1215            4 :           temp = copy_to_reg (str_rtx);
    1216            4 :           if (!store_bit_field_1 (temp, ibitsize, ibitnum, 0, 0,
    1217              :                                   int_mode, value, reverse, true, undefined_p))
    1218            0 :             gcc_unreachable ();
    1219              : 
    1220            4 :           emit_move_insn (str_rtx, temp);
    1221              :         }
    1222              : 
    1223            4 :       return;
    1224              :     }
    1225              : 
    1226              :   /* Under the C++0x memory model, we must not touch bits outside the
    1227              :      bit region.  Adjust the address to start at the beginning of the
    1228              :      bit region.  */
    1229       841814 :   if (MEM_P (str_rtx) && maybe_ne (bitregion_start, 0U))
    1230              :     {
    1231        51185 :       scalar_int_mode best_mode;
    1232        51185 :       machine_mode addr_mode = VOIDmode;
    1233              : 
    1234        51185 :       poly_uint64 offset = exact_div (bitregion_start, BITS_PER_UNIT);
    1235        51185 :       bitnum -= bitregion_start;
    1236        51185 :       poly_int64 size = bits_to_bytes_round_up (bitnum + bitsize);
    1237        51185 :       bitregion_end -= bitregion_start;
    1238        51185 :       bitregion_start = 0;
    1239        51185 :       if (bitsize.is_constant (&ibitsize)
    1240        51185 :           && bitnum.is_constant (&ibitnum)
    1241        51185 :           && get_best_mode (ibitsize, ibitnum,
    1242              :                             bitregion_start, bitregion_end,
    1243        51185 :                             MEM_ALIGN (str_rtx), INT_MAX,
    1244        51185 :                             MEM_VOLATILE_P (str_rtx), &best_mode))
    1245        47771 :         addr_mode = best_mode;
    1246        51185 :       str_rtx = adjust_bitfield_address_size (str_rtx, addr_mode,
    1247              :                                               offset, size);
    1248              :     }
    1249              : 
    1250       841814 :   if (!store_bit_field_1 (str_rtx, bitsize, bitnum,
    1251              :                           bitregion_start, bitregion_end,
    1252              :                           fieldmode, value, reverse, true, undefined_p))
    1253            0 :     gcc_unreachable ();
    1254              : }
    1255              : 
    1256              : /* Use shifts and boolean operations to store VALUE into a bit field of
    1257              :    width BITSIZE in OP0, starting at bit BITNUM.  If OP0_MODE is defined,
    1258              :    it is the mode of OP0, otherwise OP0 is a BLKmode MEM.  VALUE_MODE is
    1259              :    the mode of VALUE.
    1260              : 
    1261              :    If REVERSE is true, the store is to be done in reverse order.  */
    1262              : 
    1263              : static void
    1264       130436 : store_fixed_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
    1265              :                        unsigned HOST_WIDE_INT bitsize,
    1266              :                        unsigned HOST_WIDE_INT bitnum,
    1267              :                        poly_uint64 bitregion_start, poly_uint64 bitregion_end,
    1268              :                        rtx value, scalar_int_mode value_mode, bool reverse)
    1269              : {
    1270              :   /* There is a case not handled here:
    1271              :      a structure with a known alignment of just a halfword
    1272              :      and a field split across two aligned halfwords within the structure.
    1273              :      Or likewise a structure with a known alignment of just a byte
    1274              :      and a field split across two bytes.
    1275              :      Such cases are not supposed to be able to occur.  */
    1276              : 
    1277       130436 :   scalar_int_mode best_mode;
    1278       130436 :   if (MEM_P (op0))
    1279              :     {
    1280        84857 :       unsigned int max_bitsize = BITS_PER_WORD;
    1281        84857 :       scalar_int_mode imode;
    1282       144884 :       if (op0_mode.exists (&imode) && GET_MODE_BITSIZE (imode) < max_bitsize)
    1283        91150 :         max_bitsize = GET_MODE_BITSIZE (imode);
    1284              : 
    1285        84857 :       if (!get_best_mode (bitsize, bitnum, bitregion_start, bitregion_end,
    1286        84857 :                           MEM_ALIGN (op0), max_bitsize, MEM_VOLATILE_P (op0),
    1287              :                           &best_mode))
    1288              :         {
    1289              :           /* The only way this should occur is if the field spans word
    1290              :              boundaries.  */
    1291         6569 :           store_split_bit_field (op0, op0_mode, bitsize, bitnum,
    1292              :                                  bitregion_start, bitregion_end,
    1293              :                                  value, value_mode, reverse);
    1294         6569 :           return;
    1295              :         }
    1296              : 
    1297        78288 :       op0 = narrow_bit_field_mem (op0, best_mode, bitsize, bitnum, &bitnum);
    1298              :     }
    1299              :   else
    1300        45579 :     best_mode = op0_mode.require ();
    1301              : 
    1302       123867 :   store_fixed_bit_field_1 (op0, best_mode, bitsize, bitnum,
    1303              :                            value, value_mode, reverse);
    1304              : }
    1305              : 
    1306              : /* Helper function for store_fixed_bit_field, stores
    1307              :    the bit field always using MODE, which is the mode of OP0.  The other
    1308              :    arguments are as for store_fixed_bit_field.  */
    1309              : 
    1310              : static void
    1311       123867 : store_fixed_bit_field_1 (rtx op0, scalar_int_mode mode,
    1312              :                          unsigned HOST_WIDE_INT bitsize,
    1313              :                          unsigned HOST_WIDE_INT bitnum,
    1314              :                          rtx value, scalar_int_mode value_mode, bool reverse)
    1315              : {
    1316       123867 :   rtx temp;
    1317       123867 :   int all_zero = 0;
    1318       123867 :   int all_one = 0;
    1319              : 
    1320              :   /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
    1321              :      for invalid input, such as f5 from gcc.dg/pr48335-2.c.  */
    1322              : 
    1323       123867 :   if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    1324              :     /* BITNUM is the distance between our msb
    1325              :        and that of the containing datum.
    1326              :        Convert it to the distance from the lsb.  */
    1327         1062 :     bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
    1328              : 
    1329              :   /* Now BITNUM is always the distance between our lsb
    1330              :      and that of OP0.  */
    1331              : 
    1332              :   /* Shift VALUE left by BITNUM bits.  If VALUE is not constant,
    1333              :      we must first convert its mode to MODE.  */
    1334              : 
    1335       123867 :   if (CONST_INT_P (value))
    1336              :     {
    1337        73369 :       unsigned HOST_WIDE_INT v = UINTVAL (value);
    1338              : 
    1339        73369 :       if (bitsize < HOST_BITS_PER_WIDE_INT)
    1340        73352 :         v &= (HOST_WIDE_INT_1U << bitsize) - 1;
    1341              : 
    1342        73369 :       if (v == 0)
    1343              :         all_zero = 1;
    1344        59700 :       else if ((bitsize < HOST_BITS_PER_WIDE_INT
    1345        59692 :                 && v == (HOST_WIDE_INT_1U << bitsize) - 1)
    1346        51026 :                || (bitsize == HOST_BITS_PER_WIDE_INT
    1347        51026 :                    && v == HOST_WIDE_INT_M1U))
    1348         8674 :         all_one = 1;
    1349              : 
    1350        73369 :       value = lshift_value (mode, v, bitnum);
    1351              :     }
    1352              :   else
    1353              :     {
    1354        50498 :       int must_and = (GET_MODE_BITSIZE (value_mode) != bitsize
    1355        78000 :                       && bitnum + bitsize != GET_MODE_BITSIZE (mode));
    1356              : 
    1357        50498 :       if (value_mode != mode)
    1358        27913 :         value = convert_to_mode (mode, value, 1);
    1359              : 
    1360        50498 :       if (must_and)
    1361        20698 :         value = expand_binop (mode, and_optab, value,
    1362              :                               mask_rtx (mode, 0, bitsize, 0),
    1363              :                               NULL_RTX, 1, OPTAB_LIB_WIDEN);
    1364        50498 :       if (bitnum > 0)
    1365        14320 :         value = expand_shift (LSHIFT_EXPR, mode, value,
    1366        14320 :                               bitnum, NULL_RTX, 1);
    1367              :     }
    1368              : 
    1369       123867 :   if (reverse)
    1370          531 :     value = flip_storage_order (mode, value);
    1371              : 
    1372              :   /* A field filling the whole of a MEM has no surrounding bits to preserve,
    1373              :      so store it directly: the read of a volatile OP0 cannot be removed later
    1374              :      and would be a spurious access with side effects (PR71048).  */
    1375       183489 :   if (MEM_P (op0) && bitnum == 0 && bitsize == GET_MODE_BITSIZE (mode))
    1376              :     {
    1377        13641 :       op0 = copy_rtx (op0);
    1378        13641 :       emit_move_insn (op0, value);
    1379        13641 :       return;
    1380              :     }
    1381              : 
    1382              :   /* Now clear the chosen bits in OP0,
    1383              :      except that if VALUE is -1 we need not bother.  */
    1384              :   /* We keep the intermediates in registers to allow CSE to combine
    1385              :      consecutive bitfield assignments.  */
    1386              : 
    1387       110226 :   temp = force_reg (mode, op0);
    1388              : 
    1389       110226 :   if (! all_one)
    1390              :     {
    1391       102243 :       rtx mask = mask_rtx (mode, bitnum, bitsize, 1);
    1392       102243 :       if (reverse)
    1393          519 :         mask = flip_storage_order (mode, mask);
    1394       102243 :       temp = expand_binop (mode, and_optab, temp, mask,
    1395              :                            NULL_RTX, 1, OPTAB_LIB_WIDEN);
    1396       102243 :       temp = force_reg (mode, temp);
    1397              :     }
    1398              : 
    1399              :   /* Now logical-or VALUE into OP0, unless it is zero.  */
    1400              : 
    1401       110226 :   if (! all_zero)
    1402              :     {
    1403              :       /* Use aop_optab if the field has been cleared.  */
    1404       188699 :       temp = expand_binop (mode, all_one ? ior_optab : aop_optab,
    1405              :                            temp, value, NULL_RTX, 1, OPTAB_LIB_WIDEN);
    1406        98341 :       temp = force_reg (mode, temp);
    1407              :     }
    1408              : 
    1409       110226 :   if (op0 != temp)
    1410              :     {
    1411       110226 :       op0 = copy_rtx (op0);
    1412       110226 :       emit_move_insn (op0, temp);
    1413              :     }
    1414              : }
    1415              : 
    1416              : /* Store a bit field that is split across multiple accessible memory objects.
    1417              : 
    1418              :    OP0 is the REG, SUBREG or MEM rtx for the first of the objects.
    1419              :    BITSIZE is the field width; BITPOS the position of its first bit
    1420              :    (within the word).
    1421              :    VALUE is the value to store, which has mode VALUE_MODE.
    1422              :    If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
    1423              :    a BLKmode MEM.
    1424              : 
    1425              :    If REVERSE is true, the store is to be done in reverse order.
    1426              : 
    1427              :    This does not yet handle fields wider than BITS_PER_WORD.  */
    1428              : 
    1429              : static void
    1430         6640 : store_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
    1431              :                        unsigned HOST_WIDE_INT bitsize,
    1432              :                        unsigned HOST_WIDE_INT bitpos,
    1433              :                        poly_uint64 bitregion_start, poly_uint64 bitregion_end,
    1434              :                        rtx value, scalar_int_mode value_mode, bool reverse)
    1435              : {
    1436         6640 :   unsigned int unit, total_bits, bitsdone = 0;
    1437              : 
    1438              :   /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
    1439              :      much at a time.  */
    1440         6640 :   if (REG_P (op0) || GET_CODE (op0) == SUBREG)
    1441           71 :     unit = BITS_PER_WORD;
    1442              :   else
    1443         6569 :     unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
    1444              : 
    1445              :   /* If OP0 is a memory with a mode, then UNIT must not be larger than
    1446              :      OP0's mode as well.  Otherwise, store_fixed_bit_field will call us
    1447              :      again, and we will mutually recurse forever.  */
    1448         6640 :   if (MEM_P (op0) && op0_mode.exists ())
    1449         5111 :     unit = MIN (unit, GET_MODE_BITSIZE (op0_mode.require ()));
    1450              : 
    1451              :   /* If VALUE is a constant other than a CONST_INT, get it into a register in
    1452              :      WORD_MODE.  If we can do this using gen_lowpart_common, do so.  Note
    1453              :      that VALUE might be a floating-point constant.  */
    1454         6640 :   if (CONSTANT_P (value) && !CONST_INT_P (value))
    1455              :     {
    1456            0 :       rtx word = gen_lowpart_common (word_mode, value);
    1457              : 
    1458            0 :       if (word && (value != word))
    1459              :         value = word;
    1460              :       else
    1461            0 :         value = gen_lowpart_common (word_mode, force_reg (value_mode, value));
    1462            0 :       value_mode = word_mode;
    1463              :     }
    1464              : 
    1465         6640 :   total_bits = GET_MODE_BITSIZE (value_mode);
    1466              : 
    1467        30943 :   while (bitsdone < bitsize)
    1468              :     {
    1469        24303 :       unsigned HOST_WIDE_INT thissize;
    1470        24303 :       unsigned HOST_WIDE_INT thispos;
    1471        24303 :       unsigned HOST_WIDE_INT offset;
    1472        24303 :       rtx part;
    1473              : 
    1474        24303 :       offset = (bitpos + bitsdone) / unit;
    1475        24303 :       thispos = (bitpos + bitsdone) % unit;
    1476              : 
    1477              :       /* When region of bytes we can touch is restricted, decrease
    1478              :          UNIT close to the end of the region as needed.  If op0 is a REG
    1479              :          or SUBREG of REG, don't do this, as there can't be data races
    1480              :          on a register and we can expand shorter code in some cases.  */
    1481        31107 :       if (maybe_ne (bitregion_end, 0U)
    1482        24303 :           && unit > BITS_PER_UNIT
    1483        14219 :           && maybe_gt (bitpos + bitsdone - thispos + unit, bitregion_end + 1)
    1484         6868 :           && !REG_P (op0)
    1485        31107 :           && (GET_CODE (op0) != SUBREG || !REG_P (SUBREG_REG (op0))))
    1486              :         {
    1487         6804 :           unit = unit / 2;
    1488         6804 :           continue;
    1489              :         }
    1490              : 
    1491              :       /* THISSIZE must not overrun a word boundary.  Otherwise,
    1492              :          store_fixed_bit_field will call us again, and we will mutually
    1493              :          recurse forever.  */
    1494        17499 :       thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
    1495        17499 :       thissize = MIN (thissize, unit - thispos);
    1496              : 
    1497        17499 :       if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    1498              :         {
    1499              :           /* Fetch successively less significant portions.  */
    1500          214 :           if (CONST_INT_P (value))
    1501          108 :             part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
    1502              :                              >> (bitsize - bitsdone - thissize))
    1503              :                             & ((HOST_WIDE_INT_1 << thissize) - 1));
    1504              :           /* Likewise, but the source is little-endian.  */
    1505          106 :           else if (reverse)
    1506          106 :             part = extract_fixed_bit_field (word_mode, value, value_mode,
    1507              :                                             thissize,
    1508              :                                             bitsize - bitsdone - thissize,
    1509              :                                             NULL_RTX, 1, false);
    1510              :           else
    1511              :             /* The args are chosen so that the last part includes the
    1512              :                lsb.  Give extract_bit_field the value it needs (with
    1513              :                endianness compensation) to fetch the piece we want.  */
    1514              :             part = extract_fixed_bit_field (word_mode, value, value_mode,
    1515              :                                             thissize,
    1516              :                                             total_bits - bitsize + bitsdone,
    1517              :                                             NULL_RTX, 1, false);
    1518              :         }
    1519              :       else
    1520              :         {
    1521              :           /* Fetch successively more significant portions.  */
    1522        17285 :           if (CONST_INT_P (value))
    1523        12867 :             part = GEN_INT (((unsigned HOST_WIDE_INT) (INTVAL (value))
    1524              :                              >> bitsdone)
    1525              :                             & ((HOST_WIDE_INT_1 << thissize) - 1));
    1526              :           /* Likewise, but the source is big-endian.  */
    1527         4418 :           else if (reverse)
    1528              :             part = extract_fixed_bit_field (word_mode, value, value_mode,
    1529              :                                             thissize,
    1530              :                                             total_bits - bitsdone - thissize,
    1531              :                                             NULL_RTX, 1, false);
    1532              :           else
    1533         4418 :             part = extract_fixed_bit_field (word_mode, value, value_mode,
    1534              :                                             thissize, bitsdone, NULL_RTX,
    1535              :                                             1, false);
    1536              :         }
    1537              : 
    1538              :       /* If OP0 is a register, then handle OFFSET here.  */
    1539        17499 :       rtx op0_piece = op0;
    1540        17499 :       opt_scalar_int_mode op0_piece_mode = op0_mode;
    1541        17499 :       if (SUBREG_P (op0) || REG_P (op0))
    1542              :         {
    1543          142 :           scalar_int_mode imode;
    1544          142 :           if (op0_mode.exists (&imode)
    1545          142 :               && GET_MODE_SIZE (imode) < UNITS_PER_WORD)
    1546              :             {
    1547            0 :               if (offset)
    1548            0 :                 op0_piece = const0_rtx;
    1549              :             }
    1550              :           else
    1551              :             {
    1552          142 :               op0_piece = operand_subword_force (op0,
    1553          142 :                                                  offset * unit / BITS_PER_WORD,
    1554          142 :                                                  GET_MODE (op0));
    1555          142 :               op0_piece_mode = word_mode;
    1556              :             }
    1557          142 :           offset &= BITS_PER_WORD / unit - 1;
    1558              :         }
    1559              : 
    1560              :       /* OFFSET is in UNITs, and UNIT is in bits.  If WORD is const0_rtx,
    1561              :          it is just an out-of-bounds access.  Ignore it.  */
    1562        17499 :       if (op0_piece != const0_rtx)
    1563        17499 :         store_fixed_bit_field (op0_piece, op0_piece_mode, thissize,
    1564        17499 :                                offset * unit + thispos, bitregion_start,
    1565              :                                bitregion_end, part, word_mode, reverse);
    1566        17499 :       bitsdone += thissize;
    1567              :     }
    1568         6640 : }
    1569              : 
    1570              : /* A subroutine of extract_bit_field_1 that converts return value X
    1571              :    to either MODE or TMODE.  MODE, TMODE and UNSIGNEDP are arguments
    1572              :    to extract_bit_field.  */
    1573              : 
    1574              : static rtx
    1575       911621 : convert_extracted_bit_field (rtx x, machine_mode mode,
    1576              :                              machine_mode tmode, bool unsignedp)
    1577              : {
    1578       911621 :   if (GET_MODE (x) == tmode || GET_MODE (x) == mode)
    1579              :     return x;
    1580              : 
    1581              :   /* If the x mode is not a scalar integral, first convert to the
    1582              :      integer mode of that size and then access it as a floating-point
    1583              :      value via a SUBREG.  */
    1584        21541 :   if (!SCALAR_INT_MODE_P (tmode))
    1585              :     {
    1586        11903 :       scalar_int_mode int_mode = int_mode_for_mode (tmode).require ();
    1587        11903 :       x = convert_to_mode (int_mode, x, unsignedp);
    1588        11903 :       x = force_reg (int_mode, x);
    1589        11903 :       return gen_lowpart (tmode, x);
    1590              :     }
    1591              : 
    1592         9638 :   return convert_to_mode (tmode, x, unsignedp);
    1593              : }
    1594              : 
    1595              : /* Try to use an ext(z)v pattern to extract a field from OP0.
    1596              :    Return the extracted value on success, otherwise return null.
    1597              :    EXTV describes the extraction instruction to use.  If OP0_MODE
    1598              :    is defined, it is the mode of OP0, otherwise OP0 is a BLKmode MEM.
    1599              :    The other arguments are as for extract_bit_field.  */
    1600              : 
    1601              : static rtx
    1602       166640 : extract_bit_field_using_extv (const extraction_insn *extv, rtx op0,
    1603              :                               opt_scalar_int_mode op0_mode,
    1604              :                               unsigned HOST_WIDE_INT bitsize,
    1605              :                               unsigned HOST_WIDE_INT bitnum,
    1606              :                               int unsignedp, rtx target,
    1607              :                               machine_mode mode, machine_mode tmode)
    1608              : {
    1609       166640 :   class expand_operand ops[4];
    1610       166640 :   rtx spec_target = target;
    1611       166640 :   rtx spec_target_subreg = 0;
    1612       166640 :   scalar_int_mode ext_mode = extv->field_mode;
    1613       166640 :   unsigned unit = GET_MODE_BITSIZE (ext_mode);
    1614              : 
    1615       166640 :   if (bitsize == 0 || unit < bitsize)
    1616              :     return NULL_RTX;
    1617              : 
    1618       166640 :   if (MEM_P (op0))
    1619              :     /* Get a reference to the first byte of the field.  */
    1620            0 :     op0 = narrow_bit_field_mem (op0, extv->struct_mode, bitsize, bitnum,
    1621              :                                 &bitnum);
    1622              :   else
    1623              :     {
    1624              :       /* Convert from counting within OP0 to counting in EXT_MODE.  */
    1625       166640 :       if (BYTES_BIG_ENDIAN)
    1626              :         bitnum += unit - GET_MODE_BITSIZE (op0_mode.require ());
    1627              : 
    1628              :       /* If op0 is a register, we need it in EXT_MODE to make it
    1629              :          acceptable to the format of ext(z)v.  */
    1630       166640 :       if (GET_CODE (op0) == SUBREG && op0_mode.require () != ext_mode)
    1631            0 :         return NULL_RTX;
    1632       166640 :       if (REG_P (op0) && op0_mode.require () != ext_mode)
    1633        50563 :         op0 = gen_lowpart_SUBREG (ext_mode, op0);
    1634              :     }
    1635              : 
    1636              :   /* If BITS_BIG_ENDIAN is zero on a BYTES_BIG_ENDIAN machine, we count
    1637              :      "backwards" from the size of the unit we are extracting from.
    1638              :      Otherwise, we count bits from the most significant on a
    1639              :      BYTES/BITS_BIG_ENDIAN machine.  */
    1640              : 
    1641       166640 :   if (BITS_BIG_ENDIAN != BYTES_BIG_ENDIAN)
    1642              :     bitnum = unit - bitsize - bitnum;
    1643              : 
    1644       166640 :   if (target == 0)
    1645        46678 :     target = spec_target = gen_reg_rtx (tmode);
    1646              : 
    1647       166640 :   if (GET_MODE (target) != ext_mode)
    1648              :     {
    1649        99363 :       rtx temp;
    1650              :       /* Don't use LHS paradoxical subreg if explicit truncation is needed
    1651              :          between the mode of the extraction (word_mode) and the target
    1652              :          mode.  Instead, create a temporary and use convert_move to set
    1653              :          the target.  */
    1654        99363 :       if (REG_P (target)
    1655        98100 :           && TRULY_NOOP_TRUNCATION_MODES_P (GET_MODE (target), ext_mode)
    1656       197463 :           && (temp = gen_lowpart_if_possible (ext_mode, target)))
    1657              :         {
    1658        97588 :           target = temp;
    1659        97588 :           if (partial_subreg_p (GET_MODE (spec_target), ext_mode))
    1660        96727 :             spec_target_subreg = target;
    1661              :         }
    1662              :       else
    1663         1775 :         target = gen_reg_rtx (ext_mode);
    1664              :     }
    1665              : 
    1666       166640 :   create_output_operand (&ops[0], target, ext_mode);
    1667       166640 :   create_fixed_operand (&ops[1], op0);
    1668       166640 :   create_integer_operand (&ops[2], bitsize);
    1669       166640 :   create_integer_operand (&ops[3], bitnum);
    1670       166640 :   if (maybe_expand_insn (extv->icode, 4, ops))
    1671              :     {
    1672         1675 :       target = ops[0].value;
    1673         1675 :       if (target == spec_target)
    1674              :         return target;
    1675         1675 :       if (target == spec_target_subreg)
    1676              :         return spec_target;
    1677           66 :       return convert_extracted_bit_field (target, mode, tmode, unsignedp);
    1678              :     }
    1679              :   return NULL_RTX;
    1680              : }
    1681              : 
    1682              : /* See whether it would be valid to extract the part of OP0 with
    1683              :    mode OP0_MODE described by BITNUM and BITSIZE into a value of
    1684              :    mode MODE using a subreg operation.
    1685              :    Return the subreg if so, otherwise return null.  */
    1686              : 
    1687              : static rtx
    1688       865042 : extract_bit_field_as_subreg (machine_mode mode, rtx op0,
    1689              :                              machine_mode op0_mode,
    1690              :                              poly_uint64 bitsize, poly_uint64 bitnum)
    1691              : {
    1692       865042 :   poly_uint64 bytenum;
    1693       865042 :   if (multiple_p (bitnum, BITS_PER_UNIT, &bytenum)
    1694       826271 :       && known_eq (bitsize, GET_MODE_BITSIZE (mode))
    1695       865042 :       && lowpart_bit_field_p (bitnum, bitsize, op0_mode)
    1696      1691313 :       && TRULY_NOOP_TRUNCATION_MODES_P (mode, op0_mode))
    1697       690192 :     return force_subreg (mode, op0, op0_mode, bytenum);
    1698              :   return NULL_RTX;
    1699              : }
    1700              : 
    1701              : /* Same as extract_vector_from_vector but try with a permutation.  */
    1702              : 
    1703              : static rtx
    1704         3844 : extract_vector_from_vector_perm (rtx vec, rtx target, machine_mode mode,
    1705              :                                  machine_mode subvec_mode, poly_uint64 bitsize,
    1706              :                                  poly_uint64 bitnum, rtx *alt_rtl)
    1707              : {
    1708         3844 :   machine_mode vec_mode = GET_MODE (vec);
    1709         3844 :   poly_uint64 extract_idx;
    1710         1349 :   if (!VECTOR_MODE_P (vec_mode)
    1711         3844 :       || MEM_P (vec)
    1712         3844 :       || !VECTOR_MODE_P (subvec_mode)
    1713         3844 :       || maybe_ne (bitsize, GET_MODE_PRECISION (subvec_mode))
    1714        11529 :       || !multiple_p (bitnum, GET_MODE_UNIT_BITSIZE (subvec_mode),
    1715              :                       &extract_idx))
    1716              :     return NULL_RTX;
    1717              : 
    1718              :   /* We don't handle single-element vector modes here, they should
    1719              :      go through vector-element vec_extract.  */
    1720         3841 :   if (known_le (GET_MODE_NUNITS (subvec_mode), 1)
    1721         7669 :       || known_le (GET_MODE_NUNITS (vec_mode), 1))
    1722              :     return NULL_RTX;
    1723              : 
    1724         3812 :   machine_mode punned_vec_mode;
    1725         3812 :   if (!vec_extract_perm_p (vec_mode, subvec_mode, extract_idx).exists
    1726         3812 :       (&punned_vec_mode))
    1727         3812 :     return NULL_RTX;
    1728              : 
    1729            0 :   if (punned_vec_mode != vec_mode)
    1730            0 :     vec = gen_lowpart (punned_vec_mode, vec);
    1731              : 
    1732            0 :   poly_uint64 out_nunits = GET_MODE_NUNITS (subvec_mode);
    1733              : 
    1734            0 :   vec_perm_builder sel (out_nunits, 1, 3);
    1735            0 :   for (unsigned int i = 0; i < 3; ++i)
    1736            0 :     sel.quick_push (extract_idx + i);
    1737              : 
    1738            0 :   rtx tmp = subvec_mode == mode ? target : NULL_RTX;
    1739            0 :   rtx res = expand_vec_perm_const (subvec_mode, vec, vec, sel, BLKmode, tmp);
    1740            0 :   if (!res)
    1741              :     return NULL_RTX;
    1742              : 
    1743            0 :   if (res == target && alt_rtl)
    1744            0 :     *alt_rtl = target;
    1745              : 
    1746            0 :   if (subvec_mode != mode)
    1747            0 :     res = gen_lowpart (mode, res);
    1748              : 
    1749              :   return res;
    1750            0 : }
    1751              : 
    1752              : /* Try to extract a subvector of mode SUBVEC_MODE (which has BITSIZE bits)
    1753              :    from vector VEC, starting at bit BITNUM, then extract the MODE lowpart of
    1754              :    the subvector.  If the vec_extract-based attempt failed, build a vector
    1755              :    permute to extract the part and hand it to the target.
    1756              : 
    1757              :    Return the result on success or NULL_RTX on failure.  */
    1758              : 
    1759              : static rtx
    1760      1120127 : extract_vector_from_vector (rtx vec, rtx target, machine_mode mode,
    1761              :                             machine_mode subvec_mode, poly_uint64 bitsize,
    1762              :                             poly_uint64 bitnum, rtx *alt_rtl)
    1763              : {
    1764      1120127 :   machine_mode vec_mode = GET_MODE (vec);
    1765      1069784 :   if (!VECTOR_MODE_P (vec_mode)
    1766        72045 :       || MEM_P (vec)
    1767        70603 :       || !VECTOR_MODE_P (subvec_mode)
    1768        12875 :       || maybe_ne (bitsize, GET_MODE_PRECISION (subvec_mode))
    1769      1158752 :       || known_le (GET_MODE_SIZE (GET_MODE (vec)),
    1770              :                    GET_MODE_SIZE (subvec_mode)))
    1771              :     return NULL_RTX;
    1772              : 
    1773              :   /* Try to find a fitting mode for extracting.  */
    1774        38625 :   if (GET_MODE_INNER (vec_mode) != GET_MODE_INNER (subvec_mode))
    1775              :     {
    1776          214 :       scalar_mode inner_mode = GET_MODE_INNER (subvec_mode);
    1777          214 :       poly_uint64 nunits;
    1778          428 :       if (!multiple_p (GET_MODE_BITSIZE (vec_mode),
    1779          214 :                        GET_MODE_UNIT_BITSIZE (subvec_mode), &nunits)
    1780          412 :           || !related_vector_mode (subvec_mode, inner_mode,
    1781           16 :                                    nunits).exists (&vec_mode)
    1782          198 :           || maybe_ne (GET_MODE_SIZE (vec_mode),
    1783          396 :                        GET_MODE_SIZE (GET_MODE (vec))))
    1784           16 :         return NULL_RTX;
    1785              :     }
    1786              : 
    1787        12859 :   poly_uint64 pos;
    1788        12859 :   enum insn_code icode = convert_optab_handler (vec_extract_optab, vec_mode,
    1789              :                                                 subvec_mode);
    1790        12859 :   if (icode != CODE_FOR_nothing
    1791        21883 :       && multiple_p (bitnum, GET_MODE_BITSIZE (subvec_mode), &pos))
    1792              :     {
    1793         9015 :       if (vec_mode != GET_MODE (vec))
    1794           33 :         vec = gen_lowpart (vec_mode, vec);
    1795              : 
    1796         9015 :       class expand_operand ops[3];
    1797         9015 :       create_output_operand (&ops[0], target, subvec_mode);
    1798         9015 :       ops[0].target = 1;
    1799         9015 :       create_input_operand (&ops[1], vec, GET_MODE (vec));
    1800         9015 :       create_integer_operand (&ops[2], pos);
    1801         9015 :       if (maybe_expand_insn (icode, 3, ops))
    1802              :         {
    1803         9015 :           if (alt_rtl && ops[0].target)
    1804          207 :             *alt_rtl = target;
    1805         9015 :           target = ops[0].value;
    1806         9015 :           if (GET_MODE (target) != mode)
    1807         9015 :             return gen_lowpart (subvec_mode, target);
    1808              :           return target;
    1809              :         }
    1810              :     }
    1811              : 
    1812         3844 :   rtx res = extract_vector_from_vector_perm (vec, target, mode, subvec_mode,
    1813              :                                              bitsize, bitnum, alt_rtl);
    1814         3844 :   if (res)
    1815              :     return res;
    1816              : 
    1817              :   return NULL_RTX;
    1818              : }
    1819              : 
    1820              : /* Extract an element of OP0 of size BITSIZE, starting at bit position BITNUM.
    1821              :    Pun the result with mode TMODE if necessary.  Return the extracted element
    1822              :    if successful or NULL_RTX otherwise.  */
    1823              : 
    1824              : static rtx
    1825      1111112 : extract_element_from_vector (rtx op0, rtx target, machine_mode mode,
    1826              :                              machine_mode tmode, poly_uint64 bitsize,
    1827              :                              poly_uint64 bitnum, rtx *alt_rtl)
    1828              : {
    1829              :   /* See if we can get a better vector mode before extracting.  */
    1830      1111112 :   machine_mode outermode = GET_MODE (op0);
    1831      1062369 :   if (VECTOR_MODE_P (outermode)
    1832        63030 :       && !MEM_P (op0)
    1833      1172700 :       && GET_MODE_INNER (outermode) != tmode)
    1834              :     {
    1835         9609 :       machine_mode new_mode;
    1836              : 
    1837         9609 :       if (GET_MODE_CLASS (tmode) == MODE_FLOAT)
    1838              :         new_mode = MIN_MODE_VECTOR_FLOAT;
    1839         9052 :       else if (GET_MODE_CLASS (tmode) == MODE_FRACT)
    1840              :         new_mode = MIN_MODE_VECTOR_FRACT;
    1841         9052 :       else if (GET_MODE_CLASS (tmode) == MODE_UFRACT)
    1842              :         new_mode = MIN_MODE_VECTOR_UFRACT;
    1843         9052 :       else if (GET_MODE_CLASS (tmode) == MODE_ACCUM)
    1844              :         new_mode = MIN_MODE_VECTOR_ACCUM;
    1845         9052 :       else if (GET_MODE_CLASS (tmode) == MODE_UACCUM)
    1846              :         new_mode = MIN_MODE_VECTOR_UACCUM;
    1847              :       else
    1848         9609 :         new_mode = MIN_MODE_VECTOR_INT;
    1849              : 
    1850       152327 :       FOR_EACH_MODE_FROM (new_mode, new_mode)
    1851       303948 :         if (known_eq (GET_MODE_SIZE (new_mode), GET_MODE_SIZE (outermode))
    1852        63324 :             && known_eq (GET_MODE_UNIT_SIZE (new_mode), GET_MODE_SIZE (tmode))
    1853        19218 :             && known_eq (bitsize, GET_MODE_UNIT_PRECISION (new_mode))
    1854        28795 :             && multiple_p (bitnum, GET_MODE_UNIT_PRECISION (new_mode))
    1855         9577 :             && targetm.vector_mode_supported_p (new_mode)
    1856       161533 :             && targetm.modes_tieable_p (outermode, new_mode))
    1857              :           break;
    1858         9609 :       if (new_mode != VOIDmode)
    1859         9256 :         op0 = gen_lowpart (new_mode, op0);
    1860              :     }
    1861              : 
    1862      1111112 :   outermode = GET_MODE (op0);
    1863      1111112 :   if (VECTOR_MODE_P (outermode) && !MEM_P (op0))
    1864              :     {
    1865        61588 :       scalar_mode innermode = GET_MODE_INNER (outermode);
    1866              : 
    1867        61588 :       enum insn_code icode
    1868        61588 :         = convert_optab_handler (vec_extract_optab, outermode, innermode);
    1869              : 
    1870        61588 :       poly_uint64 pos;
    1871        61588 :       if (icode != CODE_FOR_nothing
    1872        59924 :           && known_eq (bitsize, GET_MODE_PRECISION (innermode))
    1873       121230 :           && multiple_p (bitnum, GET_MODE_PRECISION (innermode), &pos))
    1874              :         {
    1875        59639 :           class expand_operand ops[3];
    1876              : 
    1877        59639 :           create_output_operand (&ops[0], target,
    1878        59639 :                                  insn_data[icode].operand[0].mode);
    1879        59639 :           ops[0].target = 1;
    1880        59639 :           create_input_operand (&ops[1], op0, outermode);
    1881        59639 :           create_integer_operand (&ops[2], pos);
    1882        59639 :           if (maybe_expand_insn (icode, 3, ops))
    1883              :             {
    1884        59639 :               if (alt_rtl && ops[0].target)
    1885        14767 :                 *alt_rtl = target;
    1886        59639 :               target = ops[0].value;
    1887        59639 :               if (GET_MODE (target) != mode)
    1888        59639 :                 return gen_lowpart (tmode, target);
    1889              :               return target;
    1890              :             }
    1891              :         }
    1892              :       /* Using subregs is useful if we're extracting one register vector
    1893              :          from a multi-register vector.  extract_bit_field_as_subreg checks
    1894              :          for valid bitsize and bitnum, so we don't need to do that here.  */
    1895         1949 :       if (VECTOR_MODE_P (mode))
    1896              :         {
    1897           45 :           rtx sub = extract_bit_field_as_subreg (mode, op0, outermode,
    1898              :                                                  bitsize, bitnum);
    1899           45 :           if (sub)
    1900              :             return sub;
    1901              :         }
    1902              :     }
    1903              : 
    1904              :   return NULL_RTX;
    1905              : }
    1906              : 
    1907              : /* Check whether:
    1908              : 
    1909              :    (a) OP0 is a vector that spans multiple registers (which might
    1910              :        mean that it is a tuple mode) and
    1911              : 
    1912              :    (b) a sequence of BITSIZE bits, starting at bit *BITNUM, fits within
    1913              :        a single vector register.
    1914              : 
    1915              :    Return the vector register if so and update *BITNUM so that it
    1916              :    is relative to the returned register.  Return NULL_RTX otherwise.  */
    1917              : 
    1918              : static rtx
    1919      1051452 : extract_vector_from_multireg (rtx op0, machine_mode tmode,
    1920              :                               poly_uint64 bitsize, poly_uint64 *bitnum)
    1921              : {
    1922      1051452 :   machine_mode outermode = GET_MODE (op0);
    1923      1051452 :   poly_uint64 outer_natural_sz = REGMODE_NATURAL_SIZE (outermode);
    1924      1051452 :   scalar_mode elmode = GET_MODE_INNER (outermode);
    1925      1051452 :   HOST_WIDE_INT vec_idx;
    1926      1051452 :   poly_uint64 off;
    1927      1051452 :   poly_uint64 nelts;
    1928      1048211 :   if (VECTOR_MODE_P (outermode)
    1929         3370 :       && !MEM_P (op0)
    1930         1928 :       && known_gt (GET_MODE_SIZE (outermode), outer_natural_sz)
    1931         1023 :       && known_eq (bitsize, GET_MODE_PRECISION (tmode))
    1932         1023 :       && can_div_trunc_p (*bitnum, outer_natural_sz * BITS_PER_UNIT,
    1933              :                           &vec_idx, &off)
    1934         1023 :       && known_le (off + bitsize, outer_natural_sz * BITS_PER_UNIT)
    1935      1052453 :       && multiple_p (outer_natural_sz, GET_MODE_SIZE (elmode), &nelts))
    1936              :     {
    1937              :       /* Get a single-vector mode from the outer multi-vector mode.
    1938              :          If it exists and we can pun the outer mode with a subreg,
    1939              :          do so and return it.  */
    1940         1978 :       multiple_p (outer_natural_sz, GET_MODE_SIZE (elmode), &nelts);
    1941          989 :       opt_machine_mode vmode
    1942          989 :         = related_vector_mode (outermode, GET_MODE_INNER (outermode), nelts);
    1943              : 
    1944          989 :       machine_mode vmode_onevec;
    1945          989 :       rtx subreg;
    1946          989 :       if (vmode.exists (&vmode_onevec)
    1947          989 :           && (subreg = simplify_gen_subreg
    1948          989 :               (vmode_onevec, op0, outermode,
    1949         1978 :                vec_idx * GET_MODE_SIZE (vmode_onevec))))
    1950              :         {
    1951          989 :           *bitnum = off;
    1952          989 :           return subreg;
    1953              :         }
    1954              :     }
    1955              : 
    1956              :   return NULL_RTX;
    1957              : }
    1958              : 
    1959              : /* A subroutine of extract_bit_field, with the same arguments.
    1960              :    If UNSIGNEDP is -1, the result need not be sign or zero extended.
    1961              :    If FALLBACK_P is true, fall back to extract_fixed_bit_field
    1962              :    if we can find no other means of implementing the operation.
    1963              :    if FALLBACK_P is false, return NULL instead.  */
    1964              : 
    1965              : static rtx
    1966      1139771 : extract_bit_field_1 (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
    1967              :                      int unsignedp, rtx target, machine_mode mode,
    1968              :                      machine_mode tmode, bool reverse, bool fallback_p,
    1969              :                      rtx *alt_rtl)
    1970              : {
    1971      1139771 :   rtx op0 = str_rtx;
    1972      1139771 :   machine_mode mode1;
    1973              : 
    1974      1139771 :   if (tmode == VOIDmode)
    1975            0 :     tmode = mode;
    1976              : 
    1977      1150515 :   while (GET_CODE (op0) == SUBREG)
    1978              :     {
    1979        10744 :       bitnum += SUBREG_BYTE (op0) * BITS_PER_UNIT;
    1980        10744 :       op0 = SUBREG_REG (op0);
    1981              :     }
    1982              : 
    1983              :   /* If we have an out-of-bounds access to a register, just return an
    1984              :      uninitialized register of the required mode.  This can occur if the
    1985              :      source code contains an out-of-bounds access to a small array.  */
    1986      2094117 :   if (REG_P (op0) && known_ge (bitnum, GET_MODE_BITSIZE (GET_MODE (op0))))
    1987            0 :     return gen_reg_rtx (tmode);
    1988              : 
    1989      1139771 :   if (REG_P (op0)
    1990       954346 :       && mode == GET_MODE (op0)
    1991       152277 :       && known_eq (bitnum, 0U)
    1992      1382061 :       && known_eq (bitsize, GET_MODE_BITSIZE (GET_MODE (op0))))
    1993              :     {
    1994        20633 :       if (reverse)
    1995            0 :         op0 = flip_storage_order (mode, op0);
    1996              :       /* We're trying to extract a full register from itself.  */
    1997              :       return op0;
    1998              :     }
    1999              : 
    2000              :   /* First try to check for vector from vector extractions.  */
    2001      1119138 :   rtx res = extract_vector_from_vector (op0, target, mode, tmode, bitsize,
    2002              :                                         bitnum, alt_rtl);
    2003      1119138 :   if (res)
    2004              :     return res;
    2005              : 
    2006              :   /* Use vec_extract patterns for extracting parts of vectors whenever
    2007              :      available.  If that fails, see whether the current modes and bitregion
    2008              :      give a natural subreg.  */
    2009      1110123 :   res = extract_element_from_vector (op0, target, mode, tmode, bitsize, bitnum,
    2010              :                                      alt_rtl);
    2011      1110123 :   if (res)
    2012              :     return res;
    2013              : 
    2014              :   /* If this mode occupies multiple registers and we're extracting
    2015              :      a single one, we can usually get to that vector directly
    2016              :      without actual extraction.
    2017              :      This helps targets see through multi-vector modes.  */
    2018      1051452 :   poly_uint64 bitnum_new = bitnum;
    2019      1051452 :   res = extract_vector_from_multireg (op0, tmode, bitsize, &bitnum_new);
    2020      1051452 :   if (res)
    2021              :     {
    2022          989 :       rtx vec = extract_vector_from_vector (res, target, mode, tmode, bitsize,
    2023              :                                             bitnum_new, alt_rtl);
    2024          989 :       if (vec)
    2025              :         return vec;
    2026              : 
    2027          989 :       rtx el = extract_element_from_vector (res, target, mode, tmode, bitsize,
    2028              :                                             bitnum_new, alt_rtl);
    2029          989 :       if (el)
    2030              :         return el;
    2031              : 
    2032            4 :       op0 = res;
    2033            4 :       bitnum = bitnum_new;
    2034              :     }
    2035              : 
    2036              :   /* Make sure we are playing with integral modes.  Pun with subregs
    2037              :      if we aren't.  */
    2038      1050467 :   opt_scalar_int_mode op0_mode = int_mode_for_mode (GET_MODE (op0));
    2039      1050467 :   scalar_int_mode imode;
    2040      1050467 :   bool need_stack_p = false;
    2041      1050467 :   if (!op0_mode.exists (&imode) || imode != GET_MODE (op0))
    2042              :     {
    2043       197700 :       if (MEM_P (op0))
    2044       159929 :         op0 = adjust_bitfield_address_size (op0, op0_mode.else_blk (),
    2045              :                                             0, MEM_SIZE (op0));
    2046        37783 :       else if (op0_mode.exists (&imode)
    2047        38719 :                && known_ge
    2048              :                ((unsigned) UNITS_PER_WORD,
    2049              :                 (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
    2050              :         {
    2051        37752 :           op0 = gen_lowpart (imode, op0);
    2052              : 
    2053              :           /* If we got a SUBREG, force it into a register since we
    2054              :              aren't going to be able to do another SUBREG on it.  */
    2055        37752 :           if (GET_CODE (op0) == SUBREG)
    2056        37473 :             op0 = force_reg (imode, op0);
    2057              :         }
    2058              :       else
    2059              :         need_stack_p = true;
    2060              :     }
    2061       852767 :   else if (!MEM_P (op0)
    2062      1759871 :            && maybe_lt ((unsigned) UNITS_PER_WORD,
    2063       827259 :                         (poly_uint64) REGMODE_NATURAL_SIZE (GET_MODE (op0))))
    2064              :     need_stack_p = true;
    2065              : 
    2066       197390 :   if (need_stack_p)
    2067              :     {
    2068           62 :       poly_int64 size = GET_MODE_SIZE (GET_MODE (op0));
    2069           31 :       rtx mem = assign_stack_temp (GET_MODE (op0), size);
    2070           31 :       emit_move_insn (mem, op0);
    2071           31 :       op0 = adjust_bitfield_address_size (mem, BLKmode, 0, size);
    2072              :     }
    2073              : 
    2074              :   /* ??? We currently assume TARGET is at least as big as BITSIZE.
    2075              :      If that's wrong, the solution is to test for it and set TARGET to 0
    2076              :      if needed.  */
    2077              : 
    2078              :   /* Get the mode of the field to use for atomic access or subreg
    2079              :      conversion.  */
    2080      1050467 :   if (!SCALAR_INT_MODE_P (tmode)
    2081      1050467 :       || !mode_for_size (bitsize, GET_MODE_CLASS (tmode), 0).exists (&mode1))
    2082              :     mode1 = mode;
    2083      1050467 :   gcc_assert (mode1 != BLKmode);
    2084              : 
    2085              :   /* Extraction of a full MODE1 value can be done with a subreg as long
    2086              :      as the least significant bit of the value is the least significant
    2087              :      bit of either OP0 or a word of OP0.  */
    2088      1050467 :   if (!MEM_P (op0) && !reverse && op0_mode.exists (&imode))
    2089              :     {
    2090       864997 :       rtx sub = extract_bit_field_as_subreg (mode1, op0, imode,
    2091              :                                              bitsize, bitnum);
    2092       864997 :       if (sub)
    2093       683825 :         return convert_extracted_bit_field (sub, mode, tmode, unsignedp);
    2094              :     }
    2095              : 
    2096              :   /* Extraction of a full MODE1 value can be done with a load as long as
    2097              :      the field is on a byte boundary and is sufficiently aligned.  */
    2098       366642 :   poly_uint64 bytenum;
    2099       366642 :   if (simple_mem_bitfield_p (op0, bitsize, bitnum, mode1, &bytenum))
    2100              :     {
    2101        45358 :       op0 = adjust_bitfield_address (op0, mode1, bytenum);
    2102        45358 :       if (reverse)
    2103           55 :         op0 = flip_storage_order (mode1, op0);
    2104        45358 :       return convert_extracted_bit_field (op0, mode, tmode, unsignedp);
    2105              :     }
    2106              : 
    2107              :   /* If we have a memory source and a non-constant bit offset, restrict
    2108              :      the memory to the referenced bytes.  This is a worst-case fallback
    2109              :      but is useful for things like vector booleans.  */
    2110       321284 :   if (MEM_P (op0) && !bitnum.is_constant ())
    2111              :     {
    2112              :       bytenum = bits_to_bytes_round_down (bitnum);
    2113              :       bitnum = num_trailing_bits (bitnum);
    2114              :       poly_uint64 bytesize = bits_to_bytes_round_up (bitnum + bitsize);
    2115              :       op0 = adjust_bitfield_address_size (op0, BLKmode, bytenum, bytesize);
    2116              :       op0_mode = opt_scalar_int_mode ();
    2117              :     }
    2118              : 
    2119              :   /* It's possible we'll need to handle other cases here for
    2120              :      polynomial bitnum and bitsize.  */
    2121              : 
    2122              :   /* From here on we need to be looking at a fixed-size insertion.  */
    2123       321284 :   return extract_integral_bit_field (op0, op0_mode, bitsize.to_constant (),
    2124              :                                      bitnum.to_constant (), unsignedp,
    2125       321284 :                                      target, mode, tmode, reverse, fallback_p);
    2126              : }
    2127              : 
    2128              : /* Subroutine of extract_bit_field_1, with the same arguments, except
    2129              :    that BITSIZE and BITNUM are constant.  Handle cases specific to
    2130              :    integral modes.  If OP0_MODE is defined, it is the mode of OP0,
    2131              :    otherwise OP0 is a BLKmode MEM.  */
    2132              : 
    2133              : static rtx
    2134       321284 : extract_integral_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
    2135              :                             unsigned HOST_WIDE_INT bitsize,
    2136              :                             unsigned HOST_WIDE_INT bitnum, int unsignedp,
    2137              :                             rtx target, machine_mode mode, machine_mode tmode,
    2138              :                             bool reverse, bool fallback_p)
    2139              : {
    2140              :   /* Handle fields bigger than a word.  */
    2141              : 
    2142       324553 :   if (bitsize > BITS_PER_WORD)
    2143              :     {
    2144              :       /* Here we transfer the words of the field
    2145              :          in the order least significant first.
    2146              :          This is because the most significant word is the one which may
    2147              :          be less than full.  */
    2148              : 
    2149         1519 :       const bool backwards = WORDS_BIG_ENDIAN;
    2150         1519 :       unsigned int nwords = (bitsize + (BITS_PER_WORD - 1)) / BITS_PER_WORD;
    2151         1519 :       unsigned int i;
    2152         1519 :       rtx_insn *last;
    2153              : 
    2154         1519 :       if (target == 0 || !REG_P (target) || !valid_multiword_target_p (target))
    2155         1495 :         target = gen_reg_rtx (mode);
    2156              : 
    2157              :       /* In case we're about to clobber a base register or something
    2158              :          (see gcc.c-torture/execute/20040625-1.c).   */
    2159         1519 :       if (reg_mentioned_p (target, op0))
    2160            0 :         target = gen_reg_rtx (mode);
    2161              : 
    2162              :       /* Indicate for flow that the entire target reg is being set.  */
    2163         1519 :       emit_clobber (target);
    2164              : 
    2165              :       /* The mode must be fixed-size, since extract_bit_field_1 handles
    2166              :          extractions from variable-sized objects before calling this
    2167              :          function.  */
    2168         1519 :       unsigned int target_size
    2169         1519 :         = GET_MODE_SIZE (GET_MODE (target)).to_constant ();
    2170         1519 :       last = get_last_insn ();
    2171         4557 :       for (i = 0; i < nwords; i++)
    2172              :         {
    2173              :           /* If I is 0, use the low-order word in both field and target;
    2174              :              if I is 1, use the next to lowest word; and so on.  */
    2175              :           /* Word number in TARGET to use.  */
    2176         3038 :           unsigned int wordnum
    2177              :             = (backwards ? target_size / UNITS_PER_WORD - i - 1 : i);
    2178              :           /* Offset from start of field in OP0.  */
    2179         6076 :           unsigned int bit_offset = (backwards ^ reverse
    2180         3038 :                                      ? MAX ((int) bitsize - ((int) i + 1)
    2181              :                                             * BITS_PER_WORD,
    2182              :                                             0)
    2183         3106 :                                      : (int) i * BITS_PER_WORD);
    2184         3038 :           rtx target_part = operand_subword (target, wordnum, 1, VOIDmode);
    2185         3038 :           rtx result_part
    2186         3214 :             = extract_bit_field_1 (op0, MIN (BITS_PER_WORD,
    2187              :                                              bitsize - i * BITS_PER_WORD),
    2188         3038 :                                    bitnum + bit_offset,
    2189              :                                    (unsignedp ? 1 : -1), target_part,
    2190              :                                    mode, word_mode, reverse, fallback_p, NULL);
    2191              : 
    2192         3038 :           if (!result_part)
    2193              :             {
    2194            0 :               delete_insns_since (last);
    2195            0 :               return NULL;
    2196              :             }
    2197              : 
    2198         3038 :           if (!target_part)
    2199            0 :             store_bit_field (target, BITS_PER_WORD, i * BITS_PER_WORD,
    2200              :                              0, 0, word_mode, result_part, false, i == 0);
    2201         3038 :           else if (result_part != target_part)
    2202         2922 :             emit_move_insn (target_part, result_part);
    2203              :         }
    2204              : 
    2205         1519 :       if (unsignedp)
    2206              :         {
    2207              :           /* Unless we've filled TARGET, the upper regs in a multi-reg value
    2208              :              need to be zero'd out.  */
    2209         1533 :           if (target_size > nwords * UNITS_PER_WORD)
    2210              :             {
    2211            0 :               unsigned int i, total_words;
    2212              : 
    2213            0 :               total_words = target_size / UNITS_PER_WORD;
    2214            0 :               for (i = nwords; i < total_words; i++)
    2215            0 :                 emit_move_insn
    2216            0 :                   (operand_subword (target,
    2217            0 :                                     backwards ? total_words - i - 1 : i,
    2218              :                                     1, VOIDmode),
    2219              :                    const0_rtx);
    2220              :             }
    2221              :           return target;
    2222              :         }
    2223              : 
    2224              :       /* Signed bit field: sign-extend with two arithmetic shifts.  */
    2225           40 :       target = expand_shift (LSHIFT_EXPR, mode, target,
    2226           20 :                              GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
    2227           40 :       return expand_shift (RSHIFT_EXPR, mode, target,
    2228           20 :                            GET_MODE_BITSIZE (mode) - bitsize, NULL_RTX, 0);
    2229              :     }
    2230              : 
    2231              :   /* If OP0 is a multi-word register, narrow it to the affected word.
    2232              :      If the region spans two words, defer to extract_split_bit_field.  */
    2233       503481 :   if (!MEM_P (op0) && GET_MODE_SIZE (op0_mode.require ()) > UNITS_PER_WORD)
    2234              :     {
    2235         3086 :       if (bitnum % BITS_PER_WORD + bitsize > BITS_PER_WORD)
    2236              :         {
    2237         1142 :           if (!fallback_p)
    2238              :             return NULL_RTX;
    2239           60 :           target = extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
    2240              :                                             unsignedp, reverse);
    2241           60 :           return convert_extracted_bit_field (target, mode, tmode, unsignedp);
    2242              :         }
    2243              :       /* If OP0 is a hard register, copy it to a pseudo before calling
    2244              :          force_subreg.  */
    2245         1944 :       if (REG_P (op0) && HARD_REGISTER_P (op0))
    2246            1 :         op0 = copy_to_reg (op0);
    2247         1944 :       op0 = force_subreg (word_mode, op0, op0_mode.require (),
    2248         2408 :                           bitnum / BITS_PER_WORD * UNITS_PER_WORD);
    2249         1944 :       op0_mode = word_mode;
    2250         2176 :       bitnum %= BITS_PER_WORD;
    2251              :     }
    2252              : 
    2253              :   /* From here on we know the desired field is smaller than a word.
    2254              :      If OP0 is a register, it too fits within a word.  */
    2255       318623 :   enum extraction_pattern pattern = unsignedp ? EP_extzv : EP_extv;
    2256       318623 :   extraction_insn extv;
    2257       318623 :   if (!MEM_P (op0)
    2258       180035 :       && !reverse
    2259              :       /* ??? We could limit the structure size to the part of OP0 that
    2260              :          contains the field, with appropriate checks for endianness
    2261              :          and TARGET_TRULY_NOOP_TRUNCATION.  */
    2262       498644 :       && get_best_reg_extraction_insn (&extv, pattern,
    2263       512025 :                                        GET_MODE_BITSIZE (op0_mode.require ()),
    2264              :                                        tmode))
    2265              :     {
    2266       166640 :       rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
    2267              :                                                  bitsize, bitnum,
    2268              :                                                  unsignedp, target, mode,
    2269              :                                                  tmode);
    2270       166640 :       if (result)
    2271              :         return result;
    2272              :     }
    2273              : 
    2274              :   /* If OP0 is a memory, try copying it to a register and seeing if a
    2275              :      cheap register alternative is available.  */
    2276       316948 :   if (MEM_P (op0) & !reverse)
    2277              :     {
    2278       138365 :       if (get_best_mem_extraction_insn (&extv, pattern, bitsize, bitnum,
    2279              :                                         tmode))
    2280              :         {
    2281            0 :           rtx result = extract_bit_field_using_extv (&extv, op0, op0_mode,
    2282              :                                                      bitsize, bitnum,
    2283              :                                                      unsignedp, target, mode,
    2284              :                                                      tmode);
    2285            0 :           if (result)
    2286            0 :             return result;
    2287              :         }
    2288              : 
    2289       138365 :       rtx_insn *last = get_last_insn ();
    2290              : 
    2291              :       /* Try loading part of OP0 into a register and extracting the
    2292              :          bitfield from that.  */
    2293       138365 :       unsigned HOST_WIDE_INT bitpos;
    2294       138365 :       rtx xop0 = adjust_bit_field_mem_for_reg (pattern, op0, bitsize, bitnum,
    2295              :                                                0, 0, tmode, &bitpos);
    2296       138365 :       if (xop0)
    2297              :         {
    2298       135718 :           xop0 = copy_to_reg (xop0);
    2299       135718 :           rtx result = extract_bit_field_1 (xop0, bitsize, bitpos,
    2300              :                                             unsignedp, target,
    2301              :                                             mode, tmode, reverse, false, NULL);
    2302       135718 :           if (result)
    2303              :             return result;
    2304       135718 :           delete_insns_since (last);
    2305              :         }
    2306              :     }
    2307              : 
    2308       316948 :   if (!fallback_p)
    2309              :     return NULL;
    2310              : 
    2311              :   /* Find a correspondingly-sized integer field, so we can apply
    2312              :      shifts and masks to it.  */
    2313       182312 :   scalar_int_mode int_mode;
    2314       182312 :   if (!int_mode_for_mode (tmode).exists (&int_mode))
    2315              :     /* If this fails, we should probably push op0 out to memory and then
    2316              :        do a load.  */
    2317            0 :     int_mode = int_mode_for_mode (mode).require ();
    2318              : 
    2319       182312 :   target = extract_fixed_bit_field (int_mode, op0, op0_mode, bitsize,
    2320              :                                     bitnum, target, unsignedp, reverse);
    2321              : 
    2322              :   /* Complex values must be reversed piecewise, so we need to undo the global
    2323              :      reversal, convert to the complex mode and reverse again.  */
    2324       182312 :   if (reverse && COMPLEX_MODE_P (tmode))
    2325              :     {
    2326            0 :       target = flip_storage_order (int_mode, target);
    2327            0 :       target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
    2328            0 :       target = flip_storage_order (tmode, target);
    2329              :     }
    2330              :   else
    2331       182312 :     target = convert_extracted_bit_field (target, mode, tmode, unsignedp);
    2332              : 
    2333              :   return target;
    2334              : }
    2335              : 
    2336              : /* Generate code to extract a byte-field from STR_RTX
    2337              :    containing BITSIZE bits, starting at BITNUM,
    2338              :    and put it in TARGET if possible (if TARGET is nonzero).
    2339              :    Regardless of TARGET, we return the rtx for where the value is placed.
    2340              : 
    2341              :    STR_RTX is the structure containing the byte (a REG or MEM).
    2342              :    UNSIGNEDP is nonzero if this is an unsigned bit field.
    2343              :    MODE is the natural mode of the field value once extracted.
    2344              :    TMODE is the mode the caller would like the value to have;
    2345              :    but the value may be returned with type MODE instead.
    2346              : 
    2347              :    If REVERSE is true, the extraction is to be done in reverse order.
    2348              : 
    2349              :    If a TARGET is specified and we can store in it at no extra cost,
    2350              :    we do so, and return TARGET.
    2351              :    Otherwise, we return a REG of mode TMODE or MODE, with TMODE preferred
    2352              :    if they are equally easy.
    2353              : 
    2354              :    If the result can be stored at TARGET, and ALT_RTL is non-NULL,
    2355              :    then *ALT_RTL is set to TARGET (before legitimziation).  */
    2356              : 
    2357              : rtx
    2358      1001015 : extract_bit_field (rtx str_rtx, poly_uint64 bitsize, poly_uint64 bitnum,
    2359              :                    int unsignedp, rtx target, machine_mode mode,
    2360              :                    machine_mode tmode, bool reverse, rtx *alt_rtl)
    2361              : {
    2362      1001015 :   machine_mode mode1;
    2363              : 
    2364              :   /* Handle -fstrict-volatile-bitfields in the cases where it applies.  */
    2365      2002030 :   if (maybe_ne (GET_MODE_BITSIZE (GET_MODE (str_rtx)), 0))
    2366              :     mode1 = GET_MODE (str_rtx);
    2367       260927 :   else if (target && maybe_ne (GET_MODE_BITSIZE (GET_MODE (target)), 0))
    2368              :     mode1 = GET_MODE (target);
    2369              :   else
    2370              :     mode1 = tmode;
    2371              : 
    2372      1001015 :   unsigned HOST_WIDE_INT ibitsize, ibitnum;
    2373      1001015 :   scalar_int_mode int_mode;
    2374      1001015 :   if (bitsize.is_constant (&ibitsize)
    2375      1001015 :       && bitnum.is_constant (&ibitnum)
    2376      1866597 :       && is_a <scalar_int_mode> (mode1, &int_mode)
    2377       865589 :       && strict_volatile_bitfield_p (str_rtx, ibitsize, ibitnum,
    2378              :                                      int_mode, 0, 0))
    2379              :     {
    2380              :       /* Extraction of a full INT_MODE value can be done with a simple load.
    2381              :          We know here that the field can be accessed with one single
    2382              :          instruction.  For targets that support unaligned memory,
    2383              :          an unaligned access may be necessary.  */
    2384           14 :       if (ibitsize == GET_MODE_BITSIZE (int_mode))
    2385              :         {
    2386            0 :           rtx result = adjust_bitfield_address (str_rtx, int_mode,
    2387              :                                                 ibitnum / BITS_PER_UNIT);
    2388            0 :           if (reverse)
    2389            0 :             result = flip_storage_order (int_mode, result);
    2390            0 :           gcc_assert (ibitnum % BITS_PER_UNIT == 0);
    2391            0 :           return convert_extracted_bit_field (result, mode, tmode, unsignedp);
    2392              :         }
    2393              : 
    2394            7 :       str_rtx = narrow_bit_field_mem (str_rtx, int_mode, ibitsize, ibitnum,
    2395              :                                       &ibitnum);
    2396           14 :       gcc_assert (ibitnum + ibitsize <= GET_MODE_BITSIZE (int_mode));
    2397            7 :       str_rtx = copy_to_reg (str_rtx);
    2398            7 :       return extract_bit_field_1 (str_rtx, ibitsize, ibitnum, unsignedp,
    2399              :                                   target, mode, tmode, reverse, true, alt_rtl);
    2400              :     }
    2401              : 
    2402      1001008 :   return extract_bit_field_1 (str_rtx, bitsize, bitnum, unsignedp,
    2403      1001008 :                               target, mode, tmode, reverse, true, alt_rtl);
    2404              : }
    2405              : 
    2406              : /* Use shifts and boolean operations to extract a field of BITSIZE bits
    2407              :    from bit BITNUM of OP0.  If OP0_MODE is defined, it is the mode of OP0,
    2408              :    otherwise OP0 is a BLKmode MEM.
    2409              : 
    2410              :    UNSIGNEDP is nonzero for an unsigned bit field (don't sign-extend value).
    2411              :    If REVERSE is true, the extraction is to be done in reverse order.
    2412              : 
    2413              :    If TARGET is nonzero, attempts to store the value there
    2414              :    and return TARGET, but this is not guaranteed.
    2415              :    If TARGET is not used, create a pseudo-reg of mode TMODE for the value.  */
    2416              : 
    2417              : static rtx
    2418       200590 : extract_fixed_bit_field (machine_mode tmode, rtx op0,
    2419              :                          opt_scalar_int_mode op0_mode,
    2420              :                          unsigned HOST_WIDE_INT bitsize,
    2421              :                          unsigned HOST_WIDE_INT bitnum, rtx target,
    2422              :                          int unsignedp, bool reverse)
    2423              : {
    2424       200590 :   scalar_int_mode mode;
    2425       200590 :   if (MEM_P (op0))
    2426              :     {
    2427       152246 :       if (!get_best_mode (bitsize, bitnum, 0, 0, MEM_ALIGN (op0),
    2428       152246 :                           BITS_PER_WORD, MEM_VOLATILE_P (op0), &mode))
    2429              :         /* The only way this should occur is if the field spans word
    2430              :            boundaries.  */
    2431         4283 :         return extract_split_bit_field (op0, op0_mode, bitsize, bitnum,
    2432         4283 :                                         unsignedp, reverse);
    2433              : 
    2434       147963 :       op0 = narrow_bit_field_mem (op0, mode, bitsize, bitnum, &bitnum);
    2435              :     }
    2436              :   else
    2437        48344 :     mode = op0_mode.require ();
    2438              : 
    2439       196307 :   return extract_fixed_bit_field_1 (tmode, op0, mode, bitsize, bitnum,
    2440       196307 :                                     target, unsignedp, reverse);
    2441              : }
    2442              : 
    2443              : /* Helper function for extract_fixed_bit_field, extracts
    2444              :    the bit field always using MODE, which is the mode of OP0.
    2445              :    If UNSIGNEDP is -1, the result need not be sign or zero extended.
    2446              :    The other arguments are as for extract_fixed_bit_field.  */
    2447              : 
    2448              : static rtx
    2449       196307 : extract_fixed_bit_field_1 (machine_mode tmode, rtx op0, scalar_int_mode mode,
    2450              :                            unsigned HOST_WIDE_INT bitsize,
    2451              :                            unsigned HOST_WIDE_INT bitnum, rtx target,
    2452              :                            int unsignedp, bool reverse)
    2453              : {
    2454              :   /* Note that bitsize + bitnum can be greater than GET_MODE_BITSIZE (mode)
    2455              :      for invalid input, such as extract equivalent of f5 from
    2456              :      gcc.dg/pr48335-2.c.  */
    2457              : 
    2458       196307 :   if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    2459              :     /* BITNUM is the distance between our msb and that of OP0.
    2460              :        Convert it to the distance from the lsb.  */
    2461          478 :     bitnum = GET_MODE_BITSIZE (mode) - bitsize - bitnum;
    2462              : 
    2463              :   /* Now BITNUM is always the distance between the field's lsb and that of OP0.
    2464              :      We have reduced the big-endian case to the little-endian case.  */
    2465       196307 :   if (reverse)
    2466          239 :     op0 = flip_storage_order (mode, op0);
    2467              : 
    2468       196307 :   if (unsignedp)
    2469              :     {
    2470       137298 :       if (bitnum)
    2471              :         {
    2472              :           /* If the field does not already start at the lsb,
    2473              :              shift it so it does.  */
    2474              :           /* Maybe propagate the target for the shift.  */
    2475        61509 :           rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
    2476        61509 :           if (tmode != mode)
    2477        40908 :             subtarget = 0;
    2478        61509 :           op0 = expand_shift (RSHIFT_EXPR, mode, op0, bitnum, subtarget, 1);
    2479              :         }
    2480              :       /* Convert the value to the desired mode.  TMODE must also be a
    2481              :          scalar integer for this conversion to make sense, since we
    2482              :          shouldn't reinterpret the bits.  */
    2483       137298 :       scalar_int_mode new_mode = as_a <scalar_int_mode> (tmode);
    2484       137298 :       if (mode != new_mode)
    2485        60325 :         op0 = convert_to_mode (new_mode, op0, 1);
    2486              : 
    2487              :       /* Unless the msb of the field used to be the msb when we shifted,
    2488              :          mask out the upper bits.  */
    2489              : 
    2490       137298 :       if (GET_MODE_BITSIZE (mode) != bitnum + bitsize
    2491       137298 :           && unsignedp != -1)
    2492        93686 :         return expand_binop (new_mode, and_optab, op0,
    2493              :                              mask_rtx (new_mode, 0, bitsize, 0),
    2494        93686 :                              target, 1, OPTAB_LIB_WIDEN);
    2495              :       return op0;
    2496              :     }
    2497              : 
    2498              :   /* To extract a signed bit-field, first shift its msb to the msb of the word,
    2499              :      then arithmetic-shift its lsb to the lsb of the word.  */
    2500        59009 :   op0 = force_reg (mode, op0);
    2501              : 
    2502              :   /* Find the narrowest integer mode that contains the field.  */
    2503              : 
    2504        59009 :   opt_scalar_int_mode mode_iter;
    2505       145987 :   FOR_EACH_MODE_IN_CLASS (mode_iter, MODE_INT)
    2506       291974 :     if (GET_MODE_BITSIZE (mode_iter.require ()) >= bitsize + bitnum)
    2507              :       break;
    2508              : 
    2509        59009 :   mode = mode_iter.require ();
    2510        59009 :   op0 = convert_to_mode (mode, op0, 0);
    2511              : 
    2512        59009 :   if (mode != tmode)
    2513         4427 :     target = 0;
    2514              : 
    2515       118018 :   if (GET_MODE_BITSIZE (mode) != (bitsize + bitnum))
    2516              :     {
    2517        54203 :       int amount = GET_MODE_BITSIZE (mode) - (bitsize + bitnum);
    2518              :       /* Maybe propagate the target for the shift.  */
    2519        54203 :       rtx subtarget = (target != 0 && REG_P (target) ? target : 0);
    2520        54203 :       op0 = expand_shift (LSHIFT_EXPR, mode, op0, amount, subtarget, 1);
    2521              :     }
    2522              : 
    2523       118018 :   return expand_shift (RSHIFT_EXPR, mode, op0,
    2524        59009 :                        GET_MODE_BITSIZE (mode) - bitsize, target, 0);
    2525              : }
    2526              : 
    2527              : /* Return a constant integer (CONST_INT or CONST_DOUBLE) rtx with the value
    2528              :    VALUE << BITPOS.  */
    2529              : 
    2530              : static rtx
    2531        73369 : lshift_value (machine_mode mode, unsigned HOST_WIDE_INT value,
    2532              :               int bitpos)
    2533              : {
    2534        73369 :   return immed_wide_int_const (wi::lshift (value, bitpos), mode);
    2535              : }
    2536              : 
    2537              : /* Extract a bit field that is split across two words
    2538              :    and return an RTX for the result.
    2539              : 
    2540              :    OP0 is the REG, SUBREG or MEM rtx for the first of the two words.
    2541              :    BITSIZE is the field width; BITPOS, position of its first bit, in the word.
    2542              :    UNSIGNEDP is 1 if should zero-extend the contents; else sign-extend.
    2543              :    If OP0_MODE is defined, it is the mode of OP0, otherwise OP0 is
    2544              :    a BLKmode MEM.
    2545              : 
    2546              :    If REVERSE is true, the extraction is to be done in reverse order.  */
    2547              : 
    2548              : static rtx
    2549         4343 : extract_split_bit_field (rtx op0, opt_scalar_int_mode op0_mode,
    2550              :                          unsigned HOST_WIDE_INT bitsize,
    2551              :                          unsigned HOST_WIDE_INT bitpos, int unsignedp,
    2552              :                          bool reverse)
    2553              : {
    2554         4343 :   unsigned int unit;
    2555         4343 :   unsigned int bitsdone = 0;
    2556         4343 :   rtx result = NULL_RTX;
    2557         4343 :   int first = 1;
    2558              : 
    2559              :   /* Make sure UNIT isn't larger than BITS_PER_WORD, we can only handle that
    2560              :      much at a time.  */
    2561         4343 :   if (REG_P (op0) || GET_CODE (op0) == SUBREG)
    2562           60 :     unit = BITS_PER_WORD;
    2563              :   else
    2564         6362 :     unit = MIN (MEM_ALIGN (op0), BITS_PER_WORD);
    2565              : 
    2566        18097 :   while (bitsdone < bitsize)
    2567              :     {
    2568        13754 :       unsigned HOST_WIDE_INT thissize;
    2569        13754 :       rtx part;
    2570        13754 :       unsigned HOST_WIDE_INT thispos;
    2571        13754 :       unsigned HOST_WIDE_INT offset;
    2572              : 
    2573        13754 :       offset = (bitpos + bitsdone) / unit;
    2574        13754 :       thispos = (bitpos + bitsdone) % unit;
    2575              : 
    2576              :       /* THISSIZE must not overrun a word boundary.  Otherwise,
    2577              :          extract_fixed_bit_field will call us again, and we will mutually
    2578              :          recurse forever.  */
    2579        13754 :       thissize = MIN (bitsize - bitsdone, BITS_PER_WORD);
    2580        13754 :       thissize = MIN (thissize, unit - thispos);
    2581              : 
    2582              :       /* If OP0 is a register, then handle OFFSET here.  */
    2583        13754 :       rtx op0_piece = op0;
    2584        13754 :       opt_scalar_int_mode op0_piece_mode = op0_mode;
    2585        13754 :       if (SUBREG_P (op0) || REG_P (op0))
    2586              :         {
    2587          120 :           op0_piece = operand_subword_force (op0, offset, op0_mode.require ());
    2588          120 :           op0_piece_mode = word_mode;
    2589          120 :           offset = 0;
    2590              :         }
    2591              : 
    2592              :       /* Extract the parts in bit-counting order,
    2593              :          whose meaning is determined by BYTES_PER_UNIT.
    2594              :          OFFSET is in UNITs, and UNIT is in bits.  */
    2595        27508 :       part = extract_fixed_bit_field (word_mode, op0_piece, op0_piece_mode,
    2596        13754 :                                       thissize, offset * unit + thispos,
    2597              :                                       0, 1, reverse);
    2598        13754 :       bitsdone += thissize;
    2599              : 
    2600              :       /* Shift this part into place for the result.  */
    2601        13754 :       if (reverse ? !BYTES_BIG_ENDIAN : BYTES_BIG_ENDIAN)
    2602              :         {
    2603            4 :           if (bitsize != bitsdone)
    2604            2 :             part = expand_shift (LSHIFT_EXPR, word_mode, part,
    2605            2 :                                  bitsize - bitsdone, 0, 1);
    2606              :         }
    2607              :       else
    2608              :         {
    2609        13750 :           if (bitsdone != thissize)
    2610         9409 :             part = expand_shift (LSHIFT_EXPR, word_mode, part,
    2611         9409 :                                  bitsdone - thissize, 0, 1);
    2612              :         }
    2613              : 
    2614        13754 :       if (first)
    2615              :         result = part;
    2616              :       else
    2617              :         /* Combine the parts with bitwise or.  This works
    2618              :            because we extracted each part as an unsigned bit field.  */
    2619         9411 :         result = expand_binop (word_mode, ior_optab, part, result, NULL_RTX, 1,
    2620              :                                OPTAB_LIB_WIDEN);
    2621              : 
    2622        13754 :       first = 0;
    2623              :     }
    2624              : 
    2625              :   /* Unsigned bit field: we are done.  */
    2626         4343 :   if (unsignedp)
    2627              :     return result;
    2628              :   /* Signed bit field: sign-extend with two arithmetic shifts.  */
    2629         1464 :   result = expand_shift (LSHIFT_EXPR, word_mode, result,
    2630         1464 :                          BITS_PER_WORD - bitsize, NULL_RTX, 0);
    2631         1464 :   return expand_shift (RSHIFT_EXPR, word_mode, result,
    2632         1464 :                        BITS_PER_WORD - bitsize, NULL_RTX, 0);
    2633              : }
    2634              : 
    2635              : /* Try to read the low bits of SRC as an rvalue of mode MODE, preserving
    2636              :    the bit pattern.  SRC_MODE is the mode of SRC; if this is smaller than
    2637              :    MODE, fill the upper bits with zeros.  Fail if the layout of either
    2638              :    mode is unknown (as for CC modes) or if the extraction would involve
    2639              :    unprofitable mode punning.  Return the value on success, otherwise
    2640              :    return null.
    2641              : 
    2642              :    This is different from gen_lowpart* in these respects:
    2643              : 
    2644              :      - the returned value must always be considered an rvalue
    2645              : 
    2646              :      - when MODE is wider than SRC_MODE, the extraction involves
    2647              :        a zero extension
    2648              : 
    2649              :      - when MODE is smaller than SRC_MODE, the extraction involves
    2650              :        a truncation (and is thus subject to TARGET_TRULY_NOOP_TRUNCATION).
    2651              : 
    2652              :    In other words, this routine performs a computation, whereas the
    2653              :    gen_lowpart* routines are conceptually lvalue or rvalue subreg
    2654              :    operations.  */
    2655              : 
    2656              : rtx
    2657       119680 : extract_low_bits (machine_mode mode, machine_mode src_mode, rtx src)
    2658              : {
    2659       119680 :   scalar_int_mode int_mode, src_int_mode;
    2660              : 
    2661       119680 :   if (mode == src_mode)
    2662              :     return src;
    2663              : 
    2664        81937 :   if (CONSTANT_P (src))
    2665              :     {
    2666              :       /* simplify_gen_subreg can't be used here, as if simplify_subreg
    2667              :          fails, it will happily create (subreg (symbol_ref)) or similar
    2668              :          invalid SUBREGs.  */
    2669        19342 :       poly_uint64 byte = subreg_lowpart_offset (mode, src_mode);
    2670        19342 :       rtx ret = simplify_subreg (mode, src, src_mode, byte);
    2671        19342 :       if (ret)
    2672              :         return ret;
    2673              : 
    2674           22 :       if (GET_MODE (src) == VOIDmode
    2675           22 :           || !validate_subreg (mode, src_mode, src, byte))
    2676              :         return NULL_RTX;
    2677              : 
    2678           17 :       src = force_reg (GET_MODE (src), src);
    2679           17 :       return gen_rtx_SUBREG (mode, src, byte);
    2680              :     }
    2681              : 
    2682        62595 :   if (GET_MODE_CLASS (mode) == MODE_CC || GET_MODE_CLASS (src_mode) == MODE_CC)
    2683              :     return NULL_RTX;
    2684              : 
    2685       125190 :   if (known_eq (GET_MODE_BITSIZE (mode), GET_MODE_BITSIZE (src_mode))
    2686        62595 :       && targetm.modes_tieable_p (mode, src_mode))
    2687              :     {
    2688         3972 :       rtx x = gen_lowpart_common (mode, src);
    2689         3972 :       if (x)
    2690              :         return x;
    2691              :     }
    2692              : 
    2693        58634 :   if (!int_mode_for_mode (src_mode).exists (&src_int_mode)
    2694        58634 :       || !int_mode_for_mode (mode).exists (&int_mode))
    2695              :     return NULL_RTX;
    2696              : 
    2697        58621 :   if (!targetm.modes_tieable_p (src_int_mode, src_mode))
    2698              :     return NULL_RTX;
    2699        57531 :   if (!targetm.modes_tieable_p (int_mode, mode))
    2700              :     return NULL_RTX;
    2701              : 
    2702        55458 :   src = gen_lowpart (src_int_mode, src);
    2703        55458 :   if (!validate_subreg (int_mode, src_int_mode, src,
    2704              :                         subreg_lowpart_offset (int_mode, src_int_mode)))
    2705              :     return NULL_RTX;
    2706              : 
    2707        55446 :   src = convert_modes (int_mode, src_int_mode, src, true);
    2708        55446 :   src = gen_lowpart (mode, src);
    2709        55446 :   return src;
    2710              : }
    2711              : 
    2712              : /* Add INC into TARGET.  */
    2713              : 
    2714              : void
    2715         1200 : expand_inc (rtx target, rtx inc)
    2716              : {
    2717         1200 :   rtx value = expand_binop (GET_MODE (target), add_optab,
    2718              :                             target, inc,
    2719              :                             target, 0, OPTAB_LIB_WIDEN);
    2720         1200 :   if (value != target)
    2721           61 :     emit_move_insn (target, value);
    2722         1200 : }
    2723              : 
    2724              : /* Subtract DEC from TARGET.  */
    2725              : 
    2726              : void
    2727         1235 : expand_dec (rtx target, rtx dec)
    2728              : {
    2729         1235 :   rtx value = expand_binop (GET_MODE (target), sub_optab,
    2730              :                             target, dec,
    2731              :                             target, 0, OPTAB_LIB_WIDEN);
    2732         1235 :   if (value != target)
    2733            0 :     emit_move_insn (target, value);
    2734         1235 : }
    2735              : 
    2736              : /* Output a shift instruction for expression code CODE,
    2737              :    with SHIFTED being the rtx for the value to shift,
    2738              :    and AMOUNT the rtx for the amount to shift by.
    2739              :    Store the result in the rtx TARGET, if that is convenient.
    2740              :    If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
    2741              :    Return the rtx for where the value is.
    2742              :    If that cannot be done, abort the compilation unless MAY_FAIL is true,
    2743              :    in which case 0 is returned.  */
    2744              : 
    2745              : static rtx
    2746      1559195 : expand_shift_1 (enum tree_code code, machine_mode mode, rtx shifted,
    2747              :                 rtx amount, rtx target, int unsignedp, bool may_fail = false)
    2748              : {
    2749      1559195 :   rtx op1, temp = 0;
    2750      1559195 :   int left = (code == LSHIFT_EXPR || code == LROTATE_EXPR);
    2751      1559195 :   int rotate = (code == LROTATE_EXPR || code == RROTATE_EXPR);
    2752      1559195 :   optab lshift_optab = ashl_optab;
    2753      1559195 :   optab rshift_arith_optab = ashr_optab;
    2754      1559195 :   optab rshift_uns_optab = lshr_optab;
    2755      1559195 :   optab lrotate_optab = rotl_optab;
    2756      1559195 :   optab rrotate_optab = rotr_optab;
    2757      1559195 :   machine_mode op1_mode;
    2758      1559195 :   scalar_mode scalar_mode = GET_MODE_INNER (mode);
    2759      1559195 :   int attempt;
    2760      1559195 :   bool speed = optimize_insn_for_speed_p ();
    2761              : 
    2762      1559195 :   op1 = amount;
    2763      1559195 :   op1_mode = GET_MODE (op1);
    2764              : 
    2765              :   /* Determine whether the shift/rotate amount is a vector, or scalar.  If the
    2766              :      shift amount is a vector, use the vector/vector shift patterns.  */
    2767      1559195 :   if (VECTOR_MODE_P (mode) && VECTOR_MODE_P (op1_mode))
    2768              :     {
    2769      1559195 :       lshift_optab = vashl_optab;
    2770      1559195 :       rshift_arith_optab = vashr_optab;
    2771      1559195 :       rshift_uns_optab = vlshr_optab;
    2772      1559195 :       lrotate_optab = vrotl_optab;
    2773      1559195 :       rrotate_optab = vrotr_optab;
    2774              :     }
    2775              : 
    2776              :   /* Previously detected shift-counts computed by NEGATE_EXPR
    2777              :      and shifted in the other direction; but that does not work
    2778              :      on all machines.  */
    2779              : 
    2780      1559195 :   if (SHIFT_COUNT_TRUNCATED)
    2781              :     {
    2782              :       if (CONST_INT_P (op1)
    2783              :           && ((unsigned HOST_WIDE_INT) INTVAL (op1) >=
    2784              :               (unsigned HOST_WIDE_INT) GET_MODE_BITSIZE (scalar_mode)))
    2785              :         op1 = gen_int_shift_amount (mode,
    2786              :                                     (unsigned HOST_WIDE_INT) INTVAL (op1)
    2787              :                                     % GET_MODE_BITSIZE (scalar_mode));
    2788              :       else if (GET_CODE (op1) == SUBREG
    2789              :                && subreg_lowpart_p (op1)
    2790              :                && SCALAR_INT_MODE_P (GET_MODE (SUBREG_REG (op1)))
    2791              :                && SCALAR_INT_MODE_P (GET_MODE (op1)))
    2792              :         op1 = SUBREG_REG (op1);
    2793              :     }
    2794              : 
    2795              :   /* Canonicalize rotates by constant amount.  We may canonicalize
    2796              :      to reduce the immediate or if the ISA can rotate by constants
    2797              :      in only on direction.  */
    2798      1559195 :   if (rotate && reverse_rotate_by_imm_p (scalar_mode, left, op1))
    2799              :     {
    2800         3292 :       op1 = gen_int_shift_amount (mode, (GET_MODE_BITSIZE (scalar_mode)
    2801         3292 :                                          - INTVAL (op1)));
    2802         3292 :       left = !left;
    2803         3292 :       code = left ? LROTATE_EXPR : RROTATE_EXPR;
    2804              :     }
    2805              : 
    2806              :   /* Rotation of 16bit values by 8 bits is effectively equivalent to a bswaphi.
    2807              :      Note that this is not the case for bigger values.  For instance a rotation
    2808              :      of 0x01020304 by 16 bits gives 0x03040102 which is different from
    2809              :      0x04030201 (bswapsi).  */
    2810      1559195 :   if (rotate
    2811         8657 :       && CONST_INT_P (op1)
    2812         5442 :       && INTVAL (op1) == BITS_PER_UNIT
    2813         1070 :       && GET_MODE_SIZE (scalar_mode) == 2
    2814      1560090 :       && optab_handler (bswap_optab, mode) != CODE_FOR_nothing)
    2815          894 :     return expand_unop (mode, bswap_optab, shifted, NULL_RTX, unsignedp);
    2816              : 
    2817      1558301 :   if (op1 == const0_rtx)
    2818              :     return shifted;
    2819              : 
    2820              :   /* Check whether its cheaper to implement a left shift by a constant
    2821              :      bit count by a sequence of additions.  */
    2822      1508329 :   if (code == LSHIFT_EXPR
    2823       903018 :       && CONST_INT_P (op1)
    2824       872564 :       && INTVAL (op1) > 0
    2825       872539 :       && INTVAL (op1) < GET_MODE_PRECISION (scalar_mode)
    2826       872539 :       && INTVAL (op1) < MAX_BITS_PER_WORD
    2827       867766 :       && (shift_cost (speed, mode, INTVAL (op1))
    2828       867766 :           > INTVAL (op1) * add_cost (speed, mode))
    2829      1511378 :       && shift_cost (speed, mode, INTVAL (op1)) != MAX_COST)
    2830              :     {
    2831              :       int i;
    2832         6286 :       for (i = 0; i < INTVAL (op1); i++)
    2833              :         {
    2834         3237 :           temp = force_reg (mode, shifted);
    2835         3237 :           shifted = expand_binop (mode, add_optab, temp, temp, NULL_RTX,
    2836              :                                   unsignedp, OPTAB_LIB_WIDEN);
    2837              :         }
    2838              :       return shifted;
    2839              :     }
    2840              : 
    2841      3010598 :   for (attempt = 0; temp == 0 && attempt < 3; attempt++)
    2842              :     {
    2843      1505356 :       enum optab_methods methods;
    2844              : 
    2845      1505356 :       if (attempt == 0)
    2846              :         methods = OPTAB_DIRECT;
    2847           76 :       else if (attempt == 1)
    2848              :         methods = OPTAB_WIDEN;
    2849              :       else
    2850           38 :         methods = OPTAB_LIB_WIDEN;
    2851              : 
    2852      1505356 :       if (rotate)
    2853              :         {
    2854              :           /* Widening does not work for rotation.  */
    2855         7839 :           if (methods == OPTAB_WIDEN)
    2856           38 :             continue;
    2857         7801 :           else if (methods == OPTAB_LIB_WIDEN)
    2858              :             {
    2859              :               /* If we have been unable to open-code this by a rotation,
    2860              :                  do it as the IOR or PLUS of two shifts.  I.e., to rotate
    2861              :                  A by N bits, compute
    2862              :                  (A << N) | ((unsigned) A >> ((-N) & (C - 1)))
    2863              :                  where C is the bitsize of A.  If N cannot be zero,
    2864              :                  use PLUS instead of IOR.
    2865              : 
    2866              :                  It is theoretically possible that the target machine might
    2867              :                  not be able to perform either shift and hence we would
    2868              :                  be making two libcalls rather than just the one for the
    2869              :                  shift (similarly if IOR could not be done).  We will allow
    2870              :                  this extremely unlikely lossage to avoid complicating the
    2871              :                  code below.  */
    2872              : 
    2873           38 :               rtx subtarget = target == shifted ? 0 : target;
    2874           38 :               rtx new_amount, other_amount;
    2875           38 :               rtx temp1;
    2876              : 
    2877           38 :               new_amount = op1;
    2878           38 :               if (op1 == const0_rtx)
    2879              :                 return shifted;
    2880           38 :               else if (CONST_INT_P (op1))
    2881           23 :                 other_amount = gen_int_shift_amount
    2882           23 :                   (mode, GET_MODE_BITSIZE (scalar_mode) - INTVAL (op1));
    2883              :               else
    2884              :                 {
    2885           15 :                   other_amount
    2886           30 :                     = simplify_gen_unary (NEG, GET_MODE (op1),
    2887           15 :                                           op1, GET_MODE (op1));
    2888           15 :                   HOST_WIDE_INT mask = GET_MODE_PRECISION (scalar_mode) - 1;
    2889           15 :                   other_amount
    2890           15 :                     = simplify_gen_binary (AND, GET_MODE (op1), other_amount,
    2891           15 :                                            gen_int_mode (mask, GET_MODE (op1)));
    2892              :                 }
    2893              : 
    2894           38 :               shifted = force_reg (mode, shifted);
    2895              : 
    2896           51 :               temp = expand_shift_1 (left ? LSHIFT_EXPR : RSHIFT_EXPR,
    2897              :                                      mode, shifted, new_amount, 0, 1);
    2898           51 :               temp1 = expand_shift_1 (left ? RSHIFT_EXPR : LSHIFT_EXPR,
    2899              :                                       mode, shifted, other_amount,
    2900              :                                       subtarget, 1);
    2901           38 :               return expand_binop (mode,
    2902           38 :                                    CONST_INT_P (op1) ? add_optab : ior_optab,
    2903           38 :                                    temp, temp1, target, unsignedp, methods);
    2904              :             }
    2905              : 
    2906        11369 :           temp = expand_binop (mode,
    2907              :                                left ? lrotate_optab : rrotate_optab,
    2908              :                                shifted, op1, target, unsignedp, methods);
    2909              :         }
    2910      1497517 :       else if (unsignedp)
    2911      1246695 :         temp = expand_binop (mode,
    2912              :                              left ? lshift_optab : rshift_uns_optab,
    2913              :                              shifted, op1, target, unsignedp, methods);
    2914              : 
    2915              :       /* Do arithmetic shifts.
    2916              :          Also, if we are going to widen the operand, we can just as well
    2917              :          use an arithmetic right-shift instead of a logical one.  */
    2918      1505280 :       if (temp == 0 && ! rotate
    2919       600894 :           && (! unsignedp || (! left && methods == OPTAB_WIDEN)))
    2920              :         {
    2921              :           enum optab_methods methods1 = methods;
    2922              : 
    2923              :           /* If trying to widen a log shift to an arithmetic shift,
    2924              :              don't accept an arithmetic shift of the same size.  */
    2925              :           if (unsignedp)
    2926              :             methods1 = OPTAB_MUST_WIDEN;
    2927              : 
    2928              :           /* Arithmetic shift */
    2929              : 
    2930       848370 :           temp = expand_binop (mode,
    2931              :                                left ? lshift_optab : rshift_arith_optab,
    2932              :                                shifted, op1, target, unsignedp, methods1);
    2933              :         }
    2934              : 
    2935              :       /* We used to try extzv here for logical right shifts, but that was
    2936              :          only useful for one machine, the VAX, and caused poor code
    2937              :          generation there for lshrdi3, so the code was deleted and a
    2938              :          define_expand for lshrsi3 was added to vax.md.  */
    2939              :     }
    2940              : 
    2941      1505242 :   gcc_assert (temp != NULL_RTX || may_fail);
    2942              :   return temp;
    2943              : }
    2944              : 
    2945              : /* Output a shift instruction for expression code CODE,
    2946              :    with SHIFTED being the rtx for the value to shift,
    2947              :    and AMOUNT the amount to shift by.
    2948              :    Store the result in the rtx TARGET, if that is convenient.
    2949              :    If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
    2950              :    Return the rtx for where the value is.  */
    2951              : 
    2952              : rtx
    2953      1247585 : expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
    2954              :               poly_int64 amount, rtx target, int unsignedp)
    2955              : {
    2956      1247585 :   return expand_shift_1 (code, mode, shifted,
    2957              :                          gen_int_shift_amount (mode, amount),
    2958      1247585 :                          target, unsignedp);
    2959              : }
    2960              : 
    2961              : /* Likewise, but return 0 if that cannot be done.  */
    2962              : 
    2963              : rtx
    2964          325 : maybe_expand_shift (enum tree_code code, machine_mode mode, rtx shifted,
    2965              :                     int amount, rtx target, int unsignedp)
    2966              : {
    2967          325 :   return expand_shift_1 (code, mode,
    2968          325 :                          shifted, GEN_INT (amount), target, unsignedp, true);
    2969              : }
    2970              : 
    2971              : /* Output a shift instruction for expression code CODE,
    2972              :    with SHIFTED being the rtx for the value to shift,
    2973              :    and AMOUNT the tree for the amount to shift by.
    2974              :    Store the result in the rtx TARGET, if that is convenient.
    2975              :    If UNSIGNEDP is nonzero, do a logical shift; otherwise, arithmetic.
    2976              :    Return the rtx for where the value is.  */
    2977              : 
    2978              : rtx
    2979       311209 : expand_variable_shift (enum tree_code code, machine_mode mode, rtx shifted,
    2980              :                        tree amount, rtx target, int unsignedp)
    2981              : {
    2982       311209 :   return expand_shift_1 (code, mode,
    2983       311209 :                          shifted, expand_normal (amount), target, unsignedp);
    2984              : }
    2985              : 
    2986              : 
    2987              : static void synth_mult (struct algorithm *, unsigned HOST_WIDE_INT,
    2988              :                         const struct mult_cost *, machine_mode mode);
    2989              : static rtx expand_mult_const (machine_mode, rtx, HOST_WIDE_INT, rtx,
    2990              :                               const struct algorithm *, enum mult_variant);
    2991              : static unsigned HOST_WIDE_INT invert_mod2n (unsigned HOST_WIDE_INT, int);
    2992              : static rtx extract_high_half (scalar_int_mode, rtx);
    2993              : static rtx expmed_mult_highpart (scalar_int_mode, rtx, rtx, rtx, int, int);
    2994              : 
    2995              : /* Compute and return the best algorithm for multiplying by T.
    2996              :    The algorithm must cost less than cost_limit
    2997              :    If retval.cost >= COST_LIMIT, no algorithm was found and all
    2998              :    other field of the returned struct are undefined.
    2999              :    MODE is the machine mode of the multiplication.  */
    3000              : 
    3001              : static void
    3002     38852805 : synth_mult (struct algorithm *alg_out, unsigned HOST_WIDE_INT t,
    3003              :             const struct mult_cost *cost_limit, machine_mode mode)
    3004              : {
    3005     38852805 :   int m;
    3006     38852805 :   struct algorithm *alg_in, *best_alg;
    3007     38852805 :   struct mult_cost best_cost;
    3008     38852805 :   struct mult_cost new_limit;
    3009     38852805 :   int op_cost, op_latency;
    3010     38852805 :   unsigned HOST_WIDE_INT orig_t = t;
    3011     38852805 :   unsigned HOST_WIDE_INT q;
    3012     38852805 :   int maxm, hash_index;
    3013     38852805 :   bool cache_hit = false;
    3014     38852805 :   enum alg_code cache_alg = alg_zero;
    3015     38852805 :   bool speed = optimize_insn_for_speed_p ();
    3016     38852805 :   scalar_int_mode imode;
    3017     38852805 :   struct alg_hash_entry *entry_ptr;
    3018              : 
    3019              :   /* Indicate that no algorithm is yet found.  If no algorithm
    3020              :      is found, this value will be returned and indicate failure.  */
    3021     38852805 :   alg_out->cost.cost = cost_limit->cost + 1;
    3022     38852805 :   alg_out->cost.latency = cost_limit->latency + 1;
    3023              : 
    3024     38852805 :   if (cost_limit->cost < 0
    3025     31675199 :       || (cost_limit->cost == 0 && cost_limit->latency <= 0))
    3026     31410494 :     return;
    3027              : 
    3028              :   /* Be prepared for vector modes.  */
    3029     52495560 :   imode = as_a <scalar_int_mode> (GET_MODE_INNER (mode));
    3030              : 
    3031     77311683 :   maxm = MIN (BITS_PER_WORD, GET_MODE_BITSIZE (imode));
    3032              : 
    3033              :   /* Restrict the bits of "t" to the multiplication's mode.  */
    3034     26247780 :   t &= GET_MODE_MASK (imode);
    3035              : 
    3036              :   /* t == 1 can be done in zero cost.  */
    3037     26247780 :   if (t == 1)
    3038              :     {
    3039      6049887 :       alg_out->ops = 1;
    3040      6049887 :       alg_out->cost.cost = 0;
    3041      6049887 :       alg_out->cost.latency = 0;
    3042      6049887 :       alg_out->op[0] = alg_m;
    3043      6049887 :       return;
    3044              :     }
    3045              : 
    3046              :   /* t == 0 sometimes has a cost.  If it does and it exceeds our limit,
    3047              :      fail now.  */
    3048     20197893 :   if (t == 0)
    3049              :     {
    3050       535853 :       if (MULT_COST_LESS (cost_limit, zero_cost (speed)))
    3051              :         return;
    3052              :       else
    3053              :         {
    3054       535853 :           alg_out->ops = 1;
    3055       535853 :           alg_out->cost.cost = zero_cost (speed);
    3056       535853 :           alg_out->cost.latency = zero_cost (speed);
    3057       535853 :           alg_out->op[0] = alg_zero;
    3058       535853 :           return;
    3059              :         }
    3060              :     }
    3061              : 
    3062              :   /* We'll be needing a couple extra algorithm structures now.  */
    3063              : 
    3064     19662040 :   alg_in = XALLOCA (struct algorithm);
    3065     19662040 :   best_alg = XALLOCA (struct algorithm);
    3066     19662040 :   best_cost = *cost_limit;
    3067              : 
    3068              :   /* Compute the hash index.  */
    3069     19662040 :   hash_index = (t ^ (unsigned int) mode ^ (speed * 256)) % NUM_ALG_HASH_ENTRIES;
    3070              : 
    3071              :   /* See if we already know what to do for T.  */
    3072     19662040 :   entry_ptr = alg_hash_entry_ptr (hash_index);
    3073     19662040 :   if (entry_ptr->t == t
    3074     15389564 :       && entry_ptr->mode == mode
    3075     15389564 :       && entry_ptr->speed == speed
    3076     15389564 :       && entry_ptr->alg != alg_unknown)
    3077              :     {
    3078     15389564 :       cache_alg = entry_ptr->alg;
    3079              : 
    3080     15389564 :       if (cache_alg == alg_impossible)
    3081              :         {
    3082              :           /* The cache tells us that it's impossible to synthesize
    3083              :              multiplication by T within entry_ptr->cost.  */
    3084      7580459 :           if (!CHEAPER_MULT_COST (&entry_ptr->cost, cost_limit))
    3085              :             /* COST_LIMIT is at least as restrictive as the one
    3086              :                recorded in the hash table, in which case we have no
    3087              :                hope of synthesizing a multiplication.  Just
    3088              :                return.  */
    3089              :             return;
    3090              : 
    3091              :           /* If we get here, COST_LIMIT is less restrictive than the
    3092              :              one recorded in the hash table, so we may be able to
    3093              :              synthesize a multiplication.  Proceed as if we didn't
    3094              :              have the cache entry.  */
    3095              :         }
    3096              :       else
    3097              :         {
    3098      7809105 :           if (CHEAPER_MULT_COST (cost_limit, &entry_ptr->cost))
    3099              :             /* The cached algorithm shows that this multiplication
    3100              :                requires more cost than COST_LIMIT.  Just return.  This
    3101              :                way, we don't clobber this cache entry with
    3102              :                alg_impossible but retain useful information.  */
    3103              :             return;
    3104              : 
    3105      7046022 :           cache_hit = true;
    3106              : 
    3107      7046022 :           switch (cache_alg)
    3108              :             {
    3109      4640749 :             case alg_shift:
    3110      4640749 :               goto do_alg_shift;
    3111              : 
    3112       952122 :             case alg_add_t_m2:
    3113       952122 :             case alg_sub_t_m2:
    3114       952122 :               goto do_alg_addsub_t_m2;
    3115              : 
    3116       120935 :             case alg_add_factor:
    3117       120935 :             case alg_sub_factor:
    3118       120935 :               goto do_alg_addsub_factor;
    3119              : 
    3120      1332205 :             case alg_add_t2_m:
    3121      1332205 :               goto do_alg_add_t2_m;
    3122              : 
    3123           11 :             case alg_sub_t2_m:
    3124           11 :               goto do_alg_sub_t2_m;
    3125              : 
    3126            0 :             default:
    3127            0 :               gcc_unreachable ();
    3128              :             }
    3129              :         }
    3130              :     }
    3131              : 
    3132              :   /* If we have a group of zero bits at the low-order part of T, try
    3133              :      multiplying by the remaining bits and then doing a shift.  */
    3134              : 
    3135      5286620 :   if ((t & 1) == 0)
    3136              :     {
    3137      2715041 :     do_alg_shift:
    3138      7355790 :       m = ctz_or_zero (t); /* m = number of low zero bits */
    3139      7355790 :       if (m < maxm)
    3140              :         {
    3141      7354050 :           q = t >> m;
    3142              :           /* The function expand_shift will choose between a shift and
    3143              :              a sequence of additions, so the observed cost is given as
    3144              :              MIN (m * add_cost(speed, mode), shift_cost(speed, mode, m)).  */
    3145      7354050 :           op_cost = m * add_cost (speed, mode);
    3146      7354050 :           if (shift_cost (speed, mode, m) < op_cost)
    3147              :             op_cost = shift_cost (speed, mode, m);
    3148      7354050 :           new_limit.cost = best_cost.cost - op_cost;
    3149      7354050 :           new_limit.latency = best_cost.latency - op_cost;
    3150      7354050 :           synth_mult (alg_in, q, &new_limit, mode);
    3151              : 
    3152      7354050 :           alg_in->cost.cost += op_cost;
    3153      7354050 :           alg_in->cost.latency += op_cost;
    3154      7354050 :           if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3155              :             {
    3156      4269357 :               best_cost = alg_in->cost;
    3157      4269357 :               std::swap (alg_in, best_alg);
    3158      4269357 :               best_alg->log[best_alg->ops] = m;
    3159      4269357 :               best_alg->op[best_alg->ops] = alg_shift;
    3160              :             }
    3161              : 
    3162              :           /* See if treating ORIG_T as a signed number yields a better
    3163              :              sequence.  Try this sequence only for a negative ORIG_T
    3164              :              as it would be useless for a non-negative ORIG_T.  */
    3165      7354050 :           if ((HOST_WIDE_INT) orig_t < 0)
    3166              :             {
    3167              :               /* Shift ORIG_T as follows because a right shift of a
    3168              :                  negative-valued signed type is implementation
    3169              :                  defined.  */
    3170       633145 :               q = ~(~orig_t >> m);
    3171              :               /* The function expand_shift will choose between a shift
    3172              :                  and a sequence of additions, so the observed cost is
    3173              :                  given as MIN (m * add_cost(speed, mode),
    3174              :                  shift_cost(speed, mode, m)).  */
    3175       633145 :               op_cost = m * add_cost (speed, mode);
    3176       633145 :               if (shift_cost (speed, mode, m) < op_cost)
    3177              :                 op_cost = shift_cost (speed, mode, m);
    3178       633145 :               new_limit.cost = best_cost.cost - op_cost;
    3179       633145 :               new_limit.latency = best_cost.latency - op_cost;
    3180       633145 :               synth_mult (alg_in, q, &new_limit, mode);
    3181              : 
    3182       633145 :               alg_in->cost.cost += op_cost;
    3183       633145 :               alg_in->cost.latency += op_cost;
    3184       633145 :               if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3185              :                 {
    3186       600998 :                   best_cost = alg_in->cost;
    3187       600998 :                   std::swap (alg_in, best_alg);
    3188       600998 :                   best_alg->log[best_alg->ops] = m;
    3189       600998 :                   best_alg->op[best_alg->ops] = alg_shift;
    3190              :                 }
    3191              :             }
    3192              :         }
    3193         1740 :       else if (2 * BITS_PER_WORD <= HOST_BITS_PER_WIDE_INT
    3194         1740 :                && GET_MODE_BITSIZE (imode) == 2 * BITS_PER_WORD
    3195         1728 :                && m >= BITS_PER_WORD
    3196         3468 :                && imode == mode)
    3197              :         {
    3198         1728 :           q = t >> m;
    3199         1728 :           int op1_cost = shift_cost (speed, mode, m - BITS_PER_WORD);
    3200         1728 :           int op2_cost = zero_cost (speed);
    3201         1728 :           op_latency = MAX (op1_cost, op2_cost);
    3202         1728 :           op_cost = op1_cost + op2_cost;
    3203              : 
    3204         1728 :           new_limit.cost = best_cost.cost - op_cost;
    3205         1728 :           new_limit.latency = best_cost.latency - op_latency;
    3206         1728 :           synth_mult (alg_in, q, &new_limit, mode);
    3207         1728 :           alg_in->cost.cost += op_cost;
    3208         1728 :           alg_in->cost.latency += op_latency;
    3209         1728 :           if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3210              :             {
    3211         1662 :               best_cost = alg_in->cost;
    3212         1662 :               std::swap (alg_in, best_alg);
    3213         1662 :               best_alg->log[best_alg->ops] = m;
    3214         1662 :               best_alg->op[best_alg->ops] = alg_shift;
    3215              :             }
    3216              :         }
    3217      7355790 :       if (cache_hit)
    3218      4640749 :         goto done;
    3219              :     }
    3220              : 
    3221              :   /* If we have an odd number, add or subtract one.  */
    3222      2715041 :   if ((t & 1) != 0)
    3223              :     {
    3224      3523701 :       unsigned HOST_WIDE_INT w;
    3225              : 
    3226            0 :     do_alg_addsub_t_m2:
    3227     42476792 :       for (w = 1; (w & t) != 0; w <<= 1)
    3228              :         ;
    3229              :       /* If T was -1, then W will be zero after the loop.  This is another
    3230              :          case where T ends with ...111.  Handling this with (T + 1) and
    3231              :          subtract 1 produces slightly better code and results in algorithm
    3232              :          selection much faster than treating it like the ...0111 case
    3233              :          below.  */
    3234      3523701 :       if (w == 0
    3235      3104815 :           || (w > 2
    3236              :               /* Reject the case where t is 3.
    3237              :                  Thus we prefer addition in that case.  */
    3238      3104815 :               && t != 3))
    3239              :         {
    3240              :           /* T ends with ...111.  Multiply by (T + 1) and subtract T.  */
    3241              : 
    3242      1933486 :           op_cost = add_cost (speed, mode);
    3243      1933486 :           new_limit.cost = best_cost.cost - op_cost;
    3244      1933486 :           new_limit.latency = best_cost.latency - op_cost;
    3245      1933486 :           synth_mult (alg_in, t + 1, &new_limit, mode);
    3246              : 
    3247      1933486 :           alg_in->cost.cost += op_cost;
    3248      1933486 :           alg_in->cost.latency += op_cost;
    3249      1933486 :           if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3250              :             {
    3251       720385 :               best_cost = alg_in->cost;
    3252       720385 :               std::swap (alg_in, best_alg);
    3253       720385 :               best_alg->log[best_alg->ops] = 0;
    3254       720385 :               best_alg->op[best_alg->ops] = alg_sub_t_m2;
    3255              :             }
    3256              :         }
    3257              :       else
    3258              :         {
    3259              :           /* T ends with ...01 or ...011.  Multiply by (T - 1) and add T.  */
    3260              : 
    3261      1590215 :           op_cost = add_cost (speed, mode);
    3262      1590215 :           new_limit.cost = best_cost.cost - op_cost;
    3263      1590215 :           new_limit.latency = best_cost.latency - op_cost;
    3264      1590215 :           synth_mult (alg_in, t - 1, &new_limit, mode);
    3265              : 
    3266      1590215 :           alg_in->cost.cost += op_cost;
    3267      1590215 :           alg_in->cost.latency += op_cost;
    3268      1590215 :           if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3269              :             {
    3270       197738 :               best_cost = alg_in->cost;
    3271       197738 :               std::swap (alg_in, best_alg);
    3272       197738 :               best_alg->log[best_alg->ops] = 0;
    3273       197738 :               best_alg->op[best_alg->ops] = alg_add_t_m2;
    3274              :             }
    3275              :         }
    3276              : 
    3277              :       /* We may be able to calculate a * -7, a * -15, a * -31, etc
    3278              :          quickly with a - a * n for some appropriate constant n.  */
    3279      3523701 :       m = exact_log2 (-orig_t + 1);
    3280      3523701 :       if (m >= 0 && m < maxm)
    3281              :         {
    3282       743633 :           op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
    3283              :           /* If the target has a cheap shift-and-subtract insn use
    3284              :              that in preference to a shift insn followed by a sub insn.
    3285              :              Assume that the shift-and-sub is "atomic" with a latency
    3286              :              equal to it's cost, otherwise assume that on superscalar
    3287              :              hardware the shift may be executed concurrently with the
    3288              :              earlier steps in the algorithm.  */
    3289       743633 :           if (shiftsub1_cost (speed, mode, m) <= op_cost)
    3290              :             {
    3291              :               op_cost = shiftsub1_cost (speed, mode, m);
    3292              :               op_latency = op_cost;
    3293              :             }
    3294              :           else
    3295       737759 :             op_latency = add_cost (speed, mode);
    3296              : 
    3297       743633 :           new_limit.cost = best_cost.cost - op_cost;
    3298       743633 :           new_limit.latency = best_cost.latency - op_latency;
    3299       743633 :           synth_mult (alg_in, (unsigned HOST_WIDE_INT) (-orig_t + 1) >> m,
    3300              :                       &new_limit, mode);
    3301              : 
    3302       743633 :           alg_in->cost.cost += op_cost;
    3303       743633 :           alg_in->cost.latency += op_latency;
    3304       743633 :           if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3305              :             {
    3306       191358 :               best_cost = alg_in->cost;
    3307       191358 :               std::swap (alg_in, best_alg);
    3308       191358 :               best_alg->log[best_alg->ops] = m;
    3309       191358 :               best_alg->op[best_alg->ops] = alg_sub_t_m2;
    3310              :             }
    3311              :         }
    3312              : 
    3313      3523701 :       if (cache_hit)
    3314       952122 :         goto done;
    3315              :     }
    3316              : 
    3317              :   /* Look for factors of t of the form
    3318              :      t = q(2**m +- 1), 2 <= m <= floor(log2(t - 1)).
    3319              :      If we find such a factor, we can multiply by t using an algorithm that
    3320              :      multiplies by q, shift the result by m and add/subtract it to itself.
    3321              : 
    3322              :      We search for large factors first and loop down, even if large factors
    3323              :      are less probable than small; if we find a large factor we will find a
    3324              :      good sequence quickly, and therefore be able to prune (by decreasing
    3325              :      COST_LIMIT) the search.  */
    3326              : 
    3327      2715041 :  do_alg_addsub_factor:
    3328    114360357 :   for (m = floor_log2 (t - 1); m >= 2; m--)
    3329              :     {
    3330    111742992 :       unsigned HOST_WIDE_INT d;
    3331              : 
    3332    111742992 :       d = (HOST_WIDE_INT_1U << m) + 1;
    3333    111742992 :       if (t % d == 0 && t > d && m < maxm
    3334      1305287 :           && (!cache_hit || cache_alg == alg_add_factor))
    3335              :         {
    3336      1305287 :           op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
    3337      1305287 :           if (shiftadd_cost (speed, mode, m) <= op_cost)
    3338              :             op_cost = shiftadd_cost (speed, mode, m);
    3339              : 
    3340      1305287 :           op_latency = op_cost;
    3341              : 
    3342              : 
    3343      1305287 :           new_limit.cost = best_cost.cost - op_cost;
    3344      1305287 :           new_limit.latency = best_cost.latency - op_latency;
    3345      1305287 :           synth_mult (alg_in, t / d, &new_limit, mode);
    3346              : 
    3347      1305287 :           alg_in->cost.cost += op_cost;
    3348      1305287 :           alg_in->cost.latency += op_latency;
    3349      1305287 :           if (alg_in->cost.latency < op_cost)
    3350       263947 :             alg_in->cost.latency = op_cost;
    3351      1305287 :           if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3352              :             {
    3353       118733 :               best_cost = alg_in->cost;
    3354       118733 :               std::swap (alg_in, best_alg);
    3355       118733 :               best_alg->log[best_alg->ops] = m;
    3356       118733 :               best_alg->op[best_alg->ops] = alg_add_factor;
    3357              :             }
    3358              :           /* Other factors will have been taken care of in the recursion.  */
    3359              :           break;
    3360              :         }
    3361              : 
    3362    110437705 :       d = (HOST_WIDE_INT_1U << m) - 1;
    3363    110437705 :       if (t % d == 0 && t > d && m < maxm
    3364      1484903 :           && (!cache_hit || cache_alg == alg_sub_factor))
    3365              :         {
    3366      1484903 :           op_cost = add_cost (speed, mode) + shift_cost (speed, mode, m);
    3367      1484903 :           if (shiftsub0_cost (speed, mode, m) <= op_cost)
    3368              :             op_cost = shiftsub0_cost (speed, mode, m);
    3369              : 
    3370      1484903 :           op_latency = op_cost;
    3371              : 
    3372      1484903 :           new_limit.cost = best_cost.cost - op_cost;
    3373      1484903 :           new_limit.latency = best_cost.latency - op_latency;
    3374      1484903 :           synth_mult (alg_in, t / d, &new_limit, mode);
    3375              : 
    3376      1484903 :           alg_in->cost.cost += op_cost;
    3377      1484903 :           alg_in->cost.latency += op_latency;
    3378      1484903 :           if (alg_in->cost.latency < op_cost)
    3379       360987 :             alg_in->cost.latency = op_cost;
    3380      1484903 :           if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3381              :             {
    3382        33754 :               best_cost = alg_in->cost;
    3383        33754 :               std::swap (alg_in, best_alg);
    3384        33754 :               best_alg->log[best_alg->ops] = m;
    3385        33754 :               best_alg->op[best_alg->ops] = alg_sub_factor;
    3386              :             }
    3387              :           break;
    3388              :         }
    3389              :     }
    3390      5407555 :   if (cache_hit)
    3391       120935 :     goto done;
    3392              : 
    3393              :   /* Try shift-and-add (load effective address) instructions,
    3394              :      i.e. do a*3, a*5, a*9.  */
    3395      5286620 :   if ((t & 1) != 0)
    3396              :     {
    3397      2571579 :     do_alg_add_t2_m:
    3398      3903784 :       q = t - 1;
    3399      3903784 :       m = ctz_hwi (q);
    3400      3903784 :       if (q && m < maxm)
    3401              :         {
    3402      3903762 :           op_cost = shiftadd_cost (speed, mode, m);
    3403      3903762 :           new_limit.cost = best_cost.cost - op_cost;
    3404      3903762 :           new_limit.latency = best_cost.latency - op_cost;
    3405      3903762 :           synth_mult (alg_in, (t - 1) >> m, &new_limit, mode);
    3406              : 
    3407      3903762 :           alg_in->cost.cost += op_cost;
    3408      3903762 :           alg_in->cost.latency += op_cost;
    3409      3903762 :           if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3410              :             {
    3411      1387034 :               best_cost = alg_in->cost;
    3412      1387034 :               std::swap (alg_in, best_alg);
    3413      1387034 :               best_alg->log[best_alg->ops] = m;
    3414      1387034 :               best_alg->op[best_alg->ops] = alg_add_t2_m;
    3415              :             }
    3416              :         }
    3417      3903784 :       if (cache_hit)
    3418      1332205 :         goto done;
    3419              : 
    3420      2571579 :     do_alg_sub_t2_m:
    3421      2571590 :       q = t + 1;
    3422      2571590 :       m = ctz_hwi (q);
    3423      2571590 :       if (q && m < maxm)
    3424              :         {
    3425      2549073 :           op_cost = shiftsub0_cost (speed, mode, m);
    3426      2549073 :           new_limit.cost = best_cost.cost - op_cost;
    3427      2549073 :           new_limit.latency = best_cost.latency - op_cost;
    3428      2549073 :           synth_mult (alg_in, (t + 1) >> m, &new_limit, mode);
    3429              : 
    3430      2549073 :           alg_in->cost.cost += op_cost;
    3431      2549073 :           alg_in->cost.latency += op_cost;
    3432      2549073 :           if (CHEAPER_MULT_COST (&alg_in->cost, &best_cost))
    3433              :             {
    3434           95 :               best_cost = alg_in->cost;
    3435           95 :               std::swap (alg_in, best_alg);
    3436           95 :               best_alg->log[best_alg->ops] = m;
    3437           95 :               best_alg->op[best_alg->ops] = alg_sub_t2_m;
    3438              :             }
    3439              :         }
    3440              :       if (cache_hit)
    3441              :         goto done;
    3442              :     }
    3443              : 
    3444      2715041 :  done:
    3445              :   /* If best_cost has not decreased, we have not found any algorithm.  */
    3446     12332642 :   if (!CHEAPER_MULT_COST (&best_cost, cost_limit))
    3447              :     {
    3448              :       /* We failed to find an algorithm.  Record alg_impossible for
    3449              :          this case (that is, <T, MODE, COST_LIMIT>) so that next time
    3450              :          we are asked to find an algorithm for T within the same or
    3451              :          lower COST_LIMIT, we can immediately return to the
    3452              :          caller.  */
    3453      4890331 :       entry_ptr->t = t;
    3454      4890331 :       entry_ptr->mode = mode;
    3455      4890331 :       entry_ptr->speed = speed;
    3456      4890331 :       entry_ptr->alg = alg_impossible;
    3457      4890331 :       entry_ptr->cost = *cost_limit;
    3458      4890331 :       return;
    3459              :     }
    3460              : 
    3461              :   /* Cache the result.  */
    3462      7442311 :   if (!cache_hit)
    3463              :     {
    3464       722024 :       entry_ptr->t = t;
    3465       722024 :       entry_ptr->mode = mode;
    3466       722024 :       entry_ptr->speed = speed;
    3467       722024 :       entry_ptr->alg = best_alg->op[best_alg->ops];
    3468       722024 :       entry_ptr->cost.cost = best_cost.cost;
    3469       722024 :       entry_ptr->cost.latency = best_cost.latency;
    3470              :     }
    3471              : 
    3472              :   /* If we are getting a too long sequence for `struct algorithm'
    3473              :      to record, make this search fail.  */
    3474      7442311 :   if (best_alg->ops == MAX_BITS_PER_WORD)
    3475              :     return;
    3476              : 
    3477              :   /* Copy the algorithm from temporary space to the space at alg_out.
    3478              :      We avoid using structure assignment because the majority of
    3479              :      best_alg is normally undefined, and this is a critical function.  */
    3480      7442311 :   alg_out->ops = best_alg->ops + 1;
    3481      7442311 :   alg_out->cost = best_cost;
    3482      7442311 :   memcpy (alg_out->op, best_alg->op,
    3483      7442311 :           alg_out->ops * sizeof *alg_out->op);
    3484      7442311 :   memcpy (alg_out->log, best_alg->log,
    3485              :           alg_out->ops * sizeof *alg_out->log);
    3486              : }
    3487              : 
    3488              : /* Find the cheapest way of multiplying a value of mode MODE by VAL.
    3489              :    Try three variations:
    3490              : 
    3491              :        - a shift/add sequence based on VAL itself
    3492              :        - a shift/add sequence based on -VAL, followed by a negation
    3493              :        - a shift/add sequence based on VAL - 1, followed by an addition.
    3494              : 
    3495              :    Return true if the cheapest of these cost less than MULT_COST,
    3496              :    describing the algorithm in *ALG and final fixup in *VARIANT.  */
    3497              : 
    3498              : bool
    3499      7359174 : choose_mult_variant (machine_mode mode, HOST_WIDE_INT val,
    3500              :                      struct algorithm *alg, enum mult_variant *variant,
    3501              :                      int mult_cost)
    3502              : {
    3503      7359174 :   struct algorithm alg2;
    3504      7359174 :   struct mult_cost limit;
    3505      7359174 :   int op_cost;
    3506      7359174 :   bool speed = optimize_insn_for_speed_p ();
    3507              : 
    3508              :   /* Fail quickly for impossible bounds.  */
    3509      7359174 :   if (mult_cost < 0)
    3510              :     return false;
    3511              : 
    3512              :   /* Ensure that mult_cost provides a reasonable upper bound.
    3513              :      Any constant multiplication can be performed with less
    3514              :      than 2 * bits additions.  */
    3515     14715186 :   op_cost = 2 * GET_MODE_UNIT_BITSIZE (mode) * add_cost (speed, mode);
    3516      7357593 :   if (mult_cost > op_cost)
    3517              :     mult_cost = op_cost;
    3518              : 
    3519      7357593 :   *variant = basic_variant;
    3520      7357593 :   limit.cost = mult_cost;
    3521      7357593 :   limit.latency = mult_cost;
    3522      7357593 :   synth_mult (alg, val, &limit, mode);
    3523              : 
    3524              :   /* This works only if the inverted value actually fits in an
    3525              :      `unsigned int' */
    3526     14715186 :   if (HOST_BITS_PER_INT >= GET_MODE_UNIT_BITSIZE (mode))
    3527              :     {
    3528      2638337 :       op_cost = neg_cost (speed, mode);
    3529      2638337 :       if (MULT_COST_LESS (&alg->cost, mult_cost))
    3530              :         {
    3531      2530549 :           limit.cost = alg->cost.cost - op_cost;
    3532      2530549 :           limit.latency = alg->cost.latency - op_cost;
    3533              :         }
    3534              :       else
    3535              :         {
    3536       107788 :           limit.cost = mult_cost - op_cost;
    3537       107788 :           limit.latency = mult_cost - op_cost;
    3538              :         }
    3539              : 
    3540      2638337 :       synth_mult (&alg2, -val, &limit, mode);
    3541      2638337 :       alg2.cost.cost += op_cost;
    3542      2638337 :       alg2.cost.latency += op_cost;
    3543      2638337 :       if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
    3544        14756 :         *alg = alg2, *variant = negate_variant;
    3545              :     }
    3546              : 
    3547              :   /* This proves very useful for division-by-constant.  */
    3548      7357593 :   op_cost = add_cost (speed, mode);
    3549      7357593 :   if (MULT_COST_LESS (&alg->cost, mult_cost))
    3550              :     {
    3551      6502714 :       limit.cost = alg->cost.cost - op_cost;
    3552      6502714 :       limit.latency = alg->cost.latency - op_cost;
    3553              :     }
    3554              :   else
    3555              :     {
    3556       854879 :       limit.cost = mult_cost - op_cost;
    3557       854879 :       limit.latency = mult_cost - op_cost;
    3558              :     }
    3559              : 
    3560      7357593 :   if (val != HOST_WIDE_INT_MIN
    3561      7357603 :       || GET_MODE_UNIT_PRECISION (mode) == HOST_BITS_PER_WIDE_INT)
    3562              :     {
    3563      7357593 :       synth_mult (&alg2, val - HOST_WIDE_INT_1U, &limit, mode);
    3564      7357593 :       alg2.cost.cost += op_cost;
    3565      7357593 :       alg2.cost.latency += op_cost;
    3566      7357593 :       if (CHEAPER_MULT_COST (&alg2.cost, &alg->cost))
    3567         2752 :         *alg = alg2, *variant = add_variant;
    3568              :     }
    3569              : 
    3570      7357593 :   return MULT_COST_LESS (&alg->cost, mult_cost);
    3571              : }
    3572              : 
    3573              : /* A subroutine of expand_mult, used for constant multiplications.
    3574              :    Multiply OP0 by VAL in mode MODE, storing the result in TARGET if
    3575              :    convenient.  Use the shift/add sequence described by ALG and apply
    3576              :    the final fixup specified by VARIANT.  */
    3577              : 
    3578              : static rtx
    3579       141123 : expand_mult_const (machine_mode mode, rtx op0, HOST_WIDE_INT val,
    3580              :                    rtx target, const struct algorithm *alg,
    3581              :                    enum mult_variant variant)
    3582              : {
    3583       141123 :   unsigned HOST_WIDE_INT val_so_far;
    3584       141123 :   rtx_insn *insn;
    3585       141123 :   rtx accum, tem;
    3586       141123 :   int opno;
    3587       141123 :   machine_mode nmode;
    3588              : 
    3589              :   /* Avoid referencing memory over and over and invalid sharing
    3590              :      on SUBREGs.  */
    3591       141123 :   op0 = force_reg (mode, op0);
    3592              : 
    3593              :   /* ACCUM starts out either as OP0 or as a zero, depending on
    3594              :      the first operation.  */
    3595              : 
    3596       141123 :   if (alg->op[0] == alg_zero)
    3597              :     {
    3598         5492 :       accum = copy_to_mode_reg (mode, CONST0_RTX (mode));
    3599         5492 :       val_so_far = 0;
    3600              :     }
    3601       135631 :   else if (alg->op[0] == alg_m)
    3602              :     {
    3603       135631 :       accum = copy_to_mode_reg (mode, op0);
    3604       135631 :       val_so_far = 1;
    3605              :     }
    3606              :   else
    3607            0 :     gcc_unreachable ();
    3608              : 
    3609       401932 :   for (opno = 1; opno < alg->ops; opno++)
    3610              :     {
    3611       260809 :       int log = alg->log[opno];
    3612       260809 :       rtx shift_subtarget = optimize ? 0 : accum;
    3613       243923 :       rtx add_target
    3614       141123 :         = (opno == alg->ops - 1 && target != 0 && variant != add_variant
    3615        42520 :            && !optimize)
    3616       260809 :           ? target : 0;
    3617       260809 :       rtx accum_target = optimize ? 0 : accum;
    3618       260809 :       rtx accum_inner;
    3619              : 
    3620       260809 :       switch (alg->op[opno])
    3621              :         {
    3622       113944 :         case alg_shift:
    3623       113944 :           tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
    3624              :           /* REG_EQUAL note will be attached to the following insn.  */
    3625       113944 :           emit_move_insn (accum, tem);
    3626       113944 :           val_so_far <<= log;
    3627       113944 :           break;
    3628              : 
    3629         7073 :         case alg_add_t_m2:
    3630         7073 :           tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
    3631        14146 :           accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
    3632              :                                  add_target ? add_target : accum_target);
    3633         7073 :           val_so_far += HOST_WIDE_INT_1U << log;
    3634         7073 :           break;
    3635              : 
    3636        20398 :         case alg_sub_t_m2:
    3637        20398 :           tem = expand_shift (LSHIFT_EXPR, mode, op0, log, NULL_RTX, 0);
    3638        40796 :           accum = force_operand (gen_rtx_MINUS (mode, accum, tem),
    3639              :                                  add_target ? add_target : accum_target);
    3640        20398 :           val_so_far -= HOST_WIDE_INT_1U << log;
    3641        20398 :           break;
    3642              : 
    3643       118155 :         case alg_add_t2_m:
    3644       118155 :           accum = expand_shift (LSHIFT_EXPR, mode, accum,
    3645       118155 :                                 log, shift_subtarget, 0);
    3646       236310 :           accum = force_operand (gen_rtx_PLUS (mode, accum, op0),
    3647              :                                  add_target ? add_target : accum_target);
    3648       118155 :           val_so_far = (val_so_far << log) + 1;
    3649       118155 :           break;
    3650              : 
    3651            0 :         case alg_sub_t2_m:
    3652            0 :           accum = expand_shift (LSHIFT_EXPR, mode, accum,
    3653            0 :                                 log, shift_subtarget, 0);
    3654            0 :           accum = force_operand (gen_rtx_MINUS (mode, accum, op0),
    3655              :                                  add_target ? add_target : accum_target);
    3656            0 :           val_so_far = (val_so_far << log) - 1;
    3657            0 :           break;
    3658              : 
    3659         1104 :         case alg_add_factor:
    3660         1104 :           tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
    3661         2208 :           accum = force_operand (gen_rtx_PLUS (mode, accum, tem),
    3662              :                                  add_target ? add_target : accum_target);
    3663         1104 :           val_so_far += val_so_far << log;
    3664         1104 :           break;
    3665              : 
    3666          135 :         case alg_sub_factor:
    3667          135 :           tem = expand_shift (LSHIFT_EXPR, mode, accum, log, NULL_RTX, 0);
    3668          270 :           accum = force_operand (gen_rtx_MINUS (mode, tem, accum),
    3669              :                                  (add_target
    3670          135 :                                   ? add_target : (optimize ? 0 : tem)));
    3671          135 :           val_so_far = (val_so_far << log) - val_so_far;
    3672          135 :           break;
    3673              : 
    3674            0 :         default:
    3675            0 :           gcc_unreachable ();
    3676              :         }
    3677              : 
    3678       260809 :       if (SCALAR_INT_MODE_P (mode))
    3679              :         {
    3680              :           /* Write a REG_EQUAL note on the last insn so that we can cse
    3681              :              multiplication sequences.  Note that if ACCUM is a SUBREG,
    3682              :              we've set the inner register and must properly indicate that.  */
    3683       250786 :           tem = op0, nmode = mode;
    3684       250786 :           accum_inner = accum;
    3685       250786 :           if (GET_CODE (accum) == SUBREG)
    3686              :             {
    3687            0 :               accum_inner = SUBREG_REG (accum);
    3688            0 :               nmode = GET_MODE (accum_inner);
    3689            0 :               tem = gen_lowpart (nmode, op0);
    3690              :             }
    3691              : 
    3692              :           /* Don't add a REG_EQUAL note if tem is a paradoxical SUBREG.
    3693              :              In that case, only the low bits of accum would be guaranteed to
    3694              :              be equal to the content of the REG_EQUAL note, the upper bits
    3695              :              can be anything.  */
    3696       250786 :           if (!paradoxical_subreg_p (tem))
    3697              :             {
    3698       250786 :               insn = get_last_insn ();
    3699       250786 :               wide_int wval_so_far
    3700       250786 :                 = wi::uhwi (val_so_far,
    3701       250786 :                             GET_MODE_PRECISION (as_a <scalar_mode> (nmode)));
    3702       250786 :               rtx c = immed_wide_int_const (wval_so_far, nmode);
    3703       250786 :               set_dst_reg_note (insn, REG_EQUAL, gen_rtx_MULT (nmode, tem, c),
    3704              :                                 accum_inner);
    3705       250786 :             }
    3706              :         }
    3707              :     }
    3708              : 
    3709       141123 :   if (variant == negate_variant)
    3710              :     {
    3711          587 :       val_so_far = -val_so_far;
    3712          587 :       accum = expand_unop (mode, neg_optab, accum, target, 0);
    3713              :     }
    3714       140536 :   else if (variant == add_variant)
    3715              :     {
    3716           23 :       val_so_far = val_so_far + 1;
    3717           23 :       accum = force_operand (gen_rtx_PLUS (mode, accum, op0), target);
    3718              :     }
    3719              : 
    3720              :   /* Compare only the bits of val and val_so_far that are significant
    3721              :      in the result mode, to avoid sign-/zero-extension confusion.  */
    3722       141123 :   nmode = GET_MODE_INNER (mode);
    3723       141123 :   val &= GET_MODE_MASK (nmode);
    3724       141123 :   val_so_far &= GET_MODE_MASK (nmode);
    3725       141123 :   gcc_assert (val == (HOST_WIDE_INT) val_so_far);
    3726              : 
    3727       141123 :   return accum;
    3728              : }
    3729              : 
    3730              : /* Perform a multiplication and return an rtx for the result.
    3731              :    MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
    3732              :    TARGET is a suggestion for where to store the result (an rtx).
    3733              : 
    3734              :    We check specially for a constant integer as OP1.
    3735              :    If you want this check for OP0 as well, then before calling
    3736              :    you should swap the two operands if OP0 would be constant.  */
    3737              : 
    3738              : rtx
    3739      1126807 : expand_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
    3740              :              int unsignedp, bool no_libcall)
    3741              : {
    3742      1126807 :   enum mult_variant variant;
    3743      1126807 :   struct algorithm algorithm;
    3744      1126807 :   rtx scalar_op1;
    3745      1126807 :   int max_cost;
    3746      1126807 :   bool speed = optimize_insn_for_speed_p ();
    3747      1126807 :   bool do_trapv = flag_trapv && SCALAR_INT_MODE_P (mode) && !unsignedp;
    3748              : 
    3749      1126807 :   if (CONSTANT_P (op0))
    3750          275 :     std::swap (op0, op1);
    3751              : 
    3752              :   /* For vectors, there are several simplifications that can be made if
    3753              :      all elements of the vector constant are identical.  */
    3754      1126807 :   scalar_op1 = unwrap_const_vec_duplicate (op1);
    3755              : 
    3756      1126807 :   if (INTEGRAL_MODE_P (mode))
    3757              :     {
    3758      1007357 :       rtx fake_reg;
    3759      1007357 :       HOST_WIDE_INT coeff;
    3760      1007357 :       bool is_neg;
    3761      1007357 :       int mode_bitsize;
    3762              : 
    3763      1007357 :       if (op1 == CONST0_RTX (mode))
    3764              :         return op1;
    3765      1007357 :       if (op1 == CONST1_RTX (mode))
    3766              :         return op0;
    3767       968254 :       if (op1 == CONSTM1_RTX (mode))
    3768         2830 :         return expand_unop (mode, do_trapv ? negv_optab : neg_optab,
    3769         1415 :                             op0, target, 0);
    3770              : 
    3771       966839 :       if (do_trapv)
    3772           34 :         goto skip_synth;
    3773              : 
    3774              :       /* If mode is integer vector mode, check if the backend supports
    3775              :          vector lshift (by scalar or vector) at all.  If not, we can't use
    3776              :          synthesized multiply.  */
    3777       966805 :       if (GET_MODE_CLASS (mode) == MODE_VECTOR_INT
    3778        14964 :           && optab_handler (vashl_optab, mode) == CODE_FOR_nothing
    3779       979352 :           && optab_handler (ashl_optab, mode) == CODE_FOR_nothing)
    3780            0 :         goto skip_synth;
    3781              : 
    3782              :       /* These are the operations that are potentially turned into
    3783              :          a sequence of shifts and additions.  */
    3784       966805 :       mode_bitsize = GET_MODE_UNIT_BITSIZE (mode);
    3785              : 
    3786              :       /* synth_mult does an `unsigned int' multiply.  As long as the mode is
    3787              :          less than or equal in size to `unsigned int' this doesn't matter.
    3788              :          If the mode is larger than `unsigned int', then synth_mult works
    3789              :          only if the constant value exactly fits in an `unsigned int' without
    3790              :          any truncation.  This means that multiplying by negative values does
    3791              :          not work; results are off by 2^32 on a 32 bit machine.  */
    3792       966805 :       if (CONST_INT_P (scalar_op1))
    3793              :         {
    3794       705692 :           coeff = INTVAL (scalar_op1);
    3795       705692 :           is_neg = coeff < 0;
    3796              :         }
    3797              : #if TARGET_SUPPORTS_WIDE_INT
    3798       261113 :       else if (CONST_WIDE_INT_P (scalar_op1))
    3799              : #else
    3800              :       else if (CONST_DOUBLE_AS_INT_P (scalar_op1))
    3801              : #endif
    3802              :         {
    3803         1154 :           machine_mode scalar_mode = VECTOR_MODE_P (mode)
    3804         1154 :                                      ? GET_MODE_INNER (mode) : mode;
    3805         1154 :           int shift = wi::exact_log2 (rtx_mode_t (scalar_op1, scalar_mode));
    3806              :           /* Perfect power of 2 (other than 1, which is handled above).  */
    3807         1154 :           if (shift > 0)
    3808          106 :             return expand_shift (LSHIFT_EXPR, mode, op0,
    3809          106 :                                  shift, target, unsignedp);
    3810              :           else
    3811         1048 :             goto skip_synth;
    3812              :         }
    3813              :       else
    3814       259959 :         goto skip_synth;
    3815              : 
    3816              :       /* We used to test optimize here, on the grounds that it's better to
    3817              :          produce a smaller program when -O is not used.  But this causes
    3818              :          such a terrible slowdown sometimes that it seems better to always
    3819              :          use synth_mult.  */
    3820              : 
    3821              :       /* Special case powers of two.  */
    3822       705692 :       if (EXACT_POWER_OF_2_OR_ZERO_P (coeff)
    3823       475674 :           && !(is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT))
    3824       475666 :         return expand_shift (LSHIFT_EXPR, mode, op0,
    3825       951332 :                              floor_log2 (coeff), target, unsignedp);
    3826              : 
    3827       230026 :       fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
    3828              : 
    3829              :       /* Attempt to handle multiplication of DImode values by negative
    3830              :          coefficients, by performing the multiplication by a positive
    3831              :          multiplier and then inverting the result.  */
    3832       230026 :       if (is_neg && mode_bitsize > HOST_BITS_PER_WIDE_INT)
    3833              :         {
    3834              :           /* Its safe to use -coeff even for INT_MIN, as the
    3835              :              result is interpreted as an unsigned coefficient.
    3836              :              Exclude cost of op0 from max_cost to match the cost
    3837              :              calculation of the synth_mult.  */
    3838          218 :           coeff = -(unsigned HOST_WIDE_INT) coeff;
    3839          218 :           max_cost = (set_src_cost (gen_rtx_MULT (mode, fake_reg, op1),
    3840              :                                     mode, speed)
    3841          218 :                       - neg_cost (speed, mode));
    3842          218 :           if (max_cost <= 0)
    3843            0 :             goto skip_synth;
    3844              : 
    3845              :           /* Special case powers of two.  */
    3846          218 :           if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
    3847              :             {
    3848          340 :               rtx temp = expand_shift (LSHIFT_EXPR, mode, op0,
    3849          170 :                                        floor_log2 (coeff), target, unsignedp);
    3850          170 :               return expand_unop (mode, neg_optab, temp, target, 0);
    3851              :             }
    3852              : 
    3853           48 :           if (choose_mult_variant (mode, coeff, &algorithm, &variant,
    3854              :                                    max_cost))
    3855              :             {
    3856           47 :               rtx temp = expand_mult_const (mode, op0, coeff, NULL_RTX,
    3857              :                                             &algorithm, variant);
    3858           47 :               return expand_unop (mode, neg_optab, temp, target, 0);
    3859              :             }
    3860            1 :           goto skip_synth;
    3861              :         }
    3862              : 
    3863              :       /* Exclude cost of op0 from max_cost to match the cost
    3864              :          calculation of the synth_mult.  */
    3865       229808 :       max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, op1), mode, speed);
    3866       229808 :       if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
    3867       139954 :         return expand_mult_const (mode, op0, coeff, target,
    3868       139954 :                                   &algorithm, variant);
    3869              :     }
    3870        89854 :  skip_synth:
    3871              : 
    3872              :   /* Expand x*2.0 as x+x.  */
    3873        38459 :   if (CONST_DOUBLE_AS_FLOAT_P (scalar_op1)
    3874       508805 :       && real_equal (CONST_DOUBLE_REAL_VALUE (scalar_op1), &dconst2))
    3875              :     {
    3876         6016 :       op0 = force_reg (GET_MODE (op0), op0);
    3877        12032 :       return expand_binop (mode, add_optab, op0, op0,
    3878              :                            target, unsignedp,
    3879         6016 :                            no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
    3880              :     }
    3881              : 
    3882              :   /* This used to use umul_optab if unsigned, but for non-widening multiply
    3883              :      there is no difference between signed and unsigned.  */
    3884      1392956 :   op0 = expand_binop (mode, do_trapv ? smulv_optab : smul_optab,
    3885              :                       op0, op1, target, unsignedp,
    3886              :                       no_libcall ? OPTAB_WIDEN : OPTAB_LIB_WIDEN);
    3887       464330 :   gcc_assert (op0 || no_libcall);
    3888              :   return op0;
    3889              : }
    3890              : 
    3891              : /* Return a cost estimate for multiplying a register by the given
    3892              :    COEFFicient in the given MODE and SPEED.  */
    3893              : 
    3894              : int
    3895      6783926 : mult_by_coeff_cost (HOST_WIDE_INT coeff, machine_mode mode, bool speed)
    3896              : {
    3897      6783926 :   int max_cost;
    3898      6783926 :   struct algorithm algorithm;
    3899      6783926 :   enum mult_variant variant;
    3900              : 
    3901      6783926 :   rtx fake_reg = gen_raw_REG (mode, LAST_VIRTUAL_REGISTER + 1);
    3902      6783926 :   max_cost = set_src_cost (gen_rtx_MULT (mode, fake_reg, fake_reg),
    3903              :                            mode, speed);
    3904      6783926 :   if (choose_mult_variant (mode, coeff, &algorithm, &variant, max_cost))
    3905      6022802 :     return algorithm.cost.cost;
    3906              :   else
    3907              :     return max_cost;
    3908              : }
    3909              : 
    3910              : /* Perform a widening multiplication and return an rtx for the result.
    3911              :    MODE is mode of value; OP0 and OP1 are what to multiply (rtx's);
    3912              :    TARGET is a suggestion for where to store the result (an rtx).
    3913              :    THIS_OPTAB is the optab we should use, it must be either umul_widen_optab
    3914              :    or smul_widen_optab.
    3915              : 
    3916              :    We check specially for a constant integer as OP1, comparing the
    3917              :    cost of a widening multiply against the cost of a sequence of shifts
    3918              :    and adds.  */
    3919              : 
    3920              : rtx
    3921        24739 : expand_widening_mult (machine_mode mode, rtx op0, rtx op1, rtx target,
    3922              :                       int unsignedp, optab this_optab)
    3923              : {
    3924        24739 :   bool speed = optimize_insn_for_speed_p ();
    3925        24739 :   rtx cop1;
    3926              : 
    3927        24739 :   if (CONST_INT_P (op1)
    3928         5738 :       && GET_MODE (op0) != VOIDmode
    3929         5738 :       && (cop1 = convert_modes (mode, GET_MODE (op0), op1,
    3930              :                                 this_optab == umul_widen_optab))
    3931         5738 :       && CONST_INT_P (cop1)
    3932        29929 :       && (INTVAL (cop1) >= 0
    3933        27634 :           || HWI_COMPUTABLE_MODE_P (mode)))
    3934              :     {
    3935         4951 :       HOST_WIDE_INT coeff = INTVAL (cop1);
    3936         4951 :       int max_cost;
    3937         4951 :       enum mult_variant variant;
    3938         4951 :       struct algorithm algorithm;
    3939              : 
    3940         4951 :       if (coeff == 0)
    3941         2056 :         return CONST0_RTX (mode);
    3942              : 
    3943              :       /* Special case powers of two.  */
    3944         4575 :       if (EXACT_POWER_OF_2_OR_ZERO_P (coeff))
    3945              :         {
    3946          570 :           op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
    3947          570 :           return expand_shift (LSHIFT_EXPR, mode, op0,
    3948          570 :                                floor_log2 (coeff), target, unsignedp);
    3949              :         }
    3950              : 
    3951              :       /* Exclude cost of op0 from max_cost to match the cost
    3952              :          calculation of the synth_mult.  */
    3953         4005 :       max_cost = mul_widen_cost (speed, mode);
    3954         4005 :       if (choose_mult_variant (mode, coeff, &algorithm, &variant,
    3955              :                                max_cost))
    3956              :         {
    3957         1110 :           op0 = convert_to_mode (mode, op0, this_optab == umul_widen_optab);
    3958         1110 :           return expand_mult_const (mode, op0, coeff, target,
    3959         1110 :                                     &algorithm, variant);
    3960              :         }
    3961              :     }
    3962        22683 :   return expand_binop (mode, this_optab, op0, op1, target,
    3963        22683 :                        unsignedp, OPTAB_LIB_WIDEN);
    3964              : }
    3965              : 
    3966              : /* Choose a minimal N + 1 bit approximation to 2**K / D that can be used to
    3967              :    replace division by D, put the least significant N bits of the result in
    3968              :    *MULTIPLIER_PTR, the value K - N in *POST_SHIFT_PTR, and return the most
    3969              :    significant bit.
    3970              : 
    3971              :    The width of operations is N (should be <= HOST_BITS_PER_WIDE_INT), the
    3972              :    needed precision is PRECISION (should be <= N).
    3973              : 
    3974              :    PRECISION should be as small as possible so this function can choose the
    3975              :    multiplier more freely.  If PRECISION is <= N - 1, the most significant
    3976              :    bit returned by the function will be zero.
    3977              : 
    3978              :    Using this function, x / D is equal to (x*m) / 2**N >> (*POST_SHIFT_PTR),
    3979              :    where m is the full N + 1 bit multiplier.  */
    3980              : 
    3981              : unsigned HOST_WIDE_INT
    3982        75459 : choose_multiplier (unsigned HOST_WIDE_INT d, int n, int precision,
    3983              :                    unsigned HOST_WIDE_INT *multiplier_ptr,
    3984              :                    int *post_shift_ptr)
    3985              : {
    3986        75459 :   int lgup, post_shift;
    3987        75459 :   int pow1, pow2;
    3988              : 
    3989              :   /* lgup = ceil(log2(d)) */
    3990              :   /* Assuming d > 1, we have d >= 2^(lgup-1) + 1 */
    3991        75459 :   lgup = ceil_log2 (d);
    3992              : 
    3993        75459 :   gcc_assert (lgup <= n);
    3994        75459 :   gcc_assert (lgup <= precision);
    3995              : 
    3996        75459 :   pow1 = n + lgup;
    3997        75459 :   pow2 = n + lgup - precision;
    3998              : 
    3999              :   /* mlow = 2^(n + lgup)/d */
    4000              :   /* Trivially from above we have mlow < 2^(n+1) */
    4001        75459 :   wide_int val = wi::set_bit_in_zero (pow1, HOST_BITS_PER_DOUBLE_INT);
    4002        75459 :   wide_int mlow = wi::udiv_trunc (val, d);
    4003              : 
    4004              :   /* mhigh = (2^(n + lgup) + 2^(n + lgup - precision))/d */
    4005              :   /* From above we have mhigh < 2^(n+1) assuming lgup <= precision */
    4006              :   /* From precision <= n, the difference between the numerators of mhigh and
    4007              :      mlow is >= 2^lgup >= d.  Therefore the difference of the quotients in
    4008              :      the Euclidean division by d is at least 1, so we have mlow < mhigh and
    4009              :      the exact value of 2^(n + lgup)/d lies in the interval [mlow; mhigh).  */
    4010        75459 :   val |= wi::set_bit_in_zero (pow2, HOST_BITS_PER_DOUBLE_INT);
    4011        75459 :   wide_int mhigh = wi::udiv_trunc (val, d);
    4012              : 
    4013              :   /* Reduce to lowest terms.  */
    4014              :   /* If precision <= n - 1, then the difference between the numerators of
    4015              :      mhigh and mlow is >= 2^(lgup + 1) >= 2 * 2^lgup >= 2 * d.  Therefore
    4016              :      the difference of the quotients in the Euclidean division by d is at
    4017              :      least 2, which means that mhigh and mlow differ by at least one bit
    4018              :      not in the last place.  The conclusion is that the first iteration of
    4019              :      the loop below completes and shifts mhigh and mlow by 1 bit, which in
    4020              :      particular means that mhigh < 2^n, that is to say, the most significant
    4021              :      bit in the n + 1 bit value is zero.  */
    4022       269445 :   for (post_shift = lgup; post_shift > 0; post_shift--)
    4023              :     {
    4024       188212 :       unsigned HOST_WIDE_INT ml_lo = wi::extract_uhwi (mlow, 1,
    4025              :                                                        HOST_BITS_PER_WIDE_INT);
    4026       188212 :       unsigned HOST_WIDE_INT mh_lo = wi::extract_uhwi (mhigh, 1,
    4027              :                                                        HOST_BITS_PER_WIDE_INT);
    4028       188212 :       if (ml_lo >= mh_lo)
    4029              :         break;
    4030              : 
    4031       118527 :       mlow = wi::uhwi (ml_lo, HOST_BITS_PER_DOUBLE_INT);
    4032       118527 :       mhigh = wi::uhwi (mh_lo, HOST_BITS_PER_DOUBLE_INT);
    4033              :     }
    4034              : 
    4035        75459 :   *post_shift_ptr = post_shift;
    4036              : 
    4037        75459 :   if (n < HOST_BITS_PER_WIDE_INT)
    4038              :     {
    4039        42493 :       unsigned HOST_WIDE_INT mask = (HOST_WIDE_INT_1U << n) - 1;
    4040        42493 :       *multiplier_ptr = mhigh.to_uhwi () & mask;
    4041        42493 :       return mhigh.to_uhwi () > mask;
    4042              :     }
    4043              :   else
    4044              :     {
    4045        32966 :       *multiplier_ptr = mhigh.to_uhwi ();
    4046        32966 :       return wi::extract_uhwi (mhigh, HOST_BITS_PER_WIDE_INT, 1);
    4047              :     }
    4048        75459 : }
    4049              : 
    4050              : /* Compute the inverse of X mod 2**N, i.e., find Y such that X * Y is congruent
    4051              :    to 1 modulo 2**N, assuming that X is odd.  Bézout's lemma guarantees that Y
    4052              :    exists for any given positive N.  */
    4053              : 
    4054              : static unsigned HOST_WIDE_INT
    4055        46011 : invert_mod2n (unsigned HOST_WIDE_INT x, int n)
    4056              : {
    4057        46011 :   gcc_assert ((x & 1) == 1);
    4058              : 
    4059              :   /* The algorithm notes that the choice Y = X satisfies X*Y == 1 mod 2^3,
    4060              :      since X is odd.  Then each Newton-Raphson iteration doubles the number
    4061              :      of bits of significance in Y (Hensel's lemma).  */
    4062              : 
    4063        47029 :   const unsigned HOST_WIDE_INT mask
    4064              :     = (n == HOST_BITS_PER_WIDE_INT
    4065        46011 :        ? HOST_WIDE_INT_M1U
    4066         1018 :        : (HOST_WIDE_INT_1U << n) - 1);
    4067        46011 :   unsigned HOST_WIDE_INT y = x;
    4068        46011 :   int nbit = 3;
    4069              : 
    4070       275018 :   while (nbit < n)
    4071              :     {
    4072       229007 :       y = y * (2 - x*y) & mask;             /* Modulo 2^N */
    4073       229007 :       nbit *= 2;
    4074              :     }
    4075              : 
    4076        46011 :   return y;
    4077              : }
    4078              : 
    4079              : /* Emit code to adjust ADJ_OPERAND after multiplication of wrong signedness
    4080              :    flavor of OP0 and OP1.  ADJ_OPERAND is already the high half of the
    4081              :    product OP0 x OP1.  If UNSIGNEDP is nonzero, adjust the signed product
    4082              :    to become unsigned, if UNSIGNEDP is zero, adjust the unsigned product to
    4083              :    become signed.
    4084              : 
    4085              :    The result is put in TARGET if that is convenient.
    4086              : 
    4087              :    MODE is the mode of operation.  */
    4088              : 
    4089              : rtx
    4090            0 : expand_mult_highpart_adjust (scalar_int_mode mode, rtx adj_operand, rtx op0,
    4091              :                              rtx op1, rtx target, int unsignedp)
    4092              : {
    4093            0 :   rtx tem;
    4094            0 :   enum rtx_code adj_code = unsignedp ? PLUS : MINUS;
    4095              : 
    4096            0 :   tem = expand_shift (RSHIFT_EXPR, mode, op0,
    4097            0 :                       GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
    4098            0 :   tem = expand_and (mode, tem, op1, NULL_RTX);
    4099            0 :   adj_operand
    4100            0 :     = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
    4101              :                      adj_operand);
    4102              : 
    4103            0 :   tem = expand_shift (RSHIFT_EXPR, mode, op1,
    4104            0 :                       GET_MODE_BITSIZE (mode) - 1, NULL_RTX, 0);
    4105            0 :   tem = expand_and (mode, tem, op0, NULL_RTX);
    4106            0 :   target = force_operand (gen_rtx_fmt_ee (adj_code, mode, adj_operand, tem),
    4107              :                           target);
    4108              : 
    4109            0 :   return target;
    4110              : }
    4111              : 
    4112              : /* Subroutine of expmed_mult_highpart.  Return the MODE high part of OP.  */
    4113              : 
    4114              : static rtx
    4115        20035 : extract_high_half (scalar_int_mode mode, rtx op)
    4116              : {
    4117        20035 :   if (mode == word_mode)
    4118            0 :     return gen_highpart (mode, op);
    4119              : 
    4120        20035 :   scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
    4121              : 
    4122        40070 :   op = expand_shift (RSHIFT_EXPR, wider_mode, op,
    4123        20035 :                      GET_MODE_BITSIZE (mode), 0, 1);
    4124        20035 :   return convert_modes (mode, wider_mode, op, 0);
    4125              : }
    4126              : 
    4127              : /* Like expmed_mult_highpart, but only consider using multiplication optab.  */
    4128              : 
    4129              : rtx
    4130        46481 : expmed_mult_highpart_optab (scalar_int_mode mode, rtx op0, rtx op1,
    4131              :                             rtx target, int unsignedp, int max_cost)
    4132              : {
    4133        46481 :   const scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
    4134        46481 :   const bool speed = optimize_insn_for_speed_p ();
    4135        46481 :   const int size = GET_MODE_BITSIZE (mode);
    4136        46481 :   optab moptab;
    4137        46481 :   rtx tem;
    4138              : 
    4139              :   /* Firstly, try using a multiplication insn that only generates the needed
    4140              :      high part of the product, and in the sign flavor of unsignedp.  */
    4141        46481 :   if (mul_highpart_cost (speed, mode) < max_cost)
    4142              :     {
    4143        44449 :       moptab = unsignedp ? umul_highpart_optab : smul_highpart_optab;
    4144        44449 :       tem = expand_binop (mode, moptab, op0, op1, target, unsignedp,
    4145              :                           OPTAB_DIRECT);
    4146        44449 :       if (tem)
    4147              :         return tem;
    4148              :     }
    4149              : 
    4150              :   /* Secondly, same as above, but use sign flavor opposite of unsignedp.
    4151              :      Need to adjust the result after the multiplication.  */
    4152        22233 :   if (size - 1 < BITS_PER_WORD
    4153        44210 :       && (mul_highpart_cost (speed, mode)
    4154        21977 :           + 2 * shift_cost (speed, mode, size-1)
    4155        21977 :           + 4 * add_cost (speed, mode) < max_cost))
    4156              :     {
    4157         5499 :       moptab = unsignedp ? smul_highpart_optab : umul_highpart_optab;
    4158         5499 :       tem = expand_binop (mode, moptab, op0, op1, target, !unsignedp,
    4159              :                           OPTAB_DIRECT);
    4160         5499 :       if (tem)
    4161              :         /* We used the wrong signedness.  Adjust the result.  */
    4162            0 :         return expand_mult_highpart_adjust (mode, tem, op0, op1, tem,
    4163            0 :                                             unsignedp);
    4164              :     }
    4165              : 
    4166              :   /* Try widening multiplication.  */
    4167        22233 :   moptab = unsignedp ? umul_widen_optab : smul_widen_optab;
    4168        22233 :   if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
    4169        22233 :       && mul_widen_cost (speed, wider_mode) < max_cost)
    4170              :     {
    4171          388 :       tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, unsignedp,
    4172              :                           OPTAB_WIDEN);
    4173          388 :       if (tem)
    4174          388 :         return extract_high_half (mode, tem);
    4175              :     }
    4176              : 
    4177              :   /* Try widening the mode and perform a non-widening multiplication.  */
    4178        21845 :   if (optab_handler (smul_optab, wider_mode) != CODE_FOR_nothing
    4179        21240 :       && size - 1 < BITS_PER_WORD
    4180        43082 :       && (mul_cost (speed, wider_mode) + shift_cost (speed, mode, size-1)
    4181              :           < max_cost))
    4182              :     {
    4183        19635 :       rtx_insn *insns;
    4184        19635 :       rtx wop0, wop1;
    4185              : 
    4186              :       /* We need to widen the operands, for example to ensure the
    4187              :          constant multiplier is correctly sign or zero extended.
    4188              :          Use a sequence to clean-up any instructions emitted by
    4189              :          the conversions if things don't work out.  */
    4190        19635 :       start_sequence ();
    4191        19635 :       wop0 = convert_modes (wider_mode, mode, op0, unsignedp);
    4192        19635 :       wop1 = convert_modes (wider_mode, mode, op1, unsignedp);
    4193        19635 :       tem = expand_binop (wider_mode, smul_optab, wop0, wop1, 0,
    4194              :                           unsignedp, OPTAB_WIDEN);
    4195        19635 :       insns = end_sequence ();
    4196              : 
    4197        19635 :       if (tem)
    4198              :         {
    4199        19635 :           emit_insn (insns);
    4200        19635 :           return extract_high_half (mode, tem);
    4201              :         }
    4202              :     }
    4203              : 
    4204              :   /* Try widening multiplication of opposite signedness, and adjust.  */
    4205         2210 :   moptab = unsignedp ? smul_widen_optab : umul_widen_optab;
    4206         2210 :   if (convert_optab_handler (moptab, wider_mode, mode) != CODE_FOR_nothing
    4207          456 :       && size - 1 < BITS_PER_WORD
    4208         3004 :       && (mul_widen_cost (speed, wider_mode)
    4209          397 :           + 2 * shift_cost (speed, mode, size-1)
    4210          397 :           + 4 * add_cost (speed, mode) < max_cost))
    4211              :     {
    4212            0 :       tem = expand_binop (wider_mode, moptab, op0, op1, NULL_RTX, !unsignedp,
    4213              :                           OPTAB_WIDEN);
    4214            0 :       if (tem != 0)
    4215              :         {
    4216            0 :           tem = extract_high_half (mode, tem);
    4217              :           /* We used the wrong signedness.  Adjust the result.  */
    4218            0 :           return expand_mult_highpart_adjust (mode, tem, op0, op1, target,
    4219            0 :                                               unsignedp);
    4220              :         }
    4221              :     }
    4222              : 
    4223              :   return 0;
    4224              : }
    4225              : 
    4226              : /* Emit code to multiply OP0 and OP1 (where OP1 is an integer constant),
    4227              :    putting the high half of the result in TARGET if that is convenient,
    4228              :    and return where the result is.  If the operation cannot be performed,
    4229              :    0 is returned.
    4230              : 
    4231              :    MODE is the mode of operation and result.
    4232              : 
    4233              :    UNSIGNEDP nonzero means unsigned multiply.
    4234              : 
    4235              :    MAX_COST is the total allowed cost for the expanded RTL.  */
    4236              : 
    4237              : static rtx
    4238        46481 : expmed_mult_highpart (scalar_int_mode mode, rtx op0, rtx op1,
    4239              :                       rtx target, int unsignedp, int max_cost)
    4240              : {
    4241        46481 :   const bool speed = optimize_insn_for_speed_p ();
    4242        46481 :   unsigned HOST_WIDE_INT cnst1;
    4243        46481 :   int extra_cost;
    4244        46481 :   bool sign_adjust = false;
    4245        46481 :   enum mult_variant variant;
    4246        46481 :   struct algorithm alg;
    4247        46481 :   rtx narrow_op1, tem;
    4248              : 
    4249              :   /* We can't support modes wider than HOST_BITS_PER_INT.  */
    4250        46481 :   gcc_assert (HWI_COMPUTABLE_MODE_P (mode));
    4251              : 
    4252        46481 :   cnst1 = INTVAL (op1) & GET_MODE_MASK (mode);
    4253        46481 :   narrow_op1 = gen_int_mode (INTVAL (op1), mode);
    4254              : 
    4255              :   /* We can't optimize modes wider than BITS_PER_WORD.
    4256              :      ??? We might be able to perform double-word arithmetic if
    4257              :      mode == word_mode, however all the cost calculations in
    4258              :      synth_mult etc. assume single-word operations.  */
    4259        46481 :   scalar_int_mode wider_mode = GET_MODE_WIDER_MODE (mode).require ();
    4260        96079 :   if (GET_MODE_BITSIZE (wider_mode) > BITS_PER_WORD)
    4261        24856 :     return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
    4262        24856 :                                        unsignedp, max_cost);
    4263              : 
    4264        43250 :   extra_cost = shift_cost (speed, mode, GET_MODE_BITSIZE (mode) - 1);
    4265              : 
    4266              :   /* Check whether we try to multiply by a negative constant.  */
    4267        32066 :   if (!unsignedp && ((cnst1 >> (GET_MODE_BITSIZE (mode) - 1)) & 1))
    4268              :     {
    4269         2292 :       sign_adjust = true;
    4270         2292 :       extra_cost += add_cost (speed, mode);
    4271              :     }
    4272              : 
    4273              :   /* See whether shift/add multiplication is cheap enough.  */
    4274        21625 :   if (choose_mult_variant (wider_mode, cnst1, &alg, &variant,
    4275              :                            max_cost - extra_cost))
    4276              :     {
    4277              :       /* See whether the specialized multiplication optabs are
    4278              :          cheaper than the shift/add version.  */
    4279        39910 :       tem = expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
    4280              :                                         unsignedp,
    4281        19955 :                                         alg.cost.cost + extra_cost);
    4282        19955 :       if (tem)
    4283              :         return tem;
    4284              : 
    4285           12 :       tem = convert_to_mode (wider_mode, op0, unsignedp);
    4286           12 :       tem = expand_mult_const (wider_mode, tem, cnst1, 0, &alg, variant);
    4287           12 :       tem = extract_high_half (mode, tem);
    4288              : 
    4289              :       /* Adjust result for signedness.  */
    4290           12 :       if (sign_adjust)
    4291            0 :         tem = force_operand (gen_rtx_MINUS (mode, tem, op0), tem);
    4292              : 
    4293              :       return tem;
    4294              :     }
    4295         1670 :   return expmed_mult_highpart_optab (mode, op0, narrow_op1, target,
    4296         1670 :                                      unsignedp, max_cost);
    4297              : }
    4298              : 
    4299              : 
    4300              : /* Expand signed modulus of OP0 by a power of two D in mode MODE.  */
    4301              : 
    4302              : static rtx
    4303         2547 : expand_smod_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
    4304              : {
    4305         2547 :   rtx result, temp, shift;
    4306         2547 :   rtx_code_label *label;
    4307         2547 :   int logd;
    4308         2547 :   int prec = GET_MODE_PRECISION (mode);
    4309              : 
    4310         2547 :   logd = floor_log2 (d);
    4311         2547 :   result = gen_reg_rtx (mode);
    4312              : 
    4313              :   /* Avoid conditional branches when they're expensive.  */
    4314         2547 :   if (BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2
    4315         2547 :       && optimize_insn_for_speed_p ())
    4316              :     {
    4317         2546 :       rtx signmask = emit_store_flag (result, LT, op0, const0_rtx,
    4318              :                                       mode, 0, -1);
    4319         2546 :       if (signmask)
    4320              :         {
    4321         2546 :           HOST_WIDE_INT masklow = (HOST_WIDE_INT_1 << logd) - 1;
    4322         2546 :           signmask = force_reg (mode, signmask);
    4323         5092 :           shift = gen_int_shift_amount (mode, GET_MODE_BITSIZE (mode) - logd);
    4324              : 
    4325              :           /* Use the rtx_cost of a LSHIFTRT instruction to determine
    4326              :              which instruction sequence to use.  If logical right shifts
    4327              :              are expensive the use 2 XORs, 2 SUBs and an AND, otherwise
    4328              :              use a LSHIFTRT, 1 ADD, 1 SUB and an AND.  */
    4329              : 
    4330         2546 :           temp = gen_rtx_LSHIFTRT (mode, result, shift);
    4331         2546 :           if (optab_handler (lshr_optab, mode) == CODE_FOR_nothing
    4332         2546 :               || (set_src_cost (temp, mode, optimize_insn_for_speed_p ())
    4333              :                   > COSTS_N_INSNS (2)))
    4334              :             {
    4335          109 :               temp = expand_binop (mode, xor_optab, op0, signmask,
    4336              :                                    NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4337          109 :               temp = expand_binop (mode, sub_optab, temp, signmask,
    4338              :                                    NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4339          109 :               temp = expand_binop (mode, and_optab, temp,
    4340          109 :                                    gen_int_mode (masklow, mode),
    4341              :                                    NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4342          109 :               temp = expand_binop (mode, xor_optab, temp, signmask,
    4343              :                                    NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4344          109 :               temp = expand_binop (mode, sub_optab, temp, signmask,
    4345              :                                    NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4346              :             }
    4347              :           else
    4348              :             {
    4349         2437 :               signmask = expand_binop (mode, lshr_optab, signmask, shift,
    4350              :                                        NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4351         2437 :               signmask = force_reg (mode, signmask);
    4352              : 
    4353         2437 :               temp = expand_binop (mode, add_optab, op0, signmask,
    4354              :                                    NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4355         2437 :               temp = expand_binop (mode, and_optab, temp,
    4356         2437 :                                    gen_int_mode (masklow, mode),
    4357              :                                    NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4358         2437 :               temp = expand_binop (mode, sub_optab, temp, signmask,
    4359              :                                    NULL_RTX, 1, OPTAB_LIB_WIDEN);
    4360              :             }
    4361              :           return temp;
    4362              :         }
    4363              :     }
    4364              : 
    4365              :   /* Mask contains the mode's signbit and the significant bits of the
    4366              :      modulus.  By including the signbit in the operation, many targets
    4367              :      can avoid an explicit compare operation in the following comparison
    4368              :      against zero.  */
    4369            1 :   wide_int mask = wi::mask (logd, false, prec);
    4370            1 :   mask = wi::set_bit (mask, prec - 1);
    4371              : 
    4372            2 :   temp = expand_binop (mode, and_optab, op0,
    4373            1 :                        immed_wide_int_const (mask, mode),
    4374              :                        result, 1, OPTAB_LIB_WIDEN);
    4375            1 :   if (temp != result)
    4376            0 :     emit_move_insn (result, temp);
    4377              : 
    4378            1 :   label = gen_label_rtx ();
    4379            1 :   do_cmp_and_jump (result, const0_rtx, GE, mode, label);
    4380              : 
    4381            1 :   temp = expand_binop (mode, sub_optab, result, const1_rtx, result,
    4382              :                        0, OPTAB_LIB_WIDEN);
    4383              : 
    4384            1 :   mask = wi::mask (logd, true, prec);
    4385            2 :   temp = expand_binop (mode, ior_optab, temp,
    4386            1 :                        immed_wide_int_const (mask, mode),
    4387              :                        result, 1, OPTAB_LIB_WIDEN);
    4388            1 :   temp = expand_binop (mode, add_optab, temp, const1_rtx, result,
    4389              :                        0, OPTAB_LIB_WIDEN);
    4390            1 :   if (temp != result)
    4391            0 :     emit_move_insn (result, temp);
    4392            1 :   emit_label (label);
    4393            1 :   return result;
    4394            1 : }
    4395              : 
    4396              : /* Expand signed division of OP0 by a power of two D in mode MODE.
    4397              :    This routine is only called for positive values of D.  */
    4398              : 
    4399              : static rtx
    4400        10710 : expand_sdiv_pow2 (scalar_int_mode mode, rtx op0, HOST_WIDE_INT d)
    4401              : {
    4402        10710 :   rtx temp;
    4403        10710 :   rtx_code_label *label;
    4404        10710 :   int logd;
    4405              : 
    4406        10710 :   logd = floor_log2 (d);
    4407              : 
    4408        10710 :   if (d == 2
    4409        10710 :       && BRANCH_COST (optimize_insn_for_speed_p (),
    4410              :                       false) >= 1)
    4411              :     {
    4412         6982 :       temp = gen_reg_rtx (mode);
    4413         6982 :       temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, 1);
    4414         6982 :       if (temp != NULL_RTX)
    4415              :         {
    4416         6982 :           temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
    4417              :                                0, OPTAB_LIB_WIDEN);
    4418         6982 :           return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
    4419              :         }
    4420              :     }
    4421              : 
    4422         7455 :   if (HAVE_conditional_move
    4423         3728 :       && BRANCH_COST (optimize_insn_for_speed_p (), false) >= 2)
    4424              :     {
    4425         3728 :       rtx temp2;
    4426              : 
    4427         3728 :       start_sequence ();
    4428         3728 :       temp2 = copy_to_mode_reg (mode, op0);
    4429         3728 :       temp = expand_binop (mode, add_optab, temp2, gen_int_mode (d - 1, mode),
    4430              :                            NULL_RTX, 0, OPTAB_LIB_WIDEN);
    4431         3728 :       temp = force_reg (mode, temp);
    4432              : 
    4433              :       /* Construct "temp2 = (temp2 < 0) ? temp : temp2".  */
    4434         3728 :       temp2 = emit_conditional_move (temp2, { LT, temp2, const0_rtx, mode },
    4435              :                                      temp, temp2, mode, 0);
    4436         3728 :       if (temp2)
    4437              :         {
    4438         3663 :           rtx_insn *seq = end_sequence ();
    4439         3663 :           emit_insn (seq);
    4440         3663 :           return expand_shift (RSHIFT_EXPR, mode, temp2, logd, NULL_RTX, 0);
    4441              :         }
    4442           65 :       end_sequence ();
    4443              :     }
    4444              : 
    4445           65 :   if (BRANCH_COST (optimize_insn_for_speed_p (),
    4446              :                    false) >= 2)
    4447              :     {
    4448           65 :       int ushift = GET_MODE_BITSIZE (mode) - logd;
    4449              : 
    4450           65 :       temp = gen_reg_rtx (mode);
    4451           65 :       temp = emit_store_flag (temp, LT, op0, const0_rtx, mode, 0, -1);
    4452           65 :       if (temp != NULL_RTX)
    4453              :         {
    4454          130 :           if (GET_MODE_BITSIZE (mode) >= BITS_PER_WORD
    4455           65 :               || shift_cost (optimize_insn_for_speed_p (), mode, ushift)
    4456              :               > COSTS_N_INSNS (1))
    4457           65 :             temp = expand_binop (mode, and_optab, temp,
    4458           65 :                                  gen_int_mode (d - 1, mode),
    4459              :                                  NULL_RTX, 0, OPTAB_LIB_WIDEN);
    4460              :           else
    4461            0 :             temp = expand_shift (RSHIFT_EXPR, mode, temp,
    4462            0 :                                  ushift, NULL_RTX, 1);
    4463           65 :           temp = expand_binop (mode, add_optab, temp, op0, NULL_RTX,
    4464              :                                0, OPTAB_LIB_WIDEN);
    4465           65 :           return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
    4466              :         }
    4467              :     }
    4468              : 
    4469            0 :   label = gen_label_rtx ();
    4470            0 :   temp = copy_to_mode_reg (mode, op0);
    4471            0 :   do_cmp_and_jump (temp, const0_rtx, GE, mode, label);
    4472            0 :   expand_inc (temp, gen_int_mode (d - 1, mode));
    4473            0 :   emit_label (label);
    4474            0 :   return expand_shift (RSHIFT_EXPR, mode, temp, logd, NULL_RTX, 0);
    4475              : }
    4476              : 
    4477              : /* Emit the code to divide OP0 by OP1, putting the result in TARGET
    4478              :    if that is convenient, and returning where the result is.
    4479              :    You may request either the quotient or the remainder as the result;
    4480              :    specify REM_FLAG nonzero to get the remainder.
    4481              : 
    4482              :    CODE is the expression code for which kind of division this is;
    4483              :    it controls how rounding is done.  MODE is the machine mode to use.
    4484              :    UNSIGNEDP nonzero means do unsigned division.  */
    4485              : 
    4486              : /* ??? For CEIL_MOD_EXPR, can compute incorrect remainder with ANDI
    4487              :    and then correct it by or'ing in missing high bits
    4488              :    if result of ANDI is nonzero.
    4489              :    For ROUND_MOD_EXPR, can use ANDI and then sign-extend the result.
    4490              :    This could optimize to a bfexts instruction.
    4491              :    But C doesn't use these operations, so their optimizations are
    4492              :    left for later.  */
    4493              : /* ??? For modulo, we don't actually need the highpart of the first product,
    4494              :    the low part will do nicely.  And for small divisors, the second multiply
    4495              :    can also be a low-part only multiply or even be completely left out.
    4496              :    E.g. to calculate the remainder of a division by 3 with a 32 bit
    4497              :    multiply, multiply with 0x55555556 and extract the upper two bits;
    4498              :    the result is exact for inputs up to 0x1fffffff.
    4499              :    The input range can be reduced by using cross-sum rules.
    4500              :    For odd divisors >= 3, the following table gives right shift counts
    4501              :    so that if a number is shifted by an integer multiple of the given
    4502              :    amount, the remainder stays the same:
    4503              :    2, 4, 3, 6, 10, 12, 4, 8, 18, 6, 11, 20, 18, 0, 5, 10, 12, 0, 12, 20,
    4504              :    14, 12, 23, 21, 8, 0, 20, 18, 0, 0, 6, 12, 0, 22, 0, 18, 20, 30, 0, 0,
    4505              :    0, 8, 0, 11, 12, 10, 36, 0, 30, 0, 0, 12, 0, 0, 0, 0, 44, 12, 24, 0,
    4506              :    20, 0, 7, 14, 0, 18, 36, 0, 0, 46, 60, 0, 42, 0, 15, 24, 20, 0, 0, 33,
    4507              :    0, 20, 0, 0, 18, 0, 60, 0, 0, 0, 0, 0, 40, 18, 0, 0, 12
    4508              : 
    4509              :    Cross-sum rules for even numbers can be derived by leaving as many bits
    4510              :    to the right alone as the divisor has zeros to the right.
    4511              :    E.g. if x is an unsigned 32 bit number:
    4512              :    (x mod 12) == (((x & 1023) + ((x >> 8) & ~3)) * 0x15555558 >> 2 * 3) >> 28
    4513              :    */
    4514              : 
    4515              : /* Helper for expand_divmod's unsigned constant division.  For OP0 in
    4516              :    INT_MODE divided by a constant needing a (SIZE+1)-bit multiplier ML
    4517              :    with right shift POST_SHIFT (the mh != 0 case), try to obtain
    4518              :    the quotient from the high part of a single multiply in a mode twice
    4519              :    as wide as INT_MODE.  Return the quotient in INT_MODE, having emitted
    4520              :    the insns, or NULL_RTX when the transformation is unavailable or not
    4521              :    cheaper than the classic sub/shift/add sequence.  EXTRA_COST is the
    4522              :    cost of that sequence's follow-up ops, MAX_COST bounds the multiply
    4523              :    and SPEED selects the cost model.
    4524              : 
    4525              :    The magic constant occupies at most 2*SIZE bits and so must fit in a
    4526              :    HOST_WIDE_INT (always 64 bits today; checked below).  A wider INT_MODE
    4527              :    such as DImode -- which would need a 128-bit magic and a single-word
    4528              :    high-part multiply in a 2x-wide mode that common targets lack -- is
    4529              :    therefore excluded.  */
    4530              : 
    4531              : static rtx
    4532         2340 : expand_wide_mulh_udiv (scalar_int_mode int_mode, rtx op0,
    4533              :                        unsigned HOST_WIDE_INT ml, int size, int post_shift,
    4534              :                        int extra_cost, int max_cost, bool speed)
    4535              : {
    4536         2340 :   scalar_int_mode wide_mode;
    4537              : 
    4538              :   /* We need POST_SHIFT >= 1, a wider integer mode that still fits in a
    4539              :      word, and the pre-shifted magic constant to fit in a HOST_WIDE_INT.  */
    4540         2340 :   if (post_shift < 1
    4541         2340 :       || !GET_MODE_2XWIDER_MODE (int_mode).exists (&wide_mode)
    4542         2433 :       || GET_MODE_BITSIZE (wide_mode) > BITS_PER_WORD
    4543         3384 :       || GET_MODE_BITSIZE (wide_mode) > HOST_BITS_PER_WIDE_INT)
    4544              :     return NULL_RTX;
    4545              : 
    4546              :   /* The caller obtained ML and POST_SHIFT from choose_multiplier, which
    4547              :      guarantees POST_SHIFT <= ceil (log2 (d)) <= SIZE for a SIZE-bit
    4548              :      divisor d, so the shift count below is non-negative.  */
    4549         1044 :   gcc_checking_assert (post_shift <= size);
    4550              : 
    4551              :   /* Pre-shift the (SIZE+1)-bit magic constant (2^SIZE + ML) by
    4552              :      (SIZE - POST_SHIFT) so that the quotient ends up in the high part
    4553              :      of the widened product.  Since ML < 2^SIZE and POST_SHIFT >= 1, the
    4554              :      result is below 2^(2*SIZE) and thus fits in both WIDE_MODE and an
    4555              :      unsigned HOST_WIDE_INT (2*SIZE <= HOST_BITS_PER_WIDE_INT was
    4556              :      checked above).  */
    4557         1044 :   unsigned HOST_WIDE_INT magic
    4558         1044 :     = ((HOST_WIDE_INT_1U << size) + ml) << (size - post_shift);
    4559              : 
    4560         1044 :   start_sequence ();
    4561         1044 :   rtx x_wide = convert_to_mode (wide_mode, op0, 1);
    4562         1044 :   rtx hi = expmed_mult_highpart (wide_mode, x_wide,
    4563         1044 :                                  gen_int_mode (magic, wide_mode),
    4564              :                                  NULL_RTX, 1, max_cost);
    4565         1044 :   rtx quotient = hi ? convert_to_mode (int_mode, hi, 1) : NULL_RTX;
    4566         1044 :   rtx_insn *insns = end_sequence ();
    4567              : 
    4568              :   /* Use the widened multiply only when it is no more expensive than
    4569              :      the classic sub/shift/add sequence.  */
    4570         1044 :   unsigned classic_cost = mul_highpart_cost (speed, int_mode) + extra_cost;
    4571         1044 :   if (quotient == NULL_RTX || seq_cost (insns, speed) > classic_cost)
    4572              :     return NULL_RTX;
    4573              : 
    4574          967 :   emit_insn (insns);
    4575          967 :   return quotient;
    4576              : }
    4577              : 
    4578              : rtx
    4579       242366 : expand_divmod (int rem_flag, enum tree_code code, machine_mode mode,
    4580              :                rtx op0, rtx op1, rtx target, int unsignedp,
    4581              :                enum optab_methods methods)
    4582              : {
    4583       242366 :   machine_mode compute_mode;
    4584       242366 :   rtx tquotient;
    4585       242366 :   rtx quotient = 0, remainder = 0;
    4586       242366 :   rtx_insn *last;
    4587       242366 :   rtx_insn *insn;
    4588       242366 :   optab optab1, optab2;
    4589       242366 :   int op1_is_constant, op1_is_pow2 = 0;
    4590       242366 :   int max_cost, extra_cost;
    4591       242366 :   bool speed = optimize_insn_for_speed_p ();
    4592              : 
    4593       242366 :   op1_is_constant = CONST_INT_P (op1);
    4594       242366 :   if (op1_is_constant)
    4595              :     {
    4596       138637 :       wide_int ext_op1 = rtx_mode_t (op1, mode);
    4597       169071 :       op1_is_pow2 = (wi::popcount (ext_op1) == 1
    4598       277274 :                      || (! unsignedp
    4599       169071 :                          && wi::popcount (wi::neg (ext_op1)) == 1));
    4600       138637 :     }
    4601              : 
    4602              :   /*
    4603              :      This is the structure of expand_divmod:
    4604              : 
    4605              :      First comes code to fix up the operands so we can perform the operations
    4606              :      correctly and efficiently.
    4607              : 
    4608              :      Second comes a switch statement with code specific for each rounding mode.
    4609              :      For some special operands this code emits all RTL for the desired
    4610              :      operation, for other cases, it generates only a quotient and stores it in
    4611              :      QUOTIENT.  The case for trunc division/remainder might leave quotient = 0,
    4612              :      to indicate that it has not done anything.
    4613              : 
    4614              :      Last comes code that finishes the operation.  If QUOTIENT is set and
    4615              :      REM_FLAG is set, the remainder is computed as OP0 - QUOTIENT * OP1.  If
    4616              :      QUOTIENT is not set, it is computed using trunc rounding.
    4617              : 
    4618              :      We try to generate special code for division and remainder when OP1 is a
    4619              :      constant.  If |OP1| = 2**n we can use shifts and some other fast
    4620              :      operations.  For other values of OP1, we compute a carefully selected
    4621              :      fixed-point approximation m = 1/OP1, and generate code that multiplies OP0
    4622              :      by m.
    4623              : 
    4624              :      In all cases but EXACT_DIV_EXPR, this multiplication requires the upper
    4625              :      half of the product.  Different strategies for generating the product are
    4626              :      implemented in expmed_mult_highpart.
    4627              : 
    4628              :      If what we actually want is the remainder, we generate that by another
    4629              :      by-constant multiplication and a subtraction.  */
    4630              : 
    4631              :   /* We shouldn't be called with OP1 == const1_rtx, but some of the
    4632              :      code below will malfunction if we are, so check here and handle
    4633              :      the special case if so.  */
    4634       242366 :   if (op1 == const1_rtx)
    4635           15 :     return rem_flag ? const0_rtx : op0;
    4636              : 
    4637              :     /* When dividing by -1, we could get an overflow.
    4638              :      negv_optab can handle overflows.  */
    4639       242351 :   if (! unsignedp && op1 == constm1_rtx)
    4640              :     {
    4641            0 :       if (rem_flag)
    4642            0 :         return const0_rtx;
    4643            0 :       return expand_unop (mode, flag_trapv && GET_MODE_CLASS (mode) == MODE_INT
    4644            0 :                           ? negv_optab : neg_optab, op0, target, 0);
    4645              :     }
    4646              : 
    4647       242351 :   if (target
    4648              :       /* Don't use the function value register as a target
    4649              :          since we have to read it as well as write it,
    4650              :          and function-inlining gets confused by this.  */
    4651       242351 :       && ((REG_P (target) && REG_FUNCTION_VALUE_P (target))
    4652              :           /* Don't clobber an operand while doing a multi-step calculation.  */
    4653        95289 :           || ((rem_flag || op1_is_constant)
    4654        74704 :               && (reg_mentioned_p (target, op0)
    4655        72266 :                   || (MEM_P (op0) && MEM_P (target))))
    4656        92121 :           || reg_mentioned_p (target, op1)
    4657        92036 :           || (MEM_P (op1) && MEM_P (target))))
    4658              :     target = 0;
    4659              : 
    4660              :   /* Get the mode in which to perform this computation.  Normally it will
    4661              :      be MODE, but sometimes we can't do the desired operation in MODE.
    4662              :      If so, pick a wider mode in which we can do the operation.  Convert
    4663              :      to that mode at the start to avoid repeated conversions.
    4664              : 
    4665              :      First see what operations we need.  These depend on the expression
    4666              :      we are evaluating.  (We assume that divxx3 insns exist under the
    4667              :      same conditions that modxx3 insns and that these insns don't normally
    4668              :      fail.  If these assumptions are not correct, we may generate less
    4669              :      efficient code in some cases.)
    4670              : 
    4671              :      Then see if we find a mode in which we can open-code that operation
    4672              :      (either a division, modulus, or shift).  Finally, check for the smallest
    4673              :      mode for which we can do the operation with a library call.  */
    4674              : 
    4675              :   /* We might want to refine this now that we have division-by-constant
    4676              :      optimization.  Since expmed_mult_highpart tries so many variants, it is
    4677              :      not straightforward to generalize this.  Maybe we should make an array
    4678              :      of possible modes in init_expmed?  Save this for GCC 2.7.  */
    4679              : 
    4680       136440 :   optab1 = (op1_is_pow2
    4681       242351 :             ? (unsignedp ? lshr_optab : ashr_optab)
    4682       161066 :             : (unsignedp ? udiv_optab : sdiv_optab));
    4683       324961 :   optab2 = (op1_is_pow2 ? optab1
    4684       161066 :             : (unsignedp ? udivmod_optab : sdivmod_optab));
    4685              : 
    4686       242351 :   if (methods == OPTAB_WIDEN || methods == OPTAB_LIB_WIDEN)
    4687              :     {
    4688       254124 :       FOR_EACH_MODE_FROM (compute_mode, mode)
    4689       249889 :       if (optab_handler (optab1, compute_mode) != CODE_FOR_nothing
    4690       249889 :           || optab_handler (optab2, compute_mode) != CODE_FOR_nothing)
    4691              :         break;
    4692              : 
    4693       240558 :       if (compute_mode == VOIDmode && methods == OPTAB_LIB_WIDEN)
    4694         4235 :         FOR_EACH_MODE_FROM (compute_mode, mode)
    4695         4235 :           if (optab_libfunc (optab1, compute_mode)
    4696         4235 :               || optab_libfunc (optab2, compute_mode))
    4697              :             break;
    4698              :     }
    4699              :   else
    4700              :     compute_mode = mode;
    4701              : 
    4702              :   /* If we still couldn't find a mode, use MODE, but expand_binop will
    4703              :      probably die.  */
    4704         6028 :   if (compute_mode == VOIDmode)
    4705            0 :     compute_mode = mode;
    4706              : 
    4707       242351 :   if (target && GET_MODE (target) == compute_mode)
    4708              :     tquotient = target;
    4709              :   else
    4710       150491 :     tquotient = gen_reg_rtx (compute_mode);
    4711              : 
    4712              : #if 0
    4713              :   /* It should be possible to restrict the precision to GET_MODE_BITSIZE
    4714              :      (mode), and thereby get better code when OP1 is a constant.  Do that
    4715              :      later.  It will require going over all usages of SIZE below.  */
    4716              :   size = GET_MODE_BITSIZE (mode);
    4717              : #endif
    4718              : 
    4719              :   /* Only deduct something for a REM if the last divide done was
    4720              :      for a different constant.   Then set the constant of the last
    4721              :      divide.  */
    4722       242351 :   max_cost = (unsignedp
    4723       242351 :               ? udiv_cost (speed, compute_mode)
    4724       136440 :               : sdiv_cost (speed, compute_mode));
    4725       242351 :   if (rem_flag && ! (last_div_const != 0 && op1_is_constant
    4726         6289 :                      && INTVAL (op1) == last_div_const))
    4727        63126 :     max_cost -= (mul_cost (speed, compute_mode)
    4728        63126 :                  + add_cost (speed, compute_mode));
    4729              : 
    4730       242351 :   last_div_const = ! rem_flag && op1_is_constant ? INTVAL (op1) : 0;
    4731              : 
    4732              :   /* Now convert to the best mode to use.  */
    4733       242351 :   if (compute_mode != mode)
    4734              :     {
    4735            0 :       op0 = convert_modes (compute_mode, mode, op0, unsignedp);
    4736            0 :       op1 = convert_modes (compute_mode, mode, op1, unsignedp);
    4737              : 
    4738              :       /* convert_modes may have placed op1 into a register, so we
    4739              :          must recompute the following.  */
    4740            0 :       op1_is_constant = CONST_INT_P (op1);
    4741            0 :       if (op1_is_constant)
    4742              :         {
    4743            0 :           wide_int ext_op1 = rtx_mode_t (op1, compute_mode);
    4744            0 :           op1_is_pow2 = (wi::popcount (ext_op1) == 1
    4745            0 :                          || (! unsignedp
    4746            0 :                              && wi::popcount (wi::neg (ext_op1)) == 1));
    4747            0 :         }
    4748              :       else
    4749              :         op1_is_pow2 = 0;
    4750              :     }
    4751              : 
    4752              :   /* If one of the operands is a volatile MEM, copy it into a register.  */
    4753              : 
    4754       242351 :   if (MEM_P (op0) && MEM_VOLATILE_P (op0))
    4755            0 :     op0 = force_reg (compute_mode, op0);
    4756       242351 :   if (MEM_P (op1) && MEM_VOLATILE_P (op1))
    4757            0 :     op1 = force_reg (compute_mode, op1);
    4758              : 
    4759              :   /* If we need the remainder or if OP1 is constant, we need to
    4760              :      put OP0 in a register in case it has any queued subexpressions.  */
    4761       242351 :   if (rem_flag || op1_is_constant)
    4762       178453 :     op0 = force_reg (compute_mode, op0);
    4763              : 
    4764       242351 :   last = get_last_insn ();
    4765              : 
    4766              :   /* Promote floor rounding to trunc rounding for unsigned operations.  */
    4767       242351 :   if (unsignedp)
    4768              :     {
    4769       105911 :       if (code == FLOOR_DIV_EXPR)
    4770              :         code = TRUNC_DIV_EXPR;
    4771       105858 :       if (code == FLOOR_MOD_EXPR)
    4772          148 :         code = TRUNC_MOD_EXPR;
    4773       105911 :       if (code == EXACT_DIV_EXPR && op1_is_pow2)
    4774           94 :         code = TRUNC_DIV_EXPR;
    4775              :     }
    4776              : 
    4777       242351 :   if (op1 != const0_rtx)
    4778       241984 :     switch (code)
    4779              :       {
    4780       193795 :       case TRUNC_MOD_EXPR:
    4781       193795 :       case TRUNC_DIV_EXPR:
    4782       193795 :         if (op1_is_constant)
    4783              :           {
    4784        91190 :             scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
    4785        91190 :             int size = GET_MODE_BITSIZE (int_mode);
    4786        91190 :             if (unsignedp)
    4787              :               {
    4788        53301 :                 unsigned HOST_WIDE_INT mh, ml;
    4789        53301 :                 int pre_shift, post_shift;
    4790        53301 :                 wide_int wd = rtx_mode_t (op1, int_mode);
    4791        53301 :                 unsigned HOST_WIDE_INT d = wd.to_uhwi ();
    4792              : 
    4793        53301 :                 if (wi::popcount (wd) == 1)
    4794              :                   {
    4795        27455 :                     pre_shift = floor_log2 (d);
    4796        27455 :                     if (rem_flag)
    4797              :                       {
    4798          277 :                         unsigned HOST_WIDE_INT mask
    4799          277 :                           = (HOST_WIDE_INT_1U << pre_shift) - 1;
    4800          277 :                         remainder
    4801          277 :                           = expand_binop (int_mode, and_optab, op0,
    4802          277 :                                           gen_int_mode (mask, int_mode),
    4803              :                                           remainder, 1, methods);
    4804          277 :                         if (remainder)
    4805          277 :                           return gen_lowpart (mode, remainder);
    4806              :                       }
    4807        27178 :                     quotient = expand_shift (RSHIFT_EXPR, int_mode, op0,
    4808        27178 :                                              pre_shift, tquotient, 1);
    4809              :                   }
    4810        25846 :                 else if (size <= HOST_BITS_PER_WIDE_INT)
    4811              :                   {
    4812        24234 :                     if (d >= (HOST_WIDE_INT_1U << (size - 1)))
    4813              :                       {
    4814              :                         /* Most significant bit of divisor is set; emit an scc
    4815              :                            insn.  */
    4816          161 :                         quotient = emit_store_flag_force (tquotient, GEU, op0, op1,
    4817              :                                                           int_mode, 1, 1);
    4818              :                       }
    4819              :                     else
    4820              :                       {
    4821              :                         /* Find a suitable multiplier and right shift count
    4822              :                            instead of directly dividing by D.  */
    4823        24073 :                         mh = choose_multiplier (d, size, size,
    4824              :                                                 &ml, &post_shift);
    4825              : 
    4826              :                         /* If the suggested multiplier is more than SIZE bits,
    4827              :                            we can do better for even divisors, using an
    4828              :                            initial right shift.  */
    4829        24073 :                         if (mh != 0 && (d & 1) == 0)
    4830              :                           {
    4831         2380 :                             pre_shift = ctz_or_zero (d);
    4832         2380 :                             mh = choose_multiplier (d >> pre_shift, size,
    4833              :                                                     size - pre_shift,
    4834              :                                                     &ml, &post_shift);
    4835         2380 :                             gcc_assert (!mh);
    4836              :                           }
    4837              :                         else
    4838              :                           pre_shift = 0;
    4839              : 
    4840         2340 :                         if (mh != 0)
    4841              :                           {
    4842         2340 :                             rtx t1, t2, t3, t4;
    4843              : 
    4844         2433 :                             if (post_shift - 1 >= BITS_PER_WORD)
    4845            0 :                               goto fail1;
    4846              : 
    4847         2340 :                             extra_cost
    4848         2340 :                               = (shift_cost (speed, int_mode, post_shift - 1)
    4849         2340 :                                  + shift_cost (speed, int_mode, 1)
    4850         2340 :                                  + 2 * add_cost (speed, int_mode));
    4851              : 
    4852              :                             /* Try a single widened multiply first; use it when
    4853              :                                it is no more expensive.  */
    4854         2340 :                             quotient
    4855         2340 :                               = expand_wide_mulh_udiv (int_mode, op0, ml, size,
    4856              :                                                        post_shift, extra_cost,
    4857              :                                                        max_cost, speed);
    4858         2340 :                             if (quotient == NULL_RTX)
    4859              :                               {
    4860         1373 :                                 t1 = expmed_mult_highpart
    4861         1373 :                                   (int_mode, op0, gen_int_mode (ml, int_mode),
    4862              :                                    NULL_RTX, 1, max_cost - extra_cost);
    4863         1373 :                                 if (t1 == 0)
    4864          107 :                                   goto fail1;
    4865         1266 :                                 t2 = force_operand (gen_rtx_MINUS (int_mode,
    4866              :                                                                    op0, t1),
    4867              :                                                     NULL_RTX);
    4868         1266 :                                 t3 = expand_shift (RSHIFT_EXPR, int_mode,
    4869              :                                                    t2, 1, NULL_RTX, 1);
    4870         1266 :                                 t4 = force_operand (gen_rtx_PLUS (int_mode,
    4871              :                                                                   t1, t3),
    4872              :                                                     NULL_RTX);
    4873         1266 :                                 quotient = expand_shift
    4874         1266 :                                   (RSHIFT_EXPR, int_mode, t4,
    4875         1266 :                                    post_shift - 1, tquotient, 1);
    4876              :                               }
    4877              :                           }
    4878              :                         else
    4879              :                           {
    4880        21733 :                             rtx t1, t2;
    4881              : 
    4882        23556 :                             if (pre_shift >= BITS_PER_WORD
    4883        21733 :                                 || post_shift >= BITS_PER_WORD)
    4884            3 :                               goto fail1;
    4885              : 
    4886        21730 :                             t1 = expand_shift
    4887        43460 :                               (RSHIFT_EXPR, int_mode, op0,
    4888        21730 :                                pre_shift, NULL_RTX, 1);
    4889        21730 :                             extra_cost
    4890        21730 :                               = (shift_cost (speed, int_mode, pre_shift)
    4891        21730 :                                  + shift_cost (speed, int_mode, post_shift));
    4892        21730 :                             t2 = expmed_mult_highpart
    4893        21730 :                               (int_mode, t1,
    4894        21730 :                                gen_int_mode (ml, int_mode),
    4895              :                                NULL_RTX, 1, max_cost - extra_cost);
    4896        21730 :                             if (t2 == 0)
    4897          974 :                               goto fail1;
    4898        20756 :                             quotient = expand_shift
    4899        20756 :                               (RSHIFT_EXPR, int_mode, t2,
    4900        20756 :                                post_shift, tquotient, 1);
    4901              :                           }
    4902              :                       }
    4903              :                   }
    4904              :                 else            /* Too wide mode to use tricky code */
    4905              :                   break;
    4906              : 
    4907        50328 :                 insn = get_last_insn ();
    4908        50328 :                 if (insn != last)
    4909        50328 :                   set_dst_reg_note (insn, REG_EQUAL,
    4910              :                                     gen_rtx_UDIV (int_mode, op0, op1),
    4911              :                                     quotient);
    4912        51689 :               }
    4913              :             else                /* TRUNC_DIV, signed */
    4914              :               {
    4915        37889 :                 unsigned HOST_WIDE_INT ml;
    4916        37889 :                 int post_shift;
    4917        37889 :                 rtx mlr;
    4918        37889 :                 HOST_WIDE_INT d = INTVAL (op1);
    4919        37889 :                 unsigned HOST_WIDE_INT abs_d;
    4920              : 
    4921              :                 /* Not prepared to handle division/remainder by
    4922              :                    0xffffffffffffffff8000000000000000 etc.  */
    4923        37889 :                 if (d == HOST_WIDE_INT_MIN && size > HOST_BITS_PER_WIDE_INT)
    4924              :                   break;
    4925              : 
    4926              :                 /* Since d might be INT_MIN, we have to cast to
    4927              :                    unsigned HOST_WIDE_INT before negating to avoid
    4928              :                    undefined signed overflow.  */
    4929        37889 :                 abs_d = (d >= 0
    4930        37889 :                          ? (unsigned HOST_WIDE_INT) d
    4931              :                          : - (unsigned HOST_WIDE_INT) d);
    4932              : 
    4933              :                 /* n rem d = n rem -d */
    4934        37889 :                 if (rem_flag && d < 0)
    4935              :                   {
    4936          195 :                     d = abs_d;
    4937          195 :                     op1 = gen_int_mode (abs_d, int_mode);
    4938              :                   }
    4939              : 
    4940        37889 :                 if (d == 1)
    4941              :                   quotient = op0;
    4942        37889 :                 else if (d == -1)
    4943            0 :                   quotient = expand_unop (int_mode, neg_optab, op0,
    4944              :                                           tquotient, 0);
    4945        37889 :                 else if (size <= HOST_BITS_PER_WIDE_INT
    4946        36488 :                          && abs_d == HOST_WIDE_INT_1U << (size - 1))
    4947              :                   {
    4948              :                     /* This case is not handled correctly below.  */
    4949          241 :                     quotient = emit_store_flag (tquotient, EQ, op0, op1,
    4950              :                                                 int_mode, 1, 1);
    4951          241 :                     if (quotient == 0)
    4952         1302 :                       goto fail1;
    4953              :                   }
    4954        37648 :                 else if (EXACT_POWER_OF_2_OR_ZERO_P (d)
    4955        13252 :                          && (size <= HOST_BITS_PER_WIDE_INT || d >= 0)
    4956         2681 :                          && (rem_flag
    4957         2681 :                              ? smod_pow2_cheap (speed, int_mode)
    4958        10571 :                              : sdiv_pow2_cheap (speed, int_mode))
    4959              :                          /* We assume that cheap metric is true if the
    4960              :                             optab has an expander for this mode.  */
    4961        52161 :                          && ((optab_handler ((rem_flag ? smod_optab
    4962              :                                               : sdiv_optab),
    4963              :                                              int_mode)
    4964              :                               != CODE_FOR_nothing)
    4965          698 :                              || (optab_handler (sdivmod_optab, int_mode)
    4966              :                                  != CODE_FOR_nothing)))
    4967              :                   ;
    4968        36953 :                 else if (EXACT_POWER_OF_2_OR_ZERO_P (abs_d))
    4969              :                   {
    4970        13289 :                     if (rem_flag)
    4971              :                       {
    4972         2547 :                         remainder = expand_smod_pow2 (int_mode, op0, d);
    4973         2547 :                         if (remainder)
    4974         2547 :                           return gen_lowpart (mode, remainder);
    4975              :                       }
    4976              : 
    4977        10742 :                     if (sdiv_pow2_cheap (speed, int_mode)
    4978        10742 :                         && ((optab_handler (sdiv_optab, int_mode)
    4979              :                              != CODE_FOR_nothing)
    4980           35 :                             || (optab_handler (sdivmod_optab, int_mode)
    4981              :                                 != CODE_FOR_nothing)))
    4982           32 :                       quotient = expand_divmod (0, TRUNC_DIV_EXPR,
    4983              :                                                 int_mode, op0,
    4984           32 :                                                 gen_int_mode (abs_d,
    4985              :                                                               int_mode),
    4986              :                                                 NULL_RTX, 0);
    4987              :                     else
    4988        10710 :                       quotient = expand_sdiv_pow2 (int_mode, op0, abs_d);
    4989              : 
    4990              :                     /* We have computed OP0 / abs(OP1).  If OP1 is negative,
    4991              :                        negate the quotient.  */
    4992        10742 :                     if (d < 0)
    4993              :                       {
    4994          732 :                         insn = get_last_insn ();
    4995          732 :                         if (insn != last
    4996          732 :                             && abs_d < (HOST_WIDE_INT_1U
    4997              :                                         << (HOST_BITS_PER_WIDE_INT - 1)))
    4998          732 :                           set_dst_reg_note (insn, REG_EQUAL,
    4999          732 :                                             gen_rtx_DIV (int_mode, op0,
    5000              :                                                          gen_int_mode
    5001              :                                                            (abs_d,
    5002              :                                                             int_mode)),
    5003              :                                             quotient);
    5004              : 
    5005          732 :                         quotient = expand_unop (int_mode, neg_optab,
    5006              :                                                 quotient, quotient, 0);
    5007              :                       }
    5008              :                   }
    5009        23664 :                 else if (size <= HOST_BITS_PER_WIDE_INT)
    5010              :                   {
    5011        22325 :                     choose_multiplier (abs_d, size, size - 1,
    5012              :                                        &ml, &post_shift);
    5013        22325 :                     if (ml < HOST_WIDE_INT_1U << (size - 1))
    5014              :                       {
    5015        16863 :                         rtx t1, t2, t3;
    5016              : 
    5017        17978 :                         if (post_shift >= BITS_PER_WORD
    5018        16863 :                             || size - 1 >= BITS_PER_WORD)
    5019          250 :                           goto fail1;
    5020              : 
    5021        16613 :                         extra_cost = (shift_cost (speed, int_mode, post_shift)
    5022        16613 :                                       + shift_cost (speed, int_mode, size - 1)
    5023        16613 :                                       + add_cost (speed, int_mode));
    5024        16613 :                         t1 = expmed_mult_highpart
    5025        16613 :                           (int_mode, op0, gen_int_mode (ml, int_mode),
    5026              :                            NULL_RTX, 0, max_cost - extra_cost);
    5027        16613 :                         if (t1 == 0)
    5028          791 :                           goto fail1;
    5029        15822 :                         t2 = expand_shift
    5030        31644 :                           (RSHIFT_EXPR, int_mode, t1,
    5031        15822 :                            post_shift, NULL_RTX, 0);
    5032        15822 :                         t3 = expand_shift
    5033        15822 :                           (RSHIFT_EXPR, int_mode, op0,
    5034        15822 :                            size - 1, NULL_RTX, 0);
    5035        15822 :                         if (d < 0)
    5036          198 :                           quotient
    5037          198 :                             = force_operand (gen_rtx_MINUS (int_mode, t3, t2),
    5038              :                                              tquotient);
    5039              :                         else
    5040        15624 :                           quotient
    5041        15624 :                             = force_operand (gen_rtx_MINUS (int_mode, t2, t3),
    5042              :                                              tquotient);
    5043              :                       }
    5044              :                     else
    5045              :                       {
    5046         5462 :                         rtx t1, t2, t3, t4;
    5047              : 
    5048         5822 :                         if (post_shift >= BITS_PER_WORD
    5049         5457 :                             || size - 1 >= BITS_PER_WORD)
    5050           43 :                           goto fail1;
    5051              : 
    5052         5419 :                         ml |= HOST_WIDE_INT_M1U << (size - 1);
    5053         5419 :                         mlr = gen_int_mode (ml, int_mode);
    5054         5419 :                         extra_cost = (shift_cost (speed, int_mode, post_shift)
    5055         5419 :                                       + shift_cost (speed, int_mode, size - 1)
    5056         5419 :                                       + 2 * add_cost (speed, int_mode));
    5057         5419 :                         t1 = expmed_mult_highpart (int_mode, op0, mlr,
    5058              :                                                    NULL_RTX, 0,
    5059              :                                                    max_cost - extra_cost);
    5060         5419 :                         if (t1 == 0)
    5061          218 :                           goto fail1;
    5062         5201 :                         t2 = force_operand (gen_rtx_PLUS (int_mode, t1, op0),
    5063              :                                             NULL_RTX);
    5064         5201 :                         t3 = expand_shift
    5065        10402 :                           (RSHIFT_EXPR, int_mode, t2,
    5066         5201 :                            post_shift, NULL_RTX, 0);
    5067         5201 :                         t4 = expand_shift
    5068         5201 :                           (RSHIFT_EXPR, int_mode, op0,
    5069         5201 :                            size - 1, NULL_RTX, 0);
    5070         5201 :                         if (d < 0)
    5071           52 :                           quotient
    5072           52 :                             = force_operand (gen_rtx_MINUS (int_mode, t4, t3),
    5073              :                                              tquotient);
    5074              :                         else
    5075         5149 :                           quotient
    5076         5149 :                             = force_operand (gen_rtx_MINUS (int_mode, t3, t4),
    5077              :                                              tquotient);
    5078              :                       }
    5079              :                   }
    5080              :                 else            /* Too wide mode to use tricky code */
    5081              :                   break;
    5082              : 
    5083        32701 :                 insn = get_last_insn ();
    5084        32701 :                 if (insn != last)
    5085        32006 :                   set_dst_reg_note (insn, REG_EQUAL,
    5086              :                                     gen_rtx_DIV (int_mode, op0, op1),
    5087              :                                     quotient);
    5088              :               }
    5089              :             break;
    5090              :           }
    5091       102605 :       fail1:
    5092       104991 :         delete_insns_since (last);
    5093       104991 :         break;
    5094              : 
    5095         1765 :       case FLOOR_DIV_EXPR:
    5096         1765 :       case FLOOR_MOD_EXPR:
    5097              :       /* We will come here only for signed operations.  */
    5098         1765 :         if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
    5099              :           {
    5100          971 :             scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
    5101          971 :             int size = GET_MODE_BITSIZE (int_mode);
    5102          971 :             unsigned HOST_WIDE_INT mh, ml;
    5103          971 :             int pre_shift, post_shift;
    5104          971 :             HOST_WIDE_INT d = INTVAL (op1);
    5105              : 
    5106          971 :             if (d > 0)
    5107              :               {
    5108              :                 /* We could just as easily deal with negative constants here,
    5109              :                    but it does not seem worth the trouble for GCC 2.6.  */
    5110          945 :                 if (EXACT_POWER_OF_2_OR_ZERO_P (d))
    5111              :                   {
    5112          641 :                     pre_shift = floor_log2 (d);
    5113          641 :                     if (rem_flag)
    5114              :                       {
    5115           64 :                         unsigned HOST_WIDE_INT mask
    5116           64 :                           = (HOST_WIDE_INT_1U << pre_shift) - 1;
    5117           64 :                         remainder = expand_binop
    5118           64 :                           (int_mode, and_optab, op0,
    5119           64 :                            gen_int_mode (mask, int_mode),
    5120              :                            remainder, 0, methods);
    5121           64 :                         if (remainder)
    5122           64 :                           return gen_lowpart (mode, remainder);
    5123              :                       }
    5124          577 :                     quotient = expand_shift
    5125          577 :                       (RSHIFT_EXPR, int_mode, op0,
    5126          577 :                        pre_shift, tquotient, 0);
    5127              :                   }
    5128              :                 else
    5129              :                   {
    5130          304 :                     rtx t1, t2, t3, t4;
    5131              : 
    5132          304 :                     mh = choose_multiplier (d, size, size - 1,
    5133              :                                             &ml, &post_shift);
    5134          304 :                     gcc_assert (!mh);
    5135              : 
    5136          328 :                     if (post_shift < BITS_PER_WORD
    5137          304 :                         && size - 1 < BITS_PER_WORD)
    5138              :                       {
    5139          302 :                         t1 = expand_shift
    5140          302 :                           (RSHIFT_EXPR, int_mode, op0,
    5141          302 :                            size - 1, NULL_RTX, 0);
    5142          302 :                         t2 = expand_binop (int_mode, xor_optab, op0, t1,
    5143              :                                            NULL_RTX, 0, OPTAB_WIDEN);
    5144          302 :                         extra_cost = (shift_cost (speed, int_mode, post_shift)
    5145          302 :                                       + shift_cost (speed, int_mode, size - 1)
    5146          302 :                                       + 2 * add_cost (speed, int_mode));
    5147          302 :                         t3 = expmed_mult_highpart
    5148          302 :                           (int_mode, t2, gen_int_mode (ml, int_mode),
    5149              :                            NULL_RTX, 1, max_cost - extra_cost);
    5150          302 :                         if (t3 != 0)
    5151              :                           {
    5152          271 :                             t4 = expand_shift
    5153          542 :                               (RSHIFT_EXPR, int_mode, t3,
    5154          271 :                                post_shift, NULL_RTX, 1);
    5155          271 :                             quotient = expand_binop (int_mode, xor_optab,
    5156              :                                                      t4, t1, tquotient, 0,
    5157              :                                                      OPTAB_WIDEN);
    5158              :                           }
    5159              :                       }
    5160              :                   }
    5161              :               }
    5162              :             else
    5163              :               {
    5164           26 :                 rtx nsign, t1, t2, t3, t4;
    5165           26 :                 t1 = force_operand (gen_rtx_PLUS (int_mode,
    5166              :                                                   op0, constm1_rtx), NULL_RTX);
    5167           26 :                 t2 = expand_binop (int_mode, ior_optab, op0, t1, NULL_RTX,
    5168              :                                    0, OPTAB_WIDEN);
    5169           52 :                 nsign = expand_shift (RSHIFT_EXPR, int_mode, t2,
    5170           26 :                                       size - 1, NULL_RTX, 0);
    5171           26 :                 t3 = force_operand (gen_rtx_MINUS (int_mode, t1, nsign),
    5172              :                                     NULL_RTX);
    5173           26 :                 t4 = expand_divmod (0, TRUNC_DIV_EXPR, int_mode, t3, op1,
    5174              :                                     NULL_RTX, 0);
    5175           26 :                 if (t4)
    5176              :                   {
    5177           26 :                     rtx t5;
    5178           26 :                     t5 = expand_unop (int_mode, one_cmpl_optab, nsign,
    5179              :                                       NULL_RTX, 0);
    5180           26 :                     quotient = force_operand (gen_rtx_PLUS (int_mode, t4, t5),
    5181              :                                               tquotient);
    5182              :                   }
    5183              :               }
    5184              :           }
    5185              : 
    5186          907 :         if (quotient != 0)
    5187              :           break;
    5188          827 :         delete_insns_since (last);
    5189              : 
    5190              :         /* Try using an instruction that produces both the quotient and
    5191              :            remainder, using truncation.  We can easily compensate the quotient
    5192              :            or remainder to get floor rounding, once we have the remainder.
    5193              :            Notice that we compute also the final remainder value here,
    5194              :            and return the result right away.  */
    5195          827 :         if (target == 0 || GET_MODE (target) != compute_mode)
    5196          126 :           target = gen_reg_rtx (compute_mode);
    5197              : 
    5198          827 :         if (rem_flag)
    5199              :           {
    5200          329 :             remainder
    5201          329 :               = REG_P (target) ? target : gen_reg_rtx (compute_mode);
    5202          329 :             quotient = gen_reg_rtx (compute_mode);
    5203              :           }
    5204              :         else
    5205              :           {
    5206          498 :             quotient
    5207          498 :               = REG_P (target) ? target : gen_reg_rtx (compute_mode);
    5208          498 :             remainder = gen_reg_rtx (compute_mode);
    5209              :           }
    5210              : 
    5211          827 :         if (expand_twoval_binop (sdivmod_optab, op0, op1,
    5212              :                                  quotient, remainder, 0))
    5213              :           {
    5214              :             /* This could be computed with a branch-less sequence.
    5215              :                Save that for later.  */
    5216          792 :             rtx tem;
    5217          792 :             rtx_code_label *label = gen_label_rtx ();
    5218          792 :             do_cmp_and_jump (remainder, const0_rtx, EQ, compute_mode, label);
    5219          792 :             tem = expand_binop (compute_mode, xor_optab, op0, op1,
    5220              :                                 NULL_RTX, 0, OPTAB_WIDEN);
    5221          792 :             do_cmp_and_jump (tem, const0_rtx, GE, compute_mode, label);
    5222          792 :             expand_dec (quotient, const1_rtx);
    5223          792 :             expand_inc (remainder, op1);
    5224          792 :             emit_label (label);
    5225         1281 :             return gen_lowpart (mode, rem_flag ? remainder : quotient);
    5226              :           }
    5227              : 
    5228              :         /* No luck with division elimination or divmod.  Have to do it
    5229              :            by conditionally adjusting op0 *and* the result.  */
    5230           35 :         {
    5231           35 :           rtx_code_label *label1, *label2, *label3, *label4, *label5;
    5232           35 :           rtx adjusted_op0;
    5233           35 :           rtx tem;
    5234              : 
    5235           35 :           quotient = gen_reg_rtx (compute_mode);
    5236           35 :           adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
    5237           35 :           label1 = gen_label_rtx ();
    5238           35 :           label2 = gen_label_rtx ();
    5239           35 :           label3 = gen_label_rtx ();
    5240           35 :           label4 = gen_label_rtx ();
    5241           35 :           label5 = gen_label_rtx ();
    5242           35 :           do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
    5243           35 :           do_cmp_and_jump (adjusted_op0, const0_rtx, LT, compute_mode, label1);
    5244           35 :           tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
    5245              :                               quotient, 0, methods);
    5246           35 :           if (tem != quotient)
    5247           35 :             emit_move_insn (quotient, tem);
    5248           35 :           emit_jump_insn (targetm.gen_jump (label5));
    5249           35 :           emit_barrier ();
    5250           35 :           emit_label (label1);
    5251           35 :           expand_inc (adjusted_op0, const1_rtx);
    5252           35 :           emit_jump_insn (targetm.gen_jump (label4));
    5253           35 :           emit_barrier ();
    5254           35 :           emit_label (label2);
    5255           35 :           do_cmp_and_jump (adjusted_op0, const0_rtx, GT, compute_mode, label3);
    5256           35 :           tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
    5257              :                               quotient, 0, methods);
    5258           35 :           if (tem != quotient)
    5259           35 :             emit_move_insn (quotient, tem);
    5260           35 :           emit_jump_insn (targetm.gen_jump (label5));
    5261           35 :           emit_barrier ();
    5262           35 :           emit_label (label3);
    5263           35 :           expand_dec (adjusted_op0, const1_rtx);
    5264           35 :           emit_label (label4);
    5265           35 :           tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
    5266              :                               quotient, 0, methods);
    5267           35 :           if (tem != quotient)
    5268           35 :             emit_move_insn (quotient, tem);
    5269           35 :           expand_dec (quotient, const1_rtx);
    5270           35 :           emit_label (label5);
    5271              :         }
    5272           35 :         break;
    5273              : 
    5274          402 :       case CEIL_DIV_EXPR:
    5275          402 :       case CEIL_MOD_EXPR:
    5276          402 :         if (unsignedp)
    5277              :           {
    5278            0 :             if (op1_is_constant
    5279            0 :                 && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
    5280            0 :                 && (HWI_COMPUTABLE_MODE_P (compute_mode)
    5281            0 :                     || INTVAL (op1) >= 0))
    5282              :               {
    5283            0 :                 scalar_int_mode int_mode
    5284            0 :                   = as_a <scalar_int_mode> (compute_mode);
    5285            0 :                 rtx t1, t2, t3;
    5286            0 :                 unsigned HOST_WIDE_INT d = INTVAL (op1);
    5287            0 :                 t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
    5288            0 :                                    floor_log2 (d), tquotient, 1);
    5289            0 :                 t2 = expand_binop (int_mode, and_optab, op0,
    5290            0 :                                    gen_int_mode (d - 1, int_mode),
    5291              :                                    NULL_RTX, 1, methods);
    5292            0 :                 t3 = gen_reg_rtx (int_mode);
    5293            0 :                 t3 = emit_store_flag (t3, NE, t2, const0_rtx, int_mode, 1, 1);
    5294            0 :                 if (t3 == 0)
    5295              :                   {
    5296            0 :                     rtx_code_label *lab;
    5297            0 :                     lab = gen_label_rtx ();
    5298            0 :                     do_cmp_and_jump (t2, const0_rtx, EQ, int_mode, lab);
    5299            0 :                     expand_inc (t1, const1_rtx);
    5300            0 :                     emit_label (lab);
    5301            0 :                     quotient = t1;
    5302              :                   }
    5303              :                 else
    5304            0 :                   quotient = force_operand (gen_rtx_PLUS (int_mode, t1, t3),
    5305              :                                             tquotient);
    5306              :                 break;
    5307              :               }
    5308              : 
    5309              :             /* Try using an instruction that produces both the quotient and
    5310              :                remainder, using truncation.  We can easily compensate the
    5311              :                quotient or remainder to get ceiling rounding, once we have the
    5312              :                remainder.  Notice that we compute also the final remainder
    5313              :                value here, and return the result right away.  */
    5314            0 :             if (target == 0 || GET_MODE (target) != compute_mode)
    5315            0 :               target = gen_reg_rtx (compute_mode);
    5316              : 
    5317            0 :             if (rem_flag)
    5318              :               {
    5319            0 :                 remainder = (REG_P (target)
    5320            0 :                              ? target : gen_reg_rtx (compute_mode));
    5321            0 :                 quotient = gen_reg_rtx (compute_mode);
    5322              :               }
    5323              :             else
    5324              :               {
    5325            0 :                 quotient = (REG_P (target)
    5326            0 :                             ? target : gen_reg_rtx (compute_mode));
    5327            0 :                 remainder = gen_reg_rtx (compute_mode);
    5328              :               }
    5329              : 
    5330            0 :             if (expand_twoval_binop (udivmod_optab, op0, op1, quotient,
    5331              :                                      remainder, 1))
    5332              :               {
    5333              :                 /* This could be computed with a branch-less sequence.
    5334              :                    Save that for later.  */
    5335            0 :                 rtx_code_label *label = gen_label_rtx ();
    5336            0 :                 do_cmp_and_jump (remainder, const0_rtx, EQ,
    5337              :                                  compute_mode, label);
    5338            0 :                 expand_inc (quotient, const1_rtx);
    5339            0 :                 expand_dec (remainder, op1);
    5340            0 :                 emit_label (label);
    5341            0 :                 return gen_lowpart (mode, rem_flag ? remainder : quotient);
    5342              :               }
    5343              : 
    5344              :             /* No luck with division elimination or divmod.  Have to do it
    5345              :                by conditionally adjusting op0 *and* the result.  */
    5346            0 :             {
    5347            0 :               rtx_code_label *label1, *label2;
    5348            0 :               rtx adjusted_op0, tem;
    5349              : 
    5350            0 :               quotient = gen_reg_rtx (compute_mode);
    5351            0 :               adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
    5352            0 :               label1 = gen_label_rtx ();
    5353            0 :               label2 = gen_label_rtx ();
    5354            0 :               do_cmp_and_jump (adjusted_op0, const0_rtx, NE,
    5355              :                                compute_mode, label1);
    5356            0 :               emit_move_insn  (quotient, const0_rtx);
    5357            0 :               emit_jump_insn (targetm.gen_jump (label2));
    5358            0 :               emit_barrier ();
    5359            0 :               emit_label (label1);
    5360            0 :               expand_dec (adjusted_op0, const1_rtx);
    5361            0 :               tem = expand_binop (compute_mode, udiv_optab, adjusted_op0, op1,
    5362              :                                   quotient, 1, methods);
    5363            0 :               if (tem != quotient)
    5364            0 :                 emit_move_insn (quotient, tem);
    5365            0 :               expand_inc (quotient, const1_rtx);
    5366            0 :               emit_label (label2);
    5367              :             }
    5368              :           }
    5369              :         else /* signed */
    5370              :           {
    5371          402 :             if (op1_is_constant && EXACT_POWER_OF_2_OR_ZERO_P (INTVAL (op1))
    5372           31 :                 && INTVAL (op1) >= 0)
    5373              :               {
    5374              :                 /* This is extremely similar to the code for the unsigned case
    5375              :                    above.  For 2.7 we should merge these variants, but for
    5376              :                    2.6.1 I don't want to touch the code for unsigned since that
    5377              :                    get used in C.  The signed case will only be used by other
    5378              :                    languages (Ada).  */
    5379              : 
    5380           31 :                 rtx t1, t2, t3;
    5381           31 :                 unsigned HOST_WIDE_INT d = INTVAL (op1);
    5382           62 :                 t1 = expand_shift (RSHIFT_EXPR, compute_mode, op0,
    5383           31 :                                    floor_log2 (d), tquotient, 0);
    5384           31 :                 t2 = expand_binop (compute_mode, and_optab, op0,
    5385           31 :                                    gen_int_mode (d - 1, compute_mode),
    5386              :                                    NULL_RTX, 1, methods);
    5387           31 :                 t3 = gen_reg_rtx (compute_mode);
    5388           31 :                 t3 = emit_store_flag (t3, NE, t2, const0_rtx,
    5389              :                                       compute_mode, 1, 1);
    5390           31 :                 if (t3 == 0)
    5391              :                   {
    5392            0 :                     rtx_code_label *lab;
    5393            0 :                     lab = gen_label_rtx ();
    5394            0 :                     do_cmp_and_jump (t2, const0_rtx, EQ, compute_mode, lab);
    5395            0 :                     expand_inc (t1, const1_rtx);
    5396            0 :                     emit_label (lab);
    5397            0 :                     quotient = t1;
    5398              :                   }
    5399              :                 else
    5400           31 :                   quotient = force_operand (gen_rtx_PLUS (compute_mode,
    5401              :                                                           t1, t3),
    5402              :                                             tquotient);
    5403              :                 break;
    5404              :               }
    5405              : 
    5406              :             /* Try using an instruction that produces both the quotient and
    5407              :                remainder, using truncation.  We can easily compensate the
    5408              :                quotient or remainder to get ceiling rounding, once we have the
    5409              :                remainder.  Notice that we compute also the final remainder
    5410              :                value here, and return the result right away.  */
    5411          371 :             if (target == 0 || GET_MODE (target) != compute_mode)
    5412           15 :               target = gen_reg_rtx (compute_mode);
    5413          371 :             if (rem_flag)
    5414              :               {
    5415          149 :                 remainder= (REG_P (target)
    5416          149 :                             ? target : gen_reg_rtx (compute_mode));
    5417          149 :                 quotient = gen_reg_rtx (compute_mode);
    5418              :               }
    5419              :             else
    5420              :               {
    5421          222 :                 quotient = (REG_P (target)
    5422          222 :                             ? target : gen_reg_rtx (compute_mode));
    5423          222 :                 remainder = gen_reg_rtx (compute_mode);
    5424              :               }
    5425              : 
    5426          371 :             if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient,
    5427              :                                      remainder, 0))
    5428              :               {
    5429              :                 /* This could be computed with a branch-less sequence.
    5430              :                    Save that for later.  */
    5431          371 :                 rtx tem;
    5432          371 :                 rtx_code_label *label = gen_label_rtx ();
    5433          371 :                 do_cmp_and_jump (remainder, const0_rtx, EQ,
    5434              :                                  compute_mode, label);
    5435          371 :                 tem = expand_binop (compute_mode, xor_optab, op0, op1,
    5436              :                                     NULL_RTX, 0, OPTAB_WIDEN);
    5437          371 :                 do_cmp_and_jump (tem, const0_rtx, LT, compute_mode, label);
    5438          371 :                 expand_inc (quotient, const1_rtx);
    5439          371 :                 expand_dec (remainder, op1);
    5440          371 :                 emit_label (label);
    5441          593 :                 return gen_lowpart (mode, rem_flag ? remainder : quotient);
    5442              :               }
    5443              : 
    5444              :             /* No luck with division elimination or divmod.  Have to do it
    5445              :                by conditionally adjusting op0 *and* the result.  */
    5446            0 :             {
    5447            0 :               rtx_code_label *label1, *label2, *label3, *label4, *label5;
    5448            0 :               rtx adjusted_op0;
    5449            0 :               rtx tem;
    5450              : 
    5451            0 :               quotient = gen_reg_rtx (compute_mode);
    5452            0 :               adjusted_op0 = copy_to_mode_reg (compute_mode, op0);
    5453            0 :               label1 = gen_label_rtx ();
    5454            0 :               label2 = gen_label_rtx ();
    5455            0 :               label3 = gen_label_rtx ();
    5456            0 :               label4 = gen_label_rtx ();
    5457            0 :               label5 = gen_label_rtx ();
    5458            0 :               do_cmp_and_jump (op1, const0_rtx, LT, compute_mode, label2);
    5459            0 :               do_cmp_and_jump (adjusted_op0, const0_rtx, GT,
    5460              :                                compute_mode, label1);
    5461            0 :               tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
    5462              :                                   quotient, 0, methods);
    5463            0 :               if (tem != quotient)
    5464            0 :                 emit_move_insn (quotient, tem);
    5465            0 :               emit_jump_insn (targetm.gen_jump (label5));
    5466            0 :               emit_barrier ();
    5467            0 :               emit_label (label1);
    5468            0 :               expand_dec (adjusted_op0, const1_rtx);
    5469            0 :               emit_jump_insn (targetm.gen_jump (label4));
    5470            0 :               emit_barrier ();
    5471            0 :               emit_label (label2);
    5472            0 :               do_cmp_and_jump (adjusted_op0, const0_rtx, LT,
    5473              :                                compute_mode, label3);
    5474            0 :               tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
    5475              :                                   quotient, 0, methods);
    5476            0 :               if (tem != quotient)
    5477            0 :                 emit_move_insn (quotient, tem);
    5478            0 :               emit_jump_insn (targetm.gen_jump (label5));
    5479            0 :               emit_barrier ();
    5480            0 :               emit_label (label3);
    5481            0 :               expand_inc (adjusted_op0, const1_rtx);
    5482            0 :               emit_label (label4);
    5483            0 :               tem = expand_binop (compute_mode, sdiv_optab, adjusted_op0, op1,
    5484              :                                   quotient, 0, methods);
    5485            0 :               if (tem != quotient)
    5486            0 :                 emit_move_insn (quotient, tem);
    5487            0 :               expand_inc (quotient, const1_rtx);
    5488            0 :               emit_label (label5);
    5489              :             }
    5490              :           }
    5491              :         break;
    5492              : 
    5493        46020 :       case EXACT_DIV_EXPR:
    5494        46020 :         if (op1_is_constant && HWI_COMPUTABLE_MODE_P (compute_mode))
    5495              :           {
    5496        46011 :             scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
    5497        46011 :             int size = GET_MODE_BITSIZE (int_mode);
    5498        46011 :             HOST_WIDE_INT d = INTVAL (op1);
    5499        46011 :             unsigned HOST_WIDE_INT ml;
    5500        46011 :             int pre_shift;
    5501        46011 :             rtx t1;
    5502              : 
    5503        46011 :             pre_shift = ctz_or_zero (d);
    5504        46011 :             ml = invert_mod2n (d >> pre_shift, size);
    5505        46011 :             t1 = expand_shift (RSHIFT_EXPR, int_mode, op0,
    5506        46011 :                                pre_shift, NULL_RTX, unsignedp);
    5507        46011 :             quotient = expand_mult (int_mode, t1, gen_int_mode (ml, int_mode),
    5508              :                                     NULL_RTX, 1);
    5509              : 
    5510        46011 :             insn = get_last_insn ();
    5511        92022 :             set_dst_reg_note (insn, REG_EQUAL,
    5512              :                               gen_rtx_fmt_ee (unsignedp ? UDIV : DIV,
    5513              :                                               int_mode, op0, op1),
    5514              :                               quotient);
    5515              :           }
    5516              :         break;
    5517              : 
    5518            2 :       case ROUND_DIV_EXPR:
    5519            2 :       case ROUND_MOD_EXPR:
    5520            2 :         if (unsignedp)
    5521              :           {
    5522            0 :             scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
    5523            0 :             rtx tem;
    5524            0 :             rtx_code_label *label;
    5525            0 :             label = gen_label_rtx ();
    5526            0 :             quotient = gen_reg_rtx (int_mode);
    5527            0 :             remainder = gen_reg_rtx (int_mode);
    5528            0 :             if (expand_twoval_binop (udivmod_optab, op0, op1, quotient, remainder, 1) == 0)
    5529              :               {
    5530            0 :                 rtx tem;
    5531            0 :                 quotient = expand_binop (int_mode, udiv_optab, op0, op1,
    5532              :                                          quotient, 1, methods);
    5533            0 :                 tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 1);
    5534            0 :                 remainder = expand_binop (int_mode, sub_optab, op0, tem,
    5535              :                                           remainder, 1, methods);
    5536              :               }
    5537            0 :             tem = plus_constant (int_mode, op1, -1);
    5538            0 :             tem = expand_shift (RSHIFT_EXPR, int_mode, tem, 1, NULL_RTX, 1);
    5539            0 :             do_cmp_and_jump (remainder, tem, LEU, int_mode, label);
    5540            0 :             expand_inc (quotient, const1_rtx);
    5541            0 :             expand_dec (remainder, op1);
    5542            0 :             emit_label (label);
    5543              :           }
    5544              :         else
    5545              :           {
    5546            2 :             scalar_int_mode int_mode = as_a <scalar_int_mode> (compute_mode);
    5547            2 :             int size = GET_MODE_BITSIZE (int_mode);
    5548            2 :             rtx abs_rem, abs_op1, tem, mask;
    5549            2 :             rtx_code_label *label;
    5550            2 :             label = gen_label_rtx ();
    5551            2 :             quotient = gen_reg_rtx (int_mode);
    5552            2 :             remainder = gen_reg_rtx (int_mode);
    5553            2 :             if (expand_twoval_binop (sdivmod_optab, op0, op1, quotient, remainder, 0) == 0)
    5554              :               {
    5555            0 :                 rtx tem;
    5556            0 :                 quotient = expand_binop (int_mode, sdiv_optab, op0, op1,
    5557              :                                          quotient, 0, methods);
    5558            0 :                 tem = expand_mult (int_mode, quotient, op1, NULL_RTX, 0);
    5559            0 :                 remainder = expand_binop (int_mode, sub_optab, op0, tem,
    5560              :                                           remainder, 0, methods);
    5561              :               }
    5562            2 :             abs_rem = expand_abs (int_mode, remainder, NULL_RTX, 1, 0);
    5563            2 :             abs_op1 = expand_abs (int_mode, op1, NULL_RTX, 1, 0);
    5564            2 :             tem = expand_shift (LSHIFT_EXPR, int_mode, abs_rem,
    5565              :                                 1, NULL_RTX, 1);
    5566            2 :             do_cmp_and_jump (tem, abs_op1, LTU, int_mode, label);
    5567            2 :             tem = expand_binop (int_mode, xor_optab, op0, op1,
    5568              :                                 NULL_RTX, 0, OPTAB_WIDEN);
    5569            4 :             mask = expand_shift (RSHIFT_EXPR, int_mode, tem,
    5570            2 :                                  size - 1, NULL_RTX, 0);
    5571            2 :             tem = expand_binop (int_mode, xor_optab, mask, const1_rtx,
    5572              :                                 NULL_RTX, 0, OPTAB_WIDEN);
    5573            2 :             tem = expand_binop (int_mode, sub_optab, tem, mask,
    5574              :                                 NULL_RTX, 0, OPTAB_WIDEN);
    5575            2 :             expand_inc (quotient, tem);
    5576            2 :             tem = expand_binop (int_mode, xor_optab, mask, op1,
    5577              :                                 NULL_RTX, 0, OPTAB_WIDEN);
    5578            2 :             tem = expand_binop (int_mode, sub_optab, tem, mask,
    5579              :                                 NULL_RTX, 0, OPTAB_WIDEN);
    5580            2 :             expand_dec (remainder, tem);
    5581            2 :             emit_label (label);
    5582              :           }
    5583            3 :         return gen_lowpart (mode, rem_flag ? remainder : quotient);
    5584              : 
    5585            0 :       default:
    5586            0 :         gcc_unreachable ();
    5587              :       }
    5588              : 
    5589       237048 :   if (quotient == 0)
    5590              :     {
    5591       109013 :       if (target && GET_MODE (target) != compute_mode)
    5592        65577 :         target = 0;
    5593              : 
    5594       109013 :       if (rem_flag)
    5595              :         {
    5596              :           /* Try to produce the remainder without producing the quotient.
    5597              :              If we seem to have a divmod pattern that does not require widening,
    5598              :              don't try widening here.  We should really have a WIDEN argument
    5599              :              to expand_twoval_binop, since what we'd really like to do here is
    5600              :              1) try a mod insn in compute_mode
    5601              :              2) try a divmod insn in compute_mode
    5602              :              3) try a div insn in compute_mode and multiply-subtract to get
    5603              :                 remainder
    5604              :              4) try the same things with widening allowed.  */
    5605        41936 :           remainder
    5606        43430 :             = sign_expand_binop (compute_mode, umod_optab, smod_optab,
    5607              :                                  op0, op1, target,
    5608              :                                  unsignedp,
    5609        41936 :                                  ((optab_handler (optab2, compute_mode)
    5610              :                                    != CODE_FOR_nothing)
    5611              :                                   ? OPTAB_DIRECT : OPTAB_WIDEN));
    5612        41936 :           if (remainder == 0)
    5613              :             {
    5614              :               /* No luck there.  Can we do remainder and divide at once
    5615              :                  without a library call?  */
    5616        41729 :               remainder = gen_reg_rtx (compute_mode);
    5617        66257 :               if (! expand_twoval_binop ((unsignedp
    5618              :                                           ? udivmod_optab
    5619              :                                           : sdivmod_optab),
    5620              :                                          op0, op1,
    5621              :                                          NULL_RTX, remainder, unsignedp))
    5622              :                 remainder = 0;
    5623              :             }
    5624              : 
    5625        40442 :           if (remainder)
    5626        40649 :             return gen_lowpart (mode, remainder);
    5627              :         }
    5628              : 
    5629              :       /* Produce the quotient.  Try a quotient insn, but not a library call.
    5630              :          If we have a divmod in this mode, use it in preference to widening
    5631              :          the div (for this test we assume it will not fail). Note that optab2
    5632              :          is set to the one of the two optabs that the call below will use.  */
    5633        68364 :       quotient
    5634        73646 :         = sign_expand_binop (compute_mode, udiv_optab, sdiv_optab,
    5635              :                              op0, op1, rem_flag ? NULL_RTX : target,
    5636              :                              unsignedp,
    5637        68364 :                              ((optab_handler (optab2, compute_mode)
    5638              :                                != CODE_FOR_nothing)
    5639              :                               ? OPTAB_DIRECT : OPTAB_WIDEN));
    5640              : 
    5641        68364 :       if (quotient == 0)
    5642              :         {
    5643              :           /* No luck there.  Try a quotient-and-remainder insn,
    5644              :              keeping the quotient alone.  */
    5645        67929 :           quotient = gen_reg_rtx (compute_mode);
    5646        99196 :           if (! expand_twoval_binop (unsignedp ? udivmod_optab : sdivmod_optab,
    5647              :                                      op0, op1,
    5648              :                                      quotient, NULL_RTX, unsignedp))
    5649              :             {
    5650         3561 :               quotient = 0;
    5651         3561 :               if (! rem_flag)
    5652              :                 /* Still no luck.  If we are not computing the remainder,
    5653              :                    use a library call for the quotient.  */
    5654         2312 :                 quotient = sign_expand_binop (compute_mode,
    5655              :                                               udiv_optab, sdiv_optab,
    5656              :                                               op0, op1, target,
    5657              :                                               unsignedp, methods);
    5658              :             }
    5659              :         }
    5660              :     }
    5661              : 
    5662       196400 :   if (rem_flag)
    5663              :     {
    5664        22235 :       if (target && GET_MODE (target) != compute_mode)
    5665        15138 :         target = 0;
    5666              : 
    5667        22235 :       if (quotient == 0)
    5668              :         {
    5669              :           /* No divide instruction either.  Use library for remainder.  */
    5670         1249 :           remainder = sign_expand_binop (compute_mode, umod_optab, smod_optab,
    5671              :                                          op0, op1, target,
    5672              :                                          unsignedp, methods);
    5673              :           /* No remainder function.  Try a quotient-and-remainder
    5674              :              function, keeping the remainder.  */
    5675         1249 :           if (!remainder
    5676            0 :               && (methods == OPTAB_LIB || methods == OPTAB_LIB_WIDEN))
    5677              :             {
    5678            0 :               remainder = gen_reg_rtx (compute_mode);
    5679            0 :               if (!expand_twoval_binop_libfunc
    5680            0 :                   (unsignedp ? udivmod_optab : sdivmod_optab,
    5681              :                    op0, op1,
    5682              :                    NULL_RTX, remainder,
    5683              :                    unsignedp ? UMOD : MOD))
    5684            0 :                 remainder = NULL_RTX;
    5685              :             }
    5686              :         }
    5687              :       else
    5688              :         {
    5689              :           /* We divided.  Now finish doing X - Y * (X / Y).  */
    5690        20986 :           remainder = expand_mult (compute_mode, quotient, op1,
    5691              :                                    NULL_RTX, unsignedp);
    5692        20986 :           remainder = expand_binop (compute_mode, sub_optab, op0,
    5693              :                                     remainder, target, unsignedp,
    5694              :                                     methods);
    5695              :         }
    5696              :     }
    5697              : 
    5698       197649 :   if (methods != OPTAB_LIB_WIDEN
    5699         1793 :       && (rem_flag ? remainder : quotient) == NULL_RTX)
    5700              :     return NULL_RTX;
    5701              : 
    5702       373063 :   return gen_lowpart (mode, rem_flag ? remainder : quotient);
    5703              : }
    5704              : 
    5705              : /* Return a tree node with data type TYPE, describing the value of X.
    5706              :    Usually this is an VAR_DECL, if there is no obvious better choice.
    5707              :    X may be an expression, however we only support those expressions
    5708              :    generated by loop.c.  */
    5709              : 
    5710              : tree
    5711       686906 : make_tree (tree type, rtx x)
    5712              : {
    5713       686906 :   tree t;
    5714              : 
    5715       686906 :   switch (GET_CODE (x))
    5716              :     {
    5717        22976 :     case CONST_INT:
    5718        22976 :     case CONST_WIDE_INT:
    5719        22976 :       t = wide_int_to_tree (type, rtx_mode_t (x, TYPE_MODE (type)));
    5720        22976 :       return t;
    5721              : 
    5722            0 :     case CONST_POLY_INT:
    5723            0 :       return wide_int_to_tree (type, const_poly_int_value (x));
    5724              : 
    5725            0 :     case CONST_DOUBLE:
    5726            0 :       STATIC_ASSERT (HOST_BITS_PER_WIDE_INT * 2 <= MAX_BITSIZE_MODE_ANY_INT);
    5727            0 :       if (TARGET_SUPPORTS_WIDE_INT == 0 && GET_MODE (x) == VOIDmode)
    5728              :         t = wide_int_to_tree (type,
    5729              :                               wide_int::from_array (&CONST_DOUBLE_LOW (x), 2,
    5730              :                                                     HOST_BITS_PER_WIDE_INT * 2));
    5731              :       else
    5732            0 :         t = build_real (type, *CONST_DOUBLE_REAL_VALUE (x));
    5733              : 
    5734            0 :       return t;
    5735              : 
    5736            0 :     case CONST_VECTOR:
    5737            0 :       {
    5738            0 :         unsigned int npatterns = CONST_VECTOR_NPATTERNS (x);
    5739            0 :         unsigned int nelts_per_pattern = CONST_VECTOR_NELTS_PER_PATTERN (x);
    5740            0 :         tree itype = TREE_TYPE (type);
    5741              : 
    5742              :         /* Build a tree with vector elements.  */
    5743            0 :         tree_vector_builder elts (type, npatterns, nelts_per_pattern);
    5744            0 :         unsigned int count = elts.encoded_nelts ();
    5745            0 :         for (unsigned int i = 0; i < count; ++i)
    5746              :           {
    5747            0 :             rtx elt = CONST_VECTOR_ELT (x, i);
    5748            0 :             elts.quick_push (make_tree (itype, elt));
    5749              :           }
    5750              : 
    5751            0 :         return elts.build ();
    5752            0 :       }
    5753              : 
    5754            0 :     case PLUS:
    5755            0 :       return fold_build2 (PLUS_EXPR, type, make_tree (type, XEXP (x, 0)),
    5756              :                           make_tree (type, XEXP (x, 1)));
    5757              : 
    5758            0 :     case MINUS:
    5759            0 :       return fold_build2 (MINUS_EXPR, type, make_tree (type, XEXP (x, 0)),
    5760              :                           make_tree (type, XEXP (x, 1)));
    5761              : 
    5762            0 :     case NEG:
    5763            0 :       return fold_build1 (NEGATE_EXPR, type, make_tree (type, XEXP (x, 0)));
    5764              : 
    5765            0 :     case MULT:
    5766            0 :       return fold_build2 (MULT_EXPR, type, make_tree (type, XEXP (x, 0)),
    5767              :                           make_tree (type, XEXP (x, 1)));
    5768              : 
    5769            0 :     case ASHIFT:
    5770            0 :       return fold_build2 (LSHIFT_EXPR, type, make_tree (type, XEXP (x, 0)),
    5771              :                           make_tree (type, XEXP (x, 1)));
    5772              : 
    5773            0 :     case LSHIFTRT:
    5774            0 :       t = unsigned_type_for (type);
    5775            0 :       return fold_convert (type, build2 (RSHIFT_EXPR, t,
    5776              :                                          make_tree (t, XEXP (x, 0)),
    5777              :                                          make_tree (type, XEXP (x, 1))));
    5778              : 
    5779            0 :     case ASHIFTRT:
    5780            0 :       t = signed_type_for (type);
    5781            0 :       return fold_convert (type, build2 (RSHIFT_EXPR, t,
    5782              :                                          make_tree (t, XEXP (x, 0)),
    5783              :                                          make_tree (type, XEXP (x, 1))));
    5784              : 
    5785            0 :     case DIV:
    5786            0 :       if (TREE_CODE (type) != REAL_TYPE)
    5787            0 :         t = signed_type_for (type);
    5788              :       else
    5789              :         t = type;
    5790              : 
    5791            0 :       return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
    5792              :                                          make_tree (t, XEXP (x, 0)),
    5793              :                                          make_tree (t, XEXP (x, 1))));
    5794            0 :     case UDIV:
    5795            0 :       t = unsigned_type_for (type);
    5796            0 :       return fold_convert (type, build2 (TRUNC_DIV_EXPR, t,
    5797              :                                          make_tree (t, XEXP (x, 0)),
    5798              :                                          make_tree (t, XEXP (x, 1))));
    5799              : 
    5800            0 :     case SIGN_EXTEND:
    5801            0 :     case ZERO_EXTEND:
    5802            0 :       t = lang_hooks.types.type_for_mode (GET_MODE (XEXP (x, 0)),
    5803              :                                           GET_CODE (x) == ZERO_EXTEND);
    5804            0 :       return fold_convert (type, make_tree (t, XEXP (x, 0)));
    5805              : 
    5806            0 :     case CONST:
    5807            0 :       return make_tree (type, XEXP (x, 0));
    5808              : 
    5809            0 :     case SYMBOL_REF:
    5810            0 :       t = SYMBOL_REF_DECL (x);
    5811            0 :       if (t)
    5812            0 :         return fold_convert (type, build_fold_addr_expr (t));
    5813              :       /* fall through.  */
    5814              : 
    5815       663930 :     default:
    5816       663930 :       t = build_decl (RTL_LOCATION (x), VAR_DECL, NULL_TREE, type);
    5817              : 
    5818              :       /* If TYPE is a POINTER_TYPE, we might need to convert X from
    5819              :          address mode to pointer mode.  */
    5820       663930 :       if (POINTER_TYPE_P (type))
    5821       828064 :         x = convert_memory_address_addr_space
    5822       414032 :           (SCALAR_INT_TYPE_MODE (type), x, TYPE_ADDR_SPACE (TREE_TYPE (type)));
    5823              : 
    5824              :       /* Note that we do *not* use SET_DECL_RTL here, because we do not
    5825              :          want set_decl_rtl to go adjusting REG_ATTRS for this temporary.  */
    5826       663930 :       t->decl_with_rtl.rtl = x;
    5827              : 
    5828       663930 :       return t;
    5829              :     }
    5830              : }
    5831              : 
    5832              : /* Compute the logical-and of OP0 and OP1, storing it in TARGET
    5833              :    and returning TARGET.
    5834              : 
    5835              :    If TARGET is 0, a pseudo-register or constant is returned.  */
    5836              : 
    5837              : rtx
    5838        73933 : expand_and (machine_mode mode, rtx op0, rtx op1, rtx target)
    5839              : {
    5840        73933 :   rtx tem = 0;
    5841              : 
    5842        73933 :   if (GET_MODE (op0) == VOIDmode && GET_MODE (op1) == VOIDmode)
    5843           80 :     tem = simplify_binary_operation (AND, mode, op0, op1);
    5844           80 :   if (tem == 0)
    5845        73853 :     tem = expand_binop (mode, and_optab, op0, op1, target, 0, OPTAB_LIB_WIDEN);
    5846              : 
    5847        73933 :   if (target == 0)
    5848              :     target = tem;
    5849        45141 :   else if (tem != target)
    5850           13 :     emit_move_insn (target, tem);
    5851        73933 :   return target;
    5852              : }
    5853              : 
    5854              : /* Helper function for emit_store_flag.  */
    5855              : rtx
    5856       753849 : emit_cstore (rtx target, enum insn_code icode, enum rtx_code code,
    5857              :              machine_mode mode, machine_mode compare_mode,
    5858              :              int unsignedp, rtx x, rtx y, int normalizep,
    5859              :              machine_mode target_mode)
    5860              : {
    5861       753849 :   class expand_operand ops[4];
    5862       753849 :   rtx op0, comparison, subtarget;
    5863       753849 :   rtx_insn *last;
    5864       753849 :   scalar_int_mode result_mode = targetm.cstore_mode (icode);
    5865       753849 :   scalar_int_mode int_target_mode;
    5866              : 
    5867       753849 :   last = get_last_insn ();
    5868       753849 :   x = prepare_operand (icode, x, 2, mode, compare_mode, unsignedp);
    5869       753849 :   y = prepare_operand (icode, y, 3, mode, compare_mode, unsignedp);
    5870       753849 :   if (!x || !y)
    5871              :     {
    5872          284 :       delete_insns_since (last);
    5873          284 :       return NULL_RTX;
    5874              :     }
    5875              : 
    5876       753565 :   if (target_mode == VOIDmode)
    5877              :     int_target_mode = result_mode;
    5878              :   else
    5879       753549 :     int_target_mode = as_a <scalar_int_mode> (target_mode);
    5880       753565 :   if (!target)
    5881        69616 :     target = gen_reg_rtx (int_target_mode);
    5882              : 
    5883       753565 :   comparison = gen_rtx_fmt_ee (code, result_mode, x, y);
    5884              : 
    5885       753565 :   create_output_operand (&ops[0], optimize ? NULL_RTX : target, result_mode);
    5886       753565 :   create_fixed_operand (&ops[1], comparison);
    5887       753565 :   create_fixed_operand (&ops[2], x);
    5888       753565 :   create_fixed_operand (&ops[3], y);
    5889       753565 :   if (!maybe_expand_insn (icode, 4, ops))
    5890              :     {
    5891       155075 :       delete_insns_since (last);
    5892       155075 :       return NULL_RTX;
    5893              :     }
    5894       598490 :   subtarget = ops[0].value;
    5895              : 
    5896              :   /* If we are converting to a wider mode, first convert to
    5897              :      INT_TARGET_MODE, then normalize.  This produces better combining
    5898              :      opportunities on machines that have a SIGN_EXTRACT when we are
    5899              :      testing a single bit.  This mostly benefits the 68k.
    5900              : 
    5901              :      If STORE_FLAG_VALUE does not have the sign bit set when
    5902              :      interpreted in MODE, we can do this conversion as unsigned, which
    5903              :      is usually more efficient.  */
    5904       598490 :   if (GET_MODE_PRECISION (int_target_mode) > GET_MODE_PRECISION (result_mode))
    5905              :     {
    5906       115728 :       gcc_assert (GET_MODE_PRECISION (result_mode) != 1
    5907              :                   || STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1);
    5908              : 
    5909       115728 :       bool unsignedp = (STORE_FLAG_VALUE >= 0);
    5910       115728 :       convert_move (target, subtarget, unsignedp);
    5911              : 
    5912       115728 :       op0 = target;
    5913       115728 :       result_mode = int_target_mode;
    5914              :     }
    5915              :   else
    5916              :     op0 = subtarget;
    5917              : 
    5918              :   /* If we want to keep subexpressions around, don't reuse our last
    5919              :      target.  */
    5920       598490 :   if (optimize)
    5921       479834 :     subtarget = 0;
    5922              : 
    5923              :   /* Now normalize to the proper value in MODE.  Sometimes we don't
    5924              :      have to do anything.  */
    5925       598490 :   if (normalizep == 0 || normalizep == STORE_FLAG_VALUE)
    5926              :     ;
    5927              :   /* STORE_FLAG_VALUE might be the most negative number, so write
    5928              :      the comparison this way to avoid a compiler-time warning.  */
    5929          354 :   else if (- normalizep == STORE_FLAG_VALUE)
    5930          354 :     op0 = expand_unop (result_mode, neg_optab, op0, subtarget, 0);
    5931              : 
    5932              :   /* We don't want to use STORE_FLAG_VALUE < 0 below since this makes
    5933              :      it hard to use a value of just the sign bit due to ANSI integer
    5934              :      constant typing rules.  */
    5935            0 :   else if (val_signbit_known_set_p (result_mode, STORE_FLAG_VALUE))
    5936            0 :     op0 = expand_shift (RSHIFT_EXPR, result_mode, op0,
    5937            0 :                         GET_MODE_BITSIZE (result_mode) - 1, subtarget,
    5938              :                         normalizep == 1);
    5939              :   else
    5940              :     {
    5941            0 :       gcc_assert (STORE_FLAG_VALUE & 1);
    5942              : 
    5943            0 :       op0 = expand_and (result_mode, op0, const1_rtx, subtarget);
    5944            0 :       if (normalizep == -1)
    5945              :         op0 = expand_unop (result_mode, neg_optab, op0, op0, 0);
    5946              :     }
    5947              : 
    5948              :   /* If we were converting to a smaller mode, do the conversion now.  */
    5949       598490 :   if (int_target_mode != result_mode)
    5950              :     {
    5951            0 :       convert_move (target, op0, 0);
    5952            0 :       return target;
    5953              :     }
    5954              :   else
    5955              :     return op0;
    5956              : }
    5957              : 
    5958              : 
    5959              : /* A subroutine of emit_store_flag only including "tricks" that do not
    5960              :    need a recursive call.  These are kept separate to avoid infinite
    5961              :    loops.  */
    5962              : 
    5963              : static rtx
    5964       721559 : emit_store_flag_1 (rtx target, enum rtx_code code, rtx op0, rtx op1,
    5965              :                    machine_mode mode, int unsignedp, int normalizep,
    5966              :                    machine_mode target_mode)
    5967              : {
    5968       721559 :   rtx subtarget;
    5969       721559 :   enum insn_code icode;
    5970       721559 :   machine_mode compare_mode;
    5971       721559 :   enum mode_class mclass;
    5972              : 
    5973       721559 :   if (unsignedp)
    5974       166763 :     code = unsigned_condition (code);
    5975              : 
    5976              :   /* If one operand is constant, make it the second one.  Only do this
    5977              :      if the other operand is not constant as well.  */
    5978              : 
    5979       721559 :   if (swap_commutative_operands_p (op0, op1))
    5980              :     {
    5981         5355 :       std::swap (op0, op1);
    5982         5355 :       code = swap_condition (code);
    5983              :     }
    5984              : 
    5985       721559 :   if (mode == VOIDmode)
    5986        42449 :     mode = GET_MODE (op0);
    5987              : 
    5988       721559 :   if (CONST_SCALAR_INT_P (op1))
    5989       335062 :     canonicalize_comparison (mode, &code, &op1);
    5990              : 
    5991              :   /* For some comparisons with 1 and -1, we can convert this to
    5992              :      comparisons with zero.  This will often produce more opportunities for
    5993              :      store-flag insns.  */
    5994              : 
    5995       721559 :   switch (code)
    5996              :     {
    5997        37707 :     case LT:
    5998        37707 :       if (op1 == const1_rtx)
    5999           59 :         op1 = const0_rtx, code = LE;
    6000              :       break;
    6001        27139 :     case LE:
    6002        27139 :       if (op1 == constm1_rtx)
    6003            0 :         op1 = const0_rtx, code = LT;
    6004              :       break;
    6005        39857 :     case GE:
    6006        39857 :       if (op1 == const1_rtx)
    6007            0 :         op1 = const0_rtx, code = GT;
    6008              :       break;
    6009        31213 :     case GT:
    6010        31213 :       if (op1 == constm1_rtx)
    6011          145 :         op1 = const0_rtx, code = GE;
    6012              :       break;
    6013         6310 :     case GEU:
    6014         6310 :       if (op1 == const1_rtx)
    6015            0 :         op1 = const0_rtx, code = NE;
    6016              :       break;
    6017        10664 :     case LTU:
    6018        10664 :       if (op1 == const1_rtx)
    6019           10 :         op1 = const0_rtx, code = EQ;
    6020              :       break;
    6021              :     default:
    6022              :       break;
    6023              :     }
    6024              : 
    6025              :   /* If this is A < 0 or A >= 0, we can do this by taking the ones
    6026              :      complement of A (for GE) and shifting the sign bit to the low bit.  */
    6027       721559 :   scalar_int_mode int_mode;
    6028       200871 :   if (op1 == const0_rtx && (code == LT || code == GE)
    6029       721559 :       && is_int_mode (mode, &int_mode)
    6030       721559 :       && (normalizep || STORE_FLAG_VALUE == 1
    6031              :           || val_signbit_p (int_mode, STORE_FLAG_VALUE)))
    6032              :     {
    6033        41231 :       scalar_int_mode int_target_mode;
    6034        41231 :       subtarget = target;
    6035              : 
    6036        41231 :       if (!target)
    6037              :         int_target_mode = int_mode;
    6038              :       else
    6039              :         {
    6040              :           /* If the result is to be wider than OP0, it is best to convert it
    6041              :              first.  If it is to be narrower, it is *incorrect* to convert it
    6042              :              first.  */
    6043        41231 :           int_target_mode = as_a <scalar_int_mode> (target_mode);
    6044       123693 :           if (GET_MODE_SIZE (int_target_mode) > GET_MODE_SIZE (int_mode))
    6045              :             {
    6046          461 :               op0 = convert_modes (int_target_mode, int_mode, op0, 0);
    6047          461 :               int_mode = int_target_mode;
    6048              :             }
    6049              :         }
    6050              : 
    6051        41231 :       if (int_target_mode != int_mode)
    6052        27135 :         subtarget = 0;
    6053              : 
    6054        41231 :       if (code == GE)
    6055        20789 :         op0 = expand_unop (int_mode, one_cmpl_optab, op0,
    6056              :                            ((STORE_FLAG_VALUE == 1 || normalizep)
    6057              :                             ? 0 : subtarget), 0);
    6058              : 
    6059        41231 :       if (STORE_FLAG_VALUE == 1 || normalizep)
    6060              :         /* If we are supposed to produce a 0/1 value, we want to do
    6061              :            a logical shift from the sign bit to the low-order bit; for
    6062              :            a -1/0 value, we do an arithmetic shift.  */
    6063        82462 :         op0 = expand_shift (RSHIFT_EXPR, int_mode, op0,
    6064        41231 :                             GET_MODE_BITSIZE (int_mode) - 1,
    6065              :                             subtarget, normalizep != -1);
    6066              : 
    6067        41231 :       if (int_mode != int_target_mode)
    6068        27135 :         op0 = convert_modes (int_target_mode, int_mode, op0, 0);
    6069              : 
    6070              :       return op0;
    6071              :     }
    6072              : 
    6073              :   /* Next try expanding this via the backend's cstore<mode>4.  */
    6074       680328 :   mclass = GET_MODE_CLASS (mode);
    6075       692329 :   FOR_EACH_WIDER_MODE_FROM (compare_mode, mode)
    6076              :     {
    6077       685046 :      machine_mode optab_mode = mclass == MODE_CC ? CCmode : compare_mode;
    6078       685046 :      icode = optab_handler (cstore_optab, optab_mode);
    6079       685046 :      if (icode != CODE_FOR_nothing)
    6080              :         {
    6081       673045 :           do_pending_stack_adjust ();
    6082       673045 :           rtx tem = emit_cstore (target, icode, code, mode, compare_mode,
    6083              :                                  unsignedp, op0, op1, normalizep, target_mode);
    6084       673045 :           if (tem)
    6085              :             return tem;
    6086              : 
    6087        89966 :           if (GET_MODE_CLASS (mode) == MODE_FLOAT)
    6088              :             {
    6089        80784 :               enum rtx_code scode = swap_condition (code);
    6090              : 
    6091        80784 :               tem = emit_cstore (target, icode, scode, mode, compare_mode,
    6092              :                                  unsignedp, op1, op0, normalizep, target_mode);
    6093        80784 :               if (tem)
    6094              :                 return tem;
    6095              :             }
    6096              :           break;
    6097              :         }
    6098              :     }
    6099              : 
    6100              :   /* If we are comparing a double-word integer with zero or -1, we can
    6101              :      convert the comparison into one involving a single word.  */
    6102        81858 :   if (is_int_mode (mode, &int_mode)
    6103        16052 :       && GET_MODE_BITSIZE (int_mode) == BITS_PER_WORD * 2
    6104         8898 :       && (!MEM_P (op0) || ! MEM_VOLATILE_P (op0)))
    6105              :     {
    6106         8898 :       rtx tem;
    6107         8898 :       if ((code == EQ || code == NE)
    6108            0 :           && (op1 == const0_rtx || op1 == constm1_rtx))
    6109              :         {
    6110            0 :           rtx op00, op01;
    6111              : 
    6112              :           /* Do a logical OR or AND of the two words and compare the
    6113              :              result.  */
    6114            0 :           op00 = force_subreg (word_mode, op0, int_mode, 0);
    6115            0 :           op01 = force_subreg (word_mode, op0, int_mode, UNITS_PER_WORD);
    6116            0 :           tem = expand_binop (word_mode,
    6117            0 :                               op1 == const0_rtx ? ior_optab : and_optab,
    6118              :                               op00, op01, NULL_RTX, unsignedp,
    6119              :                               OPTAB_DIRECT);
    6120              : 
    6121            0 :           if (tem != 0)
    6122            0 :             tem = emit_store_flag (NULL_RTX, code, tem, op1, word_mode,
    6123              :                                    unsignedp, normalizep);
    6124              :         }
    6125         8898 :       else if ((code == LT || code == GE) && op1 == const0_rtx)
    6126              :         {
    6127            0 :           rtx op0h;
    6128              : 
    6129              :           /* If testing the sign bit, can just test on high word.  */
    6130            0 :           op0h = force_highpart_subreg (word_mode, op0, int_mode);
    6131            0 :           tem = emit_store_flag (NULL_RTX, code, op0h, op1, word_mode,
    6132              :                                  unsignedp, normalizep);
    6133            0 :         }
    6134              :       else
    6135              :         tem = NULL_RTX;
    6136              : 
    6137            0 :       if (tem)
    6138              :         {
    6139            0 :           if (target_mode == VOIDmode || GET_MODE (tem) == target_mode)
    6140              :             return tem;
    6141            0 :           if (!target)
    6142            0 :             target = gen_reg_rtx (target_mode);
    6143              : 
    6144            0 :           convert_move (target, tem,
    6145            0 :                         !val_signbit_known_set_p (word_mode,
    6146              :                                                   (normalizep ? normalizep
    6147              :                                                    : STORE_FLAG_VALUE)));
    6148            0 :           return target;
    6149              :         }
    6150              :     }
    6151              : 
    6152              :   return 0;
    6153              : }
    6154              : 
    6155              : /* Subroutine of emit_store_flag that handles cases in which the operands
    6156              :    are scalar integers.  SUBTARGET is the target to use for temporary
    6157              :    operations and TRUEVAL is the value to store when the condition is
    6158              :    true.  All other arguments are as for emit_store_flag.  */
    6159              : 
    6160              : rtx
    6161         4591 : emit_store_flag_int (rtx target, rtx subtarget, enum rtx_code code, rtx op0,
    6162              :                      rtx op1, scalar_int_mode mode, int unsignedp,
    6163              :                      int normalizep, rtx trueval)
    6164              : {
    6165         4591 :   machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
    6166         4591 :   rtx_insn *last = get_last_insn ();
    6167              : 
    6168              :   /* If this is an equality comparison of integers, we can try to exclusive-or
    6169              :      (or subtract) the two operands and use a recursive call to try the
    6170              :      comparison with zero.  Don't do any of these cases if branches are
    6171              :      very cheap.  */
    6172              : 
    6173         4591 :   if ((code == EQ || code == NE) && op1 != const0_rtx)
    6174              :     {
    6175            0 :       rtx tem = expand_binop (mode, xor_optab, op0, op1, subtarget, 1,
    6176              :                               OPTAB_WIDEN);
    6177              : 
    6178            0 :       if (tem == 0)
    6179            0 :         tem = expand_binop (mode, sub_optab, op0, op1, subtarget, 1,
    6180              :                             OPTAB_WIDEN);
    6181            0 :       if (tem != 0)
    6182            0 :         tem = emit_store_flag (target, code, tem, const0_rtx,
    6183              :                                mode, unsignedp, normalizep);
    6184            0 :       if (tem != 0)
    6185              :         return tem;
    6186              : 
    6187            0 :       delete_insns_since (last);
    6188              :     }
    6189              : 
    6190              :   /* For integer comparisons, try the reverse comparison.  However, for
    6191              :      small X and if we'd have anyway to extend, implementing "X != 0"
    6192              :      as "-(int)X >> 31" is still cheaper than inverting "(int)X == 0".  */
    6193         4591 :   rtx_code rcode = reverse_condition (code);
    6194         4591 :   if (can_compare_p (rcode, mode, ccp_store_flag)
    6195         4591 :       && ! (optab_handler (cstore_optab, mode) == CODE_FOR_nothing
    6196            0 :             && code == NE
    6197            0 :             && GET_MODE_SIZE (mode) < UNITS_PER_WORD
    6198            0 :             && op1 == const0_rtx))
    6199              :     {
    6200         4591 :       int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
    6201              :                       || (STORE_FLAG_VALUE == -1 && normalizep == 1));
    6202              : 
    6203              :       /* Again, for the reverse comparison, use either an addition or a XOR.  */
    6204         4591 :       if (want_add
    6205         4591 :           && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
    6206              :                        optimize_insn_for_speed_p ()) == 0)
    6207              :         {
    6208            0 :           rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
    6209              :                                        STORE_FLAG_VALUE, target_mode);
    6210            0 :           if (tem != 0)
    6211            0 :             tem = expand_binop (target_mode, add_optab, tem,
    6212            0 :                                 gen_int_mode (normalizep, target_mode),
    6213              :                                 target, 0, OPTAB_WIDEN);
    6214            0 :           if (tem != 0)
    6215              :             return tem;
    6216              :         }
    6217         4591 :       else if (!want_add
    6218         4591 :                && rtx_cost (trueval, mode, XOR, 1,
    6219              :                             optimize_insn_for_speed_p ()) == 0)
    6220              :         {
    6221         4591 :           rtx tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
    6222              :                                        normalizep, target_mode);
    6223         4591 :           if (tem != 0)
    6224            0 :             tem = expand_binop (target_mode, xor_optab, tem, trueval, target,
    6225            0 :                                 INTVAL (trueval) >= 0, OPTAB_WIDEN);
    6226            0 :           if (tem != 0)
    6227              :             return tem;
    6228              :         }
    6229              : 
    6230         4591 :       delete_insns_since (last);
    6231              :     }
    6232              : 
    6233              :   /* Some other cases we can do are EQ, NE, LE, and GT comparisons with
    6234              :      the constant zero.  Reject all other comparisons at this point.  Only
    6235              :      do LE and GT if branches are expensive since they are expensive on
    6236              :      2-operand machines.  */
    6237              : 
    6238         4591 :   if (op1 != const0_rtx
    6239         4591 :       || (code != EQ && code != NE
    6240           91 :           && (BRANCH_COST (optimize_insn_for_speed_p (),
    6241           91 :                            false) <= 1 || (code != LE && code != GT))))
    6242              :     return 0;
    6243              : 
    6244              :   /* Try to put the result of the comparison in the sign bit.  Assume we can't
    6245              :      do the necessary operation below.  */
    6246              : 
    6247          233 :   rtx tem = 0;
    6248              : 
    6249              :   /* To see if A <= 0, compute (A | (A - 1)).  A <= 0 iff that result has
    6250              :      the sign bit set.  */
    6251              : 
    6252          233 :   if (code == LE)
    6253              :     {
    6254              :       /* This is destructive, so SUBTARGET can't be OP0.  */
    6255           56 :       if (rtx_equal_p (subtarget, op0))
    6256            0 :         subtarget = 0;
    6257              : 
    6258           56 :       tem = expand_binop (mode, sub_optab, op0, const1_rtx, subtarget, 0,
    6259              :                           OPTAB_WIDEN);
    6260           56 :       if (tem)
    6261           56 :         tem = expand_binop (mode, ior_optab, op0, tem, subtarget, 0,
    6262              :                             OPTAB_WIDEN);
    6263              :     }
    6264              : 
    6265              :   /* To see if A > 0, compute (((signed) A) << BITS) - A, where BITS is the
    6266              :      number of bits in the mode of OP0, minus one.  */
    6267              : 
    6268          233 :   if (code == GT)
    6269              :     {
    6270           35 :       if (rtx_equal_p (subtarget, op0))
    6271            0 :         subtarget = 0;
    6272              : 
    6273           35 :       tem = maybe_expand_shift (RSHIFT_EXPR, mode, op0,
    6274           35 :                                 GET_MODE_BITSIZE (mode) - 1,
    6275              :                                 subtarget, 0);
    6276           35 :       if (tem)
    6277           35 :         tem = expand_binop (mode, sub_optab, tem, op0, subtarget, 0,
    6278              :                             OPTAB_WIDEN);
    6279              :     }
    6280              : 
    6281          233 :   if (code == EQ || code == NE)
    6282              :     {
    6283              :       /* For EQ or NE, one way to do the comparison is to apply an operation
    6284              :          that converts the operand into a positive number if it is nonzero
    6285              :          or zero if it was originally zero.  Then, for EQ, we subtract 1 and
    6286              :          for NE we negate.  This puts the result in the sign bit.  Then we
    6287              :          normalize with a shift, if needed.
    6288              : 
    6289              :          Two operations that can do the above actions are ABS and FFS, so try
    6290              :          them.  If that doesn't work, and MODE is smaller than a full word,
    6291              :          we can use zero-extension to the wider mode (an unsigned conversion)
    6292              :          as the operation.  */
    6293              : 
    6294              :       /* Note that ABS doesn't yield a positive number for INT_MIN, but
    6295              :          that is compensated by the subsequent overflow when subtracting
    6296              :          one / negating.  */
    6297              : 
    6298          142 :       if (optab_handler (abs_optab, mode) != CODE_FOR_nothing)
    6299          142 :         tem = expand_unop (mode, abs_optab, op0, subtarget, 1);
    6300            0 :       else if (optab_handler (ffs_optab, mode) != CODE_FOR_nothing)
    6301            0 :         tem = expand_unop (mode, ffs_optab, op0, subtarget, 1);
    6302            0 :       else if (GET_MODE_SIZE (mode) < UNITS_PER_WORD)
    6303              :         {
    6304            0 :           tem = convert_modes (word_mode, mode, op0, 1);
    6305            0 :           mode = word_mode;
    6306              :         }
    6307              : 
    6308          142 :       if (tem != 0)
    6309              :         {
    6310          142 :           if (code == EQ)
    6311            0 :             tem = expand_binop (mode, sub_optab, tem, const1_rtx, subtarget,
    6312              :                                 0, OPTAB_WIDEN);
    6313              :           else
    6314          142 :             tem = expand_unop (mode, neg_optab, tem, subtarget, 0);
    6315              :         }
    6316              : 
    6317              :       /* If we couldn't do it that way, for NE we can "or" the two's complement
    6318              :          of the value with itself.  For EQ, we take the one's complement of
    6319              :          that "or", which is an extra insn, so we only handle EQ if branches
    6320              :          are expensive.  */
    6321              : 
    6322          142 :       if (tem == 0
    6323          142 :           && (code == NE
    6324            0 :               || BRANCH_COST (optimize_insn_for_speed_p (),
    6325              :                               false) > 1))
    6326              :         {
    6327            0 :           if (rtx_equal_p (subtarget, op0))
    6328            0 :             subtarget = 0;
    6329              : 
    6330            0 :           tem = expand_unop (mode, neg_optab, op0, subtarget, 0);
    6331            0 :           tem = expand_binop (mode, ior_optab, tem, op0, subtarget, 0,
    6332              :                               OPTAB_WIDEN);
    6333              : 
    6334            0 :           if (tem && code == EQ)
    6335            0 :             tem = expand_unop (mode, one_cmpl_optab, tem, subtarget, 0);
    6336              :         }
    6337              :     }
    6338              : 
    6339          233 :   if (tem && normalizep)
    6340          233 :     tem = maybe_expand_shift (RSHIFT_EXPR, mode, tem,
    6341          233 :                               GET_MODE_BITSIZE (mode) - 1,
    6342              :                               subtarget, normalizep == 1);
    6343              : 
    6344          233 :   if (tem)
    6345              :     {
    6346          233 :       if (!target)
    6347              :         ;
    6348          233 :       else if (GET_MODE (tem) != target_mode)
    6349              :         {
    6350           91 :           convert_move (target, tem, 0);
    6351           91 :           tem = target;
    6352              :         }
    6353          142 :       else if (!subtarget)
    6354              :         {
    6355           73 :           emit_move_insn (target, tem);
    6356           73 :           tem = target;
    6357              :         }
    6358              :     }
    6359              :   else
    6360            0 :     delete_insns_since (last);
    6361              : 
    6362              :   return tem;
    6363              : }
    6364              : 
    6365              : /* Emit a store-flags instruction for comparison CODE on OP0 and OP1
    6366              :    and storing in TARGET.  Normally return TARGET.
    6367              :    Return 0 if that cannot be done.
    6368              : 
    6369              :    MODE is the mode to use for OP0 and OP1 should they be CONST_INTs.  If
    6370              :    it is VOIDmode, they cannot both be CONST_INT.
    6371              : 
    6372              :    UNSIGNEDP is for the case where we have to widen the operands
    6373              :    to perform the operation.  It says to use zero-extension.
    6374              : 
    6375              :    NORMALIZEP is 1 if we should convert the result to be either zero
    6376              :    or one.  Normalize is -1 if we should convert the result to be
    6377              :    either zero or -1.  If NORMALIZEP is zero, the result will be left
    6378              :    "raw" out of the scc insn.  */
    6379              : 
    6380              : rtx
    6381       651364 : emit_store_flag (rtx target, enum rtx_code code, rtx op0, rtx op1,
    6382              :                  machine_mode mode, int unsignedp, int normalizep)
    6383              : {
    6384       651364 :   machine_mode target_mode = target ? GET_MODE (target) : VOIDmode;
    6385       651364 :   enum rtx_code rcode;
    6386       651364 :   rtx subtarget;
    6387       651364 :   rtx tem, trueval;
    6388       651364 :   rtx_insn *last;
    6389              : 
    6390              :   /* If we compare constants, we shouldn't use a store-flag operation,
    6391              :      but a constant load.  We can get there via the vanilla route that
    6392              :      usually generates a compare-branch sequence, but will in this case
    6393              :      fold the comparison to a constant, and thus elide the branch.  */
    6394       651364 :   if (CONSTANT_P (op0) && CONSTANT_P (op1))
    6395              :     return NULL_RTX;
    6396              : 
    6397       651048 :   tem = emit_store_flag_1 (target, code, op0, op1, mode, unsignedp, normalizep,
    6398              :                            target_mode);
    6399       651048 :   if (tem)
    6400              :     return tem;
    6401              : 
    6402              :   /* If we reached here, we can't do this with a scc insn, however there
    6403              :      are some comparisons that can be done in other ways.  Don't do any
    6404              :      of these cases if branches are very cheap.  */
    6405        76740 :   if (BRANCH_COST (optimize_insn_for_speed_p (), false) == 0)
    6406              :     return 0;
    6407              : 
    6408              :   /* See what we need to return.  We can only return a 1, -1, or the
    6409              :      sign bit.  */
    6410              : 
    6411        76740 :   if (normalizep == 0)
    6412              :     {
    6413            0 :       if (STORE_FLAG_VALUE == 1 || STORE_FLAG_VALUE == -1)
    6414            0 :         normalizep = STORE_FLAG_VALUE;
    6415              : 
    6416              :       else if (val_signbit_p (mode, STORE_FLAG_VALUE))
    6417              :         ;
    6418              :       else
    6419              :         return 0;
    6420              :     }
    6421              : 
    6422        76740 :   last = get_last_insn ();
    6423              : 
    6424              :   /* If optimizing, use different pseudo registers for each insn, instead
    6425              :      of reusing the same pseudo.  This leads to better CSE, but slows
    6426              :      down the compiler, since there are more pseudos.  */
    6427        76671 :   subtarget = (!optimize
    6428        76740 :                && (target_mode == mode)) ? target : NULL_RTX;
    6429        76740 :   trueval = GEN_INT (normalizep ? normalizep : STORE_FLAG_VALUE);
    6430              : 
    6431              :   /* For floating-point comparisons, try the reverse comparison or try
    6432              :      changing the "orderedness" of the comparison.  */
    6433        76740 :   if (GET_MODE_CLASS (mode) == MODE_FLOAT)
    6434              :     {
    6435        68634 :       enum rtx_code first_code;
    6436        68634 :       bool and_them;
    6437              : 
    6438        68634 :       rcode = reverse_condition_maybe_unordered (code);
    6439        68634 :       if (can_compare_p (rcode, mode, ccp_store_flag)
    6440        68634 :           && (code == ORDERED || code == UNORDERED
    6441            0 :               || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
    6442            0 :               || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
    6443              :         {
    6444            0 :           int want_add = ((STORE_FLAG_VALUE == 1 && normalizep == -1)
    6445              :                           || (STORE_FLAG_VALUE == -1 && normalizep == 1));
    6446              : 
    6447              :           /* For the reverse comparison, use either an addition or a XOR.  */
    6448            0 :           if (want_add
    6449            0 :               && rtx_cost (GEN_INT (normalizep), mode, PLUS, 1,
    6450              :                            optimize_insn_for_speed_p ()) == 0)
    6451              :             {
    6452            0 :               tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
    6453              :                                        STORE_FLAG_VALUE, target_mode);
    6454            0 :               if (tem)
    6455            0 :                 return expand_binop (target_mode, add_optab, tem,
    6456            0 :                                      gen_int_mode (normalizep, target_mode),
    6457              :                                      target, 0, OPTAB_WIDEN);
    6458              :             }
    6459            0 :           else if (!want_add
    6460            0 :                    && rtx_cost (trueval, mode, XOR, 1,
    6461              :                                 optimize_insn_for_speed_p ()) == 0)
    6462              :             {
    6463            0 :               tem = emit_store_flag_1 (subtarget, rcode, op0, op1, mode, 0,
    6464              :                                        normalizep, target_mode);
    6465            0 :               if (tem)
    6466            0 :                 return expand_binop (target_mode, xor_optab, tem, trueval,
    6467            0 :                                      target, INTVAL (trueval) >= 0,
    6468            0 :                                      OPTAB_WIDEN);
    6469              :             }
    6470              :         }
    6471              : 
    6472        68634 :       delete_insns_since (last);
    6473              : 
    6474              :       /* Cannot split ORDERED and UNORDERED, only try the above trick.  */
    6475        68634 :       if (code == ORDERED || code == UNORDERED)
    6476              :         return 0;
    6477              : 
    6478        68492 :       and_them = split_comparison (code, mode, &first_code, &code);
    6479              : 
    6480              :       /* If there are no NaNs, the first comparison should always fall through.
    6481              :          Effectively change the comparison to the other one.  */
    6482        68492 :       if (!HONOR_NANS (mode))
    6483              :         {
    6484          404 :           gcc_assert (first_code == (and_them ? ORDERED : UNORDERED));
    6485          243 :           return emit_store_flag_1 (target, code, op0, op1, mode, 0, normalizep,
    6486          243 :                                     target_mode);
    6487              :         }
    6488              : 
    6489        68249 :       if (!HAVE_conditional_move)
    6490              :         return 0;
    6491              : 
    6492              :       /* x != y is split into x unord y || x ltgt y, but the conditional move
    6493              :          below evaluates its comparison unconditionally and LTGT is signaling.
    6494              :          Use the quiet x unord y || !(x uneq y) instead.  */
    6495        68249 :       bool invert = false;
    6496        68249 :       if (code == LTGT && !and_them)
    6497              :         {
    6498        62616 :           code = UNEQ;
    6499        62616 :           invert = true;
    6500              :         }
    6501              : 
    6502              :       /* Do not change whether the comparison traps: only the quiet
    6503              :          EQ, NE and UNEQ remainders come from quiet comparisons.  */
    6504        68249 :       if ((code != EQ && code != NE && code != UNEQ)
    6505         2572 :           && flag_trapping_math)
    6506              :         return 0;
    6507              : 
    6508              :       /* Try using a setcc instruction for ORDERED/UNORDERED, followed by a
    6509              :          conditional move.  */
    6510        65677 :       tem = emit_store_flag_1 (subtarget, first_code, op0, op1, mode, 0,
    6511              :                                normalizep, target_mode);
    6512        65677 :       if (tem == 0)
    6513              :         return 0;
    6514              : 
    6515        65151 :       if (and_them)
    6516         2774 :         tem = emit_conditional_move (target, { code, op0, op1, mode },
    6517         2774 :                                      tem, const0_rtx, GET_MODE (tem), 0);
    6518        62377 :       else if (invert)
    6519        62377 :         tem = emit_conditional_move (target, { code, op0, op1, mode },
    6520        62377 :                                      tem, trueval, GET_MODE (tem), 0);
    6521              :       else
    6522            0 :         tem = emit_conditional_move (target, { code, op0, op1, mode },
    6523            0 :                                      trueval, tem, GET_MODE (tem), 0);
    6524              : 
    6525        65151 :       if (tem == 0)
    6526          725 :         delete_insns_since (last);
    6527              :       return tem;
    6528              :     }
    6529              : 
    6530              :   /* The remaining tricks only apply to integer comparisons.  */
    6531              : 
    6532         8106 :   scalar_int_mode int_mode;
    6533         8106 :   if (is_int_mode (mode, &int_mode))
    6534         4591 :     return emit_store_flag_int (target, subtarget, code, op0, op1, int_mode,
    6535         4591 :                                 unsignedp, normalizep, trueval);
    6536              : 
    6537              :   return 0;
    6538              : }
    6539              : 
    6540              : /* Like emit_store_flag, but always succeeds.  */
    6541              : 
    6542              : rtx
    6543       599034 : emit_store_flag_force (rtx target, enum rtx_code code, rtx op0, rtx op1,
    6544              :                        machine_mode mode, int unsignedp, int normalizep)
    6545              : {
    6546       599034 :   rtx tem;
    6547       599034 :   rtx_code_label *label;
    6548       599034 :   rtx trueval, falseval;
    6549              : 
    6550              :   /* First see if emit_store_flag can do the job.  */
    6551       599034 :   tem = emit_store_flag (target, code, op0, op1, mode, unsignedp, normalizep);
    6552       599034 :   if (tem != 0)
    6553              :     return tem;
    6554              : 
    6555              :   /* If one operand is constant, make it the second one.  Only do this
    6556              :      if the other operand is not constant as well.  */
    6557        12155 :   if (swap_commutative_operands_p (op0, op1))
    6558              :     {
    6559            8 :       std::swap (op0, op1);
    6560            8 :       code = swap_condition (code);
    6561              :     }
    6562              : 
    6563        12155 :   if (mode == VOIDmode)
    6564            0 :     mode = GET_MODE (op0);
    6565              : 
    6566        12155 :   if (!target)
    6567            0 :     target = gen_reg_rtx (word_mode);
    6568              : 
    6569              :   /* If this failed, we have to do this with set/compare/jump/set code.
    6570              :      For foo != 0, if foo is in OP0, just replace it with 1 if nonzero.  */
    6571        12155 :   trueval = normalizep ? GEN_INT (normalizep) : const1_rtx;
    6572        12155 :   if (code == NE
    6573         1746 :       && GET_MODE_CLASS (mode) == MODE_INT
    6574           41 :       && REG_P (target)
    6575           41 :       && op0 == target
    6576            0 :       && op1 == const0_rtx)
    6577              :     {
    6578            0 :       label = gen_label_rtx ();
    6579            0 :       do_compare_rtx_and_jump (target, const0_rtx, EQ, unsignedp, mode,
    6580              :                                NULL_RTX, NULL, label,
    6581              :                                profile_probability::uninitialized ());
    6582            0 :       emit_move_insn (target, trueval);
    6583            0 :       emit_label (label);
    6584            0 :       return target;
    6585              :     }
    6586              : 
    6587        12155 :   if (!REG_P (target)
    6588        12155 :       || reg_mentioned_p (target, op0) || reg_mentioned_p (target, op1))
    6589            5 :     target = gen_reg_rtx (GET_MODE (target));
    6590              : 
    6591              :   /* Jump in the right direction if the target cannot implement CODE
    6592              :      but can jump on its reverse condition.  */
    6593        12155 :   falseval = const0_rtx;
    6594        12155 :   if (! can_compare_p (code, mode, ccp_jump)
    6595        12155 :       && (! FLOAT_MODE_P (mode)
    6596         7450 :           || code == ORDERED || code == UNORDERED
    6597         7263 :           || (! HONOR_NANS (mode) && (code == LTGT || code == UNEQ))
    6598         7263 :           || (! HONOR_SNANS (mode) && (code == EQ || code == NE))))
    6599              :     {
    6600         2787 :       enum rtx_code rcode;
    6601         2787 :       if (FLOAT_MODE_P (mode))
    6602         2787 :         rcode = reverse_condition_maybe_unordered (code);
    6603              :       else
    6604            0 :         rcode = reverse_condition (code);
    6605              : 
    6606              :       /* Canonicalize to UNORDERED for the libcall.  */
    6607         2787 :       if (can_compare_p (rcode, mode, ccp_jump)
    6608         2787 :           || (code == ORDERED && ! can_compare_p (ORDERED, mode, ccp_jump)))
    6609              :         {
    6610          155 :           falseval = trueval;
    6611          155 :           trueval = const0_rtx;
    6612          155 :           code = rcode;
    6613              :         }
    6614              :     }
    6615              : 
    6616        12155 :   emit_move_insn (target, trueval);
    6617        12155 :   label = gen_label_rtx ();
    6618        12155 :   do_compare_rtx_and_jump (op0, op1, code, unsignedp, mode, NULL_RTX, NULL,
    6619              :                            label, profile_probability::uninitialized ());
    6620              : 
    6621        12155 :   emit_move_insn (target, falseval);
    6622        12155 :   emit_label (label);
    6623              : 
    6624        12155 :   return target;
    6625              : }
    6626              : 
    6627              : /* Expand a vector (left) rotate of MODE of X by an immediate AMT as a vector
    6628              :    permute operation.  Emit code to put the result in DST if successful and
    6629              :    return it.  Otherwise return NULL.  This is intended to implement vector
    6630              :    rotates by byte amounts using vector permutes when the target does not offer
    6631              :    native vector rotate operations.  */
    6632              : rtx
    6633            0 : expand_rotate_as_vec_perm (machine_mode mode, rtx dst, rtx x, rtx amt)
    6634              : {
    6635            0 :   rtx amt_unwrap = unwrap_const_vec_duplicate (amt);
    6636              :   /* For now handle only rotate by the same integer constant in all lanes.
    6637              :      In principle rotates by any constant vector are representable through
    6638              :      permutes as long as the individual rotate amounts are multiples of
    6639              :      BITS_PER_UNIT.  */
    6640            0 :   if (!CONST_INT_P (amt_unwrap))
    6641              :     return NULL_RTX;
    6642              : 
    6643            0 :   int rotamnt = INTVAL (amt_unwrap);
    6644            0 :   if (rotamnt % BITS_PER_UNIT != 0)
    6645              :     return NULL_RTX;
    6646            0 :   machine_mode qimode;
    6647            0 :   if (!qimode_for_vec_perm (mode).exists (&qimode))
    6648            0 :     return NULL_RTX;
    6649              : 
    6650            0 :   vec_perm_builder builder;
    6651            0 :   unsigned nunits = GET_MODE_SIZE (GET_MODE_INNER (mode));
    6652            0 :   poly_uint64 total_units = GET_MODE_SIZE (mode);
    6653            0 :   builder.new_vector (total_units, nunits, 3);
    6654            0 :   unsigned rot_bytes = rotamnt / BITS_PER_UNIT;
    6655            0 :   unsigned rot_to_perm = BYTES_BIG_ENDIAN ? rot_bytes : nunits - rot_bytes;
    6656            0 :   for (unsigned j = 0; j < 3 * nunits; j += nunits)
    6657            0 :     for (unsigned i = 0; i < nunits; i++)
    6658            0 :       builder.quick_push ((rot_to_perm + i) % nunits + j);
    6659              : 
    6660            0 :   rtx perm_src = lowpart_subreg (qimode, x, mode);
    6661            0 :   rtx perm_dst = lowpart_subreg (qimode, dst, mode);
    6662            0 :   rtx res
    6663            0 :     = expand_vec_perm_const (qimode, perm_src, perm_src, builder,
    6664              :                              qimode, perm_dst);
    6665            0 :   if (!res)
    6666              :     return NULL_RTX;
    6667            0 :   if (!rtx_equal_p (res, perm_dst))
    6668            0 :     emit_move_insn (dst, lowpart_subreg (mode, res, qimode));
    6669              :   return dst;
    6670            0 : }
    6671              : 
    6672              : /* Helper function for canonicalize_cmp_for_target.  Swap between inclusive
    6673              :    and exclusive ranges in order to create an equivalent comparison.  See
    6674              :    canonicalize_cmp_for_target for the possible cases.  */
    6675              : 
    6676              : static enum rtx_code
    6677           47 : equivalent_cmp_code (enum rtx_code code)
    6678              : {
    6679           47 :   switch (code)
    6680              :     {
    6681              :     case GT:
    6682              :       return GE;
    6683            0 :     case GE:
    6684            0 :       return GT;
    6685            0 :     case LT:
    6686            0 :       return LE;
    6687            0 :     case LE:
    6688            0 :       return LT;
    6689            2 :     case GTU:
    6690            2 :       return GEU;
    6691            0 :     case GEU:
    6692            0 :       return GTU;
    6693            1 :     case LTU:
    6694            1 :       return LEU;
    6695            2 :     case LEU:
    6696            2 :       return LTU;
    6697              : 
    6698            0 :     default:
    6699            0 :       return code;
    6700              :     }
    6701              : }
    6702              : 
    6703              : /* Choose the more appropriate immediate in scalar integer comparisons.  The
    6704              :    purpose of this is to end up with an immediate which can be loaded into a
    6705              :    register in fewer moves, if possible.
    6706              : 
    6707              :    For each integer comparison there exists an equivalent choice:
    6708              :      i)   a >  b or a >= b + 1
    6709              :      ii)  a <= b or a <  b + 1
    6710              :      iii) a >= b or a >  b - 1
    6711              :      iv)  a <  b or a <= b - 1
    6712              : 
    6713              :    MODE is the mode of the first operand.
    6714              :    CODE points to the comparison code.
    6715              :    IMM points to the rtx containing the immediate.  *IMM must satisfy
    6716              :    CONST_SCALAR_INT_P on entry and continues to satisfy CONST_SCALAR_INT_P
    6717              :    on exit.  */
    6718              : 
    6719              : void
    6720      4841115 : canonicalize_comparison (machine_mode mode, enum rtx_code *code, rtx *imm)
    6721              : {
    6722      4841115 :   if (!SCALAR_INT_MODE_P (mode))
    6723      3986147 :     return;
    6724              : 
    6725      4837249 :   int to_add = 0;
    6726      4837249 :   enum signop sgn = unsigned_condition_p (*code) ? UNSIGNED : SIGNED;
    6727              : 
    6728              :   /* Extract the immediate value from the rtx.  */
    6729      4837249 :   wide_int imm_val = rtx_mode_t (*imm, mode);
    6730              : 
    6731      4837249 :   if (*code == GT || *code == GTU || *code == LE || *code == LEU)
    6732              :     to_add = 1;
    6733              :   else if (*code == GE || *code == GEU || *code == LT || *code == LTU)
    6734              :     to_add = -1;
    6735              :   else
    6736              :     return;
    6737              : 
    6738              :   /* Check for overflow/underflow in the case of signed values and
    6739              :      wrapping around in the case of unsigned values.  If any occur
    6740              :      cancel the optimization.  */
    6741       855112 :   wi::overflow_type overflow = wi::OVF_NONE;
    6742       855112 :   wide_int imm_modif;
    6743              : 
    6744       855112 :   if (to_add == 1)
    6745       616383 :     imm_modif = wi::add (imm_val, 1, sgn, &overflow);
    6746              :   else
    6747       238729 :     imm_modif = wi::sub (imm_val, 1, sgn, &overflow);
    6748              : 
    6749       855112 :   if (overflow)
    6750          144 :     return;
    6751              : 
    6752       854968 :   rtx new_imm = immed_wide_int_const (imm_modif, mode);
    6753              : 
    6754       854968 :   int old_cost = rtx_cost (*imm, mode, COMPARE, 0, true);
    6755       854968 :   int new_cost = rtx_cost (new_imm, mode, COMPARE, 0, true);
    6756              : 
    6757       854968 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6758              :     {
    6759            7 :       fprintf (dump_file, ";; cmp: %s, old cst: ",
    6760            7 :                GET_RTX_NAME (*code));
    6761            7 :       print_rtl (dump_file, *imm);
    6762            7 :       fprintf (dump_file, " new cst: ");
    6763            7 :       print_rtl (dump_file, new_imm);
    6764            7 :       fprintf (dump_file, "\n");
    6765            7 :       fprintf (dump_file, ";; old cst cost: %d, new cst cost: %d\n",
    6766              :                old_cost, new_cost);
    6767              :     }
    6768              : 
    6769              :   /* Update the immediate and the code.  */
    6770       854968 :   if (old_cost > new_cost)
    6771              :     {
    6772           47 :       *code = equivalent_cmp_code (*code);
    6773           47 :       *imm = new_imm;
    6774              :     }
    6775      4837393 : }
    6776              : 
    6777              : 
    6778              : 
    6779              : /* Perform possibly multi-word comparison and conditional jump to LABEL
    6780              :    if ARG1 OP ARG2 true where ARG1 and ARG2 are of mode MODE.  This is
    6781              :    now a thin wrapper around do_compare_rtx_and_jump.  */
    6782              : 
    6783              : static void
    6784         2434 : do_cmp_and_jump (rtx arg1, rtx arg2, enum rtx_code op, machine_mode mode,
    6785              :                  rtx_code_label *label)
    6786              : {
    6787         2434 :   int unsignedp = (op == LTU || op == LEU || op == GTU || op == GEU);
    6788         2434 :   do_compare_rtx_and_jump (arg1, arg2, op, unsignedp, mode, NULL_RTX,
    6789              :                            NULL, label, profile_probability::uninitialized ());
    6790         2434 : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.