LCOV - code coverage report
Current view: top level - gcc - gimple-lower-bitint.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 86.8 % 5579 4842
Test Date: 2026-11-14 16:39:50 Functions: 100.0 % 58 58
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Lower _BitInt(N) operations to scalar operations.
       2              :    Copyright (C) 2023-2026 Free Software Foundation, Inc.
       3              :    Contributed by Jakub Jelinek <jakub@redhat.com>.
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify it
       8              : under the terms of the GNU General Public License as published by the
       9              : Free Software Foundation; either version 3, or (at your option) any
      10              : later version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful, but WITHOUT
      13              : ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
      14              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      15              : for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : #include "config.h"
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "rtl.h"
      26              : #include "tree.h"
      27              : #include "gimple.h"
      28              : #include "cfghooks.h"
      29              : #include "tree-pass.h"
      30              : #include "ssa.h"
      31              : #include "fold-const.h"
      32              : #include "gimplify.h"
      33              : #include "gimple-iterator.h"
      34              : #include "tree-cfg.h"
      35              : #include "tree-dfa.h"
      36              : #include "cfgloop.h"
      37              : #include "cfganal.h"
      38              : #include "target.h"
      39              : #include "tree-ssa-live.h"
      40              : #include "tree-ssa-coalesce.h"
      41              : #include "domwalk.h"
      42              : #include "memmodel.h"
      43              : #include "optabs.h"
      44              : #include "varasm.h"
      45              : #include "gimple-range.h"
      46              : #include "value-range.h"
      47              : #include "langhooks.h"
      48              : #include "gimplify-me.h"
      49              : #include "diagnostic-core.h"
      50              : #include "tree-eh.h"
      51              : #include "tree-pretty-print.h"
      52              : #include "alloc-pool.h"
      53              : #include "tree-into-ssa.h"
      54              : #include "tree-cfgcleanup.h"
      55              : #include "tree-switch-conversion.h"
      56              : #include "ubsan.h"
      57              : #include "stor-layout.h"
      58              : #include "gimple-lower-bitint.h"
      59              : #include "attribs.h"
      60              : #include "asan.h"
      61              : #include "tree-outof-ssa.h"
      62              : 
      63              : /* Split BITINT_TYPE precisions in 4 categories.  Small _BitInt, where
      64              :    target hook says it is a single limb, middle _BitInt which per ABI
      65              :    does not, but there is some INTEGER_TYPE in which arithmetics can be
      66              :    performed (operations on such _BitInt are lowered to casts to that
      67              :    arithmetic type and cast back; e.g. on x86_64 limb is DImode, but
      68              :    target supports TImode, so _BitInt(65) to _BitInt(128) are middle
      69              :    ones), large _BitInt which should by straight line code and
      70              :    finally huge _BitInt which should be handled by loops over the limbs.  */
      71              : 
      72              : enum bitint_prec_kind {
      73              :   bitint_prec_small,
      74              :   bitint_prec_middle,
      75              :   bitint_prec_large,
      76              :   bitint_prec_huge
      77              : };
      78              : 
      79              : /* Caches to speed up bitint_precision_kind.  */
      80              : 
      81              : static int small_max_prec, mid_min_prec, large_min_prec, huge_min_prec;
      82              : static int limb_prec, abi_limb_prec;
      83              : static bool bitint_big_endian;
      84              : static enum bitint_ext bitint_extended;
      85              : 
      86              : /* Categorize _BitInt(PREC) as small, middle, large or huge.  */
      87              : 
      88              : static bitint_prec_kind
      89       497425 : bitint_precision_kind (int prec)
      90              : {
      91       497425 :   if (prec <= small_max_prec)
      92              :     return bitint_prec_small;
      93       475682 :   if (huge_min_prec && prec >= huge_min_prec)
      94              :     return bitint_prec_huge;
      95       268021 :   if (large_min_prec && prec >= large_min_prec)
      96              :     return bitint_prec_large;
      97        53503 :   if (mid_min_prec && prec >= mid_min_prec)
      98              :     return bitint_prec_middle;
      99              : 
     100        10473 :   struct bitint_info info;
     101        10473 :   bool ok = targetm.c.bitint_type_info (prec, &info);
     102        10473 :   gcc_assert (ok);
     103        10473 :   scalar_int_mode limb_mode = as_a <scalar_int_mode> (info.limb_mode);
     104        10473 :   if (prec <= GET_MODE_PRECISION (limb_mode))
     105              :     {
     106         2881 :       small_max_prec = prec;
     107         2881 :       return bitint_prec_small;
     108              :     }
     109         7592 :   bitint_big_endian = info.big_endian;
     110         7592 :   bitint_extended = info.extended;
     111         7592 :   if (info.limb_mode == info.abi_limb_mode && bitint_extended == bitint_ext_full)
     112            0 :     bitint_extended = bitint_ext_partial;
     113         7592 :   if (!large_min_prec
     114        15080 :       && GET_MODE_PRECISION (limb_mode) <= MAX_FIXED_MODE_SIZE)
     115        14976 :     large_min_prec = MAX_FIXED_MODE_SIZE + 1;
     116         7592 :   if (!limb_prec)
     117         7488 :     limb_prec = GET_MODE_PRECISION (limb_mode);
     118         7592 :   if (!abi_limb_prec)
     119         7488 :     abi_limb_prec
     120         7488 :       = GET_MODE_PRECISION (as_a <scalar_int_mode> (info.abi_limb_mode));
     121              :   /* For bitint_ext_full with different limb_mode from abi_limb_mode we
     122              :      currently only support only abi_limb_mode twice the precision of
     123              :      limb_mode, and don't support big endian in that case either.  */
     124         7592 :   gcc_assert (bitint_extended != bitint_ext_full
     125              :               || (abi_limb_prec == 2 * limb_prec
     126              :                   && !bitint_big_endian));
     127         7592 :   if (!huge_min_prec)
     128              :     {
     129        14976 :       if (4 * limb_prec >= MAX_FIXED_MODE_SIZE)
     130         7488 :         huge_min_prec = 4 * limb_prec;
     131              :       else
     132            0 :         huge_min_prec = MAX_FIXED_MODE_SIZE + 1;
     133              :     }
     134        15184 :   if (prec <= MAX_FIXED_MODE_SIZE)
     135              :     {
     136         1655 :       if (!mid_min_prec || prec < mid_min_prec)
     137         1655 :         mid_min_prec = prec;
     138              :       return bitint_prec_middle;
     139              :     }
     140         5937 :   if (huge_min_prec && prec >= huge_min_prec)
     141         2405 :     return bitint_prec_huge;
     142              :   return bitint_prec_large;
     143              : }
     144              : 
     145              : /* Same for a TYPE.  */
     146              : 
     147              : static bitint_prec_kind
     148       491959 : bitint_precision_kind (tree type)
     149              : {
     150       491959 :   return bitint_precision_kind (TYPE_PRECISION (type));
     151              : }
     152              : 
     153              : /* Return minimum precision needed to describe INTEGER_CST
     154              :    CST.  All bits above that precision up to precision of
     155              :    TREE_TYPE (CST) are cleared if EXT is set to 0, or set
     156              :    if EXT is set to -1.  */
     157              : 
     158              : static unsigned
     159         5502 : bitint_min_cst_precision (tree cst, int &ext)
     160              : {
     161         5502 :   ext = tree_int_cst_sgn (cst) < 0 ? -1 : 0;
     162         5502 :   wide_int w = wi::to_wide (cst);
     163         5502 :   unsigned min_prec = wi::min_precision (w, TYPE_SIGN (TREE_TYPE (cst)));
     164              :   /* For signed values, we don't need to count the sign bit,
     165              :      we'll use constant 0 or -1 for the upper bits.  */
     166         5502 :   if (!TYPE_UNSIGNED (TREE_TYPE (cst)))
     167         3321 :     --min_prec;
     168              :   else
     169              :     {
     170              :       /* For unsigned values, also try signed min_precision
     171              :          in case the constant has lots of most significant bits set.  */
     172         2181 :       unsigned min_prec2 = wi::min_precision (w, SIGNED) - 1;
     173         2181 :       if (min_prec2 < min_prec)
     174              :         {
     175         1010 :           ext = -1;
     176         1010 :           return min_prec2;
     177              :         }
     178              :     }
     179              :   return min_prec;
     180         5502 : }
     181              : 
     182              : namespace {
     183              : 
     184              : /* If OP is middle _BitInt, cast it to corresponding INTEGER_TYPE
     185              :    cached in TYPE and return it.  */
     186              : 
     187              : tree
     188         8109 : maybe_cast_middle_bitint (gimple_stmt_iterator *gsi, tree op, tree &type)
     189              : {
     190         8109 :   if (op == NULL_TREE
     191         7996 :       || !BITINT_TYPE_P (TREE_TYPE (op))
     192        15150 :       || bitint_precision_kind (TREE_TYPE (op)) != bitint_prec_middle)
     193              :     return op;
     194              : 
     195         7023 :   int prec = TYPE_PRECISION (TREE_TYPE (op));
     196         7023 :   int uns = TYPE_UNSIGNED (TREE_TYPE (op));
     197         7023 :   if (type == NULL_TREE
     198         2628 :       || TYPE_PRECISION (type) != prec
     199         9651 :       || TYPE_UNSIGNED (type) != uns)
     200         4395 :     type = build_nonstandard_integer_type (prec, uns);
     201              : 
     202         7023 :   if (TREE_CODE (op) != SSA_NAME)
     203              :     {
     204         2436 :       tree nop = fold_convert (type, op);
     205         2436 :       if (is_gimple_val (nop))
     206              :         return nop;
     207              :     }
     208              : 
     209         4587 :   tree nop = make_ssa_name (type);
     210         4587 :   gimple *g = gimple_build_assign (nop, NOP_EXPR, op);
     211         4587 :   gsi_insert_before (gsi, g, GSI_SAME_STMT);
     212         4587 :   return nop;
     213              : }
     214              : 
     215              : /* Return true if STMT can be handled in a loop from least to most
     216              :    significant limb together with its dependencies.  */
     217              : 
     218              : bool
     219        48640 : mergeable_op (gimple *stmt)
     220              : {
     221        48640 :   if (!is_gimple_assign (stmt))
     222              :     return false;
     223        39626 :   switch (gimple_assign_rhs_code (stmt))
     224              :     {
     225              :     case PLUS_EXPR:
     226              :     case MINUS_EXPR:
     227              :     case NEGATE_EXPR:
     228              :     case BIT_AND_EXPR:
     229              :     case BIT_IOR_EXPR:
     230              :     case BIT_XOR_EXPR:
     231              :     case BIT_NOT_EXPR:
     232              :     case SSA_NAME:
     233              :     case INTEGER_CST:
     234              :     case BIT_FIELD_REF:
     235              :       return true;
     236          410 :     case LSHIFT_EXPR:
     237          410 :       {
     238          410 :         tree cnt = gimple_assign_rhs2 (stmt);
     239          410 :         if (tree_fits_uhwi_p (cnt)
     240          198 :             && tree_to_uhwi (cnt) < (unsigned HOST_WIDE_INT) limb_prec)
     241              :           return true;
     242              :       }
     243              :       break;
     244         6361 :     CASE_CONVERT:
     245         6361 :     case VIEW_CONVERT_EXPR:
     246         6361 :       {
     247         6361 :         tree lhs_type = TREE_TYPE (gimple_assign_lhs (stmt));
     248         6361 :         tree rhs_type = TREE_TYPE (gimple_assign_rhs1 (stmt));
     249         6361 :         if (TREE_CODE (gimple_assign_rhs1 (stmt)) == SSA_NAME
     250         6319 :             && BITINT_TYPE_P (lhs_type)
     251         3744 :             && BITINT_TYPE_P (rhs_type)
     252         3371 :             && bitint_precision_kind (lhs_type) >= bitint_prec_large
     253         3335 :             && bitint_precision_kind (rhs_type) >= bitint_prec_large
     254         9533 :             && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
     255         3172 :                 == CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
     256              :           {
     257         2198 :             if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type))
     258              :               return true;
     259          183 :             if ((unsigned) TYPE_PRECISION (lhs_type) % (2 * limb_prec) != 0)
     260              :               return true;
     261           17 :             if (bitint_precision_kind (lhs_type) == bitint_prec_large)
     262              :               return true;
     263              :           }
     264              :         break;
     265              :       }
     266              :     default:
     267              :       break;
     268              :     }
     269              :   return false;
     270              : }
     271              : 
     272              : /* Return non-zero if stmt is .{ADD,SUB,MUL}_OVERFLOW call with
     273              :    _Complex large/huge _BitInt lhs which has at most two immediate uses,
     274              :    at most one use in REALPART_EXPR stmt in the same bb and exactly one
     275              :    IMAGPART_EXPR use in the same bb with a single use which casts it to
     276              :    non-BITINT_TYPE integral type.  If there is a REALPART_EXPR use,
     277              :    return 2.  Such cases (most common uses of those builtins) can be
     278              :    optimized by marking their lhs and lhs of IMAGPART_EXPR and maybe lhs
     279              :    of REALPART_EXPR as not needed to be backed up by a stack variable.
     280              :    For .UBSAN_CHECK_{ADD,SUB,MUL} return 3.  */
     281              : 
     282              : int
     283        21926 : optimizable_arith_overflow (gimple *stmt)
     284              : {
     285        21926 :   bool is_ubsan = false;
     286        21926 :   if (!is_gimple_call (stmt) || !gimple_call_internal_p (stmt))
     287              :     return false;
     288         5306 :   switch (gimple_call_internal_fn (stmt))
     289              :     {
     290              :     case IFN_ADD_OVERFLOW:
     291              :     case IFN_SUB_OVERFLOW:
     292              :     case IFN_MUL_OVERFLOW:
     293              :       break;
     294           63 :     case IFN_UBSAN_CHECK_ADD:
     295           63 :     case IFN_UBSAN_CHECK_SUB:
     296           63 :     case IFN_UBSAN_CHECK_MUL:
     297           63 :       is_ubsan = true;
     298           63 :       break;
     299              :     default:
     300              :       return 0;
     301              :     }
     302         5210 :   tree lhs = gimple_call_lhs (stmt);
     303         5210 :   if (!lhs)
     304              :     return 0;
     305         5210 :   if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs))
     306              :     return 0;
     307         5195 :   tree type = is_ubsan ? TREE_TYPE (lhs) : TREE_TYPE (TREE_TYPE (lhs));
     308            0 :   if (!BITINT_TYPE_P (type)
     309         5195 :       || bitint_precision_kind (type) < bitint_prec_large)
     310              :     return 0;
     311              : 
     312         5195 :   if (is_ubsan)
     313              :     {
     314           48 :       use_operand_p use_p;
     315           48 :       gimple *use_stmt;
     316           48 :       if (!single_imm_use (lhs, &use_p, &use_stmt)
     317           48 :           || gimple_bb (use_stmt) != gimple_bb (stmt)
     318           48 :           || !gimple_store_p (use_stmt)
     319           48 :           || !is_gimple_assign (use_stmt)
     320           48 :           || gimple_has_volatile_ops (use_stmt)
     321           96 :           || stmt_ends_bb_p (use_stmt))
     322            0 :         return 0;
     323              :       return 3;
     324              :     }
     325              : 
     326         5147 :   imm_use_iterator ui;
     327         5147 :   use_operand_p use_p;
     328         5147 :   int seen = 0;
     329         5147 :   gimple *realpart = NULL, *cast = NULL;
     330        14740 :   FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
     331              :     {
     332         9807 :       gimple *g = USE_STMT (use_p);
     333         9807 :       if (is_gimple_debug (g))
     334            0 :         continue;
     335         9807 :       if (!is_gimple_assign (g) || gimple_bb (g) != gimple_bb (stmt))
     336              :         return 0;
     337         9807 :       if (gimple_assign_rhs_code (g) == REALPART_EXPR)
     338              :         {
     339         4660 :           if ((seen & 1) != 0)
     340              :             return 0;
     341         4660 :           seen |= 1;
     342         4660 :           realpart = g;
     343              :         }
     344         5147 :       else if (gimple_assign_rhs_code (g) == IMAGPART_EXPR)
     345              :         {
     346         5147 :           if ((seen & 2) != 0)
     347          214 :             return 0;
     348         5147 :           seen |= 2;
     349              : 
     350         5147 :           use_operand_p use2_p;
     351         5147 :           gimple *use_stmt;
     352         5147 :           tree lhs2 = gimple_assign_lhs (g);
     353         5147 :           if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (lhs2))
     354              :             return 0;
     355         5147 :           if (!single_imm_use (lhs2, &use2_p, &use_stmt)
     356         5147 :               || gimple_bb (use_stmt) != gimple_bb (stmt)
     357        10294 :               || !gimple_assign_cast_p (use_stmt))
     358              :             return 0;
     359              : 
     360         4937 :           lhs2 = gimple_assign_lhs (use_stmt);
     361         9874 :           if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs2))
     362         9874 :               || BITINT_TYPE_P (TREE_TYPE (lhs2)))
     363              :             return 0;
     364         4933 :           cast = use_stmt;
     365              :         }
     366              :       else
     367              :         return 0;
     368          214 :     }
     369         4933 :   if ((seen & 2) == 0)
     370              :     return 0;
     371         4933 :   if (seen == 3)
     372              :     {
     373              :       /* Punt if the cast stmt appears before realpart stmt, because
     374              :          if both appear, the lowering wants to emit all the code
     375              :          at the location of realpart stmt.  */
     376         4660 :       gimple_stmt_iterator gsi = gsi_for_stmt (realpart);
     377         4660 :       unsigned int cnt = 0;
     378         4663 :       do
     379              :         {
     380         4663 :           gsi_prev_nondebug (&gsi);
     381         4663 :           if (gsi_end_p (gsi) || gsi_stmt (gsi) == cast)
     382              :             return 0;
     383         4660 :           if (gsi_stmt (gsi) == stmt)
     384              :             return 2;
     385              :           /* If realpart is too far from stmt, punt as well.
     386              :              Usually it will appear right after it.  */
     387            3 :           if (++cnt == 32)
     388              :             return 0;
     389              :         }
     390              :       while (1);
     391              :     }
     392              :   return 1;
     393              : }
     394              : 
     395              : /* If STMT is some kind of comparison (GIMPLE_COND, comparison assignment)
     396              :    comparing large/huge _BitInt types, return the comparison code and if
     397              :    non-NULL fill in the comparison operands to *POP1 and *POP2.  */
     398              : 
     399              : tree_code
     400        36315 : comparison_op (gimple *stmt, tree *pop1, tree *pop2)
     401              : {
     402        36315 :   tree op1 = NULL_TREE, op2 = NULL_TREE;
     403        36315 :   tree_code code = ERROR_MARK;
     404        36315 :   if (gimple_code (stmt) == GIMPLE_COND)
     405              :     {
     406         6667 :       code = gimple_cond_code (stmt);
     407         6667 :       op1 = gimple_cond_lhs (stmt);
     408         6667 :       op2 = gimple_cond_rhs (stmt);
     409              :     }
     410        29648 :   else if (is_gimple_assign (stmt))
     411              :     {
     412        29621 :       code = gimple_assign_rhs_code (stmt);
     413        29621 :       op1 = gimple_assign_rhs1 (stmt);
     414        29621 :       if (TREE_CODE_CLASS (code) == tcc_comparison
     415        29621 :           || TREE_CODE_CLASS (code) == tcc_binary)
     416         2275 :         op2 = gimple_assign_rhs2 (stmt);
     417              :     }
     418        36315 :   if (TREE_CODE_CLASS (code) != tcc_comparison)
     419              :     return ERROR_MARK;
     420         7449 :   tree type = TREE_TYPE (op1);
     421           10 :   if (!BITINT_TYPE_P (type)
     422         7459 :       || bitint_precision_kind (type) < bitint_prec_large)
     423              :     return ERROR_MARK;
     424         7449 :   if (pop1)
     425              :     {
     426         7387 :       *pop1 = op1;
     427         7387 :       *pop2 = op2;
     428              :     }
     429              :   return code;
     430              : }
     431              : 
     432              : /* Class used during large/huge _BitInt lowering containing all the
     433              :    state for the methods.  */
     434              : 
     435              : struct bitint_large_huge
     436              : {
     437         7486 :   bitint_large_huge ()
     438         7486 :     : m_names (NULL), m_loads (NULL), m_preserved (NULL),
     439         7486 :       m_single_use_names (NULL), m_map (NULL), m_vars (NULL),
     440         7486 :       m_limb_type (NULL_TREE), m_data (vNULL),
     441         7486 :       m_returns_twice_calls (vNULL) {}
     442              : 
     443              :   ~bitint_large_huge ();
     444              : 
     445              :   void insert_before (gimple *);
     446              :   tree limb_access_type (tree, tree);
     447              :   tree limb_access (tree, tree, tree, bool, bool = false);
     448              :   tree build_bit_field_ref (tree, tree, unsigned HOST_WIDE_INT,
     449              :                             unsigned HOST_WIDE_INT);
     450              :   void if_then (gimple *, profile_probability, edge &, edge &);
     451              :   void if_then_else (gimple *, profile_probability, edge &, edge &);
     452              :   void if_then_if_then_else (gimple *g, gimple *,
     453              :                              profile_probability, profile_probability,
     454              :                              edge &, edge &, edge &);
     455              :   tree handle_operand (tree, tree);
     456              :   tree prepare_data_in_out (tree, tree, tree *, tree = NULL_TREE);
     457              :   tree add_cast (tree, tree);
     458              :   tree handle_plus_minus (tree_code, tree, tree, tree);
     459              :   tree handle_lshift (tree, tree, tree);
     460              :   tree handle_cast (tree, tree, tree);
     461              :   tree handle_bit_field_ref (tree, tree);
     462              :   tree handle_load (gimple *, tree);
     463              :   tree handle_stmt (gimple *, tree);
     464              :   tree handle_operand_addr (tree, gimple *, int *, int *);
     465              :   tree create_loop (tree, tree *);
     466              :   tree lower_mergeable_stmt (gimple *, tree_code &, tree, tree);
     467              :   tree lower_comparison_stmt (gimple *, tree_code &, tree, tree);
     468              :   void lower_shift_stmt (tree, gimple *);
     469              :   void lower_muldiv_stmt (tree, gimple *);
     470              :   void lower_float_conv_stmt (tree, gimple *);
     471              :   tree arith_overflow_extract_bits (unsigned int, unsigned int, tree,
     472              :                                     unsigned int, bool);
     473              :   void finish_arith_overflow (tree, tree, tree, tree, tree, tree, gimple *,
     474              :                               unsigned, tree_code);
     475              :   void lower_addsub_overflow (tree, gimple *);
     476              :   void lower_mul_overflow (tree, gimple *);
     477              :   void lower_cplxpart_stmt (tree, gimple *);
     478              :   void lower_complexexpr_stmt (gimple *);
     479              :   void lower_bit_query (gimple *);
     480              :   void lower_bswap_bitreverse (tree, gimple *);
     481              :   void lower_clmul (tree, gimple *);
     482              :   void lower_call (tree, gimple *);
     483              :   void lower_asm (gimple *);
     484              :   void lower_stmt (gimple *);
     485              : 
     486              :   /* Bitmap of large/huge _BitInt SSA_NAMEs except those can be
     487              :      merged with their uses.  */
     488              :   bitmap m_names;
     489              :   /* Subset of those for lhs of load statements.  These will be
     490              :      cleared in m_names if the loads will be mergeable with all
     491              :      their uses.  */
     492              :   bitmap m_loads;
     493              :   /* Bitmap of large/huge _BitInt SSA_NAMEs that should survive
     494              :      to later passes (arguments or return values of calls).  */
     495              :   bitmap m_preserved;
     496              :   /* Subset of m_names which have a single use.  As the lowering
     497              :      can replace various original statements with their lowered
     498              :      form even before it is done iterating over all basic blocks,
     499              :      testing has_single_use for the purpose of emitting clobbers
     500              :      doesn't work properly.  */
     501              :   bitmap m_single_use_names;
     502              :   /* Used for coalescing/partitioning of large/huge _BitInt SSA_NAMEs
     503              :      set in m_names.  */
     504              :   var_map m_map;
     505              :   /* Mapping of the partitions to corresponding decls.  */
     506              :   tree *m_vars;
     507              :   /* Unsigned integer type with limb precision.  */
     508              :   tree m_limb_type;
     509              :   /* Its TYPE_SIZE_UNIT.  */
     510              :   unsigned HOST_WIDE_INT m_limb_size;
     511              :   /* Location of a gimple stmt which is being currently lowered.  */
     512              :   location_t m_loc;
     513              :   /* Current stmt iterator where code is being lowered currently.  */
     514              :   gimple_stmt_iterator m_gsi;
     515              :   /* Statement after which any clobbers should be added if non-NULL.  */
     516              :   gimple *m_after_stmt;
     517              :   /* Set when creating loops to the loop header bb and its preheader.  */
     518              :   basic_block m_bb, m_preheader_bb;
     519              :   /* Stmt iterator after which initialization statements should be emitted.  */
     520              :   gimple_stmt_iterator m_init_gsi;
     521              :   /* Decl into which a mergeable statement stores result.  */
     522              :   tree m_lhs;
     523              :   /* handle_operand/handle_stmt can be invoked in various ways.
     524              : 
     525              :      lower_mergeable_stmt for large _BitInt calls those with constant
     526              :      idx only, expanding to straight line code, for huge _BitInt
     527              :      emits a loop from least significant limb upwards, where each loop
     528              :      iteration handles 2 limbs, plus there can be up to one full limb
     529              :      and one partial limb processed after the loop, where handle_operand
     530              :      and/or handle_stmt are called with constant idx.  m_upwards_2limb
     531              :      is set for this case, false otherwise.  m_upwards is true if it
     532              :      is either large or huge _BitInt handled by lower_mergeable_stmt,
     533              :      i.e. indexes always increase.
     534              : 
     535              :      Another way is used by lower_comparison_stmt, which walks limbs
     536              :      from most significant to least significant, partial limb if any
     537              :      processed first with constant idx and then loop processing a single
     538              :      limb per iteration with non-constant idx.
     539              : 
     540              :      Another way is used in lower_shift_stmt, where for LSHIFT_EXPR
     541              :      destination limbs are processed from most significant to least
     542              :      significant or for RSHIFT_EXPR the other way around, in loops or
     543              :      straight line code, but idx usually is non-constant (so from
     544              :      handle_operand/handle_stmt POV random access).  The LSHIFT_EXPR
     545              :      handling there can access even partial limbs using non-constant
     546              :      idx (then m_var_msb should be true, for all the other cases
     547              :      including lower_mergeable_stmt/lower_comparison_stmt that is
     548              :      not the case and so m_var_msb should be false.
     549              : 
     550              :      m_first should be set the first time handle_operand/handle_stmt
     551              :      is called and clear when it is called for some other limb with
     552              :      the same argument.  If the lowering of an operand (e.g. INTEGER_CST)
     553              :      or statement (e.g. +/-/<< with < limb_prec constant) needs some
     554              :      state between the different calls, when m_first is true it should
     555              :      push some trees to m_data vector and also make sure m_data_cnt is
     556              :      incremented by how many trees were pushed, and when m_first is
     557              :      false, it can use the m_data[m_data_cnt] etc. data or update them,
     558              :      just needs to bump m_data_cnt by the same amount as when it was
     559              :      called with m_first set.  The toplevel calls to
     560              :      handle_operand/handle_stmt should set m_data_cnt to 0 and truncate
     561              :      m_data vector when setting m_first to true.
     562              : 
     563              :      m_cast_conditional and m_bitfld_load are used when handling a
     564              :      bit-field load inside of a widening cast.  handle_cast sometimes
     565              :      needs to do runtime comparisons and handle_operand only conditionally
     566              :      or even in two separate conditional blocks for one idx (once with
     567              :      constant index after comparing the runtime one for equality with the
     568              :      constant).  In these cases, m_cast_conditional is set to true and
     569              :      the bit-field load then communicates its m_data_cnt to handle_cast
     570              :      using m_bitfld_load.  */
     571              :   bool m_first;
     572              :   bool m_var_msb;
     573              :   unsigned m_upwards_2limb;
     574              :   bool m_upwards;
     575              :   bool m_cast_conditional;
     576              :   unsigned m_bitfld_load;
     577              :   vec<tree> m_data;
     578              :   unsigned int m_data_cnt;
     579              :   vec<gimple *> m_returns_twice_calls;
     580              : };
     581              : 
     582         7486 : bitint_large_huge::~bitint_large_huge ()
     583              : {
     584         7486 :   BITMAP_FREE (m_names);
     585         7486 :   BITMAP_FREE (m_loads);
     586         7486 :   BITMAP_FREE (m_preserved);
     587         7486 :   BITMAP_FREE (m_single_use_names);
     588         7486 :   if (m_map)
     589         5788 :     delete_var_map (m_map);
     590         7486 :   XDELETEVEC (m_vars);
     591         7486 :   m_data.release ();
     592         7486 :   m_returns_twice_calls.release ();
     593         7486 : }
     594              : 
     595              : /* Insert gimple statement G before current location
     596              :    and set its gimple_location.  */
     597              : 
     598              : void
     599       368806 : bitint_large_huge::insert_before (gimple *g)
     600              : {
     601       368806 :   gimple_set_location (g, m_loc);
     602       368806 :   gsi_insert_before (&m_gsi, g, GSI_SAME_STMT);
     603       368806 : }
     604              : 
     605              : /* Return type for accessing limb IDX of BITINT_TYPE TYPE.
     606              :    This is normally m_limb_type, except for a partial most
     607              :    significant limb if any.  */
     608              : 
     609              : tree
     610       131869 : bitint_large_huge::limb_access_type (tree type, tree idx)
     611              : {
     612       131869 :   if (type == NULL_TREE)
     613         5943 :     return m_limb_type;
     614       125926 :   unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
     615       125926 :   unsigned int prec = TYPE_PRECISION (type);
     616       125926 :   gcc_assert (i * limb_prec < prec
     617              :               || (bitint_extended == bitint_ext_full
     618              :                   && abi_limb_prec > limb_prec
     619              :                   && i * limb_prec
     620              :                      < CEIL (prec, abi_limb_prec) * abi_limb_prec));
     621       251852 :   if (bitint_big_endian
     622       125926 :       ? (i != 0 || (prec % limb_prec) == 0)
     623       125926 :       : (i + 1) * limb_prec <= prec)
     624        82089 :     return m_limb_type;
     625              :   else
     626        87674 :     return build_nonstandard_integer_type (prec % limb_prec,
     627        43837 :                                            TYPE_UNSIGNED (type));
     628              : }
     629              : 
     630              : /* Return a tree how to access limb IDX of VAR corresponding to BITINT_TYPE
     631              :    TYPE.  If WRITE_P is true, it will be a store, otherwise a read.  */
     632              : 
     633              : tree
     634       157784 : bitint_large_huge::limb_access (tree type, tree var, tree idx, bool write_p,
     635              :                                 bool abi_load_p)
     636              : {
     637       157784 :   tree atype = (tree_fits_uhwi_p (idx)
     638       157784 :                 ? limb_access_type (type, idx) : m_limb_type);
     639              : 
     640       157784 :   tree ltype = (bitint_extended && abi_load_p) ? atype : m_limb_type;
     641              : 
     642       157784 :   addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (var));
     643       157784 :   tree ret;
     644       157784 :   if (DECL_P (var) && tree_fits_uhwi_p (idx))
     645              :     {
     646        96728 :       if (as != TYPE_ADDR_SPACE (ltype))
     647            0 :         ltype = build_qualified_type (ltype,
     648            0 :                                       TYPE_QUALS (ltype).with_as (as));
     649        96728 :       tree ptype = build_pointer_type (strip_array_types (TREE_TYPE (var)));
     650        96728 :       unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
     651        96728 :       if (bitint_big_endian)
     652            0 :         off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
     653        96728 :       ret = build2 (MEM_REF, ltype,
     654              :                     build_fold_addr_expr (var),
     655        96728 :                     build_int_cst (ptype, off));
     656        96728 :       TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
     657        96728 :       TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
     658        96728 :     }
     659        61056 :   else if (TREE_CODE (var) == MEM_REF && tree_fits_uhwi_p (idx))
     660              :     {
     661         4316 :       if (as != TYPE_ADDR_SPACE (ltype))
     662            0 :         ltype = build_qualified_type (ltype,
     663            0 :                                       TYPE_QUALS (ltype).with_as (as));
     664         4316 :       unsigned HOST_WIDE_INT off = tree_to_uhwi (idx) * m_limb_size;
     665         4316 :       if (bitint_big_endian)
     666            0 :         off += m_limb_size - tree_to_uhwi (TYPE_SIZE_UNIT (ltype));
     667         4316 :       ret
     668         8632 :         = build2 (MEM_REF, ltype, unshare_expr (TREE_OPERAND (var, 0)),
     669         8632 :                   size_binop (PLUS_EXPR, TREE_OPERAND (var, 1),
     670              :                               build_int_cst (TREE_TYPE (TREE_OPERAND (var, 1)),
     671              :                                              off)));
     672         4316 :       TREE_THIS_VOLATILE (ret) = TREE_THIS_VOLATILE (var);
     673         4316 :       TREE_SIDE_EFFECTS (ret) = TREE_SIDE_EFFECTS (var);
     674         4316 :       TREE_THIS_NOTRAP (ret) = TREE_THIS_NOTRAP (var);
     675         4316 :     }
     676              :   else
     677              :     {
     678        56740 :       ltype = m_limb_type;
     679        56740 :       if (as != TYPE_ADDR_SPACE (ltype))
     680           17 :         ltype = build_qualified_type (ltype,
     681           17 :                                       TYPE_QUALS (ltype).with_as (as));
     682        56740 :       var = unshare_expr (var);
     683        56740 :       if (TREE_CODE (TREE_TYPE (var)) != ARRAY_TYPE
     684        85216 :           || !useless_type_conversion_p (m_limb_type,
     685        28476 :                                          TREE_TYPE (TREE_TYPE (var))))
     686              :         {
     687        29260 :           unsigned HOST_WIDE_INT nelts
     688        29260 :             = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (var))), limb_prec);
     689        29260 :           tree atype = build_array_type_nelts (ltype, nelts);
     690        29260 :           var = build1 (VIEW_CONVERT_EXPR, atype, var);
     691              :         }
     692        56740 :       ret = build4 (ARRAY_REF, ltype, var, idx, NULL_TREE, NULL_TREE);
     693              :     }
     694       157784 :   if (!write_p && !useless_type_conversion_p (atype, ltype))
     695              :     {
     696        19230 :       gimple *g = gimple_build_assign (make_ssa_name (m_limb_type), ret);
     697        19230 :       insert_before (g);
     698        19230 :       ret = gimple_assign_lhs (g);
     699        19230 :       ret = build1 (NOP_EXPR, atype, ret);
     700              :     }
     701       157784 :   return ret;
     702              : }
     703              : 
     704              : /* Build a BIT_FIELD_REF to access BITSIZE bits with FTYPE type at
     705              :    offset BITPOS inside of OBJ.  */
     706              : 
     707              : tree
     708          273 : bitint_large_huge::build_bit_field_ref (tree ftype, tree obj,
     709              :                                         unsigned HOST_WIDE_INT bitsize,
     710              :                                         unsigned HOST_WIDE_INT bitpos)
     711              : {
     712          546 :   if (INTEGRAL_TYPE_P (TREE_TYPE (obj))
     713          282 :       && !type_has_mode_precision_p (TREE_TYPE (obj)))
     714              :     {
     715            9 :       unsigned HOST_WIDE_INT nelts
     716            9 :         = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))), limb_prec);
     717            9 :       tree ltype = m_limb_type;
     718            9 :       addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (obj));
     719            9 :       if (as != TYPE_ADDR_SPACE (ltype))
     720            0 :         ltype = build_qualified_type (ltype,
     721            0 :                                       TYPE_QUALS (ltype).with_as (as));
     722            9 :       tree atype = build_array_type_nelts (ltype, nelts);
     723            9 :       obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
     724              :     }
     725          273 :   return build3 (BIT_FIELD_REF, ftype, obj, bitsize_int (bitsize),
     726          273 :                  bitsize_int (bitpos));
     727              : }
     728              : 
     729              : /* Emit a half diamond,
     730              :    if (COND)
     731              :      |\
     732              :      | \
     733              :      |  \
     734              :      | new_bb1
     735              :      |  /
     736              :      | /
     737              :      |/
     738              :    or if (COND) new_bb1;
     739              :    PROB is the probability that the condition is true.
     740              :    Updates m_gsi to start of new_bb1.
     741              :    Sets EDGE_TRUE to edge from new_bb1 to successor and
     742              :    EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb.  */
     743              : 
     744              : void
     745         4436 : bitint_large_huge::if_then (gimple *cond, profile_probability prob,
     746              :                             edge &edge_true, edge &edge_false)
     747              : {
     748         4436 :   insert_before (cond);
     749         4436 :   edge e1 = split_block (gsi_bb (m_gsi), cond);
     750         4436 :   edge e2 = split_block (e1->dest, (gimple *) NULL);
     751         4436 :   edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
     752         4436 :   e1->flags = EDGE_TRUE_VALUE;
     753         4436 :   e1->probability = prob;
     754         4436 :   e3->probability = prob.invert ();
     755         4436 :   set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
     756         4436 :   edge_true = e2;
     757         4436 :   edge_false = e3;
     758         4436 :   m_gsi = gsi_after_labels (e1->dest);
     759         4436 : }
     760              : 
     761              : /* Emit a full diamond,
     762              :        if (COND)
     763              :          /\
     764              :         /  \
     765              :        /    \
     766              :    new_bb1 new_bb2
     767              :        \    /
     768              :         \  /
     769              :          \/
     770              :    or if (COND) new_bb2; else new_bb1;
     771              :    PROB is the probability that the condition is true.
     772              :    Updates m_gsi to start of new_bb2.
     773              :    Sets EDGE_TRUE to edge from new_bb1 to successor and
     774              :    EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND) bb.  */
     775              : 
     776              : void
     777          112 : bitint_large_huge::if_then_else (gimple *cond, profile_probability prob,
     778              :                                  edge &edge_true, edge &edge_false)
     779              : {
     780          112 :   insert_before (cond);
     781          112 :   edge e1 = split_block (gsi_bb (m_gsi), cond);
     782          112 :   edge e2 = split_block (e1->dest, (gimple *) NULL);
     783          112 :   basic_block bb = create_empty_bb (e1->dest);
     784          112 :   add_bb_to_loop (bb, e1->dest->loop_father);
     785          112 :   edge e3 = make_edge (e1->src, bb, EDGE_TRUE_VALUE);
     786          112 :   e1->flags = EDGE_FALSE_VALUE;
     787          112 :   e3->probability = prob;
     788          112 :   e1->probability = prob.invert ();
     789          112 :   bb->count = e1->src->count.apply_probability (prob);
     790          112 :   set_immediate_dominator (CDI_DOMINATORS, bb, e1->src);
     791          112 :   set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
     792          112 :   edge_true = make_single_succ_edge (bb, e2->dest, EDGE_FALLTHRU);
     793          112 :   edge_false = e2;
     794          112 :   m_gsi = gsi_after_labels (bb);
     795          112 : }
     796              : 
     797              : /* Emit a half diamond with full diamond in it
     798              :    if (COND1)
     799              :      |\
     800              :      | \
     801              :      |  \
     802              :      | if (COND2)
     803              :      |    /  \
     804              :      |   /    \
     805              :      |new_bb1 new_bb2
     806              :      |   |    /
     807              :      \   |   /
     808              :       \  |  /
     809              :        \ | /
     810              :         \|/
     811              :    or if (COND1) { if (COND2) new_bb2; else new_bb1; }
     812              :    PROB1 is the probability that the condition 1 is true.
     813              :    PROB2 is the probability that the condition 2 is true.
     814              :    Updates m_gsi to start of new_bb1.
     815              :    Sets EDGE_TRUE_TRUE to edge from new_bb2 to successor,
     816              :    EDGE_TRUE_FALSE to edge from new_bb1 to successor and
     817              :    EDGE_FALSE to the EDGE_FALSE_VALUE edge from if (COND1) bb.
     818              :    If COND2 is NULL, this is equivalent to
     819              :    if_then (COND1, PROB1, EDGE_TRUE_FALSE, EDGE_FALSE);
     820              :    EDGE_TRUE_TRUE = NULL;  */
     821              : 
     822              : void
     823         2098 : bitint_large_huge::if_then_if_then_else (gimple *cond1, gimple *cond2,
     824              :                                          profile_probability prob1,
     825              :                                          profile_probability prob2,
     826              :                                          edge &edge_true_true,
     827              :                                          edge &edge_true_false,
     828              :                                          edge &edge_false)
     829              : {
     830         2098 :   edge e2, e3, e4 = NULL;
     831         2098 :   if_then (cond1, prob1, e2, e3);
     832         2098 :   if (cond2 == NULL)
     833              :     {
     834         1257 :       edge_true_true = NULL;
     835         1257 :       edge_true_false = e2;
     836         1257 :       edge_false = e3;
     837         1257 :       return;
     838              :     }
     839          841 :   insert_before (cond2);
     840          841 :   e2 = split_block (gsi_bb (m_gsi), cond2);
     841          841 :   basic_block bb = create_empty_bb (e2->dest);
     842          841 :   add_bb_to_loop (bb, e2->dest->loop_father);
     843          841 :   e4 = make_edge (e2->src, bb, EDGE_TRUE_VALUE);
     844          841 :   set_immediate_dominator (CDI_DOMINATORS, bb, e2->src);
     845          841 :   e4->probability = prob2;
     846          841 :   e2->flags = EDGE_FALSE_VALUE;
     847          841 :   e2->probability = prob2.invert ();
     848          841 :   bb->count = e2->src->count.apply_probability (prob2);
     849          841 :   e4 = make_single_succ_edge (bb, e3->dest, EDGE_FALLTHRU);
     850          841 :   e2 = find_edge (e2->dest, e3->dest);
     851          841 :   edge_true_true = e4;
     852          841 :   edge_true_false = e2;
     853          841 :   edge_false = e3;
     854          841 :   m_gsi = gsi_after_labels (e2->src);
     855              : }
     856              : 
     857              : /* Emit code to access limb IDX from OP.  */
     858              : 
     859              : tree
     860       112879 : bitint_large_huge::handle_operand (tree op, tree idx)
     861              : {
     862       112879 :   switch (TREE_CODE (op))
     863              :     {
     864        78053 :     case SSA_NAME:
     865        78053 :       if (m_names == NULL
     866        78053 :           || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
     867              :         {
     868        14619 :           if (SSA_NAME_IS_DEFAULT_DEF (op))
     869              :             {
     870           47 :               if (m_first)
     871              :                 {
     872           16 :                   tree v = create_tmp_reg (m_limb_type);
     873           16 :                   if (SSA_NAME_VAR (op) && VAR_P (SSA_NAME_VAR (op)))
     874              :                     {
     875           16 :                       DECL_NAME (v) = DECL_NAME (SSA_NAME_VAR (op));
     876           16 :                       DECL_SOURCE_LOCATION (v)
     877           16 :                         = DECL_SOURCE_LOCATION (SSA_NAME_VAR (op));
     878              :                     }
     879           16 :                   v = get_or_create_ssa_default_def (cfun, v);
     880           16 :                   m_data.safe_push (v);
     881              :                 }
     882           47 :               tree ret = m_data[m_data_cnt];
     883           47 :               m_data_cnt++;
     884           47 :               if (tree_fits_uhwi_p (idx))
     885              :                 {
     886           45 :                   tree type = limb_access_type (TREE_TYPE (op), idx);
     887           45 :                   ret = add_cast (type, ret);
     888              :                 }
     889              :               return ret;
     890              :             }
     891        14572 :           location_t loc_save = m_loc;
     892        14572 :           m_loc = gimple_location (SSA_NAME_DEF_STMT (op));
     893        14572 :           tree ret = handle_stmt (SSA_NAME_DEF_STMT (op), idx);
     894        14572 :           m_loc = loc_save;
     895        14572 :           return ret;
     896              :         }
     897        63434 :       int p;
     898        63434 :       gimple *g;
     899        63434 :       tree t;
     900        63434 :       p = var_to_partition (m_map, op);
     901        63434 :       gcc_assert (m_vars[p] != NULL_TREE);
     902        63434 :       t = limb_access (TREE_TYPE (op), m_vars[p], idx, false);
     903        63434 :       g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
     904        63434 :       insert_before (g);
     905        63434 :       t = gimple_assign_lhs (g);
     906        63434 :       if (m_first
     907        22965 :           && m_single_use_names
     908        22044 :           && m_vars[p] != m_lhs
     909        21935 :           && m_after_stmt
     910        72265 :           && bitmap_bit_p (m_single_use_names, SSA_NAME_VERSION (op)))
     911              :         {
     912         8508 :           tree clobber = build_clobber (TREE_TYPE (m_vars[p]),
     913              :                                         CLOBBER_STORAGE_END);
     914         8508 :           g = gimple_build_assign (m_vars[p], clobber);
     915         8508 :           gimple_stmt_iterator gsi = gsi_for_stmt (m_after_stmt);
     916         8508 :           gsi_insert_after (&gsi, g, GSI_SAME_STMT);
     917              :         }
     918              :       return t;
     919        34826 :     case INTEGER_CST:
     920        34826 :       if (tree_fits_uhwi_p (idx))
     921              :         {
     922        24053 :           tree c, type = limb_access_type (TREE_TYPE (op), idx);
     923        24053 :           unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
     924        24053 :           if (m_first)
     925              :             {
     926         6265 :               m_data.safe_push (NULL_TREE);
     927         6265 :               m_data.safe_push (NULL_TREE);
     928              :             }
     929        24053 :           if (bitint_big_endian)
     930            0 :             i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
     931        24053 :           if (limb_prec != HOST_BITS_PER_WIDE_INT)
     932              :             {
     933            0 :               wide_int w = wi::rshift (wi::to_wide (op), i * limb_prec,
     934            0 :                                        TYPE_SIGN (TREE_TYPE (op)));
     935            0 :               c = wide_int_to_tree (type,
     936            0 :                                     wide_int::from (w, TYPE_PRECISION (type),
     937              :                                                     UNSIGNED));
     938            0 :             }
     939        24053 :           else if (i >= TREE_INT_CST_EXT_NUNITS (op))
     940         7634 :             c = build_int_cst (type,
     941        13580 :                                tree_int_cst_sgn (op) < 0 ? -1 : 0);
     942              :           else
     943        16419 :             c = build_int_cst (type, TREE_INT_CST_ELT (op, i));
     944        24053 :           m_data_cnt += 2;
     945        24053 :           return c;
     946              :         }
     947        10773 :       if (m_first
     948        10773 :           || (m_data[m_data_cnt] == NULL_TREE
     949          159 :               && m_data[m_data_cnt + 1] == NULL_TREE))
     950              :         {
     951         5432 :           unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
     952         5432 :           unsigned int rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
     953         5432 :           int ext;
     954         5432 :           unsigned min_prec = bitint_min_cst_precision (op, ext);
     955         5432 :           if (m_first)
     956              :             {
     957         5273 :               m_data.safe_push (NULL_TREE);
     958         5273 :               m_data.safe_push (NULL_TREE);
     959              :             }
     960         5432 :           if (integer_zerop (op))
     961              :             {
     962          867 :               tree c = build_zero_cst (m_limb_type);
     963          867 :               m_data[m_data_cnt] = c;
     964          867 :               m_data[m_data_cnt + 1] = c;
     965              :             }
     966         4565 :           else if (integer_all_onesp (op))
     967              :             {
     968          679 :               tree c = build_all_ones_cst (m_limb_type);
     969          679 :               m_data[m_data_cnt] = c;
     970          679 :               m_data[m_data_cnt + 1] = c;
     971              :             }
     972         3886 :           else if (m_upwards_2limb && min_prec <= (unsigned) limb_prec)
     973              :             {
     974              :               /* Single limb constant.  Use a phi with that limb from
     975              :                  the preheader edge and 0 or -1 constant from the other edge
     976              :                  and for the second limb in the loop.  */
     977          925 :               tree out;
     978          925 :               gcc_assert (m_first);
     979          925 :               m_data.pop ();
     980          925 :               m_data.pop ();
     981          925 :               prepare_data_in_out (fold_convert (m_limb_type, op), idx, &out,
     982          925 :                                    build_int_cst (m_limb_type, ext));
     983          925 :             }
     984         2961 :           else if (min_prec > prec - rem - 2 * limb_prec)
     985              :             {
     986              :               /* Constant which has enough significant bits that it isn't
     987              :                  worth trying to save .rodata space by extending from smaller
     988              :                  number.  */
     989         2434 :               tree type;
     990         2434 :               if (m_var_msb)
     991           25 :                 type = TREE_TYPE (op);
     992              :               else
     993              :                 /* If we have a guarantee the most significant partial limb
     994              :                    (if any) will be only accessed through handle_operand
     995              :                    with INTEGER_CST idx, we don't need to include the partial
     996              :                    limb in .rodata.  */
     997         2409 :                 type = build_bitint_type (prec - rem, 1);
     998         2434 :               tree c = tree_output_constant_def (fold_convert (type, op));
     999         2434 :               m_data[m_data_cnt] = c;
    1000         2434 :               m_data[m_data_cnt + 1] = NULL_TREE;
    1001              :             }
    1002          527 :           else if (m_upwards_2limb)
    1003              :             {
    1004              :               /* Constant with smaller number of bits.  Trade conditional
    1005              :                  code for .rodata space by extending from smaller number.  */
    1006          452 :               min_prec = CEIL (min_prec, 2 * limb_prec) * (2 * limb_prec);
    1007          452 :               tree type = build_bitint_type (min_prec, 1);
    1008          452 :               tree c = tree_output_constant_def (fold_convert (type, op));
    1009          452 :               tree ridx = idx;
    1010          452 :               if (bitint_big_endian)
    1011              :                 {
    1012            0 :                   ridx = make_ssa_name (sizetype);
    1013            0 :                   g = gimple_build_assign (ridx, PLUS_EXPR, idx,
    1014            0 :                                            size_int (min_prec / limb_prec
    1015              :                                                      - ((HOST_WIDE_INT)
    1016              :                                                         CEIL (prec,
    1017              :                                                               limb_prec))));
    1018            0 :                   insert_before (g);
    1019              :                 }
    1020          452 :               tree ridx2 = make_ssa_name (sizetype);
    1021          452 :               g = gimple_build_assign (ridx2, PLUS_EXPR, ridx,
    1022              :                                        bitint_big_endian
    1023            0 :                                        ? size_int (-1) : size_one_node);
    1024          452 :               insert_before (g);
    1025          452 :               if (bitint_big_endian)
    1026            0 :                 g = gimple_build_cond (GE_EXPR, idx,
    1027            0 :                                        size_int (CEIL (prec, limb_prec)
    1028              :                                                  - min_prec / limb_prec),
    1029              :                                        NULL_TREE, NULL_TREE);
    1030              :               else
    1031          452 :                 g = gimple_build_cond (LT_EXPR, idx,
    1032          452 :                                        size_int (min_prec / limb_prec),
    1033              :                                        NULL_TREE, NULL_TREE);
    1034          452 :               edge edge_true, edge_false;
    1035          904 :               if_then (g, (min_prec >= (prec - rem) / 2
    1036          320 :                            ? profile_probability::likely ()
    1037          132 :                            : profile_probability::unlikely ()),
    1038              :                        edge_true, edge_false);
    1039          452 :               tree c1 = limb_access (TREE_TYPE (op), c, ridx, false);
    1040          452 :               g = gimple_build_assign (make_ssa_name (TREE_TYPE (c1)), c1);
    1041          452 :               insert_before (g);
    1042          452 :               c1 = gimple_assign_lhs (g);
    1043          452 :               tree c2 = limb_access (TREE_TYPE (op), c, ridx2, false);
    1044          452 :               g = gimple_build_assign (make_ssa_name (TREE_TYPE (c2)), c2);
    1045          452 :               insert_before (g);
    1046          452 :               c2 = gimple_assign_lhs (g);
    1047          452 :               tree c3 = build_int_cst (m_limb_type, ext);
    1048          452 :               m_gsi = gsi_after_labels (edge_true->dest);
    1049          452 :               m_data[m_data_cnt] = make_ssa_name (m_limb_type);
    1050          452 :               m_data[m_data_cnt + 1] = make_ssa_name (m_limb_type);
    1051          452 :               gphi *phi = create_phi_node (m_data[m_data_cnt],
    1052              :                                            edge_true->dest);
    1053          452 :               add_phi_arg (phi, c1, edge_true, UNKNOWN_LOCATION);
    1054          452 :               add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
    1055          452 :               phi = create_phi_node (m_data[m_data_cnt + 1], edge_true->dest);
    1056          452 :               add_phi_arg (phi, c2, edge_true, UNKNOWN_LOCATION);
    1057          452 :               add_phi_arg (phi, c3, edge_false, UNKNOWN_LOCATION);
    1058              :             }
    1059              :           else
    1060              :             {
    1061              :               /* Constant with smaller number of bits.  Trade conditional
    1062              :                  code for .rodata space by extending from smaller number.
    1063              :                  Version for loops with random access to the limbs or
    1064              :                  downwards loops.  */
    1065           75 :               min_prec = CEIL (min_prec, limb_prec) * limb_prec;
    1066           75 :               tree c;
    1067           75 :               if (min_prec <= (unsigned) limb_prec)
    1068           21 :                 c = fold_convert (m_limb_type, op);
    1069              :               else
    1070              :                 {
    1071           54 :                   tree type = build_bitint_type (min_prec, 1);
    1072           54 :                   c = tree_output_constant_def (fold_convert (type, op));
    1073              :                 }
    1074           75 :               m_data[m_data_cnt] = c;
    1075           75 :               m_data[m_data_cnt + 1] = integer_type_node;
    1076              :             }
    1077         5432 :           t = m_data[m_data_cnt];
    1078              :         }
    1079              :       else
    1080         5341 :         t = m_data[m_data_cnt + 1];
    1081        10773 :       if (m_data[m_data_cnt + 1] == NULL_TREE)
    1082              :         {
    1083         4812 :           tree ridx = idx;
    1084         4812 :           unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
    1085         4812 :           tree c = m_data[m_data_cnt];
    1086         4812 :           unsigned int min_prec = TYPE_PRECISION (TREE_TYPE (c));
    1087         4812 :           if (bitint_big_endian
    1088            0 :               && CEIL (min_prec, limb_prec) != CEIL (prec, limb_prec))
    1089              :             {
    1090            0 :               ridx = make_ssa_name (sizetype);
    1091            0 :               g = gimple_build_assign (ridx, PLUS_EXPR, idx,
    1092            0 :                                        size_int (CEIL (min_prec, limb_prec)
    1093              :                                                  - ((HOST_WIDE_INT)
    1094              :                                                     CEIL (prec, limb_prec))));
    1095            0 :               insert_before (g);
    1096              :             }
    1097         4812 :           t = limb_access (TREE_TYPE (op), c, ridx, false);
    1098         4812 :           g = gimple_build_assign (make_ssa_name (TREE_TYPE (t)), t);
    1099         4812 :           insert_before (g);
    1100         4812 :           t = gimple_assign_lhs (g);
    1101              :         }
    1102         5961 :       else if (m_data[m_data_cnt + 1] == integer_type_node)
    1103              :         {
    1104          115 :           unsigned int prec = TYPE_PRECISION (TREE_TYPE (op));
    1105          115 :           unsigned rem = prec % ((m_upwards_2limb ? 2 : 1) * limb_prec);
    1106          115 :           int ext = wi::neg_p (wi::to_wide (op)) ? -1 : 0;
    1107          115 :           tree c = m_data[m_data_cnt];
    1108          115 :           unsigned min_prec = TYPE_PRECISION (TREE_TYPE (c));
    1109          115 :           if (bitint_big_endian)
    1110            0 :             g = gimple_build_cond (GE_EXPR, idx,
    1111            0 :                                    size_int (CEIL (prec, limb_prec)
    1112              :                                              - min_prec / limb_prec),
    1113              :                                    NULL_TREE, NULL_TREE);
    1114              :           else
    1115          115 :             g = gimple_build_cond (LT_EXPR, idx,
    1116          115 :                                    size_int (min_prec / limb_prec),
    1117              :                                    NULL_TREE, NULL_TREE);
    1118          115 :           edge edge_true, edge_false;
    1119          230 :           if_then (g, (min_prec >= (prec - rem) / 2
    1120           29 :                        ? profile_probability::likely ()
    1121           86 :                        : profile_probability::unlikely ()),
    1122              :                    edge_true, edge_false);
    1123          115 :           if (min_prec > (unsigned) limb_prec)
    1124              :             {
    1125           70 :               tree ridx = idx;
    1126           70 :               if (bitint_big_endian)
    1127              :                 {
    1128            0 :                   ridx = make_ssa_name (sizetype);
    1129            0 :                   g = gimple_build_assign (ridx, PLUS_EXPR, idx,
    1130            0 :                                            size_int (min_prec / limb_prec
    1131              :                                                      - ((HOST_WIDE_INT)
    1132              :                                                         CEIL (prec,
    1133              :                                                               limb_prec))));
    1134            0 :                   insert_before (g);
    1135              :                 }
    1136           70 :               c = limb_access (TREE_TYPE (op), c, ridx, false);
    1137           70 :               g = gimple_build_assign (make_ssa_name (TREE_TYPE (c)), c);
    1138           70 :               insert_before (g);
    1139           70 :               c = gimple_assign_lhs (g);
    1140              :             }
    1141          115 :           tree c2 = build_int_cst (m_limb_type, ext);
    1142          115 :           m_gsi = gsi_after_labels (edge_true->dest);
    1143          115 :           t = make_ssa_name (m_limb_type);
    1144          115 :           gphi *phi = create_phi_node (t, edge_true->dest);
    1145          115 :           add_phi_arg (phi, c, edge_true, UNKNOWN_LOCATION);
    1146          115 :           add_phi_arg (phi, c2, edge_false, UNKNOWN_LOCATION);
    1147              :         }
    1148        10773 :       m_data_cnt += 2;
    1149        10773 :       return t;
    1150            0 :     default:
    1151            0 :       gcc_unreachable ();
    1152              :     }
    1153              : }
    1154              : 
    1155              : /* Helper method, add a PHI node with VAL from preheader edge if
    1156              :    inside of a loop and m_first.  Keep state in a pair of m_data
    1157              :    elements.  If VAL_OUT is non-NULL, use that as PHI argument from
    1158              :    the latch edge, otherwise create a new SSA_NAME for it and let
    1159              :    caller initialize it.  */
    1160              : 
    1161              : tree
    1162        16045 : bitint_large_huge::prepare_data_in_out (tree val, tree idx, tree *data_out,
    1163              :                                         tree val_out)
    1164              : {
    1165        16045 :   if (!m_first)
    1166              :     {
    1167         9520 :       *data_out = tree_fits_uhwi_p (idx) ? NULL_TREE : m_data[m_data_cnt + 1];
    1168         9520 :       return m_data[m_data_cnt];
    1169              :     }
    1170              : 
    1171         6525 :   *data_out = NULL_TREE;
    1172         6525 :   if (tree_fits_uhwi_p (idx))
    1173              :     {
    1174         2024 :       m_data.safe_push (val);
    1175         2024 :       m_data.safe_push (NULL_TREE);
    1176         2024 :       return val;
    1177              :     }
    1178              : 
    1179         4501 :   tree in = make_ssa_name (TREE_TYPE (val));
    1180         4501 :   gphi *phi = create_phi_node (in, m_bb);
    1181         4501 :   edge e1 = find_edge (m_preheader_bb, m_bb);
    1182         4501 :   edge e2 = EDGE_PRED (m_bb, 0);
    1183         4501 :   if (e1 == e2)
    1184         4501 :     e2 = EDGE_PRED (m_bb, 1);
    1185         4501 :   add_phi_arg (phi, val, e1, UNKNOWN_LOCATION);
    1186         4501 :   tree out = val_out ? val_out : make_ssa_name (TREE_TYPE (val));
    1187         4501 :   add_phi_arg (phi, out, e2, UNKNOWN_LOCATION);
    1188         4501 :   m_data.safe_push (in);
    1189         4501 :   m_data.safe_push (out);
    1190         4501 :   return in;
    1191              : }
    1192              : 
    1193              : /* Return VAL cast to TYPE.  If VAL is INTEGER_CST, just
    1194              :    convert it without emitting any code, otherwise emit
    1195              :    the conversion statement before the current location.  */
    1196              : 
    1197              : tree
    1198        39041 : bitint_large_huge::add_cast (tree type, tree val)
    1199              : {
    1200        39041 :   if (TREE_CODE (val) == INTEGER_CST)
    1201         4628 :     return fold_convert (type, val);
    1202              : 
    1203        34413 :   tree lhs = make_ssa_name (type);
    1204        34413 :   gimple *g = gimple_build_assign (lhs, NOP_EXPR, val);
    1205        34413 :   insert_before (g);
    1206        34413 :   return lhs;
    1207              : }
    1208              : 
    1209              : /* Helper of handle_stmt method, handle PLUS_EXPR or MINUS_EXPR.  */
    1210              : 
    1211              : tree
    1212        13495 : bitint_large_huge::handle_plus_minus (tree_code code, tree rhs1, tree rhs2,
    1213              :                                       tree idx)
    1214              : {
    1215        13495 :   tree lhs, data_out, ctype;
    1216        13495 :   tree rhs1_type = TREE_TYPE (rhs1);
    1217        13495 :   gimple *g;
    1218        13495 :   tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
    1219              :                                       &data_out);
    1220              : 
    1221        19281 :   if (optab_handler (code == PLUS_EXPR ? uaddc5_optab : usubc5_optab,
    1222        13495 :                      TYPE_MODE (m_limb_type)) != CODE_FOR_nothing)
    1223              :     {
    1224        13495 :       ctype = build_complex_type (m_limb_type);
    1225        13495 :       if (!types_compatible_p (rhs1_type, m_limb_type))
    1226              :         {
    1227         1118 :           if (!TYPE_UNSIGNED (rhs1_type))
    1228              :             {
    1229          360 :               tree type = unsigned_type_for (rhs1_type);
    1230          360 :               rhs1 = add_cast (type, rhs1);
    1231          360 :               rhs2 = add_cast (type, rhs2);
    1232              :             }
    1233         1118 :           rhs1 = add_cast (m_limb_type, rhs1);
    1234         1118 :           rhs2 = add_cast (m_limb_type, rhs2);
    1235              :         }
    1236        13495 :       lhs = make_ssa_name (ctype);
    1237        19281 :       g = gimple_build_call_internal (code == PLUS_EXPR
    1238              :                                       ? IFN_UADDC : IFN_USUBC,
    1239              :                                       3, rhs1, rhs2, data_in);
    1240        13495 :       gimple_call_set_lhs (g, lhs);
    1241        13495 :       insert_before (g);
    1242        13495 :       if (data_out == NULL_TREE)
    1243        11249 :         data_out = make_ssa_name (m_limb_type);
    1244        13495 :       g = gimple_build_assign (data_out, IMAGPART_EXPR,
    1245              :                                build1 (IMAGPART_EXPR, m_limb_type, lhs));
    1246        13495 :       insert_before (g);
    1247              :     }
    1248            0 :   else if (types_compatible_p (rhs1_type, m_limb_type))
    1249              :     {
    1250            0 :       ctype = build_complex_type (m_limb_type);
    1251            0 :       lhs = make_ssa_name (ctype);
    1252            0 :       g = gimple_build_call_internal (code == PLUS_EXPR
    1253              :                                       ? IFN_ADD_OVERFLOW : IFN_SUB_OVERFLOW,
    1254              :                                       2, rhs1, rhs2);
    1255            0 :       gimple_call_set_lhs (g, lhs);
    1256            0 :       insert_before (g);
    1257            0 :       if (data_out == NULL_TREE)
    1258            0 :         data_out = make_ssa_name (m_limb_type);
    1259            0 :       if (!integer_zerop (data_in))
    1260              :         {
    1261            0 :           rhs1 = make_ssa_name (m_limb_type);
    1262            0 :           g = gimple_build_assign (rhs1, REALPART_EXPR,
    1263              :                                    build1 (REALPART_EXPR, m_limb_type, lhs));
    1264            0 :           insert_before (g);
    1265            0 :           rhs2 = make_ssa_name (m_limb_type);
    1266            0 :           g = gimple_build_assign (rhs2, IMAGPART_EXPR,
    1267              :                                    build1 (IMAGPART_EXPR, m_limb_type, lhs));
    1268            0 :           insert_before (g);
    1269            0 :           lhs = make_ssa_name (ctype);
    1270            0 :           g = gimple_build_call_internal (code == PLUS_EXPR
    1271              :                                           ? IFN_ADD_OVERFLOW
    1272              :                                           : IFN_SUB_OVERFLOW,
    1273              :                                           2, rhs1, data_in);
    1274            0 :           gimple_call_set_lhs (g, lhs);
    1275            0 :           insert_before (g);
    1276            0 :           data_in = make_ssa_name (m_limb_type);
    1277            0 :           g = gimple_build_assign (data_in, IMAGPART_EXPR,
    1278              :                                    build1 (IMAGPART_EXPR, m_limb_type, lhs));
    1279            0 :           insert_before (g);
    1280            0 :           g = gimple_build_assign (data_out, PLUS_EXPR, rhs2, data_in);
    1281            0 :           insert_before (g);
    1282              :         }
    1283              :       else
    1284              :         {
    1285            0 :           g = gimple_build_assign (data_out, IMAGPART_EXPR,
    1286              :                                    build1 (IMAGPART_EXPR, m_limb_type, lhs));
    1287            0 :           insert_before (g);
    1288              :         }
    1289              :     }
    1290              :   else
    1291              :     {
    1292              :       /* Always perform the two additions or subtractions in
    1293              :          unsigned type, avoid introducing a temporary UB.  While
    1294              :          the end result of the 2 additions or 2 subtractions in
    1295              :          a valid program should not overflow, temporarily it can.
    1296              :          See PR126503.  */
    1297            0 :       tree utype = unsigned_type_for (rhs1_type);
    1298            0 :       tree in = add_cast (utype, data_in);
    1299            0 :       lhs = make_ssa_name (utype);
    1300            0 :       if (utype != rhs1_type)
    1301              :         {
    1302            0 :           rhs1 = add_cast (utype, rhs1);
    1303            0 :           rhs2 = add_cast (utype, rhs2);
    1304              :         }
    1305            0 :       g = gimple_build_assign (lhs, code, rhs1, rhs2);
    1306            0 :       insert_before (g);
    1307            0 :       rhs1 = make_ssa_name (utype);
    1308            0 :       g = gimple_build_assign (rhs1, code, lhs, in);
    1309            0 :       insert_before (g);
    1310            0 :       m_data[m_data_cnt] = NULL_TREE;
    1311            0 :       m_data_cnt += 2;
    1312            0 :       return utype != rhs1_type ? add_cast (rhs1_type, rhs1) : rhs1;
    1313              :     }
    1314        13495 :   rhs1 = make_ssa_name (m_limb_type);
    1315        13495 :   g = gimple_build_assign (rhs1, REALPART_EXPR,
    1316              :                            build1 (REALPART_EXPR, m_limb_type, lhs));
    1317        13495 :   insert_before (g);
    1318        13495 :   if (!types_compatible_p (rhs1_type, m_limb_type))
    1319         1118 :     rhs1 = add_cast (rhs1_type, rhs1);
    1320        13495 :   m_data[m_data_cnt] = data_out;
    1321        13495 :   m_data_cnt += 2;
    1322        13495 :   return rhs1;
    1323              : }
    1324              : 
    1325              : /* Helper function for handle_stmt method, handle LSHIFT_EXPR by
    1326              :    count in [0, limb_prec - 1] range.  */
    1327              : 
    1328              : tree
    1329          152 : bitint_large_huge::handle_lshift (tree rhs1, tree rhs2, tree idx)
    1330              : {
    1331          152 :   unsigned HOST_WIDE_INT cnt = tree_to_uhwi (rhs2);
    1332          152 :   gcc_checking_assert (cnt < (unsigned) limb_prec);
    1333          152 :   if (cnt == 0)
    1334              :     return rhs1;
    1335              : 
    1336          152 :   tree lhs, data_out, rhs1_type = TREE_TYPE (rhs1);
    1337          152 :   gimple *g;
    1338          152 :   tree data_in = prepare_data_in_out (build_zero_cst (m_limb_type), idx,
    1339              :                                       &data_out);
    1340              : 
    1341          152 :   if (!integer_zerop (data_in))
    1342              :     {
    1343          136 :       lhs = make_ssa_name (m_limb_type);
    1344          136 :       g = gimple_build_assign (lhs, RSHIFT_EXPR, data_in,
    1345              :                                build_int_cst (unsigned_type_node,
    1346          136 :                                               limb_prec - cnt));
    1347          136 :       insert_before (g);
    1348          136 :       if (!types_compatible_p (rhs1_type, m_limb_type))
    1349           35 :         lhs = add_cast (rhs1_type, lhs);
    1350              :       data_in = lhs;
    1351              :     }
    1352          152 :   if (types_compatible_p (rhs1_type, m_limb_type))
    1353              :     {
    1354          117 :       if (data_out == NULL_TREE)
    1355           82 :         data_out = make_ssa_name (m_limb_type);
    1356          117 :       g = gimple_build_assign (data_out, rhs1);
    1357          117 :       insert_before (g);
    1358              :     }
    1359          152 :   if (cnt < (unsigned) TYPE_PRECISION (rhs1_type))
    1360              :     {
    1361          137 :       lhs = make_ssa_name (rhs1_type);
    1362          137 :       g = gimple_build_assign (lhs, LSHIFT_EXPR, rhs1, rhs2);
    1363          137 :       insert_before (g);
    1364          137 :       if (!integer_zerop (data_in))
    1365              :         {
    1366          121 :           rhs1 = lhs;
    1367          121 :           lhs = make_ssa_name (rhs1_type);
    1368          121 :           g = gimple_build_assign (lhs, BIT_IOR_EXPR, rhs1, data_in);
    1369          121 :           insert_before (g);
    1370              :         }
    1371              :     }
    1372              :   else
    1373              :     lhs = data_in;
    1374          152 :   m_data[m_data_cnt] = data_out;
    1375          152 :   m_data_cnt += 2;
    1376          152 :   return lhs;
    1377              : }
    1378              : 
    1379              : /* Helper function for handle_stmt method, handle an integral
    1380              :    to integral conversion.  */
    1381              : 
    1382              : tree
    1383         7615 : bitint_large_huge::handle_cast (tree lhs_type, tree rhs1, tree idx)
    1384              : {
    1385         7615 :   tree rhs_type = TREE_TYPE (rhs1);
    1386         7615 :   gimple *g;
    1387         7615 :   if ((TREE_CODE (rhs1) == SSA_NAME || TREE_CODE (rhs1) == INTEGER_CST)
    1388         7615 :       && BITINT_TYPE_P (lhs_type)
    1389         7615 :       && BITINT_TYPE_P (rhs_type)
    1390         6568 :       && bitint_precision_kind (lhs_type) >= bitint_prec_large
    1391        14183 :       && bitint_precision_kind (rhs_type) >= bitint_prec_large)
    1392              :     {
    1393         5929 :       if (TYPE_PRECISION (rhs_type) >= TYPE_PRECISION (lhs_type)
    1394              :           /* If lhs has bigger precision than rhs, we can use
    1395              :              the simple case only if there is a guarantee that
    1396              :              the most significant limb is handled in straight
    1397              :              line code.  If m_var_msb (on left shifts) or
    1398              :              if m_upwards_2limb * limb_prec is equal to
    1399              :              lhs precision or if not m_upwards_2limb and lhs_type
    1400              :              has precision which is multiple of limb_prec that is
    1401              :              not the case.  */
    1402         5929 :           || (!m_var_msb
    1403         1524 :               && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
    1404         1524 :                   == CEIL (TYPE_PRECISION (rhs_type), limb_prec))
    1405          370 :               && ((!m_upwards_2limb
    1406          182 :                    && (TYPE_PRECISION (lhs_type) % limb_prec != 0))
    1407          267 :                   || (m_upwards_2limb
    1408          376 :                       && (m_upwards_2limb * limb_prec
    1409          188 :                           < TYPE_PRECISION (lhs_type))))))
    1410              :         {
    1411         4670 :           tree ridx = idx;
    1412         4670 :           if (bitint_big_endian
    1413         4670 :               && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
    1414            0 :                   != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
    1415              :             {
    1416            0 :               HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
    1417            0 :               diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
    1418            0 :               if (tree_fits_uhwi_p (idx))
    1419            0 :                 ridx = size_int (tree_to_uhwi (idx) + diff);
    1420              :               else
    1421              :                 {
    1422            0 :                   tree t = make_ssa_name (sizetype);
    1423            0 :                   g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
    1424            0 :                   insert_before (g);
    1425            0 :                   ridx = t;
    1426              :                 }
    1427              :             }
    1428         4670 :           rhs1 = handle_operand (rhs1, ridx);
    1429         4670 :           if (tree_fits_uhwi_p (idx))
    1430              :             {
    1431         2389 :               tree type = limb_access_type (lhs_type, idx);
    1432         2389 :               if (!types_compatible_p (type, TREE_TYPE (rhs1)))
    1433         1250 :                 rhs1 = add_cast (type, rhs1);
    1434              :             }
    1435              :           return rhs1;
    1436              :         }
    1437         1259 :       tree t;
    1438              :       /* Indexes lower than this don't need any special processing.  */
    1439         1259 :       unsigned low = ((unsigned) TYPE_PRECISION (rhs_type)
    1440         1259 :                       - !TYPE_UNSIGNED (rhs_type)) / limb_prec;
    1441              :       /* Indexes >= than this always contain an extension.  */
    1442         1259 :       unsigned high = CEIL ((unsigned) TYPE_PRECISION (rhs_type), limb_prec);
    1443         1259 :       unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
    1444         1259 :       unsigned lowe = bitint_big_endian ? lcnt - 1 - low : low;
    1445         1259 :       bool save_first = m_first;
    1446         1259 :       if (m_first)
    1447              :         {
    1448          414 :           m_data.safe_push (NULL_TREE);
    1449          414 :           m_data.safe_push (NULL_TREE);
    1450          414 :           m_data.safe_push (NULL_TREE);
    1451          414 :           if (TYPE_UNSIGNED (rhs_type))
    1452              :             /* No need to keep state between iterations.  */
    1453              :             ;
    1454          193 :           else if (m_upwards && !m_upwards_2limb)
    1455              :             /* We need to keep state between iterations, but
    1456              :                not within any loop, everything is straight line
    1457              :                code with only increasing indexes.  */
    1458              :             ;
    1459          153 :           else if (!m_upwards_2limb)
    1460              :             {
    1461            3 :               unsigned save_data_cnt = m_data_cnt;
    1462            3 :               gimple_stmt_iterator save_gsi = m_gsi;
    1463            3 :               m_gsi = m_init_gsi;
    1464            3 :               if (gsi_end_p (m_gsi))
    1465            0 :                 m_gsi = gsi_after_labels (gsi_bb (m_gsi));
    1466              :               else
    1467            3 :                 gsi_next (&m_gsi);
    1468            3 :               m_data_cnt = save_data_cnt + 3;
    1469            3 :               t = handle_operand (rhs1, size_int (bitint_big_endian
    1470              :                                                   ? high - 1 - low : low));
    1471            3 :               m_first = false;
    1472            3 :               m_data[save_data_cnt + 2]
    1473            3 :                 = build_int_cst (integer_type_node, m_data_cnt);
    1474            3 :               m_data_cnt = save_data_cnt;
    1475            3 :               t = add_cast (signed_type_for (m_limb_type), t);
    1476            3 :               tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
    1477            3 :               tree n = make_ssa_name (TREE_TYPE (t));
    1478            3 :               g = gimple_build_assign (n, RSHIFT_EXPR, t, lpm1);
    1479            3 :               insert_before (g);
    1480            3 :               m_data[save_data_cnt + 1] = add_cast (m_limb_type, n);
    1481            3 :               m_init_gsi = m_gsi;
    1482            3 :               if (gsi_end_p (m_init_gsi))
    1483            0 :                 m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
    1484              :               else
    1485            3 :                 gsi_prev (&m_init_gsi);
    1486            3 :               m_gsi = save_gsi;
    1487              :             }
    1488          150 :           else if (m_upwards_2limb * limb_prec < TYPE_PRECISION (rhs_type))
    1489              :             /* We need to keep state between iterations, but
    1490              :                fortunately not within the loop, only afterwards.  */
    1491              :             ;
    1492              :           else
    1493              :             {
    1494          146 :               tree out;
    1495          146 :               m_data.truncate (m_data_cnt);
    1496          146 :               prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
    1497          146 :               m_data.safe_push (NULL_TREE);
    1498              :             }
    1499              :         }
    1500              : 
    1501         1259 :       unsigned save_data_cnt = m_data_cnt;
    1502         1259 :       m_data_cnt += 3;
    1503         1259 :       if (!tree_fits_uhwi_p (idx))
    1504              :         {
    1505          688 :           if (m_upwards_2limb
    1506          668 :               && low >= m_upwards_2limb - m_first)
    1507              :             {
    1508          158 :               if (bitint_big_endian
    1509          158 :                   && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
    1510            0 :                       != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
    1511              :                 {
    1512            0 :                   HOST_WIDE_INT diff
    1513            0 :                     = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
    1514            0 :                   diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
    1515            0 :                   tree t = make_ssa_name (sizetype);
    1516            0 :                   g = gimple_build_assign (t, PLUS_EXPR, idx, size_int (diff));
    1517            0 :                   insert_before (g);
    1518            0 :                   idx = t;
    1519              :                 }
    1520          158 :               rhs1 = handle_operand (rhs1, idx);
    1521          158 :               if (m_first)
    1522          131 :                 m_data[save_data_cnt + 2]
    1523          262 :                   = build_int_cst (integer_type_node, m_data_cnt);
    1524          158 :               m_first = save_first;
    1525          158 :               return rhs1;
    1526              :             }
    1527         1326 :           bool single_comparison
    1528          530 :             = low == high || (m_upwards_2limb && (low & 1) == m_first);
    1529          266 :           tree idxc = idx;
    1530          266 :           if (!single_comparison
    1531          266 :               && m_upwards_2limb
    1532          246 :               && !m_first
    1533          112 :               && low + 1 == m_upwards_2limb)
    1534              :             /* In this case we know that idx <= low always,
    1535              :                so effectively we just needs a single comparison,
    1536              :                idx < low or idx == low, but we'd need to emit different
    1537              :                code for the 2 branches than single_comparison normally
    1538              :                emits.  So, instead of special-casing that, emit a
    1539              :                low <= low comparison which cfg cleanup will clean up
    1540              :                at the end of the pass.  */
    1541           89 :             idxc = size_int (lowe);
    1542          530 :           if (bitint_big_endian)
    1543            0 :             g = gimple_build_cond (single_comparison ? GT_EXPR : GE_EXPR,
    1544            0 :                                    idxc, size_int (lowe),
    1545              :                                    NULL_TREE, NULL_TREE);
    1546              :           else
    1547          796 :             g = gimple_build_cond (single_comparison ? LT_EXPR : LE_EXPR,
    1548          530 :                                    idxc, size_int (low), NULL_TREE, NULL_TREE);
    1549          530 :           edge edge_true_true, edge_true_false, edge_false;
    1550          796 :           if_then_if_then_else (g, (single_comparison ? NULL
    1551          266 :                                     : gimple_build_cond (EQ_EXPR, idx,
    1552          266 :                                                          size_int (lowe),
    1553              :                                                          NULL_TREE,
    1554              :                                                          NULL_TREE)),
    1555              :                                 profile_probability::likely (),
    1556              :                                 profile_probability::unlikely (),
    1557              :                                 edge_true_true, edge_true_false, edge_false);
    1558          530 :           bool save_cast_conditional = m_cast_conditional;
    1559          530 :           m_cast_conditional = true;
    1560          530 :           m_bitfld_load = 0;
    1561          530 :           tree t1 = idx, t2 = NULL_TREE;
    1562          530 :           if (bitint_big_endian
    1563          530 :               && (CEIL (TYPE_PRECISION (lhs_type), limb_prec)
    1564            0 :                   != CEIL (TYPE_PRECISION (rhs_type), limb_prec)))
    1565              :             {
    1566            0 :               HOST_WIDE_INT diff = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
    1567            0 :               diff -= CEIL (TYPE_PRECISION (lhs_type), limb_prec);
    1568            0 :               t1 = make_ssa_name (sizetype);
    1569            0 :               g = gimple_build_assign (t1, PLUS_EXPR, idx, size_int (diff));
    1570            0 :               insert_before (g);
    1571              :             }
    1572          530 :           t1 = handle_operand (rhs1, t1);
    1573          530 :           if (m_first)
    1574          208 :             m_data[save_data_cnt + 2]
    1575          416 :               = build_int_cst (integer_type_node, m_data_cnt);
    1576          530 :           tree ext = NULL_TREE;
    1577          530 :           tree bitfld = NULL_TREE;
    1578          530 :           if (!single_comparison)
    1579              :             {
    1580          266 :               m_gsi = gsi_after_labels (edge_true_true->src);
    1581          266 :               m_first = false;
    1582          266 :               m_data_cnt = save_data_cnt + 3;
    1583          266 :               if (m_bitfld_load)
    1584              :                 {
    1585            4 :                   bitfld = m_data[m_bitfld_load];
    1586            4 :                   m_data[m_bitfld_load] = m_data[m_bitfld_load + 2];
    1587            4 :                   m_bitfld_load = 0;
    1588              :                 }
    1589          266 :               t2 = handle_operand (rhs1, size_int (bitint_big_endian
    1590              :                                                    ? high - 1 - low : low));
    1591          266 :               if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t2)))
    1592          221 :                 t2 = add_cast (m_limb_type, t2);
    1593          266 :               if (!TYPE_UNSIGNED (rhs_type) && m_upwards_2limb)
    1594              :                 {
    1595          146 :                   ext = add_cast (signed_type_for (m_limb_type), t2);
    1596          292 :                   tree lpm1 = build_int_cst (unsigned_type_node,
    1597          146 :                                              limb_prec - 1);
    1598          146 :                   tree n = make_ssa_name (TREE_TYPE (ext));
    1599          146 :                   g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
    1600          146 :                   insert_before (g);
    1601          146 :                   ext = add_cast (m_limb_type, n);
    1602              :                 }
    1603              :             }
    1604          530 :           tree t3;
    1605          530 :           if (TYPE_UNSIGNED (rhs_type))
    1606          278 :             t3 = build_zero_cst (m_limb_type);
    1607          252 :           else if (m_upwards_2limb && (save_first || ext != NULL_TREE))
    1608          168 :             t3 = m_data[save_data_cnt];
    1609              :           else
    1610           84 :             t3 = m_data[save_data_cnt + 1];
    1611          530 :           m_gsi = gsi_after_labels (edge_true_false->dest);
    1612          530 :           t = make_ssa_name (m_limb_type);
    1613          530 :           gphi *phi = create_phi_node (t, edge_true_false->dest);
    1614          530 :           add_phi_arg (phi, t1, edge_true_false, UNKNOWN_LOCATION);
    1615          530 :           add_phi_arg (phi, t3, edge_false, UNKNOWN_LOCATION);
    1616          530 :           if (edge_true_true)
    1617          266 :             add_phi_arg (phi, t2, edge_true_true, UNKNOWN_LOCATION);
    1618          530 :           if (ext)
    1619              :             {
    1620          146 :               tree t4 = make_ssa_name (m_limb_type);
    1621          146 :               phi = create_phi_node (t4, edge_true_false->dest);
    1622          146 :               add_phi_arg (phi, build_zero_cst (m_limb_type), edge_true_false,
    1623              :                            UNKNOWN_LOCATION);
    1624          146 :               add_phi_arg (phi, m_data[save_data_cnt], edge_false,
    1625              :                            UNKNOWN_LOCATION);
    1626          146 :               add_phi_arg (phi, ext, edge_true_true, UNKNOWN_LOCATION);
    1627          146 :               if (!save_cast_conditional)
    1628              :                 {
    1629          136 :                   g = gimple_build_assign (m_data[save_data_cnt + 1], t4);
    1630          136 :                   insert_before (g);
    1631              :                 }
    1632              :               else
    1633           10 :                 for (basic_block bb = gsi_bb (m_gsi);;)
    1634              :                   {
    1635           10 :                     edge e1 = single_succ_edge (bb);
    1636           10 :                     edge e2 = find_edge (e1->dest, m_bb), e3;
    1637           10 :                     tree t5 = (e2 ? m_data[save_data_cnt + 1]
    1638           10 :                                : make_ssa_name (m_limb_type));
    1639           10 :                     phi = create_phi_node (t5, e1->dest);
    1640           10 :                     edge_iterator ei;
    1641           30 :                     FOR_EACH_EDGE (e3, ei, e1->dest->preds)
    1642           30 :                       add_phi_arg (phi, (e3 == e1 ? t4
    1643           10 :                                          : build_zero_cst (m_limb_type)),
    1644              :                                    e3, UNKNOWN_LOCATION);
    1645           10 :                     if (e2)
    1646              :                       break;
    1647            0 :                     t4 = t5;
    1648            0 :                     bb = e1->dest;
    1649            0 :                   }
    1650              :             }
    1651          530 :           if (m_bitfld_load)
    1652              :             {
    1653            8 :               tree t4;
    1654            8 :               if (!save_first && !save_cast_conditional)
    1655            2 :                 t4 = m_data[m_bitfld_load + 1];
    1656              :               else
    1657            6 :                 t4 = make_ssa_name (m_limb_type);
    1658            8 :               phi = create_phi_node (t4, edge_true_false->dest);
    1659           12 :               add_phi_arg (phi,
    1660            4 :                            edge_true_true ? bitfld : m_data[m_bitfld_load],
    1661              :                            edge_true_false, UNKNOWN_LOCATION);
    1662            8 :               add_phi_arg (phi, m_data[m_bitfld_load + 2],
    1663              :                            edge_false, UNKNOWN_LOCATION);
    1664            8 :               if (edge_true_true)
    1665            4 :                 add_phi_arg (phi, m_data[m_bitfld_load], edge_true_true,
    1666              :                              UNKNOWN_LOCATION);
    1667            8 :               if (save_cast_conditional)
    1668            4 :                 for (basic_block bb = gsi_bb (m_gsi);;)
    1669              :                   {
    1670            4 :                     edge e1 = single_succ_edge (bb);
    1671            4 :                     edge e2 = find_edge (e1->dest, m_bb), e3;
    1672            4 :                     tree t5 = ((e2 && !save_first) ? m_data[m_bitfld_load + 1]
    1673            4 :                                : make_ssa_name (m_limb_type));
    1674            4 :                     phi = create_phi_node (t5, e1->dest);
    1675            4 :                     edge_iterator ei;
    1676           14 :                     FOR_EACH_EDGE (e3, ei, e1->dest->preds)
    1677           16 :                       add_phi_arg (phi, (e3 == e1 ? t4
    1678            6 :                                          : build_zero_cst (m_limb_type)),
    1679              :                                    e3, UNKNOWN_LOCATION);
    1680            4 :                     t4 = t5;
    1681            4 :                     if (e2)
    1682              :                       break;
    1683            0 :                     bb = e1->dest;
    1684            0 :                   }
    1685            8 :               m_data[m_bitfld_load] = t4;
    1686            8 :               m_data[m_bitfld_load + 2] = t4;
    1687            8 :               m_bitfld_load = 0;
    1688              :             }
    1689          530 :           m_cast_conditional = save_cast_conditional;
    1690          530 :           m_first = save_first;
    1691          530 :           return t;
    1692              :         }
    1693              :       else
    1694              :         {
    1695          571 :           unsigned tidx = tree_to_uhwi (idx);
    1696          571 :           if (bitint_big_endian)
    1697            0 :             tidx = lcnt - 1 - tidx;
    1698          571 :           if (tidx < low)
    1699              :             {
    1700          152 :               t = handle_operand (rhs1, (bitint_big_endian
    1701            0 :                                          ? size_int (high - 1 - tidx) : idx));
    1702          152 :               if (m_first)
    1703           71 :                 m_data[save_data_cnt + 2]
    1704          142 :                   = build_int_cst (integer_type_node, m_data_cnt);
    1705              :             }
    1706          419 :           else if (tidx < high)
    1707              :             {
    1708           68 :               t = handle_operand (rhs1, size_int (bitint_big_endian
    1709              :                                                   ? high - 1 - low : low));
    1710           68 :               if (m_first)
    1711            1 :                 m_data[save_data_cnt + 2]
    1712            2 :                   = build_int_cst (integer_type_node, m_data_cnt);
    1713           68 :               if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (t)))
    1714           60 :                 t = add_cast (m_limb_type, t);
    1715           68 :               tree ext = NULL_TREE;
    1716           68 :               if (!TYPE_UNSIGNED (rhs_type) && m_upwards)
    1717              :                 {
    1718           44 :                   ext = add_cast (signed_type_for (m_limb_type), t);
    1719           88 :                   tree lpm1 = build_int_cst (unsigned_type_node,
    1720           44 :                                              limb_prec - 1);
    1721           44 :                   tree n = make_ssa_name (TREE_TYPE (ext));
    1722           44 :                   g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
    1723           44 :                   insert_before (g);
    1724           44 :                   ext = add_cast (m_limb_type, n);
    1725           44 :                   m_data[save_data_cnt + 1] = ext;
    1726              :                 }
    1727              :             }
    1728              :           else
    1729              :             {
    1730          351 :               if (TYPE_UNSIGNED (rhs_type) && m_first)
    1731              :                 {
    1732            0 :                   handle_operand (rhs1, (bitint_big_endian
    1733            0 :                                          ? size_int (high - 1)
    1734              :                                          : size_zero_node));
    1735            0 :                   m_data[save_data_cnt + 2]
    1736            0 :                     = build_int_cst (integer_type_node, m_data_cnt);
    1737              :                 }
    1738              :               else
    1739          351 :                 m_data_cnt = tree_to_uhwi (m_data[save_data_cnt + 2]);
    1740          351 :               if (TYPE_UNSIGNED (rhs_type))
    1741          184 :                 t = build_zero_cst (m_limb_type);
    1742          167 :               else if (m_bb
    1743           16 :                        && m_data[save_data_cnt]
    1744          180 :                        && ((tidx & 1) == 0 || tidx != low + 1))
    1745              :                 t = m_data[save_data_cnt];
    1746              :               else
    1747          161 :                 t = m_data[save_data_cnt + 1];
    1748              :             }
    1749          571 :           tree type = limb_access_type (lhs_type, idx);
    1750          571 :           if (!useless_type_conversion_p (type, m_limb_type))
    1751          292 :             t = add_cast (type, t);
    1752          571 :           m_first = save_first;
    1753          571 :           return t;
    1754              :         }
    1755              :     }
    1756            0 :   else if (BITINT_TYPE_P (lhs_type)
    1757         1686 :            && bitint_precision_kind (lhs_type) >= bitint_prec_large
    1758         3372 :            && INTEGRAL_TYPE_P (rhs_type))
    1759              :     {
    1760              :       /* Add support for 3 or more limbs filled in from normal integral
    1761              :          type if this assert fails.  If no target chooses limb mode smaller
    1762              :          than half of largest supported normal integral type, this will not
    1763              :          be needed.  */
    1764         1686 :       gcc_assert (TYPE_PRECISION (rhs_type) <= 2 * limb_prec);
    1765         1686 :       tree r1 = NULL_TREE, r2 = NULL_TREE, rext = NULL_TREE;
    1766         1686 :       if (m_first)
    1767              :         {
    1768          612 :           gimple_stmt_iterator save_gsi = m_gsi;
    1769          612 :           m_gsi = m_init_gsi;
    1770          612 :           if (gsi_end_p (m_gsi))
    1771           67 :             m_gsi = gsi_after_labels (gsi_bb (m_gsi));
    1772              :           else
    1773          545 :             gsi_next (&m_gsi);
    1774          393 :           if (BITINT_TYPE_P (rhs_type)
    1775          612 :               && bitint_precision_kind (rhs_type) == bitint_prec_middle)
    1776              :             {
    1777           63 :               tree type = NULL_TREE;
    1778           63 :               rhs1 = maybe_cast_middle_bitint (&m_gsi, rhs1, type);
    1779           63 :               rhs_type = TREE_TYPE (rhs1);
    1780              :             }
    1781          612 :           r1 = rhs1;
    1782          612 :           if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
    1783          544 :             r1 = add_cast (m_limb_type, rhs1);
    1784          612 :           if (TYPE_PRECISION (rhs_type) > limb_prec)
    1785              :             {
    1786          109 :               g = gimple_build_assign (make_ssa_name (rhs_type),
    1787              :                                        RSHIFT_EXPR, rhs1,
    1788              :                                        build_int_cst (unsigned_type_node,
    1789          109 :                                                       limb_prec));
    1790          109 :               insert_before (g);
    1791          109 :               r2 = add_cast (m_limb_type, gimple_assign_lhs (g));
    1792              :             }
    1793          612 :           if (TYPE_UNSIGNED (rhs_type))
    1794          305 :             rext = build_zero_cst (m_limb_type);
    1795              :           else
    1796              :             {
    1797          307 :               rext = add_cast (signed_type_for (m_limb_type), r2 ? r2 : r1);
    1798          307 :               g = gimple_build_assign (make_ssa_name (TREE_TYPE (rext)),
    1799              :                                        RSHIFT_EXPR, rext,
    1800              :                                        build_int_cst (unsigned_type_node,
    1801          307 :                                                       limb_prec - 1));
    1802          307 :               insert_before (g);
    1803          307 :               rext = add_cast (m_limb_type, gimple_assign_lhs (g));
    1804              :             }
    1805          612 :           m_init_gsi = m_gsi;
    1806          612 :           if (gsi_end_p (m_init_gsi))
    1807          616 :             m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
    1808              :           else
    1809          304 :             gsi_prev (&m_init_gsi);
    1810          612 :           m_gsi = save_gsi;
    1811              :         }
    1812         1686 :       tree t;
    1813         1686 :       if (m_upwards_2limb)
    1814              :         {
    1815          803 :           if (m_first)
    1816              :             {
    1817          306 :               tree out1, out2;
    1818          306 :               prepare_data_in_out (r1, idx, &out1, rext);
    1819          306 :               if (TYPE_PRECISION (rhs_type) > limb_prec)
    1820              :                 {
    1821           70 :                   prepare_data_in_out (r2, idx, &out2, rext);
    1822           70 :                   m_data.pop ();
    1823           70 :                   t = m_data.pop ();
    1824           70 :                   m_data[m_data_cnt + 1] = t;
    1825              :                 }
    1826              :               else
    1827          236 :                 m_data[m_data_cnt + 1] = rext;
    1828          306 :               m_data.safe_push (rext);
    1829          306 :               t = m_data[m_data_cnt];
    1830              :             }
    1831          497 :           else if (!tree_fits_uhwi_p (idx))
    1832          306 :             t = m_data[m_data_cnt + 1];
    1833              :           else
    1834              :             {
    1835          191 :               tree type = limb_access_type (lhs_type, idx);
    1836          191 :               t = m_data[m_data_cnt + 2];
    1837          191 :               if (!useless_type_conversion_p (type, m_limb_type))
    1838          162 :                 t = add_cast (type, t);
    1839              :             }
    1840          803 :           m_data_cnt += 3;
    1841          803 :           return t;
    1842              :         }
    1843          883 :       else if (m_first)
    1844              :         {
    1845          306 :           m_data.safe_push (r1);
    1846          306 :           m_data.safe_push (r2);
    1847          306 :           m_data.safe_push (rext);
    1848              :         }
    1849          883 :       unsigned lcnt = CEIL ((unsigned) TYPE_PRECISION (lhs_type), limb_prec);
    1850          883 :       if (tree_fits_uhwi_p (idx))
    1851              :         {
    1852          839 :           tree type = limb_access_type (lhs_type, idx);
    1853          839 :           if (bitint_big_endian
    1854          839 :               ? tree_to_uhwi (idx) == lcnt - 1 : integer_zerop (idx))
    1855          278 :             t = m_data[m_data_cnt];
    1856          561 :           else if (TYPE_PRECISION (rhs_type) > limb_prec
    1857          561 :                    && (bitint_big_endian
    1858           72 :                        ? tree_to_uhwi (idx) == lcnt - 2
    1859           72 :                        : integer_onep (idx)))
    1860           33 :             t = m_data[m_data_cnt + 1];
    1861              :           else
    1862          528 :             t = m_data[m_data_cnt + 2];
    1863          839 :           if (!useless_type_conversion_p (type, m_limb_type))
    1864          259 :             t = add_cast (type, t);
    1865          839 :           m_data_cnt += 3;
    1866          839 :           return t;
    1867              :         }
    1868           44 :       g = gimple_build_cond (NE_EXPR, idx,
    1869              :                              bitint_big_endian
    1870            0 :                              ? size_int (lcnt - 1) : size_zero_node,
    1871              :                              NULL_TREE, NULL_TREE);
    1872           44 :       edge e2, e3, e4 = NULL;
    1873           44 :       if_then (g, profile_probability::likely (), e2, e3);
    1874           44 :       if (m_data[m_data_cnt + 1])
    1875              :         {
    1876           14 :           g = gimple_build_cond (EQ_EXPR, idx,
    1877              :                                  bitint_big_endian
    1878            0 :                                  ? size_int (lcnt - 2) : size_one_node,
    1879              :                                  NULL_TREE, NULL_TREE);
    1880           14 :           insert_before (g);
    1881           14 :           edge e5 = split_block (gsi_bb (m_gsi), g);
    1882           14 :           e4 = make_edge (e5->src, e2->dest, EDGE_TRUE_VALUE);
    1883           14 :           e2 = find_edge (e5->dest, e2->dest);
    1884           14 :           e4->probability = profile_probability::unlikely ();
    1885           14 :           e5->flags = EDGE_FALSE_VALUE;
    1886           14 :           e5->probability = e4->probability.invert ();
    1887              :         }
    1888           44 :       m_gsi = gsi_after_labels (e2->dest);
    1889           44 :       t = make_ssa_name (m_limb_type);
    1890           44 :       gphi *phi = create_phi_node (t, e2->dest);
    1891           44 :       add_phi_arg (phi, m_data[m_data_cnt + 2], e2, UNKNOWN_LOCATION);
    1892           44 :       add_phi_arg (phi, m_data[m_data_cnt], e3, UNKNOWN_LOCATION);
    1893           44 :       if (e4)
    1894           14 :         add_phi_arg (phi, m_data[m_data_cnt + 1], e4, UNKNOWN_LOCATION);
    1895           44 :       m_data_cnt += 3;
    1896           44 :       return t;
    1897              :     }
    1898              :   return NULL_TREE;
    1899              : }
    1900              : 
    1901              : /* Helper function for handle_stmt method, handle a BIT_FIELD_REF.  */
    1902              : 
    1903              : tree
    1904           23 : bitint_large_huge::handle_bit_field_ref (tree op, tree idx)
    1905              : {
    1906           23 :   if (tree_fits_uhwi_p (idx))
    1907              :     {
    1908           13 :       if (m_first)
    1909            4 :         m_data.safe_push (NULL);
    1910           13 :       ++m_data_cnt;
    1911           13 :       unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (m_limb_type));
    1912           13 :       unsigned i = tree_to_uhwi (idx);
    1913           13 :       if (bitint_big_endian)
    1914            0 :         i = CEIL (TYPE_PRECISION (TREE_TYPE (op)), limb_prec) - 1 - i;
    1915           26 :       tree bfr = build3 (BIT_FIELD_REF, m_limb_type,
    1916           13 :                          TREE_OPERAND (op, 0),
    1917           13 :                          TYPE_SIZE (m_limb_type),
    1918           13 :                          size_binop (PLUS_EXPR, TREE_OPERAND (op, 2),
    1919              :                                      bitsize_int (i * sz)));
    1920           13 :       tree r = make_ssa_name (m_limb_type);
    1921           13 :       gimple *g = gimple_build_assign (r, bfr);
    1922           13 :       insert_before (g);
    1923           13 :       tree type = limb_access_type (TREE_TYPE (op), idx);
    1924           13 :       if (!useless_type_conversion_p (type, m_limb_type))
    1925            0 :         r = add_cast (type, r);
    1926              :       return r;
    1927              :     }
    1928           10 :   tree var;
    1929           10 :   if (m_first)
    1930              :     {
    1931            5 :       unsigned HOST_WIDE_INT sz = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op)));
    1932            5 :       machine_mode mode;
    1933            5 :       tree type, bfr;
    1934            5 :       if (bitwise_mode_for_size (sz).exists (&mode)
    1935            0 :           && known_eq (GET_MODE_BITSIZE (mode), sz))
    1936            0 :         type = bitwise_type_for_mode (mode);
    1937              :       else
    1938              :         {
    1939            5 :           mode = VOIDmode;
    1940            5 :           type = TYPE_MAIN_VARIANT (TREE_TYPE (TREE_OPERAND (op, 0)));
    1941              :         }
    1942            5 :       if (TYPE_ALIGN (type) < TYPE_ALIGN (TREE_TYPE (op)))
    1943            0 :         type = build_aligned_type (type, TYPE_ALIGN (TREE_TYPE (op)));
    1944            5 :       var = create_tmp_var (type);
    1945            5 :       TREE_ADDRESSABLE (var) = 1;
    1946            5 :       gimple *g;
    1947            5 :       if (mode != VOIDmode)
    1948              :         {
    1949            0 :           bfr = build3 (BIT_FIELD_REF, type, TREE_OPERAND (op, 0),
    1950            0 :                         TYPE_SIZE (type), TREE_OPERAND (op, 2));
    1951            0 :           g = gimple_build_assign (make_ssa_name (type),
    1952              :                                    BIT_FIELD_REF, bfr);
    1953            0 :           gimple_set_location (g, m_loc);
    1954            0 :           gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
    1955            0 :           bfr = gimple_assign_lhs (g);
    1956              :         }
    1957              :       else
    1958            5 :         bfr = TREE_OPERAND (op, 0);
    1959            5 :       g = gimple_build_assign (var, bfr);
    1960            5 :       gimple_set_location (g, m_loc);
    1961            5 :       gsi_insert_after (&m_init_gsi, g, GSI_NEW_STMT);
    1962            5 :       if (mode == VOIDmode)
    1963              :         {
    1964            5 :           unsigned HOST_WIDE_INT nelts
    1965            5 :             = CEIL (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (op))), limb_prec);
    1966            5 :           tree atype = build_array_type_nelts (m_limb_type, nelts);
    1967            5 :           var = build2 (MEM_REF, atype, build_fold_addr_expr (var),
    1968              :                         build_int_cst (build_pointer_type (type),
    1969            5 :                                        tree_to_uhwi (TREE_OPERAND (op, 2))
    1970            5 :                                        / BITS_PER_UNIT));
    1971              :         }
    1972            5 :       m_data.safe_push (var);
    1973              :     }
    1974              :   else
    1975            5 :     var = unshare_expr (m_data[m_data_cnt]);
    1976           10 :   ++m_data_cnt;
    1977           10 :   var = limb_access (TREE_TYPE (op), var, idx, false);
    1978           10 :   tree r = make_ssa_name (m_limb_type);
    1979           10 :   gimple *g = gimple_build_assign (r, var);
    1980           10 :   insert_before (g);
    1981           10 :   return r;
    1982              : }
    1983              : 
    1984              : /* Add a new EH edge from SRC to EH_EDGE->dest, where EH_EDGE
    1985              :    is an older EH edge, and except for virtual PHIs duplicate the
    1986              :    PHI argument from the EH_EDGE to the new EH edge.  */
    1987              : 
    1988              : static void
    1989           20 : add_eh_edge (basic_block src, edge eh_edge)
    1990              : {
    1991           20 :   edge e = make_edge (src, eh_edge->dest, EDGE_EH);
    1992           20 :   e->probability = profile_probability::very_unlikely ();
    1993           20 :   for (gphi_iterator gsi = gsi_start_phis (eh_edge->dest);
    1994           27 :        !gsi_end_p (gsi); gsi_next (&gsi))
    1995              :     {
    1996            7 :       gphi *phi = gsi.phi ();
    1997            7 :       tree lhs = gimple_phi_result (phi);
    1998           14 :       if (virtual_operand_p (lhs))
    1999            4 :         continue;
    2000            3 :       const phi_arg_d *arg = gimple_phi_arg (phi, eh_edge->dest_idx);
    2001            3 :       add_phi_arg (phi, arg->def, e, arg->locus);
    2002              :     }
    2003           20 : }
    2004              : 
    2005              : /* Helper function for handle_stmt method, handle a load from memory.  */
    2006              : 
    2007              : tree
    2008        21430 : bitint_large_huge::handle_load (gimple *stmt, tree idx)
    2009              : {
    2010        21430 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    2011        21430 :   tree rhs_type = TREE_TYPE (rhs1);
    2012        21430 :   bool eh = stmt_ends_bb_p (stmt);
    2013        21430 :   bool load_bitfield_p = false;
    2014        21430 :   edge eh_edge = NULL;
    2015        21430 :   gimple *g;
    2016              : 
    2017        21430 :   if (TREE_CODE (rhs1) == BIT_FIELD_REF
    2018        21430 :       && integer_zerop (TREE_OPERAND (rhs1, 2)))
    2019            2 :     rhs1 = TREE_OPERAND (rhs1, 0);
    2020              : 
    2021        21430 :   if (eh)
    2022              :     {
    2023           10 :       edge_iterator ei;
    2024           10 :       basic_block bb = gimple_bb (stmt);
    2025              : 
    2026           10 :       FOR_EACH_EDGE (eh_edge, ei, bb->succs)
    2027           10 :         if (eh_edge->flags & EDGE_EH)
    2028              :             break;
    2029              :     }
    2030              : 
    2031        21430 :   if (TREE_CODE (rhs1) == COMPONENT_REF
    2032        21430 :       && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
    2033              :     {
    2034         1267 :       tree fld = TREE_OPERAND (rhs1, 1);
    2035              :       /* For little-endian, we can allow as inputs bit-fields
    2036              :          which start at a limb boundary.  */
    2037         1267 :       gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
    2038         1267 :       if (!bitint_big_endian
    2039         1267 :           && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
    2040         2534 :           && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % limb_prec) == 0)
    2041              :         {
    2042          739 :           load_bitfield_p = true;
    2043          751 :           goto normal_load;
    2044              :         }
    2045              :       /* Even if DECL_FIELD_BIT_OFFSET (fld) is a multiple of BITS_PER_UNIT,
    2046              :          handle it normally for now.  */
    2047          528 :       if (!bitint_big_endian
    2048          528 :           && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld)) % BITS_PER_UNIT) == 0)
    2049              :         {
    2050           12 :           load_bitfield_p = true;
    2051           12 :           goto normal_load;
    2052              :         }
    2053          516 :       tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
    2054          516 :       poly_int64 bitoffset;
    2055          516 :       poly_uint64 field_offset, repr_offset;
    2056          516 :       bool var_field_off = false;
    2057          516 :       if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
    2058         1032 :           && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
    2059          516 :         bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
    2060              :       else
    2061              :         {
    2062              :           bitoffset = 0;
    2063              :           var_field_off = true;
    2064              :         }
    2065          516 :       bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
    2066          516 :                     - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
    2067         1032 :       tree nrhs1 = build3 (COMPONENT_REF, TREE_TYPE (repr),
    2068          516 :                            TREE_OPERAND (rhs1, 0), repr,
    2069            0 :                            var_field_off ? TREE_OPERAND (rhs1, 2) : NULL_TREE);
    2070          516 :       HOST_WIDE_INT bo = bitoffset.to_constant ();
    2071          516 :       unsigned bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
    2072          516 :       unsigned bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
    2073          516 :       unsigned bo_last = 0;
    2074          516 :       unsigned bo_shift = bo_bit;
    2075          516 :       unsigned nelts = CEIL (TYPE_PRECISION (rhs_type), limb_prec);
    2076          516 :       if (bitint_big_endian)
    2077              :         {
    2078            0 :           bo_last = CEIL (TYPE_PRECISION (rhs_type) + bo_bit, limb_prec) - 1;
    2079            0 :           bo_shift = (TYPE_PRECISION (rhs_type) + bo_bit) % limb_prec;
    2080            0 :           if (bo_shift)
    2081            0 :             bo_shift = limb_prec - bo_shift;
    2082              :         }
    2083          516 :       if (m_first)
    2084              :         {
    2085          153 :           if (m_upwards && bo_shift)
    2086              :             {
    2087          150 :               gimple_stmt_iterator save_gsi = m_gsi;
    2088          150 :               m_gsi = m_init_gsi;
    2089          150 :               if (gsi_end_p (m_gsi))
    2090           61 :                 m_gsi = gsi_after_labels (gsi_bb (m_gsi));
    2091              :               else
    2092           89 :                 gsi_next (&m_gsi);
    2093          150 :               tree t = limb_access (NULL_TREE, nrhs1,
    2094          150 :                                     size_int (bo_idx + bo_last), true);
    2095          150 :               tree iv = make_ssa_name (m_limb_type);
    2096          150 :               g = gimple_build_assign (iv, t);
    2097          150 :               insert_before (g);
    2098          150 :               if (eh)
    2099              :                 {
    2100            2 :                   maybe_duplicate_eh_stmt (g, stmt);
    2101            2 :                   if (eh_edge)
    2102              :                     {
    2103            2 :                       edge e = split_block (gsi_bb (m_gsi), g);
    2104            2 :                       add_eh_edge (e->src, eh_edge);
    2105            2 :                       m_gsi = gsi_after_labels (e->dest);
    2106            2 :                       if (gsi_bb (save_gsi) == e->src)
    2107              :                         {
    2108            1 :                           if (gsi_end_p (save_gsi))
    2109            0 :                             save_gsi = gsi_end_bb (e->dest);
    2110              :                           else
    2111            1 :                             save_gsi = gsi_for_stmt (gsi_stmt (save_gsi));
    2112              :                         }
    2113            2 :                       if (m_preheader_bb == e->src)
    2114            1 :                         m_preheader_bb = e->dest;
    2115              :                     }
    2116              :                 }
    2117          150 :               m_init_gsi = m_gsi;
    2118          150 :               if (gsi_end_p (m_init_gsi))
    2119          246 :                 m_init_gsi = gsi_last_bb (gsi_bb (m_init_gsi));
    2120              :               else
    2121           28 :                 gsi_prev (&m_init_gsi);
    2122          150 :               m_gsi = save_gsi;
    2123          150 :               tree out;
    2124          150 :               prepare_data_in_out (iv, idx, &out);
    2125          150 :               out = m_data[m_data_cnt];
    2126          150 :               m_data.safe_push (out);
    2127          150 :             }
    2128              :           else
    2129              :             {
    2130            3 :               m_data.safe_push (NULL_TREE);
    2131            3 :               m_data.safe_push (NULL_TREE);
    2132            3 :               m_data.safe_push (NULL_TREE);
    2133              :             }
    2134              :         }
    2135              : 
    2136          516 :       tree nidx0 = NULL_TREE, nidx1 = NULL_TREE;
    2137          516 :       tree iv = m_data[m_data_cnt];
    2138          516 :       if (m_cast_conditional && iv)
    2139              :         {
    2140           12 :           gcc_assert (!m_bitfld_load);
    2141           12 :           m_bitfld_load = m_data_cnt;
    2142              :         }
    2143          516 :       if (tree_fits_uhwi_p (idx))
    2144              :         {
    2145          290 :           unsigned prec = TYPE_PRECISION (rhs_type);
    2146          290 :           unsigned HOST_WIDE_INT i = tree_to_uhwi (idx);
    2147          290 :           if (bitint_big_endian)
    2148            0 :             i = nelts - 1 - i;
    2149          290 :           gcc_assert (i * limb_prec < prec);
    2150          290 :           if (bo_shift)
    2151          290 :             nidx1 = size_int (bo_idx + (bitint_big_endian
    2152              :                                         ? bo_last - i - 1 : i + 1));
    2153          290 :           if ((i + 1) * limb_prec > prec)
    2154              :             {
    2155          109 :               prec %= limb_prec;
    2156          109 :               if (prec + bo_bit <= (unsigned) limb_prec)
    2157          290 :                 nidx1 = NULL_TREE;
    2158              :             }
    2159          290 :           if (!iv)
    2160            4 :             nidx0 = size_int (bo_idx + (bitint_big_endian ? bo_last - i : i));
    2161              :         }
    2162              :       else
    2163              :         {
    2164          226 :           HOST_WIDE_INT adj = bo_idx;
    2165          226 :           if (bitint_big_endian)
    2166            0 :             adj += (HOST_WIDE_INT) bo_last + 1 - nelts;
    2167          226 :           if (!iv)
    2168              :             {
    2169            4 :               if (adj == 0)
    2170              :                 nidx0 = idx;
    2171              :               else
    2172              :                 {
    2173            0 :                   nidx0 = make_ssa_name (sizetype);
    2174            0 :                   g = gimple_build_assign (nidx0, PLUS_EXPR, idx,
    2175            0 :                                            size_int (adj));
    2176            0 :                   insert_before (g);
    2177              :                 }
    2178              :             }
    2179          226 :           if (bo_shift)
    2180              :             {
    2181          226 :               if (bitint_big_endian && adj == 1)
    2182              :                 nidx1 = idx;
    2183              :               else
    2184              :                 {
    2185          226 :                   nidx1 = make_ssa_name (sizetype);
    2186          226 :                   g = gimple_build_assign (nidx1, PLUS_EXPR, idx,
    2187          452 :                                            size_int (adj + (bitint_big_endian
    2188              :                                                             ? -1 : 1)));
    2189          226 :                   insert_before (g);
    2190              :                 }
    2191              :             }
    2192              :         }
    2193              : 
    2194          746 :       tree iv2 = NULL_TREE;
    2195          230 :       if (nidx0)
    2196              :         {
    2197            8 :           tree t = limb_access (NULL_TREE, nrhs1, nidx0, true);
    2198            8 :           iv = make_ssa_name (m_limb_type);
    2199            8 :           g = gimple_build_assign (iv, t);
    2200            8 :           insert_before (g);
    2201            8 :           if (eh)
    2202              :             {
    2203            0 :               maybe_duplicate_eh_stmt (g, stmt);
    2204            0 :               if (eh_edge)
    2205              :                 {
    2206            0 :                   edge e = split_block (gsi_bb (m_gsi), g);
    2207            0 :                   m_gsi = gsi_after_labels (e->dest);
    2208            0 :                   add_eh_edge (e->src, eh_edge);
    2209              :                 }
    2210              :             }
    2211              :         }
    2212          516 :       if (nidx1)
    2213              :         {
    2214          427 :           bool conditional = m_var_msb && !tree_fits_uhwi_p (idx);
    2215          427 :           unsigned prec = TYPE_PRECISION (rhs_type);
    2216          427 :           if (conditional)
    2217              :             {
    2218            3 :               if ((prec % limb_prec) == 0
    2219            3 :                   || ((prec % limb_prec) + bo_bit > (unsigned) limb_prec))
    2220          424 :                 conditional = false;
    2221              :             }
    2222          427 :           edge edge_true = NULL, edge_false = NULL;
    2223          427 :           if (conditional)
    2224              :             {
    2225            6 :               g = gimple_build_cond (NE_EXPR, idx,
    2226              :                                      bitint_big_endian
    2227              :                                      ? size_zero_node
    2228            3 :                                      : size_int (prec / limb_prec),
    2229              :                                      NULL_TREE, NULL_TREE);
    2230            3 :               if_then (g, profile_probability::likely (),
    2231              :                        edge_true, edge_false);
    2232              :             }
    2233          427 :           tree t = limb_access (NULL_TREE, nrhs1, nidx1, true);
    2234          427 :           if (m_upwards_2limb
    2235          325 :               && !m_first
    2236          214 :               && !m_bitfld_load
    2237          208 :               && !tree_fits_uhwi_p (idx))
    2238          107 :             iv2 = m_data[m_data_cnt + 1];
    2239              :           else
    2240          320 :             iv2 = make_ssa_name (m_limb_type);
    2241          427 :           g = gimple_build_assign (iv2, t);
    2242          427 :           insert_before (g);
    2243          427 :           if (eh)
    2244              :             {
    2245            5 :               maybe_duplicate_eh_stmt (g, stmt);
    2246            5 :               if (eh_edge)
    2247              :                 {
    2248            5 :                   edge e = split_block (gsi_bb (m_gsi), g);
    2249            5 :                   m_gsi = gsi_after_labels (e->dest);
    2250            5 :                   add_eh_edge (e->src, eh_edge);
    2251              :                 }
    2252              :             }
    2253          427 :           if (conditional)
    2254              :             {
    2255            3 :               tree iv3 = make_ssa_name (m_limb_type);
    2256            3 :               if (eh)
    2257            0 :                 edge_true = find_edge (gsi_bb (m_gsi), edge_false->dest);
    2258            3 :               gphi *phi = create_phi_node (iv3, edge_true->dest);
    2259            3 :               add_phi_arg (phi, iv2, edge_true, UNKNOWN_LOCATION);
    2260            3 :               add_phi_arg (phi, build_zero_cst (m_limb_type),
    2261              :                            edge_false, UNKNOWN_LOCATION);
    2262            3 :               m_gsi = gsi_after_labels (edge_true->dest);
    2263            3 :               iv2 = iv3;
    2264              :             }
    2265              :         }
    2266          516 :       if (bo_shift)
    2267              :         {
    2268          516 :           g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
    2269              :                                    iv, build_int_cst (unsigned_type_node,
    2270          516 :                                                       bo_shift));
    2271          516 :           insert_before (g);
    2272          516 :           iv = gimple_assign_lhs (g);
    2273              :         }
    2274          516 :       if (iv2)
    2275              :         {
    2276          427 :           g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
    2277              :                                    iv2, build_int_cst (unsigned_type_node,
    2278          427 :                                                        limb_prec - bo_shift));
    2279          427 :           insert_before (g);
    2280          427 :           g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
    2281              :                                    gimple_assign_lhs (g), iv);
    2282          427 :           insert_before (g);
    2283          427 :           iv = gimple_assign_lhs (g);
    2284          427 :           if (m_data[m_data_cnt])
    2285          421 :             m_data[m_data_cnt] = iv2;
    2286              :         }
    2287          516 :       if (tree_fits_uhwi_p (idx))
    2288              :         {
    2289          290 :           tree atype = limb_access_type (rhs_type, idx);
    2290          290 :           if (!useless_type_conversion_p (atype, TREE_TYPE (iv)))
    2291          109 :             iv = add_cast (atype, iv);
    2292              :         }
    2293          516 :       m_data_cnt += 3;
    2294          516 :       return iv;
    2295              :     }
    2296              : 
    2297        20914 : normal_load:
    2298              :   /* Use write_p = true for loads with EH edges to make
    2299              :      sure limb_access doesn't add a cast as separate
    2300              :      statement after it.  */
    2301        20914 :   rhs1 = limb_access (rhs_type, rhs1, idx, eh, !load_bitfield_p);
    2302        20914 :   tree ret = make_ssa_name (TREE_TYPE (rhs1));
    2303        20914 :   g = gimple_build_assign (ret, rhs1);
    2304        20914 :   insert_before (g);
    2305        20914 :   if (eh)
    2306              :     {
    2307            3 :       maybe_duplicate_eh_stmt (g, stmt);
    2308            3 :       if (eh_edge)
    2309              :         {
    2310            3 :           edge e = split_block (gsi_bb (m_gsi), g);
    2311            3 :           m_gsi = gsi_after_labels (e->dest);
    2312            3 :           add_eh_edge (e->src, eh_edge);
    2313              :         }
    2314            3 :       if (tree_fits_uhwi_p (idx))
    2315              :         {
    2316            1 :           tree atype = limb_access_type (rhs_type, idx);
    2317            1 :           if (!useless_type_conversion_p (atype, TREE_TYPE (rhs1)))
    2318            1 :             ret = add_cast (atype, ret);
    2319              :         }
    2320              :     }
    2321              :   return ret;
    2322              : }
    2323              : 
    2324              : /* Return a limb IDX from a mergeable statement STMT.  */
    2325              : 
    2326              : tree
    2327        39504 : bitint_large_huge::handle_stmt (gimple *stmt, tree idx)
    2328              : {
    2329        39504 :   tree lhs, rhs1, rhs2 = NULL_TREE;
    2330        39504 :   gimple *g;
    2331        39504 :   switch (gimple_code (stmt))
    2332              :     {
    2333        39504 :     case GIMPLE_ASSIGN:
    2334        39504 :       if (gimple_assign_load_p (stmt))
    2335        21430 :         return handle_load (stmt, idx);
    2336        18074 :       switch (gimple_assign_rhs_code (stmt))
    2337              :         {
    2338         1004 :         case BIT_AND_EXPR:
    2339         1004 :         case BIT_IOR_EXPR:
    2340         1004 :         case BIT_XOR_EXPR:
    2341         1004 :           rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
    2342              :           /* FALLTHRU */
    2343         1424 :         case BIT_NOT_EXPR:
    2344         1424 :           rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
    2345         1424 :           lhs = make_ssa_name (TREE_TYPE (rhs1));
    2346         1424 :           g = gimple_build_assign (lhs, gimple_assign_rhs_code (stmt),
    2347              :                                    rhs1, rhs2);
    2348         1424 :           insert_before (g);
    2349         1424 :           return lhs;
    2350         4161 :         case PLUS_EXPR:
    2351         4161 :         case MINUS_EXPR:
    2352         4161 :           rhs1 = handle_operand (gimple_assign_rhs1 (stmt), idx);
    2353         4161 :           rhs2 = handle_operand (gimple_assign_rhs2 (stmt), idx);
    2354         4161 :           return handle_plus_minus (gimple_assign_rhs_code (stmt),
    2355         4161 :                                     rhs1, rhs2, idx);
    2356          163 :         case NEGATE_EXPR:
    2357          163 :           rhs2 = handle_operand (gimple_assign_rhs1 (stmt), idx);
    2358          163 :           rhs1 = build_zero_cst (TREE_TYPE (rhs2));
    2359          163 :           return handle_plus_minus (MINUS_EXPR, rhs1, rhs2, idx);
    2360          152 :         case LSHIFT_EXPR:
    2361          152 :           return handle_lshift (handle_operand (gimple_assign_rhs1 (stmt),
    2362              :                                                 idx),
    2363          152 :                                 gimple_assign_rhs2 (stmt), idx);
    2364         5120 :         case SSA_NAME:
    2365         5120 :         case PAREN_EXPR:
    2366         5120 :         case INTEGER_CST:
    2367         5120 :           return handle_operand (gimple_assign_rhs1 (stmt), idx);
    2368         7023 :         CASE_CONVERT:
    2369         7023 :           return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
    2370         7023 :                               gimple_assign_rhs1 (stmt), idx);
    2371            8 :         case VIEW_CONVERT_EXPR:
    2372            8 :           return handle_cast (TREE_TYPE (gimple_assign_lhs (stmt)),
    2373            8 :                               TREE_OPERAND (gimple_assign_rhs1 (stmt), 0),
    2374            8 :                               idx);
    2375           23 :         case BIT_FIELD_REF:
    2376           23 :           return handle_bit_field_ref (gimple_assign_rhs1 (stmt), idx);
    2377              :         default:
    2378              :           break;
    2379              :         }
    2380              :       break;
    2381              :     default:
    2382              :       break;
    2383              :     }
    2384            0 :   gcc_unreachable ();
    2385              : }
    2386              : 
    2387              : /* Return minimum precision of OP at STMT.
    2388              :    Positive value is minimum precision above which all bits
    2389              :    are zero, negative means all bits above negation of the
    2390              :    value are copies of the sign bit.  */
    2391              : 
    2392              : static int
    2393         8450 : range_to_prec (tree op, gimple *stmt)
    2394              : {
    2395         8450 :   int_range_max r;
    2396         8450 :   wide_int w;
    2397         8450 :   tree type = TREE_TYPE (op);
    2398         8450 :   unsigned int prec = TYPE_PRECISION (type);
    2399              : 
    2400         8450 :   if (!optimize
    2401         7518 :       || !get_range_query (cfun)->range_of_expr (r, op, stmt)
    2402        12209 :       || r.undefined_p ())
    2403              :     {
    2404         4692 :       if (TYPE_UNSIGNED (type))
    2405         1902 :         return prec;
    2406              :       else
    2407         2790 :         return MIN ((int) -prec, -2);
    2408              :     }
    2409              : 
    2410         3758 :   if (!TYPE_UNSIGNED (TREE_TYPE (op)))
    2411              :     {
    2412         2343 :       w = r.lower_bound ();
    2413         2343 :       if (wi::neg_p (w))
    2414              :         {
    2415         1938 :           int min_prec1 = wi::min_precision (w, SIGNED);
    2416         1938 :           w = r.upper_bound ();
    2417         1938 :           int min_prec2 = wi::min_precision (w, SIGNED);
    2418         1938 :           int min_prec = MAX (min_prec1, min_prec2);
    2419         1938 :           return MIN (-min_prec, -2);
    2420              :         }
    2421              :     }
    2422              : 
    2423         1820 :   w = r.upper_bound ();
    2424         1820 :   int min_prec = wi::min_precision (w, UNSIGNED);
    2425         1820 :   return MAX (min_prec, 1);
    2426         8450 : }
    2427              : 
    2428              : /* Return address of the first limb of OP and write into *PREC
    2429              :    its precision.  If positive, the operand is zero extended
    2430              :    from that precision, if it is negative, the operand is sign-extended
    2431              :    from -*PREC.  If PREC_STORED is NULL, it is the toplevel call,
    2432              :    otherwise *PREC_STORED is prec from the innermost call without
    2433              :    range optimizations (0 for uninitialized SSA_NAME).  */
    2434              : 
    2435              : tree
    2436         3868 : bitint_large_huge::handle_operand_addr (tree op, gimple *stmt,
    2437              :                                         int *prec_stored, int *prec)
    2438              : {
    2439         3868 :   wide_int w;
    2440         3868 :   location_t loc_save = m_loc;
    2441         3868 :   tree ret = NULL_TREE;
    2442         3868 :   int precs = 0;
    2443         3903 :   if ((!BITINT_TYPE_P (TREE_TYPE (op))
    2444         3839 :        || bitint_precision_kind (TREE_TYPE (op)) < bitint_prec_large)
    2445         3992 :       && TREE_CODE (op) != INTEGER_CST)
    2446              :     {
    2447          126 :     do_int:
    2448          126 :       *prec = range_to_prec (op, stmt);
    2449          126 :       bitint_prec_kind kind = bitint_prec_small;
    2450          126 :       gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (op)));
    2451          126 :       if (BITINT_TYPE_P (TREE_TYPE (op)))
    2452          106 :         kind = bitint_precision_kind (TREE_TYPE (op));
    2453          106 :       if (kind == bitint_prec_middle)
    2454              :         {
    2455           12 :           tree type = NULL_TREE;
    2456           12 :           op = maybe_cast_middle_bitint (&m_gsi, op, type);
    2457              :         }
    2458          126 :       tree op_type = TREE_TYPE (op);
    2459          126 :       unsigned HOST_WIDE_INT nelts
    2460          126 :         = CEIL (TYPE_PRECISION (op_type), limb_prec);
    2461              :       /* Add support for 3 or more limbs filled in from normal
    2462              :          integral type if this assert fails.  If no target chooses
    2463              :          limb mode smaller than half of largest supported normal
    2464              :          integral type, this will not be needed.  */
    2465          126 :       gcc_assert (nelts <= 2);
    2466          126 :       precs = (TYPE_UNSIGNED (op_type)
    2467          126 :                ? TYPE_PRECISION (op_type) : -TYPE_PRECISION (op_type));
    2468          126 :       if (*prec <= limb_prec && *prec >= -limb_prec)
    2469              :         {
    2470          113 :           nelts = 1;
    2471          113 :           if (TYPE_UNSIGNED (op_type))
    2472              :             {
    2473           26 :               if (precs > limb_prec)
    2474          126 :                 precs = limb_prec;
    2475              :             }
    2476           87 :           else if (precs < -limb_prec)
    2477          126 :             precs = -limb_prec;
    2478              :         }
    2479          126 :       if (prec_stored)
    2480            0 :         *prec_stored = precs;
    2481          126 :       tree atype = build_array_type_nelts (m_limb_type, nelts);
    2482          126 :       tree var = create_tmp_var (atype);
    2483          126 :       tree t1 = op;
    2484          126 :       if (!useless_type_conversion_p (m_limb_type, op_type))
    2485          126 :         t1 = add_cast (m_limb_type, t1);
    2486          126 :       tree v = build4 (ARRAY_REF, m_limb_type, var,
    2487            0 :                        bitint_big_endian && nelts > 1
    2488              :                        ? size_one_node : size_zero_node,
    2489              :                        NULL_TREE, NULL_TREE);
    2490          126 :       gimple *g = gimple_build_assign (v, t1);
    2491          126 :       insert_before (g);
    2492          126 :       if (nelts > 1)
    2493              :         {
    2494           13 :           tree lp = build_int_cst (unsigned_type_node, limb_prec);
    2495           13 :           g = gimple_build_assign (make_ssa_name (op_type),
    2496              :                                    RSHIFT_EXPR, op, lp);
    2497           13 :           insert_before (g);
    2498           13 :           tree t2 = gimple_assign_lhs (g);
    2499           13 :           t2 = add_cast (m_limb_type, t2);
    2500           13 :           v = build4 (ARRAY_REF, m_limb_type, var,
    2501              :                       bitint_big_endian ? size_zero_node : size_one_node,
    2502              :                       NULL_TREE, NULL_TREE);
    2503           13 :           g = gimple_build_assign (v, t2);
    2504           13 :           insert_before (g);
    2505              :         }
    2506          126 :       ret = build_fold_addr_expr (var);
    2507          126 :       if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
    2508              :         {
    2509          125 :           tree clobber = build_clobber (atype, CLOBBER_STORAGE_END);
    2510          125 :           g = gimple_build_assign (var, clobber);
    2511          125 :           gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
    2512              :         }
    2513          126 :       m_loc = loc_save;
    2514          126 :       goto do_ret;
    2515              :     }
    2516         3792 :   switch (TREE_CODE (op))
    2517              :     {
    2518         2926 :     case SSA_NAME:
    2519         2926 :       if (m_names == NULL
    2520         2926 :           || !bitmap_bit_p (m_names, SSA_NAME_VERSION (op)))
    2521              :         {
    2522          104 :           gimple *g = SSA_NAME_DEF_STMT (op);
    2523          104 :           m_loc = gimple_location (g);
    2524          104 :           if (gimple_assign_load_p (g))
    2525              :             {
    2526           36 :               *prec = range_to_prec (op, NULL);
    2527           36 :               precs = (TYPE_UNSIGNED (TREE_TYPE (op))
    2528           36 :                        ? TYPE_PRECISION (TREE_TYPE (op))
    2529           25 :                        : -TYPE_PRECISION (TREE_TYPE (op)));
    2530           36 :               if (prec_stored)
    2531            6 :                 *prec_stored = precs;
    2532           36 :               ret = build_fold_addr_expr (gimple_assign_rhs1 (g));
    2533           36 :               ret = force_gimple_operand_gsi (&m_gsi, ret, true,
    2534              :                                               NULL_TREE, true, GSI_SAME_STMT);
    2535              :             }
    2536           68 :           else if (gimple_code (g) == GIMPLE_NOP)
    2537              :             {
    2538            2 :               *prec = TYPE_UNSIGNED (TREE_TYPE (op)) ? limb_prec : -limb_prec;
    2539            2 :               precs = *prec;
    2540            2 :               if (prec_stored)
    2541            1 :                 *prec_stored = 0;
    2542            2 :               tree var = create_tmp_var (m_limb_type);
    2543            2 :               TREE_ADDRESSABLE (var) = 1;
    2544            2 :               ret = build_fold_addr_expr (var);
    2545            2 :               if (!stmt_ends_bb_p (gsi_stmt (m_gsi)))
    2546              :                 {
    2547            2 :                   tree clobber = build_clobber (m_limb_type,
    2548              :                                                 CLOBBER_STORAGE_END);
    2549            2 :                   g = gimple_build_assign (var, clobber);
    2550            2 :                   gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
    2551              :                 }
    2552              :             }
    2553              :           else
    2554              :             {
    2555           66 :               gcc_assert (gimple_assign_cast_p (g));
    2556           66 :               tree rhs1 = gimple_assign_rhs1 (g);
    2557           66 :               bitint_prec_kind kind = bitint_prec_small;
    2558           66 :               if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
    2559            1 :                 rhs1 = TREE_OPERAND (rhs1, 0);
    2560           66 :               gcc_assert (INTEGRAL_TYPE_P (TREE_TYPE (rhs1)));
    2561           66 :               if (BITINT_TYPE_P (TREE_TYPE (rhs1)))
    2562           51 :                 kind = bitint_precision_kind (TREE_TYPE (rhs1));
    2563           51 :               if (kind >= bitint_prec_large)
    2564              :                 {
    2565           16 :                   tree lhs_type = TREE_TYPE (op);
    2566           16 :                   tree rhs_type = TREE_TYPE (rhs1);
    2567           16 :                   int prec_stored_val = 0;
    2568           16 :                   ret = handle_operand_addr (rhs1, g, &prec_stored_val, prec);
    2569           16 :                   precs = prec_stored_val;
    2570           16 :                   if (prec_stored)
    2571            0 :                     *prec_stored = prec_stored_val;
    2572           16 :                   if (precs == 0)
    2573              :                     {
    2574            1 :                       gcc_assert (*prec == limb_prec || *prec == -limb_prec);
    2575              :                       precs = *prec;
    2576              :                     }
    2577           16 :                   if (TYPE_PRECISION (lhs_type) > TYPE_PRECISION (rhs_type))
    2578              :                     {
    2579            4 :                       if (TYPE_UNSIGNED (lhs_type)
    2580            4 :                           && !TYPE_UNSIGNED (rhs_type))
    2581            1 :                         gcc_assert (*prec >= 0 || prec_stored == NULL);
    2582              :                     }
    2583              :                   else
    2584              :                     {
    2585           12 :                       if (prec_stored_val == 0)
    2586              :                         /* Non-widening cast of uninitialized value.  */;
    2587           11 :                       else if (*prec > 0 && *prec < TYPE_PRECISION (lhs_type))
    2588              :                         ;
    2589           11 :                       else if (TYPE_UNSIGNED (lhs_type))
    2590              :                         {
    2591            8 :                           gcc_assert (*prec > 0
    2592              :                                       || prec_stored_val > 0
    2593              :                                       || (-prec_stored_val
    2594              :                                           >= TYPE_PRECISION (lhs_type)));
    2595            8 :                           *prec = TYPE_PRECISION (lhs_type);
    2596              :                         }
    2597            3 :                       else if (*prec < 0 && -*prec < TYPE_PRECISION (lhs_type))
    2598              :                         ;
    2599              :                       else
    2600            3 :                         *prec = -TYPE_PRECISION (lhs_type);
    2601              :                     }
    2602              :                 }
    2603              :               else
    2604              :                 {
    2605           50 :                   op = rhs1;
    2606           50 :                   stmt = g;
    2607           50 :                   goto do_int;
    2608              :                 }
    2609              :             }
    2610           54 :           m_loc = loc_save;
    2611           54 :           goto do_ret;
    2612              :         }
    2613              :       else
    2614              :         {
    2615         2822 :           int p = var_to_partition (m_map, op);
    2616         2822 :           gcc_assert (m_vars[p] != NULL_TREE);
    2617         2822 :           *prec = range_to_prec (op, stmt);
    2618         2822 :           precs = (TYPE_UNSIGNED (TREE_TYPE (op))
    2619         2822 :                    ? TYPE_PRECISION (TREE_TYPE (op))
    2620         1671 :                    : -TYPE_PRECISION (TREE_TYPE (op)));
    2621         2822 :           if (prec_stored)
    2622            9 :             *prec_stored = precs;
    2623         2822 :           ret = build_fold_addr_expr (m_vars[p]);
    2624         2822 :           goto do_ret;
    2625              :         }
    2626          866 :     case INTEGER_CST:
    2627          866 :       unsigned int min_prec, mp;
    2628          866 :       tree type;
    2629          866 :       w = wi::to_wide (op);
    2630          866 :       if (tree_int_cst_sgn (op) >= 0)
    2631              :         {
    2632          651 :           min_prec = wi::min_precision (w, UNSIGNED);
    2633          651 :           *prec = MAX (min_prec, 1);
    2634              :         }
    2635              :       else
    2636              :         {
    2637          215 :           min_prec = wi::min_precision (w, SIGNED);
    2638          215 :           *prec = MIN ((int) -min_prec, -2);
    2639              :         }
    2640          866 :       mp = CEIL (min_prec, limb_prec) * limb_prec;
    2641          866 :       if (mp == 0)
    2642              :         mp = 1;
    2643          866 :       if (mp >= (unsigned) TYPE_PRECISION (TREE_TYPE (op))
    2644          866 :           && (BITINT_TYPE_P (TREE_TYPE (op))
    2645           24 :               || TYPE_PRECISION (TREE_TYPE (op)) <= limb_prec))
    2646          316 :         type = TREE_TYPE (op);
    2647              :       else
    2648          550 :         type = build_bitint_type (mp, 1);
    2649           20 :       if (!BITINT_TYPE_P (type)
    2650          866 :           || bitint_precision_kind (type) == bitint_prec_small)
    2651              :         {
    2652          587 :           if (TYPE_PRECISION (type) <= limb_prec)
    2653          587 :             type = m_limb_type;
    2654              :           else
    2655              :             {
    2656            0 :               while (bitint_precision_kind (mp) == bitint_prec_small)
    2657            0 :                 mp += limb_prec;
    2658              :               /* This case is for targets which e.g. have 64-bit
    2659              :                  limb but categorize up to 128-bits _BitInts as
    2660              :                  small.  We could use type of m_limb_type[2] and
    2661              :                  similar instead to save space.  */
    2662            0 :               type = build_bitint_type (mp, 1);
    2663              :             }
    2664              :         }
    2665          866 :       if (tree_int_cst_sgn (op) >= 0)
    2666          651 :         precs = MAX (TYPE_PRECISION (type), 1);
    2667              :       else
    2668          215 :         precs = MIN ((int) -TYPE_PRECISION (type), -2);
    2669          866 :       if (prec_stored)
    2670            0 :         *prec_stored = precs;
    2671          866 :       op = tree_output_constant_def (fold_convert (type, op));
    2672          866 :       ret = build_fold_addr_expr (op);
    2673          866 :       goto do_ret;
    2674            0 :     default:
    2675            0 :       gcc_unreachable ();
    2676              :     }
    2677         3868 : do_ret:
    2678         3868 :   if (bitint_big_endian && prec_stored == NULL)
    2679              :     {
    2680            0 :       int p1 = *prec < 0 ? -*prec : *prec;
    2681            0 :       int p2 = precs < 0 ? -precs : precs;
    2682            0 :       int c1 = CEIL (p1, limb_prec);
    2683            0 :       int c2 = CEIL (p2, limb_prec);
    2684            0 :       gcc_assert (c1 <= c2);
    2685            0 :       if (c1 != c2)
    2686              :         {
    2687            0 :           gimple *g
    2688            0 :             = gimple_build_assign (make_ssa_name (TREE_TYPE (ret)),
    2689              :                                    POINTER_PLUS_EXPR, ret,
    2690            0 :                                    size_int ((c2 - c1) * m_limb_size));
    2691            0 :           insert_before (g);
    2692            0 :           ret = gimple_assign_lhs (g);
    2693              :         }
    2694              :     }
    2695         3868 :   return ret;
    2696         3868 : }
    2697              : 
    2698              : /* Helper function, create a loop before the current location,
    2699              :    start with sizetype INIT value from the preheader edge.  Return
    2700              :    a PHI result and set *IDX_NEXT to SSA_NAME it creates and uses
    2701              :    from the latch edge.  */
    2702              : 
    2703              : tree
    2704        14962 : bitint_large_huge::create_loop (tree init, tree *idx_next)
    2705              : {
    2706        14962 :   if (!gsi_end_p (m_gsi))
    2707        16000 :     gsi_prev (&m_gsi);
    2708              :   else
    2709         4528 :     m_gsi = gsi_last_bb (gsi_bb (m_gsi));
    2710        14962 :   edge e1 = split_block (gsi_bb (m_gsi), gsi_stmt (m_gsi));
    2711        14962 :   edge e2 = split_block (e1->dest, (gimple *) NULL);
    2712        14962 :   edge e3 = make_edge (e1->dest, e1->dest, EDGE_TRUE_VALUE);
    2713        14962 :   e3->probability = profile_probability::very_unlikely ();
    2714        14962 :   e2->flags = EDGE_FALSE_VALUE;
    2715        14962 :   e2->probability = e3->probability.invert ();
    2716        14962 :   tree idx = make_ssa_name (sizetype);
    2717        14962 :   gphi *phi = create_phi_node (idx, e1->dest);
    2718        14962 :   add_phi_arg (phi, init, e1, UNKNOWN_LOCATION);
    2719        14962 :   *idx_next = make_ssa_name (sizetype);
    2720        14962 :   add_phi_arg (phi, *idx_next, e3, UNKNOWN_LOCATION);
    2721        14962 :   m_gsi = gsi_after_labels (e1->dest);
    2722        14962 :   m_bb = e1->dest;
    2723        14962 :   m_preheader_bb = e1->src;
    2724        14962 :   class loop *loop = alloc_loop ();
    2725        14962 :   loop->header = e1->dest;
    2726        14962 :   add_loop (loop, e1->src->loop_father);
    2727        14962 :   return idx;
    2728              : }
    2729              : 
    2730              : /* Lower large/huge _BitInt statement mergeable or similar STMT which can be
    2731              :    lowered using iteration from the least significant limb up to the most
    2732              :    significant limb.  For large _BitInt it is emitted as straight line code
    2733              :    before current location, for huge _BitInt as a loop handling two limbs
    2734              :    at once, followed by handling up to limbs in straight line code (at most
    2735              :    one full and one partial limb).  It can also handle EQ_EXPR/NE_EXPR
    2736              :    comparisons, in that case CMP_CODE should be the comparison code and
    2737              :    CMP_OP1/CMP_OP2 the comparison operands.  */
    2738              : 
    2739              : tree
    2740        23609 : bitint_large_huge::lower_mergeable_stmt (gimple *stmt, tree_code &cmp_code,
    2741              :                                          tree cmp_op1, tree cmp_op2)
    2742              : {
    2743        23609 :   bool eq_p = cmp_code != ERROR_MARK;
    2744        23609 :   tree type;
    2745        23609 :   if (eq_p)
    2746         6665 :     type = TREE_TYPE (cmp_op1);
    2747              :   else
    2748        16944 :     type = TREE_TYPE (gimple_assign_lhs (stmt));
    2749        23609 :   gcc_assert (BITINT_TYPE_P (type));
    2750        23609 :   bitint_prec_kind kind = bitint_precision_kind (type);
    2751        23609 :   gcc_assert (kind >= bitint_prec_large);
    2752        23609 :   gimple *g;
    2753        23609 :   tree lhs = gimple_get_lhs (stmt);
    2754        23609 :   tree rhs1, lhs_type = lhs ? TREE_TYPE (lhs) : NULL_TREE;
    2755        17384 :   if (lhs
    2756        17384 :       && TREE_CODE (lhs) == SSA_NAME
    2757         8815 :       && BITINT_TYPE_P (TREE_TYPE (lhs))
    2758         8375 :       && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
    2759              :     {
    2760         8375 :       int p = var_to_partition (m_map, lhs);
    2761         8375 :       gcc_assert (m_vars[p] != NULL_TREE);
    2762         8375 :       m_lhs = lhs = m_vars[p];
    2763              :     }
    2764        23609 :   unsigned cnt, rem = 0, end = 0, prec = TYPE_PRECISION (type);
    2765        23609 :   bool sext = false;
    2766        23609 :   tree ext = NULL_TREE, store_operand = NULL_TREE;
    2767        23609 :   bool eh = false;
    2768        23609 :   basic_block eh_pad = NULL;
    2769        23609 :   tree nlhs = NULL_TREE;
    2770        23609 :   unsigned HOST_WIDE_INT bo_idx = 0;
    2771        23609 :   unsigned HOST_WIDE_INT bo_bit = 0;
    2772        23609 :   unsigned bo_shift = 0;
    2773        23609 :   unsigned bo_last = 0;
    2774        23609 :   bool bo_be_p = false;
    2775        23609 :   tree bf_cur = NULL_TREE, bf_next = NULL_TREE;
    2776        23609 :   if (gimple_store_p (stmt))
    2777              :     {
    2778         8569 :       store_operand = gimple_assign_rhs1 (stmt);
    2779         8569 :       eh = stmt_ends_bb_p (stmt);
    2780         8569 :       if (eh)
    2781              :         {
    2782            2 :           edge e;
    2783            2 :           edge_iterator ei;
    2784            2 :           basic_block bb = gimple_bb (stmt);
    2785              : 
    2786            2 :           FOR_EACH_EDGE (e, ei, bb->succs)
    2787            2 :             if (e->flags & EDGE_EH)
    2788              :               {
    2789            2 :                 eh_pad = e->dest;
    2790            2 :                 break;
    2791              :               }
    2792              :         }
    2793         8569 :       if (TREE_CODE (lhs) == COMPONENT_REF
    2794         8569 :           && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
    2795              :         {
    2796          162 :           tree fld = TREE_OPERAND (lhs, 1);
    2797          162 :           gcc_assert (tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld)));
    2798          162 :           tree repr = DECL_BIT_FIELD_REPRESENTATIVE (fld);
    2799          162 :           poly_int64 bitoffset;
    2800          162 :           poly_uint64 field_offset, repr_offset;
    2801          162 :           if (!bitint_big_endian
    2802          162 :               && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
    2803          162 :                   % BITS_PER_UNIT) == 0)
    2804              :             nlhs = lhs;
    2805              :           else
    2806              :             {
    2807          143 :               bool var_field_off = false;
    2808          143 :               if (poly_int_tree_p (DECL_FIELD_OFFSET (fld), &field_offset)
    2809          286 :                   && poly_int_tree_p (DECL_FIELD_OFFSET (repr), &repr_offset))
    2810          143 :                 bitoffset = (field_offset - repr_offset) * BITS_PER_UNIT;
    2811              :               else
    2812              :                 {
    2813              :                   bitoffset = 0;
    2814              :                   var_field_off = true;
    2815              :                 }
    2816          143 :               bitoffset += (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
    2817          143 :                             - tree_to_uhwi (DECL_FIELD_BIT_OFFSET (repr)));
    2818          286 :               nlhs = build3 (COMPONENT_REF, TREE_TYPE (repr),
    2819          143 :                              TREE_OPERAND (lhs, 0), repr,
    2820              :                              var_field_off
    2821            0 :                              ? TREE_OPERAND (lhs, 2) : NULL_TREE);
    2822          143 :               HOST_WIDE_INT bo = bitoffset.to_constant ();
    2823          143 :               bo_idx = (unsigned HOST_WIDE_INT) bo / limb_prec;
    2824          143 :               bo_bit = (unsigned HOST_WIDE_INT) bo % limb_prec;
    2825          143 :               bo_shift = bo_bit;
    2826          143 :               if (bitint_big_endian)
    2827              :                 {
    2828            0 :                   bo_last = CEIL (prec + bo_bit, limb_prec) - 1;
    2829            0 :                   bo_shift = (prec + bo_bit) % limb_prec;
    2830            0 :                   bo_be_p = true;
    2831            0 :                   if (bo_shift)
    2832            0 :                     bo_shift = limb_prec - bo_shift;
    2833              :                 }
    2834              :             }
    2835              :         }
    2836              :     }
    2837         8569 :   if ((store_operand
    2838         8569 :        && TREE_CODE (store_operand) == SSA_NAME
    2839         7581 :        && (m_names == NULL
    2840         7551 :            || !bitmap_bit_p (m_names, SSA_NAME_VERSION (store_operand)))
    2841         1164 :        && gimple_assign_cast_p (SSA_NAME_DEF_STMT (store_operand)))
    2842        31622 :       || gimple_assign_cast_p (stmt))
    2843              :     {
    2844         2169 :       rhs1 = gimple_assign_rhs1 (store_operand
    2845          556 :                                  ? SSA_NAME_DEF_STMT (store_operand)
    2846              :                                  : stmt);
    2847         1613 :       if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
    2848            2 :         rhs1 = TREE_OPERAND (rhs1, 0);
    2849              :       /* Optimize mergeable ops ending with widening cast to _BitInt
    2850              :          (or followed by store).  We can lower just the limbs of the
    2851              :          cast operand and widen afterwards.  */
    2852         1613 :       if (TREE_CODE (rhs1) == SSA_NAME
    2853         1613 :           && (m_names == NULL
    2854         1607 :               || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
    2855          629 :           && BITINT_TYPE_P (TREE_TYPE (rhs1))
    2856          480 :           && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
    2857         2044 :           && (CEIL ((unsigned) TYPE_PRECISION (TREE_TYPE (rhs1)),
    2858          431 :                     limb_prec) < CEIL (prec, limb_prec)
    2859          306 :               || (kind == bitint_prec_huge
    2860          262 :                   && TYPE_PRECISION (TREE_TYPE (rhs1)) < prec)))
    2861              :         {
    2862          131 :           store_operand = rhs1;
    2863          131 :           prec = TYPE_PRECISION (TREE_TYPE (rhs1));
    2864          131 :           kind = bitint_precision_kind (TREE_TYPE (rhs1));
    2865          131 :           if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
    2866           54 :             sext = true;
    2867              :         }
    2868              :     }
    2869        23609 :   tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
    2870        23609 :   if (kind == bitint_prec_large)
    2871        12301 :     cnt = CEIL (prec, limb_prec);
    2872              :   else
    2873              :     {
    2874        11308 :       rem = (prec % (2 * limb_prec));
    2875        11308 :       end = (prec - rem) / limb_prec;
    2876        11308 :       cnt = 2 + CEIL (rem, limb_prec);
    2877        11308 :       idx = idx_first = create_loop (bitint_big_endian
    2878        11308 :                                      ? size_int (cnt - 2 + end - 1)
    2879              :                                      : size_zero_node, &idx_next);
    2880              :     }
    2881              : 
    2882        23609 :   basic_block edge_bb = NULL;
    2883        23609 :   if (eq_p)
    2884              :     {
    2885         6665 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    2886         6665 :       gsi_prev (&gsi);
    2887         6665 :       edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
    2888         6665 :       edge_bb = e->src;
    2889         6665 :       if (kind == bitint_prec_large)
    2890         7264 :         m_gsi = gsi_end_bb (edge_bb);
    2891              :     }
    2892              :   else
    2893        16944 :     m_after_stmt = stmt;
    2894        23609 :   if (kind != bitint_prec_large)
    2895        11308 :     m_upwards_2limb = end;
    2896        23609 :   m_upwards = true;
    2897              : 
    2898        23609 :   bool separate_ext
    2899        23609 :     = (prec != (unsigned) TYPE_PRECISION (type)
    2900        23609 :        && (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
    2901          131 :            > CEIL (prec, limb_prec)));
    2902        23609 :   bool zero_ms_limb = false;
    2903        23609 :   if (bitint_extended == bitint_ext_full
    2904            0 :       && !eq_p
    2905            0 :       && !nlhs
    2906            0 :       && abi_limb_prec > limb_prec
    2907        23609 :       && ((CEIL ((unsigned) TYPE_PRECISION (type), abi_limb_prec)
    2908            0 :            * abi_limb_prec / limb_prec)
    2909            0 :           > CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)))
    2910              :     {
    2911            0 :       if (prec == (unsigned) TYPE_PRECISION (type))
    2912              :         {
    2913            0 :           sext = !TYPE_UNSIGNED (type);
    2914            0 :           separate_ext = true;
    2915              :         }
    2916            0 :       else if (TYPE_UNSIGNED (type) && sext)
    2917              :         zero_ms_limb = true;
    2918              :       else
    2919              :         separate_ext = true;
    2920              :     }
    2921        23734 :   unsigned dst_idx_off = 0;
    2922        23609 :   if (separate_ext && bitint_big_endian)
    2923            0 :     dst_idx_off = (CEIL ((unsigned) TYPE_PRECISION (type), limb_prec)
    2924            0 :                    - CEIL (prec, limb_prec));
    2925              : 
    2926        94635 :   for (unsigned i = 0; i < cnt; i++)
    2927              :     {
    2928        71026 :       m_data_cnt = 0;
    2929        71026 :       if (kind == bitint_prec_large)
    2930        37517 :         idx = size_int (bitint_big_endian ? cnt - 1 - i : i);
    2931        33509 :       else if (i >= 2)
    2932        10893 :         idx = size_int (bitint_big_endian ? cnt - 1 - i : end + (i > 2));
    2933        71026 :       if (eq_p)
    2934              :         {
    2935        20164 :           rhs1 = handle_operand (cmp_op1, idx);
    2936        20164 :           tree rhs2 = handle_operand (cmp_op2, idx);
    2937        20164 :           g = gimple_build_cond (NE_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
    2938        20164 :           insert_before (g);
    2939        20164 :           edge e1 = split_block (gsi_bb (m_gsi), g);
    2940        20164 :           e1->flags = EDGE_FALSE_VALUE;
    2941        20164 :           edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
    2942        20164 :           e1->probability = profile_probability::unlikely ();
    2943        20164 :           e2->probability = e1->probability.invert ();
    2944        20164 :           if (i == 0)
    2945         6665 :             set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
    2946        20164 :           m_gsi = gsi_after_labels (e1->dest);
    2947              :         }
    2948              :       else
    2949              :         {
    2950        50862 :           if (store_operand)
    2951        25930 :             rhs1 = handle_operand (store_operand, idx);
    2952              :           else
    2953        24932 :             rhs1 = handle_stmt (stmt, idx);
    2954        50862 :           if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
    2955        12155 :             rhs1 = add_cast (m_limb_type, rhs1);
    2956        50862 :           if (sext && i == cnt - 1)
    2957        50862 :             ext = rhs1;
    2958        50862 :           tree nidx = idx;
    2959        50862 :           HOST_WIDE_INT adj = bo_idx;
    2960        50862 :           if (bo_be_p)
    2961            0 :             adj += bo_last - (CEIL (prec, limb_prec) - 1);
    2962              :           else
    2963        50862 :             adj += dst_idx_off;
    2964        50862 :           if (adj)
    2965              :             {
    2966          220 :               if (tree_fits_uhwi_p (idx))
    2967          116 :                 nidx = size_int (tree_to_uhwi (idx) + adj);
    2968              :               else
    2969              :                 {
    2970          104 :                   nidx = make_ssa_name (sizetype);
    2971          104 :                   g = gimple_build_assign (nidx, PLUS_EXPR, idx,
    2972          104 :                                            size_int (adj));
    2973          104 :                   insert_before (g);
    2974              :                 }
    2975              :             }
    2976        50862 :           bool done = false;
    2977        50862 :           basic_block new_bb = NULL;
    2978              :           /* Handle stores into bit-fields.  */
    2979        50862 :           if (bo_shift)
    2980              :             {
    2981          455 :               if (i == 0)
    2982              :                 {
    2983          143 :                   edge e2 = NULL;
    2984          143 :                   if (kind != bitint_prec_large)
    2985              :                     {
    2986          112 :                       prepare_data_in_out (build_zero_cst (m_limb_type),
    2987              :                                            idx, &bf_next);
    2988          112 :                       bf_next = m_data.pop ();
    2989          112 :                       bf_cur = m_data.pop ();
    2990          112 :                       g = gimple_build_cond (EQ_EXPR, idx,
    2991              :                                              bitint_big_endian
    2992            0 :                                              ? size_int (CEIL (prec,
    2993              :                                                                limb_prec) - 1)
    2994              :                                              : size_zero_node,
    2995              :                                              NULL_TREE, NULL_TREE);
    2996          112 :                       edge edge_true;
    2997          112 :                       if_then_else (g, profile_probability::unlikely (),
    2998              :                                     edge_true, e2);
    2999          112 :                       new_bb = e2->dest;
    3000              :                     }
    3001          143 :                   tree ftype
    3002          143 :                     = build_nonstandard_integer_type (limb_prec - bo_shift, 1);
    3003          286 :                   tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
    3004          143 :                                                   limb_prec - bo_shift,
    3005              :                                                   bitint_big_endian
    3006            0 :                                                   ? (bo_idx + bo_last)
    3007            0 :                                                     * limb_prec
    3008          143 :                                                   : bo_idx * limb_prec
    3009              :                                                     + bo_bit);
    3010          143 :                   tree t = add_cast (ftype, rhs1);
    3011          143 :                   g = gimple_build_assign (bfr, t);
    3012          143 :                   insert_before (g);
    3013          143 :                   if (eh)
    3014              :                     {
    3015            0 :                       maybe_duplicate_eh_stmt (g, stmt);
    3016            0 :                       if (eh_pad)
    3017              :                         {
    3018            0 :                           edge e = split_block (gsi_bb (m_gsi), g);
    3019            0 :                           m_gsi = gsi_after_labels (e->dest);
    3020            0 :                           add_eh_edge (e->src,
    3021              :                                        find_edge (gimple_bb (stmt), eh_pad));
    3022              :                         }
    3023              :                     }
    3024          143 :                   if (kind == bitint_prec_large)
    3025              :                     {
    3026              :                       bf_cur = rhs1;
    3027              :                       done = true;
    3028              :                     }
    3029          112 :                   else if (e2)
    3030          112 :                     m_gsi = gsi_after_labels (e2->src);
    3031              :                 }
    3032          112 :               if (!done)
    3033              :                 {
    3034          424 :                   tree t1 = make_ssa_name (m_limb_type);
    3035          424 :                   tree t2 = make_ssa_name (m_limb_type);
    3036          424 :                   tree t3 = make_ssa_name (m_limb_type);
    3037          424 :                   g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
    3038              :                                            build_int_cst (unsigned_type_node,
    3039          424 :                                                           limb_prec
    3040          424 :                                                           - bo_shift));
    3041          424 :                   insert_before (g);
    3042          424 :                   g = gimple_build_assign (t2, LSHIFT_EXPR, rhs1,
    3043              :                                            build_int_cst (unsigned_type_node,
    3044          424 :                                                           bo_shift));
    3045          424 :                   insert_before (g);
    3046          424 :                   bf_cur = rhs1;
    3047          424 :                   g = gimple_build_assign (t3, BIT_IOR_EXPR, t1, t2);
    3048          424 :                   insert_before (g);
    3049          424 :                   rhs1 = t3;
    3050          424 :                   if (bf_next && i == 1)
    3051              :                     {
    3052          112 :                       g = gimple_build_assign (bf_next, bf_cur);
    3053          112 :                       insert_before (g);
    3054              :                     }
    3055              :                 }
    3056              :             }
    3057           31 :           if (!done)
    3058              :             {
    3059              :               /* Handle bit-field access to partial last limb if needed.  */
    3060        50831 :               if (nlhs
    3061          480 :                   && i == cnt - 1
    3062          162 :                   && !separate_ext
    3063          138 :                   && tree_fits_uhwi_p (idx))
    3064              :                 {
    3065          123 :                   unsigned int tprec = TYPE_PRECISION (type);
    3066          123 :                   unsigned int rprec = (tprec - 1) % limb_prec + 1;
    3067          123 :                   if (rprec + bo_shift < (unsigned) limb_prec)
    3068              :                     {
    3069           46 :                       tree ftype
    3070           46 :                         = build_nonstandard_integer_type (rprec + bo_shift, 1);
    3071           46 :                       tree bfr
    3072           92 :                         = build_bit_field_ref (ftype, unshare_expr (nlhs),
    3073              :                                                rprec + bo_shift,
    3074              :                                                bitint_big_endian
    3075            0 :                                                ? bo_idx * limb_prec + bo_bit
    3076           46 :                                                : (bo_idx + tprec / limb_prec)
    3077           46 :                                                  * limb_prec);
    3078           46 :                       tree t = add_cast (ftype, rhs1);
    3079           46 :                       g = gimple_build_assign (bfr, t);
    3080           46 :                       done = true;
    3081           46 :                       bf_cur = NULL_TREE;
    3082              :                     }
    3083           77 :                   else if (rprec + bo_shift == (unsigned) limb_prec)
    3084        50785 :                     bf_cur = NULL_TREE;
    3085              :                 }
    3086              :               /* Otherwise, stores to any other lhs.  */
    3087           46 :               if (!done)
    3088              :                 {
    3089        51219 :                   tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
    3090              :                                         nlhs ? nlhs : lhs, nidx, true);
    3091        50785 :                   g = gimple_build_assign (l, rhs1);
    3092              :                 }
    3093        50831 :               insert_before (g);
    3094        50831 :               if (eh)
    3095              :                 {
    3096            6 :                   maybe_duplicate_eh_stmt (g, stmt);
    3097            6 :                   if (eh_pad)
    3098              :                     {
    3099            6 :                       edge e = split_block (gsi_bb (m_gsi), g);
    3100            6 :                       m_gsi = gsi_after_labels (e->dest);
    3101            6 :                       add_eh_edge (e->src,
    3102              :                                    find_edge (gimple_bb (stmt), eh_pad));
    3103              :                     }
    3104              :                 }
    3105        50831 :               if (new_bb)
    3106          112 :                 m_gsi = gsi_after_labels (new_bb);
    3107              :             }
    3108              :         }
    3109        71026 :       m_first = false;
    3110        71026 :       if (kind == bitint_prec_huge && i <= 1)
    3111              :         {
    3112        22616 :           if (i == 0)
    3113              :             {
    3114        11308 :               idx = make_ssa_name (sizetype);
    3115        11308 :               g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
    3116              :                                        bitint_big_endian
    3117            0 :                                        ? size_int (-1) : size_one_node);
    3118        11308 :               insert_before (g);
    3119              :             }
    3120              :           else
    3121              :             {
    3122        11308 :               g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
    3123        22616 :                                        size_int (bitint_big_endian ? -2 : 2));
    3124        11308 :               insert_before (g);
    3125        11308 :               if (bitint_big_endian)
    3126            0 :                 g = gimple_build_cond (NE_EXPR, idx_first, size_int (cnt - 1),
    3127              :                                        NULL_TREE, NULL_TREE);
    3128              :               else
    3129        11308 :                 g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
    3130              :                                        NULL_TREE, NULL_TREE);
    3131        11308 :               insert_before (g);
    3132        11308 :               if (eq_p)
    3133         3033 :                 m_gsi = gsi_after_labels (edge_bb);
    3134              :               else
    3135         8275 :                 m_gsi = gsi_for_stmt (stmt);
    3136        11308 :               m_bb = NULL;
    3137              :             }
    3138              :         }
    3139              :     }
    3140              : 
    3141        23609 :   if (separate_ext)
    3142              :     {
    3143          125 :       if (sext)
    3144              :         {
    3145           51 :           ext = add_cast (signed_type_for (m_limb_type), ext);
    3146          102 :           tree lpm1 = build_int_cst (unsigned_type_node,
    3147           51 :                                      limb_prec - 1);
    3148           51 :           tree n = make_ssa_name (TREE_TYPE (ext));
    3149           51 :           g = gimple_build_assign (n, RSHIFT_EXPR, ext, lpm1);
    3150           51 :           insert_before (g);
    3151           51 :           ext = add_cast (m_limb_type, n);
    3152              :         }
    3153              :       else
    3154           74 :         ext = build_zero_cst (m_limb_type);
    3155          125 :       kind = bitint_precision_kind (type);
    3156          125 :       unsigned start = CEIL (prec, limb_prec);
    3157          125 :       prec = TYPE_PRECISION (type);
    3158          125 :       if (bitint_extended == bitint_ext_full
    3159            0 :           && !nlhs
    3160            0 :           && !zero_ms_limb
    3161            0 :           && abi_limb_prec > limb_prec)
    3162              :         {
    3163            0 :           prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
    3164            0 :           kind = bitint_precision_kind (prec);
    3165              :         }
    3166          125 :       unsigned total = CEIL (prec, limb_prec);
    3167          125 :       idx = idx_first = idx_next = NULL_TREE;
    3168          125 :       if (prec <= (start + 2 + (bo_shift != 0)) * limb_prec)
    3169              :         kind = bitint_prec_large;
    3170           80 :       if (kind == bitint_prec_large)
    3171           45 :         cnt = total - start;
    3172              :       else
    3173              :         {
    3174           80 :           rem = prec % limb_prec;
    3175           80 :           end = (prec - rem) / limb_prec;
    3176          152 :           cnt = (bo_shift != 0) + 1 + (rem != 0);
    3177              :         }
    3178          125 :       if (bitint_big_endian && bo_shift != 0 && (prec % limb_prec) == 0)
    3179            0 :         ++total;
    3180          321 :       for (unsigned i = 0; i < cnt; i++)
    3181              :         {
    3182          196 :           if (kind == bitint_prec_large || (i == 0 && bo_shift != 0))
    3183           62 :             idx = size_int (bo_idx
    3184              :                             + (bitint_big_endian
    3185              :                                ? total - 1 - start - i : start + i));
    3186          134 :           else if (i == cnt - 1 && rem != 0)
    3187          108 :             idx = size_int (bo_idx + (bitint_big_endian ? 0 : end));
    3188           80 :           else if (i == (bo_shift != 0))
    3189           80 :             idx = create_loop (size_int (bo_idx
    3190              :                                          + (bitint_big_endian
    3191              :                                             ? total - 1 - start - i
    3192              :                                             : start + i)), &idx_next);
    3193          196 :           rhs1 = ext;
    3194          196 :           if (bf_cur != NULL_TREE && bf_cur != ext)
    3195              :             {
    3196           20 :               tree t1 = make_ssa_name (m_limb_type);
    3197           20 :               g = gimple_build_assign (t1, RSHIFT_EXPR, bf_cur,
    3198              :                                        build_int_cst (unsigned_type_node,
    3199           20 :                                                       limb_prec - bo_shift));
    3200           20 :               insert_before (g);
    3201           20 :               if (integer_zerop (ext))
    3202              :                 rhs1 = t1;
    3203              :               else
    3204              :                 {
    3205           10 :                   tree t2 = make_ssa_name (m_limb_type);
    3206           10 :                   rhs1 = make_ssa_name (m_limb_type);
    3207           10 :                   g = gimple_build_assign (t2, LSHIFT_EXPR, ext,
    3208              :                                            build_int_cst (unsigned_type_node,
    3209           10 :                                                           bo_shift));
    3210           10 :                   insert_before (g);
    3211           10 :                   g = gimple_build_assign (rhs1, BIT_IOR_EXPR, t1, t2);
    3212           10 :                   insert_before (g);
    3213              :                 }
    3214              :               bf_cur = ext;
    3215              :             }
    3216          196 :           bool done = false;
    3217              :           /* Handle bit-field access to partial last limb if needed.  */
    3218          196 :           if (nlhs && i == cnt - 1)
    3219              :             {
    3220           24 :               unsigned int tprec = TYPE_PRECISION (type);
    3221           24 :               unsigned int rprec = (tprec - 1) % limb_prec + 1;
    3222           24 :               if (rprec + bo_shift < (unsigned) limb_prec)
    3223              :                 {
    3224           12 :                   tree ftype
    3225           12 :                     = build_nonstandard_integer_type (rprec + bo_shift, 1);
    3226           12 :                   tree bfr
    3227           24 :                     = build_bit_field_ref (ftype, unshare_expr (nlhs),
    3228              :                                            rprec + bo_shift,
    3229              :                                            bitint_big_endian
    3230            0 :                                            ? bo_idx * limb_prec + bo_bit
    3231           12 :                                            : (bo_idx + tprec / limb_prec)
    3232           12 :                                              * limb_prec);
    3233           12 :                   tree t = add_cast (ftype, rhs1);
    3234           12 :                   g = gimple_build_assign (bfr, t);
    3235           12 :                   done = true;
    3236           12 :                   bf_cur = NULL_TREE;
    3237              :                 }
    3238           12 :               else if (rprec + bo_shift == (unsigned) limb_prec)
    3239              :                 bf_cur = NULL_TREE;
    3240              :             }
    3241              :           /* Otherwise, stores to any other lhs.  */
    3242           12 :           if (!done)
    3243              :             {
    3244          214 :               tree l = limb_access (nlhs ? NULL_TREE : lhs_type,
    3245              :                                     nlhs ? nlhs : lhs, idx, true);
    3246              : 
    3247          184 :               if (bitint_extended
    3248            0 :                   && sext
    3249            0 :                   && TYPE_UNSIGNED (lhs_type)
    3250            0 :                   && tree_fits_uhwi_p (idx)
    3251          184 :                   && !nlhs)
    3252              :                 {
    3253            0 :                   rhs1 = add_cast (limb_access_type (lhs_type, idx), rhs1);
    3254            0 :                   rhs1 = add_cast (TREE_TYPE (l), rhs1);
    3255              :                 }
    3256              : 
    3257          184 :               g = gimple_build_assign (l, rhs1);
    3258              :             }
    3259          196 :           insert_before (g);
    3260          196 :           if (eh)
    3261              :             {
    3262            0 :               maybe_duplicate_eh_stmt (g, stmt);
    3263            0 :               if (eh_pad)
    3264              :                 {
    3265            0 :                   edge e = split_block (gsi_bb (m_gsi), g);
    3266            0 :                   m_gsi = gsi_after_labels (e->dest);
    3267            0 :                   add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
    3268              :                 }
    3269              :             }
    3270          196 :           if (kind == bitint_prec_huge && i == (bo_shift != 0))
    3271              :             {
    3272           80 :               g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    3273              :                                        bitint_big_endian
    3274            0 :                                        ? size_int (-1) : size_one_node);
    3275           80 :               insert_before (g);
    3276              :               /* For big-endian, if bo_idx + total - 1 - end is all ones, then
    3277              :                  compare idx (which is equal to idx_next + 1) against 0
    3278              :                  instead.  */
    3279           80 :               if (bitint_big_endian && bo_idx + total - end == 0)
    3280            0 :                 g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
    3281              :                                        NULL_TREE, NULL_TREE);
    3282              :               else
    3283           80 :                 g = gimple_build_cond (NE_EXPR, idx_next,
    3284           80 :                                        size_int (bo_idx
    3285              :                                                  + (bitint_big_endian
    3286              :                                                     ? total - 1 - end
    3287              :                                                     : end)),
    3288              :                                        NULL_TREE, NULL_TREE);
    3289           80 :               insert_before (g);
    3290           80 :               m_gsi = gsi_for_stmt (stmt);
    3291           80 :               m_bb = NULL;
    3292              :             }
    3293              :         }
    3294              :     }
    3295        23609 :   if (bf_cur != NULL_TREE)
    3296              :     {
    3297           72 :       unsigned int tprec = TYPE_PRECISION (type);
    3298           72 :       unsigned int rprec = (tprec + bo_shift) % limb_prec;
    3299           72 :       tree ftype = build_nonstandard_integer_type (rprec, 1);
    3300          144 :       tree bfr = build_bit_field_ref (ftype, unshare_expr (nlhs),
    3301              :                                       rprec,
    3302              :                                       bitint_big_endian
    3303            0 :                                       ? bo_idx * limb_prec + bo_bit
    3304           72 :                                       : (bo_idx + (tprec + bo_bit) / limb_prec)
    3305              :                                         * limb_prec);
    3306           72 :       rhs1 = bf_cur;
    3307           72 :       if (bf_cur != ext)
    3308              :         {
    3309           64 :           rhs1 = make_ssa_name (TREE_TYPE (rhs1));
    3310           64 :           g = gimple_build_assign (rhs1, RSHIFT_EXPR, bf_cur,
    3311              :                                    build_int_cst (unsigned_type_node,
    3312           64 :                                                   limb_prec - bo_shift));
    3313           64 :           insert_before (g);
    3314              :         }
    3315           72 :       rhs1 = add_cast (ftype, rhs1);
    3316           72 :       g = gimple_build_assign (bfr, rhs1);
    3317           72 :       insert_before (g);
    3318           72 :       if (eh)
    3319              :         {
    3320            0 :           maybe_duplicate_eh_stmt (g, stmt);
    3321            0 :           if (eh_pad)
    3322              :             {
    3323            0 :               edge e = split_block (gsi_bb (m_gsi), g);
    3324            0 :               m_gsi = gsi_after_labels (e->dest);
    3325            0 :               add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
    3326              :             }
    3327              :         }
    3328              :     }
    3329        23609 :   if (zero_ms_limb)
    3330              :     {
    3331            0 :       tree p2 = build_int_cst (sizetype,
    3332            0 :                                CEIL ((unsigned) TYPE_PRECISION (type),
    3333              :                                      abi_limb_prec)
    3334            0 :                                * abi_limb_prec / limb_prec - 1);
    3335            0 :       tree l = limb_access (lhs_type, lhs, p2, true);
    3336            0 :       g = gimple_build_assign (l, build_zero_cst (m_limb_type));
    3337            0 :       insert_before (g);
    3338            0 :       if (eh)
    3339              :         {
    3340            0 :           maybe_duplicate_eh_stmt (g, stmt);
    3341            0 :           if (eh_pad)
    3342              :             {
    3343            0 :               edge e = split_block (gsi_bb (m_gsi), g);
    3344            0 :               m_gsi = gsi_after_labels (e->dest);
    3345            0 :               add_eh_edge (e->src, find_edge (gimple_bb (stmt), eh_pad));
    3346              :             }
    3347              :         }
    3348              :     }
    3349              : 
    3350        23609 :   if (gimple_store_p (stmt))
    3351              :     {
    3352         8569 :       unlink_stmt_vdef (stmt);
    3353        17138 :       release_ssa_name (gimple_vdef (stmt));
    3354         8569 :       gsi_remove (&m_gsi, true);
    3355              :     }
    3356        23609 :   if (eq_p)
    3357              :     {
    3358         6665 :       lhs = make_ssa_name (boolean_type_node);
    3359         6665 :       basic_block bb = gimple_bb (stmt);
    3360         6665 :       gphi *phi = create_phi_node (lhs, bb);
    3361         6665 :       edge e = find_edge (gsi_bb (m_gsi), bb);
    3362         6665 :       unsigned int n = EDGE_COUNT (bb->preds);
    3363        33494 :       for (unsigned int i = 0; i < n; i++)
    3364              :         {
    3365        26829 :           edge e2 = EDGE_PRED (bb, i);
    3366        26829 :           add_phi_arg (phi, e == e2 ? boolean_true_node : boolean_false_node,
    3367              :                        e2, UNKNOWN_LOCATION);
    3368              :         }
    3369         6665 :       cmp_code = cmp_code == EQ_EXPR ? NE_EXPR : EQ_EXPR;
    3370         6665 :       return lhs;
    3371              :     }
    3372              :   else
    3373              :     return NULL_TREE;
    3374              : }
    3375              : 
    3376              : /* Handle a large/huge _BitInt comparison statement STMT other than
    3377              :    EQ_EXPR/NE_EXPR.  CMP_CODE, CMP_OP1 and CMP_OP2 meaning is like in
    3378              :    lower_mergeable_stmt.  The {GT,GE,LT,LE}_EXPR comparisons are
    3379              :    lowered by iteration from the most significant limb downwards to
    3380              :    the least significant one, for large _BitInt in straight line code,
    3381              :    otherwise with most significant limb handled in
    3382              :    straight line code followed by a loop handling one limb at a time.
    3383              :    Comparisons with unsigned huge _BitInt with precisions which are
    3384              :    multiples of limb precision can use just the loop and don't need to
    3385              :    handle most significant limb before the loop.  The loop or straight
    3386              :    line code jumps to final basic block if a particular pair of limbs
    3387              :    is not equal.  */
    3388              : 
    3389              : tree
    3390          722 : bitint_large_huge::lower_comparison_stmt (gimple *stmt, tree_code &cmp_code,
    3391              :                                           tree cmp_op1, tree cmp_op2)
    3392              : {
    3393          722 :   tree type = TREE_TYPE (cmp_op1);
    3394          722 :   gcc_assert (BITINT_TYPE_P (type));
    3395          722 :   bitint_prec_kind kind = bitint_precision_kind (type);
    3396          722 :   gcc_assert (kind >= bitint_prec_large);
    3397          722 :   gimple *g;
    3398          722 :   if (!TYPE_UNSIGNED (type)
    3399          441 :       && integer_zerop (cmp_op2)
    3400          750 :       && (cmp_code == GE_EXPR || cmp_code == LT_EXPR))
    3401              :     {
    3402           28 :       unsigned end = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec) - 1;
    3403           56 :       tree idx = size_int (bitint_big_endian ? 0 : end);
    3404           28 :       m_data_cnt = 0;
    3405           28 :       tree rhs1 = handle_operand (cmp_op1, idx);
    3406           28 :       if (TYPE_UNSIGNED (TREE_TYPE (rhs1)))
    3407              :         {
    3408           24 :           tree stype = signed_type_for (TREE_TYPE (rhs1));
    3409           24 :           rhs1 = add_cast (stype, rhs1);
    3410              :         }
    3411           28 :       tree lhs = make_ssa_name (boolean_type_node);
    3412           28 :       g = gimple_build_assign (lhs, cmp_code, rhs1,
    3413           28 :                                build_zero_cst (TREE_TYPE (rhs1)));
    3414           28 :       insert_before (g);
    3415           28 :       cmp_code = NE_EXPR;
    3416           28 :       return lhs;
    3417              :     }
    3418              : 
    3419          694 :   unsigned cnt, rem = 0, end = 0;
    3420          694 :   tree idx = NULL_TREE, idx_next = NULL_TREE;
    3421          694 :   if (kind == bitint_prec_large)
    3422          377 :     cnt = CEIL ((unsigned) TYPE_PRECISION (type), limb_prec);
    3423              :   else
    3424              :     {
    3425          317 :       rem = ((unsigned) TYPE_PRECISION (type) % limb_prec);
    3426          317 :       if (rem == 0 && !TYPE_UNSIGNED (type))
    3427              :         rem = limb_prec;
    3428          317 :       end = ((unsigned) TYPE_PRECISION (type) - rem) / limb_prec;
    3429          317 :       cnt = 1 + (rem != 0);
    3430              :     }
    3431              : 
    3432          694 :   basic_block edge_bb = NULL;
    3433          694 :   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    3434          694 :   gsi_prev (&gsi);
    3435          694 :   edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
    3436          694 :   edge_bb = e->src;
    3437          694 :   m_gsi = gsi_end_bb (edge_bb);
    3438              : 
    3439          694 :   edge *edges = XALLOCAVEC (edge, cnt * 2);
    3440         2519 :   for (unsigned i = 0; i < cnt; i++)
    3441              :     {
    3442         1825 :       m_data_cnt = 0;
    3443         1825 :       if (kind == bitint_prec_large)
    3444         1246 :         idx = size_int (bitint_big_endian ? i : cnt - i - 1);
    3445          579 :       else if (i == cnt - 1)
    3446          317 :         idx = create_loop (size_int (bitint_big_endian ? cnt - 1 : end - 1),
    3447              :                            &idx_next);
    3448              :       else
    3449          524 :         idx = size_int (bitint_big_endian ? 0 : end);
    3450         1825 :       tree rhs1 = handle_operand (cmp_op1, idx);
    3451         1825 :       tree rhs2 = handle_operand (cmp_op2, idx);
    3452         1825 :       if (i == 0
    3453          694 :           && !TYPE_UNSIGNED (type)
    3454         2238 :           && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
    3455              :         {
    3456          112 :           tree stype = signed_type_for (TREE_TYPE (rhs1));
    3457          112 :           rhs1 = add_cast (stype, rhs1);
    3458          112 :           rhs2 = add_cast (stype, rhs2);
    3459              :         }
    3460         1825 :       g = gimple_build_cond (GT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
    3461         1825 :       insert_before (g);
    3462         1825 :       edge e1 = split_block (gsi_bb (m_gsi), g);
    3463         1825 :       e1->flags = EDGE_FALSE_VALUE;
    3464         1825 :       edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
    3465         1825 :       e1->probability = profile_probability::likely ();
    3466         1825 :       e2->probability = e1->probability.invert ();
    3467         1825 :       if (i == 0)
    3468          694 :         set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
    3469         1825 :       m_gsi = gsi_after_labels (e1->dest);
    3470         1825 :       edges[2 * i] = e2;
    3471         1825 :       g = gimple_build_cond (LT_EXPR, rhs1, rhs2, NULL_TREE, NULL_TREE);
    3472         1825 :       insert_before (g);
    3473         1825 :       e1 = split_block (gsi_bb (m_gsi), g);
    3474         1825 :       e1->flags = EDGE_FALSE_VALUE;
    3475         1825 :       e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
    3476         1825 :       e1->probability = profile_probability::unlikely ();
    3477         1825 :       e2->probability = e1->probability.invert ();
    3478         1825 :       m_gsi = gsi_after_labels (e1->dest);
    3479         1825 :       edges[2 * i + 1] = e2;
    3480         1825 :       m_first = false;
    3481         1825 :       if (kind == bitint_prec_huge && i == cnt - 1)
    3482              :         {
    3483          634 :           g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    3484              :                                    bitint_big_endian ? size_one_node
    3485          317 :                                    : size_int (-1));
    3486          317 :           insert_before (g);
    3487          317 :           g = gimple_build_cond (NE_EXPR, idx,
    3488              :                                  bitint_big_endian
    3489            0 :                                  ? size_int (end - 1 + (cnt != 1))
    3490              :                                  : size_zero_node,
    3491              :                                  NULL_TREE, NULL_TREE);
    3492          317 :           insert_before (g);
    3493          317 :           edge true_edge, false_edge;
    3494          317 :           extract_true_false_edges_from_block (gsi_bb (m_gsi),
    3495              :                                                &true_edge, &false_edge);
    3496          317 :           m_gsi = gsi_after_labels (false_edge->dest);
    3497          317 :           m_bb = NULL;
    3498              :         }
    3499              :     }
    3500              : 
    3501          694 :   tree lhs = make_ssa_name (boolean_type_node);
    3502          694 :   basic_block bb = gimple_bb (stmt);
    3503          694 :   gphi *phi = create_phi_node (lhs, bb);
    3504         5038 :   for (unsigned int i = 0; i < cnt * 2; i++)
    3505              :     {
    3506         3650 :       tree val = ((cmp_code == GT_EXPR || cmp_code == GE_EXPR)
    3507         3650 :                   ^ (i & 1)) ? boolean_true_node : boolean_false_node;
    3508         3650 :       add_phi_arg (phi, val, edges[i], UNKNOWN_LOCATION);
    3509              :     }
    3510          694 :   add_phi_arg (phi, (cmp_code == GE_EXPR || cmp_code == LE_EXPR)
    3511              :                     ? boolean_true_node : boolean_false_node,
    3512              :                find_edge (gsi_bb (m_gsi), bb), UNKNOWN_LOCATION);
    3513          694 :   cmp_code = NE_EXPR;
    3514          694 :   return lhs;
    3515              : }
    3516              : 
    3517              : /* Lower large/huge _BitInt left and right shift except for left
    3518              :    shift by < limb_prec constant.  */
    3519              : 
    3520              : void
    3521          602 : bitint_large_huge::lower_shift_stmt (tree obj, gimple *stmt)
    3522              : {
    3523          602 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    3524          602 :   tree lhs = gimple_assign_lhs (stmt);
    3525          602 :   tree_code rhs_code = gimple_assign_rhs_code (stmt);
    3526          602 :   tree type = TREE_TYPE (rhs1);
    3527          602 :   gimple *final_stmt = gsi_stmt (m_gsi);
    3528          602 :   gcc_assert (BITINT_TYPE_P (type)
    3529              :               && bitint_precision_kind (type) >= bitint_prec_large);
    3530          602 :   int prec = TYPE_PRECISION (type);
    3531          602 :   tree n = gimple_assign_rhs2 (stmt), n1, n2, n3, n4;
    3532          602 :   gimple *g;
    3533          602 :   if (obj == NULL_TREE)
    3534              :     {
    3535          483 :       int part = var_to_partition (m_map, lhs);
    3536          483 :       gcc_assert (m_vars[part] != NULL_TREE);
    3537              :       obj = m_vars[part];
    3538              :     }
    3539              :   /* Preparation code common for both left and right shifts.
    3540              :      unsigned n1 = n % limb_prec;
    3541              :      size_t n2 = n / limb_prec;
    3542              :      size_t n3 = n1 != 0;
    3543              :      unsigned n4 = (limb_prec - n1) % limb_prec;
    3544              :      (for power of 2 limb_prec n4 can be -n1 & limb_prec).  */
    3545          602 :   if (TREE_CODE (n) == INTEGER_CST)
    3546              :     {
    3547          267 :       tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
    3548          267 :       n1 = int_const_binop (TRUNC_MOD_EXPR, n, lp);
    3549          267 :       n2 = fold_convert (sizetype, int_const_binop (TRUNC_DIV_EXPR, n, lp));
    3550          267 :       n3 = size_int (!integer_zerop (n1));
    3551          267 :       n4 = int_const_binop (TRUNC_MOD_EXPR,
    3552          267 :                             int_const_binop (MINUS_EXPR, lp, n1), lp);
    3553              :     }
    3554              :   else
    3555              :     {
    3556          335 :       n1 = make_ssa_name (TREE_TYPE (n));
    3557          335 :       n2 = make_ssa_name (sizetype);
    3558          335 :       n3 = make_ssa_name (sizetype);
    3559          335 :       n4 = make_ssa_name (TREE_TYPE (n));
    3560          335 :       if (pow2p_hwi (limb_prec))
    3561              :         {
    3562          335 :           tree lpm1 = build_int_cst (TREE_TYPE (n), limb_prec - 1);
    3563          335 :           g = gimple_build_assign (n1, BIT_AND_EXPR, n, lpm1);
    3564          335 :           insert_before (g);
    3565          989 :           g = gimple_build_assign (useless_type_conversion_p (sizetype,
    3566          335 :                                                               TREE_TYPE (n))
    3567          319 :                                    ? n2 : make_ssa_name (TREE_TYPE (n)),
    3568              :                                    RSHIFT_EXPR, n,
    3569          335 :                                    build_int_cst (TREE_TYPE (n),
    3570          670 :                                                   exact_log2 (limb_prec)));
    3571          335 :           insert_before (g);
    3572          335 :           if (gimple_assign_lhs (g) != n2)
    3573              :             {
    3574          319 :               g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
    3575          319 :               insert_before (g);
    3576              :             }
    3577          335 :           g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
    3578              :                                    NEGATE_EXPR, n1);
    3579          335 :           insert_before (g);
    3580          335 :           g = gimple_build_assign (n4, BIT_AND_EXPR, gimple_assign_lhs (g),
    3581              :                                    lpm1);
    3582          335 :           insert_before (g);
    3583              :         }
    3584              :       else
    3585              :         {
    3586            0 :           tree lp = build_int_cst (TREE_TYPE (n), limb_prec);
    3587            0 :           g = gimple_build_assign (n1, TRUNC_MOD_EXPR, n, lp);
    3588            0 :           insert_before (g);
    3589            0 :           g = gimple_build_assign (useless_type_conversion_p (sizetype,
    3590            0 :                                                               TREE_TYPE (n))
    3591            0 :                                    ? n2 : make_ssa_name (TREE_TYPE (n)),
    3592              :                                    TRUNC_DIV_EXPR, n, lp);
    3593            0 :           insert_before (g);
    3594            0 :           if (gimple_assign_lhs (g) != n2)
    3595              :             {
    3596            0 :               g = gimple_build_assign (n2, NOP_EXPR, gimple_assign_lhs (g));
    3597            0 :               insert_before (g);
    3598              :             }
    3599            0 :           g = gimple_build_assign (make_ssa_name (TREE_TYPE (n)),
    3600              :                                    MINUS_EXPR, lp, n1);
    3601            0 :           insert_before (g);
    3602            0 :           g = gimple_build_assign (n4, TRUNC_MOD_EXPR, gimple_assign_lhs (g),
    3603              :                                    lp);
    3604            0 :           insert_before (g);
    3605              :         }
    3606          335 :       g = gimple_build_assign (make_ssa_name (boolean_type_node), NE_EXPR, n1,
    3607          335 :                                build_zero_cst (TREE_TYPE (n)));
    3608          335 :       insert_before (g);
    3609          335 :       g = gimple_build_assign (n3, NOP_EXPR, gimple_assign_lhs (g));
    3610          335 :       insert_before (g);
    3611              :     }
    3612         1204 :   tree p = build_int_cst (sizetype,
    3613          602 :                           prec / limb_prec - (prec % limb_prec == 0));
    3614          602 :   if (rhs_code == RSHIFT_EXPR)
    3615              :     {
    3616              :       /* Lower
    3617              :            dst = src >> n;
    3618              :          as
    3619              :            unsigned n1 = n % limb_prec;
    3620              :            size_t n2 = n / limb_prec;
    3621              :            size_t n3 = n1 != 0;
    3622              :            unsigned n4 = (limb_prec - n1) % limb_prec;
    3623              :            size_t idx;
    3624              :            size_t p = prec / limb_prec - (prec % limb_prec == 0);
    3625              :            int signed_p = (typeof (src) -1) < 0;
    3626              :            for (idx = n2; idx < ((!signed_p && (prec % limb_prec == 0))
    3627              :                                  ? p : p - n3); ++idx)
    3628              :              dst[idx - n2] = (src[idx] >> n1) | (src[idx + n3] << n4);
    3629              :            limb_type ext;
    3630              :            if (prec % limb_prec == 0)
    3631              :              ext = src[p];
    3632              :            else if (signed_p)
    3633              :              ext = ((signed limb_type) (src[p] << (limb_prec
    3634              :                                                    - (prec % limb_prec))))
    3635              :                    >> (limb_prec - (prec % limb_prec));
    3636              :            else
    3637              :              ext = src[p] & (((limb_type) 1 << (prec % limb_prec)) - 1);
    3638              :            if (!signed_p && (prec % limb_prec == 0))
    3639              :              ;
    3640              :            else if (idx < prec / 64)
    3641              :              {
    3642              :                dst[idx - n2] = (src[idx] >> n1) | (ext << n4);
    3643              :                ++idx;
    3644              :              }
    3645              :            idx -= n2;
    3646              :            if (signed_p)
    3647              :              {
    3648              :                dst[idx] = ((signed limb_type) ext) >> n1;
    3649              :                ext = ((signed limb_type) ext) >> (limb_prec - 1);
    3650              :              }
    3651              :            else
    3652              :              {
    3653              :                dst[idx] = ext >> n1;
    3654              :                ext = 0;
    3655              :              }
    3656              :            for (++idx; idx <= p; ++idx)
    3657              :              dst[idx] = ext;  */
    3658          405 :       tree pmn3;
    3659          405 :       if (TYPE_UNSIGNED (type) && prec % limb_prec == 0)
    3660          104 :         pmn3 = bitint_big_endian ? size_zero_node : p;
    3661          301 :       else if (bitint_big_endian)
    3662              :         pmn3 = n3;
    3663          301 :       else if (TREE_CODE (n3) == INTEGER_CST)
    3664          128 :         pmn3 = int_const_binop (MINUS_EXPR, p, n3);
    3665              :       else
    3666              :         {
    3667          173 :           pmn3 = make_ssa_name (sizetype);
    3668          173 :           g = gimple_build_assign (pmn3, MINUS_EXPR, p, n3);
    3669          173 :           insert_before (g);
    3670              :         }
    3671          405 :       tree pmn2 = NULL_TREE;
    3672          405 :       if (bitint_big_endian)
    3673              :         {
    3674            0 :           if (TREE_CODE (n2) == INTEGER_CST)
    3675            0 :             pmn2 = int_const_binop (MINUS_EXPR, p, n2);
    3676              :           else
    3677              :             {
    3678            0 :               pmn2 = make_ssa_name (sizetype);
    3679            0 :               g = gimple_build_assign (pmn2, MINUS_EXPR, p, n2);
    3680            0 :               insert_before (g);
    3681              :             }
    3682            0 :           g = gimple_build_cond (GT_EXPR, pmn2, pmn3, NULL_TREE, NULL_TREE);
    3683              :         }
    3684              :       else
    3685          405 :         g = gimple_build_cond (LT_EXPR, n2, pmn3, NULL_TREE, NULL_TREE);
    3686          405 :       edge edge_true, edge_false;
    3687          405 :       if_then (g, profile_probability::likely (), edge_true, edge_false);
    3688          405 :       tree idx_next;
    3689          810 :       tree idx = create_loop (bitint_big_endian ? pmn2 : n2, &idx_next);
    3690          405 :       tree idxmn2 = make_ssa_name (sizetype);
    3691          405 :       tree idxpn3 = make_ssa_name (sizetype);
    3692          810 :       g = gimple_build_assign (idxmn2,
    3693              :                                bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
    3694              :                                idx, n2);
    3695          405 :       insert_before (g);
    3696          810 :       g = gimple_build_assign (idxpn3,
    3697              :                                bitint_big_endian ? MINUS_EXPR : PLUS_EXPR,
    3698              :                                idx, n3);
    3699          405 :       insert_before (g);
    3700          405 :       m_data_cnt = 0;
    3701          405 :       tree t1 = handle_operand (rhs1, idx);
    3702          405 :       m_first = false;
    3703          405 :       g = gimple_build_assign (make_ssa_name (m_limb_type),
    3704              :                                RSHIFT_EXPR, t1, n1);
    3705          405 :       insert_before (g);
    3706          405 :       t1 = gimple_assign_lhs (g);
    3707          405 :       if (!integer_zerop (n3))
    3708              :         {
    3709          317 :           m_data_cnt = 0;
    3710          317 :           tree t2 = handle_operand (rhs1, idxpn3);
    3711          317 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    3712              :                                    LSHIFT_EXPR, t2, n4);
    3713          317 :           insert_before (g);
    3714          317 :           t2 = gimple_assign_lhs (g);
    3715          317 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    3716              :                                    BIT_IOR_EXPR, t1, t2);
    3717          317 :           insert_before (g);
    3718          317 :           t1 = gimple_assign_lhs (g);
    3719              :         }
    3720          405 :       tree l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
    3721          405 :       g = gimple_build_assign (l, t1);
    3722          405 :       insert_before (g);
    3723          405 :       g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    3724            0 :                                bitint_big_endian ? size_int (-1)
    3725              :                                                  : size_one_node);
    3726          405 :       insert_before (g);
    3727          810 :       g = gimple_build_cond (bitint_big_endian ? GT_EXPR : LT_EXPR,
    3728              :                              idx_next, pmn3, NULL_TREE, NULL_TREE);
    3729          405 :       insert_before (g);
    3730          405 :       idx = make_ssa_name (sizetype);
    3731          405 :       m_gsi = gsi_for_stmt (final_stmt);
    3732          405 :       gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
    3733          405 :       edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
    3734          405 :       edge_true = EDGE_PRED (gsi_bb (m_gsi),
    3735              :                              EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
    3736          810 :       add_phi_arg (phi, bitint_big_endian ? pmn2 : n2, edge_false,
    3737              :                    UNKNOWN_LOCATION);
    3738          405 :       add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
    3739          405 :       m_data_cnt = 0;
    3740          405 :       tree ms = handle_operand (rhs1, bitint_big_endian ? size_zero_node : p);
    3741          405 :       tree ext = ms;
    3742          405 :       if (!types_compatible_p (TREE_TYPE (ms), m_limb_type))
    3743          244 :         ext = add_cast (m_limb_type, ms);
    3744          623 :       if (!(TYPE_UNSIGNED (type) && prec % limb_prec == 0)
    3745          519 :           && !integer_zerop (n3))
    3746              :         {
    3747          263 :           if (bitint_big_endian)
    3748            0 :             g = gimple_build_cond (GT_EXPR, idx, size_zero_node,
    3749              :                                    NULL_TREE, NULL_TREE);
    3750              :           else
    3751          263 :             g = gimple_build_cond (LT_EXPR, idx, p, NULL_TREE, NULL_TREE);
    3752          263 :           if_then (g, profile_probability::likely (), edge_true, edge_false);
    3753          263 :           m_data_cnt = 0;
    3754          263 :           t1 = handle_operand (rhs1, idx);
    3755          263 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    3756              :                                    RSHIFT_EXPR, t1, n1);
    3757          263 :           insert_before (g);
    3758          263 :           t1 = gimple_assign_lhs (g);
    3759          263 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    3760              :                                    LSHIFT_EXPR, ext, n4);
    3761          263 :           insert_before (g);
    3762          263 :           tree t2 = gimple_assign_lhs (g);
    3763          263 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    3764              :                                    BIT_IOR_EXPR, t1, t2);
    3765          263 :           insert_before (g);
    3766          263 :           t1 = gimple_assign_lhs (g);
    3767          263 :           idxmn2 = make_ssa_name (sizetype);
    3768          526 :           g = gimple_build_assign (idxmn2, bitint_big_endian
    3769              :                                            ? PLUS_EXPR : MINUS_EXPR, idx, n2);
    3770          263 :           insert_before (g);
    3771          263 :           l = limb_access (TREE_TYPE (lhs), obj, idxmn2, true);
    3772          263 :           g = gimple_build_assign (l, t1);
    3773          263 :           insert_before (g);
    3774          263 :           idx_next = make_ssa_name (sizetype);
    3775          263 :           g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    3776              :                                    bitint_big_endian
    3777            0 :                                    ? size_int (-1) : size_one_node);
    3778          263 :           insert_before (g);
    3779          263 :           m_gsi = gsi_for_stmt (final_stmt);
    3780          263 :           tree nidx = make_ssa_name (sizetype);
    3781          263 :           phi = create_phi_node (nidx, gsi_bb (m_gsi));
    3782          263 :           edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
    3783          263 :           edge_true = EDGE_PRED (gsi_bb (m_gsi),
    3784              :                                  EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
    3785          263 :           add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
    3786          263 :           add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
    3787          263 :           idx = nidx;
    3788              :         }
    3789          810 :       g = gimple_build_assign (make_ssa_name (sizetype),
    3790              :                                bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
    3791              :                                idx, n2);
    3792          405 :       insert_before (g);
    3793          405 :       idx = gimple_assign_lhs (g);
    3794          405 :       tree sext = ext;
    3795          405 :       if (!TYPE_UNSIGNED (type))
    3796          187 :         sext = add_cast (signed_type_for (m_limb_type), ext);
    3797          405 :       g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
    3798              :                                RSHIFT_EXPR, sext, n1);
    3799          405 :       insert_before (g);
    3800          405 :       t1 = gimple_assign_lhs (g);
    3801          405 :       if (!TYPE_UNSIGNED (type))
    3802              :         {
    3803          187 :           t1 = add_cast (m_limb_type, t1);
    3804          187 :           g = gimple_build_assign (make_ssa_name (TREE_TYPE (sext)),
    3805              :                                    RSHIFT_EXPR, sext,
    3806          187 :                                    build_int_cst (TREE_TYPE (n),
    3807          187 :                                                   limb_prec - 1));
    3808          187 :           insert_before (g);
    3809          187 :           ext = add_cast (m_limb_type, gimple_assign_lhs (g));
    3810              :         }
    3811              :       else
    3812          218 :         ext = build_zero_cst (m_limb_type);
    3813          405 :       l = limb_access (TREE_TYPE (lhs), obj, idx, true);
    3814          405 :       g = gimple_build_assign (l, t1);
    3815          405 :       insert_before (g);
    3816          405 :       g = gimple_build_assign (make_ssa_name (sizetype), PLUS_EXPR, idx,
    3817              :                                bitint_big_endian
    3818            0 :                                ? size_int (-1) : size_one_node);
    3819          405 :       insert_before (g);
    3820          405 :       tree p2 = p;
    3821          405 :       if (bitint_big_endian)
    3822              :         {
    3823            0 :           tree new_idx = gimple_assign_lhs (g);
    3824            0 :           g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
    3825              :                                  NULL_TREE, NULL_TREE);
    3826            0 :           idx = new_idx;
    3827              :         }
    3828              :       else
    3829              :         {
    3830          405 :           if (bitint_extended == bitint_ext_full
    3831            0 :               && abi_limb_prec > limb_prec)
    3832            0 :             p2 = build_int_cst (sizetype,
    3833            0 :                                 CEIL (prec, abi_limb_prec)
    3834            0 :                                 * abi_limb_prec / limb_prec - 1);
    3835          405 :           idx = gimple_assign_lhs (g);
    3836          405 :           g = gimple_build_cond (LE_EXPR, idx, p2, NULL_TREE, NULL_TREE);
    3837              :         }
    3838          405 :       if_then (g, profile_probability::likely (), edge_true, edge_false);
    3839          405 :       idx = create_loop (idx, &idx_next);
    3840          405 :       l = limb_access (TREE_TYPE (lhs), obj, idx, true);
    3841          405 :       g = gimple_build_assign (l, ext);
    3842          405 :       insert_before (g);
    3843          405 :       g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    3844              :                                bitint_big_endian
    3845            0 :                                ? size_int (-1) : size_one_node);
    3846          405 :       insert_before (g);
    3847          405 :       if (bitint_big_endian)
    3848            0 :         g = gimple_build_cond (NE_EXPR, idx, size_zero_node,
    3849              :                                NULL_TREE, NULL_TREE);
    3850              :       else
    3851          405 :         g = gimple_build_cond (LE_EXPR, idx_next, p2, NULL_TREE, NULL_TREE);
    3852          405 :       insert_before (g);
    3853              :     }
    3854              :   else
    3855              :     {
    3856              :       /* Lower
    3857              :            dst = src << n;
    3858              :          as
    3859              :            unsigned n1 = n % limb_prec;
    3860              :            size_t n2 = n / limb_prec;
    3861              :            size_t n3 = n1 != 0;
    3862              :            unsigned n4 = (limb_prec - n1) % limb_prec;
    3863              :            size_t idx;
    3864              :            size_t p = prec / limb_prec - (prec % limb_prec == 0);
    3865              :            for (idx = p; (ssize_t) idx >= (ssize_t) (n2 + n3); --idx)
    3866              :              dst[idx] = (src[idx - n2] << n1) | (src[idx - n2 - n3] >> n4);
    3867              :            if (n1)
    3868              :              {
    3869              :                dst[idx] = src[idx - n2] << n1;
    3870              :                --idx;
    3871              :              }
    3872              :            for (; (ssize_t) idx >= 0; --idx)
    3873              :              dst[idx] = 0;  */
    3874          197 :       tree n2pn3;
    3875          197 :       if (TREE_CODE (n2) == INTEGER_CST && TREE_CODE (n3) == INTEGER_CST)
    3876           71 :         n2pn3 = int_const_binop (PLUS_EXPR, n2, n3);
    3877              :       else
    3878              :         {
    3879          126 :           n2pn3 = make_ssa_name (sizetype);
    3880          126 :           g = gimple_build_assign (n2pn3, PLUS_EXPR, n2, n3);
    3881          126 :           insert_before (g);
    3882              :         }
    3883          197 :       if (bitint_big_endian)
    3884              :         {
    3885            0 :           if (TREE_CODE (n2pn3) == INTEGER_CST)
    3886            0 :             n2pn3 = int_const_binop (MINUS_EXPR, p, n2pn3);
    3887              :           else
    3888              :             {
    3889            0 :               g = gimple_build_assign (make_ssa_name (sizetype),
    3890              :                                        MINUS_EXPR, p, n2pn3);
    3891            0 :               insert_before (g);
    3892            0 :               n2pn3 = gimple_assign_lhs (g);
    3893              :             }
    3894              :         }
    3895              :       /* For LSHIFT_EXPR, we can use handle_operand with non-INTEGER_CST
    3896              :          idx even to access the most significant partial limb.  */
    3897          197 :       m_var_msb = true;
    3898          197 :       if (integer_zerop (n3))
    3899              :         /* For n3 == 0 p >= n2 + n3 is always true for all valid shift
    3900              :            counts.  Emit if (true) condition that can be optimized later.  */
    3901           45 :         g = gimple_build_cond (NE_EXPR, boolean_true_node, boolean_false_node,
    3902              :                                NULL_TREE, NULL_TREE);
    3903          152 :       else if (bitint_big_endian)
    3904            0 :         g = gimple_build_cond (NE_EXPR, n2pn3, size_int (-1), NULL_TREE,
    3905              :                                NULL_TREE);
    3906              :       else
    3907          152 :         g = gimple_build_cond (LE_EXPR, n2pn3, p, NULL_TREE, NULL_TREE);
    3908          197 :       edge edge_true, edge_false;
    3909          197 :       if_then (g, profile_probability::likely (), edge_true, edge_false);
    3910          197 :       tree idx_next;
    3911          197 :       tree idx = create_loop (bitint_big_endian ? size_zero_node : p,
    3912              :                               &idx_next);
    3913          197 :       tree idxmn2 = make_ssa_name (sizetype);
    3914          197 :       tree idxmn2mn3 = make_ssa_name (sizetype);
    3915          394 :       g = gimple_build_assign (idxmn2,
    3916              :                                bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
    3917              :                                idx, n2);
    3918          197 :       insert_before (g);
    3919          394 :       g = gimple_build_assign (idxmn2mn3,
    3920              :                                bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
    3921              :                                idxmn2, n3);
    3922          197 :       insert_before (g);
    3923          197 :       m_data_cnt = 0;
    3924          197 :       tree t1 = handle_operand (rhs1, idxmn2);
    3925          197 :       m_first = false;
    3926          197 :       g = gimple_build_assign (make_ssa_name (m_limb_type),
    3927              :                                LSHIFT_EXPR, t1, n1);
    3928          197 :       insert_before (g);
    3929          197 :       t1 = gimple_assign_lhs (g);
    3930          197 :       if (!integer_zerop (n3))
    3931              :         {
    3932          152 :           m_data_cnt = 0;
    3933          152 :           tree t2 = handle_operand (rhs1, idxmn2mn3);
    3934          152 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    3935              :                                    RSHIFT_EXPR, t2, n4);
    3936          152 :           insert_before (g);
    3937          152 :           t2 = gimple_assign_lhs (g);
    3938          152 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    3939              :                                    BIT_IOR_EXPR, t1, t2);
    3940          152 :           insert_before (g);
    3941          152 :           t1 = gimple_assign_lhs (g);
    3942              :         }
    3943          197 :       tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
    3944          197 :       g = gimple_build_assign (l, t1);
    3945          197 :       insert_before (g);
    3946          394 :       g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    3947              :                                bitint_big_endian
    3948          197 :                                ? size_one_node : size_int (-1));
    3949          197 :       insert_before (g);
    3950          197 :       tree sn2pn3 = add_cast (ssizetype, n2pn3);
    3951          394 :       g = gimple_build_cond (bitint_big_endian ? LE_EXPR : GE_EXPR,
    3952              :                              add_cast (ssizetype, idx_next), sn2pn3,
    3953              :                              NULL_TREE, NULL_TREE);
    3954          197 :       insert_before (g);
    3955          197 :       idx = make_ssa_name (sizetype);
    3956          197 :       m_gsi = gsi_for_stmt (final_stmt);
    3957          197 :       gphi *phi = create_phi_node (idx, gsi_bb (m_gsi));
    3958          197 :       edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
    3959          197 :       edge_true = EDGE_PRED (gsi_bb (m_gsi),
    3960              :                              EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
    3961          197 :       add_phi_arg (phi, bitint_big_endian ? size_zero_node : p,
    3962              :                    edge_false, UNKNOWN_LOCATION);
    3963          197 :       add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
    3964          197 :       m_data_cnt = 0;
    3965          197 :       if (!integer_zerop (n3))
    3966              :         {
    3967          152 :           g = gimple_build_cond (NE_EXPR, n3, size_zero_node,
    3968              :                                  NULL_TREE, NULL_TREE);
    3969          152 :           if_then (g, profile_probability::likely (), edge_true, edge_false);
    3970          152 :           idxmn2 = make_ssa_name (sizetype);
    3971          304 :           g = gimple_build_assign (idxmn2,
    3972              :                                    bitint_big_endian ? PLUS_EXPR : MINUS_EXPR,
    3973              :                                    idx, n2);
    3974          152 :           insert_before (g);
    3975          152 :           m_data_cnt = 0;
    3976          152 :           t1 = handle_operand (rhs1, idxmn2);
    3977          152 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    3978              :                                    LSHIFT_EXPR, t1, n1);
    3979          152 :           insert_before (g);
    3980          152 :           t1 = gimple_assign_lhs (g);
    3981          152 :           l = limb_access (TREE_TYPE (lhs), obj, idx, true);
    3982          152 :           g = gimple_build_assign (l, t1);
    3983          152 :           insert_before (g);
    3984          152 :           idx_next = make_ssa_name (sizetype);
    3985          304 :           g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    3986              :                                    bitint_big_endian
    3987          152 :                                    ? size_one_node : size_int (-1));
    3988          152 :           insert_before (g);
    3989          152 :           m_gsi = gsi_for_stmt (final_stmt);
    3990          152 :           tree nidx = make_ssa_name (sizetype);
    3991          152 :           phi = create_phi_node (nidx, gsi_bb (m_gsi));
    3992          152 :           edge_false = find_edge (edge_false->src, gsi_bb (m_gsi));
    3993          152 :           edge_true = EDGE_PRED (gsi_bb (m_gsi),
    3994              :                                  EDGE_PRED (gsi_bb (m_gsi), 0) == edge_false);
    3995          152 :           add_phi_arg (phi, idx, edge_false, UNKNOWN_LOCATION);
    3996          152 :           add_phi_arg (phi, idx_next, edge_true, UNKNOWN_LOCATION);
    3997          152 :           idx = nidx;
    3998              :         }
    3999          197 :       if (bitint_big_endian)
    4000            0 :         g = gimple_build_cond (LE_EXPR, idx, p, NULL_TREE, NULL_TREE);
    4001              :       else
    4002          197 :         g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx),
    4003              :                                ssize_int (0), NULL_TREE, NULL_TREE);
    4004          197 :       if_then (g, profile_probability::likely (), edge_true, edge_false);
    4005          197 :       idx = create_loop (idx, &idx_next);
    4006          197 :       l = limb_access (TREE_TYPE (lhs), obj, idx, true);
    4007          197 :       g = gimple_build_assign (l, build_zero_cst (m_limb_type));
    4008          197 :       insert_before (g);
    4009          394 :       g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    4010              :                                bitint_big_endian
    4011          197 :                                ? size_one_node : size_int (-1));
    4012          197 :       insert_before (g);
    4013          197 :       if (bitint_big_endian)
    4014            0 :         g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
    4015              :       else
    4016          197 :         g = gimple_build_cond (GE_EXPR, add_cast (ssizetype, idx_next),
    4017              :                                ssize_int (0), NULL_TREE, NULL_TREE);
    4018          197 :       insert_before (g);
    4019          197 :       if (bitint_extended && prec % limb_prec != 0)
    4020              :         {
    4021              :           /* The most significant limb has been updated either in the
    4022              :              loop or in the if after it.  To simplify the code, just
    4023              :              read it back from memory and extend.  */
    4024            0 :           m_gsi = gsi_after_labels (edge_false->dest);
    4025            0 :           idx = bitint_big_endian ? size_zero_node : p;
    4026            0 :           tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
    4027            0 :           tree type = limb_access_type (TREE_TYPE (lhs), idx);
    4028            0 :           tree v = make_ssa_name (m_limb_type);
    4029            0 :           g = gimple_build_assign (v, l);
    4030            0 :           insert_before (g);
    4031            0 :           v = add_cast (type, v);
    4032            0 :           l = limb_access (TREE_TYPE (lhs), obj, idx, true);
    4033            0 :           v = add_cast (m_limb_type, v);
    4034            0 :           g = gimple_build_assign (l, v);
    4035            0 :           insert_before (g);
    4036            0 :           if (bitint_extended == bitint_ext_full
    4037            0 :               && abi_limb_prec > limb_prec
    4038            0 :               && (CEIL (prec, abi_limb_prec) * abi_limb_prec
    4039            0 :                   > CEIL (prec, limb_prec) * limb_prec))
    4040              :             {
    4041            0 :               tree p2 = build_int_cst (sizetype,
    4042              :                                        CEIL (prec, abi_limb_prec)
    4043            0 :                                        * abi_limb_prec / limb_prec - 1);
    4044            0 :               if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
    4045            0 :                 v = build_zero_cst (m_limb_type);
    4046              :               else
    4047              :                 {
    4048            0 :                   v = add_cast (signed_type_for (m_limb_type), v);
    4049            0 :                   g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
    4050              :                                            RSHIFT_EXPR, v,
    4051              :                                            build_int_cst (unsigned_type_node,
    4052            0 :                                                           limb_prec - 1));
    4053            0 :                   insert_before (g);
    4054            0 :                   v = add_cast (m_limb_type, gimple_assign_lhs (g));
    4055              :                 }
    4056            0 :               l = limb_access (TREE_TYPE (lhs), obj, p2, true);
    4057            0 :               g = gimple_build_assign (l, v);
    4058            0 :               insert_before (g);
    4059              :             }
    4060              :         }
    4061          197 :       else if (bitint_extended == bitint_ext_full
    4062            0 :                && abi_limb_prec > limb_prec
    4063            0 :                && (CEIL (prec, abi_limb_prec) * abi_limb_prec
    4064            0 :                    > CEIL (prec, limb_prec) * limb_prec))
    4065              :         {
    4066            0 :           m_gsi = gsi_after_labels (edge_false->dest);
    4067            0 :           tree p2 = build_int_cst (sizetype,
    4068              :                                    CEIL (prec, abi_limb_prec)
    4069            0 :                                    * abi_limb_prec / limb_prec - 1);
    4070            0 :           tree v;
    4071            0 :           if (TYPE_UNSIGNED (TREE_TYPE (lhs)))
    4072            0 :             v = build_zero_cst (m_limb_type);
    4073              :           else
    4074              :             {
    4075            0 :               tree l = limb_access (TREE_TYPE (lhs), obj, p, true);
    4076            0 :               v = make_ssa_name (m_limb_type);
    4077            0 :               g = gimple_build_assign (v, l);
    4078            0 :               insert_before (g);
    4079            0 :               v = add_cast (signed_type_for (m_limb_type), v);
    4080            0 :               g = gimple_build_assign (make_ssa_name (TREE_TYPE (v)),
    4081              :                                        RSHIFT_EXPR, v,
    4082              :                                        build_int_cst (unsigned_type_node,
    4083            0 :                                                       limb_prec - 1));
    4084            0 :               insert_before (g);
    4085            0 :               v = add_cast (m_limb_type, gimple_assign_lhs (g));
    4086              :             }
    4087            0 :           tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
    4088            0 :           g = gimple_build_assign (l, v);
    4089            0 :           insert_before (g);
    4090              :         }
    4091              :     }
    4092          602 : }
    4093              : 
    4094              : /* Lower large/huge _BitInt multiplication or division.  */
    4095              : 
    4096              : void
    4097          373 : bitint_large_huge::lower_muldiv_stmt (tree obj, gimple *stmt)
    4098              : {
    4099          373 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    4100          373 :   tree rhs2 = gimple_assign_rhs2 (stmt);
    4101          373 :   tree lhs = gimple_assign_lhs (stmt);
    4102          373 :   tree_code rhs_code = gimple_assign_rhs_code (stmt);
    4103          373 :   tree type = TREE_TYPE (rhs1);
    4104          373 :   gcc_assert (BITINT_TYPE_P (type)
    4105              :               && bitint_precision_kind (type) >= bitint_prec_large);
    4106          373 :   int prec = TYPE_PRECISION (type), prec1, prec2;
    4107          373 :   bool ext_ms_limb = false;
    4108          373 :   bool do_ext = false;
    4109          373 :   rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec1);
    4110          373 :   rhs2 = handle_operand_addr (rhs2, stmt, NULL, &prec2);
    4111          373 :   if (obj == NULL_TREE)
    4112              :     {
    4113          166 :       int part = var_to_partition (m_map, lhs);
    4114          166 :       gcc_assert (m_vars[part] != NULL_TREE);
    4115          166 :       obj = m_vars[part];
    4116          166 :       lhs = build_fold_addr_expr (obj);
    4117              :     }
    4118              :   else
    4119              :     {
    4120          207 :       lhs = build_fold_addr_expr (obj);
    4121          207 :       lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
    4122              :                                       NULL_TREE, true, GSI_SAME_STMT);
    4123              :     }
    4124          373 :   if (bitint_extended && TYPE_OVERFLOW_WRAPS (type))
    4125              :     {
    4126            0 :       if (rhs_code == MULT_EXPR)
    4127              :         do_ext = true;
    4128              :       /* For signed division with -fwrapv, minimum negative / -1 needs
    4129              :          is minimum negative and the padding bits above it should be all
    4130              :          set.  */
    4131            0 :       else if (!TYPE_UNSIGNED (type)
    4132            0 :                && (rhs_code == TRUNC_DIV_EXPR || rhs_code == EXACT_DIV_EXPR))
    4133          373 :         do_ext = true;
    4134              :     }
    4135          373 :   if (bitint_extended == bitint_ext_full
    4136            0 :       && abi_limb_prec > limb_prec
    4137            0 :       && (CEIL (prec, abi_limb_prec) * abi_limb_prec
    4138            0 :           > CEIL (prec, limb_prec) * limb_prec))
    4139              :     {
    4140              :       /* unsigned multiplication needs to wrap around, so we can't
    4141              :          increase prec.  Similarly for -fwrapv.  */
    4142            0 :       if (do_ext)
    4143              :         ext_ms_limb = true;
    4144              :       else
    4145            0 :         prec = CEIL (prec, abi_limb_prec) * abi_limb_prec;
    4146              :     }
    4147          373 :   tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
    4148          373 :   gimple *g;
    4149          373 :   switch (rhs_code)
    4150              :     {
    4151          210 :     case MULT_EXPR:
    4152          210 :       g = gimple_build_call_internal (IFN_MULBITINT, 6,
    4153          210 :                                       lhs, build_int_cst (sitype, prec),
    4154          210 :                                       rhs1, build_int_cst (sitype, prec1),
    4155          210 :                                       rhs2, build_int_cst (sitype, prec2));
    4156          210 :       insert_before (g);
    4157          210 :       break;
    4158          103 :     case TRUNC_DIV_EXPR:
    4159          103 :     case EXACT_DIV_EXPR:
    4160          103 :       g = gimple_build_call_internal (IFN_DIVMODBITINT, 8,
    4161          103 :                                       lhs, build_int_cst (sitype, prec),
    4162              :                                       null_pointer_node,
    4163              :                                       build_int_cst (sitype, 0),
    4164          103 :                                       rhs1, build_int_cst (sitype, prec1),
    4165          103 :                                       rhs2, build_int_cst (sitype, prec2));
    4166          103 :       if (!stmt_ends_bb_p (stmt))
    4167          102 :         gimple_call_set_nothrow (as_a <gcall *> (g), true);
    4168          103 :       insert_before (g);
    4169          103 :       break;
    4170           60 :     case TRUNC_MOD_EXPR:
    4171           60 :       g = gimple_build_call_internal (IFN_DIVMODBITINT, 8, null_pointer_node,
    4172              :                                       build_int_cst (sitype, 0),
    4173           60 :                                       lhs, build_int_cst (sitype, prec),
    4174           60 :                                       rhs1, build_int_cst (sitype, prec1),
    4175           60 :                                       rhs2, build_int_cst (sitype, prec2));
    4176           60 :       if (!stmt_ends_bb_p (stmt))
    4177           57 :         gimple_call_set_nothrow (as_a <gcall *> (g), true);
    4178           60 :       insert_before (g);
    4179           60 :       break;
    4180            0 :     default:
    4181            0 :       gcc_unreachable ();
    4182              :     }
    4183          373 :   if (stmt_ends_bb_p (stmt))
    4184              :     {
    4185            4 :       maybe_duplicate_eh_stmt (g, stmt);
    4186            4 :       edge e1;
    4187            4 :       edge_iterator ei;
    4188            4 :       basic_block bb = gimple_bb (stmt);
    4189              : 
    4190            4 :       FOR_EACH_EDGE (e1, ei, bb->succs)
    4191            4 :         if (e1->flags & EDGE_EH)
    4192              :           break;
    4193            4 :       if (e1)
    4194              :         {
    4195            4 :           edge e2 = split_block (gsi_bb (m_gsi), g);
    4196            4 :           m_gsi = gsi_after_labels (e2->dest);
    4197            4 :           add_eh_edge (e2->src, e1);
    4198              :         }
    4199              :     }
    4200          373 :   if (do_ext
    4201          373 :       && ((prec % limb_prec) != 0 || (ext_ms_limb && !TYPE_UNSIGNED (type))))
    4202              :     {
    4203              :       /* Unsigned multiplication wraps, but libgcc function will return the
    4204              :          bits beyond prec within the top limb as another limb of the full
    4205              :          multiplication.  So, clear the padding bits here.  */
    4206            0 :       tree idx = size_int (bitint_big_endian ? 0 : prec / limb_prec);
    4207            0 :       tree l = limb_access (type, obj, idx, true);
    4208            0 :       tree ctype = limb_access_type (type, idx);
    4209            0 :       tree v = make_ssa_name (m_limb_type);
    4210            0 :       g = gimple_build_assign (v, l);
    4211            0 :       insert_before (g);
    4212            0 :       tree v2 = v;
    4213            0 :       if ((prec % limb_prec) != 0)
    4214              :         {
    4215            0 :           v = add_cast (ctype, v);
    4216            0 :           l = limb_access (type, obj, idx, true);
    4217            0 :           v = add_cast (m_limb_type, v);
    4218            0 :           v2 = v;
    4219            0 :           g = gimple_build_assign (l, v);
    4220            0 :           insert_before (g);
    4221              :         }
    4222            0 :       if (ext_ms_limb && !TYPE_UNSIGNED (type))
    4223              :         {
    4224            0 :           v2 = add_cast (signed_type_for (m_limb_type), v2);
    4225            0 :           tree lpm1 = build_int_cst (unsigned_type_node, limb_prec - 1);
    4226            0 :           v = make_ssa_name (TREE_TYPE (v2));
    4227            0 :           g = gimple_build_assign (v, RSHIFT_EXPR, v2, lpm1);
    4228            0 :           insert_before (g);
    4229            0 :           unsigned int i
    4230            0 :             = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
    4231            0 :           v = add_cast (m_limb_type, v);
    4232            0 :           g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
    4233              :                                                 true), v);
    4234            0 :           insert_before (g);
    4235            0 :           ext_ms_limb = false;
    4236              :         }
    4237              :     }
    4238          373 :   if (ext_ms_limb)
    4239              :     {
    4240            0 :       unsigned int i = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
    4241            0 :       g = gimple_build_assign (limb_access (type, obj, size_int (i - 1), true),
    4242              :                                build_zero_cst (m_limb_type));
    4243            0 :       insert_before (g);
    4244              :     }
    4245          373 : }
    4246              : 
    4247              : /* Lower large/huge _BitInt conversion to/from floating point.  */
    4248              : 
    4249              : void
    4250          327 : bitint_large_huge::lower_float_conv_stmt (tree obj, gimple *stmt)
    4251              : {
    4252          327 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    4253          327 :   tree lhs = gimple_assign_lhs (stmt);
    4254          327 :   tree_code rhs_code = gimple_assign_rhs_code (stmt);
    4255          327 :   tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
    4256          327 :   gimple *g;
    4257          327 :   if (rhs_code == FIX_TRUNC_EXPR)
    4258              :     {
    4259          179 :       tree type = TREE_TYPE (lhs);
    4260          179 :       int prec = TYPE_PRECISION (type);
    4261          179 :       bool extend_ms_limb = false;
    4262          179 :       if (bitint_extended == bitint_ext_full
    4263            0 :           && abi_limb_prec > limb_prec
    4264            0 :           && (CEIL (prec, abi_limb_prec) * abi_limb_prec
    4265            0 :               > CEIL (prec, limb_prec) * limb_prec))
    4266          179 :         extend_ms_limb = true;
    4267          179 :       if (!TYPE_UNSIGNED (type))
    4268           93 :         prec = -prec;
    4269          179 :       if (obj == NULL_TREE)
    4270              :         {
    4271          135 :           int part = var_to_partition (m_map, lhs);
    4272          135 :           gcc_assert (m_vars[part] != NULL_TREE);
    4273          135 :           obj = m_vars[part];
    4274          135 :           lhs = build_fold_addr_expr (obj);
    4275              :         }
    4276              :       else
    4277              :         {
    4278           44 :           lhs = build_fold_addr_expr (obj);
    4279           44 :           lhs = force_gimple_operand_gsi (&m_gsi, lhs, true,
    4280              :                                           NULL_TREE, true, GSI_SAME_STMT);
    4281              :         }
    4282          179 :       scalar_mode from_mode
    4283          179 :         = as_a <scalar_mode> (TYPE_MODE (TREE_TYPE (rhs1)));
    4284              : #ifdef HAVE_SFmode
    4285              :       /* IEEE single is a full superset of both IEEE half and
    4286              :          bfloat formats, convert to float first and then to _BitInt
    4287              :          to avoid the need of another 2 library routines.  */
    4288          179 :       if ((REAL_MODE_FORMAT (from_mode) == &arm_bfloat_half_format
    4289          179 :            || REAL_MODE_FORMAT (from_mode) == &ieee_half_format)
    4290          191 :           && REAL_MODE_FORMAT (SFmode) == &ieee_single_format)
    4291              :         {
    4292           12 :           tree type = lang_hooks.types.type_for_mode (SFmode, 0);
    4293           12 :           if (type)
    4294           12 :             rhs1 = add_cast (type, rhs1);
    4295              :         }
    4296              : #endif
    4297          179 :       g = gimple_build_call_internal (IFN_FLOATTOBITINT, 3,
    4298          179 :                                       lhs, build_int_cst (sitype, prec),
    4299              :                                       rhs1);
    4300          179 :       insert_before (g);
    4301          179 :       if (extend_ms_limb)
    4302              :         {
    4303            0 :           unsigned int i
    4304            0 :             = (CEIL (prec < 0 ? -prec : prec, abi_limb_prec)
    4305            0 :                * abi_limb_prec / limb_prec);
    4306            0 :           tree val;
    4307            0 :           if (prec < 0)
    4308              :             {
    4309            0 :               g = gimple_build_assign (make_ssa_name (m_limb_type),
    4310              :                                        limb_access (type, obj,
    4311            0 :                                                     size_int (i - 2),
    4312              :                                                     true));
    4313            0 :               insert_before (g);
    4314            0 :               val = add_cast (signed_type_for (m_limb_type),
    4315              :                               gimple_assign_lhs (g));
    4316            0 :               g = gimple_build_assign (make_ssa_name (TREE_TYPE (val)),
    4317              :                                        RSHIFT_EXPR, val,
    4318              :                                        build_int_cst (unsigned_type_node,
    4319            0 :                                                       limb_prec - 1));
    4320            0 :               insert_before (g);
    4321            0 :               val = add_cast (m_limb_type, gimple_assign_lhs (g));
    4322              :             }
    4323              :           else
    4324            0 :             val = build_zero_cst (m_limb_type);
    4325            0 :           g = gimple_build_assign (limb_access (type, obj, size_int (i - 1),
    4326              :                                                 true), val);
    4327            0 :           insert_before (g);
    4328              :         }
    4329              :     }
    4330              :   else
    4331              :     {
    4332          148 :       int prec;
    4333          148 :       rhs1 = handle_operand_addr (rhs1, stmt, NULL, &prec);
    4334          148 :       g = gimple_build_call_internal (IFN_BITINTTOFLOAT, 2,
    4335          148 :                                       rhs1, build_int_cst (sitype, prec));
    4336          148 :       gimple_call_set_lhs (g, lhs);
    4337          148 :       if (!stmt_ends_bb_p (stmt))
    4338          147 :         gimple_call_set_nothrow (as_a <gcall *> (g), true);
    4339          148 :       gsi_replace (&m_gsi, g, true);
    4340              :     }
    4341          327 : }
    4342              : 
    4343              : /* Helper method for lower_addsub_overflow and lower_mul_overflow.
    4344              :    If check_zero is true, caller wants to check if all bits in [start, end)
    4345              :    are zero, otherwise if bits in [start, end) are either all zero or
    4346              :    all ones.  L is the limb with index LIMB, START and END are measured
    4347              :    in bits.  */
    4348              : 
    4349              : tree
    4350         6453 : bitint_large_huge::arith_overflow_extract_bits (unsigned int start,
    4351              :                                                 unsigned int end, tree l,
    4352              :                                                 unsigned int limb,
    4353              :                                                 bool check_zero)
    4354              : {
    4355         6453 :   unsigned startlimb = start / limb_prec;
    4356         6453 :   unsigned endlimb = (end - 1) / limb_prec;
    4357         6453 :   gimple *g;
    4358              : 
    4359         6453 :   if ((start % limb_prec) == 0 && (end % limb_prec) == 0)
    4360              :     return l;
    4361         6184 :   if (startlimb == endlimb && limb == startlimb)
    4362              :     {
    4363         2039 :       if (check_zero)
    4364              :         {
    4365         1486 :           wide_int w = wi::shifted_mask (start % limb_prec,
    4366         1486 :                                          end - start, false, limb_prec);
    4367         2972 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    4368              :                                    BIT_AND_EXPR, l,
    4369         1486 :                                    wide_int_to_tree (m_limb_type, w));
    4370         1486 :           insert_before (g);
    4371         1486 :           return gimple_assign_lhs (g);
    4372         1486 :         }
    4373          553 :       unsigned int shift = start % limb_prec;
    4374          553 :       if ((end % limb_prec) != 0)
    4375              :         {
    4376          336 :           unsigned int lshift = (-end) % limb_prec;
    4377          336 :           shift += lshift;
    4378          336 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    4379              :                                    LSHIFT_EXPR, l,
    4380              :                                    build_int_cst (unsigned_type_node,
    4381          336 :                                                   lshift));
    4382          336 :           insert_before (g);
    4383          336 :           l = gimple_assign_lhs (g);
    4384              :         }
    4385          553 :       l = add_cast (signed_type_for (m_limb_type), l);
    4386          553 :       g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
    4387              :                                RSHIFT_EXPR, l,
    4388          553 :                                build_int_cst (unsigned_type_node, shift));
    4389          553 :       insert_before (g);
    4390          553 :       return add_cast (m_limb_type, gimple_assign_lhs (g));
    4391              :     }
    4392         4145 :   else if (limb == startlimb)
    4393              :     {
    4394         1990 :       if ((start % limb_prec) == 0)
    4395              :         return l;
    4396         1876 :       if (!check_zero)
    4397          950 :         l = add_cast (signed_type_for (m_limb_type), l);
    4398         1876 :       g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
    4399              :                                RSHIFT_EXPR, l,
    4400              :                                build_int_cst (unsigned_type_node,
    4401         1876 :                                               start % limb_prec));
    4402         1876 :       insert_before (g);
    4403         1876 :       l = gimple_assign_lhs (g);
    4404         1876 :       if (!check_zero)
    4405          950 :         l = add_cast (m_limb_type, l);
    4406              :       return l;
    4407              :     }
    4408         2155 :   else if (limb == endlimb)
    4409              :     {
    4410         1704 :       if ((end % limb_prec) == 0)
    4411              :         return l;
    4412         1687 :       if (check_zero)
    4413              :         {
    4414          880 :           wide_int w = wi::mask (end % limb_prec, false, limb_prec);
    4415         1760 :           g = gimple_build_assign (make_ssa_name (m_limb_type),
    4416              :                                    BIT_AND_EXPR, l,
    4417          880 :                                    wide_int_to_tree (m_limb_type, w));
    4418          880 :           insert_before (g);
    4419          880 :           return gimple_assign_lhs (g);
    4420          880 :         }
    4421          807 :       unsigned int shift = (-end) % limb_prec;
    4422          807 :       g = gimple_build_assign (make_ssa_name (m_limb_type),
    4423              :                                LSHIFT_EXPR, l,
    4424          807 :                                build_int_cst (unsigned_type_node, shift));
    4425          807 :       insert_before (g);
    4426          807 :       l = add_cast (signed_type_for (m_limb_type), gimple_assign_lhs (g));
    4427          807 :       g = gimple_build_assign (make_ssa_name (TREE_TYPE (l)),
    4428              :                                RSHIFT_EXPR, l,
    4429          807 :                                build_int_cst (unsigned_type_node, shift));
    4430          807 :       insert_before (g);
    4431          807 :       return add_cast (m_limb_type, gimple_assign_lhs (g));
    4432              :     }
    4433              :   return l;
    4434              : }
    4435              : 
    4436              : /* Helper method for lower_addsub_overflow and lower_mul_overflow.  Store
    4437              :    result including overflow flag into the right locations.  */
    4438              : 
    4439              : void
    4440         4212 : bitint_large_huge::finish_arith_overflow (tree var, tree obj, tree type,
    4441              :                                           tree ovf, tree lhs, tree orig_obj,
    4442              :                                           gimple *stmt, unsigned nelts,
    4443              :                                           tree_code code)
    4444              : {
    4445         4212 :   gimple *g;
    4446              : 
    4447         4212 :   if (obj == NULL_TREE
    4448         4212 :       && (!BITINT_TYPE_P (type)
    4449          231 :           || bitint_precision_kind (type) < bitint_prec_large))
    4450              :     {
    4451              :       /* Add support for 3 or more limbs filled in from normal integral
    4452              :          type if this assert fails.  If no target chooses limb mode smaller
    4453              :          than half of largest supported normal integral type, this will not
    4454              :          be needed.  */
    4455          250 :       gcc_assert (TYPE_PRECISION (type) <= 2 * limb_prec);
    4456          250 :       tree lhs_type = type;
    4457          110 :       if (BITINT_TYPE_P (type)
    4458          250 :           && bitint_precision_kind (type) == bitint_prec_middle)
    4459           46 :         lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (type),
    4460           46 :                                                    TYPE_UNSIGNED (type));
    4461          250 :       tree r1 = limb_access (NULL_TREE, var,
    4462              :                              bitint_big_endian
    4463            0 :                              ? size_int (nelts - 1) : size_zero_node, true);
    4464          250 :       g = gimple_build_assign (make_ssa_name (m_limb_type), r1);
    4465          250 :       insert_before (g);
    4466          250 :       r1 = gimple_assign_lhs (g);
    4467          250 :       if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
    4468          250 :         r1 = add_cast (lhs_type, r1);
    4469          250 :       if (TYPE_PRECISION (lhs_type) > limb_prec)
    4470              :         {
    4471           90 :           tree r2 = limb_access (NULL_TREE, var,
    4472              :                                  bitint_big_endian
    4473            0 :                                  ? size_int (nelts - 2) : size_one_node, true);
    4474           90 :           g = gimple_build_assign (make_ssa_name (m_limb_type), r2);
    4475           90 :           insert_before (g);
    4476           90 :           r2 = gimple_assign_lhs (g);
    4477           90 :           r2 = add_cast (lhs_type, r2);
    4478           90 :           g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
    4479              :                                    build_int_cst (unsigned_type_node,
    4480           90 :                                                   limb_prec));
    4481           90 :           insert_before (g);
    4482           90 :           g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
    4483              :                                    gimple_assign_lhs (g));
    4484           90 :           insert_before (g);
    4485           90 :           r1 = gimple_assign_lhs (g);
    4486              :         }
    4487          250 :       if (lhs_type != type)
    4488           46 :         r1 = add_cast (type, r1);
    4489          250 :       ovf = add_cast (lhs_type, ovf);
    4490          250 :       if (lhs_type != type)
    4491           46 :         ovf = add_cast (type, ovf);
    4492          250 :       g = gimple_build_assign (lhs, COMPLEX_EXPR, r1, ovf);
    4493          250 :       m_gsi = gsi_for_stmt (stmt);
    4494          250 :       gsi_replace (&m_gsi, g, true);
    4495              :     }
    4496              :   else
    4497              :     {
    4498         3962 :       unsigned HOST_WIDE_INT obj_nelts = 0;
    4499         3962 :       tree atype = NULL_TREE;
    4500         3962 :       if (obj)
    4501              :         {
    4502         3871 :           obj_nelts = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
    4503         3871 :           if (orig_obj == NULL_TREE)
    4504         2277 :             obj_nelts >>= 1;
    4505         3871 :           atype = build_array_type_nelts (m_limb_type, obj_nelts);
    4506              :         }
    4507         3962 :       if (bitint_extended && (var || obj))
    4508              :         {
    4509            0 :           unsigned prec = TYPE_PRECISION (type);
    4510            0 :           unsigned prec_limbs = CEIL (prec, limb_prec);
    4511            0 :           bool ext_ms_limb
    4512              :             = (bitint_extended == bitint_ext_full
    4513            0 :                && abi_limb_prec > limb_prec
    4514            0 :                && (CEIL (prec, abi_limb_prec) * abi_limb_prec
    4515            0 :                    > CEIL (prec, limb_prec) * limb_prec));
    4516              :           /* For .{ADD,SUB}_OVERFLOW the partial limb if any is
    4517              :              already extended in lower_addsub_overflow.  */
    4518            0 :           if ((code == MULT_EXPR && (prec % limb_prec) != 0)
    4519            0 :               || (ext_ms_limb && !TYPE_UNSIGNED (type)))
    4520              :             {
    4521            0 :               tree plm1idx = size_int (bitint_big_endian
    4522              :                                        ? nelts - obj_nelts : prec_limbs - 1);
    4523            0 :               tree plm1type
    4524            0 :                 = limb_access_type (type, bitint_big_endian
    4525              :                                           ? size_zero_node : plm1idx);
    4526            0 :               tree l = limb_access (bitint_big_endian ? NULL_TREE : type,
    4527              :                                     var ? var : obj, plm1idx, true);
    4528            0 :               tree rhs = make_ssa_name (TREE_TYPE (l));
    4529            0 :               g = gimple_build_assign (rhs, l);
    4530            0 :               insert_before (g);
    4531            0 :               if (code == MULT_EXPR && (prec % limb_prec) != 0)
    4532              :                 {
    4533            0 :                   if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
    4534            0 :                     rhs = add_cast (plm1type, rhs);
    4535            0 :                   if (!useless_type_conversion_p (TREE_TYPE (l),
    4536            0 :                                                   TREE_TYPE (rhs)))
    4537            0 :                     rhs = add_cast (TREE_TYPE (l), rhs);
    4538            0 :                   l = limb_access (bitint_big_endian ? NULL_TREE : type,
    4539              :                                    var ? var : obj, plm1idx, true);
    4540            0 :                   g = gimple_build_assign (l, rhs);
    4541            0 :                   insert_before (g);
    4542              :                 }
    4543            0 :               if (ext_ms_limb && !TYPE_UNSIGNED (type))
    4544              :                 {
    4545            0 :                   rhs = add_cast (signed_type_for (m_limb_type), rhs);
    4546            0 :                   tree lpm1 = build_int_cst (unsigned_type_node,
    4547            0 :                                              limb_prec - 1);
    4548            0 :                   tree v = make_ssa_name (TREE_TYPE (rhs));
    4549            0 :                   g = gimple_build_assign (v, RSHIFT_EXPR, rhs, lpm1);
    4550            0 :                   insert_before (g);
    4551            0 :                   unsigned int i
    4552            0 :                     = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
    4553            0 :                   v = add_cast (m_limb_type, v);
    4554            0 :                   g = gimple_build_assign (limb_access (type, var ? var : obj,
    4555            0 :                                                         size_int (i - 1),
    4556              :                                                         true), v);
    4557            0 :                   insert_before (g);
    4558            0 :                   ext_ms_limb = false;
    4559              :                 }
    4560              :             }
    4561            0 :           if (ext_ms_limb)
    4562              :             {
    4563            0 :               unsigned int i
    4564            0 :                 = CEIL (prec, abi_limb_prec) * abi_limb_prec / limb_prec;
    4565            0 :               g = gimple_build_assign (limb_access (type, var ? var : obj,
    4566            0 :                                                     size_int (i - 1), true),
    4567              :                                        build_zero_cst (m_limb_type));
    4568            0 :               insert_before (g);
    4569              :             }
    4570              :         }
    4571         3962 :       if (var && obj)
    4572              :         {
    4573          519 :           tree v1, v2;
    4574          519 :           tree off;
    4575          519 :           if (orig_obj == NULL_TREE)
    4576              :             {
    4577           32 :               off = build_zero_cst (build_pointer_type (TREE_TYPE (obj)));
    4578           32 :               v1 = build2 (MEM_REF, atype,
    4579              :                            build_fold_addr_expr (unshare_expr (obj)), off);
    4580              :             }
    4581          487 :           else if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
    4582            9 :             v1 = build1 (VIEW_CONVERT_EXPR, atype, unshare_expr (obj));
    4583              :           else
    4584          478 :             v1 = unshare_expr (obj);
    4585          519 :           off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
    4586              :                                bitint_big_endian
    4587          519 :                                ? (nelts - obj_nelts) * m_limb_size : 0);
    4588          519 :           v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
    4589          519 :           g = gimple_build_assign (v1, v2);
    4590          519 :           insert_before (g);
    4591              :         }
    4592         3443 :       else if (obj && bitint_big_endian && nelts != obj_nelts)
    4593              :         {
    4594            0 :           gcc_assert (nelts > obj_nelts);
    4595            0 :           tree fn = builtin_decl_implicit (BUILT_IN_MEMMOVE);
    4596            0 :           tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
    4597            0 :                                     (nelts - obj_nelts) * m_limb_size);
    4598            0 :           tree src = build2 (MEM_REF, atype,
    4599              :                              build_fold_addr_expr (unshare_expr (obj)), off);
    4600            0 :           g = gimple_build_call (fn, 3,
    4601              :                                  build_fold_addr_expr (unshare_expr (obj)),
    4602              :                                  build_fold_addr_expr (src),
    4603              :                                  build_int_cst (size_type_node,
    4604            0 :                                                 obj_nelts * m_limb_size));
    4605            0 :           insert_before (g);
    4606              :         }
    4607         3962 :       if (orig_obj == NULL_TREE && obj)
    4608              :         {
    4609         2277 :           ovf = add_cast (m_limb_type, ovf);
    4610         2277 :           tree l = limb_access (NULL_TREE, obj,
    4611         2277 :                                 size_int (bitint_big_endian
    4612              :                                           ? obj_nelts * 2 - 1 : obj_nelts),
    4613              :                                 true);
    4614         2277 :           g = gimple_build_assign (l, ovf);
    4615         2277 :           insert_before (g);
    4616         2277 :           if (obj_nelts > 1)
    4617              :             {
    4618         2277 :               atype = build_array_type_nelts (m_limb_type, obj_nelts - 1);
    4619         2277 :               tree off = build_int_cst (build_pointer_type (TREE_TYPE (obj)),
    4620         2277 :                                         (obj_nelts + !bitint_big_endian)
    4621         2277 :                                         * m_limb_size);
    4622         2277 :               tree v1 = build2 (MEM_REF, atype,
    4623              :                                 build_fold_addr_expr (unshare_expr (obj)),
    4624              :                                 off);
    4625         2277 :               g = gimple_build_assign (v1, build_zero_cst (atype));
    4626         2277 :               insert_before (g);
    4627              :             }
    4628              :         }
    4629         1685 :       else if (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE)
    4630              :         {
    4631         1640 :           imm_use_iterator ui;
    4632         1640 :           use_operand_p use_p;
    4633         1640 :           FOR_EACH_IMM_USE_FAST (use_p, ui, lhs)
    4634              :             {
    4635         1640 :               g = USE_STMT (use_p);
    4636         1640 :               if (!is_gimple_assign (g)
    4637         1640 :                   || gimple_assign_rhs_code (g) != IMAGPART_EXPR)
    4638            0 :                 continue;
    4639         1640 :               tree lhs2 = gimple_assign_lhs (g);
    4640         1640 :               gimple *use_stmt;
    4641         1640 :               single_imm_use (lhs2, &use_p, &use_stmt);
    4642         1640 :               lhs2 = gimple_assign_lhs (use_stmt);
    4643         1640 :               gimple_stmt_iterator gsi = gsi_for_stmt (use_stmt);
    4644         1640 :               if (useless_type_conversion_p (TREE_TYPE (lhs2), TREE_TYPE (ovf)))
    4645         1619 :                 g = gimple_build_assign (lhs2, ovf);
    4646              :               else
    4647           21 :                 g = gimple_build_assign (lhs2, NOP_EXPR, ovf);
    4648         1640 :               gsi_replace (&gsi, g, true);
    4649         1640 :               if (gsi_stmt (m_gsi) == use_stmt)
    4650           91 :                 m_gsi = gsi_for_stmt (g);
    4651         1640 :               break;
    4652         1640 :             }
    4653              :         }
    4654           45 :       else if (ovf != boolean_false_node)
    4655              :         {
    4656           45 :           g = gimple_build_cond (NE_EXPR, ovf, boolean_false_node,
    4657              :                                  NULL_TREE, NULL_TREE);
    4658           45 :           edge edge_true, edge_false;
    4659           45 :           if_then (g, profile_probability::very_unlikely (),
    4660              :                    edge_true, edge_false);
    4661           45 :           tree zero = build_zero_cst (TREE_TYPE (lhs));
    4662           45 :           tree fn = ubsan_build_overflow_builtin (code, m_loc,
    4663           45 :                                                   TREE_TYPE (lhs),
    4664              :                                                   zero, zero, NULL);
    4665           45 :           force_gimple_operand_gsi (&m_gsi, fn, true, NULL_TREE,
    4666              :                                     true, GSI_SAME_STMT);
    4667           45 :           m_gsi = gsi_after_labels (edge_true->dest);
    4668              :         }
    4669              :     }
    4670         4212 :   if (var)
    4671              :     {
    4672          781 :       tree clobber = build_clobber (TREE_TYPE (var), CLOBBER_STORAGE_END);
    4673          781 :       g = gimple_build_assign (var, clobber);
    4674          781 :       gsi_insert_after (&m_gsi, g, GSI_SAME_STMT);
    4675              :     }
    4676         4212 : }
    4677              : 
    4678              : /* Helper function for lower_addsub_overflow and lower_mul_overflow.
    4679              :    Given precisions of result TYPE (PREC), argument 0 precision PREC0,
    4680              :    argument 1 precision PREC1 and minimum precision for the result
    4681              :    PREC2, compute *START, *END, *CHECK_ZERO and return OVF.  */
    4682              : 
    4683              : static tree
    4684         4212 : arith_overflow (tree_code code, tree type, int prec, int prec0, int prec1,
    4685              :                 int prec2, unsigned *start, unsigned *end, bool *check_zero)
    4686              : {
    4687         4212 :   *start = 0;
    4688         4212 :   *end = 0;
    4689         4212 :   *check_zero = true;
    4690              :   /* Ignore this special rule for subtraction, even if both
    4691              :      prec0 >= 0 and prec1 >= 0, their subtraction can be negative
    4692              :      in infinite precision.  */
    4693         4212 :   if (code != MINUS_EXPR && prec0 >= 0 && prec1 >= 0)
    4694              :     {
    4695              :       /* Result in [0, prec2) is unsigned, if prec > prec2,
    4696              :          all bits above it will be zero.  */
    4697          681 :       if ((prec - !TYPE_UNSIGNED (type)) >= prec2)
    4698            0 :         return boolean_false_node;
    4699              :       else
    4700              :         {
    4701              :           /* ovf if any of bits in [start, end) is non-zero.  */
    4702          681 :           *start = prec - !TYPE_UNSIGNED (type);
    4703          681 :           *end = prec2;
    4704              :         }
    4705              :     }
    4706         3531 :   else if (TYPE_UNSIGNED (type))
    4707              :     {
    4708              :       /* If result in [0, prec2) is signed and if prec > prec2,
    4709              :          all bits above it will be sign bit copies.  */
    4710         1950 :       if (prec >= prec2)
    4711              :         {
    4712              :           /* ovf if bit prec - 1 is non-zero.  */
    4713          184 :           *start = prec - 1;
    4714          184 :           *end = prec;
    4715              :         }
    4716              :       else
    4717              :         {
    4718              :           /* ovf if any of bits in [start, end) is non-zero.  */
    4719         1766 :           *start = prec;
    4720         1766 :           *end = prec2;
    4721              :         }
    4722              :     }
    4723         1581 :   else if (prec >= prec2)
    4724            2 :     return boolean_false_node;
    4725              :   else
    4726              :     {
    4727              :       /* ovf if [start, end) bits aren't all zeros or all ones.  */
    4728         1579 :       *start = prec - 1;
    4729         1579 :       *end = prec2;
    4730         1579 :       *check_zero = false;
    4731              :     }
    4732              :   return NULL_TREE;
    4733              : }
    4734              : 
    4735              : /* Lower a .{ADD,SUB}_OVERFLOW call with at least one large/huge _BitInt
    4736              :    argument or return type _Complex large/huge _BitInt.  */
    4737              : 
    4738              : void
    4739         2733 : bitint_large_huge::lower_addsub_overflow (tree obj, gimple *stmt)
    4740              : {
    4741         2733 :   tree arg0 = gimple_call_arg (stmt, 0);
    4742         2733 :   tree arg1 = gimple_call_arg (stmt, 1);
    4743         2733 :   tree lhs = gimple_call_lhs (stmt);
    4744         2733 :   gimple *g;
    4745              : 
    4746         2733 :   if (!lhs)
    4747              :     {
    4748            0 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    4749            0 :       gsi_remove (&gsi, true);
    4750            0 :       return;
    4751              :     }
    4752         2733 :   gimple *final_stmt = gsi_stmt (m_gsi);
    4753         2733 :   tree type = TREE_TYPE (lhs);
    4754         2733 :   if (TREE_CODE (type) == COMPLEX_TYPE)
    4755         2703 :     type = TREE_TYPE (type);
    4756         2733 :   int prec = TYPE_PRECISION (type);
    4757         2733 :   int prec0 = range_to_prec (arg0, stmt);
    4758         2733 :   int prec1 = range_to_prec (arg1, stmt);
    4759              :   /* If PREC0 >= 0 && PREC1 >= 0 and CODE is not MINUS_EXPR, PREC2 is
    4760              :      the minimum unsigned precision of any possible operation's
    4761              :      result, otherwise it is minimum signed precision.
    4762              :      Some examples:
    4763              :      If PREC0 or PREC1 is 8, it means that argument is [0, 0xff],
    4764              :      if PREC0 or PREC1 is 10, it means that argument is [0, 0x3ff],
    4765              :      if PREC0 or PREC1 is -8, it means that argument is [-0x80, 0x7f],
    4766              :      if PREC0 or PREC1 is -10, it means that argument is [-0x200, 0x1ff].
    4767              :      PREC0  CODE  PREC1  RESULT          PREC2  SIGNED vs. UNSIGNED
    4768              :        8      +     8    [0, 0x1fe]        9    UNSIGNED
    4769              :        8      +    10    [0, 0x4fe]       11    UNSIGNED
    4770              :       -8      +    -8    [-0x100, 0xfe]    9    SIGNED
    4771              :       -8      +   -10    [-0x280, 0x27e]  11    SIGNED
    4772              :        8      +    -8    [-0x80, 0x17e]   10    SIGNED
    4773              :        8      +   -10    [-0x200, 0x2fe]  11    SIGNED
    4774              :       10      +    -8    [-0x80, 0x47e]   12    SIGNED
    4775              :        8      -     8    [-0xff, 0xff]     9    SIGNED
    4776              :        8      -    10    [-0x3ff, 0xff]   11    SIGNED
    4777              :       10      -     8    [-0xff, 0x3ff]   11    SIGNED
    4778              :       -8      -    -8    [-0xff, 0xff]     9    SIGNED
    4779              :       -8      -   -10    [-0x27f, 0x27f]  11    SIGNED
    4780              :      -10      -    -8    [-0x27f, 0x27f]  11    SIGNED
    4781              :        8      -    -8    [-0x7f, 0x17f]   10    SIGNED
    4782              :        8      -   -10    [-0x1ff, 0x2ff]  11    SIGNED
    4783              :       10      -    -8    [-0x7f, 0x47f]   12    SIGNED
    4784              :       -8      -     8    [-0x17f, 0x7f]   10    SIGNED
    4785              :       -8      -    10    [-0x47f, 0x7f]   12    SIGNED
    4786              :      -10      -     8    [-0x2ff, 0x1ff]  11    SIGNED  */
    4787         2733 :   int prec2 = MAX (prec0 < 0 ? -prec0 : prec0,
    4788              :                    prec1 < 0 ? -prec1 : prec1);
    4789              :             /* If operands are either both signed or both unsigned,
    4790              :                we need just one additional bit.  */
    4791         3777 :   prec2 = (((prec0 < 0) == (prec1 < 0)
    4792              :                /* If one operand is signed and one unsigned and
    4793              :                   the signed one has larger precision, we need
    4794              :                   just one extra bit, otherwise two.  */
    4795          707 :             || (prec0 < 0 ? (prec2 == -prec0 && prec2 != prec1)
    4796          337 :                           : (prec2 == -prec1 && prec2 != prec0)))
    4797         2733 :            ? prec2 + 1 : prec2 + 2);
    4798         2733 :   int prec3 = MAX (prec0 < 0 ? -prec0 : prec0,
    4799              :                    prec1 < 0 ? -prec1 : prec1);
    4800         2733 :   prec3 = MAX (prec3, prec);
    4801         2733 :   tree var = NULL_TREE;
    4802         2733 :   tree orig_obj = obj;
    4803         2733 :   if (obj == NULL_TREE
    4804         1753 :       && BITINT_TYPE_P (type)
    4805         1652 :       && bitint_precision_kind (type) >= bitint_prec_large
    4806         1536 :       && m_names
    4807         4245 :       && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
    4808              :     {
    4809         1457 :       int part = var_to_partition (m_map, lhs);
    4810         1457 :       gcc_assert (m_vars[part] != NULL_TREE);
    4811         1457 :       obj = m_vars[part];
    4812         1457 :       if (TREE_TYPE (lhs) == type)
    4813           14 :         orig_obj = obj;
    4814              :     }
    4815          101 :   if (!BITINT_TYPE_P (type)
    4816         2733 :       || bitint_precision_kind (type) < bitint_prec_large)
    4817              :     {
    4818          217 :       unsigned HOST_WIDE_INT nelts = CEIL (prec, limb_prec);
    4819          217 :       tree atype = build_array_type_nelts (m_limb_type, nelts);
    4820          217 :       var = create_tmp_var (atype);
    4821              :     }
    4822              : 
    4823         2733 :   enum tree_code code;
    4824         2733 :   switch (gimple_call_internal_fn (stmt))
    4825              :     {
    4826              :     case IFN_ADD_OVERFLOW:
    4827              :     case IFN_UBSAN_CHECK_ADD:
    4828              :       code = PLUS_EXPR;
    4829              :       break;
    4830         1399 :     case IFN_SUB_OVERFLOW:
    4831         1399 :     case IFN_UBSAN_CHECK_SUB:
    4832         1399 :       code = MINUS_EXPR;
    4833         1399 :       break;
    4834            0 :     default:
    4835            0 :       gcc_unreachable ();
    4836              :     }
    4837         2733 :   unsigned start, end;
    4838         2733 :   bool check_zero;
    4839         2733 :   tree ovf = arith_overflow (code, type, prec, prec0, prec1, prec2,
    4840              :                              &start, &end, &check_zero);
    4841              : 
    4842         2733 :   unsigned startlimb, endlimb;
    4843         2733 :   if (ovf)
    4844              :     {
    4845              :       startlimb = ~0U;
    4846              :       endlimb = ~0U;
    4847              :     }
    4848              :   else
    4849              :     {
    4850         2733 :       startlimb = start / limb_prec;
    4851         2733 :       endlimb = (end - 1) / limb_prec;
    4852              :     }
    4853              : 
    4854         2733 :   int prec4 = ovf != NULL_TREE ? prec : prec3;
    4855         2733 :   bitint_prec_kind kind = bitint_precision_kind (prec4);
    4856         2733 :   unsigned cnt, rem = 0, fin = 0, nelts;
    4857         2733 :   tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
    4858         5466 :   bool last_ovf = (ovf == NULL_TREE
    4859         2733 :                    && CEIL (prec2, limb_prec) > CEIL (prec3, limb_prec));
    4860         2733 :   if (kind != bitint_prec_huge)
    4861         1539 :     nelts = cnt = CEIL (prec4, limb_prec) + last_ovf;
    4862              :   else
    4863              :     {
    4864         1194 :       rem = prec4 % (2 * limb_prec);
    4865         1194 :       fin = (prec4 - rem) / limb_prec;
    4866         1194 :       cnt = 2 + CEIL (rem, limb_prec) + last_ovf;
    4867         1194 :       nelts = fin + cnt - 2;
    4868         1194 :       idx = idx_first = create_loop (bitint_big_endian
    4869         1194 :                                      ? size_int (nelts - 1) : size_zero_node,
    4870              :                                      &idx_next);
    4871              :     }
    4872              : 
    4873         2733 :   if (kind == bitint_prec_huge)
    4874         1194 :     m_upwards_2limb = fin;
    4875         2733 :   m_upwards = true;
    4876              : 
    4877         2733 :   tree type0 = TREE_TYPE (arg0);
    4878         2733 :   tree type1 = TREE_TYPE (arg1);
    4879         2733 :   int prec5 = prec3;
    4880         2733 :   if (bitint_precision_kind (prec5) < bitint_prec_large)
    4881           10 :     prec5 = MAX (TYPE_PRECISION (type0), TYPE_PRECISION (type1));
    4882         2733 :   if (TYPE_PRECISION (type0) < prec5)
    4883              :     {
    4884          146 :       type0 = build_bitint_type (prec5, TYPE_UNSIGNED (type0));
    4885          146 :       if (TREE_CODE (arg0) == INTEGER_CST)
    4886           27 :         arg0 = fold_convert (type0, arg0);
    4887              :     }
    4888         2733 :   if (TYPE_PRECISION (type1) < prec5)
    4889              :     {
    4890          156 :       type1 = build_bitint_type (prec5, TYPE_UNSIGNED (type1));
    4891          156 :       if (TREE_CODE (arg1) == INTEGER_CST)
    4892           76 :         arg1 = fold_convert (type1, arg1);
    4893              :     }
    4894         2733 :   unsigned int data_cnt = 0;
    4895         2733 :   tree last_rhs1 = NULL_TREE, last_rhs2 = NULL_TREE;
    4896         2733 :   tree cmp = build_zero_cst (m_limb_type);
    4897         2733 :   unsigned prec_limbs = CEIL ((unsigned) prec, limb_prec);
    4898         2733 :   tree ovf_out = NULL_TREE, cmp_out = NULL_TREE;
    4899        11904 :   for (unsigned i = 0; i < cnt; i++)
    4900              :     {
    4901         9171 :       m_data_cnt = 0;
    4902         9171 :       tree rhs1, rhs2;
    4903         9171 :       if (kind != bitint_prec_huge)
    4904         5303 :         idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
    4905         3868 :       else if (i >= 2)
    4906         1480 :         idx = size_int (bitint_big_endian ? nelts + 1 - fin - i : fin + i - 2);
    4907         9171 :       if (!last_ovf || i < cnt - 1)
    4908              :         {
    4909         8225 :           tree idx0 = idx, idx1 = idx;
    4910         8225 :           if (bitint_big_endian
    4911         8225 :               && CEIL ((unsigned) TYPE_PRECISION (type0), limb_prec) != nelts)
    4912              :             {
    4913            0 :               HOST_WIDE_INT diff
    4914            0 :                 = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type0), limb_prec)
    4915            0 :                    - (HOST_WIDE_INT) nelts);
    4916            0 :               if (tree_fits_uhwi_p (idx))
    4917            0 :                 idx0 = size_int (tree_to_uhwi (idx) + diff);
    4918              :               else
    4919              :                 {
    4920            0 :                   idx0 = make_ssa_name (sizetype);
    4921            0 :                   g = gimple_build_assign (idx0, PLUS_EXPR, idx,
    4922            0 :                                            size_int (diff));
    4923            0 :                   insert_before (g);
    4924              :                 }
    4925              :             }
    4926         8225 :           if (type0 != TREE_TYPE (arg0))
    4927          334 :             rhs1 = handle_cast (type0, arg0, idx0);
    4928              :           else
    4929         7891 :             rhs1 = handle_operand (arg0, idx0);
    4930         8225 :           if (bitint_big_endian
    4931         8225 :               && CEIL ((unsigned) TYPE_PRECISION (type1), limb_prec) != nelts)
    4932              :             {
    4933            0 :               HOST_WIDE_INT diff
    4934            0 :                 = ((HOST_WIDE_INT) CEIL (TYPE_PRECISION (type1), limb_prec)
    4935            0 :                    - (HOST_WIDE_INT) nelts);
    4936            0 :               if (tree_fits_uhwi_p (idx))
    4937            0 :                 idx1 = size_int (tree_to_uhwi (idx) + diff);
    4938              :               else
    4939              :                 {
    4940            0 :                   idx1 = make_ssa_name (sizetype);
    4941            0 :                   g = gimple_build_assign (idx1, PLUS_EXPR, idx,
    4942            0 :                                            size_int (diff));
    4943            0 :                   insert_before (g);
    4944              :                 }
    4945              :             }
    4946         8225 :           if (type1 != TREE_TYPE (arg1))
    4947          250 :             rhs2 = handle_cast (type1, arg1, idx1);
    4948              :           else
    4949         7975 :             rhs2 = handle_operand (arg1, idx1);
    4950         8225 :           if (i == 0)
    4951         2733 :             data_cnt = m_data_cnt;
    4952         8225 :           if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
    4953         1983 :             rhs1 = add_cast (m_limb_type, rhs1);
    4954         8225 :           if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs2)))
    4955         1983 :             rhs2 = add_cast (m_limb_type, rhs2);
    4956              :           last_rhs1 = rhs1;
    4957              :           last_rhs2 = rhs2;
    4958              :         }
    4959              :       else
    4960              :         {
    4961          946 :           m_data_cnt = data_cnt;
    4962          946 :           if (TYPE_UNSIGNED (type0) || prec0 >= 0)
    4963          421 :             rhs1 = build_zero_cst (m_limb_type);
    4964              :           else
    4965              :             {
    4966          525 :               rhs1 = add_cast (signed_type_for (m_limb_type), last_rhs1);
    4967          525 :               if (TREE_CODE (rhs1) == INTEGER_CST)
    4968           52 :                 rhs1 = build_int_cst (m_limb_type,
    4969           74 :                                       tree_int_cst_sgn (rhs1) < 0 ? -1 : 0);
    4970              :               else
    4971              :                 {
    4972          946 :                   tree lpm1 = build_int_cst (unsigned_type_node,
    4973          473 :                                              limb_prec - 1);
    4974          473 :                   g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
    4975              :                                            RSHIFT_EXPR, rhs1, lpm1);
    4976          473 :                   insert_before (g);
    4977          473 :                   rhs1 = add_cast (m_limb_type, gimple_assign_lhs (g));
    4978              :                 }
    4979              :             }
    4980          946 :           if (TYPE_UNSIGNED (type1) || prec1 >= 0)
    4981          543 :             rhs2 = build_zero_cst (m_limb_type);
    4982              :           else
    4983              :             {
    4984          403 :               rhs2 = add_cast (signed_type_for (m_limb_type), last_rhs2);
    4985          403 :               if (TREE_CODE (rhs2) == INTEGER_CST)
    4986          114 :                 rhs2 = build_int_cst (m_limb_type,
    4987          153 :                                       tree_int_cst_sgn (rhs2) < 0 ? -1 : 0);
    4988              :               else
    4989              :                 {
    4990          578 :                   tree lpm1 = build_int_cst (unsigned_type_node,
    4991          289 :                                              limb_prec - 1);
    4992          289 :                   g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs2)),
    4993              :                                            RSHIFT_EXPR, rhs2, lpm1);
    4994          289 :                   insert_before (g);
    4995          289 :                   rhs2 = add_cast (m_limb_type, gimple_assign_lhs (g));
    4996              :                 }
    4997              :             }
    4998              :         }
    4999         9171 :       tree rhs = handle_plus_minus (code, rhs1, rhs2, idx);
    5000         9171 :       if (ovf != boolean_false_node)
    5001              :         {
    5002         9171 :           if (tree_fits_uhwi_p (idx))
    5003              :             {
    5004         6783 :               unsigned limb = tree_to_uhwi (idx);
    5005         6783 :               if (bitint_big_endian)
    5006            0 :                 limb = nelts - 1 - limb;
    5007         6783 :               if (limb >= startlimb && limb <= endlimb)
    5008              :                 {
    5009         3370 :                   tree l = arith_overflow_extract_bits (start, end, rhs,
    5010              :                                                         limb, check_zero);
    5011         3370 :                   tree this_ovf = make_ssa_name (boolean_type_node);
    5012         3370 :                   if (ovf == NULL_TREE && !check_zero)
    5013              :                     {
    5014          907 :                       cmp = l;
    5015          907 :                       g = gimple_build_assign (make_ssa_name (m_limb_type),
    5016              :                                                PLUS_EXPR, l,
    5017              :                                                build_int_cst (m_limb_type, 1));
    5018          907 :                       insert_before (g);
    5019          907 :                       g = gimple_build_assign (this_ovf, GT_EXPR,
    5020              :                                                gimple_assign_lhs (g),
    5021              :                                                build_int_cst (m_limb_type, 1));
    5022              :                     }
    5023              :                   else
    5024         2463 :                     g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
    5025         3370 :                   insert_before (g);
    5026         3370 :                   if (ovf == NULL_TREE)
    5027              :                     ovf = this_ovf;
    5028              :                   else
    5029              :                     {
    5030         1064 :                       tree b = make_ssa_name (boolean_type_node);
    5031         1064 :                       g = gimple_build_assign (b, BIT_IOR_EXPR, ovf, this_ovf);
    5032         1064 :                       insert_before (g);
    5033         1064 :                       ovf = b;
    5034              :                     }
    5035              :                 }
    5036              :             }
    5037         2388 :           else if (startlimb < fin)
    5038              :             {
    5039          854 :               if (m_first && startlimb + 2 < fin)
    5040              :                 {
    5041          324 :                   tree data_out;
    5042          324 :                   ovf = prepare_data_in_out (boolean_false_node, idx, &data_out);
    5043          324 :                   ovf_out = m_data.pop ();
    5044          324 :                   m_data.pop ();
    5045          324 :                   if (!check_zero)
    5046              :                     {
    5047          169 :                       cmp = prepare_data_in_out (cmp, idx, &data_out);
    5048          169 :                       cmp_out = m_data.pop ();
    5049          169 :                       m_data.pop ();
    5050              :                     }
    5051              :                 }
    5052          854 :               if (i != 0 || startlimb != fin - 1)
    5053              :                 {
    5054          839 :                   tree_code cmp_code;
    5055          839 :                   bool single_comparison
    5056          839 :                     = (startlimb + 2 >= fin || (startlimb & 1) != (i & 1));
    5057              :                   if (!single_comparison)
    5058              :                     cmp_code = GE_EXPR;
    5059          515 :                   else if ((startlimb & 1) == (i & 1))
    5060              :                     cmp_code = EQ_EXPR;
    5061              :                   else
    5062          412 :                     cmp_code = GT_EXPR;
    5063          839 :                   if (bitint_big_endian)
    5064            0 :                     g = gimple_build_cond (swap_tree_comparison (cmp_code),
    5065            0 :                                            idx, size_int (nelts - 1
    5066              :                                                           - startlimb),
    5067              :                                            NULL_TREE, NULL_TREE);
    5068              :                   else
    5069          839 :                     g = gimple_build_cond (cmp_code, idx, size_int (startlimb),
    5070              :                                            NULL_TREE, NULL_TREE);
    5071          839 :                   edge edge_true_true, edge_true_false, edge_false;
    5072          839 :                   gimple *g2 = NULL;
    5073          839 :                   if (!single_comparison)
    5074          324 :                     g2 = gimple_build_cond (NE_EXPR, idx,
    5075          324 :                                             size_int (bitint_big_endian
    5076              :                                                       ? nelts - 1 - startlimb
    5077              :                                                       : startlimb),
    5078              :                                             NULL_TREE, NULL_TREE);
    5079          839 :                   if_then_if_then_else (g, g2, profile_probability::likely (),
    5080              :                                         profile_probability::likely (),
    5081              :                                         edge_true_true, edge_true_false,
    5082              :                                         edge_false);
    5083          839 :                   unsigned tidx = startlimb + (cmp_code == GT_EXPR);
    5084          839 :                   tree l = arith_overflow_extract_bits (start, end, rhs, tidx,
    5085              :                                                         check_zero);
    5086          839 :                   tree this_ovf = make_ssa_name (boolean_type_node);
    5087          839 :                   if (cmp_code != GT_EXPR && !check_zero)
    5088              :                     {
    5089          173 :                       g = gimple_build_assign (make_ssa_name (m_limb_type),
    5090              :                                                PLUS_EXPR, l,
    5091              :                                                build_int_cst (m_limb_type, 1));
    5092          173 :                       insert_before (g);
    5093          173 :                       g = gimple_build_assign (this_ovf, GT_EXPR,
    5094              :                                                gimple_assign_lhs (g),
    5095              :                                                build_int_cst (m_limb_type, 1));
    5096              :                     }
    5097              :                   else
    5098          666 :                     g = gimple_build_assign (this_ovf, NE_EXPR, l, cmp);
    5099          839 :                   insert_before (g);
    5100          839 :                   if (cmp_code == GT_EXPR)
    5101              :                     {
    5102          412 :                       tree t = make_ssa_name (boolean_type_node);
    5103          412 :                       g = gimple_build_assign (t, BIT_IOR_EXPR, ovf, this_ovf);
    5104          412 :                       insert_before (g);
    5105          412 :                       this_ovf = t;
    5106              :                     }
    5107          839 :                   tree this_ovf2 = NULL_TREE;
    5108          839 :                   if (!single_comparison)
    5109              :                     {
    5110          324 :                       m_gsi = gsi_after_labels (edge_true_true->src);
    5111          324 :                       tree t = make_ssa_name (boolean_type_node);
    5112          324 :                       g = gimple_build_assign (t, NE_EXPR, rhs, cmp);
    5113          324 :                       insert_before (g);
    5114          324 :                       this_ovf2 = make_ssa_name (boolean_type_node);
    5115          324 :                       g = gimple_build_assign (this_ovf2, BIT_IOR_EXPR,
    5116              :                                                ovf, t);
    5117          324 :                       insert_before (g);
    5118              :                     }
    5119          839 :                   m_gsi = gsi_after_labels (edge_true_false->dest);
    5120          839 :                   tree t;
    5121          839 :                   if (i == 1 && ovf_out)
    5122              :                     t = ovf_out;
    5123              :                   else
    5124          515 :                     t = make_ssa_name (boolean_type_node);
    5125          839 :                   gphi *phi = create_phi_node (t, edge_true_false->dest);
    5126          839 :                   add_phi_arg (phi, this_ovf, edge_true_false,
    5127              :                                UNKNOWN_LOCATION);
    5128          839 :                   add_phi_arg (phi, ovf ? ovf
    5129              :                                     : boolean_false_node, edge_false,
    5130              :                                UNKNOWN_LOCATION);
    5131          839 :                   if (edge_true_true)
    5132          324 :                     add_phi_arg (phi, this_ovf2, edge_true_true,
    5133              :                                  UNKNOWN_LOCATION);
    5134          839 :                   ovf = t;
    5135          839 :                   if (!check_zero && cmp_code != GT_EXPR)
    5136              :                     {
    5137          173 :                       t = cmp_out ? cmp_out : make_ssa_name (m_limb_type);
    5138          173 :                       phi = create_phi_node (t, edge_true_false->dest);
    5139          173 :                       add_phi_arg (phi, l, edge_true_false, UNKNOWN_LOCATION);
    5140          173 :                       add_phi_arg (phi, cmp, edge_false, UNKNOWN_LOCATION);
    5141          173 :                       if (edge_true_true)
    5142          169 :                         add_phi_arg (phi, cmp, edge_true_true,
    5143              :                                      UNKNOWN_LOCATION);
    5144              :                       cmp = t;
    5145              :                     }
    5146              :                 }
    5147              :             }
    5148              :         }
    5149              : 
    5150         9171 :       if (var || obj)
    5151              :         {
    5152         8943 :           if (tree_fits_uhwi_p (idx)
    5153         6673 :               && (bitint_big_endian
    5154         6673 :                   ? nelts - 1 - tree_to_uhwi (idx)
    5155         6673 :                   : tree_to_uhwi (idx)) >= prec_limbs)
    5156              :             ;
    5157         7555 :           else if (!tree_fits_uhwi_p (idx)
    5158         2270 :                    && (unsigned) prec < (fin - (i == 0)) * limb_prec)
    5159              :             {
    5160         1458 :               bool single_comparison
    5161          729 :                 = (((unsigned) prec % limb_prec) == 0
    5162          571 :                    || prec_limbs + 1 >= fin
    5163         1231 :                    || (prec_limbs & 1) == (i & 1));
    5164          729 :               if (bitint_big_endian)
    5165            0 :                 g = gimple_build_cond (GE_EXPR, idx,
    5166            0 :                                        size_int (nelts - prec_limbs),
    5167              :                                        NULL_TREE, NULL_TREE);
    5168              :               else
    5169          729 :                 g = gimple_build_cond (LE_EXPR, idx, size_int (prec_limbs - 1),
    5170              :                                        NULL_TREE, NULL_TREE);
    5171          729 :               gimple *g2 = NULL;
    5172          729 :               if (!single_comparison)
    5173          251 :                 g2 = gimple_build_cond (EQ_EXPR, idx,
    5174          251 :                                         size_int (bitint_big_endian
    5175              :                                                   ? nelts - prec_limbs
    5176              :                                                   : prec_limbs - 1),
    5177              :                                         NULL_TREE, NULL_TREE);
    5178          729 :               edge edge_true_true, edge_true_false, edge_false;
    5179          729 :               if_then_if_then_else (g, g2, profile_probability::likely (),
    5180              :                                     profile_probability::unlikely (),
    5181              :                                     edge_true_true, edge_true_false,
    5182              :                                     edge_false);
    5183          729 :               tree idxl = idx;
    5184          729 :               if (bitint_big_endian && prec_limbs != nelts)
    5185              :                 {
    5186            0 :                   HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
    5187            0 :                                         - (HOST_WIDE_INT) nelts);
    5188            0 :                   if (tree_fits_uhwi_p (idx))
    5189            0 :                     idxl = size_int (tree_to_uhwi (idx) + diff);
    5190              :                   else
    5191              :                     {
    5192            0 :                       idxl = make_ssa_name (sizetype);
    5193            0 :                       g = gimple_build_assign (idxl, PLUS_EXPR, idx,
    5194            0 :                                                size_int (diff));
    5195            0 :                       insert_before (g);
    5196              :                     }
    5197              :                 }
    5198         1082 :               tree l = limb_access (type, var ? var : obj, idxl, true);
    5199          729 :               g = gimple_build_assign (l, rhs);
    5200          729 :               insert_before (g);
    5201          729 :               if (!single_comparison)
    5202              :                 {
    5203          251 :                   m_gsi = gsi_after_labels (edge_true_true->src);
    5204          251 :                   tree plm1idx = size_int (bitint_big_endian
    5205              :                                            ? 0 : prec_limbs - 1);
    5206          251 :                   tree plm1type = limb_access_type (type, plm1idx);
    5207          251 :                   l = limb_access (type, var ? var : obj, plm1idx, true);
    5208          251 :                   if (!useless_type_conversion_p (plm1type, TREE_TYPE (rhs)))
    5209          251 :                     rhs = add_cast (plm1type, rhs);
    5210          251 :                   if (!useless_type_conversion_p (TREE_TYPE (l),
    5211          251 :                                                   TREE_TYPE (rhs)))
    5212          251 :                     rhs = add_cast (TREE_TYPE (l), rhs);
    5213          251 :                   g = gimple_build_assign (l, rhs);
    5214          251 :                   insert_before (g);
    5215              :                 }
    5216          729 :               m_gsi = gsi_after_labels (edge_true_false->dest);
    5217          729 :             }
    5218              :           else
    5219              :             {
    5220         6826 :               tree idxl = idx;
    5221         6826 :               if (bitint_big_endian && prec_limbs != nelts)
    5222              :                 {
    5223            0 :                   HOST_WIDE_INT diff = ((HOST_WIDE_INT) prec_limbs
    5224            0 :                                         - (HOST_WIDE_INT) nelts);
    5225            0 :                   if (tree_fits_uhwi_p (idx))
    5226            0 :                     idxl = size_int (tree_to_uhwi (idx) + diff);
    5227              :                   else
    5228              :                     {
    5229            0 :                       idxl = make_ssa_name (sizetype);
    5230            0 :                       g = gimple_build_assign (idxl, PLUS_EXPR, idx,
    5231            0 :                                                size_int (diff));
    5232            0 :                       insert_before (g);
    5233              :                     }
    5234              :                 }
    5235        13623 :               tree l = limb_access (type, var ? var : obj, idxl, true);
    5236         6826 :               if (bitint_extended && tree_fits_uhwi_p (idxl))
    5237              :                 {
    5238            0 :                   tree atype = limb_access_type (type, idxl);
    5239            0 :                   if (!useless_type_conversion_p (atype, TREE_TYPE (rhs)))
    5240            0 :                     rhs = add_cast (atype, rhs);
    5241              :                 }
    5242         6826 :               if (!useless_type_conversion_p (TREE_TYPE (l), TREE_TYPE (rhs)))
    5243            0 :                 rhs = add_cast (TREE_TYPE (l), rhs);
    5244         6826 :               g = gimple_build_assign (l, rhs);
    5245         6826 :               insert_before (g);
    5246              :             }
    5247              :         }
    5248         9171 :       m_first = false;
    5249         9171 :       if (kind == bitint_prec_huge && i <= 1)
    5250              :         {
    5251         2388 :           if (i == 0)
    5252              :             {
    5253         1194 :               idx = make_ssa_name (sizetype);
    5254         1194 :               g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
    5255              :                                        bitint_big_endian
    5256            0 :                                        ? size_int (-1) : size_one_node);
    5257         1194 :               insert_before (g);
    5258              :             }
    5259              :           else
    5260              :             {
    5261         1194 :               g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
    5262         2388 :                                        size_int (bitint_big_endian ? -2 : 2));
    5263         1194 :               insert_before (g);
    5264         1194 :               if (bitint_big_endian)
    5265            0 :                 g = gimple_build_cond (NE_EXPR, idx_first,
    5266            0 :                                        size_int (nelts + 1 - fin),
    5267              :                                        NULL_TREE, NULL_TREE);
    5268              :               else
    5269         1194 :                 g = gimple_build_cond (NE_EXPR, idx_next, size_int (fin),
    5270              :                                        NULL_TREE, NULL_TREE);
    5271         1194 :               insert_before (g);
    5272         1194 :               m_gsi = gsi_for_stmt (final_stmt);
    5273         1194 :               m_bb = NULL;
    5274              :             }
    5275              :         }
    5276              :     }
    5277              : 
    5278         2733 :   finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
    5279              :                          prec_limbs, code);
    5280              : }
    5281              : 
    5282              : /* Lower a .MUL_OVERFLOW call with at least one large/huge _BitInt
    5283              :    argument or return type _Complex large/huge _BitInt.  */
    5284              : 
    5285              : void
    5286         1479 : bitint_large_huge::lower_mul_overflow (tree obj, gimple *stmt)
    5287              : {
    5288         1479 :   tree arg0 = gimple_call_arg (stmt, 0);
    5289         1479 :   tree arg1 = gimple_call_arg (stmt, 1);
    5290         1479 :   tree lhs = gimple_call_lhs (stmt);
    5291         1479 :   if (!lhs)
    5292              :     {
    5293            0 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    5294            0 :       gsi_remove (&gsi, true);
    5295            0 :       return;
    5296              :     }
    5297         1479 :   gimple *final_stmt = gsi_stmt (m_gsi);
    5298         1479 :   tree type = TREE_TYPE (lhs);
    5299         1479 :   if (TREE_CODE (type) == COMPLEX_TYPE)
    5300         1464 :     type = TREE_TYPE (type);
    5301         1479 :   int prec = TYPE_PRECISION (type), prec0, prec1;
    5302         1479 :   arg0 = handle_operand_addr (arg0, stmt, NULL, &prec0);
    5303         1479 :   arg1 = handle_operand_addr (arg1, stmt, NULL, &prec1);
    5304         1479 :   int prec2 = ((prec0 < 0 ? -prec0 : prec0)
    5305         1479 :                + (prec1 < 0 ? -prec1 : prec1));
    5306         1479 :   if (prec0 == 1 || prec1 == 1)
    5307           35 :     --prec2;
    5308         1479 :   tree var = NULL_TREE;
    5309         1479 :   tree orig_obj = obj;
    5310         1479 :   bool force_var = false;
    5311         1479 :   if (obj == NULL_TREE
    5312          886 :       && BITINT_TYPE_P (type)
    5313          877 :       && bitint_precision_kind (type) >= bitint_prec_large
    5314          853 :       && m_names
    5315         2320 :       && bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
    5316              :     {
    5317          841 :       int part = var_to_partition (m_map, lhs);
    5318          841 :       gcc_assert (m_vars[part] != NULL_TREE);
    5319          841 :       obj = m_vars[part];
    5320          841 :       if (TREE_TYPE (lhs) == type)
    5321            7 :         orig_obj = obj;
    5322              :     }
    5323          638 :   else if (obj != NULL_TREE && DECL_P (obj))
    5324              :     {
    5325         1755 :       for (int i = 0; i < 2; ++i)
    5326              :         {
    5327         1170 :           tree arg = i ? arg1 : arg0;
    5328         1170 :           if (TREE_CODE (arg) == ADDR_EXPR)
    5329         1170 :             arg = TREE_OPERAND (arg, 0);
    5330         1170 :           if (get_base_address (arg) == obj)
    5331              :             {
    5332              :               force_var = true;
    5333              :               break;
    5334              :             }
    5335              :         }
    5336              :     }
    5337         1479 :   if (obj == NULL_TREE
    5338         1479 :       || force_var
    5339         1434 :       || !BITINT_TYPE_P (type)
    5340         1434 :       || bitint_precision_kind (type) < bitint_prec_large
    5341         3747 :       || prec2 > (CEIL (prec, limb_prec) * limb_prec * (orig_obj ? 1 : 2)))
    5342              :     {
    5343          564 :       unsigned HOST_WIDE_INT nelts = CEIL (MAX (prec, prec2), limb_prec);
    5344          564 :       tree atype = build_array_type_nelts (m_limb_type, nelts);
    5345          564 :       var = create_tmp_var (atype);
    5346              :     }
    5347         1479 :   tree addr = build_fold_addr_expr (var ? var : obj);
    5348         1479 :   addr = force_gimple_operand_gsi (&m_gsi, addr, true,
    5349              :                                    NULL_TREE, true, GSI_SAME_STMT);
    5350         1479 :   tree sitype = lang_hooks.types.type_for_mode (SImode, 0);
    5351         1479 :   gimple *g
    5352         1479 :     = gimple_build_call_internal (IFN_MULBITINT, 6,
    5353              :                                   addr, build_int_cst (sitype,
    5354         1479 :                                                        MAX (prec2, prec)),
    5355         1479 :                                   arg0, build_int_cst (sitype, prec0),
    5356         1479 :                                   arg1, build_int_cst (sitype, prec1));
    5357         1479 :   insert_before (g);
    5358              : 
    5359         1479 :   unsigned start, end;
    5360         1479 :   bool check_zero;
    5361         1479 :   tree ovf = arith_overflow (MULT_EXPR, type, prec, prec0, prec1, prec2,
    5362              :                              &start, &end, &check_zero);
    5363         1479 :   if (ovf == NULL_TREE)
    5364              :     {
    5365         1477 :       unsigned startlimb = start / limb_prec;
    5366         1477 :       unsigned endlimb = (end - 1) / limb_prec;
    5367         1477 :       unsigned nelts = CEIL (MAX (prec, prec2), limb_prec);
    5368         1477 :       unsigned cnt;
    5369         1477 :       bool use_loop = false;
    5370         1477 :       if (startlimb == endlimb)
    5371              :         cnt = 1;
    5372         1190 :       else if (startlimb + 1 == endlimb)
    5373              :         cnt = 2;
    5374         1013 :       else if ((end % limb_prec) == 0)
    5375              :         {
    5376              :           cnt = 2;
    5377              :           use_loop = true;
    5378              :         }
    5379              :       else
    5380              :         {
    5381          767 :           cnt = 3;
    5382          767 :           use_loop = startlimb + 2 < endlimb;
    5383              :         }
    5384          767 :       if (cnt == 1)
    5385              :         {
    5386          507 :           tree l = limb_access (NULL_TREE, var ? var : obj,
    5387          287 :                                 size_int (bitint_big_endian
    5388              :                                           ? nelts - 1 - startlimb
    5389              :                                           :  startlimb), true);
    5390          287 :           g = gimple_build_assign (make_ssa_name (m_limb_type), l);
    5391          287 :           insert_before (g);
    5392          287 :           l = arith_overflow_extract_bits (start, end, gimple_assign_lhs (g),
    5393              :                                            startlimb, check_zero);
    5394          287 :           ovf = make_ssa_name (boolean_type_node);
    5395          287 :           if (check_zero)
    5396          247 :             g = gimple_build_assign (ovf, NE_EXPR, l,
    5397              :                                      build_zero_cst (m_limb_type));
    5398              :           else
    5399              :             {
    5400           40 :               g = gimple_build_assign (make_ssa_name (m_limb_type),
    5401              :                                        PLUS_EXPR, l,
    5402              :                                        build_int_cst (m_limb_type, 1));
    5403           40 :               insert_before (g);
    5404           40 :               g = gimple_build_assign (ovf, GT_EXPR, gimple_assign_lhs (g),
    5405              :                                        build_int_cst (m_limb_type, 1));
    5406              :             }
    5407          287 :           insert_before (g);
    5408              :         }
    5409              :       else
    5410              :         {
    5411         1190 :           basic_block edge_bb = NULL;
    5412         1190 :           gimple_stmt_iterator gsi = m_gsi;
    5413         1190 :           gsi_prev (&gsi);
    5414         1190 :           edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
    5415         1190 :           edge_bb = e->src;
    5416         1190 :           m_gsi = gsi_end_bb (edge_bb);
    5417              : 
    5418         1190 :           tree cmp = build_zero_cst (m_limb_type);
    5419         5527 :           for (unsigned i = 0; i < cnt; i++)
    5420              :             {
    5421         3147 :               tree idx, idx_next = NULL_TREE;
    5422         3147 :               if (i == 0)
    5423         1190 :                 idx = size_int (bitint_big_endian
    5424              :                                 ? nelts - 1 - startlimb : startlimb);
    5425         1957 :               else if (i == 2)
    5426          767 :                 idx = size_int (bitint_big_endian
    5427              :                                 ? nelts - 1 - endlimb : endlimb);
    5428         1190 :               else if (use_loop)
    5429          665 :                 idx = create_loop (size_int (bitint_big_endian
    5430              :                                              ? nelts - startlimb - 2
    5431              :                                              : startlimb + 1), &idx_next);
    5432              :               else
    5433          525 :                 idx = size_int (bitint_big_endian
    5434              :                                 ? nelts - startlimb - 2 : startlimb + 1);
    5435         4988 :               tree l = limb_access (NULL_TREE, var ? var : obj, idx, true);
    5436         3147 :               g = gimple_build_assign (make_ssa_name (m_limb_type), l);
    5437         3147 :               insert_before (g);
    5438         3147 :               l = gimple_assign_lhs (g);
    5439         3147 :               if (i == 0 || i == 2)
    5440         2724 :                 l = arith_overflow_extract_bits (start, end, l,
    5441              :                                                  i == 0 ? startlimb : endlimb,
    5442              :                                                  check_zero);
    5443         1957 :               if (i == 0 && !check_zero)
    5444              :                 {
    5445          459 :                   cmp = l;
    5446          459 :                   g = gimple_build_assign (make_ssa_name (m_limb_type),
    5447              :                                            PLUS_EXPR, l,
    5448              :                                            build_int_cst (m_limb_type, 1));
    5449          459 :                   insert_before (g);
    5450          459 :                   g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
    5451              :                                          build_int_cst (m_limb_type, 1),
    5452              :                                          NULL_TREE, NULL_TREE);
    5453              :                 }
    5454              :               else
    5455         2688 :                 g = gimple_build_cond (NE_EXPR, l, cmp, NULL_TREE, NULL_TREE);
    5456         3147 :               insert_before (g);
    5457         3147 :               edge e1 = split_block (gsi_bb (m_gsi), g);
    5458         3147 :               e1->flags = EDGE_FALSE_VALUE;
    5459         3147 :               edge e2 = make_edge (e1->src, gimple_bb (final_stmt),
    5460              :                                    EDGE_TRUE_VALUE);
    5461         3147 :               e1->probability = profile_probability::likely ();
    5462         3147 :               e2->probability = e1->probability.invert ();
    5463         3147 :               if (i == 0)
    5464         1190 :                 set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
    5465         3147 :               m_gsi = gsi_after_labels (e1->dest);
    5466         3147 :               if (i == 1 && use_loop)
    5467              :                 {
    5468          665 :                   g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    5469              :                                            bitint_big_endian
    5470            0 :                                            ? size_int (-1) : size_one_node);
    5471          665 :                   insert_before (g);
    5472          665 :                   if (bitint_big_endian)
    5473            0 :                     g = gimple_build_cond (NE_EXPR, idx,
    5474            0 :                                            size_int (nelts - endlimb
    5475              :                                                      - (cnt == 2)),
    5476              :                                            NULL_TREE, NULL_TREE);
    5477              :                   else
    5478          665 :                     g = gimple_build_cond (NE_EXPR, idx_next,
    5479          665 :                                            size_int (endlimb + (cnt == 2)),
    5480              :                                            NULL_TREE, NULL_TREE);
    5481          665 :                   insert_before (g);
    5482          665 :                   edge true_edge, false_edge;
    5483          665 :                   extract_true_false_edges_from_block (gsi_bb (m_gsi),
    5484              :                                                        &true_edge,
    5485              :                                                        &false_edge);
    5486          665 :                   m_gsi = gsi_after_labels (false_edge->dest);
    5487          665 :                   m_bb = NULL;
    5488              :                 }
    5489              :             }
    5490              : 
    5491         1190 :           ovf = make_ssa_name (boolean_type_node);
    5492         1190 :           basic_block bb = gimple_bb (final_stmt);
    5493         1190 :           gphi *phi = create_phi_node (ovf, bb);
    5494         1190 :           edge e1 = find_edge (gsi_bb (m_gsi), bb);
    5495         1190 :           edge_iterator ei;
    5496         5527 :           FOR_EACH_EDGE (e, ei, bb->preds)
    5497              :             {
    5498         4337 :               tree val = e == e1 ? boolean_false_node : boolean_true_node;
    5499         4337 :               add_phi_arg (phi, val, e, UNKNOWN_LOCATION);
    5500              :             }
    5501         1190 :           m_gsi = gsi_for_stmt (final_stmt);
    5502              :         }
    5503              :     }
    5504              : 
    5505         1479 :   finish_arith_overflow (var, obj, type, ovf, lhs, orig_obj, stmt,
    5506         1479 :                          CEIL (MAX (prec, prec2), limb_prec), MULT_EXPR);
    5507              : }
    5508              : 
    5509              : /* Lower REALPART_EXPR or IMAGPART_EXPR stmt extracting part of result from
    5510              :    .{ADD,SUB,MUL}_OVERFLOW call.  */
    5511              : 
    5512              : void
    5513         6022 : bitint_large_huge::lower_cplxpart_stmt (tree obj, gimple *stmt)
    5514              : {
    5515         6022 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    5516         6022 :   rhs1 = TREE_OPERAND (rhs1, 0);
    5517         6022 :   if (obj == NULL_TREE)
    5518              :     {
    5519         5929 :       int part = var_to_partition (m_map, gimple_assign_lhs (stmt));
    5520         5929 :       gcc_assert (m_vars[part] != NULL_TREE);
    5521              :       obj = m_vars[part];
    5522              :     }
    5523         6022 :   if (TREE_CODE (rhs1) == SSA_NAME
    5524         6022 :       && (m_names == NULL
    5525         6021 :           || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
    5526              :     {
    5527         1549 :       lower_call (obj, SSA_NAME_DEF_STMT (rhs1));
    5528         1549 :       return;
    5529              :     }
    5530         4473 :   int part = var_to_partition (m_map, rhs1);
    5531         4473 :   gcc_assert (m_vars[part] != NULL_TREE);
    5532         4473 :   tree var = m_vars[part];
    5533         4473 :   unsigned HOST_WIDE_INT nelts
    5534         4473 :     = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (obj))) / limb_prec;
    5535         4473 :   tree atype = build_array_type_nelts (m_limb_type, nelts);
    5536         4473 :   if (!useless_type_conversion_p (atype, TREE_TYPE (obj)))
    5537          122 :     obj = build1 (VIEW_CONVERT_EXPR, atype, obj);
    5538         4473 :   tree off = build_int_cst (build_pointer_type (TREE_TYPE (var)),
    5539         4473 :                             gimple_assign_rhs_code (stmt) == REALPART_EXPR
    5540         4473 :                             ? 0 : nelts * m_limb_size);
    5541         4473 :   tree v2 = build2 (MEM_REF, atype, build_fold_addr_expr (var), off);
    5542         4473 :   gimple *g = gimple_build_assign (obj, v2);
    5543         4473 :   insert_before (g);
    5544              : }
    5545              : 
    5546              : /* Lower COMPLEX_EXPR stmt.  */
    5547              : 
    5548              : void
    5549           28 : bitint_large_huge::lower_complexexpr_stmt (gimple *stmt)
    5550              : {
    5551           28 :   tree lhs = gimple_assign_lhs (stmt);
    5552           28 :   tree rhs1 = gimple_assign_rhs1 (stmt);
    5553           28 :   tree rhs2 = gimple_assign_rhs2 (stmt);
    5554           28 :   int part = var_to_partition (m_map, lhs);
    5555           28 :   gcc_assert (m_vars[part] != NULL_TREE);
    5556           28 :   lhs = m_vars[part];
    5557           28 :   unsigned HOST_WIDE_INT nelts
    5558           28 :     = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (rhs1))) / limb_prec;
    5559           28 :   tree atype = build_array_type_nelts (m_limb_type, nelts);
    5560           28 :   tree zero = build_zero_cst (build_pointer_type (TREE_TYPE (lhs)));
    5561           28 :   tree v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), zero);
    5562           28 :   tree v2;
    5563           28 :   if (TREE_CODE (rhs1) == SSA_NAME)
    5564              :     {
    5565           28 :       part = var_to_partition (m_map, rhs1);
    5566           28 :       gcc_assert (m_vars[part] != NULL_TREE);
    5567              :       v2 = m_vars[part];
    5568              :     }
    5569            0 :   else if (integer_zerop (rhs1))
    5570            0 :     v2 = build_zero_cst (atype);
    5571              :   else
    5572            0 :     v2 = tree_output_constant_def (rhs1);
    5573           28 :   if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
    5574           18 :     v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
    5575           28 :   gimple *g = gimple_build_assign (v1, v2);
    5576           28 :   insert_before (g);
    5577           28 :   tree off = fold_convert (build_pointer_type (TREE_TYPE (lhs)),
    5578              :                            TYPE_SIZE_UNIT (atype));
    5579           28 :   v1 = build2 (MEM_REF, atype, build_fold_addr_expr (lhs), off);
    5580           28 :   if (TREE_CODE (rhs2) == SSA_NAME)
    5581              :     {
    5582           10 :       part = var_to_partition (m_map, rhs2);
    5583           10 :       gcc_assert (m_vars[part] != NULL_TREE);
    5584              :       v2 = m_vars[part];
    5585              :     }
    5586           18 :   else if (integer_zerop (rhs2))
    5587           18 :     v2 = build_zero_cst (atype);
    5588              :   else
    5589            0 :     v2 = tree_output_constant_def (rhs2);
    5590           28 :   if (!useless_type_conversion_p (atype, TREE_TYPE (v2)))
    5591           10 :     v2 = build1 (VIEW_CONVERT_EXPR, atype, v2);
    5592           28 :   g = gimple_build_assign (v1, v2);
    5593           28 :   insert_before (g);
    5594           28 : }
    5595              : 
    5596              : /* Lower a .{CLZ,CTZ,CLRSB,FFS,PARITY,POPCOUNT} call with one large/huge _BitInt
    5597              :    argument.  */
    5598              : 
    5599              : void
    5600          101 : bitint_large_huge::lower_bit_query (gimple *stmt)
    5601              : {
    5602          101 :   tree arg0 = gimple_call_arg (stmt, 0);
    5603          101 :   tree arg1 = (gimple_call_num_args (stmt) == 2
    5604          101 :                ? gimple_call_arg (stmt, 1) : NULL_TREE);
    5605          101 :   tree lhs = gimple_call_lhs (stmt);
    5606          101 :   gimple *g;
    5607              : 
    5608          101 :   if (!lhs)
    5609              :     {
    5610            0 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    5611            0 :       gsi_remove (&gsi, true);
    5612            0 :       return;
    5613              :     }
    5614          101 :   tree type = TREE_TYPE (arg0);
    5615          101 :   gcc_assert (BITINT_TYPE_P (type));
    5616          101 :   bitint_prec_kind kind = bitint_precision_kind (type);
    5617          101 :   gcc_assert (kind >= bitint_prec_large);
    5618          101 :   enum internal_fn ifn = gimple_call_internal_fn (stmt);
    5619          101 :   enum built_in_function fcode = END_BUILTINS;
    5620          101 :   gcc_assert (TYPE_PRECISION (unsigned_type_node) == limb_prec
    5621              :               || TYPE_PRECISION (long_unsigned_type_node) == limb_prec
    5622              :               || TYPE_PRECISION (long_long_unsigned_type_node) == limb_prec);
    5623          101 :   switch (ifn)
    5624              :     {
    5625           26 :     case IFN_CLZ:
    5626           26 :       if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
    5627              :         fcode = BUILT_IN_CLZ;
    5628           26 :       else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
    5629              :         fcode = BUILT_IN_CLZL;
    5630              :       else
    5631            0 :         fcode = BUILT_IN_CLZLL;
    5632              :       break;
    5633           11 :     case IFN_FFS:
    5634              :       /* .FFS (X) is .CTZ (X, -1) + 1, though under the hood
    5635              :          we don't add the addend at the end.  */
    5636           11 :       arg1 = integer_zero_node;
    5637              :       /* FALLTHRU */
    5638           39 :     case IFN_CTZ:
    5639           39 :       if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
    5640              :         fcode = BUILT_IN_CTZ;
    5641           39 :       else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
    5642              :         fcode = BUILT_IN_CTZL;
    5643              :       else
    5644            0 :         fcode = BUILT_IN_CTZLL;
    5645           39 :       m_upwards = true;
    5646           39 :       break;
    5647            9 :     case IFN_CLRSB:
    5648            9 :       if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
    5649              :         fcode = BUILT_IN_CLRSB;
    5650            9 :       else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
    5651              :         fcode = BUILT_IN_CLRSBL;
    5652              :       else
    5653            0 :         fcode = BUILT_IN_CLRSBLL;
    5654              :       break;
    5655           16 :     case IFN_PARITY:
    5656           16 :       if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
    5657              :         fcode = BUILT_IN_PARITY;
    5658           16 :       else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
    5659              :         fcode = BUILT_IN_PARITYL;
    5660              :       else
    5661            0 :         fcode = BUILT_IN_PARITYLL;
    5662           16 :       m_upwards = true;
    5663           16 :       break;
    5664           11 :     case IFN_POPCOUNT:
    5665           11 :       if (TYPE_PRECISION (unsigned_type_node) == limb_prec)
    5666              :         fcode = BUILT_IN_POPCOUNT;
    5667           11 :       else if (TYPE_PRECISION (long_unsigned_type_node) == limb_prec)
    5668              :         fcode = BUILT_IN_POPCOUNTL;
    5669              :       else
    5670            0 :         fcode = BUILT_IN_POPCOUNTLL;
    5671           11 :       m_upwards = true;
    5672           11 :       break;
    5673            0 :     default:
    5674            0 :       gcc_unreachable ();
    5675              :     }
    5676          101 :   tree fndecl = builtin_decl_explicit (fcode), res = NULL_TREE;
    5677          101 :   unsigned cnt = 0, rem = 0, end = 0, prec = TYPE_PRECISION (type);
    5678          101 :   unsigned nelts = CEIL (prec, limb_prec);
    5679          101 :   struct bq_details { edge e; tree val, addend; } *bqp = NULL;
    5680          101 :   basic_block edge_bb = NULL;
    5681          101 :   if (m_upwards)
    5682              :     {
    5683           66 :       tree idx = NULL_TREE, idx_first = NULL_TREE, idx_next = NULL_TREE;
    5684           66 :       if (kind == bitint_prec_large)
    5685              :         cnt = nelts;
    5686              :       else
    5687              :         {
    5688           39 :           rem = (prec % (2 * limb_prec));
    5689           39 :           end = (prec - rem) / limb_prec;
    5690           39 :           cnt = 2 + CEIL (rem, limb_prec);
    5691           39 :           idx = idx_first = create_loop (bitint_big_endian
    5692           39 :                                          ? size_int (nelts - 1)
    5693              :                                          : size_zero_node, &idx_next);
    5694              :         }
    5695              : 
    5696           66 :       if (ifn == IFN_CTZ || ifn == IFN_FFS)
    5697              :         {
    5698           39 :           gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    5699           39 :           gsi_prev (&gsi);
    5700           39 :           edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
    5701           39 :           edge_bb = e->src;
    5702           39 :           if (kind == bitint_prec_large)
    5703           32 :             m_gsi = gsi_end_bb (edge_bb);
    5704           39 :           bqp = XALLOCAVEC (struct bq_details, cnt);
    5705              :         }
    5706              :       else
    5707           27 :         m_after_stmt = stmt;
    5708           66 :       if (kind != bitint_prec_large)
    5709           39 :         m_upwards_2limb = end;
    5710              : 
    5711          244 :       for (unsigned i = 0; i < cnt; i++)
    5712              :         {
    5713          178 :           m_data_cnt = 0;
    5714          178 :           if (kind == bitint_prec_large)
    5715           83 :             idx = size_int (bitint_big_endian ? nelts - 1 - i : i);
    5716           95 :           else if (i >= 2)
    5717           17 :             idx = size_int (bitint_big_endian
    5718              :                             ? nelts - 1 - end - (i > 2) : end + (i > 2));
    5719              : 
    5720          178 :           tree rhs1 = handle_operand (arg0, idx);
    5721          178 :           if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
    5722              :             {
    5723           32 :               if (!TYPE_UNSIGNED (TREE_TYPE (rhs1)))
    5724            5 :                 rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
    5725           32 :               rhs1 = add_cast (m_limb_type, rhs1);
    5726              :             }
    5727              : 
    5728          178 :           tree in, out, tem;
    5729          178 :           if (ifn == IFN_PARITY)
    5730           43 :             in = prepare_data_in_out (build_zero_cst (m_limb_type), idx, &out);
    5731          135 :           else if (ifn == IFN_FFS)
    5732           29 :             in = prepare_data_in_out (integer_one_node, idx, &out);
    5733              :           else
    5734          106 :             in = prepare_data_in_out (integer_zero_node, idx, &out);
    5735              : 
    5736          178 :           switch (ifn)
    5737              :             {
    5738          104 :             case IFN_CTZ:
    5739          104 :             case IFN_FFS:
    5740          104 :               g = gimple_build_cond (NE_EXPR, rhs1,
    5741              :                                      build_zero_cst (m_limb_type),
    5742              :                                      NULL_TREE, NULL_TREE);
    5743          104 :               insert_before (g);
    5744          104 :               edge e1, e2;
    5745          104 :               e1 = split_block (gsi_bb (m_gsi), g);
    5746          104 :               e1->flags = EDGE_FALSE_VALUE;
    5747          104 :               e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
    5748          104 :               e1->probability = profile_probability::unlikely ();
    5749          104 :               e2->probability = e1->probability.invert ();
    5750          104 :               if (i == 0)
    5751           39 :                 set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
    5752          104 :               m_gsi = gsi_after_labels (e1->dest);
    5753          104 :               bqp[i].e = e2;
    5754          104 :               bqp[i].val = rhs1;
    5755          104 :               if (tree_fits_uhwi_p (idx))
    5756           58 :                 bqp[i].addend
    5757           58 :                   = build_int_cst (integer_type_node,
    5758              :                                    (bitint_big_endian
    5759           58 :                                     ? nelts - 1 - tree_to_uhwi (idx)
    5760           58 :                                     : tree_to_uhwi (idx)) * limb_prec
    5761           58 :                                    + (ifn == IFN_FFS));
    5762              :               else
    5763              :                 {
    5764           46 :                   bqp[i].addend = in;
    5765           46 :                   if (i == 1)
    5766           23 :                     res = out;
    5767              :                   else
    5768           23 :                     res = make_ssa_name (integer_type_node);
    5769           46 :                   g = gimple_build_assign (res, PLUS_EXPR, in,
    5770              :                                            build_int_cst (integer_type_node,
    5771           46 :                                                           limb_prec));
    5772           46 :                   insert_before (g);
    5773           46 :                   m_data[m_data_cnt] = res;
    5774              :                 }
    5775              :               break;
    5776           43 :             case IFN_PARITY:
    5777           43 :               if (!integer_zerop (in))
    5778              :                 {
    5779           37 :                   if (kind == bitint_prec_huge && i == 1)
    5780           10 :                     res = out;
    5781              :                   else
    5782           27 :                     res = make_ssa_name (m_limb_type);
    5783           37 :                   g = gimple_build_assign (res, BIT_XOR_EXPR, in, rhs1);
    5784           37 :                   insert_before (g);
    5785              :                 }
    5786              :               else
    5787              :                 res = rhs1;
    5788           43 :               m_data[m_data_cnt] = res;
    5789           43 :               break;
    5790           31 :             case IFN_POPCOUNT:
    5791           31 :               g = gimple_build_call (fndecl, 1, rhs1);
    5792           31 :               tem = make_ssa_name (integer_type_node);
    5793           31 :               gimple_call_set_lhs (g, tem);
    5794           31 :               insert_before (g);
    5795           31 :               if (!integer_zerop (in))
    5796              :                 {
    5797           26 :                   if (kind == bitint_prec_huge && i == 1)
    5798            6 :                     res = out;
    5799              :                   else
    5800           20 :                     res = make_ssa_name (integer_type_node);
    5801           26 :                   g = gimple_build_assign (res, PLUS_EXPR, in, tem);
    5802           26 :                   insert_before (g);
    5803              :                 }
    5804              :               else
    5805              :                 res = tem;
    5806           31 :               m_data[m_data_cnt] = res;
    5807           31 :               break;
    5808            0 :             default:
    5809            0 :               gcc_unreachable ();
    5810              :             }
    5811              : 
    5812          178 :           m_first = false;
    5813          178 :           if (kind == bitint_prec_huge && i <= 1)
    5814              :             {
    5815           78 :               if (i == 0)
    5816              :                 {
    5817           39 :                   idx = make_ssa_name (sizetype);
    5818           39 :                   g = gimple_build_assign (idx, PLUS_EXPR, idx_first,
    5819              :                                            bitint_big_endian
    5820            0 :                                            ? size_int (-1) : size_one_node);
    5821           39 :                   insert_before (g);
    5822              :                 }
    5823              :               else
    5824              :                 {
    5825           39 :                   g = gimple_build_assign (idx_next, PLUS_EXPR, idx_first,
    5826           78 :                                            size_int (bitint_big_endian
    5827              :                                                      ? -2 : 2));
    5828           39 :                   insert_before (g);
    5829           39 :                   if (bitint_big_endian)
    5830            0 :                     g = gimple_build_cond (NE_EXPR, idx_first,
    5831            0 :                                            size_int (cnt - 1),
    5832              :                                            NULL_TREE, NULL_TREE);
    5833              :                   else
    5834           39 :                     g = gimple_build_cond (NE_EXPR, idx_next, size_int (end),
    5835              :                                            NULL_TREE, NULL_TREE);
    5836           39 :                   insert_before (g);
    5837           39 :                   if (ifn == IFN_CTZ || ifn == IFN_FFS)
    5838           23 :                     m_gsi = gsi_after_labels (edge_bb);
    5839              :                   else
    5840           16 :                     m_gsi = gsi_for_stmt (stmt);
    5841           39 :                   m_bb = NULL;
    5842              :                 }
    5843              :             }
    5844              :         }
    5845              :     }
    5846              :   else
    5847              :     {
    5848           35 :       tree idx = NULL_TREE, idx_next = NULL_TREE, first = NULL_TREE;
    5849           35 :       int sub_one = 0;
    5850           35 :       if (kind == bitint_prec_large)
    5851              :         cnt = nelts;
    5852              :       else
    5853              :         {
    5854           18 :           rem = prec % limb_prec;
    5855           18 :           if (rem == 0 && (!TYPE_UNSIGNED (type) || ifn == IFN_CLRSB))
    5856              :             rem = limb_prec;
    5857           18 :           end = (prec - rem) / limb_prec;
    5858           18 :           cnt = 1 + (rem != 0);
    5859           18 :           if (ifn == IFN_CLRSB)
    5860            5 :             sub_one = 1;
    5861              :         }
    5862              : 
    5863           35 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    5864           35 :       gsi_prev (&gsi);
    5865           35 :       edge e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
    5866           35 :       edge_bb = e->src;
    5867           35 :       m_gsi = gsi_end_bb (edge_bb);
    5868              : 
    5869           35 :       if (ifn == IFN_CLZ)
    5870           26 :         bqp = XALLOCAVEC (struct bq_details, cnt);
    5871              :       else
    5872              :         {
    5873            9 :           gsi = gsi_for_stmt (stmt);
    5874            9 :           gsi_prev (&gsi);
    5875            9 :           e = split_block (gsi_bb (gsi), gsi_stmt (gsi));
    5876            9 :           edge_bb = e->src;
    5877            9 :           bqp = XALLOCAVEC (struct bq_details, 2 * cnt);
    5878              :         }
    5879              : 
    5880          116 :       for (unsigned i = 0; i < cnt; i++)
    5881              :         {
    5882           81 :           m_data_cnt = 0;
    5883           81 :           if (kind == bitint_prec_large)
    5884           51 :             idx = size_int (bitint_big_endian ? i : cnt - i - 1);
    5885           30 :           else if (i == cnt - 1)
    5886           18 :             idx = create_loop (size_int (bitint_big_endian ? i : end - 1),
    5887              :                                &idx_next);
    5888              :           else
    5889           12 :             idx = bitint_big_endian ? size_zero_node : size_int (end);
    5890              : 
    5891           81 :           tree rhs1 = handle_operand (arg0, idx);
    5892           81 :           if (!useless_type_conversion_p (m_limb_type, TREE_TYPE (rhs1)))
    5893              :             {
    5894           19 :               if (ifn == IFN_CLZ && !TYPE_UNSIGNED (TREE_TYPE (rhs1)))
    5895            0 :                 rhs1 = add_cast (unsigned_type_for (TREE_TYPE (rhs1)), rhs1);
    5896           19 :               else if (ifn == IFN_CLRSB && TYPE_UNSIGNED (TREE_TYPE (rhs1)))
    5897            0 :                 rhs1 = add_cast (signed_type_for (TREE_TYPE (rhs1)), rhs1);
    5898           19 :               rhs1 = add_cast (m_limb_type, rhs1);
    5899              :             }
    5900              : 
    5901           81 :           if (ifn == IFN_CLZ)
    5902              :             {
    5903           59 :               g = gimple_build_cond (NE_EXPR, rhs1,
    5904              :                                      build_zero_cst (m_limb_type),
    5905              :                                      NULL_TREE, NULL_TREE);
    5906           59 :               insert_before (g);
    5907           59 :               edge e1 = split_block (gsi_bb (m_gsi), g);
    5908           59 :               e1->flags = EDGE_FALSE_VALUE;
    5909           59 :               edge e2 = make_edge (e1->src, gimple_bb (stmt), EDGE_TRUE_VALUE);
    5910           59 :               e1->probability = profile_probability::unlikely ();
    5911           59 :               e2->probability = e1->probability.invert ();
    5912           59 :               if (i == 0)
    5913           26 :                 set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
    5914           59 :               m_gsi = gsi_after_labels (e1->dest);
    5915           59 :               bqp[i].e = e2;
    5916           59 :               bqp[i].val = rhs1;
    5917              :             }
    5918              :           else
    5919              :             {
    5920           22 :               if (i == 0)
    5921              :                 {
    5922            9 :                   first = rhs1;
    5923            9 :                   g = gimple_build_assign (make_ssa_name (m_limb_type),
    5924              :                                            PLUS_EXPR, rhs1,
    5925              :                                            build_int_cst (m_limb_type, 1));
    5926            9 :                   insert_before (g);
    5927            9 :                   g = gimple_build_cond (GT_EXPR, gimple_assign_lhs (g),
    5928              :                                          build_int_cst (m_limb_type, 1),
    5929              :                                          NULL_TREE, NULL_TREE);
    5930            9 :                   insert_before (g);
    5931              :                 }
    5932              :               else
    5933              :                 {
    5934           13 :                   g = gimple_build_assign (make_ssa_name (m_limb_type),
    5935              :                                            BIT_XOR_EXPR, rhs1, first);
    5936           13 :                   insert_before (g);
    5937           13 :                   tree stype = signed_type_for (m_limb_type);
    5938           13 :                   g = gimple_build_cond (LT_EXPR,
    5939              :                                          add_cast (stype,
    5940              :                                                    gimple_assign_lhs (g)),
    5941              :                                          build_zero_cst (stype),
    5942              :                                          NULL_TREE, NULL_TREE);
    5943           13 :                   insert_before (g);
    5944           13 :                   edge e1 = split_block (gsi_bb (m_gsi), g);
    5945           13 :                   e1->flags = EDGE_FALSE_VALUE;
    5946           13 :                   edge e2 = make_edge (e1->src, gimple_bb (stmt),
    5947              :                                        EDGE_TRUE_VALUE);
    5948           13 :                   e1->probability = profile_probability::unlikely ();
    5949           13 :                   e2->probability = e1->probability.invert ();
    5950           13 :                   if (i == 1)
    5951            9 :                     set_immediate_dominator (CDI_DOMINATORS, e2->dest,
    5952              :                                              e2->src);
    5953           13 :                   m_gsi = gsi_after_labels (e1->dest);
    5954           13 :                   bqp[2 * i].e = e2;
    5955           13 :                   g = gimple_build_cond (NE_EXPR, rhs1, first,
    5956              :                                          NULL_TREE, NULL_TREE);
    5957           13 :                   insert_before (g);
    5958              :                 }
    5959           22 :               edge e1 = split_block (gsi_bb (m_gsi), g);
    5960           22 :               e1->flags = EDGE_FALSE_VALUE;
    5961           22 :               edge e2 = make_edge (e1->src, edge_bb, EDGE_TRUE_VALUE);
    5962           22 :               e1->probability = profile_probability::unlikely ();
    5963           22 :               e2->probability = e1->probability.invert ();
    5964           22 :               if (i == 0)
    5965            9 :                 set_immediate_dominator (CDI_DOMINATORS, e2->dest, e2->src);
    5966           22 :               m_gsi = gsi_after_labels (e1->dest);
    5967           22 :               bqp[2 * i + 1].e = e2;
    5968           22 :               bqp[i].val = rhs1;
    5969              :             }
    5970           81 :           if (tree_fits_uhwi_p (idx))
    5971          126 :             bqp[i].addend
    5972           63 :               = build_int_cst (integer_type_node,
    5973           63 :                                (int) prec
    5974           63 :                                - (((int) (bitint_big_endian
    5975            0 :                                           ? nelts - 1 - tree_to_uhwi (idx)
    5976           63 :                                           : tree_to_uhwi (idx)) + 1)
    5977           63 :                                   * limb_prec) - sub_one);
    5978              :           else
    5979              :             {
    5980           18 :               tree in, out;
    5981           18 :               in = build_int_cst (integer_type_node, rem - sub_one);
    5982           18 :               m_first = true;
    5983           18 :               in = prepare_data_in_out (in, idx, &out);
    5984           18 :               out = m_data[m_data_cnt + 1];
    5985           18 :               bqp[i].addend = in;
    5986           18 :               g = gimple_build_assign (out, PLUS_EXPR, in,
    5987              :                                        build_int_cst (integer_type_node,
    5988           18 :                                                       limb_prec));
    5989           18 :               insert_before (g);
    5990           18 :               m_data[m_data_cnt] = out;
    5991              :             }
    5992              : 
    5993           81 :           m_first = false;
    5994           81 :           if (kind == bitint_prec_huge && i == cnt - 1)
    5995              :             {
    5996           36 :               g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    5997              :                                        bitint_big_endian
    5998           18 :                                        ? size_one_node : size_int (-1));
    5999           18 :               insert_before (g);
    6000           18 :               g = gimple_build_cond (NE_EXPR, idx,
    6001              :                                      bitint_big_endian
    6002            0 :                                      ? size_int (nelts - 1) : size_zero_node,
    6003              :                                      NULL_TREE, NULL_TREE);
    6004           18 :               insert_before (g);
    6005           18 :               edge true_edge, false_edge;
    6006           18 :               extract_true_false_edges_from_block (gsi_bb (m_gsi),
    6007              :                                                    &true_edge, &false_edge);
    6008           18 :               m_gsi = gsi_after_labels (false_edge->dest);
    6009           18 :               m_bb = NULL;
    6010              :             }
    6011              :         }
    6012              :     }
    6013          101 :   switch (ifn)
    6014              :     {
    6015           65 :     case IFN_CLZ:
    6016           65 :     case IFN_CTZ:
    6017           65 :     case IFN_FFS:
    6018           65 :       gphi *phi1, *phi2, *phi3;
    6019           65 :       basic_block bb;
    6020           65 :       bb = gsi_bb (m_gsi);
    6021           65 :       remove_edge (find_edge (bb, gimple_bb (stmt)));
    6022           65 :       phi1 = create_phi_node (make_ssa_name (m_limb_type),
    6023              :                               gimple_bb (stmt));
    6024           65 :       phi2 = create_phi_node (make_ssa_name (integer_type_node),
    6025              :                               gimple_bb (stmt));
    6026          293 :       for (unsigned i = 0; i < cnt; i++)
    6027              :         {
    6028          163 :           add_phi_arg (phi1, bqp[i].val, bqp[i].e, UNKNOWN_LOCATION);
    6029          163 :           add_phi_arg (phi2, bqp[i].addend, bqp[i].e, UNKNOWN_LOCATION);
    6030              :         }
    6031           65 :       if (arg1 == NULL_TREE)
    6032              :         {
    6033           35 :           g = gimple_build_builtin_unreachable (m_loc);
    6034           35 :           insert_before (g);
    6035              :         }
    6036           65 :       m_gsi = gsi_for_stmt (stmt);
    6037           65 :       g = gimple_build_call (fndecl, 1, gimple_phi_result (phi1));
    6038           65 :       gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
    6039           65 :       insert_before (g);
    6040           65 :       if (arg1 == NULL_TREE)
    6041           35 :         g = gimple_build_assign (lhs, PLUS_EXPR,
    6042              :                                  gimple_phi_result (phi2),
    6043              :                                  gimple_call_lhs (g));
    6044              :       else
    6045              :         {
    6046           30 :           g = gimple_build_assign (make_ssa_name (integer_type_node),
    6047              :                                    PLUS_EXPR, gimple_phi_result (phi2),
    6048              :                                    gimple_call_lhs (g));
    6049           30 :           insert_before (g);
    6050           30 :           edge e1 = split_block (gimple_bb (stmt), g);
    6051           30 :           edge e2 = make_edge (bb, e1->dest, EDGE_FALLTHRU);
    6052           30 :           e2->probability = profile_probability::always ();
    6053           30 :           set_immediate_dominator (CDI_DOMINATORS, e1->dest,
    6054              :                                    get_immediate_dominator (CDI_DOMINATORS,
    6055              :                                                             e1->src));
    6056           30 :           phi3 = create_phi_node (make_ssa_name (integer_type_node), e1->dest);
    6057           30 :           add_phi_arg (phi3, gimple_assign_lhs (g), e1, UNKNOWN_LOCATION);
    6058           30 :           add_phi_arg (phi3, arg1, e2, UNKNOWN_LOCATION);
    6059           30 :           m_gsi = gsi_for_stmt (stmt);
    6060           30 :           g = gimple_build_assign (lhs, gimple_phi_result (phi3));
    6061              :         }
    6062           65 :       gsi_replace (&m_gsi, g, true);
    6063           65 :       break;
    6064            9 :     case IFN_CLRSB:
    6065            9 :       bb = gsi_bb (m_gsi);
    6066            9 :       remove_edge (find_edge (bb, edge_bb));
    6067            9 :       edge e;
    6068            9 :       e = make_edge (bb, gimple_bb (stmt), EDGE_FALLTHRU);
    6069            9 :       e->probability = profile_probability::always ();
    6070            9 :       set_immediate_dominator (CDI_DOMINATORS, gimple_bb (stmt),
    6071              :                                get_immediate_dominator (CDI_DOMINATORS,
    6072              :                                                         edge_bb));
    6073            9 :       phi1 = create_phi_node (make_ssa_name (m_limb_type),
    6074              :                               edge_bb);
    6075            9 :       phi2 = create_phi_node (make_ssa_name (integer_type_node),
    6076              :                               edge_bb);
    6077            9 :       phi3 = create_phi_node (make_ssa_name (integer_type_node),
    6078              :                               gimple_bb (stmt));
    6079           40 :       for (unsigned i = 0; i < cnt; i++)
    6080              :         {
    6081           22 :           add_phi_arg (phi1, bqp[i].val, bqp[2 * i + 1].e, UNKNOWN_LOCATION);
    6082           22 :           add_phi_arg (phi2, bqp[i].addend, bqp[2 * i + 1].e,
    6083              :                        UNKNOWN_LOCATION);
    6084           22 :           tree a = bqp[i].addend;
    6085           22 :           if (i && kind == bitint_prec_large)
    6086            8 :             a = int_const_binop (PLUS_EXPR, a, integer_minus_one_node);
    6087           22 :           if (i)
    6088           13 :             add_phi_arg (phi3, a, bqp[2 * i].e, UNKNOWN_LOCATION);
    6089              :         }
    6090            9 :       add_phi_arg (phi3, build_int_cst (integer_type_node, prec - 1), e,
    6091              :                    UNKNOWN_LOCATION);
    6092            9 :       m_gsi = gsi_after_labels (edge_bb);
    6093            9 :       g = gimple_build_call (fndecl, 1,
    6094              :                              add_cast (signed_type_for (m_limb_type),
    6095              :                                        gimple_phi_result (phi1)));
    6096            9 :       gimple_call_set_lhs (g, make_ssa_name (integer_type_node));
    6097            9 :       insert_before (g);
    6098            9 :       g = gimple_build_assign (make_ssa_name (integer_type_node),
    6099              :                                PLUS_EXPR, gimple_call_lhs (g),
    6100              :                                gimple_phi_result (phi2));
    6101            9 :       insert_before (g);
    6102            9 :       if (kind != bitint_prec_large)
    6103              :         {
    6104            5 :           g = gimple_build_assign (make_ssa_name (integer_type_node),
    6105              :                                    PLUS_EXPR, gimple_assign_lhs (g),
    6106              :                                    integer_one_node);
    6107            5 :           insert_before (g);
    6108              :         }
    6109            9 :       add_phi_arg (phi3, gimple_assign_lhs (g),
    6110              :                    find_edge (edge_bb, gimple_bb (stmt)), UNKNOWN_LOCATION);
    6111            9 :       m_gsi = gsi_for_stmt (stmt);
    6112            9 :       g = gimple_build_assign (lhs, gimple_phi_result (phi3));
    6113            9 :       gsi_replace (&m_gsi, g, true);
    6114            9 :       break;
    6115           16 :     case IFN_PARITY:
    6116           16 :       g = gimple_build_call (fndecl, 1, res);
    6117           16 :       gimple_call_set_lhs (g, lhs);
    6118           16 :       gsi_replace (&m_gsi, g, true);
    6119           16 :       break;
    6120           11 :     case IFN_POPCOUNT:
    6121           11 :       g = gimple_build_assign (lhs, res);
    6122           11 :       gsi_replace (&m_gsi, g, true);
    6123           11 :       break;
    6124            0 :     default:
    6125            0 :       gcc_unreachable ();
    6126              :     }
    6127              : }
    6128              : 
    6129              : /* Lower a .{BSWAP,BITREVERSE} call with one large/huge _BitInt argument.  */
    6130              : 
    6131              : void
    6132           47 : bitint_large_huge::lower_bswap_bitreverse (tree obj, gimple *stmt)
    6133              : {
    6134           47 :   tree arg = gimple_call_arg (stmt, 0);
    6135           47 :   tree lhs = gimple_call_lhs (stmt);
    6136           47 :   gimple *g;
    6137              : 
    6138           47 :   if (!lhs)
    6139              :     {
    6140            0 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    6141            0 :       gsi_remove (&gsi, true);
    6142            0 :       return;
    6143              :     }
    6144           47 :   tree type = TREE_TYPE (arg);
    6145           47 :   gcc_assert (BITINT_TYPE_P (type));
    6146           47 :   bitint_prec_kind kind = bitint_precision_kind (type);
    6147           47 :   gcc_assert (kind >= bitint_prec_large);
    6148           47 :   if (!obj)
    6149              :     {
    6150           34 :       int part = var_to_partition (m_map, lhs);
    6151           34 :       gcc_assert (m_vars[part] != NULL_TREE);
    6152              :       obj = m_vars[part];
    6153              :     }
    6154           47 :   enum internal_fn ifn = gimple_call_internal_fn (stmt);
    6155           47 :   enum built_in_function bcode = END_BUILTINS;
    6156           47 :   switch (limb_prec)
    6157              :     {
    6158            0 :     case 16:
    6159            0 :       bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP16 : BUILT_IN_BITREVERSE16;
    6160              :       break;
    6161            0 :     case 32:
    6162            0 :       bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP32 : BUILT_IN_BITREVERSE32;
    6163              :       break;
    6164           47 :     case 64:
    6165           47 :       bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP64 : BUILT_IN_BITREVERSE64;
    6166              :       break;
    6167            0 :     case 128:
    6168            0 :       bcode = ifn == IFN_BSWAP ? BUILT_IN_BSWAP128 : BUILT_IN_BITREVERSE128;
    6169              :       break;
    6170            0 :     default:
    6171            0 :       gcc_unreachable ();
    6172              :     }
    6173           47 :   tree fndecl = builtin_decl_explicit (bcode);
    6174           47 :   unsigned prec = TYPE_PRECISION (type);
    6175           94 :   tree p = build_int_cst (sizetype,
    6176           47 :                           prec / limb_prec - (prec % limb_prec == 0));
    6177              :   /* For .BSWAP/.BITREVERSE, we can use handle_operand with non-INTEGER_CST
    6178              :      idx even to access the most significant partial limb.  */
    6179           47 :   m_var_msb = true;
    6180              :   /* For IFN .BSWAP or .BITREVERSE and
    6181              :      FN corresponding __builtin_bswapN or __builtin_bitreverseN where N
    6182              :      is limb_prec, lower
    6183              :        dst = IFN (src);
    6184              :      as
    6185              :        size_t p = prec / limb_prec - (prec % limb_prec == 0);
    6186              :        if constexpr ((prec % limb_prec) == 0)
    6187              :          {
    6188              :            for (idx = 0; idx <= p; ++idx)
    6189              :              dst[p - idx] = FN (src[idx]);
    6190              :          }
    6191              :        else
    6192              :          {
    6193              :            unsigned n1 = prec % limb_prec;
    6194              :            unsigned n2 = limb_prec - n1;
    6195              :            dst[p] = FN (src[0]) >> n2;
    6196              :            for (idx = p - 1; (ssize_t) idx >= 0; --idx)
    6197              :              dst[idx] = FN ((src[p - idx] << n2) | (src[p - idx - 1] >> n1));
    6198              :          }  */
    6199           47 :   if (prec % limb_prec == 0)
    6200              :     {
    6201            9 :       tree idx_next;
    6202            9 :       tree idx = create_loop (size_zero_node, &idx_next);
    6203            9 :       tree pmidx = make_ssa_name (sizetype);
    6204            9 :       g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
    6205            9 :       insert_before (g);
    6206            9 :       m_data_cnt = 0;
    6207            9 :       tree t = handle_operand (arg, idx);
    6208            9 :       m_first = false;
    6209            9 :       g = gimple_build_call (fndecl, 1, t);
    6210            9 :       t = make_ssa_name (m_limb_type);
    6211            9 :       gimple_call_set_lhs (g, t);
    6212            9 :       insert_before (g);
    6213            9 :       tree l = limb_access (TREE_TYPE (lhs), obj, pmidx, true);
    6214            9 :       g = gimple_build_assign (l, t);
    6215            9 :       insert_before (g);
    6216            9 :       g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_one_node);
    6217            9 :       insert_before (g);
    6218            9 :       g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
    6219            9 :       insert_before (g);
    6220              :     }
    6221              :   else
    6222              :     {
    6223           38 :       tree n1 = build_int_cst (unsigned_type_node, prec % limb_prec);
    6224           76 :       tree n2 = build_int_cst (unsigned_type_node,
    6225           38 :                                limb_prec - (prec % limb_prec));
    6226           38 :       m_data_cnt = 0;
    6227           38 :       tree t = handle_operand (arg, bitint_big_endian ? p : size_zero_node);
    6228           38 :       m_first = false;
    6229              :       /* Nothing is needed to bswap 8 bits or bitreverse 1 bit, so just
    6230              :          mask off higher bits in that case.  */
    6231           71 :       if ((prec % limb_prec) != (ifn == IFN_BSWAP ? 8 : 1))
    6232              :         {
    6233           26 :           g = gimple_build_call (fndecl, 1, t);
    6234           26 :           t = make_ssa_name (m_limb_type);
    6235           26 :           gimple_call_set_lhs (g, t);
    6236           26 :           insert_before (g);
    6237           26 :           g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
    6238              :                                    t, n2);
    6239              :         }
    6240              :       else
    6241           12 :         g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_AND_EXPR,
    6242              :                                  t, build_int_cst (m_limb_type,
    6243              :                                                    ifn == IFN_BSWAP
    6244           22 :                                                    ? 0xff : 1));
    6245           38 :       insert_before (g);
    6246           38 :       t = gimple_assign_lhs (g);
    6247           38 :       tree l = limb_access (TREE_TYPE (lhs), obj,
    6248              :                             bitint_big_endian ? size_zero_node : p, true);
    6249           38 :       g = gimple_build_assign (l, t);
    6250           38 :       insert_before (g);
    6251           38 :       tree pm1 = build_int_cst (sizetype, prec / limb_prec - 1);
    6252           38 :       tree idx_next;
    6253           38 :       tree idx = create_loop (pm1, &idx_next);
    6254           38 :       tree pmidx = make_ssa_name (sizetype);
    6255           38 :       g = gimple_build_assign (pmidx, MINUS_EXPR, p, idx);
    6256           38 :       insert_before (g);
    6257           38 :       m_data_cnt = 0;
    6258           76 :       t = handle_operand (arg, bitint_big_endian ? idx : pmidx);
    6259           38 :       m_data_cnt = 0;
    6260           38 :       g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
    6261              :                                t, n2);
    6262           38 :       insert_before (g);
    6263           38 :       t = gimple_assign_lhs (g);
    6264           38 :       tree t2 = make_ssa_name (sizetype);
    6265           38 :       if (bitint_big_endian)
    6266            0 :         g = gimple_build_assign (t2, PLUS_EXPR, idx, size_one_node);
    6267              :       else
    6268           38 :         g = gimple_build_assign (t2, PLUS_EXPR, pmidx, size_int (-1));
    6269           38 :       insert_before (g);
    6270           38 :       t2 = handle_operand (arg, t2);
    6271           38 :       g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
    6272              :                                t2, n1);
    6273           38 :       insert_before (g);
    6274           38 :       t2 = gimple_assign_lhs (g);
    6275           38 :       g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
    6276              :                                t, t2);
    6277           38 :       insert_before (g);
    6278           38 :       t = gimple_assign_lhs (g);
    6279           38 :       g = gimple_build_call (fndecl, 1, t);
    6280           38 :       t = make_ssa_name (m_limb_type);
    6281           38 :       gimple_call_set_lhs (g, t);
    6282           38 :       insert_before (g);
    6283           76 :       l = limb_access (TREE_TYPE (lhs), obj,
    6284              :                        bitint_big_endian ? pmidx : idx, true);
    6285           38 :       g = gimple_build_assign (l, t);
    6286           38 :       insert_before (g);
    6287           38 :       g = gimple_build_assign (idx_next, PLUS_EXPR, idx, size_int (-1));
    6288           38 :       insert_before (g);
    6289           38 :       g = gimple_build_cond (NE_EXPR, idx, size_zero_node, NULL_TREE,
    6290              :                              NULL_TREE);
    6291           38 :       insert_before (g);
    6292              :     }
    6293           47 :   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    6294           47 :   if (bitint_extended == bitint_ext_full
    6295            0 :       && abi_limb_prec > limb_prec
    6296            0 :       && (CEIL (prec, abi_limb_prec) * abi_limb_prec
    6297            0 :           > CEIL (prec, limb_prec) * limb_prec))
    6298              :     {
    6299            0 :       m_gsi = gsi;
    6300            0 :       tree p2 = build_int_cst (sizetype,
    6301              :                                CEIL (prec, abi_limb_prec)
    6302            0 :                                * abi_limb_prec / limb_prec - 1);
    6303            0 :       tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
    6304            0 :       g = gimple_build_assign (l, build_zero_cst (m_limb_type));
    6305            0 :       insert_before (g);
    6306              :     }
    6307              : }
    6308              : 
    6309              : /* Lower a .CLMUL call with two large/huge _BitInt arguments.  */
    6310              : 
    6311              : void
    6312           30 : bitint_large_huge::lower_clmul (tree obj, gimple *stmt)
    6313              : {
    6314           30 :   tree arg1 = gimple_call_arg (stmt, 0);
    6315           30 :   tree arg2 = gimple_call_arg (stmt, 1);
    6316           30 :   tree lhs = gimple_call_lhs (stmt);
    6317           30 :   gimple *g;
    6318              : 
    6319           30 :   if (!lhs)
    6320              :     {
    6321            0 :       gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    6322            0 :       gsi_remove (&gsi, true);
    6323            0 :       return;
    6324              :     }
    6325           60 :   if (wi::popcount (get_nonzero_bits (arg1))
    6326           48 :       < wi::popcount (get_nonzero_bits (arg2)))
    6327            0 :     std::swap (arg1, arg2);
    6328           30 :   tree type = TREE_TYPE (arg1);
    6329           30 :   gcc_assert (BITINT_TYPE_P (type));
    6330           30 :   bitint_prec_kind kind = bitint_precision_kind (type);
    6331           30 :   gcc_assert (kind >= bitint_prec_large);
    6332           30 :   if (!obj)
    6333              :     {
    6334           20 :       int part = var_to_partition (m_map, lhs);
    6335           20 :       gcc_assert (m_vars[part] != NULL_TREE);
    6336              :       obj = m_vars[part];
    6337              :     }
    6338              :   /* For .CLMUL, we can use handle_operand with non-INTEGER_CST
    6339              :      idx even to access the most significant partial limb.  */
    6340           30 :   m_var_msb = true;
    6341           30 :   unsigned prec = TYPE_PRECISION (type);
    6342              :   /* Lower
    6343              :        dst = .CLMUL (src1, src2);
    6344              :      as
    6345              :        size_t p = prec / limb_prec - (prec % limb_prec == 0);
    6346              :        dst = {};
    6347              :        for (idx = 0; idx <= p; ++idx)
    6348              :          {
    6349              :            limb_type t = src2[idx];
    6350              :            size_t i = 0, j = 0;
    6351              :            do
    6352              :              {
    6353              :                if (t & 1)
    6354              :                  {
    6355              :                    size_t idx3 = idx + j;
    6356              :                    size_t k = -i & (limb_prec - 1);
    6357              :                    r[idx] ^= (src1[0] << i);
    6358              :                    for (idx2 = idx + 1; idx2 <= p; ++idx2)
    6359              :                      r[idx2] ^= (src1[idx2 - idx] << i)
    6360              :                                 | (src1[idx2 - idx3] >> k);
    6361              :                  }
    6362              :                t >>= 1;
    6363              :                ++i;
    6364              :                j = 1;
    6365              :              }
    6366              :            while (t);
    6367              :          }  */
    6368           60 :   tree p = build_int_cst (sizetype,
    6369           30 :                           prec / limb_prec - (prec % limb_prec == 0));
    6370           30 :   tree zero = build_zero_cst (TREE_TYPE (obj));
    6371           30 :   g = gimple_build_assign (obj, zero);
    6372           30 :   insert_before (g);
    6373           30 :   tree idx_next;
    6374           30 :   tree idx = create_loop (bitint_big_endian ? p : size_zero_node, &idx_next);
    6375           30 :   m_data_cnt = 0;
    6376           30 :   tree t_init = handle_operand (arg2, idx);
    6377           30 :   m_first = false;
    6378           30 :   tree i_next;
    6379           30 :   tree i = create_loop (size_zero_node, &i_next);
    6380           30 :   tree j = make_ssa_name (sizetype);
    6381           30 :   gphi *phi = create_phi_node (j, m_bb);
    6382           30 :   add_phi_arg (phi, size_zero_node, find_edge (m_preheader_bb, m_bb),
    6383              :                UNKNOWN_LOCATION);
    6384           30 :   add_phi_arg (phi, size_one_node, find_edge (m_bb, m_bb), UNKNOWN_LOCATION);
    6385           30 :   tree t = make_ssa_name (m_limb_type);
    6386           30 :   phi = create_phi_node (t, m_bb);
    6387           30 :   add_phi_arg (phi, t_init, find_edge (m_preheader_bb, m_bb),
    6388              :                UNKNOWN_LOCATION);
    6389           30 :   tree t_next = make_ssa_name (m_limb_type);
    6390           30 :   add_phi_arg (phi, t_next, find_edge (m_bb, m_bb), UNKNOWN_LOCATION);
    6391           30 :   tree t1 = make_ssa_name (m_limb_type);
    6392           30 :   g = gimple_build_assign (t1, BIT_AND_EXPR, t, build_one_cst (m_limb_type));
    6393           30 :   insert_before (g);
    6394           30 :   g = gimple_build_cond (NE_EXPR, t1, build_zero_cst (m_limb_type),
    6395              :                          NULL_TREE, NULL_TREE);
    6396           30 :   edge edge_true, edge_false;
    6397           30 :   if_then (g, profile_probability::even (), edge_true, edge_false);
    6398           30 :   tree idx3 = make_ssa_name (sizetype), idx4 = idx;
    6399           60 :   g = gimple_build_assign (idx3, bitint_big_endian ? MINUS_EXPR : PLUS_EXPR,
    6400              :                            idx, j);
    6401           30 :   insert_before (g);
    6402           30 :   if (bitint_big_endian)
    6403              :     {
    6404            0 :       g = gimple_build_assign (make_ssa_name (sizetype), MINUS_EXPR, p, idx3);
    6405            0 :       insert_before (g);
    6406            0 :       idx3 = gimple_assign_lhs (g);
    6407            0 :       idx4 = make_ssa_name (sizetype);
    6408            0 :       g = gimple_build_assign (idx4, MINUS_EXPR, p, idx);
    6409            0 :       insert_before (g);
    6410              :     }
    6411           30 :   g = gimple_build_assign (make_ssa_name (sizetype), NEGATE_EXPR, i);
    6412           30 :   insert_before (g);
    6413           30 :   tree k = make_ssa_name (sizetype);
    6414           30 :   g = gimple_build_assign (k, BIT_AND_EXPR, gimple_assign_lhs (g),
    6415           30 :                            size_int (limb_prec - 1));
    6416           30 :   insert_before (g);
    6417           30 :   unsigned data_cnt = m_data_cnt;
    6418           30 :   m_first = true;
    6419           30 :   tree s1 = handle_operand (arg1, bitint_big_endian ? p : size_zero_node);
    6420           30 :   m_first = false;
    6421           30 :   m_data_cnt = data_cnt;
    6422           30 :   g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR, s1, i);
    6423           30 :   insert_before (g);
    6424           30 :   s1 = gimple_assign_lhs (g);
    6425           30 :   tree r = limb_access (TREE_TYPE (lhs), obj, idx, false);
    6426           30 :   g = gimple_build_assign (make_ssa_name (m_limb_type), r);
    6427           30 :   insert_before (g);
    6428           30 :   g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_XOR_EXPR,
    6429              :                            gimple_assign_lhs (g), s1);
    6430           30 :   insert_before (g);
    6431           30 :   r = limb_access (TREE_TYPE (lhs), obj, idx, true);
    6432           30 :   g = gimple_build_assign (r, gimple_assign_lhs (g));
    6433           30 :   insert_before (g);
    6434           30 :   tree idxp1 = make_ssa_name (sizetype);
    6435           30 :   g = gimple_build_assign (idxp1, PLUS_EXPR, idx,
    6436            0 :                            bitint_big_endian ? size_int (-1) : size_one_node);
    6437           30 :   insert_before (g);
    6438           30 :   if (bitint_big_endian)
    6439            0 :     g = gimple_build_cond (NE_EXPR, idxp1, size_int (-1),
    6440              :                            NULL_TREE, NULL_TREE);
    6441              :   else
    6442           30 :     g = gimple_build_cond (LE_EXPR, idxp1, p, NULL_TREE, NULL_TREE);
    6443           30 :   edge edge_true2, edge_false2;
    6444           30 :   if_then (g, profile_probability::likely (), edge_true2, edge_false2);
    6445           30 :   tree idx2_next;
    6446           30 :   tree idx2 = create_loop (idxp1, &idx2_next);
    6447           30 :   tree idx2midx = make_ssa_name (sizetype);
    6448           30 :   g = gimple_build_assign (idx2midx, MINUS_EXPR, idx2, idx4);
    6449           30 :   insert_before (g);
    6450           30 :   tree idx2midx3 = make_ssa_name (sizetype);
    6451           30 :   g = gimple_build_assign (idx2midx3, MINUS_EXPR, idx2, idx3);
    6452           30 :   insert_before (g);
    6453           30 :   m_data_cnt = data_cnt;
    6454           30 :   tree s2 = handle_operand (arg1, idx2midx);
    6455           30 :   g = gimple_build_assign (make_ssa_name (m_limb_type), LSHIFT_EXPR,
    6456              :                            s2, i);
    6457           30 :   insert_before (g);
    6458           30 :   s2 = gimple_assign_lhs (g);
    6459           30 :   m_data_cnt = data_cnt;
    6460           30 :   tree s3 = handle_operand (arg1, idx2midx3);
    6461           30 :   g = gimple_build_assign (make_ssa_name (m_limb_type), RSHIFT_EXPR,
    6462              :                            s3, k);
    6463           30 :   insert_before (g);
    6464           30 :   g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_IOR_EXPR,
    6465              :                            s2, gimple_assign_lhs (g));
    6466           30 :   insert_before (g);
    6467           30 :   s2 = gimple_assign_lhs (g);
    6468           30 :   r = limb_access (TREE_TYPE (lhs), obj, idx2, false);
    6469           30 :   g = gimple_build_assign (make_ssa_name (m_limb_type), r);
    6470           30 :   insert_before (g);
    6471           30 :   g = gimple_build_assign (make_ssa_name (m_limb_type), BIT_XOR_EXPR,
    6472              :                            gimple_assign_lhs (g), s2);
    6473           30 :   insert_before (g);
    6474           30 :   r = limb_access (TREE_TYPE (lhs), obj, idx2, true);
    6475           30 :   g = gimple_build_assign (r, gimple_assign_lhs (g));
    6476           30 :   insert_before (g);
    6477           30 :   g = gimple_build_assign (idx2_next, PLUS_EXPR, idx2,
    6478            0 :                            bitint_big_endian ? size_int (-1) : size_one_node);
    6479           30 :   insert_before (g);
    6480           30 :   if (bitint_big_endian)
    6481            0 :     g = gimple_build_cond (NE_EXPR, idx2_next, size_int (-1),
    6482              :                            NULL_TREE, NULL_TREE);
    6483              :   else
    6484           30 :     g = gimple_build_cond (LE_EXPR, idx2_next, p, NULL_TREE, NULL_TREE);
    6485           30 :   insert_before (g);
    6486           30 :   m_gsi = gsi_after_labels (edge_false->dest);
    6487           30 :   g = gimple_build_assign (t_next, RSHIFT_EXPR, t, integer_one_node);
    6488           30 :   insert_before (g);
    6489           30 :   g = gimple_build_assign (i_next, PLUS_EXPR, i, size_one_node);
    6490           30 :   insert_before (g);
    6491           30 :   g = gimple_build_cond (NE_EXPR, t_next, build_zero_cst (m_limb_type),
    6492              :                          NULL_TREE, NULL_TREE);
    6493           30 :   insert_before (g);
    6494           30 :   if ((EDGE_SUCC (edge_false->dest, 0)->flags & EDGE_FALSE_VALUE) != 0)
    6495           30 :     m_gsi = gsi_after_labels (EDGE_SUCC (edge_false->dest, 0)->dest);
    6496              :   else
    6497            0 :     m_gsi = gsi_after_labels (EDGE_SUCC (edge_false->dest, 1)->dest);
    6498           30 :   g = gimple_build_assign (idx_next, PLUS_EXPR, idx,
    6499            0 :                            bitint_big_endian ? size_int (-1) : size_one_node);
    6500           30 :   insert_before (g);
    6501           30 :   if (bitint_big_endian)
    6502            0 :     g = gimple_build_cond (NE_EXPR, idx_next, size_int (-1),
    6503              :                            NULL_TREE, NULL_TREE);
    6504              :   else
    6505           30 :     g = gimple_build_cond (LE_EXPR, idx_next, p, NULL_TREE, NULL_TREE);
    6506           30 :   insert_before (g);
    6507           30 :   m_gsi = gsi_for_stmt (stmt);
    6508           30 :   if (bitint_extended && prec % limb_prec != 0)
    6509              :     {
    6510              :       /* The most significant limb has been updated either in the
    6511              :          loop or in the if after it.  To simplify the code, just
    6512              :          read it back from memory and extend.  */
    6513            0 :       idx = bitint_big_endian ? size_zero_node : p;
    6514            0 :       tree l = limb_access (TREE_TYPE (lhs), obj, idx, true);
    6515            0 :       tree type = limb_access_type (TREE_TYPE (lhs), idx);
    6516            0 :       tree v = make_ssa_name (m_limb_type);
    6517            0 :       g = gimple_build_assign (v, l);
    6518            0 :       insert_before (g);
    6519            0 :       v = add_cast (type, v);
    6520            0 :       l = limb_access (TREE_TYPE (lhs), obj, idx, true);
    6521            0 :       v = add_cast (m_limb_type, v);
    6522            0 :       g = gimple_build_assign (l, v);
    6523            0 :       insert_before (g);
    6524              :     }
    6525           30 :   if (bitint_extended == bitint_ext_full
    6526            0 :       && abi_limb_prec > limb_prec
    6527            0 :       && (CEIL (prec, abi_limb_prec) * abi_limb_prec
    6528            0 :           > CEIL (prec, limb_prec) * limb_prec))
    6529              :     {
    6530            0 :       tree p2 = build_int_cst (sizetype,
    6531              :                                CEIL (prec, abi_limb_prec)
    6532            0 :                                * abi_limb_prec / limb_prec - 1);
    6533            0 :       tree l = limb_access (TREE_TYPE (lhs), obj, p2, true);
    6534            0 :       g = gimple_build_assign (l, build_zero_cst (m_limb_type));
    6535            0 :       insert_before (g);
    6536              :     }
    6537              : }
    6538              : 
    6539              : /* Lower a call statement with one or more large/huge _BitInt
    6540              :    arguments or large/huge _BitInt return value.  */
    6541              : 
    6542              : void
    6543         9012 : bitint_large_huge::lower_call (tree obj, gimple *stmt)
    6544              : {
    6545         9012 :   gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
    6546         9012 :   unsigned int nargs = gimple_call_num_args (stmt);
    6547         9012 :   if (gimple_call_internal_p (stmt))
    6548         4390 :     switch (gimple_call_internal_fn (stmt))
    6549              :       {
    6550         2733 :       case IFN_ADD_OVERFLOW:
    6551         2733 :       case IFN_SUB_OVERFLOW:
    6552         2733 :       case IFN_UBSAN_CHECK_ADD:
    6553         2733 :       case IFN_UBSAN_CHECK_SUB:
    6554         2733 :         lower_addsub_overflow (obj, stmt);
    6555         7123 :         return;
    6556         1479 :       case IFN_MUL_OVERFLOW:
    6557         1479 :       case IFN_UBSAN_CHECK_MUL:
    6558         1479 :         lower_mul_overflow (obj, stmt);
    6559         1479 :         return;
    6560          101 :       case IFN_CLZ:
    6561          101 :       case IFN_CTZ:
    6562          101 :       case IFN_CLRSB:
    6563          101 :       case IFN_FFS:
    6564          101 :       case IFN_PARITY:
    6565          101 :       case IFN_POPCOUNT:
    6566          101 :         lower_bit_query (stmt);
    6567          101 :         return;
    6568           47 :       case IFN_BSWAP:
    6569           47 :       case IFN_BITREVERSE:
    6570           47 :         lower_bswap_bitreverse (obj, stmt);
    6571           47 :         return;
    6572           30 :       case IFN_CLMUL:
    6573           30 :         lower_clmul (obj, stmt);
    6574           30 :         return;
    6575              :       default:
    6576              :         break;
    6577              :       }
    6578         4622 :   bool returns_twice = (gimple_call_flags (stmt) & ECF_RETURNS_TWICE) != 0;
    6579        10542 :   for (unsigned int i = 0; i < nargs; ++i)
    6580              :     {
    6581         5920 :       tree arg = gimple_call_arg (stmt, i);
    6582         9497 :       if (TREE_CODE (arg) != SSA_NAME
    6583         2440 :           || !BITINT_TYPE_P (TREE_TYPE (arg))
    6584         8298 :           || bitint_precision_kind (TREE_TYPE (arg)) <= bitint_prec_middle)
    6585         3577 :         continue;
    6586         2343 :       if (SSA_NAME_IS_DEFAULT_DEF (arg)
    6587         2343 :           && (!SSA_NAME_VAR (arg) || VAR_P (SSA_NAME_VAR (arg))))
    6588              :         {
    6589            1 :           tree var = create_tmp_reg (TREE_TYPE (arg));
    6590            1 :           arg = get_or_create_ssa_default_def (cfun, var);
    6591              :         }
    6592              :       else
    6593              :         {
    6594         2342 :           int p = var_to_partition (m_map, arg);
    6595         2342 :           tree v = m_vars[p];
    6596         2342 :           gcc_assert (v != NULL_TREE);
    6597         2342 :           if (!types_compatible_p (TREE_TYPE (arg), TREE_TYPE (v)))
    6598         2322 :             v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (arg), v);
    6599         2342 :           arg = make_ssa_name (TREE_TYPE (arg));
    6600         2342 :           gimple *g = gimple_build_assign (arg, v);
    6601         2342 :           gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    6602         2342 :           if (returns_twice && bb_has_abnormal_pred (gimple_bb (stmt)))
    6603              :             {
    6604           11 :               m_returns_twice_calls.safe_push (stmt);
    6605           11 :               returns_twice = false;
    6606              :             }
    6607              :         }
    6608         2343 :       gimple_call_set_arg (stmt, i, arg);
    6609         2343 :       if (m_preserved == NULL)
    6610          412 :         m_preserved = BITMAP_ALLOC (NULL);
    6611         2343 :       bitmap_set_bit (m_preserved, SSA_NAME_VERSION (arg));
    6612              :     }
    6613         4622 :   tree lhs = gimple_call_lhs (stmt);
    6614         4622 :   if (lhs
    6615         4491 :       && TREE_CODE (lhs) == SSA_NAME
    6616         4491 :       && BITINT_TYPE_P (TREE_TYPE (lhs))
    6617         9035 :       && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large)
    6618              :     {
    6619         4387 :       int p = var_to_partition (m_map, lhs);
    6620         4387 :       tree v = m_vars[p];
    6621         4387 :       gcc_assert (v != NULL_TREE);
    6622         4387 :       if (!types_compatible_p (TREE_TYPE (lhs), TREE_TYPE (v)))
    6623         4387 :         v = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), v);
    6624         4387 :       gimple_call_set_lhs (stmt, v);
    6625         4387 :       SSA_NAME_DEF_STMT (lhs) = gimple_build_nop ();
    6626              :     }
    6627         4622 :   update_stmt (stmt);
    6628              : }
    6629              : 
    6630              : /* Lower __asm STMT which involves large/huge _BitInt values.  */
    6631              : 
    6632              : void
    6633            3 : bitint_large_huge::lower_asm (gimple *stmt)
    6634              : {
    6635            3 :   gasm *g = as_a <gasm *> (stmt);
    6636            3 :   unsigned noutputs = gimple_asm_noutputs (g);
    6637            3 :   unsigned ninputs = gimple_asm_ninputs (g);
    6638              : 
    6639            5 :   for (unsigned i = 0; i < noutputs; ++i)
    6640              :     {
    6641            2 :       tree t = gimple_asm_output_op (g, i);
    6642            2 :       tree s = TREE_VALUE (t);
    6643            2 :       if (TREE_CODE (s) == SSA_NAME
    6644            1 :           && BITINT_TYPE_P (TREE_TYPE (s))
    6645            3 :           && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
    6646              :         {
    6647            1 :           int part = var_to_partition (m_map, s);
    6648            1 :           gcc_assert (m_vars[part] != NULL_TREE);
    6649            1 :           TREE_VALUE (t) = m_vars[part];
    6650              :         }
    6651              :     }
    6652            8 :   for (unsigned i = 0; i < ninputs; ++i)
    6653              :     {
    6654            5 :       tree t = gimple_asm_input_op (g, i);
    6655            5 :       tree s = TREE_VALUE (t);
    6656            5 :       if (TREE_CODE (s) == SSA_NAME
    6657            4 :           && BITINT_TYPE_P (TREE_TYPE (s))
    6658            9 :           && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
    6659              :         {
    6660            4 :           if (SSA_NAME_IS_DEFAULT_DEF (s)
    6661            4 :               && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
    6662              :             {
    6663            1 :               TREE_VALUE (t) = create_tmp_var (TREE_TYPE (s), "bitint");
    6664            1 :               mark_addressable (TREE_VALUE (t));
    6665              :             }
    6666              :           else
    6667              :             {
    6668            3 :               int part = var_to_partition (m_map, s);
    6669            3 :               gcc_assert (m_vars[part] != NULL_TREE);
    6670            3 :               TREE_VALUE (t) = m_vars[part];
    6671              :             }
    6672              :         }
    6673              :     }
    6674            3 :   update_stmt (stmt);
    6675            3 : }
    6676              : 
    6677              : /* Lower statement STMT which involves large/huge _BitInt values
    6678              :    into code accessing individual limbs.  */
    6679              : 
    6680              : void
    6681        43332 : bitint_large_huge::lower_stmt (gimple *stmt)
    6682              : {
    6683        43332 :   m_first = true;
    6684        43332 :   m_lhs = NULL_TREE;
    6685        43332 :   m_data.truncate (0);
    6686        43332 :   m_data_cnt = 0;
    6687        43332 :   m_gsi = gsi_for_stmt (stmt);
    6688        43332 :   m_after_stmt = NULL;
    6689        43332 :   m_bb = NULL;
    6690        43332 :   m_init_gsi = m_gsi;
    6691        43332 :   gsi_prev (&m_init_gsi);
    6692        43332 :   m_preheader_bb = NULL;
    6693        43332 :   m_upwards_2limb = 0;
    6694        43332 :   m_upwards = false;
    6695        43332 :   m_var_msb = false;
    6696        43332 :   m_cast_conditional = false;
    6697        43332 :   m_bitfld_load = 0;
    6698        43332 :   m_loc = gimple_location (stmt);
    6699        43332 :   if (is_gimple_call (stmt))
    6700              :     {
    6701         7325 :       lower_call (NULL_TREE, stmt);
    6702         7325 :       return;
    6703              :     }
    6704        36007 :   if (gimple_code (stmt) == GIMPLE_ASM)
    6705              :     {
    6706            3 :       lower_asm (stmt);
    6707            3 :       return;
    6708              :     }
    6709        36004 :   tree lhs = NULL_TREE, cmp_op1 = NULL_TREE, cmp_op2 = NULL_TREE;
    6710        36004 :   tree_code cmp_code = comparison_op (stmt, &cmp_op1, &cmp_op2);
    6711        36004 :   bool eq_p = (cmp_code == EQ_EXPR || cmp_code == NE_EXPR);
    6712        36004 :   bool mergeable_cast_p = false;
    6713        36004 :   bool final_cast_p = false;
    6714        36004 :   if (gimple_assign_cast_p (stmt))
    6715              :     {
    6716         5459 :       lhs = gimple_assign_lhs (stmt);
    6717         5459 :       tree rhs1 = gimple_assign_rhs1 (stmt);
    6718         5459 :       if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
    6719           46 :         rhs1 = TREE_OPERAND (rhs1, 0);
    6720         9680 :       if (BITINT_TYPE_P (TREE_TYPE (lhs))
    6721         1238 :           && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
    6722         6661 :           && INTEGRAL_TYPE_P (TREE_TYPE (rhs1)))
    6723              :         mergeable_cast_p = true;
    6724         4547 :       else if (BITINT_TYPE_P (TREE_TYPE (rhs1))
    6725         4257 :                && bitint_precision_kind (TREE_TYPE (rhs1)) >= bitint_prec_large
    6726         8659 :                && (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
    6727           36 :                    || POINTER_TYPE_P (TREE_TYPE (lhs))
    6728           35 :                    || gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR))
    6729              :         {
    6730         4257 :           final_cast_p = true;
    6731         4257 :           if (((TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
    6732          572 :                 && TYPE_PRECISION (TREE_TYPE (lhs)) > MAX_FIXED_MODE_SIZE)
    6733         4257 :                || (!INTEGRAL_TYPE_P (TREE_TYPE (lhs))
    6734           36 :                    && !POINTER_TYPE_P (TREE_TYPE (lhs))))
    6735         4292 :               && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
    6736              :             {
    6737              :               /* Handle VIEW_CONVERT_EXPRs to not generally supported
    6738              :                  huge INTEGER_TYPEs like uint256_t or uint512_t.  These
    6739              :                  are usually emitted from memcpy folding and backends
    6740              :                  support moves with them but that is usually it.
    6741              :                  Similarly handle VCEs to vector/complex types etc.  */
    6742           35 :               gcc_assert (TREE_CODE (rhs1) == SSA_NAME);
    6743           35 :               if (SSA_NAME_IS_DEFAULT_DEF (rhs1)
    6744           35 :                   && (!SSA_NAME_VAR (rhs1) || VAR_P (SSA_NAME_VAR (rhs1))))
    6745              :                 {
    6746            0 :                   tree var = create_tmp_reg (TREE_TYPE (lhs));
    6747            0 :                   rhs1 = get_or_create_ssa_default_def (cfun, var);
    6748            0 :                   gimple_assign_set_rhs1 (stmt, rhs1);
    6749            0 :                   gimple_assign_set_rhs_code (stmt, SSA_NAME);
    6750              :                 }
    6751           35 :               else if (m_names == NULL
    6752           35 :                        || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1)))
    6753              :                 {
    6754            0 :                   gimple *g = SSA_NAME_DEF_STMT (rhs1);
    6755            0 :                   gcc_assert (gimple_assign_load_p (g));
    6756            0 :                   tree mem = gimple_assign_rhs1 (g);
    6757            0 :                   tree ltype = TREE_TYPE (lhs);
    6758            0 :                   addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (mem));
    6759            0 :                   if (as != TYPE_ADDR_SPACE (ltype))
    6760            0 :                     ltype
    6761            0 :                       = build_qualified_type (ltype,
    6762            0 :                                               TYPE_QUALS (ltype).with_as (as));
    6763            0 :                   rhs1 = build1 (VIEW_CONVERT_EXPR, ltype, unshare_expr (mem));
    6764            0 :                   gimple_assign_set_rhs1 (stmt, rhs1);
    6765              :                 }
    6766              :               else
    6767              :                 {
    6768           35 :                   int part = var_to_partition (m_map, rhs1);
    6769           35 :                   gcc_assert (m_vars[part] != NULL_TREE);
    6770           35 :                   rhs1 = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs),
    6771              :                                  m_vars[part]);
    6772           35 :                   gimple_assign_set_rhs1 (stmt, rhs1);
    6773              :                 }
    6774           35 :               update_stmt (stmt);
    6775           35 :               return;
    6776              :             }
    6777         4222 :           if (TREE_CODE (rhs1) == SSA_NAME
    6778         4222 :               && (m_names == NULL
    6779         4185 :                   || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
    6780              :             {
    6781         1723 :               gimple *g = SSA_NAME_DEF_STMT (rhs1);
    6782         1723 :               if (is_gimple_assign (g)
    6783         1723 :                   && gimple_assign_rhs_code (g) == IMAGPART_EXPR)
    6784              :                 {
    6785         1640 :                   tree rhs2 = TREE_OPERAND (gimple_assign_rhs1 (g), 0);
    6786         1640 :                   if (TREE_CODE (rhs2) == SSA_NAME
    6787         1640 :                       && (m_names == NULL
    6788         1603 :                           || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs2))))
    6789              :                     {
    6790         1640 :                       g = SSA_NAME_DEF_STMT (rhs2);
    6791         1640 :                       int ovf = optimizable_arith_overflow (g);
    6792         1640 :                       if (ovf == 2)
    6793              :                         /* If .{ADD,SUB,MUL}_OVERFLOW has both REALPART_EXPR
    6794              :                            and IMAGPART_EXPR uses, where the latter is cast to
    6795              :                            non-_BitInt, it will be optimized when handling
    6796              :                            the REALPART_EXPR.  */
    6797              :                         return;
    6798           91 :                       if (ovf == 1)
    6799              :                         {
    6800           91 :                           lower_call (NULL_TREE, g);
    6801           91 :                           return;
    6802              :                         }
    6803              :                     }
    6804              :                 }
    6805              :             }
    6806              :         }
    6807          145 :       else if (BITINT_TYPE_P (TREE_TYPE (lhs))
    6808          145 :                && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
    6809          145 :                && !INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
    6810          145 :                && !POINTER_TYPE_P (TREE_TYPE (rhs1))
    6811          290 :                && gimple_assign_rhs_code (stmt) == VIEW_CONVERT_EXPR)
    6812              :         {
    6813           10 :           int part = var_to_partition (m_map, lhs);
    6814           10 :           gcc_assert (m_vars[part] != NULL_TREE);
    6815           10 :           lhs = build1 (VIEW_CONVERT_EXPR, TREE_TYPE (rhs1), m_vars[part]);
    6816           10 :           insert_before (gimple_build_assign (lhs, rhs1));
    6817           10 :           return;
    6818              :         }
    6819              :     }
    6820        34319 :   if (gimple_store_p (stmt))
    6821              :     {
    6822         9086 :       tree rhs1 = gimple_assign_rhs1 (stmt);
    6823         9086 :       if (TREE_CODE (rhs1) == SSA_NAME
    6824         9086 :           && (m_names == NULL
    6825         8016 :               || !bitmap_bit_p (m_names, SSA_NAME_VERSION (rhs1))))
    6826              :         {
    6827         1681 :           gimple *g = SSA_NAME_DEF_STMT (rhs1);
    6828         1681 :           m_loc = gimple_location (g);
    6829         1681 :           lhs = gimple_assign_lhs (stmt);
    6830         1681 :           if (is_gimple_assign (g) && !mergeable_op (g))
    6831          667 :             switch (gimple_assign_rhs_code (g))
    6832              :               {
    6833          119 :               case LSHIFT_EXPR:
    6834          119 :               case RSHIFT_EXPR:
    6835          119 :                 lower_shift_stmt (lhs, g);
    6836          510 :               handled:
    6837          510 :                 m_gsi = gsi_for_stmt (stmt);
    6838          510 :                 unlink_stmt_vdef (stmt);
    6839         1020 :                 release_ssa_name (gimple_vdef (stmt));
    6840          510 :                 gsi_remove (&m_gsi, true);
    6841          510 :                 return;
    6842          207 :               case MULT_EXPR:
    6843          207 :               case TRUNC_DIV_EXPR:
    6844          207 :               case EXACT_DIV_EXPR:
    6845          207 :               case TRUNC_MOD_EXPR:
    6846          207 :                 lower_muldiv_stmt (lhs, g);
    6847          207 :                 goto handled;
    6848           44 :               case FIX_TRUNC_EXPR:
    6849           44 :                 lower_float_conv_stmt (lhs, g);
    6850           44 :                 goto handled;
    6851           93 :               case REALPART_EXPR:
    6852           93 :               case IMAGPART_EXPR:
    6853           93 :                 lower_cplxpart_stmt (lhs, g);
    6854           93 :                 goto handled;
    6855            8 :               case VIEW_CONVERT_EXPR:
    6856            8 :                 {
    6857            8 :                   tree rhs1 = gimple_assign_rhs1 (g);
    6858            8 :                   rhs1 = TREE_OPERAND (rhs1, 0);
    6859            8 :                   if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
    6860            7 :                       && !POINTER_TYPE_P (TREE_TYPE (rhs1)))
    6861              :                     {
    6862            7 :                       tree ltype = TREE_TYPE (rhs1);
    6863            7 :                       addr_space_t as = TYPE_ADDR_SPACE (TREE_TYPE (lhs));
    6864            7 :                       ltype
    6865            7 :                         = build_qualified_type (ltype,
    6866            7 :                                                 TYPE_QUALS (TREE_TYPE (lhs))
    6867              :                                                 .with_as (as));
    6868            7 :                       lhs = build1 (VIEW_CONVERT_EXPR, ltype, lhs);
    6869            7 :                       gimple_assign_set_lhs (stmt, lhs);
    6870            7 :                       gimple_assign_set_rhs1 (stmt, rhs1);
    6871            7 :                       gimple_assign_set_rhs_code (stmt, TREE_CODE (rhs1));
    6872            7 :                       update_stmt (stmt);
    6873            7 :                       return;
    6874              :                     }
    6875              :                 }
    6876              :                 break;
    6877              :               default:
    6878              :                 break;
    6879              :               }
    6880         1014 :           else if (optimizable_arith_overflow (g) == 3)
    6881              :             {
    6882           24 :               lower_call (lhs, g);
    6883           24 :               goto handled;
    6884              :             }
    6885          990 :           else if (is_gimple_call (g) && gimple_call_internal_p (g))
    6886           23 :             switch (gimple_call_internal_fn (g))
    6887              :               {
    6888           23 :               case IFN_BSWAP:
    6889           23 :               case IFN_BITREVERSE:
    6890           23 :               case IFN_CLMUL:
    6891           23 :                 lower_call (lhs, g);
    6892           23 :                 goto handled;
    6893              :               default:
    6894              :                 break;
    6895              :               }
    6896              : 
    6897         1164 :           m_loc = gimple_location (stmt);
    6898              :         }
    6899              :     }
    6900        33802 :   if (mergeable_op (stmt)
    6901        22277 :       || gimple_store_p (stmt)
    6902        22277 :       || gimple_assign_load_p (stmt)
    6903              :       || eq_p
    6904        17451 :       || mergeable_cast_p
    6905        43997 :       || (is_gimple_assign (stmt)
    6906         9807 :           && gimple_assign_rhs_code (stmt) == PAREN_EXPR))
    6907              :     {
    6908        23609 :       lhs = lower_mergeable_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
    6909        23609 :       if (!eq_p)
    6910              :         return;
    6911              :     }
    6912        10193 :   else if (cmp_code != ERROR_MARK)
    6913          722 :     lhs = lower_comparison_stmt (stmt, cmp_code, cmp_op1, cmp_op2);
    6914        16858 :   if (cmp_code != ERROR_MARK)
    6915              :     {
    6916         7387 :       if (gimple_code (stmt) == GIMPLE_COND)
    6917              :         {
    6918         6613 :           gcond *cstmt = as_a <gcond *> (stmt);
    6919         6613 :           gimple_cond_set_lhs (cstmt, lhs);
    6920         6613 :           gimple_cond_set_rhs (cstmt, boolean_false_node);
    6921         6613 :           gimple_cond_set_code (cstmt, cmp_code);
    6922         6613 :           update_stmt (stmt);
    6923         6613 :           return;
    6924              :         }
    6925          774 :       if (gimple_assign_rhs_code (stmt) == COND_EXPR)
    6926              :         {
    6927            0 :           tree cond = build2 (cmp_code, boolean_type_node, lhs,
    6928              :                               boolean_false_node);
    6929            0 :           gimple_assign_set_rhs1 (stmt, cond);
    6930            0 :           lhs = gimple_assign_lhs (stmt);
    6931            0 :           gcc_assert (!BITINT_TYPE_P (TREE_TYPE (lhs))
    6932              :                       || (bitint_precision_kind (TREE_TYPE (lhs))
    6933              :                           <= bitint_prec_middle));
    6934            0 :           update_stmt (stmt);
    6935            0 :           return;
    6936              :         }
    6937          774 :       gimple_assign_set_rhs1 (stmt, lhs);
    6938          774 :       gimple_assign_set_rhs2 (stmt, boolean_false_node);
    6939          774 :       gimple_assign_set_rhs_code (stmt, cmp_code);
    6940          774 :       update_stmt (stmt);
    6941          774 :       return;
    6942              :     }
    6943         9471 :   if (final_cast_p)
    6944              :     {
    6945         2582 :       tree lhs_type = TREE_TYPE (lhs);
    6946              :       /* Add support for 3 or more limbs filled in from normal integral
    6947              :          type if this assert fails.  If no target chooses limb mode smaller
    6948              :          than half of largest supported normal integral type, this will not
    6949              :          be needed.  */
    6950         2582 :       gcc_assert (TYPE_PRECISION (lhs_type) <= 2 * limb_prec);
    6951         2582 :       gimple *g;
    6952         2546 :       if ((BITINT_TYPE_P (lhs_type)
    6953           36 :            && bitint_precision_kind (lhs_type) == bitint_prec_middle)
    6954         5149 :           || POINTER_TYPE_P (lhs_type))
    6955           16 :         lhs_type = build_nonstandard_integer_type (TYPE_PRECISION (lhs_type),
    6956           16 :                                                    TYPE_UNSIGNED (lhs_type));
    6957         2582 :       m_data_cnt = 0;
    6958         2582 :       tree rhs1 = gimple_assign_rhs1 (stmt);
    6959         2582 :       unsigned int prec = TYPE_PRECISION (TREE_TYPE (rhs1));
    6960         2582 :       unsigned int cnt = CEIL (prec, limb_prec);
    6961         2582 :       tree r1 = handle_operand (rhs1, size_int (bitint_big_endian
    6962              :                                                 ? cnt - 1 : 0));
    6963         2582 :       if (!useless_type_conversion_p (lhs_type, TREE_TYPE (r1)))
    6964         2473 :         r1 = add_cast (lhs_type, r1);
    6965         2582 :       if (TYPE_PRECISION (lhs_type) > limb_prec)
    6966              :         {
    6967           70 :           m_data_cnt = 0;
    6968           70 :           m_first = false;
    6969           70 :           tree r2 = handle_operand (rhs1, size_int (bitint_big_endian
    6970              :                                                     ? cnt - 2 : 1));
    6971           70 :           r2 = add_cast (lhs_type, r2);
    6972           70 :           g = gimple_build_assign (make_ssa_name (lhs_type), LSHIFT_EXPR, r2,
    6973              :                                    build_int_cst (unsigned_type_node,
    6974           70 :                                                   limb_prec));
    6975           70 :           insert_before (g);
    6976           70 :           g = gimple_build_assign (make_ssa_name (lhs_type), BIT_IOR_EXPR, r1,
    6977              :                                    gimple_assign_lhs (g));
    6978           70 :           insert_before (g);
    6979           70 :           r1 = gimple_assign_lhs (g);
    6980              :         }
    6981         2582 :       if (lhs_type != TREE_TYPE (lhs))
    6982           16 :         g = gimple_build_assign (lhs, NOP_EXPR, r1);
    6983              :       else
    6984         2566 :         g = gimple_build_assign (lhs, r1);
    6985         2582 :       gsi_replace (&m_gsi, g, true);
    6986         2582 :       return;
    6987              :     }
    6988         6889 :   if (is_gimple_assign (stmt))
    6989         6889 :     switch (gimple_assign_rhs_code (stmt))
    6990              :       {
    6991          483 :       case LSHIFT_EXPR:
    6992          483 :       case RSHIFT_EXPR:
    6993          483 :         lower_shift_stmt (NULL_TREE, stmt);
    6994          483 :         return;
    6995          166 :       case MULT_EXPR:
    6996          166 :       case TRUNC_DIV_EXPR:
    6997          166 :       case EXACT_DIV_EXPR:
    6998          166 :       case TRUNC_MOD_EXPR:
    6999          166 :         lower_muldiv_stmt (NULL_TREE, stmt);
    7000          166 :         return;
    7001          283 :       case FIX_TRUNC_EXPR:
    7002          283 :       case FLOAT_EXPR:
    7003          283 :         lower_float_conv_stmt (NULL_TREE, stmt);
    7004          283 :         return;
    7005         5929 :       case REALPART_EXPR:
    7006         5929 :       case IMAGPART_EXPR:
    7007         5929 :         lower_cplxpart_stmt (NULL_TREE, stmt);
    7008         5929 :         return;
    7009           28 :       case COMPLEX_EXPR:
    7010           28 :         lower_complexexpr_stmt (stmt);
    7011           28 :         return;
    7012              :       default:
    7013              :         break;
    7014              :       }
    7015            0 :   gcc_unreachable ();
    7016              : }
    7017              : 
    7018              : /* Helper for walk_non_aliased_vuses.  Determine if we arrived at
    7019              :    the desired memory state.  */
    7020              : 
    7021              : void *
    7022         2190 : vuse_eq (ao_ref *, tree vuse1, void *data)
    7023              : {
    7024         2190 :   tree vuse2 = (tree) data;
    7025         2190 :   if (vuse1 == vuse2)
    7026          820 :     return data;
    7027              : 
    7028              :   return NULL;
    7029              : }
    7030              : 
    7031              : /* Return true if STMT uses a library function and needs to take
    7032              :    address of its inputs.  We need to avoid bit-fields in those
    7033              :    cases.  Similarly, we need to avoid overlap between destination
    7034              :    and source limb arrays.  */
    7035              : 
    7036              : bool
    7037        15561 : stmt_needs_operand_addr (gimple *stmt)
    7038              : {
    7039        15561 :   if (is_gimple_assign (stmt))
    7040        10374 :     switch (gimple_assign_rhs_code (stmt))
    7041              :       {
    7042          585 :       case MULT_EXPR:
    7043          585 :       case TRUNC_DIV_EXPR:
    7044          585 :       case EXACT_DIV_EXPR:
    7045          585 :       case TRUNC_MOD_EXPR:
    7046          585 :       case FLOAT_EXPR:
    7047          585 :         return true;
    7048              :       default:
    7049              :         break;
    7050              :       }
    7051         5187 :   else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
    7052           46 :     switch (gimple_call_internal_fn (stmt))
    7053              :       {
    7054           17 :       case IFN_MUL_OVERFLOW:
    7055           17 :       case IFN_UBSAN_CHECK_MUL:
    7056           17 :         return true;
    7057              :       /* These three actually don't take address, but reshuffle
    7058              :          all bytes or bits, so need similar treatment.  */
    7059            4 :       case IFN_BSWAP:
    7060            4 :       case IFN_BITREVERSE:
    7061            4 :       case IFN_CLMUL:
    7062            4 :         return true;
    7063              :       default:
    7064              :         break;
    7065              :       }
    7066              :   return false;
    7067              : }
    7068              : 
    7069              : /* Dominator walker used to discover which large/huge _BitInt
    7070              :    loads could be sunk into all their uses.  */
    7071              : 
    7072          634 : class bitint_dom_walker : public dom_walker
    7073              : {
    7074              : public:
    7075          317 :   bitint_dom_walker (bitmap names, bitmap loads)
    7076          634 :     : dom_walker (CDI_DOMINATORS), m_names (names), m_loads (loads) {}
    7077              : 
    7078              :   edge before_dom_children (basic_block) final override;
    7079              : 
    7080              : private:
    7081              :   bitmap m_names, m_loads;
    7082              : };
    7083              : 
    7084              : edge
    7085         4784 : bitint_dom_walker::before_dom_children (basic_block bb)
    7086              : {
    7087         4784 :   gphi *phi = get_virtual_phi (bb);
    7088         4784 :   tree vop;
    7089         4784 :   if (phi)
    7090          800 :     vop = gimple_phi_result (phi);
    7091         3984 :   else if (bb == ENTRY_BLOCK_PTR_FOR_FN (cfun))
    7092              :     vop = NULL_TREE;
    7093              :   else
    7094         3667 :     vop = (tree) get_immediate_dominator (CDI_DOMINATORS, bb)->aux;
    7095              : 
    7096         4784 :   auto_vec<tree, 16> worklist;
    7097         9568 :   for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
    7098        20734 :        !gsi_end_p (gsi); gsi_next (&gsi))
    7099              :     {
    7100        15950 :       gimple *stmt = gsi_stmt (gsi);
    7101        15950 :       if (is_gimple_debug (stmt))
    7102         2769 :         continue;
    7103              : 
    7104        16000 :       if (!vop && gimple_vuse (stmt))
    7105              :         vop = gimple_vuse (stmt);
    7106              : 
    7107        15561 :       tree cvop = vop;
    7108        29098 :       if (gimple_vdef (stmt))
    7109              :         vop = gimple_vdef (stmt);
    7110              : 
    7111        15561 :       tree lhs = gimple_get_lhs (stmt);
    7112        17941 :       if (lhs
    7113        11163 :           && TREE_CODE (lhs) == SSA_NAME
    7114         8912 :           && BITINT_TYPE_P (TREE_TYPE (lhs))
    7115         5937 :           && bitint_precision_kind (TREE_TYPE (lhs)) >= bitint_prec_large
    7116        21308 :           && !bitmap_bit_p (m_names, SSA_NAME_VERSION (lhs)))
    7117              :         /* If lhs of stmt is large/huge _BitInt SSA_NAME not in m_names,
    7118              :            it means it will be handled in a loop or straight line code
    7119              :            at the location of its (ultimate) immediate use, so for
    7120              :            vop checking purposes check these only at the ultimate
    7121              :            immediate use.  */
    7122         2380 :         continue;
    7123              : 
    7124        13181 :       ssa_op_iter oi;
    7125        13181 :       use_operand_p use_p;
    7126        22193 :       FOR_EACH_SSA_USE_OPERAND (use_p, stmt, oi, SSA_OP_USE)
    7127              :         {
    7128         9012 :           tree s = USE_FROM_PTR (use_p);
    7129        14534 :           if (BITINT_TYPE_P (TREE_TYPE (s))
    7130         9012 :               && bitint_precision_kind (TREE_TYPE (s)) >= bitint_prec_large)
    7131         3293 :             worklist.safe_push (s);
    7132              :         }
    7133              : 
    7134        13181 :       bool needs_operand_addr = stmt_needs_operand_addr (stmt);
    7135        32495 :       while (worklist.length () > 0)
    7136              :         {
    7137         6133 :           tree s = worklist.pop ();
    7138              : 
    7139         6133 :           if (!bitmap_bit_p (m_names, SSA_NAME_VERSION (s)))
    7140              :             {
    7141         2380 :               gimple *g = SSA_NAME_DEF_STMT (s);
    7142         2380 :               needs_operand_addr |= stmt_needs_operand_addr (g);
    7143         5459 :               FOR_EACH_SSA_USE_OPERAND (use_p, g, oi, SSA_OP_USE)
    7144              :                 {
    7145         3079 :                   tree s2 = USE_FROM_PTR (use_p);
    7146         3269 :                   if (BITINT_TYPE_P (TREE_TYPE (s2))
    7147         3079 :                       && (bitint_precision_kind (TREE_TYPE (s2))
    7148              :                           >= bitint_prec_large))
    7149         2840 :                     worklist.safe_push (s2);
    7150              :                 }
    7151         3116 :               continue;
    7152         2380 :             }
    7153         3753 :           if (!SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
    7154         3753 :               && gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
    7155              :             {
    7156          226 :               tree rhs = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
    7157          396 :               if (TREE_CODE (rhs) == SSA_NAME
    7158          226 :                   && bitmap_bit_p (m_loads, SSA_NAME_VERSION (rhs)))
    7159              :                 s = rhs;
    7160              :               else
    7161          170 :                 continue;
    7162              :             }
    7163         3527 :           else if (!bitmap_bit_p (m_loads, SSA_NAME_VERSION (s)))
    7164          564 :             continue;
    7165              : 
    7166         3019 :           gimple *g = SSA_NAME_DEF_STMT (s);
    7167         3019 :           tree rhs1 = gimple_assign_rhs1 (g);
    7168         3019 :           if (needs_operand_addr
    7169          218 :               && TREE_CODE (rhs1) == COMPONENT_REF
    7170         3036 :               && DECL_BIT_FIELD_TYPE (TREE_OPERAND (rhs1, 1)))
    7171              :             {
    7172            4 :               tree fld = TREE_OPERAND (rhs1, 1);
    7173              :               /* For little-endian, we can allow as inputs bit-fields
    7174              :                  which start at a limb boundary.  */
    7175            6 :               if (!bitint_big_endian
    7176            4 :                   && DECL_OFFSET_ALIGN (fld) >= TYPE_ALIGN (TREE_TYPE (rhs1))
    7177            4 :                   && tree_fits_uhwi_p (DECL_FIELD_BIT_OFFSET (fld))
    7178            4 :                   && (tree_to_uhwi (DECL_FIELD_BIT_OFFSET (fld))
    7179            4 :                       % limb_prec) == 0)
    7180              :                 ;
    7181              :               else
    7182              :                 {
    7183            2 :                   bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
    7184            2 :                   continue;
    7185              :                 }
    7186              :             }
    7187              : 
    7188         3017 :           ao_ref ref;
    7189         3017 :           ao_ref_init (&ref, rhs1);
    7190         3017 :           tree lvop = gimple_vuse (g);
    7191         3017 :           unsigned limit = 64;
    7192         3017 :           tree vuse = cvop;
    7193         3017 :           if (vop != cvop
    7194         1338 :               && is_gimple_assign (stmt)
    7195         1336 :               && gimple_store_p (stmt)
    7196         4353 :               && (needs_operand_addr
    7197         1143 :                   || !operand_equal_p (lhs, gimple_assign_rhs1 (g), 0)))
    7198              :             vuse = vop;
    7199         3017 :           if (vuse != lvop
    7200         3017 :               && walk_non_aliased_vuses (&ref, vuse, false, vuse_eq,
    7201              :                                          NULL, NULL, NULL, limit, lvop) == NULL)
    7202          532 :             bitmap_clear_bit (m_loads, SSA_NAME_VERSION (s));
    7203              :         }
    7204              :     }
    7205              : 
    7206         4784 :   bb->aux = (void *) vop;
    7207         4784 :   return NULL;
    7208         4784 : }
    7209              : 
    7210              : }
    7211              : 
    7212              : /* Replacement for normal processing of STMT in tree-ssa-coalesce.cc
    7213              :    build_ssa_conflict_graph.
    7214              :    The differences are:
    7215              :    1) don't process assignments with large/huge _BitInt lhs not in NAMES
    7216              :    2) for large/huge _BitInt multiplication/division/modulo process def
    7217              :       only after processing uses rather than before to make uses conflict
    7218              :       with the definition
    7219              :    3) for large/huge _BitInt uses not in NAMES mark the uses of their
    7220              :       SSA_NAME_DEF_STMT (recursively), because those uses will be sunk into
    7221              :       the final statement.  */
    7222              : 
    7223              : void
    7224        86418 : build_bitint_stmt_ssa_conflicts (gimple *stmt, live_track *live,
    7225              :                                  ssa_conflicts *graph, bitmap names,
    7226              :                                  void (*def) (live_track *, tree,
    7227              :                                               ssa_conflicts *),
    7228              :                                  void (*use) (live_track *, tree),
    7229              :                                  void (*clear) (live_track *, tree))
    7230              : {
    7231        86418 :   bool muldiv_p = false;
    7232        86418 :   tree lhs = NULL_TREE;
    7233        86418 :   if (is_gimple_assign (stmt))
    7234              :     {
    7235        47347 :       lhs = gimple_assign_lhs (stmt);
    7236        47347 :       if (TREE_CODE (lhs) == SSA_NAME)
    7237              :         {
    7238        33823 :           tree type = TREE_TYPE (lhs);
    7239        33823 :           if (TREE_CODE (type) == COMPLEX_TYPE)
    7240           73 :             type = TREE_TYPE (type);
    7241        13031 :           if (BITINT_TYPE_P (type)
    7242        33828 :               && bitint_precision_kind (type) >= bitint_prec_large)
    7243              :             {
    7244        20114 :               if (!bitmap_bit_p (names, SSA_NAME_VERSION (lhs)))
    7245         4955 :                 return;
    7246              : 
    7247              :               /* A copy between 2 partitions does not introduce an interference
    7248              :                  by itself.  If they did, you would never be able to coalesce
    7249              :                  two things which are copied.  If the two variables really do
    7250              :                  conflict, they will conflict elsewhere in the program.
    7251              : 
    7252              :                  This is handled by simply removing the SRC of the copy from
    7253              :                  the live list, and processing the stmt normally.
    7254              : 
    7255              :                  Don't do this if lhs is not in names though, in such cases
    7256              :                  it is actually used at some point later in the basic
    7257              :                  block.  */
    7258        15159 :               if (gimple_assign_copy_p (stmt))
    7259              :                 {
    7260         1697 :                   tree rhs1 = gimple_assign_rhs1 (stmt);
    7261         1697 :                   if (TREE_CODE (rhs1) == SSA_NAME)
    7262           52 :                     clear (live, rhs1);
    7263              :                 }
    7264              : 
    7265        15159 :               switch (gimple_assign_rhs_code (stmt))
    7266              :                 {
    7267          166 :                 case MULT_EXPR:
    7268          166 :                 case TRUNC_DIV_EXPR:
    7269          166 :                 case EXACT_DIV_EXPR:
    7270          166 :                 case TRUNC_MOD_EXPR:
    7271          166 :                   muldiv_p = true;
    7272              :                 default:
    7273              :                   break;
    7274              :                 }
    7275              :             }
    7276              :         }
    7277              :     }
    7278        39071 :   else if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
    7279         4363 :     switch (gimple_call_internal_fn (stmt))
    7280              :       {
    7281         2699 :       case IFN_ADD_OVERFLOW:
    7282         2699 :       case IFN_SUB_OVERFLOW:
    7283         2699 :       case IFN_UBSAN_CHECK_ADD:
    7284         2699 :       case IFN_UBSAN_CHECK_SUB:
    7285         2699 :         if (bitint_big_endian)
    7286              :           {
    7287            0 :             lhs = gimple_call_lhs (stmt);
    7288            0 :             if (lhs)
    7289        81463 :               muldiv_p = true;
    7290              :           }
    7291              :         break;
    7292         1540 :       case IFN_MUL_OVERFLOW:
    7293         1540 :       case IFN_UBSAN_CHECK_MUL:
    7294         1540 :       case IFN_BSWAP:
    7295         1540 :       case IFN_BITREVERSE:
    7296         1540 :       case IFN_CLMUL:
    7297         1540 :         lhs = gimple_call_lhs (stmt);
    7298         1540 :         if (lhs)
    7299        81463 :           muldiv_p = true;
    7300              :         break;
    7301              :       default:
    7302              :         break;
    7303              :       }
    7304              : 
    7305       162926 :   auto_vec<tree, 16> worklist;
    7306        81463 :   ssa_op_iter iter;
    7307        81463 :   tree var;
    7308              :   /* On little-endian, mergeable ops process limbs from 0 up so except
    7309              :      for multiplication/division/modulo there is no risk in using the
    7310              :      same underlying variable for lhs and some operand, even when casts
    7311              :      are involved, the lhs limb is stored only after processing the source
    7312              :      limbs with the same index.
    7313              :      For multiplication/division/modulo, the libgcc library function requires
    7314              :      no aliasing between result and sources.
    7315              :      On big-endian, even mergeable ops limb processing can be problematic
    7316              :      though, because it can apply various index corrections e.g. when there
    7317              :      is a cast from operand with different number of limbs.  So, make the
    7318              :      lhs conflict with all the operands which are (for now virtually) used on
    7319              :      the current stmt if there is any mismatch in the number of limbs between
    7320              :      operands and the lhs.  */
    7321        81463 :   if (bitint_big_endian && lhs && !muldiv_p)
    7322              :     {
    7323            0 :       tree ltype = TREE_TYPE (lhs);
    7324            0 :       if (TREE_CODE (ltype) == COMPLEX_TYPE)
    7325              :         muldiv_p = true;
    7326            0 :       else if (TREE_CODE (lhs) == SSA_NAME
    7327            0 :                && BITINT_TYPE_P (ltype)
    7328            0 :                && bitint_precision_kind (ltype) >= bitint_prec_large)
    7329              :         {
    7330            0 :           unsigned lnelts = CEIL (TYPE_PRECISION (ltype), limb_prec);
    7331            0 :           FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
    7332              :             {
    7333            0 :               tree type = TREE_TYPE (var);
    7334            0 :               if (TREE_CODE (type) == COMPLEX_TYPE)
    7335            0 :                 type = TREE_TYPE (type);
    7336            0 :               if (BITINT_TYPE_P (type)
    7337            0 :                   && bitint_precision_kind (type) >= bitint_prec_large)
    7338              :                 {
    7339            0 :                   if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
    7340              :                     {
    7341            0 :                       unsigned nelts = CEIL (TYPE_PRECISION (type), limb_prec);
    7342            0 :                       if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
    7343            0 :                           || lnelts != nelts)
    7344              :                         {
    7345            0 :                           muldiv_p = true;
    7346              :                           break;
    7347              :                         }
    7348              :                     }
    7349              :                   else
    7350            0 :                     worklist.safe_push (var);
    7351              :                 }
    7352              :             }
    7353              : 
    7354            0 :           while (!muldiv_p && worklist.length () > 0)
    7355              :             {
    7356            0 :               tree s = worklist.pop ();
    7357            0 :               FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter,
    7358              :                                          SSA_OP_USE)
    7359              :                 {
    7360            0 :                   tree type = TREE_TYPE (var);
    7361            0 :                   if (TREE_CODE (type) == COMPLEX_TYPE)
    7362            0 :                     type = TREE_TYPE (type);
    7363            0 :                   if (BITINT_TYPE_P (type)
    7364            0 :                       && bitint_precision_kind (type) >= bitint_prec_large)
    7365              :                     {
    7366            0 :                       if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
    7367              :                         {
    7368            0 :                           unsigned nelts = CEIL (TYPE_PRECISION (type),
    7369              :                                                  limb_prec);
    7370            0 :                           if (TREE_CODE (TREE_TYPE (var)) == COMPLEX_TYPE
    7371            0 :                               || lnelts != nelts)
    7372              :                             {
    7373              :                               muldiv_p = true;
    7374              :                               break;
    7375              :                             }
    7376              :                         }
    7377              :                       else
    7378            0 :                         worklist.safe_push (var);
    7379              :                     }
    7380              :                 }
    7381              :             }
    7382            0 :           worklist.truncate (0);
    7383              :         }
    7384              :     }
    7385              : 
    7386        81463 :   if (!muldiv_p)
    7387              :     {
    7388              :       /* For stmts with more than one SSA_NAME definition pretend all the
    7389              :          SSA_NAME outputs but the first one are live at this point, so
    7390              :          that conflicts are added in between all those even when they are
    7391              :          actually not really live after the asm, because expansion might
    7392              :          copy those into pseudos after the asm and if multiple outputs
    7393              :          share the same partition, it might overwrite those that should
    7394              :          be live.  E.g.
    7395              :          asm volatile (".." : "=r" (a) : "=r" (b) : "0" (a), "1" (a));
    7396              :          return a;
    7397              :          See PR70593.  */
    7398        79757 :       bool first = true;
    7399       116303 :       FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
    7400        36546 :         if (first)
    7401              :           first = false;
    7402              :         else
    7403            0 :           use (live, var);
    7404              : 
    7405       116303 :       FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_DEF)
    7406        36546 :         def (live, var, graph);
    7407              :     }
    7408              : 
    7409       138309 :   FOR_EACH_SSA_TREE_OPERAND (var, stmt, iter, SSA_OP_USE)
    7410              :     {
    7411        56846 :       tree type = TREE_TYPE (var);
    7412        56846 :       if (TREE_CODE (type) == COMPLEX_TYPE)
    7413         6313 :         type = TREE_TYPE (type);
    7414        19311 :       if (BITINT_TYPE_P (type)
    7415        56863 :           && bitint_precision_kind (type) >= bitint_prec_large)
    7416              :         {
    7417        36446 :           if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
    7418        31290 :             use (live, var);
    7419              :           else
    7420         5156 :             worklist.safe_push (var);
    7421              :         }
    7422              :     }
    7423              : 
    7424        89629 :   while (worklist.length () > 0)
    7425              :     {
    7426         8166 :       tree s = worklist.pop ();
    7427        17092 :       FOR_EACH_SSA_TREE_OPERAND (var, SSA_NAME_DEF_STMT (s), iter, SSA_OP_USE)
    7428              :         {
    7429         8926 :           tree type = TREE_TYPE (var);
    7430         8926 :           if (TREE_CODE (type) == COMPLEX_TYPE)
    7431         1695 :             type = TREE_TYPE (type);
    7432          544 :           if (BITINT_TYPE_P (type)
    7433         8931 :               && bitint_precision_kind (type) >= bitint_prec_large)
    7434              :             {
    7435         8250 :               if (bitmap_bit_p (names, SSA_NAME_VERSION (var)))
    7436         5240 :                 use (live, var);
    7437              :               else
    7438         3010 :                 worklist.safe_push (var);
    7439              :             }
    7440              :         }
    7441              :     }
    7442              : 
    7443        81463 :   if (muldiv_p)
    7444         1706 :     def (live, lhs, graph);
    7445              : }
    7446              : 
    7447              : /* If STMT is .{ADD,SUB,MUL}_OVERFLOW with INTEGER_CST arguments,
    7448              :    return the largest bitint_prec_kind of them, otherwise return
    7449              :    bitint_prec_small.  */
    7450              : 
    7451              : static bitint_prec_kind
    7452       210531 : arith_overflow_arg_kind (gimple *stmt)
    7453              : {
    7454       210531 :   bitint_prec_kind ret = bitint_prec_small;
    7455       210531 :   if (is_gimple_call (stmt) && gimple_call_internal_p (stmt))
    7456        89168 :     switch (gimple_call_internal_fn (stmt))
    7457              :       {
    7458              :       case IFN_ADD_OVERFLOW:
    7459              :       case IFN_SUB_OVERFLOW:
    7460              :       case IFN_MUL_OVERFLOW:
    7461       225999 :         for (int i = 0; i < 2; ++i)
    7462              :           {
    7463       150666 :             tree a = gimple_call_arg (stmt, i);
    7464       150666 :             if (TREE_CODE (a) == INTEGER_CST
    7465       150666 :                 && BITINT_TYPE_P (TREE_TYPE (a)))
    7466              :               {
    7467         5951 :                 bitint_prec_kind kind = bitint_precision_kind (TREE_TYPE (a));
    7468       150666 :                 ret = MAX (ret, kind);
    7469              :               }
    7470              :           }
    7471              :         break;
    7472              :       default:
    7473              :         break;
    7474              :       }
    7475       210531 :   return ret;
    7476              : }
    7477              : 
    7478              : /* Entry point for _BitInt(N) operation lowering during optimization.  */
    7479              : 
    7480              : static unsigned int
    7481      1518056 : gimple_lower_bitint (void)
    7482              : {
    7483      1518056 :   small_max_prec = mid_min_prec = large_min_prec = huge_min_prec = 0;
    7484      1518056 :   limb_prec = abi_limb_prec = 0;
    7485      1518056 :   bitint_big_endian = false;
    7486      1518056 :   bool loop_init_done = false;
    7487              : 
    7488      1518056 :   unsigned int i;
    7489     65169656 :   for (i = 0; i < num_ssa_names; ++i)
    7490              :     {
    7491     63659086 :       tree s = ssa_name (i);
    7492     63659086 :       if (s == NULL)
    7493     13592634 :         continue;
    7494     50066452 :       tree type = TREE_TYPE (s);
    7495     50066452 :       if (TREE_CODE (type) == COMPLEX_TYPE)
    7496              :         {
    7497       199821 :           if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
    7498              :               != bitint_prec_small)
    7499              :             break;
    7500       199701 :           type = TREE_TYPE (type);
    7501              :         }
    7502     50048364 :       if (BITINT_TYPE_P (type)
    7503     50066339 :           && bitint_precision_kind (type) != bitint_prec_small)
    7504              :         break;
    7505              :       /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
    7506              :          into memory.  Such functions could have no large/huge SSA_NAMEs.  */
    7507     50059036 :       if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
    7508              :         {
    7509     21945073 :           gimple *g = SSA_NAME_DEF_STMT (s);
    7510     21945073 :           if (is_gimple_assign (g) && gimple_store_p (g))
    7511              :             {
    7512     11163373 :               tree t = gimple_assign_rhs1 (g);
    7513     22323949 :               if (BITINT_TYPE_P (TREE_TYPE (t))
    7514     11163373 :                   && (bitint_precision_kind (TREE_TYPE (t))
    7515              :                       >= bitint_prec_large))
    7516              :                 break;
    7517              :             }
    7518              :         }
    7519              :       /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
    7520              :          to floating point types need to be rewritten.  */
    7521     28113963 :       else if (SCALAR_FLOAT_TYPE_P (type))
    7522              :         {
    7523      2337326 :           gimple *g = SSA_NAME_DEF_STMT (s);
    7524      2337326 :           if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
    7525              :             {
    7526       129583 :               tree t = gimple_assign_rhs1 (g);
    7527       129583 :               if (TREE_CODE (t) == INTEGER_CST
    7528          110 :                   && BITINT_TYPE_P (TREE_TYPE (t))
    7529       129584 :                   && (bitint_precision_kind (TREE_TYPE (t))
    7530              :                       != bitint_prec_small))
    7531              :                 break;
    7532              :             }
    7533              :         }
    7534              :     }
    7535      3036112 :   if (i == num_ssa_names)
    7536              :     return 0;
    7537              : 
    7538         7486 :   basic_block bb;
    7539         7486 :   auto_vec<gimple *, 4> switch_statements;
    7540        47328 :   FOR_EACH_BB_FN (bb, cfun)
    7541              :     {
    7542       118728 :       if (gswitch *swtch = safe_dyn_cast <gswitch *> (*gsi_last_bb (bb)))
    7543              :         {
    7544           32 :           tree idx = gimple_switch_index (swtch);
    7545           40 :           if (!BITINT_TYPE_P (TREE_TYPE (idx))
    7546           32 :               || bitint_precision_kind (TREE_TYPE (idx)) < bitint_prec_large)
    7547           21 :             continue;
    7548              : 
    7549           11 :           if (optimize)
    7550            6 :             group_case_labels_stmt (swtch);
    7551           11 :           if (gimple_switch_num_labels (swtch) == 1)
    7552              :             {
    7553            0 :               single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
    7554            0 :               gimple_stmt_iterator gsi = gsi_for_stmt (swtch);
    7555            0 :               gsi_remove (&gsi, true);
    7556              :             }
    7557              :           else
    7558           11 :             switch_statements.safe_push (swtch);
    7559              :         }
    7560              :     }
    7561              : 
    7562         7486 :   if (!switch_statements.is_empty ())
    7563              :     {
    7564           11 :       bool expanded = false;
    7565           11 :       gimple *stmt;
    7566           11 :       unsigned int j;
    7567           11 :       i = 0;
    7568           22 :       FOR_EACH_VEC_ELT (switch_statements, j, stmt)
    7569              :         {
    7570           11 :           gswitch *swtch = as_a<gswitch *> (stmt);
    7571           11 :           tree_switch_conversion::switch_decision_tree dt (swtch);
    7572           11 :           expanded |= dt.analyze_switch_statement ();
    7573           11 :         }
    7574              : 
    7575           11 :       if (expanded)
    7576              :         {
    7577           11 :           free_dominance_info (CDI_DOMINATORS);
    7578           11 :           free_dominance_info (CDI_POST_DOMINATORS);
    7579           11 :           mark_virtual_operands_for_renaming (cfun);
    7580           11 :           cleanup_tree_cfg (TODO_update_ssa);
    7581              :         }
    7582              :     }
    7583              : 
    7584         7486 :   if (flag_sanitize & (SANITIZE_ADDRESS | SANITIZE_HWADDRESS))
    7585              :     {
    7586           14 :       hash_map<tree, tree> shadow_vars_mapping;
    7587           14 :       bool need_commit_edge_insert = false;
    7588           14 :       bool any_asan_poison = false;
    7589          129 :       for (unsigned j = 0; j < num_ssa_names; ++j)
    7590              :         {
    7591          115 :           tree s = ssa_name (j);
    7592          115 :           if (s == NULL)
    7593           31 :             continue;
    7594           84 :           tree type = TREE_TYPE (s);
    7595           37 :           if (BITINT_TYPE_P (type)
    7596           47 :               && gimple_call_internal_p (SSA_NAME_DEF_STMT (s),
    7597              :                                          IFN_ASAN_POISON)
    7598          112 :               && bitint_precision_kind (type) >= bitint_prec_large)
    7599              :             {
    7600           28 :               any_asan_poison = true;
    7601           28 :               gimple_stmt_iterator gsi = gsi_for_stmt (SSA_NAME_DEF_STMT (s));
    7602           28 :               asan_expand_poison_ifn (&gsi, &need_commit_edge_insert,
    7603              :                                       shadow_vars_mapping);
    7604              :             }
    7605              :         }
    7606           14 :       if (any_asan_poison)
    7607              :         {
    7608           14 :           if (need_commit_edge_insert)
    7609            0 :             gsi_commit_edge_inserts ();
    7610           14 :           i = 0;
    7611           14 :           free_dominance_info (CDI_DOMINATORS);
    7612           14 :           free_dominance_info (CDI_POST_DOMINATORS);
    7613           14 :           mark_virtual_operands_for_renaming (cfun);
    7614           14 :           cleanup_tree_cfg (TODO_update_ssa);
    7615              :         }
    7616           14 :     }
    7617              : 
    7618         7486 :   struct bitint_large_huge large_huge;
    7619         7486 :   bool has_large_huge_parm_result = false;
    7620         7486 :   bool has_large_huge = false;
    7621         7486 :   unsigned int ret = 0, first_large_huge = ~0U;
    7622         7486 :   bool edge_insertions = false;
    7623       129462 :   for (; i < num_ssa_names; ++i)
    7624              :     {
    7625       121976 :       tree s = ssa_name (i);
    7626       121976 :       if (s == NULL)
    7627         3140 :         continue;
    7628       118836 :       tree type = TREE_TYPE (s);
    7629       118836 :       if (TREE_CODE (type) == COMPLEX_TYPE)
    7630              :         {
    7631         5375 :           if (arith_overflow_arg_kind (SSA_NAME_DEF_STMT (s))
    7632              :               >= bitint_prec_large)
    7633         1958 :             has_large_huge = true;
    7634         5375 :           type = TREE_TYPE (type);
    7635              :         }
    7636        72015 :       if (BITINT_TYPE_P (type)
    7637       118862 :           && bitint_precision_kind (type) >= bitint_prec_large)
    7638              :         {
    7639        36648 :           if (first_large_huge == ~0U)
    7640         5911 :             first_large_huge = i;
    7641        36648 :           gimple *stmt = SSA_NAME_DEF_STMT (s), *g;
    7642        36648 :           gimple_stmt_iterator gsi;
    7643        36648 :           tree_code rhs_code;
    7644              :           /* Unoptimize certain constructs to simpler alternatives to
    7645              :              avoid having to lower all of them.  */
    7646        36648 :           if (is_gimple_assign (stmt) && gimple_bb (stmt))
    7647        22419 :             switch (rhs_code = gimple_assign_rhs_code (stmt))
    7648              :               {
    7649              :               default:
    7650              :                 break;
    7651          373 :               case MULT_EXPR:
    7652          373 :               case TRUNC_DIV_EXPR:
    7653          373 :               case EXACT_DIV_EXPR:
    7654          373 :               case TRUNC_MOD_EXPR:
    7655          373 :                 if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s))
    7656              :                   {
    7657            2 :                     location_t loc = gimple_location (stmt);
    7658            2 :                     gsi = gsi_for_stmt (stmt);
    7659            2 :                     tree rhs1 = gimple_assign_rhs1 (stmt);
    7660            2 :                     tree rhs2 = gimple_assign_rhs2 (stmt);
    7661              :                     /* For multiplication and division with (ab)
    7662              :                        lhs and one or both operands force the operands
    7663              :                        into new SSA_NAMEs to avoid coalescing failures.  */
    7664            2 :                     if (TREE_CODE (rhs1) == SSA_NAME
    7665            2 :                         && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1))
    7666              :                       {
    7667            2 :                         first_large_huge = 0;
    7668            2 :                         tree t = make_ssa_name (TREE_TYPE (rhs1));
    7669            2 :                         g = gimple_build_assign (t, SSA_NAME, rhs1);
    7670            2 :                         gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7671            2 :                         gimple_set_location (g, loc);
    7672            2 :                         gimple_assign_set_rhs1 (stmt, t);
    7673            2 :                         if (rhs1 == rhs2)
    7674              :                           {
    7675            0 :                             gimple_assign_set_rhs2 (stmt, t);
    7676            0 :                             rhs2 = t;
    7677              :                           }
    7678            2 :                         update_stmt (stmt);
    7679              :                       }
    7680            2 :                     if (TREE_CODE (rhs2) == SSA_NAME
    7681            2 :                         && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs2))
    7682              :                       {
    7683            0 :                         first_large_huge = 0;
    7684            0 :                         tree t = make_ssa_name (TREE_TYPE (rhs2));
    7685            0 :                         g = gimple_build_assign (t, SSA_NAME, rhs2);
    7686            0 :                         gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7687            0 :                         gimple_set_location (g, loc);
    7688            0 :                         gimple_assign_set_rhs2 (stmt, t);
    7689            0 :                         update_stmt (stmt);
    7690              :                       }
    7691              :                   }
    7692              :                 break;
    7693            3 :               case LROTATE_EXPR:
    7694            3 :               case RROTATE_EXPR:
    7695            3 :                 {
    7696            3 :                   first_large_huge = 0;
    7697            3 :                   location_t loc = gimple_location (stmt);
    7698            3 :                   gsi = gsi_for_stmt (stmt);
    7699            3 :                   tree rhs1 = gimple_assign_rhs1 (stmt);
    7700            3 :                   tree type = TREE_TYPE (rhs1);
    7701            3 :                   tree n = gimple_assign_rhs2 (stmt), m;
    7702            3 :                   tree p = build_int_cst (TREE_TYPE (n),
    7703            3 :                                           TYPE_PRECISION (type));
    7704            3 :                   if (TREE_CODE (n) == INTEGER_CST)
    7705              :                     {
    7706            0 :                       if (integer_zerop (n))
    7707              :                         m = n;
    7708              :                       else
    7709            0 :                         m = fold_build2 (MINUS_EXPR, TREE_TYPE (n), p, n);
    7710              :                     }
    7711              :                   else
    7712              :                     {
    7713            3 :                       tree tem = make_ssa_name (TREE_TYPE (n));
    7714            3 :                       g = gimple_build_assign (tem, MINUS_EXPR, p, n);
    7715            3 :                       gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7716            3 :                       gimple_set_location (g, loc);
    7717            3 :                       m = make_ssa_name (TREE_TYPE (n));
    7718            3 :                       g = gimple_build_assign (m, TRUNC_MOD_EXPR, tem, p);
    7719            3 :                       gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7720            3 :                       gimple_set_location (g, loc);
    7721              :                     }
    7722            3 :                   if (!TYPE_UNSIGNED (type))
    7723              :                     {
    7724            0 :                       tree utype = build_bitint_type (TYPE_PRECISION (type),
    7725              :                                                       1);
    7726            0 :                       if (TREE_CODE (rhs1) == INTEGER_CST)
    7727            0 :                         rhs1 = fold_convert (utype, rhs1);
    7728              :                       else
    7729              :                         {
    7730            0 :                           tree t = make_ssa_name (utype);
    7731            0 :                           g = gimple_build_assign (t, NOP_EXPR, rhs1);
    7732            0 :                           gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7733            0 :                           gimple_set_location (g, loc);
    7734            0 :                           rhs1 = t;
    7735              :                         }
    7736              :                     }
    7737            4 :                   g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
    7738              :                                            rhs_code == LROTATE_EXPR
    7739              :                                            ? LSHIFT_EXPR : RSHIFT_EXPR,
    7740              :                                            rhs1, n);
    7741            3 :                   gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7742            3 :                   gimple_set_location (g, loc);
    7743            3 :                   tree op1 = gimple_assign_lhs (g);
    7744            4 :                   g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
    7745              :                                            rhs_code == LROTATE_EXPR
    7746              :                                            ? RSHIFT_EXPR : LSHIFT_EXPR,
    7747              :                                            rhs1, m);
    7748            3 :                   gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7749            3 :                   gimple_set_location (g, loc);
    7750            3 :                   tree op2 = gimple_assign_lhs (g);
    7751            3 :                   tree lhs = gimple_assign_lhs (stmt);
    7752            3 :                   if (!TYPE_UNSIGNED (type))
    7753              :                     {
    7754            0 :                       g = gimple_build_assign (make_ssa_name (TREE_TYPE (op1)),
    7755              :                                                BIT_IOR_EXPR, op1, op2);
    7756            0 :                       gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7757            0 :                       gimple_set_location (g, loc);
    7758            0 :                       g = gimple_build_assign (lhs, NOP_EXPR,
    7759              :                                                gimple_assign_lhs (g));
    7760              :                     }
    7761              :                   else
    7762            3 :                     g = gimple_build_assign (lhs, BIT_IOR_EXPR, op1, op2);
    7763            3 :                   gsi_replace (&gsi, g, true);
    7764            3 :                   gimple_set_location (g, loc);
    7765              :                 }
    7766            3 :                 break;
    7767           17 :               case ABS_EXPR:
    7768           17 :               case ABSU_EXPR:
    7769           17 :               case MIN_EXPR:
    7770           17 :               case MAX_EXPR:
    7771           17 :               case COND_EXPR:
    7772           17 :                 first_large_huge = 0;
    7773           17 :                 gsi = gsi_for_stmt (stmt);
    7774           17 :                 tree lhs = gimple_assign_lhs (stmt);
    7775           17 :                 tree rhs1 = gimple_assign_rhs1 (stmt), rhs2 = NULL_TREE;
    7776           17 :                 location_t loc = gimple_location (stmt);
    7777           17 :                 if (rhs_code == ABS_EXPR)
    7778            4 :                   g = gimple_build_cond (LT_EXPR, rhs1,
    7779            4 :                                          build_zero_cst (TREE_TYPE (rhs1)),
    7780              :                                          NULL_TREE, NULL_TREE);
    7781           13 :                 else if (rhs_code == ABSU_EXPR)
    7782              :                   {
    7783            4 :                     rhs2 = make_ssa_name (TREE_TYPE (lhs));
    7784            4 :                     g = gimple_build_assign (rhs2, NOP_EXPR, rhs1);
    7785            4 :                     gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7786            4 :                     gimple_set_location (g, loc);
    7787            4 :                     g = gimple_build_cond (LT_EXPR, rhs1,
    7788            4 :                                            build_zero_cst (TREE_TYPE (rhs1)),
    7789              :                                            NULL_TREE, NULL_TREE);
    7790            4 :                     rhs1 = rhs2;
    7791              :                   }
    7792            9 :                 else if (rhs_code == MIN_EXPR || rhs_code == MAX_EXPR)
    7793              :                   {
    7794            9 :                     rhs2 = gimple_assign_rhs2 (stmt);
    7795            9 :                     if (TREE_CODE (rhs1) == INTEGER_CST)
    7796            0 :                       std::swap (rhs1, rhs2);
    7797            9 :                     g = gimple_build_cond (LT_EXPR, rhs1, rhs2,
    7798              :                                            NULL_TREE, NULL_TREE);
    7799            9 :                     if (rhs_code == MAX_EXPR)
    7800            5 :                       std::swap (rhs1, rhs2);
    7801              :                   }
    7802              :                 else
    7803              :                   {
    7804            0 :                     g = gimple_build_cond (NE_EXPR, rhs1,
    7805            0 :                                            build_zero_cst (TREE_TYPE (rhs1)),
    7806              :                                            NULL_TREE, NULL_TREE);
    7807            0 :                     rhs1 = gimple_assign_rhs2 (stmt);
    7808            0 :                     rhs2 = gimple_assign_rhs3 (stmt);
    7809              :                   }
    7810           17 :                 gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7811           17 :                 gimple_set_location (g, loc);
    7812           17 :                 edge e1 = split_block (gsi_bb (gsi), g);
    7813           17 :                 edge e2 = split_block (e1->dest, (gimple *) NULL);
    7814           17 :                 edge e3 = make_edge (e1->src, e2->dest, EDGE_FALSE_VALUE);
    7815           17 :                 e3->probability = profile_probability::even ();
    7816           17 :                 e1->flags = EDGE_TRUE_VALUE;
    7817           17 :                 e1->probability = e3->probability.invert ();
    7818           17 :                 if (dom_info_available_p (CDI_DOMINATORS))
    7819            9 :                   set_immediate_dominator (CDI_DOMINATORS, e2->dest, e1->src);
    7820           17 :                 if (rhs_code == ABS_EXPR || rhs_code == ABSU_EXPR)
    7821              :                   {
    7822            8 :                     gsi = gsi_after_labels (e1->dest);
    7823            8 :                     g = gimple_build_assign (make_ssa_name (TREE_TYPE (rhs1)),
    7824              :                                              NEGATE_EXPR, rhs1);
    7825            8 :                     gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7826            8 :                     gimple_set_location (g, loc);
    7827            8 :                     rhs2 = gimple_assign_lhs (g);
    7828            8 :                     std::swap (rhs1, rhs2);
    7829              :                   }
    7830           17 :                 gsi = gsi_for_stmt (stmt);
    7831           17 :                 gsi_remove (&gsi, true);
    7832           17 :                 gphi *phi = create_phi_node (lhs, e2->dest);
    7833           17 :                 add_phi_arg (phi, rhs1, e2, UNKNOWN_LOCATION);
    7834           17 :                 add_phi_arg (phi, rhs2, e3, UNKNOWN_LOCATION);
    7835           17 :                 break;
    7836              :               }
    7837        14229 :           else if (gimple_call_internal_p (stmt, IFN_CLMUL_WIDEN))
    7838              :             {
    7839              :               /* For now, lower this to two IFN_CLMUL and three
    7840              :                  IFN_BITREVERSE calls.  Could be implemented in
    7841              :                  lower_clmul too, just needs special care on padding
    7842              :                  bits and split the shifting into up to two disjunct
    7843              :                  results, or treat result as 2*prec array and adjust
    7844              :                  afterwards.  */
    7845           10 :               first_large_huge = 0;
    7846           10 :               location_t loc = gimple_location (stmt);
    7847           10 :               tree arg0 = gimple_call_arg (stmt, 0);
    7848           10 :               tree arg1 = gimple_call_arg (stmt, 1);
    7849           10 :               tree type = TREE_TYPE (arg0);
    7850           10 :               gsi = gsi_for_stmt (stmt);
    7851           10 :               gimple *g = gimple_build_call_internal (IFN_CLMUL, 2,
    7852              :                                                       arg0, arg1);
    7853           10 :               tree real = make_ssa_name (type);
    7854           10 :               gimple_call_set_lhs (g, real);
    7855           10 :               gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7856           10 :               gimple_set_location (g, loc);
    7857           10 :               g = gimple_build_call_internal (IFN_BITREVERSE, 1, arg0);
    7858           10 :               tree rarg0 = make_ssa_name (type);
    7859           10 :               gimple_call_set_lhs (g, rarg0);
    7860           10 :               gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7861           10 :               gimple_set_location (g, loc);
    7862           10 :               g = gimple_build_call_internal (IFN_BITREVERSE, 1, arg1);
    7863           10 :               tree rarg1 = make_ssa_name (type);
    7864           10 :               gimple_call_set_lhs (g, rarg1);
    7865           10 :               gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7866           10 :               gimple_set_location (g, loc);
    7867           10 :               g = gimple_build_call_internal (IFN_CLMUL, 2, rarg0, rarg1);
    7868           10 :               tree rimag = make_ssa_name (type);
    7869           10 :               gimple_call_set_lhs (g, rimag);
    7870           10 :               gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7871           10 :               gimple_set_location (g, loc);
    7872           10 :               g = gimple_build_call_internal (IFN_BITREVERSE, 1, rimag);
    7873           10 :               tree imag = make_ssa_name (type);
    7874           10 :               gimple_call_set_lhs (g, imag);
    7875           10 :               gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7876           10 :               gimple_set_location (g, loc);
    7877           10 :               g = gimple_build_assign (make_ssa_name (type), RSHIFT_EXPR, imag,
    7878              :                                        build_one_cst (unsigned_type_node));
    7879           10 :               gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7880           10 :               gimple_set_location (g, loc);
    7881           10 :               g = gimple_build_assign (gimple_call_lhs (stmt), COMPLEX_EXPR,
    7882              :                                        real, gimple_assign_lhs (g));
    7883           10 :               gimple_set_location (g, loc);
    7884           10 :               gsi_replace (&gsi, g, true);
    7885              :             }
    7886        14219 :           else if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
    7887           76 :                    && is_gimple_call (stmt)
    7888        14239 :                    && gimple_call_internal_p (stmt))
    7889           20 :             switch (gimple_call_internal_fn (stmt))
    7890              :               {
    7891           12 :               case IFN_UBSAN_CHECK_ADD:
    7892           12 :               case IFN_UBSAN_CHECK_SUB:
    7893           12 :                 if (!bitint_big_endian)
    7894              :                   break;
    7895              :                 /* FALLTHRU */
    7896              :               case IFN_UBSAN_CHECK_MUL:
    7897              :               case IFN_BSWAP:
    7898              :               case IFN_BITREVERSE:
    7899              :               case IFN_CLMUL:
    7900           22 :                 for (unsigned i = 0; i < gimple_call_num_args (stmt); ++i)
    7901              :                   {
    7902           14 :                     location_t loc = gimple_location (stmt);
    7903           14 :                     gsi = gsi_for_stmt (stmt);
    7904              :                     /* Similar case to multiplication/division with (ab)
    7905              :                        above for internal functions which set muldiv_p.  */
    7906           14 :                     tree arg = gimple_call_arg (stmt, i);
    7907           14 :                     if (TREE_CODE (arg) == SSA_NAME
    7908           14 :                         && SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg))
    7909              :                       {
    7910            8 :                         first_large_huge = 0;
    7911            8 :                         tree t = make_ssa_name (TREE_TYPE (arg));
    7912            8 :                         g = gimple_build_assign (t, SSA_NAME, arg);
    7913            8 :                         gsi_insert_before (&gsi, g, GSI_SAME_STMT);
    7914            8 :                         gimple_set_location (g, loc);
    7915            8 :                         gimple_call_set_arg (stmt, i, t);
    7916            8 :                         if (i == 0
    7917            8 :                             && gimple_call_num_args (stmt) >= 2
    7918           14 :                             && gimple_call_arg (stmt, 1) == arg)
    7919            0 :                           gimple_call_set_arg (stmt, 1, t);
    7920            8 :                         update_stmt (stmt);
    7921              :                       }
    7922              :                   }
    7923              :                 break;
    7924              :               default:
    7925              :                 break;
    7926              :               }
    7927        36903 :           else if (gphi *phi = dyn_cast <gphi *> (stmt))
    7928              :             {
    7929              :               /* Remove PHIs with no uses, coalescing apparently
    7930              :                  relies on this.  */
    7931          255 :               if (has_zero_uses (s))
    7932              :                 {
    7933            8 :                   remove_gimple_phi_args (phi);
    7934            8 :                   gsi = gsi_for_stmt (stmt);
    7935            8 :                   remove_phi_node (&gsi, true);
    7936              :                 }
    7937              :             }
    7938              :         }
    7939              :       /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
    7940              :          into memory.  Such functions could have no large/huge SSA_NAMEs.  */
    7941        82188 :       else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
    7942              :         {
    7943        52623 :           gimple *g = SSA_NAME_DEF_STMT (s);
    7944        52623 :           if (is_gimple_assign (g) && gimple_store_p (g))
    7945              :             {
    7946        17176 :               tree t = gimple_assign_rhs1 (g);
    7947        19503 :               if (BITINT_TYPE_P (TREE_TYPE (t))
    7948        17176 :                   && (bitint_precision_kind (TREE_TYPE (t))
    7949              :                       >= bitint_prec_large))
    7950              :                 has_large_huge = true;
    7951              :             }
    7952              :         }
    7953              :       /* Similarly, e.g. with -frounding-math casts from _BitInt INTEGER_CSTs
    7954              :          to floating point types need to be rewritten.  */
    7955        29565 :       else if (SCALAR_FLOAT_TYPE_P (type))
    7956              :         {
    7957          742 :           gimple *g = SSA_NAME_DEF_STMT (s);
    7958          742 :           if (is_gimple_assign (g) && gimple_assign_rhs_code (g) == FLOAT_EXPR)
    7959              :             {
    7960          187 :               tree t = gimple_assign_rhs1 (g);
    7961          187 :               if (TREE_CODE (t) == INTEGER_CST
    7962            1 :                   && BITINT_TYPE_P (TREE_TYPE (t))
    7963          188 :                   && (bitint_precision_kind (TREE_TYPE (t))
    7964              :                       >= bitint_prec_large))
    7965              :                 has_large_huge = true;
    7966              :             }
    7967              :         }
    7968              :     }
    7969       105674 :   for (i = first_large_huge; i < num_ssa_names; ++i)
    7970              :     {
    7971        98188 :       tree s = ssa_name (i);
    7972        98188 :       if (s == NULL)
    7973         2808 :         continue;
    7974        95380 :       tree type = TREE_TYPE (s);
    7975        95380 :       if (TREE_CODE (type) == COMPLEX_TYPE)
    7976         4134 :         type = TREE_TYPE (type);
    7977        58064 :       if (BITINT_TYPE_P (type)
    7978        95404 :           && bitint_precision_kind (type) >= bitint_prec_large)
    7979              :         {
    7980        36640 :           use_operand_p use_p;
    7981        36640 :           gimple *use_stmt;
    7982        36640 :           has_large_huge = true;
    7983        38304 :           if (optimize
    7984        54098 :               && optimizable_arith_overflow (SSA_NAME_DEF_STMT (s)))
    7985         6635 :             continue;
    7986              :           /* Ignore large/huge _BitInt SSA_NAMEs which have single use in
    7987              :              the same bb and could be handled in the same loop with the
    7988              :              immediate use.  */
    7989        34976 :           if (optimize
    7990        15794 :               && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
    7991        15727 :               && single_imm_use (s, &use_p, &use_stmt)
    7992        50036 :               && gimple_bb (SSA_NAME_DEF_STMT (s)) == gimple_bb (use_stmt))
    7993              :             {
    7994        10587 :               if (mergeable_op (SSA_NAME_DEF_STMT (s)))
    7995              :                 {
    7996         2238 :                   if (mergeable_op (use_stmt))
    7997         1927 :                     continue;
    7998          311 :                   tree_code cmp_code = comparison_op (use_stmt, NULL, NULL);
    7999          311 :                   if (cmp_code == EQ_EXPR || cmp_code == NE_EXPR)
    8000           26 :                     continue;
    8001          285 :                   if (gimple_assign_cast_p (use_stmt))
    8002              :                     {
    8003          130 :                       tree lhs = gimple_assign_lhs (use_stmt);
    8004          260 :                       if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
    8005              :                           /* Don't merge with VIEW_CONVERT_EXPRs to
    8006              :                              huge INTEGER_TYPEs used sometimes in memcpy
    8007              :                              expansion.  */
    8008          248 :                           && (TREE_CODE (TREE_TYPE (lhs)) != INTEGER_TYPE
    8009            6 :                               || (TYPE_PRECISION (TREE_TYPE (lhs))
    8010           12 :                                   <= MAX_FIXED_MODE_SIZE)))
    8011          118 :                         continue;
    8012              :                     }
    8013          155 :                   else if (gimple_store_p (use_stmt)
    8014            0 :                            && is_gimple_assign (use_stmt)
    8015            0 :                            && !gimple_has_volatile_ops (use_stmt)
    8016          155 :                            && !stmt_ends_bb_p (use_stmt))
    8017            0 :                     continue;
    8018              :                 }
    8019         8516 :               if (gimple_assign_cast_p (SSA_NAME_DEF_STMT (s)))
    8020              :                 {
    8021          874 :                   tree rhs1 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s));
    8022          874 :                   if (TREE_CODE (rhs1) == VIEW_CONVERT_EXPR)
    8023              :                     {
    8024           20 :                       rhs1 = TREE_OPERAND (rhs1, 0);
    8025           20 :                       if (!INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
    8026           16 :                           && !POINTER_TYPE_P (TREE_TYPE (rhs1))
    8027           16 :                           && gimple_store_p (use_stmt))
    8028            7 :                         continue;
    8029              :                     }
    8030         1734 :                   if (INTEGRAL_TYPE_P (TREE_TYPE (rhs1))
    8031          776 :                       && ((is_gimple_assign (use_stmt)
    8032          729 :                            && (gimple_assign_rhs_code (use_stmt)
    8033              :                                != COMPLEX_EXPR))
    8034           47 :                           || gimple_code (use_stmt) == GIMPLE_COND)
    8035          766 :                       && (!gimple_store_p (use_stmt)
    8036          132 :                           || (is_gimple_assign (use_stmt)
    8037          132 :                               && !gimple_has_volatile_ops (use_stmt)
    8038          132 :                               && !stmt_ends_bb_p (use_stmt)))
    8039         1633 :                       && (TREE_CODE (rhs1) != SSA_NAME
    8040          766 :                           || !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (rhs1)))
    8041              :                     {
    8042          766 :                       if (is_gimple_assign (use_stmt))
    8043          729 :                         switch (gimple_assign_rhs_code (use_stmt))
    8044              :                           {
    8045           54 :                           case TRUNC_DIV_EXPR:
    8046           54 :                           case EXACT_DIV_EXPR:
    8047           54 :                           case TRUNC_MOD_EXPR:
    8048           54 :                           case FLOAT_EXPR:
    8049              :                             /* For division, modulo and casts to floating
    8050              :                                point, avoid representing unsigned operands
    8051              :                                using negative prec if they were sign-extended
    8052              :                                from narrower precision.  */
    8053           54 :                             if (TYPE_UNSIGNED (TREE_TYPE (s))
    8054           28 :                                 && !TYPE_UNSIGNED (TREE_TYPE (rhs1))
    8055           63 :                                 && (TYPE_PRECISION (TREE_TYPE (s))
    8056            9 :                                     > TYPE_PRECISION (TREE_TYPE (rhs1))))
    8057            8 :                               goto force_name;
    8058              :                             /* FALLTHRU */
    8059          115 :                           case MULT_EXPR:
    8060          130 :                             if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
    8061          115 :                                 || (bitint_precision_kind (TREE_TYPE (rhs1))
    8062              :                                     < bitint_prec_large))
    8063           50 :                               continue;
    8064              :                             /* Uses which use handle_operand_addr can't
    8065              :                                deal with nested casts.  */
    8066           65 :                             if (TREE_CODE (rhs1) == SSA_NAME
    8067           65 :                                 && gimple_assign_cast_p
    8068           65 :                                      (SSA_NAME_DEF_STMT (rhs1))
    8069           43 :                                 && has_single_use (rhs1)
    8070          108 :                                 && (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
    8071           43 :                                     == gimple_bb (SSA_NAME_DEF_STMT (s))))
    8072           43 :                               goto force_name;
    8073              :                             break;
    8074            0 :                           case VIEW_CONVERT_EXPR:
    8075            0 :                             {
    8076            0 :                               tree lhs = gimple_assign_lhs (use_stmt);
    8077              :                               /* Don't merge with VIEW_CONVERT_EXPRs to
    8078              :                                  non-integral types.  */
    8079            0 :                               if (!INTEGRAL_TYPE_P (TREE_TYPE (lhs)))
    8080            0 :                                 goto force_name;
    8081              :                               /* Don't merge with VIEW_CONVERT_EXPRs to
    8082              :                                  huge INTEGER_TYPEs used sometimes in memcpy
    8083              :                                  expansion.  */
    8084            0 :                               if (TREE_CODE (TREE_TYPE (lhs)) == INTEGER_TYPE
    8085            0 :                                   && (TYPE_PRECISION (TREE_TYPE (lhs))
    8086            0 :                                       > MAX_FIXED_MODE_SIZE))
    8087            0 :                                 goto force_name;
    8088              :                             }
    8089              :                             break;
    8090              :                           default:
    8091              :                             break;
    8092              :                         }
    8093          850 :                       if (!BITINT_TYPE_P (TREE_TYPE (rhs1))
    8094          665 :                           || (bitint_precision_kind (TREE_TYPE (rhs1))
    8095              :                               < bitint_prec_large))
    8096          247 :                         continue;
    8097          418 :                       if ((TYPE_PRECISION (TREE_TYPE (rhs1))
    8098          418 :                            >= TYPE_PRECISION (TREE_TYPE (s)))
    8099          418 :                           && mergeable_op (use_stmt))
    8100           60 :                         continue;
    8101              :                       /* Prevent merging a widening non-mergeable cast
    8102              :                          on result of some narrower mergeable op
    8103              :                          together with later mergeable operations.  E.g.
    8104              :                          result of _BitInt(223) addition shouldn't be
    8105              :                          sign-extended to _BitInt(513) and have another
    8106              :                          _BitInt(513) added to it, as handle_plus_minus
    8107              :                          with its PHI node handling inside of handle_cast
    8108              :                          will not work correctly.  An exception is if
    8109              :                          use_stmt is a store, this is handled directly
    8110              :                          in lower_mergeable_stmt.  */
    8111          709 :                       if (TREE_CODE (rhs1) != SSA_NAME
    8112          358 :                           || !has_single_use (rhs1)
    8113          272 :                           || (gimple_bb (SSA_NAME_DEF_STMT (rhs1))
    8114          272 :                               != gimple_bb (SSA_NAME_DEF_STMT (s)))
    8115          216 :                           || !mergeable_op (SSA_NAME_DEF_STMT (rhs1))
    8116          445 :                           || gimple_store_p (use_stmt))
    8117          351 :                         continue;
    8118            7 :                       if ((TYPE_PRECISION (TREE_TYPE (rhs1))
    8119            7 :                            < TYPE_PRECISION (TREE_TYPE (s)))
    8120            9 :                           && gimple_assign_cast_p (SSA_NAME_DEF_STMT (rhs1)))
    8121              :                         {
    8122              :                           /* Another exception is if the widening cast is
    8123              :                              from mergeable same precision cast from something
    8124              :                              not mergeable.  */
    8125            0 :                           tree rhs2
    8126            0 :                             = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (rhs1));
    8127            0 :                           if (BITINT_TYPE_P (TREE_TYPE (rhs2))
    8128            0 :                               && (TYPE_PRECISION (TREE_TYPE (rhs1))
    8129            0 :                                   == TYPE_PRECISION (TREE_TYPE (rhs2))))
    8130              :                             {
    8131            0 :                               if (TREE_CODE (rhs2) != SSA_NAME
    8132            0 :                                   || !has_single_use (rhs2)
    8133            0 :                                   || (gimple_bb (SSA_NAME_DEF_STMT (rhs2))
    8134            0 :                                       != gimple_bb (SSA_NAME_DEF_STMT (s)))
    8135            0 :                                   || !mergeable_op (SSA_NAME_DEF_STMT (rhs2)))
    8136            0 :                                 continue;
    8137              :                             }
    8138              :                         }
    8139              :                     }
    8140              :                 }
    8141         7750 :               if (is_gimple_assign (SSA_NAME_DEF_STMT (s)))
    8142         5464 :                 switch (gimple_assign_rhs_code (SSA_NAME_DEF_STMT (s)))
    8143              :                   {
    8144         1814 :                   case REALPART_EXPR:
    8145         1814 :                   case IMAGPART_EXPR:
    8146         1814 :                     {
    8147         1814 :                       gimple *ds = SSA_NAME_DEF_STMT (s);
    8148         1814 :                       tree rhs1 = gimple_assign_rhs1 (ds);
    8149         1814 :                       rhs1 = TREE_OPERAND (rhs1, 0);
    8150         1814 :                       if (TREE_CODE (rhs1) == SSA_NAME)
    8151              :                         {
    8152         1814 :                           gimple *g = SSA_NAME_DEF_STMT (rhs1);
    8153         1814 :                           if (optimizable_arith_overflow (g))
    8154              :                             {
    8155         1650 :                               if (gimple_assign_rhs_code (ds) == IMAGPART_EXPR)
    8156         1640 :                                 continue;
    8157           10 :                               if (gimple_store_p (use_stmt))
    8158              :                                 {
    8159              :                                   /* Punt if the cast use of IMAGPART_EXPR stmt
    8160              :                                      appears before the store use_stmt, because
    8161              :                                      optimizable arith overflow can't be
    8162              :                                      lowered at the store location in that case.
    8163              :                                      See PR121828.  */
    8164           10 :                                   gimple_stmt_iterator gsi
    8165           10 :                                     = gsi_for_stmt (use_stmt);
    8166           10 :                                   unsigned int cnt = 0;
    8167           12 :                                   do
    8168              :                                     {
    8169           12 :                                       gsi_prev_nondebug (&gsi);
    8170           12 :                                       if (gsi_end_p (gsi))
    8171              :                                         break;
    8172           12 :                                       gimple *g2 = gsi_stmt (gsi);
    8173           12 :                                       if (g2 == ds)
    8174              :                                         break;
    8175            3 :                                       if (++cnt == 64)
    8176              :                                         break;
    8177            3 :                                       if (!gimple_assign_cast_p (g2))
    8178            2 :                                         continue;
    8179            1 :                                       tree rhs2 = gimple_assign_rhs1 (g2);
    8180            1 :                                       if (TREE_CODE (rhs2) != SSA_NAME)
    8181            0 :                                         continue;
    8182            1 :                                       gimple *g3 = SSA_NAME_DEF_STMT (rhs2);
    8183            1 :                                       if (!is_gimple_assign (g3))
    8184            0 :                                         continue;
    8185            1 :                                       if (gimple_assign_rhs_code (g3)
    8186              :                                           != IMAGPART_EXPR)
    8187            0 :                                         continue;
    8188            1 :                                       rhs2 = gimple_assign_rhs1 (g3);
    8189            1 :                                       rhs2 = TREE_OPERAND (rhs2, 0);
    8190            1 :                                       if (rhs2 != rhs1)
    8191            0 :                                         continue;
    8192              :                                       cnt = 64;
    8193              :                                       break;
    8194              :                                     }
    8195              :                                   while (1);
    8196           10 :                                   if (cnt == 64)
    8197              :                                     break;
    8198              :                                 }
    8199              :                             }
    8200              :                         }
    8201              :                     }
    8202              :                     /* FALLTHRU */
    8203          876 :                   case LSHIFT_EXPR:
    8204          876 :                   case RSHIFT_EXPR:
    8205          876 :                   case MULT_EXPR:
    8206          876 :                   case TRUNC_DIV_EXPR:
    8207          876 :                   case EXACT_DIV_EXPR:
    8208          876 :                   case TRUNC_MOD_EXPR:
    8209          876 :                   case FIX_TRUNC_EXPR:
    8210          876 :                     if (gimple_store_p (use_stmt)
    8211          466 :                         && is_gimple_assign (use_stmt)
    8212          466 :                         && !gimple_has_volatile_ops (use_stmt)
    8213         1342 :                         && !stmt_ends_bb_p (use_stmt))
    8214              :                       {
    8215          466 :                         tree lhs = gimple_assign_lhs (use_stmt);
    8216              :                         /* As multiply/division passes address of the lhs
    8217              :                            to library function and that assumes it can extend
    8218              :                            it to whole number of limbs, avoid merging those
    8219              :                            with bit-field stores.  Don't allow it for
    8220              :                            shifts etc. either, so that the bit-field store
    8221              :                            handling doesn't have to be done everywhere.  */
    8222          466 :                         if (TREE_CODE (lhs) == COMPONENT_REF
    8223          466 :                             && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
    8224              :                           break;
    8225          463 :                         continue;
    8226          463 :                       }
    8227              :                     break;
    8228              :                   default:
    8229              :                     break;
    8230              :                   }
    8231         2286 :               else if (is_gimple_call (SSA_NAME_DEF_STMT (s))
    8232         2286 :                        && gimple_call_internal_p (SSA_NAME_DEF_STMT (s)))
    8233           75 :                 switch (gimple_call_internal_fn (SSA_NAME_DEF_STMT (s)))
    8234              :                   {
    8235           73 :                   case IFN_BSWAP:
    8236           73 :                   case IFN_BITREVERSE:
    8237           73 :                   case IFN_CLMUL:
    8238           73 :                     if (gimple_store_p (use_stmt)
    8239           23 :                         && is_gimple_assign (use_stmt)
    8240           23 :                         && !gimple_has_volatile_ops (use_stmt)
    8241           96 :                         && !stmt_ends_bb_p (use_stmt))
    8242              :                       {
    8243           23 :                         tree lhs = gimple_assign_lhs (use_stmt);
    8244           23 :                         if (TREE_CODE (lhs) == COMPONENT_REF
    8245           23 :                             && DECL_BIT_FIELD_TYPE (TREE_OPERAND (lhs, 1)))
    8246              :                           break;
    8247           23 :                         continue;
    8248           23 :                       }
    8249              :                     break;
    8250              :                   default:
    8251              :                     break;
    8252              :                   }
    8253              :             }
    8254              : 
    8255              :           /* Also ignore uninitialized uses.  */
    8256        30013 :           if (SSA_NAME_IS_DEFAULT_DEF (s)
    8257        30013 :               && (!SSA_NAME_VAR (s) || VAR_P (SSA_NAME_VAR (s))))
    8258           59 :             continue;
    8259              : 
    8260        30005 :         force_name:
    8261        30005 :           if (!large_huge.m_names)
    8262         5788 :             large_huge.m_names = BITMAP_ALLOC (NULL);
    8263        30005 :           bitmap_set_bit (large_huge.m_names, SSA_NAME_VERSION (s));
    8264        30005 :           if (has_single_use (s))
    8265              :             {
    8266              :               tree s2 = s, s3 = s;
    8267              :               /* The coalescing hook special cases SSA_NAME copies.
    8268              :                  Make sure not to mark in m_single_use_names single
    8269              :                  use SSA_NAMEs copied from non-single use SSA_NAMEs.  */
    8270        26243 :               while (gimple_assign_copy_p (SSA_NAME_DEF_STMT (s2)))
    8271              :                 {
    8272          948 :                   s2 = gimple_assign_rhs1 (SSA_NAME_DEF_STMT (s2));
    8273          948 :                   if (TREE_CODE (s2) != SSA_NAME)
    8274              :                     break;
    8275           48 :                   if (!has_single_use (s2))
    8276              :                     {
    8277              :                       s2 = NULL_TREE;
    8278              :                       break;
    8279              :                     }
    8280              :                   s3 = s2;
    8281              :                 }
    8282        26229 :               if (s2)
    8283              :                 {
    8284              :                   /* Only set m_single_use_names for SSA_NAMEs where
    8285              :                      the definition and use are within the same loop,
    8286              :                      otherwise if the use could be executed more times
    8287              :                      than the def stmt, we can't emit clobbers.
    8288              :                      Or if irreducible loops are involved, only handle
    8289              :                      same bb definitions.  */
    8290        26195 :                   gimple *def = SSA_NAME_DEF_STMT (s3), *use;
    8291        26195 :                   use_operand_p use_p;
    8292        26195 :                   single_imm_use (s, &use_p, &use);
    8293        26195 :                   basic_block def_bb, use_bb;
    8294        26195 :                   if (SSA_NAME_IS_DEFAULT_DEF (s3))
    8295              :                     def_bb = NULL;
    8296              :                   else
    8297        20960 :                     def_bb = gimple_bb (def);
    8298        26195 :                   use_bb = gimple_bb (use);
    8299        26195 :                   if (def_bb != use_bb)
    8300              :                     {
    8301         7069 :                       if (!loop_init_done)
    8302              :                         {
    8303         4218 :                           unsigned flags
    8304              :                             = (AVOID_CFG_MODIFICATIONS
    8305              :                                | LOOPS_HAVE_MARKED_IRREDUCIBLE_REGIONS);
    8306         4218 :                           loop_optimizer_init (flags);
    8307         4218 :                           loop_init_done = true;
    8308              :                         }
    8309         7069 :                       if ((use_bb->flags & BB_IRREDUCIBLE_LOOP) == 0)
    8310              :                         {
    8311         7021 :                           class loop *def_loop
    8312         7021 :                             = def_bb ? def_bb->loop_father
    8313         7021 :                                      : get_loop (cfun, 0);
    8314         7021 :                           class loop *use_loop = use_bb->loop_father;
    8315         7021 :                           if (def_loop == use_loop)
    8316        26195 :                             def_bb = use_bb;
    8317              :                         }
    8318              :                     }
    8319        26195 :                   if (def_bb == use_bb)
    8320              :                     {
    8321        26078 :                       if (!large_huge.m_single_use_names)
    8322         5651 :                         large_huge.m_single_use_names = BITMAP_ALLOC (NULL);
    8323        26078 :                       bitmap_set_bit (large_huge.m_single_use_names,
    8324        26078 :                                       SSA_NAME_VERSION (s));
    8325              :                     }
    8326              :                 }
    8327              :             }
    8328        30005 :           if (SSA_NAME_VAR (s)
    8329         7585 :               && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
    8330         5515 :                    && SSA_NAME_IS_DEFAULT_DEF (s))
    8331         2107 :                   || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
    8332              :             has_large_huge_parm_result = true;
    8333        30005 :           if (optimize
    8334        10823 :               && !SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s)
    8335        10756 :               && gimple_assign_load_p (SSA_NAME_DEF_STMT (s))
    8336         6090 :               && !gimple_has_volatile_ops (SSA_NAME_DEF_STMT (s))
    8337        33001 :               && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
    8338              :             {
    8339         2996 :               use_operand_p use_p;
    8340         2996 :               imm_use_iterator iter;
    8341         2996 :               bool optimizable_load = true;
    8342         6057 :               FOR_EACH_IMM_USE_FAST (use_p, iter, s)
    8343              :                 {
    8344         3159 :                   gimple *use_stmt = USE_STMT (use_p);
    8345         3159 :                   if (is_gimple_debug (use_stmt))
    8346            0 :                     continue;
    8347         3159 :                   if (is_gimple_call (use_stmt)
    8348         3159 :                       && gimple_call_internal_p (use_stmt))
    8349           66 :                     switch (gimple_call_internal_fn (use_stmt))
    8350              :                       {
    8351           10 :                       case IFN_CLZ:
    8352           10 :                       case IFN_CTZ:
    8353           10 :                       case IFN_CLRSB:
    8354           10 :                       case IFN_FFS:
    8355           10 :                       case IFN_PARITY:
    8356           10 :                       case IFN_POPCOUNT:
    8357           10 :                       case IFN_BSWAP:
    8358           10 :                       case IFN_BITREVERSE:
    8359           10 :                       case IFN_CLMUL:
    8360           10 :                         continue;
    8361              :                       default:
    8362              :                         break;
    8363              :                       }
    8364         3149 :                   if (gimple_code (use_stmt) == GIMPLE_PHI
    8365              :                       || is_gimple_call (use_stmt)
    8366              :                       || gimple_code (use_stmt) == GIMPLE_ASM
    8367              :                       || (is_gimple_assign (use_stmt)
    8368         1907 :                           && (gimple_assign_rhs_code (use_stmt)
    8369              :                               == COMPLEX_EXPR)))
    8370              :                     {
    8371              :                       optimizable_load = false;
    8372              :                       break;
    8373              :                     }
    8374         2996 :                 }
    8375              : 
    8376         2996 :               ssa_op_iter oi;
    8377         4045 :               FOR_EACH_SSA_USE_OPERAND (use_p, SSA_NAME_DEF_STMT (s),
    8378              :                                         oi, SSA_OP_USE)
    8379              :                 {
    8380         1049 :                   tree s2 = USE_FROM_PTR (use_p);
    8381         1049 :                   if (SSA_NAME_OCCURS_IN_ABNORMAL_PHI (s2))
    8382              :                     {
    8383              :                       optimizable_load = false;
    8384              :                       break;
    8385              :                     }
    8386              :                 }
    8387              : 
    8388         2996 :               if (optimizable_load && !stmt_ends_bb_p (SSA_NAME_DEF_STMT (s)))
    8389              :                 {
    8390         2898 :                   if (!large_huge.m_loads)
    8391          317 :                     large_huge.m_loads = BITMAP_ALLOC (NULL);
    8392         2898 :                   bitmap_set_bit (large_huge.m_loads, SSA_NAME_VERSION (s));
    8393              :                 }
    8394              :             }
    8395              :         }
    8396              :       /* We need to also rewrite stores of large/huge _BitInt INTEGER_CSTs
    8397              :          into memory.  Such functions could have no large/huge SSA_NAMEs.  */
    8398        58740 :       else if (SSA_NAME_IS_VIRTUAL_OPERAND (s))
    8399              :         {
    8400        41687 :           gimple *g = SSA_NAME_DEF_STMT (s);
    8401        41687 :           if (is_gimple_assign (g) && gimple_store_p (g))
    8402              :             {
    8403        14443 :               tree t = gimple_assign_rhs1 (g);
    8404        16394 :               if (BITINT_TYPE_P (TREE_TYPE (t))
    8405        14443 :                   && bitint_precision_kind (TREE_TYPE (t)) >= bitint_prec_large)
    8406              :                 has_large_huge = true;
    8407              :             }
    8408              :         }
    8409              :     }
    8410              : 
    8411         7486 :   if (large_huge.m_names || has_large_huge)
    8412              :     {
    8413         5988 :       ret = TODO_update_ssa_only_virtuals | TODO_cleanup_cfg;
    8414         5988 :       calculate_dominance_info (CDI_DOMINATORS);
    8415         5988 :       if (optimize)
    8416         3192 :         enable_ranger (cfun);
    8417         5988 :       if (large_huge.m_loads)
    8418              :         {
    8419          317 :           basic_block entry = ENTRY_BLOCK_PTR_FOR_FN (cfun);
    8420          317 :           entry->aux = NULL;
    8421          634 :           bitint_dom_walker (large_huge.m_names,
    8422          317 :                              large_huge.m_loads).walk (entry);
    8423          317 :           bitmap_and_compl_into (large_huge.m_names, large_huge.m_loads);
    8424          317 :           clear_aux_for_blocks ();
    8425          317 :           BITMAP_FREE (large_huge.m_loads);
    8426              :         }
    8427         5988 :       large_huge.m_limb_type = build_nonstandard_integer_type (limb_prec, 1);
    8428         5988 :       large_huge.m_limb_size
    8429         5988 :         = tree_to_uhwi (TYPE_SIZE_UNIT (large_huge.m_limb_type));
    8430              :     }
    8431         7486 :   if (large_huge.m_names)
    8432              :     {
    8433         5788 :       large_huge.m_map
    8434        11576 :         = init_var_map (num_ssa_names, NULL, large_huge.m_names);
    8435         5788 :       coalesce_ssa_name (large_huge.m_map);
    8436         5788 :       partition_view_normal (large_huge.m_map);
    8437         5788 :       if (dump_file && (dump_flags & TDF_DETAILS))
    8438              :         {
    8439            0 :           fprintf (dump_file, "After Coalescing:\n");
    8440            0 :           dump_var_map (dump_file, large_huge.m_map);
    8441              :         }
    8442         5788 :       large_huge.m_vars
    8443         5788 :         = XCNEWVEC (tree, num_var_partitions (large_huge.m_map));
    8444         5788 :       bitmap_iterator bi;
    8445         5788 :       if (has_large_huge_parm_result)
    8446        20084 :         EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
    8447              :           {
    8448        15815 :             tree s = ssa_name (i);
    8449        15815 :             if (SSA_NAME_VAR (s)
    8450         6436 :                 && ((TREE_CODE (SSA_NAME_VAR (s)) == PARM_DECL
    8451         5515 :                      && SSA_NAME_IS_DEFAULT_DEF (s))
    8452          958 :                     || TREE_CODE (SSA_NAME_VAR (s)) == RESULT_DECL))
    8453              :               {
    8454         5478 :                 int p = var_to_partition (large_huge.m_map, s);
    8455         5478 :                 if (large_huge.m_vars[p] == NULL_TREE)
    8456              :                   {
    8457         5478 :                     large_huge.m_vars[p] = SSA_NAME_VAR (s);
    8458         5478 :                     mark_addressable (SSA_NAME_VAR (s));
    8459              :                   }
    8460              :               }
    8461              :           }
    8462         5788 :       tree atype = NULL_TREE;
    8463         5788 :       if (dump_file && (dump_flags & TDF_DETAILS))
    8464            0 :         fprintf (dump_file, "Mapping SSA_NAMEs to decls:\n");
    8465        33429 :       EXECUTE_IF_SET_IN_BITMAP (large_huge.m_names, 0, i, bi)
    8466              :         {
    8467        27641 :           tree s = ssa_name (i);
    8468        27641 :           int p = var_to_partition (large_huge.m_map, s);
    8469        27641 :           if (large_huge.m_vars[p] == NULL_TREE)
    8470              :             {
    8471        19068 :               if (atype == NULL_TREE
    8472        33265 :                   || !tree_int_cst_equal (TYPE_SIZE (atype),
    8473        14197 :                                           TYPE_SIZE (TREE_TYPE (s))))
    8474              :                 {
    8475         7862 :                   unsigned HOST_WIDE_INT nelts
    8476         7862 :                     = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (s))) / limb_prec;
    8477         7862 :                   atype = build_array_type_nelts (large_huge.m_limb_type,
    8478         7862 :                                                   nelts);
    8479              :                 }
    8480        19068 :               large_huge.m_vars[p] = create_tmp_var (atype, "bitint");
    8481        19068 :               mark_addressable (large_huge.m_vars[p]);
    8482              :             }
    8483        27641 :           if (dump_file && (dump_flags & TDF_DETAILS))
    8484              :             {
    8485            0 :               print_generic_expr (dump_file, s, TDF_SLIM);
    8486            0 :               fprintf (dump_file, " -> ");
    8487            0 :               print_generic_expr (dump_file, large_huge.m_vars[p], TDF_SLIM);
    8488            0 :               fprintf (dump_file, "\n");
    8489              :             }
    8490              :         }
    8491              :     }
    8492              : 
    8493        47615 :   FOR_EACH_BB_REVERSE_FN (bb, cfun)
    8494              :     {
    8495        40129 :       gimple_stmt_iterator prev;
    8496       202661 :       for (gimple_stmt_iterator gsi = gsi_last_bb (bb); !gsi_end_p (gsi);
    8497       122403 :            gsi = prev)
    8498              :         {
    8499       122403 :           prev = gsi;
    8500       122403 :           gsi_prev (&prev);
    8501       122403 :           ssa_op_iter iter;
    8502       122403 :           gimple *stmt = gsi_stmt (gsi);
    8503       122403 :           if (is_gimple_debug (stmt))
    8504        79071 :             continue;
    8505       117242 :           bitint_prec_kind kind = bitint_prec_small;
    8506       117242 :           tree t;
    8507       352448 :           FOR_EACH_SSA_TREE_OPERAND (t, stmt, iter, SSA_OP_ALL_OPERANDS)
    8508       235206 :             if (BITINT_TYPE_P (TREE_TYPE (t)))
    8509              :               {
    8510        80397 :                 bitint_prec_kind this_kind
    8511        80397 :                   = bitint_precision_kind (TREE_TYPE (t));
    8512       235425 :                 kind = MAX (kind, this_kind);
    8513              :               }
    8514       117242 :           if (is_gimple_assign (stmt) && gimple_store_p (stmt))
    8515              :             {
    8516        17198 :               t = gimple_assign_rhs1 (stmt);
    8517        17198 :               if (BITINT_TYPE_P (TREE_TYPE (t)))
    8518              :                 {
    8519        14853 :                   bitint_prec_kind this_kind
    8520        14853 :                     = bitint_precision_kind (TREE_TYPE (t));
    8521        14853 :                   kind = MAX (kind, this_kind);
    8522              :                 }
    8523              :             }
    8524       117242 :           if (is_gimple_assign (stmt)
    8525       117242 :               && gimple_assign_rhs_code (stmt) == FLOAT_EXPR)
    8526              :             {
    8527          189 :               t = gimple_assign_rhs1 (stmt);
    8528          218 :               if (BITINT_TYPE_P (TREE_TYPE (t))
    8529          191 :                   && TREE_CODE (t) == INTEGER_CST)
    8530              :                 {
    8531            1 :                   bitint_prec_kind this_kind
    8532            1 :                     = bitint_precision_kind (TREE_TYPE (t));
    8533            1 :                   kind = MAX (kind, this_kind);
    8534              :                 }
    8535              :             }
    8536       117242 :           if (is_gimple_call (stmt))
    8537              :             {
    8538        26413 :               t = gimple_call_lhs (stmt);
    8539        26413 :               if (t && TREE_CODE (TREE_TYPE (t)) == COMPLEX_TYPE)
    8540              :                 {
    8541         5335 :                   bitint_prec_kind this_kind = arith_overflow_arg_kind (stmt);
    8542         5335 :                   kind = MAX (kind, this_kind);
    8543         5335 :                   if (BITINT_TYPE_P (TREE_TYPE (TREE_TYPE (t))))
    8544              :                     {
    8545         5210 :                       this_kind
    8546         5210 :                         = bitint_precision_kind (TREE_TYPE (TREE_TYPE (t)));
    8547         5210 :                       kind = MAX (kind, this_kind);
    8548              :                     }
    8549              :                 }
    8550              :             }
    8551       116920 :           if (kind == bitint_prec_small)
    8552        47029 :             continue;
    8553        70213 :           switch (gimple_code (stmt))
    8554              :             {
    8555        11310 :             case GIMPLE_CALL:
    8556              :               /* For now.  We'll need to handle some internal functions and
    8557              :                  perhaps some builtins.  */
    8558        11310 :               if (kind == bitint_prec_middle)
    8559         2298 :                 continue;
    8560              :               break;
    8561            4 :             case GIMPLE_ASM:
    8562            4 :               if (kind == bitint_prec_middle)
    8563            1 :                 continue;
    8564              :               break;
    8565         1129 :             case GIMPLE_RETURN:
    8566         1129 :               continue;
    8567        49013 :             case GIMPLE_ASSIGN:
    8568        49013 :               if (gimple_clobber_p (stmt))
    8569         3521 :                 continue;
    8570        45492 :               if (kind >= bitint_prec_large)
    8571              :                 break;
    8572         8848 :               if (gimple_assign_single_p (stmt))
    8573              :                 /* No need to lower copies, loads or stores.  */
    8574         5826 :                 continue;
    8575         3022 :               if (gimple_assign_cast_p (stmt))
    8576              :                 {
    8577         2443 :                   tree lhs = gimple_assign_lhs (stmt);
    8578         2443 :                   tree rhs = gimple_assign_rhs1 (stmt);
    8579         4886 :                   if (INTEGRAL_TYPE_P (TREE_TYPE (lhs))
    8580         2443 :                       && INTEGRAL_TYPE_P (TREE_TYPE (rhs))
    8581         4880 :                       && (TYPE_PRECISION (TREE_TYPE (lhs))
    8582         2437 :                           == TYPE_PRECISION (TREE_TYPE (rhs))))
    8583              :                     /* No need to lower casts to same precision.  */
    8584           35 :                     continue;
    8585              :                 }
    8586              :               break;
    8587              :             default:
    8588              :               break;
    8589         1129 :             }
    8590              : 
    8591        11744 :           if (kind == bitint_prec_middle)
    8592              :             {
    8593         5131 :               tree type = NULL_TREE;
    8594              :               /* Middle _BitInt(N) is rewritten to casts to INTEGER_TYPEs
    8595              :                  with the same precision and back.  */
    8596         5131 :               unsigned int nops = gimple_num_ops (stmt);
    8597        17227 :               for (unsigned int i = is_gimple_assign (stmt) ? 1 : 0;
    8598        17227 :                    i < nops; ++i)
    8599        12096 :                 if (tree op = gimple_op (stmt, i))
    8600              :                   {
    8601         7834 :                     tree nop = maybe_cast_middle_bitint (&gsi, op, type);
    8602         7834 :                     if (nop != op)
    8603         6861 :                       gimple_set_op (stmt, i, nop);
    8604          973 :                     else if (COMPARISON_CLASS_P (op))
    8605              :                       {
    8606            0 :                         TREE_OPERAND (op, 0)
    8607            0 :                           = maybe_cast_middle_bitint (&gsi,
    8608            0 :                                                       TREE_OPERAND (op, 0),
    8609              :                                                       type);
    8610            0 :                         TREE_OPERAND (op, 1)
    8611            0 :                           = maybe_cast_middle_bitint (&gsi,
    8612            0 :                                                       TREE_OPERAND (op, 1),
    8613              :                                                       type);
    8614              :                       }
    8615          973 :                     else if (TREE_CODE (op) == CASE_LABEL_EXPR)
    8616              :                       {
    8617          100 :                         CASE_LOW (op)
    8618          100 :                           = maybe_cast_middle_bitint (&gsi, CASE_LOW (op),
    8619              :                                                       type);
    8620          200 :                         CASE_HIGH (op)
    8621          200 :                           = maybe_cast_middle_bitint (&gsi, CASE_HIGH (op),
    8622              :                                                       type);
    8623              :                       }
    8624              :                   }
    8625         5131 :               if (tree lhs = gimple_get_lhs (stmt))
    8626         4559 :                 if (BITINT_TYPE_P (TREE_TYPE (lhs))
    8627         2987 :                     && (bitint_precision_kind (TREE_TYPE (lhs))
    8628              :                         == bitint_prec_middle))
    8629              :                   {
    8630         1403 :                     int prec = TYPE_PRECISION (TREE_TYPE (lhs));
    8631         1403 :                     int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
    8632         1403 :                     type = build_nonstandard_integer_type (prec, uns);
    8633         1403 :                     tree lhs2 = make_ssa_name (type);
    8634         1403 :                     gimple_set_lhs (stmt, lhs2);
    8635         1403 :                     gimple *g = gimple_build_assign (lhs, NOP_EXPR, lhs2);
    8636         1403 :                     if (stmt_ends_bb_p (stmt))
    8637              :                       {
    8638            4 :                         edge e = find_fallthru_edge (gsi_bb (gsi)->succs);
    8639            4 :                         gsi_insert_on_edge (e, g);
    8640            4 :                         edge_insertions = true;
    8641              :                       }
    8642              :                     else
    8643         1399 :                       gsi_insert_after (&gsi, g, GSI_SAME_STMT);
    8644              :                   }
    8645         5131 :               update_stmt (stmt);
    8646         5131 :               continue;
    8647         5131 :             }
    8648              : 
    8649        52272 :           if (tree lhs = gimple_get_lhs (stmt))
    8650        45525 :             if (TREE_CODE (lhs) == SSA_NAME)
    8651              :               {
    8652        36439 :                 tree type = TREE_TYPE (lhs);
    8653        36439 :                 if (TREE_CODE (type) == COMPLEX_TYPE)
    8654         4207 :                   type = TREE_TYPE (type);
    8655         5444 :                 if (BITINT_TYPE_P (type)
    8656        31007 :                     && bitint_precision_kind (type) >= bitint_prec_large
    8657        67244 :                     && (large_huge.m_names == NULL
    8658        30600 :                         || !bitmap_bit_p (large_huge.m_names,
    8659        30600 :                                           SSA_NAME_VERSION (lhs))))
    8660         8940 :                   continue;
    8661              :               }
    8662              : 
    8663        43332 :           large_huge.lower_stmt (stmt);
    8664              :         }
    8665              : 
    8666        40129 :       tree atype = NULL_TREE;
    8667        48943 :       for (gphi_iterator gsi = gsi_start_phis (bb); !gsi_end_p (gsi);
    8668         8814 :            gsi_next (&gsi))
    8669              :         {
    8670         8814 :           gphi *phi = gsi.phi ();
    8671         8814 :           tree lhs = gimple_phi_result (phi);
    8672        17316 :           if (!BITINT_TYPE_P (TREE_TYPE (lhs))
    8673         8814 :               || bitint_precision_kind (TREE_TYPE (lhs)) < bitint_prec_large)
    8674         8550 :             continue;
    8675          264 :           int p1 = var_to_partition (large_huge.m_map, lhs);
    8676          264 :           gcc_assert (large_huge.m_vars[p1] != NULL_TREE);
    8677              :           tree v1 = large_huge.m_vars[p1];
    8678          899 :           for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
    8679              :             {
    8680          635 :               tree arg = gimple_phi_arg_def (phi, i);
    8681          635 :               edge e = gimple_phi_arg_edge (phi, i);
    8682          635 :               gimple *g;
    8683          635 :               switch (TREE_CODE (arg))
    8684              :                 {
    8685          130 :                 case INTEGER_CST:
    8686          130 :                   if (integer_zerop (arg) && VAR_P (v1))
    8687              :                     {
    8688           60 :                       tree zero = build_zero_cst (TREE_TYPE (v1));
    8689           60 :                       g = gimple_build_assign (v1, zero);
    8690           60 :                       gsi_insert_on_edge (e, g);
    8691           60 :                       edge_insertions = true;
    8692          232 :                       break;
    8693              :                     }
    8694           70 :                   int ext;
    8695           70 :                   unsigned int min_prec, prec, rem;
    8696           70 :                   tree c;
    8697           70 :                   prec = TYPE_PRECISION (TREE_TYPE (arg));
    8698           70 :                   rem = prec % (2 * limb_prec);
    8699           70 :                   min_prec = bitint_min_cst_precision (arg, ext);
    8700           70 :                   if (min_prec > prec - rem - 2 * limb_prec
    8701           28 :                       && min_prec > (unsigned) limb_prec)
    8702              :                     /* Constant which has enough significant bits that it
    8703              :                        isn't worth trying to save .rodata space by extending
    8704              :                        from smaller number.  */
    8705              :                     min_prec = prec;
    8706              :                   else
    8707              :                     {
    8708           61 :                       min_prec = CEIL (min_prec, limb_prec) * limb_prec;
    8709           61 :                       if (min_prec > (unsigned) limb_prec
    8710            3 :                           && abi_limb_prec > limb_prec)
    8711              :                         {
    8712              :                           /* For targets with ABI limb precision higher than
    8713              :                              limb precision round to ABI limb precision,
    8714              :                              otherwise c can contain padding bits.  */
    8715            0 :                           min_prec
    8716            0 :                             = CEIL (min_prec, abi_limb_prec) * abi_limb_prec;
    8717            0 :                           if (min_prec > prec - rem - 2 * limb_prec)
    8718            9 :                             min_prec = prec;
    8719              :                         }
    8720              :                     }
    8721           70 :                   if (min_prec == 0)
    8722              :                     c = NULL_TREE;
    8723           67 :                   else if (min_prec == prec)
    8724            9 :                     c = tree_output_constant_def (arg);
    8725           58 :                   else if (min_prec == (unsigned) limb_prec)
    8726           55 :                     c = fold_convert (large_huge.m_limb_type, arg);
    8727              :                   else
    8728              :                     {
    8729            3 :                       tree ctype = build_bitint_type (min_prec, 1);
    8730            3 :                       c = tree_output_constant_def (fold_convert (ctype, arg));
    8731              :                     }
    8732           67 :                   if (c)
    8733              :                     {
    8734           67 :                       if (VAR_P (v1) && min_prec == prec)
    8735              :                         {
    8736            8 :                           tree v2 = build1 (VIEW_CONVERT_EXPR,
    8737            8 :                                             TREE_TYPE (v1), c);
    8738            8 :                           g = gimple_build_assign (v1, v2);
    8739            8 :                           gsi_insert_on_edge (e, g);
    8740            8 :                           edge_insertions = true;
    8741            8 :                           break;
    8742              :                         }
    8743           59 :                       if (TREE_CODE (TREE_TYPE (c)) == INTEGER_TYPE)
    8744              :                         {
    8745           55 :                           if (bitint_big_endian)
    8746              :                             {
    8747            0 :                               tree ptype = build_pointer_type (TREE_TYPE (v1));
    8748            0 :                               tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
    8749            0 :                               tree sz2 = TYPE_SIZE_UNIT (TREE_TYPE (c));
    8750            0 :                               tree off = build_int_cst (ptype,
    8751            0 :                                                         tree_to_uhwi (sz1)
    8752            0 :                                                         - tree_to_uhwi (sz2));
    8753            0 :                               tree vd = build2 (MEM_REF, TREE_TYPE (c),
    8754              :                                                 build_fold_addr_expr (v1),
    8755              :                                                 off);
    8756            0 :                               g = gimple_build_assign (vd, c);
    8757              :                             }
    8758              :                           else
    8759           55 :                             g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
    8760           55 :                                                              TREE_TYPE (c),
    8761              :                                                              v1), c);
    8762              :                         }
    8763              :                       else
    8764              :                         {
    8765            4 :                           unsigned HOST_WIDE_INT nelts
    8766            4 :                             = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (c)))
    8767            4 :                               / limb_prec;
    8768            4 :                           tree vtype
    8769            8 :                             = build_array_type_nelts (large_huge.m_limb_type,
    8770            4 :                                                       nelts);
    8771            4 :                           tree vd;
    8772            4 :                           if (bitint_big_endian)
    8773              :                             {
    8774            0 :                               tree ptype = build_pointer_type (TREE_TYPE (v1));
    8775            0 :                               tree sz1 = TYPE_SIZE_UNIT (TREE_TYPE (v1));
    8776            0 :                               tree sz2 = TYPE_SIZE_UNIT (vtype);
    8777            0 :                               tree off = build_int_cst (ptype,
    8778            0 :                                                         tree_to_uhwi (sz1)
    8779            0 :                                                         - tree_to_uhwi (sz2));
    8780            0 :                               vd = build2 (MEM_REF, vtype,
    8781              :                                            build_fold_addr_expr (v1), off);
    8782              :                             }
    8783              :                           else
    8784            4 :                             vd = build1 (VIEW_CONVERT_EXPR, vtype, v1);
    8785            4 :                           g = gimple_build_assign (vd,
    8786              :                                                    build1 (VIEW_CONVERT_EXPR,
    8787              :                                                            vtype, c));
    8788              :                         }
    8789           59 :                       gsi_insert_on_edge (e, g);
    8790           59 :                       if (min_prec == prec)
    8791              :                         {
    8792              :                           edge_insertions = true;
    8793              :                           break;
    8794              :                         }
    8795              :                     }
    8796           61 :                   if (ext == 0)
    8797              :                     {
    8798           55 :                       unsigned HOST_WIDE_INT nelts
    8799           55 :                         = (tree_to_uhwi (TYPE_SIZE (TREE_TYPE (v1)))
    8800           55 :                            - min_prec) / limb_prec;
    8801           55 :                       tree vtype
    8802          110 :                         = build_array_type_nelts (large_huge.m_limb_type,
    8803           55 :                                                   nelts);
    8804           55 :                       tree ptype = build_pointer_type (TREE_TYPE (v1));
    8805           55 :                       tree off;
    8806           55 :                       if (c && !bitint_big_endian)
    8807           54 :                         off = fold_convert (ptype,
    8808              :                                             TYPE_SIZE_UNIT (TREE_TYPE (c)));
    8809              :                       else
    8810            1 :                         off = build_zero_cst (ptype);
    8811           55 :                       tree vd = build2 (MEM_REF, vtype,
    8812              :                                         build_fold_addr_expr (v1), off);
    8813           55 :                       g = gimple_build_assign (vd, build_zero_cst (vtype));
    8814              :                     }
    8815              :                   else
    8816              :                     {
    8817            6 :                       tree vd = v1;
    8818            6 :                       if (c && !bitint_big_endian)
    8819              :                         {
    8820            4 :                           tree ptype = build_pointer_type (TREE_TYPE (v1));
    8821            4 :                           tree off
    8822            4 :                             = fold_convert (ptype,
    8823              :                                             TYPE_SIZE_UNIT (TREE_TYPE (c)));
    8824            4 :                           vd = build2 (MEM_REF, large_huge.m_limb_type,
    8825              :                                        build_fold_addr_expr (v1), off);
    8826              :                         }
    8827            6 :                       vd = build_fold_addr_expr (vd);
    8828            6 :                       unsigned HOST_WIDE_INT nbytes
    8829            6 :                         = tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (v1)));
    8830            6 :                       if (c)
    8831            4 :                         nbytes
    8832            4 :                           -= tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (c)));
    8833            6 :                       tree fn = builtin_decl_implicit (BUILT_IN_MEMSET);
    8834            6 :                       g = gimple_build_call (fn, 3, vd,
    8835              :                                              integer_minus_one_node,
    8836              :                                              build_int_cst (sizetype,
    8837            6 :                                                             nbytes));
    8838              :                     }
    8839           61 :                   gsi_insert_on_edge (e, g);
    8840           61 :                   edge_insertions = true;
    8841           61 :                   break;
    8842            0 :                 default:
    8843            0 :                   gcc_unreachable ();
    8844          505 :                 case SSA_NAME:
    8845          505 :                   if (gimple_code (SSA_NAME_DEF_STMT (arg)) == GIMPLE_NOP)
    8846              :                     {
    8847           49 :                       if (large_huge.m_names == NULL
    8848           98 :                           || !bitmap_bit_p (large_huge.m_names,
    8849           49 :                                             SSA_NAME_VERSION (arg)))
    8850          463 :                         continue;
    8851              :                     }
    8852          505 :                   int p2 = var_to_partition (large_huge.m_map, arg);
    8853          505 :                   if (p1 == p2)
    8854          463 :                     continue;
    8855           42 :                   gcc_assert (large_huge.m_vars[p2] != NULL_TREE);
    8856           42 :                   tree v2 = large_huge.m_vars[p2];
    8857           42 :                   if (VAR_P (v1) && VAR_P (v2))
    8858           18 :                     g = gimple_build_assign (v1, v2);
    8859           24 :                   else if (VAR_P (v1))
    8860           16 :                     g = gimple_build_assign (v1, build1 (VIEW_CONVERT_EXPR,
    8861           16 :                                                          TREE_TYPE (v1), v2));
    8862            8 :                   else if (VAR_P (v2))
    8863            0 :                     g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
    8864            0 :                                                      TREE_TYPE (v2), v1), v2);
    8865              :                   else
    8866              :                     {
    8867            8 :                       if (atype == NULL_TREE
    8868            8 :                           || !tree_int_cst_equal (TYPE_SIZE (atype),
    8869            0 :                                                   TYPE_SIZE (TREE_TYPE (lhs))))
    8870              :                         {
    8871            8 :                           unsigned HOST_WIDE_INT nelts
    8872            8 :                             = tree_to_uhwi (TYPE_SIZE (TREE_TYPE (lhs)))
    8873            8 :                               / limb_prec;
    8874            8 :                           atype
    8875            8 :                             = build_array_type_nelts (large_huge.m_limb_type,
    8876            8 :                                                       nelts);
    8877              :                         }
    8878            8 :                       g = gimple_build_assign (build1 (VIEW_CONVERT_EXPR,
    8879              :                                                        atype, v1),
    8880              :                                                build1 (VIEW_CONVERT_EXPR,
    8881              :                                                        atype, v2));
    8882              :                     }
    8883           42 :                   gsi_insert_on_edge (e, g);
    8884           42 :                   edge_insertions = true;
    8885           42 :                   break;
    8886              :                 }
    8887              :             }
    8888              :         }
    8889              :     }
    8890              : 
    8891         7486 :   if (large_huge.m_names || has_large_huge)
    8892              :     {
    8893         5988 :       gimple *nop = NULL;
    8894       393588 :       for (i = 0; i < num_ssa_names; ++i)
    8895              :         {
    8896       387600 :           tree s = ssa_name (i);
    8897       387600 :           if (s == NULL_TREE)
    8898        15924 :             continue;
    8899       371676 :           tree type = TREE_TYPE (s);
    8900       371676 :           if (TREE_CODE (type) == COMPLEX_TYPE)
    8901        17757 :             type = TREE_TYPE (type);
    8902       331200 :           if (BITINT_TYPE_P (type)
    8903       371703 :               && bitint_precision_kind (type) >= bitint_prec_large)
    8904              :             {
    8905        41326 :               if (large_huge.m_preserved
    8906        45824 :                   && bitmap_bit_p (large_huge.m_preserved,
    8907         6841 :                                    SSA_NAME_VERSION (s)))
    8908         2343 :                 continue;
    8909        36640 :               gimple *g = SSA_NAME_DEF_STMT (s);
    8910        36640 :               if (gimple_code (g) == GIMPLE_NOP)
    8911              :                 {
    8912         9924 :                   if (SSA_NAME_VAR (s))
    8913         5547 :                     set_ssa_default_def (cfun, SSA_NAME_VAR (s), NULL_TREE);
    8914         9924 :                   release_ssa_name (s);
    8915         9924 :                   continue;
    8916              :                 }
    8917        26716 :               if (gimple_bb (g) == NULL)
    8918              :                 {
    8919            0 :                   release_ssa_name (s);
    8920            0 :                   continue;
    8921              :                 }
    8922        26716 :               if (gimple_code (g) != GIMPLE_ASM)
    8923              :                 {
    8924        26715 :                   gimple_stmt_iterator gsi = gsi_for_stmt (g);
    8925        26715 :                   bool save_vta = flag_var_tracking_assignments;
    8926        26715 :                   flag_var_tracking_assignments = false;
    8927        26715 :                   gsi_remove (&gsi, true);
    8928        26715 :                   flag_var_tracking_assignments = save_vta;
    8929              :                 }
    8930        26716 :               if (nop == NULL)
    8931         4976 :                 nop = gimple_build_nop ();
    8932        26716 :               SSA_NAME_DEF_STMT (s) = nop;
    8933        26716 :               release_ssa_name (s);
    8934              :             }
    8935              :         }
    8936         5988 :       if (optimize)
    8937         3192 :         disable_ranger (cfun);
    8938              :     }
    8939              : 
    8940         7486 :   if (edge_insertions)
    8941           80 :     gsi_commit_edge_inserts ();
    8942              : 
    8943              :   /* Fix up arguments of ECF_RETURNS_TWICE calls.  Those were temporarily
    8944              :      inserted before the call, but that is invalid IL, so move them to the
    8945              :      right place and add corresponding PHIs.  */
    8946         7486 :   if (!large_huge.m_returns_twice_calls.is_empty ())
    8947              :     {
    8948            9 :       auto_vec<gimple *, 16> arg_stmts;
    8949           29 :       while (!large_huge.m_returns_twice_calls.is_empty ())
    8950              :         {
    8951           11 :           gimple *stmt = large_huge.m_returns_twice_calls.pop ();
    8952           11 :           gimple_stmt_iterator gsi = gsi_after_labels (gimple_bb (stmt));
    8953           36 :           while (gsi_stmt (gsi) != stmt)
    8954              :             {
    8955           25 :               if (is_gimple_debug (gsi_stmt (gsi)))
    8956            2 :                 gsi_next (&gsi);
    8957              :               else
    8958              :                 {
    8959           23 :                   arg_stmts.safe_push (gsi_stmt (gsi));
    8960           23 :                   gsi_remove (&gsi, false);
    8961              :                 }
    8962              :             }
    8963           11 :           gimple *g;
    8964           11 :           basic_block bb = NULL;
    8965           11 :           edge e = NULL, ead = NULL;
    8966           34 :           FOR_EACH_VEC_ELT (arg_stmts, i, g)
    8967              :             {
    8968           23 :               gsi_safe_insert_before (&gsi, g);
    8969           23 :               if (i == 0)
    8970              :                 {
    8971           11 :                   bb = gimple_bb (stmt);
    8972           11 :                   gcc_checking_assert (EDGE_COUNT (bb->preds) == 2);
    8973           11 :                   e = EDGE_PRED (bb, 0);
    8974           11 :                   ead = EDGE_PRED (bb, 1);
    8975           11 :                   if ((ead->flags & EDGE_ABNORMAL) == 0)
    8976            0 :                     std::swap (e, ead);
    8977           11 :                   gcc_checking_assert ((e->flags & EDGE_ABNORMAL) == 0
    8978              :                                        && (ead->flags & EDGE_ABNORMAL));
    8979              :                 }
    8980           23 :               tree lhs = gimple_assign_lhs (g);
    8981           23 :               tree arg = lhs;
    8982           23 :               gphi *phi = create_phi_node (copy_ssa_name (arg), bb);
    8983           23 :               add_phi_arg (phi, arg, e, UNKNOWN_LOCATION);
    8984           23 :               tree var = create_tmp_reg (TREE_TYPE (arg));
    8985           23 :               suppress_warning (var, OPT_Wuninitialized);
    8986           23 :               arg = get_or_create_ssa_default_def (cfun, var);
    8987           23 :               SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
    8988           23 :               add_phi_arg (phi, arg, ead, UNKNOWN_LOCATION);
    8989           23 :               arg = gimple_phi_result (phi);
    8990           23 :               SSA_NAME_OCCURS_IN_ABNORMAL_PHI (arg) = 1;
    8991           23 :               imm_use_iterator iter;
    8992           23 :               gimple *use_stmt;
    8993           69 :               FOR_EACH_IMM_USE_STMT (use_stmt, iter, lhs)
    8994              :                 {
    8995           46 :                   if (use_stmt == phi)
    8996           23 :                     continue;
    8997           23 :                   gcc_checking_assert (use_stmt == stmt);
    8998           23 :                   use_operand_p use_p;
    8999           46 :                   FOR_EACH_IMM_USE_ON_STMT (use_p, iter)
    9000           23 :                     SET_USE (use_p, arg);
    9001           23 :                 }
    9002              :             }
    9003           11 :           update_stmt (stmt);
    9004           11 :           arg_stmts.truncate (0);
    9005              :         }
    9006            9 :     }
    9007              : 
    9008         7486 :   if (loop_init_done)
    9009         4218 :     loop_optimizer_finalize ();
    9010              : 
    9011         7486 :   return ret;
    9012         7486 : }
    9013              : 
    9014              : namespace {
    9015              : 
    9016              : const pass_data pass_data_lower_bitint =
    9017              : {
    9018              :   GIMPLE_PASS, /* type */
    9019              :   "bitintlower", /* name */
    9020              :   OPTGROUP_NONE, /* optinfo_flags */
    9021              :   TV_NONE, /* tv_id */
    9022              :   PROP_ssa, /* properties_required */
    9023              :   PROP_gimple_lbitint, /* properties_provided */
    9024              :   0, /* properties_destroyed */
    9025              :   0, /* todo_flags_start */
    9026              :   0, /* todo_flags_finish */
    9027              : };
    9028              : 
    9029              : class pass_lower_bitint : public gimple_opt_pass
    9030              : {
    9031              : public:
    9032       594360 :   pass_lower_bitint (gcc::context *ctxt)
    9033      1188720 :     : gimple_opt_pass (pass_data_lower_bitint, ctxt)
    9034              :   {}
    9035              : 
    9036              :   /* opt_pass methods: */
    9037       297180 :   opt_pass * clone () final override { return new pass_lower_bitint (m_ctxt); }
    9038      1065442 :   unsigned int execute (function *) final override
    9039              :   {
    9040      1065442 :     return gimple_lower_bitint ();
    9041              :   }
    9042              : 
    9043              : }; // class pass_lower_bitint
    9044              : 
    9045              : } // anon namespace
    9046              : 
    9047              : gimple_opt_pass *
    9048       297180 : make_pass_lower_bitint (gcc::context *ctxt)
    9049              : {
    9050       297180 :   return new pass_lower_bitint (ctxt);
    9051              : }
    9052              : 
    9053              : 
    9054              : namespace {
    9055              : 
    9056              : const pass_data pass_data_lower_bitint_O0 =
    9057              : {
    9058              :   GIMPLE_PASS, /* type */
    9059              :   "bitintlower0", /* name */
    9060              :   OPTGROUP_NONE, /* optinfo_flags */
    9061              :   TV_NONE, /* tv_id */
    9062              :   PROP_cfg, /* properties_required */
    9063              :   PROP_gimple_lbitint, /* properties_provided */
    9064              :   0, /* properties_destroyed */
    9065              :   0, /* todo_flags_start */
    9066              :   0, /* todo_flags_finish */
    9067              : };
    9068              : 
    9069              : class pass_lower_bitint_O0 : public gimple_opt_pass
    9070              : {
    9071              : public:
    9072       297180 :   pass_lower_bitint_O0 (gcc::context *ctxt)
    9073       594360 :     : gimple_opt_pass (pass_data_lower_bitint_O0, ctxt)
    9074              :   {}
    9075              : 
    9076              :   /* opt_pass methods: */
    9077      1517917 :   bool gate (function *fun) final override
    9078              :     {
    9079              :       /* With errors, normal optimization passes are not run.  If we don't
    9080              :          lower bitint operations at all, rtl expansion will abort.  */
    9081      1517917 :       return !(fun->curr_properties & PROP_gimple_lbitint);
    9082              :     }
    9083              : 
    9084       452614 :   unsigned int execute (function *) final override
    9085              :   {
    9086       452614 :     return gimple_lower_bitint ();
    9087              :   }
    9088              : 
    9089              : }; // class pass_lower_bitint_O0
    9090              : 
    9091              : } // anon namespace
    9092              : 
    9093              : gimple_opt_pass *
    9094       297180 : make_pass_lower_bitint_O0 (gcc::context *ctxt)
    9095              : {
    9096       297180 :   return new pass_lower_bitint_O0 (ctxt);
    9097              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.