LCOV - code coverage report
Current view: top level - gcc - internal-fn.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 72.4 % 2836 2054
Test Date: 2026-11-14 16:39:50 Functions: 59.9 % 182 109
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Internal functions.
       2              :    Copyright (C) 2011-2026 Free Software Foundation, Inc.
       3              : 
       4              : This file is part of GCC.
       5              : 
       6              : GCC is free software; you can redistribute it and/or modify it under
       7              : the terms of the GNU General Public License as published by the Free
       8              : Software Foundation; either version 3, or (at your option) any later
       9              : version.
      10              : 
      11              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      12              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      13              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      14              : for more details.
      15              : 
      16              : You should have received a copy of the GNU General Public License
      17              : along with GCC; see the file COPYING3.  If not see
      18              : <http://www.gnu.org/licenses/>.  */
      19              : 
      20              : #include "config.h"
      21              : #define INCLUDE_MEMORY
      22              : #include "system.h"
      23              : #include "coretypes.h"
      24              : #include "backend.h"
      25              : #include "target.h"
      26              : #include "rtl.h"
      27              : #include "tree.h"
      28              : #include "tree-eh.h"
      29              : #include "gimple.h"
      30              : #include "predict.h"
      31              : #include "stringpool.h"
      32              : #include "tree-vrp.h"
      33              : #include "tree-ssanames.h"
      34              : #include "expmed.h"
      35              : #include "memmodel.h"
      36              : #include "optabs.h"
      37              : #include "emit-rtl.h"
      38              : #include "diagnostic-core.h"
      39              : #include "fold-const.h"
      40              : #include "internal-fn.h"
      41              : #include "stor-layout.h"
      42              : #include "dojump.h"
      43              : #include "expr.h"
      44              : #include "attribs.h"
      45              : #include "asan.h"
      46              : #include "ubsan.h"
      47              : #include "recog.h"
      48              : #include "builtins.h"
      49              : #include "optabs-tree.h"
      50              : #include "gimple-ssa.h"
      51              : #include "tree-phinodes.h"
      52              : #include "ssa-iterators.h"
      53              : #include "explow.h"
      54              : #include "rtl-iter.h"
      55              : #include "gimple-range.h"
      56              : #include "fold-const-call.h"
      57              : #include "tree-ssa-live.h"
      58              : #include "tree-outof-ssa.h"
      59              : #include "gcc-urlifier.h"
      60              : 
      61              : /* For lang_hooks.types.type_for_mode.  */
      62              : #include "langhooks.h"
      63              : 
      64              : /* The names of each internal function, indexed by function number.  */
      65              : const char *const internal_fn_name_array[] = {
      66              : #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) #CODE,
      67              : #include "internal-fn.def"
      68              :   "<invalid-fn>"
      69              : };
      70              : 
      71              : /* The ECF_* flags of each internal function, indexed by function number.  */
      72              : const int internal_fn_flags_array[] = {
      73              : #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) FLAGS,
      74              : #include "internal-fn.def"
      75              :   0
      76              : };
      77              : 
      78              : /* Return the internal function called NAME, or IFN_LAST if there's
      79              :    no such function.  */
      80              : 
      81              : internal_fn
      82          152 : lookup_internal_fn (const char *name)
      83              : {
      84          152 :   typedef hash_map<nofree_string_hash, internal_fn> name_to_fn_map_type;
      85          152 :   static name_to_fn_map_type *name_to_fn_map;
      86              : 
      87          152 :   if (!name_to_fn_map)
      88              :     {
      89           20 :       name_to_fn_map = new name_to_fn_map_type (IFN_LAST);
      90         5980 :       for (unsigned int i = 0; i < IFN_LAST; ++i)
      91        11920 :         name_to_fn_map->put (internal_fn_name (internal_fn (i)),
      92         5960 :                              internal_fn (i));
      93              :     }
      94          152 :   internal_fn *entry = name_to_fn_map->get (name);
      95          152 :   return entry ? *entry : IFN_LAST;
      96              : }
      97              : 
      98              : /* Geven an internal_fn IFN that is a widening function, return its
      99              :    corresponding LO and HI internal_fns.  */
     100              : 
     101              : extern void
     102        71116 : lookup_hilo_internal_fn (internal_fn ifn, internal_fn *lo, internal_fn *hi)
     103              : {
     104        71116 :   gcc_assert (widening_fn_p (ifn));
     105              : 
     106        71116 :   switch (ifn)
     107              :     {
     108            0 :     default:
     109            0 :       gcc_unreachable ();
     110              : #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
     111              : #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T)   \
     112              :     case IFN_##NAME:                                            \
     113              :       *lo = internal_fn (IFN_##NAME##_LO);                      \
     114              :       *hi = internal_fn (IFN_##NAME##_HI);                      \
     115              :       break;
     116              : #include "internal-fn.def"
     117              :     }
     118        71116 : }
     119              : 
     120              : /* Given an internal_fn IFN that is a widening function, return its
     121              :    corresponding _EVEN and _ODD internal_fns in *EVEN and *ODD.  */
     122              : 
     123              : extern void
     124        71116 : lookup_evenodd_internal_fn (internal_fn ifn, internal_fn *even,
     125              :                             internal_fn *odd)
     126              : {
     127        71116 :   gcc_assert (widening_fn_p (ifn));
     128              : 
     129        71116 :   switch (ifn)
     130              :     {
     131            0 :     default:
     132            0 :       gcc_unreachable ();
     133              : #define DEF_INTERNAL_FN(NAME, FLAGS, TYPE)
     134              : #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T)   \
     135              :     case IFN_##NAME:                                            \
     136              :       *even = internal_fn (IFN_##NAME##_EVEN);                  \
     137              :       *odd = internal_fn (IFN_##NAME##_ODD);                    \
     138              :       break;
     139              : #include "internal-fn.def"
     140              :     }
     141        71116 : }
     142              : 
     143              : 
     144              : /* Fnspec of each internal function, indexed by function number.  */
     145              : const_tree internal_fn_fnspec_array[IFN_LAST + 1];
     146              : 
     147              : void
     148       296862 : init_internal_fns ()
     149              : {
     150              : #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
     151              :   if (FNSPEC) internal_fn_fnspec_array[IFN_##CODE] = \
     152              :     build_string ((int) sizeof (FNSPEC) - 1, FNSPEC ? FNSPEC : "");
     153              : #include "internal-fn.def"
     154       296862 :   internal_fn_fnspec_array[IFN_LAST] = 0;
     155       296862 : }
     156              : 
     157              : /* Create static initializers for the information returned by
     158              :    direct_internal_fn.  */
     159              : #define not_direct { -2, -2, false, 0, 0 }
     160              : #define mask_load_direct { -1, 2, false, 4, 4 }
     161              : #define load_lanes_direct { -1, -1, false, 1, 1 }
     162              : #define mask_load_lanes_direct { -1, -1, false, 4, 4 }
     163              : #define mask_len_load_lanes_direct { -1, -1, false, 6, 6 }
     164              : #define gather_load_direct { 3, 1, false, 5, 5 }
     165              : #define mask_gather_load_direct { 3, 1, false, 7, 7 }
     166              : #define mask_len_gather_load_direct { 3, 1, false, 9, 9 }
     167              : #define strided_load_direct { -1, -1, false, 7, 7 }
     168              : #define len_load_direct { -1, -1, false, 5, 5 }
     169              : #define mask_len_load_direct { -1, 4, false, 6, 6 }
     170              : #define mask_store_direct { 3, 2, false, 4, 4 }
     171              : #define store_lanes_direct { 0, 0, false, 1, 1 }
     172              : #define mask_store_lanes_direct { 0, 0, false, 4, 4 }
     173              : #define mask_len_store_lanes_direct { 0, 0, false, 6, 6 }
     174              : #define vec_cond_mask_direct { 1, 0, false, 3, 3 }
     175              : #define vec_cond_mask_len_direct { 1, 1, false, 5, 5 }
     176              : #define vec_cond_direct { 2, 0, false, 5, 5 }
     177              : #define scatter_store_direct { 3, 1, false, 5, 5 }
     178              : #define mask_scatter_store_direct { 3, 1, false, 6, 6 }
     179              : #define mask_len_scatter_store_direct { 3, 1, false, 8, 8 }
     180              : #define strided_store_direct { 1, 1, false, 6, 6 }
     181              : #define len_store_direct { 3, 3, false, 5, 5 }
     182              : #define mask_len_store_direct { 4, 5, false, 6, 6 }
     183              : #define vec_set_direct { 3, 3, false, 3, 3 }
     184              : #define vec_extract_direct { 0, -1, false, 2, 2 }
     185              : #define unary_direct { 0, 0, true, 1, 1 }
     186              : #define unary_opt_direct { 0, 0, true, 1, 2 }
     187              : #define unary_convert_direct { -1, 0, true, 1, 1 }
     188              : #define binary_direct { 0, 0, true, 2, 2 }
     189              : #define ternary_direct { 0, 0, true, 3, 3 }
     190              : #define cond_unary_direct { 1, 1, true, 3, 3 }
     191              : #define cond_binary_direct { 1, 1, true, 4, 4 }
     192              : #define cond_ternary_direct { 1, 1, true, 5, 5 }
     193              : #define cond_len_unary_direct { 1, 1, true, 5, 5 }
     194              : #define cond_len_binary_direct { 1, 1, true, 6, 6 }
     195              : #define cond_len_ternary_direct { 1, 1, true, 7, 7 }
     196              : #define while_direct { 0, 2, false, 3, 3 }
     197              : #define fold_extract_direct { 2, 2, false, 3, 3 }
     198              : #define fold_len_extract_direct { 2, 2, false, 5, 5 }
     199              : #define fold_left_direct { 1, 1, false, 2, 2 }
     200              : #define mask_fold_left_direct { 1, 1, false, 3, 3 }
     201              : #define mask_len_fold_left_direct { 1, 1, false, 5, 5 }
     202              : #define check_ptrs_direct { 0, 0, false, 4, 4 }
     203              : #define crc_direct { 1, -1, true, 3, 3 }
     204              : #define reduc_sbool_direct { 0, 0, true, 1, 1 }
     205              : #define select_vl_direct { 2, 0, false, 3, 3 }
     206              : 
     207              : const direct_internal_fn_info direct_internal_fn_array[IFN_LAST + 1] = {
     208              : #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) not_direct,
     209              : #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) TYPE##_direct,
     210              : #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
     211              :                                      UNSIGNED_OPTAB, TYPE) TYPE##_direct,
     212              : #include "internal-fn.def"
     213              :   not_direct
     214              : };
     215              : 
     216              : /* Like create_output_operand, but for callers that will use
     217              :    assign_call_lhs afterwards.  */
     218              : 
     219              : static void
     220        88204 : create_call_lhs_operand (expand_operand *op, rtx lhs_rtx, machine_mode mode)
     221              : {
     222              :   /* Do not assign directly to a promoted subreg, since there is no
     223              :      guarantee that the instruction will leave the upper bits of the
     224              :      register in the state required by SUBREG_PROMOTED_SIGN.  */
     225        88204 :   rtx dest = lhs_rtx;
     226        88204 :   if (dest && GET_CODE (dest) == SUBREG && SUBREG_PROMOTED_VAR_P (dest))
     227        88204 :     dest = NULL_RTX;
     228        88204 :   create_output_operand (op, dest, mode);
     229        88204 : }
     230              : 
     231              : /* Move the result of an expanded instruction into the lhs of a gimple call.
     232              :    LHS is the lhs of the call, LHS_RTX is its expanded form, and OP is the
     233              :    result of the expanded instruction.  OP should have been set up by
     234              :    create_call_lhs_operand.  */
     235              : 
     236              : static void
     237        88204 : assign_call_lhs (tree lhs, rtx lhs_rtx, expand_operand *op)
     238              : {
     239        88204 :   if (rtx_equal_p (lhs_rtx, op->value))
     240              :     return;
     241              : 
     242              :   /* If the return value has an integral type, convert the instruction
     243              :      result to that type.  This is useful for things that return an
     244              :      int regardless of the size of the input.  If the instruction result
     245              :      is smaller than required, assume that it is signed.
     246              : 
     247              :      If the return value has a nonintegral type, its mode must match
     248              :      the instruction result.  */
     249         3503 :   if (GET_CODE (lhs_rtx) == SUBREG && SUBREG_PROMOTED_VAR_P (lhs_rtx))
     250              :     {
     251              :       /* If this is a scalar in a register that is stored in a wider
     252              :          mode than the declared mode, compute the result into its
     253              :          declared mode and then convert to the wider mode.  */
     254            0 :       gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
     255            0 :       rtx tmp = convert_to_mode (GET_MODE (lhs_rtx), op->value, 0);
     256            0 :       convert_move (SUBREG_REG (lhs_rtx), tmp,
     257            0 :                     SUBREG_PROMOTED_SIGN (lhs_rtx));
     258              :     }
     259         3503 :   else if (GET_MODE (lhs_rtx) == GET_MODE (op->value))
     260           40 :     emit_move_insn (lhs_rtx, op->value);
     261              :   else
     262              :     {
     263         3463 :       gcc_checking_assert (INTEGRAL_TYPE_P (TREE_TYPE (lhs)));
     264         3463 :       convert_move (lhs_rtx, op->value, 0);
     265              :     }
     266              : }
     267              : 
     268              : /* Expand STMT using instruction ICODE.  The instruction has NOUTPUTS
     269              :    output operands and NINPUTS input operands, where NOUTPUTS is either
     270              :    0 or 1.  The output operand (if any) comes first, followed by the
     271              :    NINPUTS input operands.  */
     272              : 
     273              : static void
     274        77458 : expand_fn_using_insn (gcall *stmt, insn_code icode, unsigned int noutputs,
     275              :                       unsigned int ninputs)
     276              : {
     277        77458 :   gcc_assert (icode != CODE_FOR_nothing);
     278              : 
     279        77458 :   expand_operand *ops = XALLOCAVEC (expand_operand, noutputs + ninputs);
     280        77458 :   unsigned int opno = 0;
     281        77458 :   rtx lhs_rtx = NULL_RTX;
     282        77458 :   tree lhs = gimple_call_lhs (stmt);
     283              : 
     284        77458 :   if (noutputs)
     285              :     {
     286        77458 :       gcc_assert (noutputs == 1);
     287        77458 :       if (lhs)
     288        77458 :         lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     289        77458 :       create_call_lhs_operand (&ops[opno], lhs_rtx,
     290        77458 :                                insn_data[icode].operand[opno].mode);
     291        77458 :       opno += 1;
     292              :     }
     293              :   else
     294            0 :     gcc_assert (!lhs);
     295              : 
     296       221342 :   for (unsigned int i = 0; i < ninputs; ++i)
     297              :     {
     298       143884 :       tree rhs = gimple_call_arg (stmt, i);
     299       143884 :       tree rhs_type = TREE_TYPE (rhs);
     300       143884 :       rtx rhs_rtx = expand_normal (rhs);
     301       143884 :       if (INTEGRAL_TYPE_P (rhs_type))
     302         7342 :         create_convert_operand_from (&ops[opno], rhs_rtx,
     303         7342 :                                      TYPE_MODE (rhs_type),
     304         7342 :                                      TYPE_UNSIGNED (rhs_type));
     305       136542 :       else if (TREE_CODE (rhs) == SSA_NAME
     306       111310 :                && SSA_NAME_IS_DEFAULT_DEF (rhs)
     307       141076 :                && VAR_P (SSA_NAME_VAR (rhs)))
     308          249 :         create_undefined_input_operand (&ops[opno], TYPE_MODE (rhs_type));
     309        55998 :       else if (VECTOR_BOOLEAN_TYPE_P (rhs_type)
     310         1579 :                && SCALAR_INT_MODE_P (TYPE_MODE (rhs_type))
     311       137872 :                && maybe_ne (GET_MODE_PRECISION (TYPE_MODE (rhs_type)),
     312         1579 :                             TYPE_VECTOR_SUBPARTS (rhs_type).to_constant ()))
     313              :         {
     314              :           /* Ensure that the vector bitmasks do not have excess bits.  */
     315          427 :           int nunits = TYPE_VECTOR_SUBPARTS (rhs_type).to_constant ();
     316          427 :           rtx tmp = expand_binop (TYPE_MODE (rhs_type), and_optab, rhs_rtx,
     317          427 :                                   GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
     318              :                                   NULL_RTX, true, OPTAB_WIDEN);
     319          427 :           create_input_operand (&ops[opno], tmp, TYPE_MODE (rhs_type));
     320              :         }
     321              :       else
     322       135866 :         create_input_operand (&ops[opno], rhs_rtx, TYPE_MODE (rhs_type));
     323       143884 :       opno += 1;
     324              :     }
     325              : 
     326        77458 :   gcc_assert (opno == noutputs + ninputs);
     327        77458 :   expand_insn (icode, opno, ops);
     328        77458 :   if (lhs_rtx)
     329        77458 :     assign_call_lhs (lhs, lhs_rtx, &ops[0]);
     330        77458 : }
     331              : 
     332              : /* ARRAY_TYPE is an array of vector modes.  Return the associated insn
     333              :    for load-lanes-style optab OPTAB, or CODE_FOR_nothing if none.  */
     334              : 
     335              : static enum insn_code
     336            0 : get_multi_vector_move (tree array_type, convert_optab optab)
     337              : {
     338            0 :   machine_mode imode;
     339            0 :   machine_mode vmode;
     340              : 
     341            0 :   gcc_assert (TREE_CODE (array_type) == ARRAY_TYPE);
     342            0 :   imode = TYPE_MODE (array_type);
     343            0 :   vmode = TYPE_MODE (TREE_TYPE (array_type));
     344              : 
     345            0 :   return convert_optab_handler (optab, imode, vmode);
     346              : }
     347              : 
     348              : /* Add mask, else, and len arguments according to the STMT.  */
     349              : 
     350              : static unsigned int
     351         1435 : add_mask_else_and_len_args (expand_operand *ops, unsigned int opno, gcall *stmt)
     352              : {
     353         1435 :   internal_fn ifn = gimple_call_internal_fn (stmt);
     354         1435 :   int len_index = internal_fn_len_index (ifn);
     355              :   /* BIAS is always consecutive next of LEN.  */
     356         1435 :   int bias_index = len_index + 1;
     357         1435 :   int mask_index = internal_fn_mask_index (ifn);
     358              : 
     359              :   /* The order of arguments is always {mask, else, len, bias}.  */
     360         1435 :   if (mask_index >= 0)
     361              :     {
     362         1435 :       tree mask = gimple_call_arg (stmt, mask_index);
     363         1435 :       rtx mask_rtx = expand_normal (mask);
     364              : 
     365         1435 :       tree mask_type = TREE_TYPE (mask);
     366         1435 :       if (VECTOR_BOOLEAN_TYPE_P (mask_type)
     367         1434 :           && SCALAR_INT_MODE_P (TYPE_MODE (mask_type))
     368         2495 :           && maybe_ne (GET_MODE_PRECISION (TYPE_MODE (mask_type)),
     369         2248 :                        TYPE_VECTOR_SUBPARTS (mask_type).to_constant ()))
     370              :         {
     371              :           /* Ensure that the vector bitmasks do not have excess bits.  */
     372          247 :           int nunits = TYPE_VECTOR_SUBPARTS (mask_type).to_constant ();
     373          247 :           mask_rtx = expand_binop (TYPE_MODE (mask_type), and_optab, mask_rtx,
     374          247 :                                    GEN_INT ((HOST_WIDE_INT_1U << nunits) - 1),
     375              :                                    NULL_RTX, true, OPTAB_WIDEN);
     376              :         }
     377              : 
     378         1435 :       create_input_operand (&ops[opno++], mask_rtx,
     379         1435 :                             TYPE_MODE (TREE_TYPE (mask)));
     380              :     }
     381              : 
     382         1435 :   int els_index = internal_fn_else_index (ifn);
     383         1435 :   if (els_index >= 0)
     384              :     {
     385          692 :       tree els = gimple_call_arg (stmt, els_index);
     386          692 :       tree els_type = TREE_TYPE (els);
     387          692 :       if (TREE_CODE (els) == SSA_NAME
     388            0 :           && SSA_NAME_IS_DEFAULT_DEF (els)
     389          692 :           && VAR_P (SSA_NAME_VAR (els)))
     390            0 :         create_undefined_input_operand (&ops[opno++], TYPE_MODE (els_type));
     391              :       else
     392              :         {
     393          692 :           rtx els_rtx = expand_normal (els);
     394          692 :           create_input_operand (&ops[opno++], els_rtx, TYPE_MODE (els_type));
     395              :         }
     396              :     }
     397         1435 :   if (len_index >= 0)
     398              :     {
     399            0 :       tree len = gimple_call_arg (stmt, len_index);
     400            0 :       rtx len_rtx = expand_normal (len);
     401            0 :       create_convert_operand_from (&ops[opno++], len_rtx,
     402            0 :                                    TYPE_MODE (TREE_TYPE (len)),
     403            0 :                                    TYPE_UNSIGNED (TREE_TYPE (len)));
     404            0 :       tree biast = gimple_call_arg (stmt, bias_index);
     405            0 :       rtx bias = expand_normal (biast);
     406            0 :       create_input_operand (&ops[opno++], bias, QImode);
     407              :     }
     408         1435 :   return opno;
     409              : }
     410              : 
     411              : /* Expand LOAD_LANES call STMT using optab OPTAB.  */
     412              : 
     413              : static void
     414            0 : expand_load_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
     415              : {
     416            0 :   class expand_operand ops[2];
     417            0 :   tree type, lhs, rhs;
     418            0 :   rtx target, mem;
     419              : 
     420            0 :   lhs = gimple_call_lhs (stmt);
     421            0 :   rhs = gimple_call_arg (stmt, 0);
     422            0 :   type = TREE_TYPE (lhs);
     423              : 
     424            0 :   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     425            0 :   mem = expand_normal (rhs);
     426              : 
     427            0 :   gcc_assert (MEM_P (mem));
     428            0 :   PUT_MODE (mem, TYPE_MODE (type));
     429              : 
     430            0 :   create_call_lhs_operand (&ops[0], target, TYPE_MODE (type));
     431            0 :   create_fixed_operand (&ops[1], mem);
     432            0 :   expand_insn (get_multi_vector_move (type, optab), 2, ops);
     433            0 :   assign_call_lhs (lhs, target, &ops[0]);
     434            0 : }
     435              : 
     436              : /* Expand STORE_LANES call STMT using optab OPTAB.  */
     437              : 
     438              : static void
     439            0 : expand_store_lanes_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
     440              : {
     441            0 :   class expand_operand ops[2];
     442            0 :   tree type, lhs, rhs;
     443            0 :   rtx target, reg;
     444              : 
     445            0 :   lhs = gimple_call_lhs (stmt);
     446            0 :   rhs = gimple_call_arg (stmt, 0);
     447            0 :   type = TREE_TYPE (rhs);
     448              : 
     449            0 :   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     450            0 :   reg = expand_normal (rhs);
     451              : 
     452            0 :   gcc_assert (MEM_P (target));
     453            0 :   PUT_MODE (target, TYPE_MODE (type));
     454              : 
     455            0 :   create_fixed_operand (&ops[0], target);
     456            0 :   create_input_operand (&ops[1], reg, TYPE_MODE (type));
     457            0 :   expand_insn (get_multi_vector_move (type, optab), 2, ops);
     458            0 : }
     459              : 
     460              : static void
     461            0 : expand_ANNOTATE (internal_fn, gcall *)
     462              : {
     463            0 :   gcc_unreachable ();
     464              : }
     465              : 
     466              : /* This should get expanded in omp_device_lower pass.  */
     467              : 
     468              : static void
     469            0 : expand_GOMP_USE_SIMT (internal_fn, gcall *)
     470              : {
     471            0 :   gcc_unreachable ();
     472              : }
     473              : 
     474              : /* This should get expanded in omp_device_lower pass.  */
     475              : 
     476              : static void
     477            0 : expand_GOMP_SIMT_ENTER (internal_fn, gcall *)
     478              : {
     479            0 :   gcc_unreachable ();
     480              : }
     481              : 
     482              : /* Allocate per-lane storage and begin non-uniform execution region.  */
     483              : 
     484              : static void
     485            0 : expand_GOMP_SIMT_ENTER_ALLOC (internal_fn, gcall *stmt)
     486              : {
     487            0 :   rtx target;
     488            0 :   tree lhs = gimple_call_lhs (stmt);
     489            0 :   if (lhs)
     490            0 :     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     491              :   else
     492            0 :     target = gen_reg_rtx (Pmode);
     493            0 :   rtx size = expand_normal (gimple_call_arg (stmt, 0));
     494            0 :   rtx align = expand_normal (gimple_call_arg (stmt, 1));
     495            0 :   class expand_operand ops[3];
     496            0 :   create_call_lhs_operand (&ops[0], target, Pmode);
     497            0 :   create_input_operand (&ops[1], size, Pmode);
     498            0 :   create_input_operand (&ops[2], align, Pmode);
     499            0 :   gcc_assert (targetm.have_omp_simt_enter ());
     500            0 :   expand_insn (targetm.code_for_omp_simt_enter, 3, ops);
     501            0 :   assign_call_lhs (lhs, target, &ops[0]);
     502            0 : }
     503              : 
     504              : /* Deallocate per-lane storage and leave non-uniform execution region.  */
     505              : 
     506              : static void
     507            0 : expand_GOMP_SIMT_EXIT (internal_fn, gcall *stmt)
     508              : {
     509            0 :   gcc_checking_assert (!gimple_call_lhs (stmt));
     510            0 :   rtx arg = expand_normal (gimple_call_arg (stmt, 0));
     511            0 :   class expand_operand ops[1];
     512            0 :   create_input_operand (&ops[0], arg, Pmode);
     513            0 :   gcc_assert (targetm.have_omp_simt_exit ());
     514            0 :   expand_insn (targetm.code_for_omp_simt_exit, 1, ops);
     515            0 : }
     516              : 
     517              : /* Lane index on SIMT targets: thread index in the warp on NVPTX.  On targets
     518              :    without SIMT execution this should be expanded in omp_device_lower pass.  */
     519              : 
     520              : static void
     521            0 : expand_GOMP_SIMT_LANE (internal_fn, gcall *stmt)
     522              : {
     523            0 :   tree lhs = gimple_call_lhs (stmt);
     524            0 :   if (!lhs)
     525              :     return;
     526              : 
     527            0 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     528            0 :   gcc_assert (targetm.have_omp_simt_lane ());
     529            0 :   emit_insn (targetm.gen_omp_simt_lane (target));
     530              : }
     531              : 
     532              : /* This should get expanded in omp_device_lower pass.  */
     533              : 
     534              : static void
     535            0 : expand_GOMP_SIMT_VF (internal_fn, gcall *)
     536              : {
     537            0 :   gcc_unreachable ();
     538              : }
     539              : 
     540              : /* This should get expanded in omp_device_lower pass.  */
     541              : 
     542              : static void
     543            0 : expand_GOMP_MAX_VF (internal_fn, gcall *)
     544              : {
     545            0 :   gcc_unreachable ();
     546              : }
     547              : 
     548              : /* This should get expanded in omp_device_lower pass.  */
     549              : 
     550              : static void
     551            0 : expand_GOMP_TARGET_REV (internal_fn, gcall *)
     552              : {
     553            0 :   gcc_unreachable ();
     554              : }
     555              : 
     556              : /* Lane index of the first SIMT lane that supplies a non-zero argument.
     557              :    This is a SIMT counterpart to GOMP_SIMD_LAST_LANE, used to represent the
     558              :    lane that executed the last iteration for handling OpenMP lastprivate.  */
     559              : 
     560              : static void
     561            0 : expand_GOMP_SIMT_LAST_LANE (internal_fn, gcall *stmt)
     562              : {
     563            0 :   tree lhs = gimple_call_lhs (stmt);
     564            0 :   if (!lhs)
     565            0 :     return;
     566              : 
     567            0 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     568            0 :   rtx cond = expand_normal (gimple_call_arg (stmt, 0));
     569            0 :   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
     570            0 :   class expand_operand ops[2];
     571            0 :   create_call_lhs_operand (&ops[0], target, mode);
     572            0 :   create_input_operand (&ops[1], cond, mode);
     573            0 :   gcc_assert (targetm.have_omp_simt_last_lane ());
     574            0 :   expand_insn (targetm.code_for_omp_simt_last_lane, 2, ops);
     575            0 :   assign_call_lhs (lhs, target, &ops[0]);
     576              : }
     577              : 
     578              : /* Non-transparent predicate used in SIMT lowering of OpenMP "ordered".  */
     579              : 
     580              : static void
     581            0 : expand_GOMP_SIMT_ORDERED_PRED (internal_fn, gcall *stmt)
     582              : {
     583            0 :   tree lhs = gimple_call_lhs (stmt);
     584            0 :   if (!lhs)
     585            0 :     return;
     586              : 
     587            0 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     588            0 :   rtx ctr = expand_normal (gimple_call_arg (stmt, 0));
     589            0 :   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
     590            0 :   class expand_operand ops[2];
     591            0 :   create_call_lhs_operand (&ops[0], target, mode);
     592            0 :   create_input_operand (&ops[1], ctr, mode);
     593            0 :   gcc_assert (targetm.have_omp_simt_ordered ());
     594            0 :   expand_insn (targetm.code_for_omp_simt_ordered, 2, ops);
     595            0 :   assign_call_lhs (lhs, target, &ops[0]);
     596              : }
     597              : 
     598              : /* "Or" boolean reduction across SIMT lanes: return non-zero in all lanes if
     599              :    any lane supplies a non-zero argument.  */
     600              : 
     601              : static void
     602            0 : expand_GOMP_SIMT_VOTE_ANY (internal_fn, gcall *stmt)
     603              : {
     604            0 :   tree lhs = gimple_call_lhs (stmt);
     605            0 :   if (!lhs)
     606            0 :     return;
     607              : 
     608            0 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     609            0 :   rtx cond = expand_normal (gimple_call_arg (stmt, 0));
     610            0 :   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
     611            0 :   class expand_operand ops[2];
     612            0 :   create_call_lhs_operand (&ops[0], target, mode);
     613            0 :   create_input_operand (&ops[1], cond, mode);
     614            0 :   gcc_assert (targetm.have_omp_simt_vote_any ());
     615            0 :   expand_insn (targetm.code_for_omp_simt_vote_any, 2, ops);
     616            0 :   assign_call_lhs (lhs, target, &ops[0]);
     617              : }
     618              : 
     619              : /* Exchange between SIMT lanes with a "butterfly" pattern: source lane index
     620              :    is destination lane index XOR given offset.  */
     621              : 
     622              : static void
     623            0 : expand_GOMP_SIMT_XCHG_BFLY (internal_fn, gcall *stmt)
     624              : {
     625            0 :   tree lhs = gimple_call_lhs (stmt);
     626            0 :   if (!lhs)
     627            0 :     return;
     628              : 
     629            0 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     630            0 :   rtx src = expand_normal (gimple_call_arg (stmt, 0));
     631            0 :   rtx idx = expand_normal (gimple_call_arg (stmt, 1));
     632            0 :   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
     633            0 :   class expand_operand ops[3];
     634            0 :   create_call_lhs_operand (&ops[0], target, mode);
     635            0 :   create_input_operand (&ops[1], src, mode);
     636            0 :   create_input_operand (&ops[2], idx, SImode);
     637            0 :   gcc_assert (targetm.have_omp_simt_xchg_bfly ());
     638            0 :   expand_insn (targetm.code_for_omp_simt_xchg_bfly, 3, ops);
     639            0 :   assign_call_lhs (lhs, target, &ops[0]);
     640              : }
     641              : 
     642              : /* Exchange between SIMT lanes according to given source lane index.  */
     643              : 
     644              : static void
     645            0 : expand_GOMP_SIMT_XCHG_IDX (internal_fn, gcall *stmt)
     646              : {
     647            0 :   tree lhs = gimple_call_lhs (stmt);
     648            0 :   if (!lhs)
     649            0 :     return;
     650              : 
     651            0 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
     652            0 :   rtx src = expand_normal (gimple_call_arg (stmt, 0));
     653            0 :   rtx idx = expand_normal (gimple_call_arg (stmt, 1));
     654            0 :   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
     655            0 :   class expand_operand ops[3];
     656            0 :   create_call_lhs_operand (&ops[0], target, mode);
     657            0 :   create_input_operand (&ops[1], src, mode);
     658            0 :   create_input_operand (&ops[2], idx, SImode);
     659            0 :   gcc_assert (targetm.have_omp_simt_xchg_idx ());
     660            0 :   expand_insn (targetm.code_for_omp_simt_xchg_idx, 3, ops);
     661            0 :   assign_call_lhs (lhs, target, &ops[0]);
     662              : }
     663              : 
     664              : /* This should get expanded in adjust_simduid_builtins.  */
     665              : 
     666              : static void
     667            0 : expand_GOMP_SIMD_LANE (internal_fn, gcall *)
     668              : {
     669            0 :   gcc_unreachable ();
     670              : }
     671              : 
     672              : /* This should get expanded in adjust_simduid_builtins.  */
     673              : 
     674              : static void
     675            0 : expand_GOMP_SIMD_VF (internal_fn, gcall *)
     676              : {
     677            0 :   gcc_unreachable ();
     678              : }
     679              : 
     680              : /* This should get expanded in adjust_simduid_builtins.  */
     681              : 
     682              : static void
     683            0 : expand_GOMP_SIMD_LAST_LANE (internal_fn, gcall *)
     684              : {
     685            0 :   gcc_unreachable ();
     686              : }
     687              : 
     688              : /* This should get expanded in adjust_simduid_builtins.  */
     689              : 
     690              : static void
     691            0 : expand_GOMP_SIMD_ORDERED_START (internal_fn, gcall *)
     692              : {
     693            0 :   gcc_unreachable ();
     694              : }
     695              : 
     696              : /* This should get expanded in adjust_simduid_builtins.  */
     697              : 
     698              : static void
     699            0 : expand_GOMP_SIMD_ORDERED_END (internal_fn, gcall *)
     700              : {
     701            0 :   gcc_unreachable ();
     702              : }
     703              : 
     704              : /* This should get expanded in gimplify_omp_dispatch.  */
     705              : 
     706              : static void
     707            0 : expand_GOMP_DISPATCH (internal_fn, gcall *)
     708              : {
     709            0 :   gcc_unreachable ();
     710              : }
     711              : 
     712              : /* This should get expanded in the sanopt pass.  */
     713              : 
     714              : static void
     715            0 : expand_UBSAN_NULL (internal_fn, gcall *)
     716              : {
     717            0 :   gcc_unreachable ();
     718              : }
     719              : 
     720              : /* This should get expanded in the sanopt pass.  */
     721              : 
     722              : static void
     723            0 : expand_UBSAN_BOUNDS (internal_fn, gcall *)
     724              : {
     725            0 :   gcc_unreachable ();
     726              : }
     727              : 
     728              : /* This should get expanded in the sanopt pass.  */
     729              : 
     730              : static void
     731            0 : expand_UBSAN_VPTR (internal_fn, gcall *)
     732              : {
     733            0 :   gcc_unreachable ();
     734              : }
     735              : 
     736              : /* This should get expanded in the sanopt pass.  */
     737              : 
     738              : static void
     739            0 : expand_UBSAN_PTR (internal_fn, gcall *)
     740              : {
     741            0 :   gcc_unreachable ();
     742              : }
     743              : 
     744              : /* This should get expanded in the sanopt pass.  */
     745              : 
     746              : static void
     747            0 : expand_UBSAN_OBJECT_SIZE (internal_fn, gcall *)
     748              : {
     749            0 :   gcc_unreachable ();
     750              : }
     751              : 
     752              : /* This should get expanded in the sanopt pass.  */
     753              : 
     754              : static void
     755            0 : expand_HWASAN_CHECK (internal_fn, gcall *)
     756              : {
     757            0 :   gcc_unreachable ();
     758              : }
     759              : 
     760              : /* For hwasan stack tagging:
     761              :    Tag memory which is dynamically allocated.  */
     762              : static void
     763            0 : expand_HWASAN_ALLOCA_POISON (internal_fn, gcall *gc)
     764              : {
     765            0 :   gcc_assert (ptr_mode == Pmode);
     766            0 :   tree g_target = gimple_call_lhs (gc);
     767            0 :   tree g_ptr = gimple_call_arg (gc, 0);
     768            0 :   tree g_size = gimple_call_arg (gc, 1);
     769              : 
     770              :   /* There is no target; this happens, usually, when we have an alloca of zero
     771              :      size.  */
     772            0 :   if (!g_target)
     773              :     return;
     774            0 :   rtx target = expand_normal (g_target);
     775            0 :   rtx ptr = expand_normal (g_ptr);
     776            0 :   rtx size = expand_normal (g_size);
     777              : 
     778              :   /* No size, nothing to do.  */
     779            0 :   if (size == const0_rtx)
     780              :     return;
     781              : 
     782              :   /* Get new tag for the alloca'd memory.
     783              :      Doing a regular add_tag () like so:
     784              :         rtx tag = targetm.memtag.add_tag (hwasan_frame_base (), 0,
     785              :                                           hwasan_current_frame_tag ());
     786              :      gets a new tag, which can be used for tagging memory.  But for alloca, we
     787              :      need both tagged memory and a tagged pointer to pass to consumers.  Invoke
     788              :      insert_random_tag () instead to add a random tag to ptr to get a tagged
     789              :      pointer that will work for both purposes.  */
     790            0 :   rtx tagged_ptr
     791            0 :     = force_reg (Pmode, targetm.memtag.insert_random_tag (ptr, NULL_RTX));
     792            0 :   rtx tag = targetm.memtag.extract_tag (tagged_ptr, NULL_RTX);
     793              : 
     794            0 :   if (memtag_sanitize_p ())
     795              :     {
     796              :       /* Need to put the tagged ptr into the `target` RTX for consumers
     797              :          of alloca'd memory.  */
     798            0 :       if (tagged_ptr != target)
     799            0 :         emit_move_insn (target, tagged_ptr);
     800              :       /* Tag the memory.  */
     801            0 :       emit_insn (targetm.gen_tag_memory (ptr, tag, size));
     802            0 :       hwasan_increment_frame_tag ();
     803              :     }
     804              :   else
     805            0 :     gcc_unreachable ();
     806              : }
     807              : 
     808              : /* For hwasan stack tagging:
     809              :    Clear tags on the dynamically allocated space.
     810              :    For use after an object dynamically allocated on the stack goes out of
     811              :    scope.  */
     812              : static void
     813            0 : expand_HWASAN_ALLOCA_UNPOISON (internal_fn, gcall *gc)
     814              : {
     815            0 :   gcc_assert (Pmode == ptr_mode);
     816            0 :   tree restored_position = gimple_call_arg (gc, 0);
     817            0 :   rtx restored_rtx = expand_expr (restored_position, NULL_RTX, VOIDmode,
     818              :                                   EXPAND_NORMAL);
     819            0 :   rtx off = expand_simple_binop (Pmode, MINUS, restored_rtx,
     820              :                                  stack_pointer_rtx, NULL_RTX, 0,
     821              :                                  OPTAB_WIDEN);
     822              : 
     823            0 :   if (memtag_sanitize_p ())
     824            0 :     emit_insn (targetm.gen_tag_memory (virtual_stack_dynamic_rtx,
     825              :                                        HWASAN_STACK_BACKGROUND,
     826              :                                        off));
     827              :   else
     828              :     {
     829            0 :       rtx func = init_one_libfunc ("__hwasan_tag_memory");
     830            0 :       emit_library_call_value (func, NULL_RTX, LCT_NORMAL, VOIDmode,
     831            0 :                                virtual_stack_dynamic_rtx, Pmode,
     832              :                                HWASAN_STACK_BACKGROUND, QImode,
     833            0 :                                off, Pmode);
     834              :     }
     835            0 : }
     836              : 
     837              : /* For hwasan stack tagging:
     838              :    Return a tag to be used for a dynamic allocation.  */
     839              : static void
     840            0 : expand_HWASAN_CHOOSE_TAG (internal_fn, gcall *gc)
     841              : {
     842            0 :   tree tag = gimple_call_lhs (gc);
     843            0 :   rtx target = expand_expr (tag, NULL_RTX, VOIDmode, EXPAND_NORMAL);
     844            0 :   machine_mode mode = GET_MODE (target);
     845            0 :   gcc_assert (mode == QImode);
     846              : 
     847            0 :   rtx base_tag = targetm.memtag.extract_tag (hwasan_frame_base (), NULL_RTX);
     848            0 :   gcc_assert (base_tag);
     849            0 :   rtx tag_offset = gen_int_mode (hwasan_current_frame_tag (), QImode);
     850            0 :   rtx chosen_tag = expand_simple_binop (QImode, PLUS, base_tag, tag_offset,
     851              :                                         target, /* unsignedp = */1,
     852              :                                         OPTAB_WIDEN);
     853            0 :   chosen_tag = hwasan_truncate_to_tag_size (chosen_tag, target);
     854              : 
     855              :   /* Really need to put the tag into the `target` RTX.  */
     856            0 :   if (chosen_tag != target)
     857              :     {
     858            0 :       rtx temp = chosen_tag;
     859            0 :       gcc_assert (GET_MODE (chosen_tag) == mode);
     860            0 :       emit_move_insn (target, temp);
     861              :     }
     862              : 
     863            0 :   hwasan_increment_frame_tag ();
     864            0 : }
     865              : 
     866              : /* For hwasan stack tagging:
     867              :    Tag a region of space in the shadow stack according to the base pointer of
     868              :    an object on the stack.  N.b. the length provided in the internal call is
     869              :    required to be aligned to HWASAN_TAG_GRANULE_SIZE.  */
     870              : static void
     871            0 : expand_HWASAN_MARK (internal_fn, gcall *gc)
     872              : {
     873            0 :   gcc_assert (ptr_mode == Pmode);
     874            0 :   HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (gc, 0));
     875            0 :   bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
     876              : 
     877            0 :   tree base = gimple_call_arg (gc, 1);
     878            0 :   gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
     879            0 :   rtx base_rtx = expand_normal (base);
     880              : 
     881            0 :   rtx tag = is_poison ? HWASAN_STACK_BACKGROUND
     882            0 :     : targetm.memtag.extract_tag (base_rtx, NULL_RTX);
     883            0 :   rtx address = targetm.memtag.untagged_pointer (base_rtx, NULL_RTX);
     884              : 
     885            0 :   tree len = gimple_call_arg (gc, 2);
     886            0 :   rtx r_len = expand_normal (len);
     887              : 
     888            0 :   if (memtag_sanitize_p ())
     889            0 :     emit_insn (targetm.gen_tag_memory (address, tag, r_len));
     890              :   else
     891              :     {
     892            0 :       rtx func = init_one_libfunc ("__hwasan_tag_memory");
     893            0 :       emit_library_call (func, LCT_NORMAL, VOIDmode, address, Pmode,
     894            0 :                          tag, QImode, r_len, Pmode);
     895              :     }
     896            0 : }
     897              : 
     898              : /* For hwasan stack tagging:
     899              :    Store a tag into a pointer.  */
     900              : static void
     901            0 : expand_HWASAN_SET_TAG (internal_fn, gcall *gc)
     902              : {
     903            0 :   gcc_assert (ptr_mode == Pmode);
     904            0 :   tree g_target = gimple_call_lhs (gc);
     905            0 :   tree g_ptr = gimple_call_arg (gc, 0);
     906            0 :   tree g_tag = gimple_call_arg (gc, 1);
     907              : 
     908            0 :   rtx ptr = expand_normal (g_ptr);
     909            0 :   rtx tag = expand_expr (g_tag, NULL_RTX, QImode, EXPAND_NORMAL);
     910            0 :   rtx target = expand_normal (g_target);
     911              : 
     912            0 :   rtx untagged = targetm.memtag.untagged_pointer (ptr, target);
     913            0 :   rtx tagged_value = targetm.memtag.set_tag (untagged, tag, target);
     914            0 :   if (tagged_value != target)
     915            0 :     emit_move_insn (target, tagged_value);
     916            0 : }
     917              : 
     918              : /* This should get expanded in the sanopt pass.  */
     919              : 
     920              : static void
     921            0 : expand_ASAN_CHECK (internal_fn, gcall *)
     922              : {
     923            0 :   gcc_unreachable ();
     924              : }
     925              : 
     926              : /* This should get expanded in the sanopt pass.  */
     927              : 
     928              : static void
     929            0 : expand_ASAN_MARK (internal_fn, gcall *)
     930              : {
     931            0 :   gcc_unreachable ();
     932              : }
     933              : 
     934              : /* This should get expanded in the sanopt pass.  */
     935              : 
     936              : static void
     937            0 : expand_ASAN_POISON (internal_fn, gcall *)
     938              : {
     939            0 :   gcc_unreachable ();
     940              : }
     941              : 
     942              : /* This should get expanded in the sanopt pass.  */
     943              : 
     944              : static void
     945            0 : expand_ASAN_POISON_USE (internal_fn, gcall *)
     946              : {
     947            0 :   gcc_unreachable ();
     948              : }
     949              : 
     950              : /* This should get expanded in the tsan pass.  */
     951              : 
     952              : static void
     953            0 : expand_TSAN_FUNC_EXIT (internal_fn, gcall *)
     954              : {
     955            0 :   gcc_unreachable ();
     956              : }
     957              : 
     958              : /* This should get expanded in the lower pass.  */
     959              : 
     960              : static void
     961           25 : expand_FALLTHROUGH (internal_fn, gcall *call)
     962              : {
     963           25 :   auto_urlify_attributes sentinel;
     964           25 :   error_at (gimple_location (call),
     965              :             "invalid use of attribute %<fallthrough%>");
     966           25 : }
     967              : 
     968              : /* Return minimum precision needed to represent all values
     969              :    of ARG in SIGNed integral type.  */
     970              : 
     971              : static int
     972       157580 : get_min_precision (tree arg, signop sign)
     973              : {
     974       157580 :   int prec = TYPE_PRECISION (TREE_TYPE (arg));
     975       157580 :   int cnt = 0;
     976       157580 :   signop orig_sign = sign;
     977       157580 :   if (TREE_CODE (arg) == INTEGER_CST)
     978              :     {
     979        40770 :       int p;
     980        40770 :       if (TYPE_SIGN (TREE_TYPE (arg)) != sign)
     981              :         {
     982        14623 :           widest_int w = wi::to_widest (arg);
     983        14623 :           w = wi::ext (w, prec, sign);
     984        14623 :           p = wi::min_precision (w, sign);
     985        14623 :         }
     986              :       else
     987        26147 :         p = wi::min_precision (wi::to_wide (arg), sign);
     988        40770 :       return MIN (p, prec);
     989              :     }
     990       116765 :   while (CONVERT_EXPR_P (arg)
     991         7371 :          && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
     992       131507 :          && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) <= prec)
     993              :     {
     994         7371 :       arg = TREE_OPERAND (arg, 0);
     995         7371 :       if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
     996              :         {
     997         1109 :           if (TYPE_UNSIGNED (TREE_TYPE (arg)))
     998              :             sign = UNSIGNED;
     999          805 :           else if (sign == UNSIGNED
    1000          805 :                    && (get_range_pos_neg (arg,
    1001              :                                           currently_expanding_gimple_stmt)
    1002              :                        != 1))
    1003           45 :             return prec + (orig_sign != sign);
    1004         1064 :           prec = TYPE_PRECISION (TREE_TYPE (arg));
    1005              :         }
    1006         7326 :       if (++cnt > 30)
    1007            0 :         return prec + (orig_sign != sign);
    1008              :     }
    1009       116765 :   if (CONVERT_EXPR_P (arg)
    1010            0 :       && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (arg, 0)))
    1011       116765 :       && TYPE_PRECISION (TREE_TYPE (TREE_OPERAND (arg, 0))) > prec)
    1012              :     {
    1013              :       /* We have e.g. (unsigned short) y_2 where int y_2 = (int) x_1(D);
    1014              :          If y_2's min precision is smaller than prec, return that.  */
    1015            0 :       int oprec = get_min_precision (TREE_OPERAND (arg, 0), sign);
    1016            0 :       if (oprec < prec)
    1017            0 :         return oprec + (orig_sign != sign);
    1018              :     }
    1019       116765 :   if (TREE_CODE (arg) != SSA_NAME)
    1020            0 :     return prec + (orig_sign != sign);
    1021       116765 :   int_range_max r;
    1022       116765 :   gimple *cg = currently_expanding_gimple_stmt;
    1023       239126 :   while (!get_range_query (cfun)->range_of_expr (r, arg, cg)
    1024       122361 :          || r.varying_p ()
    1025       254083 :          || r.undefined_p ())
    1026              :     {
    1027       113001 :       gimple *g = SSA_NAME_DEF_STMT (arg);
    1028       113001 :       if (is_gimple_assign (g)
    1029       113001 :           && CONVERT_EXPR_CODE_P (gimple_assign_rhs_code (g)))
    1030              :         {
    1031         6706 :           tree t = gimple_assign_rhs1 (g);
    1032        13412 :           if (INTEGRAL_TYPE_P (TREE_TYPE (t))
    1033        12415 :               && TYPE_PRECISION (TREE_TYPE (t)) <= prec)
    1034              :             {
    1035         5603 :               arg = t;
    1036         5603 :               if (TYPE_PRECISION (TREE_TYPE (arg)) < prec)
    1037              :                 {
    1038         5209 :                   if (TYPE_UNSIGNED (TREE_TYPE (arg)))
    1039              :                     sign = UNSIGNED;
    1040         2885 :                   else if (sign == UNSIGNED
    1041         2885 :                            && get_range_pos_neg (arg, g) != 1)
    1042            7 :                     return prec + (orig_sign != sign);
    1043         5202 :                   prec = TYPE_PRECISION (TREE_TYPE (arg));
    1044              :                 }
    1045         5596 :               if (++cnt > 30)
    1046            0 :                 return prec + (orig_sign != sign);
    1047         5596 :               continue;
    1048              :             }
    1049              :         }
    1050       107398 :       return prec + (orig_sign != sign);
    1051              :     }
    1052         9360 :   if (sign == TYPE_SIGN (TREE_TYPE (arg)))
    1053              :     {
    1054         8778 :       int p1 = wi::min_precision (r.lower_bound (), sign);
    1055         8778 :       int p2 = wi::min_precision (r.upper_bound (), sign);
    1056         8778 :       p1 = MAX (p1, p2);
    1057         8778 :       prec = MIN (prec, p1);
    1058              :     }
    1059         1140 :   else if (sign == UNSIGNED && !wi::neg_p (r.lower_bound (), SIGNED))
    1060              :     {
    1061          554 :       int p = wi::min_precision (r.upper_bound (), UNSIGNED);
    1062          582 :       prec = MIN (prec, p);
    1063              :     }
    1064         9360 :   return prec + (orig_sign != sign);
    1065       116765 : }
    1066              : 
    1067              : /* Helper for expand_*_overflow.  Set the __imag__ part to true
    1068              :    (1 except for signed:1 type, in which case store -1).  */
    1069              : 
    1070              : static void
    1071        82332 : expand_arith_set_overflow (tree lhs, rtx target)
    1072              : {
    1073        82332 :   if (TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs))) == 1
    1074        82332 :       && !TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs))))
    1075          202 :     write_complex_part (target, constm1_rtx, true, false);
    1076              :   else
    1077        82130 :     write_complex_part (target, const1_rtx, true, false);
    1078        82332 : }
    1079              : 
    1080              : /* Helper for expand_*_overflow.  Store RES into the __real__ part
    1081              :    of TARGET.  If RES has larger MODE than __real__ part of TARGET,
    1082              :    set the __imag__ part to 1 if RES doesn't fit into it.  Similarly
    1083              :    if LHS has smaller precision than its mode, unless OVERFLOW_CHECKED
    1084              :    is true in which case the caller has already performed the
    1085              :    overflow check and recorded the result.  */
    1086              : 
    1087              : static void
    1088        72249 : expand_arith_overflow_result_store (tree lhs, rtx target,
    1089              :                                     scalar_int_mode mode, rtx res,
    1090              :                                     bool overflow_checked = false)
    1091              : {
    1092        72249 :   scalar_int_mode tgtmode
    1093       144498 :     = as_a <scalar_int_mode> (GET_MODE_INNER (GET_MODE (target)));
    1094        72249 :   rtx lres = res;
    1095        72249 :   if (tgtmode != mode)
    1096              :     {
    1097        18892 :       rtx_code_label *done_label = gen_label_rtx ();
    1098        18892 :       int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
    1099        18892 :       lres = convert_modes (tgtmode, mode, res, uns);
    1100        18892 :       gcc_assert (GET_MODE_PRECISION (tgtmode) < GET_MODE_PRECISION (mode));
    1101        18892 :       do_compare_rtx_and_jump (res, convert_modes (mode, tgtmode, lres, uns),
    1102              :                                EQ, true, mode, NULL_RTX, NULL, done_label,
    1103              :                                profile_probability::very_likely ());
    1104        18892 :       expand_arith_set_overflow (lhs, target);
    1105        18892 :       emit_label (done_label);
    1106              :     }
    1107        72249 :   int prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
    1108        72249 :   int tgtprec = GET_MODE_PRECISION (tgtmode);
    1109        72249 :   if (prec < tgtprec && !overflow_checked)
    1110              :     {
    1111         3569 :       rtx_code_label *done_label = gen_label_rtx ();
    1112         3569 :       int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
    1113         3569 :       res = lres;
    1114         3569 :       if (uns)
    1115              :         {
    1116         2024 :           rtx mask
    1117         2024 :             = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
    1118              :                                     tgtmode);
    1119         2024 :           lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
    1120              :                                       true, OPTAB_LIB_WIDEN);
    1121              :         }
    1122              :       else
    1123              :         {
    1124         1545 :           lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
    1125              :                                NULL_RTX, 1);
    1126         1545 :           lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
    1127              :                                NULL_RTX, 0);
    1128              :         }
    1129         3569 :       do_compare_rtx_and_jump (res, lres,
    1130              :                                EQ, true, tgtmode, NULL_RTX, NULL, done_label,
    1131              :                                profile_probability::very_likely ());
    1132         3569 :       expand_arith_set_overflow (lhs, target);
    1133         3569 :       emit_label (done_label);
    1134              :     }
    1135        68680 :   else if (prec < tgtprec)
    1136              :     {
    1137              :       /* overflow_checked: the caller already detected and recorded overflow
    1138              :          via the hipart comparison.  */
    1139            0 :       int uns = TYPE_UNSIGNED (TREE_TYPE (TREE_TYPE (lhs)));
    1140            0 :       res = lres;
    1141            0 :       if (uns)
    1142              :         {
    1143            0 :           rtx mask
    1144            0 :             = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
    1145              :                                     tgtmode);
    1146            0 :           lres = expand_simple_binop (tgtmode, AND, res, mask, NULL_RTX,
    1147              :                                       true, OPTAB_LIB_WIDEN);
    1148              :         }
    1149              :       else
    1150              :         {
    1151            0 :           lres = expand_shift (LSHIFT_EXPR, tgtmode, res, tgtprec - prec,
    1152              :                                NULL_RTX, 1);
    1153            0 :           lres = expand_shift (RSHIFT_EXPR, tgtmode, lres, tgtprec - prec,
    1154              :                                NULL_RTX, 0);
    1155              :         }
    1156              :     }
    1157        72249 :   write_complex_part (target, lres, false, false);
    1158        72249 : }
    1159              : 
    1160              : /* Helper for expand_*_overflow.  Store RES into TARGET.  */
    1161              : 
    1162              : static void
    1163         4675 : expand_ubsan_result_store (tree lhs, rtx target, scalar_int_mode mode,
    1164              :                            rtx res, rtx_code_label *do_error)
    1165              : {
    1166         9296 :   if (BITINT_TYPE_P (TREE_TYPE (lhs))
    1167         4675 :       && TYPE_PRECISION (TREE_TYPE (lhs)) < GET_MODE_PRECISION (mode))
    1168              :     {
    1169           54 :       int uns = TYPE_UNSIGNED (TREE_TYPE (lhs));
    1170           54 :       int prec = TYPE_PRECISION (TREE_TYPE (lhs));
    1171           54 :       int tgtprec = GET_MODE_PRECISION (mode);
    1172           54 :       rtx resc = gen_reg_rtx (mode), lres;
    1173           54 :       emit_move_insn (resc, res);
    1174           54 :       if (uns)
    1175              :         {
    1176            0 :           rtx mask
    1177            0 :             = immed_wide_int_const (wi::shifted_mask (0, prec, false, tgtprec),
    1178              :                                     mode);
    1179            0 :           lres = expand_simple_binop (mode, AND, res, mask, NULL_RTX,
    1180              :                                       true, OPTAB_LIB_WIDEN);
    1181              :         }
    1182              :       else
    1183              :         {
    1184           54 :           lres = expand_shift (LSHIFT_EXPR, mode, res, tgtprec - prec,
    1185              :                                NULL_RTX, 1);
    1186           54 :           lres = expand_shift (RSHIFT_EXPR, mode, lres, tgtprec - prec,
    1187              :                                NULL_RTX, 0);
    1188              :         }
    1189           54 :       if (lres != res)
    1190           54 :         emit_move_insn (res, lres);
    1191           54 :       do_compare_rtx_and_jump (res, resc,
    1192              :                                NE, true, mode, NULL_RTX, NULL, do_error,
    1193              :                                profile_probability::very_unlikely ());
    1194              :     }
    1195         4675 :   if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
    1196              :     /* If this is a scalar in a register that is stored in a wider mode
    1197              :        than the declared mode, compute the result into its declared mode
    1198              :        and then convert to the wider mode.  Our value is the computed
    1199              :        expression.  */
    1200            0 :     convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
    1201              :   else
    1202         4675 :     emit_move_insn (target, res);
    1203         4675 : }
    1204              : 
    1205              : /* Add sub/add overflow checking to the statement STMT.
    1206              :    CODE says whether the operation is +, or -.  */
    1207              : 
    1208              : void
    1209        39263 : expand_addsub_overflow (location_t loc, tree_code code, tree lhs,
    1210              :                         tree arg0, tree arg1, bool unsr_p, bool uns0_p,
    1211              :                         bool uns1_p, bool is_ubsan, tree *datap)
    1212              : {
    1213        39263 :   rtx res, target = NULL_RTX;
    1214        39263 :   tree fn;
    1215        39263 :   rtx_code_label *done_label = gen_label_rtx ();
    1216        39263 :   rtx_code_label *do_error = gen_label_rtx ();
    1217        39263 :   do_pending_stack_adjust ();
    1218        39263 :   rtx op0 = expand_normal (arg0);
    1219        39263 :   rtx op1 = expand_normal (arg1);
    1220        39263 :   scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
    1221        39263 :   int prec = GET_MODE_PRECISION (mode);
    1222        39263 :   rtx sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
    1223        39263 :   bool do_xor = false;
    1224              : 
    1225        39263 :   if (is_ubsan)
    1226         3391 :     gcc_assert (!unsr_p && !uns0_p && !uns1_p);
    1227              : 
    1228        39263 :   if (lhs)
    1229              :     {
    1230        38735 :       target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    1231        38735 :       if (!is_ubsan)
    1232        35872 :         write_complex_part (target, const0_rtx, true, false);
    1233              :     }
    1234              : 
    1235              :   /* We assume both operands and result have the same precision
    1236              :      here (GET_MODE_BITSIZE (mode)), S stands for signed type
    1237              :      with that precision, U for unsigned type with that precision,
    1238              :      sgn for unsigned most significant bit in that precision.
    1239              :      s1 is signed first operand, u1 is unsigned first operand,
    1240              :      s2 is signed second operand, u2 is unsigned second operand,
    1241              :      sr is signed result, ur is unsigned result and the following
    1242              :      rules say how to compute result (which is always result of
    1243              :      the operands as if both were unsigned, cast to the right
    1244              :      signedness) and how to compute whether operation overflowed.
    1245              : 
    1246              :      s1 + s2 -> sr
    1247              :         res = (S) ((U) s1 + (U) s2)
    1248              :         ovf = s2 < 0 ? res > s1 : res < s1 (or jump on overflow)
    1249              :      s1 - s2 -> sr
    1250              :         res = (S) ((U) s1 - (U) s2)
    1251              :         ovf = s2 < 0 ? res < s1 : res > s2 (or jump on overflow)
    1252              :      u1 + u2 -> ur
    1253              :         res = u1 + u2
    1254              :         ovf = res < u1 (or jump on carry, but RTL opts will handle it)
    1255              :      u1 - u2 -> ur
    1256              :         res = u1 - u2
    1257              :         ovf = res > u1 (or jump on carry, but RTL opts will handle it)
    1258              :      s1 + u2 -> sr
    1259              :         res = (S) ((U) s1 + u2)
    1260              :         ovf = ((U) res ^ sgn) < u2
    1261              :      s1 + u2 -> ur
    1262              :         t1 = (S) (u2 ^ sgn)
    1263              :         t2 = s1 + t1
    1264              :         res = (U) t2 ^ sgn
    1265              :         ovf = t1 < 0 ? t2 > s1 : t2 < s1 (or jump on overflow)
    1266              :      s1 - u2 -> sr
    1267              :         res = (S) ((U) s1 - u2)
    1268              :         ovf = u2 > ((U) s1 ^ sgn)
    1269              :      s1 - u2 -> ur
    1270              :         res = (U) s1 - u2
    1271              :         ovf = s1 < 0 || u2 > (U) s1
    1272              :      u1 - s2 -> sr
    1273              :         res = u1 - (U) s2
    1274              :         ovf = u1 >= ((U) s2 ^ sgn)
    1275              :      u1 - s2 -> ur
    1276              :         t1 = u1 ^ sgn
    1277              :         t2 = t1 - (U) s2
    1278              :         res = t2 ^ sgn
    1279              :         ovf = s2 < 0 ? (S) t2 < (S) t1 : (S) t2 > (S) t1 (or jump on overflow)
    1280              :      s1 + s2 -> ur
    1281              :         res = (U) s1 + (U) s2
    1282              :         ovf = s2 < 0 ? (s1 | (S) res) < 0) : (s1 & (S) res) < 0)
    1283              :      u1 + u2 -> sr
    1284              :         res = (S) (u1 + u2)
    1285              :         ovf = (U) res < u2 || res < 0
    1286              :      u1 - u2 -> sr
    1287              :         res = (S) (u1 - u2)
    1288              :         ovf = u1 >= u2 ? res < 0 : res >= 0
    1289              :      s1 - s2 -> ur
    1290              :         res = (U) s1 - (U) s2
    1291              :         ovf = s2 >= 0 ? ((s1 | (S) res) < 0) : ((s1 & (S) res) < 0)  */
    1292              : 
    1293        39263 :   if (code == PLUS_EXPR && uns0_p && !uns1_p)
    1294              :     {
    1295              :       /* PLUS_EXPR is commutative, if operand signedness differs,
    1296              :          canonicalize to the first operand being signed and second
    1297              :          unsigned to simplify following code.  */
    1298              :       std::swap (op0, op1);
    1299              :       std::swap (arg0, arg1);
    1300              :       uns0_p = false;
    1301              :       uns1_p = true;
    1302              :     }
    1303              : 
    1304              :   /* u1 +- u2 -> ur  */
    1305        38480 :   if (uns0_p && uns1_p && unsr_p)
    1306              :     {
    1307        20577 :       insn_code icode = optab_handler (code == PLUS_EXPR ? uaddv4_optab
    1308              :                                        : usubv4_optab, mode);
    1309        15470 :       if (icode != CODE_FOR_nothing)
    1310              :         {
    1311        14641 :           class expand_operand ops[4];
    1312        14641 :           rtx_insn *last = get_last_insn ();
    1313              : 
    1314        14641 :           res = gen_reg_rtx (mode);
    1315        14641 :           create_output_operand (&ops[0], res, mode);
    1316        14641 :           create_input_operand (&ops[1], op0, mode);
    1317        14641 :           create_input_operand (&ops[2], op1, mode);
    1318        14641 :           create_fixed_operand (&ops[3], do_error);
    1319        14641 :           if (maybe_expand_insn (icode, 4, ops))
    1320              :             {
    1321        14641 :               last = get_last_insn ();
    1322        14641 :               if (profile_status_for_fn (cfun) != PROFILE_ABSENT
    1323        12303 :                   && JUMP_P (last)
    1324        12303 :                   && any_condjump_p (last)
    1325        26944 :                   && !find_reg_note (last, REG_BR_PROB, 0))
    1326        12303 :                 add_reg_br_prob_note (last,
    1327              :                                       profile_probability::very_unlikely ());
    1328        14641 :               emit_jump (done_label);
    1329        14641 :               goto do_error_label;
    1330              :             }
    1331              : 
    1332            0 :           delete_insns_since (last);
    1333              :         }
    1334              : 
    1335              :       /* Compute the operation.  On RTL level, the addition is always
    1336              :          unsigned.  */
    1337         1658 :       res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
    1338              :                           op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
    1339          829 :       rtx tem = op0;
    1340              :       /* For PLUS_EXPR, the operation is commutative, so we can pick
    1341              :          operand to compare against.  For prec <= BITS_PER_WORD, I think
    1342              :          preferring REG operand is better over CONST_INT, because
    1343              :          the CONST_INT might enlarge the instruction or CSE would need
    1344              :          to figure out we'd already loaded it into a register before.
    1345              :          For prec > BITS_PER_WORD, I think CONST_INT might be more beneficial,
    1346              :          as then the multi-word comparison can be perhaps simplified.  */
    1347          829 :       if (code == PLUS_EXPR
    1348            0 :           && (prec <= BITS_PER_WORD
    1349            0 :               ? (CONST_SCALAR_INT_P (op0) && REG_P (op1))
    1350            0 :               : CONST_SCALAR_INT_P (op1)))
    1351          829 :         tem = op1;
    1352         1658 :       do_compare_rtx_and_jump (res, tem, code == PLUS_EXPR ? GEU : LEU,
    1353              :                                true, mode, NULL_RTX, NULL, done_label,
    1354              :                                profile_probability::very_likely ());
    1355          829 :       goto do_error_label;
    1356              :     }
    1357              : 
    1358              :   /* s1 +- u2 -> sr  */
    1359        23793 :   if (!uns0_p && uns1_p && !unsr_p)
    1360              :     {
    1361              :       /* Compute the operation.  On RTL level, the addition is always
    1362              :          unsigned.  */
    1363         2514 :       res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
    1364              :                           op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
    1365         2514 :       rtx tem = expand_binop (mode, add_optab,
    1366              :                               code == PLUS_EXPR ? res : op0, sgn,
    1367              :                               NULL_RTX, false, OPTAB_LIB_WIDEN);
    1368         1582 :       do_compare_rtx_and_jump (tem, op1, GEU, true, mode, NULL_RTX, NULL,
    1369              :                                done_label, profile_probability::very_likely ());
    1370         1582 :       goto do_error_label;
    1371              :     }
    1372              : 
    1373              :   /* s1 + u2 -> ur  */
    1374        22211 :   if (code == PLUS_EXPR && !uns0_p && uns1_p && unsr_p)
    1375              :     {
    1376         1691 :       op1 = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
    1377              :                           OPTAB_LIB_WIDEN);
    1378              :       /* As we've changed op1, we have to avoid using the value range
    1379              :          for the original argument.  */
    1380         1691 :       arg1 = error_mark_node;
    1381         1691 :       do_xor = true;
    1382         1691 :       goto do_signed;
    1383              :     }
    1384              : 
    1385              :   /* u1 - s2 -> ur  */
    1386        20520 :   if (code == MINUS_EXPR && uns0_p && !uns1_p && unsr_p)
    1387              :     {
    1388          846 :       op0 = expand_binop (mode, add_optab, op0, sgn, NULL_RTX, false,
    1389              :                           OPTAB_LIB_WIDEN);
    1390              :       /* As we've changed op0, we have to avoid using the value range
    1391              :          for the original argument.  */
    1392          846 :       arg0 = error_mark_node;
    1393          846 :       do_xor = true;
    1394          846 :       goto do_signed;
    1395              :     }
    1396              : 
    1397              :   /* s1 - u2 -> ur  */
    1398        19674 :   if (code == MINUS_EXPR && !uns0_p && uns1_p && unsr_p)
    1399              :     {
    1400              :       /* Compute the operation.  On RTL level, the addition is always
    1401              :          unsigned.  */
    1402         1374 :       res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
    1403              :                           OPTAB_LIB_WIDEN);
    1404         1374 :       int pos_neg = get_range_pos_neg (arg0, currently_expanding_gimple_stmt);
    1405         1374 :       if (pos_neg == 2)
    1406              :         /* If ARG0 is known to be always negative, this is always overflow.  */
    1407           39 :         emit_jump (do_error);
    1408         1335 :       else if (pos_neg == 3)
    1409              :         /* If ARG0 is not known to be always positive, check at runtime.  */
    1410         1335 :         do_compare_rtx_and_jump (op0, const0_rtx, LT, false, mode, NULL_RTX,
    1411              :                                  NULL, do_error, profile_probability::very_unlikely ());
    1412         1374 :       do_compare_rtx_and_jump (op1, op0, LEU, true, mode, NULL_RTX, NULL,
    1413              :                                done_label, profile_probability::very_likely ());
    1414         1374 :       goto do_error_label;
    1415              :     }
    1416              : 
    1417              :   /* u1 - s2 -> sr  */
    1418        18300 :   if (code == MINUS_EXPR && uns0_p && !uns1_p && !unsr_p)
    1419              :     {
    1420              :       /* Compute the operation.  On RTL level, the addition is always
    1421              :          unsigned.  */
    1422          481 :       res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
    1423              :                           OPTAB_LIB_WIDEN);
    1424          481 :       rtx tem = expand_binop (mode, add_optab, op1, sgn, NULL_RTX, false,
    1425              :                               OPTAB_LIB_WIDEN);
    1426          481 :       do_compare_rtx_and_jump (op0, tem, LTU, true, mode, NULL_RTX, NULL,
    1427              :                                done_label, profile_probability::very_likely ());
    1428          481 :       goto do_error_label;
    1429              :     }
    1430              : 
    1431              :   /* u1 + u2 -> sr  */
    1432        17819 :   if (code == PLUS_EXPR && uns0_p && uns1_p && !unsr_p)
    1433              :     {
    1434              :       /* Compute the operation.  On RTL level, the addition is always
    1435              :          unsigned.  */
    1436         1192 :       res = expand_binop (mode, add_optab, op0, op1, NULL_RTX, false,
    1437              :                           OPTAB_LIB_WIDEN);
    1438         1192 :       do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
    1439              :                                NULL, do_error, profile_probability::very_unlikely ());
    1440         1192 :       rtx tem = op1;
    1441              :       /* The operation is commutative, so we can pick operand to compare
    1442              :          against.  For prec <= BITS_PER_WORD, I think preferring REG operand
    1443              :          is better over CONST_INT, because the CONST_INT might enlarge the
    1444              :          instruction or CSE would need to figure out we'd already loaded it
    1445              :          into a register before.  For prec > BITS_PER_WORD, I think CONST_INT
    1446              :          might be more beneficial, as then the multi-word comparison can be
    1447              :          perhaps simplified.  */
    1448         1192 :       if (prec <= BITS_PER_WORD
    1449         1192 :           ? (CONST_SCALAR_INT_P (op1) && REG_P (op0))
    1450          408 :           : CONST_SCALAR_INT_P (op0))
    1451         1192 :         tem = op0;
    1452         1192 :       do_compare_rtx_and_jump (res, tem, GEU, true, mode, NULL_RTX, NULL,
    1453              :                                done_label, profile_probability::very_likely ());
    1454         1192 :       goto do_error_label;
    1455              :     }
    1456              : 
    1457              :   /* s1 +- s2 -> ur  */
    1458        16627 :   if (!uns0_p && !uns1_p && unsr_p)
    1459              :     {
    1460              :       /* Compute the operation.  On RTL level, the addition is always
    1461              :          unsigned.  */
    1462         3825 :       res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
    1463              :                           op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
    1464         2589 :       int pos_neg = get_range_pos_neg (arg1, currently_expanding_gimple_stmt);
    1465         2589 :       if (code == PLUS_EXPR)
    1466              :         {
    1467         1353 :           int pos_neg0 = get_range_pos_neg (arg0,
    1468              :                                             currently_expanding_gimple_stmt);
    1469         1353 :           if (pos_neg0 != 3 && pos_neg == 3)
    1470              :             {
    1471              :               std::swap (op0, op1);
    1472              :               pos_neg = pos_neg0;
    1473              :             }
    1474              :         }
    1475         2589 :       rtx tem;
    1476         2589 :       if (pos_neg != 3)
    1477              :         {
    1478         1585 :           tem = expand_binop (mode, ((pos_neg == 1) ^ (code == MINUS_EXPR))
    1479              :                                     ? and_optab : ior_optab,
    1480              :                               op0, res, NULL_RTX, false, OPTAB_LIB_WIDEN);
    1481          935 :           do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL,
    1482              :                                    NULL, done_label, profile_probability::very_likely ());
    1483              :         }
    1484              :       else
    1485              :         {
    1486         1654 :           rtx_code_label *do_ior_label = gen_label_rtx ();
    1487         2357 :           do_compare_rtx_and_jump (op1, const0_rtx,
    1488              :                                    code == MINUS_EXPR ? GE : LT, false, mode,
    1489              :                                    NULL_RTX, NULL, do_ior_label,
    1490              :                                    profile_probability::even ());
    1491         1654 :           tem = expand_binop (mode, and_optab, op0, res, NULL_RTX, false,
    1492              :                               OPTAB_LIB_WIDEN);
    1493         1654 :           do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
    1494              :                                    NULL, done_label, profile_probability::very_likely ());
    1495         1654 :           emit_jump (do_error);
    1496         1654 :           emit_label (do_ior_label);
    1497         1654 :           tem = expand_binop (mode, ior_optab, op0, res, NULL_RTX, false,
    1498              :                               OPTAB_LIB_WIDEN);
    1499         1654 :           do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
    1500              :                                    NULL, done_label, profile_probability::very_likely ());
    1501              :         }
    1502         2589 :       goto do_error_label;
    1503              :     }
    1504              : 
    1505              :   /* u1 - u2 -> sr  */
    1506        14038 :   if (code == MINUS_EXPR && uns0_p && uns1_p && !unsr_p)
    1507              :     {
    1508              :       /* Compute the operation.  On RTL level, the addition is always
    1509              :          unsigned.  */
    1510         1729 :       res = expand_binop (mode, sub_optab, op0, op1, NULL_RTX, false,
    1511              :                           OPTAB_LIB_WIDEN);
    1512         1729 :       rtx_code_label *op0_geu_op1 = gen_label_rtx ();
    1513         1729 :       do_compare_rtx_and_jump (op0, op1, GEU, true, mode, NULL_RTX, NULL,
    1514              :                                op0_geu_op1, profile_probability::even ());
    1515         1729 :       do_compare_rtx_and_jump (res, const0_rtx, LT, false, mode, NULL_RTX,
    1516              :                                NULL, done_label, profile_probability::very_likely ());
    1517         1729 :       emit_jump (do_error);
    1518         1729 :       emit_label (op0_geu_op1);
    1519         1729 :       do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
    1520              :                                NULL, done_label, profile_probability::very_likely ());
    1521         1729 :       goto do_error_label;
    1522              :     }
    1523              : 
    1524        12309 :   gcc_assert (!uns0_p && !uns1_p && !unsr_p);
    1525              : 
    1526              :   /* s1 +- s2 -> sr  */
    1527        12309 :  do_signed:
    1528        14846 :   {
    1529        21520 :     insn_code icode = optab_handler (code == PLUS_EXPR ? addv4_optab
    1530              :                                      : subv4_optab, mode);
    1531        14846 :     if (icode != CODE_FOR_nothing)
    1532              :       {
    1533        14846 :         class expand_operand ops[4];
    1534        14846 :         rtx_insn *last = get_last_insn ();
    1535              : 
    1536        14846 :         res = gen_reg_rtx (mode);
    1537        14846 :         create_output_operand (&ops[0], res, mode);
    1538        14846 :         create_input_operand (&ops[1], op0, mode);
    1539        14846 :         create_input_operand (&ops[2], op1, mode);
    1540        14846 :         create_fixed_operand (&ops[3], do_error);
    1541        14846 :         if (maybe_expand_insn (icode, 4, ops))
    1542              :           {
    1543        14846 :             last = get_last_insn ();
    1544        14846 :             if (profile_status_for_fn (cfun) != PROFILE_ABSENT
    1545         9776 :                 && JUMP_P (last)
    1546         9776 :                 && any_condjump_p (last)
    1547        24622 :                 && !find_reg_note (last, REG_BR_PROB, 0))
    1548         9776 :               add_reg_br_prob_note (last,
    1549              :                                     profile_probability::very_unlikely ());
    1550        14846 :             emit_jump (done_label);
    1551        14846 :             goto do_error_label;
    1552              :           }
    1553              : 
    1554            0 :         delete_insns_since (last);
    1555              :       }
    1556              : 
    1557              :     /* Compute the operation.  On RTL level, the addition is always
    1558              :        unsigned.  */
    1559            0 :     res = expand_binop (mode, code == PLUS_EXPR ? add_optab : sub_optab,
    1560              :                         op0, op1, NULL_RTX, false, OPTAB_LIB_WIDEN);
    1561              : 
    1562              :     /* If we can prove that one of the arguments (for MINUS_EXPR only
    1563              :        the second operand, as subtraction is not commutative) is always
    1564              :        non-negative or always negative, we can do just one comparison
    1565              :        and conditional jump.  */
    1566            0 :     int pos_neg = get_range_pos_neg (arg1, currently_expanding_gimple_stmt);
    1567            0 :     if (code == PLUS_EXPR)
    1568              :       {
    1569            0 :         int pos_neg0 = get_range_pos_neg (arg0,
    1570              :                                           currently_expanding_gimple_stmt);
    1571            0 :         if (pos_neg0 != 3 && pos_neg == 3)
    1572              :           {
    1573              :             std::swap (op0, op1);
    1574              :             pos_neg = pos_neg0;
    1575              :           }
    1576              :       }
    1577              : 
    1578              :     /* Addition overflows if and only if the two operands have the same sign,
    1579              :        and the result has the opposite sign.  Subtraction overflows if and
    1580              :        only if the two operands have opposite sign, and the subtrahend has
    1581              :        the same sign as the result.  Here 0 is counted as positive.  */
    1582            0 :     if (pos_neg == 3)
    1583              :       {
    1584              :         /* Compute op0 ^ op1 (operands have opposite sign).  */
    1585            0 :         rtx op_xor = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
    1586              :                                    OPTAB_LIB_WIDEN);
    1587              : 
    1588              :         /* Compute res ^ op1 (result and 2nd operand have opposite sign).  */
    1589            0 :         rtx res_xor = expand_binop (mode, xor_optab, res, op1, NULL_RTX, false,
    1590              :                                     OPTAB_LIB_WIDEN);
    1591              : 
    1592            0 :         rtx tem;
    1593            0 :         if (code == PLUS_EXPR)
    1594              :           {
    1595              :             /* Compute (res ^ op1) & ~(op0 ^ op1).  */
    1596            0 :             tem = expand_unop (mode, one_cmpl_optab, op_xor, NULL_RTX, false);
    1597            0 :             tem = expand_binop (mode, and_optab, res_xor, tem, NULL_RTX, false,
    1598              :                                 OPTAB_LIB_WIDEN);
    1599              :           }
    1600              :         else
    1601              :           {
    1602              :             /* Compute (op0 ^ op1) & ~(res ^ op1).  */
    1603            0 :             tem = expand_unop (mode, one_cmpl_optab, res_xor, NULL_RTX, false);
    1604            0 :             tem = expand_binop (mode, and_optab, op_xor, tem, NULL_RTX, false,
    1605              :                                 OPTAB_LIB_WIDEN);
    1606              :           }
    1607              : 
    1608              :         /* No overflow if the result has bit sign cleared.  */
    1609            0 :         do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
    1610              :                                  NULL, done_label, profile_probability::very_likely ());
    1611              :       }
    1612              : 
    1613              :     /* Compare the result of the operation with the first operand.
    1614              :        No overflow for addition if second operand is positive and result
    1615              :        is larger or second operand is negative and result is smaller.
    1616              :        Likewise for subtraction with sign of second operand flipped.  */
    1617              :     else
    1618            0 :       do_compare_rtx_and_jump (res, op0,
    1619            0 :                                (pos_neg == 1) ^ (code == MINUS_EXPR) ? GE : LE,
    1620              :                                false, mode, NULL_RTX, NULL, done_label,
    1621              :                                profile_probability::very_likely ());
    1622              :   }
    1623              : 
    1624        39263 :  do_error_label:
    1625        39263 :   emit_label (do_error);
    1626        39263 :   if (is_ubsan)
    1627              :     {
    1628              :       /* Expand the ubsan builtin call.  */
    1629         3391 :       push_temp_slots ();
    1630         3391 :       fn = ubsan_build_overflow_builtin (code, loc, TREE_TYPE (arg0),
    1631              :                                          arg0, arg1, datap);
    1632         3391 :       expand_normal (fn);
    1633         3391 :       pop_temp_slots ();
    1634         3391 :       do_pending_stack_adjust ();
    1635              :     }
    1636        35872 :   else if (lhs)
    1637        35872 :     expand_arith_set_overflow (lhs, target);
    1638              : 
    1639              :   /* We're done.  */
    1640        39263 :   emit_label (done_label);
    1641              : 
    1642        39263 :   if (lhs)
    1643              :     {
    1644        38735 :       if (is_ubsan)
    1645         2863 :         expand_ubsan_result_store (lhs, target, mode, res, do_error);
    1646              :       else
    1647              :         {
    1648        35872 :           if (do_xor)
    1649         2537 :             res = expand_binop (mode, add_optab, res, sgn, NULL_RTX, false,
    1650              :                                 OPTAB_LIB_WIDEN);
    1651              : 
    1652        35872 :           expand_arith_overflow_result_store (lhs, target, mode, res, false);
    1653              :         }
    1654              :     }
    1655        39263 : }
    1656              : 
    1657              : /* Add negate overflow checking to the statement STMT.  */
    1658              : 
    1659              : static void
    1660         1052 : expand_neg_overflow (location_t loc, tree lhs, tree arg1, bool is_ubsan,
    1661              :                      tree *datap)
    1662              : {
    1663         1052 :   rtx res, op1;
    1664         1052 :   tree fn;
    1665         1052 :   rtx_code_label *done_label, *do_error;
    1666         1052 :   rtx target = NULL_RTX;
    1667              : 
    1668         1052 :   done_label = gen_label_rtx ();
    1669         1052 :   do_error = gen_label_rtx ();
    1670              : 
    1671         1052 :   do_pending_stack_adjust ();
    1672         1052 :   op1 = expand_normal (arg1);
    1673              : 
    1674         1052 :   scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg1));
    1675         1052 :   if (lhs)
    1676              :     {
    1677          848 :       target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    1678          848 :       if (!is_ubsan)
    1679          402 :         write_complex_part (target, const0_rtx, true, false);
    1680              :     }
    1681              : 
    1682         1052 :   enum insn_code icode = optab_handler (negv3_optab, mode);
    1683         1052 :   if (icode != CODE_FOR_nothing)
    1684              :     {
    1685          974 :       class expand_operand ops[3];
    1686          974 :       rtx_insn *last = get_last_insn ();
    1687              : 
    1688          974 :       res = gen_reg_rtx (mode);
    1689          974 :       create_output_operand (&ops[0], res, mode);
    1690          974 :       create_input_operand (&ops[1], op1, mode);
    1691          974 :       create_fixed_operand (&ops[2], do_error);
    1692          974 :       if (maybe_expand_insn (icode, 3, ops))
    1693              :         {
    1694          974 :           last = get_last_insn ();
    1695          974 :           if (profile_status_for_fn (cfun) != PROFILE_ABSENT
    1696          685 :               && JUMP_P (last)
    1697          685 :               && any_condjump_p (last)
    1698         1659 :               && !find_reg_note (last, REG_BR_PROB, 0))
    1699          685 :             add_reg_br_prob_note (last,
    1700              :                                   profile_probability::very_unlikely ());
    1701          974 :           emit_jump (done_label);
    1702              :         }
    1703              :       else
    1704              :         {
    1705            0 :           delete_insns_since (last);
    1706            0 :           icode = CODE_FOR_nothing;
    1707              :         }
    1708              :     }
    1709              : 
    1710          974 :   if (icode == CODE_FOR_nothing)
    1711              :     {
    1712              :       /* Compute the operation.  On RTL level, the addition is always
    1713              :          unsigned.  */
    1714           78 :       res = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
    1715              : 
    1716              :       /* Compare the operand with the most negative value.  */
    1717           78 :       rtx minv = expand_normal (TYPE_MIN_VALUE (TREE_TYPE (arg1)));
    1718           78 :       do_compare_rtx_and_jump (op1, minv, NE, true, mode, NULL_RTX, NULL,
    1719              :                                done_label, profile_probability::very_likely ());
    1720              :     }
    1721              : 
    1722         1052 :   emit_label (do_error);
    1723         1052 :   if (is_ubsan)
    1724              :     {
    1725              :       /* Expand the ubsan builtin call.  */
    1726          650 :       push_temp_slots ();
    1727          650 :       fn = ubsan_build_overflow_builtin (NEGATE_EXPR, loc, TREE_TYPE (arg1),
    1728              :                                          arg1, NULL_TREE, datap);
    1729          650 :       expand_normal (fn);
    1730          650 :       pop_temp_slots ();
    1731          650 :       do_pending_stack_adjust ();
    1732              :     }
    1733          402 :   else if (lhs)
    1734          402 :     expand_arith_set_overflow (lhs, target);
    1735              : 
    1736              :   /* We're done.  */
    1737         1052 :   emit_label (done_label);
    1738              : 
    1739         1052 :   if (lhs)
    1740              :     {
    1741          848 :       if (is_ubsan)
    1742          446 :         expand_ubsan_result_store (lhs, target, mode, res, do_error);
    1743              :       else
    1744          402 :         expand_arith_overflow_result_store (lhs, target, mode, res, false);
    1745              :     }
    1746         1052 : }
    1747              : 
    1748              : /* Return true if UNS WIDEN_MULT_EXPR with result mode WMODE and operand
    1749              :    mode MODE can be expanded without using a libcall.  */
    1750              : 
    1751              : static bool
    1752            0 : can_widen_mult_without_libcall (scalar_int_mode wmode, scalar_int_mode mode,
    1753              :                                 rtx op0, rtx op1, bool uns)
    1754              : {
    1755            0 :   if (find_widening_optab_handler (umul_widen_optab, wmode, mode)
    1756              :       != CODE_FOR_nothing)
    1757              :     return true;
    1758              : 
    1759            0 :   if (find_widening_optab_handler (smul_widen_optab, wmode, mode)
    1760              :       != CODE_FOR_nothing)
    1761              :     return true;
    1762              : 
    1763            0 :   rtx_insn *last = get_last_insn ();
    1764            0 :   if (CONSTANT_P (op0))
    1765            0 :     op0 = convert_modes (wmode, mode, op0, uns);
    1766              :   else
    1767            0 :     op0 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 1);
    1768            0 :   if (CONSTANT_P (op1))
    1769            0 :     op1 = convert_modes (wmode, mode, op1, uns);
    1770              :   else
    1771            0 :     op1 = gen_raw_REG (wmode, LAST_VIRTUAL_REGISTER + 2);
    1772            0 :   rtx ret = expand_mult (wmode, op0, op1, NULL_RTX, uns, true);
    1773            0 :   delete_insns_since (last);
    1774            0 :   return ret != NULL_RTX;
    1775              : }
    1776              : 
    1777              : /* Add mul overflow checking to the statement STMT.  */
    1778              : 
    1779              : static void
    1780        18959 : expand_mul_overflow (location_t loc, tree lhs, tree arg0, tree arg1,
    1781              :                      bool unsr_p, bool uns0_p, bool uns1_p, bool is_ubsan,
    1782              :                      tree *datap)
    1783              : {
    1784        18959 :   rtx res, op0, op1;
    1785        18959 :   tree fn, type, orig_arg0 = arg0, orig_arg1 = arg1;
    1786        18959 :   rtx_code_label *done_label, *do_error;
    1787        18959 :   rtx target = NULL_RTX;
    1788        18959 :   signop sign;
    1789        18959 :   enum insn_code icode;
    1790        18959 :   int save_flag_trapv = flag_trapv;
    1791              : 
    1792              :   /* We don't want any __mulv?i3 etc. calls from the expansion of
    1793              :      these internal functions, so disable -ftrapv temporarily.  */
    1794        18959 :   flag_trapv = 0;
    1795        18959 :   done_label = gen_label_rtx ();
    1796        18959 :   do_error = gen_label_rtx ();
    1797              : 
    1798        18959 :   do_pending_stack_adjust ();
    1799              : 
    1800        18959 :   scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
    1801              :   /* If the operand types don't have mode precision, extend them
    1802              :      to mode precision.  */
    1803        18959 :   if (TYPE_PRECISION (TREE_TYPE (arg0)) < GET_MODE_PRECISION (mode))
    1804              :     {
    1805          627 :       tree type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
    1806              :                                                   uns0_p);
    1807          627 :       arg0 = fold_convert_loc (loc, type, arg0);
    1808              :     }
    1809        18959 :   if (TYPE_PRECISION (TREE_TYPE (arg1)) < GET_MODE_PRECISION (mode))
    1810              :     {
    1811          627 :       tree type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
    1812              :                                                   uns1_p);
    1813          627 :       arg1 = fold_convert_loc (loc, type, arg1);
    1814              :     }
    1815              : 
    1816        18959 :   op0 = expand_normal (arg0);
    1817        18959 :   op1 = expand_normal (arg1);
    1818              : 
    1819        18959 :   bool uns = unsr_p;
    1820        18959 :   bool overflow_checked = false;
    1821        18959 :   if (lhs)
    1822              :     {
    1823        18815 :       target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    1824        18815 :       if (!is_ubsan)
    1825        17449 :         write_complex_part (target, const0_rtx, true, false);
    1826              :     }
    1827              : 
    1828        17593 :   if (is_ubsan)
    1829         1510 :     gcc_assert (!unsr_p && !uns0_p && !uns1_p);
    1830              : 
    1831              :   /* We assume both operands and result have the same precision
    1832              :      here (GET_MODE_BITSIZE (mode)), S stands for signed type
    1833              :      with that precision, U for unsigned type with that precision,
    1834              :      sgn for unsigned most significant bit in that precision.
    1835              :      s1 is signed first operand, u1 is unsigned first operand,
    1836              :      s2 is signed second operand, u2 is unsigned second operand,
    1837              :      sr is signed result, ur is unsigned result and the following
    1838              :      rules say how to compute result (which is always result of
    1839              :      the operands as if both were unsigned, cast to the right
    1840              :      signedness) and how to compute whether operation overflowed.
    1841              :      main_ovf (false) stands for jump on signed multiplication
    1842              :      overflow or the main algorithm with uns == false.
    1843              :      main_ovf (true) stands for jump on unsigned multiplication
    1844              :      overflow or the main algorithm with uns == true.
    1845              : 
    1846              :      s1 * s2 -> sr
    1847              :         res = (S) ((U) s1 * (U) s2)
    1848              :         ovf = main_ovf (false)
    1849              :      u1 * u2 -> ur
    1850              :         res = u1 * u2
    1851              :         ovf = main_ovf (true)
    1852              :      s1 * u2 -> ur
    1853              :         res = (U) s1 * u2
    1854              :         ovf = (s1 < 0 && u2) || main_ovf (true)
    1855              :      u1 * u2 -> sr
    1856              :         res = (S) (u1 * u2)
    1857              :         ovf = res < 0 || main_ovf (true)
    1858              :      s1 * u2 -> sr
    1859              :         res = (S) ((U) s1 * u2)
    1860              :         ovf = (S) u2 >= 0 ? main_ovf (false)
    1861              :                           : (s1 != 0 && (s1 != -1 || u2 != (U) res))
    1862              :      s1 * s2 -> ur
    1863              :         t1 = (s1 & s2) < 0 ? (-(U) s1) : ((U) s1)
    1864              :         t2 = (s1 & s2) < 0 ? (-(U) s2) : ((U) s2)
    1865              :         res = t1 * t2
    1866              :         ovf = (s1 ^ s2) < 0 ? (s1 && s2) : main_ovf (true)  */
    1867              : 
    1868        18959 :   if (uns0_p && !uns1_p)
    1869              :     {
    1870              :       /* Multiplication is commutative, if operand signedness differs,
    1871              :          canonicalize to the first operand being signed and second
    1872              :          unsigned to simplify following code.  */
    1873         1558 :       std::swap (op0, op1);
    1874         1558 :       std::swap (arg0, arg1);
    1875         1558 :       uns0_p = false;
    1876         1558 :       uns1_p = true;
    1877              :     }
    1878              : 
    1879        18959 :   int pos_neg0 = get_range_pos_neg (arg0, currently_expanding_gimple_stmt);
    1880        18959 :   int pos_neg1 = get_range_pos_neg (arg1, currently_expanding_gimple_stmt);
    1881              : 
    1882              :   /* s1 * u2 -> ur  */
    1883        18959 :   if (!uns0_p && uns1_p && unsr_p)
    1884              :     {
    1885         2254 :       switch (pos_neg0)
    1886              :         {
    1887            0 :         case 1:
    1888              :           /* If s1 is non-negative, just perform normal u1 * u2 -> ur.  */
    1889         1821 :           goto do_main;
    1890          328 :         case 2:
    1891              :           /* If s1 is negative, avoid the main code, just multiply and
    1892              :              signal overflow if op1 is not 0.  */
    1893          328 :           struct separate_ops ops;
    1894          328 :           ops.code = MULT_EXPR;
    1895          328 :           ops.type = TREE_TYPE (arg1);
    1896          328 :           ops.op0 = make_tree (ops.type, op0);
    1897          328 :           ops.op1 = make_tree (ops.type, op1);
    1898          328 :           ops.op2 = NULL_TREE;
    1899          328 :           ops.location = loc;
    1900          328 :           res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    1901          328 :           do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
    1902              :                                    NULL, done_label, profile_probability::very_likely ());
    1903          433 :           goto do_error_label;
    1904         1926 :         case 3:
    1905         3852 :           if (get_min_precision (arg1, UNSIGNED)
    1906         1926 :               + get_min_precision (arg0, SIGNED) <= GET_MODE_PRECISION (mode))
    1907              :             {
    1908              :               /* If the first operand is sign extended from narrower type, the
    1909              :                  second operand is zero extended from narrower type and
    1910              :                  the sum of the two precisions is smaller or equal to the
    1911              :                  result precision: if the first argument is at runtime
    1912              :                  non-negative, maximum result will be 0x7e81 or 0x7f..fe80..01
    1913              :                  and there will be no overflow, if the first argument is
    1914              :                  negative and the second argument zero, the result will be
    1915              :                  0 and there will be no overflow, if the first argument is
    1916              :                  negative and the second argument positive, the result when
    1917              :                  treated as signed will be negative (minimum -0x7f80 or
    1918              :                  -0x7f..f80..0) there will be always overflow.  So, do
    1919              :                  res = (U) (s1 * u2)
    1920              :                  ovf = (S) res < 0  */
    1921          105 :               struct separate_ops ops;
    1922          105 :               ops.code = MULT_EXPR;
    1923          105 :               ops.type
    1924          105 :                 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
    1925              :                                                   1);
    1926          105 :               ops.op0 = make_tree (ops.type, op0);
    1927          105 :               ops.op1 = make_tree (ops.type, op1);
    1928          105 :               ops.op2 = NULL_TREE;
    1929          105 :               ops.location = loc;
    1930          105 :               res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    1931          105 :               do_compare_rtx_and_jump (res, const0_rtx, GE, false,
    1932              :                                        mode, NULL_RTX, NULL, done_label,
    1933              :                                        profile_probability::very_likely ());
    1934          105 :               goto do_error_label;
    1935              :             }
    1936         1821 :           rtx_code_label *do_main_label;
    1937         1821 :           do_main_label = gen_label_rtx ();
    1938         1821 :           do_compare_rtx_and_jump (op0, const0_rtx, GE, false, mode, NULL_RTX,
    1939              :                                    NULL, do_main_label, profile_probability::very_likely ());
    1940         1821 :           do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
    1941              :                                    NULL, do_main_label, profile_probability::very_likely ());
    1942         1821 :           expand_arith_set_overflow (lhs, target);
    1943         1821 :           emit_label (do_main_label);
    1944         1821 :           goto do_main;
    1945            0 :         default:
    1946            0 :           gcc_unreachable ();
    1947              :         }
    1948              :     }
    1949              : 
    1950              :   /* u1 * u2 -> sr  */
    1951        16705 :   if (uns0_p && uns1_p && !unsr_p)
    1952              :     {
    1953         1369 :       if ((pos_neg0 | pos_neg1) == 1)
    1954              :         {
    1955              :           /* If both arguments are zero extended from narrower types,
    1956              :              the MSB will be clear on both and so we can pretend it is
    1957              :              a normal s1 * s2 -> sr multiplication.  */
    1958              :           uns0_p = false;
    1959              :           uns1_p = false;
    1960              :         }
    1961              :       else
    1962          957 :         uns = true;
    1963              :       /* Rest of handling of this case after res is computed.  */
    1964         1369 :       goto do_main;
    1965              :     }
    1966              : 
    1967              :   /* s1 * u2 -> sr  */
    1968        15336 :   if (!uns0_p && uns1_p && !unsr_p)
    1969              :     {
    1970         1788 :       switch (pos_neg1)
    1971              :         {
    1972           66 :         case 1:
    1973         1339 :           goto do_main;
    1974          449 :         case 2:
    1975              :           /* If (S) u2 is negative (i.e. u2 is larger than maximum of S,
    1976              :              avoid the main code, just multiply and signal overflow
    1977              :              unless 0 * u2 or -1 * ((U) Smin).  */
    1978          449 :           struct separate_ops ops;
    1979          449 :           ops.code = MULT_EXPR;
    1980          449 :           ops.type = TREE_TYPE (arg1);
    1981          449 :           ops.op0 = make_tree (ops.type, op0);
    1982          449 :           ops.op1 = make_tree (ops.type, op1);
    1983          449 :           ops.op2 = NULL_TREE;
    1984          449 :           ops.location = loc;
    1985          449 :           res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    1986          449 :           do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
    1987              :                                    NULL, done_label, profile_probability::very_likely ());
    1988          449 :           do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
    1989              :                                    NULL, do_error, profile_probability::very_unlikely ());
    1990          449 :           int prec;
    1991          449 :           prec = GET_MODE_PRECISION (mode);
    1992          449 :           rtx sgn;
    1993          449 :           sgn = immed_wide_int_const (wi::min_value (prec, SIGNED), mode);
    1994          449 :           do_compare_rtx_and_jump (op1, sgn, EQ, true, mode, NULL_RTX,
    1995              :                                    NULL, done_label, profile_probability::very_likely ());
    1996          449 :           goto do_error_label;
    1997         1273 :         case 3:
    1998              :           /* Rest of handling of this case after res is computed.  */
    1999         1273 :           goto do_main;
    2000            0 :         default:
    2001            0 :           gcc_unreachable ();
    2002              :         }
    2003              :     }
    2004              : 
    2005              :   /* s1 * s2 -> ur  */
    2006        13548 :   if (!uns0_p && !uns1_p && unsr_p)
    2007              :     {
    2008         2329 :       rtx tem;
    2009         2329 :       switch (pos_neg0 | pos_neg1)
    2010              :         {
    2011            0 :         case 1: /* Both operands known to be non-negative.  */
    2012            0 :           goto do_main;
    2013           12 :         case 2: /* Both operands known to be negative.  */
    2014           12 :           op0 = expand_unop (mode, neg_optab, op0, NULL_RTX, false);
    2015           12 :           op1 = expand_unop (mode, neg_optab, op1, NULL_RTX, false);
    2016              :           /* Avoid looking at arg0/arg1 ranges, as we've changed
    2017              :              the arguments.  */
    2018           12 :           arg0 = error_mark_node;
    2019           12 :           arg1 = error_mark_node;
    2020           12 :           goto do_main;
    2021         2317 :         case 3:
    2022         2317 :           if ((pos_neg0 ^ pos_neg1) == 3)
    2023              :             {
    2024              :               /* If one operand is known to be negative and the other
    2025              :                  non-negative, this overflows always, unless the non-negative
    2026              :                  one is 0.  Just do normal multiply and set overflow
    2027              :                  unless one of the operands is 0.  */
    2028            0 :               struct separate_ops ops;
    2029            0 :               ops.code = MULT_EXPR;
    2030            0 :               ops.type
    2031            0 :                 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
    2032              :                                                   1);
    2033            0 :               ops.op0 = make_tree (ops.type, op0);
    2034            0 :               ops.op1 = make_tree (ops.type, op1);
    2035            0 :               ops.op2 = NULL_TREE;
    2036            0 :               ops.location = loc;
    2037            0 :               res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2038            0 :               do_compare_rtx_and_jump (pos_neg0 == 1 ? op0 : op1, const0_rtx, EQ,
    2039              :                                        true, mode, NULL_RTX, NULL, done_label,
    2040              :                                        profile_probability::very_likely ());
    2041            0 :               goto do_error_label;
    2042              :             }
    2043         4634 :           if (get_min_precision (arg0, SIGNED)
    2044         2317 :               + get_min_precision (arg1, SIGNED) <= GET_MODE_PRECISION (mode))
    2045              :             {
    2046              :               /* If both operands are sign extended from narrower types and
    2047              :                  the sum of the two precisions is smaller or equal to the
    2048              :                  result precision: if both arguments are at runtime
    2049              :                  non-negative, maximum result will be 0x3f01 or 0x3f..f0..01
    2050              :                  and there will be no overflow, if both arguments are negative,
    2051              :                  maximum result will be 0x40..00 and there will be no overflow
    2052              :                  either, if one argument is positive and the other argument
    2053              :                  negative, the result when treated as signed will be negative
    2054              :                  and there will be always overflow, and if one argument is
    2055              :                  zero and the other negative the result will be zero and no
    2056              :                  overflow.  So, do
    2057              :                  res = (U) (s1 * s2)
    2058              :                  ovf = (S) res < 0  */
    2059          220 :               struct separate_ops ops;
    2060          220 :               ops.code = MULT_EXPR;
    2061          220 :               ops.type
    2062          220 :                 = build_nonstandard_integer_type (GET_MODE_PRECISION (mode),
    2063              :                                                   1);
    2064          220 :               ops.op0 = make_tree (ops.type, op0);
    2065          220 :               ops.op1 = make_tree (ops.type, op1);
    2066          220 :               ops.op2 = NULL_TREE;
    2067          220 :               ops.location = loc;
    2068          220 :               res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2069          220 :               do_compare_rtx_and_jump (res, const0_rtx, GE, false,
    2070              :                                        mode, NULL_RTX, NULL, done_label,
    2071              :                                        profile_probability::very_likely ());
    2072          220 :               goto do_error_label;
    2073              :             }
    2074              :           /* The general case, do all the needed comparisons at runtime.  */
    2075         2097 :           rtx_code_label *do_main_label, *after_negate_label;
    2076         2097 :           rtx rop0, rop1;
    2077         2097 :           rop0 = gen_reg_rtx (mode);
    2078         2097 :           rop1 = gen_reg_rtx (mode);
    2079         2097 :           emit_move_insn (rop0, op0);
    2080         2097 :           emit_move_insn (rop1, op1);
    2081         2097 :           op0 = rop0;
    2082         2097 :           op1 = rop1;
    2083         2097 :           do_main_label = gen_label_rtx ();
    2084         2097 :           after_negate_label = gen_label_rtx ();
    2085         2097 :           tem = expand_binop (mode, and_optab, op0, op1, NULL_RTX, false,
    2086              :                               OPTAB_LIB_WIDEN);
    2087         2097 :           do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
    2088              :                                    NULL, after_negate_label, profile_probability::very_likely ());
    2089              :           /* Both arguments negative here, negate them and continue with
    2090              :              normal unsigned overflow checking multiplication.  */
    2091         2097 :           emit_move_insn (op0, expand_unop (mode, neg_optab, op0,
    2092              :                                             NULL_RTX, false));
    2093         2097 :           emit_move_insn (op1, expand_unop (mode, neg_optab, op1,
    2094              :                                             NULL_RTX, false));
    2095              :           /* Avoid looking at arg0/arg1 ranges, as we might have changed
    2096              :              the arguments.  */
    2097         2097 :           arg0 = error_mark_node;
    2098         2097 :           arg1 = error_mark_node;
    2099         2097 :           emit_jump (do_main_label);
    2100         2097 :           emit_label (after_negate_label);
    2101         2097 :           tem = expand_binop (mode, xor_optab, op0, op1, NULL_RTX, false,
    2102              :                               OPTAB_LIB_WIDEN);
    2103         2097 :           do_compare_rtx_and_jump (tem, const0_rtx, GE, false, mode, NULL_RTX,
    2104              :                                    NULL, do_main_label,
    2105              :                                    profile_probability::very_likely ());
    2106              :           /* One argument is negative here, the other positive.  This
    2107              :              overflows always, unless one of the arguments is 0.  But
    2108              :              if e.g. s2 is 0, (U) s1 * 0 doesn't overflow, whatever s1
    2109              :              is, thus we can keep do_main code oring in overflow as is.  */
    2110         2097 :           if (pos_neg0 != 2)
    2111         2097 :             do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
    2112              :                                      NULL, do_main_label,
    2113              :                                      profile_probability::very_unlikely ());
    2114         2097 :           if (pos_neg1 != 2)
    2115         1080 :             do_compare_rtx_and_jump (op1, const0_rtx, EQ, true, mode, NULL_RTX,
    2116              :                                      NULL, do_main_label,
    2117              :                                      profile_probability::very_unlikely ());
    2118         2097 :           expand_arith_set_overflow (lhs, target);
    2119         2097 :           emit_label (do_main_label);
    2120         2097 :           goto do_main;
    2121            0 :         default:
    2122            0 :           gcc_unreachable ();
    2123              :         }
    2124              :     }
    2125              : 
    2126        11219 :  do_main:
    2127        17857 :   type = build_nonstandard_integer_type (GET_MODE_PRECISION (mode), uns);
    2128        17857 :   sign = uns ? UNSIGNED : SIGNED;
    2129        17857 :   icode = optab_handler (uns ? umulv4_optab : mulv4_optab, mode);
    2130        17857 :   if (uns
    2131        10496 :       && (integer_pow2p (arg0) || integer_pow2p (arg1))
    2132        18729 :       && (optimize_insn_for_speed_p () || icode == CODE_FOR_nothing))
    2133              :     {
    2134              :       /* Optimize unsigned multiplication by power of 2 constant
    2135              :          using 2 shifts, one for result, one to extract the shifted
    2136              :          out bits to see if they are all zero.
    2137              :          Don't do this if optimizing for size and we have umulv4_optab,
    2138              :          in that case assume multiplication will be shorter.
    2139              :          This is heuristics based on the single target that provides
    2140              :          umulv4 right now (i?86/x86_64), if further targets add it, this
    2141              :          might need to be revisited.
    2142              :          Cases where both operands are constant should be folded already
    2143              :          during GIMPLE, and cases where one operand is constant but not
    2144              :          power of 2 are questionable, either the WIDEN_MULT_EXPR case
    2145              :          below can be done without multiplication, just by shifts and adds,
    2146              :          or we'd need to divide the result (and hope it actually doesn't
    2147              :          really divide nor multiply) and compare the result of the division
    2148              :          with the original operand.  */
    2149          870 :       rtx opn0 = op0;
    2150          870 :       rtx opn1 = op1;
    2151          870 :       tree argn0 = arg0;
    2152          870 :       tree argn1 = arg1;
    2153          870 :       if (integer_pow2p (arg0))
    2154              :         {
    2155            0 :           std::swap (opn0, opn1);
    2156            0 :           std::swap (argn0, argn1);
    2157              :         }
    2158          870 :       int cnt = tree_log2 (argn1);
    2159          870 :       if (cnt >= 0 && cnt < GET_MODE_PRECISION (mode))
    2160              :         {
    2161          870 :           rtx upper = const0_rtx;
    2162          870 :           res = expand_shift (LSHIFT_EXPR, mode, opn0, cnt, NULL_RTX, uns);
    2163          870 :           if (cnt != 0)
    2164          698 :             upper = expand_shift (RSHIFT_EXPR, mode, opn0,
    2165          698 :                                   GET_MODE_PRECISION (mode) - cnt,
    2166              :                                   NULL_RTX, uns);
    2167          870 :           do_compare_rtx_and_jump (upper, const0_rtx, EQ, true, mode,
    2168              :                                    NULL_RTX, NULL, done_label,
    2169              :                                    profile_probability::very_likely ());
    2170          870 :           goto do_error_label;
    2171              :         }
    2172              :     }
    2173        16987 :   if (icode != CODE_FOR_nothing)
    2174              :     {
    2175        13888 :       class expand_operand ops[4];
    2176        13888 :       rtx_insn *last = get_last_insn ();
    2177              : 
    2178        13888 :       res = gen_reg_rtx (mode);
    2179        13888 :       create_output_operand (&ops[0], res, mode);
    2180        13888 :       create_input_operand (&ops[1], op0, mode);
    2181        13888 :       create_input_operand (&ops[2], op1, mode);
    2182        13888 :       create_fixed_operand (&ops[3], do_error);
    2183        13888 :       if (maybe_expand_insn (icode, 4, ops))
    2184              :         {
    2185        13888 :           last = get_last_insn ();
    2186        13888 :           if (profile_status_for_fn (cfun) != PROFILE_ABSENT
    2187         9202 :               && JUMP_P (last)
    2188         9202 :               && any_condjump_p (last)
    2189        23090 :               && !find_reg_note (last, REG_BR_PROB, 0))
    2190         9202 :             add_reg_br_prob_note (last,
    2191              :                                   profile_probability::very_unlikely ());
    2192        13888 :           emit_jump (done_label);
    2193              :         }
    2194              :       else
    2195              :         {
    2196            0 :           delete_insns_since (last);
    2197            0 :           icode = CODE_FOR_nothing;
    2198              :         }
    2199              :     }
    2200              : 
    2201            0 :   if (icode == CODE_FOR_nothing)
    2202              :     {
    2203         3099 :       struct separate_ops ops;
    2204         3099 :       int prec = GET_MODE_PRECISION (mode);
    2205         3099 :       scalar_int_mode hmode, wmode;
    2206         3099 :       ops.op0 = make_tree (type, op0);
    2207         3099 :       ops.op1 = make_tree (type, op1);
    2208         3099 :       ops.op2 = NULL_TREE;
    2209         3099 :       ops.location = loc;
    2210              : 
    2211              :       /* Optimize unsigned overflow check where we don't use the
    2212              :          multiplication result, just whether overflow happened.
    2213              :          If we can do MULT_HIGHPART_EXPR, that followed by
    2214              :          comparison of the result against zero is cheapest.
    2215              :          We'll still compute res, but it should be DCEd later.  */
    2216         3099 :       use_operand_p use;
    2217         3099 :       gimple *use_stmt;
    2218         3099 :       if (!is_ubsan
    2219         3099 :           && lhs
    2220         3079 :           && uns
    2221         1973 :           && !(uns0_p && uns1_p && !unsr_p)
    2222         1692 :           && can_mult_highpart_p (mode, uns) == 1
    2223            0 :           && single_imm_use (lhs, &use, &use_stmt)
    2224            0 :           && is_gimple_assign (use_stmt)
    2225         3099 :           && gimple_assign_rhs_code (use_stmt) == IMAGPART_EXPR)
    2226            0 :         goto highpart;
    2227              : 
    2228         3099 :       if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
    2229         3099 :           && targetm.scalar_mode_supported_p (wmode)
    2230            0 :           && can_widen_mult_without_libcall (wmode, mode, op0, op1, uns))
    2231              :         {
    2232            0 :         twoxwider:
    2233            0 :           int ovf_prec = prec;
    2234            0 :           if (!is_ubsan && lhs && unsr_p)
    2235            0 :             ovf_prec = TYPE_PRECISION (TREE_TYPE (TREE_TYPE (lhs)));
    2236            0 :           ops.code = WIDEN_MULT_EXPR;
    2237            0 :           ops.type
    2238            0 :             = build_nonstandard_integer_type (GET_MODE_PRECISION (wmode), uns);
    2239              : 
    2240            0 :           res = expand_expr_real_2 (&ops, NULL_RTX, wmode, EXPAND_NORMAL);
    2241            0 :           rtx hipart = expand_shift (RSHIFT_EXPR, wmode, res, ovf_prec,
    2242              :                                      NULL_RTX, uns);
    2243            0 :           if (uns)
    2244              :             {
    2245              :               /* For the unsigned multiplication, there was overflow if
    2246              :                  HIPART is non-zero. Convert RES before the branch so that
    2247              :                  both the no-overflow fall-through and the overflow path see
    2248              :                  RES in MODE.  */
    2249            0 :               res = convert_modes (mode, wmode, res, uns);
    2250            0 :               if (ovf_prec < prec)
    2251              :                 {
    2252            0 :                   overflow_checked = true;
    2253            0 :                   do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, wmode,
    2254              :                                            NULL_RTX, NULL, done_label,
    2255              :                                            profile_probability::very_likely ());
    2256              :                 }
    2257              :               else
    2258              :                 {
    2259            0 :                   hipart = convert_modes (mode, wmode, hipart, uns);
    2260            0 :                   do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
    2261              :                                            NULL_RTX, NULL, done_label,
    2262              :                                            profile_probability::very_likely ());
    2263              :                 }
    2264              :             }
    2265              :           else
    2266              :             {
    2267            0 :               hipart = convert_modes (mode, wmode, hipart, uns);
    2268            0 :               res = convert_modes (mode, wmode, res, uns);
    2269              :               /* RES is used more than once, place it in a pseudo.  */
    2270            0 :               res = force_reg (mode, res);
    2271              : 
    2272            0 :               rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
    2273              :                                           NULL_RTX, 0);
    2274              :               /* RES is low half of the double width result, HIPART
    2275              :                  the high half.  There was overflow if
    2276              :                  HIPART is different from RES < 0 ? -1 : 0.  */
    2277            0 :               do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
    2278              :                                        NULL_RTX, NULL, done_label,
    2279              :                                        profile_probability::very_likely ());
    2280              :             }
    2281              :         }
    2282         3099 :       else if (can_mult_highpart_p (mode, uns) == 1)
    2283              :         {
    2284            0 :         highpart:
    2285            0 :           ops.code = MULT_HIGHPART_EXPR;
    2286            0 :           ops.type = type;
    2287              : 
    2288            0 :           rtx hipart = expand_expr_real_2 (&ops, NULL_RTX, mode,
    2289              :                                            EXPAND_NORMAL);
    2290            0 :           ops.code = MULT_EXPR;
    2291            0 :           res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2292            0 :           if (uns)
    2293              :             /* For the unsigned multiplication, there was overflow if
    2294              :                HIPART is non-zero.  */
    2295            0 :             do_compare_rtx_and_jump (hipart, const0_rtx, EQ, true, mode,
    2296              :                                      NULL_RTX, NULL, done_label,
    2297              :                                      profile_probability::very_likely ());
    2298              :           else
    2299              :             {
    2300            0 :               rtx signbit = expand_shift (RSHIFT_EXPR, mode, res, prec - 1,
    2301              :                                           NULL_RTX, 0);
    2302              :               /* RES is low half of the double width result, HIPART
    2303              :                  the high half.  There was overflow if
    2304              :                  HIPART is different from RES < 0 ? -1 : 0.  */
    2305            0 :               do_compare_rtx_and_jump (signbit, hipart, EQ, true, mode,
    2306              :                                        NULL_RTX, NULL, done_label,
    2307              :                                        profile_probability::very_likely ());
    2308              :             }
    2309              : 
    2310              :         }
    2311         3099 :       else if (int_mode_for_size (prec / 2, 1).exists (&hmode)
    2312         3099 :                && 2 * GET_MODE_PRECISION (hmode) == prec)
    2313              :         {
    2314         3099 :           rtx_code_label *large_op0 = gen_label_rtx ();
    2315         3099 :           rtx_code_label *small_op0_large_op1 = gen_label_rtx ();
    2316         3099 :           rtx_code_label *one_small_one_large = gen_label_rtx ();
    2317         3099 :           rtx_code_label *both_ops_large = gen_label_rtx ();
    2318         3099 :           rtx_code_label *after_hipart_neg = uns ? NULL : gen_label_rtx ();
    2319         1126 :           rtx_code_label *after_lopart_neg = uns ? NULL : gen_label_rtx ();
    2320         3099 :           rtx_code_label *do_overflow = gen_label_rtx ();
    2321         3099 :           rtx_code_label *hipart_different = uns ? NULL : gen_label_rtx ();
    2322              : 
    2323         3099 :           unsigned int hprec = GET_MODE_PRECISION (hmode);
    2324         3099 :           rtx hipart0 = expand_shift (RSHIFT_EXPR, mode, op0, hprec,
    2325              :                                       NULL_RTX, uns);
    2326         3099 :           hipart0 = convert_modes (hmode, mode, hipart0, uns);
    2327         3099 :           rtx lopart0 = convert_modes (hmode, mode, op0, uns);
    2328         3099 :           rtx signbit0 = const0_rtx;
    2329         3099 :           if (!uns)
    2330         1126 :             signbit0 = expand_shift (RSHIFT_EXPR, hmode, lopart0, hprec - 1,
    2331              :                                      NULL_RTX, 0);
    2332         3099 :           rtx hipart1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec,
    2333              :                                       NULL_RTX, uns);
    2334         3099 :           hipart1 = convert_modes (hmode, mode, hipart1, uns);
    2335         3099 :           rtx lopart1 = convert_modes (hmode, mode, op1, uns);
    2336         3099 :           rtx signbit1 = const0_rtx;
    2337         3099 :           if (!uns)
    2338         1126 :             signbit1 = expand_shift (RSHIFT_EXPR, hmode, lopart1, hprec - 1,
    2339              :                                      NULL_RTX, 0);
    2340              : 
    2341         3099 :           res = gen_reg_rtx (mode);
    2342              : 
    2343              :           /* True if op0 resp. op1 are known to be in the range of
    2344              :              halfstype.  */
    2345         3099 :           bool op0_small_p = false;
    2346         3099 :           bool op1_small_p = false;
    2347              :           /* True if op0 resp. op1 are known to have all zeros or all ones
    2348              :              in the upper half of bits, but are not known to be
    2349              :              op{0,1}_small_p.  */
    2350         3099 :           bool op0_medium_p = false;
    2351         3099 :           bool op1_medium_p = false;
    2352              :           /* -1 if op{0,1} is known to be negative, 0 if it is known to be
    2353              :              nonnegative, 1 if unknown.  */
    2354         3099 :           int op0_sign = 1;
    2355         3099 :           int op1_sign = 1;
    2356              : 
    2357         3099 :           if (pos_neg0 == 1)
    2358              :             op0_sign = 0;
    2359         2950 :           else if (pos_neg0 == 2)
    2360          188 :             op0_sign = -1;
    2361         3099 :           if (pos_neg1 == 1)
    2362              :             op1_sign = 0;
    2363         2002 :           else if (pos_neg1 == 2)
    2364          428 :             op1_sign = -1;
    2365              : 
    2366         3099 :           unsigned int mprec0 = prec;
    2367         3099 :           if (arg0 != error_mark_node)
    2368         2350 :             mprec0 = get_min_precision (arg0, sign);
    2369         3099 :           if (mprec0 <= hprec)
    2370              :             op0_small_p = true;
    2371         2923 :           else if (!uns && mprec0 <= hprec + 1)
    2372         3099 :             op0_medium_p = true;
    2373         3099 :           unsigned int mprec1 = prec;
    2374         3099 :           if (arg1 != error_mark_node)
    2375         2350 :             mprec1 = get_min_precision (arg1, sign);
    2376         3099 :           if (mprec1 <= hprec)
    2377              :             op1_small_p = true;
    2378         2582 :           else if (!uns && mprec1 <= hprec + 1)
    2379         3099 :             op1_medium_p = true;
    2380              : 
    2381         3099 :           int smaller_sign = 1;
    2382         3099 :           int larger_sign = 1;
    2383         3099 :           if (op0_small_p)
    2384              :             {
    2385              :               smaller_sign = op0_sign;
    2386              :               larger_sign = op1_sign;
    2387              :             }
    2388         2923 :           else if (op1_small_p)
    2389              :             {
    2390              :               smaller_sign = op1_sign;
    2391              :               larger_sign = op0_sign;
    2392              :             }
    2393         2406 :           else if (op0_sign == op1_sign)
    2394              :             {
    2395         1494 :               smaller_sign = op0_sign;
    2396         1494 :               larger_sign = op0_sign;
    2397              :             }
    2398              : 
    2399         2923 :           if (!op0_small_p)
    2400         2923 :             do_compare_rtx_and_jump (signbit0, hipart0, NE, true, hmode,
    2401              :                                      NULL_RTX, NULL, large_op0,
    2402              :                                      profile_probability::unlikely ());
    2403              : 
    2404         3099 :           if (!op1_small_p)
    2405         2582 :             do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
    2406              :                                      NULL_RTX, NULL, small_op0_large_op1,
    2407              :                                      profile_probability::unlikely ());
    2408              : 
    2409              :           /* If both op0 and op1 are sign (!uns) or zero (uns) extended from
    2410              :              hmode to mode, the multiplication will never overflow.  We can
    2411              :              do just one hmode x hmode => mode widening multiplication.  */
    2412         3099 :           tree halfstype = build_nonstandard_integer_type (hprec, uns);
    2413         3099 :           ops.op0 = make_tree (halfstype, lopart0);
    2414         3099 :           ops.op1 = make_tree (halfstype, lopart1);
    2415         3099 :           ops.code = WIDEN_MULT_EXPR;
    2416         3099 :           ops.type = type;
    2417         3099 :           rtx thisres
    2418         3099 :             = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2419         3099 :           emit_move_insn (res, thisres);
    2420         3099 :           emit_jump (done_label);
    2421              : 
    2422         3099 :           emit_label (small_op0_large_op1);
    2423              : 
    2424              :           /* If op0 is sign (!uns) or zero (uns) extended from hmode to mode,
    2425              :              but op1 is not, just swap the arguments and handle it as op1
    2426              :              sign/zero extended, op0 not.  */
    2427         3099 :           rtx larger = gen_reg_rtx (mode);
    2428         3099 :           rtx hipart = gen_reg_rtx (hmode);
    2429         3099 :           rtx lopart = gen_reg_rtx (hmode);
    2430         3099 :           emit_move_insn (larger, op1);
    2431         3099 :           emit_move_insn (hipart, hipart1);
    2432         3099 :           emit_move_insn (lopart, lopart0);
    2433         3099 :           emit_jump (one_small_one_large);
    2434              : 
    2435         3099 :           emit_label (large_op0);
    2436              : 
    2437         3099 :           if (!op1_small_p)
    2438         2582 :             do_compare_rtx_and_jump (signbit1, hipart1, NE, true, hmode,
    2439              :                                      NULL_RTX, NULL, both_ops_large,
    2440              :                                      profile_probability::unlikely ());
    2441              : 
    2442              :           /* If op1 is sign (!uns) or zero (uns) extended from hmode to mode,
    2443              :              but op0 is not, prepare larger, hipart and lopart pseudos and
    2444              :              handle it together with small_op0_large_op1.  */
    2445         3099 :           emit_move_insn (larger, op0);
    2446         3099 :           emit_move_insn (hipart, hipart0);
    2447         3099 :           emit_move_insn (lopart, lopart1);
    2448              : 
    2449         3099 :           emit_label (one_small_one_large);
    2450              : 
    2451              :           /* lopart is the low part of the operand that is sign extended
    2452              :              to mode, larger is the other operand, hipart is the
    2453              :              high part of larger and lopart0 and lopart1 are the low parts
    2454              :              of both operands.
    2455              :              We perform lopart0 * lopart1 and lopart * hipart widening
    2456              :              multiplications.  */
    2457         3099 :           tree halfutype = build_nonstandard_integer_type (hprec, 1);
    2458         3099 :           ops.op0 = make_tree (halfutype, lopart0);
    2459         3099 :           ops.op1 = make_tree (halfutype, lopart1);
    2460         3099 :           rtx lo0xlo1
    2461         3099 :             = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2462              : 
    2463         3099 :           ops.op0 = make_tree (halfutype, lopart);
    2464         3099 :           ops.op1 = make_tree (halfutype, hipart);
    2465         3099 :           rtx loxhi = gen_reg_rtx (mode);
    2466         3099 :           rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2467         3099 :           emit_move_insn (loxhi, tem);
    2468              : 
    2469         3099 :           if (!uns)
    2470              :             {
    2471              :               /* if (hipart < 0) loxhi -= lopart << (bitsize / 2);  */
    2472         1126 :               if (larger_sign == 0)
    2473           72 :                 emit_jump (after_hipart_neg);
    2474         1054 :               else if (larger_sign != -1)
    2475         1054 :                 do_compare_rtx_and_jump (hipart, const0_rtx, GE, false, hmode,
    2476              :                                          NULL_RTX, NULL, after_hipart_neg,
    2477              :                                          profile_probability::even ());
    2478              : 
    2479         1126 :               tem = convert_modes (mode, hmode, lopart, 1);
    2480         1126 :               tem = expand_shift (LSHIFT_EXPR, mode, tem, hprec, NULL_RTX, 1);
    2481         1126 :               tem = expand_simple_binop (mode, MINUS, loxhi, tem, NULL_RTX,
    2482              :                                          1, OPTAB_WIDEN);
    2483         1126 :               emit_move_insn (loxhi, tem);
    2484              : 
    2485         1126 :               emit_label (after_hipart_neg);
    2486              : 
    2487              :               /* if (lopart < 0) loxhi -= larger;  */
    2488         1126 :               if (smaller_sign == 0)
    2489          251 :                 emit_jump (after_lopart_neg);
    2490          875 :               else if (smaller_sign != -1)
    2491          705 :                 do_compare_rtx_and_jump (lopart, const0_rtx, GE, false, hmode,
    2492              :                                          NULL_RTX, NULL, after_lopart_neg,
    2493              :                                          profile_probability::even ());
    2494              : 
    2495         1126 :               tem = expand_simple_binop (mode, MINUS, loxhi, larger, NULL_RTX,
    2496              :                                          1, OPTAB_WIDEN);
    2497         1126 :               emit_move_insn (loxhi, tem);
    2498              : 
    2499         1126 :               emit_label (after_lopart_neg);
    2500              :             }
    2501              : 
    2502              :           /* loxhi += (uns) lo0xlo1 >> (bitsize / 2);  */
    2503         3099 :           tem = expand_shift (RSHIFT_EXPR, mode, lo0xlo1, hprec, NULL_RTX, 1);
    2504         3099 :           tem = expand_simple_binop (mode, PLUS, loxhi, tem, NULL_RTX,
    2505              :                                      1, OPTAB_WIDEN);
    2506         3099 :           emit_move_insn (loxhi, tem);
    2507              : 
    2508              :           /* if (loxhi >> (bitsize / 2)
    2509              :                  == (hmode) loxhi >> (bitsize / 2 - 1))  (if !uns)
    2510              :              if (loxhi >> (bitsize / 2) == 0               (if uns).  */
    2511         3099 :           rtx hipartloxhi = expand_shift (RSHIFT_EXPR, mode, loxhi, hprec,
    2512              :                                           NULL_RTX, 0);
    2513         3099 :           hipartloxhi = convert_modes (hmode, mode, hipartloxhi, 0);
    2514         3099 :           rtx signbitloxhi = const0_rtx;
    2515         3099 :           if (!uns)
    2516         1126 :             signbitloxhi = expand_shift (RSHIFT_EXPR, hmode,
    2517              :                                          convert_modes (hmode, mode,
    2518              :                                                         loxhi, 0),
    2519         1126 :                                          hprec - 1, NULL_RTX, 0);
    2520              : 
    2521         3099 :           do_compare_rtx_and_jump (signbitloxhi, hipartloxhi, NE, true, hmode,
    2522              :                                    NULL_RTX, NULL, do_overflow,
    2523              :                                    profile_probability::very_unlikely ());
    2524              : 
    2525              :           /* res = (loxhi << (bitsize / 2)) | (hmode) lo0xlo1;  */
    2526         3099 :           rtx loxhishifted = expand_shift (LSHIFT_EXPR, mode, loxhi, hprec,
    2527              :                                            NULL_RTX, 1);
    2528         3099 :           tem = convert_modes (mode, hmode,
    2529              :                                convert_modes (hmode, mode, lo0xlo1, 1), 1);
    2530              : 
    2531         3099 :           tem = expand_simple_binop (mode, IOR, loxhishifted, tem, res,
    2532              :                                      1, OPTAB_WIDEN);
    2533         3099 :           if (tem != res)
    2534            0 :             emit_move_insn (res, tem);
    2535         3099 :           emit_jump (done_label);
    2536              : 
    2537         3099 :           emit_label (both_ops_large);
    2538              : 
    2539              :           /* If both operands are large (not sign (!uns) or zero (uns)
    2540              :              extended from hmode), then perform the full multiplication
    2541              :              which will be the result of the operation.
    2542              :              The only cases which don't overflow are for signed multiplication
    2543              :              some cases where both hipart0 and highpart1 are 0 or -1.
    2544              :              For unsigned multiplication when high parts are both non-zero
    2545              :              this overflows always.  */
    2546         3099 :           ops.code = MULT_EXPR;
    2547         3099 :           ops.op0 = make_tree (type, op0);
    2548         3099 :           ops.op1 = make_tree (type, op1);
    2549         3099 :           tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2550         3099 :           emit_move_insn (res, tem);
    2551              : 
    2552         3099 :           if (!uns)
    2553              :             {
    2554         1126 :               if (!op0_medium_p)
    2555              :                 {
    2556         1126 :                   tem = expand_simple_binop (hmode, PLUS, hipart0, const1_rtx,
    2557              :                                              NULL_RTX, 1, OPTAB_WIDEN);
    2558         1126 :                   do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
    2559              :                                            NULL_RTX, NULL, do_error,
    2560              :                                            profile_probability::very_unlikely ());
    2561              :                 }
    2562              : 
    2563         1126 :               if (!op1_medium_p)
    2564              :                 {
    2565         1126 :                   tem = expand_simple_binop (hmode, PLUS, hipart1, const1_rtx,
    2566              :                                              NULL_RTX, 1, OPTAB_WIDEN);
    2567         1126 :                   do_compare_rtx_and_jump (tem, const1_rtx, GTU, true, hmode,
    2568              :                                            NULL_RTX, NULL, do_error,
    2569              :                                            profile_probability::very_unlikely ());
    2570              :                 }
    2571              : 
    2572              :               /* At this point hipart{0,1} are both in [-1, 0].  If they are
    2573              :                  the same, overflow happened if res is non-positive, if they
    2574              :                  are different, overflow happened if res is positive.  */
    2575         1126 :               if (op0_sign != 1 && op1_sign != 1 && op0_sign != op1_sign)
    2576           12 :                 emit_jump (hipart_different);
    2577         1114 :               else if (op0_sign == 1 || op1_sign == 1)
    2578         1058 :                 do_compare_rtx_and_jump (hipart0, hipart1, NE, true, hmode,
    2579              :                                          NULL_RTX, NULL, hipart_different,
    2580              :                                          profile_probability::even ());
    2581              : 
    2582         1126 :               do_compare_rtx_and_jump (res, const0_rtx, LE, false, mode,
    2583              :                                        NULL_RTX, NULL, do_error,
    2584              :                                        profile_probability::very_unlikely ());
    2585         1126 :               emit_jump (done_label);
    2586              : 
    2587         1126 :               emit_label (hipart_different);
    2588              : 
    2589         1126 :               do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode,
    2590              :                                        NULL_RTX, NULL, do_error,
    2591              :                                        profile_probability::very_unlikely ());
    2592         1126 :               emit_jump (done_label);
    2593              :             }
    2594              : 
    2595         3099 :           emit_label (do_overflow);
    2596              : 
    2597              :           /* Overflow, do full multiplication and fallthru into do_error.  */
    2598         3099 :           ops.op0 = make_tree (type, op0);
    2599         3099 :           ops.op1 = make_tree (type, op1);
    2600         3099 :           tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2601         3099 :           emit_move_insn (res, tem);
    2602              :         }
    2603            0 :       else if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
    2604            0 :                && targetm.scalar_mode_supported_p (wmode))
    2605              :         /* Even emitting a libcall is better than not detecting overflow
    2606              :            at all.  */
    2607            0 :         goto twoxwider;
    2608              :       else
    2609              :         {
    2610            0 :           gcc_assert (!is_ubsan);
    2611            0 :           ops.code = MULT_EXPR;
    2612            0 :           ops.type = type;
    2613            0 :           res = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2614            0 :           emit_jump (done_label);
    2615              :         }
    2616              :     }
    2617              : 
    2618        13888 :  do_error_label:
    2619        18959 :   emit_label (do_error);
    2620        18959 :   if (is_ubsan)
    2621              :     {
    2622              :       /* Expand the ubsan builtin call.  */
    2623         1510 :       push_temp_slots ();
    2624         1510 :       fn = ubsan_build_overflow_builtin (MULT_EXPR, loc, TREE_TYPE (orig_arg0),
    2625              :                                          orig_arg0, orig_arg1, datap);
    2626         1510 :       expand_normal (fn);
    2627         1510 :       pop_temp_slots ();
    2628         1510 :       do_pending_stack_adjust ();
    2629              :     }
    2630        17449 :   else if (lhs)
    2631        17449 :     expand_arith_set_overflow (lhs, target);
    2632              : 
    2633              :   /* We're done.  */
    2634        18959 :   emit_label (done_label);
    2635              : 
    2636              :   /* u1 * u2 -> sr  */
    2637        18959 :   if (uns0_p && uns1_p && !unsr_p)
    2638              :     {
    2639          957 :       rtx_code_label *all_done_label = gen_label_rtx ();
    2640          957 :       do_compare_rtx_and_jump (res, const0_rtx, GE, false, mode, NULL_RTX,
    2641              :                                NULL, all_done_label, profile_probability::very_likely ());
    2642          957 :       expand_arith_set_overflow (lhs, target);
    2643          957 :       emit_label (all_done_label);
    2644              :     }
    2645              : 
    2646              :   /* s1 * u2 -> sr  */
    2647        18959 :   if (!uns0_p && uns1_p && !unsr_p && pos_neg1 == 3)
    2648              :     {
    2649         1273 :       rtx_code_label *all_done_label = gen_label_rtx ();
    2650         1273 :       rtx_code_label *set_noovf = gen_label_rtx ();
    2651         1273 :       do_compare_rtx_and_jump (op1, const0_rtx, GE, false, mode, NULL_RTX,
    2652              :                                NULL, all_done_label, profile_probability::very_likely ());
    2653         1273 :       expand_arith_set_overflow (lhs, target);
    2654         1273 :       do_compare_rtx_and_jump (op0, const0_rtx, EQ, true, mode, NULL_RTX,
    2655              :                                NULL, set_noovf, profile_probability::very_likely ());
    2656         1273 :       do_compare_rtx_and_jump (op0, constm1_rtx, NE, true, mode, NULL_RTX,
    2657              :                                NULL, all_done_label, profile_probability::very_unlikely ());
    2658         1273 :       do_compare_rtx_and_jump (op1, res, NE, true, mode, NULL_RTX, NULL,
    2659              :                                all_done_label, profile_probability::very_unlikely ());
    2660         1273 :       emit_label (set_noovf);
    2661         1273 :       write_complex_part (target, const0_rtx, true, false);
    2662         1273 :       emit_label (all_done_label);
    2663              :     }
    2664              : 
    2665        18959 :   if (lhs)
    2666              :     {
    2667        18815 :       if (is_ubsan)
    2668         1366 :         expand_ubsan_result_store (lhs, target, mode, res, do_error);
    2669              :       else
    2670        17449 :         expand_arith_overflow_result_store (lhs, target, mode, res,
    2671              :                                             overflow_checked);
    2672              :     }
    2673        18959 :   flag_trapv = save_flag_trapv;
    2674        18959 : }
    2675              : 
    2676              : /* Expand UBSAN_CHECK_* internal function if it has vector operands.  */
    2677              : 
    2678              : static void
    2679          567 : expand_vector_ubsan_overflow (location_t loc, enum tree_code code, tree lhs,
    2680              :                               tree arg0, tree arg1)
    2681              : {
    2682          567 :   poly_uint64 cnt = TYPE_VECTOR_SUBPARTS (TREE_TYPE (arg0));
    2683          567 :   rtx_code_label *loop_lab = NULL;
    2684          567 :   rtx cntvar = NULL_RTX;
    2685          567 :   tree cntv = NULL_TREE;
    2686          567 :   tree eltype = TREE_TYPE (TREE_TYPE (arg0));
    2687          567 :   tree sz = TYPE_SIZE (eltype);
    2688          567 :   tree data = NULL_TREE;
    2689          567 :   tree resv = NULL_TREE;
    2690          567 :   rtx lhsr = NULL_RTX;
    2691          567 :   rtx resvr = NULL_RTX;
    2692          567 :   unsigned HOST_WIDE_INT const_cnt = 0;
    2693          567 :   bool use_loop_p = (!cnt.is_constant (&const_cnt) || const_cnt > 4);
    2694          567 :   int save_flag_trapv = flag_trapv;
    2695              : 
    2696              :   /* We don't want any __mulv?i3 etc. calls from the expansion of
    2697              :      these internal functions, so disable -ftrapv temporarily.  */
    2698          567 :   flag_trapv = 0;
    2699          567 :   if (lhs)
    2700              :     {
    2701          567 :       optab op;
    2702          567 :       lhsr = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    2703          141 :       if (!VECTOR_MODE_P (GET_MODE (lhsr))
    2704          426 :           || (op = optab_for_tree_code (code, TREE_TYPE (arg0),
    2705              :                                         optab_default)) == unknown_optab
    2706          993 :           || (optab_handler (op, TYPE_MODE (TREE_TYPE (arg0)))
    2707              :               == CODE_FOR_nothing))
    2708              :         {
    2709          141 :           if (MEM_P (lhsr))
    2710          141 :             resv = make_tree (TREE_TYPE (lhs), lhsr);
    2711              :           else
    2712              :             {
    2713            0 :               resvr = assign_temp (TREE_TYPE (lhs), 1, 1);
    2714            0 :               resv = make_tree (TREE_TYPE (lhs), resvr);
    2715              :             }
    2716              :         }
    2717              :     }
    2718          567 :   if (use_loop_p)
    2719              :     {
    2720          417 :       do_pending_stack_adjust ();
    2721          417 :       loop_lab = gen_label_rtx ();
    2722          417 :       cntvar = gen_reg_rtx (TYPE_MODE (sizetype));
    2723          417 :       cntv = make_tree (sizetype, cntvar);
    2724          417 :       emit_move_insn (cntvar, const0_rtx);
    2725          417 :       emit_label (loop_lab);
    2726              :     }
    2727          567 :   if (TREE_CODE (arg0) != VECTOR_CST)
    2728              :     {
    2729          461 :       rtx arg0r = expand_normal (arg0);
    2730          461 :       arg0 = make_tree (TREE_TYPE (arg0), arg0r);
    2731              :     }
    2732          567 :   if (TREE_CODE (arg1) != VECTOR_CST)
    2733              :     {
    2734          555 :       rtx arg1r = expand_normal (arg1);
    2735          555 :       arg1 = make_tree (TREE_TYPE (arg1), arg1r);
    2736              :     }
    2737         2334 :   for (unsigned int i = 0; i < (use_loop_p ? 1 : const_cnt); i++)
    2738              :     {
    2739         1017 :       tree op0, op1, res = NULL_TREE;
    2740         1017 :       if (use_loop_p)
    2741              :         {
    2742          417 :           tree atype = build_array_type_nelts (eltype, cnt);
    2743          417 :           op0 = uniform_vector_p (arg0);
    2744          417 :           if (op0 == NULL_TREE)
    2745              :             {
    2746          351 :               op0 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg0);
    2747          351 :               op0 = build4_loc (loc, ARRAY_REF, eltype, op0, cntv,
    2748              :                                 NULL_TREE, NULL_TREE);
    2749              :             }
    2750          417 :           op1 = uniform_vector_p (arg1);
    2751          417 :           if (op1 == NULL_TREE)
    2752              :             {
    2753          411 :               op1 = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, arg1);
    2754          411 :               op1 = build4_loc (loc, ARRAY_REF, eltype, op1, cntv,
    2755              :                                 NULL_TREE, NULL_TREE);
    2756              :             }
    2757          417 :           if (resv)
    2758              :             {
    2759          141 :               res = fold_build1_loc (loc, VIEW_CONVERT_EXPR, atype, resv);
    2760          141 :               res = build4_loc (loc, ARRAY_REF, eltype, res, cntv,
    2761              :                                 NULL_TREE, NULL_TREE);
    2762              :             }
    2763              :         }
    2764              :       else
    2765              :         {
    2766          600 :           tree bitpos = bitsize_int (tree_to_uhwi (sz) * i);
    2767          600 :           op0 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg0, sz, bitpos);
    2768          600 :           op1 = fold_build3_loc (loc, BIT_FIELD_REF, eltype, arg1, sz, bitpos);
    2769          600 :           if (resv)
    2770            0 :             res = fold_build3_loc (loc, BIT_FIELD_REF, eltype, resv, sz,
    2771              :                                    bitpos);
    2772              :         }
    2773         1017 :       switch (code)
    2774              :         {
    2775          317 :         case PLUS_EXPR:
    2776          317 :           expand_addsub_overflow (loc, PLUS_EXPR, res, op0, op1,
    2777              :                                   false, false, false, true, &data);
    2778          317 :           break;
    2779          534 :         case MINUS_EXPR:
    2780          534 :           if (use_loop_p ? integer_zerop (arg0) : integer_zerop (op0))
    2781          226 :             expand_neg_overflow (loc, res, op1, true, &data);
    2782              :           else
    2783          308 :             expand_addsub_overflow (loc, MINUS_EXPR, res, op0, op1,
    2784              :                                     false, false, false, true, &data);
    2785              :           break;
    2786          166 :         case MULT_EXPR:
    2787          166 :           expand_mul_overflow (loc, res, op0, op1, false, false, false,
    2788              :                                true, &data);
    2789          166 :           break;
    2790            0 :         default:
    2791            0 :           gcc_unreachable ();
    2792              :         }
    2793              :     }
    2794          567 :   if (use_loop_p)
    2795              :     {
    2796          417 :       struct separate_ops ops;
    2797          417 :       ops.code = PLUS_EXPR;
    2798          417 :       ops.type = TREE_TYPE (cntv);
    2799          417 :       ops.op0 = cntv;
    2800          417 :       ops.op1 = build_int_cst (TREE_TYPE (cntv), 1);
    2801          417 :       ops.op2 = NULL_TREE;
    2802          417 :       ops.location = loc;
    2803          417 :       rtx ret = expand_expr_real_2 (&ops, cntvar, TYPE_MODE (sizetype),
    2804              :                                     EXPAND_NORMAL);
    2805          417 :       if (ret != cntvar)
    2806            0 :         emit_move_insn (cntvar, ret);
    2807          417 :       rtx cntrtx = gen_int_mode (cnt, TYPE_MODE (sizetype));
    2808          834 :       do_compare_rtx_and_jump (cntvar, cntrtx, NE, false,
    2809          417 :                                TYPE_MODE (sizetype), NULL_RTX, NULL, loop_lab,
    2810              :                                profile_probability::very_likely ());
    2811              :     }
    2812          567 :   if (lhs && resv == NULL_TREE)
    2813              :     {
    2814          426 :       struct separate_ops ops;
    2815          426 :       ops.code = code;
    2816          426 :       ops.type = TREE_TYPE (arg0);
    2817          426 :       ops.op0 = arg0;
    2818          426 :       ops.op1 = arg1;
    2819          426 :       ops.op2 = NULL_TREE;
    2820          426 :       ops.location = loc;
    2821          426 :       rtx ret = expand_expr_real_2 (&ops, lhsr, TYPE_MODE (TREE_TYPE (arg0)),
    2822              :                                     EXPAND_NORMAL);
    2823          426 :       if (ret != lhsr)
    2824            5 :         emit_move_insn (lhsr, ret);
    2825              :     }
    2826          141 :   else if (resvr)
    2827            0 :     emit_move_insn (lhsr, resvr);
    2828          567 :   flag_trapv = save_flag_trapv;
    2829          567 : }
    2830              : 
    2831              : /* Expand UBSAN_CHECK_ADD call STMT.  */
    2832              : 
    2833              : static void
    2834         1792 : expand_UBSAN_CHECK_ADD (internal_fn, gcall *stmt)
    2835              : {
    2836         1792 :   location_t loc = gimple_location (stmt);
    2837         1792 :   tree lhs = gimple_call_lhs (stmt);
    2838         1792 :   tree arg0 = gimple_call_arg (stmt, 0);
    2839         1792 :   tree arg1 = gimple_call_arg (stmt, 1);
    2840         1792 :   if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
    2841          185 :     expand_vector_ubsan_overflow (loc, PLUS_EXPR, lhs, arg0, arg1);
    2842              :   else
    2843         1607 :     expand_addsub_overflow (loc, PLUS_EXPR, lhs, arg0, arg1,
    2844              :                             false, false, false, true, NULL);
    2845         1792 : }
    2846              : 
    2847              : /* Expand UBSAN_CHECK_SUB call STMT.  */
    2848              : 
    2849              : static void
    2850         1865 : expand_UBSAN_CHECK_SUB (internal_fn, gcall *stmt)
    2851              : {
    2852         1865 :   location_t loc = gimple_location (stmt);
    2853         1865 :   tree lhs = gimple_call_lhs (stmt);
    2854         1865 :   tree arg0 = gimple_call_arg (stmt, 0);
    2855         1865 :   tree arg1 = gimple_call_arg (stmt, 1);
    2856         1865 :   if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
    2857          282 :     expand_vector_ubsan_overflow (loc, MINUS_EXPR, lhs, arg0, arg1);
    2858         1583 :   else if (integer_zerop (arg0))
    2859          424 :     expand_neg_overflow (loc, lhs, arg1, true, NULL);
    2860              :   else
    2861         1159 :     expand_addsub_overflow (loc, MINUS_EXPR, lhs, arg0, arg1,
    2862              :                             false, false, false, true, NULL);
    2863         1865 : }
    2864              : 
    2865              : /* Expand UBSAN_CHECK_MUL call STMT.  */
    2866              : 
    2867              : static void
    2868         1444 : expand_UBSAN_CHECK_MUL (internal_fn, gcall *stmt)
    2869              : {
    2870         1444 :   location_t loc = gimple_location (stmt);
    2871         1444 :   tree lhs = gimple_call_lhs (stmt);
    2872         1444 :   tree arg0 = gimple_call_arg (stmt, 0);
    2873         1444 :   tree arg1 = gimple_call_arg (stmt, 1);
    2874         1444 :   if (VECTOR_TYPE_P (TREE_TYPE (arg0)))
    2875          100 :     expand_vector_ubsan_overflow (loc, MULT_EXPR, lhs, arg0, arg1);
    2876              :   else
    2877         1344 :     expand_mul_overflow (loc, lhs, arg0, arg1, false, false, false, true,
    2878              :                          NULL);
    2879         1444 : }
    2880              : 
    2881              : /* Helper function for {ADD,SUB,MUL}_OVERFLOW call stmt expansion.  */
    2882              : 
    2883              : static void
    2884        72036 : expand_arith_overflow (enum tree_code code, gimple *stmt)
    2885              : {
    2886        72036 :   tree lhs = gimple_call_lhs (stmt);
    2887        72036 :   if (lhs == NULL_TREE)
    2888              :     return;
    2889        72036 :   tree arg0 = gimple_call_arg (stmt, 0);
    2890        72036 :   tree arg1 = gimple_call_arg (stmt, 1);
    2891        72036 :   tree type = TREE_TYPE (TREE_TYPE (lhs));
    2892        72036 :   int uns0_p = TYPE_UNSIGNED (TREE_TYPE (arg0));
    2893        72036 :   int uns1_p = TYPE_UNSIGNED (TREE_TYPE (arg1));
    2894        72036 :   int unsr_p = TYPE_UNSIGNED (type);
    2895        72036 :   int prec0 = TYPE_PRECISION (TREE_TYPE (arg0));
    2896        72036 :   int prec1 = TYPE_PRECISION (TREE_TYPE (arg1));
    2897        72036 :   int precres = TYPE_PRECISION (type);
    2898        72036 :   location_t loc = gimple_location (stmt);
    2899        72036 :   if (!uns0_p && get_range_pos_neg (arg0, stmt) == 1)
    2900              :     uns0_p = true;
    2901        72036 :   if (!uns1_p && get_range_pos_neg (arg1, stmt) == 1)
    2902              :     uns1_p = true;
    2903       106378 :   int pr = get_min_precision (arg0, uns0_p ? UNSIGNED : SIGNED);
    2904        72036 :   prec0 = MIN (prec0, pr);
    2905        97212 :   pr = get_min_precision (arg1, uns1_p ? UNSIGNED : SIGNED);
    2906        72036 :   prec1 = MIN (prec1, pr);
    2907        72036 :   int save_flag_trapv = flag_trapv;
    2908              : 
    2909              :   /* We don't want any __mulv?i3 etc. calls from the expansion of
    2910              :      these internal functions, so disable -ftrapv temporarily.  */
    2911        72036 :   flag_trapv = 0;
    2912              :   /* If uns0_p && uns1_p, precop is minimum needed precision
    2913              :      of unsigned type to hold the exact result, otherwise
    2914              :      precop is minimum needed precision of signed type to
    2915              :      hold the exact result.  */
    2916        72036 :   int precop;
    2917        72036 :   if (code == MULT_EXPR)
    2918        23457 :     precop = prec0 + prec1 + (uns0_p != uns1_p);
    2919              :   else
    2920              :     {
    2921        48579 :       if (uns0_p == uns1_p)
    2922        31411 :         precop = MAX (prec0, prec1) + 1;
    2923        17168 :       else if (uns0_p)
    2924         5815 :         precop = MAX (prec0 + 1, prec1) + 1;
    2925              :       else
    2926        11353 :         precop = MAX (prec0, prec1 + 1) + 1;
    2927              :     }
    2928        72036 :   int orig_precres = precres;
    2929              : 
    2930        91476 :   do
    2931              :     {
    2932        91476 :       if ((uns0_p && uns1_p)
    2933        91476 :           ? ((precop + !unsr_p) <= precres
    2934              :              /* u1 - u2 -> ur can overflow, no matter what precision
    2935              :                 the result has.  */
    2936        33719 :              && (code != MINUS_EXPR || !unsr_p))
    2937        57757 :           : (!unsr_p && precop <= precres))
    2938              :         {
    2939              :           /* The infinity precision result will always fit into result.  */
    2940        18526 :           rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    2941        18526 :           write_complex_part (target, const0_rtx, true, false);
    2942        18526 :           scalar_int_mode mode = SCALAR_INT_TYPE_MODE (type);
    2943        18526 :           struct separate_ops ops;
    2944        18526 :           ops.code = code;
    2945        18526 :           ops.type = type;
    2946        18526 :           ops.op0 = fold_convert_loc (loc, type, arg0);
    2947        18526 :           ops.op1 = fold_convert_loc (loc, type, arg1);
    2948        18526 :           ops.op2 = NULL_TREE;
    2949        18526 :           ops.location = loc;
    2950        18526 :           rtx tem = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    2951        18526 :           expand_arith_overflow_result_store (lhs, target, mode, tem);
    2952        18526 :           flag_trapv = save_flag_trapv;
    2953        18526 :           return;
    2954              :         }
    2955              : 
    2956              :       /* For operations with low precision, if target doesn't have them, start
    2957              :          with precres widening right away, otherwise do it only if the most
    2958              :          simple cases can't be used.  */
    2959        72950 :       const int min_precision = targetm.min_arithmetic_precision ();
    2960        72950 :       if (orig_precres == precres && precres < min_precision)
    2961              :         ;
    2962        71683 :       else if ((uns0_p && uns1_p && unsr_p && prec0 <= precres
    2963        20869 :                 && prec1 <= precres)
    2964        50817 :           || ((!uns0_p || !uns1_p) && !unsr_p
    2965        20780 :               && prec0 + uns0_p <= precres
    2966        17578 :               && prec1 + uns1_p <= precres))
    2967              :         {
    2968        34286 :           arg0 = fold_convert_loc (loc, type, arg0);
    2969        34286 :           arg1 = fold_convert_loc (loc, type, arg1);
    2970        34286 :           switch (code)
    2971              :             {
    2972         9870 :             case MINUS_EXPR:
    2973         9870 :               if (integer_zerop (arg0) && !unsr_p)
    2974              :                 {
    2975          402 :                   expand_neg_overflow (loc, lhs, arg1, false, NULL);
    2976          402 :                   flag_trapv = save_flag_trapv;
    2977          402 :                   return;
    2978              :                 }
    2979              :               /* FALLTHRU */
    2980        24175 :             case PLUS_EXPR:
    2981        24175 :               expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
    2982              :                                       unsr_p, unsr_p, false, NULL);
    2983        24175 :               flag_trapv = save_flag_trapv;
    2984        24175 :               return;
    2985         9709 :             case MULT_EXPR:
    2986         9709 :               expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
    2987              :                                    unsr_p, unsr_p, false, NULL);
    2988         9709 :               flag_trapv = save_flag_trapv;
    2989         9709 :               return;
    2990            0 :             default:
    2991            0 :               gcc_unreachable ();
    2992              :             }
    2993              :         }
    2994              : 
    2995              :       /* For sub-word operations, retry with a wider type first.  */
    2996        38665 :       if (orig_precres == precres && precop <= BITS_PER_WORD)
    2997              :         {
    2998        19508 :           int p = MAX (min_precision, precop);
    2999        19508 :           scalar_int_mode m = smallest_int_mode_for_size (p).require ();
    3000        19508 :           tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
    3001              :                                                         uns0_p && uns1_p
    3002        19508 :                                                         && unsr_p);
    3003        19508 :           p = TYPE_PRECISION (optype);
    3004        19508 :           if (p > precres)
    3005              :             {
    3006        18805 :               precres = p;
    3007        18805 :               unsr_p = TYPE_UNSIGNED (optype);
    3008        18805 :               type = optype;
    3009        18805 :               continue;
    3010              :             }
    3011              :         }
    3012              : 
    3013        19859 :       if (prec0 <= precres && prec1 <= precres)
    3014              :         {
    3015        19224 :           tree types[2];
    3016        19224 :           if (unsr_p)
    3017              :             {
    3018        11083 :               types[0] = build_nonstandard_integer_type (precres, 0);
    3019        11083 :               types[1] = type;
    3020              :             }
    3021              :           else
    3022              :             {
    3023         8141 :               types[0] = type;
    3024         8141 :               types[1] = build_nonstandard_integer_type (precres, 1);
    3025              :             }
    3026        19224 :           arg0 = fold_convert_loc (loc, types[uns0_p], arg0);
    3027        19224 :           arg1 = fold_convert_loc (loc, types[uns1_p], arg1);
    3028        19224 :           if (code != MULT_EXPR)
    3029        11484 :             expand_addsub_overflow (loc, code, lhs, arg0, arg1, unsr_p,
    3030              :                                     uns0_p, uns1_p, false, NULL);
    3031              :           else
    3032         7740 :             expand_mul_overflow (loc, lhs, arg0, arg1, unsr_p,
    3033              :                                  uns0_p, uns1_p, false, NULL);
    3034        19224 :           flag_trapv = save_flag_trapv;
    3035        19224 :           return;
    3036              :         }
    3037              : 
    3038              :       /* Retry with a wider type.  */
    3039          635 :       if (orig_precres == precres)
    3040              :         {
    3041          635 :           int p = MAX (prec0, prec1);
    3042          635 :           scalar_int_mode m = smallest_int_mode_for_size (p).require ();
    3043          635 :           tree optype = build_nonstandard_integer_type (GET_MODE_PRECISION (m),
    3044              :                                                         uns0_p && uns1_p
    3045          635 :                                                         && unsr_p);
    3046          635 :           p = TYPE_PRECISION (optype);
    3047          635 :           if (p > precres)
    3048              :             {
    3049          635 :               precres = p;
    3050          635 :               unsr_p = TYPE_UNSIGNED (optype);
    3051          635 :               type = optype;
    3052          635 :               continue;
    3053              :             }
    3054              :         }
    3055              : 
    3056            0 :       gcc_unreachable ();
    3057              :     }
    3058              :   while (1);
    3059              : }
    3060              : 
    3061              : /* Expand ADD_OVERFLOW STMT.  */
    3062              : 
    3063              : static void
    3064        25078 : expand_ADD_OVERFLOW (internal_fn, gcall *stmt)
    3065              : {
    3066        25078 :   expand_arith_overflow (PLUS_EXPR, stmt);
    3067        25078 : }
    3068              : 
    3069              : /* Expand SUB_OVERFLOW STMT.  */
    3070              : 
    3071              : static void
    3072        23501 : expand_SUB_OVERFLOW (internal_fn, gcall *stmt)
    3073              : {
    3074        23501 :   expand_arith_overflow (MINUS_EXPR, stmt);
    3075        23501 : }
    3076              : 
    3077              : /* Expand MUL_OVERFLOW STMT.  */
    3078              : 
    3079              : static void
    3080        23457 : expand_MUL_OVERFLOW (internal_fn, gcall *stmt)
    3081              : {
    3082        23457 :   expand_arith_overflow (MULT_EXPR, stmt);
    3083        23457 : }
    3084              : 
    3085              : /* Expand UADDC STMT.  */
    3086              : 
    3087              : static void
    3088        16070 : expand_UADDC (internal_fn ifn, gcall *stmt)
    3089              : {
    3090        16070 :   tree lhs = gimple_call_lhs (stmt);
    3091        16070 :   tree arg1 = gimple_call_arg (stmt, 0);
    3092        16070 :   tree arg2 = gimple_call_arg (stmt, 1);
    3093        16070 :   tree arg3 = gimple_call_arg (stmt, 2);
    3094        16070 :   tree type = TREE_TYPE (arg1);
    3095        16070 :   machine_mode mode = TYPE_MODE (type);
    3096        22565 :   insn_code icode = optab_handler (ifn == IFN_UADDC
    3097              :                                    ? uaddc5_optab : usubc5_optab, mode);
    3098        16070 :   rtx op1 = expand_normal (arg1);
    3099        16070 :   rtx op2 = expand_normal (arg2);
    3100        16070 :   rtx op3 = expand_normal (arg3);
    3101        16070 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3102        16070 :   rtx re = gen_reg_rtx (mode);
    3103        16070 :   rtx im = gen_reg_rtx (mode);
    3104        16070 :   class expand_operand ops[5];
    3105        16070 :   create_output_operand (&ops[0], re, mode);
    3106        16070 :   create_output_operand (&ops[1], im, mode);
    3107        16070 :   create_input_operand (&ops[2], op1, mode);
    3108        16070 :   create_input_operand (&ops[3], op2, mode);
    3109        16070 :   create_input_operand (&ops[4], op3, mode);
    3110        16070 :   expand_insn (icode, 5, ops);
    3111        16070 :   write_complex_part (target, re, false, false);
    3112        16070 :   write_complex_part (target, im, true, false);
    3113        16070 : }
    3114              : 
    3115              : /* Expand USUBC STMT.  */
    3116              : 
    3117              : static void
    3118         6495 : expand_USUBC (internal_fn ifn, gcall *stmt)
    3119              : {
    3120         6495 :   expand_UADDC (ifn, stmt);
    3121         6495 : }
    3122              : 
    3123              : /* This should get folded in tree-vectorizer.cc.  */
    3124              : 
    3125              : static void
    3126            0 : expand_LOOP_VECTORIZED (internal_fn, gcall *)
    3127              : {
    3128            0 :   gcc_unreachable ();
    3129              : }
    3130              : 
    3131              : /* This should get folded in tree-vectorizer.cc.  */
    3132              : 
    3133              : static void
    3134            0 : expand_LOOP_DIST_ALIAS (internal_fn, gcall *)
    3135              : {
    3136            0 :   gcc_unreachable ();
    3137              : }
    3138              : 
    3139              : /* Return a memory reference of type TYPE for argument INDEX of STMT.
    3140              :    Use argument INDEX + 1 to derive the second (TBAA) operand.  */
    3141              : 
    3142              : static tree
    3143         1435 : expand_call_mem_ref (tree type, gcall *stmt, int index)
    3144              : {
    3145         1435 :   tree addr = gimple_call_arg (stmt, index);
    3146         1435 :   tree alias_ptr_type = TREE_TYPE (gimple_call_arg (stmt, index + 1));
    3147         1435 :   unsigned int align = tree_to_shwi (gimple_call_arg (stmt, index + 1));
    3148         1435 :   if (TYPE_ALIGN (type) != align)
    3149         1167 :     type = build_aligned_type (type, align);
    3150              : 
    3151         1435 :   tree tmp = addr;
    3152         1435 :   if (TREE_CODE (tmp) == SSA_NAME)
    3153              :     {
    3154         1297 :       gimple *def = get_gimple_for_ssa_name (tmp);
    3155         1297 :       if (def && gimple_assign_single_p (def))
    3156          581 :         tmp = gimple_assign_rhs1 (def);
    3157              :     }
    3158              : 
    3159         1435 :   if (TREE_CODE (tmp) == ADDR_EXPR)
    3160              :     {
    3161          719 :       tree mem = TREE_OPERAND (tmp, 0);
    3162          719 :       if (TREE_CODE (mem) == TARGET_MEM_REF
    3163          719 :           && types_compatible_p (TREE_TYPE (mem), type))
    3164              :         {
    3165          581 :           tree offset = TMR_OFFSET (mem);
    3166          581 :           if (type != TREE_TYPE (mem)
    3167          132 :               || alias_ptr_type != TREE_TYPE (offset)
    3168          713 :               || !integer_zerop (offset))
    3169              :             {
    3170          476 :               mem = copy_node (mem);
    3171          952 :               TMR_OFFSET (mem) = wide_int_to_tree (alias_ptr_type,
    3172          476 :                                                    wi::to_poly_wide (offset));
    3173          476 :               TREE_TYPE (mem) = type;
    3174              :             }
    3175              :           return mem;
    3176              :         }
    3177              :     }
    3178              : 
    3179          854 :   return fold_build2 (MEM_REF, type, addr, build_int_cst (alias_ptr_type, 0));
    3180              : }
    3181              : 
    3182              : /* Expand MASK_LOAD{,_LANES}, MASK_LEN_LOAD or LEN_LOAD call STMT using optab
    3183              :  * OPTAB.  */
    3184              : 
    3185              : static void
    3186          692 : expand_partial_load_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
    3187              : {
    3188          692 :   int i = 0;
    3189          692 :   class expand_operand ops[6];
    3190          692 :   tree type, lhs, rhs, maskt;
    3191          692 :   rtx mem, target;
    3192          692 :   insn_code icode;
    3193              : 
    3194          692 :   maskt = gimple_call_arg (stmt, internal_fn_mask_index (ifn));
    3195          692 :   lhs = gimple_call_lhs (stmt);
    3196          692 :   if (lhs == NULL_TREE)
    3197            0 :     return;
    3198          692 :   type = TREE_TYPE (lhs);
    3199          692 :   rhs = expand_call_mem_ref (type, stmt, 0);
    3200              : 
    3201          692 :   if (optab == vec_mask_load_lanes_optab
    3202          692 :       || optab == vec_mask_len_load_lanes_optab)
    3203            0 :     icode = get_multi_vector_move (type, optab);
    3204          692 :   else if (optab == len_load_optab)
    3205            0 :     icode = direct_optab_handler (optab, TYPE_MODE (type));
    3206              :   else
    3207          692 :     icode = convert_optab_handler (optab, TYPE_MODE (type),
    3208          692 :                                    TYPE_MODE (TREE_TYPE (maskt)));
    3209              : 
    3210          692 :   mem = expand_expr (rhs, NULL_RTX, VOIDmode, EXPAND_MEMORY);
    3211          692 :   gcc_assert (MEM_P (mem));
    3212          692 :   MEM_NOTRAP_P (mem) = !tree_could_trap_p (rhs);
    3213              :   /* The built MEM_REF does not accurately reflect that the load
    3214              :      is only partial.  Clear it.  */
    3215          692 :   set_mem_expr (mem, NULL_TREE);
    3216          692 :   clear_mem_offset (mem);
    3217          692 :   target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3218          692 :   create_call_lhs_operand (&ops[i++], target, TYPE_MODE (type));
    3219          692 :   create_fixed_operand (&ops[i++], mem);
    3220          692 :   i = add_mask_else_and_len_args (ops, i, stmt);
    3221          692 :   expand_insn (icode, i, ops);
    3222              : 
    3223          692 :   assign_call_lhs (lhs, target, &ops[0]);
    3224              : }
    3225              : 
    3226              : #define expand_mask_load_optab_fn expand_partial_load_optab_fn
    3227              : #define expand_mask_load_lanes_optab_fn expand_mask_load_optab_fn
    3228              : #define expand_mask_len_load_lanes_optab_fn expand_mask_load_optab_fn
    3229              : #define expand_len_load_optab_fn expand_partial_load_optab_fn
    3230              : #define expand_mask_len_load_optab_fn expand_partial_load_optab_fn
    3231              : 
    3232              : /* Expand MASK_STORE{,_LANES}, MASK_LEN_STORE or LEN_STORE call STMT using optab
    3233              :  * OPTAB.  */
    3234              : 
    3235              : static void
    3236          743 : expand_partial_store_optab_fn (internal_fn ifn, gcall *stmt, convert_optab optab)
    3237              : {
    3238          743 :   int i = 0;
    3239          743 :   class expand_operand ops[5];
    3240          743 :   tree type, lhs, rhs, maskt;
    3241          743 :   rtx mem, reg;
    3242          743 :   insn_code icode;
    3243              : 
    3244          743 :   maskt = gimple_call_arg (stmt, internal_fn_mask_index (ifn));
    3245          743 :   rhs = gimple_call_arg (stmt, internal_fn_stored_value_index (ifn));
    3246          743 :   type = TREE_TYPE (rhs);
    3247          743 :   lhs = expand_call_mem_ref (type, stmt, 0);
    3248              : 
    3249          743 :   if (optab == vec_mask_store_lanes_optab
    3250          743 :       || optab == vec_mask_len_store_lanes_optab)
    3251            0 :     icode = get_multi_vector_move (type, optab);
    3252          743 :   else if (optab == len_store_optab)
    3253            0 :     icode = direct_optab_handler (optab, TYPE_MODE (type));
    3254              :   else
    3255          743 :     icode = convert_optab_handler (optab, TYPE_MODE (type),
    3256          743 :                                    TYPE_MODE (TREE_TYPE (maskt)));
    3257              : 
    3258          743 :   mem = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3259          743 :   gcc_assert (MEM_P (mem));
    3260              :   /* The built MEM_REF does not accurately reflect that the store
    3261              :      is only partial.  Clear it.  */
    3262          743 :   set_mem_expr (mem, NULL_TREE);
    3263          743 :   clear_mem_offset (mem);
    3264          743 :   reg = expand_normal (rhs);
    3265          743 :   create_fixed_operand (&ops[i++], mem);
    3266          743 :   create_input_operand (&ops[i++], reg, TYPE_MODE (type));
    3267          743 :   i = add_mask_else_and_len_args (ops, i, stmt);
    3268          743 :   expand_insn (icode, i, ops);
    3269          743 : }
    3270              : 
    3271              : #define expand_mask_store_optab_fn expand_partial_store_optab_fn
    3272              : #define expand_mask_store_lanes_optab_fn expand_mask_store_optab_fn
    3273              : #define expand_mask_len_store_lanes_optab_fn expand_mask_store_optab_fn
    3274              : #define expand_len_store_optab_fn expand_partial_store_optab_fn
    3275              : #define expand_mask_len_store_optab_fn expand_partial_store_optab_fn
    3276              : 
    3277              : /* Expand VCOND_MASK optab internal function.
    3278              :    The expansion of STMT happens based on OPTAB table associated.  */
    3279              : 
    3280              : static void
    3281         9321 : expand_vec_cond_mask_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
    3282              : {
    3283         9321 :   class expand_operand ops[4];
    3284              : 
    3285         9321 :   tree lhs = gimple_call_lhs (stmt);
    3286         9321 :   tree op0 = gimple_call_arg (stmt, 0);
    3287         9321 :   tree op1 = gimple_call_arg (stmt, 1);
    3288         9321 :   tree op2 = gimple_call_arg (stmt, 2);
    3289         9321 :   tree vec_cond_type = TREE_TYPE (lhs);
    3290              : 
    3291         9321 :   machine_mode mode = TYPE_MODE (vec_cond_type);
    3292         9321 :   machine_mode mask_mode = TYPE_MODE (TREE_TYPE (op0));
    3293         9321 :   enum insn_code icode = convert_optab_handler (optab, mode, mask_mode);
    3294         9321 :   rtx mask, rtx_op1, rtx_op2;
    3295              : 
    3296         9321 :   gcc_assert (icode != CODE_FOR_nothing);
    3297              : 
    3298              :   /* Find the comparison generating the mask OP0.  */
    3299         9321 :   tree cmp_op0 = NULL_TREE;
    3300         9321 :   tree cmp_op1 = NULL_TREE;
    3301         9321 :   enum tree_code cmp_code = TREE_CODE (op0);
    3302         9321 :   if (TREE_CODE_CLASS (cmp_code) == tcc_comparison)
    3303              :     {
    3304            0 :       cmp_op0 = TREE_OPERAND (op0, 0);
    3305            0 :       cmp_op1 = TREE_OPERAND (op0, 1);
    3306              :     }
    3307         9321 :   else if (cmp_code == SSA_NAME)
    3308              :     {
    3309         7985 :       gimple *def_stmt = get_gimple_for_ssa_name (op0);
    3310         7985 :       if (def_stmt && is_gimple_assign (def_stmt))
    3311              :         {
    3312         6343 :           cmp_code = gimple_assign_rhs_code (def_stmt);
    3313         6343 :           if (TREE_CODE_CLASS (cmp_code) == tcc_comparison)
    3314              :             {
    3315         4625 :               cmp_op0 = gimple_assign_rhs1 (def_stmt);
    3316         4625 :               cmp_op1 = gimple_assign_rhs2 (def_stmt);
    3317              :             }
    3318              :         }
    3319              :     }
    3320              : 
    3321              :   /* Decide whether to invert comparison based on rtx_cost.  */
    3322         4625 :   if (cmp_op0)
    3323              :     {
    3324         4625 :       enum tree_code rev_code;
    3325         4625 :       tree op_type = TREE_TYPE (cmp_op0);
    3326         4625 :       int unsignedp = TYPE_UNSIGNED (op_type);
    3327         4625 :       rev_code = invert_tree_comparison (cmp_code, HONOR_NANS (op_type));
    3328              : 
    3329         4625 :       if (rev_code != ERROR_MARK)
    3330              :         {
    3331         3384 :           tree cmp_type = TREE_TYPE (op0);
    3332         3384 :           machine_mode cmp_mode = TYPE_MODE (cmp_type);
    3333         3384 :           machine_mode op_mode = TYPE_MODE (op_type);
    3334         3384 :           bool speed_p = optimize_insn_for_speed_p ();
    3335         3384 :           rtx reg = gen_raw_REG (op_mode, LAST_VIRTUAL_REGISTER + 1);
    3336         3384 :           enum rtx_code cmp_rtx_code = convert_tree_comp_to_rtx (cmp_code,
    3337              :                                                                  unsignedp);
    3338         3384 :           rtx veccmp = gen_rtx_fmt_ee (cmp_rtx_code, cmp_mode, reg, reg);
    3339         3384 :           int old_cost = rtx_cost (veccmp, cmp_mode, SET, 0, speed_p);
    3340         3384 :           enum rtx_code rev_rtx_code = convert_tree_comp_to_rtx (rev_code,
    3341              :                                                                  unsignedp);
    3342         3384 :           PUT_CODE (veccmp, rev_rtx_code);
    3343         3384 :           int new_cost = rtx_cost (veccmp, cmp_mode, SET, 0, speed_p);
    3344         3384 :           if (new_cost < old_cost)
    3345              :             {
    3346          369 :               op0 = fold_build2_loc (EXPR_LOCATION (op0), rev_code,
    3347              :                                      cmp_type, cmp_op0, cmp_op1);
    3348          369 :               std::swap (op1, op2);
    3349              :             }
    3350              : 
    3351         3384 :           if (dump_file && (dump_flags & TDF_DETAILS))
    3352              :             {
    3353            0 :               fprintf (dump_file,
    3354              :                        ";; %sswapping operands of .VCOND_MASK\n",
    3355              :                        new_cost >= old_cost ? "not " : "");
    3356            0 :               fprintf (dump_file,
    3357              :                        ";; cost of original %s: %d\n",
    3358            0 :                        GET_RTX_NAME (cmp_rtx_code), old_cost);
    3359            0 :               fprintf (dump_file,
    3360              :                        ";; cost of replacement %s: %d\n",
    3361            0 :                        GET_RTX_NAME (rev_rtx_code), new_cost);
    3362              :             }
    3363              :         }
    3364              :     }
    3365              : 
    3366         9321 :   mask = expand_normal (op0);
    3367         9321 :   rtx_op1 = expand_normal (op1);
    3368         9321 :   rtx_op2 = expand_normal (op2);
    3369              : 
    3370         9321 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3371         9321 :   create_call_lhs_operand (&ops[0], target, mode);
    3372         9321 :   create_input_operand (&ops[1], rtx_op1, mode);
    3373         9321 :   create_input_operand (&ops[2], rtx_op2, mode);
    3374         9321 :   create_input_operand (&ops[3], mask, mask_mode);
    3375         9321 :   expand_insn (icode, 4, ops);
    3376         9321 :   assign_call_lhs (lhs, target, &ops[0]);
    3377         9321 : }
    3378              : 
    3379              : /* Expand VEC_SET internal functions.  */
    3380              : 
    3381              : static void
    3382          123 : expand_vec_set_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
    3383              : {
    3384          123 :   tree lhs = gimple_call_lhs (stmt);
    3385          123 :   tree op0 = gimple_call_arg (stmt, 0);
    3386          123 :   tree op1 = gimple_call_arg (stmt, 1);
    3387          123 :   tree op2 = gimple_call_arg (stmt, 2);
    3388          123 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3389          123 :   rtx src = expand_normal (op0);
    3390              : 
    3391          123 :   machine_mode outermode = TYPE_MODE (TREE_TYPE (op0));
    3392          123 :   scalar_mode innermode = GET_MODE_INNER (outermode);
    3393              : 
    3394          123 :   rtx value = expand_normal (op1);
    3395          123 :   rtx pos = expand_normal (op2);
    3396              : 
    3397          123 :   class expand_operand ops[3];
    3398          123 :   enum insn_code icode = optab_handler (optab, outermode);
    3399              : 
    3400          123 :   if (icode != CODE_FOR_nothing)
    3401              :     {
    3402          123 :       rtx temp = gen_reg_rtx (outermode);
    3403          123 :       emit_move_insn (temp, src);
    3404              : 
    3405          123 :       create_fixed_operand (&ops[0], temp);
    3406          123 :       create_input_operand (&ops[1], value, innermode);
    3407          123 :       create_convert_operand_from (&ops[2], pos, TYPE_MODE (TREE_TYPE (op2)),
    3408              :                                    true);
    3409          123 :       if (maybe_expand_insn (icode, 3, ops))
    3410              :         {
    3411          123 :           emit_move_insn (target, temp);
    3412          123 :           return;
    3413              :         }
    3414              :     }
    3415            0 :   gcc_unreachable ();
    3416              : }
    3417              : 
    3418              : static void
    3419         1916 : expand_ABNORMAL_DISPATCHER (internal_fn, gcall *)
    3420              : {
    3421         1916 : }
    3422              : 
    3423              : static void
    3424            0 : expand_BUILTIN_EXPECT (internal_fn, gcall *stmt)
    3425              : {
    3426              :   /* When guessing was done, the hints should be already stripped away.  */
    3427            0 :   gcc_assert (!flag_guess_branch_prob || optimize == 0 || seen_error ());
    3428              : 
    3429            0 :   rtx target;
    3430            0 :   tree lhs = gimple_call_lhs (stmt);
    3431            0 :   if (lhs)
    3432            0 :     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3433              :   else
    3434            0 :     target = const0_rtx;
    3435            0 :   rtx val = expand_expr (gimple_call_arg (stmt, 0), target, VOIDmode, EXPAND_NORMAL);
    3436            0 :   if (lhs && val != target)
    3437            0 :     emit_move_insn (target, val);
    3438            0 : }
    3439              : 
    3440              : /* IFN_VA_ARG is supposed to be expanded at pass_stdarg.  So this dummy function
    3441              :    should never be called.  */
    3442              : 
    3443              : static void
    3444            0 : expand_VA_ARG (internal_fn, gcall *)
    3445              : {
    3446            0 :   gcc_unreachable ();
    3447              : }
    3448              : 
    3449              : /* IFN_VEC_CONVERT is supposed to be expanded at pass_lower_vector.  So this
    3450              :    dummy function should never be called.  */
    3451              : 
    3452              : static void
    3453            0 : expand_VEC_CONVERT (internal_fn, gcall *)
    3454              : {
    3455            0 :   gcc_unreachable ();
    3456              : }
    3457              : 
    3458              : /* Expand IFN_RAWMEMCHR internal function.  */
    3459              : 
    3460              : void
    3461            0 : expand_RAWMEMCHR (internal_fn, gcall *stmt)
    3462              : {
    3463            0 :   expand_operand ops[3];
    3464              : 
    3465            0 :   tree lhs = gimple_call_lhs (stmt);
    3466            0 :   if (!lhs)
    3467            0 :     return;
    3468            0 :   machine_mode lhs_mode = TYPE_MODE (TREE_TYPE (lhs));
    3469            0 :   rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3470            0 :   create_call_lhs_operand (&ops[0], lhs_rtx, lhs_mode);
    3471              : 
    3472            0 :   tree mem = gimple_call_arg (stmt, 0);
    3473            0 :   rtx mem_rtx = get_memory_rtx (mem, NULL, false);
    3474            0 :   create_fixed_operand (&ops[1], mem_rtx);
    3475              : 
    3476            0 :   tree pattern = gimple_call_arg (stmt, 1);
    3477            0 :   machine_mode mode = TYPE_MODE (TREE_TYPE (pattern));
    3478            0 :   rtx pattern_rtx = expand_normal (pattern);
    3479            0 :   create_input_operand (&ops[2], pattern_rtx, mode);
    3480              : 
    3481            0 :   insn_code icode = direct_optab_handler (rawmemchr_optab, mode);
    3482              : 
    3483            0 :   expand_insn (icode, 3, ops);
    3484            0 :   assign_call_lhs (lhs, lhs_rtx, &ops[0]);
    3485              : }
    3486              : 
    3487              : /* Expand the IFN_UNIQUE function according to its first argument.  */
    3488              : 
    3489              : static void
    3490            0 : expand_UNIQUE (internal_fn, gcall *stmt)
    3491              : {
    3492            0 :   rtx pattern = NULL_RTX;
    3493            0 :   enum ifn_unique_kind kind
    3494            0 :     = (enum ifn_unique_kind) TREE_INT_CST_LOW (gimple_call_arg (stmt, 0));
    3495              : 
    3496            0 :   switch (kind)
    3497              :     {
    3498            0 :     default:
    3499            0 :       gcc_unreachable ();
    3500              : 
    3501            0 :     case IFN_UNIQUE_UNSPEC:
    3502            0 :       if (targetm.have_unique ())
    3503            0 :         pattern = targetm.gen_unique ();
    3504              :       break;
    3505              : 
    3506            0 :     case IFN_UNIQUE_OACC_FORK:
    3507            0 :     case IFN_UNIQUE_OACC_JOIN:
    3508            0 :       if (targetm.have_oacc_fork () && targetm.have_oacc_join ())
    3509              :         {
    3510            0 :           tree lhs = gimple_call_lhs (stmt);
    3511            0 :           rtx target = const0_rtx;
    3512              : 
    3513            0 :           if (lhs)
    3514            0 :             target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3515              : 
    3516            0 :           rtx data_dep = expand_normal (gimple_call_arg (stmt, 1));
    3517            0 :           rtx axis = expand_normal (gimple_call_arg (stmt, 2));
    3518              : 
    3519            0 :           if (kind == IFN_UNIQUE_OACC_FORK)
    3520            0 :             pattern = targetm.gen_oacc_fork (target, data_dep, axis);
    3521              :           else
    3522            0 :             pattern = targetm.gen_oacc_join (target, data_dep, axis);
    3523              :         }
    3524              :       else
    3525            0 :         gcc_unreachable ();
    3526              :       break;
    3527              :     }
    3528              : 
    3529            0 :   if (pattern)
    3530            0 :     emit_insn (pattern);
    3531            0 : }
    3532              : 
    3533              : /* Expand the IFN_DEFERRED_INIT function:
    3534              :    LHS = DEFERRED_INIT (SIZE of the DECL, INIT_TYPE, NAME of the DECL);
    3535              : 
    3536              :    Initialize the LHS with zero/pattern according to its second argument
    3537              :    INIT_TYPE:
    3538              :    if INIT_TYPE is AUTO_INIT_ZERO, use zeroes to initialize;
    3539              :    if INIT_TYPE is AUTO_INIT_PATTERN, use 0xFE byte-repeatable pattern
    3540              :      to initialize;
    3541              :    The LHS variable is initialized including paddings.
    3542              :    The reasons to choose 0xFE for pattern initialization are:
    3543              :      1. It is a non-canonical virtual address on x86_64, and at the
    3544              :         high end of the i386 kernel address space.
    3545              :      2. It is a very large float value (-1.694739530317379e+38).
    3546              :      3. It is also an unusual number for integers.  */
    3547              : #define INIT_PATTERN_VALUE  0xFE
    3548              : static void
    3549        40901 : expand_DEFERRED_INIT (internal_fn, gcall *stmt)
    3550              : {
    3551        40901 :   tree lhs = gimple_call_lhs (stmt);
    3552        40901 :   tree var_size = gimple_call_arg (stmt, 0);
    3553        40901 :   enum auto_init_type init_type
    3554        40901 :     = (enum auto_init_type) TREE_INT_CST_LOW (gimple_call_arg (stmt, 1));
    3555        40901 :   bool reg_lhs = true;
    3556              : 
    3557        40901 :   tree var_type = TREE_TYPE (lhs);
    3558        40901 :   gcc_assert (init_type > AUTO_INIT_UNINITIALIZED);
    3559              : 
    3560        40901 :   if (TREE_CODE (lhs) == SSA_NAME)
    3561              :     reg_lhs = true;
    3562              :   else
    3563              :     {
    3564              :       tree lhs_base = lhs;
    3565        27690 :       while (handled_component_p (lhs_base))
    3566            4 :         lhs_base = TREE_OPERAND (lhs_base, 0);
    3567        27686 :       reg_lhs = (mem_ref_refers_to_non_mem_p (lhs_base)
    3568        27686 :                  || non_mem_decl_p (lhs_base));
    3569              :       /* If this expands to a register and the underlying decl is wrapped in
    3570              :          a MEM_REF that just serves as an access type change expose the decl
    3571              :          if it is of correct size.  This avoids a situation as in PR103271
    3572              :          if the target does not support a direct move to the registers mode.  */
    3573              :       if (reg_lhs
    3574         1833 :           && TREE_CODE (lhs_base) == MEM_REF
    3575            4 :           && TREE_CODE (TREE_OPERAND (lhs_base, 0)) == ADDR_EXPR
    3576            4 :           && DECL_P (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))
    3577            4 :           && integer_zerop (TREE_OPERAND (lhs_base, 1))
    3578            4 :           && tree_fits_uhwi_p (var_size)
    3579            4 :           && tree_int_cst_equal
    3580            4 :                (var_size,
    3581            4 :                 DECL_SIZE_UNIT (TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0))))
    3582              :         {
    3583            4 :           lhs = TREE_OPERAND (TREE_OPERAND (lhs_base, 0), 0);
    3584            4 :           var_type = TREE_TYPE (lhs);
    3585              :         }
    3586              :     }
    3587              : 
    3588        27686 :   if (!reg_lhs)
    3589              :     {
    3590              :       /* If the variable is not in register, expand to a memset
    3591              :          to initialize it.  */
    3592        25853 :       mark_addressable (lhs);
    3593        25853 :       tree var_addr = build_fold_addr_expr (lhs);
    3594              : 
    3595        25853 :       tree value = ((init_type & ~AUTO_INIT_CXX26) == AUTO_INIT_PATTERN)
    3596        25853 :                     ? build_int_cst (integer_type_node,
    3597              :                                      INIT_PATTERN_VALUE)
    3598        25776 :                     : integer_zero_node;
    3599        51706 :       tree m_call = build_call_expr (builtin_decl_implicit (BUILT_IN_MEMSET),
    3600              :                                      3, var_addr, value, var_size);
    3601              :       /* Expand this memset call.  */
    3602        25853 :       expand_builtin_memset (m_call, NULL_RTX, TYPE_MODE (var_type));
    3603              :     }
    3604              :   else
    3605              :     {
    3606              :       /* If this variable is in a register use expand_assignment.
    3607              :          For boolean scalars force zero-init.  */
    3608        15048 :       tree init;
    3609        15048 :       scalar_int_mode var_mode;
    3610        15048 :       if (TREE_CODE (TREE_TYPE (lhs)) != BOOLEAN_TYPE
    3611        14675 :           && tree_fits_uhwi_p (var_size)
    3612        14675 :           && ((init_type & ~AUTO_INIT_CXX26) == AUTO_INIT_PATTERN
    3613        14589 :               || !is_gimple_reg_type (var_type))
    3614         1919 :           && int_mode_for_size (tree_to_uhwi (var_size) * BITS_PER_UNIT,
    3615        15053 :                                 0).exists (&var_mode)
    3616        16967 :           && have_insn_for (SET, var_mode))
    3617              :         {
    3618         1914 :           unsigned HOST_WIDE_INT total_bytes = tree_to_uhwi (var_size);
    3619         1914 :           unsigned char *buf = XALLOCAVEC (unsigned char, total_bytes);
    3620         1914 :           memset (buf, ((init_type & ~AUTO_INIT_CXX26) == AUTO_INIT_PATTERN
    3621              :                         ? INIT_PATTERN_VALUE : 0), total_bytes);
    3622         1914 :           tree itype
    3623         1914 :             = build_nonstandard_integer_type (total_bytes * BITS_PER_UNIT, 1);
    3624         1914 :           wide_int w = wi::from_buffer (buf, total_bytes);
    3625         1914 :           init = wide_int_to_tree (itype, w);
    3626              :           /* Pun the LHS to make sure its type has constant size
    3627              :              unless it is an SSA name where that's already known.  */
    3628         1914 :           if (TREE_CODE (lhs) != SSA_NAME)
    3629         1833 :             lhs = build1 (VIEW_CONVERT_EXPR, itype, lhs);
    3630              :           else
    3631           81 :             init = fold_build1 (VIEW_CONVERT_EXPR, TREE_TYPE (lhs), init);
    3632         1914 :         }
    3633              :       else
    3634              :         /* Use zero-init also for variable-length sizes.  */
    3635        13134 :         init = build_zero_cst (var_type);
    3636              : 
    3637        15048 :       expand_assignment (lhs, init, false);
    3638              :     }
    3639        40901 : }
    3640              : 
    3641              : /* Expand the IFN_ACCESS_WITH_SIZE function:
    3642              :    ACCESS_WITH_SIZE (REF_TO_OBJ, REF_TO_SIZE,
    3643              :                      TYPE_OF_SIZE + ACCESS_MODE, TYPE_SIZE_UNIT for element)
    3644              :    which returns the REF_TO_OBJ same as the 1st argument;
    3645              : 
    3646              :    1st argument REF_TO_OBJ: The reference to the object;
    3647              :    2nd argument REF_TO_SIZE: The reference to the size of the object,
    3648              :    3rd argument TYPE_OF_SIZE + ACCESS_MODE: An integer constant with a pointer
    3649              :      TYPE.
    3650              :      The pointee TYPE of the pointer TYPE is the TYPE of the object referenced
    3651              :         by REF_TO_SIZE.
    3652              :      The integer constant value represents the ACCESS_MODE:
    3653              :         0: none
    3654              :         1: read_only
    3655              :         2: write_only
    3656              :         3: read_write
    3657              : 
    3658              :    4th argument: The TYPE_SIZE_UNIT of the element TYPE of the array.
    3659              : 
    3660              :    Both the return type and the type of the first argument of this
    3661              :    function have been converted from the incomplete array type to
    3662              :    the corresponding pointer type.
    3663              : 
    3664              :    For each call to a .ACCESS_WITH_SIZE, replace it with its 1st argument.  */
    3665              : 
    3666              : static void
    3667          521 : expand_ACCESS_WITH_SIZE (internal_fn, gcall *stmt)
    3668              : {
    3669          521 :   tree lhs = gimple_call_lhs (stmt);
    3670          521 :   tree ref_to_obj = gimple_call_arg (stmt, 0);
    3671          521 :   if (lhs)
    3672          521 :     expand_assignment (lhs, ref_to_obj, false);
    3673          521 : }
    3674              : 
    3675              : /* The size of an OpenACC compute dimension.  */
    3676              : 
    3677              : static void
    3678         5695 : expand_GOACC_DIM_SIZE (internal_fn, gcall *stmt)
    3679              : {
    3680         5695 :   tree lhs = gimple_call_lhs (stmt);
    3681              : 
    3682         5695 :   if (!lhs)
    3683              :     return;
    3684              : 
    3685         5695 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3686         5695 :   if (targetm.have_oacc_dim_size ())
    3687              :     {
    3688            0 :       rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
    3689              :                              VOIDmode, EXPAND_NORMAL);
    3690            0 :       emit_insn (targetm.gen_oacc_dim_size (target, dim));
    3691              :     }
    3692              :   else
    3693         5695 :     emit_move_insn (target, GEN_INT (1));
    3694              : }
    3695              : 
    3696              : /* The position of an OpenACC execution engine along one compute axis.  */
    3697              : 
    3698              : static void
    3699         4567 : expand_GOACC_DIM_POS (internal_fn, gcall *stmt)
    3700              : {
    3701         4567 :   tree lhs = gimple_call_lhs (stmt);
    3702              : 
    3703         4567 :   if (!lhs)
    3704              :     return;
    3705              : 
    3706         4567 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3707         4567 :   if (targetm.have_oacc_dim_pos ())
    3708              :     {
    3709            0 :       rtx dim = expand_expr (gimple_call_arg (stmt, 0), NULL_RTX,
    3710              :                              VOIDmode, EXPAND_NORMAL);
    3711            0 :       emit_insn (targetm.gen_oacc_dim_pos (target, dim));
    3712              :     }
    3713              :   else
    3714         4567 :     emit_move_insn (target, const0_rtx);
    3715              : }
    3716              : 
    3717              : /* This is expanded by oacc_device_lower pass.  */
    3718              : 
    3719              : static void
    3720            0 : expand_GOACC_LOOP (internal_fn, gcall *)
    3721              : {
    3722            0 :   gcc_unreachable ();
    3723              : }
    3724              : 
    3725              : /* This is expanded by oacc_device_lower pass.  */
    3726              : 
    3727              : static void
    3728            0 : expand_GOACC_REDUCTION (internal_fn, gcall *)
    3729              : {
    3730            0 :   gcc_unreachable ();
    3731              : }
    3732              : 
    3733              : /* This is expanded by oacc_device_lower pass.  */
    3734              : 
    3735              : static void
    3736            0 : expand_GOACC_TILE (internal_fn, gcall *)
    3737              : {
    3738            0 :   gcc_unreachable ();
    3739              : }
    3740              : 
    3741              : /* Set errno to EDOM.  */
    3742              : 
    3743              : static void
    3744            0 : expand_SET_EDOM (internal_fn, gcall *)
    3745              : {
    3746              : #ifdef TARGET_EDOM
    3747              : #ifdef GEN_ERRNO_RTX
    3748              :   rtx errno_rtx = GEN_ERRNO_RTX;
    3749              : #else
    3750              :   rtx errno_rtx = gen_rtx_MEM (word_mode, gen_rtx_SYMBOL_REF (Pmode, "errno"));
    3751              : #endif
    3752              :   emit_move_insn (errno_rtx,
    3753              :                   gen_int_mode (TARGET_EDOM, GET_MODE (errno_rtx)));
    3754              : #else
    3755            0 :   gcc_unreachable ();
    3756              : #endif
    3757              : }
    3758              : 
    3759              : /* Expand atomic bit test and set.  */
    3760              : 
    3761              : static void
    3762          152 : expand_ATOMIC_BIT_TEST_AND_SET (internal_fn, gcall *call)
    3763              : {
    3764          152 :   expand_ifn_atomic_bit_test_and (call);
    3765          152 : }
    3766              : 
    3767              : /* Expand atomic bit test and complement.  */
    3768              : 
    3769              : static void
    3770          146 : expand_ATOMIC_BIT_TEST_AND_COMPLEMENT (internal_fn, gcall *call)
    3771              : {
    3772          146 :   expand_ifn_atomic_bit_test_and (call);
    3773          146 : }
    3774              : 
    3775              : /* Expand atomic bit test and reset.  */
    3776              : 
    3777              : static void
    3778          135 : expand_ATOMIC_BIT_TEST_AND_RESET (internal_fn, gcall *call)
    3779              : {
    3780          135 :   expand_ifn_atomic_bit_test_and (call);
    3781          135 : }
    3782              : 
    3783              : /* Expand atomic bit test and set.  */
    3784              : 
    3785              : static void
    3786        14041 : expand_ATOMIC_COMPARE_EXCHANGE (internal_fn, gcall *call)
    3787              : {
    3788        14041 :   expand_ifn_atomic_compare_exchange (call);
    3789        14041 : }
    3790              : 
    3791              : /* Expand atomic add fetch and cmp with 0.  */
    3792              : 
    3793              : static void
    3794          942 : expand_ATOMIC_ADD_FETCH_CMP_0 (internal_fn, gcall *call)
    3795              : {
    3796          942 :   expand_ifn_atomic_op_fetch_cmp_0 (call);
    3797          942 : }
    3798              : 
    3799              : /* Expand atomic sub fetch and cmp with 0.  */
    3800              : 
    3801              : static void
    3802          745 : expand_ATOMIC_SUB_FETCH_CMP_0 (internal_fn, gcall *call)
    3803              : {
    3804          745 :   expand_ifn_atomic_op_fetch_cmp_0 (call);
    3805          745 : }
    3806              : 
    3807              : /* Expand atomic and fetch and cmp with 0.  */
    3808              : 
    3809              : static void
    3810          176 : expand_ATOMIC_AND_FETCH_CMP_0 (internal_fn, gcall *call)
    3811              : {
    3812          176 :   expand_ifn_atomic_op_fetch_cmp_0 (call);
    3813          176 : }
    3814              : 
    3815              : /* Expand atomic or fetch and cmp with 0.  */
    3816              : 
    3817              : static void
    3818          112 : expand_ATOMIC_OR_FETCH_CMP_0 (internal_fn, gcall *call)
    3819              : {
    3820          112 :   expand_ifn_atomic_op_fetch_cmp_0 (call);
    3821          112 : }
    3822              : 
    3823              : /* Expand atomic xor fetch and cmp with 0.  */
    3824              : 
    3825              : static void
    3826          176 : expand_ATOMIC_XOR_FETCH_CMP_0 (internal_fn, gcall *call)
    3827              : {
    3828          176 :   expand_ifn_atomic_op_fetch_cmp_0 (call);
    3829          176 : }
    3830              : 
    3831              : /* Expand LAUNDER to assignment, lhs = arg0.  */
    3832              : 
    3833              : static void
    3834           29 : expand_LAUNDER (internal_fn, gcall *call)
    3835              : {
    3836           29 :   tree lhs = gimple_call_lhs (call);
    3837              : 
    3838           29 :   if (!lhs)
    3839              :     return;
    3840              : 
    3841           28 :   expand_assignment (lhs, gimple_call_arg (call, 0), false);
    3842              : }
    3843              : 
    3844              : /* Expand {MASK_,}SCATTER_STORE{S,U} call CALL using optab OPTAB.  */
    3845              : 
    3846              : static void
    3847            0 : expand_scatter_store_optab_fn (internal_fn, gcall *stmt, direct_optab optab)
    3848              : {
    3849            0 :   internal_fn ifn = gimple_call_internal_fn (stmt);
    3850            0 :   int rhs_index = internal_fn_stored_value_index (ifn);
    3851            0 :   tree base = gimple_call_arg (stmt, 0);
    3852            0 :   tree offset = gimple_call_arg (stmt, internal_fn_offset_index (ifn));
    3853            0 :   tree scale = gimple_call_arg (stmt, internal_fn_scale_index (ifn));
    3854            0 :   tree rhs = gimple_call_arg (stmt, rhs_index);
    3855              : 
    3856            0 :   rtx base_rtx = expand_normal (base);
    3857            0 :   rtx offset_rtx = expand_normal (offset);
    3858            0 :   HOST_WIDE_INT scale_int = tree_to_shwi (scale);
    3859            0 :   rtx rhs_rtx = expand_normal (rhs);
    3860              : 
    3861            0 :   class expand_operand ops[8];
    3862            0 :   int i = 0;
    3863            0 :   create_address_operand (&ops[i++], base_rtx);
    3864            0 :   create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
    3865            0 :   create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
    3866            0 :   create_integer_operand (&ops[i++], scale_int);
    3867            0 :   create_input_operand (&ops[i++], rhs_rtx, TYPE_MODE (TREE_TYPE (rhs)));
    3868            0 :   i = add_mask_else_and_len_args (ops, i, stmt);
    3869              : 
    3870            0 :   insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (rhs)),
    3871            0 :                                            TYPE_MODE (TREE_TYPE (offset)));
    3872            0 :   expand_insn (icode, i, ops);
    3873            0 : }
    3874              : 
    3875              : #define expand_mask_scatter_store_optab_fn expand_scatter_store_optab_fn
    3876              : #define expand_mask_len_scatter_store_optab_fn expand_scatter_store_optab_fn
    3877              : 
    3878              : /* Expand {MASK_,}GATHER_LOAD call CALL using optab OPTAB.  */
    3879              : 
    3880              : static void
    3881            0 : expand_gather_load_optab_fn (internal_fn ifn, gcall *stmt, direct_optab optab)
    3882              : {
    3883            0 :   tree lhs = gimple_call_lhs (stmt);
    3884            0 :   tree base = gimple_call_arg (stmt, 0);
    3885            0 :   tree offset = gimple_call_arg (stmt, internal_fn_offset_index (ifn));
    3886            0 :   tree scale = gimple_call_arg (stmt, internal_fn_scale_index (ifn));
    3887              : 
    3888            0 :   rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3889            0 :   rtx base_rtx = expand_normal (base);
    3890            0 :   rtx offset_rtx = expand_normal (offset);
    3891            0 :   HOST_WIDE_INT scale_int = tree_to_shwi (scale);
    3892              : 
    3893            0 :   int i = 0;
    3894            0 :   class expand_operand ops[9];
    3895            0 :   create_call_lhs_operand (&ops[i++], lhs_rtx, TYPE_MODE (TREE_TYPE (lhs)));
    3896            0 :   create_address_operand (&ops[i++], base_rtx);
    3897            0 :   create_input_operand (&ops[i++], offset_rtx, TYPE_MODE (TREE_TYPE (offset)));
    3898            0 :   create_integer_operand (&ops[i++], TYPE_UNSIGNED (TREE_TYPE (offset)));
    3899            0 :   create_integer_operand (&ops[i++], scale_int);
    3900            0 :   i = add_mask_else_and_len_args (ops, i, stmt);
    3901            0 :   insn_code icode = convert_optab_handler (optab, TYPE_MODE (TREE_TYPE (lhs)),
    3902            0 :                                            TYPE_MODE (TREE_TYPE (offset)));
    3903            0 :   expand_insn (icode, i, ops);
    3904            0 :   assign_call_lhs (lhs, lhs_rtx, &ops[0]);
    3905            0 : }
    3906              : 
    3907              : #define expand_mask_gather_load_optab_fn expand_gather_load_optab_fn
    3908              : #define expand_mask_len_gather_load_optab_fn expand_gather_load_optab_fn
    3909              : 
    3910              : /* Expand MASK_LEN_STRIDED_LOAD call CALL by optab OPTAB.  */
    3911              : 
    3912              : static void
    3913            0 : expand_strided_load_optab_fn (ATTRIBUTE_UNUSED internal_fn, gcall *stmt,
    3914              :                               direct_optab optab)
    3915              : {
    3916            0 :   tree lhs = gimple_call_lhs (stmt);
    3917            0 :   tree base = gimple_call_arg (stmt, 0);
    3918            0 :   tree stride = gimple_call_arg (stmt, 1);
    3919              : 
    3920            0 :   rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    3921            0 :   rtx base_rtx = expand_normal (base);
    3922            0 :   rtx stride_rtx = expand_normal (stride);
    3923              : 
    3924            0 :   unsigned i = 0;
    3925            0 :   class expand_operand ops[7];
    3926            0 :   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
    3927              : 
    3928            0 :   create_output_operand (&ops[i++], lhs_rtx, mode);
    3929            0 :   create_address_operand (&ops[i++], base_rtx);
    3930            0 :   create_address_operand (&ops[i++], stride_rtx);
    3931              : 
    3932            0 :   i = add_mask_else_and_len_args (ops, i, stmt);
    3933            0 :   expand_insn (direct_optab_handler (optab, mode), i, ops);
    3934              : 
    3935            0 :   if (!rtx_equal_p (lhs_rtx, ops[0].value))
    3936            0 :     emit_move_insn (lhs_rtx, ops[0].value);
    3937            0 : }
    3938              : 
    3939              : /* Expand MASK_LEN_STRIDED_STORE call CALL by optab OPTAB.  */
    3940              : 
    3941              : static void
    3942            0 : expand_strided_store_optab_fn (ATTRIBUTE_UNUSED internal_fn, gcall *stmt,
    3943              :                                direct_optab optab)
    3944              : {
    3945            0 :   internal_fn fn = gimple_call_internal_fn (stmt);
    3946            0 :   int rhs_index = internal_fn_stored_value_index (fn);
    3947              : 
    3948            0 :   tree base = gimple_call_arg (stmt, 0);
    3949            0 :   tree stride = gimple_call_arg (stmt, 1);
    3950            0 :   tree rhs = gimple_call_arg (stmt, rhs_index);
    3951              : 
    3952            0 :   rtx base_rtx = expand_normal (base);
    3953            0 :   rtx stride_rtx = expand_normal (stride);
    3954            0 :   rtx rhs_rtx = expand_normal (rhs);
    3955              : 
    3956            0 :   unsigned i = 0;
    3957            0 :   class expand_operand ops[6];
    3958            0 :   machine_mode mode = TYPE_MODE (TREE_TYPE (rhs));
    3959              : 
    3960            0 :   create_address_operand (&ops[i++], base_rtx);
    3961            0 :   create_address_operand (&ops[i++], stride_rtx);
    3962            0 :   create_input_operand (&ops[i++], rhs_rtx, mode);
    3963              : 
    3964            0 :   i = add_mask_else_and_len_args (ops, i, stmt);
    3965            0 :   expand_insn (direct_optab_handler (optab, mode), i, ops);
    3966            0 : }
    3967              : 
    3968              : /* Helper for expand_DIVMOD.  Return true if the sequence starting with
    3969              :    INSN contains any call insns or insns with {,U}{DIV,MOD} rtxes.  */
    3970              : 
    3971              : static bool
    3972          838 : contains_call_div_mod (rtx_insn *insn)
    3973              : {
    3974          838 :   subrtx_iterator::array_type array;
    3975         4821 :   for (; insn; insn = NEXT_INSN (insn))
    3976         4821 :     if (CALL_P (insn))
    3977              :       return true;
    3978         3983 :     else if (INSN_P (insn))
    3979        17804 :       FOR_EACH_SUBRTX (iter, array, PATTERN (insn), NONCONST)
    3980        13821 :         switch (GET_CODE (*iter))
    3981              :           {
    3982            0 :           case CALL:
    3983            0 :           case DIV:
    3984            0 :           case UDIV:
    3985            0 :           case MOD:
    3986            0 :           case UMOD:
    3987            0 :             return true;
    3988        13821 :           default:
    3989        13821 :             break;
    3990              :           }
    3991              :   return false;
    3992          838 :  }
    3993              : 
    3994              : /* Expand DIVMOD() using:
    3995              :  a) optab handler for udivmod/sdivmod if it is available.
    3996              :  b) If optab_handler doesn't exist, generate call to
    3997              :     target-specific divmod libfunc.  */
    3998              : 
    3999              : static void
    4000        11531 : expand_DIVMOD (internal_fn, gcall *call_stmt)
    4001              : {
    4002        11531 :   tree lhs = gimple_call_lhs (call_stmt);
    4003        11531 :   tree arg0 = gimple_call_arg (call_stmt, 0);
    4004        11531 :   tree arg1 = gimple_call_arg (call_stmt, 1);
    4005              : 
    4006        11531 :   gcc_assert (TREE_CODE (TREE_TYPE (lhs)) == COMPLEX_TYPE);
    4007        11531 :   tree type = TREE_TYPE (TREE_TYPE (lhs));
    4008        11531 :   machine_mode mode = TYPE_MODE (type);
    4009        11531 :   bool unsignedp = TYPE_UNSIGNED (type);
    4010        11531 :   optab tab = (unsignedp) ? udivmod_optab : sdivmod_optab;
    4011              : 
    4012        11531 :   rtx op0 = expand_normal (arg0);
    4013        11531 :   rtx op1 = expand_normal (arg1);
    4014        11531 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    4015              : 
    4016        11531 :   rtx quotient = NULL_RTX, remainder = NULL_RTX;
    4017        11531 :   rtx_insn *insns = NULL;
    4018              : 
    4019        11531 :   if (TREE_CODE (arg1) == INTEGER_CST)
    4020              :     {
    4021              :       /* For DIVMOD by integral constants, there could be efficient code
    4022              :          expanded inline e.g. using shifts and plus/minus.  Try to expand
    4023              :          the division and modulo and if it emits any library calls or any
    4024              :          {,U}{DIV,MOD} rtxes throw it away and use a divmod optab or
    4025              :          divmod libcall.  */
    4026         1182 :       scalar_int_mode int_mode;
    4027         1182 :       if (remainder == NULL_RTX
    4028         1182 :           && optimize
    4029         1182 :           && CONST_INT_P (op1)
    4030         1178 :           && !pow2p_hwi (INTVAL (op1))
    4031         2356 :           && is_int_mode (TYPE_MODE (type), &int_mode)
    4032         1342 :           && GET_MODE_SIZE (int_mode) == 2 * UNITS_PER_WORD
    4033         1178 :           && optab_handler (and_optab, word_mode) != CODE_FOR_nothing
    4034         1178 :           && optab_handler (add_optab, word_mode) != CODE_FOR_nothing
    4035         1178 :           && optimize_insn_for_speed_p ())
    4036              :         {
    4037         1178 :           rtx_insn *last = get_last_insn ();
    4038         1178 :           remainder = NULL_RTX;
    4039         3534 :           quotient = expand_doubleword_divmod (int_mode, op0, op1, &remainder,
    4040         1178 :                                                TYPE_UNSIGNED (type));
    4041         1178 :           if (quotient != NULL_RTX)
    4042              :             {
    4043          344 :               if (optab_handler (mov_optab, int_mode) != CODE_FOR_nothing)
    4044              :                 {
    4045          344 :                   rtx_insn *move = emit_move_insn (quotient, quotient);
    4046          688 :                   set_dst_reg_note (move, REG_EQUAL,
    4047          344 :                                     gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
    4048              :                                                     ? UDIV : DIV, int_mode,
    4049              :                                                     copy_rtx (op0), op1),
    4050              :                                     quotient);
    4051          344 :                   move = emit_move_insn (remainder, remainder);
    4052          688 :                   set_dst_reg_note (move, REG_EQUAL,
    4053          344 :                                     gen_rtx_fmt_ee (TYPE_UNSIGNED (type)
    4054              :                                                     ? UMOD : MOD, int_mode,
    4055              :                                                     copy_rtx (op0), op1),
    4056              :                                     quotient);
    4057              :                 }
    4058              :             }
    4059              :           else
    4060          834 :             delete_insns_since (last);
    4061              :         }
    4062              : 
    4063         1182 :       if (remainder == NULL_RTX)
    4064              :         {
    4065          838 :           struct separate_ops ops;
    4066          838 :           ops.code = TRUNC_DIV_EXPR;
    4067          838 :           ops.type = type;
    4068          838 :           ops.op0 = make_tree (ops.type, op0);
    4069          838 :           ops.op1 = arg1;
    4070          838 :           ops.op2 = NULL_TREE;
    4071          838 :           ops.location = gimple_location (call_stmt);
    4072          838 :           start_sequence ();
    4073          838 :           quotient = expand_expr_real_2 (&ops, NULL_RTX, mode, EXPAND_NORMAL);
    4074          838 :           if (contains_call_div_mod (get_insns ()))
    4075          838 :             quotient = NULL_RTX;
    4076              :           else
    4077              :             {
    4078            0 :               ops.code = TRUNC_MOD_EXPR;
    4079            0 :               remainder = expand_expr_real_2 (&ops, NULL_RTX, mode,
    4080              :                                               EXPAND_NORMAL);
    4081            0 :               if (contains_call_div_mod (get_insns ()))
    4082            0 :                 remainder = NULL_RTX;
    4083              :             }
    4084          838 :           if (remainder)
    4085            0 :             insns = get_insns ();
    4086          838 :           end_sequence ();
    4087              :         }
    4088              :     }
    4089              : 
    4090        11531 :   if (remainder)
    4091          344 :     emit_insn (insns);
    4092              : 
    4093              :   /* Check if optab_handler exists for divmod_optab for given mode.  */
    4094        11187 :   else if (optab_handler (tab, mode) != CODE_FOR_nothing)
    4095              :     {
    4096        10295 :       quotient = gen_reg_rtx (mode);
    4097        10295 :       remainder = gen_reg_rtx (mode);
    4098        10295 :       expand_twoval_binop (tab, op0, op1, quotient, remainder, unsignedp);
    4099              :     }
    4100              : 
    4101              :   /* Generate call to divmod libfunc if it exists.  */
    4102          892 :   else if (rtx libfunc = optab_libfunc (tab, mode))
    4103          892 :     targetm.expand_divmod_libfunc (libfunc, mode, op0, op1,
    4104              :                                    &quotient, &remainder);
    4105              : 
    4106              :   else
    4107            0 :     gcc_unreachable ();
    4108              : 
    4109              :   /* Wrap the return value (quotient, remainder) within COMPLEX_EXPR.  */
    4110        23062 :   expand_expr (build2 (COMPLEX_EXPR, TREE_TYPE (lhs),
    4111        11531 :                        make_tree (TREE_TYPE (arg0), quotient),
    4112        11531 :                        make_tree (TREE_TYPE (arg1), remainder)),
    4113              :                target, VOIDmode, EXPAND_NORMAL);
    4114        11531 : }
    4115              : 
    4116              : /* Expand a NOP.  */
    4117              : 
    4118              : static void
    4119            1 : expand_NOP (internal_fn, gcall *)
    4120              : {
    4121              :   /* Nothing.  But it shouldn't really prevail.  */
    4122            1 : }
    4123              : 
    4124              : /* Coroutines, all should have been processed at this stage.  */
    4125              : 
    4126              : static void
    4127            0 : expand_CO_FRAME (internal_fn, gcall *)
    4128              : {
    4129            0 :   gcc_unreachable ();
    4130              : }
    4131              : 
    4132              : static void
    4133            0 : expand_CO_YIELD (internal_fn, gcall *)
    4134              : {
    4135            0 :   gcc_unreachable ();
    4136              : }
    4137              : 
    4138              : static void
    4139            0 : expand_CO_SUSPN (internal_fn, gcall *)
    4140              : {
    4141            0 :   gcc_unreachable ();
    4142              : }
    4143              : 
    4144              : static void
    4145            0 : expand_CO_ACTOR (internal_fn, gcall *)
    4146              : {
    4147            0 :   gcc_unreachable ();
    4148              : }
    4149              : 
    4150              : /* Expand a call to FN using the operands in STMT.  FN has a single
    4151              :    output operand and NARGS input operands.  */
    4152              : 
    4153              : static void
    4154        76977 : expand_direct_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab,
    4155              :                         unsigned int nargs)
    4156              : {
    4157        76977 :   gcc_checking_assert (direct_internal_fn_nargs_ok_p
    4158              :                        (fn, gimple_call_num_args (stmt)));
    4159        76977 :   tree_pair types = direct_internal_fn_types (fn, stmt);
    4160        76977 :   insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
    4161        76977 :   expand_fn_using_insn (stmt, icode, 1, nargs);
    4162        76977 : }
    4163              : 
    4164              : /* Expand WHILE_ULT call STMT using optab OPTAB.  */
    4165              : 
    4166              : static void
    4167            0 : expand_while_optab_fn (internal_fn, gcall *stmt, convert_optab optab)
    4168              : {
    4169            0 :   expand_operand ops[4];
    4170            0 :   tree rhs_type[2];
    4171              : 
    4172            0 :   tree lhs = gimple_call_lhs (stmt);
    4173            0 :   tree lhs_type = TREE_TYPE (lhs);
    4174            0 :   rtx lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    4175            0 :   create_call_lhs_operand (&ops[0], lhs_rtx, TYPE_MODE (lhs_type));
    4176              : 
    4177            0 :   for (unsigned int i = 0; i < 2; ++i)
    4178              :     {
    4179            0 :       tree rhs = gimple_call_arg (stmt, i);
    4180            0 :       rhs_type[i] = TREE_TYPE (rhs);
    4181            0 :       rtx rhs_rtx = expand_normal (rhs);
    4182            0 :       create_input_operand (&ops[i + 1], rhs_rtx, TYPE_MODE (rhs_type[i]));
    4183              :     }
    4184              : 
    4185            0 :   int opcnt;
    4186            0 :   if (!VECTOR_MODE_P (TYPE_MODE (lhs_type)))
    4187              :     {
    4188              :       /* When the mask is an integer mode the exact vector length may not
    4189              :          be clear to the backend, so we pass it in operand[3].
    4190              :          Use the vector in arg2 for the most reliable intended size.  */
    4191            0 :       tree type = TREE_TYPE (gimple_call_arg (stmt, 2));
    4192            0 :       create_integer_operand (&ops[3], TYPE_VECTOR_SUBPARTS (type));
    4193            0 :       opcnt = 4;
    4194              :     }
    4195              :   else
    4196              :     /* The mask has a vector type so the length operand is unnecessary.  */
    4197              :     opcnt = 3;
    4198              : 
    4199            0 :   insn_code icode = convert_optab_handler (optab, TYPE_MODE (rhs_type[0]),
    4200            0 :                                            TYPE_MODE (lhs_type));
    4201              : 
    4202            0 :   expand_insn (icode, opcnt, ops);
    4203            0 :   assign_call_lhs (lhs, lhs_rtx, &ops[0]);
    4204            0 : }
    4205              : 
    4206              : /* Expand a call to a convert-like optab using the operands in STMT.
    4207              :    FN has a single output operand and NARGS input operands.  */
    4208              : 
    4209              : static void
    4210          481 : expand_convert_optab_fn (internal_fn fn, gcall *stmt, convert_optab optab,
    4211              :                          unsigned int nargs)
    4212              : {
    4213          481 :   gcc_checking_assert (direct_internal_fn_nargs_ok_p
    4214              :                        (fn, gimple_call_num_args (stmt)));
    4215          481 :   tree_pair types = direct_internal_fn_types (fn, stmt);
    4216          481 :   insn_code icode = convert_optab_handler (optab, TYPE_MODE (types.first),
    4217          481 :                                           TYPE_MODE (types.second));
    4218          481 :   expand_fn_using_insn (stmt, icode, 1, nargs);
    4219          481 : }
    4220              : 
    4221              : /* Expand CRC call STMT.  */
    4222              : 
    4223              : static void
    4224          351 : expand_crc_optab_fn (internal_fn fn, gcall *stmt, convert_optab optab)
    4225              : {
    4226          351 :   tree lhs = gimple_call_lhs (stmt);
    4227          351 :   tree rhs1 = gimple_call_arg (stmt, 0); // crc
    4228          351 :   tree rhs2 = gimple_call_arg (stmt, 1); // data
    4229          351 :   tree rhs3 = gimple_call_arg (stmt, 2); // polynomial
    4230              : 
    4231          351 :   tree result_type = TREE_TYPE (lhs);
    4232          351 :   tree data_type = TREE_TYPE (rhs2);
    4233              : 
    4234          351 :   gcc_assert (TYPE_MODE (result_type) >= TYPE_MODE (data_type));
    4235              : 
    4236          351 :   rtx dest = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    4237          351 :   rtx crc = expand_normal (rhs1);
    4238          351 :   rtx data = expand_normal (rhs2);
    4239          351 :   rtx polynomial;
    4240          351 :   if (TREE_CODE (rhs3) != INTEGER_CST)
    4241              :     {
    4242            0 :       error ("third argument to %<crc%> builtins must be a constant");
    4243            0 :       polynomial = const0_rtx;
    4244              :     }
    4245              :   else
    4246          351 :     polynomial = convert_to_mode (TYPE_MODE (result_type), expand_normal (rhs3), 0);
    4247              : 
    4248              :   /* Use target specific expansion if it exists.
    4249              :      Otherwise, generate table-based CRC.  */
    4250          351 :   if (direct_internal_fn_supported_p (fn, tree_pair (data_type, result_type),
    4251              :                                       OPTIMIZE_FOR_SPEED))
    4252              :     {
    4253           43 :       class expand_operand ops[4];
    4254              : 
    4255           43 :       if (dump_file && (dump_flags & TDF_DETAILS))
    4256              :         {
    4257            0 :           fprintf (dump_file,
    4258              :                    ";; using optab for crc_%u_polynomial_"
    4259              :                    HOST_WIDE_INT_PRINT_HEX "\n",
    4260            0 :                    GET_MODE_BITSIZE (GET_MODE (dest)).to_constant (),
    4261            0 :                    TREE_INT_CST_LOW (rhs3));
    4262              :         }
    4263              : 
    4264           43 :       create_call_lhs_operand (&ops[0], dest, TYPE_MODE (result_type));
    4265           43 :       create_input_operand (&ops[1], crc, TYPE_MODE (result_type));
    4266           43 :       create_input_operand (&ops[2], data, TYPE_MODE (data_type));
    4267           43 :       create_input_operand (&ops[3], polynomial, TYPE_MODE (result_type));
    4268           43 :       insn_code icode = convert_optab_handler (optab, TYPE_MODE (data_type),
    4269           43 :                                                TYPE_MODE (result_type));
    4270           43 :       expand_insn (icode, 4, ops);
    4271           43 :       assign_call_lhs (lhs, dest, &ops[0]);
    4272              :     }
    4273              :   else
    4274              :     {
    4275              :       /* We're bypassing all the operand conversions that are done in the
    4276              :          case when we get an icode, operands and pass that off to expand_insn.
    4277              : 
    4278              :          That path has special case handling for promoted return values which
    4279              :          we must emulate here (is the same kind of special treatment ever
    4280              :          needed for input arguments here?).
    4281              : 
    4282              :          In particular we do not want to store directly into a promoted
    4283              :          SUBREG destination, instead store into a suitably sized pseudo.  */
    4284          308 :       rtx orig_dest = dest;
    4285          308 :       if (SUBREG_P (dest) && SUBREG_PROMOTED_VAR_P (dest))
    4286            0 :         dest = gen_reg_rtx (GET_MODE (dest));
    4287              : 
    4288              :       /* If it's IFN_CRC generate bit-forward CRC.  */
    4289          308 :       if (fn == IFN_CRC)
    4290          151 :         expand_crc_table_based (dest, crc, data, polynomial,
    4291          151 :                                 TYPE_MODE (data_type));
    4292              :       else
    4293              :         /* If it's IFN_CRC_REV generate bit-reversed CRC.  */
    4294          157 :         expand_reversed_crc_table_based (dest, crc, data, polynomial,
    4295          157 :                                          TYPE_MODE (data_type));
    4296              : 
    4297              :       /* Now get the return value where it needs to be, taking care to
    4298              :          ensure it's promoted appropriately if the ABI demands it.
    4299              : 
    4300              :          Re-use assign_call_lhs to handle the details.  */
    4301          308 :       class expand_operand ops[4];
    4302          308 :       create_call_lhs_operand (&ops[0], dest, TYPE_MODE (result_type));
    4303          308 :       ops[0].value = dest;
    4304          308 :       assign_call_lhs (lhs, orig_dest, &ops[0]);
    4305              :     }
    4306          351 : }
    4307              : 
    4308              : /* Expand .REDUC_SBOOL_{AND,IOR,XOR}.  */
    4309              : 
    4310              : static void
    4311           49 : expand_reduc_sbool_optab_fn (internal_fn fn, gcall *stmt, direct_optab optab)
    4312              : {
    4313           49 :   tree_pair types = direct_internal_fn_types (fn, stmt);
    4314           49 :   insn_code icode = direct_optab_handler (optab, TYPE_MODE (types.first));
    4315              : 
    4316              :   /* Below copied from expand_fn_using_insn.  */
    4317              : 
    4318           49 :   gcc_assert (icode != CODE_FOR_nothing);
    4319              : 
    4320           49 :   expand_operand *ops = XALLOCAVEC (expand_operand, 3);
    4321           49 :   rtx lhs_rtx = NULL_RTX;
    4322           49 :   tree lhs = gimple_call_lhs (stmt);
    4323           49 :   if (lhs)
    4324           49 :     lhs_rtx = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    4325           49 :   create_call_lhs_operand (&ops[0], lhs_rtx,
    4326           49 :                            insn_data[icode].operand[0].mode);
    4327              : 
    4328           49 :   tree rhs = gimple_call_arg (stmt, 0);
    4329           49 :   tree rhs_type = TREE_TYPE (rhs);
    4330           49 :   rtx rhs_rtx = expand_normal (rhs);
    4331           49 :   gcc_assert (VECTOR_BOOLEAN_TYPE_P (rhs_type));
    4332           49 :   create_input_operand (&ops[1], rhs_rtx, TYPE_MODE (rhs_type));
    4333           49 :   if (SCALAR_INT_MODE_P (TYPE_MODE (rhs_type)))
    4334              :     {
    4335           24 :       rtx nunits = GEN_INT (TYPE_VECTOR_SUBPARTS (rhs_type).to_constant ());
    4336           24 :       gcc_assert (insn_operand_matches (icode, 2, nunits));
    4337           24 :       create_input_operand (&ops[2], nunits, SImode);
    4338              :     }
    4339           73 :   expand_insn (icode, SCALAR_INT_MODE_P (TYPE_MODE (rhs_type)) ? 3 : 2, ops);
    4340           49 :   if (lhs_rtx)
    4341           49 :     assign_call_lhs (lhs, lhs_rtx, &ops[0]);
    4342           49 : }
    4343              : 
    4344              : /* Expanders for optabs that can use expand_direct_optab_fn.  */
    4345              : 
    4346              : #define expand_unary_optab_fn(FN, STMT, OPTAB) \
    4347              :   expand_direct_optab_fn (FN, STMT, OPTAB, 1)
    4348              : 
    4349              : #define expand_unary_opt_optab_fn expand_unary_optab_fn
    4350              : 
    4351              : #define expand_binary_optab_fn(FN, STMT, OPTAB) \
    4352              :   expand_direct_optab_fn (FN, STMT, OPTAB, 2)
    4353              : 
    4354              : #define expand_ternary_optab_fn(FN, STMT, OPTAB) \
    4355              :   expand_direct_optab_fn (FN, STMT, OPTAB, 3)
    4356              : 
    4357              : #define expand_cond_unary_optab_fn(FN, STMT, OPTAB) \
    4358              :   expand_direct_optab_fn (FN, STMT, OPTAB, 3)
    4359              : 
    4360              : #define expand_cond_binary_optab_fn(FN, STMT, OPTAB) \
    4361              :   expand_direct_optab_fn (FN, STMT, OPTAB, 4)
    4362              : 
    4363              : #define expand_cond_ternary_optab_fn(FN, STMT, OPTAB) \
    4364              :   expand_direct_optab_fn (FN, STMT, OPTAB, 5)
    4365              : 
    4366              : #define expand_cond_len_unary_optab_fn(FN, STMT, OPTAB) \
    4367              :   expand_direct_optab_fn (FN, STMT, OPTAB, 5)
    4368              : 
    4369              : #define expand_cond_len_binary_optab_fn(FN, STMT, OPTAB) \
    4370              :   expand_direct_optab_fn (FN, STMT, OPTAB, 6)
    4371              : 
    4372              : #define expand_cond_len_ternary_optab_fn(FN, STMT, OPTAB) \
    4373              :   expand_direct_optab_fn (FN, STMT, OPTAB, 7)
    4374              : 
    4375              : #define expand_fold_extract_optab_fn(FN, STMT, OPTAB) \
    4376              :   expand_direct_optab_fn (FN, STMT, OPTAB, 3)
    4377              : 
    4378              : #define expand_fold_len_extract_optab_fn(FN, STMT, OPTAB) \
    4379              :   expand_direct_optab_fn (FN, STMT, OPTAB, 5)
    4380              : 
    4381              : #define expand_fold_left_optab_fn(FN, STMT, OPTAB) \
    4382              :   expand_direct_optab_fn (FN, STMT, OPTAB, 2)
    4383              : 
    4384              : #define expand_mask_fold_left_optab_fn(FN, STMT, OPTAB) \
    4385              :   expand_direct_optab_fn (FN, STMT, OPTAB, 3)
    4386              : 
    4387              : #define expand_mask_len_fold_left_optab_fn(FN, STMT, OPTAB) \
    4388              :   expand_direct_optab_fn (FN, STMT, OPTAB, 5)
    4389              : 
    4390              : #define expand_check_ptrs_optab_fn(FN, STMT, OPTAB) \
    4391              :   expand_direct_optab_fn (FN, STMT, OPTAB, 4)
    4392              : 
    4393              : #define expand_select_vl_optab_fn(FN, STMT, OPTAB) \
    4394              :   expand_convert_optab_fn (FN, STMT, OPTAB, 3)
    4395              : 
    4396              : /* Expanders for optabs that can use expand_convert_optab_fn.  */
    4397              : 
    4398              : #define expand_unary_convert_optab_fn(FN, STMT, OPTAB) \
    4399              :   expand_convert_optab_fn (FN, STMT, OPTAB, 1)
    4400              : 
    4401              : #define expand_vec_extract_optab_fn(FN, STMT, OPTAB) \
    4402              :   expand_convert_optab_fn (FN, STMT, OPTAB, 2)
    4403              : 
    4404              : /* RETURN_TYPE and ARGS are a return type and argument list that are
    4405              :    in principle compatible with FN (which satisfies direct_internal_fn_p).
    4406              :    Return the types that should be used to determine whether the
    4407              :    target supports FN.  */
    4408              : 
    4409              : tree_pair
    4410         7365 : direct_internal_fn_types (internal_fn fn, tree return_type, tree *args)
    4411              : {
    4412         7365 :   const direct_internal_fn_info &info = direct_internal_fn (fn);
    4413         7365 :   tree type0 = (info.type0 < 0 ? return_type : TREE_TYPE (args[info.type0]));
    4414         7365 :   tree type1 = (info.type1 < 0 ? return_type : TREE_TYPE (args[info.type1]));
    4415         7365 :   return tree_pair (type0, type1);
    4416              : }
    4417              : 
    4418              : /* CALL is a call whose return type and arguments are in principle
    4419              :    compatible with FN (which satisfies direct_internal_fn_p).  Return the
    4420              :    types that should be used to determine whether the target supports FN.  */
    4421              : 
    4422              : tree_pair
    4423       138948 : direct_internal_fn_types (internal_fn fn, gcall *call)
    4424              : {
    4425       138948 :   const direct_internal_fn_info &info = direct_internal_fn (fn);
    4426       138948 :   tree op0 = (info.type0 < 0
    4427       138948 :               ? gimple_call_lhs (call)
    4428       137426 :               : gimple_call_arg (call, info.type0));
    4429       138948 :   tree op1 = (info.type1 < 0
    4430       138948 :               ? gimple_call_lhs (call)
    4431       138895 :               : gimple_call_arg (call, info.type1));
    4432       138948 :   return tree_pair (TREE_TYPE (op0), TREE_TYPE (op1));
    4433              : }
    4434              : 
    4435              : /* Return true if OPTAB is supported for TYPES (whose modes should be
    4436              :    the same) when the optimization type is OPT_TYPE.  Used for simple
    4437              :    direct optabs.  */
    4438              : 
    4439              : static bool
    4440      4118998 : direct_optab_supported_p (direct_optab optab, tree_pair types,
    4441              :                           optimization_type opt_type)
    4442              : {
    4443      4118998 :   machine_mode mode = TYPE_MODE (types.first);
    4444      4118998 :   gcc_checking_assert (mode == TYPE_MODE (types.second));
    4445      4118998 :   return direct_optab_handler (optab, mode, opt_type) != CODE_FOR_nothing;
    4446              : }
    4447              : 
    4448              : /* Return true if OPTAB is supported for TYPES, where the first type
    4449              :    is the destination and the second type is the source.  Used for
    4450              :    convert optabs.  */
    4451              : 
    4452              : static bool
    4453       115634 : convert_optab_supported_p (convert_optab optab, tree_pair types,
    4454              :                            optimization_type opt_type)
    4455              : {
    4456       115634 :   return (convert_optab_handler (optab, TYPE_MODE (types.first),
    4457       115634 :                                  TYPE_MODE (types.second), opt_type)
    4458       115634 :           != CODE_FOR_nothing);
    4459              : }
    4460              : 
    4461              : /* Return true if load/store lanes optab OPTAB is supported for
    4462              :    array type TYPES.first when the optimization type is OPT_TYPE.  */
    4463              : 
    4464              : static bool
    4465            0 : multi_vector_optab_supported_p (convert_optab optab, tree_pair types,
    4466              :                                 optimization_type opt_type)
    4467              : {
    4468            0 :   gcc_assert (TREE_CODE (types.first) == ARRAY_TYPE);
    4469            0 :   machine_mode imode = TYPE_MODE (types.first);
    4470            0 :   machine_mode vmode = TYPE_MODE (TREE_TYPE (types.first));
    4471            0 :   return (convert_optab_handler (optab, imode, vmode, opt_type)
    4472            0 :           != CODE_FOR_nothing);
    4473              : }
    4474              : 
    4475              : #define direct_unary_optab_supported_p direct_optab_supported_p
    4476              : #define direct_unary_opt_optab_supported_p direct_optab_supported_p
    4477              : #define direct_unary_convert_optab_supported_p convert_optab_supported_p
    4478              : #define direct_binary_optab_supported_p direct_optab_supported_p
    4479              : #define direct_ternary_optab_supported_p direct_optab_supported_p
    4480              : #define direct_cond_unary_optab_supported_p direct_optab_supported_p
    4481              : #define direct_cond_binary_optab_supported_p direct_optab_supported_p
    4482              : #define direct_cond_ternary_optab_supported_p direct_optab_supported_p
    4483              : #define direct_cond_len_unary_optab_supported_p direct_optab_supported_p
    4484              : #define direct_cond_len_binary_optab_supported_p direct_optab_supported_p
    4485              : #define direct_cond_len_ternary_optab_supported_p direct_optab_supported_p
    4486              : #define direct_crc_optab_supported_p convert_optab_supported_p
    4487              : #define direct_mask_load_optab_supported_p convert_optab_supported_p
    4488              : #define direct_load_lanes_optab_supported_p multi_vector_optab_supported_p
    4489              : #define direct_mask_load_lanes_optab_supported_p multi_vector_optab_supported_p
    4490              : #define direct_mask_len_load_lanes_optab_supported_p \
    4491              :   multi_vector_optab_supported_p
    4492              : #define direct_gather_load_optab_supported_p convert_optab_supported_p
    4493              : #define direct_mask_gather_load_optab_supported_p convert_optab_supported_p
    4494              : #define direct_mask_len_gather_load_optab_supported_p convert_optab_supported_p
    4495              : #define direct_strided_load_optab_supported_p direct_optab_supported_p
    4496              : #define direct_len_load_optab_supported_p direct_optab_supported_p
    4497              : #define direct_mask_len_load_optab_supported_p convert_optab_supported_p
    4498              : #define direct_mask_store_optab_supported_p convert_optab_supported_p
    4499              : #define direct_store_lanes_optab_supported_p multi_vector_optab_supported_p
    4500              : #define direct_mask_store_lanes_optab_supported_p multi_vector_optab_supported_p
    4501              : #define direct_mask_len_store_lanes_optab_supported_p \
    4502              :   multi_vector_optab_supported_p
    4503              : #define direct_vec_cond_mask_optab_supported_p convert_optab_supported_p
    4504              : #define direct_vec_cond_optab_supported_p convert_optab_supported_p
    4505              : #define direct_scatter_store_optab_supported_p convert_optab_supported_p
    4506              : #define direct_mask_scatter_store_optab_supported_p convert_optab_supported_p
    4507              : #define direct_mask_len_scatter_store_optab_supported_p \
    4508              :   convert_optab_supported_p
    4509              : #define direct_strided_store_optab_supported_p direct_optab_supported_p
    4510              : #define direct_len_store_optab_supported_p direct_optab_supported_p
    4511              : #define direct_mask_len_store_optab_supported_p convert_optab_supported_p
    4512              : #define direct_while_optab_supported_p convert_optab_supported_p
    4513              : #define direct_fold_extract_optab_supported_p direct_optab_supported_p
    4514              : #define direct_fold_len_extract_optab_supported_p direct_optab_supported_p
    4515              : #define direct_fold_left_optab_supported_p direct_optab_supported_p
    4516              : #define direct_mask_fold_left_optab_supported_p direct_optab_supported_p
    4517              : #define direct_mask_len_fold_left_optab_supported_p direct_optab_supported_p
    4518              : #define direct_check_ptrs_optab_supported_p direct_optab_supported_p
    4519              : #define direct_vec_set_optab_supported_p direct_optab_supported_p
    4520              : #define direct_vec_extract_optab_supported_p convert_optab_supported_p
    4521              : #define direct_reduc_sbool_optab_supported_p direct_optab_supported_p
    4522              : #define direct_select_vl_optab_supported_p convert_optab_supported_p
    4523              : 
    4524              : /* Return the optab used by internal function FN.  */
    4525              : 
    4526              : optab
    4527       285824 : direct_internal_fn_optab (internal_fn fn, tree_pair types)
    4528              : {
    4529       285824 :   switch (fn)
    4530              :     {
    4531              : #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
    4532              :     case IFN_##CODE: break;
    4533              : #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
    4534              :     case IFN_##CODE: return OPTAB##_optab;
    4535              : #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
    4536              :                                      UNSIGNED_OPTAB, TYPE)              \
    4537              :     case IFN_##CODE: return (TYPE_UNSIGNED (types.SELECTOR)             \
    4538              :                              ? UNSIGNED_OPTAB ## _optab                 \
    4539              :                              : SIGNED_OPTAB ## _optab);
    4540              : #include "internal-fn.def"
    4541              : 
    4542              :     case IFN_LAST:
    4543              :       break;
    4544              :     }
    4545            0 :   gcc_unreachable ();
    4546              : }
    4547              : 
    4548              : /* Return the optab used by internal function FN.  */
    4549              : 
    4550              : static optab
    4551       983246 : direct_internal_fn_optab (internal_fn fn)
    4552              : {
    4553       983246 :   switch (fn)
    4554              :     {
    4555              : #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
    4556              :     case IFN_##CODE: break;
    4557              : #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
    4558              :     case IFN_##CODE: return OPTAB##_optab;
    4559              : #include "internal-fn.def"
    4560              : 
    4561              :     case IFN_LAST:
    4562              :       break;
    4563              :     }
    4564            0 :   gcc_unreachable ();
    4565              : }
    4566              : 
    4567              : /* Return true if TYPE's mode has the same format as TYPE, and if there is
    4568              :    a 1:1 correspondence between the values that the mode can store and the
    4569              :    values that the type can store.  */
    4570              : 
    4571              : static bool
    4572      8621310 : type_strictly_matches_mode_p (const_tree type)
    4573              : {
    4574              :   /* The masked vector operations have both vector data operands and vector
    4575              :      boolean operands.  The vector data operands are expected to have a vector
    4576              :      mode,  but the vector boolean operands can be an integer mode rather than
    4577              :      a vector mode,  depending on how TARGET_VECTORIZE_GET_MASK_MODE is
    4578              :      defined.  PR116103.  */
    4579      1009723 :   if (VECTOR_BOOLEAN_TYPE_P (type)
    4580       263287 :       && SCALAR_INT_MODE_P (TYPE_MODE (type))
    4581      8714182 :       && TYPE_PRECISION (TREE_TYPE (type)) == 1)
    4582              :     return true;
    4583              : 
    4584      8528439 :   if (VECTOR_TYPE_P (type))
    4585       916852 :     return VECTOR_MODE_P (TYPE_MODE (type));
    4586              : 
    4587      7611587 :   if (INTEGRAL_TYPE_P (type))
    4588      7204057 :     return type_has_mode_precision_p (type);
    4589              : 
    4590       407530 :   if (SCALAR_FLOAT_TYPE_P (type) || COMPLEX_FLOAT_TYPE_P (type))
    4591              :     return true;
    4592              : 
    4593              :   return false;
    4594              : }
    4595              : 
    4596              : /* Returns true if both types of TYPE_PAIR strictly match their modes,
    4597              :    else returns false.  */
    4598              : 
    4599              : static bool
    4600      4386678 : type_pair_strictly_matches_mode_p (tree_pair type_pair)
    4601              : {
    4602      4386678 :   return type_strictly_matches_mode_p (type_pair.first)
    4603      4386678 :     && type_strictly_matches_mode_p (type_pair.second);
    4604              : }
    4605              : 
    4606              : /* Return true if FN is supported for the types in TYPES when the
    4607              :    optimization type is OPT_TYPE.  The types are those associated with
    4608              :    the "type0" and "type1" fields of FN's direct_internal_fn_info
    4609              :    structure.  */
    4610              : 
    4611              : bool
    4612      4386678 : direct_internal_fn_supported_p (internal_fn fn, tree_pair types,
    4613              :                                 optimization_type opt_type)
    4614              : {
    4615      4386678 :   if (!type_pair_strictly_matches_mode_p (types))
    4616              :     return false;
    4617              : 
    4618      4234632 :   switch (fn)
    4619              :     {
    4620              : #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) \
    4621              :     case IFN_##CODE: break;
    4622              : #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
    4623              :     case IFN_##CODE: \
    4624              :       return direct_##TYPE##_optab_supported_p (OPTAB##_optab, types, \
    4625              :                                                 opt_type);
    4626              : #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
    4627              :                                      UNSIGNED_OPTAB, TYPE)              \
    4628              :     case IFN_##CODE:                                                    \
    4629              :       {                                                                 \
    4630              :         optab which_optab = (TYPE_UNSIGNED (types.SELECTOR)             \
    4631              :                              ? UNSIGNED_OPTAB ## _optab                 \
    4632              :                              : SIGNED_OPTAB ## _optab);                 \
    4633              :         return direct_##TYPE##_optab_supported_p (which_optab, types,   \
    4634              :                                                   opt_type);            \
    4635              :       }
    4636              : #include "internal-fn.def"
    4637              : 
    4638              :     case IFN_LAST:
    4639              :       break;
    4640              :     }
    4641            0 :   gcc_unreachable ();
    4642              : }
    4643              : 
    4644              : /* Return true if FN is supported for type TYPE when the optimization
    4645              :    type is OPT_TYPE.  The caller knows that the "type0" and "type1"
    4646              :    fields of FN's direct_internal_fn_info structure are the same.  */
    4647              : 
    4648              : bool
    4649      4190058 : direct_internal_fn_supported_p (internal_fn fn, tree type,
    4650              :                                 optimization_type opt_type)
    4651              : {
    4652      4190058 :   const direct_internal_fn_info &info = direct_internal_fn (fn);
    4653      4190058 :   gcc_checking_assert (info.type0 == info.type1);
    4654      4190058 :   return direct_internal_fn_supported_p (fn, tree_pair (type, type), opt_type);
    4655              : }
    4656              : 
    4657              : /* Return true if the STMT is supported when the optimization type is OPT_TYPE,
    4658              :    given that STMT is a call to a direct internal function.  */
    4659              : 
    4660              : bool
    4661          388 : direct_internal_fn_supported_p (gcall *stmt, optimization_type opt_type)
    4662              : {
    4663          388 :   internal_fn fn = gimple_call_internal_fn (stmt);
    4664          388 :   tree_pair types = direct_internal_fn_types (fn, stmt);
    4665          388 :   return direct_internal_fn_supported_p (fn, types, opt_type);
    4666              : }
    4667              : 
    4668              : /* Return true if FN is a binary operation and if FN is commutative.  */
    4669              : 
    4670              : bool
    4671     34509709 : commutative_binary_fn_p (internal_fn fn)
    4672              : {
    4673     34509709 :   switch (fn)
    4674              :     {
    4675              :     case IFN_AVG_FLOOR:
    4676              :     case IFN_AVG_CEIL:
    4677              :     case IFN_MULH:
    4678              :     case IFN_MULHS:
    4679              :     case IFN_MULHRS:
    4680              :     case IFN_FMIN:
    4681              :     case IFN_FMAX:
    4682              :     case IFN_COMPLEX_MUL:
    4683              :     case IFN_UBSAN_CHECK_ADD:
    4684              :     case IFN_UBSAN_CHECK_MUL:
    4685              :     case IFN_ADD_OVERFLOW:
    4686              :     case IFN_MUL_OVERFLOW:
    4687              :     case IFN_CLMUL:
    4688              :     case IFN_CLMUL_WIDEN:
    4689              :     case IFN_SAT_ADD:
    4690              :     case IFN_SAT_MUL:
    4691              :     case IFN_VEC_WIDEN_PLUS:
    4692              :     case IFN_VEC_WIDEN_PLUS_LO:
    4693              :     case IFN_VEC_WIDEN_PLUS_HI:
    4694              :     case IFN_VEC_WIDEN_PLUS_EVEN:
    4695              :     case IFN_VEC_WIDEN_PLUS_ODD:
    4696              :       return true;
    4697              : 
    4698     33593052 :     default:
    4699     33593052 :       return false;
    4700              :     }
    4701              : }
    4702              : 
    4703              : /* Return true if FN is a ternary operation and if its first two arguments
    4704              :    are commutative.  */
    4705              : 
    4706              : bool
    4707     28126718 : commutative_ternary_fn_p (internal_fn fn)
    4708              : {
    4709     28126718 :   switch (fn)
    4710              :     {
    4711              :     case IFN_FMA:
    4712              :     case IFN_FMS:
    4713              :     case IFN_FNMA:
    4714              :     case IFN_FNMS:
    4715              :     case IFN_UADDC:
    4716              :       return true;
    4717              : 
    4718     27993214 :     default:
    4719     27993214 :       return false;
    4720              :     }
    4721              : }
    4722              : 
    4723              : /* Return true if FN is an associative binary operation.  */
    4724              : 
    4725              : bool
    4726           52 : associative_binary_fn_p (internal_fn fn)
    4727              : {
    4728           52 :   switch (fn)
    4729              :     {
    4730              :     case IFN_FMIN:
    4731              :     case IFN_FMAX:
    4732              :       return true;
    4733              : 
    4734            0 :     default:
    4735            0 :       return false;
    4736              :     }
    4737              : }
    4738              : 
    4739              : /* If FN is commutative in two consecutive arguments, return the
    4740              :    index of the first, otherwise return -1.  */
    4741              : 
    4742              : int
    4743     28464059 : first_commutative_argument (internal_fn fn)
    4744              : {
    4745     28464059 :   switch (fn)
    4746              :     {
    4747              :     case IFN_COND_ADD:
    4748              :     case IFN_COND_MUL:
    4749              :     case IFN_COND_MIN:
    4750              :     case IFN_COND_MAX:
    4751              :     case IFN_COND_FMIN:
    4752              :     case IFN_COND_FMAX:
    4753              :     case IFN_COND_AND:
    4754              :     case IFN_COND_IOR:
    4755              :     case IFN_COND_XOR:
    4756              :     case IFN_COND_FMA:
    4757              :     case IFN_COND_FMS:
    4758              :     case IFN_COND_FNMA:
    4759              :     case IFN_COND_FNMS:
    4760              :     case IFN_COND_LEN_ADD:
    4761              :     case IFN_COND_LEN_MUL:
    4762              :     case IFN_COND_LEN_MIN:
    4763              :     case IFN_COND_LEN_MAX:
    4764              :     case IFN_COND_LEN_FMIN:
    4765              :     case IFN_COND_LEN_FMAX:
    4766              :     case IFN_COND_LEN_AND:
    4767              :     case IFN_COND_LEN_IOR:
    4768              :     case IFN_COND_LEN_XOR:
    4769              :     case IFN_COND_LEN_FMA:
    4770              :     case IFN_COND_LEN_FMS:
    4771              :     case IFN_COND_LEN_FNMA:
    4772              :     case IFN_COND_LEN_FNMS:
    4773              :       return 1;
    4774              : 
    4775     28439533 :     default:
    4776     28439533 :       if (commutative_binary_fn_p (fn)
    4777     28439533 :           || commutative_ternary_fn_p (fn))
    4778       451309 :         return 0;
    4779              :       return -1;
    4780              :     }
    4781              : }
    4782              : 
    4783              : /* Return true if this CODE describes an internal_fn that returns a vector with
    4784              :    elements twice as wide as the element size of the input vectors.  */
    4785              : 
    4786              : bool
    4787      2120483 : widening_fn_p (code_helper code)
    4788              : {
    4789      2120483 :   if (!code.is_fn_code ())
    4790              :     return false;
    4791              : 
    4792       221175 :   if (!internal_fn_p ((combined_fn) code))
    4793              :     return false;
    4794              : 
    4795       221175 :   internal_fn fn = as_internal_fn ((combined_fn) code);
    4796       221175 :   switch (fn)
    4797              :     {
    4798              :     #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
    4799              :     case IFN_##NAME:                                              \
    4800              :     case IFN_##NAME##_HI:                                         \
    4801              :     case IFN_##NAME##_LO:                                         \
    4802              :     case IFN_##NAME##_EVEN:                                       \
    4803              :     case IFN_##NAME##_ODD:                                        \
    4804              :       return true;
    4805              :     #include "internal-fn.def"
    4806              : 
    4807              :     default:
    4808              :       return false;
    4809              :     }
    4810              : }
    4811              : 
    4812              : /* Return true if this CODE describes an internal_fn that returns a vector with
    4813              :    elements twice as wide as the element size of the input vectors and operates
    4814              :    on even/odd parts of the input.  */
    4815              : 
    4816              : bool
    4817        21214 : widening_evenodd_fn_p (code_helper code)
    4818              : {
    4819        21214 :   if (!code.is_fn_code ())
    4820              :     return false;
    4821              : 
    4822            0 :   if (!internal_fn_p ((combined_fn) code))
    4823              :     return false;
    4824              : 
    4825            0 :   internal_fn fn = as_internal_fn ((combined_fn) code);
    4826            0 :   switch (fn)
    4827              :     {
    4828              :     #define DEF_INTERNAL_WIDENING_OPTAB_FN(NAME, F, S, SO, UO, T) \
    4829              :     case IFN_##NAME##_EVEN:                                       \
    4830              :     case IFN_##NAME##_ODD:                                        \
    4831              :       return true;
    4832              :     #include "internal-fn.def"
    4833              : 
    4834              :     default:
    4835              :       return false;
    4836              :     }
    4837              : }
    4838              : 
    4839              : /* Check the NARGS arguments ARGS of a call to internal function FN.
    4840              :    Return true if they are valid.  Otherwise report an error at LOC
    4841              :    and return false.  */
    4842              : 
    4843              : bool
    4844     42057655 : check_internal_fn_args (location_t loc, internal_fn fn, unsigned int nargs,
    4845              :                         const tree *args)
    4846              : {
    4847     42057655 :   const char *name = internal_fn_name (fn);
    4848     42057655 :   if (direct_internal_fn_p (fn))
    4849              :     {
    4850      1595504 :       const direct_internal_fn_info &info = direct_internal_fn (fn);
    4851      1595504 :       if (nargs < info.nargs_min)
    4852              :         {
    4853            2 :           error_at (loc, "%<.%s%> call with too few arguments", name);
    4854            2 :           return false;
    4855              :         }
    4856      1595502 :       if (nargs > info.nargs_max)
    4857              :         {
    4858            2 :           error_at (loc, "%<.%s%> call with too many arguments", name);
    4859            2 :           return false;
    4860              :         }
    4861              :     }
    4862              : 
    4863     42057651 :   switch (fn)
    4864              :     {
    4865        42686 :     case IFN_CLZ:
    4866        42686 :     case IFN_CTZ:
    4867        42686 :       {
    4868        42686 :         tree type = TREE_TYPE (args[0]);
    4869         3497 :         if (!(INTEGRAL_TYPE_P (type) || VECTOR_INTEGER_TYPE_P (type))
    4870        46183 :             || !TYPE_UNSIGNED (type))
    4871              :           {
    4872            3 :             error_at (loc, "%<.%s%> call with an argument that is not "
    4873              :                       "unsigned integer", name);
    4874            3 :             return false;
    4875              :           }
    4876        42683 :         if (nargs == 2 && TREE_CODE (args[1]) != INTEGER_CST)
    4877              :           {
    4878            1 :             error_at (loc, "%<.%s%> call with a value at zero that is not "
    4879              :                       "constant", name);
    4880            1 :             return false;
    4881              :           }
    4882              :         break;
    4883              :       }
    4884              : 
    4885              :     default:
    4886              :       break;
    4887              :     }
    4888              :   return true;
    4889              : }
    4890              : 
    4891              : /* Return true if IFN_SET_EDOM is supported.  */
    4892              : 
    4893              : bool
    4894          140 : set_edom_supported_p (void)
    4895              : {
    4896              : #ifdef TARGET_EDOM
    4897              :   return true;
    4898              : #else
    4899          140 :   return false;
    4900              : #endif
    4901              : }
    4902              : 
    4903              : #define DEF_INTERNAL_OPTAB_FN(CODE, FLAGS, OPTAB, TYPE) \
    4904              :   static void                                           \
    4905              :   expand_##CODE (internal_fn fn, gcall *stmt)           \
    4906              :   {                                                     \
    4907              :     expand_##TYPE##_optab_fn (fn, stmt, OPTAB##_optab); \
    4908              :   }
    4909              : #define DEF_INTERNAL_INT_EXT_FN(CODE, FLAGS, OPTAB, TYPE)
    4910              : #define DEF_INTERNAL_INTSZ_EXT_FN(CODE, FLAGS, OPTAB, TYPE)
    4911              : #define DEF_INTERNAL_SIGNED_OPTAB_FN(CODE, FLAGS, SELECTOR, SIGNED_OPTAB, \
    4912              :                                      UNSIGNED_OPTAB, TYPE)              \
    4913              :   static void                                                           \
    4914              :   expand_##CODE (internal_fn fn, gcall *stmt)                           \
    4915              :   {                                                                     \
    4916              :     tree_pair types = direct_internal_fn_types (fn, stmt);              \
    4917              :     optab which_optab = direct_internal_fn_optab (fn, types);           \
    4918              :     expand_##TYPE##_optab_fn (fn, stmt, which_optab);                   \
    4919              :   }
    4920              : #include "internal-fn.def"
    4921              : 
    4922              : /* Routines to expand each internal function, indexed by function number.
    4923              :    Each routine has the prototype:
    4924              : 
    4925              :        expand_<NAME> (gcall *stmt)
    4926              : 
    4927              :    where STMT is the statement that performs the call. */
    4928              : static void (*const internal_fn_expanders[]) (internal_fn, gcall *) = {
    4929              : 
    4930              : #define DEF_INTERNAL_FN(CODE, FLAGS, FNSPEC) expand_##CODE,
    4931              : #include "internal-fn.def"
    4932              :   0
    4933              : };
    4934              : 
    4935              : /* Invoke T(CODE, SUFFIX) for each conditional function IFN_COND_##SUFFIX
    4936              :    that maps to a tree code CODE.  There is also an IFN_COND_LEN_##SUFFIX
    4937              :    for each such IFN_COND_##SUFFIX.  */
    4938              : #define FOR_EACH_CODE_MAPPING(T) \
    4939              :   T (PLUS_EXPR, ADD) \
    4940              :   T (MINUS_EXPR, SUB) \
    4941              :   T (MULT_EXPR, MUL) \
    4942              :   T (TRUNC_DIV_EXPR, DIV) \
    4943              :   T (TRUNC_MOD_EXPR, MOD) \
    4944              :   T (RDIV_EXPR, RDIV) \
    4945              :   T (MIN_EXPR, MIN) \
    4946              :   T (MAX_EXPR, MAX) \
    4947              :   T (BIT_AND_EXPR, AND) \
    4948              :   T (BIT_IOR_EXPR, IOR) \
    4949              :   T (BIT_XOR_EXPR, XOR) \
    4950              :   T (LSHIFT_EXPR, SHL) \
    4951              :   T (RSHIFT_EXPR, SHR) \
    4952              :   T (NEGATE_EXPR, NEG)
    4953              : 
    4954              : /* Return a function that only performs CODE when a certain condition is met
    4955              :    and that uses a given fallback value otherwise.  For example, if CODE is
    4956              :    a binary operation associated with conditional function FN:
    4957              : 
    4958              :      LHS = FN (COND, A, B, ELSE)
    4959              : 
    4960              :    is equivalent to the C expression:
    4961              : 
    4962              :      LHS = COND ? A CODE B : ELSE;
    4963              : 
    4964              :    operating elementwise if the operands are vectors.
    4965              : 
    4966              :    Return IFN_LAST if no such function exists.  */
    4967              : 
    4968              : internal_fn
    4969       567375 : get_conditional_internal_fn (tree_code code)
    4970              : {
    4971       567375 :   switch (code)
    4972              :     {
    4973              : #define CASE(CODE, IFN) case CODE: return IFN_COND_##IFN;
    4974              :       FOR_EACH_CODE_MAPPING(CASE)
    4975              : #undef CASE
    4976              :     default:
    4977              :       return IFN_LAST;
    4978              :     }
    4979              : }
    4980              : 
    4981              : /* If IFN implements the conditional form of a tree code, return that
    4982              :    tree code, otherwise return ERROR_MARK.  */
    4983              : 
    4984              : tree_code
    4985      3511679 : conditional_internal_fn_code (internal_fn ifn)
    4986              : {
    4987      3511679 :   switch (ifn)
    4988              :     {
    4989              : #define CASE(CODE, IFN)                                                        \
    4990              :   case IFN_COND_##IFN:                                                         \
    4991              :   case IFN_COND_LEN_##IFN:                                                     \
    4992              :     return CODE;
    4993              :       FOR_EACH_CODE_MAPPING (CASE)
    4994              : #undef CASE
    4995              :       default:
    4996              :         return ERROR_MARK;
    4997              :     }
    4998              : }
    4999              : 
    5000              : /* Like get_conditional_internal_fn, but return a function that
    5001              :    additionally restricts the operation to the leading elements
    5002              :    of a vector.  The number of elements to process is given by a length
    5003              :    and bias pair, as for IFN_LOAD_LEN.  The values of the remaining
    5004              :    elements are taken from the fallback ("else") argument.
    5005              : 
    5006              :    For example, if CODE is a binary operation associated with FN:
    5007              : 
    5008              :      LHS = FN (COND, A, B, ELSE, LEN, BIAS)
    5009              : 
    5010              :    is equivalent to the C code:
    5011              : 
    5012              :      for (int i = 0; i < NUNITS; i++)
    5013              :       {
    5014              :         if (i < LEN + BIAS && COND[i])
    5015              :           LHS[i] = A[i] CODE B[i];
    5016              :         else
    5017              :           LHS[i] = ELSE[i];
    5018              :       }
    5019              : */
    5020              : 
    5021              : internal_fn
    5022       542800 : get_conditional_len_internal_fn (tree_code code)
    5023              : {
    5024       542800 :   switch (code)
    5025              :     {
    5026              : #define CASE(CODE, IFN) case CODE: return IFN_COND_LEN_##IFN;
    5027              :       FOR_EACH_CODE_MAPPING(CASE)
    5028              : #undef CASE
    5029              :     default:
    5030              :       return IFN_LAST;
    5031              :     }
    5032              : }
    5033              : 
    5034              : /* Invoke T(IFN) for each internal function IFN that also has an
    5035              :    IFN_COND_* form.  */
    5036              : #define FOR_EACH_COND_FN_PAIR(T) \
    5037              :   T (FMAX) \
    5038              :   T (FMIN) \
    5039              :   T (FMA) \
    5040              :   T (FMS) \
    5041              :   T (FNMA) \
    5042              :   T (FNMS) \
    5043              :   T (SQRT) \
    5044              :   T (ROUND) \
    5045              :   T (FLOOR) \
    5046              :   T (RINT) \
    5047              :   T (CEIL)
    5048              : 
    5049              : /* Return a function that only performs internal function FN when a
    5050              :    certain condition is met and that uses a given fallback value otherwise.
    5051              :    In other words, the returned function FN' is such that:
    5052              : 
    5053              :      LHS = FN' (COND, A1, ... An, ELSE)
    5054              : 
    5055              :    is equivalent to the C expression:
    5056              : 
    5057              :      LHS = COND ? FN (A1, ..., An) : ELSE;
    5058              : 
    5059              :    operating elementwise if the operands are vectors.
    5060              : 
    5061              :    Return IFN_LAST if no such function exists.  */
    5062              : 
    5063              : internal_fn
    5064        11080 : get_conditional_internal_fn (internal_fn fn)
    5065              : {
    5066        11080 :   switch (fn)
    5067              :     {
    5068              : #define CASE(NAME) case IFN_##NAME: return IFN_COND_##NAME;
    5069         2250 :       FOR_EACH_COND_FN_PAIR(CASE)
    5070              : #undef CASE
    5071         8829 :     default:
    5072         8829 :       return IFN_LAST;
    5073              :     }
    5074              : }
    5075              : 
    5076              : /* If there exists an internal function like IFN that operates on vectors,
    5077              :    but with additional length and bias parameters, return the internal_fn
    5078              :    for that function, otherwise return IFN_LAST.  */
    5079              : internal_fn
    5080        13229 : get_len_internal_fn (internal_fn fn)
    5081              : {
    5082        13229 :   switch (fn)
    5083              :     {
    5084              : #define DEF_INTERNAL_COND_FN(NAME, ...)                                        \
    5085              :   case IFN_COND_##NAME:                                                        \
    5086              :     return IFN_COND_LEN_##NAME;
    5087              : #define DEF_INTERNAL_SIGNED_COND_FN(NAME, ...)                                 \
    5088              :   case IFN_COND_##NAME:                                                        \
    5089              :     return IFN_COND_LEN_##NAME;
    5090              : #include "internal-fn.def"
    5091         8791 :     default:
    5092         8791 :       break;
    5093              :     }
    5094              : 
    5095         8791 :   switch (fn)
    5096              :     {
    5097              :     case IFN_MASK_LOAD:
    5098              :       return IFN_MASK_LEN_LOAD;
    5099              :     case IFN_MASK_LOAD_LANES:
    5100              :       return IFN_MASK_LEN_LOAD_LANES;
    5101              :     case IFN_MASK_GATHER_LOAD:
    5102              :       return IFN_MASK_LEN_GATHER_LOAD;
    5103              :     default:
    5104              :       return IFN_LAST;
    5105              :     }
    5106              : }
    5107              : 
    5108              : /* If IFN implements the conditional form of an unconditional internal
    5109              :    function, return that unconditional function, otherwise return IFN_LAST.  */
    5110              : 
    5111              : internal_fn
    5112      3456021 : get_unconditional_internal_fn (internal_fn ifn)
    5113              : {
    5114      3456021 :   switch (ifn)
    5115              :     {
    5116              : #define CASE(NAME)                                                             \
    5117              :     case IFN_COND_##NAME:                                                      \
    5118              :     case IFN_COND_LEN_##NAME:                                                  \
    5119              :       return IFN_##NAME;
    5120              : FOR_EACH_COND_FN_PAIR (CASE)
    5121              : #undef CASE
    5122              :     default:
    5123              :       return IFN_LAST;
    5124              :     }
    5125              : }
    5126              : 
    5127              : /* Return true if STMT can be interpreted as a conditional tree code
    5128              :    operation of the form:
    5129              : 
    5130              :      LHS = COND ? OP (RHS1, ...) : ELSE;
    5131              : 
    5132              :    operating elementwise if the operands are vectors.  This includes
    5133              :    the case of an all-true COND, so that the operation always happens.
    5134              : 
    5135              :    There is an alternative approach to interpret the STMT when the operands
    5136              :    are vectors which is the operation predicated by both conditional mask
    5137              :    and loop control length, the equivalent C code:
    5138              : 
    5139              :      for (int i = 0; i < NUNTIS; i++)
    5140              :       {
    5141              :         if (i < LEN + BIAS && COND[i])
    5142              :           LHS[i] = A[i] CODE B[i];
    5143              :         else
    5144              :           LHS[i] = ELSE[i];
    5145              :       }
    5146              : 
    5147              :    When returning true, set:
    5148              : 
    5149              :    - *COND_OUT to the condition COND, or to NULL_TREE if the condition
    5150              :      is known to be all-true
    5151              :    - *CODE_OUT to the tree code
    5152              :    - OPS[I] to operand I of *CODE_OUT
    5153              :    - *ELSE_OUT to the fallback value ELSE, or to NULL_TREE if the
    5154              :      condition is known to be all true.
    5155              :    - *LEN to the len argument if it COND_LEN_* operations or to NULL_TREE.
    5156              :    - *BIAS to the bias argument if it COND_LEN_* operations or to NULL_TREE.  */
    5157              : 
    5158              : bool
    5159        40421 : can_interpret_as_conditional_op_p (gimple *stmt, tree *cond_out,
    5160              :                                    tree_code *code_out,
    5161              :                                    tree (&ops)[3], tree *else_out,
    5162              :                                    tree *len, tree *bias)
    5163              : {
    5164        40421 :   *len = NULL_TREE;
    5165        40421 :   *bias = NULL_TREE;
    5166        40421 :   if (gassign *assign = dyn_cast <gassign *> (stmt))
    5167              :     {
    5168        37734 :       *cond_out = NULL_TREE;
    5169        37734 :       *code_out = gimple_assign_rhs_code (assign);
    5170        37734 :       ops[0] = gimple_assign_rhs1 (assign);
    5171        37734 :       ops[1] = gimple_assign_rhs2 (assign);
    5172        37734 :       ops[2] = gimple_assign_rhs3 (assign);
    5173        37734 :       *else_out = NULL_TREE;
    5174        37734 :       return true;
    5175              :     }
    5176         2687 :   if (gcall *call = dyn_cast <gcall *> (stmt))
    5177         2110 :     if (gimple_call_internal_p (call))
    5178              :       {
    5179         2053 :         internal_fn ifn = gimple_call_internal_fn (call);
    5180         2053 :         tree_code code = conditional_internal_fn_code (ifn);
    5181         2053 :         int len_index = internal_fn_len_index (ifn);
    5182         2053 :         int cond_nargs = len_index >= 0 ? 4 : 2;
    5183         2053 :         if (code != ERROR_MARK)
    5184              :           {
    5185          328 :             *cond_out = gimple_call_arg (call, 0);
    5186          328 :             *code_out = code;
    5187          328 :             unsigned int nops = gimple_call_num_args (call) - cond_nargs;
    5188         1312 :             for (unsigned int i = 0; i < 3; ++i)
    5189          984 :               ops[i] = i < nops ? gimple_call_arg (call, i + 1) : NULL_TREE;
    5190          328 :             *else_out = gimple_call_arg (call, nops + 1);
    5191          328 :             if (len_index < 0)
    5192              :               {
    5193          328 :                 if (integer_truep (*cond_out))
    5194              :                   {
    5195            0 :                     *cond_out = NULL_TREE;
    5196            0 :                     *else_out = NULL_TREE;
    5197              :                   }
    5198              :               }
    5199              :             else
    5200              :               {
    5201            0 :                 *len = gimple_call_arg (call, len_index);
    5202            0 :                 *bias = gimple_call_arg (call, len_index + 1);
    5203              :               }
    5204              :             return true;
    5205              :           }
    5206              :       }
    5207              :   return false;
    5208              : }
    5209              : 
    5210              : /* Return true if IFN is some form of load from memory.  */
    5211              : 
    5212              : bool
    5213       999776 : internal_load_fn_p (internal_fn fn)
    5214              : {
    5215       999776 :   switch (fn)
    5216              :     {
    5217              :     case IFN_MASK_LOAD:
    5218              :     case IFN_LOAD_LANES:
    5219              :     case IFN_MASK_LOAD_LANES:
    5220              :     case IFN_MASK_LEN_LOAD_LANES:
    5221              :     case IFN_GATHER_LOAD:
    5222              :     case IFN_MASK_GATHER_LOAD:
    5223              :     case IFN_MASK_LEN_GATHER_LOAD:
    5224              :     case IFN_LEN_LOAD:
    5225              :     case IFN_MASK_LEN_LOAD:
    5226              :       return true;
    5227              : 
    5228       261922 :     default:
    5229       261922 :       return false;
    5230              :     }
    5231              : }
    5232              : 
    5233              : /* Return true if IFN is some form of store to memory.  */
    5234              : 
    5235              : bool
    5236       562482 : internal_store_fn_p (internal_fn fn)
    5237              : {
    5238       562482 :   switch (fn)
    5239              :     {
    5240              :     case IFN_MASK_STORE:
    5241              :     case IFN_STORE_LANES:
    5242              :     case IFN_MASK_STORE_LANES:
    5243              :     case IFN_MASK_LEN_STORE_LANES:
    5244              :     case IFN_SCATTER_STORE:
    5245              :     case IFN_MASK_SCATTER_STORE:
    5246              :     case IFN_MASK_LEN_SCATTER_STORE:
    5247              :     case IFN_LEN_STORE:
    5248              :     case IFN_MASK_LEN_STORE:
    5249              :       return true;
    5250              : 
    5251       549684 :     default:
    5252       549684 :       return false;
    5253              :     }
    5254              : }
    5255              : 
    5256              : /* Return true if IFN is some form of gather load or scatter store.  */
    5257              : 
    5258              : bool
    5259        50056 : internal_gather_scatter_fn_p (internal_fn fn)
    5260              : {
    5261        50056 :   switch (fn)
    5262              :     {
    5263              :     case IFN_GATHER_LOAD:
    5264              :     case IFN_MASK_GATHER_LOAD:
    5265              :     case IFN_MASK_LEN_GATHER_LOAD:
    5266              :     case IFN_SCATTER_STORE:
    5267              :     case IFN_MASK_SCATTER_STORE:
    5268              :     case IFN_MASK_LEN_SCATTER_STORE:
    5269              :       return true;
    5270              : 
    5271        50056 :     default:
    5272        50056 :       return false;
    5273              :     }
    5274              : }
    5275              : 
    5276              : /* If FN takes a vector len argument, return the index of that argument,
    5277              :    otherwise return -1.  */
    5278              : 
    5279              : int
    5280      3340441 : internal_fn_len_index (internal_fn fn)
    5281              : {
    5282      3340441 :   switch (fn)
    5283              :     {
    5284              :     case IFN_LEN_LOAD:
    5285              :       return 3;
    5286              :     case IFN_LEN_STORE:
    5287              :       return 2;
    5288              : 
    5289              :     case IFN_MASK_LEN_SCATTER_STORE:
    5290              :       return 6;
    5291              : 
    5292              :     case IFN_MASK_LEN_STRIDED_LOAD:
    5293              :       return 5;
    5294              : 
    5295              :     case IFN_MASK_LEN_GATHER_LOAD:
    5296              :       return 7;
    5297              : 
    5298              :     case IFN_COND_LEN_FMA:
    5299              :     case IFN_COND_LEN_FMS:
    5300              :     case IFN_COND_LEN_FNMA:
    5301              :     case IFN_COND_LEN_FNMS:
    5302              :       return 5;
    5303              : 
    5304              :     case IFN_COND_LEN_ADD:
    5305              :     case IFN_COND_LEN_SUB:
    5306              :     case IFN_COND_LEN_MUL:
    5307              :     case IFN_COND_LEN_DIV:
    5308              :     case IFN_COND_LEN_MOD:
    5309              :     case IFN_COND_LEN_RDIV:
    5310              :     case IFN_COND_LEN_MIN:
    5311              :     case IFN_COND_LEN_MAX:
    5312              :     case IFN_COND_LEN_FMIN:
    5313              :     case IFN_COND_LEN_FMAX:
    5314              :     case IFN_COND_LEN_AND:
    5315              :     case IFN_COND_LEN_IOR:
    5316              :     case IFN_COND_LEN_XOR:
    5317              :     case IFN_COND_LEN_SHL:
    5318              :     case IFN_COND_LEN_SHR:
    5319              :     case IFN_MASK_LEN_STRIDED_STORE:
    5320              :       return 4;
    5321              : 
    5322              :     case IFN_COND_LEN_NEG:
    5323              :     case IFN_MASK_LEN_STORE:
    5324              :     case IFN_MASK_LEN_STORE_LANES:
    5325              :     case IFN_VCOND_MASK_LEN:
    5326              :       return 3;
    5327              : 
    5328              :     case IFN_MASK_LEN_LOAD:
    5329              :     case IFN_MASK_LEN_LOAD_LANES:
    5330              :       return 4;
    5331              : 
    5332              :     default:
    5333              :       return -1;
    5334              :     }
    5335              : }
    5336              : 
    5337              : /* If FN is an IFN_COND_* or IFN_COND_LEN_* function, return the index of the
    5338              :    argument that is used when the condition is false.  Return -1 otherwise.  */
    5339              : 
    5340              : int
    5341        85746 : internal_fn_else_index (internal_fn fn)
    5342              : {
    5343        85746 :   switch (fn)
    5344              :     {
    5345              :     case IFN_COND_NEG:
    5346              :     case IFN_COND_NOT:
    5347              :     case IFN_COND_SQRT:
    5348              :     case IFN_COND_CEIL:
    5349              :     case IFN_COND_FLOOR:
    5350              :     case IFN_COND_ROUND:
    5351              :     case IFN_COND_RINT:
    5352              :     case IFN_COND_LEN_NEG:
    5353              :     case IFN_COND_LEN_NOT:
    5354              :     case IFN_COND_LEN_SQRT:
    5355              :     case IFN_COND_LEN_CEIL:
    5356              :     case IFN_COND_LEN_FLOOR:
    5357              :     case IFN_COND_LEN_ROUND:
    5358              :     case IFN_COND_LEN_RINT:
    5359              :       return 2;
    5360              : 
    5361              :     case IFN_LEN_LOAD:
    5362              :       return 2;
    5363              : 
    5364              :     case IFN_COND_ADD:
    5365              :     case IFN_COND_SUB:
    5366              :     case IFN_COND_MUL:
    5367              :     case IFN_COND_DIV:
    5368              :     case IFN_COND_MOD:
    5369              :     case IFN_COND_MIN:
    5370              :     case IFN_COND_MAX:
    5371              :     case IFN_COND_FMIN:
    5372              :     case IFN_COND_FMAX:
    5373              :     case IFN_COND_AND:
    5374              :     case IFN_COND_IOR:
    5375              :     case IFN_COND_XOR:
    5376              :     case IFN_COND_SHL:
    5377              :     case IFN_COND_SHR:
    5378              :     case IFN_COND_LEN_ADD:
    5379              :     case IFN_COND_LEN_SUB:
    5380              :     case IFN_COND_LEN_MUL:
    5381              :     case IFN_COND_LEN_DIV:
    5382              :     case IFN_COND_LEN_MOD:
    5383              :     case IFN_COND_LEN_MIN:
    5384              :     case IFN_COND_LEN_MAX:
    5385              :     case IFN_COND_LEN_FMIN:
    5386              :     case IFN_COND_LEN_FMAX:
    5387              :     case IFN_COND_LEN_AND:
    5388              :     case IFN_COND_LEN_IOR:
    5389              :     case IFN_COND_LEN_XOR:
    5390              :     case IFN_COND_LEN_SHL:
    5391              :     case IFN_COND_LEN_SHR:
    5392              :       return 3;
    5393              : 
    5394              :     case IFN_MASK_LOAD:
    5395              :     case IFN_MASK_LEN_LOAD:
    5396              :     case IFN_MASK_LOAD_LANES:
    5397              :     case IFN_MASK_LEN_LOAD_LANES:
    5398              :       return 3;
    5399              : 
    5400              :     case IFN_COND_FMA:
    5401              :     case IFN_COND_FMS:
    5402              :     case IFN_COND_FNMA:
    5403              :     case IFN_COND_FNMS:
    5404              :     case IFN_COND_LEN_FMA:
    5405              :     case IFN_COND_LEN_FMS:
    5406              :     case IFN_COND_LEN_FNMA:
    5407              :     case IFN_COND_LEN_FNMS:
    5408              :     case IFN_MASK_LEN_STRIDED_LOAD:
    5409              :       return 4;
    5410              : 
    5411              :     case IFN_MASK_GATHER_LOAD:
    5412              :     case IFN_MASK_LEN_GATHER_LOAD:
    5413              :       return 6;
    5414              : 
    5415              :     default:
    5416              :       return -1;
    5417              :     }
    5418              : 
    5419              :   return -1;
    5420              : }
    5421              : 
    5422              : /* If FN takes a vector mask argument, return the index of that argument,
    5423              :    otherwise return -1.  */
    5424              : 
    5425              : int
    5426       212677 : internal_fn_mask_index (internal_fn fn)
    5427              : {
    5428       212677 :   switch (fn)
    5429              :     {
    5430              :     case IFN_MASK_LOAD:
    5431              :     case IFN_MASK_LOAD_LANES:
    5432              :     case IFN_MASK_LEN_LOAD_LANES:
    5433              :     case IFN_MASK_STORE:
    5434              :     case IFN_MASK_STORE_LANES:
    5435              :     case IFN_MASK_LEN_STORE_LANES:
    5436              :     case IFN_MASK_LEN_LOAD:
    5437              :     case IFN_MASK_LEN_STORE:
    5438              :       return 2;
    5439              : 
    5440            0 :     case IFN_MASK_LEN_STRIDED_LOAD:
    5441            0 :     case IFN_MASK_LEN_STRIDED_STORE:
    5442            0 :       return 3;
    5443              : 
    5444            0 :     case IFN_MASK_GATHER_LOAD:
    5445            0 :     case IFN_MASK_SCATTER_STORE:
    5446            0 :     case IFN_MASK_LEN_GATHER_LOAD:
    5447            0 :     case IFN_MASK_LEN_SCATTER_STORE:
    5448            0 :       return 5;
    5449              : 
    5450              :     case IFN_VCOND_MASK:
    5451              :     case IFN_VCOND_MASK_LEN:
    5452              :       return 0;
    5453              : 
    5454       168309 :     default:
    5455       168309 :       return (conditional_internal_fn_code (fn) != ERROR_MARK
    5456       168309 :               || get_unconditional_internal_fn (fn) != IFN_LAST ? 0 : -1);
    5457              :     }
    5458              : }
    5459              : 
    5460              : /* If FN takes a value that should be stored to memory, return the index
    5461              :    of that argument, otherwise return -1.  */
    5462              : 
    5463              : int
    5464        26312 : internal_fn_stored_value_index (internal_fn fn)
    5465              : {
    5466        26312 :   switch (fn)
    5467              :     {
    5468              :     case IFN_MASK_LEN_STRIDED_STORE:
    5469              :       return 2;
    5470              : 
    5471              :     case IFN_MASK_STORE:
    5472              :     case IFN_MASK_STORE_LANES:
    5473              :       return 3;
    5474              :     case IFN_SCATTER_STORE:
    5475              :     case IFN_MASK_SCATTER_STORE:
    5476              :     case IFN_MASK_LEN_SCATTER_STORE:
    5477              :       return 4;
    5478              : 
    5479              :     case IFN_LEN_STORE:
    5480              :       return 4;
    5481              : 
    5482              :     case IFN_MASK_LEN_STORE:
    5483              :     case IFN_MASK_LEN_STORE_LANES:
    5484              :       return 5;
    5485              : 
    5486              :     default:
    5487              :       return -1;
    5488              :     }
    5489              : }
    5490              : 
    5491              : /* If FN has an alias pointer return its index, otherwise return -1.  */
    5492              : 
    5493              : int
    5494            0 : internal_fn_alias_ptr_index (internal_fn fn)
    5495              : {
    5496            0 :   switch (fn)
    5497              :     {
    5498              :     case IFN_MASK_LOAD:
    5499              :     case IFN_MASK_LEN_LOAD:
    5500              :     case IFN_GATHER_LOAD:
    5501              :     case IFN_MASK_GATHER_LOAD:
    5502              :     case IFN_MASK_LEN_GATHER_LOAD:
    5503              :     case IFN_SCATTER_STORE:
    5504              :     case IFN_MASK_SCATTER_STORE:
    5505              :     case IFN_MASK_LEN_SCATTER_STORE:
    5506              :       return 1;
    5507              : 
    5508            0 :     default:
    5509            0 :       return -1;
    5510              :     }
    5511              : }
    5512              : 
    5513              : /* If FN is a gather/scatter return the index of its offset argument,
    5514              :    otherwise return -1.  */
    5515              : 
    5516              : int
    5517            0 : internal_fn_offset_index (internal_fn fn)
    5518              : {
    5519            0 :   if (!internal_gather_scatter_fn_p (fn))
    5520              :     return -1;
    5521              : 
    5522            0 :   switch (fn)
    5523              :     {
    5524              :     case IFN_GATHER_LOAD:
    5525              :     case IFN_MASK_GATHER_LOAD:
    5526              :     case IFN_MASK_LEN_GATHER_LOAD:
    5527              :     case IFN_SCATTER_STORE:
    5528              :     case IFN_MASK_SCATTER_STORE:
    5529              :     case IFN_MASK_LEN_SCATTER_STORE:
    5530              :       return 2;
    5531              : 
    5532              :     default:
    5533              :       return -1;
    5534              :     }
    5535              : }
    5536              : 
    5537              : /* If FN is a gather/scatter return the index of its scale argument,
    5538              :    otherwise return -1.  */
    5539              : 
    5540              : int
    5541            0 : internal_fn_scale_index (internal_fn fn)
    5542              : {
    5543            0 :   if (!internal_gather_scatter_fn_p (fn))
    5544              :     return -1;
    5545              : 
    5546            0 :   switch (fn)
    5547              :     {
    5548              :     case IFN_GATHER_LOAD:
    5549              :     case IFN_MASK_GATHER_LOAD:
    5550              :     case IFN_MASK_LEN_GATHER_LOAD:
    5551              :     case IFN_SCATTER_STORE:
    5552              :     case IFN_MASK_SCATTER_STORE:
    5553              :     case IFN_MASK_LEN_SCATTER_STORE:
    5554              :       return 3;
    5555              : 
    5556              :     default:
    5557              :       return -1;
    5558              :     }
    5559              : }
    5560              : 
    5561              : /* Store all supported else values for the optab referred to by ICODE
    5562              :    in ELSE_VALS.  The index of the else operand must be specified in
    5563              :    ELSE_INDEX.  */
    5564              : 
    5565              : void
    5566        63857 : get_supported_else_vals (enum insn_code icode, unsigned else_index,
    5567              :                          vec<int> &else_vals)
    5568              : {
    5569        63857 :   const struct insn_data_d *data = &insn_data[icode];
    5570        63857 :   if ((int) else_index >= data->n_operands || (int) else_index == -1)
    5571              :     return;
    5572              : 
    5573        63857 :   machine_mode else_mode = data->operand[else_index].mode;
    5574              : 
    5575        63857 :   else_vals.truncate (0);
    5576              : 
    5577              :   /* For now we only support else values of 0, -1, and "undefined".  */
    5578        63857 :   if (insn_operand_matches (icode, else_index, CONST0_RTX (else_mode)))
    5579        63857 :     else_vals.safe_push (MASK_LOAD_ELSE_ZERO);
    5580              : 
    5581        63857 :   if (insn_operand_matches (icode, else_index, gen_rtx_SCRATCH (else_mode)))
    5582            0 :     else_vals.safe_push (MASK_LOAD_ELSE_UNDEFINED);
    5583              : 
    5584        63857 :   if (GET_MODE_CLASS (else_mode) == MODE_VECTOR_INT
    5585        63857 :       && insn_operand_matches (icode, else_index, CONSTM1_RTX (else_mode)))
    5586            0 :     else_vals.safe_push (MASK_LOAD_ELSE_M1);
    5587              : }
    5588              : 
    5589              : /* Return true if the else value ELSE_VAL (one of MASK_LOAD_ELSE_ZERO,
    5590              :    MASK_LOAD_ELSE_M1, and MASK_LOAD_ELSE_UNDEFINED) is valid fo the optab
    5591              :    referred to by ICODE.  The index of the else operand must be specified
    5592              :    in ELSE_INDEX.  */
    5593              : 
    5594              : bool
    5595            0 : supported_else_val_p (enum insn_code icode, unsigned else_index, int else_val)
    5596              : {
    5597            0 :   if (else_val != MASK_LOAD_ELSE_ZERO && else_val != MASK_LOAD_ELSE_M1
    5598            0 :       && else_val != MASK_LOAD_ELSE_UNDEFINED)
    5599            0 :     gcc_unreachable ();
    5600              : 
    5601            0 :   auto_vec<int> else_vals;
    5602            0 :   get_supported_else_vals (icode, else_index, else_vals);
    5603            0 :   return else_vals.contains (else_val);
    5604            0 : }
    5605              : 
    5606              : /* Return true if the target supports gather load or scatter store function
    5607              :    IFN.  For loads, VECTOR_TYPE is the vector type of the load result,
    5608              :    while for stores it is the vector type of the stored data argument.
    5609              :    MEMORY_ELEMENT_TYPE is the type of the memory elements being loaded
    5610              :    or stored.  OFFSET_VECTOR_TYPE is the vector type that holds the
    5611              :    offset from the shared base address of each loaded or stored element.
    5612              :    SCALE is the amount by which these offsets should be multiplied
    5613              :    *after* they have been extended to address width.
    5614              :    If the target supports the gather load the supported else values
    5615              :    will be added to the vector ELSVAL points to if it is nonzero.  */
    5616              : 
    5617              : bool
    5618      3584378 : internal_gather_scatter_fn_supported_p (internal_fn ifn, tree vector_type,
    5619              :                                         tree memory_element_type,
    5620              :                                         tree offset_vector_type, int scale,
    5621              :                                         vec<int> *elsvals)
    5622              : {
    5623      3584378 :   if (!tree_int_cst_equal (TYPE_SIZE (TREE_TYPE (vector_type)),
    5624      3584378 :                            TYPE_SIZE (memory_element_type)))
    5625              :     return false;
    5626      3584378 :   if (maybe_ne (TYPE_VECTOR_SUBPARTS (vector_type),
    5627      7168756 :                 TYPE_VECTOR_SUBPARTS (offset_vector_type)))
    5628              :     return false;
    5629       975990 :   optab optab = direct_internal_fn_optab (ifn);
    5630       975990 :   insn_code icode = convert_optab_handler (optab, TYPE_MODE (vector_type),
    5631       975990 :                                            TYPE_MODE (offset_vector_type));
    5632       975990 :   int output_ops = internal_load_fn_p (ifn) ? 1 : 0;
    5633       975990 :   bool unsigned_p = TYPE_UNSIGNED (TREE_TYPE (offset_vector_type));
    5634       975990 :   bool ok = icode != CODE_FOR_nothing
    5635            0 :     && insn_operand_matches (icode, 2 + output_ops, GEN_INT (unsigned_p))
    5636       975990 :     && insn_operand_matches (icode, 3 + output_ops, GEN_INT (scale));
    5637              : 
    5638       975990 :   if (ok && elsvals)
    5639            0 :     get_supported_else_vals
    5640            0 :       (icode, internal_fn_else_index (IFN_MASK_GATHER_LOAD), *elsvals);
    5641              : 
    5642              :   return ok;
    5643              : }
    5644              : 
    5645              : /* Return true if the target supports a strided load/store function IFN
    5646              :    with VECTOR_TYPE.  If supported and ELSVALS is nonzero the supported else
    5647              :    values will be added to the vector ELSVALS points to.  */
    5648              : 
    5649              : bool
    5650         1868 : internal_strided_fn_supported_p (internal_fn ifn, tree vector_type,
    5651              :                                  vec<int> *elsvals)
    5652              : {
    5653         1868 :   machine_mode mode = TYPE_MODE (vector_type);
    5654         1868 :   optab optab = direct_internal_fn_optab (ifn);
    5655         1868 :   insn_code icode = direct_optab_handler (optab, mode);
    5656              : 
    5657         1868 :   bool ok = icode != CODE_FOR_nothing;
    5658              : 
    5659         1868 :   if (ok && elsvals)
    5660            0 :     get_supported_else_vals (icode, internal_fn_else_index (ifn), *elsvals);
    5661              : 
    5662         1868 :   return ok;
    5663              : }
    5664              : 
    5665              : /* Return true if the target supports IFN_CHECK_{RAW,WAR}_PTRS function IFN
    5666              :    for pointers of type TYPE when the accesses have LENGTH bytes and their
    5667              :    common byte alignment is ALIGN.  */
    5668              : 
    5669              : bool
    5670         5388 : internal_check_ptrs_fn_supported_p (internal_fn ifn, tree type,
    5671              :                                     poly_uint64 length, unsigned int align)
    5672              : {
    5673         5388 :   machine_mode mode = TYPE_MODE (type);
    5674         5388 :   optab optab = direct_internal_fn_optab (ifn);
    5675         5388 :   insn_code icode = direct_optab_handler (optab, mode);
    5676         5388 :   if (icode == CODE_FOR_nothing)
    5677              :     return false;
    5678            0 :   rtx length_rtx = immed_wide_int_const (length, mode);
    5679            0 :   return (insn_operand_matches (icode, 3, length_rtx)
    5680            0 :           && insn_operand_matches (icode, 4, GEN_INT (align)));
    5681              : }
    5682              : 
    5683              : /* Return the supported bias for IFN which is either IFN_{LEN_,MASK_LEN_,}LOAD
    5684              :    or IFN_{LEN_,MASK_LEN_,}STORE.  For now we only support the biases of 0 and
    5685              :    -1 (in case 0 is not an allowable length for {len_,mask_len_}load or
    5686              :    {len_,mask_len_}store). If none of the biases match what the backend
    5687              :    provides, return VECT_PARTIAL_BIAS_UNSUPPORTED.  */
    5688              : 
    5689              : signed char
    5690            0 : internal_len_load_store_bias (internal_fn ifn, machine_mode mode)
    5691              : {
    5692            0 :   optab optab = direct_internal_fn_optab (ifn);
    5693            0 :   insn_code icode = direct_optab_handler (optab, mode);
    5694            0 :   int bias_idx = internal_fn_len_index (ifn) + 1;
    5695              : 
    5696            0 :   if (icode == CODE_FOR_nothing)
    5697              :     {
    5698            0 :       machine_mode mask_mode;
    5699            0 :       if (!targetm.vectorize.get_mask_mode (mode).exists (&mask_mode))
    5700            0 :         return VECT_PARTIAL_BIAS_UNSUPPORTED;
    5701            0 :       if (ifn == IFN_LEN_LOAD)
    5702              :         {
    5703              :           /* Try MASK_LEN_LOAD.  */
    5704            0 :           optab = direct_internal_fn_optab (IFN_MASK_LEN_LOAD);
    5705            0 :           bias_idx = internal_fn_len_index (IFN_MASK_LEN_LOAD) + 1;
    5706              :         }
    5707              :       else
    5708              :         {
    5709              :           /* Try MASK_LEN_STORE.  */
    5710            0 :           optab = direct_internal_fn_optab (IFN_MASK_LEN_STORE);
    5711            0 :           bias_idx = internal_fn_len_index (IFN_MASK_LEN_STORE) + 1;
    5712              :         }
    5713            0 :       icode = convert_optab_handler (optab, mode, mask_mode);
    5714              :     }
    5715              : 
    5716            0 :   if (icode != CODE_FOR_nothing)
    5717              :     {
    5718              :       /* For now we only support biases of 0 or -1.  Try both of them.  */
    5719            0 :       if (insn_operand_matches (icode, bias_idx, GEN_INT (0)))
    5720              :         return 0;
    5721            0 :       if (insn_operand_matches (icode, bias_idx, GEN_INT (-1)))
    5722            0 :         return -1;
    5723              :     }
    5724              : 
    5725              :   return VECT_PARTIAL_BIAS_UNSUPPORTED;
    5726              : }
    5727              : 
    5728              : /* Expand STMT as though it were a call to internal function FN.  */
    5729              : 
    5730              : void
    5731       266625 : expand_internal_call (internal_fn fn, gcall *stmt)
    5732              : {
    5733       266625 :   internal_fn_expanders[fn] (fn, stmt);
    5734       266625 : }
    5735              : 
    5736              : /* Expand STMT, which is a call to internal function FN.  */
    5737              : 
    5738              : void
    5739       233038 : expand_internal_call (gcall *stmt)
    5740              : {
    5741       233038 :   expand_internal_call (gimple_call_internal_fn (stmt), stmt);
    5742       233038 : }
    5743              : 
    5744              : /* If TYPE is a vector type, return true if IFN is a direct internal
    5745              :    function that is supported for that type.  If TYPE is a scalar type,
    5746              :    return true if IFN is a direct internal function that is supported for
    5747              :    the target's preferred vector version of TYPE.  */
    5748              : 
    5749              : bool
    5750        17306 : vectorized_internal_fn_supported_p (internal_fn ifn, tree type)
    5751              : {
    5752        17306 :   if (VECTOR_MODE_P (TYPE_MODE (type)))
    5753        10924 :     return direct_internal_fn_supported_p (ifn, type, OPTIMIZE_FOR_SPEED);
    5754              : 
    5755         6382 :   scalar_mode smode;
    5756         6382 :   if (VECTOR_TYPE_P (type)
    5757         6382 :       || !is_a <scalar_mode> (TYPE_MODE (type), &smode))
    5758              :     return false;
    5759              : 
    5760         6371 :   machine_mode vmode = targetm.vectorize.preferred_simd_mode (smode);
    5761         6371 :   if (VECTOR_MODE_P (vmode))
    5762              :     {
    5763         6334 :       tree vectype = build_vector_type_for_mode (type, vmode);
    5764         6334 :       if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
    5765              :         return true;
    5766              :     }
    5767              : 
    5768         4296 :   auto_vector_modes vector_modes;
    5769         4296 :   targetm.vectorize.autovectorize_vector_modes (&vector_modes, true);
    5770        26795 :   for (machine_mode base_mode : vector_modes)
    5771        13907 :     if (related_vector_mode (base_mode, smode).exists (&vmode))
    5772              :       {
    5773        12202 :         tree vectype = build_vector_type_for_mode (type, vmode);
    5774        12202 :         if (direct_internal_fn_supported_p (ifn, vectype, OPTIMIZE_FOR_SPEED))
    5775              :           return true;
    5776              :       }
    5777              : 
    5778              :   return false;
    5779         4296 : }
    5780              : 
    5781              : void
    5782            0 : expand_SHUFFLEVECTOR (internal_fn, gcall *)
    5783              : {
    5784            0 :   gcc_unreachable ();
    5785              : }
    5786              : 
    5787              : void
    5788            0 : expand_PHI (internal_fn, gcall *)
    5789              : {
    5790            0 :   gcc_unreachable ();
    5791              : }
    5792              : 
    5793              : void
    5794          333 : expand_SPACESHIP (internal_fn, gcall *stmt)
    5795              : {
    5796          333 :   tree lhs = gimple_call_lhs (stmt);
    5797          333 :   tree rhs1 = gimple_call_arg (stmt, 0);
    5798          333 :   tree rhs2 = gimple_call_arg (stmt, 1);
    5799          333 :   tree rhs3 = gimple_call_arg (stmt, 2);
    5800          333 :   tree type = TREE_TYPE (rhs1);
    5801              : 
    5802          333 :   do_pending_stack_adjust ();
    5803              : 
    5804          333 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    5805          333 :   rtx op1 = expand_normal (rhs1);
    5806          333 :   rtx op2 = expand_normal (rhs2);
    5807          333 :   rtx op3 = expand_normal (rhs3);
    5808              : 
    5809          333 :   class expand_operand ops[4];
    5810          333 :   create_call_lhs_operand (&ops[0], target, TYPE_MODE (TREE_TYPE (lhs)));
    5811          333 :   create_input_operand (&ops[1], op1, TYPE_MODE (type));
    5812          333 :   create_input_operand (&ops[2], op2, TYPE_MODE (type));
    5813          333 :   create_input_operand (&ops[3], op3, TYPE_MODE (TREE_TYPE (rhs3)));
    5814          333 :   insn_code icode = optab_handler (spaceship_optab, TYPE_MODE (type));
    5815          333 :   expand_insn (icode, 4, ops);
    5816          333 :   assign_call_lhs (lhs, target, &ops[0]);
    5817          333 : }
    5818              : 
    5819              : void
    5820            0 : expand_ASSUME (internal_fn, gcall *)
    5821              : {
    5822            0 : }
    5823              : 
    5824              : void
    5825            0 : expand_MASK_CALL (internal_fn, gcall *)
    5826              : {
    5827              :   /* This IFN should only exist between ifcvt and vect passes.  */
    5828            0 :   gcc_unreachable ();
    5829              : }
    5830              : 
    5831              : void
    5832            0 : expand_VARYING (internal_fn, gcall *)
    5833              : {
    5834              :   /* This IFN should reach expand.  */
    5835            0 :   gcc_unreachable ();
    5836              : }
    5837              : 
    5838              : void
    5839         1691 : expand_MULBITINT (internal_fn, gcall *stmt)
    5840              : {
    5841         1691 :   rtx_mode_t args[6];
    5842        11837 :   for (int i = 0; i < 6; i++)
    5843        10146 :     args[i] = rtx_mode_t (expand_normal (gimple_call_arg (stmt, i)),
    5844        20292 :                           (i & 1) ? SImode : ptr_mode);
    5845         1691 :   rtx fun = init_one_libfunc ("__mulbitint3");
    5846         1691 :   emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 6, args);
    5847         1691 : }
    5848              : 
    5849              : void
    5850          163 : expand_DIVMODBITINT (internal_fn, gcall *stmt)
    5851              : {
    5852          163 :   rtx_mode_t args[8];
    5853         1467 :   for (int i = 0; i < 8; i++)
    5854         1304 :     args[i] = rtx_mode_t (expand_normal (gimple_call_arg (stmt, i)),
    5855         2608 :                           (i & 1) ? SImode : ptr_mode);
    5856          163 :   rtx fun = init_one_libfunc ("__divmodbitint4");
    5857          163 :   emit_library_call_value_1 (0, fun, NULL_RTX, LCT_NORMAL, VOIDmode, 8, args);
    5858          163 : }
    5859              : 
    5860              : void
    5861          179 : expand_FLOATTOBITINT (internal_fn, gcall *stmt)
    5862              : {
    5863          179 :   machine_mode mode = TYPE_MODE (TREE_TYPE (gimple_call_arg (stmt, 2)));
    5864          179 :   rtx arg0 = expand_normal (gimple_call_arg (stmt, 0));
    5865          179 :   rtx arg1 = expand_normal (gimple_call_arg (stmt, 1));
    5866          179 :   rtx arg2 = expand_normal (gimple_call_arg (stmt, 2));
    5867          179 :   rtx fun = optab_libfunc (bitintfromfp_optab, mode);
    5868          179 :   emit_library_call (fun, LCT_NORMAL, VOIDmode, arg0, ptr_mode, arg1,
    5869              :                      SImode, arg2, mode);
    5870          179 : }
    5871              : 
    5872              : void
    5873          148 : expand_BITINTTOFLOAT (internal_fn, gcall *stmt)
    5874              : {
    5875          148 :   tree lhs = gimple_call_lhs (stmt);
    5876          148 :   if (!lhs)
    5877              :     return;
    5878          148 :   machine_mode mode = TYPE_MODE (TREE_TYPE (lhs));
    5879          148 :   rtx arg0 = expand_normal (gimple_call_arg (stmt, 0));
    5880          148 :   rtx arg1 = expand_normal (gimple_call_arg (stmt, 1));
    5881          148 :   rtx fun = optab_libfunc (bitinttofp_optab, mode);
    5882          148 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    5883          148 :   rtx val = emit_library_call_value (fun, target, LCT_PURE, mode,
    5884              :                                      arg0, ptr_mode, arg1, SImode);
    5885          148 :   if (val != target)
    5886            0 :     emit_move_insn (target, val);
    5887              : }
    5888              : 
    5889              : static bool
    5890         6716 : expand_bitquery (internal_fn fn, gcall *stmt)
    5891              : {
    5892         6716 :   tree lhs = gimple_call_lhs (stmt);
    5893         6716 :   if (lhs == NULL_TREE)
    5894              :     return false;
    5895         6710 :   tree arg = gimple_call_arg (stmt, 0);
    5896         6710 :   if (TREE_CODE (arg) == INTEGER_CST)
    5897              :     {
    5898            1 :       tree ret;
    5899              :       /* A second argument of IFN_CLZ or IFN_CTZ is the result for zero.  */
    5900            1 :       if ((fn == IFN_CLZ || fn == IFN_CTZ) && gimple_call_num_args (stmt) == 2)
    5901            1 :         ret = fold_const_call (as_combined_fn (fn), TREE_TYPE (lhs), arg,
    5902              :                                gimple_call_arg (stmt, 1));
    5903              :       else
    5904            0 :         ret = fold_const_call (as_combined_fn (fn), TREE_TYPE (arg), arg);
    5905            1 :       gcc_checking_assert (ret && TREE_CODE (ret) == INTEGER_CST);
    5906            1 :       expand_assignment (lhs, ret, false);
    5907            1 :       return false;
    5908              :     }
    5909              :   return true;
    5910              : }
    5911              : 
    5912              : void
    5913            1 : expand_CLRSB (internal_fn fn, gcall *stmt)
    5914              : {
    5915            1 :   if (expand_bitquery (fn, stmt))
    5916            0 :     expand_unary_optab_fn (fn, stmt, clrsb_optab);
    5917            1 : }
    5918              : 
    5919              : /* Expand STMT, a call to the IFN_CLZ or IFN_CTZ internal function FN,
    5920              :    using OPTAB.  The optional second argument of STMT is the result for a
    5921              :    zero first argument.  If it is not the value that the instruction gives
    5922              :    for zero, test the first argument for zero and use the second argument
    5923              :    as the result.  If the target does not define the instruction for zero,
    5924              :    execute it only for a nonzero first argument.  */
    5925              : 
    5926              : static void
    5927         6165 : expand_clz_ctz (internal_fn fn, gcall *stmt, direct_optab optab)
    5928              : {
    5929         6165 :   if (!expand_bitquery (fn, stmt))
    5930         5977 :     return;
    5931         6162 :   tree arg = gimple_call_arg (stmt, 0);
    5932         6162 :   machine_mode mode = TYPE_MODE (TREE_TYPE (arg));
    5933         6162 :   int val = 0, defined = 0;
    5934         6162 :   if (gimple_call_num_args (stmt) == 2)
    5935          542 :     defined = (fn == IFN_CLZ
    5936          464 :                ? CLZ_DEFINED_VALUE_AT_ZERO (mode, val)
    5937           78 :                : CTZ_DEFINED_VALUE_AT_ZERO (mode, val));
    5938         6162 :   if (gimple_call_num_args (stmt) == 1
    5939         6245 :       || (defined == 2 && wi::to_widest (gimple_call_arg (stmt, 1)) == val))
    5940              :     {
    5941         5974 :       expand_unary_optab_fn (fn, stmt, optab);
    5942         5974 :       return;
    5943              :     }
    5944              :   /* The vectorizer creates vector calls only with the value of the
    5945              :      target.  */
    5946          188 :   gcc_checking_assert (!VECTOR_TYPE_P (TREE_TYPE (arg)));
    5947              : 
    5948          188 :   tree lhs = gimple_call_lhs (stmt);
    5949          188 :   rtx op0 = force_reg (mode, expand_normal (arg));
    5950          188 :   rtx zero_val = expand_normal (fold_convert (TREE_TYPE (lhs),
    5951              :                                               gimple_call_arg (stmt, 1)));
    5952          188 :   rtx res = gen_reg_rtx (TYPE_MODE (TREE_TYPE (lhs)));
    5953          188 :   rtx_code_label *done = gen_label_rtx ();
    5954          188 :   if (defined)
    5955              :     {
    5956            0 :       convert_move (res, expand_unop (mode, optab, op0, NULL_RTX, 1), 1);
    5957            0 :       emit_cmp_and_jump_insns (op0, const0_rtx, NE, NULL_RTX, mode, true,
    5958              :                                done, profile_probability::likely ());
    5959            0 :       emit_move_insn (res, zero_val);
    5960              :     }
    5961              :   else
    5962              :     {
    5963          188 :       emit_move_insn (res, zero_val);
    5964          188 :       emit_cmp_and_jump_insns (op0, const0_rtx, EQ, NULL_RTX, mode, true,
    5965              :                                done, profile_probability::unlikely ());
    5966          188 :       convert_move (res, expand_unop (mode, optab, op0, NULL_RTX, 1), 1);
    5967              :     }
    5968          188 :   emit_label (done);
    5969          188 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    5970          188 :   if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
    5971            0 :     convert_move (SUBREG_REG (target), res, SUBREG_PROMOTED_SIGN (target));
    5972              :   else
    5973          188 :     emit_move_insn (target, res);
    5974              : }
    5975              : 
    5976              : void
    5977         4879 : expand_CLZ (internal_fn fn, gcall *stmt)
    5978              : {
    5979         4879 :   expand_clz_ctz (fn, stmt, clz_optab);
    5980         4879 : }
    5981              : 
    5982              : void
    5983         1286 : expand_CTZ (internal_fn fn, gcall *stmt)
    5984              : {
    5985         1286 :   expand_clz_ctz (fn, stmt, ctz_optab);
    5986         1286 : }
    5987              : 
    5988              : void
    5989          133 : expand_FFS (internal_fn fn, gcall *stmt)
    5990              : {
    5991          133 :   if (expand_bitquery (fn, stmt))
    5992          132 :     expand_unary_optab_fn (fn, stmt, ffs_optab);
    5993          133 : }
    5994              : 
    5995              : void
    5996          200 : expand_PARITY (internal_fn fn, gcall *stmt)
    5997              : {
    5998          200 :   if (expand_bitquery (fn, stmt))
    5999          199 :     expand_unary_optab_fn (fn, stmt, parity_optab);
    6000          200 : }
    6001              : 
    6002              : void
    6003          217 : expand_POPCOUNT (internal_fn fn, gcall *stmt)
    6004              : {
    6005          217 :   if (!expand_bitquery (fn, stmt))
    6006              :     return;
    6007          216 :   if (gimple_call_num_args (stmt) == 1)
    6008              :     {
    6009          208 :       expand_unary_optab_fn (fn, stmt, popcount_optab);
    6010          208 :       return;
    6011              :     }
    6012              :   /* If .POPCOUNT call has 2 arguments, match_single_bit_test marked it
    6013              :      because the result is only used in an equality comparison against 1.
    6014              :      Use rtx costs in that case to determine if .POPCOUNT (arg) == 1
    6015              :      or (arg ^ (arg - 1)) > arg - 1 is cheaper.
    6016              :      If .POPCOUNT second argument is 0, we additionally know that arg
    6017              :      is non-zero, so use arg & (arg - 1) == 0 instead.
    6018              :      If .POPCOUNT second argument is -1, the comparison was either `<= 1`
    6019              :      or `> 1`.  */
    6020            8 :   bool speed_p = optimize_insn_for_speed_p ();
    6021            8 :   tree lhs = gimple_call_lhs (stmt);
    6022            8 :   tree arg = gimple_call_arg (stmt, 0);
    6023            8 :   bool nonzero_arg = integer_zerop (gimple_call_arg (stmt, 1));
    6024            8 :   bool was_le = integer_minus_onep (gimple_call_arg (stmt, 1));
    6025            8 :   if (was_le)
    6026            0 :     nonzero_arg = true;
    6027            8 :   tree type = TREE_TYPE (arg);
    6028            8 :   machine_mode mode = TYPE_MODE (type);
    6029            8 :   machine_mode lhsmode = TYPE_MODE (TREE_TYPE (lhs));
    6030            8 :   do_pending_stack_adjust ();
    6031            8 :   start_sequence ();
    6032            8 :   expand_unary_optab_fn (fn, stmt, popcount_optab);
    6033            8 :   rtx_insn *popcount_insns = end_sequence ();
    6034            8 :   start_sequence ();
    6035            8 :   rtx plhs = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    6036            8 :   rtx pcmp = emit_store_flag (NULL_RTX, EQ, plhs, const1_rtx, lhsmode, 0, 0);
    6037            8 :   if (pcmp == NULL_RTX)
    6038              :     {
    6039            0 :     fail:
    6040            0 :       end_sequence ();
    6041            0 :       emit_insn (popcount_insns);
    6042            0 :       return;
    6043              :     }
    6044            8 :   rtx_insn *popcount_cmp_insns = end_sequence ();
    6045            8 :   start_sequence ();
    6046            8 :   rtx op0 = expand_normal (arg);
    6047            8 :   rtx argm1 = expand_simple_binop (mode, PLUS, op0, constm1_rtx, NULL_RTX,
    6048              :                                    1, OPTAB_WIDEN);
    6049            8 :   if (argm1 == NULL_RTX)
    6050            0 :     goto fail;
    6051           12 :   rtx argxorargm1 = expand_simple_binop (mode, nonzero_arg ? AND : XOR, op0,
    6052              :                                          argm1, NULL_RTX, 1, OPTAB_WIDEN);
    6053            8 :   if (argxorargm1 == NULL_RTX)
    6054            0 :     goto fail;
    6055            8 :   rtx cmp;
    6056            8 :   if (nonzero_arg)
    6057            4 :     cmp = emit_store_flag (NULL_RTX, EQ, argxorargm1, const0_rtx, mode, 1, 1);
    6058              :   else
    6059            4 :     cmp = emit_store_flag (NULL_RTX, GTU, argxorargm1, argm1, mode, 1, 1);
    6060            8 :   if (cmp == NULL_RTX)
    6061            0 :     goto fail;
    6062            8 :   rtx_insn *cmp_insns = end_sequence ();
    6063            8 :   unsigned popcount_cost = (seq_cost (popcount_insns, speed_p)
    6064            8 :                             + seq_cost (popcount_cmp_insns, speed_p));
    6065            8 :   unsigned cmp_cost = seq_cost (cmp_insns, speed_p);
    6066              : 
    6067            8 :   if (dump_file && (dump_flags & TDF_DETAILS))
    6068            0 :     fprintf(dump_file, "popcount == 1: popcount cost: %u; cmp cost: %u\n",
    6069              :             popcount_cost, cmp_cost);
    6070              : 
    6071            8 :   if (popcount_cost <= cmp_cost)
    6072            8 :     emit_insn (popcount_insns);
    6073              :   else
    6074              :     {
    6075            0 :       start_sequence ();
    6076            0 :       emit_insn (cmp_insns);
    6077            0 :       plhs = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    6078            0 :       if (GET_MODE (cmp) != GET_MODE (plhs))
    6079            0 :         cmp = convert_to_mode (GET_MODE (plhs), cmp, 1);
    6080              :       /* For `<= 1`, we need to produce `2 - cmp` or `cmp ? 1 : 2` as that
    6081              :          then gets compared against 1 and we need the false case to be 2.  */
    6082            0 :       if (was_le)
    6083              :         {
    6084            0 :           cmp = expand_simple_binop (GET_MODE (cmp), MINUS, const2_rtx,
    6085              :                                      cmp, NULL_RTX, 1, OPTAB_WIDEN);
    6086            0 :           if (!cmp)
    6087            0 :             goto fail;
    6088              :         }
    6089            0 :       emit_move_insn (plhs, cmp);
    6090            0 :       rtx_insn *all_insns = end_sequence ();
    6091            0 :       emit_insn (all_insns);
    6092              :     }
    6093              : }
    6094              : 
    6095              : void
    6096            0 : expand_BSWAP (internal_fn fn, gcall *stmt)
    6097              : {
    6098            0 :   if (expand_bitquery (fn, stmt))
    6099            0 :     expand_unary_optab_fn (fn, stmt, bswap_optab);
    6100            0 : }
    6101              : 
    6102              : void
    6103            0 : expand_BITREVERSE (internal_fn fn, gcall *stmt)
    6104              : {
    6105            0 :   if (expand_bitquery (fn, stmt))
    6106            0 :     expand_unary_optab_fn (fn, stmt, bitreverse_optab);
    6107            0 : }
    6108              : 
    6109              : /* Expand CLMUL (OP0, OP1) where operands are in OP_MODE and have
    6110              :    minimum unsigned precision PREC0 and PREC1.  Compute result in MODE.  */
    6111              : 
    6112              : static rtx
    6113          175 : expand_clmul (machine_mode mode, machine_mode op_mode, rtx op0, rtx op1,
    6114              :               unsigned int prec0, unsigned int prec1)
    6115              : {
    6116          175 :   unsigned int op_prec = GET_MODE_PRECISION (as_a <scalar_int_mode> (op_mode));
    6117          175 :   unsigned int prec = GET_MODE_PRECISION (as_a <scalar_int_mode> (mode));
    6118          175 :   machine_mode cmode;
    6119          175 :   optab_methods olw = OPTAB_LIB_WIDEN;
    6120          175 :   if (MIN (prec0, prec1) == 1)
    6121              :     {
    6122              :       /* If one operand has [0, 1] range, we can do just
    6123              :          multiplication and if both are [0, 1], this is equivalent to AND.  */
    6124            0 :       rtx res = expand_binop (op_mode, prec0 == prec1 ? and_optab : smul_optab,
    6125              :                               op0, op1, NULL_RTX, 1, olw);
    6126            0 :       return convert_modes (mode, op_mode, res, 1);
    6127              :     }
    6128         1308 :   FOR_EACH_MODE_IN_CLASS (cmode, MODE_INT)
    6129              :     {
    6130         1156 :       unsigned int this_prec
    6131         1156 :         = GET_MODE_PRECISION (as_a <scalar_int_mode> (cmode));
    6132         1156 :       if (prec0 > this_prec || prec1 > this_prec)
    6133          371 :         continue;
    6134          785 :       if (optab_handler (clmul_optab, cmode) != CODE_FOR_nothing
    6135          785 :           && (prec0 + prec1 - 1 <= this_prec || prec <= this_prec))
    6136              :         {
    6137           16 :           rtx_insn *start = get_last_insn ();
    6138           16 :           rtx op0c = convert_modes (cmode, op_mode, op0, 1);
    6139           16 :           rtx op1c = convert_modes (cmode, op_mode, op1, 1);
    6140           16 :           if (rtx res = expand_binop (cmode, clmul_optab, op0c, op1c,
    6141              :                                       NULL_RTX, 1, OPTAB_DIRECT))
    6142           16 :             return convert_modes (mode, cmode, res, 1);
    6143            0 :           delete_insns_since (start);
    6144              :         }
    6145          769 :       if (optab_handler (clmul_widen_optab, cmode) != CODE_FOR_nothing)
    6146              :         {
    6147            7 :           rtx_insn *start = get_last_insn ();
    6148            7 :           rtx op0c = convert_modes (cmode, op_mode, op0, 1);
    6149            7 :           rtx op1c = convert_modes (cmode, op_mode, op1, 1);
    6150            7 :           rtx reslo = gen_reg_rtx (cmode);
    6151            7 :           rtx reshi = gen_reg_rtx (cmode);
    6152            7 :           class expand_operand ops[4];
    6153            7 :           create_output_operand (&ops[0], reslo, cmode);
    6154            7 :           create_output_operand (&ops[1], reshi, cmode);
    6155            7 :           create_input_operand (&ops[2], op0c, cmode);
    6156            7 :           create_input_operand (&ops[3], op1c, cmode);
    6157            7 :           if (maybe_expand_insn (optab_handler (clmul_widen_optab, cmode),
    6158              :                                  4, ops))
    6159              :             {
    6160            7 :               if (prec0 + prec1 - 1 <= this_prec || prec <= this_prec)
    6161            0 :                 return convert_modes (mode, cmode, reslo, 1);
    6162            7 :               machine_mode wmode;
    6163            7 :               if (!GET_MODE_2XWIDER_MODE (cmode).exists (&wmode))
    6164              :                 wmode = mode;
    6165            7 :               reslo = convert_modes (wmode, cmode, reslo, 1);
    6166            7 :               reshi = convert_modes (wmode, cmode, reshi, 1);
    6167            7 :               reshi = expand_shift (LSHIFT_EXPR, wmode, reshi, this_prec,
    6168              :                                     NULL_RTX, 1);
    6169            7 :               rtx res = expand_binop (wmode, ior_optab, reslo, reshi,
    6170              :                                       NULL_RTX, 1, olw);
    6171            7 :               return convert_modes (mode, wmode, res, 1);
    6172              :             }
    6173            0 :           delete_insns_since (start);
    6174              :         }
    6175              :     }
    6176          152 :   if (CONST_SCALAR_INT_P (op1))
    6177              :     {
    6178           40 :       wide_int wop1 = rtx_mode_t (op1, op_mode);
    6179           40 :       unsigned lz = wi::clz (wop1);
    6180           40 :       unsigned tz = wi::ctz (wop1);
    6181           40 :       unsigned cnt = wi::popcount (wop1);
    6182           40 :       if (lz + cnt + tz == op_prec
    6183           56 :           && (pow2p_hwi (cnt) || (lz == 0 && prec == op_prec)))
    6184              :         {
    6185              :           /* Optimize the cases where one argument is a constant
    6186              :              equal to ((((T) 1) << M) - 1) << N for some M and N
    6187              :              constants.
    6188              :              In this case the clmul (x, ((((T) 1) << M) - 1) << N)
    6189              :              result can be computed as
    6190              :              T r = x;
    6191              :              for (unsigned i = 1; i < M; i = i + i)
    6192              :                r ^= r << i;
    6193              :              return r << N;
    6194              :              If M is not a power of two, it shouldn't be a widening
    6195              :              shift and most significant bit of the constant should be
    6196              :              set.  */
    6197           20 :           rtx x = convert_modes (mode, op_mode, op0, 1);
    6198           20 :           rtx res = gen_reg_rtx (mode);
    6199           20 :           emit_move_insn (res, x);
    6200           20 :           rtx i = gen_reg_rtx (TYPE_MODE (integer_type_node));
    6201           20 :           emit_move_insn (i, const1_rtx);
    6202           20 :           rtx_code_label *loop_label = gen_label_rtx ();
    6203           20 :           emit_label (loop_label);
    6204           20 :           rtx tem = expand_variable_shift (LSHIFT_EXPR, mode, res,
    6205              :                                            make_tree (integer_type_node, i),
    6206              :                                            NULL_RTX, 1);
    6207           20 :           tem = expand_binop (mode, xor_optab, res, tem, res, 1, olw);
    6208           20 :           if (tem != res)
    6209            0 :             emit_move_insn (res, tem);
    6210           20 :           tem = expand_binop (GET_MODE (i), add_optab, i, i, i, 1, olw);
    6211           20 :           if (tem != res)
    6212           20 :             emit_move_insn (i, tem);
    6213           40 :           do_compare_rtx_and_jump (i, gen_int_mode (cnt, GET_MODE (i)),
    6214           20 :                                    LTU, true, GET_MODE (i), NULL_RTX, NULL,
    6215              :                                    loop_label,
    6216              :                                    profile_probability::very_likely ());
    6217           20 :           if (tz)
    6218              :             {
    6219           12 :               tem = expand_shift (LSHIFT_EXPR, mode, res, tz, res, 1);
    6220           12 :               if (tem != res)
    6221            0 :                 emit_move_insn (res, tem);
    6222              :             }
    6223           20 :           return res;
    6224              :         }
    6225           40 :     }
    6226              :   /* Fallback using Kronecker substitution.
    6227              :      Depending on the number of active bits in the operand with smaller
    6228              :      number of those and the result precision we choose smallest spacing
    6229              :      S in [2, 13], such that MIN (prec0, prec1) <= S * ((1 << S) - 1),
    6230              :      or prec <= S * (1 << S) or prec0 + prec1 - 1 <= S * (1 << S).
    6231              :      The algorithm then uses S * S multiplications, 3 * S bitwise ANDs,
    6232              :      S * (S - 1) XORs and S - 1 IORs.
    6233              :      For e.g. S = 4 this is:
    6234              :      uint64_t x0 = x & 0x1111111111111111ULL, y0 = y & 0x1111111111111111ULL;
    6235              :      uint64_t x1 = x & 0x2222222222222222ULL, y1 = y & 0x2222222222222222ULL;
    6236              :      uint64_t x2 = x & 0x4444444444444444ULL, y2 = y & 0x4444444444444444ULL;
    6237              :      uint64_t x3 = x & 0x8888888888888888ULL, y3 = y & 0x8888888888888888ULL;
    6238              :      uint64_t z0 = (x0 * y0) ^ (x1 * y3) ^ (x2 * y2) ^ (x3 * y1);
    6239              :      uint64_t z1 = (x0 * y1) ^ (x1 * y0) ^ (x2 * y3) ^ (x3 * y2);
    6240              :      uint64_t z2 = (x0 * y2) ^ (x1 * y1) ^ (x2 * y0) ^ (x3 * y3);
    6241              :      uint64_t z3 = (x0 * y3) ^ (x1 * y2) ^ (x2 * y1) ^ (x3 * y0);
    6242              :      z0 &= 0x1111111111111111ULL;
    6243              :      z1 &= 0x2222222222222222ULL;
    6244              :      z2 &= 0x4444444444444444ULL;
    6245              :      z3 &= 0x8888888888888888ULL;
    6246              :      return (z0 | z1) | (z2 | z3);  */
    6247          132 :   unsigned int s;
    6248          368 :   for (s = 2; s <= 13; ++s)
    6249          368 :     if (MIN (prec0, prec1) <= s * ((1 << s) - 1)
    6250          255 :         || MIN (prec, prec0 + prec1 - 1) <= s * (1 << s))
    6251              :       break;
    6252              :   /* S = 13 should handle up to _BitInt maximum 65535 (106483 bits).  */
    6253          132 :   gcc_assert (s <= 13);
    6254          132 :   rtx mask[13], x[13], y[13], t[13], r;
    6255          132 :   bool do_widen_mult = false;
    6256          132 :   if (mode != op_mode
    6257          132 :       && (convert_optab_handler (umul_widen_optab, mode, op_mode)
    6258              :           != CODE_FOR_nothing))
    6259              :     do_widen_mult = true;
    6260          632 :   for (unsigned int i = 0; i < s; ++i)
    6261              :     {
    6262          500 :       mask[i]
    6263          500 :         = immed_wide_int_const (wi::bit_repeat (wide_int::from (1 << i, s,
    6264              :                                                                 UNSIGNED),
    6265              :                                                 op_prec), op_mode);
    6266          500 :       if (CONST_INT_P (op0) && CONST_INT_P (mask[i]))
    6267            0 :         x[i] = gen_int_mode (INTVAL (op0) & INTVAL (mask[i]), op_mode);
    6268              :       else
    6269          500 :         x[i] = expand_binop (op_mode, and_optab, op0, mask[i], NULL_RTX, 1,
    6270              :                              olw);
    6271          500 :       if (mode != op_mode && !do_widen_mult)
    6272           39 :         x[i] = convert_modes (mode, op_mode, x[i], 1);
    6273          500 :       if (CONST_INT_P (op1) && CONST_INT_P (mask[i]))
    6274           88 :         y[i] = gen_int_mode (INTVAL (op1) & INTVAL (mask[i]), op_mode);
    6275              :       else
    6276          412 :         y[i] = expand_binop (op_mode, and_optab, op1, mask[i], NULL_RTX, 1,
    6277              :                              olw);
    6278          500 :       if (mode != op_mode && !do_widen_mult)
    6279           39 :         y[i] = convert_modes (mode, op_mode, y[i], 1);
    6280              :     }
    6281          632 :   for (unsigned int i = 0; i < s; ++i)
    6282              :     {
    6283         2552 :       for (unsigned int j = 0; j < s; ++j)
    6284              :         {
    6285         2052 :           unsigned int k = (i + s - j) % s;
    6286         2052 :           if (do_widen_mult)
    6287          592 :             t[j] = expand_widening_mult (mode, x[j], y[k], NULL_RTX, 1,
    6288              :                                          umul_widen_optab);
    6289              :           else
    6290         1460 :             t[j] = expand_binop (mode, smul_optab, x[j], y[k], NULL_RTX, 1,
    6291              :                                  olw);
    6292         2052 :           if ((j & 1) == 1)
    6293              :             {
    6294          880 :               t[j - 1] = expand_binop (mode, xor_optab, t[j - 1], t[j],
    6295              :                                        NULL_RTX, 1, olw);
    6296          880 :               t[j] = NULL_RTX;
    6297              :             }
    6298          880 :           if ((j & 3) == 3)
    6299              :             {
    6300          380 :               t[j - 3] = expand_binop (mode, xor_optab, t[j - 3], t[j - 1],
    6301              :                                        NULL_RTX, 1, olw);
    6302          380 :               t[j - 1] = NULL_RTX;
    6303              :             }
    6304              :         }
    6305         2052 :       for (unsigned int j = 1; j < s; ++j)
    6306         1552 :         if (t[j])
    6307          292 :           t[0] = expand_binop (mode, xor_optab, t[0], t[j], NULL_RTX, 1, olw);
    6308          500 :       if (mode != op_mode)
    6309          171 :         mask[i]
    6310          171 :           = immed_wide_int_const (wi::bit_repeat (wide_int::from (1 << i, s,
    6311              :                                                                   UNSIGNED),
    6312              :                                                   prec), mode);
    6313          500 :       t[0] = expand_binop (mode, and_optab, t[0], mask[i], NULL_RTX, 1, olw);
    6314          500 :       if (i == 0)
    6315              :         r = t[0];
    6316              :       else
    6317          368 :         r = expand_binop (mode, ior_optab, r, t[0], NULL_RTX, 1, olw);
    6318              :     }
    6319              :   return r;
    6320              : }
    6321              : 
    6322              : void
    6323           88 : expand_CLMUL (internal_fn, gcall *stmt)
    6324              : {
    6325           88 :   tree lhs = gimple_call_lhs (stmt);
    6326           88 :   tree arg0 = gimple_call_arg (stmt, 0);
    6327           88 :   tree arg1 = gimple_call_arg (stmt, 1);
    6328           88 :   scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
    6329           88 :   unsigned int prec0 = get_min_precision (arg0, UNSIGNED);
    6330           88 :   unsigned int prec1 = get_min_precision (arg1, UNSIGNED);
    6331           88 :   rtx op0 = expand_normal (arg0);
    6332           88 :   rtx op1 = expand_normal (arg1);
    6333           88 :   rtx res = expand_clmul (mode, mode, op0, op1, prec0, prec1);
    6334           88 :   if (TYPE_PRECISION (TREE_TYPE (arg0)) < GET_MODE_PRECISION (mode))
    6335              :     {
    6336            5 :       rtx mask
    6337            5 :         = immed_wide_int_const (wi::mask (TYPE_PRECISION (TREE_TYPE (arg0)),
    6338              :                                           false,
    6339            5 :                                           GET_MODE_PRECISION (mode)), mode);
    6340            5 :       res = expand_and (mode, res, mask, NULL_RTX);
    6341              :     }
    6342           88 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    6343           88 :   emit_move_insn (target, res);
    6344           88 : }
    6345              : 
    6346              : void
    6347           73 : expand_CLMUL_WIDEN (internal_fn, gcall *stmt)
    6348              : {
    6349           73 :   tree lhs = gimple_call_lhs (stmt);
    6350           73 :   tree arg0 = gimple_call_arg (stmt, 0);
    6351           73 :   tree arg1 = gimple_call_arg (stmt, 1);
    6352           73 :   scalar_int_mode mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (arg0));
    6353           73 :   unsigned int prec0 = get_min_precision (arg0, UNSIGNED);
    6354           73 :   unsigned int prec1 = get_min_precision (arg1, UNSIGNED);
    6355           73 :   unsigned int prec = GET_MODE_PRECISION (mode);
    6356           73 :   unsigned int type_prec = TYPE_PRECISION (TREE_TYPE (arg0));
    6357           73 :   rtx op0 = expand_normal (arg0);
    6358           73 :   rtx op1 = expand_normal (arg1);
    6359           73 :   rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    6360           73 :   rtx bitint_mask = NULL_RTX;
    6361           73 :   if (type_prec != prec)
    6362           15 :     bitint_mask
    6363           15 :       = immed_wide_int_const (wi::mask (type_prec, false, prec), mode);
    6364           73 :   if (prec0 + prec1 - 1 <= prec)
    6365              :     {
    6366              :       /* The whole product fits in MODE.  For a type narrower than MODE
    6367              :          the bits above its precision are the high half.  */
    6368            8 :       rtx reslo = expand_clmul (mode, mode, op0, op1, prec0, prec1);
    6369            8 :       rtx reshi = const0_rtx;
    6370            8 :       if (prec0 + prec1 - 1 > type_prec)
    6371            8 :         reshi = expand_shift (RSHIFT_EXPR, mode, reslo, type_prec, NULL_RTX, 1);
    6372            8 :       if (bitint_mask)
    6373            8 :         reslo = expand_and (mode, reslo, bitint_mask, NULL_RTX);
    6374            8 :       write_complex_part (target, reslo, false, true);
    6375            8 :       write_complex_part (target, reshi, true, false);
    6376            8 :       return;
    6377              :     }
    6378           65 :   if (optab_handler (clmul_widen_optab, mode) != CODE_FOR_nothing
    6379           65 :       && type_prec == prec)
    6380              :     {
    6381            2 :       rtx_insn *start = get_last_insn ();
    6382            2 :       rtx reslo = gen_reg_rtx (mode);
    6383            2 :       rtx reshi = gen_reg_rtx (mode);
    6384            2 :       class expand_operand ops[4];
    6385            2 :       create_output_operand (&ops[0], reslo, mode);
    6386            2 :       create_output_operand (&ops[1], reshi, mode);
    6387            2 :       create_input_operand (&ops[2], op0, mode);
    6388            2 :       create_input_operand (&ops[3], op1, mode);
    6389            2 :       if (maybe_expand_insn (optab_handler (clmul_widen_optab, mode),
    6390              :                              4, ops))
    6391              :         {
    6392            2 :           write_complex_part (target, reslo, false, true);
    6393            2 :           write_complex_part (target, reshi, true, false);
    6394            2 :           return;
    6395              :         }
    6396            0 :       delete_insns_since (start);
    6397              :     }
    6398           63 :   scalar_int_mode wmode;
    6399           63 :   if (GET_MODE_2XWIDER_MODE (mode).exists (&wmode)
    6400           63 :       && targetm.scalar_mode_supported_p (wmode))
    6401              :     {
    6402           55 :       rtx res = expand_clmul (wmode, mode, op0, op1, prec0, prec1);
    6403           55 :       rtx reslo = convert_modes (mode, wmode, res, 1);
    6404           55 :       if (bitint_mask)
    6405            7 :         reslo = expand_and (mode, reslo, bitint_mask, NULL_RTX);
    6406           55 :       write_complex_part (target, reslo, false, true);
    6407           55 :       rtx reshi = expand_shift (RSHIFT_EXPR, wmode, res, type_prec,
    6408              :                                 NULL_RTX, 1);
    6409           55 :       reshi = convert_modes (mode, wmode, reshi, 1);
    6410           55 :       write_complex_part (target, reshi, true, false);
    6411           55 :       return;
    6412              :     }
    6413              :   /* If we have native bitreverse in mode, fallback using:
    6414              :      uint64_t l = clmul (x, y);
    6415              :      uint64_t h = clmul (__builtin_bitreverseg (x),
    6416              :                          __builtin_bitreverseg (y));
    6417              :      return { l, __builtin_bitreverseg (h) >> 1 };  */
    6418            8 :   if (optab_handler (bitreverse_optab, mode) != CODE_FOR_nothing
    6419            8 :       || (prec == 2 * UNITS_PER_WORD
    6420            0 :           && optab_handler (bitreverse_optab, word_mode) != CODE_FOR_nothing))
    6421              :     {
    6422            0 :       rtx reslo = expand_clmul (mode, mode, op0, op1, prec0, prec1);
    6423            0 :       if (bitint_mask)
    6424            0 :         reslo = expand_and (mode, reslo, bitint_mask, NULL_RTX);
    6425            0 :       write_complex_part (target, reslo, false, true);
    6426            0 :       rtx revx = expand_unop (mode, bitreverse_optab, op0, NULL_RTX, 1);
    6427            0 :       rtx revy = expand_unop (mode, bitreverse_optab, op1, NULL_RTX, 1);
    6428            0 :       rtx reshi = expand_clmul (mode, mode, revx, revy, prec, prec);
    6429            0 :       reshi = expand_unop (mode, bitreverse_optab, reshi, NULL_RTX, 1);
    6430            0 :       if (bitint_mask)
    6431            0 :         reshi = expand_and (mode, reshi, bitint_mask, NULL_RTX);
    6432            0 :       reshi = expand_shift (RSHIFT_EXPR, mode, reshi, 1, NULL_RTX, 1);
    6433            0 :       write_complex_part (target, reshi, true, false);
    6434            0 :       return;
    6435              :     }
    6436              :   /* Fallback to modified Karatsuba multiplication step
    6437              :      using 3x expand_clmul in mode.
    6438              :      For 64x64 -> 128bit CLMUL this would be:
    6439              :      uint64_t x0 = (uint32_t) x, x1 = (x >> 32);
    6440              :      uint64_t y0 = (uint32_t) y, y1 = (y >> 32);
    6441              :      uint64_t l = clmul (x0, y0);
    6442              :      uint64_t m = clmul (x0 ^ x1, y0 ^ y1);
    6443              :      uint64_t h = clmul (x1, y1);
    6444              :      m ^= l ^ h;
    6445              :      return { l ^ (m << 32), h ^ (m >> 32) };  */
    6446            8 :   unsigned int hprec = prec / 2;
    6447            8 :   rtx mask = immed_wide_int_const (wi::mask (hprec, false, prec), mode);
    6448            8 :   optab_methods olw = OPTAB_LIB_WIDEN;
    6449            8 :   rtx x0 = expand_binop (mode, and_optab, op0, mask, NULL_RTX, 1, olw);
    6450            8 :   rtx x1 = expand_shift (RSHIFT_EXPR, mode, op0, hprec, NULL_RTX, 1);
    6451            8 :   rtx y0 = expand_binop (mode, and_optab, op1, mask, NULL_RTX, 1, olw);
    6452            8 :   rtx y1 = expand_shift (RSHIFT_EXPR, mode, op1, hprec, NULL_RTX, 1);
    6453            8 :   rtx l = expand_clmul (mode, mode, x0, y0, hprec, hprec);
    6454            8 :   rtx t0 = expand_binop (mode, xor_optab, x0, x1, NULL_RTX, 1, olw);
    6455            8 :   rtx t1 = expand_binop (mode, xor_optab, y0, y1, NULL_RTX, 1, olw);
    6456            8 :   rtx m = expand_clmul (mode, mode, t0, t1, hprec, hprec);
    6457            8 :   rtx h;
    6458            8 :   if (prec0 <= hprec || prec1 <= hprec)
    6459            0 :     h = const0_rtx;
    6460              :   else
    6461            8 :     h = expand_clmul (mode, mode, x1, y1, MIN (hprec, prec0 - hprec),
    6462            8 :                       MIN (hprec, prec1 - hprec));
    6463            8 :   t0 = expand_binop (mode, xor_optab, l, h, NULL_RTX, 1, olw);
    6464            8 :   m = expand_binop (mode, xor_optab, m, t0, NULL_RTX, 1, olw);
    6465            8 :   t0 = expand_shift (LSHIFT_EXPR, mode, m, hprec, NULL_RTX, 1);
    6466            8 :   t1 = expand_shift (RSHIFT_EXPR, mode, m, hprec, NULL_RTX, 1);
    6467            8 :   rtx reslo = expand_binop (mode, xor_optab, l, t0, NULL_RTX, 1, olw);
    6468            8 :   if (bitint_mask)
    6469              :     {
    6470            0 :       rtx reslom = expand_and (mode, reslo, bitint_mask, NULL_RTX);
    6471            0 :       write_complex_part (target, reslom, false, true);
    6472            0 :       rtx reshi = expand_binop (mode, xor_optab, h, t1, NULL_RTX, 1, olw);
    6473            0 :       reshi = expand_shift (LSHIFT_EXPR, mode, reshi,
    6474            0 :                             prec - type_prec, NULL_RTX, 1);
    6475            0 :       reslo = expand_shift (RSHIFT_EXPR, mode, reslo, type_prec, NULL_RTX, 1);
    6476            0 :       reshi = expand_binop (mode, ior_optab, reshi, reslo, NULL_RTX, 1, olw);
    6477            0 :       reshi = expand_and (mode, reshi, bitint_mask, NULL_RTX);
    6478            0 :       write_complex_part (target, reshi, true, false);
    6479              :     }
    6480              :   else
    6481              :     {
    6482            8 :       write_complex_part (target, reslo, false, true);
    6483            8 :       rtx reshi = expand_binop (mode, xor_optab, h, t1, NULL_RTX, 1, olw);
    6484            8 :       write_complex_part (target, reshi, true, false);
    6485              :     }
    6486              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.