LCOV - code coverage report
Current view: top level - gcc - builtins.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 85.7 % 5858 5023
Test Date: 2026-11-14 16:39:50 Functions: 93.7 % 222 208
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* Expand builtin functions.
       2              :    Copyright (C) 1988-2026 Free Software Foundation, Inc.
       3              : 
       4              : This file is part of GCC.
       5              : 
       6              : GCC is free software; you can redistribute it and/or modify it under
       7              : the terms of the GNU General Public License as published by the Free
       8              : Software Foundation; either version 3, or (at your option) any later
       9              : version.
      10              : 
      11              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      12              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      13              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      14              : for more details.
      15              : 
      16              : You should have received a copy of the GNU General Public License
      17              : along with GCC; see the file COPYING3.  If not see
      18              : <http://www.gnu.org/licenses/>.  */
      19              : 
      20              : /* Legacy warning!  Please add no further builtin simplifications here
      21              :    (apart from pure constant folding) - builtin simplifications should go
      22              :    to match.pd or gimple-fold.cc instead.  */
      23              : 
      24              : #include "config.h"
      25              : #include "system.h"
      26              : #include "coretypes.h"
      27              : #include "backend.h"
      28              : #include "target.h"
      29              : #include "rtl.h"
      30              : #include "tree.h"
      31              : #include "memmodel.h"
      32              : #include "gimple.h"
      33              : #include "tree-eh.h"
      34              : #include "predict.h"
      35              : #include "tm_p.h"
      36              : #include "stringpool.h"
      37              : #include "tree-vrp.h"
      38              : #include "tree-ssanames.h"
      39              : #include "expmed.h"
      40              : #include "optabs.h"
      41              : #include "emit-rtl.h"
      42              : #include "recog.h"
      43              : #include "diagnostic-core.h"
      44              : #include "alias.h"
      45              : #include "fold-const.h"
      46              : #include "fold-const-call.h"
      47              : #include "gimple-ssa-warn-access.h"
      48              : #include "stor-layout.h"
      49              : #include "calls.h"
      50              : #include "varasm.h"
      51              : #include "tree-object-size.h"
      52              : #include "tree-ssa-strlen.h"
      53              : #include "realmpfr.h"
      54              : #include "cfgrtl.h"
      55              : #include "except.h"
      56              : #include "dojump.h"
      57              : #include "explow.h"
      58              : #include "stmt.h"
      59              : #include "expr.h"
      60              : #include "libfuncs.h"
      61              : #include "output.h"
      62              : #include "typeclass.h"
      63              : #include "langhooks.h"
      64              : #include "value-prof.h"
      65              : #include "builtins.h"
      66              : #include "attribs.h"
      67              : #include "asan.h"
      68              : #include "internal-fn.h"
      69              : #include "case-cfn-macros.h"
      70              : #include "gimple-iterator.h"
      71              : #include "gimple-fold.h"
      72              : #include "intl.h"
      73              : #include "file-prefix-map.h" /* remap_macro_filename()  */
      74              : #include "ipa-strub.h" /* strub_watermark_parm()  */
      75              : #include "gomp-constants.h"
      76              : #include "omp-general.h"
      77              : #include "tree-dfa.h"
      78              : #include "gimple-ssa.h"
      79              : #include "tree-ssa-live.h"
      80              : #include "tree-outof-ssa.h"
      81              : #include "attr-fnspec.h"
      82              : #include "demangle.h"
      83              : #include "gimple-range.h"
      84              : #include "pointer-query.h"
      85              : 
      86              : struct target_builtins default_target_builtins;
      87              : #if SWITCHABLE_TARGET
      88              : struct target_builtins *this_target_builtins = &default_target_builtins;
      89              : #endif
      90              : 
      91              : /* Define the names of the builtin function types and codes.  */
      92              : const char *const built_in_class_names[BUILT_IN_LAST]
      93              :   = {"NOT_BUILT_IN", "BUILT_IN_FRONTEND", "BUILT_IN_MD", "BUILT_IN_NORMAL"};
      94              : 
      95              : #define DEF_BUILTIN(X, N, C, T, LT, B, F, NA, AT, IM, COND) #X,
      96              : const char * built_in_names[(int) END_BUILTINS] =
      97              : {
      98              : #include "builtins.def"
      99              : };
     100              : 
     101              : /* Setup an array of builtin_info_type, make sure each element decl is
     102              :    initialized to NULL_TREE.  */
     103              : builtin_info_type builtin_info[(int)END_BUILTINS];
     104              : 
     105              : /* Non-zero if __builtin_constant_p should be folded right away.  */
     106              : bool force_folding_builtin_constant_p;
     107              : 
     108              : static int target_char_cast (tree, char *);
     109              : static int apply_args_size (void);
     110              : static int apply_result_size (void);
     111              : static rtx result_vector (int, rtx);
     112              : static void expand_builtin_prefetch (tree);
     113              : static rtx expand_builtin_apply_args (void);
     114              : static rtx expand_builtin_apply_args_1 (void);
     115              : static rtx expand_builtin_apply (rtx, rtx, rtx);
     116              : static void expand_builtin_return (rtx);
     117              : static rtx expand_builtin_classify_type (tree);
     118              : static rtx expand_builtin_mathfn_3 (tree, rtx, rtx);
     119              : static rtx expand_builtin_mathfn_ternary (tree, rtx, rtx);
     120              : static rtx expand_builtin_interclass_mathfn (tree, rtx);
     121              : static rtx expand_builtin_sincos (tree);
     122              : static rtx expand_builtin_fegetround (tree, rtx, machine_mode);
     123              : static rtx expand_builtin_feclear_feraise_except (tree, rtx, machine_mode,
     124              :                                                   optab);
     125              : static rtx expand_builtin_cexpi (tree, rtx);
     126              : static rtx expand_builtin_issignaling (tree, rtx);
     127              : static rtx expand_builtin_int_roundingfn (tree, rtx);
     128              : static rtx expand_builtin_int_roundingfn_2 (tree, rtx);
     129              : static rtx expand_builtin_next_arg (void);
     130              : static rtx expand_builtin_va_start (tree);
     131              : static rtx expand_builtin_va_end (tree);
     132              : static rtx expand_builtin_va_copy (tree);
     133              : static rtx inline_expand_builtin_bytecmp (tree, rtx);
     134              : static rtx expand_builtin_strcmp (tree, rtx);
     135              : static rtx expand_builtin_strncmp (tree, rtx, machine_mode);
     136              : static rtx expand_builtin_memcpy (tree, rtx);
     137              : static rtx expand_builtin_memory_copy_args (tree dest, tree src, tree len,
     138              :                                             rtx target, tree exp,
     139              :                                             memop_ret retmode,
     140              :                                             bool might_overlap);
     141              : static rtx expand_builtin_memmove (tree, rtx);
     142              : static rtx expand_builtin_mempcpy (tree, rtx);
     143              : static rtx expand_builtin_mempcpy_args (tree, tree, tree, rtx, tree, memop_ret);
     144              : static rtx expand_builtin_strcpy (tree, rtx);
     145              : static rtx expand_builtin_strcpy_args (tree, tree, tree, rtx);
     146              : static rtx expand_builtin_stpcpy (tree, rtx, machine_mode);
     147              : static rtx expand_builtin_strncpy (tree, rtx);
     148              : static rtx expand_builtin_memset_args (tree, tree, tree, rtx, machine_mode, tree);
     149              : static rtx expand_builtin_bzero (tree);
     150              : static rtx expand_builtin_strlen (tree, rtx, machine_mode);
     151              : static rtx expand_builtin_strnlen (tree, rtx, machine_mode);
     152              : static rtx expand_builtin_alloca (tree);
     153              : static rtx expand_builtin_unop (machine_mode, tree, rtx, rtx, optab);
     154              : static rtx expand_builtin_frame_address (tree, tree);
     155              : static rtx expand_builtin_stack_address ();
     156              : static tree stabilize_va_list_loc (location_t, tree, int);
     157              : static rtx expand_builtin_expect (tree, rtx);
     158              : static rtx expand_builtin_expect_with_probability (tree, rtx);
     159              : static tree fold_builtin_classify_type (tree);
     160              : static tree fold_builtin_strlen (location_t, tree, tree, tree);
     161              : static tree fold_builtin_inf (location_t, tree, int);
     162              : static tree rewrite_call_expr (location_t, tree, int, tree, int, ...);
     163              : static bool validate_arg (const_tree, enum tree_code code);
     164              : static rtx expand_builtin_fabs (tree, rtx, rtx);
     165              : static rtx expand_builtin_signbit (tree, rtx);
     166              : static tree fold_builtin_memcmp (location_t, tree, tree, tree);
     167              : static tree fold_builtin_isascii (location_t, tree);
     168              : static tree fold_builtin_toascii (location_t, tree);
     169              : static tree fold_builtin_isdigit (location_t, tree);
     170              : static tree fold_builtin_fabs (location_t, tree, tree);
     171              : static tree fold_builtin_abs (location_t, tree, tree);
     172              : static tree fold_builtin_unordered_cmp (location_t, tree, tree, tree, enum tree_code,
     173              :                                         enum tree_code);
     174              : static tree fold_builtin_iseqsig (location_t, tree, tree);
     175              : static tree fold_builtin_varargs (location_t, tree, tree*, int);
     176              : 
     177              : static tree fold_builtin_strpbrk (location_t, tree, tree, tree, tree);
     178              : static tree fold_builtin_strspn (location_t, tree, tree, tree, tree);
     179              : static tree fold_builtin_strcspn (location_t, tree, tree, tree, tree);
     180              : 
     181              : static rtx expand_builtin_object_size (tree);
     182              : static rtx expand_builtin_memory_chk (tree, rtx, machine_mode,
     183              :                                       enum built_in_function);
     184              : static void maybe_emit_chk_warning (tree, enum built_in_function);
     185              : static void maybe_emit_sprintf_chk_warning (tree, enum built_in_function);
     186              : static tree fold_builtin_object_size (tree, tree, enum built_in_function);
     187              : 
     188              : unsigned HOST_WIDE_INT target_newline;
     189              : unsigned HOST_WIDE_INT target_percent;
     190              : static unsigned HOST_WIDE_INT target_c;
     191              : static unsigned HOST_WIDE_INT target_s;
     192              : char target_percent_c[3];
     193              : char target_percent_s[3];
     194              : char target_percent_s_newline[4];
     195              : static tree do_mpfr_remquo (tree, tree, tree);
     196              : static tree do_mpfr_lgamma_r (tree, tree, tree);
     197              : static void expand_builtin_sync_synchronize (void);
     198              : 
     199              : /* Return true if NAME starts with __builtin_ or __sync_.  */
     200              : 
     201              : static bool
     202       543401 : is_builtin_name (const char *name)
     203              : {
     204       543401 :   return (startswith (name, "__builtin_")
     205       195048 :           || startswith (name, "__sync_")
     206       737584 :           || startswith (name, "__atomic_"));
     207              : }
     208              : 
     209              : /* Return true if NODE should be considered for inline expansion regardless
     210              :    of the optimization level.  This means whenever a function is invoked with
     211              :    its "internal" name, which normally contains the prefix "__builtin".  */
     212              : 
     213              : bool
     214       543401 : called_as_built_in (tree node)
     215              : {
     216              :   /* Note that we must use DECL_NAME, not DECL_ASSEMBLER_NAME_SET_P since
     217              :      we want the name used to call the function, not the name it
     218              :      will have. */
     219       543401 :   const char *name = IDENTIFIER_POINTER (DECL_NAME (node));
     220       543401 :   return is_builtin_name (name);
     221              : }
     222              : 
     223              : /* Compute values M and N such that M divides (address of EXP - N) and such
     224              :    that N < M.  If these numbers can be determined, store M in alignp and N in
     225              :    *BITPOSP and return true.  Otherwise return false and store BITS_PER_UNIT to
     226              :    *alignp and any bit-offset to *bitposp.
     227              : 
     228              :    Note that the address (and thus the alignment) computed here is based
     229              :    on the address to which a symbol resolves, whereas DECL_ALIGN is based
     230              :    on the address at which an object is actually located.  These two
     231              :    addresses are not always the same.  For example, on ARM targets,
     232              :    the address &foo of a Thumb function foo() has the lowest bit set,
     233              :    whereas foo() itself starts on an even address.
     234              : 
     235              :    If ADDR_P is true we are taking the address of the memory reference EXP
     236              :    and thus cannot rely on the access taking place.  */
     237              : 
     238              : bool
     239    112874270 : get_object_alignment_2 (tree exp, unsigned int *alignp,
     240              :                         unsigned HOST_WIDE_INT *bitposp, bool addr_p)
     241              : {
     242    112874270 :   poly_int64 bitsize, bitpos;
     243    112874270 :   tree offset;
     244    112874270 :   machine_mode mode;
     245    112874270 :   int unsignedp, reversep, volatilep;
     246    112874270 :   unsigned int align = BITS_PER_UNIT;
     247    112874270 :   bool known_alignment = false;
     248              : 
     249              :   /* Get the innermost object and the constant (bitpos) and possibly
     250              :      variable (offset) offset of the access.  */
     251    112874270 :   exp = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode,
     252              :                              &unsignedp, &reversep, &volatilep);
     253              : 
     254              :   /* Extract alignment information from the innermost object and
     255              :      possibly adjust bitpos and offset.  */
     256    112874270 :   if (TREE_CODE (exp) == FUNCTION_DECL)
     257              :     {
     258              :       /* Function addresses can encode extra information besides their
     259              :          alignment.  However, if TARGET_PTRMEMFUNC_VBIT_LOCATION
     260              :          allows the low bit to be used as a virtual bit, we know
     261              :          that the address itself must be at least 2-byte aligned.  */
     262              :       if (TARGET_PTRMEMFUNC_VBIT_LOCATION == ptrmemfunc_vbit_in_pfn)
     263              :         align = 2 * BITS_PER_UNIT;
     264              :     }
     265    112421566 :   else if (TREE_CODE (exp) == LABEL_DECL)
     266              :     ;
     267    112384620 :   else if (TREE_CODE (exp) == CONST_DECL)
     268              :     {
     269              :       /* The alignment of a CONST_DECL is determined by its initializer.  */
     270        59935 :       exp = DECL_INITIAL (exp);
     271        59935 :       align = TYPE_ALIGN (TREE_TYPE (exp));
     272        59935 :       if (CONSTANT_CLASS_P (exp))
     273        59934 :         align = targetm.constant_alignment (exp, align);
     274              : 
     275              :       known_alignment = true;
     276              :     }
     277    112324685 :   else if (DECL_P (exp))
     278              :     {
     279     67491749 :       align = DECL_ALIGN (exp);
     280              :       known_alignment = true;
     281              :     }
     282     44832936 :   else if (TREE_CODE (exp) == INDIRECT_REF
     283     44831880 :            || TREE_CODE (exp) == MEM_REF
     284      5350840 :            || TREE_CODE (exp) == TARGET_MEM_REF)
     285              :     {
     286     43178185 :       tree addr = TREE_OPERAND (exp, 0);
     287     43178185 :       unsigned ptr_align;
     288     43178185 :       unsigned HOST_WIDE_INT ptr_bitpos;
     289     43178185 :       unsigned HOST_WIDE_INT ptr_bitmask = ~0;
     290              : 
     291              :       /* If the address is explicitly aligned, handle that.  */
     292     43178185 :       if (TREE_CODE (addr) == BIT_AND_EXPR
     293     43178185 :           && TREE_CODE (TREE_OPERAND (addr, 1)) == INTEGER_CST)
     294              :         {
     295           88 :           ptr_bitmask = TREE_INT_CST_LOW (TREE_OPERAND (addr, 1));
     296           88 :           ptr_bitmask *= BITS_PER_UNIT;
     297           88 :           align = least_bit_hwi (ptr_bitmask);
     298           88 :           addr = TREE_OPERAND (addr, 0);
     299              :         }
     300              : 
     301     43178185 :       known_alignment
     302     43178185 :         = get_pointer_alignment_1 (addr, &ptr_align, &ptr_bitpos);
     303     43178185 :       align = MAX (ptr_align, align);
     304              : 
     305              :       /* Re-apply explicit alignment to the bitpos.  */
     306     43178185 :       ptr_bitpos &= ptr_bitmask;
     307              : 
     308              :       /* The alignment of the pointer operand in a TARGET_MEM_REF
     309              :          has to take the variable offset parts into account.  */
     310     43178185 :       if (TREE_CODE (exp) == TARGET_MEM_REF)
     311              :         {
     312      3696089 :           if (TMR_INDEX (exp))
     313              :             {
     314      1618882 :               unsigned HOST_WIDE_INT step = 1;
     315      1618882 :               if (TMR_STEP (exp))
     316      1485969 :                 step = TREE_INT_CST_LOW (TMR_STEP (exp));
     317      1618882 :               align = MIN (align, least_bit_hwi (step) * BITS_PER_UNIT);
     318              :             }
     319      3696089 :           if (TMR_INDEX2 (exp))
     320        41885 :             align = BITS_PER_UNIT;
     321              :           known_alignment = false;
     322              :         }
     323              : 
     324              :       /* When EXP is an actual memory reference then we can use
     325              :          TYPE_ALIGN of a pointer indirection to derive alignment.
     326              :          Do so only if get_pointer_alignment_1 did not reveal absolute
     327              :          alignment knowledge and if using that alignment would
     328              :          improve the situation.  */
     329     43178185 :       unsigned int talign;
     330     43178185 :       if (!addr_p && !known_alignment
     331     40747855 :           && (talign = min_align_of_type (TREE_TYPE (exp)) * BITS_PER_UNIT)
     332     83926034 :           && talign > align)
     333              :         align = talign;
     334              :       else
     335              :         {
     336              :           /* Else adjust bitpos accordingly.  */
     337     11825803 :           bitpos += ptr_bitpos;
     338     11825803 :           if (TREE_CODE (exp) == MEM_REF
     339     11825803 :               || TREE_CODE (exp) == TARGET_MEM_REF)
     340     11824747 :             bitpos += mem_ref_offset (exp).force_shwi () * BITS_PER_UNIT;
     341              :         }
     342     43178185 :     }
     343      1654751 :   else if (TREE_CODE (exp) == STRING_CST)
     344              :     {
     345              :       /* STRING_CST are the only constant objects we allow to be not
     346              :          wrapped inside a CONST_DECL.  */
     347      1654131 :       align = TYPE_ALIGN (TREE_TYPE (exp));
     348      1654131 :       if (CONSTANT_CLASS_P (exp))
     349      1654131 :         align = targetm.constant_alignment (exp, align);
     350              : 
     351              :       known_alignment = true;
     352              :     }
     353              : 
     354              :   /* If there is a non-constant offset part extract the maximum
     355              :      alignment that can prevail.  */
     356    112874270 :   if (offset)
     357              :     {
     358     10219948 :       unsigned int trailing_zeros = tree_ctz (offset);
     359     10219948 :       if (trailing_zeros < HOST_BITS_PER_INT)
     360              :         {
     361     10219870 :           unsigned int inner = (1U << trailing_zeros) * BITS_PER_UNIT;
     362     10219870 :           if (inner)
     363     10219870 :             align = MIN (align, inner);
     364              :         }
     365              :     }
     366              : 
     367              :   /* Account for the alignment of runtime coefficients, so that the constant
     368              :      bitpos is guaranteed to be accurate.  */
     369    112874270 :   unsigned int alt_align = ::known_alignment (bitpos - bitpos.coeffs[0]);
     370    112874270 :   if (alt_align != 0 && alt_align < align)
     371              :     {
     372              :       align = alt_align;
     373              :       known_alignment = false;
     374              :     }
     375              : 
     376    112874270 :   *alignp = align;
     377    112874270 :   *bitposp = bitpos.coeffs[0] & (align - 1);
     378    112874270 :   return known_alignment;
     379              : }
     380              : 
     381              : /* For a memory reference expression EXP compute values M and N such that M
     382              :    divides (&EXP - N) and such that N < M.  If these numbers can be determined,
     383              :    store M in alignp and N in *BITPOSP and return true.  Otherwise return false
     384              :    and store BITS_PER_UNIT to *alignp and any bit-offset to *bitposp.  */
     385              : 
     386              : bool
     387     89646785 : get_object_alignment_1 (tree exp, unsigned int *alignp,
     388              :                         unsigned HOST_WIDE_INT *bitposp)
     389              : {
     390              :   /* Strip a WITH_SIZE_EXPR, get_inner_reference doesn't know how to deal
     391              :      with it.  */
     392     89646785 :   if (TREE_CODE (exp) == WITH_SIZE_EXPR)
     393            0 :     exp = TREE_OPERAND (exp, 0);
     394     89646785 :   return get_object_alignment_2 (exp, alignp, bitposp, false);
     395              : }
     396              : 
     397              : /* Return the alignment in bits of EXP, an object.  */
     398              : 
     399              : unsigned int
     400     37203882 : get_object_alignment (tree exp)
     401              : {
     402     37203882 :   unsigned HOST_WIDE_INT bitpos = 0;
     403     37203882 :   unsigned int align;
     404              : 
     405     37203882 :   get_object_alignment_1 (exp, &align, &bitpos);
     406              : 
     407              :   /* align and bitpos now specify known low bits of the pointer.
     408              :      ptr & (align - 1) == bitpos.  */
     409              : 
     410     37203882 :   if (bitpos != 0)
     411      2781276 :     align = least_bit_hwi (bitpos);
     412     37203882 :   return align;
     413              : }
     414              : 
     415              : /* For a pointer valued expression EXP compute values M and N such that M
     416              :    divides (EXP - N) and such that N < M.  If these numbers can be determined,
     417              :    store M in alignp and N in *BITPOSP and return true.  Return false if
     418              :    the results are just a conservative approximation.
     419              : 
     420              :    If EXP is not a pointer, false is returned too.  */
     421              : 
     422              : bool
     423     85224520 : get_pointer_alignment_1 (tree exp, unsigned int *alignp,
     424              :                          unsigned HOST_WIDE_INT *bitposp)
     425              : {
     426     85224520 :   STRIP_NOPS (exp);
     427              : 
     428     85224520 :   if (TREE_CODE (exp) == ADDR_EXPR)
     429     23225845 :     return get_object_alignment_2 (TREE_OPERAND (exp, 0),
     430     23225845 :                                    alignp, bitposp, true);
     431     61998675 :   else if (TREE_CODE (exp) == POINTER_PLUS_EXPR)
     432              :     {
     433      1031537 :       unsigned int align;
     434      1031537 :       unsigned HOST_WIDE_INT bitpos;
     435      1031537 :       bool res = get_pointer_alignment_1 (TREE_OPERAND (exp, 0),
     436              :                                           &align, &bitpos);
     437      1031537 :       if (TREE_CODE (TREE_OPERAND (exp, 1)) == INTEGER_CST)
     438         3764 :         bitpos += TREE_INT_CST_LOW (TREE_OPERAND (exp, 1)) * BITS_PER_UNIT;
     439              :       else
     440              :         {
     441      1027773 :           unsigned int trailing_zeros = tree_ctz (TREE_OPERAND (exp, 1));
     442      1027773 :           if (trailing_zeros < HOST_BITS_PER_INT)
     443              :             {
     444      1027765 :               unsigned int inner = (1U << trailing_zeros) * BITS_PER_UNIT;
     445      1027765 :               if (inner)
     446      1027765 :                 align = MIN (align, inner);
     447              :             }
     448              :         }
     449      1031537 :       *alignp = align;
     450      1031537 :       *bitposp = bitpos & (align - 1);
     451      1031537 :       return res;
     452              :     }
     453     60967138 :   else if (TREE_CODE (exp) == SSA_NAME
     454     60967138 :            && POINTER_TYPE_P (TREE_TYPE (exp)))
     455              :     {
     456     58622998 :       unsigned int ptr_align, ptr_misalign;
     457              : 
     458     58622998 :       if (get_ssa_name_alignment (exp, &ptr_align, &ptr_misalign))
     459              :         {
     460      6550548 :           *bitposp = ptr_misalign * BITS_PER_UNIT;
     461      6550548 :           *alignp = ptr_align * BITS_PER_UNIT;
     462              :           /* Make sure to return a sensible alignment when the multiplication
     463              :              by BITS_PER_UNIT overflowed.  */
     464      6550548 :           if (*alignp == 0)
     465          128 :             *alignp = 1u << (HOST_BITS_PER_INT - 1);
     466              :           /* We cannot really tell whether this result is an approximation.  */
     467              :           return false;
     468              :         }
     469              :       else
     470              :         {
     471     52072450 :           *bitposp = 0;
     472     52072450 :           *alignp = BITS_PER_UNIT;
     473     52072450 :           return false;
     474              :         }
     475              :     }
     476      2344140 :   else if (TREE_CODE (exp) == INTEGER_CST)
     477              :     {
     478        18375 :       *alignp = BIGGEST_ALIGNMENT;
     479        18375 :       *bitposp = ((TREE_INT_CST_LOW (exp) * BITS_PER_UNIT)
     480        18375 :                   & (BIGGEST_ALIGNMENT - 1));
     481        18375 :       return true;
     482              :     }
     483              : 
     484      2325765 :   *bitposp = 0;
     485      2325765 :   *alignp = BITS_PER_UNIT;
     486      2325765 :   return false;
     487              : }
     488              : 
     489              : /* Return the alignment in bits of EXP, a pointer valued expression.
     490              :    The alignment returned is, by default, the alignment of the thing that
     491              :    EXP points to.  If it is not a POINTER_TYPE, 0 is returned.
     492              : 
     493              :    Otherwise, look at the expression to see if we can do better, i.e., if the
     494              :    expression is actually pointing at an object whose alignment is tighter.  */
     495              : 
     496              : unsigned int
     497     11527631 : get_pointer_alignment (tree exp)
     498              : {
     499     11527631 :   unsigned HOST_WIDE_INT bitpos = 0;
     500     11527631 :   unsigned int align;
     501              : 
     502     11527631 :   get_pointer_alignment_1 (exp, &align, &bitpos);
     503              : 
     504              :   /* align and bitpos now specify known low bits of the pointer.
     505              :      ptr & (align - 1) == bitpos.  */
     506              : 
     507     11527631 :   if (bitpos != 0)
     508        81651 :     align = least_bit_hwi (bitpos);
     509              : 
     510     11527631 :   return align;
     511              : }
     512              : 
     513              : /* Return the number of leading non-zero elements in the sequence
     514              :    [ PTR, PTR + MAXELTS ) where each element's size is ELTSIZE bytes.
     515              :    ELTSIZE must be a power of 2 less than 8.  Used by c_strlen.  */
     516              : 
     517              : unsigned
     518       764132 : string_length (const void *ptr, unsigned eltsize, unsigned maxelts)
     519              : {
     520       764132 :   gcc_checking_assert (eltsize == 1 || eltsize == 2 || eltsize == 4);
     521              : 
     522       764132 :   unsigned n;
     523              : 
     524       764132 :   if (eltsize == 1)
     525              :     {
     526              :       /* Optimize the common case of plain char.  */
     527    244782163 :       for (n = 0; n < maxelts; n++)
     528              :         {
     529    244691373 :           const char *elt = (const char*) ptr + n;
     530    244691373 :           if (!*elt)
     531              :             break;
     532              :         }
     533              :     }
     534              :   else
     535              :     {
     536          590 :       for (n = 0; n < maxelts; n++)
     537              :         {
     538          566 :           const char *elt = (const char*) ptr + n * eltsize;
     539          566 :           if (!memcmp (elt, "\0\0\0\0", eltsize))
     540              :             break;
     541              :         }
     542              :     }
     543       764132 :   return n;
     544              : }
     545              : 
     546              : /* Compute the length of a null-terminated character string or wide
     547              :    character string handling character sizes of 1, 2, and 4 bytes.
     548              :    TREE_STRING_LENGTH is not the right way because it evaluates to
     549              :    the size of the character array in bytes (as opposed to characters)
     550              :    and because it can contain a zero byte in the middle.
     551              : 
     552              :    ONLY_VALUE should be nonzero if the result is not going to be emitted
     553              :    into the instruction stream and zero if it is going to be expanded.
     554              :    E.g. with i++ ? "foo" : "bar", if ONLY_VALUE is nonzero, constant 3
     555              :    is returned, otherwise NULL, since
     556              :    len = c_strlen (ARG, 1); if (len) expand_expr (len, ...); would not
     557              :    evaluate the side-effects.
     558              : 
     559              :    If ONLY_VALUE is two then we do not emit warnings about out-of-bound
     560              :    accesses.  Note that this implies the result is not going to be emitted
     561              :    into the instruction stream.
     562              : 
     563              :    Additional information about the string accessed may be recorded
     564              :    in DATA.  For example, if ARG references an unterminated string,
     565              :    then the declaration will be stored in the DECL field.   If the
     566              :    length of the unterminated string can be determined, it'll be
     567              :    stored in the LEN field.  Note this length could well be different
     568              :    than what a C strlen call would return.
     569              : 
     570              :    ELTSIZE is 1 for normal single byte character strings, and 2 or
     571              :    4 for wide character strings.  ELTSIZE is by default 1.
     572              : 
     573              :    The value returned is of type `ssizetype'.  */
     574              : 
     575              : tree
     576      2589047 : c_strlen (tree arg, int only_value, c_strlen_data *data, unsigned eltsize)
     577              : {
     578              :   /* If we were not passed a DATA pointer, then get one to a local
     579              :      structure.  That avoids having to check DATA for NULL before
     580              :      each time we want to use it.  */
     581      2589047 :   c_strlen_data local_strlen_data = { };
     582      2589047 :   if (!data)
     583       594479 :     data = &local_strlen_data;
     584              : 
     585      2589047 :   gcc_checking_assert (eltsize == 1 || eltsize == 2 || eltsize == 4);
     586              : 
     587      2589047 :   tree src = STRIP_NOPS (arg);
     588      2589047 :   if (TREE_CODE (src) == COND_EXPR
     589      2589047 :       && (only_value || !TREE_SIDE_EFFECTS (TREE_OPERAND (src, 0))))
     590              :     {
     591          690 :       tree len1, len2;
     592              : 
     593          690 :       len1 = c_strlen (TREE_OPERAND (src, 1), only_value, data, eltsize);
     594          690 :       len2 = c_strlen (TREE_OPERAND (src, 2), only_value, data, eltsize);
     595          690 :       if (tree_int_cst_equal (len1, len2))
     596              :         return len1;
     597              :     }
     598              : 
     599      2588881 :   if (TREE_CODE (src) == COMPOUND_EXPR
     600      2588881 :       && (only_value || !TREE_SIDE_EFFECTS (TREE_OPERAND (src, 0))))
     601           45 :     return c_strlen (TREE_OPERAND (src, 1), only_value, data, eltsize);
     602              : 
     603      2588836 :   location_t loc = EXPR_LOC_OR_LOC (src, input_location);
     604              : 
     605              :   /* Offset from the beginning of the string in bytes.  */
     606      2588836 :   tree byteoff;
     607      2588836 :   tree memsize;
     608      2588836 :   tree decl;
     609      2588836 :   src = string_constant (src, &byteoff, &memsize, &decl);
     610      2588836 :   if (src == 0)
     611              :     return NULL_TREE;
     612              : 
     613              :   /* Determine the size of the string element.  */
     614       780381 :   if (eltsize != tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (TREE_TYPE (src)))))
     615              :     return NULL_TREE;
     616              : 
     617              :   /* Set MAXELTS to ARRAY_SIZE (SRC) - 1, the maximum possible
     618              :      length of SRC.  Prefer TYPE_SIZE() to TREE_STRING_LENGTH() if possible
     619              :      in case the latter is less than the size of the array, such as when
     620              :      SRC refers to a short string literal used to initialize a large array.
     621              :      In that case, the elements of the array after the terminating NUL are
     622              :      all NUL.  */
     623       767960 :   HOST_WIDE_INT strelts = TREE_STRING_LENGTH (src);
     624       767960 :   strelts = strelts / eltsize;
     625              : 
     626       767960 :   if (!tree_fits_uhwi_p (memsize))
     627              :     return NULL_TREE;
     628              : 
     629       767960 :   HOST_WIDE_INT maxelts = tree_to_uhwi (memsize) / eltsize;
     630              : 
     631              :   /* PTR can point to the byte representation of any string type, including
     632              :      char* and wchar_t*.  */
     633       767960 :   const char *ptr = TREE_STRING_POINTER (src);
     634              : 
     635       767960 :   if (byteoff && TREE_CODE (byteoff) != INTEGER_CST)
     636              :     {
     637              :       /* The code below works only for single byte character types.  */
     638         5050 :       if (eltsize != 1)
     639              :         return NULL_TREE;
     640              : 
     641              :       /* If the string has an internal NUL character followed by any
     642              :          non-NUL characters (e.g., "foo\0bar"), we can't compute
     643              :          the offset to the following NUL if we don't know where to
     644              :          start searching for it.  */
     645         5050 :       unsigned len = string_length (ptr, eltsize, strelts);
     646              : 
     647              :       /* Return when an embedded null character is found or none at all.
     648              :          In the latter case, set the DECL/LEN field in the DATA structure
     649              :          so that callers may examine them.  */
     650         5050 :       if (len + 1 < strelts)
     651              :         return NULL_TREE;
     652         4480 :       else if (len >= maxelts)
     653              :         {
     654         1102 :           data->decl = decl;
     655         1102 :           data->off = byteoff;
     656         1102 :           data->minlen = ssize_int (len);
     657         1102 :           return NULL_TREE;
     658              :         }
     659              : 
     660              :       /* For empty strings the result should be zero.  */
     661         3378 :       if (len == 0)
     662           39 :         return ssize_int (0);
     663              : 
     664              :       /* We don't know the starting offset, but we do know that the string
     665              :          has no internal zero bytes.  If the offset falls within the bounds
     666              :          of the string subtract the offset from the length of the string,
     667              :          and return that.  Otherwise the length is zero.  Take care to
     668              :          use SAVE_EXPR in case the OFFSET has side-effects.  */
     669         3339 :       tree offsave = TREE_SIDE_EFFECTS (byteoff) ? save_expr (byteoff)
     670              :                                                  : byteoff;
     671         3339 :       offsave = fold_convert_loc (loc, sizetype, offsave);
     672         3339 :       tree condexp = fold_build2_loc (loc, LE_EXPR, boolean_type_node, offsave,
     673         3339 :                                       size_int (len));
     674         3339 :       tree lenexp = fold_build2_loc (loc, MINUS_EXPR, sizetype, size_int (len),
     675              :                                      offsave);
     676         3339 :       lenexp = fold_convert_loc (loc, ssizetype, lenexp);
     677         3339 :       return fold_build3_loc (loc, COND_EXPR, ssizetype, condexp, lenexp,
     678         3339 :                               build_zero_cst (ssizetype));
     679              :     }
     680              : 
     681              :   /* Offset from the beginning of the string in elements.  */
     682       762910 :   HOST_WIDE_INT eltoff;
     683              : 
     684              :   /* We have a known offset into the string.  Start searching there for
     685              :      a null character if we can represent it as a single HOST_WIDE_INT.  */
     686       762910 :   if (byteoff == 0)
     687              :     eltoff = 0;
     688       762910 :   else if (! tree_fits_uhwi_p (byteoff) || tree_to_uhwi (byteoff) % eltsize)
     689              :     eltoff = -1;
     690              :   else
     691       761722 :     eltoff = tree_to_uhwi (byteoff) / eltsize;
     692              : 
     693              :   /* If the offset is known to be out of bounds, warn, and call strlen at
     694              :      runtime.  */
     695       762910 :   if (eltoff < 0 || eltoff >= maxelts)
     696              :     {
     697              :       /* Suppress multiple warnings for propagated constant strings.  */
     698         3828 :       if (only_value != 2
     699         3828 :           && !warning_suppressed_p (arg, OPT_Warray_bounds_)
     700         7656 :           && warning_at (loc, OPT_Warray_bounds_,
     701              :                          "offset %qwi outside bounds of constant string",
     702              :                          eltoff))
     703              :         {
     704          679 :           if (decl)
     705          678 :             inform (DECL_SOURCE_LOCATION (decl), "%qE declared here", decl);
     706          679 :           suppress_warning (arg, OPT_Warray_bounds_);
     707              :         }
     708              :       return NULL_TREE;
     709              :     }
     710              : 
     711              :   /* If eltoff is larger than strelts but less than maxelts the
     712              :      string length is zero, since the excess memory will be zero.  */
     713       759082 :   if (eltoff > strelts)
     714            0 :     return ssize_int (0);
     715              : 
     716              :   /* Use strlen to search for the first zero byte.  Since any strings
     717              :      constructed with build_string will have nulls appended, we win even
     718              :      if we get handed something like (char[4])"abcd".
     719              : 
     720              :      Since ELTOFF is our starting index into the string, no further
     721              :      calculation is needed.  */
     722       759082 :   unsigned len = string_length (ptr + eltoff * eltsize, eltsize,
     723       759082 :                                 strelts - eltoff);
     724              : 
     725              :   /* Don't know what to return if there was no zero termination.
     726              :      Ideally this would turn into a gcc_checking_assert over time.
     727              :      Set DECL/LEN so callers can examine them.  */
     728       759082 :   if (len >= maxelts - eltoff)
     729              :     {
     730        85192 :       data->decl = decl;
     731        85192 :       data->off = byteoff;
     732        85192 :       data->minlen = ssize_int (len);
     733        85192 :       return NULL_TREE;
     734              :     }
     735              : 
     736       673890 :   return ssize_int (len);
     737              : }
     738              : 
     739              : /* Return a constant integer corresponding to target reading
     740              :    GET_MODE_BITSIZE (MODE) bits from string constant STR.  If
     741              :    NULL_TERMINATED_P, reading stops after '\0' character, all further ones
     742              :    are assumed to be zero, otherwise it reads as many characters
     743              :    as needed.  */
     744              : 
     745              : rtx
     746       288750 : c_readstr (const char *str, fixed_size_mode mode,
     747              :            bool null_terminated_p/*=true*/)
     748              : {
     749       288750 :   auto_vec<target_unit, MAX_BITSIZE_MODE_ANY_INT / BITS_PER_UNIT> bytes;
     750              : 
     751       577500 :   bytes.reserve (GET_MODE_SIZE (mode));
     752              : 
     753       288750 :   target_unit ch = 1;
     754      4823170 :   for (unsigned int i = 0; i < GET_MODE_SIZE (mode); ++i)
     755              :     {
     756      2122835 :       if (ch || !null_terminated_p)
     757      2031707 :         ch = (unsigned char) str[i];
     758      2122835 :       bytes.quick_push (ch);
     759              :     }
     760              : 
     761       288750 :   return native_decode_rtx (mode, bytes, 0);
     762       288750 : }
     763              : 
     764              : /* Cast a target constant CST to target CHAR and if that value fits into
     765              :    host char type, return zero and put that value into variable pointed to by
     766              :    P.  */
     767              : 
     768              : static int
     769        61181 : target_char_cast (tree cst, char *p)
     770              : {
     771        61181 :   unsigned HOST_WIDE_INT val, hostval;
     772              : 
     773        61181 :   if (TREE_CODE (cst) != INTEGER_CST
     774              :       || CHAR_TYPE_SIZE > HOST_BITS_PER_WIDE_INT)
     775              :     return 1;
     776              : 
     777              :   /* Do not care if it fits or not right here.  */
     778        61181 :   val = TREE_INT_CST_LOW (cst);
     779              : 
     780        61181 :   if (CHAR_TYPE_SIZE < HOST_BITS_PER_WIDE_INT)
     781        61181 :     val &= (HOST_WIDE_INT_1U << CHAR_TYPE_SIZE) - 1;
     782              : 
     783        61181 :   hostval = val;
     784        61181 :   if (HOST_BITS_PER_CHAR < HOST_BITS_PER_WIDE_INT)
     785        61181 :     hostval &= (HOST_WIDE_INT_1U << HOST_BITS_PER_CHAR) - 1;
     786              : 
     787        61181 :   if (val != hostval)
     788              :     return 1;
     789              : 
     790        61181 :   *p = hostval;
     791        61181 :   return 0;
     792              : }
     793              : 
     794              : /* Similar to save_expr, but assumes that arbitrary code is not executed
     795              :    in between the multiple evaluations.  In particular, we assume that a
     796              :    non-addressable local variable will not be modified.  */
     797              : 
     798              : static tree
     799      1133338 : builtin_save_expr (tree exp)
     800              : {
     801      1133338 :   if (TREE_CODE (exp) == SSA_NAME
     802       969048 :       || (TREE_ADDRESSABLE (exp) == 0
     803       968856 :           && (TREE_CODE (exp) == PARM_DECL
     804       434791 :               || (VAR_P (exp) && !TREE_STATIC (exp)))))
     805              :     return exp;
     806              : 
     807       432982 :   return save_expr (exp);
     808              : }
     809              : 
     810              : /* Given TEM, a pointer to a stack frame, follow the dynamic chain COUNT
     811              :    times to get the address of either a higher stack frame, or a return
     812              :    address located within it (depending on FNDECL_CODE).  */
     813              : 
     814              : static rtx
     815        15936 : expand_builtin_return_addr (enum built_in_function fndecl_code, int count)
     816              : {
     817        15936 :   int i;
     818        15936 :   rtx tem = INITIAL_FRAME_ADDRESS_RTX;
     819        15936 :   if (tem == NULL_RTX)
     820              :     {
     821              :       /* For a zero count with __builtin_return_address, we don't care what
     822              :          frame address we return, because target-specific definitions will
     823              :          override us.  Therefore frame pointer elimination is OK, and using
     824              :          the soft frame pointer is OK.
     825              : 
     826              :          For a nonzero count, or a zero count with __builtin_frame_address,
     827              :          we require a stable offset from the current frame pointer to the
     828              :          previous one, so we must use the hard frame pointer, and
     829              :          we must disable frame pointer elimination.  */
     830        15936 :       if (count == 0 && fndecl_code == BUILT_IN_RETURN_ADDRESS)
     831         4333 :         tem = frame_pointer_rtx;
     832              :       else
     833              :         {
     834        11603 :           tem = hard_frame_pointer_rtx;
     835              : 
     836              :           /* Tell reload not to eliminate the frame pointer.  */
     837        11603 :           crtl->accesses_prior_frames = 1;
     838              :         }
     839              :     }
     840              : 
     841        15936 :   if (count > 0)
     842          938 :     SETUP_FRAME_ADDRESSES ();
     843              : 
     844              :   /* On the SPARC, the return address is not in the frame, it is in a
     845              :      register.  There is no way to access it off of the current frame
     846              :      pointer, but it can be accessed off the previous frame pointer by
     847              :      reading the value from the register window save area.  */
     848        15936 :   if (RETURN_ADDR_IN_PREVIOUS_FRAME && fndecl_code == BUILT_IN_RETURN_ADDRESS)
     849              :     count--;
     850              : 
     851              :   /* Scan back COUNT frames to the specified frame.  */
     852        51818 :   for (i = 0; i < count; i++)
     853              :     {
     854              :       /* Assume the dynamic chain pointer is in the word that the
     855              :          frame address points to, unless otherwise specified.  */
     856        35882 :       tem = DYNAMIC_CHAIN_ADDRESS (tem);
     857        35882 :       tem = memory_address (Pmode, tem);
     858        35882 :       tem = gen_frame_mem (Pmode, tem);
     859        35882 :       tem = copy_to_reg (tem);
     860              :     }
     861              : 
     862              :   /* For __builtin_frame_address, return what we've got.  But, on
     863              :      the SPARC for example, we may have to add a bias.  */
     864        15936 :   if (fndecl_code == BUILT_IN_FRAME_ADDRESS)
     865              :     return FRAME_ADDR_RTX (count, tem);
     866              : 
     867              :   /* For __builtin_return_address, get the return address from that frame.  */
     868              : #ifdef RETURN_ADDR_RTX
     869         7402 :   tem = RETURN_ADDR_RTX (count, tem);
     870              : #else
     871              :   tem = memory_address (Pmode,
     872              :                         plus_constant (Pmode, tem, GET_MODE_SIZE (Pmode)));
     873              :   tem = gen_frame_mem (Pmode, tem);
     874              : #endif
     875         5173 :   return tem;
     876              : }
     877              : 
     878              : /* Alias set used for setjmp buffer.  */
     879              : static alias_set_type setjmp_alias_set = -1;
     880              : 
     881              : /* Construct the leading half of a __builtin_setjmp call.  Control will
     882              :    return to RECEIVER_LABEL.  This is also called directly by the SJLJ
     883              :    exception handling code.  */
     884              : 
     885              : void
     886          842 : expand_builtin_setjmp_setup (rtx buf_addr, rtx_insn *receiver_label)
     887              : {
     888          842 :   machine_mode sa_mode = STACK_SAVEAREA_MODE (SAVE_NONLOCAL);
     889          842 :   rtx stack_save;
     890          842 :   rtx mem;
     891              : 
     892          842 :   if (setjmp_alias_set == -1)
     893          241 :     setjmp_alias_set = new_alias_set ();
     894              : 
     895          843 :   buf_addr = convert_memory_address (Pmode, buf_addr);
     896              : 
     897          843 :   buf_addr = force_reg (Pmode, force_operand (buf_addr, NULL_RTX));
     898              : 
     899              :   /* We store the frame pointer and the address of receiver_label in
     900              :      the buffer and use the rest of it for the stack save area, which
     901              :      is machine-dependent.  */
     902              : 
     903          843 :   mem = gen_rtx_MEM (Pmode, buf_addr);
     904          842 :   set_mem_alias_set (mem, setjmp_alias_set);
     905          842 :   emit_move_insn (mem, hard_frame_pointer_rtx);
     906              : 
     907          843 :   mem = gen_rtx_MEM (Pmode, plus_constant (Pmode, buf_addr,
     908         1684 :                                            GET_MODE_SIZE (Pmode))),
     909          842 :   set_mem_alias_set (mem, setjmp_alias_set);
     910              : 
     911         1684 :   emit_move_insn (validize_mem (mem),
     912         1684 :                   force_reg (Pmode, gen_rtx_LABEL_REF (Pmode, receiver_label)));
     913              : 
     914          842 :   stack_save = gen_rtx_MEM (sa_mode,
     915              :                             plus_constant (Pmode, buf_addr,
     916         1684 :                                            2 * GET_MODE_SIZE (Pmode)));
     917          842 :   set_mem_alias_set (stack_save, setjmp_alias_set);
     918          842 :   emit_stack_save (SAVE_NONLOCAL, &stack_save);
     919              : 
     920              :   /* If there is further processing to do, do it.  */
     921          842 :   if (targetm.have_builtin_setjmp_setup ())
     922            0 :     emit_insn (targetm.gen_builtin_setjmp_setup (buf_addr));
     923              : 
     924              :   /* We have a nonlocal label.   */
     925          842 :   cfun->has_nonlocal_label = 1;
     926          842 : }
     927              : 
     928              : /* Construct the trailing part of a __builtin_setjmp call.  This is
     929              :    also called directly by the SJLJ exception handling code.
     930              :    If RECEIVER_LABEL is NULL, instead construct a nonlocal goto handler.  */
     931              : 
     932              : void
     933         1367 : expand_builtin_setjmp_receiver (rtx_insn *receiver_label)
     934              : {
     935         1367 :   rtx chain;
     936              : 
     937              :   /* Mark the FP as used when we get here, so we have to make sure it's
     938              :      marked as used by this function.  */
     939         1367 :   emit_use (hard_frame_pointer_rtx);
     940              : 
     941              :   /* Mark the static chain as clobbered here so life information
     942              :      doesn't get messed up for it.  */
     943         1367 :   chain = rtx_for_static_chain (current_function_decl, true);
     944         1367 :   if (chain && REG_P (chain))
     945            2 :     emit_clobber (chain);
     946              : 
     947         1367 :   if (!HARD_FRAME_POINTER_IS_ARG_POINTER && fixed_regs[ARG_POINTER_REGNUM])
     948              :     {
     949              :       /* If the argument pointer can be eliminated in favor of the
     950              :          frame pointer, we don't need to restore it.  We assume here
     951              :          that if such an elimination is present, it can always be used.
     952              :          This is the case on all known machines; if we don't make this
     953              :          assumption, we do unnecessary saving on many machines.  */
     954              :       size_t i;
     955              :       static const struct elims {const int from, to;} elim_regs[] = ELIMINABLE_REGS;
     956              : 
     957         2734 :       for (i = 0; i < ARRAY_SIZE (elim_regs); i++)
     958         2734 :         if (elim_regs[i].from == ARG_POINTER_REGNUM
     959         2734 :             && elim_regs[i].to == HARD_FRAME_POINTER_REGNUM)
     960              :           break;
     961              : 
     962         1367 :       if (i == ARRAY_SIZE (elim_regs))
     963              :         {
     964              :           /* Now restore our arg pointer from the address at which it
     965              :              was saved in our stack frame.  */
     966            0 :           emit_move_insn (crtl->args.internal_arg_pointer,
     967              :                           copy_to_reg (get_arg_pointer_save_area ()));
     968              :         }
     969              :     }
     970              : 
     971         1367 :   if (receiver_label != NULL && targetm.have_builtin_setjmp_receiver ())
     972            0 :     emit_insn (targetm.gen_builtin_setjmp_receiver (receiver_label));
     973         1367 :   else if (targetm.have_nonlocal_goto_receiver ())
     974            0 :     emit_insn (targetm.gen_nonlocal_goto_receiver ());
     975              :   else
     976              :     { /* Nothing */ }
     977              : 
     978              :   /* We must not allow the code we just generated to be reordered by
     979              :      scheduling.  Specifically, the update of the frame pointer must
     980              :      happen immediately, not later.  */
     981         1367 :   emit_insn (gen_blockage ());
     982         1367 : }
     983              : 
     984              : /* __builtin_longjmp is passed a pointer to an array of five words (not
     985              :    all will be used on all machines).  It operates similarly to the C
     986              :    library function of the same name, but is more efficient.  Much of
     987              :    the code below is copied from the handling of non-local gotos.  */
     988              : 
     989              : static void
     990          392 : expand_builtin_longjmp (rtx buf_addr, rtx value)
     991              : {
     992          392 :   rtx fp, lab, stack;
     993          392 :   rtx_insn *insn, *last;
     994          392 :   machine_mode sa_mode = STACK_SAVEAREA_MODE (SAVE_NONLOCAL);
     995              : 
     996              :   /* DRAP is needed for stack realign if longjmp is expanded to current
     997              :      function  */
     998          392 :   if (SUPPORTS_STACK_ALIGNMENT)
     999          392 :     crtl->need_drap = true;
    1000              : 
    1001          392 :   if (setjmp_alias_set == -1)
    1002          331 :     setjmp_alias_set = new_alias_set ();
    1003              : 
    1004          393 :   buf_addr = convert_memory_address (Pmode, buf_addr);
    1005              : 
    1006          393 :   buf_addr = force_reg (Pmode, buf_addr);
    1007              : 
    1008              :   /* We require that the user must pass a second argument of 1, because
    1009              :      that is what builtin_setjmp will return.  */
    1010          392 :   gcc_assert (value == const1_rtx);
    1011              : 
    1012          392 :   last = get_last_insn ();
    1013          392 :   if (targetm.have_builtin_longjmp ())
    1014            0 :     emit_insn (targetm.gen_builtin_longjmp (buf_addr));
    1015              :   else
    1016              :     {
    1017          393 :       fp = gen_rtx_MEM (Pmode, buf_addr);
    1018          393 :       lab = gen_rtx_MEM (Pmode, plus_constant (Pmode, buf_addr,
    1019          784 :                                                GET_MODE_SIZE (Pmode)));
    1020              : 
    1021          392 :       stack = gen_rtx_MEM (sa_mode, plus_constant (Pmode, buf_addr,
    1022          784 :                                                    2 * GET_MODE_SIZE (Pmode)));
    1023          392 :       set_mem_alias_set (fp, setjmp_alias_set);
    1024          392 :       set_mem_alias_set (lab, setjmp_alias_set);
    1025          392 :       set_mem_alias_set (stack, setjmp_alias_set);
    1026              : 
    1027              :       /* Pick up FP, label, and SP from the block and jump.  This code is
    1028              :          from expand_goto in stmt.cc; see there for detailed comments.  */
    1029          392 :       if (targetm.have_nonlocal_goto ())
    1030              :         /* We have to pass a value to the nonlocal_goto pattern that will
    1031              :            get copied into the static_chain pointer, but it does not matter
    1032              :            what that value is, because builtin_setjmp does not use it.  */
    1033            0 :         emit_insn (targetm.gen_nonlocal_goto (value, lab, stack, fp));
    1034              :       else
    1035              :         {
    1036          392 :           emit_clobber (gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode)));
    1037          392 :           emit_clobber (gen_rtx_MEM (BLKmode, hard_frame_pointer_rtx));
    1038              : 
    1039          392 :           lab = copy_to_reg (lab);
    1040              : 
    1041              :           /* Restore the frame pointer and stack pointer.  We must use a
    1042              :              temporary since the setjmp buffer may be a local.  */
    1043          392 :           fp = copy_to_reg (fp);
    1044          392 :           emit_stack_restore (SAVE_NONLOCAL, stack);
    1045              : 
    1046              :           /* Ensure the frame pointer move is not optimized.  */
    1047          392 :           emit_insn (gen_blockage ());
    1048          392 :           emit_clobber (hard_frame_pointer_rtx);
    1049          392 :           emit_clobber (frame_pointer_rtx);
    1050          392 :           emit_move_insn (hard_frame_pointer_rtx, fp);
    1051              : 
    1052          392 :           emit_use (hard_frame_pointer_rtx);
    1053          392 :           emit_use (stack_pointer_rtx);
    1054          392 :           emit_indirect_jump (lab);
    1055              :         }
    1056              :     }
    1057              : 
    1058              :   /* Search backwards and mark the jump insn as a non-local goto.
    1059              :      Note that this precludes the use of __builtin_longjmp to a
    1060              :      __builtin_setjmp target in the same function.  However, we've
    1061              :      already cautioned the user that these functions are for
    1062              :      internal exception handling use only.  */
    1063          784 :   for (insn = get_last_insn (); insn; insn = PREV_INSN (insn))
    1064              :     {
    1065          784 :       gcc_assert (insn != last);
    1066              : 
    1067          784 :       if (JUMP_P (insn))
    1068              :         {
    1069          392 :           add_reg_note (insn, REG_NON_LOCAL_GOTO, const0_rtx);
    1070          392 :           break;
    1071              :         }
    1072          392 :       else if (CALL_P (insn))
    1073              :         break;
    1074              :     }
    1075          392 : }
    1076              : 
    1077              : static inline bool
    1078      1950779 : more_const_call_expr_args_p (const const_call_expr_arg_iterator *iter)
    1079              : {
    1080      1950779 :   return (iter->i < iter->n);
    1081              : }
    1082              : 
    1083              : /* This function validates the types of a function call argument list
    1084              :    against a specified list of tree_codes.  If the last specifier is a 0,
    1085              :    that represents an ellipsis, otherwise the last specifier must be a
    1086              :    VOID_TYPE.  */
    1087              : 
    1088              : static bool
    1089       503550 : validate_arglist (const_tree callexpr, ...)
    1090              : {
    1091       503550 :   enum tree_code code;
    1092       503550 :   bool res = 0;
    1093       503550 :   va_list ap;
    1094       503550 :   const_call_expr_arg_iterator iter;
    1095       503550 :   const_tree arg;
    1096              : 
    1097       503550 :   va_start (ap, callexpr);
    1098       503550 :   init_const_call_expr_arg_iterator (callexpr, &iter);
    1099              : 
    1100              :   /* Get a bitmap of pointer argument numbers declared attribute nonnull.  */
    1101       503550 :   tree fn = CALL_EXPR_FN (callexpr);
    1102       503550 :   bitmap argmap = get_nonnull_args (TREE_TYPE (TREE_TYPE (fn)));
    1103              : 
    1104      1756053 :   for (unsigned argno = 1; ; ++argno)
    1105              :     {
    1106      1756053 :       code = (enum tree_code) va_arg (ap, int);
    1107              : 
    1108      1756053 :       switch (code)
    1109              :         {
    1110         2061 :         case 0:
    1111              :           /* This signifies an ellipses, any further arguments are all ok.  */
    1112         2061 :           res = true;
    1113         2061 :           goto end;
    1114       501257 :         case VOID_TYPE:
    1115              :           /* This signifies an endlink, if no arguments remain, return
    1116              :              true, otherwise return false.  */
    1117       501257 :           res = !more_const_call_expr_args_p (&iter);
    1118       501257 :           goto end;
    1119       823970 :         case POINTER_TYPE:
    1120              :           /* The actual argument must be nonnull when either the whole
    1121              :              called function has been declared nonnull, or when the formal
    1122              :              argument corresponding to the actual argument has been.  */
    1123       823970 :           if (argmap
    1124       823970 :               && (bitmap_empty_p (argmap) || bitmap_bit_p (argmap, argno)))
    1125              :             {
    1126       280968 :               arg = next_const_call_expr_arg (&iter);
    1127       280968 :               if (!validate_arg (arg, code) || integer_zerop (arg))
    1128           58 :                 goto end;
    1129              :               break;
    1130              :             }
    1131              :           /* FALLTHRU */
    1132       971767 :         default:
    1133              :           /* If no parameters remain or the parameter's code does not
    1134              :              match the specified code, return false.  Otherwise continue
    1135              :              checking any remaining arguments.  */
    1136       971767 :           arg = next_const_call_expr_arg (&iter);
    1137       971767 :           if (!validate_arg (arg, code))
    1138          174 :             goto end;
    1139              :           break;
    1140              :         }
    1141              :     }
    1142              : 
    1143              :   /* We need gotos here since we can only have one VA_CLOSE in a
    1144              :      function.  */
    1145       503550 :  end: ;
    1146       503550 :   va_end (ap);
    1147              : 
    1148       503550 :   BITMAP_FREE (argmap);
    1149              : 
    1150       503550 :   if (res)
    1151       503318 :     for (tree attrs = TYPE_ATTRIBUTES (TREE_TYPE (TREE_TYPE (fn)));
    1152       727474 :          (attrs = lookup_attribute ("nonnull_if_nonzero", attrs));
    1153       224156 :          attrs = TREE_CHAIN (attrs))
    1154              :       {
    1155       224178 :         tree args = TREE_VALUE (attrs);
    1156       224178 :         unsigned int idx = TREE_INT_CST_LOW (TREE_VALUE (args)) - 1;
    1157       224178 :         unsigned int idx2
    1158       224178 :           = TREE_INT_CST_LOW (TREE_VALUE (TREE_CHAIN (args))) - 1;
    1159       224178 :         unsigned int idx3 = idx2;
    1160       224178 :         if (tree chain2 = TREE_CHAIN (TREE_CHAIN (args)))
    1161            0 :           idx3 = TREE_INT_CST_LOW (TREE_VALUE (chain2)) - 1;
    1162       224178 :         if (idx < (unsigned) call_expr_nargs (callexpr)
    1163       224178 :             && idx2 < (unsigned) call_expr_nargs (callexpr)
    1164       224178 :             && idx3 < (unsigned) call_expr_nargs (callexpr)
    1165       224178 :             && POINTER_TYPE_P (TREE_TYPE (CALL_EXPR_ARG (callexpr, idx)))
    1166       224178 :             && integer_zerop (CALL_EXPR_ARG (callexpr, idx))
    1167          101 :             && INTEGRAL_TYPE_P (TREE_TYPE (CALL_EXPR_ARG (callexpr, idx2)))
    1168          101 :             && integer_nonzerop (CALL_EXPR_ARG (callexpr, idx2))
    1169           22 :             && INTEGRAL_TYPE_P (TREE_TYPE (CALL_EXPR_ARG (callexpr, idx3)))
    1170       224200 :             && integer_nonzerop (CALL_EXPR_ARG (callexpr, idx3)))
    1171              :           return false;
    1172              :       }
    1173              : 
    1174              :   return res;
    1175              : }
    1176              : 
    1177              : /* Expand a call to __builtin_nonlocal_goto.  We're passed the target label
    1178              :    and the address of the save area.  */
    1179              : 
    1180              : static rtx
    1181          524 : expand_builtin_nonlocal_goto (tree exp)
    1182              : {
    1183          524 :   tree t_label, t_save_area;
    1184          524 :   rtx r_label, r_save_area, r_fp, r_sp;
    1185          524 :   rtx_insn *insn;
    1186              : 
    1187          524 :   if (!validate_arglist (exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
    1188              :     return NULL_RTX;
    1189              : 
    1190          524 :   t_label = CALL_EXPR_ARG (exp, 0);
    1191          524 :   t_save_area = CALL_EXPR_ARG (exp, 1);
    1192              : 
    1193          524 :   r_label = expand_normal (t_label);
    1194          524 :   r_label = convert_memory_address (Pmode, r_label);
    1195          524 :   r_save_area = expand_normal (t_save_area);
    1196          524 :   r_save_area = convert_memory_address (Pmode, r_save_area);
    1197              :   /* Copy the address of the save location to a register just in case it was
    1198              :      based on the frame pointer.   */
    1199          524 :   r_save_area = copy_to_reg (r_save_area);
    1200          524 :   r_fp = gen_rtx_MEM (Pmode, r_save_area);
    1201          524 :   r_sp = gen_rtx_MEM (STACK_SAVEAREA_MODE (SAVE_NONLOCAL),
    1202              :                       plus_constant (Pmode, r_save_area,
    1203         1048 :                                      GET_MODE_SIZE (Pmode)));
    1204              : 
    1205          524 :   crtl->has_nonlocal_goto = 1;
    1206              : 
    1207              :   /* ??? We no longer need to pass the static chain value, afaik.  */
    1208          524 :   if (targetm.have_nonlocal_goto ())
    1209            0 :     emit_insn (targetm.gen_nonlocal_goto (const0_rtx, r_label, r_sp, r_fp));
    1210              :   else
    1211              :     {
    1212          524 :       emit_clobber (gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode)));
    1213          524 :       emit_clobber (gen_rtx_MEM (BLKmode, hard_frame_pointer_rtx));
    1214              : 
    1215          524 :       r_label = copy_to_reg (r_label);
    1216              : 
    1217              :       /* Restore the frame pointer and stack pointer.  We must use a
    1218              :          temporary since the setjmp buffer may be a local.  */
    1219          524 :       r_fp = copy_to_reg (r_fp);
    1220          524 :       emit_stack_restore (SAVE_NONLOCAL, r_sp);
    1221              : 
    1222              :       /* Ensure the frame pointer move is not optimized.  */
    1223          524 :       emit_insn (gen_blockage ());
    1224          524 :       emit_clobber (hard_frame_pointer_rtx);
    1225          524 :       emit_clobber (frame_pointer_rtx);
    1226          524 :       emit_move_insn (hard_frame_pointer_rtx, r_fp);
    1227              : 
    1228              :       /* USE of hard_frame_pointer_rtx added for consistency;
    1229              :          not clear if really needed.  */
    1230          524 :       emit_use (hard_frame_pointer_rtx);
    1231          524 :       emit_use (stack_pointer_rtx);
    1232              : 
    1233              :       /* If the architecture is using a GP register, we must
    1234              :          conservatively assume that the target function makes use of it.
    1235              :          The prologue of functions with nonlocal gotos must therefore
    1236              :          initialize the GP register to the appropriate value, and we
    1237              :          must then make sure that this value is live at the point
    1238              :          of the jump.  (Note that this doesn't necessarily apply
    1239              :          to targets with a nonlocal_goto pattern; they are free
    1240              :          to implement it in their own way.  Note also that this is
    1241              :          a no-op if the GP register is a global invariant.)  */
    1242          524 :       unsigned regnum = PIC_OFFSET_TABLE_REGNUM;
    1243            0 :       if (regnum != INVALID_REGNUM && fixed_regs[regnum])
    1244            0 :         emit_use (pic_offset_table_rtx);
    1245              : 
    1246          524 :       emit_indirect_jump (r_label);
    1247              :     }
    1248              : 
    1249              :   /* Search backwards to the jump insn and mark it as a
    1250              :      non-local goto.  */
    1251         1048 :   for (insn = get_last_insn (); insn; insn = PREV_INSN (insn))
    1252              :     {
    1253         1048 :       if (JUMP_P (insn))
    1254              :         {
    1255          524 :           add_reg_note (insn, REG_NON_LOCAL_GOTO, const0_rtx);
    1256          524 :           break;
    1257              :         }
    1258          524 :       else if (CALL_P (insn))
    1259              :         break;
    1260              :     }
    1261              : 
    1262          524 :   return const0_rtx;
    1263              : }
    1264              : 
    1265              : /* __builtin_update_setjmp_buf is passed a pointer to an array of five words
    1266              :    (not all will be used on all machines) that was passed to __builtin_setjmp.
    1267              :    It updates the stack pointer in that block to the current value.  This is
    1268              :    also called directly by the SJLJ exception handling code.  */
    1269              : 
    1270              : void
    1271            0 : expand_builtin_update_setjmp_buf (rtx buf_addr)
    1272              : {
    1273            0 :   machine_mode sa_mode = STACK_SAVEAREA_MODE (SAVE_NONLOCAL);
    1274            0 :   buf_addr = convert_memory_address (Pmode, buf_addr);
    1275            0 :   rtx stack_save
    1276            0 :     = gen_rtx_MEM (sa_mode,
    1277            0 :                    memory_address
    1278              :                    (sa_mode,
    1279              :                     plus_constant (Pmode, buf_addr,
    1280            0 :                                    2 * GET_MODE_SIZE (Pmode))));
    1281              : 
    1282            0 :   emit_stack_save (SAVE_NONLOCAL, &stack_save);
    1283            0 : }
    1284              : 
    1285              : /* Expand a call to __builtin_prefetch.  For a target that does not support
    1286              :    data prefetch, evaluate the memory address argument in case it has side
    1287              :    effects.  */
    1288              : 
    1289              : static void
    1290         2061 : expand_builtin_prefetch (tree exp)
    1291              : {
    1292         2061 :   tree arg0, arg1, arg2;
    1293         2061 :   int nargs;
    1294         2061 :   rtx op0, op1, op2;
    1295              : 
    1296         2061 :   if (!validate_arglist (exp, POINTER_TYPE, 0))
    1297              :     return;
    1298              : 
    1299         2061 :   arg0 = CALL_EXPR_ARG (exp, 0);
    1300              : 
    1301              :   /* Arguments 1 and 2 are optional; argument 1 (read/write) defaults to
    1302              :      zero (read) and argument 2 (locality) defaults to 3 (high degree of
    1303              :      locality).  */
    1304         2061 :   nargs = call_expr_nargs (exp);
    1305         2061 :   arg1 = nargs > 1 ? CALL_EXPR_ARG (exp, 1) : NULL_TREE;
    1306         2021 :   arg2 = nargs > 2 ? CALL_EXPR_ARG (exp, 2) : NULL_TREE;
    1307              : 
    1308              :   /* Argument 0 is an address.  */
    1309         2157 :   op0 = expand_expr (arg0, NULL_RTX, Pmode, EXPAND_NORMAL);
    1310              : 
    1311              :   /* Argument 1 (read/write flag) must be a compile-time constant int.  */
    1312         2061 :   if (arg1 == NULL_TREE)
    1313           40 :     op1 = const0_rtx;
    1314         2021 :   else if (TREE_CODE (arg1) != INTEGER_CST)
    1315              :     {
    1316            0 :       error ("second argument to %<__builtin_prefetch%> must be a constant");
    1317            0 :       op1 = const0_rtx;
    1318              :     }
    1319              :   else
    1320         2021 :     op1 = expand_normal (arg1);
    1321              :   /* Argument 1 must be 0, 1 or 2.  */
    1322         2061 :   if (!IN_RANGE (INTVAL (op1), 0, 2))
    1323              :     {
    1324            3 :       warning (0, "invalid second argument to %<__builtin_prefetch%>;"
    1325              :                " using zero");
    1326            3 :       op1 = const0_rtx;
    1327              :     }
    1328              : 
    1329              :   /* Argument 2 (locality) must be a compile-time constant int.  */
    1330         2061 :   if (arg2 == NULL_TREE)
    1331           77 :     op2 = GEN_INT (3);
    1332         1984 :   else if (TREE_CODE (arg2) != INTEGER_CST)
    1333              :     {
    1334            0 :       error ("third argument to %<__builtin_prefetch%> must be a constant");
    1335            0 :       op2 = const0_rtx;
    1336              :     }
    1337              :   else
    1338         1984 :     op2 = expand_normal (arg2);
    1339              :   /* Argument 2 must be 0, 1, 2, or 3.  */
    1340         2061 :   if (!IN_RANGE (INTVAL (op2), 0, 3))
    1341              :     {
    1342            3 :       warning (0, "invalid third argument to %<__builtin_prefetch%>; using zero");
    1343            3 :       op2 = const0_rtx;
    1344              :     }
    1345              : 
    1346         2061 :   if (targetm.have_prefetch ())
    1347              :     {
    1348         2061 :       class expand_operand ops[3];
    1349              : 
    1350         2061 :       create_address_operand (&ops[0], op0);
    1351         2061 :       create_integer_operand (&ops[1], INTVAL (op1));
    1352         2061 :       create_integer_operand (&ops[2], INTVAL (op2));
    1353         2061 :       if (maybe_expand_insn (targetm.code_for_prefetch, 3, ops))
    1354         2061 :         return;
    1355              :     }
    1356              : 
    1357              :   /* Don't do anything with direct references to volatile memory, but
    1358              :      generate code to handle other side effects.  */
    1359            0 :   if (!MEM_P (op0) && side_effects_p (op0))
    1360            0 :     emit_insn (op0);
    1361              : }
    1362              : 
    1363              : /* Get a MEM rtx for expression EXP which is the address of an operand
    1364              :    to be used in a string instruction (cmpstrsi, cpymemsi, ..).  LEN is
    1365              :    the maximum length of the block of memory that might be accessed or
    1366              :    NULL if unknown.  STORE_P is true when EXP is the destination.  */
    1367              : 
    1368              : rtx
    1369       756257 : get_memory_rtx (tree exp, tree len, bool store_p)
    1370              : {
    1371       756257 :   tree orig_exp = exp, base;
    1372       756257 :   rtx addr, mem;
    1373              : 
    1374       756257 :   gcc_checking_assert
    1375              :     (ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)))));
    1376              : 
    1377              :   /* When EXP is not resolved SAVE_EXPR, MEM_ATTRS can be still derived
    1378              :      from its expression, for expr->a.b only <variable>.a.b is recorded.  */
    1379       756257 :   if (TREE_CODE (exp) == SAVE_EXPR && !SAVE_EXPR_RESOLVED_P (exp))
    1380         1640 :     exp = TREE_OPERAND (exp, 0);
    1381              : 
    1382       756257 :   addr = expand_expr (orig_exp, NULL_RTX, ptr_mode, EXPAND_NORMAL);
    1383       756257 :   mem = gen_rtx_MEM (BLKmode, memory_address (BLKmode, addr));
    1384              : 
    1385              :   /* Get an expression we can use to find the attributes to assign to MEM.
    1386              :      First remove any nops.  */
    1387      1512514 :   while (CONVERT_EXPR_P (exp)
    1388       756260 :          && POINTER_TYPE_P (TREE_TYPE (TREE_OPERAND (exp, 0))))
    1389            3 :     exp = TREE_OPERAND (exp, 0);
    1390              : 
    1391              :   /* Build a MEM_REF representing the whole accessed area as a byte blob,
    1392              :      (as builtin stringops may alias with anything).  */
    1393       756257 :   exp = fold_build2 (MEM_REF,
    1394              :                      build_array_type (char_type_node,
    1395              :                                        build_range_type (sizetype,
    1396              :                                                          size_one_node, len)),
    1397              :                      exp, build_int_cst (ptr_type_node, 0));
    1398              : 
    1399              :   /* If the MEM_REF has no acceptable address, try to get the base object
    1400              :      from the original address we got, and build an all-aliasing
    1401              :      unknown-sized access to that one.  */
    1402       756257 :   if (is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
    1403       756245 :     set_mem_attributes (mem, exp, 0, store_p);
    1404           12 :   else if (TREE_CODE (TREE_OPERAND (exp, 0)) == ADDR_EXPR
    1405           12 :            && (base = get_base_address (TREE_OPERAND (TREE_OPERAND (exp, 0),
    1406              :                                                       0))))
    1407              :     {
    1408           12 :       unsigned int align = get_pointer_alignment (TREE_OPERAND (exp, 0));
    1409           12 :       exp = build_fold_addr_expr (base);
    1410           12 :       exp = fold_build2 (MEM_REF,
    1411              :                          build_array_type (char_type_node,
    1412              :                                            build_range_type (sizetype,
    1413              :                                                              size_zero_node,
    1414              :                                                              NULL)),
    1415              :                          exp, build_int_cst (ptr_type_node, 0));
    1416           12 :       set_mem_attributes (mem, exp, 0, store_p);
    1417              :       /* Since we stripped parts make sure the offset is unknown and the
    1418              :          alignment is computed from the original address.  */
    1419           12 :       clear_mem_offset (mem);
    1420           12 :       set_mem_align (mem, align);
    1421              :     }
    1422       756257 :   set_mem_alias_set (mem, 0);
    1423       756257 :   return mem;
    1424              : }
    1425              : 
    1426              : /* Built-in functions to perform an untyped call and return.  */
    1427              : 
    1428              : /* Wrapper that implicitly applies a delta when getting or setting the
    1429              :    enclosed value.  */
    1430              : template <typename T>
    1431              : class delta_type
    1432              : {
    1433              :   T &value; T const delta;
    1434              : public:
    1435         2524 :   delta_type (T &val, T dlt) : value (val), delta (dlt) {}
    1436         1809 :   operator T () const { return value + delta; }
    1437          715 :   T operator = (T val) const { value = val - delta; return val; }
    1438              : };
    1439              : 
    1440              : #define saved_apply_args_size \
    1441              :   (delta_type<int> (this_target_builtins->x_apply_args_size_plus_one, -1))
    1442              : #define apply_args_mode \
    1443              :   (this_target_builtins->x_apply_args_mode)
    1444              : #define saved_apply_result_size \
    1445              :   (delta_type<int> (this_target_builtins->x_apply_result_size_plus_one, -1))
    1446              : #define apply_result_mode \
    1447              :   (this_target_builtins->x_apply_result_mode)
    1448              : 
    1449              : /* Return the size required for the block returned by __builtin_apply_args,
    1450              :    and initialize apply_args_mode.  */
    1451              : 
    1452              : static int
    1453          947 : apply_args_size (void)
    1454              : {
    1455          947 :   int size = saved_apply_args_size;
    1456          947 :   int align;
    1457          947 :   unsigned int regno;
    1458              : 
    1459              :   /* The values computed by this function never change.  */
    1460          947 :   if (size < 0)
    1461              :     {
    1462              :       /* The first value is the incoming arg-pointer.  */
    1463          360 :       size = GET_MODE_SIZE (Pmode);
    1464              : 
    1465              :       /* The second value is the structure value address unless this is
    1466              :          passed as an "invisible" first argument.  */
    1467          360 :       if (targetm.calls.struct_value_rtx (cfun ? TREE_TYPE (cfun->decl) : 0, 0))
    1468            0 :         size += GET_MODE_SIZE (Pmode);
    1469              : 
    1470        34200 :       for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    1471        33840 :         if (FUNCTION_ARG_REGNO_P (regno))
    1472              :           {
    1473         5400 :             fixed_size_mode mode = targetm.calls.get_raw_arg_mode (regno);
    1474              : 
    1475         5400 :             if (mode != VOIDmode)
    1476              :               {
    1477         5400 :                 align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
    1478         5400 :                 if (size % align != 0)
    1479            1 :                   size = CEIL (size, align) * align;
    1480         5400 :                 size += GET_MODE_SIZE (mode);
    1481         5400 :                 apply_args_mode[regno] = mode;
    1482              :               }
    1483              :             else
    1484            0 :               apply_args_mode[regno] = as_a <fixed_size_mode> (VOIDmode);
    1485              :           }
    1486              :         else
    1487        28440 :           apply_args_mode[regno] = as_a <fixed_size_mode> (VOIDmode);
    1488              : 
    1489          360 :       saved_apply_args_size = size;
    1490              :     }
    1491          947 :   return size;
    1492              : }
    1493              : 
    1494              : /* Return the size required for the block returned by __builtin_apply,
    1495              :    and initialize apply_result_mode.  */
    1496              : 
    1497              : static int
    1498          862 : apply_result_size (void)
    1499              : {
    1500          862 :   int size = saved_apply_result_size;
    1501          862 :   int align, regno;
    1502              : 
    1503              :   /* The values computed by this function never change.  */
    1504          862 :   if (size < 0)
    1505              :     {
    1506              :       size = 0;
    1507              : 
    1508        33725 :       for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    1509        33370 :         if (targetm.calls.function_value_regno_p (regno))
    1510              :           {
    1511         2130 :             fixed_size_mode mode = targetm.calls.get_raw_result_mode (regno);
    1512              : 
    1513         2130 :             if (mode != VOIDmode)
    1514              :               {
    1515         2130 :                 align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
    1516         2130 :                 if (size % align != 0)
    1517              :                   size = CEIL (size, align) * align;
    1518         2130 :                 size += GET_MODE_SIZE (mode);
    1519         2130 :                 apply_result_mode[regno] = mode;
    1520              :               }
    1521              :             else
    1522            0 :               apply_result_mode[regno] = as_a <fixed_size_mode> (VOIDmode);
    1523              :           }
    1524              :         else
    1525        31240 :           apply_result_mode[regno] = as_a <fixed_size_mode> (VOIDmode);
    1526              : 
    1527              :       /* Allow targets that use untyped_call and untyped_return to override
    1528              :          the size so that machine-specific information can be stored here.  */
    1529              : #ifdef APPLY_RESULT_SIZE
    1530          355 :       size = APPLY_RESULT_SIZE;
    1531              : #endif
    1532              : 
    1533          355 :       saved_apply_result_size = size;
    1534              :     }
    1535          862 :   return size;
    1536              : }
    1537              : 
    1538              : /* Create a vector describing the result block RESULT.  If SAVEP is true,
    1539              :    the result block is used to save the values; otherwise it is used to
    1540              :    restore the values.  */
    1541              : 
    1542              : static rtx
    1543          483 : result_vector (int savep, rtx result)
    1544              : {
    1545          483 :   int regno, size, align, nelts;
    1546          483 :   fixed_size_mode mode;
    1547          483 :   rtx reg, mem;
    1548          483 :   rtx *savevec = XALLOCAVEC (rtx, FIRST_PSEUDO_REGISTER);
    1549              : 
    1550          483 :   size = nelts = 0;
    1551        45885 :   for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    1552        45402 :     if ((mode = apply_result_mode[regno]) != VOIDmode)
    1553              :       {
    1554         2898 :         align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
    1555         2898 :         if (size % align != 0)
    1556            0 :           size = CEIL (size, align) * align;
    1557         2898 :         reg = gen_rtx_REG (mode, savep ? regno : INCOMING_REGNO (regno));
    1558         2898 :         mem = adjust_address (result, mode, size);
    1559         5796 :         savevec[nelts++] = (savep
    1560         5796 :                             ? gen_rtx_SET (mem, reg)
    1561            0 :                             : gen_rtx_SET (reg, mem));
    1562         5796 :         size += GET_MODE_SIZE (mode);
    1563              :       }
    1564          483 :   return gen_rtx_PARALLEL (VOIDmode, gen_rtvec_v (nelts, savevec));
    1565              : }
    1566              : 
    1567              : /* Save the state required to perform an untyped call with the same
    1568              :    arguments as were passed to the current function.  */
    1569              : 
    1570              : static rtx
    1571          464 : expand_builtin_apply_args_1 (void)
    1572              : {
    1573          464 :   rtx registers, tem;
    1574          464 :   int size, align, regno;
    1575          464 :   fixed_size_mode mode;
    1576          464 :   rtx struct_incoming_value = targetm.calls.struct_value_rtx (cfun ? TREE_TYPE (cfun->decl) : 0, 1);
    1577              : 
    1578              :   /* Create a block where the arg-pointer, structure value address,
    1579              :      and argument registers can be saved.  */
    1580          464 :   registers = assign_stack_local (BLKmode, apply_args_size (), -1);
    1581              : 
    1582              :   /* Walk past the arg-pointer and structure value address.  */
    1583          464 :   size = GET_MODE_SIZE (Pmode);
    1584          464 :   if (targetm.calls.struct_value_rtx (cfun ? TREE_TYPE (cfun->decl) : 0, 0))
    1585            0 :     size += GET_MODE_SIZE (Pmode);
    1586              : 
    1587              :   /* Save each register used in calling a function to the block.  */
    1588        44080 :   for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    1589        43616 :     if ((mode = apply_args_mode[regno]) != VOIDmode)
    1590              :       {
    1591         6960 :         align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
    1592         6960 :         if (size % align != 0)
    1593            1 :           size = CEIL (size, align) * align;
    1594              : 
    1595         6960 :         tem = gen_rtx_REG (mode, INCOMING_REGNO (regno));
    1596              : 
    1597         6960 :         emit_move_insn (adjust_address (registers, mode, size), tem);
    1598        13920 :         size += GET_MODE_SIZE (mode);
    1599              :       }
    1600              : 
    1601              :   /* Save the arg pointer to the block.  */
    1602          464 :   tem = copy_to_reg (crtl->args.internal_arg_pointer);
    1603              :   /* We need the pointer as the caller actually passed them to us, not
    1604              :      as we might have pretended they were passed.  Make sure it's a valid
    1605              :      operand, as emit_move_insn isn't expected to handle a PLUS.  */
    1606          464 :   if (STACK_GROWS_DOWNWARD)
    1607          464 :     tem
    1608          464 :       = force_operand (plus_constant (Pmode, tem,
    1609          464 :                                       crtl->args.pretend_args_size),
    1610              :                        NULL_RTX);
    1611          464 :   emit_move_insn (adjust_address (registers, Pmode, 0), tem);
    1612              : 
    1613          464 :   size = GET_MODE_SIZE (Pmode);
    1614              : 
    1615              :   /* Save the structure value address unless this is passed as an
    1616              :      "invisible" first argument.  */
    1617          464 :   if (struct_incoming_value)
    1618            0 :     emit_move_insn (adjust_address (registers, Pmode, size),
    1619              :                     copy_to_reg (struct_incoming_value));
    1620              : 
    1621              :   /* Return the address of the block.  */
    1622          464 :   return copy_addr_to_reg (XEXP (registers, 0));
    1623              : }
    1624              : 
    1625              : /* __builtin_apply_args returns block of memory allocated on
    1626              :    the stack into which is stored the arg pointer, structure
    1627              :    value address, static chain, and all the registers that might
    1628              :    possibly be used in performing a function call.  The code is
    1629              :    moved to the start of the function so the incoming values are
    1630              :    saved.  */
    1631              : 
    1632              : static rtx
    1633          464 : expand_builtin_apply_args (void)
    1634              : {
    1635              :   /* Don't do __builtin_apply_args more than once in a function.
    1636              :      Save the result of the first call and reuse it.  */
    1637          464 :   if (apply_args_value != 0)
    1638              :     return apply_args_value;
    1639          464 :   {
    1640              :     /* When this function is called, it means that registers must be
    1641              :        saved on entry to this function.  So we migrate the
    1642              :        call to the first insn of this function.  */
    1643          464 :     rtx temp;
    1644              : 
    1645          464 :     start_sequence ();
    1646          464 :     temp = expand_builtin_apply_args_1 ();
    1647          464 :     rtx_insn *seq = end_sequence ();
    1648              : 
    1649          464 :     apply_args_value = temp;
    1650              : 
    1651              :     /* Put the insns after the NOTE that starts the function.
    1652              :        If this is inside a start_sequence, make the outer-level insn
    1653              :        chain current, so the code is placed at the start of the
    1654              :        function.  If internal_arg_pointer is a non-virtual pseudo,
    1655              :        it needs to be placed after the function that initializes
    1656              :        that pseudo.  */
    1657          464 :     push_topmost_sequence ();
    1658          464 :     if (REG_P (crtl->args.internal_arg_pointer)
    1659          464 :         && REGNO (crtl->args.internal_arg_pointer) > LAST_VIRTUAL_REGISTER)
    1660            0 :       emit_insn_before (seq, parm_birth_insn);
    1661              :     else
    1662          464 :       emit_insn_before (seq, NEXT_INSN (entry_of_function ()));
    1663          464 :     pop_topmost_sequence ();
    1664          464 :     return temp;
    1665              :   }
    1666              : }
    1667              : 
    1668              : /* Perform an untyped call and save the state required to perform an
    1669              :    untyped return of whatever value was returned by the given function.  */
    1670              : 
    1671              : static rtx
    1672          483 : expand_builtin_apply (rtx function, rtx arguments, rtx argsize)
    1673              : {
    1674          483 :   int size, align, regno;
    1675          483 :   fixed_size_mode mode;
    1676          483 :   rtx incoming_args, result, reg, dest, src;
    1677          483 :   rtx_call_insn *call_insn;
    1678          483 :   rtx old_stack_level = 0;
    1679          483 :   rtx call_fusage = 0;
    1680          483 :   rtx struct_value = targetm.calls.struct_value_rtx (cfun ? TREE_TYPE (cfun->decl) : 0, 0);
    1681              : 
    1682          483 :   arguments = convert_memory_address (Pmode, arguments);
    1683              : 
    1684              :   /* Create a block where the return registers can be saved.  */
    1685          483 :   result = assign_stack_local (BLKmode, apply_result_size (), -1);
    1686              : 
    1687              :   /* Fetch the arg pointer from the ARGUMENTS block.  */
    1688          483 :   incoming_args = gen_reg_rtx (Pmode);
    1689          483 :   emit_move_insn (incoming_args, gen_rtx_MEM (Pmode, arguments));
    1690          483 :   if (!STACK_GROWS_DOWNWARD)
    1691              :     incoming_args = expand_simple_binop (Pmode, MINUS, incoming_args, argsize,
    1692              :                                          incoming_args, 0, OPTAB_LIB_WIDEN);
    1693              : 
    1694              :   /* Push a new argument block and copy the arguments.  Do not allow
    1695              :      the (potential) memcpy call below to interfere with our stack
    1696              :      manipulations.  */
    1697          483 :   do_pending_stack_adjust ();
    1698          483 :   NO_DEFER_POP;
    1699              : 
    1700              :   /* Save the stack with nonlocal if available.  */
    1701          483 :   if (targetm.have_save_stack_nonlocal ())
    1702          483 :     emit_stack_save (SAVE_NONLOCAL, &old_stack_level);
    1703              :   else
    1704            0 :     emit_stack_save (SAVE_BLOCK, &old_stack_level);
    1705              : 
    1706              :   /* Allocate a block of memory onto the stack and copy the memory
    1707              :      arguments to the outgoing arguments address.  We can pass TRUE
    1708              :      as the 4th argument because we just saved the stack pointer
    1709              :      and will restore it right after the call.  */
    1710          966 :   allocate_dynamic_stack_space (argsize, 0, BIGGEST_ALIGNMENT, -1, true);
    1711              : 
    1712              :   /* Set DRAP flag to true, even though allocate_dynamic_stack_space
    1713              :      may have already set current_function_calls_alloca to true.
    1714              :      current_function_calls_alloca won't be set if argsize is zero,
    1715              :      so we have to guarantee need_drap is true here.  */
    1716          483 :   if (SUPPORTS_STACK_ALIGNMENT)
    1717          483 :     crtl->need_drap = true;
    1718              : 
    1719          483 :   dest = virtual_outgoing_args_rtx;
    1720          483 :   if (!STACK_GROWS_DOWNWARD)
    1721              :     {
    1722              :       if (CONST_INT_P (argsize))
    1723              :         dest = plus_constant (Pmode, dest, -INTVAL (argsize));
    1724              :       else
    1725              :         dest = gen_rtx_PLUS (Pmode, dest, negate_rtx (Pmode, argsize));
    1726              :     }
    1727          483 :   dest = gen_rtx_MEM (BLKmode, dest);
    1728          483 :   set_mem_align (dest, PARM_BOUNDARY);
    1729          483 :   src = gen_rtx_MEM (BLKmode, incoming_args);
    1730          483 :   set_mem_align (src, PARM_BOUNDARY);
    1731          483 :   emit_block_move (dest, src, argsize, BLOCK_OP_NORMAL);
    1732              : 
    1733              :   /* Refer to the argument block.  */
    1734          483 :   apply_args_size ();
    1735          483 :   arguments = gen_rtx_MEM (BLKmode, arguments);
    1736          483 :   set_mem_align (arguments, PARM_BOUNDARY);
    1737              : 
    1738              :   /* Walk past the arg-pointer and structure value address.  */
    1739          483 :   size = GET_MODE_SIZE (Pmode);
    1740          483 :   if (struct_value)
    1741            0 :     size += GET_MODE_SIZE (Pmode);
    1742              : 
    1743              :   /* Restore each of the registers previously saved.  Make USE insns
    1744              :      for each of these registers for use in making the call.  */
    1745        45885 :   for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    1746        45402 :     if ((mode = apply_args_mode[regno]) != VOIDmode)
    1747              :       {
    1748         7245 :         align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
    1749         7245 :         if (size % align != 0)
    1750            0 :           size = CEIL (size, align) * align;
    1751         7245 :         reg = gen_rtx_REG (mode, regno);
    1752         7245 :         emit_move_insn (reg, adjust_address (arguments, mode, size));
    1753         7245 :         use_reg (&call_fusage, reg);
    1754        14490 :         size += GET_MODE_SIZE (mode);
    1755              :       }
    1756              : 
    1757              :   /* Restore the structure value address unless this is passed as an
    1758              :      "invisible" first argument.  */
    1759          483 :   size = GET_MODE_SIZE (Pmode);
    1760          483 :   if (struct_value)
    1761              :     {
    1762            0 :       rtx value = gen_reg_rtx (Pmode);
    1763            0 :       emit_move_insn (value, adjust_address (arguments, Pmode, size));
    1764            0 :       emit_move_insn (struct_value, value);
    1765            0 :       if (REG_P (struct_value))
    1766            0 :         use_reg (&call_fusage, struct_value);
    1767              :     }
    1768              : 
    1769              :   /* All arguments and registers used for the call are set up by now!  */
    1770          483 :   function = prepare_call_address (NULL, function, NULL, &call_fusage, 0, 0);
    1771              : 
    1772              :   /* Ensure address is valid.  SYMBOL_REF is already valid, so no need,
    1773              :      and we don't want to load it into a register as an optimization,
    1774              :      because prepare_call_address already did it if it should be done.  */
    1775          483 :   if (GET_CODE (function) != SYMBOL_REF)
    1776           29 :     function = memory_address (FUNCTION_MODE, function);
    1777              : 
    1778              :   /* Generate the actual call instruction and save the return value.  */
    1779          483 :   if (targetm.have_untyped_call ())
    1780              :     {
    1781          483 :       rtx mem = gen_rtx_MEM (FUNCTION_MODE, function);
    1782          483 :       rtx_insn *seq = targetm.gen_untyped_call (mem, result,
    1783              :                                                 result_vector (1, result));
    1784         5317 :       for (rtx_insn *insn = seq; insn; insn = NEXT_INSN (insn))
    1785         4351 :         if (CALL_P (insn))
    1786          483 :           add_reg_note (insn, REG_UNTYPED_CALL, NULL_RTX);
    1787          483 :       emit_insn (seq);
    1788              :     }
    1789            0 :   else if (targetm.have_call_value ())
    1790              :     {
    1791              :       rtx valreg = 0;
    1792              : 
    1793              :       /* Locate the unique return register.  It is not possible to
    1794              :          express a call that sets more than one return register using
    1795              :          call_value; use untyped_call for that.  In fact, untyped_call
    1796              :          only needs to save the return registers in the given block.  */
    1797            0 :       for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    1798            0 :         if ((mode = apply_result_mode[regno]) != VOIDmode)
    1799              :           {
    1800            0 :             gcc_assert (!valreg); /* have_untyped_call required.  */
    1801              : 
    1802            0 :             valreg = gen_rtx_REG (mode, regno);
    1803              :           }
    1804              : 
    1805            0 :       emit_insn (targetm.gen_call_value (valreg,
    1806              :                                          gen_rtx_MEM (FUNCTION_MODE, function),
    1807              :                                          const0_rtx, NULL_RTX, const0_rtx));
    1808              : 
    1809            0 :       emit_move_insn (adjust_address (result, GET_MODE (valreg), 0), valreg);
    1810              :     }
    1811              :   else
    1812            0 :     gcc_unreachable ();
    1813              : 
    1814              :   /* Find the CALL insn we just emitted, and attach the register usage
    1815              :      information.  */
    1816          483 :   call_insn = last_call_insn ();
    1817          483 :   add_function_usage_to (call_insn, call_fusage);
    1818              : 
    1819              :   /* Restore the stack.  */
    1820          483 :   if (targetm.have_save_stack_nonlocal ())
    1821          483 :     emit_stack_restore (SAVE_NONLOCAL, old_stack_level);
    1822              :   else
    1823            0 :     emit_stack_restore (SAVE_BLOCK, old_stack_level);
    1824          483 :   fixup_args_size_notes (call_insn, get_last_insn (), 0);
    1825              : 
    1826          483 :   OK_DEFER_POP;
    1827              : 
    1828              :   /* Return the address of the result block.  */
    1829          483 :   result = copy_addr_to_reg (XEXP (result, 0));
    1830          483 :   return convert_memory_address (ptr_mode, result);
    1831              : }
    1832              : 
    1833              : /* Perform an untyped return.  */
    1834              : 
    1835              : static void
    1836          379 : expand_builtin_return (rtx result)
    1837              : {
    1838          379 :   int size, align, regno;
    1839          379 :   fixed_size_mode mode;
    1840          379 :   rtx reg;
    1841          379 :   rtx_insn *call_fusage = 0;
    1842              : 
    1843          379 :   result = convert_memory_address (Pmode, result);
    1844              : 
    1845          379 :   apply_result_size ();
    1846          379 :   result = gen_rtx_MEM (BLKmode, result);
    1847              : 
    1848          379 :   if (targetm.have_untyped_return ())
    1849              :     {
    1850            0 :       rtx vector = result_vector (0, result);
    1851            0 :       emit_jump_insn (targetm.gen_untyped_return (result, vector));
    1852            0 :       emit_barrier ();
    1853            0 :       return;
    1854              :     }
    1855              : 
    1856              :   /* Restore the return value and note that each value is used.  */
    1857              :   size = 0;
    1858        36005 :   for (regno = 0; regno < FIRST_PSEUDO_REGISTER; regno++)
    1859        35626 :     if ((mode = apply_result_mode[regno]) != VOIDmode)
    1860              :       {
    1861         2274 :         align = GET_MODE_ALIGNMENT (mode) / BITS_PER_UNIT;
    1862         2274 :         if (size % align != 0)
    1863            7 :           size = CEIL (size, align) * align;
    1864         2274 :         reg = gen_rtx_REG (mode, INCOMING_REGNO (regno));
    1865         2274 :         emit_move_insn (reg, adjust_address (result, mode, size));
    1866              : 
    1867         2274 :         push_to_sequence (call_fusage);
    1868         2274 :         emit_use (reg);
    1869         2274 :         call_fusage = end_sequence ();
    1870         4548 :         size += GET_MODE_SIZE (mode);
    1871              :       }
    1872              : 
    1873              :   /* Put the USE insns before the return.  */
    1874          379 :   emit_insn (call_fusage);
    1875              : 
    1876              :   /* Return whatever values was restored by jumping directly to the end
    1877              :      of the function.  */
    1878          379 :   expand_naked_return ();
    1879              : }
    1880              : 
    1881              : /* Used by expand_builtin_classify_type and fold_builtin_classify_type.  */
    1882              : 
    1883              : int
    1884         2371 : type_to_class (tree type)
    1885              : {
    1886         2371 :   switch (TREE_CODE (type))
    1887              :     {
    1888              :     case VOID_TYPE:        return void_type_class;
    1889          899 :     case INTEGER_TYPE:     return integer_type_class;
    1890           39 :     case ENUMERAL_TYPE:    return enumeral_type_class;
    1891           38 :     case BOOLEAN_TYPE:     return boolean_type_class;
    1892          985 :     case POINTER_TYPE:     return pointer_type_class;
    1893           27 :     case REFERENCE_TYPE:   return reference_type_class;
    1894           36 :     case OFFSET_TYPE:      return offset_type_class;
    1895          132 :     case REAL_TYPE:        return real_type_class;
    1896           40 :     case COMPLEX_TYPE:     return complex_type_class;
    1897           20 :     case FUNCTION_TYPE:    return function_type_class;
    1898            0 :     case METHOD_TYPE:      return method_type_class;
    1899           44 :     case RECORD_TYPE:      return record_type_class;
    1900           44 :     case UNION_TYPE:
    1901           44 :     case QUAL_UNION_TYPE:  return union_type_class;
    1902           24 :     case ARRAY_TYPE:       return (TYPE_STRING_FLAG (type)
    1903           24 :                                    ? string_type_class : array_type_class);
    1904            0 :     case LANG_TYPE:        return lang_type_class;
    1905            0 :     case OPAQUE_TYPE:      return opaque_type_class;
    1906            0 :     case BITINT_TYPE:      return bitint_type_class;
    1907           30 :     case VECTOR_TYPE:      return vector_type_class;
    1908            3 :     default:               return no_type_class;
    1909              :     }
    1910              : }
    1911              : 
    1912              : /* Expand a call EXP to __builtin_classify_type.  */
    1913              : 
    1914              : static rtx
    1915            0 : expand_builtin_classify_type (tree exp)
    1916              : {
    1917            0 :   if (call_expr_nargs (exp))
    1918            0 :     return GEN_INT (type_to_class (TREE_TYPE (CALL_EXPR_ARG (exp, 0))));
    1919            0 :   return GEN_INT (no_type_class);
    1920              : }
    1921              : 
    1922              : /* This helper macro, meant to be used in mathfn_built_in below, determines
    1923              :    which among a set of builtin math functions is appropriate for a given type
    1924              :    mode.  The `F' (float) and `L' (long double) are automatically generated
    1925              :    from the 'double' case.  If a function supports the _Float<N> and _Float<N>X
    1926              :    types, there are additional types that are considered with 'F32', 'F64',
    1927              :    'F128', etc. suffixes.  */
    1928              : #define CASE_MATHFN(MATHFN) \
    1929              :   CASE_CFN_##MATHFN: \
    1930              :   fcode = BUILT_IN_##MATHFN; fcodef = BUILT_IN_##MATHFN##F ; \
    1931              :   fcodel = BUILT_IN_##MATHFN##L ; break;
    1932              : /* Similar to the above, but also add support for the _Float<N> and _Float<N>X
    1933              :    types.  */
    1934              : #define CASE_MATHFN_FLOATN(MATHFN) \
    1935              :   CASE_CFN_##MATHFN: \
    1936              :   fcode = BUILT_IN_##MATHFN; fcodef = BUILT_IN_##MATHFN##F ; \
    1937              :   fcodel = BUILT_IN_##MATHFN##L ; fcodef16 = BUILT_IN_##MATHFN##F16 ; \
    1938              :   fcodef32 = BUILT_IN_##MATHFN##F32; fcodef64 = BUILT_IN_##MATHFN##F64 ; \
    1939              :   fcodef128 = BUILT_IN_##MATHFN##F128 ; fcodef32x = BUILT_IN_##MATHFN##F32X ; \
    1940              :   fcodef64x = BUILT_IN_##MATHFN##F64X ; fcodef128x = BUILT_IN_##MATHFN##F128X ;\
    1941              :   break;
    1942              : /* Similar to above, but appends _R after any F/L suffix.  */
    1943              : #define CASE_MATHFN_REENT(MATHFN) \
    1944              :   case CFN_BUILT_IN_##MATHFN##_R: \
    1945              :   case CFN_BUILT_IN_##MATHFN##F_R: \
    1946              :   case CFN_BUILT_IN_##MATHFN##L_R: \
    1947              :   fcode = BUILT_IN_##MATHFN##_R; fcodef = BUILT_IN_##MATHFN##F_R ; \
    1948              :   fcodel = BUILT_IN_##MATHFN##L_R ; break;
    1949              : 
    1950              : /* Return a function equivalent to FN but operating on floating-point
    1951              :    values of type TYPE, or END_BUILTINS if no such function exists.
    1952              :    This is purely an operation on function codes; it does not guarantee
    1953              :    that the target actually has an implementation of the function.  */
    1954              : 
    1955              : static built_in_function
    1956       612260 : mathfn_built_in_2 (tree type, combined_fn fn)
    1957              : {
    1958       612260 :   tree mtype;
    1959       612260 :   built_in_function fcode, fcodef, fcodel;
    1960       612260 :   built_in_function fcodef16 = END_BUILTINS;
    1961       612260 :   built_in_function fcodef32 = END_BUILTINS;
    1962       612260 :   built_in_function fcodef64 = END_BUILTINS;
    1963       612260 :   built_in_function fcodef128 = END_BUILTINS;
    1964       612260 :   built_in_function fcodef32x = END_BUILTINS;
    1965       612260 :   built_in_function fcodef64x = END_BUILTINS;
    1966       612260 :   built_in_function fcodef128x = END_BUILTINS;
    1967              : 
    1968              :   /* If <math.h> has been included somehow, HUGE_VAL and NAN definitions
    1969              :      break the uses below.  */
    1970              : #undef HUGE_VAL
    1971              : #undef NAN
    1972              : 
    1973       612260 :   switch (fn)
    1974              :     {
    1975              : #define SEQ_OF_CASE_MATHFN                      \
    1976              :     CASE_MATHFN_FLOATN (ACOS)                   \
    1977              :     CASE_MATHFN_FLOATN (ACOSH)                  \
    1978              :     CASE_MATHFN_FLOATN (ACOSPI)                 \
    1979              :     CASE_MATHFN_FLOATN (ASIN)                   \
    1980              :     CASE_MATHFN_FLOATN (ASINH)                  \
    1981              :     CASE_MATHFN_FLOATN (ASINPI)                 \
    1982              :     CASE_MATHFN_FLOATN (ATAN)                   \
    1983              :     CASE_MATHFN_FLOATN (ATAN2)                  \
    1984              :     CASE_MATHFN_FLOATN (ATAN2PI)                \
    1985              :     CASE_MATHFN_FLOATN (ATANH)                  \
    1986              :     CASE_MATHFN_FLOATN (ATANPI)                 \
    1987              :     CASE_MATHFN_FLOATN (CBRT)                   \
    1988              :     CASE_MATHFN_FLOATN (CEIL)                   \
    1989              :     CASE_MATHFN (CEXPI)                         \
    1990              :     CASE_MATHFN_FLOATN (COPYSIGN)               \
    1991              :     CASE_MATHFN_FLOATN (COS)                    \
    1992              :     CASE_MATHFN_FLOATN (COSH)                   \
    1993              :     CASE_MATHFN_FLOATN (COSPI)                  \
    1994              :     CASE_MATHFN (DREM)                          \
    1995              :     CASE_MATHFN_FLOATN (ERF)                    \
    1996              :     CASE_MATHFN_FLOATN (ERFC)                   \
    1997              :     CASE_MATHFN_FLOATN (EXP)                    \
    1998              :     CASE_MATHFN (EXP10)                         \
    1999              :     CASE_MATHFN_FLOATN (EXP2)                   \
    2000              :     CASE_MATHFN_FLOATN (EXPM1)                  \
    2001              :     CASE_MATHFN_FLOATN (FABS)                   \
    2002              :     CASE_MATHFN_FLOATN (FDIM)                   \
    2003              :     CASE_MATHFN_FLOATN (FLOOR)                  \
    2004              :     CASE_MATHFN_FLOATN (FMA)                    \
    2005              :     CASE_MATHFN_FLOATN (FMAX)                   \
    2006              :     CASE_MATHFN_FLOATN (FMIN)                   \
    2007              :     CASE_MATHFN_FLOATN (FMOD)                   \
    2008              :     CASE_MATHFN_FLOATN (FREXP)                  \
    2009              :     CASE_MATHFN (GAMMA)                         \
    2010              :     CASE_MATHFN_REENT (GAMMA) /* GAMMA_R */     \
    2011              :     CASE_MATHFN_FLOATN (HUGE_VAL)               \
    2012              :     CASE_MATHFN_FLOATN (HYPOT)                  \
    2013              :     CASE_MATHFN_FLOATN (ILOGB)                  \
    2014              :     CASE_MATHFN (ICEIL)                         \
    2015              :     CASE_MATHFN (IFLOOR)                        \
    2016              :     CASE_MATHFN_FLOATN (INF)                    \
    2017              :     CASE_MATHFN (IRINT)                         \
    2018              :     CASE_MATHFN (IROUND)                        \
    2019              :     CASE_MATHFN (ISINF)                         \
    2020              :     CASE_MATHFN (J0)                            \
    2021              :     CASE_MATHFN (J1)                            \
    2022              :     CASE_MATHFN (JN)                            \
    2023              :     CASE_MATHFN (LCEIL)                         \
    2024              :     CASE_MATHFN_FLOATN (LDEXP)                  \
    2025              :     CASE_MATHFN (LFLOOR)                        \
    2026              :     CASE_MATHFN_FLOATN (LGAMMA)                 \
    2027              :     CASE_MATHFN_REENT (LGAMMA) /* LGAMMA_R */   \
    2028              :     CASE_MATHFN (LLCEIL)                        \
    2029              :     CASE_MATHFN (LLFLOOR)                       \
    2030              :     CASE_MATHFN_FLOATN (LLRINT)                 \
    2031              :     CASE_MATHFN_FLOATN (LLROUND)                \
    2032              :     CASE_MATHFN_FLOATN (LOG)                    \
    2033              :     CASE_MATHFN_FLOATN (LOG10)                  \
    2034              :     CASE_MATHFN_FLOATN (LOG1P)                  \
    2035              :     CASE_MATHFN_FLOATN (LOG2)                   \
    2036              :     CASE_MATHFN_FLOATN (LOGB)                   \
    2037              :     CASE_MATHFN_FLOATN (LRINT)                  \
    2038              :     CASE_MATHFN_FLOATN (LROUND)                 \
    2039              :     CASE_MATHFN_FLOATN (MODF)                   \
    2040              :     CASE_MATHFN_FLOATN (NAN)                    \
    2041              :     CASE_MATHFN_FLOATN (NANS)                   \
    2042              :     CASE_MATHFN_FLOATN (NEARBYINT)              \
    2043              :     CASE_MATHFN_FLOATN (NEXTAFTER)              \
    2044              :     CASE_MATHFN (NEXTTOWARD)                    \
    2045              :     CASE_MATHFN_FLOATN (POW)                    \
    2046              :     CASE_MATHFN (POWI)                          \
    2047              :     CASE_MATHFN (POW10)                         \
    2048              :     CASE_MATHFN_FLOATN (REMAINDER)              \
    2049              :     CASE_MATHFN_FLOATN (REMQUO)                 \
    2050              :     CASE_MATHFN_FLOATN (RINT)                   \
    2051              :     CASE_MATHFN_FLOATN (ROUND)                  \
    2052              :     CASE_MATHFN_FLOATN (ROUNDEVEN)              \
    2053              :     CASE_MATHFN (SCALB)                         \
    2054              :     CASE_MATHFN_FLOATN (SCALBLN)                \
    2055              :     CASE_MATHFN_FLOATN (SCALBN)                 \
    2056              :     CASE_MATHFN (SIGNBIT)                       \
    2057              :     CASE_MATHFN (SIGNIFICAND)                   \
    2058              :     CASE_MATHFN_FLOATN (SIN)                    \
    2059              :     CASE_MATHFN (SINCOS)                        \
    2060              :     CASE_MATHFN_FLOATN (SINH)                   \
    2061              :     CASE_MATHFN_FLOATN (SINPI)                  \
    2062              :     CASE_MATHFN_FLOATN (SQRT)                   \
    2063              :     CASE_MATHFN_FLOATN (TAN)                    \
    2064              :     CASE_MATHFN_FLOATN (TANH)                   \
    2065              :     CASE_MATHFN_FLOATN (TANPI)                  \
    2066              :     CASE_MATHFN_FLOATN (TGAMMA)                 \
    2067              :     CASE_MATHFN_FLOATN (TRUNC)                  \
    2068              :     CASE_MATHFN (Y0)                            \
    2069              :     CASE_MATHFN (Y1)                            \
    2070              :     CASE_MATHFN (YN)
    2071              : 
    2072       612164 :     SEQ_OF_CASE_MATHFN
    2073              : 
    2074              :     default:
    2075              :       return END_BUILTINS;
    2076              :     }
    2077              : 
    2078       612260 :   mtype = TYPE_MAIN_VARIANT (type);
    2079       612260 :   if (mtype == double_type_node)
    2080              :     return fcode;
    2081       572425 :   else if (mtype == float_type_node)
    2082              :     return fcodef;
    2083       526573 :   else if (mtype == long_double_type_node)
    2084              :     return fcodel;
    2085       494224 :   else if (mtype == float16_type_node)
    2086              :     return fcodef16;
    2087       494174 :   else if (mtype == float32_type_node)
    2088              :     return fcodef32;
    2089       494174 :   else if (mtype == float64_type_node)
    2090              :     return fcodef64;
    2091       494173 :   else if (mtype == float128_type_node)
    2092              :     return fcodef128;
    2093       486649 :   else if (mtype == float32x_type_node)
    2094              :     return fcodef32x;
    2095       486649 :   else if (mtype == float64x_type_node)
    2096              :     return fcodef64x;
    2097       486649 :   else if (mtype == float128x_type_node)
    2098              :     return fcodef128x;
    2099              :   else
    2100       486649 :     return END_BUILTINS;
    2101              : }
    2102              : 
    2103              : #undef CASE_MATHFN
    2104              : #undef CASE_MATHFN_FLOATN
    2105              : #undef CASE_MATHFN_REENT
    2106              : 
    2107              : /* Return mathematic function equivalent to FN but operating directly on TYPE,
    2108              :    if available.  If IMPLICIT_P is true use the implicit builtin declaration,
    2109              :    otherwise use the explicit declaration.  If we can't do the conversion,
    2110              :    return null.  */
    2111              : 
    2112              : static tree
    2113       612115 : mathfn_built_in_1 (tree type, combined_fn fn, bool implicit_p)
    2114              : {
    2115       612115 :   built_in_function fcode2 = mathfn_built_in_2 (type, fn);
    2116       612115 :   if (fcode2 == END_BUILTINS)
    2117              :     return NULL_TREE;
    2118              : 
    2119       121815 :   if (implicit_p && !builtin_decl_implicit_p (fcode2))
    2120              :     return NULL_TREE;
    2121              : 
    2122       117842 :   return builtin_decl_explicit (fcode2);
    2123              : }
    2124              : 
    2125              : /* Like mathfn_built_in_1, but always use the implicit array.  */
    2126              : 
    2127              : tree
    2128         1039 : mathfn_built_in (tree type, combined_fn fn)
    2129              : {
    2130         1039 :   return mathfn_built_in_1 (type, fn, /*implicit=*/ 1);
    2131              : }
    2132              : 
    2133              : /* Like mathfn_built_in_1, but always use the explicit array.  */
    2134              : 
    2135              : tree
    2136            0 : mathfn_built_in_explicit (tree type, combined_fn fn)
    2137              : {
    2138            0 :   return mathfn_built_in_1 (type, fn, /*implicit=*/ 0);
    2139              : }
    2140              : 
    2141              : /* Like mathfn_built_in_1, but take a built_in_function and
    2142              :    always use the implicit array.  */
    2143              : 
    2144              : tree
    2145       610860 : mathfn_built_in (tree type, enum built_in_function fn)
    2146              : {
    2147       610860 :   return mathfn_built_in_1 (type, as_combined_fn (fn), /*implicit=*/ 1);
    2148              : }
    2149              : 
    2150              : /* Return the type associated with a built in function, i.e., the one
    2151              :    to be passed to mathfn_built_in to get the type-specific
    2152              :    function.  */
    2153              : 
    2154              : tree
    2155         1293 : mathfn_built_in_type (combined_fn fn)
    2156              : {
    2157              : #define CASE_MATHFN(MATHFN)                     \
    2158              :   case CFN_BUILT_IN_##MATHFN:                   \
    2159              :     return double_type_node;                    \
    2160              :   case CFN_BUILT_IN_##MATHFN##F:                \
    2161              :     return float_type_node;                     \
    2162              :   case CFN_BUILT_IN_##MATHFN##L:                \
    2163              :     return long_double_type_node;
    2164              : 
    2165              : #define CASE_MATHFN_FLOATN(MATHFN)              \
    2166              :   CASE_MATHFN(MATHFN)                           \
    2167              :   case CFN_BUILT_IN_##MATHFN##F16:              \
    2168              :     return float16_type_node;                   \
    2169              :   case CFN_BUILT_IN_##MATHFN##F32:              \
    2170              :     return float32_type_node;                   \
    2171              :   case CFN_BUILT_IN_##MATHFN##F64:              \
    2172              :     return float64_type_node;                   \
    2173              :   case CFN_BUILT_IN_##MATHFN##F128:             \
    2174              :     return float128_type_node;                  \
    2175              :   case CFN_BUILT_IN_##MATHFN##F32X:             \
    2176              :     return float32x_type_node;                  \
    2177              :   case CFN_BUILT_IN_##MATHFN##F64X:             \
    2178              :     return float64x_type_node;                  \
    2179              :   case CFN_BUILT_IN_##MATHFN##F128X:            \
    2180              :     return float128x_type_node;
    2181              : 
    2182              : /* Similar to above, but appends _R after any F/L suffix.  */
    2183              : #define CASE_MATHFN_REENT(MATHFN) \
    2184              :   case CFN_BUILT_IN_##MATHFN##_R:               \
    2185              :     return double_type_node;                    \
    2186              :   case CFN_BUILT_IN_##MATHFN##F_R:              \
    2187              :     return float_type_node;                     \
    2188              :   case CFN_BUILT_IN_##MATHFN##L_R:              \
    2189              :     return long_double_type_node;
    2190              : 
    2191         1293 :   switch (fn)
    2192              :     {
    2193         1291 :     SEQ_OF_CASE_MATHFN
    2194              : 
    2195              :     default:
    2196              :       return NULL_TREE;
    2197              :     }
    2198              : 
    2199              : #undef CASE_MATHFN
    2200              : #undef CASE_MATHFN_FLOATN
    2201              : #undef CASE_MATHFN_REENT
    2202              : #undef SEQ_OF_CASE_MATHFN
    2203              : }
    2204              : 
    2205              : /* Check whether there is an internal function associated with function FN
    2206              :    and return type RETURN_TYPE.  Return the function if so, otherwise return
    2207              :    IFN_LAST.
    2208              : 
    2209              :    Note that this function only tests whether the function is defined in
    2210              :    internals.def, not whether it is actually available on the target.  */
    2211              : 
    2212              : static internal_fn
    2213     14070863 : associated_internal_fn (built_in_function fn, tree return_type)
    2214              : {
    2215     14070863 :   switch (fn)
    2216              :     {
    2217              : #define DEF_INTERNAL_FLT_FN(NAME, FLAGS, OPTAB, TYPE) \
    2218              :     CASE_FLT_FN (BUILT_IN_##NAME): return IFN_##NAME;
    2219              : #define DEF_INTERNAL_FLT_FLOATN_FN(NAME, FLAGS, OPTAB, TYPE) \
    2220              :     CASE_FLT_FN (BUILT_IN_##NAME): return IFN_##NAME; \
    2221              :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_##NAME): return IFN_##NAME;
    2222              : #define DEF_INTERNAL_INT_FN(NAME, FLAGS, OPTAB, TYPE) \
    2223              :     CASE_INT_FN (BUILT_IN_##NAME): return IFN_##NAME;
    2224              : #include "internal-fn.def"
    2225              : 
    2226              :     CASE_FLT_FN (BUILT_IN_POW10):
    2227              :       return IFN_EXP10;
    2228              : 
    2229              :     CASE_FLT_FN (BUILT_IN_DREM):
    2230              :       return IFN_REMAINDER;
    2231              : 
    2232        19608 :     CASE_FLT_FN (BUILT_IN_SCALBN):
    2233        19608 :     CASE_FLT_FN (BUILT_IN_SCALBLN):
    2234        19608 :       if (REAL_MODE_FORMAT (TYPE_MODE (return_type))->b == 2)
    2235              :         return IFN_LDEXP;
    2236              :       return IFN_LAST;
    2237          727 :     case BUILT_IN_CRC8_DATA8:
    2238          727 :     case BUILT_IN_CRC16_DATA8:
    2239          727 :     case BUILT_IN_CRC16_DATA16:
    2240          727 :     case BUILT_IN_CRC32_DATA8:
    2241          727 :     case BUILT_IN_CRC32_DATA16:
    2242          727 :     case BUILT_IN_CRC32_DATA32:
    2243          727 :     case BUILT_IN_CRC64_DATA8:
    2244          727 :     case BUILT_IN_CRC64_DATA16:
    2245          727 :     case BUILT_IN_CRC64_DATA32:
    2246          727 :     case BUILT_IN_CRC64_DATA64:
    2247          727 :       return IFN_CRC;
    2248          565 :     case BUILT_IN_REV_CRC8_DATA8:
    2249          565 :     case BUILT_IN_REV_CRC16_DATA8:
    2250          565 :     case BUILT_IN_REV_CRC16_DATA16:
    2251          565 :     case BUILT_IN_REV_CRC32_DATA8:
    2252          565 :     case BUILT_IN_REV_CRC32_DATA16:
    2253          565 :     case BUILT_IN_REV_CRC32_DATA32:
    2254          565 :     case BUILT_IN_REV_CRC64_DATA8:
    2255          565 :     case BUILT_IN_REV_CRC64_DATA16:
    2256          565 :     case BUILT_IN_REV_CRC64_DATA32:
    2257          565 :     case BUILT_IN_REV_CRC64_DATA64:
    2258          565 :       return IFN_CRC_REV;
    2259              :     default:
    2260              :       return IFN_LAST;
    2261              :     }
    2262              : }
    2263              : 
    2264              : /* If BUILT_IN_NORMAL function FNDECL has an associated internal function,
    2265              :    return its code, otherwise return IFN_LAST.  Note that this function
    2266              :    only tests whether the function is defined in internals.def, not whether
    2267              :    it is actually available on the target.  */
    2268              : 
    2269              : internal_fn
    2270       640268 : associated_internal_fn (tree fndecl)
    2271              : {
    2272       640268 :   gcc_checking_assert (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL);
    2273       640268 :   return associated_internal_fn (DECL_FUNCTION_CODE (fndecl),
    2274       640268 :                                  TREE_TYPE (TREE_TYPE (fndecl)));
    2275              : }
    2276              : 
    2277              : /* Check whether there is an internal function associated with function CFN
    2278              :    and return type RETURN_TYPE.  Return the function if so, otherwise return
    2279              :    IFN_LAST.
    2280              : 
    2281              :    Note that this function only tests whether the function is defined in
    2282              :    internals.def, not whether it is actually available on the target.  */
    2283              : 
    2284              : internal_fn
    2285     34504665 : associated_internal_fn (combined_fn cfn, tree return_type)
    2286              : {
    2287     34504665 :   if (internal_fn_p (cfn))
    2288     21074070 :     return as_internal_fn (cfn);
    2289     13430595 :   return associated_internal_fn (as_builtin_fn (cfn), return_type);
    2290              : }
    2291              : 
    2292              : /* If CALL is a call to a BUILT_IN_NORMAL function that could be replaced
    2293              :    on the current target by a call to an internal function, return the
    2294              :    code of that internal function, otherwise return IFN_LAST.  The caller
    2295              :    is responsible for ensuring that any side-effects of the built-in
    2296              :    call are dealt with correctly.  E.g. if CALL sets errno, the caller
    2297              :    must decide that the errno result isn't needed or make it available
    2298              :    in some other way.  */
    2299              : 
    2300              : internal_fn
    2301       871422 : replacement_internal_fn (gcall *call)
    2302              : {
    2303       871422 :   if (gimple_call_builtin_p (call, BUILT_IN_NORMAL))
    2304              :     {
    2305       636164 :       internal_fn ifn = associated_internal_fn (gimple_call_fndecl (call));
    2306       636164 :       if (ifn != IFN_LAST)
    2307              :         {
    2308        59596 :           tree_pair types = direct_internal_fn_types (ifn, call);
    2309        59596 :           optimization_type opt_type = bb_optimization_type (gimple_bb (call));
    2310        59596 :           if (direct_internal_fn_supported_p (ifn, types, opt_type))
    2311        37474 :             return ifn;
    2312              :         }
    2313              :     }
    2314              :   return IFN_LAST;
    2315              : }
    2316              : 
    2317              : /* Expand a call to the builtin trinary math functions (fma).
    2318              :    Return NULL_RTX if a normal call should be emitted rather than expanding the
    2319              :    function in-line.  EXP is the expression that is a call to the builtin
    2320              :    function; if convenient, the result should be placed in TARGET.
    2321              :    SUBTARGET may be used as the target for computing one of EXP's
    2322              :    operands.  */
    2323              : 
    2324              : static rtx
    2325          372 : expand_builtin_mathfn_ternary (tree exp, rtx target, rtx subtarget)
    2326              : {
    2327          372 :   optab builtin_optab;
    2328          372 :   rtx op0, op1, op2, result;
    2329          372 :   rtx_insn *insns;
    2330          372 :   tree fndecl = get_callee_fndecl (exp);
    2331          372 :   tree arg0, arg1, arg2;
    2332          372 :   machine_mode mode;
    2333              : 
    2334          372 :   if (!validate_arglist (exp, REAL_TYPE, REAL_TYPE, REAL_TYPE, VOID_TYPE))
    2335              :     return NULL_RTX;
    2336              : 
    2337          372 :   arg0 = CALL_EXPR_ARG (exp, 0);
    2338          372 :   arg1 = CALL_EXPR_ARG (exp, 1);
    2339          372 :   arg2 = CALL_EXPR_ARG (exp, 2);
    2340              : 
    2341          372 :   switch (DECL_FUNCTION_CODE (fndecl))
    2342              :     {
    2343          372 :     CASE_FLT_FN (BUILT_IN_FMA):
    2344          372 :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMA):
    2345          372 :       builtin_optab = fma_optab; break;
    2346            0 :     default:
    2347            0 :       gcc_unreachable ();
    2348              :     }
    2349              : 
    2350              :   /* Make a suitable register to place result in.  */
    2351          372 :   mode = TYPE_MODE (TREE_TYPE (exp));
    2352              : 
    2353              :   /* Before working hard, check whether the instruction is available.  */
    2354          372 :   if (optab_handler (builtin_optab, mode) == CODE_FOR_nothing)
    2355              :     return NULL_RTX;
    2356              : 
    2357            0 :   result = gen_reg_rtx (mode);
    2358              : 
    2359              :   /* Always stabilize the argument list.  */
    2360            0 :   CALL_EXPR_ARG (exp, 0) = arg0 = builtin_save_expr (arg0);
    2361            0 :   CALL_EXPR_ARG (exp, 1) = arg1 = builtin_save_expr (arg1);
    2362            0 :   CALL_EXPR_ARG (exp, 2) = arg2 = builtin_save_expr (arg2);
    2363              : 
    2364            0 :   op0 = expand_expr (arg0, subtarget, VOIDmode, EXPAND_NORMAL);
    2365            0 :   op1 = expand_normal (arg1);
    2366            0 :   op2 = expand_normal (arg2);
    2367              : 
    2368            0 :   start_sequence ();
    2369              : 
    2370              :   /* Compute into RESULT.
    2371              :      Set RESULT to wherever the result comes back.  */
    2372            0 :   result = expand_ternary_op (mode, builtin_optab, op0, op1, op2,
    2373              :                               result, 0);
    2374              : 
    2375              :   /* If we were unable to expand via the builtin, stop the sequence
    2376              :      (without outputting the insns) and call to the library function
    2377              :      with the stabilized argument list.  */
    2378            0 :   if (result == 0)
    2379              :     {
    2380            0 :       end_sequence ();
    2381            0 :       return expand_call (exp, target, target == const0_rtx);
    2382              :     }
    2383              : 
    2384              :   /* Output the entire sequence.  */
    2385            0 :   insns = end_sequence ();
    2386            0 :   emit_insn (insns);
    2387              : 
    2388            0 :   return result;
    2389              : }
    2390              : 
    2391              : /* Expand a call to the builtin sin and cos math functions.
    2392              :    Return NULL_RTX if a normal call should be emitted rather than expanding the
    2393              :    function in-line.  EXP is the expression that is a call to the builtin
    2394              :    function; if convenient, the result should be placed in TARGET.
    2395              :    SUBTARGET may be used as the target for computing one of EXP's
    2396              :    operands.  */
    2397              : 
    2398              : static rtx
    2399           47 : expand_builtin_mathfn_3 (tree exp, rtx target, rtx subtarget)
    2400              : {
    2401           47 :   optab builtin_optab;
    2402           47 :   rtx op0;
    2403           47 :   rtx_insn *insns;
    2404           47 :   tree fndecl = get_callee_fndecl (exp);
    2405           47 :   machine_mode mode;
    2406           47 :   tree arg;
    2407              : 
    2408           47 :   if (!validate_arglist (exp, REAL_TYPE, VOID_TYPE))
    2409              :     return NULL_RTX;
    2410              : 
    2411           39 :   arg = CALL_EXPR_ARG (exp, 0);
    2412              : 
    2413           39 :   switch (DECL_FUNCTION_CODE (fndecl))
    2414              :     {
    2415           39 :     CASE_FLT_FN (BUILT_IN_SIN):
    2416           39 :     CASE_FLT_FN (BUILT_IN_COS):
    2417           39 :       builtin_optab = sincos_optab; break;
    2418            0 :     default:
    2419            0 :       gcc_unreachable ();
    2420              :     }
    2421              : 
    2422              :   /* Make a suitable register to place result in.  */
    2423           39 :   mode = TYPE_MODE (TREE_TYPE (exp));
    2424              : 
    2425              :   /* Check if sincos insn is available, otherwise fallback
    2426              :      to sin or cos insn.  */
    2427           39 :   if (optab_handler (builtin_optab, mode) == CODE_FOR_nothing)
    2428           39 :     switch (DECL_FUNCTION_CODE (fndecl))
    2429              :       {
    2430              :       CASE_FLT_FN (BUILT_IN_SIN):
    2431              :         builtin_optab = sin_optab; break;
    2432           14 :       CASE_FLT_FN (BUILT_IN_COS):
    2433           14 :         builtin_optab = cos_optab; break;
    2434            0 :       default:
    2435            0 :         gcc_unreachable ();
    2436              :       }
    2437              : 
    2438              :   /* Before working hard, check whether the instruction is available.  */
    2439           39 :   if (optab_handler (builtin_optab, mode) != CODE_FOR_nothing)
    2440              :     {
    2441            0 :       rtx result = gen_reg_rtx (mode);
    2442              : 
    2443              :       /* Wrap the computation of the argument in a SAVE_EXPR, as we may
    2444              :          need to expand the argument again.  This way, we will not perform
    2445              :          side-effects more the once.  */
    2446            0 :       CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (arg);
    2447              : 
    2448            0 :       op0 = expand_expr (arg, subtarget, VOIDmode, EXPAND_NORMAL);
    2449              : 
    2450            0 :       start_sequence ();
    2451              : 
    2452              :       /* Compute into RESULT.
    2453              :          Set RESULT to wherever the result comes back.  */
    2454            0 :       if (builtin_optab == sincos_optab)
    2455              :         {
    2456            0 :           int ok;
    2457              : 
    2458            0 :           switch (DECL_FUNCTION_CODE (fndecl))
    2459              :             {
    2460            0 :             CASE_FLT_FN (BUILT_IN_SIN):
    2461            0 :               ok = expand_twoval_unop (builtin_optab, op0, 0, result, 0);
    2462            0 :               break;
    2463            0 :             CASE_FLT_FN (BUILT_IN_COS):
    2464            0 :               ok = expand_twoval_unop (builtin_optab, op0, result, 0, 0);
    2465            0 :               break;
    2466            0 :             default:
    2467            0 :               gcc_unreachable ();
    2468              :             }
    2469            0 :           gcc_assert (ok);
    2470              :         }
    2471              :       else
    2472            0 :         result = expand_unop (mode, builtin_optab, op0, result, 0);
    2473              : 
    2474            0 :       if (result != 0)
    2475              :         {
    2476              :           /* Output the entire sequence.  */
    2477            0 :           insns = end_sequence ();
    2478            0 :           emit_insn (insns);
    2479            0 :           return result;
    2480              :         }
    2481              : 
    2482              :       /* If we were unable to expand via the builtin, stop the sequence
    2483              :          (without outputting the insns) and call to the library function
    2484              :          with the stabilized argument list.  */
    2485            0 :       end_sequence ();
    2486              :     }
    2487              : 
    2488           39 :   return expand_call (exp, target, target == const0_rtx);
    2489              : }
    2490              : 
    2491              : /* Given an interclass math builtin decl FNDECL and it's argument ARG
    2492              :    return an RTL instruction code that implements the functionality.
    2493              :    If that isn't possible or available return CODE_FOR_nothing.  */
    2494              : 
    2495              : static enum insn_code
    2496      1815853 : interclass_mathfn_icode (tree arg, tree fndecl)
    2497              : {
    2498      1815853 :   bool errno_set = false;
    2499      1815853 :   optab builtin_optab = unknown_optab;
    2500      1815853 :   machine_mode mode;
    2501              : 
    2502      1815853 :   switch (DECL_FUNCTION_CODE (fndecl))
    2503              :     {
    2504            4 :     CASE_FLT_FN (BUILT_IN_ILOGB):
    2505            4 :       errno_set = true;
    2506            4 :       builtin_optab = ilogb_optab;
    2507            4 :       break;
    2508       705048 :     CASE_FLT_FN (BUILT_IN_ISINF):
    2509       705048 :       builtin_optab = isinf_optab;
    2510       705048 :       break;
    2511       627943 :     case BUILT_IN_ISFINITE:
    2512       627943 :       builtin_optab = isfinite_optab;
    2513       627943 :       break;
    2514       239084 :     case BUILT_IN_ISNORMAL:
    2515       239084 :       builtin_optab = isnormal_optab;
    2516       239084 :       break;
    2517       241697 :     CASE_FLT_FN (BUILT_IN_ISNAN):
    2518       241697 :       builtin_optab = isnan_optab;
    2519       241697 :       break;
    2520              :     CASE_FLT_FN (BUILT_IN_FINITE):
    2521              :     case BUILT_IN_FINITED32:
    2522              :     case BUILT_IN_FINITED64:
    2523              :     case BUILT_IN_FINITED128:
    2524              :     case BUILT_IN_ISINFD32:
    2525              :     case BUILT_IN_ISINFD64:
    2526              :     case BUILT_IN_ISINFD128:
    2527              :     case BUILT_IN_ISNAND32:
    2528              :     case BUILT_IN_ISNAND64:
    2529              :     case BUILT_IN_ISNAND128:
    2530              :       /* These builtins have no optabs (yet).  */
    2531              :       break;
    2532            0 :     default:
    2533            0 :       gcc_unreachable ();
    2534              :     }
    2535              : 
    2536              :   /* There's no easy way to detect the case we need to set EDOM.  */
    2537      1815853 :   if (flag_errno_math && errno_set)
    2538              :     return CODE_FOR_nothing;
    2539              : 
    2540              :   /* Optab mode depends on the mode of the input argument.  */
    2541      1815853 :   mode = TYPE_MODE (TREE_TYPE (arg));
    2542              : 
    2543      1815853 :   if (builtin_optab)
    2544      1813776 :     return optab_handler (builtin_optab, mode);
    2545              :   return CODE_FOR_nothing;
    2546              : }
    2547              : 
    2548              : /* Expand a call to one of the builtin math functions that operate on
    2549              :    floating point argument and output an integer result (ilogb, isinf,
    2550              :    isnan, etc).
    2551              :    Return 0 if a normal call should be emitted rather than expanding the
    2552              :    function in-line.  EXP is the expression that is a call to the builtin
    2553              :    function; if convenient, the result should be placed in TARGET.  */
    2554              : 
    2555              : static rtx
    2556            4 : expand_builtin_interclass_mathfn (tree exp, rtx target)
    2557              : {
    2558            4 :   enum insn_code icode = CODE_FOR_nothing;
    2559            4 :   rtx op0;
    2560            4 :   tree fndecl = get_callee_fndecl (exp);
    2561            4 :   machine_mode mode;
    2562            4 :   tree arg;
    2563              : 
    2564            4 :   if (!validate_arglist (exp, REAL_TYPE, VOID_TYPE))
    2565              :     return NULL_RTX;
    2566              : 
    2567            4 :   arg = CALL_EXPR_ARG (exp, 0);
    2568            4 :   icode = interclass_mathfn_icode (arg, fndecl);
    2569            4 :   mode = TYPE_MODE (TREE_TYPE (arg));
    2570              : 
    2571            4 :   if (icode != CODE_FOR_nothing)
    2572              :     {
    2573            2 :       class expand_operand ops[1];
    2574            2 :       rtx_insn *last = get_last_insn ();
    2575            2 :       tree orig_arg = arg;
    2576              : 
    2577              :       /* Wrap the computation of the argument in a SAVE_EXPR, as we may
    2578              :          need to expand the argument again.  This way, we will not perform
    2579              :          side-effects more the once.  */
    2580            2 :       CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (arg);
    2581              : 
    2582            2 :       op0 = expand_expr (arg, NULL_RTX, VOIDmode, EXPAND_NORMAL);
    2583              : 
    2584            2 :       if (mode != GET_MODE (op0))
    2585            0 :         op0 = convert_to_mode (mode, op0, 0);
    2586              : 
    2587            2 :       create_output_operand (&ops[0], target, TYPE_MODE (TREE_TYPE (exp)));
    2588            2 :       if (maybe_legitimize_operands (icode, 0, 1, ops)
    2589            2 :           && maybe_emit_unop_insn (icode, ops[0].value, op0, UNKNOWN))
    2590            1 :         return ops[0].value;
    2591              : 
    2592            1 :       delete_insns_since (last);
    2593            1 :       CALL_EXPR_ARG (exp, 0) = orig_arg;
    2594              :     }
    2595              : 
    2596              :   return NULL_RTX;
    2597              : }
    2598              : 
    2599              : /* Expand a call to the builtin sincos math function.
    2600              :    Return NULL_RTX if a normal call should be emitted rather than expanding the
    2601              :    function in-line.  EXP is the expression that is a call to the builtin
    2602              :    function.  */
    2603              : 
    2604              : static rtx
    2605            3 : expand_builtin_sincos (tree exp)
    2606              : {
    2607            3 :   rtx op0, op1, op2, target1, target2;
    2608            3 :   machine_mode mode;
    2609            3 :   tree arg, sinp, cosp;
    2610            3 :   int result;
    2611            3 :   location_t loc = EXPR_LOCATION (exp);
    2612            3 :   tree alias_type, alias_off;
    2613              : 
    2614            3 :   if (!validate_arglist (exp, REAL_TYPE,
    2615              :                          POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
    2616              :     return NULL_RTX;
    2617              : 
    2618            3 :   arg = CALL_EXPR_ARG (exp, 0);
    2619            3 :   sinp = CALL_EXPR_ARG (exp, 1);
    2620            3 :   cosp = CALL_EXPR_ARG (exp, 2);
    2621              : 
    2622              :   /* Make a suitable register to place result in.  */
    2623            3 :   mode = TYPE_MODE (TREE_TYPE (arg));
    2624              : 
    2625              :   /* Check if sincos insn is available, otherwise emit the call.  */
    2626            3 :   if (optab_handler (sincos_optab, mode) == CODE_FOR_nothing)
    2627              :     return NULL_RTX;
    2628              : 
    2629            0 :   target1 = gen_reg_rtx (mode);
    2630            0 :   target2 = gen_reg_rtx (mode);
    2631              : 
    2632            0 :   op0 = expand_normal (arg);
    2633            0 :   alias_type = build_pointer_type_for_mode (TREE_TYPE (arg), ptr_mode, true);
    2634            0 :   alias_off = build_int_cst (alias_type, 0);
    2635            0 :   op1 = expand_normal (fold_build2_loc (loc, MEM_REF, TREE_TYPE (arg),
    2636              :                                         sinp, alias_off));
    2637            0 :   op2 = expand_normal (fold_build2_loc (loc, MEM_REF, TREE_TYPE (arg),
    2638              :                                         cosp, alias_off));
    2639              : 
    2640              :   /* Compute into target1 and target2.
    2641              :      Set TARGET to wherever the result comes back.  */
    2642            0 :   result = expand_twoval_unop (sincos_optab, op0, target2, target1, 0);
    2643            0 :   gcc_assert (result);
    2644              : 
    2645              :   /* Move target1 and target2 to the memory locations indicated
    2646              :      by op1 and op2.  */
    2647            0 :   emit_move_insn (op1, target1);
    2648            0 :   emit_move_insn (op2, target2);
    2649              : 
    2650            0 :   return const0_rtx;
    2651              : }
    2652              : 
    2653              : /* Expand call EXP to the fegetround builtin (from C99 fenv.h), returning the
    2654              :    result and setting it in TARGET.  Otherwise return NULL_RTX on failure.  */
    2655              : static rtx
    2656           60 : expand_builtin_fegetround (tree exp, rtx target, machine_mode target_mode)
    2657              : {
    2658           60 :   if (!validate_arglist (exp, VOID_TYPE))
    2659              :     return NULL_RTX;
    2660              : 
    2661           60 :   insn_code icode = direct_optab_handler (fegetround_optab, SImode);
    2662           60 :   if (icode == CODE_FOR_nothing)
    2663              :     return NULL_RTX;
    2664              : 
    2665            0 :   if (target == 0
    2666            0 :       || GET_MODE (target) != target_mode
    2667            0 :       || !(*insn_data[icode].operand[0].predicate) (target, target_mode))
    2668            0 :     target = gen_reg_rtx (target_mode);
    2669              : 
    2670            0 :   rtx pat = GEN_FCN (icode) (target);
    2671            0 :   if (!pat)
    2672              :     return NULL_RTX;
    2673            0 :   emit_insn (pat);
    2674              : 
    2675            0 :   return target;
    2676              : }
    2677              : 
    2678              : /* Expand call EXP to either feclearexcept or feraiseexcept builtins (from C99
    2679              :    fenv.h), returning the result and setting it in TARGET.  Otherwise return
    2680              :    NULL_RTX on failure.  */
    2681              : static rtx
    2682         2052 : expand_builtin_feclear_feraise_except (tree exp, rtx target,
    2683              :                                        machine_mode target_mode, optab op_optab)
    2684              : {
    2685         2052 :   if (!validate_arglist (exp, INTEGER_TYPE, VOID_TYPE))
    2686              :     return NULL_RTX;
    2687         2052 :   rtx op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
    2688              : 
    2689         2052 :   insn_code icode = direct_optab_handler (op_optab, SImode);
    2690         2052 :   if (icode == CODE_FOR_nothing)
    2691              :     return NULL_RTX;
    2692              : 
    2693            0 :   if (!(*insn_data[icode].operand[1].predicate) (op0, GET_MODE (op0)))
    2694              :     return NULL_RTX;
    2695              : 
    2696            0 :   if (target == 0
    2697            0 :       || GET_MODE (target) != target_mode
    2698            0 :       || !(*insn_data[icode].operand[0].predicate) (target, target_mode))
    2699            0 :     target = gen_reg_rtx (target_mode);
    2700              : 
    2701            0 :   rtx pat = GEN_FCN (icode) (target, op0);
    2702            0 :   if (!pat)
    2703              :     return NULL_RTX;
    2704            0 :   emit_insn (pat);
    2705              : 
    2706            0 :   return target;
    2707              : }
    2708              : 
    2709              : /* Expand a call to the internal cexpi builtin to the sincos math function.
    2710              :    EXP is the expression that is a call to the builtin function; if convenient,
    2711              :    the result should be placed in TARGET.  */
    2712              : 
    2713              : static rtx
    2714          173 : expand_builtin_cexpi (tree exp, rtx target)
    2715              : {
    2716          173 :   tree fndecl = get_callee_fndecl (exp);
    2717          173 :   tree arg, type;
    2718          173 :   machine_mode mode;
    2719          173 :   rtx op0, op1, op2;
    2720          173 :   location_t loc = EXPR_LOCATION (exp);
    2721              : 
    2722          173 :   if (!validate_arglist (exp, REAL_TYPE, VOID_TYPE))
    2723              :     return NULL_RTX;
    2724              : 
    2725          173 :   arg = CALL_EXPR_ARG (exp, 0);
    2726          173 :   type = TREE_TYPE (arg);
    2727          173 :   mode = TYPE_MODE (TREE_TYPE (arg));
    2728              : 
    2729              :   /* Try expanding via a sincos optab, fall back to emitting a libcall
    2730              :      to sincos or cexp.  We are sure we have sincos or cexp because cexpi
    2731              :      is only generated from sincos, cexp or if we have either of them.  */
    2732          173 :   if (optab_handler (sincos_optab, mode) != CODE_FOR_nothing)
    2733              :     {
    2734            6 :       op1 = gen_reg_rtx (mode);
    2735            6 :       op2 = gen_reg_rtx (mode);
    2736              : 
    2737            6 :       op0 = expand_expr (arg, NULL_RTX, VOIDmode, EXPAND_NORMAL);
    2738              : 
    2739              :       /* Compute into op1 and op2.  */
    2740            6 :       expand_twoval_unop (sincos_optab, op0, op2, op1, 0);
    2741              :     }
    2742          167 :   else if (targetm.libc_has_function (function_sincos, type))
    2743              :     {
    2744          167 :       tree call, fn = NULL_TREE;
    2745          167 :       tree top1, top2;
    2746          167 :       rtx op1a, op2a;
    2747              : 
    2748          167 :       if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIF)
    2749           54 :         fn = builtin_decl_explicit (BUILT_IN_SINCOSF);
    2750          113 :       else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPI)
    2751           89 :         fn = builtin_decl_explicit (BUILT_IN_SINCOS);
    2752           24 :       else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIL)
    2753           24 :         fn = builtin_decl_explicit (BUILT_IN_SINCOSL);
    2754              :       else
    2755            0 :         gcc_unreachable ();
    2756              : 
    2757          167 :       op1 = assign_temp (TREE_TYPE (arg), 1, 1);
    2758          167 :       op2 = assign_temp (TREE_TYPE (arg), 1, 1);
    2759          167 :       op1a = copy_addr_to_reg (XEXP (op1, 0));
    2760          167 :       op2a = copy_addr_to_reg (XEXP (op2, 0));
    2761          167 :       top1 = make_tree (build_pointer_type (TREE_TYPE (arg)), op1a);
    2762          167 :       top2 = make_tree (build_pointer_type (TREE_TYPE (arg)), op2a);
    2763              : 
    2764              :       /* Make sure not to fold the sincos call again.  */
    2765          167 :       call = build1 (ADDR_EXPR, build_pointer_type (TREE_TYPE (fn)), fn);
    2766          167 :       expand_normal (build_call_nary (TREE_TYPE (TREE_TYPE (fn)),
    2767              :                                       call, 3, arg, top1, top2));
    2768              :     }
    2769              :   else
    2770              :     {
    2771            0 :       tree call, fn = NULL_TREE, narg;
    2772            0 :       tree ctype = build_complex_type (type);
    2773              : 
    2774            0 :       if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIF)
    2775            0 :         fn = builtin_decl_explicit (BUILT_IN_CEXPF);
    2776            0 :       else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPI)
    2777            0 :         fn = builtin_decl_explicit (BUILT_IN_CEXP);
    2778            0 :       else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIL)
    2779            0 :         fn = builtin_decl_explicit (BUILT_IN_CEXPL);
    2780              :       else
    2781            0 :         gcc_unreachable ();
    2782              : 
    2783              :       /* If we don't have a decl for cexp create one.  This is the
    2784              :          friendliest fallback if the user calls __builtin_cexpi
    2785              :          without full target C99 function support.  */
    2786            0 :       if (fn == NULL_TREE)
    2787              :         {
    2788            0 :           tree fntype;
    2789            0 :           const char *name = NULL;
    2790              : 
    2791            0 :           if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIF)
    2792              :             name = "cexpf";
    2793            0 :           else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPI)
    2794              :             name = "cexp";
    2795            0 :           else if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_CEXPIL)
    2796            0 :             name = "cexpl";
    2797              : 
    2798            0 :           fntype = build_function_type_list (ctype, ctype, NULL_TREE);
    2799            0 :           fn = build_fn_decl (name, fntype);
    2800              :         }
    2801              : 
    2802            0 :       narg = fold_build2_loc (loc, COMPLEX_EXPR, ctype,
    2803              :                           build_real (type, dconst0), arg);
    2804              : 
    2805              :       /* Make sure not to fold the cexp call again.  */
    2806            0 :       call = build1 (ADDR_EXPR, build_pointer_type (TREE_TYPE (fn)), fn);
    2807            0 :       return expand_expr (build_call_nary (ctype, call, 1, narg),
    2808            0 :                           target, VOIDmode, EXPAND_NORMAL);
    2809              :     }
    2810              : 
    2811              :   /* Now build the proper return type.  */
    2812          346 :   return expand_expr (build2 (COMPLEX_EXPR, build_complex_type (type),
    2813          173 :                               make_tree (TREE_TYPE (arg), op2),
    2814          173 :                               make_tree (TREE_TYPE (arg), op1)),
    2815          173 :                       target, VOIDmode, EXPAND_NORMAL);
    2816              : }
    2817              : 
    2818              : /* Conveniently construct a function call expression.  FNDECL names the
    2819              :    function to be called, N is the number of arguments, and the "..."
    2820              :    parameters are the argument expressions.  Unlike build_call_exr
    2821              :    this doesn't fold the call, hence it will always return a CALL_EXPR.  */
    2822              : 
    2823              : static tree
    2824       136189 : build_call_nofold_loc (location_t loc, tree fndecl, int n, ...)
    2825              : {
    2826       136189 :   va_list ap;
    2827       136189 :   tree fntype = TREE_TYPE (fndecl);
    2828       136189 :   tree fn = build1 (ADDR_EXPR, build_pointer_type (fntype), fndecl);
    2829              : 
    2830       136189 :   va_start (ap, n);
    2831       136189 :   fn = build_call_valist (TREE_TYPE (fntype), fn, n, ap);
    2832       136189 :   va_end (ap);
    2833       136189 :   SET_EXPR_LOCATION (fn, loc);
    2834       136189 :   return fn;
    2835              : }
    2836              : 
    2837              : /* Expand the __builtin_issignaling builtin.  This needs to handle
    2838              :    all floating point formats that do support NaNs (for those that
    2839              :    don't it just sets target to 0).  */
    2840              : 
    2841              : static rtx
    2842          784 : expand_builtin_issignaling (tree exp, rtx target)
    2843              : {
    2844          784 :   if (!validate_arglist (exp, REAL_TYPE, VOID_TYPE))
    2845              :     return NULL_RTX;
    2846              : 
    2847          784 :   tree arg = CALL_EXPR_ARG (exp, 0);
    2848          784 :   scalar_float_mode fmode = SCALAR_FLOAT_TYPE_MODE (TREE_TYPE (arg));
    2849          784 :   const struct real_format *fmt = REAL_MODE_FORMAT (fmode);
    2850              : 
    2851              :   /* Expand the argument yielding a RTX expression. */
    2852          784 :   rtx temp = expand_normal (arg);
    2853              : 
    2854              :   /* If mode doesn't support NaN, always return 0.
    2855              :      Don't use !HONOR_SNANS (fmode) here, so there is some possibility of
    2856              :      __builtin_issignaling working without -fsignaling-nans.  Especially
    2857              :      when -fno-signaling-nans is the default.
    2858              :      On the other side, MODE_HAS_NANS (fmode) is unnecessary, with
    2859              :      -ffinite-math-only even __builtin_isnan or __builtin_fpclassify
    2860              :      fold to 0 or non-NaN/Inf classification.  */
    2861          784 :   if (!HONOR_NANS (fmode))
    2862              :     {
    2863            0 :       emit_move_insn (target, const0_rtx);
    2864            0 :       return target;
    2865              :     }
    2866              : 
    2867              :   /* Check if the back end provides an insn that handles issignaling for the
    2868              :      argument's mode. */
    2869          784 :   enum insn_code icode = optab_handler (issignaling_optab, fmode);
    2870          784 :   if (icode != CODE_FOR_nothing)
    2871              :     {
    2872          142 :       rtx_insn *last = get_last_insn ();
    2873          142 :       rtx this_target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
    2874          142 :       if (maybe_emit_unop_insn (icode, this_target, temp, UNKNOWN))
    2875              :         return this_target;
    2876            0 :       delete_insns_since (last);
    2877              :     }
    2878              : 
    2879          642 :   if (DECIMAL_FLOAT_MODE_P (fmode))
    2880              :     {
    2881           63 :       scalar_int_mode imode;
    2882           63 :       rtx hi;
    2883           63 :       switch (fmt->ieee_bits)
    2884              :         {
    2885           42 :         case 32:
    2886           42 :         case 64:
    2887           42 :           imode = int_mode_for_mode (fmode).require ();
    2888           42 :           temp = gen_lowpart (imode, temp);
    2889           42 :           break;
    2890           21 :         case 128:
    2891           21 :           imode = int_mode_for_size (64, 1).require ();
    2892           21 :           hi = NULL_RTX;
    2893              :           /* For decimal128, TImode support isn't always there and even when
    2894              :              it is, working on the DImode high part is usually better.  */
    2895           21 :           if (!MEM_P (temp))
    2896              :             {
    2897            6 :               if (rtx t = force_highpart_subreg (imode, temp, fmode))
    2898              :                 hi = t;
    2899              :               else
    2900              :                 {
    2901            0 :                   scalar_int_mode imode2;
    2902            0 :                   if (int_mode_for_mode (fmode).exists (&imode2))
    2903              :                     {
    2904            0 :                       rtx temp2 = gen_lowpart (imode2, temp);
    2905            0 :                       if (rtx t = force_highpart_subreg (imode, temp2, imode2))
    2906              :                         hi = t;
    2907              :                     }
    2908              :                 }
    2909            0 :               if (!hi)
    2910              :                 {
    2911            0 :                   rtx mem = assign_stack_temp (fmode, GET_MODE_SIZE (fmode));
    2912            0 :                   emit_move_insn (mem, temp);
    2913            0 :                   temp = mem;
    2914              :                 }
    2915              :             }
    2916            0 :           if (!hi)
    2917              :             {
    2918           15 :               poly_int64 offset
    2919           15 :                 = subreg_highpart_offset (imode, GET_MODE (temp));
    2920           15 :               hi = adjust_address (temp, imode, offset);
    2921              :             }
    2922              :           temp = hi;
    2923              :           break;
    2924            0 :         default:
    2925            0 :           gcc_unreachable ();
    2926              :         }
    2927              :       /* In all of decimal{32,64,128}, there is MSB sign bit and sNaN
    2928              :          have 6 bits below it all set.  */
    2929           63 :       rtx val
    2930           63 :         = GEN_INT (HOST_WIDE_INT_C (0x3f) << (GET_MODE_BITSIZE (imode) - 7));
    2931           63 :       temp = expand_binop (imode, and_optab, temp, val,
    2932              :                            NULL_RTX, 1, OPTAB_LIB_WIDEN);
    2933           63 :       temp = emit_store_flag_force (target, EQ, temp, val, imode, 1, 1);
    2934           63 :       return temp;
    2935              :     }
    2936              : 
    2937              :   /* Only PDP11 has these defined differently but doesn't support NaNs.  */
    2938          579 :   gcc_assert (FLOAT_WORDS_BIG_ENDIAN == WORDS_BIG_ENDIAN);
    2939          579 :   gcc_assert (fmt->signbit_ro > 0 && fmt->b == 2);
    2940         4053 :   gcc_assert (MODE_COMPOSITE_P (fmode)
    2941              :               || (fmt->pnan == fmt->p
    2942              :                   && fmt->signbit_ro == fmt->signbit_rw));
    2943              : 
    2944          579 :   switch (fmt->p)
    2945              :     {
    2946            0 :     case 106: /* IBM double double  */
    2947              :       /* For IBM double double, recurse on the most significant double.  */
    2948            0 :       gcc_assert (MODE_COMPOSITE_P (fmode));
    2949            0 :       temp = convert_modes (DFmode, fmode, temp, 0);
    2950            0 :       fmode = DFmode;
    2951            0 :       fmt = REAL_MODE_FORMAT (DFmode);
    2952              :       /* FALLTHRU */
    2953          479 :     case 8: /* bfloat */
    2954          479 :     case 11: /* IEEE half */
    2955          479 :     case 24: /* IEEE single */
    2956          479 :     case 53: /* IEEE double or Intel extended with rounding to double */
    2957          479 :       if (fmt->p == 53 && fmt->signbit_ro == 79)
    2958            0 :         goto extended;
    2959          479 :       {
    2960          479 :         scalar_int_mode imode = int_mode_for_mode (fmode).require ();
    2961          479 :         temp = gen_lowpart (imode, temp);
    2962          479 :         rtx val = GEN_INT ((HOST_WIDE_INT_M1U << (fmt->p - 2))
    2963              :                            & ~(HOST_WIDE_INT_M1U << fmt->signbit_ro));
    2964          479 :         if (fmt->qnan_msb_set)
    2965              :           {
    2966          479 :             rtx mask = GEN_INT (~(HOST_WIDE_INT_M1U << fmt->signbit_ro));
    2967          479 :             rtx bit = GEN_INT (HOST_WIDE_INT_1U << (fmt->p - 2));
    2968              :             /* For non-MIPS/PA IEEE single/double/half or bfloat, expand to:
    2969              :                ((temp ^ bit) & mask) > val.  */
    2970          479 :             temp = expand_binop (imode, xor_optab, temp, bit,
    2971              :                                  NULL_RTX, 1, OPTAB_LIB_WIDEN);
    2972          479 :             temp = expand_binop (imode, and_optab, temp, mask,
    2973              :                                  NULL_RTX, 1, OPTAB_LIB_WIDEN);
    2974          479 :             temp = emit_store_flag_force (target, GTU, temp, val, imode,
    2975              :                                           1, 1);
    2976              :           }
    2977              :         else
    2978              :           {
    2979              :             /* For MIPS/PA IEEE single/double, expand to:
    2980              :                (temp & val) == val.  */
    2981            0 :             temp = expand_binop (imode, and_optab, temp, val,
    2982              :                                  NULL_RTX, 1, OPTAB_LIB_WIDEN);
    2983            0 :             temp = emit_store_flag_force (target, EQ, temp, val, imode,
    2984              :                                           1, 1);
    2985              :           }
    2986              :       }
    2987              :       break;
    2988          100 :     case 113: /* IEEE quad */
    2989          100 :       {
    2990          100 :         rtx hi = NULL_RTX, lo = NULL_RTX;
    2991          100 :         scalar_int_mode imode = int_mode_for_size (64, 1).require ();
    2992              :         /* For IEEE quad, TImode support isn't always there and even when
    2993              :            it is, working on DImode parts is usually better.  */
    2994          100 :         if (!MEM_P (temp))
    2995              :           {
    2996           85 :             hi = force_highpart_subreg (imode, temp, fmode);
    2997           85 :             lo = force_lowpart_subreg (imode, temp, fmode);
    2998           85 :             if (!hi || !lo)
    2999              :               {
    3000            0 :                 scalar_int_mode imode2;
    3001            0 :                 if (int_mode_for_mode (fmode).exists (&imode2))
    3002              :                   {
    3003            0 :                     rtx temp2 = gen_lowpart (imode2, temp);
    3004            0 :                     hi = force_highpart_subreg (imode, temp2, imode2);
    3005            0 :                     lo = force_lowpart_subreg (imode, temp2, imode2);
    3006              :                   }
    3007              :               }
    3008           85 :             if (!hi || !lo)
    3009              :               {
    3010            0 :                 rtx mem = assign_stack_temp (fmode, GET_MODE_SIZE (fmode));
    3011            0 :                 emit_move_insn (mem, temp);
    3012            0 :                 temp = mem;
    3013              :               }
    3014              :           }
    3015          100 :         if (!hi || !lo)
    3016              :           {
    3017           15 :             poly_int64 offset
    3018           15 :               = subreg_highpart_offset (imode, GET_MODE (temp));
    3019           15 :             hi = adjust_address (temp, imode, offset);
    3020           15 :             offset = subreg_lowpart_offset (imode, GET_MODE (temp));
    3021           15 :             lo = adjust_address (temp, imode, offset);
    3022              :           }
    3023          100 :         rtx val = GEN_INT ((HOST_WIDE_INT_M1U << (fmt->p - 2 - 64))
    3024              :                            & ~(HOST_WIDE_INT_M1U << (fmt->signbit_ro - 64)));
    3025          100 :         if (fmt->qnan_msb_set)
    3026              :           {
    3027          100 :             rtx mask = GEN_INT (~(HOST_WIDE_INT_M1U << (fmt->signbit_ro
    3028              :                                                         - 64)));
    3029          100 :             rtx bit = GEN_INT (HOST_WIDE_INT_1U << (fmt->p - 2 - 64));
    3030              :             /* For non-MIPS/PA IEEE quad, expand to:
    3031              :                (((hi ^ bit) | ((lo | -lo) >> 63)) & mask) > val.  */
    3032          100 :             rtx nlo = expand_unop (imode, neg_optab, lo, NULL_RTX, 0);
    3033          100 :             lo = expand_binop (imode, ior_optab, lo, nlo,
    3034              :                                NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3035          100 :             lo = expand_shift (RSHIFT_EXPR, imode, lo, 63, NULL_RTX, 1);
    3036          100 :             temp = expand_binop (imode, xor_optab, hi, bit,
    3037              :                                  NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3038          100 :             temp = expand_binop (imode, ior_optab, temp, lo,
    3039              :                                  NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3040          100 :             temp = expand_binop (imode, and_optab, temp, mask,
    3041              :                                  NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3042          100 :             temp = emit_store_flag_force (target, GTU, temp, val, imode,
    3043              :                                           1, 1);
    3044              :           }
    3045              :         else
    3046              :           {
    3047              :             /* For MIPS/PA IEEE quad, expand to:
    3048              :                (hi & val) == val.  */
    3049            0 :             temp = expand_binop (imode, and_optab, hi, val,
    3050              :                                  NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3051            0 :             temp = emit_store_flag_force (target, EQ, temp, val, imode,
    3052              :                                           1, 1);
    3053              :           }
    3054              :       }
    3055              :       break;
    3056            0 :     case 64: /* Intel or Motorola extended */
    3057            0 :     extended:
    3058            0 :       {
    3059            0 :         rtx ex, hi, lo;
    3060            0 :         scalar_int_mode imode = int_mode_for_size (32, 1).require ();
    3061            0 :         scalar_int_mode iemode = int_mode_for_size (16, 1).require ();
    3062            0 :         if (!MEM_P (temp))
    3063              :           {
    3064            0 :             rtx mem = assign_stack_temp (fmode, GET_MODE_SIZE (fmode));
    3065            0 :             emit_move_insn (mem, temp);
    3066            0 :             temp = mem;
    3067              :           }
    3068            0 :         if (fmt->signbit_ro == 95)
    3069              :           {
    3070              :             /* Motorola, always big endian, with 16-bit gap in between
    3071              :                16-bit sign+exponent and 64-bit mantissa.  */
    3072            0 :             ex = adjust_address (temp, iemode, 0);
    3073            0 :             hi = adjust_address (temp, imode, 4);
    3074            0 :             lo = adjust_address (temp, imode, 8);
    3075              :           }
    3076            0 :         else if (!WORDS_BIG_ENDIAN)
    3077              :           {
    3078              :             /* Intel little endian, 64-bit mantissa followed by 16-bit
    3079              :                sign+exponent and then either 16 or 48 bits of gap.  */
    3080            0 :             ex = adjust_address (temp, iemode, 8);
    3081            0 :             hi = adjust_address (temp, imode, 4);
    3082            0 :             lo = adjust_address (temp, imode, 0);
    3083              :           }
    3084              :         else
    3085              :           {
    3086              :             /* Big endian Itanium.  */
    3087              :             ex = adjust_address (temp, iemode, 0);
    3088              :             hi = adjust_address (temp, imode, 2);
    3089              :             lo = adjust_address (temp, imode, 6);
    3090              :           }
    3091            0 :         rtx val = GEN_INT (HOST_WIDE_INT_M1U << 30);
    3092            0 :         gcc_assert (fmt->qnan_msb_set);
    3093            0 :         rtx mask = GEN_INT (0x7fff);
    3094            0 :         rtx bit = GEN_INT (HOST_WIDE_INT_1U << 30);
    3095              :         /* For Intel/Motorola extended format, expand to:
    3096              :            (ex & mask) == mask && ((hi ^ bit) | ((lo | -lo) >> 31)) > val.  */
    3097            0 :         rtx nlo = expand_unop (imode, neg_optab, lo, NULL_RTX, 0);
    3098            0 :         lo = expand_binop (imode, ior_optab, lo, nlo,
    3099              :                            NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3100            0 :         lo = expand_shift (RSHIFT_EXPR, imode, lo, 31, NULL_RTX, 1);
    3101            0 :         temp = expand_binop (imode, xor_optab, hi, bit,
    3102              :                              NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3103            0 :         temp = expand_binop (imode, ior_optab, temp, lo,
    3104              :                              NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3105            0 :         temp = emit_store_flag_force (target, GTU, temp, val, imode, 1, 1);
    3106            0 :         ex = expand_binop (iemode, and_optab, ex, mask,
    3107              :                            NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3108            0 :         ex = emit_store_flag_force (gen_reg_rtx (GET_MODE (temp)), EQ,
    3109              :                                     ex, mask, iemode, 1, 1);
    3110            0 :         temp = expand_binop (GET_MODE (temp), and_optab, temp, ex,
    3111              :                              NULL_RTX, 1, OPTAB_LIB_WIDEN);
    3112              :       }
    3113            0 :       break;
    3114            0 :     default:
    3115            0 :       gcc_unreachable ();
    3116              :     }
    3117              : 
    3118              :   return temp;
    3119              : }
    3120              : 
    3121              : /* Expand a call to one of the builtin rounding functions gcc defines
    3122              :    as an extension (lfloor and lceil).  As these are gcc extensions we
    3123              :    do not need to worry about setting errno to EDOM.
    3124              :    If expanding via optab fails, lower expression to (int)(floor(x)).
    3125              :    EXP is the expression that is a call to the builtin function;
    3126              :    if convenient, the result should be placed in TARGET.  */
    3127              : 
    3128              : static rtx
    3129          223 : expand_builtin_int_roundingfn (tree exp, rtx target)
    3130              : {
    3131          223 :   convert_optab builtin_optab;
    3132          223 :   rtx op0, tmp;
    3133          223 :   rtx_insn *insns;
    3134          223 :   tree fndecl = get_callee_fndecl (exp);
    3135          223 :   enum built_in_function fallback_fn;
    3136          223 :   tree fallback_fndecl;
    3137          223 :   machine_mode mode;
    3138          223 :   tree arg;
    3139              : 
    3140          223 :   if (!validate_arglist (exp, REAL_TYPE, VOID_TYPE))
    3141              :     return NULL_RTX;
    3142              : 
    3143          223 :   arg = CALL_EXPR_ARG (exp, 0);
    3144              : 
    3145          223 :   switch (DECL_FUNCTION_CODE (fndecl))
    3146              :     {
    3147              :     CASE_FLT_FN (BUILT_IN_ICEIL):
    3148              :     CASE_FLT_FN (BUILT_IN_LCEIL):
    3149              :     CASE_FLT_FN (BUILT_IN_LLCEIL):
    3150              :       builtin_optab = lceil_optab;
    3151              :       fallback_fn = BUILT_IN_CEIL;
    3152              :       break;
    3153              : 
    3154          112 :     CASE_FLT_FN (BUILT_IN_IFLOOR):
    3155          112 :     CASE_FLT_FN (BUILT_IN_LFLOOR):
    3156          112 :     CASE_FLT_FN (BUILT_IN_LLFLOOR):
    3157          112 :       builtin_optab = lfloor_optab;
    3158          112 :       fallback_fn = BUILT_IN_FLOOR;
    3159          112 :       break;
    3160              : 
    3161            0 :     default:
    3162            0 :       gcc_unreachable ();
    3163              :     }
    3164              : 
    3165              :   /* Make a suitable register to place result in.  */
    3166          223 :   mode = TYPE_MODE (TREE_TYPE (exp));
    3167              : 
    3168          223 :   target = gen_reg_rtx (mode);
    3169              : 
    3170              :   /* Wrap the computation of the argument in a SAVE_EXPR, as we may
    3171              :      need to expand the argument again.  This way, we will not perform
    3172              :      side-effects more the once.  */
    3173          223 :   CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (arg);
    3174              : 
    3175          223 :   op0 = expand_expr (arg, NULL, VOIDmode, EXPAND_NORMAL);
    3176              : 
    3177          223 :   start_sequence ();
    3178              : 
    3179              :   /* Compute into TARGET.  */
    3180          223 :   if (expand_sfix_optab (target, op0, builtin_optab))
    3181              :     {
    3182              :       /* Output the entire sequence.  */
    3183           32 :       insns = end_sequence ();
    3184           32 :       emit_insn (insns);
    3185           32 :       return target;
    3186              :     }
    3187              : 
    3188              :   /* If we were unable to expand via the builtin, stop the sequence
    3189              :      (without outputting the insns).  */
    3190          191 :   end_sequence ();
    3191              : 
    3192              :   /* Fall back to floating point rounding optab.  */
    3193          191 :   fallback_fndecl = mathfn_built_in (TREE_TYPE (arg), fallback_fn);
    3194              : 
    3195              :   /* For non-C99 targets we may end up without a fallback fndecl here
    3196              :      if the user called __builtin_lfloor directly.  In this case emit
    3197              :      a call to the floor/ceil variants nevertheless.  This should result
    3198              :      in the best user experience for not full C99 targets.  */
    3199          191 :   if (fallback_fndecl == NULL_TREE)
    3200              :     {
    3201            0 :       tree fntype;
    3202            0 :       const char *name = NULL;
    3203              : 
    3204            0 :       switch (DECL_FUNCTION_CODE (fndecl))
    3205              :         {
    3206              :         case BUILT_IN_ICEIL:
    3207              :         case BUILT_IN_LCEIL:
    3208              :         case BUILT_IN_LLCEIL:
    3209              :           name = "ceil";
    3210              :           break;
    3211            0 :         case BUILT_IN_ICEILF:
    3212            0 :         case BUILT_IN_LCEILF:
    3213            0 :         case BUILT_IN_LLCEILF:
    3214            0 :           name = "ceilf";
    3215            0 :           break;
    3216            0 :         case BUILT_IN_ICEILL:
    3217            0 :         case BUILT_IN_LCEILL:
    3218            0 :         case BUILT_IN_LLCEILL:
    3219            0 :           name = "ceill";
    3220            0 :           break;
    3221            0 :         case BUILT_IN_IFLOOR:
    3222            0 :         case BUILT_IN_LFLOOR:
    3223            0 :         case BUILT_IN_LLFLOOR:
    3224            0 :           name = "floor";
    3225            0 :           break;
    3226            0 :         case BUILT_IN_IFLOORF:
    3227            0 :         case BUILT_IN_LFLOORF:
    3228            0 :         case BUILT_IN_LLFLOORF:
    3229            0 :           name = "floorf";
    3230            0 :           break;
    3231            0 :         case BUILT_IN_IFLOORL:
    3232            0 :         case BUILT_IN_LFLOORL:
    3233            0 :         case BUILT_IN_LLFLOORL:
    3234            0 :           name = "floorl";
    3235            0 :           break;
    3236            0 :         default:
    3237            0 :           gcc_unreachable ();
    3238              :         }
    3239              : 
    3240            0 :       fntype = build_function_type_list (TREE_TYPE (arg),
    3241            0 :                                          TREE_TYPE (arg), NULL_TREE);
    3242            0 :       fallback_fndecl = build_fn_decl (name, fntype);
    3243              :     }
    3244              : 
    3245          191 :   exp = build_call_nofold_loc (EXPR_LOCATION (exp), fallback_fndecl, 1, arg);
    3246              : 
    3247          191 :   tmp = expand_normal (exp);
    3248          191 :   tmp = maybe_emit_group_store (tmp, TREE_TYPE (exp));
    3249              : 
    3250              :   /* Truncate the result of floating point optab to integer
    3251              :      via expand_fix ().  */
    3252          191 :   target = gen_reg_rtx (mode);
    3253          191 :   expand_fix (target, tmp, 0);
    3254              : 
    3255          191 :   return target;
    3256              : }
    3257              : 
    3258              : /* Expand a call to one of the builtin math functions doing integer
    3259              :    conversion (lrint).
    3260              :    Return 0 if a normal call should be emitted rather than expanding the
    3261              :    function in-line.  EXP is the expression that is a call to the builtin
    3262              :    function; if convenient, the result should be placed in TARGET.  */
    3263              : 
    3264              : static rtx
    3265          581 : expand_builtin_int_roundingfn_2 (tree exp, rtx target)
    3266              : {
    3267          581 :   convert_optab builtin_optab;
    3268          581 :   rtx op0;
    3269          581 :   rtx_insn *insns;
    3270          581 :   tree fndecl = get_callee_fndecl (exp);
    3271          581 :   tree arg;
    3272          581 :   machine_mode mode;
    3273          581 :   enum built_in_function fallback_fn = BUILT_IN_NONE;
    3274              : 
    3275          581 :   if (!validate_arglist (exp, REAL_TYPE, VOID_TYPE))
    3276              :     return NULL_RTX;
    3277              : 
    3278          481 :   arg = CALL_EXPR_ARG (exp, 0);
    3279              : 
    3280          481 :   switch (DECL_FUNCTION_CODE (fndecl))
    3281              :     {
    3282            4 :     CASE_FLT_FN (BUILT_IN_IRINT):
    3283            4 :       fallback_fn = BUILT_IN_LRINT;
    3284              :       gcc_fallthrough ();
    3285              :     CASE_FLT_FN (BUILT_IN_LRINT):
    3286              :     CASE_FLT_FN (BUILT_IN_LLRINT):
    3287              :       builtin_optab = lrint_optab;
    3288              :       break;
    3289              : 
    3290          216 :     CASE_FLT_FN (BUILT_IN_IROUND):
    3291          216 :       fallback_fn = BUILT_IN_LROUND;
    3292              :       gcc_fallthrough ();
    3293              :     CASE_FLT_FN (BUILT_IN_LROUND):
    3294              :     CASE_FLT_FN (BUILT_IN_LLROUND):
    3295              :       builtin_optab = lround_optab;
    3296              :       break;
    3297              : 
    3298            0 :     default:
    3299            0 :       gcc_unreachable ();
    3300              :     }
    3301              : 
    3302              :   /* There's no easy way to detect the case we need to set EDOM.  */
    3303          481 :   if (flag_errno_math && fallback_fn == BUILT_IN_NONE)
    3304              :     return NULL_RTX;
    3305              : 
    3306              :   /* Make a suitable register to place result in.  */
    3307          250 :   mode = TYPE_MODE (TREE_TYPE (exp));
    3308              : 
    3309              :   /* There's no easy way to detect the case we need to set EDOM.  */
    3310          250 :   if (!flag_errno_math)
    3311              :     {
    3312          250 :       rtx result = gen_reg_rtx (mode);
    3313              : 
    3314              :       /* Wrap the computation of the argument in a SAVE_EXPR, as we may
    3315              :          need to expand the argument again.  This way, we will not perform
    3316              :          side-effects more the once.  */
    3317          250 :       CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (arg);
    3318              : 
    3319          250 :       op0 = expand_expr (arg, NULL, VOIDmode, EXPAND_NORMAL);
    3320              : 
    3321          250 :       start_sequence ();
    3322              : 
    3323          250 :       if (expand_sfix_optab (result, op0, builtin_optab))
    3324              :         {
    3325              :           /* Output the entire sequence.  */
    3326           20 :           insns = end_sequence ();
    3327           20 :           emit_insn (insns);
    3328           20 :           return result;
    3329              :         }
    3330              : 
    3331              :       /* If we were unable to expand via the builtin, stop the sequence
    3332              :          (without outputting the insns) and call to the library function
    3333              :          with the stabilized argument list.  */
    3334          230 :       end_sequence ();
    3335              :     }
    3336              : 
    3337          230 :   if (fallback_fn != BUILT_IN_NONE)
    3338              :     {
    3339              :       /* Fall back to rounding to long int.  Use implicit_p 0 - for non-C99
    3340              :          targets, (int) round (x) should never be transformed into
    3341              :          BUILT_IN_IROUND and if __builtin_iround is called directly, emit
    3342              :          a call to lround in the hope that the target provides at least some
    3343              :          C99 functions.  This should result in the best user experience for
    3344              :          not full C99 targets.
    3345              :          As scalar float conversions with same mode are useless in GIMPLE,
    3346              :          we can end up e.g. with _Float32 argument passed to float builtin,
    3347              :          try to get the type from the builtin prototype first.  */
    3348          216 :       tree fallback_fndecl = NULL_TREE;
    3349          216 :       if (tree argtypes = TYPE_ARG_TYPES (TREE_TYPE (fndecl)))
    3350          216 :         fallback_fndecl
    3351          216 :           = mathfn_built_in_1 (TREE_VALUE (argtypes),
    3352              :                                as_combined_fn (fallback_fn), 0);
    3353          216 :       if (fallback_fndecl == NULL_TREE)
    3354            0 :         fallback_fndecl
    3355            0 :           = mathfn_built_in_1 (TREE_TYPE (arg),
    3356              :                                as_combined_fn (fallback_fn), 0);
    3357            0 :       if (fallback_fndecl)
    3358              :         {
    3359          216 :           exp = build_call_nofold_loc (EXPR_LOCATION (exp),
    3360              :                                        fallback_fndecl, 1, arg);
    3361              : 
    3362          216 :           target = expand_call (exp, NULL_RTX, target == const0_rtx);
    3363          216 :           target = maybe_emit_group_store (target, TREE_TYPE (exp));
    3364          216 :           return convert_to_mode (mode, target, 0);
    3365              :         }
    3366              :     }
    3367              : 
    3368           14 :   return expand_call (exp, target, target == const0_rtx);
    3369              : }
    3370              : 
    3371              : /* Expand a call to the powi built-in mathematical function.  Return NULL_RTX if
    3372              :    a normal call should be emitted rather than expanding the function
    3373              :    in-line.  EXP is the expression that is a call to the builtin
    3374              :    function; if convenient, the result should be placed in TARGET.  */
    3375              : 
    3376              : static rtx
    3377          293 : expand_builtin_powi (tree exp, rtx target)
    3378              : {
    3379          293 :   tree arg0, arg1;
    3380          293 :   rtx op0, op1;
    3381          293 :   machine_mode mode;
    3382          293 :   machine_mode mode2;
    3383              : 
    3384          293 :   if (! validate_arglist (exp, REAL_TYPE, INTEGER_TYPE, VOID_TYPE))
    3385              :     return NULL_RTX;
    3386              : 
    3387          293 :   arg0 = CALL_EXPR_ARG (exp, 0);
    3388          293 :   arg1 = CALL_EXPR_ARG (exp, 1);
    3389          293 :   mode = TYPE_MODE (TREE_TYPE (exp));
    3390              : 
    3391              :   /* Emit a libcall to libgcc.  */
    3392              : 
    3393              :   /* Mode of the 2nd argument must match that of an int.  */
    3394          293 :   mode2 = int_mode_for_size (INT_TYPE_SIZE, 0).require ();
    3395              : 
    3396          293 :   if (target == NULL_RTX)
    3397            0 :     target = gen_reg_rtx (mode);
    3398              : 
    3399          293 :   op0 = expand_expr (arg0, NULL_RTX, mode, EXPAND_NORMAL);
    3400          293 :   if (GET_MODE (op0) != mode)
    3401            0 :     op0 = convert_to_mode (mode, op0, 0);
    3402          293 :   op1 = expand_expr (arg1, NULL_RTX, mode2, EXPAND_NORMAL);
    3403          293 :   if (GET_MODE (op1) != mode2)
    3404           52 :     op1 = convert_to_mode (mode2, op1, 0);
    3405              : 
    3406          293 :   target = emit_library_call_value (optab_libfunc (powi_optab, mode),
    3407              :                                     target, LCT_CONST, mode,
    3408              :                                     op0, mode, op1, mode2);
    3409              : 
    3410          293 :   return target;
    3411              : }
    3412              : 
    3413              : /* Expand expression EXP which is a call to the strlen builtin.  Return
    3414              :    NULL_RTX if we failed and the caller should emit a normal call, otherwise
    3415              :    try to get the result in TARGET, if convenient.  */
    3416              : 
    3417              : static rtx
    3418        13808 : expand_builtin_strlen (tree exp, rtx target,
    3419              :                        machine_mode target_mode)
    3420              : {
    3421        13808 :   if (!validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    3422              :     return NULL_RTX;
    3423              : 
    3424        13789 :   tree src = CALL_EXPR_ARG (exp, 0);
    3425              : 
    3426              :   /* If the length can be computed at compile-time, return it.  */
    3427        13789 :   if (tree len = c_strlen (src, 0))
    3428          141 :     return expand_expr (len, target, target_mode, EXPAND_NORMAL);
    3429              : 
    3430              :   /* If the length can be computed at compile-time and is constant
    3431              :      integer, but there are side-effects in src, evaluate
    3432              :      src for side-effects, then return len.
    3433              :      E.g. x = strlen (i++ ? "xfoo" + 1 : "bar");
    3434              :      can be optimized into: i++; x = 3;  */
    3435        13648 :   tree len = c_strlen (src, 1);
    3436        13648 :   if (len && TREE_CODE (len) == INTEGER_CST)
    3437              :     {
    3438            0 :       expand_expr (src, const0_rtx, VOIDmode, EXPAND_NORMAL);
    3439            0 :       return expand_expr (len, target, target_mode, EXPAND_NORMAL);
    3440              :     }
    3441              : 
    3442        13648 :   unsigned int align = get_pointer_alignment (src) / BITS_PER_UNIT;
    3443              : 
    3444              :   /* If SRC is not a pointer type, don't do this operation inline.  */
    3445        13648 :   if (align == 0)
    3446              :     return NULL_RTX;
    3447              : 
    3448              :   /* Bail out if we can't compute strlen in the right mode.  */
    3449              :   machine_mode insn_mode;
    3450              :   enum insn_code icode = CODE_FOR_nothing;
    3451        13648 :   FOR_EACH_MODE_FROM (insn_mode, target_mode)
    3452              :     {
    3453        13648 :       icode = optab_handler (strlen_optab, insn_mode);
    3454        13648 :       if (icode != CODE_FOR_nothing)
    3455              :         break;
    3456              :     }
    3457        13648 :   if (insn_mode == VOIDmode)
    3458              :     return NULL_RTX;
    3459              : 
    3460              :   /* Make a place to hold the source address.  We will not expand
    3461              :      the actual source until we are sure that the expansion will
    3462              :      not fail -- there are trees that cannot be expanded twice.  */
    3463        14263 :   rtx src_reg = gen_reg_rtx (Pmode);
    3464              : 
    3465              :   /* Mark the beginning of the strlen sequence so we can emit the
    3466              :      source operand later.  */
    3467        13648 :   rtx_insn *before_strlen = get_last_insn ();
    3468              : 
    3469        13648 :   class expand_operand ops[4];
    3470        13648 :   create_output_operand (&ops[0], target, insn_mode);
    3471        13648 :   create_fixed_operand (&ops[1], gen_rtx_MEM (BLKmode, src_reg));
    3472        13648 :   create_integer_operand (&ops[2], 0);
    3473        13648 :   create_integer_operand (&ops[3], align);
    3474        13648 :   if (!maybe_expand_insn (icode, 4, ops))
    3475              :     return NULL_RTX;
    3476              : 
    3477              :   /* Check to see if the argument was declared attribute nonstring
    3478              :      and if so, issue a warning since at this point it's not known
    3479              :      to be nul-terminated.  */
    3480           11 :   maybe_warn_nonstring_arg (get_callee_fndecl (exp), exp);
    3481              : 
    3482              :   /* Now that we are assured of success, expand the source.  */
    3483           11 :   start_sequence ();
    3484           15 :   rtx pat = expand_expr (src, src_reg, Pmode, EXPAND_NORMAL);
    3485           11 :   if (pat != src_reg)
    3486              :     {
    3487              : #ifdef POINTERS_EXTEND_UNSIGNED
    3488           15 :       if (GET_MODE (pat) != Pmode)
    3489            0 :         pat = convert_to_mode (Pmode, pat,
    3490              :                                POINTERS_EXTEND_UNSIGNED);
    3491              : #endif
    3492           11 :       emit_move_insn (src_reg, pat);
    3493              :     }
    3494           11 :   pat = end_sequence ();
    3495              : 
    3496           11 :   if (before_strlen)
    3497           11 :     emit_insn_after (pat, before_strlen);
    3498              :   else
    3499            0 :     emit_insn_before (pat, get_insns ());
    3500              : 
    3501              :   /* Return the value in the proper mode for this function.  */
    3502           11 :   if (GET_MODE (ops[0].value) == target_mode)
    3503              :     target = ops[0].value;
    3504            0 :   else if (target != 0)
    3505            0 :     convert_move (target, ops[0].value, 0);
    3506              :   else
    3507            0 :     target = convert_to_mode (target_mode, ops[0].value, 0);
    3508              : 
    3509              :   return target;
    3510              : }
    3511              : 
    3512              : /* Expand call EXP to the strnlen built-in, returning the result
    3513              :    and setting it in TARGET.  Otherwise return NULL_RTX on failure.  */
    3514              : 
    3515              : static rtx
    3516          580 : expand_builtin_strnlen (tree exp, rtx target, machine_mode target_mode)
    3517              : {
    3518          580 :   if (!validate_arglist (exp, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    3519              :     return NULL_RTX;
    3520              : 
    3521          571 :   tree src = CALL_EXPR_ARG (exp, 0);
    3522          571 :   tree bound = CALL_EXPR_ARG (exp, 1);
    3523              : 
    3524          571 :   if (!bound)
    3525              :     return NULL_RTX;
    3526              : 
    3527          571 :   location_t loc = UNKNOWN_LOCATION;
    3528          571 :   if (EXPR_HAS_LOCATION (exp))
    3529          571 :     loc = EXPR_LOCATION (exp);
    3530              : 
    3531              :   /* FIXME: Change c_strlen() to return sizetype instead of ssizetype
    3532              :      so these conversions aren't necessary.  */
    3533          571 :   c_strlen_data lendata = { };
    3534          571 :   tree len = c_strlen (src, 0, &lendata, 1);
    3535          571 :   if (len)
    3536           63 :     len = fold_convert_loc (loc, TREE_TYPE (bound), len);
    3537              : 
    3538          571 :   if (TREE_CODE (bound) == INTEGER_CST)
    3539              :     {
    3540          304 :       if (!len)
    3541              :         return NULL_RTX;
    3542              : 
    3543           23 :       len = fold_build2_loc (loc, MIN_EXPR, size_type_node, len, bound);
    3544           23 :       return expand_expr (len, target, target_mode, EXPAND_NORMAL);
    3545              :     }
    3546              : 
    3547          267 :   if (TREE_CODE (bound) != SSA_NAME)
    3548              :     return NULL_RTX;
    3549              : 
    3550          267 :   wide_int min, max;
    3551          267 :   int_range_max r;
    3552          534 :   get_range_query (cfun)->range_of_expr (r, bound,
    3553              :                                          currently_expanding_gimple_stmt);
    3554          267 :   if (r.varying_p () || r.undefined_p ())
    3555              :     return NULL_RTX;
    3556          152 :   min = r.lower_bound ();
    3557          152 :   max = r.upper_bound ();
    3558              : 
    3559          152 :   if (!len || TREE_CODE (len) != INTEGER_CST)
    3560              :     {
    3561          115 :       bool exact;
    3562          115 :       lendata.decl = unterminated_array (src, &len, &exact);
    3563          115 :       if (!lendata.decl)
    3564           95 :         return NULL_RTX;
    3565              :     }
    3566              : 
    3567           57 :   if (lendata.decl)
    3568              :     return NULL_RTX;
    3569              : 
    3570           37 :   if (wi::gtu_p (min, wi::to_wide (len)))
    3571            7 :     return expand_expr (len, target, target_mode, EXPAND_NORMAL);
    3572              : 
    3573           30 :   len = fold_build2_loc (loc, MIN_EXPR, TREE_TYPE (len), len, bound);
    3574           30 :   return expand_expr (len, target, target_mode, EXPAND_NORMAL);
    3575          267 : }
    3576              : 
    3577              : /* Callback routine for store_by_pieces.  Read GET_MODE_BITSIZE (MODE)
    3578              :    bytes from bytes at DATA + OFFSET and return it reinterpreted as
    3579              :    a target constant.  */
    3580              : 
    3581              : static rtx
    3582       162160 : builtin_memcpy_read_str (void *data, void *, HOST_WIDE_INT offset,
    3583              :                          fixed_size_mode mode)
    3584              : {
    3585              :   /* The REPresentation pointed to by DATA need not be a nul-terminated
    3586              :      string but the caller guarantees it's large enough for MODE.  */
    3587       162160 :   const char *rep = (const char *) data;
    3588              : 
    3589       162160 :   return c_readstr (rep + offset, mode, /*nul_terminated=*/false);
    3590              : }
    3591              : 
    3592              : /* LEN specify length of the block of memcpy/memset operation.
    3593              :    Figure out its range and put it into MIN_SIZE/MAX_SIZE.
    3594              :    In some cases we can make very likely guess on max size, then we
    3595              :    set it into PROBABLE_MAX_SIZE.  */
    3596              : 
    3597              : static void
    3598       187747 : determine_block_size (tree len, rtx &len_rtx,
    3599              :                       unsigned HOST_WIDE_INT *min_size,
    3600              :                       unsigned HOST_WIDE_INT *max_size,
    3601              :                       unsigned HOST_WIDE_INT *probable_max_size)
    3602              : {
    3603       187747 :   if (CONST_INT_P (len_rtx))
    3604              :     {
    3605        94680 :       *min_size = *max_size = *probable_max_size = UINTVAL (len_rtx);
    3606        94680 :       return;
    3607              :     }
    3608              :   else
    3609              :     {
    3610        93067 :       wide_int min, max;
    3611        93067 :       enum value_range_kind range_type = VR_UNDEFINED;
    3612              : 
    3613              :       /* Determine bounds from the type.  */
    3614        93067 :       if (tree_fits_uhwi_p (TYPE_MIN_VALUE (TREE_TYPE (len))))
    3615        93066 :         *min_size = tree_to_uhwi (TYPE_MIN_VALUE (TREE_TYPE (len)));
    3616              :       else
    3617              :         *min_size = 0;
    3618        93067 :       if (tree_fits_uhwi_p (TYPE_MAX_VALUE (TREE_TYPE (len))))
    3619        93067 :         *probable_max_size = *max_size
    3620        93067 :           = tree_to_uhwi (TYPE_MAX_VALUE (TREE_TYPE (len)));
    3621              :       else
    3622            0 :         *probable_max_size = *max_size = GET_MODE_MASK (GET_MODE (len_rtx));
    3623              : 
    3624        93067 :       if (TREE_CODE (len) == SSA_NAME)
    3625              :         {
    3626        93067 :           int_range_max r;
    3627        93067 :           tree tmin, tmax;
    3628        93067 :           gimple *cg = currently_expanding_gimple_stmt;
    3629       186134 :           get_range_query (cfun)->range_of_expr (r, len, cg);
    3630        93067 :           range_type = get_legacy_range (r, tmin, tmax);
    3631        93067 :           if (range_type != VR_UNDEFINED)
    3632              :             {
    3633        93065 :               min = wi::to_wide (tmin);
    3634        93065 :               max = wi::to_wide (tmax);
    3635              :             }
    3636        93067 :         }
    3637        93067 :       if (range_type == VR_RANGE)
    3638              :         {
    3639        66514 :           if (wi::fits_uhwi_p (min) && *min_size < min.to_uhwi ())
    3640        53680 :             *min_size = min.to_uhwi ();
    3641        66514 :           if (wi::fits_uhwi_p (max) && *max_size > max.to_uhwi ())
    3642        60508 :             *probable_max_size = *max_size = max.to_uhwi ();
    3643              :         }
    3644        26553 :       else if (range_type == VR_ANTI_RANGE)
    3645              :         {
    3646              :           /* Code like
    3647              : 
    3648              :              int n;
    3649              :              if (n < 100)
    3650              :                memcpy (a, b, n)
    3651              : 
    3652              :              Produce anti range allowing negative values of N.  We still
    3653              :              can use the information and make a guess that N is not negative.
    3654              :              */
    3655         3092 :           if (!wi::leu_p (max, 1 << 30) && wi::fits_uhwi_p (min))
    3656         2943 :             *probable_max_size = min.to_uhwi () - 1;
    3657              :         }
    3658        93067 :     }
    3659              : 
    3660        93067 :   if (*min_size == *max_size)
    3661           13 :     len_rtx = GEN_INT (*min_size);
    3662              :   else
    3663        93054 :     gcc_checking_assert (*max_size <=
    3664              :                          (unsigned HOST_WIDE_INT)
    3665              :                          GET_MODE_MASK (GET_MODE (len_rtx)));
    3666              : }
    3667              : 
    3668              : /* Expand a call EXP to the memcpy builtin.
    3669              :    Return NULL_RTX if we failed, the caller should emit a normal call,
    3670              :    otherwise try to get the result in TARGET, if convenient (and in
    3671              :    mode MODE if that's convenient).  */
    3672              : 
    3673              : static rtx
    3674       106053 : expand_builtin_memcpy (tree exp, rtx target)
    3675              : {
    3676       106053 :   if (!validate_arglist (exp,
    3677              :                          POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    3678              :     return NULL_RTX;
    3679              : 
    3680       106035 :   tree dest = CALL_EXPR_ARG (exp, 0);
    3681       106035 :   tree src = CALL_EXPR_ARG (exp, 1);
    3682       106035 :   tree len = CALL_EXPR_ARG (exp, 2);
    3683              : 
    3684       106035 :   return expand_builtin_memory_copy_args (dest, src, len, target, exp,
    3685       106035 :                                           /*retmode=*/ RETURN_BEGIN, false);
    3686              : }
    3687              : 
    3688              : /* Check a call EXP to the memmove built-in for validity.
    3689              :    Return NULL_RTX on both success and failure.  */
    3690              : 
    3691              : static rtx
    3692        17405 : expand_builtin_memmove (tree exp, rtx target)
    3693              : {
    3694        17405 :   if (!validate_arglist (exp,
    3695              :                          POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    3696              :     return NULL_RTX;
    3697              : 
    3698        17390 :   tree dest = CALL_EXPR_ARG (exp, 0);
    3699        17390 :   tree src = CALL_EXPR_ARG (exp, 1);
    3700        17390 :   tree len = CALL_EXPR_ARG (exp, 2);
    3701              : 
    3702        17390 :   return expand_builtin_memory_copy_args (dest, src, len, target, exp,
    3703        17390 :                                           /*retmode=*/ RETURN_BEGIN, true);
    3704              : }
    3705              : 
    3706              : /* Expand a call EXP to the mempcpy builtin.
    3707              :    Return NULL_RTX if we failed; the caller should emit a normal call,
    3708              :    otherwise try to get the result in TARGET, if convenient (and in
    3709              :    mode MODE if that's convenient).  */
    3710              : 
    3711              : static rtx
    3712         1395 : expand_builtin_mempcpy (tree exp, rtx target)
    3713              : {
    3714         1395 :   if (!validate_arglist (exp,
    3715              :                          POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    3716              :     return NULL_RTX;
    3717              : 
    3718         1390 :   tree dest = CALL_EXPR_ARG (exp, 0);
    3719         1390 :   tree src = CALL_EXPR_ARG (exp, 1);
    3720         1390 :   tree len = CALL_EXPR_ARG (exp, 2);
    3721              : 
    3722              :   /* Policy does not generally allow using compute_objsize (which
    3723              :      is used internally by check_memop_size) to change code generation
    3724              :      or drive optimization decisions.
    3725              : 
    3726              :      In this instance it is safe because the code we generate has
    3727              :      the same semantics regardless of the return value of
    3728              :      check_memop_sizes.   Exactly the same amount of data is copied
    3729              :      and the return value is exactly the same in both cases.
    3730              : 
    3731              :      Furthermore, check_memop_size always uses mode 0 for the call to
    3732              :      compute_objsize, so the imprecise nature of compute_objsize is
    3733              :      avoided.  */
    3734              : 
    3735              :   /* Avoid expanding mempcpy into memcpy when the call is determined
    3736              :      to overflow the buffer.  This also prevents the same overflow
    3737              :      from being diagnosed again when expanding memcpy.  */
    3738              : 
    3739         1390 :   return expand_builtin_mempcpy_args (dest, src, len,
    3740         1390 :                                       target, exp, /*retmode=*/ RETURN_END);
    3741              : }
    3742              : 
    3743              : /* Helper function to do the actual work for expand of memory copy family
    3744              :    functions (memcpy, mempcpy, stpcpy).  Expansing should assign LEN bytes
    3745              :    of memory from SRC to DEST and assign to TARGET if convenient.  Return
    3746              :    value is based on RETMODE argument.  */
    3747              : 
    3748              : static rtx
    3749       124838 : expand_builtin_memory_copy_args (tree dest, tree src, tree len,
    3750              :                                  rtx target, tree exp, memop_ret retmode,
    3751              :                                  bool might_overlap)
    3752              : {
    3753       124838 :   unsigned int src_align = get_pointer_alignment (src);
    3754       124838 :   unsigned int dest_align = get_pointer_alignment (dest);
    3755       124838 :   rtx dest_mem, src_mem, dest_addr, len_rtx;
    3756       124838 :   HOST_WIDE_INT expected_size = -1;
    3757       124838 :   unsigned int expected_align = 0;
    3758       124838 :   unsigned HOST_WIDE_INT min_size;
    3759       124838 :   unsigned HOST_WIDE_INT max_size;
    3760       124838 :   unsigned HOST_WIDE_INT probable_max_size;
    3761              : 
    3762       124838 :   bool is_move_done;
    3763              : 
    3764              :   /* If DEST is not a pointer type, call the normal function.  */
    3765       124838 :   if (dest_align == 0)
    3766              :     return NULL_RTX;
    3767              : 
    3768              :   /* If either SRC is not a pointer type, don't do this
    3769              :      operation in-line.  */
    3770       124838 :   if (src_align == 0)
    3771              :     return NULL_RTX;
    3772              : 
    3773       124838 :   if (currently_expanding_gimple_stmt)
    3774       124838 :     stringop_block_profile (currently_expanding_gimple_stmt,
    3775              :                             &expected_align, &expected_size);
    3776              : 
    3777       124838 :   if (expected_align < dest_align)
    3778       124816 :     expected_align = dest_align;
    3779       124838 :   dest_mem = get_memory_rtx (dest, len, true);
    3780       124838 :   set_mem_align (dest_mem, dest_align);
    3781       124838 :   len_rtx = expand_normal (len);
    3782       124838 :   determine_block_size (len, len_rtx, &min_size, &max_size,
    3783              :                         &probable_max_size);
    3784              : 
    3785              :   /* Try to get the byte representation of the constant SRC points to,
    3786              :      with its byte size in NBYTES.  */
    3787       124838 :   unsigned HOST_WIDE_INT nbytes;
    3788       124838 :   const char *rep = getbyterep (src, &nbytes);
    3789              : 
    3790              :   /* If the function's constant bound LEN_RTX is less than or equal
    3791              :      to the byte size of the representation of the constant argument,
    3792              :      and if block move would be done by pieces, we can avoid loading
    3793              :      the bytes from memory and only store the computed constant.
    3794              :      This works in the overlap (memmove) case as well because
    3795              :      store_by_pieces just generates a series of stores of constants
    3796              :      from the representation returned by getbyterep().  */
    3797       124838 :   if (rep
    3798        29502 :       && CONST_INT_P (len_rtx)
    3799        27834 :       && (unsigned HOST_WIDE_INT) INTVAL (len_rtx) <= nbytes
    3800       152385 :       && can_store_by_pieces (INTVAL (len_rtx), builtin_memcpy_read_str,
    3801              :                               const_cast<char *> (rep),
    3802              :                               dest_align, false))
    3803              :     {
    3804        27067 :       dest_mem = store_by_pieces (dest_mem, INTVAL (len_rtx),
    3805              :                                   builtin_memcpy_read_str,
    3806              :                                   const_cast<char *> (rep),
    3807              :                                   dest_align, false, retmode);
    3808        27067 :       dest_mem = force_operand (XEXP (dest_mem, 0), target);
    3809        27067 :       dest_mem = convert_memory_address (ptr_mode, dest_mem);
    3810        27067 :       return dest_mem;
    3811              :     }
    3812              : 
    3813        97771 :   src_mem = get_memory_rtx (src, len, false);
    3814        97771 :   set_mem_align (src_mem, src_align);
    3815              : 
    3816              :   /* Copy word part most expediently.  */
    3817        97771 :   enum block_op_methods method = BLOCK_OP_NORMAL;
    3818        97771 :   if (CALL_EXPR_TAILCALL (exp)
    3819        97771 :       && (retmode == RETURN_BEGIN || target == const0_rtx))
    3820              :     method = BLOCK_OP_TAILCALL;
    3821        97771 :   bool use_mempcpy_call = (targetm.libc_has_fast_function (BUILT_IN_MEMPCPY)
    3822        97771 :                            && retmode == RETURN_END
    3823        97771 :                            && !might_overlap
    3824        98846 :                            && target != const0_rtx);
    3825              :   if (use_mempcpy_call)
    3826        97771 :     method = BLOCK_OP_NO_LIBCALL_RET;
    3827        97771 :   dest_addr = emit_block_move_hints (dest_mem, src_mem, len_rtx, method,
    3828              :                                      expected_align, expected_size,
    3829              :                                      min_size, max_size, probable_max_size,
    3830              :                                      use_mempcpy_call, &is_move_done,
    3831              :                                      might_overlap, tree_ctz (len));
    3832              : 
    3833              :   /* Bail out when a mempcpy call would be expanded as libcall and when
    3834              :      we have a target that provides a fast implementation
    3835              :      of mempcpy routine.  */
    3836        97771 :   if (!is_move_done)
    3837              :     return NULL_RTX;
    3838              : 
    3839        84860 :   if (dest_addr == pc_rtx)
    3840              :     return NULL_RTX;
    3841              : 
    3842        84860 :   if (dest_addr == 0)
    3843              :     {
    3844        33178 :       dest_addr = force_operand (XEXP (dest_mem, 0), target);
    3845        33178 :       dest_addr = convert_memory_address (ptr_mode, dest_addr);
    3846              :     }
    3847              : 
    3848        84860 :   if (retmode != RETURN_BEGIN && target != const0_rtx)
    3849              :     {
    3850          815 :       dest_addr = gen_rtx_PLUS (ptr_mode, dest_addr, len_rtx);
    3851              :       /* stpcpy pointer to last byte.  */
    3852          815 :       if (retmode == RETURN_END_MINUS_ONE)
    3853            0 :         dest_addr = gen_rtx_MINUS (ptr_mode, dest_addr, const1_rtx);
    3854              :     }
    3855              : 
    3856              :   return dest_addr;
    3857              : }
    3858              : 
    3859              : static rtx
    3860         1413 : expand_builtin_mempcpy_args (tree dest, tree src, tree len,
    3861              :                              rtx target, tree orig_exp, memop_ret retmode)
    3862              : {
    3863         1390 :   return expand_builtin_memory_copy_args (dest, src, len, target, orig_exp,
    3864            0 :                                           retmode, false);
    3865              : }
    3866              : 
    3867              : /* Expand into a movstr instruction, if one is available.  Return NULL_RTX if
    3868              :    we failed, the caller should emit a normal call, otherwise try to
    3869              :    get the result in TARGET, if convenient.
    3870              :    Return value is based on RETMODE argument.  */
    3871              : 
    3872              : static rtx
    3873         2267 : expand_movstr (tree dest, tree src, rtx target, memop_ret retmode)
    3874              : {
    3875         2267 :   class expand_operand ops[3];
    3876         2267 :   rtx dest_mem;
    3877         2267 :   rtx src_mem;
    3878              : 
    3879         2267 :   if (!targetm.have_movstr ())
    3880              :     return NULL_RTX;
    3881              : 
    3882            0 :   dest_mem = get_memory_rtx (dest, NULL, true);
    3883            0 :   src_mem = get_memory_rtx (src, NULL, false);
    3884            0 :   if (retmode == RETURN_BEGIN)
    3885              :     {
    3886            0 :       target = force_reg (Pmode, XEXP (dest_mem, 0));
    3887            0 :       dest_mem = replace_equiv_address (dest_mem, target);
    3888              :     }
    3889              : 
    3890            0 :   create_output_operand (&ops[0],
    3891            0 :                          retmode != RETURN_BEGIN ? target : NULL_RTX, Pmode);
    3892            0 :   create_fixed_operand (&ops[1], dest_mem);
    3893            0 :   create_fixed_operand (&ops[2], src_mem);
    3894            0 :   if (!maybe_expand_insn (targetm.code_for_movstr, 3, ops))
    3895              :     return NULL_RTX;
    3896              : 
    3897            0 :   if (retmode != RETURN_BEGIN && target != const0_rtx)
    3898              :     {
    3899            0 :       target = ops[0].value;
    3900              :       /* movstr is supposed to set end to the address of the NUL
    3901              :          terminator.  If the caller requested a mempcpy-like return value,
    3902              :          adjust it.  */
    3903            0 :       if (retmode == RETURN_END)
    3904              :         {
    3905            0 :           rtx tem = plus_constant (GET_MODE (target),
    3906            0 :                                    gen_lowpart (GET_MODE (target), target), 1);
    3907            0 :           emit_move_insn (target, force_operand (tem, NULL_RTX));
    3908              :         }
    3909              :     }
    3910              :   return target;
    3911              : }
    3912              : 
    3913              : /* Expand expression EXP, which is a call to the strcpy builtin.  Return
    3914              :    NULL_RTX if we failed the caller should emit a normal call, otherwise
    3915              :    try to get the result in TARGET, if convenient (and in mode MODE if that's
    3916              :    convenient).  */
    3917              : 
    3918              : static rtx
    3919         1860 : expand_builtin_strcpy (tree exp, rtx target)
    3920              : {
    3921         1860 :   if (!validate_arglist (exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
    3922              :     return NULL_RTX;
    3923              : 
    3924         1845 :   tree dest = CALL_EXPR_ARG (exp, 0);
    3925         1845 :   tree src = CALL_EXPR_ARG (exp, 1);
    3926              : 
    3927         1845 :   return expand_builtin_strcpy_args (exp, dest, src, target);
    3928              : }
    3929              : 
    3930              : /* Helper function to do the actual work for expand_builtin_strcpy.  The
    3931              :    arguments to the builtin_strcpy call DEST and SRC are broken out
    3932              :    so that this can also be called without constructing an actual CALL_EXPR.
    3933              :    The other arguments and return value are the same as for
    3934              :    expand_builtin_strcpy.  */
    3935              : 
    3936              : static rtx
    3937         1845 : expand_builtin_strcpy_args (tree, tree dest, tree src, rtx target)
    3938              : {
    3939         1845 :   return expand_movstr (dest, src, target, /*retmode=*/ RETURN_BEGIN);
    3940              : }
    3941              : 
    3942              : /* Expand a call EXP to the stpcpy builtin.
    3943              :    Return NULL_RTX if we failed the caller should emit a normal call,
    3944              :    otherwise try to get the result in TARGET, if convenient (and in
    3945              :    mode MODE if that's convenient).  */
    3946              : 
    3947              : static rtx
    3948          454 : expand_builtin_stpcpy_1 (tree exp, rtx target, machine_mode mode)
    3949              : {
    3950          454 :   tree dst, src;
    3951          454 :   location_t loc = EXPR_LOCATION (exp);
    3952              : 
    3953          454 :   if (!validate_arglist (exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
    3954              :     return NULL_RTX;
    3955              : 
    3956          445 :   dst = CALL_EXPR_ARG (exp, 0);
    3957          445 :   src = CALL_EXPR_ARG (exp, 1);
    3958              : 
    3959              :   /* If return value is ignored, transform stpcpy into strcpy.  */
    3960          445 :   if (target == const0_rtx && builtin_decl_implicit (BUILT_IN_STRCPY))
    3961              :     {
    3962            0 :       tree fn = builtin_decl_implicit (BUILT_IN_STRCPY);
    3963            0 :       tree result = build_call_nofold_loc (loc, fn, 2, dst, src);
    3964            0 :       return expand_expr (result, target, mode, EXPAND_NORMAL);
    3965              :     }
    3966              :   else
    3967              :     {
    3968          445 :       tree len, lenp1;
    3969          445 :       rtx ret;
    3970              : 
    3971              :       /* Ensure we get an actual string whose length can be evaluated at
    3972              :          compile-time, not an expression containing a string.  This is
    3973              :          because the latter will potentially produce pessimized code
    3974              :          when used to produce the return value.  */
    3975          445 :       c_strlen_data lendata = { };
    3976          445 :       if (!c_getstr (src)
    3977          445 :           || !(len = c_strlen (src, 0, &lendata, 1)))
    3978          422 :         return expand_movstr (dst, src, target,
    3979          422 :                               /*retmode=*/ RETURN_END_MINUS_ONE);
    3980              : 
    3981           23 :       lenp1 = size_binop_loc (loc, PLUS_EXPR, len, ssize_int (1));
    3982           23 :       ret = expand_builtin_mempcpy_args (dst, src, lenp1,
    3983              :                                          target, exp,
    3984              :                                          /*retmode=*/ RETURN_END_MINUS_ONE);
    3985              : 
    3986           23 :       if (ret)
    3987              :         return ret;
    3988              : 
    3989            0 :       if (TREE_CODE (len) == INTEGER_CST)
    3990              :         {
    3991            0 :           rtx len_rtx = expand_normal (len);
    3992              : 
    3993            0 :           if (CONST_INT_P (len_rtx))
    3994              :             {
    3995            0 :               ret = expand_builtin_strcpy_args (exp, dst, src, target);
    3996              : 
    3997            0 :               if (ret)
    3998              :                 {
    3999            0 :                   if (! target)
    4000              :                     {
    4001            0 :                       if (mode != VOIDmode)
    4002            0 :                         target = gen_reg_rtx (mode);
    4003              :                       else
    4004            0 :                         target = gen_reg_rtx (GET_MODE (ret));
    4005              :                     }
    4006            0 :                   if (GET_MODE (target) != GET_MODE (ret))
    4007            0 :                     ret = gen_lowpart (GET_MODE (target), ret);
    4008              : 
    4009            0 :                   ret = plus_constant (GET_MODE (ret), ret, INTVAL (len_rtx));
    4010            0 :                   ret = emit_move_insn (target, force_operand (ret, NULL_RTX));
    4011            0 :                   gcc_assert (ret);
    4012              : 
    4013              :                   return target;
    4014              :                 }
    4015              :             }
    4016              :         }
    4017              : 
    4018            0 :       return expand_movstr (dst, src, target,
    4019            0 :                             /*retmode=*/ RETURN_END_MINUS_ONE);
    4020              :     }
    4021              : }
    4022              : 
    4023              : /* Expand a call EXP to the stpcpy builtin and diagnose uses of nonstring
    4024              :    arguments while being careful to avoid duplicate warnings (which could
    4025              :    be issued if the expander were to expand the call, resulting in it
    4026              :    being emitted in expand_call().  */
    4027              : 
    4028              : static rtx
    4029          454 : expand_builtin_stpcpy (tree exp, rtx target, machine_mode mode)
    4030              : {
    4031          454 :   if (rtx ret = expand_builtin_stpcpy_1 (exp, target, mode))
    4032              :     {
    4033              :       /* The call has been successfully expanded.  Check for nonstring
    4034              :          arguments and issue warnings as appropriate.  */
    4035           23 :       maybe_warn_nonstring_arg (get_callee_fndecl (exp), exp);
    4036           23 :       return ret;
    4037              :     }
    4038              : 
    4039              :   return NULL_RTX;
    4040              : }
    4041              : 
    4042              : /* Callback routine for store_by_pieces.  Read GET_MODE_BITSIZE (MODE)
    4043              :    bytes from constant string DATA + OFFSET and return it as target
    4044              :    constant.  */
    4045              : 
    4046              : rtx
    4047        10031 : builtin_strncpy_read_str (void *data, void *, HOST_WIDE_INT offset,
    4048              :                           fixed_size_mode mode)
    4049              : {
    4050        10031 :   const char *str = (const char *) data;
    4051              : 
    4052        10031 :   if ((unsigned HOST_WIDE_INT) offset > strlen (str))
    4053          260 :     return const0_rtx;
    4054              : 
    4055         9771 :   return c_readstr (str + offset, mode);
    4056              : }
    4057              : 
    4058              : /* Helper to check the sizes of sequences and the destination of calls
    4059              :    to __builtin_strncat and __builtin___strncat_chk.  Returns true on
    4060              :    success (no overflow or invalid sizes), false otherwise.  */
    4061              : 
    4062              : static bool
    4063          108 : check_strncat_sizes (tree exp, tree objsize)
    4064              : {
    4065          108 :   tree dest = CALL_EXPR_ARG (exp, 0);
    4066          108 :   tree src = CALL_EXPR_ARG (exp, 1);
    4067          108 :   tree maxread = CALL_EXPR_ARG (exp, 2);
    4068              : 
    4069              :   /* Try to determine the range of lengths that the source expression
    4070              :      refers to.  */
    4071          108 :   c_strlen_data lendata = { };
    4072          108 :   get_range_strlen (src, &lendata, /* eltsize = */ 1);
    4073              : 
    4074              :   /* Try to verify that the destination is big enough for the shortest
    4075              :      string.  */
    4076              : 
    4077          108 :   access_data data (nullptr, exp, access_read_write, maxread, true);
    4078          108 :   if (!objsize && warn_stringop_overflow)
    4079              :     {
    4080              :       /* If it hasn't been provided by __strncat_chk, try to determine
    4081              :          the size of the destination object into which the source is
    4082              :          being copied.  */
    4083            0 :       objsize = compute_objsize (dest, warn_stringop_overflow - 1, &data.dst);
    4084              :     }
    4085              : 
    4086              :   /* Add one for the terminating nul.  */
    4087          108 :   tree srclen = (lendata.minlen
    4088          108 :                  ? fold_build2 (PLUS_EXPR, size_type_node, lendata.minlen,
    4089              :                                 size_one_node)
    4090              :                  : NULL_TREE);
    4091              : 
    4092              :   /* The strncat function copies at most MAXREAD bytes and always appends
    4093              :      the terminating nul so the specified upper bound should never be equal
    4094              :      to (or greater than) the size of the destination.  */
    4095           55 :   if (tree_fits_uhwi_p (maxread) && tree_fits_uhwi_p (objsize)
    4096          155 :       && tree_int_cst_equal (objsize, maxread))
    4097              :     {
    4098           23 :       location_t loc = EXPR_LOCATION (exp);
    4099           23 :       warning_at (loc, OPT_Wstringop_overflow_,
    4100              :                   "%qD specified bound %E equals destination size",
    4101              :                   get_callee_fndecl (exp), maxread);
    4102              : 
    4103           23 :       return false;
    4104              :     }
    4105              : 
    4106           85 :   if (!srclen
    4107           85 :       || (maxread && tree_fits_uhwi_p (maxread)
    4108           32 :           && tree_fits_uhwi_p (srclen)
    4109           32 :           && tree_int_cst_lt (maxread, srclen)))
    4110              :     srclen = maxread;
    4111              : 
    4112              :   /* The number of bytes to write is LEN but check_access will alsoa
    4113              :      check SRCLEN if LEN's value isn't known.  */
    4114           85 :   return check_access (exp, /*dstwrite=*/NULL_TREE, maxread, srclen,
    4115           85 :                        objsize, data.mode, &data);
    4116              : }
    4117              : 
    4118              : /* Expand expression EXP, which is a call to the strncpy builtin.  Return
    4119              :    NULL_RTX if we failed the caller should emit a normal call.  */
    4120              : 
    4121              : static rtx
    4122         2177 : expand_builtin_strncpy (tree exp, rtx target)
    4123              : {
    4124         2177 :   location_t loc = EXPR_LOCATION (exp);
    4125              : 
    4126         2177 :   if (!validate_arglist (exp,
    4127              :                          POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    4128              :     return NULL_RTX;
    4129         2165 :   tree dest = CALL_EXPR_ARG (exp, 0);
    4130         2165 :   tree src = CALL_EXPR_ARG (exp, 1);
    4131              :   /* The number of bytes to write (not the maximum).  */
    4132         2165 :   tree len = CALL_EXPR_ARG (exp, 2);
    4133              : 
    4134              :   /* The length of the source sequence.  */
    4135         2165 :   tree slen = c_strlen (src, 1);
    4136              : 
    4137              :   /* We must be passed a constant len and src parameter.  */
    4138         2165 :   if (!tree_fits_uhwi_p (len) || !slen || !tree_fits_uhwi_p (slen))
    4139              :     return NULL_RTX;
    4140              : 
    4141          237 :   slen = size_binop_loc (loc, PLUS_EXPR, slen, ssize_int (1));
    4142              : 
    4143              :   /* We're required to pad with trailing zeros if the requested
    4144              :      len is greater than strlen(s2)+1.  In that case try to
    4145              :      use store_by_pieces, if it fails, punt.  */
    4146          237 :   if (tree_int_cst_lt (slen, len))
    4147              :     {
    4148          142 :       unsigned int dest_align = get_pointer_alignment (dest);
    4149          142 :       const char *p = c_getstr (src);
    4150          142 :       rtx dest_mem;
    4151              : 
    4152          136 :       if (!p || dest_align == 0 || !tree_fits_uhwi_p (len)
    4153          278 :           || !can_store_by_pieces (tree_to_uhwi (len),
    4154              :                                    builtin_strncpy_read_str,
    4155              :                                    const_cast<char *> (p),
    4156              :                                    dest_align, false))
    4157              :         return NULL_RTX;
    4158              : 
    4159          128 :       dest_mem = get_memory_rtx (dest, len, true);
    4160          128 :       store_by_pieces (dest_mem, tree_to_uhwi (len),
    4161              :                        builtin_strncpy_read_str,
    4162              :                        const_cast<char *> (p), dest_align, false,
    4163              :                        RETURN_BEGIN);
    4164          128 :       dest_mem = force_operand (XEXP (dest_mem, 0), target);
    4165          128 :       dest_mem = convert_memory_address (ptr_mode, dest_mem);
    4166          128 :       return dest_mem;
    4167              :     }
    4168              : 
    4169              :   return NULL_RTX;
    4170              : }
    4171              : 
    4172              : /* Return the RTL of a register in MODE generated from PREV in the
    4173              :    previous iteration.  */
    4174              : 
    4175              : static rtx
    4176       256265 : gen_memset_value_from_prev (by_pieces_prev *prev, fixed_size_mode mode)
    4177              : {
    4178       256265 :   rtx target = nullptr;
    4179       256265 :   if (prev != nullptr && prev->data != nullptr)
    4180              :     {
    4181              :       /* Use the previous data in the same mode.  */
    4182       141471 :       if (prev->mode == mode)
    4183       256265 :         return prev->data;
    4184              : 
    4185        32578 :       fixed_size_mode prev_mode = prev->mode;
    4186              : 
    4187              :       /* Don't use the previous data to write QImode if it is in a
    4188              :          vector mode.  */
    4189        32578 :       if (VECTOR_MODE_P (prev_mode) && mode == QImode)
    4190              :         return target;
    4191              : 
    4192        32179 :       rtx prev_rtx = prev->data;
    4193              : 
    4194        32179 :       if (REG_P (prev_rtx)
    4195        23773 :           && HARD_REGISTER_P (prev_rtx)
    4196        32179 :           && lowpart_subreg_regno (REGNO (prev_rtx), prev_mode, mode) < 0)
    4197              :         {
    4198              :           /* This case occurs when PREV_MODE is a vector and when
    4199              :              MODE is too small to store using vector operations.
    4200              :              After register allocation, the code will need to move the
    4201              :              lowpart of the vector register into a non-vector register.
    4202              : 
    4203              :              Also, the target has chosen to use a hard register
    4204              :              instead of going with the default choice of using a
    4205              :              pseudo register.  We should respect that choice and try to
    4206              :              avoid creating a pseudo register with the same mode as the
    4207              :              current hard register.
    4208              : 
    4209              :              In principle, we could just use a lowpart MODE subreg of
    4210              :              the vector register.  However, the vector register mode might
    4211              :              be too wide for non-vector registers, and we already know
    4212              :              that the non-vector mode is too small for vector registers.
    4213              :              It's therefore likely that we'd need to spill to memory in
    4214              :              the vector mode and reload the non-vector value from there.
    4215              : 
    4216              :              Try to avoid that by reducing the vector register to the
    4217              :              smallest size that it can hold.  This should increase the
    4218              :              chances that non-vector registers can hold both the inner
    4219              :              and outer modes of the subreg that we generate later.  */
    4220            0 :           machine_mode m;
    4221            0 :           fixed_size_mode candidate;
    4222            0 :           FOR_EACH_MODE_IN_CLASS (m, GET_MODE_CLASS (mode))
    4223            0 :             if (is_a<fixed_size_mode> (m, &candidate))
    4224              :               {
    4225            0 :                 if (GET_MODE_SIZE (candidate)
    4226            0 :                     >= GET_MODE_SIZE (prev_mode))
    4227              :                   break;
    4228            0 :                 if (GET_MODE_SIZE (candidate) >= GET_MODE_SIZE (mode)
    4229            0 :                     && lowpart_subreg_regno (REGNO (prev_rtx),
    4230              :                                              prev_mode, candidate) >= 0)
    4231              :                   {
    4232            0 :                     target = lowpart_subreg (candidate, prev_rtx,
    4233              :                                              prev_mode);
    4234            0 :                     prev_rtx = target;
    4235            0 :                     prev_mode = candidate;
    4236            0 :                     break;
    4237              :                   }
    4238              :               }
    4239            0 :           if (target == nullptr)
    4240            0 :             prev_rtx = copy_to_reg (prev_rtx);
    4241              :         }
    4242              : 
    4243        32179 :       target = lowpart_subreg (mode, prev_rtx, prev_mode);
    4244              :     }
    4245              :   return target;
    4246              : }
    4247              : 
    4248              : /* Callback routine for store_by_pieces.  Read GET_MODE_BITSIZE (MODE)
    4249              :    bytes from constant string DATA + OFFSET and return it as target
    4250              :    constant.  If PREV isn't nullptr, it has the RTL info from the
    4251              :    previous iteration.  */
    4252              : 
    4253              : rtx
    4254       254989 : builtin_memset_read_str (void *data, void *prev,
    4255              :                          HOST_WIDE_INT offset ATTRIBUTE_UNUSED,
    4256              :                          fixed_size_mode mode)
    4257              : {
    4258       254989 :   const char *c = (const char *) data;
    4259       254989 :   unsigned int size = GET_MODE_SIZE (mode);
    4260              : 
    4261       254989 :   rtx target = gen_memset_value_from_prev ((by_pieces_prev *) prev,
    4262              :                                            mode);
    4263       254989 :   if (target != nullptr)
    4264              :     return target;
    4265       114466 :   rtx src = gen_int_mode (*c, QImode);
    4266              : 
    4267       114466 :   if (VECTOR_MODE_P (mode))
    4268              :     {
    4269       165954 :       gcc_assert (GET_MODE_INNER (mode) == QImode);
    4270              : 
    4271        82977 :       rtx const_vec = gen_const_vec_duplicate (mode, src);
    4272        82977 :       if (prev == NULL)
    4273              :         /* Return CONST_VECTOR when called by a query function.  */
    4274              :         return const_vec;
    4275              : 
    4276              :       /* Use the move expander with CONST_VECTOR.  */
    4277        61243 :       target = gen_reg_rtx (mode);
    4278        61243 :       emit_move_insn (target, const_vec);
    4279        61243 :       return target;
    4280              :     }
    4281              : 
    4282        31489 :   char *p = XALLOCAVEC (char, size);
    4283              : 
    4284        31489 :   memset (p, *c, size);
    4285              : 
    4286        31489 :   return c_readstr (p, mode);
    4287              : }
    4288              : 
    4289              : /* Callback routine for store_by_pieces.  Return the RTL of a register
    4290              :    containing GET_MODE_SIZE (MODE) consecutive copies of the unsigned
    4291              :    char value given in the RTL register data.  For example, if mode is
    4292              :    4 bytes wide, return the RTL for 0x01010101*data.  If PREV isn't
    4293              :    nullptr, it has the RTL info from the previous iteration.  */
    4294              : 
    4295              : static rtx
    4296         1415 : builtin_memset_gen_str (void *data, void *prev,
    4297              :                         HOST_WIDE_INT offset ATTRIBUTE_UNUSED,
    4298              :                         fixed_size_mode mode)
    4299              : {
    4300         1415 :   rtx target, coeff;
    4301         1415 :   size_t size;
    4302         1415 :   char *p;
    4303              : 
    4304         1415 :   size = GET_MODE_SIZE (mode);
    4305         1415 :   if (size == 1)
    4306              :     return (rtx) data;
    4307              : 
    4308         1276 :   target = gen_memset_value_from_prev ((by_pieces_prev *) prev, mode);
    4309         1276 :   if (target != nullptr)
    4310              :     return target;
    4311              : 
    4312          727 :   if (VECTOR_MODE_P (mode))
    4313              :     {
    4314          366 :       gcc_assert (GET_MODE_INNER (mode) == QImode);
    4315              : 
    4316              :       /* vec_duplicate_optab is a precondition to pick a vector mode for
    4317              :          the memset expander.  */
    4318          183 :       insn_code icode = optab_handler (vec_duplicate_optab, mode);
    4319              : 
    4320          183 :       target = gen_reg_rtx (mode);
    4321          183 :       class expand_operand ops[2];
    4322          183 :       create_output_operand (&ops[0], target, mode);
    4323          183 :       create_input_operand (&ops[1], (rtx) data, QImode);
    4324          183 :       expand_insn (icode, 2, ops);
    4325          183 :       if (!rtx_equal_p (target, ops[0].value))
    4326            0 :         emit_move_insn (target, ops[0].value);
    4327              : 
    4328          183 :       return target;
    4329              :     }
    4330              : 
    4331          544 :   p = XALLOCAVEC (char, size);
    4332          544 :   memset (p, 1, size);
    4333          544 :   coeff = c_readstr (p, mode);
    4334              : 
    4335          544 :   target = convert_to_mode (mode, (rtx) data, 1);
    4336          544 :   target = expand_mult (mode, target, coeff, NULL_RTX, 1);
    4337          544 :   return force_reg (mode, target);
    4338              : }
    4339              : 
    4340              : /* Expand expression EXP, which is a call to the memset builtin.  Return
    4341              :    NULL_RTX if we failed the caller should emit a normal call, otherwise
    4342              :    try to get the result in TARGET, if convenient (and in mode MODE if that's
    4343              :    convenient).  */
    4344              : 
    4345              : rtx
    4346        62918 : expand_builtin_memset (tree exp, rtx target, machine_mode mode)
    4347              : {
    4348        62918 :   if (!validate_arglist (exp,
    4349              :                          POINTER_TYPE, INTEGER_TYPE, INTEGER_TYPE, VOID_TYPE))
    4350              :     return NULL_RTX;
    4351              : 
    4352        62909 :   tree dest = CALL_EXPR_ARG (exp, 0);
    4353        62909 :   tree val = CALL_EXPR_ARG (exp, 1);
    4354        62909 :   tree len = CALL_EXPR_ARG (exp, 2);
    4355              : 
    4356        62909 :   return expand_builtin_memset_args (dest, val, len, target, mode, exp);
    4357              : }
    4358              : 
    4359              : /* Check that store_by_pieces allows BITS + LEN (so that we don't
    4360              :    expand something too unreasonably long), and every power of 2 in
    4361              :    BITS.  It is assumed that LEN has already been tested by
    4362              :    itself.  */
    4363              : static bool
    4364        24867 : can_store_by_multiple_pieces (unsigned HOST_WIDE_INT bits,
    4365              :                               by_pieces_constfn constfun,
    4366              :                               void *constfundata, unsigned int align,
    4367              :                               bool memsetp,
    4368              :                               unsigned HOST_WIDE_INT len)
    4369              : {
    4370        24867 :   if (bits
    4371        24867 :       && !can_store_by_pieces (bits + len, constfun, constfundata,
    4372              :                                align, memsetp))
    4373              :     return false;
    4374              : 
    4375              :   /* BITS set are expected to be generally in the low range and
    4376              :      contiguous.  We do NOT want to repeat the test above in case BITS
    4377              :      has a single bit set, so we terminate the loop when BITS == BIT.
    4378              :      In the unlikely case that BITS has the MSB set, also terminate in
    4379              :      case BIT gets shifted out.  */
    4380         1122 :   for (unsigned HOST_WIDE_INT bit = 1; bit < bits && bit; bit <<= 1)
    4381              :     {
    4382          840 :       if ((bits & bit) == 0)
    4383          504 :         continue;
    4384              : 
    4385          336 :       if (!can_store_by_pieces (bit, constfun, constfundata,
    4386              :                                 align, memsetp))
    4387              :         return false;
    4388              :     }
    4389              : 
    4390              :   return true;
    4391              : }
    4392              : 
    4393              : /* Try to store VAL (or, if NULL_RTX, VALC) in LEN bytes starting at TO.
    4394              :    Return TRUE if successful, FALSE otherwise.  TO is assumed to be
    4395              :    aligned at an ALIGN-bits boundary.  LEN must be a multiple of
    4396              :    1<<CTZ_LEN between MIN_LEN and MAX_LEN.
    4397              : 
    4398              :    The strategy is to issue one store_by_pieces for each power of two,
    4399              :    from most to least significant, guarded by a test on whether there
    4400              :    are at least that many bytes left to copy in LEN.
    4401              : 
    4402              :    ??? Should we skip some powers of two in favor of loops?  Maybe start
    4403              :    at the max of TO/LEN/word alignment, at least when optimizing for
    4404              :    size, instead of ensuring O(log len) dynamic compares?  */
    4405              : 
    4406              : bool
    4407        22726 : try_store_by_multiple_pieces (rtx to, rtx len, unsigned int ctz_len,
    4408              :                               unsigned HOST_WIDE_INT min_len,
    4409              :                               unsigned HOST_WIDE_INT max_len,
    4410              :                               rtx val, char valc, unsigned int align)
    4411              : {
    4412        22726 :   int max_bits = floor_log2 (max_len);
    4413        22726 :   int min_bits = floor_log2 (min_len);
    4414        22726 :   int sctz_len = ctz_len;
    4415              : 
    4416        22726 :   gcc_checking_assert (sctz_len >= 0);
    4417              : 
    4418        22726 :   if (val)
    4419          637 :     valc = 1;
    4420              : 
    4421              :   /* Bits more significant than TST_BITS are part of the shared prefix
    4422              :      in the binary representation of both min_len and max_len.  Since
    4423              :      they're identical, we don't need to test them in the loop.  */
    4424        22726 :   int tst_bits = (max_bits != min_bits ? max_bits
    4425        10147 :                   : floor_log2 (max_len ^ min_len));
    4426              : 
    4427              :   /* Save the pre-blksize values.  */
    4428        22726 :   int orig_max_bits = max_bits;
    4429        22726 :   int orig_tst_bits = tst_bits;
    4430              : 
    4431              :   /* Check whether it's profitable to start by storing a fixed BLKSIZE
    4432              :      bytes, to lower max_bits.  In the unlikely case of a constant LEN
    4433              :      (implied by identical MAX_LEN and MIN_LEN), we want to issue a
    4434              :      single store_by_pieces, but otherwise, select the minimum multiple
    4435              :      of the ALIGN (in bytes) and of the MCD of the possible LENs, that
    4436              :      brings MAX_LEN below TST_BITS, if that's lower than min_len.  */
    4437        22726 :   unsigned HOST_WIDE_INT blksize;
    4438        22726 :   if (max_len > min_len)
    4439              :     {
    4440        12592 :       unsigned HOST_WIDE_INT alrng = MAX (HOST_WIDE_INT_1U << ctz_len,
    4441              :                                           align / BITS_PER_UNIT);
    4442        12592 :       blksize = max_len - (HOST_WIDE_INT_1U << tst_bits) + alrng;
    4443        12592 :       blksize &= ~(alrng - 1);
    4444              :     }
    4445        10134 :   else if (max_len == min_len)
    4446              :     blksize = max_len;
    4447              :   else
    4448              :     /* Huh, max_len < min_len?  Punt.  See pr100843.c.  */
    4449              :     return false;
    4450        22725 :   if (min_len >= blksize
    4451              :       /* ??? Maybe try smaller fixed-prefix blksizes before
    4452              :          punting?  */
    4453        22725 :       && can_store_by_pieces (blksize, builtin_memset_read_str,
    4454              :                               &valc, align, true))
    4455              :     {
    4456          309 :       min_len -= blksize;
    4457          309 :       min_bits = floor_log2 (min_len);
    4458          309 :       max_len -= blksize;
    4459          309 :       max_bits = floor_log2 (max_len);
    4460              : 
    4461          309 :       tst_bits = (max_bits != min_bits ? max_bits
    4462          175 :                  : floor_log2 (max_len ^ min_len));
    4463              :     }
    4464              :   else
    4465              :     blksize = 0;
    4466              : 
    4467              :   /* Check that we can use store by pieces for the maximum store count
    4468              :      we may issue (initial fixed-size block, plus conditional
    4469              :      power-of-two-sized from max_bits to ctz_len.  */
    4470        22725 :   unsigned HOST_WIDE_INT xlenest = blksize;
    4471        22725 :   if (max_bits >= 0)
    4472        22550 :     xlenest += ((HOST_WIDE_INT_1U << max_bits) * 2
    4473        22550 :                 - (HOST_WIDE_INT_1U << ctz_len));
    4474        22725 :   bool max_loop = false;
    4475        22725 :   bool use_store_by_pieces = true;
    4476              :   /* Skip the test in case of overflow in xlenest.  It shouldn't
    4477              :      happen because of the way max_bits and blksize are related, but
    4478              :      it doesn't hurt to test.  */
    4479        22725 :   if (blksize > xlenest
    4480        22725 :       || !can_store_by_multiple_pieces (xlenest - blksize,
    4481              :                                         builtin_memset_read_str,
    4482              :                                         &valc, align, true, blksize))
    4483              :     {
    4484        22550 :       if (!(flag_inline_stringops & ILSOP_MEMSET))
    4485              :         return false;
    4486              : 
    4487         2067 :       for (max_bits = orig_max_bits;
    4488         2174 :            max_bits >= sctz_len;
    4489              :            --max_bits)
    4490              :         {
    4491         2174 :           xlenest = ((HOST_WIDE_INT_1U << max_bits) * 2
    4492         2174 :                      - (HOST_WIDE_INT_1U << ctz_len));
    4493              :           /* Check that blksize plus the bits to be stored as blocks
    4494              :              sized at powers of two can be stored by pieces.  This is
    4495              :              like the test above, but with smaller max_bits.  Skip
    4496              :              orig_max_bits (it would be redundant).  Also skip in case
    4497              :              of overflow.  */
    4498         2174 :           if (max_bits < orig_max_bits
    4499         2067 :               && xlenest + blksize >= xlenest
    4500         4241 :               && can_store_by_multiple_pieces (xlenest,
    4501              :                                                builtin_memset_read_str,
    4502              :                                                &valc, align, true, blksize))
    4503              :             {
    4504              :               max_loop = true;
    4505              :               break;
    4506              :             }
    4507         2067 :           if (blksize
    4508         2067 :               && can_store_by_multiple_pieces (xlenest,
    4509              :                                                builtin_memset_read_str,
    4510              :                                                &valc, align, true, 0))
    4511              :             {
    4512            0 :               max_len += blksize;
    4513            0 :               min_len += blksize;
    4514            0 :               tst_bits = orig_tst_bits;
    4515            0 :               blksize = 0;
    4516            0 :               max_loop = true;
    4517            0 :               break;
    4518              :             }
    4519         2067 :           if (max_bits == sctz_len)
    4520              :             {
    4521              :               /* We'll get here if can_store_by_pieces refuses to
    4522              :                  store even a single QImode.  We'll fall back to
    4523              :                  QImode stores then.  */
    4524           98 :               if (!sctz_len)
    4525              :                 {
    4526              :                   blksize = 0;
    4527              :                   max_loop = true;
    4528              :                   use_store_by_pieces = false;
    4529              :                   break;
    4530              :                 }
    4531           98 :               --sctz_len;
    4532           98 :               --ctz_len;
    4533              :             }
    4534              :         }
    4535          107 :       if (!max_loop)
    4536              :         return false;
    4537              :       /* If the boundaries are such that min and max may run a
    4538              :          different number of trips in the initial loop, the remainder
    4539              :          needs not be between the moduli, so set tst_bits to cover all
    4540              :          bits.  Otherwise, if the trip counts are the same, max_len
    4541              :          has the common prefix, and the previously-computed tst_bits
    4542              :          is usable.  */
    4543          107 :       if (max_len >> max_bits > min_len >> max_bits)
    4544           64 :         tst_bits = max_bits;
    4545              :     }
    4546              : 
    4547          282 :   by_pieces_constfn constfun;
    4548          282 :   void *constfundata;
    4549          282 :   if (val)
    4550              :     {
    4551            1 :       constfun = builtin_memset_gen_str;
    4552            1 :       constfundata = val = force_reg (TYPE_MODE (unsigned_char_type_node),
    4553              :                                       val);
    4554              :     }
    4555              :   else
    4556              :     {
    4557              :       constfun = builtin_memset_read_str;
    4558              :       constfundata = &valc;
    4559              :     }
    4560              : 
    4561          282 :   rtx ptr = copy_addr_to_reg (XEXP (to, 0));
    4562          282 :   rtx rem = copy_to_mode_reg (ptr_mode, convert_to_mode (ptr_mode, len, 0));
    4563          282 :   to = replace_equiv_address (to, ptr);
    4564          282 :   set_mem_align (to, align);
    4565              : 
    4566          282 :   if (blksize)
    4567              :     {
    4568          363 :       to = store_by_pieces (to, blksize,
    4569              :                             constfun, constfundata,
    4570              :                             align, true,
    4571              :                             max_len != 0 ? RETURN_END : RETURN_BEGIN);
    4572          189 :       if (max_len == 0)
    4573              :         return true;
    4574              : 
    4575              :       /* Adjust PTR, TO and REM.  Since TO's address is likely
    4576              :          PTR+offset, we have to replace it.  */
    4577           15 :       emit_move_insn (ptr, force_operand (XEXP (to, 0), NULL_RTX));
    4578           15 :       to = replace_equiv_address (to, ptr);
    4579           15 :       rtx rem_minus_blksize = plus_constant (ptr_mode, rem, -blksize);
    4580           15 :       emit_move_insn (rem, force_operand (rem_minus_blksize, NULL_RTX));
    4581              :     }
    4582              : 
    4583              :   /* Iterate over power-of-two block sizes from the maximum length to
    4584              :      the least significant bit possibly set in the length.  */
    4585          493 :   for (int i = max_bits; i >= sctz_len; i--)
    4586              :     {
    4587          385 :       rtx_code_label *loop_label = NULL;
    4588          385 :       rtx_code_label *label = NULL;
    4589              : 
    4590          385 :       blksize = HOST_WIDE_INT_1U << i;
    4591              : 
    4592              :       /* If we're past the bits shared between min_ and max_len, expand
    4593              :          a test on the dynamic length, comparing it with the
    4594              :          BLKSIZE.  */
    4595          385 :       if (i <= tst_bits)
    4596              :         {
    4597          275 :           label = gen_label_rtx ();
    4598          275 :           emit_cmp_and_jump_insns (rem, GEN_INT (blksize), LT, NULL,
    4599              :                                    ptr_mode, 1, label,
    4600              :                                    profile_probability::even ());
    4601              :         }
    4602              :       /* If we are at a bit that is in the prefix shared by min_ and
    4603              :          max_len, skip the current BLKSIZE if the bit is clear, but do
    4604              :          not skip the loop, even if it doesn't require
    4605              :          prechecking.  */
    4606          110 :       else if ((max_len & blksize) == 0
    4607           69 :                && !(max_loop && i == max_bits))
    4608           39 :         continue;
    4609              : 
    4610          346 :       if (max_loop && i == max_bits)
    4611              :         {
    4612          107 :           loop_label = gen_label_rtx ();
    4613          107 :           emit_label (loop_label);
    4614              :           /* Since we may run this multiple times, don't assume we
    4615              :              know anything about the offset.  */
    4616          107 :           clear_mem_offset (to);
    4617              :         }
    4618              : 
    4619          346 :       bool update_needed = i != sctz_len || loop_label;
    4620          346 :       rtx next_ptr = NULL_RTX;
    4621          346 :       if (!use_store_by_pieces)
    4622              :         {
    4623            0 :           gcc_checking_assert (blksize == 1);
    4624            0 :           if (!val)
    4625            0 :             val = gen_int_mode (valc, QImode);
    4626            0 :           to = change_address (to, QImode, 0);
    4627            0 :           emit_move_insn (to, val);
    4628            0 :           if (update_needed)
    4629            0 :             next_ptr = plus_constant (GET_MODE (ptr), ptr, blksize);
    4630              :         }
    4631              :       else
    4632              :         {
    4633              :           /* Issue a store of BLKSIZE bytes.  */
    4634          404 :           to = store_by_pieces (to, blksize,
    4635              :                                 constfun, constfundata,
    4636              :                                 align, true,
    4637              :                                 update_needed ? RETURN_END : RETURN_BEGIN);
    4638          346 :           next_ptr = XEXP (to, 0);
    4639              :         }
    4640              :       /* Adjust REM and PTR, unless this is the last iteration.  */
    4641          346 :       if (update_needed)
    4642              :         {
    4643          288 :           emit_move_insn (ptr, force_operand (next_ptr, NULL_RTX));
    4644          288 :           to = replace_equiv_address (to, ptr);
    4645          288 :           rtx rem_minus_blksize = plus_constant (ptr_mode, rem, -blksize);
    4646          288 :           emit_move_insn (rem, force_operand (rem_minus_blksize, NULL_RTX));
    4647              :         }
    4648              : 
    4649          288 :       if (loop_label)
    4650          107 :         emit_cmp_and_jump_insns (rem, GEN_INT (blksize), GE, NULL,
    4651              :                                  ptr_mode, 1, loop_label,
    4652              :                                  profile_probability::likely ());
    4653              : 
    4654          346 :       if (label)
    4655              :         {
    4656          275 :           emit_label (label);
    4657              : 
    4658              :           /* Given conditional stores, the offset can no longer be
    4659              :              known, so clear it.  */
    4660          275 :           clear_mem_offset (to);
    4661              :         }
    4662              :     }
    4663              : 
    4664              :   return true;
    4665              : }
    4666              : 
    4667              : /* Helper function to do the actual work for expand_builtin_memset.  The
    4668              :    arguments to the builtin_memset call DEST, VAL, and LEN are broken out
    4669              :    so that this can also be called without constructing an actual CALL_EXPR.
    4670              :    The other arguments and return value are the same as for
    4671              :    expand_builtin_memset.  */
    4672              : 
    4673              : static rtx
    4674        62909 : expand_builtin_memset_args (tree dest, tree val, tree len,
    4675              :                             rtx target, machine_mode mode, tree orig_exp)
    4676              : {
    4677        62909 :   tree fndecl, fn;
    4678        62909 :   enum built_in_function fcode;
    4679        62909 :   machine_mode val_mode;
    4680        62909 :   char c;
    4681        62909 :   unsigned int dest_align;
    4682        62909 :   rtx dest_mem, dest_addr, len_rtx;
    4683        62909 :   HOST_WIDE_INT expected_size = -1;
    4684        62909 :   unsigned int expected_align = 0;
    4685        62909 :   unsigned HOST_WIDE_INT min_size;
    4686        62909 :   unsigned HOST_WIDE_INT max_size;
    4687        62909 :   unsigned HOST_WIDE_INT probable_max_size;
    4688              : 
    4689        62909 :   dest_align = get_pointer_alignment (dest);
    4690              : 
    4691              :   /* If DEST is not a pointer type, don't do this operation in-line.  */
    4692        62909 :   if (dest_align == 0)
    4693              :     return NULL_RTX;
    4694              : 
    4695        62909 :   if (currently_expanding_gimple_stmt)
    4696        62909 :     stringop_block_profile (currently_expanding_gimple_stmt,
    4697              :                             &expected_align, &expected_size);
    4698              : 
    4699        62909 :   if (expected_align < dest_align)
    4700        62901 :     expected_align = dest_align;
    4701              : 
    4702              :   /* If the LEN parameter is zero, return DEST.  */
    4703        62909 :   if (integer_zerop (len))
    4704              :     {
    4705              :       /* Evaluate and ignore VAL in case it has side-effects.  */
    4706            0 :       expand_expr (val, const0_rtx, VOIDmode, EXPAND_NORMAL);
    4707            0 :       return expand_expr (dest, target, mode, EXPAND_NORMAL);
    4708              :     }
    4709              : 
    4710              :   /* Stabilize the arguments in case we fail.  */
    4711        62909 :   dest = builtin_save_expr (dest);
    4712        62909 :   val = builtin_save_expr (val);
    4713        62909 :   len = builtin_save_expr (len);
    4714              : 
    4715        62909 :   len_rtx = expand_normal (len);
    4716        62909 :   determine_block_size (len, len_rtx, &min_size, &max_size,
    4717              :                         &probable_max_size);
    4718        62909 :   dest_mem = get_memory_rtx (dest, len, true);
    4719        62909 :   val_mode = TYPE_MODE (unsigned_char_type_node);
    4720              : 
    4721        62909 :   if (TREE_CODE (val) != INTEGER_CST
    4722        62909 :       || target_char_cast (val, &c))
    4723              :     {
    4724         1728 :       rtx val_rtx;
    4725              : 
    4726         1728 :       val_rtx = expand_normal (val);
    4727         1728 :       val_rtx = convert_to_mode (val_mode, val_rtx, 0);
    4728              : 
    4729              :       /* Assume that we can memset by pieces if we can store
    4730              :        * the coefficients by pieces (in the required modes).
    4731              :        * We can't pass builtin_memset_gen_str as that emits RTL.  */
    4732         1728 :       c = 1;
    4733         1728 :       if (tree_fits_uhwi_p (len)
    4734         1728 :           && can_store_by_pieces (tree_to_uhwi (len),
    4735              :                                   builtin_memset_read_str, &c, dest_align,
    4736              :                                   true))
    4737              :         {
    4738          730 :           val_rtx = force_reg (val_mode, val_rtx);
    4739          730 :           store_by_pieces (dest_mem, tree_to_uhwi (len),
    4740              :                            builtin_memset_gen_str, val_rtx, dest_align,
    4741              :                            true, RETURN_BEGIN);
    4742              :         }
    4743          998 :       else if (!set_storage_via_setmem (dest_mem, len_rtx, val_rtx,
    4744              :                                         dest_align, expected_align,
    4745              :                                         expected_size, min_size, max_size,
    4746              :                                         probable_max_size)
    4747          998 :                && !try_store_by_multiple_pieces (dest_mem, len_rtx,
    4748              :                                                  tree_ctz (len),
    4749              :                                                  min_size, max_size,
    4750              :                                                  val_rtx, 0,
    4751              :                                                  dest_align))
    4752          636 :         goto do_libcall;
    4753              : 
    4754         1092 :       dest_mem = force_operand (XEXP (dest_mem, 0), NULL_RTX);
    4755         1092 :       dest_mem = convert_memory_address (ptr_mode, dest_mem);
    4756         1092 :       return dest_mem;
    4757              :     }
    4758              : 
    4759        61181 :   if (c)
    4760              :     {
    4761        13681 :       if (tree_fits_uhwi_p (len)
    4762        13681 :           && can_store_by_pieces (tree_to_uhwi (len),
    4763              :                                   builtin_memset_read_str, &c, dest_align,
    4764              :                                   true))
    4765         5869 :         store_by_pieces (dest_mem, tree_to_uhwi (len),
    4766              :                          builtin_memset_read_str, &c, dest_align, true,
    4767              :                          RETURN_BEGIN);
    4768         9760 :       else if (!set_storage_via_setmem (dest_mem, len_rtx,
    4769         7812 :                                         gen_int_mode (c, val_mode),
    4770              :                                         dest_align, expected_align,
    4771              :                                         expected_size, min_size, max_size,
    4772              :                                         probable_max_size)
    4773         7812 :                && !try_store_by_multiple_pieces (dest_mem, len_rtx,
    4774              :                                                  tree_ctz (len),
    4775              :                                                  min_size, max_size,
    4776              :                                                  NULL_RTX, c,
    4777              :                                                  dest_align))
    4778         5864 :         goto do_libcall;
    4779              : 
    4780         7817 :       dest_mem = force_operand (XEXP (dest_mem, 0), NULL_RTX);
    4781         7817 :       dest_mem = convert_memory_address (ptr_mode, dest_mem);
    4782         7817 :       return dest_mem;
    4783              :     }
    4784              : 
    4785        47500 :   set_mem_align (dest_mem, dest_align);
    4786        95000 :   dest_addr = clear_storage_hints (dest_mem, len_rtx,
    4787        47500 :                                    CALL_EXPR_TAILCALL (orig_exp)
    4788              :                                    ? BLOCK_OP_TAILCALL : BLOCK_OP_NORMAL,
    4789              :                                    expected_align, expected_size,
    4790              :                                    min_size, max_size,
    4791              :                                    probable_max_size, tree_ctz (len));
    4792              : 
    4793        47500 :   if (dest_addr == 0)
    4794              :     {
    4795        37482 :       dest_addr = force_operand (XEXP (dest_mem, 0), NULL_RTX);
    4796        37482 :       dest_addr = convert_memory_address (ptr_mode, dest_addr);
    4797              :     }
    4798              : 
    4799              :   return dest_addr;
    4800              : 
    4801         6500 :  do_libcall:
    4802         6500 :   fndecl = get_callee_fndecl (orig_exp);
    4803         6500 :   fcode = DECL_FUNCTION_CODE (fndecl);
    4804         6500 :   if (fcode == BUILT_IN_MEMSET)
    4805         6500 :     fn = build_call_nofold_loc (EXPR_LOCATION (orig_exp), fndecl, 3,
    4806              :                                 dest, val, len);
    4807            0 :   else if (fcode == BUILT_IN_BZERO)
    4808            0 :     fn = build_call_nofold_loc (EXPR_LOCATION (orig_exp), fndecl, 2,
    4809              :                                 dest, len);
    4810              :   else
    4811            0 :     gcc_unreachable ();
    4812         6500 :   gcc_assert (TREE_CODE (fn) == CALL_EXPR);
    4813         6500 :   CALL_EXPR_TAILCALL (fn) = CALL_EXPR_TAILCALL (orig_exp);
    4814         6500 :   return expand_call (fn, target, target == const0_rtx);
    4815              : }
    4816              : 
    4817              : /* Expand expression EXP, which is a call to the bzero builtin.  Return
    4818              :    NULL_RTX if we failed the caller should emit a normal call.  */
    4819              : 
    4820              : static rtx
    4821            0 : expand_builtin_bzero (tree exp)
    4822              : {
    4823            0 :   if (!validate_arglist (exp, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    4824              :     return NULL_RTX;
    4825              : 
    4826            0 :   tree dest = CALL_EXPR_ARG (exp, 0);
    4827            0 :   tree size = CALL_EXPR_ARG (exp, 1);
    4828              : 
    4829              :   /* New argument list transforming bzero(ptr x, int y) to
    4830              :      memset(ptr x, int 0, size_t y).   This is done this way
    4831              :      so that if it isn't expanded inline, we fallback to
    4832              :      calling bzero instead of memset.  */
    4833              : 
    4834            0 :   location_t loc = EXPR_LOCATION (exp);
    4835              : 
    4836            0 :   return expand_builtin_memset_args (dest, integer_zero_node,
    4837              :                                      fold_convert_loc (loc,
    4838              :                                                        size_type_node, size),
    4839            0 :                                      const0_rtx, VOIDmode, exp);
    4840              : }
    4841              : 
    4842              : /* Try to expand cmpstr operation ICODE with the given operands.
    4843              :    Return the result rtx on success, otherwise return null.  */
    4844              : 
    4845              : static rtx
    4846            0 : expand_cmpstr (insn_code icode, rtx target, rtx arg1_rtx, rtx arg2_rtx,
    4847              :                HOST_WIDE_INT align)
    4848              : {
    4849            0 :   machine_mode insn_mode = insn_data[icode].operand[0].mode;
    4850              : 
    4851            0 :   if (target && (!REG_P (target) || HARD_REGISTER_P (target)))
    4852              :     target = NULL_RTX;
    4853              : 
    4854            0 :   class expand_operand ops[4];
    4855            0 :   create_output_operand (&ops[0], target, insn_mode);
    4856            0 :   create_fixed_operand (&ops[1], arg1_rtx);
    4857            0 :   create_fixed_operand (&ops[2], arg2_rtx);
    4858            0 :   create_integer_operand (&ops[3], align);
    4859            0 :   if (maybe_expand_insn (icode, 4, ops))
    4860            0 :     return ops[0].value;
    4861              :   return NULL_RTX;
    4862              : }
    4863              : 
    4864              : /* Expand expression EXP, which is a call to the memcmp built-in function.
    4865              :    Return NULL_RTX if we failed and the caller should emit a normal call,
    4866              :    otherwise try to get the result in TARGET, if convenient.
    4867              :    RESULT_EQ is true if we can relax the returned value to be either zero
    4868              :    or nonzero, without caring about the sign.  */
    4869              : 
    4870              : static rtx
    4871       105674 : expand_builtin_memcmp (tree exp, rtx target, bool result_eq)
    4872              : {
    4873       105674 :   if (!validate_arglist (exp,
    4874              :                          POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    4875              :     return NULL_RTX;
    4876              : 
    4877       105670 :   tree arg1 = CALL_EXPR_ARG (exp, 0);
    4878       105670 :   tree arg2 = CALL_EXPR_ARG (exp, 1);
    4879       105670 :   tree len = CALL_EXPR_ARG (exp, 2);
    4880              : 
    4881              :   /* Due to the performance benefit, always inline the calls first
    4882              :      when result_eq is false.  */
    4883       105670 :   rtx result = NULL_RTX;
    4884       105670 :   enum built_in_function fcode = DECL_FUNCTION_CODE (get_callee_fndecl (exp));
    4885       105670 :   if (!result_eq && fcode != BUILT_IN_BCMP)
    4886              :     {
    4887        15274 :       result = inline_expand_builtin_bytecmp (exp, target);
    4888        15274 :       if (result)
    4889              :         return result;
    4890              :     }
    4891              : 
    4892       105652 :   machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
    4893       105652 :   location_t loc = EXPR_LOCATION (exp);
    4894              : 
    4895       105652 :   unsigned int arg1_align = get_pointer_alignment (arg1) / BITS_PER_UNIT;
    4896       105652 :   unsigned int arg2_align = get_pointer_alignment (arg2) / BITS_PER_UNIT;
    4897              : 
    4898              :   /* If we don't have POINTER_TYPE, call the function.  */
    4899       105652 :   if (arg1_align == 0 || arg2_align == 0)
    4900              :     return NULL_RTX;
    4901              : 
    4902       105652 :   rtx arg1_rtx = get_memory_rtx (arg1, len, false);
    4903       105652 :   rtx arg2_rtx = get_memory_rtx (arg2, len, false);
    4904       105652 :   rtx len_rtx = expand_normal (fold_convert_loc (loc, sizetype, len));
    4905              : 
    4906              :   /* Set MEM_SIZE as appropriate.  */
    4907       105652 :   if (CONST_INT_P (len_rtx))
    4908              :     {
    4909        80615 :       set_mem_size (arg1_rtx, INTVAL (len_rtx));
    4910        80615 :       set_mem_size (arg2_rtx, INTVAL (len_rtx));
    4911              :     }
    4912              : 
    4913       105652 :   by_pieces_constfn constfn = NULL;
    4914              : 
    4915              :   /* Try to get the byte representation of the constant ARG2 (or, only
    4916              :      when the function's result is used for equality to zero, ARG1)
    4917              :      points to, with its byte size in NBYTES.  */
    4918       105652 :   unsigned HOST_WIDE_INT nbytes;
    4919       105652 :   const char *rep = getbyterep (arg2, &nbytes);
    4920       105652 :   if (result_eq && rep == NULL)
    4921              :     {
    4922              :       /* For equality to zero the arguments are interchangeable.  */
    4923        63222 :       rep = getbyterep (arg1, &nbytes);
    4924        63222 :       if (rep != NULL)
    4925              :         std::swap (arg1_rtx, arg2_rtx);
    4926              :     }
    4927              : 
    4928              :   /* If the function's constant bound LEN_RTX is less than or equal
    4929              :      to the byte size of the representation of the constant argument,
    4930              :      and if block move would be done by pieces, we can avoid loading
    4931              :      the bytes from memory and only store the computed constant result.  */
    4932        42430 :   if (rep
    4933        29265 :       && CONST_INT_P (len_rtx)
    4934        29166 :       && (unsigned HOST_WIDE_INT) INTVAL (len_rtx) <= nbytes)
    4935       105652 :     constfn = builtin_memcpy_read_str;
    4936              : 
    4937       211304 :   result = emit_block_cmp_hints (arg1_rtx, arg2_rtx, len_rtx,
    4938       105652 :                                  TREE_TYPE (len), target,
    4939              :                                  result_eq, constfn,
    4940              :                                  const_cast<char *> (rep),
    4941              :                                  tree_ctz (len));
    4942              : 
    4943       105652 :   if (result)
    4944              :     {
    4945              :       /* Return the value in the proper mode for this function.  */
    4946        68090 :       if (GET_MODE (result) == mode)
    4947              :         return result;
    4948              : 
    4949            0 :       if (target != 0)
    4950              :         {
    4951            0 :           convert_move (target, result, 0);
    4952            0 :           return target;
    4953              :         }
    4954              : 
    4955            0 :       return convert_to_mode (mode, result, 0);
    4956              :     }
    4957              : 
    4958              :   return NULL_RTX;
    4959              : }
    4960              : 
    4961              : /* Expand expression EXP, which is a call to the strcmp builtin.  Return NULL_RTX
    4962              :    if we failed the caller should emit a normal call, otherwise try to get
    4963              :    the result in TARGET, if convenient.  */
    4964              : 
    4965              : static rtx
    4966       127947 : expand_builtin_strcmp (tree exp, ATTRIBUTE_UNUSED rtx target)
    4967              : {
    4968       127947 :   if (!validate_arglist (exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
    4969              :     return NULL_RTX;
    4970              : 
    4971       127932 :   tree arg1 = CALL_EXPR_ARG (exp, 0);
    4972       127932 :   tree arg2 = CALL_EXPR_ARG (exp, 1);
    4973              : 
    4974              :   /* Due to the performance benefit, always inline the calls first.  */
    4975       127932 :   rtx result = NULL_RTX;
    4976       127932 :   result = inline_expand_builtin_bytecmp (exp, target);
    4977       127932 :   if (result)
    4978              :     return result;
    4979              : 
    4980       127533 :   insn_code cmpstr_icode = direct_optab_handler (cmpstr_optab, SImode);
    4981       127533 :   insn_code cmpstrn_icode = direct_optab_handler (cmpstrn_optab, SImode);
    4982       127533 :   if (cmpstr_icode == CODE_FOR_nothing && cmpstrn_icode == CODE_FOR_nothing)
    4983              :     return NULL_RTX;
    4984              : 
    4985       127533 :   unsigned int arg1_align = get_pointer_alignment (arg1) / BITS_PER_UNIT;
    4986       127533 :   unsigned int arg2_align = get_pointer_alignment (arg2) / BITS_PER_UNIT;
    4987              : 
    4988              :   /* If we don't have POINTER_TYPE, call the function.  */
    4989       127533 :   if (arg1_align == 0 || arg2_align == 0)
    4990              :     return NULL_RTX;
    4991              : 
    4992              :   /* Stabilize the arguments in case gen_cmpstr(n)si fail.  */
    4993       127533 :   arg1 = builtin_save_expr (arg1);
    4994       127533 :   arg2 = builtin_save_expr (arg2);
    4995              : 
    4996       127533 :   rtx arg1_rtx = get_memory_rtx (arg1, NULL, false);
    4997       127533 :   rtx arg2_rtx = get_memory_rtx (arg2, NULL, false);
    4998              : 
    4999              :   /* Try to call cmpstrsi.  */
    5000       127533 :   if (cmpstr_icode != CODE_FOR_nothing)
    5001            0 :     result = expand_cmpstr (cmpstr_icode, target, arg1_rtx, arg2_rtx,
    5002            0 :                             MIN (arg1_align, arg2_align));
    5003              : 
    5004              :   /* Try to determine at least one length and call cmpstrnsi.  */
    5005       127533 :   if (!result && cmpstrn_icode != CODE_FOR_nothing)
    5006              :     {
    5007       127533 :       tree len;
    5008       127533 :       rtx arg3_rtx;
    5009              : 
    5010       127533 :       tree len1 = c_strlen (arg1, 1);
    5011       127533 :       tree len2 = c_strlen (arg2, 1);
    5012              : 
    5013       127533 :       if (len1)
    5014          224 :         len1 = size_binop (PLUS_EXPR, ssize_int (1), len1);
    5015       127533 :       if (len2)
    5016       125909 :         len2 = size_binop (PLUS_EXPR, ssize_int (1), len2);
    5017              : 
    5018              :       /* If we don't have a constant length for the first, use the length
    5019              :          of the second, if we know it.  We don't require a constant for
    5020              :          this case; some cost analysis could be done if both are available
    5021              :          but neither is constant.  For now, assume they're equally cheap,
    5022              :          unless one has side effects.  If both strings have constant lengths,
    5023              :          use the smaller.  */
    5024              : 
    5025       127533 :       if (!len1)
    5026              :         len = len2;
    5027          224 :       else if (!len2)
    5028              :         len = len1;
    5029           17 :       else if (TREE_SIDE_EFFECTS (len1))
    5030              :         len = len2;
    5031           17 :       else if (TREE_SIDE_EFFECTS (len2))
    5032              :         len = len1;
    5033           17 :       else if (TREE_CODE (len1) != INTEGER_CST)
    5034              :         len = len2;
    5035           17 :       else if (TREE_CODE (len2) != INTEGER_CST)
    5036              :         len = len1;
    5037           10 :       else if (tree_int_cst_lt (len1, len2))
    5038              :         len = len1;
    5039              :       else
    5040       127533 :         len = len2;
    5041              : 
    5042              :       /* If both arguments have side effects, we cannot optimize.  */
    5043       127533 :       if (len && !TREE_SIDE_EFFECTS (len))
    5044              :         {
    5045       126116 :           arg3_rtx = expand_normal (len);
    5046       126116 :           result = expand_cmpstrn_or_cmpmem
    5047       126116 :             (cmpstrn_icode, target, arg1_rtx, arg2_rtx, TREE_TYPE (len),
    5048       126116 :              arg3_rtx, MIN (arg1_align, arg2_align));
    5049              :         }
    5050              :     }
    5051              : 
    5052       127533 :   tree fndecl = get_callee_fndecl (exp);
    5053       127533 :   if (result)
    5054              :     {
    5055              :       /* Return the value in the proper mode for this function.  */
    5056           52 :       machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
    5057           52 :       if (GET_MODE (result) == mode)
    5058              :         return result;
    5059            0 :       if (target == 0)
    5060            0 :         return convert_to_mode (mode, result, 0);
    5061            0 :       convert_move (target, result, 0);
    5062            0 :       return target;
    5063              :     }
    5064              : 
    5065              :   /* Expand the library call ourselves using a stabilized argument
    5066              :      list to avoid re-evaluating the function's arguments twice.  */
    5067       127481 :   tree fn = build_call_nofold_loc (EXPR_LOCATION (exp), fndecl, 2, arg1, arg2);
    5068       127481 :   copy_warning (fn, exp);
    5069       127481 :   gcc_assert (TREE_CODE (fn) == CALL_EXPR);
    5070       127481 :   CALL_EXPR_TAILCALL (fn) = CALL_EXPR_TAILCALL (exp);
    5071       127481 :   return expand_call (fn, target, target == const0_rtx);
    5072              : }
    5073              : 
    5074              : /* Expand expression EXP, which is a call to the strncmp builtin. Return
    5075              :    NULL_RTX if we failed the caller should emit a normal call, otherwise
    5076              :    try to get the result in TARGET, if convenient.  */
    5077              : 
    5078              : static rtx
    5079         2027 : expand_builtin_strncmp (tree exp, ATTRIBUTE_UNUSED rtx target,
    5080              :                         ATTRIBUTE_UNUSED machine_mode mode)
    5081              : {
    5082         2027 :   if (!validate_arglist (exp,
    5083              :                          POINTER_TYPE, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    5084              :     return NULL_RTX;
    5085              : 
    5086         2021 :   tree arg1 = CALL_EXPR_ARG (exp, 0);
    5087         2021 :   tree arg2 = CALL_EXPR_ARG (exp, 1);
    5088         2021 :   tree arg3 = CALL_EXPR_ARG (exp, 2);
    5089              : 
    5090         2021 :   location_t loc = EXPR_LOCATION (exp);
    5091         2021 :   tree len1 = c_strlen (arg1, 1);
    5092         2021 :   tree len2 = c_strlen (arg2, 1);
    5093              : 
    5094              :   /* Due to the performance benefit, always inline the calls first.  */
    5095         2021 :   rtx result = NULL_RTX;
    5096         2021 :   result = inline_expand_builtin_bytecmp (exp, target);
    5097         2021 :   if (result)
    5098              :     return result;
    5099              : 
    5100              :   /* If c_strlen can determine an expression for one of the string
    5101              :      lengths, and it doesn't have side effects, then emit cmpstrnsi
    5102              :      using length MIN(strlen(string)+1, arg3).  */
    5103         1803 :   insn_code cmpstrn_icode = direct_optab_handler (cmpstrn_optab, SImode);
    5104         1803 :   if (cmpstrn_icode == CODE_FOR_nothing)
    5105              :     return NULL_RTX;
    5106              : 
    5107         1803 :   tree len;
    5108              : 
    5109         1803 :   unsigned int arg1_align = get_pointer_alignment (arg1) / BITS_PER_UNIT;
    5110         1803 :   unsigned int arg2_align = get_pointer_alignment (arg2) / BITS_PER_UNIT;
    5111              : 
    5112         1803 :   if (len1)
    5113          160 :     len1 = size_binop_loc (loc, PLUS_EXPR, ssize_int (1), len1);
    5114         1803 :   if (len2)
    5115          674 :     len2 = size_binop_loc (loc, PLUS_EXPR, ssize_int (1), len2);
    5116              : 
    5117         1803 :   tree len3 = fold_convert_loc (loc, sizetype, arg3);
    5118              : 
    5119              :   /* If we don't have a constant length for the first, use the length
    5120              :      of the second, if we know it.  If neither string is constant length,
    5121              :      use the given length argument.  We don't require a constant for
    5122              :      this case; some cost analysis could be done if both are available
    5123              :      but neither is constant.  For now, assume they're equally cheap,
    5124              :      unless one has side effects.  If both strings have constant lengths,
    5125              :      use the smaller.  */
    5126              : 
    5127         1803 :   if (!len1 && !len2)
    5128              :     len = len3;
    5129          717 :   else if (!len1)
    5130              :     len = len2;
    5131          160 :   else if (!len2)
    5132              :     len = len1;
    5133          117 :   else if (TREE_SIDE_EFFECTS (len1))
    5134              :     len = len2;
    5135          117 :   else if (TREE_SIDE_EFFECTS (len2))
    5136              :     len = len1;
    5137          117 :   else if (TREE_CODE (len1) != INTEGER_CST)
    5138              :     len = len2;
    5139          117 :   else if (TREE_CODE (len2) != INTEGER_CST)
    5140              :     len = len1;
    5141          117 :   else if (tree_int_cst_lt (len1, len2))
    5142              :     len = len1;
    5143              :   else
    5144         1803 :     len = len2;
    5145              : 
    5146              :   /* If we are not using the given length, we must incorporate it here.
    5147              :      The actual new length parameter will be MIN(len,arg3) in this case.  */
    5148         1803 :   if (len != len3)
    5149              :     {
    5150          717 :       len = fold_convert_loc (loc, sizetype, len);
    5151          717 :       len = fold_build2_loc (loc, MIN_EXPR, TREE_TYPE (len), len, len3);
    5152              :     }
    5153         1803 :   rtx arg1_rtx = get_memory_rtx (arg1, len, false);
    5154         1803 :   rtx arg2_rtx = get_memory_rtx (arg2, len, false);
    5155         1803 :   rtx arg3_rtx = expand_normal (len);
    5156         1803 :   result = expand_cmpstrn_or_cmpmem (cmpstrn_icode, target, arg1_rtx,
    5157         1803 :                                      arg2_rtx, TREE_TYPE (len), arg3_rtx,
    5158         1803 :                                      MIN (arg1_align, arg2_align));
    5159              : 
    5160         1803 :   tree fndecl = get_callee_fndecl (exp);
    5161         1803 :   if (result)
    5162              :     {
    5163              :       /* Return the value in the proper mode for this function.  */
    5164           17 :       mode = TYPE_MODE (TREE_TYPE (exp));
    5165           17 :       if (GET_MODE (result) == mode)
    5166              :         return result;
    5167            0 :       if (target == 0)
    5168            0 :         return convert_to_mode (mode, result, 0);
    5169            0 :       convert_move (target, result, 0);
    5170            0 :       return target;
    5171              :     }
    5172              : 
    5173              :   /* Expand the library call ourselves using a stabilized argument
    5174              :      list to avoid re-evaluating the function's arguments twice.  */
    5175         1786 :   tree call = build_call_nofold_loc (loc, fndecl, 3, arg1, arg2, len);
    5176         1786 :   copy_warning (call, exp);
    5177         1786 :   gcc_assert (TREE_CODE (call) == CALL_EXPR);
    5178         1786 :   CALL_EXPR_TAILCALL (call) = CALL_EXPR_TAILCALL (exp);
    5179         1786 :   return expand_call (call, target, target == const0_rtx);
    5180              : }
    5181              : 
    5182              : /* Expand a call to __builtin_saveregs, generating the result in TARGET,
    5183              :    if that's convenient.  */
    5184              : 
    5185              : rtx
    5186            0 : expand_builtin_saveregs (void)
    5187              : {
    5188            0 :   rtx val;
    5189            0 :   rtx_insn *seq;
    5190              : 
    5191              :   /* Don't do __builtin_saveregs more than once in a function.
    5192              :      Save the result of the first call and reuse it.  */
    5193            0 :   if (saveregs_value != 0)
    5194              :     return saveregs_value;
    5195              : 
    5196              :   /* When this function is called, it means that registers must be
    5197              :      saved on entry to this function.  So we migrate the call to the
    5198              :      first insn of this function.  */
    5199              : 
    5200            0 :   start_sequence ();
    5201              : 
    5202              :   /* Do whatever the machine needs done in this case.  */
    5203            0 :   val = targetm.calls.expand_builtin_saveregs ();
    5204              : 
    5205            0 :   seq = end_sequence ();
    5206              : 
    5207            0 :   saveregs_value = val;
    5208              : 
    5209              :   /* Put the insns after the NOTE that starts the function.  If this
    5210              :      is inside a start_sequence, make the outer-level insn chain current, so
    5211              :      the code is placed at the start of the function.  */
    5212            0 :   push_topmost_sequence ();
    5213            0 :   emit_insn_after (seq, entry_of_function ());
    5214            0 :   pop_topmost_sequence ();
    5215              : 
    5216            0 :   return val;
    5217              : }
    5218              : 
    5219              : /* Expand a call to __builtin_next_arg.  */
    5220              : 
    5221              : static rtx
    5222        21243 : expand_builtin_next_arg (void)
    5223              : {
    5224              :   /* Checking arguments is already done in fold_builtin_next_arg
    5225              :      that must be called before this function.  */
    5226        21243 :   return expand_binop (ptr_mode, add_optab,
    5227              :                        crtl->args.internal_arg_pointer,
    5228              :                        crtl->args.arg_offset_rtx,
    5229        21243 :                        NULL_RTX, 0, OPTAB_LIB_WIDEN);
    5230              : }
    5231              : 
    5232              : /* Make it easier for the backends by protecting the valist argument
    5233              :    from multiple evaluations.  */
    5234              : 
    5235              : static tree
    5236        21565 : stabilize_va_list_loc (location_t loc, tree valist, int needs_lvalue)
    5237              : {
    5238        21565 :   tree vatype = targetm.canonical_va_list_type (TREE_TYPE (valist));
    5239              : 
    5240              :   /* The current way of determining the type of valist is completely
    5241              :      bogus.  We should have the information on the va builtin instead.  */
    5242        21565 :   if (!vatype)
    5243        21441 :     vatype = targetm.fn_abi_va_list (cfun->decl);
    5244              : 
    5245        21565 :   if (TREE_CODE (vatype) == ARRAY_TYPE)
    5246              :     {
    5247        15897 :       if (TREE_SIDE_EFFECTS (valist))
    5248            0 :         valist = save_expr (valist);
    5249              : 
    5250              :       /* For this case, the backends will be expecting a pointer to
    5251              :          vatype, but it's possible we've actually been given an array
    5252              :          (an actual TARGET_CANONICAL_VA_LIST_TYPE (valist)).
    5253              :          So fix it.  */
    5254        15897 :       if (TREE_CODE (TREE_TYPE (valist)) == ARRAY_TYPE)
    5255              :         {
    5256            0 :           tree p1 = build_pointer_type (TREE_TYPE (vatype));
    5257            0 :           valist = build_fold_addr_expr_with_type_loc (loc, valist, p1);
    5258              :         }
    5259              :     }
    5260              :   else
    5261              :     {
    5262         5668 :       tree pt = build_pointer_type (vatype);
    5263              : 
    5264         5668 :       if (! needs_lvalue)
    5265              :         {
    5266            6 :           if (! TREE_SIDE_EFFECTS (valist))
    5267              :             return valist;
    5268              : 
    5269            0 :           valist = fold_build1_loc (loc, ADDR_EXPR, pt, valist);
    5270            0 :           TREE_SIDE_EFFECTS (valist) = 1;
    5271              :         }
    5272              : 
    5273         5662 :       if (TREE_SIDE_EFFECTS (valist))
    5274            0 :         valist = save_expr (valist);
    5275         5662 :       valist = fold_build2_loc (loc, MEM_REF,
    5276              :                                 vatype, valist, build_int_cst (pt, 0));
    5277              :     }
    5278              : 
    5279              :   return valist;
    5280              : }
    5281              : 
    5282              : /* The "standard" definition of va_list is void*.  */
    5283              : 
    5284              : tree
    5285            0 : std_build_builtin_va_list (void)
    5286              : {
    5287            0 :   return ptr_type_node;
    5288              : }
    5289              : 
    5290              : /* The "standard" abi va_list is va_list_type_node.  */
    5291              : 
    5292              : tree
    5293            0 : std_fn_abi_va_list (tree fndecl ATTRIBUTE_UNUSED)
    5294              : {
    5295            0 :   return va_list_type_node;
    5296              : }
    5297              : 
    5298              : /* The "standard" type of va_list is va_list_type_node.  */
    5299              : 
    5300              : tree
    5301          502 : std_canonical_va_list_type (tree type)
    5302              : {
    5303          502 :   tree wtype, htype;
    5304              : 
    5305          502 :   wtype = va_list_type_node;
    5306          502 :   htype = type;
    5307              : 
    5308          502 :   if (TREE_CODE (wtype) == ARRAY_TYPE)
    5309              :     {
    5310              :       /* If va_list is an array type, the argument may have decayed
    5311              :          to a pointer type, e.g. by being passed to another function.
    5312              :          In that case, unwrap both types so that we can compare the
    5313              :          underlying records.  */
    5314            0 :       if (TREE_CODE (htype) == ARRAY_TYPE
    5315            0 :           || POINTER_TYPE_P (htype))
    5316              :         {
    5317            0 :           wtype = TREE_TYPE (wtype);
    5318            0 :           htype = TREE_TYPE (htype);
    5319              :         }
    5320              :     }
    5321          502 :   if (TYPE_MAIN_VARIANT (wtype) == TYPE_MAIN_VARIANT (htype))
    5322          332 :     return va_list_type_node;
    5323              : 
    5324              :   return NULL_TREE;
    5325              : }
    5326              : 
    5327              : /* The "standard" implementation of va_start: just assign `nextarg' to
    5328              :    the variable.  */
    5329              : 
    5330              : void
    5331         5658 : std_expand_builtin_va_start (tree valist, rtx nextarg)
    5332              : {
    5333         5658 :   rtx va_r = expand_expr (valist, NULL_RTX, VOIDmode, EXPAND_WRITE);
    5334         5658 :   convert_move (va_r, nextarg, 0);
    5335         5658 : }
    5336              : 
    5337              : /* Expand EXP, a call to __builtin_va_start.  */
    5338              : 
    5339              : static rtx
    5340        21085 : expand_builtin_va_start (tree exp)
    5341              : {
    5342        21085 :   rtx nextarg;
    5343        21085 :   tree valist;
    5344        21085 :   location_t loc = EXPR_LOCATION (exp);
    5345              : 
    5346        21085 :   if (call_expr_nargs (exp) < 2)
    5347              :     {
    5348            0 :       error_at (loc, "too few arguments to function %<va_start%>");
    5349            0 :       return const0_rtx;
    5350              :     }
    5351              : 
    5352        21085 :   if (fold_builtin_next_arg (exp, true))
    5353            0 :     return const0_rtx;
    5354              : 
    5355        21085 :   nextarg = expand_builtin_next_arg ();
    5356        21085 :   valist = stabilize_va_list_loc (loc, CALL_EXPR_ARG (exp, 0), 1);
    5357              : 
    5358        21085 :   if (targetm.expand_builtin_va_start)
    5359        21085 :     targetm.expand_builtin_va_start (valist, nextarg);
    5360              :   else
    5361            0 :     std_expand_builtin_va_start (valist, nextarg);
    5362              : 
    5363        21085 :   return const0_rtx;
    5364              : }
    5365              : 
    5366              : /* Expand EXP, a call to __builtin_va_end.  */
    5367              : 
    5368              : static rtx
    5369        12168 : expand_builtin_va_end (tree exp)
    5370              : {
    5371        12168 :   tree valist = CALL_EXPR_ARG (exp, 0);
    5372              : 
    5373              :   /* Evaluate for side effects, if needed.  I hate macros that don't
    5374              :      do that.  */
    5375        12168 :   if (TREE_SIDE_EFFECTS (valist))
    5376            0 :     expand_expr (valist, const0_rtx, VOIDmode, EXPAND_NORMAL);
    5377              : 
    5378        12168 :   return const0_rtx;
    5379              : }
    5380              : 
    5381              : /* Expand EXP, a call to __builtin_va_copy.  We do this as a
    5382              :    builtin rather than just as an assignment in stdarg.h because of the
    5383              :    nastiness of array-type va_list types.  */
    5384              : 
    5385              : static rtx
    5386          240 : expand_builtin_va_copy (tree exp)
    5387              : {
    5388          240 :   tree dst, src, t;
    5389          240 :   location_t loc = EXPR_LOCATION (exp);
    5390              : 
    5391          240 :   dst = CALL_EXPR_ARG (exp, 0);
    5392          240 :   src = CALL_EXPR_ARG (exp, 1);
    5393              : 
    5394          240 :   dst = stabilize_va_list_loc (loc, dst, 1);
    5395          240 :   src = stabilize_va_list_loc (loc, src, 0);
    5396              : 
    5397          240 :   gcc_assert (cfun != NULL && cfun->decl != NULL_TREE);
    5398              : 
    5399          240 :   if (TREE_CODE (targetm.fn_abi_va_list (cfun->decl)) != ARRAY_TYPE)
    5400              :     {
    5401            0 :       t = build2 (MODIFY_EXPR, targetm.fn_abi_va_list (cfun->decl), dst, src);
    5402            0 :       TREE_SIDE_EFFECTS (t) = 1;
    5403            0 :       expand_expr (t, const0_rtx, VOIDmode, EXPAND_NORMAL);
    5404              :     }
    5405              :   else
    5406              :     {
    5407          240 :       rtx dstb, srcb, size;
    5408              : 
    5409              :       /* Evaluate to pointers.  */
    5410          240 :       dstb = expand_expr (dst, NULL_RTX, Pmode, EXPAND_NORMAL);
    5411          240 :       srcb = expand_expr (src, NULL_RTX, Pmode, EXPAND_NORMAL);
    5412          240 :       size = expand_expr (TYPE_SIZE_UNIT (targetm.fn_abi_va_list (cfun->decl)),
    5413              :                   NULL_RTX, VOIDmode, EXPAND_NORMAL);
    5414              : 
    5415          240 :       dstb = convert_memory_address (Pmode, dstb);
    5416          240 :       srcb = convert_memory_address (Pmode, srcb);
    5417              : 
    5418              :       /* "Dereference" to BLKmode memories.  */
    5419          240 :       dstb = gen_rtx_MEM (BLKmode, dstb);
    5420          240 :       set_mem_alias_set (dstb, get_alias_set (TREE_TYPE (TREE_TYPE (dst))));
    5421          240 :       set_mem_align (dstb, TYPE_ALIGN (targetm.fn_abi_va_list (cfun->decl)));
    5422          240 :       srcb = gen_rtx_MEM (BLKmode, srcb);
    5423          240 :       set_mem_alias_set (srcb, get_alias_set (TREE_TYPE (TREE_TYPE (src))));
    5424          240 :       set_mem_align (srcb, TYPE_ALIGN (targetm.fn_abi_va_list (cfun->decl)));
    5425              : 
    5426              :       /* Copy.  */
    5427          240 :       emit_block_move (dstb, srcb, size, BLOCK_OP_NORMAL);
    5428              :     }
    5429              : 
    5430          240 :   return const0_rtx;
    5431              : }
    5432              : 
    5433              : /* Expand a call to one of the builtin functions __builtin_frame_address or
    5434              :    __builtin_return_address.  */
    5435              : 
    5436              : static rtx
    5437        15936 : expand_builtin_frame_address (tree fndecl, tree exp)
    5438              : {
    5439              :   /* The argument must be a nonnegative integer constant.
    5440              :      It counts the number of frames to scan up the stack.
    5441              :      The value is either the frame pointer value or the return
    5442              :      address saved in that frame.  */
    5443        15936 :   if (call_expr_nargs (exp) == 0)
    5444              :     /* Warning about missing arg was already issued.  */
    5445            0 :     return const0_rtx;
    5446        15936 :   else if (! tree_fits_uhwi_p (CALL_EXPR_ARG (exp, 0)))
    5447              :     {
    5448            0 :       error ("invalid argument to %qD", fndecl);
    5449            0 :       return const0_rtx;
    5450              :     }
    5451              :   else
    5452              :     {
    5453              :       /* Number of frames to scan up the stack.  */
    5454        15936 :       unsigned HOST_WIDE_INT count = tree_to_uhwi (CALL_EXPR_ARG (exp, 0));
    5455              : 
    5456        15936 :       rtx tem = expand_builtin_return_addr (DECL_FUNCTION_CODE (fndecl), count);
    5457              : 
    5458              :       /* Some ports cannot access arbitrary stack frames.  */
    5459        15936 :       if (tem == NULL)
    5460              :         {
    5461            0 :           warning (0, "unsupported argument to %qD", fndecl);
    5462            0 :           return const0_rtx;
    5463              :         }
    5464              : 
    5465        15936 :       if (count)
    5466              :         {
    5467              :           /* Warn since no effort is made to ensure that any frame
    5468              :              beyond the current one exists or can be safely reached.  */
    5469          938 :           warning (OPT_Wframe_address, "calling %qD with "
    5470              :                    "a nonzero argument is unsafe", fndecl);
    5471              :         }
    5472              : 
    5473              :       /* For __builtin_frame_address, return what we've got.  */
    5474        15936 :       if (DECL_FUNCTION_CODE (fndecl) == BUILT_IN_FRAME_ADDRESS)
    5475              :         return tem;
    5476              : 
    5477         5173 :       if (!REG_P (tem)
    5478         5173 :           && ! CONSTANT_P (tem))
    5479         5173 :         tem = copy_addr_to_reg (tem);
    5480              :       return tem;
    5481              :     }
    5482              : }
    5483              : 
    5484              : #if ! STACK_GROWS_DOWNWARD
    5485              : # define STACK_TOPS GT
    5486              : #else
    5487              : # define STACK_TOPS LT
    5488              : #endif
    5489              : 
    5490              : #ifdef POINTERS_EXTEND_UNSIGNED
    5491              : # define STACK_UNSIGNED POINTERS_EXTEND_UNSIGNED
    5492              : #else
    5493              : # define STACK_UNSIGNED true
    5494              : #endif
    5495              : 
    5496              : /* Expand a call to builtin function __builtin_stack_address.  */
    5497              : 
    5498              : static rtx
    5499         3455 : expand_builtin_stack_address ()
    5500              : {
    5501         3455 :   rtx ret = convert_to_mode (ptr_mode, copy_to_reg (stack_pointer_rtx),
    5502              :                              STACK_UNSIGNED);
    5503              : 
    5504              : #ifdef STACK_ADDRESS_OFFSET
    5505              :   /* Unbias the stack pointer, bringing it to the boundary between the
    5506              :      stack area claimed by the active function calling this builtin,
    5507              :      and stack ranges that could get clobbered if it called another
    5508              :      function.  It should NOT encompass any stack red zone, that is
    5509              :      used in leaf functions.
    5510              : 
    5511              :      On SPARC, the register save area is *not* considered active or
    5512              :      used by the active function, but rather as akin to the area in
    5513              :      which call-preserved registers are saved by callees.  This
    5514              :      enables __strub_leave to clear what would otherwise overlap with
    5515              :      its own register save area.
    5516              : 
    5517              :      If the address is computed too high or too low, parts of a stack
    5518              :      range that should be scrubbed may be left unscrubbed, scrubbing
    5519              :      may corrupt active portions of the stack frame, and stack ranges
    5520              :      may be doubly-scrubbed by caller and callee.
    5521              : 
    5522              :      In order for it to be just right, the area delimited by
    5523              :      @code{__builtin_stack_address} and @code{__builtin_frame_address
    5524              :      (0)} should encompass caller's registers saved by the function,
    5525              :      local on-stack variables and @code{alloca} stack areas.
    5526              :      Accumulated outgoing on-stack arguments, preallocated as part of
    5527              :      a function's own prologue, are to be regarded as part of the
    5528              :      (caller) function's active area as well, whereas those pushed or
    5529              :      allocated temporarily for a call are regarded as part of the
    5530              :      callee's stack range, rather than the caller's.  */
    5531              :   ret = plus_constant (ptr_mode, ret, STACK_ADDRESS_OFFSET);
    5532              : #endif
    5533              : 
    5534         3455 :   return force_reg (ptr_mode, ret);
    5535              : }
    5536              : 
    5537              : /* Expand a call to builtin function __builtin_strub_enter.  */
    5538              : 
    5539              : static rtx
    5540         2170 : expand_builtin_strub_enter (tree exp)
    5541              : {
    5542         2170 :   if (!validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    5543              :     return NULL_RTX;
    5544              : 
    5545         2170 :   if (optimize < 1 || flag_no_inline)
    5546              :     return NULL_RTX;
    5547              : 
    5548         1526 :   rtx stktop = expand_builtin_stack_address ();
    5549              : 
    5550         1526 :   tree wmptr = CALL_EXPR_ARG (exp, 0);
    5551         1526 :   tree wmtype = TREE_TYPE (TREE_TYPE (wmptr));
    5552         1526 :   tree wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
    5553              :                              build_int_cst (TREE_TYPE (wmptr), 0));
    5554         1526 :   rtx wmark = expand_expr (wmtree, NULL_RTX, ptr_mode, EXPAND_MEMORY);
    5555              : 
    5556         1526 :   emit_move_insn (wmark, stktop);
    5557              : 
    5558         1526 :   return const0_rtx;
    5559              : }
    5560              : 
    5561              : /* Expand a call to builtin function __builtin_strub_update.  */
    5562              : 
    5563              : static rtx
    5564         1085 : expand_builtin_strub_update (tree exp)
    5565              : {
    5566         1085 :   if (!validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    5567              :     return NULL_RTX;
    5568              : 
    5569         1085 :   if (optimize < 2 || flag_no_inline)
    5570              :     return NULL_RTX;
    5571              : 
    5572          663 :   rtx stktop = expand_builtin_stack_address ();
    5573              : 
    5574              : #ifdef RED_ZONE_SIZE
    5575              :   /* Here's how the strub enter, update and leave functions deal with red zones.
    5576              : 
    5577              :      If it weren't for red zones, update, called from within a strub context,
    5578              :      would bump the watermark to the top of the stack.  Enter and leave, running
    5579              :      in the caller, would use the caller's top of stack address both to
    5580              :      initialize the watermark passed to the callee, and to start strubbing the
    5581              :      stack afterwards.
    5582              : 
    5583              :      Ideally, we'd update the watermark so as to cover the used amount of red
    5584              :      zone, and strub starting at the caller's other end of the (presumably
    5585              :      unused) red zone.  Normally, only leaf functions use the red zone, but at
    5586              :      this point we can't tell whether a function is a leaf, nor can we tell how
    5587              :      much of the red zone it uses.  Furthermore, some strub contexts may have
    5588              :      been inlined so that update and leave are called from the same stack frame,
    5589              :      and the strub builtins may all have been inlined, turning a strub function
    5590              :      into a leaf.
    5591              : 
    5592              :      So cleaning the range from the caller's stack pointer (one end of the red
    5593              :      zone) to the (potentially inlined) callee's (other end of the) red zone
    5594              :      could scribble over the caller's own red zone.
    5595              : 
    5596              :      We avoid this possibility by arranging for callers that are strub contexts
    5597              :      to use their own watermark as the strub starting point.  So, if A calls B,
    5598              :      and B calls C, B will tell A to strub up to the end of B's red zone, and
    5599              :      will strub itself only the part of C's stack frame and red zone that
    5600              :      doesn't overlap with B's.  With that, we don't need to know who's leaf and
    5601              :      who isn't: inlined calls will shrink their strub window to zero, each
    5602              :      remaining call will strub some portion of the stack, and eventually the
    5603              :      strub context will return to a caller that isn't a strub context itself,
    5604              :      that will therefore use its own stack pointer as the strub starting point.
    5605              :      It's not a leaf, because strub contexts can't be inlined into non-strub
    5606              :      contexts, so it doesn't use the red zone, and it will therefore correctly
    5607              :      strub up the callee's stack frame up to the end of the callee's red zone.
    5608              :      Neat!  */
    5609          663 :   if (true /* (flags_from_decl_or_type (current_function_decl) & ECF_LEAF) */)
    5610              :     {
    5611          663 :       poly_int64 red_zone_size = RED_ZONE_SIZE;
    5612              : #if STACK_GROWS_DOWNWARD
    5613          663 :       red_zone_size = -red_zone_size;
    5614              : #endif
    5615          663 :       stktop = plus_constant (ptr_mode, stktop, red_zone_size);
    5616          663 :       stktop = force_reg (ptr_mode, stktop);
    5617              :     }
    5618              : #endif
    5619              : 
    5620          663 :   tree wmptr = CALL_EXPR_ARG (exp, 0);
    5621          663 :   tree wmtype = TREE_TYPE (TREE_TYPE (wmptr));
    5622          663 :   tree wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
    5623              :                              build_int_cst (TREE_TYPE (wmptr), 0));
    5624          663 :   rtx wmark = expand_expr (wmtree, NULL_RTX, ptr_mode, EXPAND_MEMORY);
    5625              : 
    5626          663 :   rtx wmark_load = shallow_copy_rtx (wmark);
    5627          663 :   MEM_NOTRAP_P (wmark_load) = !tree_could_trap_p (wmtree);
    5628          663 :   rtx wmarkr = force_reg (ptr_mode, wmark_load);
    5629              : 
    5630          663 :   rtx_code_label *lab = gen_label_rtx ();
    5631          663 :   do_compare_rtx_and_jump (stktop, wmarkr, STACK_TOPS, STACK_UNSIGNED,
    5632              :                            ptr_mode, NULL_RTX, lab, NULL,
    5633              :                            profile_probability::very_likely ());
    5634          663 :   emit_move_insn (wmark, stktop);
    5635              : 
    5636              :   /* If this is an inlined strub function, also bump the watermark for the
    5637              :      enclosing function.  This avoids a problem with the following scenario: A
    5638              :      calls B and B calls C, and both B and C get inlined into A.  B allocates
    5639              :      temporary stack space before calling C.  If we don't update A's watermark,
    5640              :      we may use an outdated baseline for the post-C strub_leave, erasing B's
    5641              :      temporary stack allocation.  We only need this if we're fully expanding
    5642              :      strub_leave inline.  */
    5643          663 :   tree xwmptr = (optimize > 2
    5644          663 :                  ? strub_watermark_parm (current_function_decl)
    5645              :                  : wmptr);
    5646          663 :   if (wmptr != xwmptr)
    5647              :     {
    5648          156 :       wmptr = xwmptr;
    5649          156 :       wmtype = TREE_TYPE (TREE_TYPE (wmptr));
    5650          156 :       wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
    5651              :                             build_int_cst (TREE_TYPE (wmptr), 0));
    5652          156 :       wmark = expand_expr (wmtree, NULL_RTX, ptr_mode, EXPAND_MEMORY);
    5653          156 :       wmark_load = shallow_copy_rtx (wmark);
    5654          156 :       MEM_NOTRAP_P (wmark_load) = !tree_could_trap_p (wmtree);
    5655          156 :       wmarkr = force_reg (ptr_mode, wmark_load);
    5656              : 
    5657          156 :       do_compare_rtx_and_jump (stktop, wmarkr, STACK_TOPS, STACK_UNSIGNED,
    5658              :                                ptr_mode, NULL_RTX, lab, NULL,
    5659              :                                profile_probability::very_likely ());
    5660          156 :       emit_move_insn (wmark, stktop);
    5661              :     }
    5662              : 
    5663          663 :   emit_label (lab);
    5664              : 
    5665          663 :   return const0_rtx;
    5666              : }
    5667              : 
    5668              : 
    5669              : /* Expand a call to builtin function __builtin_strub_leave.  */
    5670              : 
    5671              : static rtx
    5672         2740 : expand_builtin_strub_leave (tree exp)
    5673              : {
    5674         2740 :   if (!validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    5675              :     return NULL_RTX;
    5676              : 
    5677         2740 :   if (optimize < 2 || optimize_size || flag_no_inline)
    5678              :     return NULL_RTX;
    5679              : 
    5680         1239 :   rtx stktop = NULL_RTX;
    5681              : 
    5682         1239 :   if (tree wmptr = (optimize
    5683         1239 :                     ? strub_watermark_parm (current_function_decl)
    5684              :                     : NULL_TREE))
    5685              :     {
    5686          513 :       tree wmtype = TREE_TYPE (TREE_TYPE (wmptr));
    5687          513 :       tree wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
    5688              :                                  build_int_cst (TREE_TYPE (wmptr), 0));
    5689          513 :       rtx wmark = expand_expr (wmtree, NULL_RTX, ptr_mode, EXPAND_MEMORY);
    5690          513 :       MEM_NOTRAP_P (wmark) = !tree_could_trap_p (wmtree);
    5691          513 :       stktop = force_reg (ptr_mode, wmark);
    5692              :     }
    5693              : 
    5694          513 :   if (!stktop)
    5695          726 :     stktop = expand_builtin_stack_address ();
    5696              : 
    5697         1239 :   tree wmptr = CALL_EXPR_ARG (exp, 0);
    5698         1239 :   tree wmtype = TREE_TYPE (TREE_TYPE (wmptr));
    5699         1239 :   tree wmtree = fold_build2 (MEM_REF, wmtype, wmptr,
    5700              :                              build_int_cst (TREE_TYPE (wmptr), 0));
    5701         1239 :   rtx wmark = expand_expr (wmtree, NULL_RTX, ptr_mode, EXPAND_MEMORY);
    5702         1239 :   MEM_NOTRAP_P (wmark) = !tree_could_trap_p (wmtree);
    5703              : 
    5704         1239 :   rtx wmarkr = force_reg (ptr_mode, wmark);
    5705              : 
    5706              : #if ! STACK_GROWS_DOWNWARD
    5707              :   rtx base = stktop;
    5708              :   rtx end = wmarkr;
    5709              : #else
    5710         1239 :   rtx base = wmarkr;
    5711         1239 :   rtx end = stktop;
    5712              : #endif
    5713              : 
    5714              :   /* We're going to modify it, so make sure it's not e.g. the stack pointer.  */
    5715         1239 :   base = copy_to_reg (base);
    5716              : 
    5717         1239 :   rtx_code_label *done = gen_label_rtx ();
    5718         1239 :   do_compare_rtx_and_jump (base, end, LT, STACK_UNSIGNED,
    5719              :                            ptr_mode, NULL_RTX, done, NULL,
    5720              :                            profile_probability::very_likely ());
    5721              : 
    5722         1239 :   if (optimize < 3)
    5723          919 :     expand_call (exp, NULL_RTX, true);
    5724              :   else
    5725              :     {
    5726              :       /* Ok, now we've determined we want to copy the block, so convert the
    5727              :          addresses to Pmode, as needed to dereference them to access ptr_mode
    5728              :          memory locations, so that we don't have to convert anything within the
    5729              :          loop.  */
    5730          320 :       base = memory_address (ptr_mode, base);
    5731          320 :       end = memory_address (ptr_mode, end);
    5732              : 
    5733          320 :       rtx zero = force_operand (const0_rtx, NULL_RTX);
    5734          320 :       int ulen = GET_MODE_SIZE (ptr_mode);
    5735              : 
    5736              :       /* ??? It would be nice to use setmem or similar patterns here,
    5737              :          but they do not necessarily obey the stack growth direction,
    5738              :          which has security implications.  We also have to avoid calls
    5739              :          (memset, bzero or any machine-specific ones), which are
    5740              :          likely unsafe here (see TARGET_STRUB_MAY_USE_MEMSET).  */
    5741              : #if ! STACK_GROWS_DOWNWARD
    5742              :       rtx incr = plus_constant (Pmode, base, ulen);
    5743              :       rtx dstm = gen_rtx_MEM (ptr_mode, base);
    5744              : 
    5745              :       rtx_code_label *loop = gen_label_rtx ();
    5746              :       emit_label (loop);
    5747              :       emit_move_insn (dstm, zero);
    5748              :       emit_move_insn (base, force_operand (incr, NULL_RTX));
    5749              : #else
    5750          320 :       rtx decr = plus_constant (Pmode, end, -ulen);
    5751          320 :       rtx dstm = gen_rtx_MEM (ptr_mode, end);
    5752              : 
    5753          320 :       rtx_code_label *loop = gen_label_rtx ();
    5754          320 :       emit_label (loop);
    5755          320 :       emit_move_insn (end, force_operand (decr, NULL_RTX));
    5756          320 :       emit_move_insn (dstm, zero);
    5757              : #endif
    5758          640 :       do_compare_rtx_and_jump (base, end, LT, STACK_UNSIGNED,
    5759          320 :                                Pmode, NULL_RTX, NULL, loop,
    5760              :                                profile_probability::very_likely ());
    5761              :     }
    5762              : 
    5763         1239 :   emit_label (done);
    5764              : 
    5765         1239 :   return const0_rtx;
    5766              : }
    5767              : 
    5768              : /* Expand EXP, a call to the alloca builtin.  Return NULL_RTX if we
    5769              :    failed and the caller should emit a normal call.  */
    5770              : 
    5771              : static rtx
    5772        27680 : expand_builtin_alloca (tree exp)
    5773              : {
    5774        27680 :   rtx op0;
    5775        27680 :   rtx result;
    5776        27680 :   unsigned int align;
    5777        27680 :   tree fndecl = get_callee_fndecl (exp);
    5778        27680 :   HOST_WIDE_INT max_size;
    5779        27680 :   enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
    5780        27680 :   bool alloca_for_var = CALL_ALLOCA_FOR_VAR_P (exp);
    5781        27680 :   bool valid_arglist
    5782              :     = (fcode == BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX
    5783        27680 :        ? validate_arglist (exp, INTEGER_TYPE, INTEGER_TYPE, INTEGER_TYPE,
    5784              :                            VOID_TYPE)
    5785              :        : fcode == BUILT_IN_ALLOCA_WITH_ALIGN
    5786        27678 :          ? validate_arglist (exp, INTEGER_TYPE, INTEGER_TYPE, VOID_TYPE)
    5787        27680 :          : validate_arglist (exp, INTEGER_TYPE, VOID_TYPE));
    5788              : 
    5789        27680 :   if (!valid_arglist)
    5790              :     return NULL_RTX;
    5791              : 
    5792              :   /* Compute the argument.  */
    5793        27675 :   op0 = expand_normal (CALL_EXPR_ARG (exp, 0));
    5794              : 
    5795              :   /* Compute the alignment.  */
    5796        51348 :   align = (fcode == BUILT_IN_ALLOCA
    5797        23674 :            ? BIGGEST_ALIGNMENT
    5798         4001 :            : TREE_INT_CST_LOW (CALL_EXPR_ARG (exp, 1)));
    5799              : 
    5800              :   /* Compute the maximum size.  */
    5801            2 :   max_size = (fcode == BUILT_IN_ALLOCA_WITH_ALIGN_AND_MAX
    5802        27677 :               ? TREE_INT_CST_LOW (CALL_EXPR_ARG (exp, 2))
    5803              :               : -1);
    5804              : 
    5805              :   /* Allocate the desired space.  If the allocation stems from the declaration
    5806              :      of a variable-sized object, it cannot accumulate.  */
    5807        27675 :   result
    5808        27675 :     = allocate_dynamic_stack_space (op0, 0, align, max_size, alloca_for_var);
    5809        27675 :   result = convert_memory_address (ptr_mode, result);
    5810              : 
    5811              :   /* Dynamic allocations for variables are recorded during gimplification.  */
    5812        27675 :   if (!alloca_for_var && (flag_callgraph_info & CALLGRAPH_INFO_DYNAMIC_ALLOC))
    5813            0 :     record_dynamic_alloc (exp);
    5814              : 
    5815              :   return result;
    5816              : }
    5817              : 
    5818              : /* Emit a call to __asan_allocas_unpoison call in EXP.  Add to second argument
    5819              :    of the call virtual_stack_dynamic_rtx - stack_pointer_rtx, which is the
    5820              :    STACK_DYNAMIC_OFFSET value.  See motivation for this in comment to
    5821              :    handle_builtin_stack_restore function.  */
    5822              : 
    5823              : static rtx
    5824          204 : expand_asan_emit_allocas_unpoison (tree exp)
    5825              : {
    5826          204 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
    5827          204 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
    5828          204 :   rtx top = expand_expr (arg0, NULL_RTX, ptr_mode, EXPAND_NORMAL);
    5829          204 :   rtx bot = expand_expr (arg1, NULL_RTX, ptr_mode, EXPAND_NORMAL);
    5830          204 :   rtx off = expand_simple_binop (Pmode, MINUS, virtual_stack_dynamic_rtx,
    5831              :                                  stack_pointer_rtx, NULL_RTX, 0,
    5832              :                                  OPTAB_LIB_WIDEN);
    5833          204 :   off = convert_modes (ptr_mode, Pmode, off, 0);
    5834          204 :   bot = expand_simple_binop (ptr_mode, PLUS, bot, off, NULL_RTX, 0,
    5835              :                              OPTAB_LIB_WIDEN);
    5836          204 :   rtx ret = init_one_libfunc ("__asan_allocas_unpoison");
    5837          204 :   ret = emit_library_call_value (ret, NULL_RTX, LCT_NORMAL, ptr_mode,
    5838              :                                  top, ptr_mode, bot, ptr_mode);
    5839          204 :   return ret;
    5840              : }
    5841              : 
    5842              : /* Expand a call to bswap builtin in EXP.
    5843              :    Return NULL_RTX if a normal call should be emitted rather than expanding the
    5844              :    function in-line.  If convenient, the result should be placed in TARGET.
    5845              :    SUBTARGET may be used as the target for computing one of EXP's operands.  */
    5846              : 
    5847              : static rtx
    5848         1276 : expand_builtin_bswap (machine_mode target_mode, tree exp, rtx target,
    5849              :                       rtx subtarget)
    5850              : {
    5851         1276 :   tree arg;
    5852         1276 :   rtx op0;
    5853              : 
    5854         1276 :   if (!validate_arglist (exp, INTEGER_TYPE, VOID_TYPE))
    5855              :     return NULL_RTX;
    5856              : 
    5857         1276 :   arg = CALL_EXPR_ARG (exp, 0);
    5858         1276 :   op0 = expand_expr (arg,
    5859           56 :                      subtarget && GET_MODE (subtarget) == target_mode
    5860              :                      ? subtarget : NULL_RTX,
    5861              :                      target_mode, EXPAND_NORMAL);
    5862         1276 :   if (GET_MODE (op0) != target_mode)
    5863            0 :     op0 = convert_to_mode (target_mode, op0, 1);
    5864              : 
    5865         1276 :   target = expand_unop (target_mode, bswap_optab, op0, target, 1);
    5866              : 
    5867         1276 :   gcc_assert (target);
    5868              : 
    5869         1276 :   return convert_to_mode (target_mode, target, 1);
    5870              : }
    5871              : 
    5872              : /* Expand a call to a unary builtin in EXP.
    5873              :    Return NULL_RTX if a normal call should be emitted rather than expanding the
    5874              :    function in-line.  If convenient, the result should be placed in TARGET.
    5875              :    SUBTARGET may be used as the target for computing one of EXP's operands.  */
    5876              : 
    5877              : static rtx
    5878          969 : expand_builtin_unop (machine_mode target_mode, tree exp, rtx target,
    5879              :                      rtx subtarget, optab op_optab)
    5880              : {
    5881          969 :   rtx op0;
    5882              : 
    5883          969 :   if (!validate_arglist (exp, INTEGER_TYPE, VOID_TYPE))
    5884              :     return NULL_RTX;
    5885              : 
    5886              :   /* Compute the argument.  */
    5887         1938 :   op0 = expand_expr (CALL_EXPR_ARG (exp, 0),
    5888              :                      (subtarget
    5889          136 :                       && (TYPE_MODE (TREE_TYPE (CALL_EXPR_ARG (exp, 0)))
    5890          136 :                           == GET_MODE (subtarget))) ? subtarget : NULL_RTX,
    5891              :                      VOIDmode, EXPAND_NORMAL);
    5892              :   /* Compute op, into TARGET if possible.
    5893              :      Set TARGET to wherever the result comes back.  */
    5894          969 :   target = expand_unop (TYPE_MODE (TREE_TYPE (CALL_EXPR_ARG (exp, 0))),
    5895              :                         op_optab, op0, target, op_optab != clrsb_optab);
    5896          969 :   gcc_assert (target);
    5897              : 
    5898          969 :   return convert_to_mode (target_mode, target, 0);
    5899              : }
    5900              : 
    5901              : /* Expand a call to __builtin_expect.  We just return our argument
    5902              :    as the builtin_expect semantic should've been already executed by
    5903              :    tree branch prediction pass. */
    5904              : 
    5905              : static rtx
    5906         1407 : expand_builtin_expect (tree exp, rtx target)
    5907              : {
    5908         1407 :   tree arg;
    5909              : 
    5910         1407 :   if (call_expr_nargs (exp) < 2)
    5911            0 :     return const0_rtx;
    5912         1407 :   arg = CALL_EXPR_ARG (exp, 0);
    5913              : 
    5914         1407 :   target = expand_expr (arg, target, VOIDmode, EXPAND_NORMAL);
    5915              :   /* When guessing was done, the hints should be already stripped away.  */
    5916         1407 :   gcc_assert (!flag_guess_branch_prob
    5917              :               || optimize == 0 || seen_error ());
    5918              :   return target;
    5919              : }
    5920              : 
    5921              : /* Expand a call to __builtin_expect_with_probability.  We just return our
    5922              :    argument as the builtin_expect semantic should've been already executed by
    5923              :    tree branch prediction pass.  */
    5924              : 
    5925              : static rtx
    5926            5 : expand_builtin_expect_with_probability (tree exp, rtx target)
    5927              : {
    5928            5 :   tree arg;
    5929              : 
    5930            5 :   if (call_expr_nargs (exp) < 3)
    5931            0 :     return const0_rtx;
    5932            5 :   arg = CALL_EXPR_ARG (exp, 0);
    5933              : 
    5934            5 :   target = expand_expr (arg, target, VOIDmode, EXPAND_NORMAL);
    5935              :   /* When guessing was done, the hints should be already stripped away.  */
    5936            5 :   gcc_assert (!flag_guess_branch_prob
    5937              :               || optimize == 0 || seen_error ());
    5938              :   return target;
    5939              : }
    5940              : 
    5941              : 
    5942              : /* Expand a call to __builtin_assume_aligned.  We just return our first
    5943              :    argument as the builtin_assume_aligned semantic should've been already
    5944              :    executed by CCP.  */
    5945              : 
    5946              : static rtx
    5947          662 : expand_builtin_assume_aligned (tree exp, rtx target)
    5948              : {
    5949          662 :   if (call_expr_nargs (exp) < 2)
    5950            0 :     return const0_rtx;
    5951          662 :   target = expand_expr (CALL_EXPR_ARG (exp, 0), target, VOIDmode,
    5952              :                         EXPAND_NORMAL);
    5953          662 :   gcc_assert (!TREE_SIDE_EFFECTS (CALL_EXPR_ARG (exp, 1))
    5954              :               && (call_expr_nargs (exp) < 3
    5955              :                   || !TREE_SIDE_EFFECTS (CALL_EXPR_ARG (exp, 2))));
    5956              :   return target;
    5957              : }
    5958              : 
    5959              : void
    5960        38826 : expand_builtin_trap (void)
    5961              : {
    5962        38826 :   if (targetm.have_trap ())
    5963              :     {
    5964        38826 :       rtx_insn *insn = emit_insn (targetm.gen_trap ());
    5965              :       /* For trap insns when not accumulating outgoing args force
    5966              :          REG_ARGS_SIZE note to prevent crossjumping of calls with
    5967              :          different args sizes.  */
    5968        38826 :       if (!ACCUMULATE_OUTGOING_ARGS)
    5969        38824 :         add_args_size_note (insn, stack_pointer_delta);
    5970              :     }
    5971              :   else
    5972              :     {
    5973            0 :       tree fn = builtin_decl_implicit (BUILT_IN_ABORT);
    5974            0 :       tree call_expr = build_call_expr (fn, 0);
    5975            0 :       expand_call (call_expr, NULL_RTX, false);
    5976              :     }
    5977              : 
    5978        38826 :   emit_barrier ();
    5979        38826 : }
    5980              : 
    5981              : /* Expand a call to __builtin_unreachable.  We do nothing except emit
    5982              :    a barrier saying that control flow will not pass here.
    5983              : 
    5984              :    It is the responsibility of the program being compiled to ensure
    5985              :    that control flow does never reach __builtin_unreachable.  */
    5986              : static void
    5987         2668 : expand_builtin_unreachable (void)
    5988              : {
    5989              :   /* Use gimple_build_builtin_unreachable or builtin_decl_unreachable
    5990              :      to avoid this.  */
    5991         2668 :   gcc_checking_assert (!sanitize_flags_p (SANITIZE_UNREACHABLE));
    5992         2668 :   emit_barrier ();
    5993         2668 : }
    5994              : 
    5995              : /* Expand EXP, a call to fabs, fabsf or fabsl.
    5996              :    Return NULL_RTX if a normal call should be emitted rather than expanding
    5997              :    the function inline.  If convenient, the result should be placed
    5998              :    in TARGET.  SUBTARGET may be used as the target for computing
    5999              :    the operand.  */
    6000              : 
    6001              : static rtx
    6002            4 : expand_builtin_fabs (tree exp, rtx target, rtx subtarget)
    6003              : {
    6004            4 :   machine_mode mode;
    6005            4 :   tree arg;
    6006            4 :   rtx op0;
    6007              : 
    6008            4 :   if (!validate_arglist (exp, REAL_TYPE, VOID_TYPE))
    6009              :     return NULL_RTX;
    6010              : 
    6011            0 :   arg = CALL_EXPR_ARG (exp, 0);
    6012            0 :   CALL_EXPR_ARG (exp, 0) = arg = builtin_save_expr (arg);
    6013            0 :   mode = TYPE_MODE (TREE_TYPE (arg));
    6014            0 :   op0 = expand_expr (arg, subtarget, VOIDmode, EXPAND_NORMAL);
    6015            0 :   return expand_abs (mode, op0, target, 0, safe_from_p (target, arg, 1));
    6016              : }
    6017              : 
    6018              : /* Expand EXP, a call to copysign, copysignf, or copysignl.
    6019              :    Return NULL is a normal call should be emitted rather than expanding the
    6020              :    function inline.  If convenient, the result should be placed in TARGET.
    6021              :    SUBTARGET may be used as the target for computing the operand.  */
    6022              : 
    6023              : static rtx
    6024        11677 : expand_builtin_copysign (tree exp, rtx target, rtx subtarget)
    6025              : {
    6026        11677 :   rtx op0, op1;
    6027        11677 :   tree arg;
    6028              : 
    6029        11677 :   if (!validate_arglist (exp, REAL_TYPE, REAL_TYPE, VOID_TYPE))
    6030              :     return NULL_RTX;
    6031              : 
    6032        11676 :   arg = CALL_EXPR_ARG (exp, 0);
    6033        11676 :   op0 = expand_expr (arg, subtarget, VOIDmode, EXPAND_NORMAL);
    6034              : 
    6035        11676 :   arg = CALL_EXPR_ARG (exp, 1);
    6036        11676 :   op1 = expand_normal (arg);
    6037              : 
    6038        11676 :   return expand_copysign (op0, op1, target);
    6039              : }
    6040              : 
    6041              : /* Emit a call to __builtin___clear_cache.  */
    6042              : 
    6043              : void
    6044            0 : default_emit_call_builtin___clear_cache (rtx begin, rtx end)
    6045              : {
    6046            0 :   rtx callee = gen_rtx_SYMBOL_REF (Pmode,
    6047              :                                    BUILTIN_ASM_NAME_PTR
    6048              :                                    (BUILT_IN_CLEAR_CACHE));
    6049              : 
    6050            0 :   emit_library_call (callee,
    6051              :                      LCT_NORMAL, VOIDmode,
    6052              :                      convert_memory_address (ptr_mode, begin), ptr_mode,
    6053              :                      convert_memory_address (ptr_mode, end), ptr_mode);
    6054            0 : }
    6055              : 
    6056              : /* Emit a call to __builtin___clear_cache, unless the target specifies
    6057              :    it as do-nothing.  This function can be used by trampoline
    6058              :    finalizers to duplicate the effects of expanding a call to the
    6059              :    clear_cache builtin.  */
    6060              : 
    6061              : void
    6062           28 : maybe_emit_call_builtin___clear_cache (rtx begin, rtx end)
    6063              : {
    6064           28 :   gcc_assert ((GET_MODE (begin) == ptr_mode || GET_MODE (begin) == Pmode
    6065              :                || CONST_INT_P (begin))
    6066              :               && (GET_MODE (end) == ptr_mode || GET_MODE (end) == Pmode
    6067              :                   || CONST_INT_P (end)));
    6068              : 
    6069           28 :   if (targetm.have_clear_cache ())
    6070              :     {
    6071              :       /* We have a "clear_cache" insn, and it will handle everything.  */
    6072            0 :       class expand_operand ops[2];
    6073              : 
    6074            0 :       create_address_operand (&ops[0], begin);
    6075            0 :       create_address_operand (&ops[1], end);
    6076              : 
    6077            0 :       if (maybe_expand_insn (targetm.code_for_clear_cache, 2, ops))
    6078            0 :         return;
    6079              :     }
    6080              :   else
    6081              :     {
    6082              : #ifndef CLEAR_INSN_CACHE
    6083              :       /* There is no "clear_cache" insn, and __clear_cache() in libgcc
    6084              :          does nothing.  There is no need to call it.  Do nothing.  */
    6085              :       return;
    6086              : #endif /* CLEAR_INSN_CACHE */
    6087              :     }
    6088              : 
    6089            0 :   targetm.calls.emit_call_builtin___clear_cache (begin, end);
    6090              : }
    6091              : 
    6092              : /* Expand a call to __builtin___clear_cache.  */
    6093              : 
    6094              : static void
    6095           28 : expand_builtin___clear_cache (tree exp)
    6096              : {
    6097           28 :   tree begin, end;
    6098           28 :   rtx begin_rtx, end_rtx;
    6099              : 
    6100              :   /* We must not expand to a library call.  If we did, any
    6101              :      fallback library function in libgcc that might contain a call to
    6102              :      __builtin___clear_cache() would recurse infinitely.  */
    6103           28 :   if (!validate_arglist (exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
    6104              :     {
    6105            0 :       error ("both arguments to %<__builtin___clear_cache%> must be pointers");
    6106            0 :       return;
    6107              :     }
    6108              : 
    6109           28 :   begin = CALL_EXPR_ARG (exp, 0);
    6110           30 :   begin_rtx = expand_expr (begin, NULL_RTX, Pmode, EXPAND_NORMAL);
    6111              : 
    6112           28 :   end = CALL_EXPR_ARG (exp, 1);
    6113           30 :   end_rtx = expand_expr (end, NULL_RTX, Pmode, EXPAND_NORMAL);
    6114              : 
    6115           28 :   maybe_emit_call_builtin___clear_cache (begin_rtx, end_rtx);
    6116              : }
    6117              : 
    6118              : /* Given a trampoline address, make sure it satisfies TRAMPOLINE_ALIGNMENT.  */
    6119              : 
    6120              : static rtx
    6121          663 : round_trampoline_addr (rtx tramp)
    6122              : {
    6123          663 :   rtx temp, addend, mask;
    6124              : 
    6125              :   /* If we don't need too much alignment, we'll have been guaranteed
    6126              :      proper alignment by get_trampoline_type.  */
    6127          663 :   if (TRAMPOLINE_ALIGNMENT <= STACK_BOUNDARY)
    6128              :     return tramp;
    6129              : 
    6130              :   /* Round address up to desired boundary.  */
    6131            0 :   temp = gen_reg_rtx (Pmode);
    6132            0 :   addend = gen_int_mode (TRAMPOLINE_ALIGNMENT / BITS_PER_UNIT - 1, Pmode);
    6133            0 :   mask = gen_int_mode (-TRAMPOLINE_ALIGNMENT / BITS_PER_UNIT, Pmode);
    6134              : 
    6135            0 :   temp  = expand_simple_binop (Pmode, PLUS, tramp, addend,
    6136              :                                temp, 0, OPTAB_LIB_WIDEN);
    6137            0 :   tramp = expand_simple_binop (Pmode, AND, temp, mask,
    6138              :                                temp, 0, OPTAB_LIB_WIDEN);
    6139              : 
    6140            0 :   return tramp;
    6141              : }
    6142              : 
    6143              : static rtx
    6144          308 : expand_builtin_init_trampoline (tree exp, bool onstack)
    6145              : {
    6146          308 :   tree t_tramp, t_func, t_chain;
    6147          308 :   rtx m_tramp, r_tramp, r_chain, tmp;
    6148              : 
    6149          308 :   if (!validate_arglist (exp, POINTER_TYPE, POINTER_TYPE,
    6150              :                          POINTER_TYPE, VOID_TYPE))
    6151              :     return NULL_RTX;
    6152              : 
    6153          308 :   t_tramp = CALL_EXPR_ARG (exp, 0);
    6154          308 :   t_func = CALL_EXPR_ARG (exp, 1);
    6155          308 :   t_chain = CALL_EXPR_ARG (exp, 2);
    6156              : 
    6157          308 :   r_tramp = expand_normal (t_tramp);
    6158          308 :   m_tramp = gen_rtx_MEM (BLKmode, r_tramp);
    6159          308 :   MEM_NOTRAP_P (m_tramp) = 1;
    6160              : 
    6161              :   /* If ONSTACK, the TRAMP argument should be the address of a field
    6162              :      within the local function's FRAME decl.  Either way, let's see if
    6163              :      we can fill in the MEM_ATTRs for this memory.  */
    6164          308 :   if (TREE_CODE (t_tramp) == ADDR_EXPR)
    6165          308 :     set_mem_attributes (m_tramp, TREE_OPERAND (t_tramp, 0), true);
    6166              : 
    6167              :   /* Creator of a heap trampoline is responsible for making sure the
    6168              :      address is aligned to at least STACK_BOUNDARY.  Normally malloc
    6169              :      will ensure this anyhow.  */
    6170          308 :   tmp = round_trampoline_addr (r_tramp);
    6171          308 :   if (tmp != r_tramp)
    6172              :     {
    6173            0 :       m_tramp = change_address (m_tramp, BLKmode, tmp);
    6174            0 :       set_mem_align (m_tramp, TRAMPOLINE_ALIGNMENT);
    6175            0 :       set_mem_size (m_tramp, TRAMPOLINE_SIZE);
    6176              :     }
    6177              : 
    6178              :   /* The FUNC argument should be the address of the nested function.
    6179              :      Extract the actual function decl to pass to the hook.  */
    6180          308 :   gcc_assert (TREE_CODE (t_func) == ADDR_EXPR);
    6181          308 :   t_func = TREE_OPERAND (t_func, 0);
    6182          308 :   gcc_assert (TREE_CODE (t_func) == FUNCTION_DECL);
    6183              : 
    6184          308 :   r_chain = expand_normal (t_chain);
    6185              : 
    6186              :   /* Generate insns to initialize the trampoline.  */
    6187          308 :   targetm.calls.trampoline_init (m_tramp, t_func, r_chain);
    6188              : 
    6189          308 :   if (onstack)
    6190              :     {
    6191          308 :       trampolines_created = 1;
    6192              : 
    6193          308 :       if (targetm.calls.custom_function_descriptors != 0)
    6194          308 :         warning_at (DECL_SOURCE_LOCATION (t_func), OPT_Wtrampolines,
    6195              :                     "trampoline generated for nested function %qD", t_func);
    6196              :     }
    6197              : 
    6198          308 :   return const0_rtx;
    6199              : }
    6200              : 
    6201              : static rtx
    6202          355 : expand_builtin_adjust_trampoline (tree exp)
    6203              : {
    6204          355 :   rtx tramp;
    6205              : 
    6206          355 :   if (!validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    6207              :     return NULL_RTX;
    6208              : 
    6209          355 :   tramp = expand_normal (CALL_EXPR_ARG (exp, 0));
    6210          355 :   tramp = round_trampoline_addr (tramp);
    6211          355 :   if (targetm.calls.trampoline_adjust_address)
    6212            0 :     tramp = targetm.calls.trampoline_adjust_address (tramp);
    6213              : 
    6214              :   return tramp;
    6215              : }
    6216              : 
    6217              : /* Expand a call to the builtin descriptor initialization routine.
    6218              :    A descriptor is made up of a couple of pointers to the static
    6219              :    chain and the code entry in this order.  */
    6220              : 
    6221              : static rtx
    6222            0 : expand_builtin_init_descriptor (tree exp)
    6223              : {
    6224            0 :   tree t_descr, t_func, t_chain;
    6225            0 :   rtx m_descr, r_descr, r_func, r_chain;
    6226              : 
    6227            0 :   if (!validate_arglist (exp, POINTER_TYPE, POINTER_TYPE, POINTER_TYPE,
    6228              :                          VOID_TYPE))
    6229              :     return NULL_RTX;
    6230              : 
    6231            0 :   t_descr = CALL_EXPR_ARG (exp, 0);
    6232            0 :   t_func = CALL_EXPR_ARG (exp, 1);
    6233            0 :   t_chain = CALL_EXPR_ARG (exp, 2);
    6234              : 
    6235            0 :   r_descr = expand_normal (t_descr);
    6236            0 :   m_descr = gen_rtx_MEM (BLKmode, r_descr);
    6237            0 :   MEM_NOTRAP_P (m_descr) = 1;
    6238            0 :   set_mem_align (m_descr, GET_MODE_ALIGNMENT (ptr_mode));
    6239              : 
    6240            0 :   r_func = expand_normal (t_func);
    6241            0 :   r_chain = expand_normal (t_chain);
    6242              : 
    6243              :   /* Generate insns to initialize the descriptor.  */
    6244            0 :   emit_move_insn (adjust_address_nv (m_descr, ptr_mode, 0), r_chain);
    6245            0 :   emit_move_insn (adjust_address_nv (m_descr, ptr_mode,
    6246              :                                      POINTER_SIZE / BITS_PER_UNIT), r_func);
    6247              : 
    6248            0 :   return const0_rtx;
    6249              : }
    6250              : 
    6251              : /* Expand a call to the builtin descriptor adjustment routine.  */
    6252              : 
    6253              : static rtx
    6254            0 : expand_builtin_adjust_descriptor (tree exp)
    6255              : {
    6256            0 :   rtx tramp;
    6257              : 
    6258            0 :   if (!validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    6259              :     return NULL_RTX;
    6260              : 
    6261            0 :   tramp = expand_normal (CALL_EXPR_ARG (exp, 0));
    6262              : 
    6263              :   /* Unalign the descriptor to allow runtime identification.  */
    6264            0 :   tramp = plus_constant (ptr_mode, tramp,
    6265            0 :                          targetm.calls.custom_function_descriptors);
    6266              : 
    6267            0 :   return force_operand (tramp, NULL_RTX);
    6268              : }
    6269              : 
    6270              : /* Expand the call EXP to the built-in signbit, signbitf or signbitl
    6271              :    function.  The function first checks whether the back end provides
    6272              :    an insn to implement signbit for the respective mode.  If not, it
    6273              :    checks whether the floating point format of the value is such that
    6274              :    the sign bit can be extracted.  If that is not the case, error out.
    6275              :    EXP is the expression that is a call to the builtin function; if
    6276              :    convenient, the result should be placed in TARGET.  */
    6277              : static rtx
    6278         1385 : expand_builtin_signbit (tree exp, rtx target)
    6279              : {
    6280         1385 :   const struct real_format *fmt;
    6281         1385 :   scalar_float_mode fmode;
    6282         1385 :   scalar_int_mode rmode, imode;
    6283         1385 :   tree arg;
    6284         1385 :   int word, bitpos;
    6285         1385 :   enum insn_code icode;
    6286         1385 :   rtx temp;
    6287         1385 :   location_t loc = EXPR_LOCATION (exp);
    6288              : 
    6289         1385 :   if (!validate_arglist (exp, REAL_TYPE, VOID_TYPE))
    6290              :     return NULL_RTX;
    6291              : 
    6292         1385 :   arg = CALL_EXPR_ARG (exp, 0);
    6293         1385 :   fmode = SCALAR_FLOAT_TYPE_MODE (TREE_TYPE (arg));
    6294         1385 :   rmode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
    6295         1385 :   fmt = REAL_MODE_FORMAT (fmode);
    6296              : 
    6297         1385 :   arg = builtin_save_expr (arg);
    6298              : 
    6299              :   /* Expand the argument yielding a RTX expression. */
    6300         1385 :   temp = expand_normal (arg);
    6301              : 
    6302              :   /* Check if the back end provides an insn that handles signbit for the
    6303              :      argument's mode. */
    6304         1385 :   icode = optab_handler (signbit_optab, fmode);
    6305         1385 :   if (icode != CODE_FOR_nothing)
    6306              :     {
    6307           10 :       rtx_insn *last = get_last_insn ();
    6308           10 :       rtx this_target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
    6309           10 :       if (maybe_emit_unop_insn (icode, this_target, temp, UNKNOWN))
    6310              :         return this_target;
    6311            0 :       delete_insns_since (last);
    6312              :     }
    6313              : 
    6314              :   /* For floating point formats without a sign bit, implement signbit
    6315              :      as "ARG < 0.0".  */
    6316         1375 :   bitpos = fmt->signbit_ro;
    6317         1375 :   if (bitpos < 0)
    6318              :   {
    6319              :     /* But we can't do this if the format supports signed zero.  */
    6320            0 :     gcc_assert (!fmt->has_signed_zero || !HONOR_SIGNED_ZEROS (fmode));
    6321              : 
    6322            0 :     arg = fold_build2_loc (loc, LT_EXPR, TREE_TYPE (exp), arg,
    6323            0 :                        build_real (TREE_TYPE (arg), dconst0));
    6324            0 :     return expand_expr (arg, target, VOIDmode, EXPAND_NORMAL);
    6325              :   }
    6326              : 
    6327         2754 :   if (GET_MODE_SIZE (fmode) <= UNITS_PER_WORD)
    6328              :     {
    6329         1271 :       imode = int_mode_for_mode (fmode).require ();
    6330         1271 :       temp = gen_lowpart (imode, temp);
    6331              :     }
    6332              :   else
    6333              :     {
    6334          104 :       imode = word_mode;
    6335              :       /* Handle targets with different FP word orders.  */
    6336          104 :       if (FLOAT_WORDS_BIG_ENDIAN)
    6337              :         word = (GET_MODE_BITSIZE (fmode) - bitpos) / BITS_PER_WORD;
    6338              :       else
    6339          104 :         word = bitpos / BITS_PER_WORD;
    6340          104 :       temp = operand_subword_force (temp, word, fmode);
    6341          104 :       bitpos = bitpos % BITS_PER_WORD;
    6342              :     }
    6343              : 
    6344              :   /* Force the intermediate word_mode (or narrower) result into a
    6345              :      register.  This avoids attempting to create paradoxical SUBREGs
    6346              :      of floating point modes below.  */
    6347         1375 :   temp = force_reg (imode, temp);
    6348              : 
    6349              :   /* If the bitpos is within the "result mode" lowpart, the operation
    6350              :      can be implement with a single bitwise AND.  Otherwise, we need
    6351              :      a right shift and an AND.  */
    6352              : 
    6353         2750 :   if (bitpos < GET_MODE_BITSIZE (rmode))
    6354              :     {
    6355         1167 :       wide_int mask = wi::set_bit_in_zero (bitpos, GET_MODE_PRECISION (rmode));
    6356              : 
    6357         3501 :       if (GET_MODE_SIZE (imode) > GET_MODE_SIZE (rmode))
    6358            0 :         temp = gen_lowpart (rmode, temp);
    6359         2334 :       temp = expand_binop (rmode, and_optab, temp,
    6360         2334 :                            immed_wide_int_const (mask, rmode),
    6361              :                            NULL_RTX, 1, OPTAB_LIB_WIDEN);
    6362         1167 :     }
    6363              :   else
    6364              :     {
    6365              :       /* Perform a logical right shift to place the signbit in the least
    6366              :          significant bit, then truncate the result to the desired mode
    6367              :          and mask just this bit.  */
    6368          208 :       temp = expand_shift (RSHIFT_EXPR, imode, temp, bitpos, NULL_RTX, 1);
    6369          208 :       temp = gen_lowpart (rmode, temp);
    6370          208 :       temp = expand_binop (rmode, and_optab, temp, const1_rtx,
    6371              :                            NULL_RTX, 1, OPTAB_LIB_WIDEN);
    6372              :     }
    6373              : 
    6374              :   return temp;
    6375              : }
    6376              : 
    6377              : /* Expand fork or exec calls.  TARGET is the desired target of the
    6378              :    call.  EXP is the call. FN is the
    6379              :    identificator of the actual function.  IGNORE is nonzero if the
    6380              :    value is to be ignored.  */
    6381              : 
    6382              : static rtx
    6383          112 : expand_builtin_fork_or_exec (tree fn, tree exp, rtx target, int ignore)
    6384              : {
    6385          112 :   tree id, decl;
    6386          112 :   tree call;
    6387              : 
    6388              :   /* If we are not profiling, just call the function.  */
    6389          112 :   if (!coverage_instrumentation_p ())
    6390              :     return NULL_RTX;
    6391              : 
    6392              :   /* Otherwise call the wrapper.  This should be equivalent for the rest of
    6393              :      compiler, so the code does not diverge, and the wrapper may run the
    6394              :      code necessary for keeping the profiling sane.  */
    6395              : 
    6396            4 :   switch (DECL_FUNCTION_CODE (fn))
    6397              :     {
    6398            4 :     case BUILT_IN_FORK:
    6399            4 :       id = get_identifier ("__gcov_fork");
    6400            4 :       break;
    6401              : 
    6402            0 :     case BUILT_IN_EXECL:
    6403            0 :       id = get_identifier ("__gcov_execl");
    6404            0 :       break;
    6405              : 
    6406            0 :     case BUILT_IN_EXECV:
    6407            0 :       id = get_identifier ("__gcov_execv");
    6408            0 :       break;
    6409              : 
    6410            0 :     case BUILT_IN_EXECLP:
    6411            0 :       id = get_identifier ("__gcov_execlp");
    6412            0 :       break;
    6413              : 
    6414            0 :     case BUILT_IN_EXECLE:
    6415            0 :       id = get_identifier ("__gcov_execle");
    6416            0 :       break;
    6417              : 
    6418            0 :     case BUILT_IN_EXECVP:
    6419            0 :       id = get_identifier ("__gcov_execvp");
    6420            0 :       break;
    6421              : 
    6422            0 :     case BUILT_IN_EXECVE:
    6423            0 :       id = get_identifier ("__gcov_execve");
    6424            0 :       break;
    6425              : 
    6426            0 :     default:
    6427            0 :       gcc_unreachable ();
    6428              :     }
    6429              : 
    6430            4 :   decl = build_decl (DECL_SOURCE_LOCATION (fn),
    6431            4 :                      FUNCTION_DECL, id, TREE_TYPE (fn));
    6432            4 :   DECL_EXTERNAL (decl) = 1;
    6433            4 :   TREE_PUBLIC (decl) = 1;
    6434            4 :   DECL_ARTIFICIAL (decl) = 1;
    6435            4 :   TREE_NOTHROW (decl) = 1;
    6436            4 :   DECL_VISIBILITY (decl) = VISIBILITY_DEFAULT;
    6437            4 :   DECL_VISIBILITY_SPECIFIED (decl) = 1;
    6438            4 :   call = rewrite_call_expr (EXPR_LOCATION (exp), exp, 0, decl, 0);
    6439            4 :   return expand_call (call, target, ignore);
    6440              :  }
    6441              : 
    6442              : 
    6443              : 
    6444              : /* Reconstitute a mode for a __sync intrinsic operation.  Since the type of
    6445              :    the pointer in these functions is void*, the tree optimizers may remove
    6446              :    casts.  The mode computed in expand_builtin isn't reliable either, due
    6447              :    to __sync_bool_compare_and_swap.
    6448              : 
    6449              :    FCODE_DIFF should be fcode - base, where base is the FOO_1 code for the
    6450              :    group of builtins.  This gives us log2 of the mode size.  */
    6451              : 
    6452              : static inline machine_mode
    6453       135801 : get_builtin_sync_mode (int fcode_diff)
    6454              : {
    6455              :   /* The size is not negotiable, so ask not to get BLKmode in return
    6456              :      if the target indicates that a smaller size would be better.  */
    6457       135801 :   return int_mode_for_size (BITS_PER_UNIT << fcode_diff, 0).require ();
    6458              : }
    6459              : 
    6460              : /* Expand the memory expression LOC and return the appropriate memory operand
    6461              :    for the builtin_sync operations.  */
    6462              : 
    6463              : static rtx
    6464       152541 : get_builtin_sync_mem (tree loc, machine_mode mode)
    6465              : {
    6466       152541 :   rtx addr, mem;
    6467       152541 :   int addr_space = TYPE_ADDR_SPACE (POINTER_TYPE_P (TREE_TYPE (loc))
    6468              :                                     ? TREE_TYPE (TREE_TYPE (loc))
    6469              :                                     : TREE_TYPE (loc));
    6470       152541 :   scalar_int_mode addr_mode = targetm.addr_space.address_mode (addr_space);
    6471              : 
    6472       152541 :   addr = expand_expr (loc, NULL_RTX, addr_mode, EXPAND_SUM);
    6473       152541 :   addr = convert_memory_address (addr_mode, addr);
    6474              : 
    6475              :   /* Note that we explicitly do not want any alias information for this
    6476              :      memory, so that we kill all other live memories.  Otherwise we don't
    6477              :      satisfy the full barrier semantics of the intrinsic.  */
    6478       152541 :   mem = gen_rtx_MEM (mode, addr);
    6479              : 
    6480       152541 :   set_mem_addr_space (mem, addr_space);
    6481              : 
    6482       152541 :   mem = validize_mem (mem);
    6483              : 
    6484              :   /* The alignment needs to be at least according to that of the mode.  */
    6485       152541 :   set_mem_align (mem, MAX (GET_MODE_ALIGNMENT (mode),
    6486              :                            get_pointer_alignment (loc)));
    6487       152541 :   set_mem_alias_set (mem, ALIAS_SET_MEMORY_BARRIER);
    6488       152541 :   MEM_VOLATILE_P (mem) = 1;
    6489              : 
    6490       152541 :   return mem;
    6491              : }
    6492              : 
    6493              : /* Make sure an argument is in the right mode.
    6494              :    EXP is the tree argument.
    6495              :    MODE is the mode it should be in.  */
    6496              : 
    6497              : static rtx
    6498        95144 : expand_expr_force_mode (tree exp, machine_mode mode)
    6499              : {
    6500        95144 :   rtx val;
    6501        95144 :   machine_mode old_mode;
    6502              : 
    6503        95144 :   if (TREE_CODE (exp) == SSA_NAME
    6504        95144 :       && TYPE_MODE (TREE_TYPE (exp)) != mode)
    6505              :     {
    6506              :       /* Undo argument promotion if possible, as combine might not
    6507              :          be able to do it later due to MEM_VOLATILE_P uses in the
    6508              :          patterns.  */
    6509           25 :       gimple *g = get_gimple_for_ssa_name (exp);
    6510           25 :       if (g && gimple_assign_cast_p (g))
    6511              :         {
    6512            0 :           tree rhs = gimple_assign_rhs1 (g);
    6513            0 :           tree_code code = gimple_assign_rhs_code (g);
    6514            0 :           if (CONVERT_EXPR_CODE_P (code)
    6515            0 :               && TYPE_MODE (TREE_TYPE (rhs)) == mode
    6516            0 :               && INTEGRAL_TYPE_P (TREE_TYPE (exp))
    6517            0 :               && INTEGRAL_TYPE_P (TREE_TYPE (rhs))
    6518            0 :               && (TYPE_PRECISION (TREE_TYPE (exp))
    6519            0 :                   > TYPE_PRECISION (TREE_TYPE (rhs))))
    6520              :             exp = rhs;
    6521              :         }
    6522              :     }
    6523              : 
    6524        95144 :   val = expand_expr (exp, NULL_RTX, mode, EXPAND_NORMAL);
    6525              :   /* If VAL is promoted to a wider mode, convert it back to MODE.  Take care
    6526              :      of CONST_INTs, where we know the old_mode only from the call argument.  */
    6527              : 
    6528        95144 :   old_mode = GET_MODE (val);
    6529        95144 :   if (old_mode == VOIDmode)
    6530        44767 :     old_mode = TYPE_MODE (TREE_TYPE (exp));
    6531        95144 :   val = convert_modes (mode, old_mode, val, 1);
    6532        95144 :   return val;
    6533              : }
    6534              : 
    6535              : 
    6536              : /* Expand the __sync_xxx_and_fetch and __sync_fetch_and_xxx intrinsics.
    6537              :    EXP is the CALL_EXPR.  CODE is the rtx code
    6538              :    that corresponds to the arithmetic or logical operation from the name;
    6539              :    an exception here is that NOT actually means NAND.  TARGET is an optional
    6540              :    place for us to store the results; AFTER is true if this is the
    6541              :    fetch_and_xxx form.  */
    6542              : 
    6543              : static rtx
    6544         4028 : expand_builtin_sync_operation (machine_mode mode, tree exp,
    6545              :                                enum rtx_code code, bool after,
    6546              :                                rtx target)
    6547              : {
    6548         4028 :   rtx val, mem;
    6549         4028 :   location_t loc = EXPR_LOCATION (exp);
    6550              : 
    6551         4028 :   if (code == NOT && warn_sync_nand)
    6552              :     {
    6553          523 :       tree fndecl = get_callee_fndecl (exp);
    6554          523 :       enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
    6555              : 
    6556          523 :       static bool warned_f_a_n, warned_n_a_f;
    6557              : 
    6558          523 :       switch (fcode)
    6559              :         {
    6560          434 :         case BUILT_IN_SYNC_FETCH_AND_NAND_1:
    6561          434 :         case BUILT_IN_SYNC_FETCH_AND_NAND_2:
    6562          434 :         case BUILT_IN_SYNC_FETCH_AND_NAND_4:
    6563          434 :         case BUILT_IN_SYNC_FETCH_AND_NAND_8:
    6564          434 :         case BUILT_IN_SYNC_FETCH_AND_NAND_16:
    6565          434 :           if (warned_f_a_n)
    6566              :             break;
    6567              : 
    6568           30 :           fndecl = builtin_decl_implicit (BUILT_IN_SYNC_FETCH_AND_NAND_N);
    6569           30 :           inform (loc, "%qD changed semantics in GCC 4.4", fndecl);
    6570           30 :           warned_f_a_n = true;
    6571           30 :           break;
    6572              : 
    6573           89 :         case BUILT_IN_SYNC_NAND_AND_FETCH_1:
    6574           89 :         case BUILT_IN_SYNC_NAND_AND_FETCH_2:
    6575           89 :         case BUILT_IN_SYNC_NAND_AND_FETCH_4:
    6576           89 :         case BUILT_IN_SYNC_NAND_AND_FETCH_8:
    6577           89 :         case BUILT_IN_SYNC_NAND_AND_FETCH_16:
    6578           89 :           if (warned_n_a_f)
    6579              :             break;
    6580              : 
    6581           22 :          fndecl = builtin_decl_implicit (BUILT_IN_SYNC_NAND_AND_FETCH_N);
    6582           22 :           inform (loc, "%qD changed semantics in GCC 4.4", fndecl);
    6583           22 :           warned_n_a_f = true;
    6584           22 :           break;
    6585              : 
    6586            0 :         default:
    6587            0 :           gcc_unreachable ();
    6588              :         }
    6589              :     }
    6590              : 
    6591              :   /* Expand the operands.  */
    6592         4028 :   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    6593         4028 :   val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
    6594              : 
    6595         4028 :   return expand_atomic_fetch_op (target, mem, val, code, MEMMODEL_SYNC_SEQ_CST,
    6596         4028 :                                  after);
    6597              : }
    6598              : 
    6599              : /* Expand the __sync_val_compare_and_swap and __sync_bool_compare_and_swap
    6600              :    intrinsics. EXP is the CALL_EXPR.  IS_BOOL is
    6601              :    true if this is the boolean form.  TARGET is a place for us to store the
    6602              :    results; this is NOT optional if IS_BOOL is true.  */
    6603              : 
    6604              : static rtx
    6605          460 : expand_builtin_compare_and_swap (machine_mode mode, tree exp,
    6606              :                                  bool is_bool, rtx target)
    6607              : {
    6608          460 :   rtx old_val, new_val, mem;
    6609          460 :   rtx *pbool, *poval;
    6610              : 
    6611              :   /* Expand the operands.  */
    6612          460 :   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    6613          460 :   old_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
    6614          460 :   new_val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode);
    6615              : 
    6616          460 :   pbool = poval = NULL;
    6617          460 :   if (target != const0_rtx)
    6618              :     {
    6619          434 :       if (is_bool)
    6620              :         pbool = &target;
    6621              :       else
    6622          230 :         poval = &target;
    6623              :     }
    6624          460 :   if (!expand_atomic_compare_and_swap (pbool, poval, mem, old_val, new_val,
    6625              :                                        false, MEMMODEL_SYNC_SEQ_CST,
    6626              :                                        MEMMODEL_SYNC_SEQ_CST))
    6627              :     return NULL_RTX;
    6628              : 
    6629          458 :   return target;
    6630              : }
    6631              : 
    6632              : /* Expand the __sync_lock_test_and_set intrinsic.  Note that the most
    6633              :    general form is actually an atomic exchange, and some targets only
    6634              :    support a reduced form with the second argument being a constant 1.
    6635              :    EXP is the CALL_EXPR; TARGET is an optional place for us to store
    6636              :    the results.  */
    6637              : 
    6638              : static rtx
    6639          326 : expand_builtin_sync_lock_test_and_set (machine_mode mode, tree exp,
    6640              :                                        rtx target)
    6641              : {
    6642          326 :   rtx val, mem;
    6643              : 
    6644              :   /* Expand the operands.  */
    6645          326 :   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    6646          326 :   val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
    6647              : 
    6648          326 :   return expand_sync_lock_test_and_set (target, mem, val);
    6649              : }
    6650              : 
    6651              : /* Expand the __sync_lock_release intrinsic.  EXP is the CALL_EXPR.  */
    6652              : 
    6653              : static rtx
    6654          146 : expand_builtin_sync_lock_release (machine_mode mode, tree exp)
    6655              : {
    6656          146 :   rtx mem;
    6657              : 
    6658              :   /* Expand the operands.  */
    6659          146 :   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    6660              : 
    6661          146 :   return expand_atomic_store (mem, const0_rtx, MEMMODEL_SYNC_RELEASE, true);
    6662              : }
    6663              : 
    6664              : /* Given an integer representing an ``enum memmodel'', verify its
    6665              :    correctness and return the memory model enum.  */
    6666              : 
    6667              : static enum memmodel
    6668       171434 : get_memmodel (tree exp)
    6669              : {
    6670              :   /* If the parameter is not a constant, it's a run time value so we'll just
    6671              :      convert it to MEMMODEL_SEQ_CST to avoid annoying runtime checking.  */
    6672       171434 :   if (TREE_CODE (exp) != INTEGER_CST)
    6673              :     return MEMMODEL_SEQ_CST;
    6674              : 
    6675       170588 :   rtx op = expand_normal (exp);
    6676              : 
    6677       170588 :   unsigned HOST_WIDE_INT val = INTVAL (op);
    6678       170588 :   if (targetm.memmodel_check)
    6679       170588 :     val = targetm.memmodel_check (val);
    6680            0 :   else if (val & ~MEMMODEL_MASK)
    6681              :     return MEMMODEL_SEQ_CST;
    6682              : 
    6683              :   /* Should never see a user explicit SYNC memodel model, so >= LAST works. */
    6684       170588 :   if (memmodel_base (val) >= MEMMODEL_LAST)
    6685              :     return MEMMODEL_SEQ_CST;
    6686              : 
    6687              :   /* Workaround for Bugzilla 59448. GCC doesn't track consume properly, so
    6688              :      be conservative and promote consume to acquire.  */
    6689       170585 :   if (val == MEMMODEL_CONSUME)
    6690          755 :     val = MEMMODEL_ACQUIRE;
    6691              : 
    6692       170585 :   return (enum memmodel) val;
    6693              : }
    6694              : 
    6695              : /* Expand the __atomic_exchange intrinsic:
    6696              :         TYPE __atomic_exchange (TYPE *object, TYPE desired, enum memmodel)
    6697              :    EXP is the CALL_EXPR.
    6698              :    TARGET is an optional place for us to store the results.  */
    6699              : 
    6700              : static rtx
    6701         2937 : expand_builtin_atomic_exchange (machine_mode mode, tree exp, rtx target)
    6702              : {
    6703         2937 :   rtx val, mem;
    6704         2937 :   enum memmodel model;
    6705              : 
    6706         2937 :   model = get_memmodel (CALL_EXPR_ARG (exp, 2));
    6707              : 
    6708         2937 :   if (!flag_inline_atomics)
    6709              :     return NULL_RTX;
    6710              : 
    6711              :   /* Expand the operands.  */
    6712         2886 :   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    6713         2886 :   val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
    6714              : 
    6715         2886 :   return expand_atomic_exchange (target, mem, val, model);
    6716              : }
    6717              : 
    6718              : /* Expand the __atomic_compare_exchange intrinsic:
    6719              :         bool __atomic_compare_exchange (TYPE *object, TYPE *expect,
    6720              :                                         TYPE desired, BOOL weak,
    6721              :                                         enum memmodel success,
    6722              :                                         enum memmodel failure)
    6723              :    EXP is the CALL_EXPR.
    6724              :    TARGET is an optional place for us to store the results.  */
    6725              : 
    6726              : static rtx
    6727         9398 : expand_builtin_atomic_compare_exchange (machine_mode mode, tree exp,
    6728              :                                         rtx target)
    6729              : {
    6730         9398 :   rtx expect, desired, mem, oldval;
    6731         9398 :   rtx_code_label *label;
    6732         9398 :   tree weak;
    6733         9398 :   bool is_weak;
    6734              : 
    6735         9398 :   memmodel success = get_memmodel (CALL_EXPR_ARG (exp, 4));
    6736         9398 :   memmodel failure = get_memmodel (CALL_EXPR_ARG (exp, 5));
    6737              : 
    6738         9398 :   if (failure > success)
    6739           22 :     success = MEMMODEL_SEQ_CST;
    6740              : 
    6741         9398 :   if (is_mm_release (failure) || is_mm_acq_rel (failure))
    6742              :     {
    6743              :       failure = MEMMODEL_SEQ_CST;
    6744              :       success = MEMMODEL_SEQ_CST;
    6745              :     }
    6746              : 
    6747              : 
    6748         9398 :   if (!flag_inline_atomics)
    6749              :     return NULL_RTX;
    6750              : 
    6751              :   /* Expand the operands.  */
    6752         9347 :   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    6753              : 
    6754         9347 :   expect = expand_normal (CALL_EXPR_ARG (exp, 1));
    6755         9383 :   expect = convert_memory_address (Pmode, expect);
    6756         9347 :   expect = gen_rtx_MEM (mode, expect);
    6757         9347 :   desired = expand_expr_force_mode (CALL_EXPR_ARG (exp, 2), mode);
    6758              : 
    6759         9347 :   weak = CALL_EXPR_ARG (exp, 3);
    6760         9347 :   is_weak = false;
    6761         9347 :   if (tree_fits_shwi_p (weak) && tree_to_shwi (weak) != 0)
    6762         9347 :     is_weak = true;
    6763              : 
    6764         9347 :   if (target == const0_rtx)
    6765          288 :     target = NULL;
    6766              : 
    6767              :   /* Lest the rtl backend create a race condition with an imporoper store
    6768              :      to memory, always create a new pseudo for OLDVAL.  */
    6769         9347 :   oldval = NULL;
    6770              : 
    6771         9347 :   if (!expand_atomic_compare_and_swap (&target, &oldval, mem, expect, desired,
    6772              :                                        is_weak, success, failure))
    6773              :     return NULL_RTX;
    6774              : 
    6775              :   /* Conditionally store back to EXPECT, lest we create a race condition
    6776              :      with an improper store to memory.  */
    6777              :   /* ??? With a rearrangement of atomics at the gimple level, we can handle
    6778              :      the normal case where EXPECT is totally private, i.e. a register.  At
    6779              :      which point the store can be unconditional.  */
    6780         7598 :   label = gen_label_rtx ();
    6781         7598 :   emit_cmp_and_jump_insns (target, const0_rtx, NE, NULL,
    6782         7598 :                            GET_MODE (target), 1, label);
    6783         7598 :   emit_move_insn (expect, oldval);
    6784         7598 :   emit_label (label);
    6785              : 
    6786         7598 :   return target;
    6787              : }
    6788              : 
    6789              : /* Helper function for expand_ifn_atomic_compare_exchange - expand
    6790              :    internal ATOMIC_COMPARE_EXCHANGE call into __atomic_compare_exchange_N
    6791              :    call.  The weak parameter must be dropped to match the expected parameter
    6792              :    list and the expected argument changed from value to pointer to memory
    6793              :    slot.  */
    6794              : 
    6795              : static void
    6796            0 : expand_ifn_atomic_compare_exchange_into_call (gcall *call, machine_mode mode)
    6797              : {
    6798            0 :   unsigned int z;
    6799            0 :   vec<tree, va_gc> *vec;
    6800              : 
    6801            0 :   vec_alloc (vec, 5);
    6802            0 :   vec->quick_push (gimple_call_arg (call, 0));
    6803            0 :   tree expected = gimple_call_arg (call, 1);
    6804            0 :   rtx x = assign_stack_temp_for_type (mode, GET_MODE_SIZE (mode),
    6805            0 :                                       TREE_TYPE (expected));
    6806            0 :   rtx expd = expand_expr (expected, x, mode, EXPAND_NORMAL);
    6807            0 :   if (expd != x)
    6808            0 :     emit_move_insn (x, expd);
    6809            0 :   tree v = make_tree (TREE_TYPE (expected), x);
    6810            0 :   vec->quick_push (build1 (ADDR_EXPR,
    6811            0 :                            build_pointer_type (TREE_TYPE (expected)), v));
    6812            0 :   vec->quick_push (gimple_call_arg (call, 2));
    6813              :   /* Skip the boolean weak parameter.  */
    6814            0 :   for (z = 4; z < 6; z++)
    6815            0 :     vec->quick_push (gimple_call_arg (call, z));
    6816              :   /* At present we only have BUILT_IN_ATOMIC_COMPARE_EXCHANGE_{1,2,4,8,16}.  */
    6817            0 :   unsigned int bytes_log2 = exact_log2 (GET_MODE_SIZE (mode).to_constant ());
    6818            0 :   gcc_assert (bytes_log2 < 5);
    6819            0 :   built_in_function fncode
    6820              :     = (built_in_function) ((int) BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1
    6821              :                            + bytes_log2);
    6822            0 :   tree fndecl = builtin_decl_explicit (fncode);
    6823            0 :   tree fn = build1 (ADDR_EXPR, build_pointer_type (TREE_TYPE (fndecl)),
    6824              :                     fndecl);
    6825            0 :   tree exp = build_call_vec (boolean_type_node, fn, vec);
    6826            0 :   tree lhs = gimple_call_lhs (call);
    6827            0 :   rtx boolret = expand_call (exp, NULL_RTX, lhs == NULL_TREE);
    6828            0 :   if (lhs)
    6829              :     {
    6830            0 :       rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    6831            0 :       if (GET_MODE (boolret) != mode)
    6832            0 :         boolret = convert_modes (mode, GET_MODE (boolret), boolret, 1);
    6833            0 :       x = force_reg (mode, x);
    6834            0 :       write_complex_part (target, boolret, true, true);
    6835            0 :       write_complex_part (target, x, false, false);
    6836              :     }
    6837            0 : }
    6838              : 
    6839              : /* Expand IFN_ATOMIC_COMPARE_EXCHANGE internal function.  */
    6840              : 
    6841              : void
    6842        14041 : expand_ifn_atomic_compare_exchange (gcall *call)
    6843              : {
    6844        14041 :   int size = tree_to_shwi (gimple_call_arg (call, 3)) & 255;
    6845        14041 :   gcc_assert (size == 1 || size == 2 || size == 4 || size == 8 || size == 16);
    6846        14041 :   machine_mode mode = int_mode_for_size (BITS_PER_UNIT * size, 0).require ();
    6847              : 
    6848        14041 :   memmodel success = get_memmodel (gimple_call_arg (call, 4));
    6849        14041 :   memmodel failure = get_memmodel (gimple_call_arg (call, 5));
    6850              : 
    6851        14041 :   if (failure > success)
    6852            0 :     success = MEMMODEL_SEQ_CST;
    6853              : 
    6854        14041 :   if (is_mm_release (failure) || is_mm_acq_rel (failure))
    6855              :     {
    6856              :       failure = MEMMODEL_SEQ_CST;
    6857              :       success = MEMMODEL_SEQ_CST;
    6858              :     }
    6859              : 
    6860        14041 :   if (!flag_inline_atomics)
    6861              :     {
    6862            0 :       expand_ifn_atomic_compare_exchange_into_call (call, mode);
    6863            0 :       return;
    6864              :     }
    6865              : 
    6866              :   /* Expand the operands.  */
    6867        14041 :   rtx mem = get_builtin_sync_mem (gimple_call_arg (call, 0), mode);
    6868              : 
    6869        14041 :   rtx expect = expand_expr_force_mode (gimple_call_arg (call, 1), mode);
    6870        14041 :   rtx desired = expand_expr_force_mode (gimple_call_arg (call, 2), mode);
    6871              : 
    6872        14041 :   bool is_weak = (tree_to_shwi (gimple_call_arg (call, 3)) & 256) != 0;
    6873              : 
    6874        14041 :   rtx boolret = NULL;
    6875        14041 :   rtx oldval = NULL;
    6876              : 
    6877        14041 :   if (!expand_atomic_compare_and_swap (&boolret, &oldval, mem, expect, desired,
    6878              :                                        is_weak, success, failure))
    6879              :     {
    6880            0 :       expand_ifn_atomic_compare_exchange_into_call (call, mode);
    6881            0 :       return;
    6882              :     }
    6883              : 
    6884        14041 :   tree lhs = gimple_call_lhs (call);
    6885        14041 :   if (lhs)
    6886              :     {
    6887        13832 :       rtx target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    6888        13832 :       if (GET_MODE (boolret) != mode)
    6889        12297 :         boolret = convert_modes (mode, GET_MODE (boolret), boolret, 1);
    6890        13832 :       write_complex_part (target, boolret, true, true);
    6891        13832 :       write_complex_part (target, oldval, false, false);
    6892              :     }
    6893              : }
    6894              : 
    6895              : /* Expand the __atomic_load intrinsic:
    6896              :         TYPE __atomic_load (TYPE *object, enum memmodel)
    6897              :    EXP is the CALL_EXPR.
    6898              :    TARGET is an optional place for us to store the results.  */
    6899              : 
    6900              : static rtx
    6901        71474 : expand_builtin_atomic_load (machine_mode mode, tree exp, rtx target)
    6902              : {
    6903        71474 :   memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 1));
    6904        71474 :   if (is_mm_release (model) || is_mm_acq_rel (model))
    6905              :     model = MEMMODEL_SEQ_CST;
    6906              : 
    6907        71474 :   if (!flag_inline_atomics)
    6908              :     return NULL_RTX;
    6909              : 
    6910              :   /* Expand the operand.  */
    6911        71433 :   rtx mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    6912              : 
    6913        71433 :   return expand_atomic_load (target, mem, model);
    6914              : }
    6915              : 
    6916              : 
    6917              : /* Expand the __atomic_store intrinsic:
    6918              :         void __atomic_store (TYPE *object, TYPE desired, enum memmodel)
    6919              :    EXP is the CALL_EXPR.
    6920              :    TARGET is an optional place for us to store the results.  */
    6921              : 
    6922              : static rtx
    6923        17608 : expand_builtin_atomic_store (machine_mode mode, tree exp)
    6924              : {
    6925        17608 :   memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 2));
    6926        17608 :   if (!(is_mm_relaxed (model) || is_mm_seq_cst (model)
    6927         1672 :         || is_mm_release (model)))
    6928              :     model = MEMMODEL_SEQ_CST;
    6929              : 
    6930        17608 :   if (!flag_inline_atomics)
    6931              :     return NULL_RTX;
    6932              : 
    6933              :   /* Expand the operands.  */
    6934        17577 :   rtx mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    6935        17577 :   rtx val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
    6936              : 
    6937        17577 :   return expand_atomic_store (mem, val, model, false);
    6938              : }
    6939              : 
    6940              : /* Expand the __atomic_fetch_XXX intrinsic:
    6941              :         TYPE __atomic_fetch_XXX (TYPE *object, TYPE val, enum memmodel)
    6942              :    EXP is the CALL_EXPR.
    6943              :    TARGET is an optional place for us to store the results.
    6944              :    CODE is the operation, PLUS, MINUS, ADD, XOR, or IOR.
    6945              :    FETCH_AFTER is true if returning the result of the operation.
    6946              :    FETCH_AFTER is false if returning the value before the operation.
    6947              :    IGNORE is true if the result is not used.
    6948              :    EXT_CALL is the correct builtin for an external call if this cannot be
    6949              :    resolved to an instruction sequence.  */
    6950              : 
    6951              : static rtx
    6952        29394 : expand_builtin_atomic_fetch_op (machine_mode mode, tree exp, rtx target,
    6953              :                                 enum rtx_code code, bool fetch_after,
    6954              :                                 bool ignore, enum built_in_function ext_call)
    6955              : {
    6956        29394 :   rtx val, mem, ret;
    6957        29394 :   enum memmodel model;
    6958        29394 :   tree fndecl, addr, oldval;
    6959              : 
    6960        29394 :   model = get_memmodel (CALL_EXPR_ARG (exp, 2));
    6961              : 
    6962              :   /* Expand the operands.  */
    6963        29394 :   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    6964        29394 :   val = expand_expr_force_mode (CALL_EXPR_ARG (exp, 1), mode);
    6965              : 
    6966              :   /* Only try generating instructions if inlining is turned on.  */
    6967        29394 :   if (flag_inline_atomics)
    6968              :     {
    6969        28851 :       ret = expand_atomic_fetch_op (target, mem, val, code, model, fetch_after);
    6970        28851 :       if (ret)
    6971              :         return ret;
    6972              :     }
    6973              : 
    6974              :   /* Return if a different routine isn't needed for the library call.  */
    6975         1159 :   if (ext_call == BUILT_IN_NONE)
    6976              :     return NULL_RTX;
    6977              : 
    6978              :   /* Change the call to the specified function.  */
    6979          288 :   fndecl = get_callee_fndecl (exp);
    6980          288 :   addr = CALL_EXPR_FN (exp);
    6981          288 :   STRIP_NOPS (addr);
    6982              : 
    6983          288 :   gcc_assert (TREE_OPERAND (addr, 0) == fndecl);
    6984          288 :   TREE_OPERAND (addr, 0) = builtin_decl_explicit (ext_call);
    6985          288 :   oldval = CALL_EXPR_ARG (exp, 1);
    6986              : 
    6987          288 :   if (!ignore && reg_overlap_mentioned_p (mem, val))
    6988              :     {
    6989            1 :       val = force_reg (mode, val);
    6990            1 :       CALL_EXPR_ARG (exp, 1) = make_tree (TREE_TYPE (oldval), val);
    6991              :     }
    6992              : 
    6993              :   /* If we will emit code after the call, the call cannot be a tail call.
    6994              :      If it is emitted as a tail call, a barrier is emitted after it, and
    6995              :      then all trailing code is removed.  */
    6996          288 :   if (!ignore)
    6997          191 :     CALL_EXPR_TAILCALL (exp) = 0;
    6998              : 
    6999              :   /* Expand the call here so we can emit trailing code.  */
    7000          288 :   ret = expand_call (exp, target, ignore);
    7001              : 
    7002              :   /* Replace the original function just in case it matters.  */
    7003          288 :   TREE_OPERAND (addr, 0) = fndecl;
    7004              : 
    7005              :   /* Then issue the arithmetic correction to return the right result.  */
    7006          288 :   if (!ignore)
    7007              :     {
    7008          191 :       CALL_EXPR_ARG (exp, 1) = oldval;
    7009          191 :       if (code == NOT)
    7010              :         {
    7011           33 :           ret = expand_simple_binop (mode, AND, ret, val, NULL_RTX, true,
    7012              :                                      OPTAB_LIB_WIDEN);
    7013           33 :           ret = expand_simple_unop (mode, NOT, ret, target, true);
    7014              :         }
    7015              :       else
    7016          158 :         ret = expand_simple_binop (mode, code, ret, val, target, true,
    7017              :                                    OPTAB_LIB_WIDEN);
    7018              :     }
    7019              :   return ret;
    7020              : }
    7021              : 
    7022              : /* Expand IFN_ATOMIC_BIT_TEST_AND_* internal function.  */
    7023              : 
    7024              : void
    7025          433 : expand_ifn_atomic_bit_test_and (gcall *call)
    7026              : {
    7027          433 :   tree ptr = gimple_call_arg (call, 0);
    7028          433 :   tree bit = gimple_call_arg (call, 1);
    7029          433 :   tree flag = gimple_call_arg (call, 2);
    7030          433 :   tree lhs = gimple_call_lhs (call);
    7031          433 :   enum memmodel model = MEMMODEL_SYNC_SEQ_CST;
    7032          433 :   machine_mode mode = TYPE_MODE (TREE_TYPE (flag));
    7033          433 :   enum rtx_code code;
    7034          433 :   optab optab;
    7035          433 :   class expand_operand ops[5];
    7036              : 
    7037          433 :   gcc_assert (flag_inline_atomics);
    7038              : 
    7039          433 :   if (gimple_call_num_args (call) == 5)
    7040          302 :     model = get_memmodel (gimple_call_arg (call, 3));
    7041              : 
    7042          433 :   rtx mem = get_builtin_sync_mem (ptr, mode);
    7043          433 :   rtx val = expand_expr_force_mode (bit, mode);
    7044              : 
    7045          433 :   switch (gimple_call_internal_fn (call))
    7046              :     {
    7047              :     case IFN_ATOMIC_BIT_TEST_AND_SET:
    7048              :       code = IOR;
    7049              :       optab = atomic_bit_test_and_set_optab;
    7050              :       break;
    7051              :     case IFN_ATOMIC_BIT_TEST_AND_COMPLEMENT:
    7052              :       code = XOR;
    7053              :       optab = atomic_bit_test_and_complement_optab;
    7054              :       break;
    7055              :     case IFN_ATOMIC_BIT_TEST_AND_RESET:
    7056              :       code = AND;
    7057              :       optab = atomic_bit_test_and_reset_optab;
    7058              :       break;
    7059            0 :     default:
    7060            0 :       gcc_unreachable ();
    7061              :     }
    7062              : 
    7063          433 :   if (lhs == NULL_TREE)
    7064              :     {
    7065            0 :       rtx val2 = expand_simple_binop (mode, ASHIFT, const1_rtx,
    7066              :                                       val, NULL_RTX, true, OPTAB_DIRECT);
    7067            0 :       if (code == AND)
    7068            0 :         val2 = expand_simple_unop (mode, NOT, val2, NULL_RTX, true);
    7069            0 :       if (expand_atomic_fetch_op (const0_rtx, mem, val2, code, model, false))
    7070          433 :         return;
    7071              :     }
    7072              : 
    7073            0 :   rtx target;
    7074            0 :   if (lhs)
    7075          433 :     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    7076              :   else
    7077            0 :     target = gen_reg_rtx (mode);
    7078          433 :   enum insn_code icode = direct_optab_handler (optab, mode);
    7079          433 :   gcc_assert (icode != CODE_FOR_nothing);
    7080          433 :   create_output_operand (&ops[0], target, mode);
    7081          433 :   create_fixed_operand (&ops[1], mem);
    7082          433 :   create_convert_operand_to (&ops[2], val, mode, true);
    7083          433 :   create_integer_operand (&ops[3], model);
    7084          433 :   create_integer_operand (&ops[4], integer_onep (flag));
    7085          433 :   if (maybe_expand_insn (icode, 5, ops))
    7086              :     return;
    7087              : 
    7088            0 :   if (reg_overlap_mentioned_p (mem, val))
    7089            0 :     val = force_reg (mode, val);
    7090              : 
    7091            0 :   rtx bitval = val;
    7092            0 :   val = expand_simple_binop (mode, ASHIFT, const1_rtx,
    7093              :                              val, NULL_RTX, true, OPTAB_DIRECT);
    7094            0 :   rtx maskval = val;
    7095            0 :   if (code == AND)
    7096            0 :     val = expand_simple_unop (mode, NOT, val, NULL_RTX, true);
    7097            0 :   rtx result = expand_atomic_fetch_op (gen_reg_rtx (mode), mem, val,
    7098              :                                        code, model, false);
    7099            0 :   if (!result)
    7100              :     {
    7101            0 :       bool is_atomic = gimple_call_num_args (call) == 5;
    7102            0 :       tree tcall = gimple_call_arg (call, 3 + is_atomic);
    7103            0 :       tree fndecl = gimple_call_addr_fndecl (tcall);
    7104            0 :       tree type = TREE_TYPE (TREE_TYPE (fndecl));
    7105            0 :       tree exp;
    7106            0 :       if (is_atomic)
    7107            0 :         exp = build_call_nary (type, tcall, 3,
    7108              :                                ptr, make_tree (type, val),
    7109              :                                gimple_call_arg (call, 3));
    7110              :       else
    7111            0 :         exp = build_call_nary (type, tcall, 2, ptr, make_tree (type, val));
    7112            0 :       result = expand_builtin (exp, gen_reg_rtx (mode), NULL_RTX,
    7113              :                                mode, !lhs);
    7114              :     }
    7115            0 :   if (!lhs)
    7116              :     return;
    7117            0 :   if (integer_onep (flag))
    7118              :     {
    7119            0 :       result = expand_simple_binop (mode, ASHIFTRT, result, bitval,
    7120              :                                     NULL_RTX, true, OPTAB_DIRECT);
    7121            0 :       result = expand_simple_binop (mode, AND, result, const1_rtx, target,
    7122              :                                     true, OPTAB_DIRECT);
    7123              :     }
    7124              :   else
    7125            0 :     result = expand_simple_binop (mode, AND, result, maskval, target, true,
    7126              :                                   OPTAB_DIRECT);
    7127            0 :   if (result != target)
    7128            0 :     emit_move_insn (target, result);
    7129              : }
    7130              : 
    7131              : /* Expand IFN_ATOMIC_*_FETCH_CMP_0 internal function.  */
    7132              : 
    7133              : void
    7134         2151 : expand_ifn_atomic_op_fetch_cmp_0 (gcall *call)
    7135              : {
    7136         2151 :   tree cmp = gimple_call_arg (call, 0);
    7137         2151 :   tree ptr = gimple_call_arg (call, 1);
    7138         2151 :   tree arg = gimple_call_arg (call, 2);
    7139         2151 :   tree lhs = gimple_call_lhs (call);
    7140         2151 :   enum memmodel model = MEMMODEL_SYNC_SEQ_CST;
    7141         2151 :   machine_mode mode = TYPE_MODE (TREE_TYPE (cmp));
    7142         2151 :   optab optab;
    7143         2151 :   rtx_code code;
    7144         2151 :   class expand_operand ops[5];
    7145              : 
    7146         2151 :   gcc_assert (flag_inline_atomics);
    7147              : 
    7148         2151 :   if (gimple_call_num_args (call) == 5)
    7149         1743 :     model = get_memmodel (gimple_call_arg (call, 3));
    7150              : 
    7151         2151 :   rtx mem = get_builtin_sync_mem (ptr, mode);
    7152         2151 :   rtx op = expand_expr_force_mode (arg, mode);
    7153              : 
    7154         2151 :   switch (gimple_call_internal_fn (call))
    7155              :     {
    7156              :     case IFN_ATOMIC_ADD_FETCH_CMP_0:
    7157              :       code = PLUS;
    7158              :       optab = atomic_add_fetch_cmp_0_optab;
    7159              :       break;
    7160              :     case IFN_ATOMIC_SUB_FETCH_CMP_0:
    7161              :       code = MINUS;
    7162              :       optab = atomic_sub_fetch_cmp_0_optab;
    7163              :       break;
    7164              :     case IFN_ATOMIC_AND_FETCH_CMP_0:
    7165              :       code = AND;
    7166              :       optab = atomic_and_fetch_cmp_0_optab;
    7167              :       break;
    7168              :     case IFN_ATOMIC_OR_FETCH_CMP_0:
    7169              :       code = IOR;
    7170              :       optab = atomic_or_fetch_cmp_0_optab;
    7171              :       break;
    7172              :     case IFN_ATOMIC_XOR_FETCH_CMP_0:
    7173              :       code = XOR;
    7174              :       optab = atomic_xor_fetch_cmp_0_optab;
    7175              :       break;
    7176            0 :     default:
    7177            0 :       gcc_unreachable ();
    7178              :     }
    7179              : 
    7180         2151 :   enum rtx_code comp = UNKNOWN;
    7181         2151 :   switch (tree_to_uhwi (cmp))
    7182              :     {
    7183              :     case ATOMIC_OP_FETCH_CMP_0_EQ: comp = EQ; break;
    7184              :     case ATOMIC_OP_FETCH_CMP_0_NE: comp = NE; break;
    7185              :     case ATOMIC_OP_FETCH_CMP_0_GT: comp = GT; break;
    7186              :     case ATOMIC_OP_FETCH_CMP_0_GE: comp = GE; break;
    7187              :     case ATOMIC_OP_FETCH_CMP_0_LT: comp = LT; break;
    7188              :     case ATOMIC_OP_FETCH_CMP_0_LE: comp = LE; break;
    7189            0 :     default: gcc_unreachable ();
    7190              :     }
    7191              : 
    7192         2151 :   rtx target;
    7193         2151 :   if (lhs == NULL_TREE)
    7194            0 :     target = gen_reg_rtx (TYPE_MODE (boolean_type_node));
    7195              :   else
    7196         2151 :     target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
    7197         2151 :   enum insn_code icode = direct_optab_handler (optab, mode);
    7198         2151 :   gcc_assert (icode != CODE_FOR_nothing);
    7199         2151 :   create_output_operand (&ops[0], target, TYPE_MODE (boolean_type_node));
    7200         2151 :   create_fixed_operand (&ops[1], mem);
    7201         2151 :   create_convert_operand_to (&ops[2], op, mode, true);
    7202         2151 :   create_integer_operand (&ops[3], model);
    7203         2151 :   create_integer_operand (&ops[4], comp);
    7204         2151 :   if (maybe_expand_insn (icode, 5, ops))
    7205         2119 :     return;
    7206              : 
    7207           32 :   rtx result = expand_atomic_fetch_op (gen_reg_rtx (mode), mem, op,
    7208              :                                        code, model, true);
    7209           32 :   if (!result)
    7210              :     {
    7211            0 :       bool is_atomic = gimple_call_num_args (call) == 5;
    7212            0 :       tree tcall = gimple_call_arg (call, 3 + is_atomic);
    7213            0 :       tree fndecl = gimple_call_addr_fndecl (tcall);
    7214            0 :       tree type = TREE_TYPE (TREE_TYPE (fndecl));
    7215            0 :       tree exp;
    7216            0 :       if (is_atomic)
    7217            0 :         exp = build_call_nary (type, tcall, 3,
    7218              :                                ptr, arg,
    7219              :                                gimple_call_arg (call, 3));
    7220              :       else
    7221            0 :         exp = build_call_nary (type, tcall, 2, ptr, arg);
    7222            0 :       result = expand_builtin (exp, gen_reg_rtx (mode), NULL_RTX,
    7223              :                                mode, !lhs);
    7224              :     }
    7225              : 
    7226           32 :   if (lhs)
    7227              :     {
    7228           32 :       result = emit_store_flag_force (target, comp, result, const0_rtx, mode,
    7229              :                                       0, 1);
    7230           32 :       if (result != target)
    7231           32 :         emit_move_insn (target, result);
    7232              :     }
    7233              : }
    7234              : 
    7235              : /* Expand an atomic clear operation.
    7236              :         void _atomic_clear (BOOL *obj, enum memmodel)
    7237              :    EXP is the call expression.  */
    7238              : 
    7239              : static rtx
    7240           53 : expand_builtin_atomic_clear (tree exp)
    7241              : {
    7242           53 :   machine_mode mode = int_mode_for_size (BOOL_TYPE_SIZE, 0).require ();
    7243           53 :   rtx mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    7244           53 :   memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 1));
    7245              : 
    7246           53 :   if (is_mm_consume (model) || is_mm_acquire (model) || is_mm_acq_rel (model))
    7247              :     model = MEMMODEL_SEQ_CST;
    7248              : 
    7249              :   /* Try issuing an __atomic_store, and allow fallback to __sync_lock_release.
    7250              :      Failing that, a store is issued by __atomic_store.  The only way this can
    7251              :      fail is if the bool type is larger than a word size.  Unlikely, but
    7252              :      handle it anyway for completeness.  Assume a single threaded model since
    7253              :      there is no atomic support in this case, and no barriers are required.  */
    7254           53 :   rtx ret = expand_atomic_store (mem, const0_rtx, model, true);
    7255           53 :   if (!ret)
    7256            0 :     emit_move_insn (mem, const0_rtx);
    7257           53 :   return const0_rtx;
    7258              : }
    7259              : 
    7260              : /* Expand an atomic test_and_set operation.
    7261              :         bool _atomic_test_and_set (BOOL *obj, enum memmodel)
    7262              :    EXP is the call expression.  */
    7263              : 
    7264              : static rtx
    7265          266 : expand_builtin_atomic_test_and_set (tree exp, rtx target)
    7266              : {
    7267          266 :   rtx mem;
    7268          266 :   enum memmodel model;
    7269          266 :   machine_mode mode;
    7270              : 
    7271          266 :   mode = int_mode_for_size (BOOL_TYPE_SIZE, 0).require ();
    7272          266 :   mem = get_builtin_sync_mem (CALL_EXPR_ARG (exp, 0), mode);
    7273          266 :   model = get_memmodel (CALL_EXPR_ARG (exp, 1));
    7274              : 
    7275          266 :   return expand_atomic_test_and_set (target, mem, model);
    7276              : }
    7277              : 
    7278              : 
    7279              : /* Return true if (optional) argument ARG1 of size ARG0 is always lock free on
    7280              :    this architecture.  If ARG1 is NULL, use typical alignment for size ARG0.  */
    7281              : 
    7282              : static tree
    7283       143460 : fold_builtin_atomic_always_lock_free (tree arg0, tree arg1)
    7284              : {
    7285       143460 :   HOST_WIDE_INT size;
    7286       143460 :   machine_mode mode;
    7287       143460 :   unsigned int mode_align, type_align;
    7288              : 
    7289       143460 :   if (!tree_fits_shwi_p (arg0))
    7290              :     return NULL_TREE;
    7291              : 
    7292              :   /* We need a corresponding integer mode for the access to be lock-free.  */
    7293       143307 :   size = tree_to_shwi (arg0) * BITS_PER_UNIT;
    7294       143307 :   if (!int_mode_for_size (size, 0).exists (&mode))
    7295            9 :     return boolean_false_node;
    7296              : 
    7297       143298 :   mode_align = GET_MODE_ALIGNMENT (mode);
    7298              : 
    7299       143298 :   if (tree_fits_uhwi_p (arg1))
    7300              :     {
    7301       123674 :       unsigned HOST_WIDE_INT val = tree_to_uhwi (arg1);
    7302              : 
    7303              :       /* Either this argument is null, or it's a fake pointer encoding
    7304              :          the alignment of the object.  */
    7305       123674 :       val = least_bit_hwi (val);
    7306       123674 :       val *= BITS_PER_UNIT;
    7307              : 
    7308       123674 :       if (val == 0 || mode_align < val)
    7309              :         type_align = mode_align;
    7310              :       else
    7311        18867 :         type_align = val;
    7312              :     }
    7313              :   else
    7314              :     {
    7315        19624 :       tree ttype = TREE_TYPE (arg1);
    7316              : 
    7317              :       /* This function is usually invoked and folded immediately by the front
    7318              :          end before anything else has a chance to look at it.  The pointer
    7319              :          parameter at this point is usually cast to a void *, so check for that
    7320              :          and look past the cast.  */
    7321          195 :       if (CONVERT_EXPR_P (arg1)
    7322        19460 :           && POINTER_TYPE_P (ttype)
    7323        19460 :           && VOID_TYPE_P (TREE_TYPE (ttype))
    7324        39084 :           && POINTER_TYPE_P (TREE_TYPE (TREE_OPERAND (arg1, 0))))
    7325        19429 :         arg1 = TREE_OPERAND (arg1, 0);
    7326              : 
    7327        19624 :       ttype = TREE_TYPE (arg1);
    7328        19624 :       gcc_assert (POINTER_TYPE_P (ttype));
    7329              : 
    7330              :       /* Get the underlying type of the object.  */
    7331        19624 :       ttype = TREE_TYPE (ttype);
    7332        19624 :       type_align = TYPE_ALIGN (ttype);
    7333              :     }
    7334              : 
    7335              :   /* If the object has smaller alignment, the lock free routines cannot
    7336              :      be used.  */
    7337       143298 :   if (type_align < mode_align)
    7338          247 :     return boolean_false_node;
    7339              : 
    7340              :   /* Check if a compare_and_swap pattern exists for the mode which represents
    7341              :      the required size.  The pattern is not allowed to fail, so the existence
    7342              :      of the pattern indicates support is present.  Also require that an
    7343              :      atomic load exists for the required size.  */
    7344       143051 :   if (can_compare_and_swap_p (mode, true) && can_atomic_load_p (mode))
    7345       143009 :     return boolean_true_node;
    7346              :   else
    7347           42 :     return boolean_false_node;
    7348              : }
    7349              : 
    7350              : /* Return true if the parameters to call EXP represent an object which will
    7351              :    always generate lock free instructions.  The first argument represents the
    7352              :    size of the object, and the second parameter is a pointer to the object
    7353              :    itself.  If NULL is passed for the object, then the result is based on
    7354              :    typical alignment for an object of the specified size.  Otherwise return
    7355              :    false.  */
    7356              : 
    7357              : static rtx
    7358            1 : expand_builtin_atomic_always_lock_free (tree exp)
    7359              : {
    7360            1 :   tree size;
    7361            1 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
    7362            1 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
    7363              : 
    7364            1 :   if (TREE_CODE (arg0) != INTEGER_CST)
    7365              :     {
    7366            1 :       error ("non-constant argument 1 to %qs", "__atomic_always_lock_free");
    7367            1 :       return const0_rtx;
    7368              :     }
    7369              : 
    7370            0 :   size = fold_builtin_atomic_always_lock_free (arg0, arg1);
    7371            0 :   if (size == boolean_true_node)
    7372            0 :     return const1_rtx;
    7373            0 :   return const0_rtx;
    7374              : }
    7375              : 
    7376              : /* Return a one or zero if it can be determined that object ARG1 of size ARG
    7377              :    is lock free on this architecture.  */
    7378              : 
    7379              : static tree
    7380        38590 : fold_builtin_atomic_is_lock_free (tree arg0, tree arg1)
    7381              : {
    7382        38590 :   if (!flag_inline_atomics)
    7383              :     return NULL_TREE;
    7384              : 
    7385              :   /* If it isn't always lock free, don't generate a result.  */
    7386        38581 :   if (fold_builtin_atomic_always_lock_free (arg0, arg1) == boolean_true_node)
    7387        38222 :     return boolean_true_node;
    7388              : 
    7389              :   return NULL_TREE;
    7390              : }
    7391              : 
    7392              : /* Return true if the parameters to call EXP represent an object which will
    7393              :    always generate lock free instructions.  The first argument represents the
    7394              :    size of the object, and the second parameter is a pointer to the object
    7395              :    itself.  If NULL is passed for the object, then the result is based on
    7396              :    typical alignment for an object of the specified size.  Otherwise return
    7397              :    NULL*/
    7398              : 
    7399              : static rtx
    7400           10 : expand_builtin_atomic_is_lock_free (tree exp)
    7401              : {
    7402           10 :   tree size;
    7403           10 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
    7404           10 :   tree arg1 = CALL_EXPR_ARG (exp, 1);
    7405              : 
    7406           10 :   if (!INTEGRAL_TYPE_P (TREE_TYPE (arg0)))
    7407              :     {
    7408            0 :       error ("non-integer argument 1 to %qs", "__atomic_is_lock_free");
    7409            0 :       return NULL_RTX;
    7410              :     }
    7411              : 
    7412           10 :   if (!flag_inline_atomics)
    7413              :     return NULL_RTX;
    7414              : 
    7415              :   /* If the value is known at compile time, return the RTX for it.  */
    7416            9 :   size = fold_builtin_atomic_is_lock_free (arg0, arg1);
    7417            9 :   if (size == boolean_true_node)
    7418            0 :     return const1_rtx;
    7419              : 
    7420              :   return NULL_RTX;
    7421              : }
    7422              : 
    7423              : /* Expand the __atomic_thread_fence intrinsic:
    7424              :         void __atomic_thread_fence (enum memmodel)
    7425              :    EXP is the CALL_EXPR.  */
    7426              : 
    7427              : static void
    7428          719 : expand_builtin_atomic_thread_fence (tree exp)
    7429              : {
    7430          719 :   enum memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 0));
    7431          719 :   expand_mem_thread_fence (model);
    7432          719 : }
    7433              : 
    7434              : /* Expand the __atomic_signal_fence intrinsic:
    7435              :         void __atomic_signal_fence (enum memmodel)
    7436              :    EXP is the CALL_EXPR.  */
    7437              : 
    7438              : static void
    7439           60 : expand_builtin_atomic_signal_fence (tree exp)
    7440              : {
    7441           60 :   enum memmodel model = get_memmodel (CALL_EXPR_ARG (exp, 0));
    7442           60 :   expand_mem_signal_fence (model);
    7443           60 : }
    7444              : 
    7445              : /* Expand the __sync_synchronize intrinsic.  */
    7446              : 
    7447              : static void
    7448          275 : expand_builtin_sync_synchronize (void)
    7449              : {
    7450            0 :   expand_mem_thread_fence (MEMMODEL_SYNC_SEQ_CST);
    7451            0 : }
    7452              : 
    7453              : static rtx
    7454            3 : expand_builtin_thread_pointer (tree exp, rtx target)
    7455              : {
    7456            3 :   enum insn_code icode;
    7457            3 :   if (!validate_arglist (exp, VOID_TYPE))
    7458            0 :     return const0_rtx;
    7459            3 :   icode = direct_optab_handler (get_thread_pointer_optab, Pmode);
    7460            3 :   if (icode != CODE_FOR_nothing)
    7461              :     {
    7462            3 :       class expand_operand op;
    7463              :       /* If the target is not sutitable then create a new target. */
    7464            3 :       if (target == NULL_RTX
    7465            3 :           || !REG_P (target)
    7466            6 :           || GET_MODE (target) != Pmode)
    7467            0 :         target = gen_reg_rtx (Pmode);
    7468            3 :       create_output_operand (&op, target, Pmode);
    7469            3 :       expand_insn (icode, 1, &op);
    7470            3 :       return target;
    7471              :     }
    7472            0 :   error ("%<__builtin_thread_pointer%> is not supported on this target");
    7473            0 :   return const0_rtx;
    7474              : }
    7475              : 
    7476              : static void
    7477            0 : expand_builtin_set_thread_pointer (tree exp)
    7478              : {
    7479            0 :   enum insn_code icode;
    7480            0 :   if (!validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    7481              :     return;
    7482            0 :   icode = direct_optab_handler (set_thread_pointer_optab, Pmode);
    7483            0 :   if (icode != CODE_FOR_nothing)
    7484              :     {
    7485            0 :       class expand_operand op;
    7486            0 :       rtx val = expand_expr (CALL_EXPR_ARG (exp, 0), NULL_RTX,
    7487            0 :                              Pmode, EXPAND_NORMAL);
    7488            0 :       create_input_operand (&op, val, Pmode);
    7489            0 :       expand_insn (icode, 1, &op);
    7490            0 :       return;
    7491              :     }
    7492            0 :   error ("%<__builtin_set_thread_pointer%> is not supported on this target");
    7493              : }
    7494              : 
    7495              : 
    7496              : /* Emit code to restore the current value of stack.  */
    7497              : 
    7498              : static void
    7499         1762 : expand_stack_restore (tree var)
    7500              : {
    7501         1762 :   rtx_insn *prev;
    7502         1762 :   rtx sa = expand_normal (var);
    7503              : 
    7504         1818 :   sa = convert_memory_address (Pmode, sa);
    7505              : 
    7506         1762 :   prev = get_last_insn ();
    7507         1762 :   emit_stack_restore (SAVE_BLOCK, sa);
    7508              : 
    7509         1762 :   record_new_stack_level ();
    7510              : 
    7511         1762 :   fixup_args_size_notes (prev, get_last_insn (), 0);
    7512         1762 : }
    7513              : 
    7514              : /* Emit code to save the current value of stack.  */
    7515              : 
    7516              : static rtx
    7517         1863 : expand_stack_save (void)
    7518              : {
    7519         1863 :   rtx ret = NULL_RTX;
    7520              : 
    7521            0 :   emit_stack_save (SAVE_BLOCK, &ret);
    7522         1863 :   return ret;
    7523              : }
    7524              : 
    7525              : /* Emit code to get the openacc gang, worker or vector id or size.  */
    7526              : 
    7527              : static rtx
    7528          356 : expand_builtin_goacc_parlevel_id_size (tree exp, rtx target, int ignore)
    7529              : {
    7530          356 :   const char *name;
    7531          356 :   rtx fallback_retval;
    7532          356 :   rtx_insn *(*gen_fn) (rtx, rtx);
    7533          356 :   switch (DECL_FUNCTION_CODE (get_callee_fndecl (exp)))
    7534              :     {
    7535          244 :     case BUILT_IN_GOACC_PARLEVEL_ID:
    7536          244 :       name = "__builtin_goacc_parlevel_id";
    7537          244 :       fallback_retval = const0_rtx;
    7538          244 :       gen_fn = targetm.gen_oacc_dim_pos;
    7539          244 :       break;
    7540          112 :     case BUILT_IN_GOACC_PARLEVEL_SIZE:
    7541          112 :       name = "__builtin_goacc_parlevel_size";
    7542          112 :       fallback_retval = const1_rtx;
    7543          112 :       gen_fn = targetm.gen_oacc_dim_size;
    7544          112 :       break;
    7545            0 :     default:
    7546            0 :       gcc_unreachable ();
    7547              :     }
    7548              : 
    7549          356 :   if (oacc_get_fn_attrib (current_function_decl) == NULL_TREE)
    7550              :     {
    7551            8 :       error ("%qs only supported in OpenACC code", name);
    7552            8 :       return const0_rtx;
    7553              :     }
    7554              : 
    7555          348 :   tree arg = CALL_EXPR_ARG (exp, 0);
    7556          348 :   if (TREE_CODE (arg) != INTEGER_CST)
    7557              :     {
    7558            8 :       error ("non-constant argument 0 to %qs", name);
    7559            8 :       return const0_rtx;
    7560              :     }
    7561              : 
    7562          340 :   int dim = TREE_INT_CST_LOW (arg);
    7563          340 :   switch (dim)
    7564              :     {
    7565          324 :     case GOMP_DIM_GANG:
    7566          324 :     case GOMP_DIM_WORKER:
    7567          324 :     case GOMP_DIM_VECTOR:
    7568          324 :       break;
    7569           16 :     default:
    7570           16 :       error ("illegal argument 0 to %qs", name);
    7571           16 :       return const0_rtx;
    7572              :     }
    7573              : 
    7574          324 :   if (ignore)
    7575              :     return target;
    7576              : 
    7577          180 :   if (target == NULL_RTX)
    7578            0 :     target = gen_reg_rtx (TYPE_MODE (TREE_TYPE (exp)));
    7579              : 
    7580          180 :   if (!targetm.have_oacc_dim_size ())
    7581              :     {
    7582          180 :       emit_move_insn (target, fallback_retval);
    7583          180 :       return target;
    7584              :     }
    7585              : 
    7586            0 :   rtx reg = MEM_P (target) ? gen_reg_rtx (GET_MODE (target)) : target;
    7587            0 :   emit_insn (gen_fn (reg, GEN_INT (dim)));
    7588            0 :   if (reg != target)
    7589            0 :     emit_move_insn (target, reg);
    7590              : 
    7591              :   return target;
    7592              : }
    7593              : 
    7594              : /* Expand a string compare operation using a sequence of char comparison
    7595              :    to get rid of the calling overhead, with result going to TARGET if
    7596              :    that's convenient.
    7597              : 
    7598              :    VAR_STR is the variable string source;
    7599              :    CONST_STR is the constant string source;
    7600              :    LENGTH is the number of chars to compare;
    7601              :    CONST_STR_N indicates which source string is the constant string;
    7602              :    IS_MEMCMP indicates whether it's a memcmp or strcmp.
    7603              : 
    7604              :    to: (assume const_str_n is 2, i.e., arg2 is a constant string)
    7605              : 
    7606              :    target = (int) (unsigned char) var_str[0]
    7607              :             - (int) (unsigned char) const_str[0];
    7608              :    if (target != 0)
    7609              :      goto ne_label;
    7610              :      ...
    7611              :    target = (int) (unsigned char) var_str[length - 2]
    7612              :             - (int) (unsigned char) const_str[length - 2];
    7613              :    if (target != 0)
    7614              :      goto ne_label;
    7615              :    target = (int) (unsigned char) var_str[length - 1]
    7616              :             - (int) (unsigned char) const_str[length - 1];
    7617              :    ne_label:
    7618              :   */
    7619              : 
    7620              : static rtx
    7621          635 : inline_string_cmp (rtx target, tree var_str, const char *const_str,
    7622              :                    unsigned HOST_WIDE_INT length,
    7623              :                    int const_str_n, machine_mode mode)
    7624              : {
    7625          635 :   HOST_WIDE_INT offset = 0;
    7626          635 :   rtx var_rtx_array
    7627          635 :     = get_memory_rtx (var_str, build_int_cst (unsigned_type_node, length),
    7628              :                       false);
    7629          635 :   rtx var_rtx = NULL_RTX;
    7630          635 :   rtx const_rtx = NULL_RTX;
    7631          635 :   rtx result = target ? target : gen_reg_rtx (mode);
    7632          635 :   rtx_code_label *ne_label = gen_label_rtx ();
    7633          635 :   tree unit_type_node = unsigned_char_type_node;
    7634          635 :   scalar_int_mode unit_mode
    7635          635 :     = as_a <scalar_int_mode> TYPE_MODE (unit_type_node);
    7636              : 
    7637          635 :   start_sequence ();
    7638              : 
    7639         2964 :   for (unsigned HOST_WIDE_INT i = 0; i < length; i++)
    7640              :     {
    7641         1694 :       var_rtx
    7642         1694 :         = adjust_address (var_rtx_array, TYPE_MODE (unit_type_node), offset);
    7643         1694 :       const_rtx = c_readstr (const_str + offset, unit_mode);
    7644         1694 :       rtx op0 = (const_str_n == 1) ? const_rtx : var_rtx;
    7645         1544 :       rtx op1 = (const_str_n == 1) ? var_rtx : const_rtx;
    7646              : 
    7647         1694 :       op0 = convert_modes (mode, unit_mode, op0, 1);
    7648         1694 :       op1 = convert_modes (mode, unit_mode, op1, 1);
    7649         1694 :       rtx diff = expand_simple_binop (mode, MINUS, op0, op1,
    7650              :                                       result, 1, OPTAB_WIDEN);
    7651              : 
    7652              :       /* Force the difference into result register.  We cannot reassign
    7653              :          result here ("result = diff") or we may end up returning
    7654              :          uninitialized result when expand_simple_binop allocates a new
    7655              :          pseudo-register for returning.  */
    7656         1694 :       if (diff != result)
    7657            0 :         emit_move_insn (result, diff);
    7658              : 
    7659         1694 :       if (i < length - 1)
    7660         1059 :         emit_cmp_and_jump_insns (result, CONST0_RTX (mode), NE, NULL_RTX,
    7661              :                                  mode, true, ne_label);
    7662         3388 :       offset += GET_MODE_SIZE (unit_mode);
    7663              :     }
    7664              : 
    7665          635 :   emit_label (ne_label);
    7666          635 :   rtx_insn *insns = end_sequence ();
    7667          635 :   emit_insn (insns);
    7668              : 
    7669          635 :   return result;
    7670              : }
    7671              : 
    7672              : /* Inline expansion of a call to str(n)cmp and memcmp, with result going
    7673              :    to TARGET if that's convenient.
    7674              :    If the call is not been inlined, return NULL_RTX.  */
    7675              : 
    7676              : static rtx
    7677       145227 : inline_expand_builtin_bytecmp (tree exp, rtx target)
    7678              : {
    7679       145227 :   tree fndecl = get_callee_fndecl (exp);
    7680       145227 :   enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
    7681       145227 :   bool is_ncmp = (fcode == BUILT_IN_STRNCMP || fcode == BUILT_IN_MEMCMP);
    7682              : 
    7683              :   /* Do NOT apply this inlining expansion when optimizing for size or
    7684              :      optimization level below 2 or if unused *cmp hasn't been DCEd.  */
    7685       145227 :   if (optimize < 2 || optimize_insn_for_size_p () || target == const0_rtx)
    7686              :     return NULL_RTX;
    7687              : 
    7688        23076 :   gcc_checking_assert (fcode == BUILT_IN_STRCMP
    7689              :                        || fcode == BUILT_IN_STRNCMP
    7690              :                        || fcode == BUILT_IN_MEMCMP);
    7691              : 
    7692              :   /* On a target where the type of the call (int) has same or narrower precision
    7693              :      than unsigned char, give up the inlining expansion.  */
    7694        23076 :   if (TYPE_PRECISION (unsigned_char_type_node)
    7695        23076 :       >= TYPE_PRECISION (TREE_TYPE (exp)))
    7696              :     return NULL_RTX;
    7697              : 
    7698        23076 :   tree arg1 = CALL_EXPR_ARG (exp, 0);
    7699        23076 :   tree arg2 = CALL_EXPR_ARG (exp, 1);
    7700        23076 :   tree len3_tree = is_ncmp ? CALL_EXPR_ARG (exp, 2) : NULL_TREE;
    7701              : 
    7702        23076 :   unsigned HOST_WIDE_INT len1 = 0;
    7703        23076 :   unsigned HOST_WIDE_INT len2 = 0;
    7704        23076 :   unsigned HOST_WIDE_INT len3 = 0;
    7705              : 
    7706              :   /* Get the object representation of the initializers of ARG1 and ARG2
    7707              :      as strings, provided they refer to constant objects, with their byte
    7708              :      sizes in LEN1 and LEN2, respectively.  */
    7709        23076 :   const char *bytes1 = getbyterep (arg1, &len1);
    7710        23076 :   const char *bytes2 = getbyterep (arg2, &len2);
    7711              : 
    7712              :   /* Fail if neither argument refers to an initialized constant.  */
    7713        23076 :   if (!bytes1 && !bytes2)
    7714              :     return NULL_RTX;
    7715              : 
    7716        18437 :   if (is_ncmp)
    7717              :     {
    7718              :       /* Fail if the memcmp/strncmp bound is not a constant.  */
    7719         1328 :       if (!tree_fits_uhwi_p (len3_tree))
    7720              :         return NULL_RTX;
    7721              : 
    7722          962 :       len3 = tree_to_uhwi (len3_tree);
    7723              : 
    7724          962 :       if (fcode == BUILT_IN_MEMCMP)
    7725              :         {
    7726              :           /* Fail if the memcmp bound is greater than the size of either
    7727              :              of the two constant objects.  */
    7728          462 :           if ((bytes1 && len1 < len3)
    7729          462 :               || (bytes2 && len2 < len3))
    7730              :             return NULL_RTX;
    7731              :         }
    7732              :     }
    7733              : 
    7734              :   if (fcode != BUILT_IN_MEMCMP)
    7735              :     {
    7736              :       /* For string functions (i.e., strcmp and strncmp) reduce LEN1
    7737              :          and LEN2 to the length of the nul-terminated string stored
    7738              :          in each.  */
    7739        17609 :       if (bytes1 != NULL)
    7740          236 :         len1 = strnlen (bytes1, len1) + 1;
    7741        17609 :       if (bytes2 != NULL)
    7742        17376 :         len2 = strnlen (bytes2, len2) + 1;
    7743              :     }
    7744              : 
    7745              :   /* See inline_string_cmp.  */
    7746        18067 :   int const_str_n;
    7747        18067 :   if (!len1)
    7748              :     const_str_n = 2;
    7749          241 :   else if (!len2)
    7750              :     const_str_n = 1;
    7751            3 :   else if (len2 > len1)
    7752              :     const_str_n = 1;
    7753              :   else
    7754              :     const_str_n = 2;
    7755              : 
    7756              :   /* For strncmp only, compute the new bound as the smallest of
    7757              :      the lengths of the two strings (plus 1) and the bound provided
    7758              :      to the function.  */
    7759        17828 :   unsigned HOST_WIDE_INT bound = (const_str_n == 1) ? len1 : len2;
    7760        18067 :   if (is_ncmp && len3 < bound)
    7761          534 :     bound = len3;
    7762              : 
    7763              :   /* If the bound of the comparison is larger than the threshold,
    7764              :      do nothing.  */
    7765        18067 :   if (bound > (unsigned HOST_WIDE_INT) param_builtin_string_cmp_inline_length)
    7766              :     return NULL_RTX;
    7767              : 
    7768          635 :   machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
    7769              : 
    7770              :   /* Now, start inline expansion the call.  */
    7771         1213 :   return inline_string_cmp (target, (const_str_n == 1) ? arg2 : arg1,
    7772              :                             (const_str_n == 1) ? bytes1 : bytes2, bound,
    7773          635 :                             const_str_n, mode);
    7774              : }
    7775              : 
    7776              : /* Expand a call to __builtin_speculation_safe_value_<N>.  MODE
    7777              :    represents the size of the first argument to that call, or VOIDmode
    7778              :    if the argument is a pointer.  IGNORE will be true if the result
    7779              :    isn't used.  */
    7780              : static rtx
    7781           34 : expand_speculation_safe_value (machine_mode mode, tree exp, rtx target,
    7782              :                                bool ignore)
    7783              : {
    7784           34 :   rtx val, failsafe;
    7785           34 :   unsigned nargs = call_expr_nargs (exp);
    7786              : 
    7787           34 :   tree arg0 = CALL_EXPR_ARG (exp, 0);
    7788              : 
    7789           34 :   if (mode == VOIDmode)
    7790              :     {
    7791            4 :       mode = TYPE_MODE (TREE_TYPE (arg0));
    7792            4 :       gcc_assert (GET_MODE_CLASS (mode) == MODE_INT);
    7793              :     }
    7794              : 
    7795           34 :   val = expand_expr (arg0, NULL_RTX, mode, EXPAND_NORMAL);
    7796              : 
    7797              :   /* An optional second argument can be used as a failsafe value on
    7798              :      some machines.  If it isn't present, then the failsafe value is
    7799              :      assumed to be 0.  */
    7800           34 :   if (nargs > 1)
    7801              :     {
    7802            4 :       tree arg1 = CALL_EXPR_ARG (exp, 1);
    7803            4 :       failsafe = expand_expr (arg1, NULL_RTX, mode, EXPAND_NORMAL);
    7804              :     }
    7805              :   else
    7806           30 :     failsafe = const0_rtx;
    7807              : 
    7808              :   /* If the result isn't used, the behavior is undefined.  It would be
    7809              :      nice to emit a warning here, but path splitting means this might
    7810              :      happen with legitimate code.  So simply drop the builtin
    7811              :      expansion in that case; we've handled any side-effects above.  */
    7812           34 :   if (ignore)
    7813            0 :     return const0_rtx;
    7814              : 
    7815              :   /* If we don't have a suitable target, create one to hold the result.  */
    7816           34 :   if (target == NULL || GET_MODE (target) != mode)
    7817            0 :     target = gen_reg_rtx (mode);
    7818              : 
    7819           34 :   if (GET_MODE (val) != mode && GET_MODE (val) != VOIDmode)
    7820            0 :     val = convert_modes (mode, VOIDmode, val, false);
    7821              : 
    7822           34 :   return targetm.speculation_safe_value (mode, target, val, failsafe);
    7823              : }
    7824              : 
    7825              : /* Expand CRC* or REV_CRC* built-ins.  */
    7826              : 
    7827              : rtx
    7828           46 : expand_builtin_crc_table_based (internal_fn fn, scalar_mode crc_mode,
    7829              :                                 scalar_mode data_mode, machine_mode mode,
    7830              :                                 tree exp, rtx target)
    7831              : {
    7832           46 :   tree rhs1 = CALL_EXPR_ARG (exp, 0); // crc
    7833           46 :   tree rhs2 = CALL_EXPR_ARG (exp, 1); // data
    7834           46 :   tree rhs3 = CALL_EXPR_ARG (exp, 2); // polynomial
    7835              : 
    7836           46 :   if (!target || mode == VOIDmode)
    7837            0 :     target = gen_reg_rtx (crc_mode);
    7838              : 
    7839           46 :   rtx op1 = expand_normal (rhs1);
    7840           46 :   rtx op2 = expand_normal (rhs2);
    7841           46 :   rtx op3;
    7842           46 :   if (TREE_CODE (rhs3) != INTEGER_CST)
    7843              :     {
    7844            1 :       error ("third argument to %<crc%> builtins must be a constant");
    7845            1 :       op3 = const0_rtx;
    7846              :     }
    7847              :   else
    7848           45 :     op3 = convert_to_mode (crc_mode, expand_normal (rhs3), 0);
    7849              : 
    7850           46 :   if (CONST_INT_P (op2))
    7851           32 :     op2 = convert_to_mode (crc_mode, op2, 0);
    7852              : 
    7853           46 :   if (fn == IFN_CRC)
    7854           29 :     expand_crc_table_based (target, op1, op2, op3, data_mode);
    7855              :   else
    7856              :     /* If it's IFN_CRC_REV generate bit-reversed CRC.  */
    7857           17 :     expand_reversed_crc_table_based (target, op1, op2, op3,
    7858              :                                      data_mode);
    7859           46 :   return target;
    7860              : }
    7861              : 
    7862              : /* Expand an expression EXP that calls a built-in function,
    7863              :    with result going to TARGET if that's convenient
    7864              :    (and in mode MODE if that's convenient).
    7865              :    SUBTARGET may be used as the target for computing one of EXP's operands.
    7866              :    IGNORE is nonzero if the value is to be ignored.  */
    7867              : 
    7868              : rtx
    7869      2035299 : expand_builtin (tree exp, rtx target, rtx subtarget, machine_mode mode,
    7870              :                 int ignore)
    7871              : {
    7872      2035299 :   tree fndecl = get_callee_fndecl (exp);
    7873      2035299 :   machine_mode target_mode = TYPE_MODE (TREE_TYPE (exp));
    7874      2035299 :   int flags;
    7875              : 
    7876      2035299 :   if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
    7877       177680 :     return targetm.expand_builtin (exp, target, subtarget, mode, ignore);
    7878              : 
    7879              :   /* When ASan is enabled, we don't want to expand some memory/string
    7880              :      builtins and rely on libsanitizer's hooks.  This allows us to avoid
    7881              :      redundant checks and be sure, that possible overflow will be detected
    7882              :      by ASan.  */
    7883              : 
    7884      1857619 :   enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
    7885      1857619 :   if (param_asan_kernel_mem_intrinsic_prefix
    7886      1857619 :       && sanitize_flags_p (SANITIZE_KERNEL_ADDRESS
    7887              :                            | SANITIZE_KERNEL_HWADDRESS))
    7888           42 :     switch (fcode)
    7889              :       {
    7890           42 :         rtx save_decl_rtl, ret;
    7891           42 :       case BUILT_IN_MEMCPY:
    7892           42 :       case BUILT_IN_MEMMOVE:
    7893           42 :       case BUILT_IN_MEMSET:
    7894           42 :         save_decl_rtl = DECL_RTL (fndecl);
    7895           42 :         DECL_RTL (fndecl) = asan_memfn_rtl (fndecl);
    7896           42 :         ret = expand_call (exp, target, ignore);
    7897           42 :         DECL_RTL (fndecl) = save_decl_rtl;
    7898           42 :         return ret;
    7899              :       default:
    7900              :         break;
    7901              :       }
    7902      1857577 :   if (sanitize_flags_p (SANITIZE_ADDRESS | SANITIZE_HWADDRESS)
    7903      1857577 :                   && asan_intercepted_p (fcode))
    7904          971 :     return expand_call (exp, target, ignore);
    7905              : 
    7906              :   /* When not optimizing, generate calls to library functions for a certain
    7907              :      set of builtins.  */
    7908      1856606 :   if (!optimize
    7909       436490 :       && !called_as_built_in (fndecl)
    7910       145910 :       && fcode != BUILT_IN_FORK
    7911       145910 :       && fcode != BUILT_IN_EXECL
    7912       145896 :       && fcode != BUILT_IN_EXECV
    7913       145896 :       && fcode != BUILT_IN_EXECLP
    7914       145894 :       && fcode != BUILT_IN_EXECLE
    7915       145894 :       && fcode != BUILT_IN_EXECVP
    7916       145893 :       && fcode != BUILT_IN_EXECVE
    7917       145893 :       && fcode != BUILT_IN_CLEAR_CACHE
    7918       145890 :       && !ALLOCA_FUNCTION_CODE_P (fcode)
    7919       145836 :       && fcode != BUILT_IN_FREE
    7920       143691 :       && (fcode != BUILT_IN_MEMSET
    7921        55895 :           || !(flag_inline_stringops & ILSOP_MEMSET))
    7922       143688 :       && (fcode != BUILT_IN_MEMCPY
    7923          918 :           || !(flag_inline_stringops & ILSOP_MEMCPY))
    7924       143552 :       && (fcode != BUILT_IN_MEMMOVE
    7925          348 :           || !(flag_inline_stringops & ILSOP_MEMMOVE))
    7926      2000158 :       && (fcode != BUILT_IN_MEMCMP
    7927         2529 :           || !(flag_inline_stringops & ILSOP_MEMCMP)))
    7928       143504 :     return expand_call (exp, target, ignore);
    7929              : 
    7930              :   /* The built-in function expanders test for target == const0_rtx
    7931              :      to determine whether the function's result will be ignored.  */
    7932      1713102 :   if (ignore)
    7933      1015885 :     target = const0_rtx;
    7934              : 
    7935              :   /* If the result of a pure or const built-in function is ignored, and
    7936              :      none of its arguments are volatile, we can avoid expanding the
    7937              :      built-in call and just evaluate the arguments for side-effects.  */
    7938      1713102 :   if (target == const0_rtx
    7939      1015885 :       && ((flags = flags_from_decl_or_type (fndecl)) & (ECF_CONST | ECF_PURE))
    7940      1716873 :       && !(flags & ECF_LOOPING_CONST_OR_PURE))
    7941              :     {
    7942            6 :       bool volatilep = false;
    7943            6 :       tree arg;
    7944            6 :       call_expr_arg_iterator iter;
    7945              : 
    7946           20 :       FOR_EACH_CALL_EXPR_ARG (arg, iter, exp)
    7947            8 :         if (TREE_THIS_VOLATILE (arg))
    7948              :           {
    7949              :             volatilep = true;
    7950              :             break;
    7951              :           }
    7952              : 
    7953            6 :       if (! volatilep)
    7954              :         {
    7955           20 :           FOR_EACH_CALL_EXPR_ARG (arg, iter, exp)
    7956            8 :             expand_expr (arg, const0_rtx, VOIDmode, EXPAND_NORMAL);
    7957            6 :           return const0_rtx;
    7958              :         }
    7959              :     }
    7960              : 
    7961      1713096 :   switch (fcode)
    7962              :     {
    7963            4 :     CASE_FLT_FN (BUILT_IN_FABS):
    7964            4 :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_FABS):
    7965            4 :     case BUILT_IN_FABSD32:
    7966            4 :     case BUILT_IN_FABSD64:
    7967            4 :     case BUILT_IN_FABSD128:
    7968            4 :     case BUILT_IN_FABSD64X:
    7969            4 :       target = expand_builtin_fabs (exp, target, subtarget);
    7970            4 :       if (target)
    7971       848478 :         return target;
    7972       864618 :       break;
    7973              : 
    7974        11677 :     CASE_FLT_FN (BUILT_IN_COPYSIGN):
    7975        11677 :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_COPYSIGN):
    7976        11677 :       target = expand_builtin_copysign (exp, target, subtarget);
    7977        11677 :       if (target)
    7978              :         return target;
    7979              :       break;
    7980              : 
    7981              :       /* Just do a normal library call if we were unable to fold
    7982              :          the values.  */
    7983              :     CASE_FLT_FN (BUILT_IN_CABS):
    7984              :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_CABS):
    7985              :       break;
    7986              : 
    7987          372 :     CASE_FLT_FN (BUILT_IN_FMA):
    7988          372 :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMA):
    7989          372 :       target = expand_builtin_mathfn_ternary (exp, target, subtarget);
    7990          372 :       if (target)
    7991              :         return target;
    7992              :       break;
    7993              : 
    7994          232 :     CASE_FLT_FN (BUILT_IN_ILOGB):
    7995          232 :       if (! flag_unsafe_math_optimizations)
    7996              :         break;
    7997            4 :       gcc_fallthrough ();
    7998            4 :     CASE_FLT_FN (BUILT_IN_ISINF):
    7999            4 :     CASE_FLT_FN (BUILT_IN_FINITE):
    8000            4 :     CASE_FLT_FN (BUILT_IN_ISNAN):
    8001            4 :     case BUILT_IN_ISFINITE:
    8002            4 :     case BUILT_IN_ISNORMAL:
    8003            4 :       target = expand_builtin_interclass_mathfn (exp, target);
    8004            4 :       if (target)
    8005              :         return target;
    8006              :       break;
    8007              : 
    8008          784 :     case BUILT_IN_ISSIGNALING:
    8009          784 :       target = expand_builtin_issignaling (exp, target);
    8010          784 :       if (target)
    8011              :         return target;
    8012              :       break;
    8013              : 
    8014          223 :     CASE_FLT_FN (BUILT_IN_ICEIL):
    8015          223 :     CASE_FLT_FN (BUILT_IN_LCEIL):
    8016          223 :     CASE_FLT_FN (BUILT_IN_LLCEIL):
    8017          223 :     CASE_FLT_FN (BUILT_IN_LFLOOR):
    8018          223 :     CASE_FLT_FN (BUILT_IN_IFLOOR):
    8019          223 :     CASE_FLT_FN (BUILT_IN_LLFLOOR):
    8020          223 :       target = expand_builtin_int_roundingfn (exp, target);
    8021          223 :       if (target)
    8022              :         return target;
    8023              :       break;
    8024              : 
    8025          581 :     CASE_FLT_FN (BUILT_IN_IRINT):
    8026          581 :     CASE_FLT_FN (BUILT_IN_LRINT):
    8027          581 :     CASE_FLT_FN (BUILT_IN_LLRINT):
    8028          581 :     CASE_FLT_FN (BUILT_IN_IROUND):
    8029          581 :     CASE_FLT_FN (BUILT_IN_LROUND):
    8030          581 :     CASE_FLT_FN (BUILT_IN_LLROUND):
    8031          581 :       target = expand_builtin_int_roundingfn_2 (exp, target);
    8032          581 :       if (target)
    8033              :         return target;
    8034              :       break;
    8035              : 
    8036          293 :     CASE_FLT_FN (BUILT_IN_POWI):
    8037          293 :       target = expand_builtin_powi (exp, target);
    8038          293 :       if (target)
    8039              :         return target;
    8040              :       break;
    8041              : 
    8042          173 :     CASE_FLT_FN (BUILT_IN_CEXPI):
    8043          173 :       target = expand_builtin_cexpi (exp, target);
    8044          173 :       gcc_assert (target);
    8045              :       return target;
    8046              : 
    8047         1010 :     CASE_FLT_FN (BUILT_IN_SIN):
    8048         1010 :     CASE_FLT_FN (BUILT_IN_COS):
    8049         1010 :       if (! flag_unsafe_math_optimizations)
    8050              :         break;
    8051           47 :       target = expand_builtin_mathfn_3 (exp, target, subtarget);
    8052           47 :       if (target)
    8053              :         return target;
    8054              :       break;
    8055              : 
    8056          167 :     CASE_FLT_FN (BUILT_IN_SINCOS):
    8057          167 :       if (! flag_unsafe_math_optimizations)
    8058              :         break;
    8059            3 :       target = expand_builtin_sincos (exp);
    8060            3 :       if (target)
    8061              :         return target;
    8062              :       break;
    8063              : 
    8064           60 :     case BUILT_IN_FEGETROUND:
    8065           60 :       target = expand_builtin_fegetround (exp, target, target_mode);
    8066           60 :       if (target)
    8067              :         return target;
    8068              :       break;
    8069              : 
    8070         1221 :     case BUILT_IN_FECLEAREXCEPT:
    8071         1221 :       target = expand_builtin_feclear_feraise_except (exp, target, target_mode,
    8072              :                                                       feclearexcept_optab);
    8073         1221 :       if (target)
    8074              :         return target;
    8075              :       break;
    8076              : 
    8077          831 :     case BUILT_IN_FERAISEEXCEPT:
    8078          831 :       target = expand_builtin_feclear_feraise_except (exp, target, target_mode,
    8079              :                                                       feraiseexcept_optab);
    8080          831 :       if (target)
    8081              :         return target;
    8082              :       break;
    8083              : 
    8084          464 :     case BUILT_IN_APPLY_ARGS:
    8085          464 :       return expand_builtin_apply_args ();
    8086              : 
    8087              :       /* __builtin_apply (FUNCTION, ARGUMENTS, ARGSIZE) invokes
    8088              :          FUNCTION with a copy of the parameters described by
    8089              :          ARGUMENTS, and ARGSIZE.  It returns a block of memory
    8090              :          allocated on the stack into which is stored all the registers
    8091              :          that might possibly be used for returning the result of a
    8092              :          function.  ARGUMENTS is the value returned by
    8093              :          __builtin_apply_args.  ARGSIZE is the number of bytes of
    8094              :          arguments that must be copied.  ??? How should this value be
    8095              :          computed?  We'll also need a safe worst case value for varargs
    8096              :          functions.  */
    8097          483 :     case BUILT_IN_APPLY:
    8098          483 :       if (!validate_arglist (exp, POINTER_TYPE,
    8099              :                              POINTER_TYPE, INTEGER_TYPE, VOID_TYPE)
    8100          483 :           && !validate_arglist (exp, REFERENCE_TYPE,
    8101              :                                 POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    8102            0 :         return const0_rtx;
    8103              :       else
    8104              :         {
    8105          483 :           rtx ops[3];
    8106              : 
    8107          483 :           ops[0] = expand_normal (CALL_EXPR_ARG (exp, 0));
    8108          483 :           ops[1] = expand_normal (CALL_EXPR_ARG (exp, 1));
    8109          483 :           ops[2] = expand_normal (CALL_EXPR_ARG (exp, 2));
    8110              : 
    8111          483 :           return expand_builtin_apply (ops[0], ops[1], ops[2]);
    8112              :         }
    8113              : 
    8114              :       /* __builtin_return (RESULT) causes the function to return the
    8115              :          value described by RESULT.  RESULT is address of the block of
    8116              :          memory returned by __builtin_apply.  */
    8117          379 :     case BUILT_IN_RETURN:
    8118          379 :       if (validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    8119          379 :         expand_builtin_return (expand_normal (CALL_EXPR_ARG (exp, 0)));
    8120          379 :       return const0_rtx;
    8121              : 
    8122            0 :     case BUILT_IN_CALL_CODE_ADDRESS:
    8123              :       /* All valid uses of __builtin_call_code_address () are removed in
    8124              :          in tree-nested.cc or gimple-fold.cc.  */
    8125            0 :       error ("argument to %<__builtin_call_code_address%> must be a function");
    8126            0 :       return const0_rtx;
    8127              : 
    8128            0 :     case BUILT_IN_CALL_STATIC_CHAIN:
    8129              :        /* All valid uses of __builtin_call_static_chain () are removed in
    8130              :           in tree-nested.cc or gimple-fold.cc.  */
    8131            0 :       error ("argument to %<__builtin_call_static_chain%> must be a function");
    8132            0 :       return const0_rtx;
    8133              : 
    8134            0 :     case BUILT_IN_SAVEREGS:
    8135            0 :       return expand_builtin_saveregs ();
    8136              : 
    8137           16 :     case BUILT_IN_VA_ARG_PACK:
    8138              :       /* All valid uses of __builtin_va_arg_pack () are removed during
    8139              :          inlining.  */
    8140           16 :       error ("invalid use of %<__builtin_va_arg_pack ()%>");
    8141           16 :       return const0_rtx;
    8142              : 
    8143            0 :     case BUILT_IN_VA_ARG_PACK_LEN:
    8144              :       /* All valid uses of __builtin_va_arg_pack_len () are removed during
    8145              :          inlining.  */
    8146            0 :       error ("invalid use of %<__builtin_va_arg_pack_len ()%>");
    8147            0 :       return const0_rtx;
    8148              : 
    8149              :       /* Return the address of the first anonymous stack arg.  */
    8150          158 :     case BUILT_IN_NEXT_ARG:
    8151          158 :       if (fold_builtin_next_arg (exp, false))
    8152            0 :         return const0_rtx;
    8153          158 :       return expand_builtin_next_arg ();
    8154              : 
    8155           28 :     case BUILT_IN_CLEAR_CACHE:
    8156           28 :       expand_builtin___clear_cache (exp);
    8157           28 :       return const0_rtx;
    8158              : 
    8159            0 :     case BUILT_IN_CLASSIFY_TYPE:
    8160            0 :       return expand_builtin_classify_type (exp);
    8161              : 
    8162            0 :     case BUILT_IN_CONSTANT_P:
    8163            0 :       return const0_rtx;
    8164              : 
    8165        15936 :     case BUILT_IN_FRAME_ADDRESS:
    8166        15936 :     case BUILT_IN_RETURN_ADDRESS:
    8167        15936 :       return expand_builtin_frame_address (fndecl, exp);
    8168              : 
    8169          540 :     case BUILT_IN_STACK_ADDRESS:
    8170          540 :       return expand_builtin_stack_address ();
    8171              : 
    8172         2170 :     case BUILT_IN___STRUB_ENTER:
    8173         2170 :       target = expand_builtin_strub_enter (exp);
    8174         2170 :       if (target)
    8175              :         return target;
    8176              :       break;
    8177              : 
    8178         1085 :     case BUILT_IN___STRUB_UPDATE:
    8179         1085 :       target = expand_builtin_strub_update (exp);
    8180         1085 :       if (target)
    8181              :         return target;
    8182              :       break;
    8183              : 
    8184         2740 :     case BUILT_IN___STRUB_LEAVE:
    8185         2740 :       target = expand_builtin_strub_leave (exp);
    8186         2740 :       if (target)
    8187              :         return target;
    8188              :       break;
    8189              : 
    8190              :     /* Returns the address of the area where the structure is returned.
    8191              :        0 otherwise.  */
    8192            0 :     case BUILT_IN_AGGREGATE_INCOMING_ADDRESS:
    8193            0 :       if (call_expr_nargs (exp) != 0
    8194            0 :           || ! AGGREGATE_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl)))
    8195            0 :           || !MEM_P (DECL_RTL (DECL_RESULT (current_function_decl))))
    8196            0 :         return const0_rtx;
    8197              :       else
    8198            0 :         return XEXP (DECL_RTL (DECL_RESULT (current_function_decl)), 0);
    8199              : 
    8200        27680 :     CASE_BUILT_IN_ALLOCA:
    8201        27680 :       target = expand_builtin_alloca (exp);
    8202        27680 :       if (target)
    8203              :         return target;
    8204              :       break;
    8205              : 
    8206          204 :     case BUILT_IN_ASAN_ALLOCAS_UNPOISON:
    8207          204 :       return expand_asan_emit_allocas_unpoison (exp);
    8208              : 
    8209         1863 :     case BUILT_IN_STACK_SAVE:
    8210         1863 :       return expand_stack_save ();
    8211              : 
    8212         1762 :     case BUILT_IN_STACK_RESTORE:
    8213         1762 :       expand_stack_restore (CALL_EXPR_ARG (exp, 0));
    8214         1762 :       return const0_rtx;
    8215              : 
    8216         1276 :     case BUILT_IN_BSWAP16:
    8217         1276 :     case BUILT_IN_BSWAP32:
    8218         1276 :     case BUILT_IN_BSWAP64:
    8219         1276 :     case BUILT_IN_BSWAP128:
    8220         1276 :       target = expand_builtin_bswap (target_mode, exp, target, subtarget);
    8221         1276 :       if (target)
    8222              :         return target;
    8223              :       break;
    8224              : 
    8225          122 :     case BUILT_IN_BITREVERSE8:
    8226          122 :     case BUILT_IN_BITREVERSE16:
    8227          122 :     case BUILT_IN_BITREVERSE32:
    8228          122 :     case BUILT_IN_BITREVERSE64:
    8229          122 :     case BUILT_IN_BITREVERSE128:
    8230          122 :       target = expand_builtin_unop (target_mode, exp, target, subtarget,
    8231              :                                     bitreverse_optab);
    8232          122 :       if (target)
    8233              :         return target;
    8234              :       break;
    8235              : 
    8236            0 :     CASE_INT_FN (BUILT_IN_FFS):
    8237            0 :       target = expand_builtin_unop (target_mode, exp, target,
    8238              :                                     subtarget, ffs_optab);
    8239            0 :       if (target)
    8240              :         return target;
    8241              :       break;
    8242              : 
    8243          206 :     CASE_INT_FN (BUILT_IN_CLZ):
    8244          206 :       target = expand_builtin_unop (target_mode, exp, target,
    8245              :                                     subtarget, clz_optab);
    8246          206 :       if (target)
    8247              :         return target;
    8248              :       break;
    8249              : 
    8250           48 :     CASE_INT_FN (BUILT_IN_CTZ):
    8251           48 :       target = expand_builtin_unop (target_mode, exp, target,
    8252              :                                     subtarget, ctz_optab);
    8253           48 :       if (target)
    8254              :         return target;
    8255              :       break;
    8256              : 
    8257           91 :     CASE_INT_FN (BUILT_IN_CLRSB):
    8258           91 :       target = expand_builtin_unop (target_mode, exp, target,
    8259              :                                     subtarget, clrsb_optab);
    8260           91 :       if (target)
    8261              :         return target;
    8262              :       break;
    8263              : 
    8264          491 :     CASE_INT_FN (BUILT_IN_POPCOUNT):
    8265          491 :       target = expand_builtin_unop (target_mode, exp, target,
    8266              :                                     subtarget, popcount_optab);
    8267          491 :       if (target)
    8268              :         return target;
    8269              :       break;
    8270              : 
    8271           11 :     CASE_INT_FN (BUILT_IN_PARITY):
    8272           11 :       target = expand_builtin_unop (target_mode, exp, target,
    8273              :                                     subtarget, parity_optab);
    8274           11 :       if (target)
    8275              :         return target;
    8276              :       break;
    8277              : 
    8278        13808 :     case BUILT_IN_STRLEN:
    8279        13808 :       target = expand_builtin_strlen (exp, target, target_mode);
    8280        13808 :       if (target)
    8281              :         return target;
    8282              :       break;
    8283              : 
    8284          580 :     case BUILT_IN_STRNLEN:
    8285          580 :       target = expand_builtin_strnlen (exp, target, target_mode);
    8286          580 :       if (target)
    8287              :         return target;
    8288              :       break;
    8289              : 
    8290         1860 :     case BUILT_IN_STRCPY:
    8291         1860 :       target = expand_builtin_strcpy (exp, target);
    8292         1860 :       if (target)
    8293              :         return target;
    8294              :       break;
    8295              : 
    8296         2177 :     case BUILT_IN_STRNCPY:
    8297         2177 :       target = expand_builtin_strncpy (exp, target);
    8298         2177 :       if (target)
    8299              :         return target;
    8300              :       break;
    8301              : 
    8302          454 :     case BUILT_IN_STPCPY:
    8303          454 :       target = expand_builtin_stpcpy (exp, target, mode);
    8304          454 :       if (target)
    8305              :         return target;
    8306              :       break;
    8307              : 
    8308       106053 :     case BUILT_IN_MEMCPY:
    8309       106053 :       target = expand_builtin_memcpy (exp, target);
    8310       106053 :       if (target)
    8311              :         return target;
    8312              :       break;
    8313              : 
    8314        17405 :     case BUILT_IN_MEMMOVE:
    8315        17405 :       target = expand_builtin_memmove (exp, target);
    8316        17405 :       if (target)
    8317              :         return target;
    8318              :       break;
    8319              : 
    8320         1395 :     case BUILT_IN_MEMPCPY:
    8321         1395 :       target = expand_builtin_mempcpy (exp, target);
    8322         1395 :       if (target)
    8323              :         return target;
    8324              :       break;
    8325              : 
    8326        37065 :     case BUILT_IN_MEMSET:
    8327        37065 :       target = expand_builtin_memset (exp, target, mode);
    8328        37065 :       if (target)
    8329              :         return target;
    8330              :       break;
    8331              : 
    8332            0 :     case BUILT_IN_BZERO:
    8333            0 :       target = expand_builtin_bzero (exp);
    8334            0 :       if (target)
    8335              :         return target;
    8336              :       break;
    8337              : 
    8338              :     /* Expand it as BUILT_IN_MEMCMP_EQ first. If not successful, change it
    8339              :        back to a BUILT_IN_STRCMP. Remember to delete the 3rd parameter
    8340              :        when changing it to a strcmp call.  */
    8341          371 :     case BUILT_IN_STRCMP_EQ:
    8342          371 :       target = expand_builtin_memcmp (exp, target, true);
    8343          371 :       if (target)
    8344              :         return target;
    8345              : 
    8346              :       /* Change this call back to a BUILT_IN_STRCMP.  */
    8347           25 :       TREE_OPERAND (exp, 1)
    8348           25 :         = build_fold_addr_expr (builtin_decl_explicit (BUILT_IN_STRCMP));
    8349              : 
    8350              :       /* Delete the last parameter.  */
    8351           25 :       unsigned int i;
    8352           25 :       vec<tree, va_gc> *arg_vec;
    8353           25 :       vec_alloc (arg_vec, 2);
    8354          100 :       for (i = 0; i < 2; i++)
    8355           50 :         arg_vec->quick_push (CALL_EXPR_ARG (exp, i));
    8356           25 :       exp = build_call_vec (TREE_TYPE (exp), CALL_EXPR_FN (exp), arg_vec);
    8357              :       /* FALLTHROUGH */
    8358              : 
    8359       127947 :     case BUILT_IN_STRCMP:
    8360       127947 :       target = expand_builtin_strcmp (exp, target);
    8361       127947 :       if (target)
    8362              :         return target;
    8363              :       break;
    8364              : 
    8365              :     /* Expand it as BUILT_IN_MEMCMP_EQ first. If not successful, change it
    8366              :        back to a BUILT_IN_STRNCMP.  */
    8367           38 :     case BUILT_IN_STRNCMP_EQ:
    8368           38 :       target = expand_builtin_memcmp (exp, target, true);
    8369           38 :       if (target)
    8370              :         return target;
    8371              : 
    8372              :       /* Change it back to a BUILT_IN_STRNCMP.  */
    8373            0 :       TREE_OPERAND (exp, 1)
    8374            0 :         = build_fold_addr_expr (builtin_decl_explicit (BUILT_IN_STRNCMP));
    8375              :       /* FALLTHROUGH */
    8376              : 
    8377         2027 :     case BUILT_IN_STRNCMP:
    8378         2027 :       target = expand_builtin_strncmp (exp, target, mode);
    8379         2027 :       if (target)
    8380              :         return target;
    8381              :       break;
    8382              : 
    8383       105265 :     case BUILT_IN_BCMP:
    8384       105265 :     case BUILT_IN_MEMCMP:
    8385       105265 :     case BUILT_IN_MEMCMP_EQ:
    8386       105265 :       target = expand_builtin_memcmp (exp, target, fcode == BUILT_IN_MEMCMP_EQ);
    8387       105265 :       if (target)
    8388              :         return target;
    8389        37541 :       if (fcode == BUILT_IN_MEMCMP_EQ)
    8390              :         {
    8391        22654 :           tree newdecl = builtin_decl_explicit (BUILT_IN_MEMCMP);
    8392        22654 :           TREE_OPERAND (exp, 1) = build_fold_addr_expr (newdecl);
    8393              :         }
    8394              :       break;
    8395              : 
    8396            0 :     case BUILT_IN_SETJMP:
    8397              :       /* This should have been lowered to the builtins below.  */
    8398            0 :       gcc_unreachable ();
    8399              : 
    8400          842 :     case BUILT_IN_SETJMP_SETUP:
    8401              :       /* __builtin_setjmp_setup is passed a pointer to an array of five words
    8402              :           and the receiver label.  */
    8403          842 :       if (validate_arglist (exp, POINTER_TYPE, POINTER_TYPE, VOID_TYPE))
    8404              :         {
    8405          842 :           rtx buf_addr = expand_expr (CALL_EXPR_ARG (exp, 0), subtarget,
    8406              :                                       VOIDmode, EXPAND_NORMAL);
    8407          842 :           tree label = TREE_OPERAND (CALL_EXPR_ARG (exp, 1), 0);
    8408          842 :           rtx_insn *label_r = label_rtx (label);
    8409              : 
    8410          842 :           expand_builtin_setjmp_setup (buf_addr, label_r);
    8411          842 :           return const0_rtx;
    8412              :         }
    8413              :       break;
    8414              : 
    8415          842 :     case BUILT_IN_SETJMP_RECEIVER:
    8416              :        /* __builtin_setjmp_receiver is passed the receiver label.  */
    8417          842 :       if (validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    8418              :         {
    8419          842 :           tree label = TREE_OPERAND (CALL_EXPR_ARG (exp, 0), 0);
    8420          842 :           rtx_insn *label_r = label_rtx (label);
    8421              : 
    8422          842 :           expand_builtin_setjmp_receiver (label_r);
    8423          842 :           nonlocal_goto_handler_labels
    8424         1684 :             = gen_rtx_INSN_LIST (VOIDmode, label_r,
    8425          842 :                                  nonlocal_goto_handler_labels);
    8426              :           /* ??? Do not let expand_label treat us as such since we would
    8427              :              not want to be both on the list of non-local labels and on
    8428              :              the list of forced labels.  */
    8429          842 :           FORCED_LABEL (label) = 0;
    8430          842 :           return const0_rtx;
    8431              :         }
    8432              :       break;
    8433              : 
    8434              :       /* __builtin_longjmp is passed a pointer to an array of five words.
    8435              :          It's similar to the C library longjmp function but works with
    8436              :          __builtin_setjmp above.  */
    8437          392 :     case BUILT_IN_LONGJMP:
    8438          392 :       if (validate_arglist (exp, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
    8439              :         {
    8440          392 :           rtx buf_addr = expand_expr (CALL_EXPR_ARG (exp, 0), subtarget,
    8441              :                                       VOIDmode, EXPAND_NORMAL);
    8442          392 :           rtx value = expand_normal (CALL_EXPR_ARG (exp, 1));
    8443              : 
    8444          392 :           if (value != const1_rtx)
    8445              :             {
    8446            0 :               error ("%<__builtin_longjmp%> second argument must be 1");
    8447            0 :               return const0_rtx;
    8448              :             }
    8449              : 
    8450          392 :           expand_builtin_longjmp (buf_addr, value);
    8451          392 :           return const0_rtx;
    8452              :         }
    8453              :       break;
    8454              : 
    8455          524 :     case BUILT_IN_NONLOCAL_GOTO:
    8456          524 :       target = expand_builtin_nonlocal_goto (exp);
    8457          524 :       if (target)
    8458              :         return target;
    8459              :       break;
    8460              : 
    8461              :       /* This updates the setjmp buffer that is its argument with the value
    8462              :          of the current stack pointer.  */
    8463            0 :     case BUILT_IN_UPDATE_SETJMP_BUF:
    8464            0 :       if (validate_arglist (exp, POINTER_TYPE, VOID_TYPE))
    8465              :         {
    8466            0 :           rtx buf_addr
    8467            0 :             = expand_normal (CALL_EXPR_ARG (exp, 0));
    8468              : 
    8469            0 :           expand_builtin_update_setjmp_buf (buf_addr);
    8470            0 :           return const0_rtx;
    8471              :         }
    8472              :       break;
    8473              : 
    8474        38824 :     case BUILT_IN_TRAP:
    8475        38824 :     case BUILT_IN_UNREACHABLE_TRAP:
    8476        38824 :       expand_builtin_trap ();
    8477        38824 :       return const0_rtx;
    8478              : 
    8479         2668 :     case BUILT_IN_UNREACHABLE:
    8480         2668 :       expand_builtin_unreachable ();
    8481         2668 :       return const0_rtx;
    8482              : 
    8483          417 :     case BUILT_IN_OBSERVABLE_CHKPT:
    8484              :       /* Generate no code.  */
    8485          417 :       return const0_rtx;
    8486              : 
    8487         1385 :     CASE_FLT_FN (BUILT_IN_SIGNBIT):
    8488         1385 :     case BUILT_IN_SIGNBITD32:
    8489         1385 :     case BUILT_IN_SIGNBITD64:
    8490         1385 :     case BUILT_IN_SIGNBITD128:
    8491         1385 :       target = expand_builtin_signbit (exp, target);
    8492         1385 :       if (target)
    8493              :         return target;
    8494              :       break;
    8495              : 
    8496              :       /* Various hooks for the DWARF 2 __throw routine.  */
    8497           34 :     case BUILT_IN_UNWIND_INIT:
    8498           34 :       expand_builtin_unwind_init ();
    8499           34 :       return const0_rtx;
    8500         1237 :     case BUILT_IN_DWARF_CFA:
    8501         1237 :       return virtual_cfa_rtx;
    8502              : #ifdef DWARF2_UNWIND_INFO
    8503           48 :     case BUILT_IN_DWARF_SP_COLUMN:
    8504           48 :       return expand_builtin_dwarf_sp_column ();
    8505            8 :     case BUILT_IN_INIT_DWARF_REG_SIZES:
    8506            8 :       expand_builtin_init_dwarf_reg_sizes (CALL_EXPR_ARG (exp, 0));
    8507            8 :       return const0_rtx;
    8508              : #endif
    8509           19 :     case BUILT_IN_FROB_RETURN_ADDR:
    8510           19 :       return expand_builtin_frob_return_addr (CALL_EXPR_ARG (exp, 0));
    8511         2352 :     case BUILT_IN_EXTRACT_RETURN_ADDR:
    8512         2352 :       return expand_builtin_extract_return_addr (CALL_EXPR_ARG (exp, 0));
    8513           31 :     case BUILT_IN_EH_RETURN:
    8514           62 :       expand_builtin_eh_return (CALL_EXPR_ARG (exp, 0),
    8515           31 :                                 CALL_EXPR_ARG (exp, 1));
    8516           31 :       return const0_rtx;
    8517           21 :     case BUILT_IN_EH_RETURN_DATA_REGNO:
    8518           21 :       return expand_builtin_eh_return_data_regno (exp);
    8519            2 :     case BUILT_IN_EXTEND_POINTER:
    8520            2 :       return expand_builtin_extend_pointer (CALL_EXPR_ARG (exp, 0));
    8521       103560 :     case BUILT_IN_EH_POINTER:
    8522       103560 :       return expand_builtin_eh_pointer (exp);
    8523         7141 :     case BUILT_IN_EH_FILTER:
    8524         7141 :       return expand_builtin_eh_filter (exp);
    8525       103875 :     case BUILT_IN_EH_COPY_VALUES:
    8526       103875 :       return expand_builtin_eh_copy_values (exp);
    8527              : 
    8528        21085 :     case BUILT_IN_VA_START:
    8529        21085 :       return expand_builtin_va_start (exp);
    8530        12168 :     case BUILT_IN_VA_END:
    8531        12168 :       return expand_builtin_va_end (exp);
    8532          240 :     case BUILT_IN_VA_COPY:
    8533          240 :       return expand_builtin_va_copy (exp);
    8534         1407 :     case BUILT_IN_EXPECT:
    8535         1407 :       return expand_builtin_expect (exp, target);
    8536            5 :     case BUILT_IN_EXPECT_WITH_PROBABILITY:
    8537            5 :       return expand_builtin_expect_with_probability (exp, target);
    8538          662 :     case BUILT_IN_ASSUME_ALIGNED:
    8539          662 :       return expand_builtin_assume_aligned (exp, target);
    8540         2061 :     case BUILT_IN_PREFETCH:
    8541         2061 :       expand_builtin_prefetch (exp);
    8542         2061 :       return const0_rtx;
    8543              : 
    8544          308 :     case BUILT_IN_INIT_TRAMPOLINE:
    8545          308 :       return expand_builtin_init_trampoline (exp, true);
    8546            0 :     case BUILT_IN_INIT_HEAP_TRAMPOLINE:
    8547            0 :       return expand_builtin_init_trampoline (exp, false);
    8548          355 :     case BUILT_IN_ADJUST_TRAMPOLINE:
    8549          355 :       return expand_builtin_adjust_trampoline (exp);
    8550              : 
    8551            0 :     case BUILT_IN_INIT_DESCRIPTOR:
    8552            0 :       return expand_builtin_init_descriptor (exp);
    8553            0 :     case BUILT_IN_ADJUST_DESCRIPTOR:
    8554            0 :       return expand_builtin_adjust_descriptor (exp);
    8555              : 
    8556              :     case BUILT_IN_GCC_NESTED_PTR_CREATED:
    8557              :     case BUILT_IN_GCC_NESTED_PTR_DELETED:
    8558              :       break; /* At present, no expansion, just call the function.  */
    8559              : 
    8560          112 :     case BUILT_IN_FORK:
    8561          112 :     case BUILT_IN_EXECL:
    8562          112 :     case BUILT_IN_EXECV:
    8563          112 :     case BUILT_IN_EXECLP:
    8564          112 :     case BUILT_IN_EXECLE:
    8565          112 :     case BUILT_IN_EXECVP:
    8566          112 :     case BUILT_IN_EXECVE:
    8567          112 :       target = expand_builtin_fork_or_exec (fndecl, exp, target, ignore);
    8568          112 :       if (target)
    8569              :         return target;
    8570              :       break;
    8571              : 
    8572          712 :     case BUILT_IN_SYNC_FETCH_AND_ADD_1:
    8573          712 :     case BUILT_IN_SYNC_FETCH_AND_ADD_2:
    8574          712 :     case BUILT_IN_SYNC_FETCH_AND_ADD_4:
    8575          712 :     case BUILT_IN_SYNC_FETCH_AND_ADD_8:
    8576          712 :     case BUILT_IN_SYNC_FETCH_AND_ADD_16:
    8577          712 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_ADD_1);
    8578          712 :       target = expand_builtin_sync_operation (mode, exp, PLUS, false, target);
    8579          712 :       if (target)
    8580              :         return target;
    8581              :       break;
    8582              : 
    8583          585 :     case BUILT_IN_SYNC_FETCH_AND_SUB_1:
    8584          585 :     case BUILT_IN_SYNC_FETCH_AND_SUB_2:
    8585          585 :     case BUILT_IN_SYNC_FETCH_AND_SUB_4:
    8586          585 :     case BUILT_IN_SYNC_FETCH_AND_SUB_8:
    8587          585 :     case BUILT_IN_SYNC_FETCH_AND_SUB_16:
    8588          585 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_SUB_1);
    8589          585 :       target = expand_builtin_sync_operation (mode, exp, MINUS, false, target);
    8590          585 :       if (target)
    8591              :         return target;
    8592              :       break;
    8593              : 
    8594          512 :     case BUILT_IN_SYNC_FETCH_AND_OR_1:
    8595          512 :     case BUILT_IN_SYNC_FETCH_AND_OR_2:
    8596          512 :     case BUILT_IN_SYNC_FETCH_AND_OR_4:
    8597          512 :     case BUILT_IN_SYNC_FETCH_AND_OR_8:
    8598          512 :     case BUILT_IN_SYNC_FETCH_AND_OR_16:
    8599          512 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_OR_1);
    8600          512 :       target = expand_builtin_sync_operation (mode, exp, IOR, false, target);
    8601          512 :       if (target)
    8602              :         return target;
    8603              :       break;
    8604              : 
    8605          488 :     case BUILT_IN_SYNC_FETCH_AND_AND_1:
    8606          488 :     case BUILT_IN_SYNC_FETCH_AND_AND_2:
    8607          488 :     case BUILT_IN_SYNC_FETCH_AND_AND_4:
    8608          488 :     case BUILT_IN_SYNC_FETCH_AND_AND_8:
    8609          488 :     case BUILT_IN_SYNC_FETCH_AND_AND_16:
    8610          488 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_AND_1);
    8611          488 :       target = expand_builtin_sync_operation (mode, exp, AND, false, target);
    8612          488 :       if (target)
    8613              :         return target;
    8614              :       break;
    8615              : 
    8616          581 :     case BUILT_IN_SYNC_FETCH_AND_XOR_1:
    8617          581 :     case BUILT_IN_SYNC_FETCH_AND_XOR_2:
    8618          581 :     case BUILT_IN_SYNC_FETCH_AND_XOR_4:
    8619          581 :     case BUILT_IN_SYNC_FETCH_AND_XOR_8:
    8620          581 :     case BUILT_IN_SYNC_FETCH_AND_XOR_16:
    8621          581 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_XOR_1);
    8622          581 :       target = expand_builtin_sync_operation (mode, exp, XOR, false, target);
    8623          581 :       if (target)
    8624              :         return target;
    8625              :       break;
    8626              : 
    8627          434 :     case BUILT_IN_SYNC_FETCH_AND_NAND_1:
    8628          434 :     case BUILT_IN_SYNC_FETCH_AND_NAND_2:
    8629          434 :     case BUILT_IN_SYNC_FETCH_AND_NAND_4:
    8630          434 :     case BUILT_IN_SYNC_FETCH_AND_NAND_8:
    8631          434 :     case BUILT_IN_SYNC_FETCH_AND_NAND_16:
    8632          434 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_FETCH_AND_NAND_1);
    8633          434 :       target = expand_builtin_sync_operation (mode, exp, NOT, false, target);
    8634          434 :       if (target)
    8635              :         return target;
    8636              :       break;
    8637              : 
    8638          161 :     case BUILT_IN_SYNC_ADD_AND_FETCH_1:
    8639          161 :     case BUILT_IN_SYNC_ADD_AND_FETCH_2:
    8640          161 :     case BUILT_IN_SYNC_ADD_AND_FETCH_4:
    8641          161 :     case BUILT_IN_SYNC_ADD_AND_FETCH_8:
    8642          161 :     case BUILT_IN_SYNC_ADD_AND_FETCH_16:
    8643          161 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_ADD_AND_FETCH_1);
    8644          161 :       target = expand_builtin_sync_operation (mode, exp, PLUS, true, target);
    8645          161 :       if (target)
    8646              :         return target;
    8647              :       break;
    8648              : 
    8649          121 :     case BUILT_IN_SYNC_SUB_AND_FETCH_1:
    8650          121 :     case BUILT_IN_SYNC_SUB_AND_FETCH_2:
    8651          121 :     case BUILT_IN_SYNC_SUB_AND_FETCH_4:
    8652          121 :     case BUILT_IN_SYNC_SUB_AND_FETCH_8:
    8653          121 :     case BUILT_IN_SYNC_SUB_AND_FETCH_16:
    8654          121 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_SUB_AND_FETCH_1);
    8655          121 :       target = expand_builtin_sync_operation (mode, exp, MINUS, true, target);
    8656          121 :       if (target)
    8657              :         return target;
    8658              :       break;
    8659              : 
    8660          115 :     case BUILT_IN_SYNC_OR_AND_FETCH_1:
    8661          115 :     case BUILT_IN_SYNC_OR_AND_FETCH_2:
    8662          115 :     case BUILT_IN_SYNC_OR_AND_FETCH_4:
    8663          115 :     case BUILT_IN_SYNC_OR_AND_FETCH_8:
    8664          115 :     case BUILT_IN_SYNC_OR_AND_FETCH_16:
    8665          115 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_OR_AND_FETCH_1);
    8666          115 :       target = expand_builtin_sync_operation (mode, exp, IOR, true, target);
    8667          115 :       if (target)
    8668              :         return target;
    8669              :       break;
    8670              : 
    8671          116 :     case BUILT_IN_SYNC_AND_AND_FETCH_1:
    8672          116 :     case BUILT_IN_SYNC_AND_AND_FETCH_2:
    8673          116 :     case BUILT_IN_SYNC_AND_AND_FETCH_4:
    8674          116 :     case BUILT_IN_SYNC_AND_AND_FETCH_8:
    8675          116 :     case BUILT_IN_SYNC_AND_AND_FETCH_16:
    8676          116 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_AND_AND_FETCH_1);
    8677          116 :       target = expand_builtin_sync_operation (mode, exp, AND, true, target);
    8678          116 :       if (target)
    8679              :         return target;
    8680              :       break;
    8681              : 
    8682          114 :     case BUILT_IN_SYNC_XOR_AND_FETCH_1:
    8683          114 :     case BUILT_IN_SYNC_XOR_AND_FETCH_2:
    8684          114 :     case BUILT_IN_SYNC_XOR_AND_FETCH_4:
    8685          114 :     case BUILT_IN_SYNC_XOR_AND_FETCH_8:
    8686          114 :     case BUILT_IN_SYNC_XOR_AND_FETCH_16:
    8687          114 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_XOR_AND_FETCH_1);
    8688          114 :       target = expand_builtin_sync_operation (mode, exp, XOR, true, target);
    8689          114 :       if (target)
    8690              :         return target;
    8691              :       break;
    8692              : 
    8693           89 :     case BUILT_IN_SYNC_NAND_AND_FETCH_1:
    8694           89 :     case BUILT_IN_SYNC_NAND_AND_FETCH_2:
    8695           89 :     case BUILT_IN_SYNC_NAND_AND_FETCH_4:
    8696           89 :     case BUILT_IN_SYNC_NAND_AND_FETCH_8:
    8697           89 :     case BUILT_IN_SYNC_NAND_AND_FETCH_16:
    8698           89 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_NAND_AND_FETCH_1);
    8699           89 :       target = expand_builtin_sync_operation (mode, exp, NOT, true, target);
    8700           89 :       if (target)
    8701              :         return target;
    8702              :       break;
    8703              : 
    8704          204 :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_1:
    8705          204 :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_2:
    8706          204 :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_4:
    8707          204 :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_8:
    8708          204 :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_16:
    8709          204 :       if (mode == VOIDmode)
    8710           40 :         mode = TYPE_MODE (boolean_type_node);
    8711          204 :       if (!target || !register_operand (target, mode))
    8712           40 :         target = gen_reg_rtx (mode);
    8713              : 
    8714          204 :       mode = get_builtin_sync_mode
    8715          204 :                                 (fcode - BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_1);
    8716          204 :       target = expand_builtin_compare_and_swap (mode, exp, true, target);
    8717          204 :       if (target)
    8718              :         return target;
    8719              :       break;
    8720              : 
    8721          256 :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_1:
    8722          256 :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_2:
    8723          256 :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_4:
    8724          256 :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_8:
    8725          256 :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_16:
    8726          256 :       mode = get_builtin_sync_mode
    8727          256 :                                 (fcode - BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_1);
    8728          256 :       target = expand_builtin_compare_and_swap (mode, exp, false, target);
    8729          256 :       if (target)
    8730              :         return target;
    8731              :       break;
    8732              : 
    8733          326 :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_1:
    8734          326 :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_2:
    8735          326 :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_4:
    8736          326 :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_8:
    8737          326 :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_16:
    8738          326 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_LOCK_TEST_AND_SET_1);
    8739          326 :       target = expand_builtin_sync_lock_test_and_set (mode, exp, target);
    8740          326 :       if (target)
    8741              :         return target;
    8742              :       break;
    8743              : 
    8744          146 :     case BUILT_IN_SYNC_LOCK_RELEASE_1:
    8745          146 :     case BUILT_IN_SYNC_LOCK_RELEASE_2:
    8746          146 :     case BUILT_IN_SYNC_LOCK_RELEASE_4:
    8747          146 :     case BUILT_IN_SYNC_LOCK_RELEASE_8:
    8748          146 :     case BUILT_IN_SYNC_LOCK_RELEASE_16:
    8749          146 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SYNC_LOCK_RELEASE_1);
    8750          146 :       if (expand_builtin_sync_lock_release (mode, exp))
    8751          144 :         return const0_rtx;
    8752              :       break;
    8753              : 
    8754          275 :     case BUILT_IN_SYNC_SYNCHRONIZE:
    8755          275 :       expand_builtin_sync_synchronize ();
    8756          275 :       return const0_rtx;
    8757              : 
    8758         2937 :     case BUILT_IN_ATOMIC_EXCHANGE_1:
    8759         2937 :     case BUILT_IN_ATOMIC_EXCHANGE_2:
    8760         2937 :     case BUILT_IN_ATOMIC_EXCHANGE_4:
    8761         2937 :     case BUILT_IN_ATOMIC_EXCHANGE_8:
    8762         2937 :     case BUILT_IN_ATOMIC_EXCHANGE_16:
    8763         2937 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_EXCHANGE_1);
    8764         2937 :       target = expand_builtin_atomic_exchange (mode, exp, target);
    8765         2937 :       if (target)
    8766              :         return target;
    8767              :       break;
    8768              : 
    8769         9398 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1:
    8770         9398 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_2:
    8771         9398 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4:
    8772         9398 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8:
    8773         9398 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_16:
    8774         9398 :       {
    8775         9398 :         unsigned int nargs, z;
    8776         9398 :         vec<tree, va_gc> *vec;
    8777              : 
    8778         9398 :         mode =
    8779         9398 :             get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1);
    8780         9398 :         target = expand_builtin_atomic_compare_exchange (mode, exp, target);
    8781         9398 :         if (target)
    8782         7598 :           return target;
    8783              : 
    8784              :         /* If this is turned into an external library call, the weak parameter
    8785              :            must be dropped to match the expected parameter list.  */
    8786         1800 :         nargs = call_expr_nargs (exp);
    8787         1800 :         vec_alloc (vec, nargs - 1);
    8788         9000 :         for (z = 0; z < 3; z++)
    8789         5400 :           vec->quick_push (CALL_EXPR_ARG (exp, z));
    8790              :         /* Skip the boolean weak parameter.  */
    8791         5400 :         for (z = 4; z < 6; z++)
    8792         3600 :           vec->quick_push (CALL_EXPR_ARG (exp, z));
    8793         1800 :         exp = build_call_vec (TREE_TYPE (exp), CALL_EXPR_FN (exp), vec);
    8794         1800 :         break;
    8795              :       }
    8796              : 
    8797        71474 :     case BUILT_IN_ATOMIC_LOAD_1:
    8798        71474 :     case BUILT_IN_ATOMIC_LOAD_2:
    8799        71474 :     case BUILT_IN_ATOMIC_LOAD_4:
    8800        71474 :     case BUILT_IN_ATOMIC_LOAD_8:
    8801        71474 :     case BUILT_IN_ATOMIC_LOAD_16:
    8802        71474 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_LOAD_1);
    8803        71474 :       target = expand_builtin_atomic_load (mode, exp, target);
    8804        71474 :       if (target)
    8805              :         return target;
    8806              :       break;
    8807              : 
    8808        17608 :     case BUILT_IN_ATOMIC_STORE_1:
    8809        17608 :     case BUILT_IN_ATOMIC_STORE_2:
    8810        17608 :     case BUILT_IN_ATOMIC_STORE_4:
    8811        17608 :     case BUILT_IN_ATOMIC_STORE_8:
    8812        17608 :     case BUILT_IN_ATOMIC_STORE_16:
    8813        17608 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_STORE_1);
    8814        17608 :       target = expand_builtin_atomic_store (mode, exp);
    8815        17608 :       if (target)
    8816        15976 :         return const0_rtx;
    8817              :       break;
    8818              : 
    8819         4408 :     case BUILT_IN_ATOMIC_ADD_FETCH_1:
    8820         4408 :     case BUILT_IN_ATOMIC_ADD_FETCH_2:
    8821         4408 :     case BUILT_IN_ATOMIC_ADD_FETCH_4:
    8822         4408 :     case BUILT_IN_ATOMIC_ADD_FETCH_8:
    8823         4408 :     case BUILT_IN_ATOMIC_ADD_FETCH_16:
    8824         4408 :       {
    8825         4408 :         enum built_in_function lib;
    8826         4408 :         mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_ADD_FETCH_1);
    8827         4408 :         lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_ADD_1 +
    8828              :                                        (fcode - BUILT_IN_ATOMIC_ADD_FETCH_1));
    8829         4408 :         target = expand_builtin_atomic_fetch_op (mode, exp, target, PLUS, true,
    8830              :                                                  ignore, lib);
    8831         4408 :         if (target)
    8832              :           return target;
    8833              :         break;
    8834              :       }
    8835         2172 :     case BUILT_IN_ATOMIC_SUB_FETCH_1:
    8836         2172 :     case BUILT_IN_ATOMIC_SUB_FETCH_2:
    8837         2172 :     case BUILT_IN_ATOMIC_SUB_FETCH_4:
    8838         2172 :     case BUILT_IN_ATOMIC_SUB_FETCH_8:
    8839         2172 :     case BUILT_IN_ATOMIC_SUB_FETCH_16:
    8840         2172 :       {
    8841         2172 :         enum built_in_function lib;
    8842         2172 :         mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_SUB_FETCH_1);
    8843         2172 :         lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_SUB_1 +
    8844              :                                        (fcode - BUILT_IN_ATOMIC_SUB_FETCH_1));
    8845         2172 :         target = expand_builtin_atomic_fetch_op (mode, exp, target, MINUS, true,
    8846              :                                                  ignore, lib);
    8847         2172 :         if (target)
    8848              :           return target;
    8849              :         break;
    8850              :       }
    8851          902 :     case BUILT_IN_ATOMIC_AND_FETCH_1:
    8852          902 :     case BUILT_IN_ATOMIC_AND_FETCH_2:
    8853          902 :     case BUILT_IN_ATOMIC_AND_FETCH_4:
    8854          902 :     case BUILT_IN_ATOMIC_AND_FETCH_8:
    8855          902 :     case BUILT_IN_ATOMIC_AND_FETCH_16:
    8856          902 :       {
    8857          902 :         enum built_in_function lib;
    8858          902 :         mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_AND_FETCH_1);
    8859          902 :         lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_AND_1 +
    8860              :                                        (fcode - BUILT_IN_ATOMIC_AND_FETCH_1));
    8861          902 :         target = expand_builtin_atomic_fetch_op (mode, exp, target, AND, true,
    8862              :                                                  ignore, lib);
    8863          902 :         if (target)
    8864              :           return target;
    8865              :         break;
    8866              :       }
    8867          162 :     case BUILT_IN_ATOMIC_NAND_FETCH_1:
    8868          162 :     case BUILT_IN_ATOMIC_NAND_FETCH_2:
    8869          162 :     case BUILT_IN_ATOMIC_NAND_FETCH_4:
    8870          162 :     case BUILT_IN_ATOMIC_NAND_FETCH_8:
    8871          162 :     case BUILT_IN_ATOMIC_NAND_FETCH_16:
    8872          162 :       {
    8873          162 :         enum built_in_function lib;
    8874          162 :         mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_NAND_FETCH_1);
    8875          162 :         lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_NAND_1 +
    8876              :                                        (fcode - BUILT_IN_ATOMIC_NAND_FETCH_1));
    8877          162 :         target = expand_builtin_atomic_fetch_op (mode, exp, target, NOT, true,
    8878              :                                                  ignore, lib);
    8879          162 :         if (target)
    8880              :           return target;
    8881              :         break;
    8882              :       }
    8883          854 :     case BUILT_IN_ATOMIC_XOR_FETCH_1:
    8884          854 :     case BUILT_IN_ATOMIC_XOR_FETCH_2:
    8885          854 :     case BUILT_IN_ATOMIC_XOR_FETCH_4:
    8886          854 :     case BUILT_IN_ATOMIC_XOR_FETCH_8:
    8887          854 :     case BUILT_IN_ATOMIC_XOR_FETCH_16:
    8888          854 :       {
    8889          854 :         enum built_in_function lib;
    8890          854 :         mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_XOR_FETCH_1);
    8891          854 :         lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_XOR_1 +
    8892              :                                        (fcode - BUILT_IN_ATOMIC_XOR_FETCH_1));
    8893          854 :         target = expand_builtin_atomic_fetch_op (mode, exp, target, XOR, true,
    8894              :                                                  ignore, lib);
    8895          854 :         if (target)
    8896              :           return target;
    8897              :         break;
    8898              :       }
    8899          965 :     case BUILT_IN_ATOMIC_OR_FETCH_1:
    8900          965 :     case BUILT_IN_ATOMIC_OR_FETCH_2:
    8901          965 :     case BUILT_IN_ATOMIC_OR_FETCH_4:
    8902          965 :     case BUILT_IN_ATOMIC_OR_FETCH_8:
    8903          965 :     case BUILT_IN_ATOMIC_OR_FETCH_16:
    8904          965 :       {
    8905          965 :         enum built_in_function lib;
    8906          965 :         mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_OR_FETCH_1);
    8907          965 :         lib = (enum built_in_function)((int)BUILT_IN_ATOMIC_FETCH_OR_1 +
    8908              :                                        (fcode - BUILT_IN_ATOMIC_OR_FETCH_1));
    8909          965 :         target = expand_builtin_atomic_fetch_op (mode, exp, target, IOR, true,
    8910              :                                                  ignore, lib);
    8911          965 :         if (target)
    8912              :           return target;
    8913              :         break;
    8914              :       }
    8915        14681 :     case BUILT_IN_ATOMIC_FETCH_ADD_1:
    8916        14681 :     case BUILT_IN_ATOMIC_FETCH_ADD_2:
    8917        14681 :     case BUILT_IN_ATOMIC_FETCH_ADD_4:
    8918        14681 :     case BUILT_IN_ATOMIC_FETCH_ADD_8:
    8919        14681 :     case BUILT_IN_ATOMIC_FETCH_ADD_16:
    8920        14681 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_ADD_1);
    8921        14681 :       target = expand_builtin_atomic_fetch_op (mode, exp, target, PLUS, false,
    8922              :                                                ignore, BUILT_IN_NONE);
    8923        14681 :       if (target)
    8924              :         return target;
    8925              :       break;
    8926              : 
    8927         2134 :     case BUILT_IN_ATOMIC_FETCH_SUB_1:
    8928         2134 :     case BUILT_IN_ATOMIC_FETCH_SUB_2:
    8929         2134 :     case BUILT_IN_ATOMIC_FETCH_SUB_4:
    8930         2134 :     case BUILT_IN_ATOMIC_FETCH_SUB_8:
    8931         2134 :     case BUILT_IN_ATOMIC_FETCH_SUB_16:
    8932         2134 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_SUB_1);
    8933         2134 :       target = expand_builtin_atomic_fetch_op (mode, exp, target, MINUS, false,
    8934              :                                                ignore, BUILT_IN_NONE);
    8935         2134 :       if (target)
    8936              :         return target;
    8937              :       break;
    8938              : 
    8939          829 :     case BUILT_IN_ATOMIC_FETCH_AND_1:
    8940          829 :     case BUILT_IN_ATOMIC_FETCH_AND_2:
    8941          829 :     case BUILT_IN_ATOMIC_FETCH_AND_4:
    8942          829 :     case BUILT_IN_ATOMIC_FETCH_AND_8:
    8943          829 :     case BUILT_IN_ATOMIC_FETCH_AND_16:
    8944          829 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_AND_1);
    8945          829 :       target = expand_builtin_atomic_fetch_op (mode, exp, target, AND, false,
    8946              :                                                ignore, BUILT_IN_NONE);
    8947          829 :       if (target)
    8948              :         return target;
    8949              :       break;
    8950              : 
    8951          165 :     case BUILT_IN_ATOMIC_FETCH_NAND_1:
    8952          165 :     case BUILT_IN_ATOMIC_FETCH_NAND_2:
    8953          165 :     case BUILT_IN_ATOMIC_FETCH_NAND_4:
    8954          165 :     case BUILT_IN_ATOMIC_FETCH_NAND_8:
    8955          165 :     case BUILT_IN_ATOMIC_FETCH_NAND_16:
    8956          165 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_NAND_1);
    8957          165 :       target = expand_builtin_atomic_fetch_op (mode, exp, target, NOT, false,
    8958              :                                                ignore, BUILT_IN_NONE);
    8959          165 :       if (target)
    8960              :         return target;
    8961              :       break;
    8962              : 
    8963          908 :     case BUILT_IN_ATOMIC_FETCH_XOR_1:
    8964          908 :     case BUILT_IN_ATOMIC_FETCH_XOR_2:
    8965          908 :     case BUILT_IN_ATOMIC_FETCH_XOR_4:
    8966          908 :     case BUILT_IN_ATOMIC_FETCH_XOR_8:
    8967          908 :     case BUILT_IN_ATOMIC_FETCH_XOR_16:
    8968          908 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_XOR_1);
    8969          908 :       target = expand_builtin_atomic_fetch_op (mode, exp, target, XOR, false,
    8970              :                                                ignore, BUILT_IN_NONE);
    8971          908 :       if (target)
    8972              :         return target;
    8973              :       break;
    8974              : 
    8975         1214 :     case BUILT_IN_ATOMIC_FETCH_OR_1:
    8976         1214 :     case BUILT_IN_ATOMIC_FETCH_OR_2:
    8977         1214 :     case BUILT_IN_ATOMIC_FETCH_OR_4:
    8978         1214 :     case BUILT_IN_ATOMIC_FETCH_OR_8:
    8979         1214 :     case BUILT_IN_ATOMIC_FETCH_OR_16:
    8980         1214 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_ATOMIC_FETCH_OR_1);
    8981         1214 :       target = expand_builtin_atomic_fetch_op (mode, exp, target, IOR, false,
    8982              :                                                ignore, BUILT_IN_NONE);
    8983         1214 :       if (target)
    8984              :         return target;
    8985              :       break;
    8986              : 
    8987          266 :     case BUILT_IN_ATOMIC_TEST_AND_SET:
    8988          266 :       target = expand_builtin_atomic_test_and_set (exp, target);
    8989          266 :       if (target)
    8990              :         return target;
    8991              :       break;
    8992              : 
    8993           53 :     case BUILT_IN_ATOMIC_CLEAR:
    8994           53 :       return expand_builtin_atomic_clear (exp);
    8995              : 
    8996            1 :     case BUILT_IN_ATOMIC_ALWAYS_LOCK_FREE:
    8997            1 :       return expand_builtin_atomic_always_lock_free (exp);
    8998              : 
    8999           10 :     case BUILT_IN_ATOMIC_IS_LOCK_FREE:
    9000           10 :       target = expand_builtin_atomic_is_lock_free (exp);
    9001           10 :       if (target)
    9002              :         return target;
    9003              :       break;
    9004              : 
    9005          719 :     case BUILT_IN_ATOMIC_THREAD_FENCE:
    9006          719 :       expand_builtin_atomic_thread_fence (exp);
    9007          719 :       return const0_rtx;
    9008              : 
    9009           60 :     case BUILT_IN_ATOMIC_SIGNAL_FENCE:
    9010           60 :       expand_builtin_atomic_signal_fence (exp);
    9011           60 :       return const0_rtx;
    9012              : 
    9013          620 :     case BUILT_IN_OBJECT_SIZE:
    9014          620 :     case BUILT_IN_DYNAMIC_OBJECT_SIZE:
    9015          620 :       return expand_builtin_object_size (exp);
    9016              : 
    9017          837 :     case BUILT_IN_MEMCPY_CHK:
    9018          837 :     case BUILT_IN_MEMPCPY_CHK:
    9019          837 :     case BUILT_IN_MEMMOVE_CHK:
    9020          837 :     case BUILT_IN_MEMSET_CHK:
    9021          837 :       target = expand_builtin_memory_chk (exp, target, mode, fcode);
    9022          837 :       if (target)
    9023              :         return target;
    9024              :       break;
    9025              : 
    9026         1086 :     case BUILT_IN_STRCPY_CHK:
    9027         1086 :     case BUILT_IN_STPCPY_CHK:
    9028         1086 :     case BUILT_IN_STRNCPY_CHK:
    9029         1086 :     case BUILT_IN_STPNCPY_CHK:
    9030         1086 :     case BUILT_IN_STRCAT_CHK:
    9031         1086 :     case BUILT_IN_STRNCAT_CHK:
    9032         1086 :     case BUILT_IN_SNPRINTF_CHK:
    9033         1086 :     case BUILT_IN_VSNPRINTF_CHK:
    9034         1086 :       maybe_emit_chk_warning (exp, fcode);
    9035         1086 :       break;
    9036              : 
    9037         1328 :     case BUILT_IN_SPRINTF_CHK:
    9038         1328 :     case BUILT_IN_VSPRINTF_CHK:
    9039         1328 :       maybe_emit_sprintf_chk_warning (exp, fcode);
    9040         1328 :       break;
    9041              : 
    9042            3 :     case BUILT_IN_THREAD_POINTER:
    9043            3 :       return expand_builtin_thread_pointer (exp, target);
    9044              : 
    9045            0 :     case BUILT_IN_SET_THREAD_POINTER:
    9046            0 :       expand_builtin_set_thread_pointer (exp);
    9047            0 :       return const0_rtx;
    9048              : 
    9049              :     case BUILT_IN_ACC_ON_DEVICE:
    9050              :       /* Do library call, if we failed to expand the builtin when
    9051              :          folding.  */
    9052              :       break;
    9053              : 
    9054          356 :     case BUILT_IN_GOACC_PARLEVEL_ID:
    9055          356 :     case BUILT_IN_GOACC_PARLEVEL_SIZE:
    9056          356 :       return expand_builtin_goacc_parlevel_id_size (exp, target, ignore);
    9057              : 
    9058            4 :     case BUILT_IN_SPECULATION_SAFE_VALUE_PTR:
    9059            4 :       return expand_speculation_safe_value (VOIDmode, exp, target, ignore);
    9060              : 
    9061           30 :     case BUILT_IN_SPECULATION_SAFE_VALUE_1:
    9062           30 :     case BUILT_IN_SPECULATION_SAFE_VALUE_2:
    9063           30 :     case BUILT_IN_SPECULATION_SAFE_VALUE_4:
    9064           30 :     case BUILT_IN_SPECULATION_SAFE_VALUE_8:
    9065           30 :     case BUILT_IN_SPECULATION_SAFE_VALUE_16:
    9066           30 :       mode = get_builtin_sync_mode (fcode - BUILT_IN_SPECULATION_SAFE_VALUE_1);
    9067           30 :       return expand_speculation_safe_value (mode, exp, target, ignore);
    9068              : 
    9069            2 :     case BUILT_IN_CRC8_DATA8:
    9070            2 :       return expand_builtin_crc_table_based (IFN_CRC, QImode, QImode, mode,
    9071              :                                                exp, target);
    9072            2 :     case BUILT_IN_CRC16_DATA8:
    9073            2 :       return expand_builtin_crc_table_based (IFN_CRC, HImode, QImode, mode,
    9074              :                                                exp, target);
    9075            3 :     case BUILT_IN_CRC16_DATA16:
    9076            3 :       return expand_builtin_crc_table_based (IFN_CRC, HImode, HImode, mode,
    9077              :                                                exp, target);
    9078            2 :     case BUILT_IN_CRC32_DATA8:
    9079            2 :       return expand_builtin_crc_table_based (IFN_CRC, SImode, QImode, mode,
    9080              :                                                exp, target);
    9081            2 :     case BUILT_IN_CRC32_DATA16:
    9082            2 :       return expand_builtin_crc_table_based (IFN_CRC, SImode, HImode, mode,
    9083              :                                                exp, target);
    9084           14 :     case BUILT_IN_CRC32_DATA32:
    9085           14 :       return expand_builtin_crc_table_based (IFN_CRC, SImode, SImode, mode,
    9086              :                                                exp, target);
    9087            1 :     case BUILT_IN_CRC64_DATA8:
    9088            1 :       return expand_builtin_crc_table_based (IFN_CRC, DImode, QImode, mode,
    9089              :                                                exp, target);
    9090            1 :     case BUILT_IN_CRC64_DATA16:
    9091            1 :       return expand_builtin_crc_table_based (IFN_CRC, DImode, HImode, mode,
    9092              :                                                exp, target);
    9093            1 :     case BUILT_IN_CRC64_DATA32:
    9094            1 :       return expand_builtin_crc_table_based (IFN_CRC, DImode, SImode, mode,
    9095              :                                                exp, target);
    9096            1 :     case BUILT_IN_CRC64_DATA64:
    9097            1 :       return expand_builtin_crc_table_based (IFN_CRC, DImode, DImode, mode,
    9098              :                                                exp, target);
    9099            2 :     case BUILT_IN_REV_CRC8_DATA8:
    9100            2 :       return expand_builtin_crc_table_based (IFN_CRC_REV, QImode, QImode,
    9101              :                                                mode, exp, target);
    9102            2 :     case BUILT_IN_REV_CRC16_DATA8:
    9103            2 :       return expand_builtin_crc_table_based (IFN_CRC_REV, HImode, QImode,
    9104              :                                                mode, exp, target);
    9105            2 :     case BUILT_IN_REV_CRC16_DATA16:
    9106            2 :       return expand_builtin_crc_table_based (IFN_CRC_REV, HImode, HImode,
    9107              :                                                mode, exp, target);
    9108            3 :     case BUILT_IN_REV_CRC32_DATA8:
    9109            3 :       return expand_builtin_crc_table_based (IFN_CRC_REV, SImode, QImode,
    9110              :                                                mode, exp, target);
    9111            2 :     case BUILT_IN_REV_CRC32_DATA16:
    9112            2 :       return expand_builtin_crc_table_based (IFN_CRC_REV, SImode, HImode,
    9113              :                                                mode, exp, target);
    9114            2 :     case BUILT_IN_REV_CRC32_DATA32:
    9115            2 :       return expand_builtin_crc_table_based (IFN_CRC_REV, SImode, SImode,
    9116              :                                                mode, exp, target);
    9117            1 :     case BUILT_IN_REV_CRC64_DATA8:
    9118            1 :       return expand_builtin_crc_table_based (IFN_CRC_REV, DImode, QImode,
    9119              :                                                mode, exp, target);
    9120            1 :     case BUILT_IN_REV_CRC64_DATA16:
    9121            1 :       return expand_builtin_crc_table_based (IFN_CRC_REV, DImode, HImode,
    9122              :                                                mode, exp, target);
    9123            1 :     case BUILT_IN_REV_CRC64_DATA32:
    9124            1 :       return expand_builtin_crc_table_based (IFN_CRC_REV, DImode, SImode,
    9125              :                                                mode, exp, target);
    9126            1 :     case BUILT_IN_REV_CRC64_DATA64:
    9127            1 :       return expand_builtin_crc_table_based (IFN_CRC_REV, DImode, DImode,
    9128              :                                                mode, exp, target);
    9129              :     default:    /* just do library call, if unknown builtin */
    9130              :       break;
    9131              :     }
    9132              : 
    9133              :   /* The switch statement above can drop through to cause the function
    9134              :      to be called normally.  */
    9135       864618 :   return expand_call (exp, target, ignore);
    9136              : }
    9137              : 
    9138              : /* Determine whether a tree node represents a call to a built-in
    9139              :    function.  If the tree T is a call to a built-in function with
    9140              :    the right number of arguments of the appropriate types, return
    9141              :    the DECL_FUNCTION_CODE of the call, e.g. BUILT_IN_SQRT.
    9142              :    Otherwise the return value is END_BUILTINS.  */
    9143              : 
    9144              : enum built_in_function
    9145    108133197 : builtin_mathfn_code (const_tree t)
    9146              : {
    9147    108133197 :   const_tree fndecl, arg, parmlist;
    9148    108133197 :   const_tree argtype, parmtype;
    9149    108133197 :   const_call_expr_arg_iterator iter;
    9150              : 
    9151    108133197 :   if (TREE_CODE (t) != CALL_EXPR)
    9152              :     return END_BUILTINS;
    9153              : 
    9154      1453122 :   fndecl = get_callee_fndecl (t);
    9155      1453122 :   if (fndecl == NULL_TREE || !fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
    9156              :       return END_BUILTINS;
    9157              : 
    9158       658948 :   parmlist = TYPE_ARG_TYPES (TREE_TYPE (fndecl));
    9159       658948 :   init_const_call_expr_arg_iterator (t, &iter);
    9160      2127933 :   for (; parmlist; parmlist = TREE_CHAIN (parmlist))
    9161              :     {
    9162              :       /* If a function doesn't take a variable number of arguments,
    9163              :          the last element in the list will have type `void'.  */
    9164      1449522 :       parmtype = TREE_VALUE (parmlist);
    9165      1449522 :       if (VOID_TYPE_P (parmtype))
    9166              :         {
    9167       639471 :           if (more_const_call_expr_args_p (&iter))
    9168              :             return END_BUILTINS;
    9169       639471 :           return DECL_FUNCTION_CODE (fndecl);
    9170              :         }
    9171              : 
    9172       810051 :       if (! more_const_call_expr_args_p (&iter))
    9173              :         return END_BUILTINS;
    9174              : 
    9175       810051 :       arg = next_const_call_expr_arg (&iter);
    9176       810051 :       argtype = TREE_TYPE (arg);
    9177              : 
    9178       810051 :       if (SCALAR_FLOAT_TYPE_P (parmtype))
    9179              :         {
    9180       549173 :           if (! SCALAR_FLOAT_TYPE_P (argtype))
    9181              :             return END_BUILTINS;
    9182              :         }
    9183       260878 :       else if (COMPLEX_FLOAT_TYPE_P (parmtype))
    9184              :         {
    9185         7817 :           if (! COMPLEX_FLOAT_TYPE_P (argtype))
    9186              :             return END_BUILTINS;
    9187              :         }
    9188       253061 :       else if (POINTER_TYPE_P (parmtype))
    9189              :         {
    9190        75501 :           if (! POINTER_TYPE_P (argtype))
    9191              :             return END_BUILTINS;
    9192              :         }
    9193       177560 :       else if (INTEGRAL_TYPE_P (parmtype))
    9194              :         {
    9195       177560 :           if (! INTEGRAL_TYPE_P (argtype))
    9196              :             return END_BUILTINS;
    9197              :         }
    9198              :       else
    9199              :         return END_BUILTINS;
    9200              :     }
    9201              : 
    9202              :   /* Variable-length argument list.  */
    9203        19463 :   return DECL_FUNCTION_CODE (fndecl);
    9204              : }
    9205              : 
    9206              : /* Fold a call to __builtin_constant_p, if we know its argument ARG will
    9207              :    evaluate to a constant.  */
    9208              : 
    9209              : tree
    9210      3261200 : fold_builtin_constant_p (tree arg)
    9211              : {
    9212              :   /* We return 1 for a numeric type that's known to be a constant
    9213              :      value at compile-time or for an aggregate type that's a
    9214              :      literal constant.  */
    9215      3261200 :   STRIP_NOPS (arg);
    9216              : 
    9217              :   /* If we know this is a constant, emit the constant of one.  */
    9218      3261200 :   if (CONSTANT_CLASS_P (arg)
    9219      3261200 :       || (TREE_CODE (arg) == CONSTRUCTOR
    9220            6 :           && TREE_CONSTANT (arg)))
    9221      1835743 :     return integer_one_node;
    9222      1425457 :   if (TREE_CODE (arg) == ADDR_EXPR)
    9223              :     {
    9224          110 :        tree op = TREE_OPERAND (arg, 0);
    9225          110 :        if (TREE_CODE (op) == STRING_CST
    9226          110 :            || (TREE_CODE (op) == ARRAY_REF
    9227           57 :                && integer_zerop (TREE_OPERAND (op, 1))
    9228           57 :                && TREE_CODE (TREE_OPERAND (op, 0)) == STRING_CST))
    9229           87 :          return integer_one_node;
    9230              :     }
    9231              : 
    9232              :   /* If this expression has side effects, show we don't know it to be a
    9233              :      constant.  Likewise if it's a pointer or aggregate type since in
    9234              :      those case we only want literals, since those are only optimized
    9235              :      when generating RTL, not later.
    9236              :      And finally, if we are compiling an initializer, not code, we
    9237              :      need to return a definite result now; there's not going to be any
    9238              :      more optimization done.  */
    9239      1425370 :   if (TREE_SIDE_EFFECTS (arg)
    9240      1425245 :       || AGGREGATE_TYPE_P (TREE_TYPE (arg))
    9241      1425239 :       || POINTER_TYPE_P (TREE_TYPE (arg))
    9242      1425068 :       || cfun == 0
    9243      1424975 :       || folding_initializer
    9244      2850338 :       || force_folding_builtin_constant_p)
    9245          497 :     return integer_zero_node;
    9246              : 
    9247              :   return NULL_TREE;
    9248              : }
    9249              : 
    9250              : /* Create builtin_expect or builtin_expect_with_probability
    9251              :    with PRED and EXPECTED as its arguments and return it as a truthvalue.
    9252              :    Fortran FE can also produce builtin_expect with PREDICTOR as third argument.
    9253              :    builtin_expect_with_probability instead uses third argument as PROBABILITY
    9254              :    value.  */
    9255              : 
    9256              : static tree
    9257       227798 : build_builtin_expect_predicate (location_t loc, tree pred, tree expected,
    9258              :                                 tree predictor, tree probability)
    9259              : {
    9260       227798 :   tree fn, arg_types, pred_type, expected_type, call_expr, ret_type;
    9261              : 
    9262       227798 :   fn = builtin_decl_explicit (probability == NULL_TREE ? BUILT_IN_EXPECT
    9263              :                               : BUILT_IN_EXPECT_WITH_PROBABILITY);
    9264       227798 :   arg_types = TYPE_ARG_TYPES (TREE_TYPE (fn));
    9265       227798 :   ret_type = TREE_TYPE (TREE_TYPE (fn));
    9266       227798 :   pred_type = TREE_VALUE (arg_types);
    9267       227798 :   expected_type = TREE_VALUE (TREE_CHAIN (arg_types));
    9268              : 
    9269       227798 :   pred = fold_convert_loc (loc, pred_type, pred);
    9270       227798 :   expected = fold_convert_loc (loc, expected_type, expected);
    9271              : 
    9272       227798 :   if (probability)
    9273            0 :     call_expr = build_call_expr_loc (loc, fn, 3, pred, expected, probability);
    9274              :   else
    9275       455596 :     call_expr = build_call_expr_loc (loc, fn, predictor ? 3 : 2, pred, expected,
    9276              :                                      predictor);
    9277              : 
    9278       227798 :   return build2 (NE_EXPR, TREE_TYPE (pred), call_expr,
    9279       227798 :                  build_int_cst (ret_type, 0));
    9280              : }
    9281              : 
    9282              : /* Fold a call to builtin_expect with arguments ARG0, ARG1, ARG2, ARG3.  Return
    9283              :    NULL_TREE if no simplification is possible.  */
    9284              : 
    9285              : tree
    9286      7360610 : fold_builtin_expect (location_t loc, tree arg0, tree arg1, tree arg2,
    9287              :                      tree arg3)
    9288              : {
    9289      7360610 :   tree inner, fndecl, inner_arg0;
    9290      7360610 :   enum tree_code code;
    9291              : 
    9292              :   /* Distribute the expected value over short-circuiting operators.
    9293              :      See through the cast from truthvalue_type_node to long.  */
    9294      7360610 :   inner_arg0 = arg0;
    9295     15035637 :   while (CONVERT_EXPR_P (inner_arg0)
    9296      1073918 :          && INTEGRAL_TYPE_P (TREE_TYPE (inner_arg0))
    9297      9508445 :          && INTEGRAL_TYPE_P (TREE_TYPE (TREE_OPERAND (inner_arg0, 0))))
    9298      1073917 :     inner_arg0 = TREE_OPERAND (inner_arg0, 0);
    9299              : 
    9300              :   /* If this is a builtin_expect within a builtin_expect keep the
    9301              :      inner one.  See through a comparison against a constant.  It
    9302              :      might have been added to create a thruthvalue.  */
    9303      7360610 :   inner = inner_arg0;
    9304              : 
    9305      7360610 :   if (COMPARISON_CLASS_P (inner)
    9306      7360610 :       && TREE_CODE (TREE_OPERAND (inner, 1)) == INTEGER_CST)
    9307      1413749 :     inner = TREE_OPERAND (inner, 0);
    9308              : 
    9309      7360610 :   if (TREE_CODE (inner) == CALL_EXPR
    9310       175341 :       && (fndecl = get_callee_fndecl (inner))
    9311      7535951 :       && fndecl_built_in_p (fndecl, BUILT_IN_EXPECT,
    9312              :                             BUILT_IN_EXPECT_WITH_PROBABILITY))
    9313              :     return arg0;
    9314              : 
    9315      7360602 :   inner = inner_arg0;
    9316      7360602 :   code = TREE_CODE (inner);
    9317      7360602 :   if (code == TRUTH_ANDIF_EXPR || code == TRUTH_ORIF_EXPR)
    9318              :     {
    9319       113899 :       tree op0 = TREE_OPERAND (inner, 0);
    9320       113899 :       tree op1 = TREE_OPERAND (inner, 1);
    9321       113899 :       arg1 = save_expr (arg1);
    9322              : 
    9323       113899 :       op0 = build_builtin_expect_predicate (loc, op0, arg1, arg2, arg3);
    9324       113899 :       op1 = build_builtin_expect_predicate (loc, op1, arg1, arg2, arg3);
    9325       113899 :       inner = build2 (code, TREE_TYPE (inner), op0, op1);
    9326              : 
    9327       113899 :       return fold_convert_loc (loc, TREE_TYPE (arg0), inner);
    9328              :     }
    9329              : 
    9330              :   /* If the argument isn't invariant then there's nothing else we can do.  */
    9331      7246703 :   if (!TREE_CONSTANT (inner_arg0))
    9332              :     return NULL_TREE;
    9333              : 
    9334              :   /* If we expect that a comparison against the argument will fold to
    9335              :      a constant return the constant.  In practice, this means a true
    9336              :      constant or the address of a non-weak symbol.  */
    9337       253852 :   inner = inner_arg0;
    9338       253852 :   STRIP_NOPS (inner);
    9339       253852 :   if (TREE_CODE (inner) == ADDR_EXPR)
    9340              :     {
    9341            1 :       do
    9342              :         {
    9343            1 :           inner = TREE_OPERAND (inner, 0);
    9344              :         }
    9345            1 :       while (TREE_CODE (inner) == COMPONENT_REF
    9346            1 :              || TREE_CODE (inner) == ARRAY_REF);
    9347            1 :       if (VAR_OR_FUNCTION_DECL_P (inner) && DECL_WEAK (inner))
    9348            0 :         return NULL_TREE;
    9349              :     }
    9350              : 
    9351              :   /* Otherwise, ARG0 already has the proper type for the return value.  */
    9352              :   return arg0;
    9353              : }
    9354              : 
    9355              : /* Fold a call to __builtin_classify_type with argument ARG.  */
    9356              : 
    9357              : static tree
    9358         2084 : fold_builtin_classify_type (tree arg)
    9359              : {
    9360         2084 :   if (arg == 0)
    9361            0 :     return build_int_cst (integer_type_node, no_type_class);
    9362              : 
    9363         2084 :   return build_int_cst (integer_type_node, type_to_class (TREE_TYPE (arg)));
    9364              : }
    9365              : 
    9366              : /* Fold a call EXPR (which may be null) to __builtin_strlen with argument
    9367              :    ARG.  */
    9368              : 
    9369              : static tree
    9370       461577 : fold_builtin_strlen (location_t loc, tree expr, tree type, tree arg)
    9371              : {
    9372       461577 :   if (!validate_arg (arg, POINTER_TYPE))
    9373              :     return NULL_TREE;
    9374              :   else
    9375              :     {
    9376       461574 :       c_strlen_data lendata = { };
    9377       461574 :       tree len = c_strlen (arg, 0, &lendata);
    9378              : 
    9379       461574 :       if (len)
    9380         2223 :         return fold_convert_loc (loc, type, len);
    9381              : 
    9382              :       /* TODO: Move this to gimple-ssa-warn-access once the pass runs
    9383              :          also early enough to detect invalid reads in multimensional
    9384              :          arrays and struct members.  */
    9385       459351 :       if (!lendata.decl)
    9386       452173 :          c_strlen (arg, 1, &lendata);
    9387              : 
    9388       459351 :       if (lendata.decl)
    9389              :         {
    9390         7192 :           if (EXPR_HAS_LOCATION (arg))
    9391         2800 :             loc = EXPR_LOCATION (arg);
    9392         4392 :           else if (loc == UNKNOWN_LOCATION)
    9393            0 :             loc = input_location;
    9394         7192 :           warn_string_no_nul (loc, expr, "strlen", arg, lendata.decl);
    9395              :         }
    9396              : 
    9397              :       return NULL_TREE;
    9398              :     }
    9399              : }
    9400              : 
    9401              : /* Fold a call to __builtin_inf or __builtin_huge_val.  */
    9402              : 
    9403              : static tree
    9404       258405 : fold_builtin_inf (location_t loc, tree type, int warn)
    9405              : {
    9406              :   /* __builtin_inff is intended to be usable to define INFINITY on all
    9407              :      targets.  If an infinity is not available, INFINITY expands "to a
    9408              :      positive constant of type float that overflows at translation
    9409              :      time", footnote "In this case, using INFINITY will violate the
    9410              :      constraint in 6.4.4 and thus require a diagnostic." (C99 7.12#4).
    9411              :      Thus we pedwarn to ensure this constraint violation is
    9412              :      diagnosed.  */
    9413      1032811 :   if (!MODE_HAS_INFINITIES (TYPE_MODE (type)) && warn)
    9414            0 :     pedwarn (loc, 0, "target format does not support infinity");
    9415              : 
    9416       258405 :   return build_real (type, dconstinf);
    9417              : }
    9418              : 
    9419              : /* Fold function call to builtin sincos, sincosf, or sincosl.  Return
    9420              :    NULL_TREE if no simplification can be made.  */
    9421              : 
    9422              : static tree
    9423          145 : fold_builtin_sincos (location_t loc,
    9424              :                      tree arg0, tree arg1, tree arg2)
    9425              : {
    9426          145 :   tree type;
    9427          145 :   tree fndecl, call = NULL_TREE;
    9428              : 
    9429          145 :   if (!validate_arg (arg0, REAL_TYPE)
    9430          145 :       || !validate_arg (arg1, POINTER_TYPE)
    9431          290 :       || !validate_arg (arg2, POINTER_TYPE))
    9432              :     return NULL_TREE;
    9433              : 
    9434          145 :   type = TREE_TYPE (arg0);
    9435              : 
    9436              :   /* Calculate the result when the argument is a constant.  */
    9437          145 :   built_in_function fn = mathfn_built_in_2 (type, CFN_BUILT_IN_CEXPI);
    9438          145 :   if (fn == END_BUILTINS)
    9439              :     return NULL_TREE;
    9440              : 
    9441              :   /* Canonicalize sincos to cexpi.  */
    9442          145 :   if (TREE_CODE (arg0) == REAL_CST)
    9443              :     {
    9444           86 :       tree complex_type = build_complex_type (type);
    9445           86 :       call = fold_const_call (as_combined_fn (fn), complex_type, arg0);
    9446              :     }
    9447           86 :   if (!call)
    9448              :     {
    9449           59 :       if (!targetm.libc_has_function (function_c99_math_complex, type)
    9450           59 :           || !builtin_decl_implicit_p (fn))
    9451              :         return NULL_TREE;
    9452           59 :       fndecl = builtin_decl_explicit (fn);
    9453           59 :       call = build_call_expr_loc (loc, fndecl, 1, arg0);
    9454           59 :       call = builtin_save_expr (call);
    9455              :     }
    9456              : 
    9457          145 :   tree ptype = build_pointer_type (type);
    9458          145 :   arg1 = fold_convert (ptype, arg1);
    9459          145 :   arg2 = fold_convert (ptype, arg2);
    9460          145 :   return build2 (COMPOUND_EXPR, void_type_node,
    9461              :                  build2 (MODIFY_EXPR, void_type_node,
    9462              :                          build_fold_indirect_ref_loc (loc, arg1),
    9463              :                          fold_build1_loc (loc, IMAGPART_EXPR, type, call)),
    9464              :                  build2 (MODIFY_EXPR, void_type_node,
    9465              :                          build_fold_indirect_ref_loc (loc, arg2),
    9466          145 :                          fold_build1_loc (loc, REALPART_EXPR, type, call)));
    9467              : }
    9468              : 
    9469              : /* Fold function call to builtin memcmp with arguments ARG1 and ARG2.
    9470              :    Return NULL_TREE if no simplification can be made.  */
    9471              : 
    9472              : static tree
    9473      2547918 : fold_builtin_memcmp (location_t loc, tree arg1, tree arg2, tree len)
    9474              : {
    9475      2547918 :   if (!validate_arg (arg1, POINTER_TYPE)
    9476      2547918 :       || !validate_arg (arg2, POINTER_TYPE)
    9477      5095836 :       || !validate_arg (len, INTEGER_TYPE))
    9478              :     return NULL_TREE;
    9479              : 
    9480              :   /* If the LEN parameter is zero, return zero.  */
    9481      2547918 :   if (integer_zerop (len))
    9482            0 :     return omit_two_operands_loc (loc, integer_type_node, integer_zero_node,
    9483            0 :                               arg1, arg2);
    9484              : 
    9485              :   /* If ARG1 and ARG2 are the same (and not volatile), return zero.  */
    9486      2547918 :   if (operand_equal_p (arg1, arg2, 0))
    9487          705 :     return omit_one_operand_loc (loc, integer_type_node, integer_zero_node, len);
    9488              : 
    9489              :   /* If len parameter is one, return an expression corresponding to
    9490              :      (*(const unsigned char*)arg1 - (const unsigned char*)arg2).  */
    9491      2547213 :   if (tree_fits_uhwi_p (len) && tree_to_uhwi (len) == 1)
    9492              :     {
    9493        54799 :       tree cst_uchar_node = build_type_variant (unsigned_char_type_node, 1, 0);
    9494        54799 :       tree cst_uchar_ptr_node
    9495        54799 :         = build_pointer_type_for_mode (cst_uchar_node, ptr_mode, true);
    9496              : 
    9497        54799 :       tree ind1
    9498        54799 :         = fold_convert_loc (loc, integer_type_node,
    9499              :                             build1 (INDIRECT_REF, cst_uchar_node,
    9500              :                                     fold_convert_loc (loc,
    9501              :                                                       cst_uchar_ptr_node,
    9502              :                                                       arg1)));
    9503        54799 :       tree ind2
    9504        54799 :         = fold_convert_loc (loc, integer_type_node,
    9505              :                             build1 (INDIRECT_REF, cst_uchar_node,
    9506              :                                     fold_convert_loc (loc,
    9507              :                                                       cst_uchar_ptr_node,
    9508              :                                                       arg2)));
    9509        54799 :       return fold_build2_loc (loc, MINUS_EXPR, integer_type_node, ind1, ind2);
    9510              :     }
    9511              : 
    9512              :   return NULL_TREE;
    9513              : }
    9514              : 
    9515              : /* Fold a call to builtin isascii with argument ARG.  */
    9516              : 
    9517              : static tree
    9518          211 : fold_builtin_isascii (location_t loc, tree arg)
    9519              : {
    9520          211 :   if (!validate_arg (arg, INTEGER_TYPE))
    9521              :     return NULL_TREE;
    9522              :   else
    9523              :     {
    9524              :       /* Transform isascii(c) -> ((c & ~0x7f) == 0).  */
    9525          211 :       arg = fold_build2 (BIT_AND_EXPR, integer_type_node, arg,
    9526              :                          build_int_cst (integer_type_node,
    9527              :                                         ~ HOST_WIDE_INT_UC (0x7f)));
    9528          211 :       return fold_build2_loc (loc, EQ_EXPR, integer_type_node,
    9529          211 :                               arg, integer_zero_node);
    9530              :     }
    9531              : }
    9532              : 
    9533              : /* Fold a call to builtin toascii with argument ARG.  */
    9534              : 
    9535              : static tree
    9536          168 : fold_builtin_toascii (location_t loc, tree arg)
    9537              : {
    9538          168 :   if (!validate_arg (arg, INTEGER_TYPE))
    9539              :     return NULL_TREE;
    9540              : 
    9541              :   /* Transform toascii(c) -> (c & 0x7f).  */
    9542          168 :   return fold_build2_loc (loc, BIT_AND_EXPR, integer_type_node, arg,
    9543              :                           build_int_cst (integer_type_node, 0x7f));
    9544              : }
    9545              : 
    9546              : /* Fold a call to builtin isdigit with argument ARG.  */
    9547              : 
    9548              : static tree
    9549          325 : fold_builtin_isdigit (location_t loc, tree arg)
    9550              : {
    9551          325 :   if (!validate_arg (arg, INTEGER_TYPE))
    9552              :     return NULL_TREE;
    9553              :   else
    9554              :     {
    9555              :       /* Transform isdigit(c) -> (unsigned)(c) - '0' <= 9.  */
    9556              :       /* According to the C standard, isdigit is unaffected by locale.
    9557              :          However, it definitely is affected by the target character set.  */
    9558          313 :       unsigned HOST_WIDE_INT target_digit0
    9559          313 :         = lang_hooks.to_target_charset ('0');
    9560              : 
    9561          313 :       if (target_digit0 == 0)
    9562              :         return NULL_TREE;
    9563              : 
    9564          313 :       arg = fold_convert_loc (loc, unsigned_type_node, arg);
    9565          313 :       arg = fold_build2 (MINUS_EXPR, unsigned_type_node, arg,
    9566              :                          build_int_cst (unsigned_type_node, target_digit0));
    9567          313 :       return fold_build2_loc (loc, LE_EXPR, integer_type_node, arg,
    9568              :                           build_int_cst (unsigned_type_node, 9));
    9569              :     }
    9570              : }
    9571              : 
    9572              : /* Fold a call to fabs, fabsf or fabsl with argument ARG.  */
    9573              : 
    9574              : static tree
    9575       372999 : fold_builtin_fabs (location_t loc, tree arg, tree type)
    9576              : {
    9577       372999 :   if (!validate_arg (arg, REAL_TYPE))
    9578              :     return NULL_TREE;
    9579              : 
    9580       372914 :   arg = fold_convert_loc (loc, type, arg);
    9581       372914 :   return fold_build1_loc (loc, ABS_EXPR, type, arg);
    9582              : }
    9583              : 
    9584              : /* Fold a call to abs, labs, llabs, imaxabs, uabs, ulabs, ullabs or uimaxabs
    9585              :    with argument ARG.  */
    9586              : 
    9587              : static tree
    9588        98708 : fold_builtin_abs (location_t loc, tree arg, tree type)
    9589              : {
    9590        98708 :   if (!validate_arg (arg, INTEGER_TYPE) || !INTEGRAL_TYPE_P (type))
    9591              :     return NULL_TREE;
    9592              : 
    9593        98660 :   if (TYPE_UNSIGNED (type))
    9594              :     {
    9595         1168 :       if (TYPE_PRECISION (TREE_TYPE (arg))
    9596         1168 :           != TYPE_PRECISION (type)
    9597         1168 :           || TYPE_UNSIGNED (TREE_TYPE (arg)))
    9598              :         return NULL_TREE;
    9599         1168 :       return fold_build1_loc (loc, ABSU_EXPR, type, arg);
    9600              :     }
    9601        97492 :   arg = fold_convert_loc (loc, type, arg);
    9602        97492 :   return fold_build1_loc (loc, ABS_EXPR, type, arg);
    9603              : }
    9604              : 
    9605              : /* Fold a call to builtin carg(a+bi) -> atan2(b,a).  */
    9606              : 
    9607              : static tree
    9608       104102 : fold_builtin_carg (location_t loc, tree arg, tree type)
    9609              : {
    9610       104102 :   if (validate_arg (arg, COMPLEX_TYPE)
    9611       104102 :       && SCALAR_FLOAT_TYPE_P (TREE_TYPE (TREE_TYPE (arg))))
    9612              :     {
    9613       104102 :       tree atan2_fn = mathfn_built_in (type, BUILT_IN_ATAN2);
    9614              : 
    9615       104102 :       if (atan2_fn)
    9616              :         {
    9617       100178 :           tree new_arg = builtin_save_expr (arg);
    9618       100178 :           tree r_arg = fold_build1_loc (loc, REALPART_EXPR, type, new_arg);
    9619       100178 :           tree i_arg = fold_build1_loc (loc, IMAGPART_EXPR, type, new_arg);
    9620       100178 :           return build_call_expr_loc (loc, atan2_fn, 2, i_arg, r_arg);
    9621              :         }
    9622              :     }
    9623              : 
    9624              :   return NULL_TREE;
    9625              : }
    9626              : 
    9627              : /* Fold a call to builtin frexp, we can assume the base is 2.  */
    9628              : 
    9629              : static tree
    9630       115827 : fold_builtin_frexp (location_t loc, tree arg0, tree arg1, tree rettype)
    9631              : {
    9632       115827 :   if (! validate_arg (arg0, REAL_TYPE) || ! validate_arg (arg1, POINTER_TYPE))
    9633              :     return NULL_TREE;
    9634              : 
    9635       115827 :   STRIP_NOPS (arg0);
    9636              : 
    9637       115827 :   if (!(TREE_CODE (arg0) == REAL_CST && ! TREE_OVERFLOW (arg0)))
    9638              :     return NULL_TREE;
    9639              : 
    9640         2210 :   arg1 = build_fold_indirect_ref_loc (loc, arg1);
    9641              : 
    9642              :   /* Proceed if a valid pointer type was passed in.  */
    9643         2210 :   if (TYPE_MAIN_VARIANT (TREE_TYPE (arg1)) == integer_type_node)
    9644              :     {
    9645         2210 :       const REAL_VALUE_TYPE *const value = TREE_REAL_CST_PTR (arg0);
    9646         2210 :       tree frac, exp, res;
    9647              : 
    9648         2210 :       switch (value->cl)
    9649              :       {
    9650          276 :       case rvc_zero:
    9651          276 :       case rvc_nan:
    9652          276 :       case rvc_inf:
    9653              :         /* For +-0, return (*exp = 0, +-0).  */
    9654              :         /* For +-NaN or +-Inf, *exp is unspecified, but something should
    9655              :            be stored there so that it isn't read from uninitialized object.
    9656              :            As glibc and newlib store *exp = 0 for +-Inf/NaN, storing
    9657              :            0 here as well is easiest.  */
    9658          276 :         exp = integer_zero_node;
    9659          276 :         frac = arg0;
    9660          276 :         break;
    9661         1934 :       case rvc_normal:
    9662         1934 :         {
    9663              :           /* Since the frexp function always expects base 2, and in
    9664              :              GCC normalized significands are already in the range
    9665              :              [0.5, 1.0), we have exactly what frexp wants.  */
    9666         1934 :           REAL_VALUE_TYPE frac_rvt = *value;
    9667         1934 :           SET_REAL_EXP (&frac_rvt, 0);
    9668         1934 :           frac = build_real (rettype, frac_rvt);
    9669         1934 :           exp = build_int_cst (integer_type_node, REAL_EXP (value));
    9670              :         }
    9671         1934 :         break;
    9672            0 :       default:
    9673            0 :         gcc_unreachable ();
    9674              :       }
    9675              : 
    9676              :       /* Create the COMPOUND_EXPR (*arg1 = trunc, frac). */
    9677         2210 :       arg1 = fold_build2_loc (loc, MODIFY_EXPR, rettype, arg1, exp);
    9678         2210 :       TREE_SIDE_EFFECTS (arg1) = 1;
    9679         2210 :       res = fold_build2_loc (loc, COMPOUND_EXPR, rettype, arg1, frac);
    9680         2210 :       suppress_warning (res, OPT_Wunused_value);
    9681         2210 :       return res;
    9682              :     }
    9683              : 
    9684              :   return NULL_TREE;
    9685              : }
    9686              : 
    9687              : /* Fold a call to builtin modf.  */
    9688              : 
    9689              : static tree
    9690        92498 : fold_builtin_modf (location_t loc, tree arg0, tree arg1, tree rettype)
    9691              : {
    9692        92498 :   if (! validate_arg (arg0, REAL_TYPE) || ! validate_arg (arg1, POINTER_TYPE))
    9693              :     return NULL_TREE;
    9694              : 
    9695        92498 :   STRIP_NOPS (arg0);
    9696              : 
    9697        92498 :   if (!(TREE_CODE (arg0) == REAL_CST && ! TREE_OVERFLOW (arg0)))
    9698              :     return NULL_TREE;
    9699              : 
    9700         2474 :   arg1 = build_fold_indirect_ref_loc (loc, arg1);
    9701              : 
    9702              :   /* Proceed if a valid pointer type was passed in.  */
    9703         2474 :   if (TYPE_MAIN_VARIANT (TREE_TYPE (arg1)) == TYPE_MAIN_VARIANT (rettype))
    9704              :     {
    9705         2474 :       const REAL_VALUE_TYPE *const value = TREE_REAL_CST_PTR (arg0);
    9706         2474 :       REAL_VALUE_TYPE trunc, frac;
    9707         2474 :       tree res;
    9708              : 
    9709         2474 :       switch (value->cl)
    9710              :       {
    9711          228 :       case rvc_nan:
    9712          228 :       case rvc_zero:
    9713              :         /* For +-NaN or +-0, return (*arg1 = arg0, arg0).  */
    9714          228 :         trunc = frac = *value;
    9715          228 :         break;
    9716           96 :       case rvc_inf:
    9717              :         /* For +-Inf, return (*arg1 = arg0, +-0).  */
    9718           96 :         frac = dconst0;
    9719           96 :         frac.sign = value->sign;
    9720           96 :         trunc = *value;
    9721           96 :         break;
    9722         2150 :       case rvc_normal:
    9723              :         /* Return (*arg1 = trunc(arg0), arg0-trunc(arg0)).  */
    9724         2150 :         real_trunc (&trunc, VOIDmode, value);
    9725         2150 :         real_arithmetic (&frac, MINUS_EXPR, value, &trunc);
    9726              :         /* If the original number was negative and already
    9727              :            integral, then the fractional part is -0.0.  */
    9728         2150 :         if (value->sign && frac.cl == rvc_zero)
    9729           69 :           frac.sign = value->sign;
    9730              :         break;
    9731              :       }
    9732              : 
    9733              :       /* Create the COMPOUND_EXPR (*arg1 = trunc, frac). */
    9734         2474 :       arg1 = fold_build2_loc (loc, MODIFY_EXPR, rettype, arg1,
    9735              :                           build_real (rettype, trunc));
    9736         2474 :       TREE_SIDE_EFFECTS (arg1) = 1;
    9737         2474 :       res = fold_build2_loc (loc, COMPOUND_EXPR, rettype, arg1,
    9738              :                              build_real (rettype, frac));
    9739         2474 :       suppress_warning (res, OPT_Wunused_value);
    9740         2474 :       return res;
    9741              :     }
    9742              : 
    9743              :   return NULL_TREE;
    9744              : }
    9745              : 
    9746              : /* Given a location LOC, an interclass builtin function decl FNDECL
    9747              :    and its single argument ARG, return an folded expression computing
    9748              :    the same, or NULL_TREE if we either couldn't or didn't want to fold
    9749              :    (the latter happen if there's an RTL instruction available).  */
    9750              : 
    9751              : static tree
    9752      1815849 : fold_builtin_interclass_mathfn (location_t loc, tree fndecl, tree arg)
    9753              : {
    9754      1815849 :   machine_mode mode;
    9755              : 
    9756      1815849 :   if (!validate_arg (arg, REAL_TYPE))
    9757              :     return NULL_TREE;
    9758              : 
    9759      1815849 :   if (interclass_mathfn_icode (arg, fndecl) != CODE_FOR_nothing)
    9760              :     return NULL_TREE;
    9761              : 
    9762      1815849 :   mode = TYPE_MODE (TREE_TYPE (arg));
    9763              : 
    9764     12707101 :   bool is_ibm_extended = MODE_COMPOSITE_P (mode);
    9765              : 
    9766              :   /* If there is no optab, try generic code.  */
    9767      1815849 :   switch (DECL_FUNCTION_CODE (fndecl))
    9768              :     {
    9769       705048 :       tree result;
    9770              : 
    9771       705048 :     CASE_FLT_FN (BUILT_IN_ISINF):
    9772       705048 :       {
    9773              :         /* isinf(x) -> isgreater(fabs(x),DBL_MAX).  */
    9774       705048 :         tree const isgr_fn = builtin_decl_explicit (BUILT_IN_ISGREATER);
    9775       705048 :         tree type = TREE_TYPE (arg);
    9776       705048 :         REAL_VALUE_TYPE r;
    9777       705048 :         char buf[128];
    9778              : 
    9779       705048 :         if (is_ibm_extended)
    9780              :           {
    9781              :             /* NaN and Inf are encoded in the high-order double value
    9782              :                only.  The low-order value is not significant.  */
    9783            0 :             type = double_type_node;
    9784            0 :             mode = DFmode;
    9785            0 :             arg = fold_build1_loc (loc, NOP_EXPR, type, arg);
    9786              :           }
    9787       705048 :         get_max_float (REAL_MODE_FORMAT (mode), buf, sizeof (buf), false);
    9788       705048 :         real_from_string3 (&r, buf, mode);
    9789       705048 :         result = build_call_expr (isgr_fn, 2,
    9790              :                                   fold_build1_loc (loc, ABS_EXPR, type, arg),
    9791              :                                   build_real (type, r));
    9792       705048 :         return result;
    9793              :       }
    9794       628111 :     CASE_FLT_FN (BUILT_IN_FINITE):
    9795       628111 :     case BUILT_IN_ISFINITE:
    9796       628111 :       {
    9797              :         /* isfinite(x) -> islessequal(fabs(x),DBL_MAX).  */
    9798       628111 :         tree const isle_fn = builtin_decl_explicit (BUILT_IN_ISLESSEQUAL);
    9799       628111 :         tree type = TREE_TYPE (arg);
    9800       628111 :         REAL_VALUE_TYPE r;
    9801       628111 :         char buf[128];
    9802              : 
    9803       628111 :         if (is_ibm_extended)
    9804              :           {
    9805              :             /* NaN and Inf are encoded in the high-order double value
    9806              :                only.  The low-order value is not significant.  */
    9807            0 :             type = double_type_node;
    9808            0 :             mode = DFmode;
    9809            0 :             arg = fold_build1_loc (loc, NOP_EXPR, type, arg);
    9810              :           }
    9811       628111 :         get_max_float (REAL_MODE_FORMAT (mode), buf, sizeof (buf), false);
    9812       628111 :         real_from_string3 (&r, buf, mode);
    9813       628111 :         result = build_call_expr (isle_fn, 2,
    9814              :                                   fold_build1_loc (loc, ABS_EXPR, type, arg),
    9815              :                                   build_real (type, r));
    9816              :         /*result = fold_build2_loc (loc, UNGT_EXPR,
    9817              :                                   TREE_TYPE (TREE_TYPE (fndecl)),
    9818              :                                   fold_build1_loc (loc, ABS_EXPR, type, arg),
    9819              :                                   build_real (type, r));
    9820              :         result = fold_build1_loc (loc, TRUTH_NOT_EXPR,
    9821              :                                   TREE_TYPE (TREE_TYPE (fndecl)),
    9822              :                                   result);*/
    9823       628111 :         return result;
    9824              :       }
    9825       239084 :     case BUILT_IN_ISNORMAL:
    9826       239084 :       {
    9827              :         /* isnormal(x) -> isgreaterequal(fabs(x),DBL_MIN) &
    9828              :            islessequal(fabs(x),DBL_MAX).  */
    9829       239084 :         tree const isle_fn = builtin_decl_explicit (BUILT_IN_ISLESSEQUAL);
    9830       239084 :         tree type = TREE_TYPE (arg);
    9831       239084 :         tree orig_arg, max_exp, min_exp;
    9832       239084 :         machine_mode orig_mode = mode;
    9833       239084 :         REAL_VALUE_TYPE rmax, rmin;
    9834       239084 :         char buf[128];
    9835              : 
    9836       239084 :         orig_arg = arg = builtin_save_expr (arg);
    9837       239084 :         if (is_ibm_extended)
    9838              :           {
    9839              :             /* Use double to test the normal range of IBM extended
    9840              :                precision.  Emin for IBM extended precision is
    9841              :                different to emin for IEEE double, being 53 higher
    9842              :                since the low double exponent is at least 53 lower
    9843              :                than the high double exponent.  */
    9844            0 :             type = double_type_node;
    9845            0 :             mode = DFmode;
    9846            0 :             arg = fold_build1_loc (loc, NOP_EXPR, type, arg);
    9847              :           }
    9848       239084 :         arg = fold_build1_loc (loc, ABS_EXPR, type, arg);
    9849              : 
    9850       239084 :         get_max_float (REAL_MODE_FORMAT (mode), buf, sizeof (buf), false);
    9851       239084 :         real_from_string3 (&rmax, buf, mode);
    9852       239084 :         if (DECIMAL_FLOAT_MODE_P (mode))
    9853            1 :           sprintf (buf, "1E%d", REAL_MODE_FORMAT (orig_mode)->emin - 1);
    9854              :         else
    9855       239083 :           sprintf (buf, "0x1p%d", REAL_MODE_FORMAT (orig_mode)->emin - 1);
    9856       239084 :         real_from_string3 (&rmin, buf, orig_mode);
    9857       239084 :         max_exp = build_real (type, rmax);
    9858       239084 :         min_exp = build_real (type, rmin);
    9859              : 
    9860       239084 :         max_exp = build_call_expr (isle_fn, 2, arg, max_exp);
    9861       239084 :         if (is_ibm_extended)
    9862              :           {
    9863              :             /* Testing the high end of the range is done just using
    9864              :                the high double, using the same test as isfinite().
    9865              :                For the subnormal end of the range we first test the
    9866              :                high double, then if its magnitude is equal to the
    9867              :                limit of 0x1p-969, we test whether the low double is
    9868              :                non-zero and opposite sign to the high double.  */
    9869            0 :             tree const islt_fn = builtin_decl_explicit (BUILT_IN_ISLESS);
    9870            0 :             tree const isgt_fn = builtin_decl_explicit (BUILT_IN_ISGREATER);
    9871            0 :             tree gt_min = build_call_expr (isgt_fn, 2, arg, min_exp);
    9872            0 :             tree eq_min = fold_build2 (EQ_EXPR, integer_type_node,
    9873              :                                        arg, min_exp);
    9874            0 :             tree as_complex = build1 (VIEW_CONVERT_EXPR,
    9875              :                                       complex_double_type_node, orig_arg);
    9876            0 :             tree hi_dbl = build1 (REALPART_EXPR, type, as_complex);
    9877            0 :             tree lo_dbl = build1 (IMAGPART_EXPR, type, as_complex);
    9878            0 :             tree zero = build_real (type, dconst0);
    9879            0 :             tree hilt = build_call_expr (islt_fn, 2, hi_dbl, zero);
    9880            0 :             tree lolt = build_call_expr (islt_fn, 2, lo_dbl, zero);
    9881            0 :             tree logt = build_call_expr (isgt_fn, 2, lo_dbl, zero);
    9882            0 :             tree ok_lo = fold_build1 (TRUTH_NOT_EXPR, integer_type_node,
    9883              :                                       fold_build3 (COND_EXPR,
    9884              :                                                    integer_type_node,
    9885              :                                                    hilt, logt, lolt));
    9886            0 :             eq_min = fold_build2 (TRUTH_ANDIF_EXPR, integer_type_node,
    9887              :                                   eq_min, ok_lo);
    9888            0 :             min_exp = fold_build2 (TRUTH_ORIF_EXPR, integer_type_node,
    9889              :                                    gt_min, eq_min);
    9890              :           }
    9891              :         else
    9892              :           {
    9893       239084 :             tree const isge_fn
    9894       239084 :               = builtin_decl_explicit (BUILT_IN_ISGREATEREQUAL);
    9895       239084 :             min_exp = build_call_expr (isge_fn, 2, arg, min_exp);
    9896              :           }
    9897       239084 :         result = fold_build2 (BIT_AND_EXPR, integer_type_node,
    9898              :                               max_exp, min_exp);
    9899       239084 :         return result;
    9900              :       }
    9901       242072 :     CASE_FLT_FN (BUILT_IN_ISNAN):
    9902       242072 :     case BUILT_IN_ISNAND32:
    9903       242072 :     case BUILT_IN_ISNAND64:
    9904       242072 :     case BUILT_IN_ISNAND128:
    9905       242072 :       {
    9906              :         /* In IBM extended NaN and Inf are encoded in the high-order double
    9907              :            value only.  The low-order value is not significant.  */
    9908       242072 :         if (is_ibm_extended)
    9909            0 :           arg = fold_build1_loc (loc, NOP_EXPR, double_type_node, arg);
    9910       242072 :         arg = builtin_save_expr (arg);
    9911       242072 :         tree type = TREE_TYPE (TREE_TYPE (fndecl));
    9912       242072 :         return fold_build2_loc (loc, UNORDERED_EXPR, type, arg, arg);
    9913              :       }
    9914              :     default:
    9915              :       break;
    9916              :     }
    9917              : 
    9918              :   return NULL_TREE;
    9919              : }
    9920              : 
    9921              : /* Fold a call to __builtin_isnan(), __builtin_isinf, __builtin_finite.
    9922              :    ARG is the argument for the call.  */
    9923              : 
    9924              : static tree
    9925      1606059 : fold_builtin_classify (location_t loc, tree fndecl, tree arg, int builtin_index)
    9926              : {
    9927      1606059 :   tree type = TREE_TYPE (TREE_TYPE (fndecl));
    9928              : 
    9929      1606059 :   if (!validate_arg (arg, REAL_TYPE))
    9930              :     return NULL_TREE;
    9931              : 
    9932      1606059 :   switch (builtin_index)
    9933              :     {
    9934       706833 :     case BUILT_IN_ISINF:
    9935       706833 :       if (tree_expr_infinite_p (arg))
    9936            0 :         return omit_one_operand_loc (loc, type, integer_one_node, arg);
    9937       706833 :       if (!tree_expr_maybe_infinite_p (arg))
    9938          251 :         return omit_one_operand_loc (loc, type, integer_zero_node, arg);
    9939              :       return NULL_TREE;
    9940              : 
    9941          708 :     case BUILT_IN_ISINF_SIGN:
    9942          708 :       {
    9943              :         /* isinf_sign(x) -> isinf(x) ? (signbit(x) ? -1 : 1) : 0 */
    9944              :         /* In a boolean context, GCC will fold the inner COND_EXPR to
    9945              :            1.  So e.g. "if (isinf_sign(x))" would be folded to just
    9946              :            "if (isinf(x) ? 1 : 0)" which becomes "if (isinf(x))". */
    9947          708 :         tree signbit_fn = builtin_decl_explicit (BUILT_IN_SIGNBIT);
    9948          708 :         tree isinf_fn = builtin_decl_explicit (BUILT_IN_ISINF);
    9949          708 :         tree tmp = NULL_TREE;
    9950              : 
    9951          708 :         arg = builtin_save_expr (arg);
    9952              : 
    9953          708 :         if (signbit_fn && isinf_fn)
    9954              :           {
    9955          708 :             tree signbit_call = build_call_expr_loc (loc, signbit_fn, 1, arg);
    9956          708 :             tree isinf_call = build_call_expr_loc (loc, isinf_fn, 1, arg);
    9957              : 
    9958          708 :             signbit_call = fold_build2_loc (loc, NE_EXPR, integer_type_node,
    9959              :                                         signbit_call, integer_zero_node);
    9960          708 :             isinf_call = fold_build2_loc (loc, NE_EXPR, integer_type_node,
    9961              :                                       isinf_call, integer_zero_node);
    9962              : 
    9963          708 :             tmp = fold_build3_loc (loc, COND_EXPR, integer_type_node, signbit_call,
    9964              :                                integer_minus_one_node, integer_one_node);
    9965          708 :             tmp = fold_build3_loc (loc, COND_EXPR, integer_type_node,
    9966              :                                isinf_call, tmp,
    9967              :                                integer_zero_node);
    9968              :           }
    9969              : 
    9970              :         return tmp;
    9971              :       }
    9972              : 
    9973       628443 :     case BUILT_IN_ISFINITE:
    9974       628443 :       if (tree_expr_finite_p (arg))
    9975          332 :         return omit_one_operand_loc (loc, type, integer_one_node, arg);
    9976       628111 :       if (tree_expr_nan_p (arg) || tree_expr_infinite_p (arg))
    9977            0 :         return omit_one_operand_loc (loc, type, integer_zero_node, arg);
    9978              :       return NULL_TREE;
    9979              : 
    9980       242196 :     case BUILT_IN_ISNAN:
    9981       242196 :       if (tree_expr_nan_p (arg))
    9982            0 :         return omit_one_operand_loc (loc, type, integer_one_node, arg);
    9983       242196 :       if (!tree_expr_maybe_nan_p (arg))
    9984          124 :         return omit_one_operand_loc (loc, type, integer_zero_node, arg);
    9985              :       return NULL_TREE;
    9986              : 
    9987        27879 :     case BUILT_IN_ISSIGNALING:
    9988              :       /* Folding to true for REAL_CST is done in fold_const_call_ss.
    9989              :          Don't use tree_expr_signaling_nan_p (arg) -> integer_one_node
    9990              :          and !tree_expr_maybe_signaling_nan_p (arg) -> integer_zero_node
    9991              :          here, so there is some possibility of __builtin_issignaling working
    9992              :          without -fsignaling-nans.  Especially when -fno-signaling-nans is
    9993              :          the default.  */
    9994        27879 :       if (!tree_expr_maybe_nan_p (arg))
    9995            0 :         return omit_one_operand_loc (loc, type, integer_zero_node, arg);
    9996              :       return NULL_TREE;
    9997              : 
    9998            0 :     default:
    9999            0 :       gcc_unreachable ();
   10000              :     }
   10001              : }
   10002              : 
   10003              : /* Fold a call to __builtin_fpclassify(int, int, int, int, int, ...).
   10004              :    This builtin will generate code to return the appropriate floating
   10005              :    point classification depending on the value of the floating point
   10006              :    number passed in.  The possible return values must be supplied as
   10007              :    int arguments to the call in the following order: FP_NAN, FP_INFINITE,
   10008              :    FP_NORMAL, FP_SUBNORMAL and FP_ZERO.  The ellipses is for exactly
   10009              :    one floating point argument which is "type generic".  */
   10010              : 
   10011              : static tree
   10012       104166 : fold_builtin_fpclassify (location_t loc, tree *args, int nargs)
   10013              : {
   10014       104166 :   tree fp_nan, fp_infinite, fp_normal, fp_subnormal, fp_zero,
   10015              :     arg, type, res, tmp;
   10016       104166 :   machine_mode mode;
   10017       104166 :   REAL_VALUE_TYPE r;
   10018       104166 :   char buf[128];
   10019              : 
   10020              :   /* Verify the required arguments in the original call.  */
   10021       104166 :   if (nargs != 6
   10022       104166 :       || !validate_arg (args[0], INTEGER_TYPE)
   10023       104166 :       || !validate_arg (args[1], INTEGER_TYPE)
   10024       104166 :       || !validate_arg (args[2], INTEGER_TYPE)
   10025       104166 :       || !validate_arg (args[3], INTEGER_TYPE)
   10026       104166 :       || !validate_arg (args[4], INTEGER_TYPE)
   10027       208332 :       || !validate_arg (args[5], REAL_TYPE))
   10028              :     return NULL_TREE;
   10029              : 
   10030       104166 :   fp_nan = args[0];
   10031       104166 :   fp_infinite = args[1];
   10032       104166 :   fp_normal = args[2];
   10033       104166 :   fp_subnormal = args[3];
   10034       104166 :   fp_zero = args[4];
   10035       104166 :   arg = args[5];
   10036       104166 :   type = TREE_TYPE (arg);
   10037       104166 :   mode = TYPE_MODE (type);
   10038              : 
   10039              :   /* A target defines the classification optabs when FP comparisons are
   10040              :      unsuitable for classification, for instance because they raise
   10041              :      FE_INVALID for a signaling NaN (PR middle-end/66462).  Build on the
   10042              :      built-ins in that case rather than on comparisons.  */
   10043       104166 :   if (optab_handler (isnan_optab, mode) != CODE_FOR_nothing
   10044            0 :       && optab_handler (isinf_optab, mode) != CODE_FOR_nothing
   10045       104166 :       && optab_handler (isnormal_optab, mode) != CODE_FOR_nothing)
   10046              :     {
   10047            0 :       tree isnan_fn = builtin_decl_explicit (BUILT_IN_ISNAN);
   10048            0 :       tree isinf_fn = builtin_decl_explicit (BUILT_IN_ISINF);
   10049            0 :       tree isnormal_fn = builtin_decl_explicit (BUILT_IN_ISNORMAL);
   10050            0 :       const struct real_format *fmt = REAL_MODE_FORMAT (mode);
   10051            0 :       tree itype = NULL_TREE;
   10052            0 :       scalar_int_mode imode;
   10053              : 
   10054              :       /* The remaining zero versus subnormal test must not use a comparison
   10055              :          either.  Testing the magnitude of the encoding for zero works
   10056              :          whenever the format is binary, has a matching integer mode and
   10057              :          stores its sign in the high bit of it; that excludes e.g. XFmode,
   10058              :          whose encoding is narrower than its integer mode, and the composite
   10059              :          formats, whose sign is not a single bit that can be shifted out.  */
   10060            0 :       if (fmt->b == 2
   10061            0 :           && int_mode_for_mode (mode).exists (&imode)
   10062            0 :           && fmt->signbit_rw == fmt->signbit_ro
   10063            0 :           && fmt->signbit_rw == (int) GET_MODE_PRECISION (imode) - 1)
   10064              :         {
   10065            0 :           itype = lang_hooks.types.type_for_mode (imode, 1);
   10066            0 :           if (itype && TYPE_PRECISION (itype) != GET_MODE_PRECISION (imode))
   10067              :             itype = NULL_TREE;
   10068              :         }
   10069              : 
   10070            0 :       if (isnan_fn && isinf_fn && isnormal_fn && itype)
   10071              :         {
   10072            0 :           arg = builtin_save_expr (arg);
   10073              : 
   10074              :           /* Shifting the sign bit out leaves zero iff ARG is +-0.  */
   10075            0 :           tmp = fold_build1_loc (loc, VIEW_CONVERT_EXPR, itype, arg);
   10076            0 :           tmp = fold_build2_loc (loc, LSHIFT_EXPR, itype, tmp,
   10077              :                                  build_int_cst (integer_type_node, 1));
   10078            0 :           tmp = fold_build2_loc (loc, EQ_EXPR, integer_type_node, tmp,
   10079              :                                  build_int_cst (itype, 0));
   10080            0 :           res = fold_build3_loc (loc, COND_EXPR, integer_type_node,
   10081              :                                  tmp, fp_zero, fp_subnormal);
   10082              : 
   10083            0 :           tmp = build_call_expr_loc (loc, isnormal_fn, 1, arg);
   10084            0 :           tmp = fold_build2_loc (loc, NE_EXPR, integer_type_node, tmp,
   10085              :                                  integer_zero_node);
   10086            0 :           res = fold_build3_loc (loc, COND_EXPR, integer_type_node, tmp,
   10087              :                                  fp_normal, res);
   10088              : 
   10089            0 :           if (tree_expr_maybe_infinite_p (arg))
   10090              :             {
   10091            0 :               tmp = build_call_expr_loc (loc, isinf_fn, 1, arg);
   10092            0 :               tmp = fold_build2_loc (loc, NE_EXPR, integer_type_node, tmp,
   10093              :                                      integer_zero_node);
   10094            0 :               res = fold_build3_loc (loc, COND_EXPR, integer_type_node, tmp,
   10095              :                                      fp_infinite, res);
   10096              :             }
   10097              : 
   10098            0 :           if (tree_expr_maybe_nan_p (arg))
   10099              :             {
   10100            0 :               tmp = build_call_expr_loc (loc, isnan_fn, 1, arg);
   10101            0 :               tmp = fold_build2_loc (loc, NE_EXPR, integer_type_node, tmp,
   10102              :                                      integer_zero_node);
   10103            0 :               res = fold_build3_loc (loc, COND_EXPR, integer_type_node, tmp,
   10104              :                                      fp_nan, res);
   10105              :             }
   10106              : 
   10107       104166 :           return res;
   10108              :         }
   10109              :     }
   10110              : 
   10111       104166 :   arg = builtin_save_expr (fold_build1_loc (loc, ABS_EXPR, type, arg));
   10112              : 
   10113              :   /* fpclassify(x) ->
   10114              :        isnan(x) ? FP_NAN :
   10115              :          (fabs(x) == Inf ? FP_INFINITE :
   10116              :            (fabs(x) >= DBL_MIN ? FP_NORMAL :
   10117              :              (x == 0 ? FP_ZERO : FP_SUBNORMAL))).  */
   10118              : 
   10119       104166 :   tmp = fold_build2_loc (loc, EQ_EXPR, integer_type_node, arg,
   10120              :                          build_real (type, dconst0));
   10121       104166 :   res = fold_build3_loc (loc, COND_EXPR, integer_type_node,
   10122              :                          tmp, fp_zero, fp_subnormal);
   10123              : 
   10124       104166 :   if (DECIMAL_FLOAT_MODE_P (mode))
   10125            3 :     sprintf (buf, "1E%d", REAL_MODE_FORMAT (mode)->emin - 1);
   10126              :   else
   10127       104163 :     sprintf (buf, "0x1p%d", REAL_MODE_FORMAT (mode)->emin - 1);
   10128       104166 :   real_from_string3 (&r, buf, mode);
   10129       104166 :   tmp = fold_build2_loc (loc, GE_EXPR, integer_type_node,
   10130              :                          arg, build_real (type, r));
   10131       104166 :   res = fold_build3_loc (loc, COND_EXPR, integer_type_node, tmp,
   10132              :                          fp_normal, res);
   10133              : 
   10134       104166 :   if (tree_expr_maybe_infinite_p (arg))
   10135              :     {
   10136       104078 :       tmp = fold_build2_loc (loc, EQ_EXPR, integer_type_node, arg,
   10137              :                              build_real (type, dconstinf));
   10138       104078 :       res = fold_build3_loc (loc, COND_EXPR, integer_type_node, tmp,
   10139              :                              fp_infinite, res);
   10140              :     }
   10141              : 
   10142       104166 :   if (tree_expr_maybe_nan_p (arg))
   10143              :     {
   10144       104076 :       tmp = fold_build2_loc (loc, ORDERED_EXPR, integer_type_node, arg, arg);
   10145       104076 :       res = fold_build3_loc (loc, COND_EXPR, integer_type_node, tmp,
   10146              :                              res, fp_nan);
   10147              :     }
   10148              : 
   10149              :   return res;
   10150              : }
   10151              : 
   10152              : /* Fold a call to an unordered comparison function such as
   10153              :    __builtin_isgreater().  FNDECL is the FUNCTION_DECL for the function
   10154              :    being called and ARG0 and ARG1 are the arguments for the call.
   10155              :    UNORDERED_CODE and ORDERED_CODE are comparison codes that give
   10156              :    the opposite of the desired result.  UNORDERED_CODE is used
   10157              :    for modes that can hold NaNs and ORDERED_CODE is used for
   10158              :    the rest.  */
   10159              : 
   10160              : static tree
   10161      3250321 : fold_builtin_unordered_cmp (location_t loc, tree fndecl, tree arg0, tree arg1,
   10162              :                             enum tree_code unordered_code,
   10163              :                             enum tree_code ordered_code)
   10164              : {
   10165      3250321 :   tree type = TREE_TYPE (TREE_TYPE (fndecl));
   10166      3250321 :   enum tree_code code;
   10167      3250321 :   tree type0, type1;
   10168      3250321 :   enum tree_code code0, code1;
   10169      3250321 :   tree cmp_type = NULL_TREE;
   10170              : 
   10171      3250321 :   type0 = TREE_TYPE (arg0);
   10172      3250321 :   type1 = TREE_TYPE (arg1);
   10173              : 
   10174      3250321 :   code0 = TREE_CODE (type0);
   10175      3250321 :   code1 = TREE_CODE (type1);
   10176              : 
   10177      3250321 :   if (code0 == REAL_TYPE && code1 == REAL_TYPE)
   10178              :     /* Choose the wider of two real types.  */
   10179      3250141 :     cmp_type = TYPE_PRECISION (type0) >= TYPE_PRECISION (type1)
   10180      3250141 :       ? type0 : type1;
   10181          180 :   else if (code0 == REAL_TYPE
   10182           91 :            && (code1 == INTEGER_TYPE || code1 == BITINT_TYPE))
   10183              :     cmp_type = type0;
   10184           89 :   else if ((code0 == INTEGER_TYPE || code0 == BITINT_TYPE)
   10185           89 :            && code1 == REAL_TYPE)
   10186          135 :     cmp_type = type1;
   10187              : 
   10188      3250321 :   arg0 = fold_convert_loc (loc, cmp_type, arg0);
   10189      3250321 :   arg1 = fold_convert_loc (loc, cmp_type, arg1);
   10190              : 
   10191      3250321 :   if (unordered_code == UNORDERED_EXPR)
   10192              :     {
   10193       240131 :       if (tree_expr_nan_p (arg0) || tree_expr_nan_p (arg1))
   10194           16 :         return omit_two_operands_loc (loc, type, integer_one_node, arg0, arg1);
   10195       240115 :       if (!tree_expr_maybe_nan_p (arg0) && !tree_expr_maybe_nan_p (arg1))
   10196          141 :         return omit_two_operands_loc (loc, type, integer_zero_node, arg0, arg1);
   10197       239974 :       return fold_build2_loc (loc, UNORDERED_EXPR, type, arg0, arg1);
   10198              :     }
   10199              : 
   10200      3013041 :   code = (tree_expr_maybe_nan_p (arg0) || tree_expr_maybe_nan_p (arg1))
   10201      3010190 :          ? unordered_code : ordered_code;
   10202      3010190 :   return fold_build1_loc (loc, TRUTH_NOT_EXPR, type,
   10203      3010190 :                       fold_build2_loc (loc, code, type, arg0, arg1));
   10204              : }
   10205              : 
   10206              : /* Fold a call to __builtin_iseqsig().  ARG0 and ARG1 are the arguments.
   10207              :    After choosing the wider floating-point type for the comparison,
   10208              :    the code is folded to:
   10209              :      SAVE_EXPR<ARG0> >= SAVE_EXPR<ARG1> && SAVE_EXPR<ARG0> <= SAVE_EXPR<ARG1>  */
   10210              : 
   10211              : static tree
   10212          709 : fold_builtin_iseqsig (location_t loc, tree arg0, tree arg1)
   10213              : {
   10214          709 :   tree type0, type1;
   10215          709 :   enum tree_code code0, code1;
   10216          709 :   tree cmp1, cmp2, cmp_type = NULL_TREE;
   10217              : 
   10218          709 :   type0 = TREE_TYPE (arg0);
   10219          709 :   type1 = TREE_TYPE (arg1);
   10220              : 
   10221          709 :   code0 = TREE_CODE (type0);
   10222          709 :   code1 = TREE_CODE (type1);
   10223              : 
   10224          709 :   if (code0 == REAL_TYPE && code1 == REAL_TYPE)
   10225              :     /* Choose the wider of two real types.  */
   10226          690 :     cmp_type = TYPE_PRECISION (type0) >= TYPE_PRECISION (type1)
   10227          690 :       ? type0 : type1;
   10228           19 :   else if (code0 == REAL_TYPE
   10229            6 :            && (code1 == INTEGER_TYPE || code1 == BITINT_TYPE))
   10230              :     cmp_type = type0;
   10231           13 :   else if ((code0 == INTEGER_TYPE || code0 == BITINT_TYPE)
   10232           13 :            && code1 == REAL_TYPE)
   10233           13 :     cmp_type = type1;
   10234              : 
   10235          709 :   arg0 = builtin_save_expr (fold_convert_loc (loc, cmp_type, arg0));
   10236          709 :   arg1 = builtin_save_expr (fold_convert_loc (loc, cmp_type, arg1));
   10237              : 
   10238          709 :   cmp1 = fold_build2_loc (loc, GE_EXPR, integer_type_node, arg0, arg1);
   10239          709 :   cmp2 = fold_build2_loc (loc, LE_EXPR, integer_type_node, arg0, arg1);
   10240              : 
   10241          709 :   return fold_build2_loc (loc, TRUTH_AND_EXPR, integer_type_node, cmp1, cmp2);
   10242              : }
   10243              : 
   10244              : /* Fold __builtin_{,s,u}{add,sub,mul}{,l,ll}_overflow, either into normal
   10245              :    arithmetics if it can never overflow, or into internal functions that
   10246              :    return both result of arithmetics and overflowed boolean flag in
   10247              :    a complex integer result, or some other check for overflow.
   10248              :    Similarly fold __builtin_{add,sub,mul}_overflow_p to just the overflow
   10249              :    checking part of that.  */
   10250              : 
   10251              : static tree
   10252       503397 : fold_builtin_arith_overflow (location_t loc, enum built_in_function fcode,
   10253              :                              tree arg0, tree arg1, tree arg2)
   10254              : {
   10255       503397 :   enum internal_fn ifn = IFN_LAST;
   10256              :   /* The code of the expression corresponding to the built-in.  */
   10257       503397 :   enum tree_code opcode = ERROR_MARK;
   10258       503397 :   bool ovf_only = false;
   10259              : 
   10260       503397 :   switch (fcode)
   10261              :     {
   10262              :     case BUILT_IN_ADD_OVERFLOW_P:
   10263              :       ovf_only = true;
   10264              :       /* FALLTHRU */
   10265              :     case BUILT_IN_ADD_OVERFLOW:
   10266              :     case BUILT_IN_SADD_OVERFLOW:
   10267              :     case BUILT_IN_SADDL_OVERFLOW:
   10268              :     case BUILT_IN_SADDLL_OVERFLOW:
   10269              :     case BUILT_IN_UADD_OVERFLOW:
   10270              :     case BUILT_IN_UADDL_OVERFLOW:
   10271              :     case BUILT_IN_UADDLL_OVERFLOW:
   10272              :       opcode = PLUS_EXPR;
   10273              :       ifn = IFN_ADD_OVERFLOW;
   10274              :       break;
   10275        15591 :     case BUILT_IN_SUB_OVERFLOW_P:
   10276        15591 :       ovf_only = true;
   10277              :       /* FALLTHRU */
   10278        37647 :     case BUILT_IN_SUB_OVERFLOW:
   10279        37647 :     case BUILT_IN_SSUB_OVERFLOW:
   10280        37647 :     case BUILT_IN_SSUBL_OVERFLOW:
   10281        37647 :     case BUILT_IN_SSUBLL_OVERFLOW:
   10282        37647 :     case BUILT_IN_USUB_OVERFLOW:
   10283        37647 :     case BUILT_IN_USUBL_OVERFLOW:
   10284        37647 :     case BUILT_IN_USUBLL_OVERFLOW:
   10285        37647 :       opcode = MINUS_EXPR;
   10286        37647 :       ifn = IFN_SUB_OVERFLOW;
   10287        37647 :       break;
   10288        15878 :     case BUILT_IN_MUL_OVERFLOW_P:
   10289        15878 :       ovf_only = true;
   10290              :       /* FALLTHRU */
   10291       426226 :     case BUILT_IN_MUL_OVERFLOW:
   10292       426226 :     case BUILT_IN_SMUL_OVERFLOW:
   10293       426226 :     case BUILT_IN_SMULL_OVERFLOW:
   10294       426226 :     case BUILT_IN_SMULLL_OVERFLOW:
   10295       426226 :     case BUILT_IN_UMUL_OVERFLOW:
   10296       426226 :     case BUILT_IN_UMULL_OVERFLOW:
   10297       426226 :     case BUILT_IN_UMULLL_OVERFLOW:
   10298       426226 :       opcode = MULT_EXPR;
   10299       426226 :       ifn = IFN_MUL_OVERFLOW;
   10300       426226 :       break;
   10301            0 :     default:
   10302            0 :       gcc_unreachable ();
   10303              :     }
   10304              : 
   10305              :   /* For the "generic" overloads, the first two arguments can have different
   10306              :      types and the last argument determines the target type to use to check
   10307              :      for overflow.  The arguments of the other overloads all have the same
   10308              :      type.  */
   10309       503397 :   tree type = ovf_only ? TREE_TYPE (arg2) : TREE_TYPE (TREE_TYPE (arg2));
   10310              : 
   10311              :   /* For the __builtin_{add,sub,mul}_overflow_p builtins, when the first two
   10312              :      arguments are constant, attempt to fold the built-in call into a constant
   10313              :      expression indicating whether or not it detected an overflow.  */
   10314       503397 :   if (ovf_only
   10315        43279 :       && TREE_CODE (arg0) == INTEGER_CST
   10316        13131 :       && TREE_CODE (arg1) == INTEGER_CST)
   10317              :     /* Perform the computation in the target type and check for overflow.  */
   10318         7347 :     return omit_one_operand_loc (loc, boolean_type_node,
   10319         7347 :                                  arith_overflowed_p (opcode, type, arg0, arg1)
   10320              :                                  ? boolean_true_node : boolean_false_node,
   10321         7347 :                                  arg2);
   10322              : 
   10323       496050 :   tree intres, ovfres;
   10324       496050 :   if (TREE_CODE (arg0) == INTEGER_CST && TREE_CODE (arg1) == INTEGER_CST)
   10325              :     {
   10326        89138 :       intres = fold_binary_loc (loc, opcode, type,
   10327              :                                 fold_convert_loc (loc, type, arg0),
   10328              :                                 fold_convert_loc (loc, type, arg1));
   10329        89138 :       if (TREE_OVERFLOW (intres))
   10330         1629 :         intres = drop_tree_overflow (intres);
   10331       178276 :       ovfres = (arith_overflowed_p (opcode, type, arg0, arg1)
   10332        89138 :                 ? boolean_true_node : boolean_false_node);
   10333              :     }
   10334              :   else
   10335              :     {
   10336       406912 :       tree ctype = build_complex_type (type);
   10337       406912 :       tree call = build_call_expr_internal_loc (loc, ifn, ctype, 2,
   10338              :                                                 arg0, arg1);
   10339       406912 :       tree tgt;
   10340       406912 :       if (ovf_only)
   10341              :         {
   10342              :           tgt = call;
   10343       406912 :           intres = NULL_TREE;
   10344              :         }
   10345              :       else
   10346              :         {
   10347              :           /* Force SAVE_EXPR even for calls which satisfy tree_invariant_p_1,
   10348              :              as while the call itself is const, the REALPART_EXPR store is
   10349              :              certainly not.  And in any case, we want just one call,
   10350              :              not multiple and trying to CSE them later.  */
   10351       370980 :           TREE_SIDE_EFFECTS (call) = 1;
   10352       370980 :           tgt = save_expr (call);
   10353              :         }
   10354       406912 :       intres = build1_loc (loc, REALPART_EXPR, type, tgt);
   10355       406912 :       ovfres = build1_loc (loc, IMAGPART_EXPR, type, tgt);
   10356       406912 :       ovfres = fold_convert_loc (loc, boolean_type_node, ovfres);
   10357              :     }
   10358              : 
   10359       496050 :   if (ovf_only)
   10360        35932 :     return omit_one_operand_loc (loc, boolean_type_node, ovfres, arg2);
   10361              : 
   10362       460118 :   tree mem_arg2 = build_fold_indirect_ref_loc (loc, arg2);
   10363       460118 :   tree store
   10364       460118 :     = fold_build2_loc (loc, MODIFY_EXPR, void_type_node, mem_arg2, intres);
   10365       460118 :   return build2_loc (loc, COMPOUND_EXPR, boolean_type_node, store, ovfres);
   10366              : }
   10367              : 
   10368              : /* Fold __builtin_{clz,ctz,clrsb,ffs,parity,popcount}g into corresponding
   10369              :    internal function.  */
   10370              : 
   10371              : static tree
   10372       262666 : fold_builtin_bit_query (location_t loc, enum built_in_function fcode,
   10373              :                         tree arg0, tree arg1)
   10374              : {
   10375       262666 :   enum internal_fn ifn;
   10376       262666 :   enum built_in_function fcodei, fcodel, fcodell;
   10377       262666 :   tree arg0_type = TREE_TYPE (arg0);
   10378       262666 :   tree cast_type = NULL_TREE;
   10379       262666 :   int addend = 0;
   10380              : 
   10381       262666 :   switch (fcode)
   10382              :     {
   10383       184363 :     case BUILT_IN_CLZG:
   10384       184363 :       if (arg1 && TREE_CODE (arg1) != INTEGER_CST)
   10385              :         return NULL_TREE;
   10386              :       ifn = IFN_CLZ;
   10387              :       fcodei = BUILT_IN_CLZ;
   10388              :       fcodel = BUILT_IN_CLZL;
   10389              :       fcodell = BUILT_IN_CLZLL;
   10390              :       break;
   10391        66764 :     case BUILT_IN_CTZG:
   10392        66764 :       if (arg1 && TREE_CODE (arg1) != INTEGER_CST)
   10393              :         return NULL_TREE;
   10394              :       ifn = IFN_CTZ;
   10395              :       fcodei = BUILT_IN_CTZ;
   10396              :       fcodel = BUILT_IN_CTZL;
   10397              :       fcodell = BUILT_IN_CTZLL;
   10398              :       break;
   10399              :     case BUILT_IN_CLRSBG:
   10400              :       ifn = IFN_CLRSB;
   10401              :       fcodei = BUILT_IN_CLRSB;
   10402              :       fcodel = BUILT_IN_CLRSBL;
   10403              :       fcodell = BUILT_IN_CLRSBLL;
   10404              :       break;
   10405           84 :     case BUILT_IN_FFSG:
   10406           84 :       ifn = IFN_FFS;
   10407           84 :       fcodei = BUILT_IN_FFS;
   10408           84 :       fcodel = BUILT_IN_FFSL;
   10409           84 :       fcodell = BUILT_IN_FFSLL;
   10410           84 :       break;
   10411           82 :     case BUILT_IN_PARITYG:
   10412           82 :       ifn = IFN_PARITY;
   10413           82 :       fcodei = BUILT_IN_PARITY;
   10414           82 :       fcodel = BUILT_IN_PARITYL;
   10415           82 :       fcodell = BUILT_IN_PARITYLL;
   10416           82 :       break;
   10417        11291 :     case BUILT_IN_POPCOUNTG:
   10418        11291 :       ifn = IFN_POPCOUNT;
   10419        11291 :       fcodei = BUILT_IN_POPCOUNT;
   10420        11291 :       fcodel = BUILT_IN_POPCOUNTL;
   10421        11291 :       fcodell = BUILT_IN_POPCOUNTLL;
   10422        11291 :       break;
   10423            0 :     default:
   10424            0 :       gcc_unreachable ();
   10425              :     }
   10426              : 
   10427       262463 :   if (TYPE_PRECISION (arg0_type)
   10428       262463 :       <= TYPE_PRECISION (long_long_unsigned_type_node))
   10429              :     {
   10430       224411 :       if (TYPE_PRECISION (arg0_type) <= TYPE_PRECISION (unsigned_type_node))
   10431              : 
   10432        79399 :         cast_type = (TYPE_UNSIGNED (arg0_type)
   10433        79345 :                      ? unsigned_type_node : integer_type_node);
   10434       145066 :       else if (TYPE_PRECISION (arg0_type)
   10435       145066 :                <= TYPE_PRECISION (long_unsigned_type_node))
   10436              :         {
   10437       145015 :           cast_type = (TYPE_UNSIGNED (arg0_type)
   10438       144943 :                        ? long_unsigned_type_node : long_integer_type_node);
   10439              :           fcodei = fcodel;
   10440              :         }
   10441              :       else
   10442              :         {
   10443          123 :           cast_type = (TYPE_UNSIGNED (arg0_type)
   10444          123 :                        ? long_long_unsigned_type_node
   10445              :                        : long_long_integer_type_node);
   10446              :           fcodei = fcodell;
   10447              :         }
   10448              :     }
   10449        76104 :   else if (TYPE_PRECISION (arg0_type) <= MAX_FIXED_MODE_SIZE)
   10450              :     {
   10451        37942 :       cast_type
   10452        37942 :         = build_nonstandard_integer_type (MAX_FIXED_MODE_SIZE,
   10453        37942 :                                           TYPE_UNSIGNED (arg0_type));
   10454        37942 :       gcc_assert (TYPE_PRECISION (cast_type)
   10455              :                   == 2 * TYPE_PRECISION (long_long_unsigned_type_node));
   10456              :       fcodei = END_BUILTINS;
   10457              :     }
   10458              :   else
   10459              :     fcodei = END_BUILTINS;
   10460       262353 :   if (cast_type)
   10461              :     {
   10462       262353 :       switch (fcode)
   10463              :         {
   10464       184312 :         case BUILT_IN_CLZG:
   10465       184312 :         case BUILT_IN_CLRSBG:
   10466       184312 :           addend = TYPE_PRECISION (arg0_type) - TYPE_PRECISION (cast_type);
   10467       184312 :           break;
   10468              :         default:
   10469              :           break;
   10470              :         }
   10471       262353 :       arg0 = fold_convert (cast_type, arg0);
   10472       262353 :       arg0_type = cast_type;
   10473              :     }
   10474              : 
   10475       262463 :   if (arg1)
   10476       174934 :     arg1 = fold_convert (integer_type_node, arg1);
   10477              : 
   10478       262463 :   tree arg2 = arg1;
   10479       262463 :   if (fcode == BUILT_IN_CLZG && addend)
   10480              :     {
   10481         4413 :       if (arg1)
   10482         4380 :         arg0 = save_expr (arg0);
   10483              :       arg2 = NULL_TREE;
   10484              :     }
   10485       262463 :   tree call = NULL_TREE, tem;
   10486       262463 :   if (TYPE_PRECISION (arg0_type) == MAX_FIXED_MODE_SIZE
   10487        38065 :       && (TYPE_PRECISION (arg0_type)
   10488        38065 :           == 2 * TYPE_PRECISION (long_long_unsigned_type_node))
   10489              :       /* If the target supports the optab, then don't do the expansion. */
   10490       300405 :       && !direct_internal_fn_supported_p (ifn, arg0_type, OPTIMIZE_FOR_BOTH))
   10491              :     {
   10492              :       /* __int128 expansions using up to 2 long long builtins.  */
   10493        37942 :       arg0 = save_expr (arg0);
   10494        37942 :       tree type = (TYPE_UNSIGNED (arg0_type)
   10495        37942 :                    ? long_long_unsigned_type_node
   10496        37942 :                    : long_long_integer_type_node);
   10497        75884 :       tree hi = fold_build2 (RSHIFT_EXPR, arg0_type, arg0,
   10498              :                              build_int_cst (integer_type_node,
   10499              :                                             MAX_FIXED_MODE_SIZE / 2));
   10500        37942 :       hi = fold_convert (type, hi);
   10501        37942 :       tree lo = fold_convert (type, arg0);
   10502        37942 :       switch (fcode)
   10503              :         {
   10504        37853 :         case BUILT_IN_CLZG:
   10505        37853 :           call = fold_builtin_bit_query (loc, fcode, lo, NULL_TREE);
   10506        75706 :           call = fold_build2 (PLUS_EXPR, integer_type_node, call,
   10507              :                               build_int_cst (integer_type_node,
   10508              :                                              MAX_FIXED_MODE_SIZE / 2));
   10509        37853 :           if (arg2)
   10510        37833 :             call = fold_build3 (COND_EXPR, integer_type_node,
   10511              :                                 fold_build2 (NE_EXPR, boolean_type_node,
   10512              :                                              lo, build_zero_cst (type)),
   10513              :                                 call, arg2);
   10514        37853 :           call = fold_build3 (COND_EXPR, integer_type_node,
   10515              :                               fold_build2 (NE_EXPR, boolean_type_node,
   10516              :                                            hi, build_zero_cst (type)),
   10517              :                               fold_builtin_bit_query (loc, fcode, hi,
   10518              :                                                       NULL_TREE),
   10519              :                               call);
   10520        37853 :           break;
   10521           27 :         case BUILT_IN_CTZG:
   10522           27 :           call = fold_builtin_bit_query (loc, fcode, hi, NULL_TREE);
   10523           54 :           call = fold_build2 (PLUS_EXPR, integer_type_node, call,
   10524              :                               build_int_cst (integer_type_node,
   10525              :                                              MAX_FIXED_MODE_SIZE / 2));
   10526           27 :           if (arg2)
   10527           18 :             call = fold_build3 (COND_EXPR, integer_type_node,
   10528              :                                 fold_build2 (NE_EXPR, boolean_type_node,
   10529              :                                              hi, build_zero_cst (type)),
   10530              :                                 call, arg2);
   10531           27 :           call = fold_build3 (COND_EXPR, integer_type_node,
   10532              :                               fold_build2 (NE_EXPR, boolean_type_node,
   10533              :                                            lo, build_zero_cst (type)),
   10534              :                               fold_builtin_bit_query (loc, fcode, lo,
   10535              :                                                       NULL_TREE),
   10536              :                               call);
   10537           27 :           break;
   10538            9 :         case BUILT_IN_CLRSBG:
   10539            9 :           tem = fold_builtin_bit_query (loc, fcode, lo, NULL_TREE);
   10540           18 :           tem = fold_build2 (PLUS_EXPR, integer_type_node, tem,
   10541              :                              build_int_cst (integer_type_node,
   10542              :                                             MAX_FIXED_MODE_SIZE / 2));
   10543           18 :           tem = fold_build3 (COND_EXPR, integer_type_node,
   10544              :                              fold_build2 (LT_EXPR, boolean_type_node,
   10545              :                                           fold_build2 (BIT_XOR_EXPR, type,
   10546              :                                                        lo, hi),
   10547              :                                           build_zero_cst (type)),
   10548              :                              build_int_cst (integer_type_node,
   10549              :                                             MAX_FIXED_MODE_SIZE / 2 - 1),
   10550              :                              tem);
   10551            9 :           call = fold_builtin_bit_query (loc, fcode, hi, NULL_TREE);
   10552            9 :           call = save_expr (call);
   10553           18 :           call = fold_build3 (COND_EXPR, integer_type_node,
   10554              :                               fold_build2 (NE_EXPR, boolean_type_node,
   10555              :                                            call,
   10556              :                                            build_int_cst (integer_type_node,
   10557              :                                                           MAX_FIXED_MODE_SIZE
   10558              :                                                           / 2 - 1)),
   10559              :                               call, tem);
   10560            9 :           break;
   10561            9 :         case BUILT_IN_FFSG:
   10562            9 :           call = fold_builtin_bit_query (loc, fcode, hi, NULL_TREE);
   10563           18 :           call = fold_build2 (PLUS_EXPR, integer_type_node, call,
   10564              :                               build_int_cst (integer_type_node,
   10565              :                                              MAX_FIXED_MODE_SIZE / 2));
   10566            9 :           call = fold_build3 (COND_EXPR, integer_type_node,
   10567              :                               fold_build2 (NE_EXPR, boolean_type_node,
   10568              :                                            hi, build_zero_cst (type)),
   10569              :                               call, integer_zero_node);
   10570            9 :           call = fold_build3 (COND_EXPR, integer_type_node,
   10571              :                               fold_build2 (NE_EXPR, boolean_type_node,
   10572              :                                            lo, build_zero_cst (type)),
   10573              :                               fold_builtin_bit_query (loc, fcode, lo,
   10574              :                                                       NULL_TREE),
   10575              :                               call);
   10576            9 :           break;
   10577            9 :         case BUILT_IN_PARITYG:
   10578            9 :           call = fold_builtin_bit_query (loc, fcode,
   10579              :                                          fold_build2 (BIT_XOR_EXPR, type,
   10580              :                                                       lo, hi), NULL_TREE);
   10581            9 :           break;
   10582           35 :         case BUILT_IN_POPCOUNTG:
   10583           35 :           call = fold_build2 (PLUS_EXPR, integer_type_node,
   10584              :                               fold_builtin_bit_query (loc, fcode, hi,
   10585              :                                                       NULL_TREE),
   10586              :                               fold_builtin_bit_query (loc, fcode, lo,
   10587              :                                                       NULL_TREE));
   10588           35 :           break;
   10589              :         default:
   10590              :           gcc_unreachable ();
   10591              :         }
   10592              :     }
   10593              :   else
   10594              :     {
   10595              :       /* Only keep second argument to IFN_CLZ/IFN_CTZ if it is the
   10596              :          value defined at zero during GIMPLE, or for large/huge _BitInt
   10597              :          (which are then lowered during bitint lowering).  */
   10598       224521 :       if (arg2 && !BITINT_TYPE_P (TREE_TYPE (arg0)))
   10599              :         {
   10600       132683 :           int val;
   10601       132683 :           if (fcode == BUILT_IN_CLZG)
   10602              :             {
   10603        66213 :               if (CLZ_DEFINED_VALUE_AT_ZERO (SCALAR_TYPE_MODE (arg0_type),
   10604              :                                              val) != 2
   10605        66213 :                   || wi::to_widest (arg2) != val)
   10606              :                 arg2 = NULL_TREE;
   10607              :             }
   10608        66470 :           else if (CTZ_DEFINED_VALUE_AT_ZERO (SCALAR_TYPE_MODE (arg0_type),
   10609              :                                               val) != 2
   10610        66470 :                    || wi::to_widest (arg2) != val)
   10611              :             arg2 = NULL_TREE;
   10612       132683 :           if (!direct_internal_fn_supported_p (ifn, arg0_type,
   10613              :                                                OPTIMIZE_FOR_BOTH))
   10614              :             arg2 = NULL_TREE;
   10615       132572 :           if (arg2 == NULL_TREE)
   10616       132635 :             arg0 = save_expr (arg0);
   10617              :         }
   10618       224521 :       if (fcodei == END_BUILTINS || arg2)
   10619          248 :         call = build_call_expr_internal_loc (loc, ifn, integer_type_node,
   10620              :                                              arg2 ? 2 : 1, arg0, arg2);
   10621              :       else
   10622       224363 :         call = build_call_expr_loc (loc, builtin_decl_explicit (fcodei), 1,
   10623              :                                     arg0);
   10624              :     }
   10625       262463 :   if (addend)
   10626         4431 :     call = fold_build2 (PLUS_EXPR, integer_type_node, call,
   10627              :                         build_int_cst (integer_type_node, addend));
   10628       262463 :   if (arg1 && arg2 == NULL_TREE)
   10629       137015 :     call = fold_build3 (COND_EXPR, integer_type_node,
   10630              :                         fold_build2 (NE_EXPR, boolean_type_node,
   10631              :                                      arg0, build_zero_cst (arg0_type)),
   10632              :                         call, arg1);
   10633              : 
   10634              :   return call;
   10635              : }
   10636              : 
   10637              : /* Fold or build a __builtin_bswapg (ARG) (if IFN is IFN_BSWAP) or
   10638              :    __builtin_bitreverseg (ARG (otherwise) call.  The FE should have
   10639              :    verified earlier that the argument type is unsigned INTEGER_TYPE
   10640              :    or BITINT_TYPE, for __builtin_bswapg with precision divisible by 8.  */
   10641              : 
   10642              : tree
   10643          218 : fold_build_builtin_bswapg_bitreverseg (location_t loc, enum internal_fn ifn,
   10644              :                                        tree arg)
   10645              : {
   10646          218 :   tree type = TYPE_MAIN_VARIANT (TREE_TYPE (arg));
   10647          218 :   if (ifn == IFN_BSWAP && TYPE_PRECISION (type) == 8)
   10648           11 :     return fold_convert_loc (loc, type, arg);
   10649          207 :   struct { tree type; built_in_function bswap, bitreverse; } fns[]
   10650          207 :     = { { unsigned_char_type_node, END_BUILTINS, BUILT_IN_BITREVERSE8 },
   10651          207 :         { uint16_type_node, BUILT_IN_BSWAP16, BUILT_IN_BITREVERSE16 },
   10652          207 :         { uint32_type_node, BUILT_IN_BSWAP32, BUILT_IN_BITREVERSE32 },
   10653          207 :         { uint64_type_node, BUILT_IN_BSWAP64, BUILT_IN_BITREVERSE64 },
   10654          207 :         { uint128_type_node, BUILT_IN_BSWAP128, BUILT_IN_BITREVERSE128 } };
   10655          207 :   if (TREE_CODE (arg) == INTEGER_CST)
   10656              :     {
   10657          127 :       wide_int res;
   10658          127 :       if (ifn == IFN_BSWAP)
   10659           37 :         res = wi::bswap (wi::to_wide (arg));
   10660              :       else
   10661           96 :         res = wi::bitreverse (wi::to_wide (arg));
   10662          127 :       return wide_int_to_tree (type, res);
   10663          127 :     }
   10664          346 :   for (unsigned i = 0; i < ARRAY_SIZE (fns); ++i)
   10665          314 :     if (fns[i].type
   10666          314 :         && TYPE_PRECISION (fns[i].type) >= TYPE_PRECISION (type))
   10667              :       {
   10668           48 :         arg = fold_convert_loc (loc, fns[i].type, arg);
   10669           48 :         tree fndecl
   10670           48 :           = builtin_decl_explicit (ifn == IFN_BSWAP
   10671              :                                    ? fns[i].bswap : fns[i].bitreverse);
   10672           48 :         tree ret = build_call_expr_loc (loc, fndecl, 1, arg);
   10673           48 :         if (TYPE_PRECISION (type) != TYPE_PRECISION (fns[i].type))
   10674            4 :           ret = fold_build2_loc (loc, RSHIFT_EXPR, fns[i].type, ret,
   10675              :                                  build_int_cst (unsigned_type_node,
   10676            4 :                                                 TYPE_PRECISION (fns[i].type)
   10677            4 :                                                 - TYPE_PRECISION (type)));
   10678           48 :         return fold_convert_loc (loc, type, ret);
   10679              :       }
   10680           32 :   gcc_assert (TREE_CODE (type) == BITINT_TYPE);
   10681           32 :   return build_call_expr_internal_loc (loc, ifn, type, 1, arg);
   10682              : }
   10683              : 
   10684              : /* Fold __builtin_{add,sub}c{,l,ll} into pair of internal functions
   10685              :    that return both result of arithmetics and overflowed boolean
   10686              :    flag in a complex integer result.  */
   10687              : 
   10688              : static tree
   10689           55 : fold_builtin_addc_subc (location_t loc, enum built_in_function fcode,
   10690              :                         tree *args)
   10691              : {
   10692           55 :   enum internal_fn ifn;
   10693              : 
   10694           55 :   switch (fcode)
   10695              :     {
   10696              :     case BUILT_IN_ADDC:
   10697              :     case BUILT_IN_ADDCL:
   10698              :     case BUILT_IN_ADDCLL:
   10699              :       ifn = IFN_ADD_OVERFLOW;
   10700              :       break;
   10701           29 :     case BUILT_IN_SUBC:
   10702           29 :     case BUILT_IN_SUBCL:
   10703           29 :     case BUILT_IN_SUBCLL:
   10704           29 :       ifn = IFN_SUB_OVERFLOW;
   10705           29 :       break;
   10706            0 :     default:
   10707            0 :       gcc_unreachable ();
   10708              :     }
   10709              : 
   10710           55 :   tree type = TREE_TYPE (args[0]);
   10711           55 :   tree ctype = build_complex_type (type);
   10712           55 :   tree call = build_call_expr_internal_loc (loc, ifn, ctype, 2,
   10713              :                                             args[0], args[1]);
   10714              :   /* Force SAVE_EXPR even for calls which satisfy tree_invariant_p_1,
   10715              :      as while the call itself is const, the REALPART_EXPR store is
   10716              :      certainly not.  And in any case, we want just one call,
   10717              :      not multiple and trying to CSE them later.  */
   10718           55 :   TREE_SIDE_EFFECTS (call) = 1;
   10719           55 :   tree tgt = save_expr (call);
   10720           55 :   tree intres = build1_loc (loc, REALPART_EXPR, type, tgt);
   10721           55 :   tree ovfres = build1_loc (loc, IMAGPART_EXPR, type, tgt);
   10722           55 :   call = build_call_expr_internal_loc (loc, ifn, ctype, 2,
   10723              :                                        intres, args[2]);
   10724           55 :   TREE_SIDE_EFFECTS (call) = 1;
   10725           55 :   tgt = save_expr (call);
   10726           55 :   intres = build1_loc (loc, REALPART_EXPR, type, tgt);
   10727           55 :   tree ovfres2 = build1_loc (loc, IMAGPART_EXPR, type, tgt);
   10728           55 :   ovfres = build2_loc (loc, BIT_IOR_EXPR, type, ovfres, ovfres2);
   10729           55 :   tree mem_arg3 = build_fold_indirect_ref_loc (loc, args[3]);
   10730           55 :   tree store
   10731           55 :     = fold_build2_loc (loc, MODIFY_EXPR, void_type_node, mem_arg3, ovfres);
   10732           55 :   return build2_loc (loc, COMPOUND_EXPR, type, store, intres);
   10733              : }
   10734              : 
   10735              : /* Fold __builtin_clmulg (arg1, arg2) or __builtin_clmulg (arg1, arg2, arg3)
   10736              :    into IFN_CLMUL or IFN_CLMUL_WIDEN internal call.  */
   10737              : 
   10738              : tree
   10739          430 : fold_build_builtin_clmulg (location_t loc, tree arg1, tree arg2, tree arg3)
   10740              : {
   10741          430 :   tree type = TYPE_MAIN_VARIANT (TREE_TYPE (arg1));
   10742          430 :   if (TREE_CODE (arg1) == INTEGER_CST
   10743          132 :       && !TREE_OVERFLOW (arg1)
   10744          132 :       && TREE_CODE (arg2) == INTEGER_CST
   10745          530 :       && !TREE_OVERFLOW (arg2))
   10746              :     {
   10747          100 :       unsigned prec = TYPE_PRECISION (type);
   10748          100 :       wide_int res = wi::clmul (wi::to_wide (arg1), wi::to_wide (arg2),
   10749          200 :                                 prec * (arg3 == NULL_TREE ? 1 : 2));
   10750          100 :       if (arg3 == NULL_TREE)
   10751          100 :         return wide_int_to_tree (type, res);
   10752            0 :       wide_int lo = wide_int::from (res, prec, UNSIGNED);
   10753            0 :       wide_int hi = wide_int::from (wi::rshift (res, prec, UNSIGNED),
   10754            0 :                                     prec, UNSIGNED);
   10755            0 :       tree mem_arg3 = build_fold_indirect_ref_loc (loc, arg3);
   10756            0 :       tree store
   10757            0 :         = fold_build2_loc (loc, MODIFY_EXPR, void_type_node, mem_arg3,
   10758            0 :                            wide_int_to_tree (type, hi));
   10759            0 :       return build2_loc (loc, COMPOUND_EXPR, type, store,
   10760            0 :                          wide_int_to_tree (type, lo));
   10761          100 :     }
   10762          330 :   if (arg3 == NULL_TREE)
   10763          199 :     return build_call_expr_internal_loc (loc, IFN_CLMUL, type, 2, arg1, arg2);
   10764          131 :   tree ctype = build_complex_type (type);
   10765          131 :   tree call = build_call_expr_internal_loc (loc, IFN_CLMUL_WIDEN, ctype, 2,
   10766              :                                             arg1, arg2);
   10767              :   /* Force SAVE_EXPR even for calls which satisfy tree_invariant_p_1,
   10768              :      as while the call itself is const, the REALPART_EXPR store is
   10769              :      certainly not.  And in any case, we want just one call,
   10770              :      not multiple and trying to CSE them later.  */
   10771          131 :   TREE_SIDE_EFFECTS (call) = 1;
   10772          131 :   tree tgt = save_expr (call);
   10773          131 :   tree intres = build1_loc (loc, REALPART_EXPR, type, tgt);
   10774          131 :   tree ovfres = build1_loc (loc, IMAGPART_EXPR, type, tgt);
   10775          131 :   tree mem_arg3 = build_fold_indirect_ref_loc (loc, arg3);
   10776          131 :   tree store
   10777          131 :     = fold_build2_loc (loc, MODIFY_EXPR, void_type_node, mem_arg3, ovfres);
   10778          131 :   return build2_loc (loc, COMPOUND_EXPR, type, store, intres);
   10779              : }
   10780              : 
   10781              : /* Fold a call to __builtin_FILE to a constant string.  */
   10782              : 
   10783              : static inline tree
   10784         5946 : fold_builtin_FILE (location_t loc)
   10785              : {
   10786         5946 :   if (const char *fname = LOCATION_FILE (loc))
   10787              :     {
   10788              :       /* The documentation says this builtin is equivalent to the preprocessor
   10789              :          __FILE__ macro so it appears appropriate to use the same file prefix
   10790              :          mappings.  */
   10791         5946 :       fname = remap_macro_filename (fname);
   10792         5946 :       return build_string_literal (fname);
   10793              :     }
   10794              : 
   10795            0 :   return build_string_literal ("");
   10796              : }
   10797              : 
   10798              : /* Fold a call to __builtin_FUNCTION to a constant string.  */
   10799              : 
   10800              : static inline tree
   10801           68 : fold_builtin_FUNCTION ()
   10802              : {
   10803           68 :   const char *name = "";
   10804              : 
   10805           68 :   if (current_function_decl)
   10806           43 :     name = lang_hooks.decl_printable_name (current_function_decl, 0);
   10807              : 
   10808           68 :   return build_string_literal (name);
   10809              : }
   10810              : 
   10811              : /* Fold a call to __builtin_LINE to an integer constant.  */
   10812              : 
   10813              : static inline tree
   10814        11859 : fold_builtin_LINE (location_t loc, tree type)
   10815              : {
   10816        11859 :   return build_int_cst (type, LOCATION_LINE (loc));
   10817              : }
   10818              : 
   10819              : /* Fold a call to built-in function FNDECL with 0 arguments.
   10820              :    This function returns NULL_TREE if no simplification was possible.  */
   10821              : 
   10822              : static tree
   10823     23747437 : fold_builtin_0 (location_t loc, tree fndecl)
   10824              : {
   10825     23747437 :   tree type = TREE_TYPE (TREE_TYPE (fndecl));
   10826     23747437 :   enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
   10827     23747437 :   switch (fcode)
   10828              :     {
   10829         5946 :     case BUILT_IN_FILE:
   10830         5946 :       return fold_builtin_FILE (loc);
   10831              : 
   10832           68 :     case BUILT_IN_FUNCTION:
   10833           68 :       return fold_builtin_FUNCTION ();
   10834              : 
   10835        11859 :     case BUILT_IN_LINE:
   10836        11859 :       return fold_builtin_LINE (loc, type);
   10837              : 
   10838        34101 :     CASE_FLT_FN (BUILT_IN_INF):
   10839        34101 :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_INF):
   10840        34101 :     case BUILT_IN_INFD32:
   10841        34101 :     case BUILT_IN_INFD64:
   10842        34101 :     case BUILT_IN_INFD128:
   10843        34101 :     case BUILT_IN_INFD64X:
   10844        34101 :       return fold_builtin_inf (loc, type, true);
   10845              : 
   10846       224304 :     CASE_FLT_FN (BUILT_IN_HUGE_VAL):
   10847       224304 :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_HUGE_VAL):
   10848       224304 :       return fold_builtin_inf (loc, type, false);
   10849              : 
   10850            0 :     case BUILT_IN_CLASSIFY_TYPE:
   10851            0 :       return fold_builtin_classify_type (NULL_TREE);
   10852              : 
   10853     20251235 :     case BUILT_IN_UNREACHABLE:
   10854              :       /* Rewrite any explicit calls to __builtin_unreachable.  */
   10855     20251235 :       if (sanitize_flags_p (SANITIZE_UNREACHABLE))
   10856           91 :         return build_builtin_unreachable (loc);
   10857              :       break;
   10858              : 
   10859              :     default:
   10860              :       break;
   10861              :     }
   10862              :   return NULL_TREE;
   10863              : }
   10864              : 
   10865              : /* Fold a call to built-in function FNDECL with 1 argument, ARG0.
   10866              :    This function returns NULL_TREE if no simplification was possible.  */
   10867              : 
   10868              : static tree
   10869     20261775 : fold_builtin_1 (location_t loc, tree expr, tree fndecl, tree arg0)
   10870              : {
   10871     20261775 :   tree type = TREE_TYPE (TREE_TYPE (fndecl));
   10872     20261775 :   enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
   10873              : 
   10874     20261775 :   if (error_operand_p (arg0))
   10875              :     return NULL_TREE;
   10876              : 
   10877     20261775 :   if (tree ret = fold_const_call (as_combined_fn (fcode), type, arg0))
   10878              :     return ret;
   10879              : 
   10880     19648458 :   switch (fcode)
   10881              :     {
   10882      3090175 :     case BUILT_IN_CONSTANT_P:
   10883      3090175 :       {
   10884      3090175 :         tree val = fold_builtin_constant_p (arg0);
   10885              : 
   10886              :         /* Gimplification will pull the CALL_EXPR for the builtin out of
   10887              :            an if condition.  When not optimizing, we'll not CSE it back.
   10888              :            To avoid link error types of regressions, return false now.  */
   10889      3090175 :         if (!val && !optimize)
   10890         1558 :           val = integer_zero_node;
   10891              : 
   10892              :         return val;
   10893              :       }
   10894              : 
   10895         2084 :     case BUILT_IN_CLASSIFY_TYPE:
   10896         2084 :       return fold_builtin_classify_type (arg0);
   10897              : 
   10898       461577 :     case BUILT_IN_STRLEN:
   10899       461577 :       return fold_builtin_strlen (loc, expr, type, arg0);
   10900              : 
   10901       372999 :     CASE_FLT_FN (BUILT_IN_FABS):
   10902       372999 :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_FABS):
   10903       372999 :     case BUILT_IN_FABSD32:
   10904       372999 :     case BUILT_IN_FABSD64:
   10905       372999 :     case BUILT_IN_FABSD128:
   10906       372999 :     case BUILT_IN_FABSD64X:
   10907       372999 :       return fold_builtin_fabs (loc, arg0, type);
   10908              : 
   10909        98708 :     case BUILT_IN_ABS:
   10910        98708 :     case BUILT_IN_LABS:
   10911        98708 :     case BUILT_IN_LLABS:
   10912        98708 :     case BUILT_IN_IMAXABS:
   10913        98708 :     case BUILT_IN_UABS:
   10914        98708 :     case BUILT_IN_ULABS:
   10915        98708 :     case BUILT_IN_ULLABS:
   10916        98708 :     case BUILT_IN_UMAXABS:
   10917        98708 :       return fold_builtin_abs (loc, arg0, type);
   10918              : 
   10919        25461 :     CASE_FLT_FN (BUILT_IN_CONJ):
   10920        25461 :       if (validate_arg (arg0, COMPLEX_TYPE)
   10921        25461 :         && TREE_CODE (TREE_TYPE (TREE_TYPE (arg0))) == REAL_TYPE)
   10922        25461 :         return fold_build1_loc (loc, CONJ_EXPR, type, arg0);
   10923              :     break;
   10924              : 
   10925          764 :     CASE_FLT_FN (BUILT_IN_CREAL):
   10926          764 :       if (validate_arg (arg0, COMPLEX_TYPE)
   10927          764 :         && TREE_CODE (TREE_TYPE (TREE_TYPE (arg0))) == REAL_TYPE)
   10928          764 :         return non_lvalue_loc (loc, fold_build1_loc (loc, REALPART_EXPR, type, arg0));
   10929              :     break;
   10930              : 
   10931         1884 :     CASE_FLT_FN (BUILT_IN_CIMAG):
   10932         1884 :       if (validate_arg (arg0, COMPLEX_TYPE)
   10933         1884 :           && TREE_CODE (TREE_TYPE (TREE_TYPE (arg0))) == REAL_TYPE)
   10934         1884 :         return non_lvalue_loc (loc, fold_build1_loc (loc, IMAGPART_EXPR, type, arg0));
   10935              :     break;
   10936              : 
   10937       104102 :     CASE_FLT_FN (BUILT_IN_CARG):
   10938       104102 :     CASE_FLT_FN_FLOATN_NX (BUILT_IN_CARG):
   10939       104102 :       return fold_builtin_carg (loc, arg0, type);
   10940              : 
   10941          211 :     case BUILT_IN_ISASCII:
   10942          211 :       return fold_builtin_isascii (loc, arg0);
   10943              : 
   10944          168 :     case BUILT_IN_TOASCII:
   10945          168 :       return fold_builtin_toascii (loc, arg0);
   10946              : 
   10947          325 :     case BUILT_IN_ISDIGIT:
   10948          325 :       return fold_builtin_isdigit (loc, arg0);
   10949              : 
   10950       628443 :     CASE_FLT_FN (BUILT_IN_FINITE):
   10951       628443 :     case BUILT_IN_FINITED32:
   10952       628443 :     case BUILT_IN_FINITED64:
   10953       628443 :     case BUILT_IN_FINITED128:
   10954       628443 :     case BUILT_IN_ISFINITE:
   10955       628443 :       {
   10956       628443 :         tree ret = fold_builtin_classify (loc, fndecl, arg0, BUILT_IN_ISFINITE);
   10957       628443 :         if (ret)
   10958              :           return ret;
   10959       628111 :         return fold_builtin_interclass_mathfn (loc, fndecl, arg0);
   10960              :       }
   10961              : 
   10962       706833 :     CASE_FLT_FN (BUILT_IN_ISINF):
   10963       706833 :     case BUILT_IN_ISINFD32:
   10964       706833 :     case BUILT_IN_ISINFD64:
   10965       706833 :     case BUILT_IN_ISINFD128:
   10966       706833 :       {
   10967       706833 :         tree ret = fold_builtin_classify (loc, fndecl, arg0, BUILT_IN_ISINF);
   10968       706833 :         if (ret)
   10969              :           return ret;
   10970       706582 :         return fold_builtin_interclass_mathfn (loc, fndecl, arg0);
   10971              :       }
   10972              : 
   10973       239084 :     case BUILT_IN_ISNORMAL:
   10974       239084 :       return fold_builtin_interclass_mathfn (loc, fndecl, arg0);
   10975              : 
   10976          708 :     case BUILT_IN_ISINF_SIGN:
   10977          708 :       return fold_builtin_classify (loc, fndecl, arg0, BUILT_IN_ISINF_SIGN);
   10978              : 
   10979       242196 :     CASE_FLT_FN (BUILT_IN_ISNAN):
   10980       242196 :     case BUILT_IN_ISNAND32:
   10981       242196 :     case BUILT_IN_ISNAND64:
   10982       242196 :     case BUILT_IN_ISNAND128:
   10983       242196 :       {
   10984       242196 :         tree ret = fold_builtin_classify (loc, fndecl, arg0, BUILT_IN_ISNAN);
   10985       242196 :         if (ret)
   10986              :           return ret;
   10987       242072 :         return fold_builtin_interclass_mathfn (loc, fndecl, arg0);
   10988              :       }
   10989              : 
   10990        27879 :     case BUILT_IN_ISSIGNALING:
   10991        27879 :       return fold_builtin_classify (loc, fndecl, arg0, BUILT_IN_ISSIGNALING);
   10992              : 
   10993       651295 :     case BUILT_IN_FREE:
   10994       651295 :       if (integer_zerop (arg0))
   10995         1090 :         return build_empty_stmt (loc);
   10996              :       break;
   10997              : 
   10998        11654 :     case BUILT_IN_CLZG:
   10999        11654 :     case BUILT_IN_CTZG:
   11000        11654 :     case BUILT_IN_CLRSBG:
   11001        11654 :     case BUILT_IN_FFSG:
   11002        11654 :     case BUILT_IN_PARITYG:
   11003        11654 :     case BUILT_IN_POPCOUNTG:
   11004        11654 :       return fold_builtin_bit_query (loc, fcode, arg0, NULL_TREE);
   11005              : 
   11006              :     default:
   11007              :       break;
   11008              :     }
   11009              : 
   11010              :   return NULL_TREE;
   11011              : 
   11012              : }
   11013              : 
   11014              : /* Folds a call EXPR (which may be null) to built-in function FNDECL
   11015              :    with 2 arguments, ARG0 and ARG1.  This function returns NULL_TREE
   11016              :    if no simplification was possible.  */
   11017              : 
   11018              : static tree
   11019     19388963 : fold_builtin_2 (location_t loc, tree expr, tree fndecl, tree arg0, tree arg1)
   11020              : {
   11021     19388963 :   tree type = TREE_TYPE (TREE_TYPE (fndecl));
   11022     19388963 :   enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
   11023              : 
   11024     19388963 :   if (error_operand_p (arg0)
   11025     19388963 :       || error_operand_p (arg1))
   11026              :     return NULL_TREE;
   11027              : 
   11028     19388959 :   if (tree ret = fold_const_call (as_combined_fn (fcode), type, arg0, arg1))
   11029              :     return ret;
   11030              : 
   11031     19173446 :   switch (fcode)
   11032              :     {
   11033         6504 :     CASE_FLT_FN_REENT (BUILT_IN_GAMMA): /* GAMMA_R */
   11034         6504 :     CASE_FLT_FN_REENT (BUILT_IN_LGAMMA): /* LGAMMA_R */
   11035         6504 :       if (validate_arg (arg0, REAL_TYPE)
   11036         6504 :           && validate_arg (arg1, POINTER_TYPE))
   11037         6504 :         return do_mpfr_lgamma_r (arg0, arg1, type);
   11038              :     break;
   11039              : 
   11040       115827 :     CASE_FLT_FN (BUILT_IN_FREXP):
   11041       115827 :       return fold_builtin_frexp (loc, arg0, arg1, type);
   11042              : 
   11043        92498 :     CASE_FLT_FN (BUILT_IN_MODF):
   11044        92498 :       return fold_builtin_modf (loc, arg0, arg1, type);
   11045              : 
   11046         2560 :     case BUILT_IN_STRSPN:
   11047         2560 :       return fold_builtin_strspn (loc, expr, arg0, arg1, type);
   11048              : 
   11049         2465 :     case BUILT_IN_STRCSPN:
   11050         2465 :       return fold_builtin_strcspn (loc, expr, arg0, arg1, type);
   11051              : 
   11052        99377 :     case BUILT_IN_STRPBRK:
   11053        99377 :       return fold_builtin_strpbrk (loc, expr, arg0, arg1, type);
   11054              : 
   11055      6661398 :     case BUILT_IN_EXPECT:
   11056      6661398 :       return fold_builtin_expect (loc, arg0, arg1, NULL_TREE, NULL_TREE);
   11057              : 
   11058       945825 :     case BUILT_IN_ISGREATER:
   11059       945825 :       return fold_builtin_unordered_cmp (loc, fndecl,
   11060       945825 :                                          arg0, arg1, UNLE_EXPR, LE_EXPR);
   11061       478302 :     case BUILT_IN_ISGREATEREQUAL:
   11062       478302 :       return fold_builtin_unordered_cmp (loc, fndecl,
   11063       478302 :                                          arg0, arg1, UNLT_EXPR, LT_EXPR);
   11064       240747 :     case BUILT_IN_ISLESS:
   11065       240747 :       return fold_builtin_unordered_cmp (loc, fndecl,
   11066       240747 :                                          arg0, arg1, UNGE_EXPR, GE_EXPR);
   11067      1106403 :     case BUILT_IN_ISLESSEQUAL:
   11068      1106403 :       return fold_builtin_unordered_cmp (loc, fndecl,
   11069      1106403 :                                          arg0, arg1, UNGT_EXPR, GT_EXPR);
   11070       238913 :     case BUILT_IN_ISLESSGREATER:
   11071       238913 :       return fold_builtin_unordered_cmp (loc, fndecl,
   11072       238913 :                                          arg0, arg1, UNEQ_EXPR, EQ_EXPR);
   11073       240131 :     case BUILT_IN_ISUNORDERED:
   11074       240131 :       return fold_builtin_unordered_cmp (loc, fndecl,
   11075              :                                          arg0, arg1, UNORDERED_EXPR,
   11076       240131 :                                          NOP_EXPR);
   11077              : 
   11078          709 :     case BUILT_IN_ISEQSIG:
   11079          709 :       return fold_builtin_iseqsig (loc, arg0, arg1);
   11080              : 
   11081              :       /* We do the folding for va_start in the expander.  */
   11082              :     case BUILT_IN_VA_START:
   11083              :       break;
   11084              : 
   11085       201177 :     case BUILT_IN_OBJECT_SIZE:
   11086       201177 :     case BUILT_IN_DYNAMIC_OBJECT_SIZE:
   11087       201177 :       return fold_builtin_object_size (arg0, arg1, fcode);
   11088              : 
   11089       104879 :     case BUILT_IN_ATOMIC_ALWAYS_LOCK_FREE:
   11090       104879 :       return fold_builtin_atomic_always_lock_free (arg0, arg1);
   11091              : 
   11092        38581 :     case BUILT_IN_ATOMIC_IS_LOCK_FREE:
   11093        38581 :       return fold_builtin_atomic_is_lock_free (arg0, arg1);
   11094              : 
   11095       175137 :     case BUILT_IN_CLZG:
   11096       175137 :     case BUILT_IN_CTZG:
   11097       175137 :       return fold_builtin_bit_query (loc, fcode, arg0, arg1);
   11098              : 
   11099              :     default:
   11100              :       break;
   11101              :     }
   11102              :   return NULL_TREE;
   11103              : }
   11104              : 
   11105              : /* Fold a call to built-in function FNDECL with 3 arguments, ARG0, ARG1,
   11106              :    and ARG2.
   11107              :    This function returns NULL_TREE if no simplification was possible.  */
   11108              : 
   11109              : static tree
   11110      7366737 : fold_builtin_3 (location_t loc, tree fndecl,
   11111              :                 tree arg0, tree arg1, tree arg2)
   11112              : {
   11113      7366737 :   tree type = TREE_TYPE (TREE_TYPE (fndecl));
   11114      7366737 :   enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
   11115              : 
   11116      7366737 :   if (error_operand_p (arg0)
   11117      7366737 :       || error_operand_p (arg1)
   11118     14733474 :       || error_operand_p (arg2))
   11119              :     return NULL_TREE;
   11120              : 
   11121      7366735 :   if (tree ret = fold_const_call (as_combined_fn (fcode), type,
   11122              :                                   arg0, arg1, arg2))
   11123              :     return ret;
   11124              : 
   11125      7349665 :   switch (fcode)
   11126              :     {
   11127              : 
   11128          145 :     CASE_FLT_FN (BUILT_IN_SINCOS):
   11129          145 :       return fold_builtin_sincos (loc, arg0, arg1, arg2);
   11130              : 
   11131        99724 :     CASE_FLT_FN (BUILT_IN_REMQUO):
   11132        99724 :       if (validate_arg (arg0, REAL_TYPE)
   11133        99724 :           && validate_arg (arg1, REAL_TYPE)
   11134       199448 :           && validate_arg (arg2, POINTER_TYPE))
   11135        99724 :         return do_mpfr_remquo (arg0, arg1, arg2);
   11136              :     break;
   11137              : 
   11138      2547918 :     case BUILT_IN_MEMCMP:
   11139      2547918 :       return fold_builtin_memcmp (loc, arg0, arg1, arg2);
   11140              : 
   11141       528938 :     case BUILT_IN_EXPECT:
   11142       528938 :       return fold_builtin_expect (loc, arg0, arg1, arg2, NULL_TREE);
   11143              : 
   11144          395 :     case BUILT_IN_EXPECT_WITH_PROBABILITY:
   11145          395 :       return fold_builtin_expect (loc, arg0, arg1, NULL_TREE, arg2);
   11146              : 
   11147       503397 :     case BUILT_IN_ADD_OVERFLOW:
   11148       503397 :     case BUILT_IN_SUB_OVERFLOW:
   11149       503397 :     case BUILT_IN_MUL_OVERFLOW:
   11150       503397 :     case BUILT_IN_ADD_OVERFLOW_P:
   11151       503397 :     case BUILT_IN_SUB_OVERFLOW_P:
   11152       503397 :     case BUILT_IN_MUL_OVERFLOW_P:
   11153       503397 :     case BUILT_IN_SADD_OVERFLOW:
   11154       503397 :     case BUILT_IN_SADDL_OVERFLOW:
   11155       503397 :     case BUILT_IN_SADDLL_OVERFLOW:
   11156       503397 :     case BUILT_IN_SSUB_OVERFLOW:
   11157       503397 :     case BUILT_IN_SSUBL_OVERFLOW:
   11158       503397 :     case BUILT_IN_SSUBLL_OVERFLOW:
   11159       503397 :     case BUILT_IN_SMUL_OVERFLOW:
   11160       503397 :     case BUILT_IN_SMULL_OVERFLOW:
   11161       503397 :     case BUILT_IN_SMULLL_OVERFLOW:
   11162       503397 :     case BUILT_IN_UADD_OVERFLOW:
   11163       503397 :     case BUILT_IN_UADDL_OVERFLOW:
   11164       503397 :     case BUILT_IN_UADDLL_OVERFLOW:
   11165       503397 :     case BUILT_IN_USUB_OVERFLOW:
   11166       503397 :     case BUILT_IN_USUBL_OVERFLOW:
   11167       503397 :     case BUILT_IN_USUBLL_OVERFLOW:
   11168       503397 :     case BUILT_IN_UMUL_OVERFLOW:
   11169       503397 :     case BUILT_IN_UMULL_OVERFLOW:
   11170       503397 :     case BUILT_IN_UMULLL_OVERFLOW:
   11171       503397 :       return fold_builtin_arith_overflow (loc, fcode, arg0, arg1, arg2);
   11172              : 
   11173              :     default:
   11174              :       break;
   11175              :     }
   11176              :   return NULL_TREE;
   11177              : }
   11178              : 
   11179              : /* Folds a call EXPR (which may be null) to built-in function FNDECL.
   11180              :    ARGS is an array of NARGS arguments.  IGNORE is true if the result
   11181              :    of the function call is ignored.  This function returns NULL_TREE
   11182              :    if no simplification was possible.  */
   11183              : 
   11184              : static tree
   11185     73412978 : fold_builtin_n (location_t loc, tree expr, tree fndecl, tree *args,
   11186              :                 int nargs, bool)
   11187              : {
   11188     73412978 :   tree ret = NULL_TREE;
   11189              : 
   11190     73412978 :   switch (nargs)
   11191              :     {
   11192     23747437 :     case 0:
   11193     23747437 :       ret = fold_builtin_0 (loc, fndecl);
   11194     23747437 :       break;
   11195     20261775 :     case 1:
   11196     20261775 :       ret = fold_builtin_1 (loc, expr, fndecl, args[0]);
   11197     20261775 :       break;
   11198     19388963 :     case 2:
   11199     19388963 :       ret = fold_builtin_2 (loc, expr, fndecl, args[0], args[1]);
   11200     19388963 :       break;
   11201      7366737 :     case 3:
   11202      7366737 :       ret = fold_builtin_3 (loc, fndecl, args[0], args[1], args[2]);
   11203      7366737 :       break;
   11204      2648066 :     default:
   11205      2648066 :       ret = fold_builtin_varargs (loc, fndecl, args, nargs);
   11206      2648066 :       break;
   11207              :     }
   11208     73412978 :   if (ret)
   11209              :     {
   11210     10014044 :       ret = build1 (NOP_EXPR, TREE_TYPE (ret), ret);
   11211     10014044 :       SET_EXPR_LOCATION (ret, loc);
   11212     10014044 :       return ret;
   11213              :     }
   11214              :   return NULL_TREE;
   11215              : }
   11216              : 
   11217              : /* Construct a new CALL_EXPR to FNDECL using the tail of the argument
   11218              :    list ARGS along with N new arguments in NEWARGS.  SKIP is the number
   11219              :    of arguments in ARGS to be omitted.  OLDNARGS is the number of
   11220              :    elements in ARGS.  */
   11221              : 
   11222              : static tree
   11223            4 : rewrite_call_expr_valist (location_t loc, int oldnargs, tree *args,
   11224              :                           int skip, tree fndecl, int n, va_list newargs)
   11225              : {
   11226            4 :   int nargs = oldnargs - skip + n;
   11227            4 :   tree *buffer;
   11228              : 
   11229            4 :   if (n > 0)
   11230              :     {
   11231            0 :       int i, j;
   11232              : 
   11233            0 :       buffer = XALLOCAVEC (tree, nargs);
   11234            0 :       for (i = 0; i < n; i++)
   11235            0 :         buffer[i] = va_arg (newargs, tree);
   11236            0 :       for (j = skip; j < oldnargs; j++, i++)
   11237            0 :         buffer[i] = args[j];
   11238              :     }
   11239              :   else
   11240            4 :     buffer = args + skip;
   11241              : 
   11242            4 :   return build_call_expr_loc_array (loc, fndecl, nargs, buffer);
   11243              : }
   11244              : 
   11245              : /* Return true if FNDECL shouldn't be folded right now.
   11246              :    If a built-in function has an inline attribute always_inline
   11247              :    wrapper, defer folding it after always_inline functions have
   11248              :    been inlined, otherwise e.g. -D_FORTIFY_SOURCE checking
   11249              :    might not be performed.  */
   11250              : 
   11251              : bool
   11252    154535517 : avoid_folding_inline_builtin (tree fndecl)
   11253              : {
   11254    154535517 :   return (DECL_DECLARED_INLINE_P (fndecl)
   11255        13320 :           && DECL_DISREGARD_INLINE_LIMITS (fndecl)
   11256        13278 :           && cfun
   11257        13278 :           && !cfun->always_inline_functions_inlined
   11258    154548795 :           && lookup_attribute ("always_inline", DECL_ATTRIBUTES (fndecl)));
   11259              : }
   11260              : 
   11261              : /* A wrapper function for builtin folding that prevents warnings for
   11262              :    "statement without effect" and the like, caused by removing the
   11263              :    call node earlier than the warning is generated.  */
   11264              : 
   11265              : tree
   11266    202621148 : fold_call_expr (location_t loc, tree exp, bool ignore)
   11267              : {
   11268    202621148 :   tree ret = NULL_TREE;
   11269    202621148 :   tree fndecl = get_callee_fndecl (exp);
   11270    200861766 :   if (fndecl && fndecl_built_in_p (fndecl)
   11271              :       /* If CALL_EXPR_VA_ARG_PACK is set, the arguments aren't finalized
   11272              :          yet.  Defer folding until we see all the arguments
   11273              :          (after inlining).  */
   11274    262693134 :       && !CALL_EXPR_VA_ARG_PACK (exp))
   11275              :     {
   11276     60071955 :       int nargs = call_expr_nargs (exp);
   11277              : 
   11278              :       /* Before gimplification CALL_EXPR_VA_ARG_PACK is not set, but
   11279              :          instead last argument is __builtin_va_arg_pack ().  Defer folding
   11280              :          even in that case, until arguments are finalized.  */
   11281     60071955 :       if (nargs && TREE_CODE (CALL_EXPR_ARG (exp, nargs - 1)) == CALL_EXPR)
   11282              :         {
   11283       221343 :           tree fndecl2 = get_callee_fndecl (CALL_EXPR_ARG (exp, nargs - 1));
   11284       221343 :           if (fndecl2 && fndecl_built_in_p (fndecl2, BUILT_IN_VA_ARG_PACK))
   11285              :             return NULL_TREE;
   11286              :         }
   11287              : 
   11288     60071886 :       if (avoid_folding_inline_builtin (fndecl))
   11289              :         return NULL_TREE;
   11290              : 
   11291     60068348 :       if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
   11292     72217486 :         return targetm.fold_builtin (fndecl, call_expr_nargs (exp),
   11293     72217486 :                                      CALL_EXPR_ARGP (exp), ignore);
   11294              :       else
   11295              :         {
   11296     23959605 :           tree *args = CALL_EXPR_ARGP (exp);
   11297     23959605 :           ret = fold_builtin_n (loc, exp, fndecl, args, nargs, ignore);
   11298     23959605 :           if (ret)
   11299              :             return ret;
   11300              :         }
   11301              :     }
   11302              :   return NULL_TREE;
   11303              : }
   11304              : 
   11305              : /* Fold a CALL_EXPR with type TYPE with FN as the function expression.
   11306              :    N arguments are passed in the array ARGARRAY.  Return a folded
   11307              :    expression or NULL_TREE if no simplification was possible.  */
   11308              : 
   11309              : tree
   11310     79215021 : fold_builtin_call_array (location_t loc, tree,
   11311              :                          tree fn,
   11312              :                          int n,
   11313              :                          tree *argarray)
   11314              : {
   11315     79215021 :   if (TREE_CODE (fn) != ADDR_EXPR)
   11316              :     return NULL_TREE;
   11317              : 
   11318     79215021 :   tree fndecl = TREE_OPERAND (fn, 0);
   11319     79215021 :   if (TREE_CODE (fndecl) == FUNCTION_DECL
   11320     79215021 :       && fndecl_built_in_p (fndecl))
   11321              :     {
   11322              :       /* If last argument is __builtin_va_arg_pack (), arguments to this
   11323              :          function are not finalized yet.  Defer folding until they are.  */
   11324     78597590 :       if (n && TREE_CODE (argarray[n - 1]) == CALL_EXPR)
   11325              :         {
   11326       137270 :           tree fndecl2 = get_callee_fndecl (argarray[n - 1]);
   11327       137270 :           if (fndecl2 && fndecl_built_in_p (fndecl2, BUILT_IN_VA_ARG_PACK))
   11328              :             return NULL_TREE;
   11329              :         }
   11330     78597551 :       if (avoid_folding_inline_builtin (fndecl))
   11331              :         return NULL_TREE;
   11332     78597551 :       if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
   11333     35343970 :         return targetm.fold_builtin (fndecl, n, argarray, false);
   11334              :       else
   11335     43253581 :         return fold_builtin_n (loc, NULL_TREE, fndecl, argarray, n, false);
   11336              :     }
   11337              : 
   11338              :   return NULL_TREE;
   11339              : }
   11340              : 
   11341              : /* Construct a new CALL_EXPR using the tail of the argument list of EXP
   11342              :    along with N new arguments specified as the "..." parameters.  SKIP
   11343              :    is the number of arguments in EXP to be omitted.  This function is used
   11344              :    to do varargs-to-varargs transformations.  */
   11345              : 
   11346              : static tree
   11347            4 : rewrite_call_expr (location_t loc, tree exp, int skip, tree fndecl, int n, ...)
   11348              : {
   11349            4 :   va_list ap;
   11350            4 :   tree t;
   11351              : 
   11352            4 :   va_start (ap, n);
   11353            8 :   t = rewrite_call_expr_valist (loc, call_expr_nargs (exp),
   11354            4 :                                 CALL_EXPR_ARGP (exp), skip, fndecl, n, ap);
   11355            4 :   va_end (ap);
   11356              : 
   11357            4 :   return t;
   11358              : }
   11359              : 
   11360              : /* Validate a single argument ARG against a tree code CODE representing
   11361              :    a type.  Return true when argument is valid.  */
   11362              : 
   11363              : static bool
   11364     15350423 : validate_arg (const_tree arg, enum tree_code code)
   11365              : {
   11366     15350423 :   if (!arg)
   11367              :     return false;
   11368     15350400 :   else if (code == POINTER_TYPE)
   11369      7106421 :     return POINTER_TYPE_P (TREE_TYPE (arg));
   11370      8243979 :   else if (code == INTEGER_TYPE)
   11371      3770308 :     return INTEGRAL_TYPE_P (TREE_TYPE (arg));
   11372      4473671 :   return code == TREE_CODE (TREE_TYPE (arg));
   11373              : }
   11374              : 
   11375              : /* This function validates the types of a function call argument list
   11376              :    against a specified list of tree_codes.  If the last specifier is a 0,
   11377              :    that represents an ellipses, otherwise the last specifier must be a
   11378              :    VOID_TYPE.
   11379              : 
   11380              :    This is the GIMPLE version of validate_arglist.  Eventually we want to
   11381              :    completely convert builtins.cc to work from GIMPLEs and the tree based
   11382              :    validate_arglist will then be removed.  */
   11383              : 
   11384              : bool
   11385          136 : validate_gimple_arglist (const gcall *call, ...)
   11386              : {
   11387          136 :   enum tree_code code;
   11388          136 :   bool res = 0;
   11389          136 :   va_list ap;
   11390          136 :   const_tree arg;
   11391          136 :   size_t i;
   11392              : 
   11393          136 :   va_start (ap, call);
   11394          136 :   i = 0;
   11395              : 
   11396          544 :   do
   11397              :     {
   11398          544 :       code = (enum tree_code) va_arg (ap, int);
   11399          544 :       switch (code)
   11400              :         {
   11401            0 :         case 0:
   11402              :           /* This signifies an ellipses, any further arguments are all ok.  */
   11403            0 :           res = true;
   11404            0 :           goto end;
   11405          136 :         case VOID_TYPE:
   11406              :           /* This signifies an endlink, if no arguments remain, return
   11407              :              true, otherwise return false.  */
   11408          136 :           res = (i == gimple_call_num_args (call));
   11409          136 :           goto end;
   11410          408 :         default:
   11411              :           /* If no parameters remain or the parameter's code does not
   11412              :              match the specified code, return false.  Otherwise continue
   11413              :              checking any remaining arguments.  */
   11414          408 :           arg = gimple_call_arg (call, i++);
   11415          408 :           if (!validate_arg (arg, code))
   11416            0 :             goto end;
   11417              :           break;
   11418              :         }
   11419              :     }
   11420              :   while (1);
   11421              : 
   11422              :   /* We need gotos here since we can only have one VA_CLOSE in a
   11423              :      function.  */
   11424          136 :  end: ;
   11425          136 :   va_end (ap);
   11426              : 
   11427          136 :   return res;
   11428              : }
   11429              : 
   11430              : /* Default target-specific builtin expander that does nothing.  */
   11431              : 
   11432              : rtx
   11433            0 : default_expand_builtin (tree exp ATTRIBUTE_UNUSED,
   11434              :                         rtx target ATTRIBUTE_UNUSED,
   11435              :                         rtx subtarget ATTRIBUTE_UNUSED,
   11436              :                         machine_mode mode ATTRIBUTE_UNUSED,
   11437              :                         int ignore ATTRIBUTE_UNUSED)
   11438              : {
   11439            0 :   return NULL_RTX;
   11440              : }
   11441              : 
   11442              : /* Returns true is EXP represents data that would potentially reside
   11443              :    in a readonly section.  */
   11444              : 
   11445              : bool
   11446       209040 : readonly_data_expr (tree exp)
   11447              : {
   11448       209040 :   STRIP_NOPS (exp);
   11449              : 
   11450       209040 :   if (TREE_CODE (exp) != ADDR_EXPR)
   11451              :     return false;
   11452              : 
   11453        26954 :   exp = get_base_address (TREE_OPERAND (exp, 0));
   11454        26954 :   if (!exp)
   11455              :     return false;
   11456              : 
   11457              :   /* Make sure we call decl_readonly_section only for trees it
   11458              :      can handle (since it returns true for everything it doesn't
   11459              :      understand).  */
   11460        26954 :   if (TREE_CODE (exp) == STRING_CST
   11461         5796 :       || TREE_CODE (exp) == CONSTRUCTOR
   11462         5796 :       || (VAR_P (exp) && TREE_STATIC (exp)))
   11463        23795 :     return decl_readonly_section (exp, 0);
   11464              :   else
   11465              :     return false;
   11466              : }
   11467              : 
   11468              : /* Simplify a call to the strpbrk builtin.  S1 and S2 are the arguments
   11469              :    to the call, and TYPE is its return type.
   11470              : 
   11471              :    Return NULL_TREE if no simplification was possible, otherwise return the
   11472              :    simplified form of the call as a tree.
   11473              : 
   11474              :    The simplified form may be a constant or other expression which
   11475              :    computes the same value, but in a more efficient manner (including
   11476              :    calls to other builtin functions).
   11477              : 
   11478              :    The call may contain arguments which need to be evaluated, but
   11479              :    which are not useful to determine the result of the call.  In
   11480              :    this case we return a chain of COMPOUND_EXPRs.  The LHS of each
   11481              :    COMPOUND_EXPR will be an argument which must be evaluated.
   11482              :    COMPOUND_EXPRs are chained through their RHS.  The RHS of the last
   11483              :    COMPOUND_EXPR in the chain will contain the tree for the simplified
   11484              :    form of the builtin function call.  */
   11485              : 
   11486              : static tree
   11487        99377 : fold_builtin_strpbrk (location_t loc, tree, tree s1, tree s2, tree type)
   11488              : {
   11489        99377 :   if (!validate_arg (s1, POINTER_TYPE)
   11490        99377 :       || !validate_arg (s2, POINTER_TYPE))
   11491              :     return NULL_TREE;
   11492              : 
   11493        99377 :   tree fn;
   11494        99377 :   const char *p1, *p2;
   11495              : 
   11496        99377 :   p2 = c_getstr (s2);
   11497        99377 :   if (p2 == NULL)
   11498              :     return NULL_TREE;
   11499              : 
   11500           92 :   p1 = c_getstr (s1);
   11501           92 :   if (p1 != NULL)
   11502              :     {
   11503           22 :       const char *r = strpbrk (p1, p2);
   11504           22 :       tree tem;
   11505              : 
   11506           22 :       if (r == NULL)
   11507            0 :         return build_int_cst (TREE_TYPE (s1), 0);
   11508              : 
   11509              :       /* Return an offset into the constant string argument.  */
   11510           22 :       tem = fold_build_pointer_plus_hwi_loc (loc, s1, r - p1);
   11511           22 :       return fold_convert_loc (loc, type, tem);
   11512              :     }
   11513              : 
   11514           70 :   if (p2[0] == '\0')
   11515              :     /* strpbrk(x, "") == NULL.
   11516              :        Evaluate and ignore s1 in case it had side-effects.  */
   11517           26 :     return omit_one_operand_loc (loc, type, integer_zero_node, s1);
   11518              : 
   11519           44 :   if (p2[1] != '\0')
   11520              :     return NULL_TREE;  /* Really call strpbrk.  */
   11521              : 
   11522           41 :   fn = builtin_decl_implicit (BUILT_IN_STRCHR);
   11523           41 :   if (!fn)
   11524              :     return NULL_TREE;
   11525              : 
   11526              :   /* New argument list transforming strpbrk(s1, s2) to
   11527              :      strchr(s1, s2[0]).  */
   11528           41 :   return build_call_expr_loc (loc, fn, 2, s1,
   11529           41 :                               build_int_cst (integer_type_node, p2[0]));
   11530              : }
   11531              : 
   11532              : /* Simplify a call to the strspn builtin.  S1 and S2 are the arguments
   11533              :    to the call.
   11534              : 
   11535              :    Return NULL_TREE if no simplification was possible, otherwise return the
   11536              :    simplified form of the call as a tree.
   11537              : 
   11538              :    The simplified form may be a constant or other expression which
   11539              :    computes the same value, but in a more efficient manner (including
   11540              :    calls to other builtin functions).
   11541              : 
   11542              :    The call may contain arguments which need to be evaluated, but
   11543              :    which are not useful to determine the result of the call.  In
   11544              :    this case we return a chain of COMPOUND_EXPRs.  The LHS of each
   11545              :    COMPOUND_EXPR will be an argument which must be evaluated.
   11546              :    COMPOUND_EXPRs are chained through their RHS.  The RHS of the last
   11547              :    COMPOUND_EXPR in the chain will contain the tree for the simplified
   11548              :    form of the builtin function call.  */
   11549              : 
   11550              : static tree
   11551         2560 : fold_builtin_strspn (location_t loc, tree expr, tree s1, tree s2, tree type)
   11552              : {
   11553         2560 :   if (!validate_arg (s1, POINTER_TYPE)
   11554         2560 :       || !validate_arg (s2, POINTER_TYPE))
   11555              :     return NULL_TREE;
   11556              : 
   11557         2560 :   if (!check_nul_terminated_array (expr, s1)
   11558         2560 :       || !check_nul_terminated_array (expr, s2))
   11559              :     return NULL_TREE;
   11560              : 
   11561         2502 :   const char *p1 = c_getstr (s1), *p2 = c_getstr (s2);
   11562              : 
   11563              :   /* If either argument is "", return NULL_TREE.  */
   11564         2502 :   if ((p1 && *p1 == '\0') || (p2 && *p2 == '\0'))
   11565              :     /* Evaluate and ignore both arguments in case either one has
   11566              :        side-effects.  */
   11567          147 :     return omit_two_operands_loc (loc, type, size_zero_node, s1, s2);
   11568              :   return NULL_TREE;
   11569              : }
   11570              : 
   11571              : /* Simplify a call to the strcspn builtin.  S1 and S2 are the arguments
   11572              :    to the call.
   11573              : 
   11574              :    Return NULL_TREE if no simplification was possible, otherwise return the
   11575              :    simplified form of the call as a tree.
   11576              : 
   11577              :    The simplified form may be a constant or other expression which
   11578              :    computes the same value, but in a more efficient manner (including
   11579              :    calls to other builtin functions).
   11580              : 
   11581              :    The call may contain arguments which need to be evaluated, but
   11582              :    which are not useful to determine the result of the call.  In
   11583              :    this case we return a chain of COMPOUND_EXPRs.  The LHS of each
   11584              :    COMPOUND_EXPR will be an argument which must be evaluated.
   11585              :    COMPOUND_EXPRs are chained through their RHS.  The RHS of the last
   11586              :    COMPOUND_EXPR in the chain will contain the tree for the simplified
   11587              :    form of the builtin function call.  */
   11588              : 
   11589              : static tree
   11590         2465 : fold_builtin_strcspn (location_t loc, tree expr, tree s1, tree s2, tree type)
   11591              : {
   11592         2465 :   if (!validate_arg (s1, POINTER_TYPE)
   11593         2465 :       || !validate_arg (s2, POINTER_TYPE))
   11594              :     return NULL_TREE;
   11595              : 
   11596         2465 :   if (!check_nul_terminated_array (expr, s1)
   11597         2465 :       || !check_nul_terminated_array (expr, s2))
   11598              :     return NULL_TREE;
   11599              : 
   11600              :   /* If the first argument is "", return NULL_TREE.  */
   11601         2407 :   const char *p1 = c_getstr (s1);
   11602         2407 :   if (p1 && *p1 == '\0')
   11603              :     {
   11604              :       /* Evaluate and ignore argument s2 in case it has
   11605              :          side-effects.  */
   11606           65 :       return omit_one_operand_loc (loc, type, size_zero_node, s2);
   11607              :     }
   11608              : 
   11609              :   /* If the second argument is "", return __builtin_strlen(s1).  */
   11610         2342 :   const char *p2 = c_getstr (s2);
   11611         2342 :   if (p2 && *p2 == '\0')
   11612              :     {
   11613           81 :       tree fn = builtin_decl_implicit (BUILT_IN_STRLEN);
   11614              : 
   11615              :       /* If the replacement _DECL isn't initialized, don't do the
   11616              :          transformation.  */
   11617           81 :       if (!fn)
   11618              :         return NULL_TREE;
   11619              : 
   11620           81 :       return fold_convert_loc (loc, type,
   11621           81 :                                build_call_expr_loc (loc, fn, 1, s1));
   11622              :     }
   11623              :   return NULL_TREE;
   11624              : }
   11625              : 
   11626              : /* Fold the next_arg or va_start call EXP. Returns true if there was an error
   11627              :    produced.  False otherwise.  This is done so that we don't output the error
   11628              :    or warning twice or three times.  */
   11629              : 
   11630              : bool
   11631        42147 : fold_builtin_next_arg (tree exp, bool va_start_p)
   11632              : {
   11633        42147 :   tree fntype = TREE_TYPE (current_function_decl);
   11634        42147 :   int nargs = call_expr_nargs (exp);
   11635        42147 :   tree arg;
   11636              :   /* There is good chance the current input_location points inside the
   11637              :      definition of the va_start macro (perhaps on the token for
   11638              :      builtin) in a system header, so warnings will not be emitted.
   11639              :      Use the location in real source code.  */
   11640        42147 :   location_t current_location =
   11641        42147 :     linemap_unwind_to_first_non_reserved_loc (line_table, input_location,
   11642              :                                               NULL);
   11643              : 
   11644        42147 :   if (!stdarg_p (fntype))
   11645              :     {
   11646            8 :       error ("%<va_start%> used in function with fixed arguments");
   11647            8 :       return true;
   11648              :     }
   11649              : 
   11650        42139 :   if (va_start_p)
   11651              :     {
   11652        41981 :       if (va_start_p && (nargs != 2))
   11653              :         {
   11654            0 :           error ("wrong number of arguments to function %<va_start%>");
   11655            0 :           return true;
   11656              :         }
   11657        41981 :       arg = CALL_EXPR_ARG (exp, 1);
   11658              :     }
   11659              :   /* We use __builtin_va_start (ap, 0, 0) or __builtin_next_arg (0, 0)
   11660              :      when we checked the arguments and if needed issued a warning.  */
   11661              :   else
   11662              :     {
   11663          158 :       if (nargs == 0)
   11664              :         {
   11665              :           /* Evidently an out of date version of <stdarg.h>; can't validate
   11666              :              va_start's second argument, but can still work as intended.  */
   11667            0 :           warning_at (current_location,
   11668            0 :                       OPT_Wvarargs,
   11669              :                    "%<__builtin_next_arg%> called without an argument");
   11670            0 :           return true;
   11671              :         }
   11672          158 :       else if (nargs > 1)
   11673              :         {
   11674            0 :           error ("wrong number of arguments to function %<__builtin_next_arg%>");
   11675            0 :           return true;
   11676              :         }
   11677          158 :       arg = CALL_EXPR_ARG (exp, 0);
   11678              :     }
   11679              : 
   11680        42139 :   if (TREE_CODE (arg) == SSA_NAME
   11681        42139 :       && SSA_NAME_VAR (arg))
   11682              :     arg = SSA_NAME_VAR (arg);
   11683              : 
   11684              :   /* We destructively modify the call to be __builtin_va_start (ap, 0)
   11685              :      or __builtin_next_arg (0) the first time we see it, after checking
   11686              :      the arguments and if needed issuing a warning.  */
   11687        42139 :   if (!integer_zerop (arg))
   11688              :     {
   11689         7031 :       tree last_parm = tree_last (DECL_ARGUMENTS (current_function_decl));
   11690              : 
   11691              :       /* Strip off all nops for the sake of the comparison.  This
   11692              :          is not quite the same as STRIP_NOPS.  It does more.
   11693              :          We must also strip off INDIRECT_EXPR for C++ reference
   11694              :          parameters.  */
   11695        14064 :       while (CONVERT_EXPR_P (arg)
   11696        14075 :              || INDIRECT_REF_P (arg))
   11697           11 :         arg = TREE_OPERAND (arg, 0);
   11698         7031 :       if (arg != last_parm)
   11699              :         {
   11700              :           /* FIXME: Sometimes with the tree optimizers we can get the
   11701              :              not the last argument even though the user used the last
   11702              :              argument.  We just warn and set the arg to be the last
   11703              :              argument so that we will get wrong-code because of
   11704              :              it.  */
   11705           12 :           warning_at (current_location,
   11706           12 :                       OPT_Wvarargs,
   11707              :                       "second parameter of %<va_start%> not last named argument");
   11708              :         }
   11709              : 
   11710              :       /* Undefined by C99 7.15.1.4p4 (va_start):
   11711              :          "If the parameter parmN is declared with the register storage
   11712              :          class, with a function or array type, or with a type that is
   11713              :          not compatible with the type that results after application of
   11714              :          the default argument promotions, the behavior is undefined."
   11715              :       */
   11716         7019 :       else if (DECL_REGISTER (arg))
   11717              :         {
   11718           12 :           warning_at (current_location,
   11719           12 :                       OPT_Wvarargs,
   11720              :                       "undefined behavior when second parameter of "
   11721              :                       "%<va_start%> is declared with %<register%> storage");
   11722              :         }
   11723              : 
   11724              :       /* We want to verify the second parameter just once before the tree
   11725              :          optimizers are run and then avoid keeping it in the tree,
   11726              :          as otherwise we could warn even for correct code like:
   11727              :          void foo (int i, ...)
   11728              :          { va_list ap; i++; va_start (ap, i); va_end (ap); }  */
   11729         7031 :       if (va_start_p)
   11730         7029 :         CALL_EXPR_ARG (exp, 1) = integer_zero_node;
   11731              :       else
   11732            2 :         CALL_EXPR_ARG (exp, 0) = integer_zero_node;
   11733              :     }
   11734              :   return false;
   11735              : }
   11736              : 
   11737              : 
   11738              : /* Expand a call EXP to __builtin_object_size.  */
   11739              : 
   11740              : static rtx
   11741          620 : expand_builtin_object_size (tree exp)
   11742              : {
   11743          620 :   tree ost;
   11744          620 :   int object_size_type;
   11745          620 :   tree fndecl = get_callee_fndecl (exp);
   11746              : 
   11747          620 :   if (!validate_arglist (exp, POINTER_TYPE, INTEGER_TYPE, VOID_TYPE))
   11748              :     {
   11749            0 :       error ("first argument of %qD must be a pointer, second integer constant",
   11750              :              fndecl);
   11751            0 :       expand_builtin_trap ();
   11752            0 :       return const0_rtx;
   11753              :     }
   11754              : 
   11755          620 :   ost = CALL_EXPR_ARG (exp, 1);
   11756          620 :   STRIP_NOPS (ost);
   11757              : 
   11758          620 :   if (TREE_CODE (ost) != INTEGER_CST
   11759          620 :       || tree_int_cst_sgn (ost) < 0
   11760         1240 :       || compare_tree_int (ost, 3) > 0)
   11761              :     {
   11762            0 :       error ("last argument of %qD is not integer constant between 0 and 3",
   11763              :               fndecl);
   11764            0 :       expand_builtin_trap ();
   11765            0 :       return const0_rtx;
   11766              :     }
   11767              : 
   11768          620 :   object_size_type = tree_to_shwi (ost);
   11769              : 
   11770          620 :   return object_size_type < 2 ? constm1_rtx : const0_rtx;
   11771              : }
   11772              : 
   11773              : /* Expand EXP, a call to the __mem{cpy,pcpy,move,set}_chk builtin.
   11774              :    FCODE is the BUILT_IN_* to use.
   11775              :    Return NULL_RTX if we failed; the caller should emit a normal call,
   11776              :    otherwise try to get the result in TARGET, if convenient (and in
   11777              :    mode MODE if that's convenient).  */
   11778              : 
   11779              : static rtx
   11780          837 : expand_builtin_memory_chk (tree exp, rtx target, machine_mode mode,
   11781              :                            enum built_in_function fcode)
   11782              : {
   11783         1410 :   if (!validate_arglist (exp,
   11784              :                          POINTER_TYPE,
   11785              :                          fcode == BUILT_IN_MEMSET_CHK
   11786              :                          ? INTEGER_TYPE : POINTER_TYPE,
   11787              :                          INTEGER_TYPE, INTEGER_TYPE, VOID_TYPE))
   11788              :     return NULL_RTX;
   11789              : 
   11790          837 :   tree dest = CALL_EXPR_ARG (exp, 0);
   11791          837 :   tree src = CALL_EXPR_ARG (exp, 1);
   11792          837 :   tree len = CALL_EXPR_ARG (exp, 2);
   11793          837 :   tree size = CALL_EXPR_ARG (exp, 3);
   11794              : 
   11795              :   /* FIXME: Set access mode to write only for memset et al.  */
   11796          837 :   bool sizes_ok = check_access (exp, len, /*maxread=*/NULL_TREE,
   11797              :                                 /*srcstr=*/NULL_TREE, size, access_read_write);
   11798              : 
   11799          837 :   if (!tree_fits_uhwi_p (size))
   11800              :     return NULL_RTX;
   11801              : 
   11802          627 :   if (tree_fits_uhwi_p (len) || integer_all_onesp (size))
   11803              :     {
   11804              :       /* Avoid transforming the checking call to an ordinary one when
   11805              :          an overflow has been detected or when the call couldn't be
   11806              :          validated because the size is not constant.  */
   11807          186 :       if (!sizes_ok && !integer_all_onesp (size) && tree_int_cst_lt (size, len))
   11808              :         return NULL_RTX;
   11809              : 
   11810            0 :       tree fn = NULL_TREE;
   11811              :       /* If __builtin_mem{cpy,pcpy,move,set}_chk is used, assume
   11812              :          mem{cpy,pcpy,move,set} is available.  */
   11813            0 :       switch (fcode)
   11814              :         {
   11815            0 :         case BUILT_IN_MEMCPY_CHK:
   11816            0 :           fn = builtin_decl_explicit (BUILT_IN_MEMCPY);
   11817            0 :           break;
   11818            0 :         case BUILT_IN_MEMPCPY_CHK:
   11819            0 :           fn = builtin_decl_explicit (BUILT_IN_MEMPCPY);
   11820            0 :           break;
   11821            0 :         case BUILT_IN_MEMMOVE_CHK:
   11822            0 :           fn = builtin_decl_explicit (BUILT_IN_MEMMOVE);
   11823            0 :           break;
   11824            0 :         case BUILT_IN_MEMSET_CHK:
   11825            0 :           fn = builtin_decl_explicit (BUILT_IN_MEMSET);
   11826            0 :           break;
   11827              :         default:
   11828              :           break;
   11829              :         }
   11830              : 
   11831            0 :       if (! fn)
   11832              :         return NULL_RTX;
   11833              : 
   11834            0 :       fn = build_call_nofold_loc (EXPR_LOCATION (exp), fn, 3, dest, src, len);
   11835            0 :       gcc_assert (TREE_CODE (fn) == CALL_EXPR);
   11836            0 :       CALL_EXPR_TAILCALL (fn) = CALL_EXPR_TAILCALL (exp);
   11837            0 :       return expand_expr (fn, target, mode, EXPAND_NORMAL);
   11838              :     }
   11839          441 :   else if (fcode == BUILT_IN_MEMSET_CHK)
   11840              :     return NULL_RTX;
   11841              :   else
   11842              :     {
   11843          293 :       unsigned int dest_align = get_pointer_alignment (dest);
   11844              : 
   11845              :       /* If DEST is not a pointer type, call the normal function.  */
   11846          293 :       if (dest_align == 0)
   11847              :         return NULL_RTX;
   11848              : 
   11849              :       /* If SRC and DEST are the same (and not volatile), do nothing.  */
   11850          293 :       if (operand_equal_p (src, dest, 0))
   11851              :         {
   11852            0 :           tree expr;
   11853              : 
   11854            0 :           if (fcode != BUILT_IN_MEMPCPY_CHK)
   11855              :             {
   11856              :               /* Evaluate and ignore LEN in case it has side-effects.  */
   11857            0 :               expand_expr (len, const0_rtx, VOIDmode, EXPAND_NORMAL);
   11858            0 :               return expand_expr (dest, target, mode, EXPAND_NORMAL);
   11859              :             }
   11860              : 
   11861            0 :           expr = fold_build_pointer_plus (dest, len);
   11862            0 :           return expand_expr (expr, target, mode, EXPAND_NORMAL);
   11863              :         }
   11864              : 
   11865              :       /* __memmove_chk special case.  */
   11866          293 :       if (fcode == BUILT_IN_MEMMOVE_CHK)
   11867              :         {
   11868           79 :           unsigned int src_align = get_pointer_alignment (src);
   11869              : 
   11870           79 :           if (src_align == 0)
   11871              :             return NULL_RTX;
   11872              : 
   11873              :           /* If src is categorized for a readonly section we can use
   11874              :              normal __memcpy_chk.  */
   11875           79 :           if (readonly_data_expr (src))
   11876              :             {
   11877           15 :               tree fn = builtin_decl_explicit (BUILT_IN_MEMCPY_CHK);
   11878           15 :               if (!fn)
   11879              :                 return NULL_RTX;
   11880           15 :               fn = build_call_nofold_loc (EXPR_LOCATION (exp), fn, 4,
   11881              :                                           dest, src, len, size);
   11882           15 :               gcc_assert (TREE_CODE (fn) == CALL_EXPR);
   11883           15 :               CALL_EXPR_TAILCALL (fn) = CALL_EXPR_TAILCALL (exp);
   11884           15 :               return expand_expr (fn, target, mode, EXPAND_NORMAL);
   11885              :             }
   11886              :         }
   11887              :       return NULL_RTX;
   11888              :     }
   11889              : }
   11890              : 
   11891              : /* Emit warning if a buffer overflow is detected at compile time.  */
   11892              : 
   11893              : static void
   11894         1086 : maybe_emit_chk_warning (tree exp, enum built_in_function fcode)
   11895              : {
   11896              :   /* The source string.  */
   11897         1086 :   tree srcstr = NULL_TREE;
   11898              :   /* The size of the destination object returned by __builtin_object_size.  */
   11899         1086 :   tree objsize = NULL_TREE;
   11900              :   /* The string that is being concatenated with (as in __strcat_chk)
   11901              :      or null if it isn't.  */
   11902         1086 :   tree catstr = NULL_TREE;
   11903              :   /* The maximum length of the source sequence in a bounded operation
   11904              :      (such as __strncat_chk) or null if the operation isn't bounded
   11905              :      (such as __strcat_chk).  */
   11906         1086 :   tree maxread = NULL_TREE;
   11907              :   /* The exact size of the access (such as in __strncpy_chk).  */
   11908         1086 :   tree size = NULL_TREE;
   11909              :   /* The access by the function that's checked.  Except for snprintf
   11910              :      both writing and reading is checked.  */
   11911         1086 :   access_mode mode = access_read_write;
   11912              : 
   11913         1086 :   switch (fcode)
   11914              :     {
   11915          275 :     case BUILT_IN_STRCPY_CHK:
   11916          275 :     case BUILT_IN_STPCPY_CHK:
   11917          275 :       srcstr = CALL_EXPR_ARG (exp, 1);
   11918          275 :       objsize = CALL_EXPR_ARG (exp, 2);
   11919          275 :       break;
   11920              : 
   11921          151 :     case BUILT_IN_STRCAT_CHK:
   11922              :       /* For __strcat_chk the warning will be emitted only if overflowing
   11923              :          by at least strlen (dest) + 1 bytes.  */
   11924          151 :       catstr = CALL_EXPR_ARG (exp, 0);
   11925          151 :       srcstr = CALL_EXPR_ARG (exp, 1);
   11926          151 :       objsize = CALL_EXPR_ARG (exp, 2);
   11927          151 :       break;
   11928              : 
   11929          108 :     case BUILT_IN_STRNCAT_CHK:
   11930          108 :       catstr = CALL_EXPR_ARG (exp, 0);
   11931          108 :       srcstr = CALL_EXPR_ARG (exp, 1);
   11932          108 :       maxread = CALL_EXPR_ARG (exp, 2);
   11933          108 :       objsize = CALL_EXPR_ARG (exp, 3);
   11934          108 :       break;
   11935              : 
   11936          243 :     case BUILT_IN_STRNCPY_CHK:
   11937          243 :     case BUILT_IN_STPNCPY_CHK:
   11938          243 :       srcstr = CALL_EXPR_ARG (exp, 1);
   11939          243 :       size = CALL_EXPR_ARG (exp, 2);
   11940          243 :       objsize = CALL_EXPR_ARG (exp, 3);
   11941          243 :       break;
   11942              : 
   11943          309 :     case BUILT_IN_SNPRINTF_CHK:
   11944          309 :     case BUILT_IN_VSNPRINTF_CHK:
   11945          309 :       maxread = CALL_EXPR_ARG (exp, 1);
   11946          309 :       objsize = CALL_EXPR_ARG (exp, 3);
   11947              :       /* The only checked access the write to the destination.  */
   11948          309 :       mode = access_write_only;
   11949          309 :       break;
   11950            0 :     default:
   11951            0 :       gcc_unreachable ();
   11952              :     }
   11953              : 
   11954         1086 :   if (catstr && maxread)
   11955              :     {
   11956              :       /* Check __strncat_chk.  There is no way to determine the length
   11957              :          of the string to which the source string is being appended so
   11958              :          just warn when the length of the source string is not known.  */
   11959          108 :       check_strncat_sizes (exp, objsize);
   11960          108 :       return;
   11961              :     }
   11962              : 
   11963          978 :   check_access (exp, size, maxread, srcstr, objsize, mode);
   11964              : }
   11965              : 
   11966              : /* Emit warning if a buffer overflow is detected at compile time
   11967              :    in __sprintf_chk/__vsprintf_chk calls.  */
   11968              : 
   11969              : static void
   11970         1328 : maybe_emit_sprintf_chk_warning (tree exp, enum built_in_function fcode)
   11971              : {
   11972         1328 :   tree size, len, fmt;
   11973         1328 :   const char *fmt_str;
   11974         1328 :   int nargs = call_expr_nargs (exp);
   11975              : 
   11976              :   /* Verify the required arguments in the original call.  */
   11977              : 
   11978         1328 :   if (nargs < 4)
   11979              :     return;
   11980         1328 :   size = CALL_EXPR_ARG (exp, 2);
   11981         1328 :   fmt = CALL_EXPR_ARG (exp, 3);
   11982              : 
   11983         1328 :   if (! tree_fits_uhwi_p (size) || integer_all_onesp (size))
   11984              :     return;
   11985              : 
   11986              :   /* Check whether the format is a literal string constant.  */
   11987         1307 :   fmt_str = c_getstr (fmt);
   11988         1307 :   if (fmt_str == NULL)
   11989              :     return;
   11990              : 
   11991         1271 :   if (!init_target_chars ())
   11992              :     return;
   11993              : 
   11994              :   /* If the format doesn't contain % args or %%, we know its size.  */
   11995         1271 :   if (strchr (fmt_str, target_percent) == 0)
   11996           22 :     len = build_int_cstu (size_type_node, strlen (fmt_str));
   11997              :   /* If the format is "%s" and first ... argument is a string literal,
   11998              :      we know it too.  */
   11999         1249 :   else if (fcode == BUILT_IN_SPRINTF_CHK
   12000         1095 :            && strcmp (fmt_str, target_percent_s) == 0)
   12001              :     {
   12002           49 :       tree arg;
   12003              : 
   12004           49 :       if (nargs < 5)
   12005              :         return;
   12006           49 :       arg = CALL_EXPR_ARG (exp, 4);
   12007           49 :       if (! POINTER_TYPE_P (TREE_TYPE (arg)))
   12008              :         return;
   12009              : 
   12010           45 :       len = c_strlen (arg, 1);
   12011           45 :       if (!len || ! tree_fits_uhwi_p (len))
   12012              :         return;
   12013              :     }
   12014              :   else
   12015              :     return;
   12016              : 
   12017              :   /* Add one for the terminating nul.  */
   12018           34 :   len = fold_build2 (PLUS_EXPR, TREE_TYPE (len), len, size_one_node);
   12019              : 
   12020           34 :   check_access (exp, /*size=*/NULL_TREE, /*maxread=*/NULL_TREE, len, size,
   12021              :                 access_write_only);
   12022              : }
   12023              : 
   12024              : /* Fold a call to __builtin_object_size with arguments PTR and OST,
   12025              :    if possible.  */
   12026              : 
   12027              : static tree
   12028       201177 : fold_builtin_object_size (tree ptr, tree ost, enum built_in_function fcode)
   12029              : {
   12030       201177 :   tree bytes;
   12031       201177 :   int object_size_type;
   12032              : 
   12033       201177 :   if (!validate_arg (ptr, POINTER_TYPE)
   12034       201177 :       || !validate_arg (ost, INTEGER_TYPE))
   12035              :     return NULL_TREE;
   12036              : 
   12037       201177 :   STRIP_NOPS (ost);
   12038              : 
   12039       201177 :   if (TREE_CODE (ost) != INTEGER_CST
   12040       201177 :       || tree_int_cst_sgn (ost) < 0
   12041       402354 :       || compare_tree_int (ost, 3) > 0)
   12042              :     return NULL_TREE;
   12043              : 
   12044       201177 :   object_size_type = tree_to_shwi (ost);
   12045              : 
   12046              :   /* __builtin_object_size doesn't evaluate side-effects in its arguments;
   12047              :      if there are any side-effects, it returns (size_t) -1 for types 0 and 1
   12048              :      and (size_t) 0 for types 2 and 3.  */
   12049       201177 :   if (TREE_SIDE_EFFECTS (ptr))
   12050          603 :     return build_int_cst_type (size_type_node, object_size_type < 2 ? -1 : 0);
   12051              : 
   12052       200574 :   if (fcode == BUILT_IN_DYNAMIC_OBJECT_SIZE)
   12053        67524 :     object_size_type |= OST_DYNAMIC;
   12054              : 
   12055       200574 :   if (TREE_CODE (ptr) == ADDR_EXPR)
   12056              :     {
   12057        12244 :       compute_builtin_object_size (ptr, object_size_type, &bytes);
   12058        12244 :       if ((object_size_type & OST_DYNAMIC)
   12059        10892 :           || int_fits_type_p (bytes, size_type_node))
   12060        12244 :         return fold_convert (size_type_node, bytes);
   12061              :     }
   12062       188330 :   else if (TREE_CODE (ptr) == SSA_NAME)
   12063              :     {
   12064              :       /* If object size is not known yet, delay folding until
   12065              :        later.  Maybe subsequent passes will help determining
   12066              :        it.  */
   12067       102272 :       if (compute_builtin_object_size (ptr, object_size_type, &bytes)
   12068       102272 :           && ((object_size_type & OST_DYNAMIC)
   12069         2538 :               || int_fits_type_p (bytes, size_type_node)))
   12070         4808 :         return fold_convert (size_type_node, bytes);
   12071              :     }
   12072              : 
   12073              :   return NULL_TREE;
   12074              : }
   12075              : 
   12076              : /* Builtins with folding operations that operate on "..." arguments
   12077              :    need special handling; we need to store the arguments in a convenient
   12078              :    data structure before attempting any folding.  Fortunately there are
   12079              :    only a few builtins that fall into this category.  FNDECL is the
   12080              :    function, EXP is the CALL_EXPR for the call.  */
   12081              : 
   12082              : static tree
   12083      2648066 : fold_builtin_varargs (location_t loc, tree fndecl, tree *args, int nargs)
   12084              : {
   12085      2648066 :   enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
   12086      2648066 :   tree ret = NULL_TREE;
   12087              : 
   12088      2648066 :   switch (fcode)
   12089              :     {
   12090       104166 :     case BUILT_IN_FPCLASSIFY:
   12091       104166 :       ret = fold_builtin_fpclassify (loc, args, nargs);
   12092       104166 :       break;
   12093              : 
   12094           55 :     case BUILT_IN_ADDC:
   12095           55 :     case BUILT_IN_ADDCL:
   12096           55 :     case BUILT_IN_ADDCLL:
   12097           55 :     case BUILT_IN_SUBC:
   12098           55 :     case BUILT_IN_SUBCL:
   12099           55 :     case BUILT_IN_SUBCLL:
   12100           55 :       return fold_builtin_addc_subc (loc, fcode, args);
   12101              : 
   12102              :     default:
   12103              :       break;
   12104              :     }
   12105       104166 :   if (ret)
   12106              :     {
   12107       104166 :       ret = build1 (NOP_EXPR, TREE_TYPE (ret), ret);
   12108       104166 :       SET_EXPR_LOCATION (ret, loc);
   12109       104166 :       suppress_warning (ret);
   12110       104166 :       return ret;
   12111              :     }
   12112              :   return NULL_TREE;
   12113              : }
   12114              : 
   12115              : /* Initialize format string characters in the target charset.  */
   12116              : 
   12117              : bool
   12118       291692 : init_target_chars (void)
   12119              : {
   12120       291692 :   static bool init;
   12121       291692 :   if (!init)
   12122              :     {
   12123       116496 :       target_newline = lang_hooks.to_target_charset ('\n');
   12124       116496 :       target_percent = lang_hooks.to_target_charset ('%');
   12125       116496 :       target_c = lang_hooks.to_target_charset ('c');
   12126       116496 :       target_s = lang_hooks.to_target_charset ('s');
   12127       116496 :       if (target_newline == 0 || target_percent == 0 || target_c == 0
   12128       116496 :           || target_s == 0)
   12129              :         return false;
   12130              : 
   12131       116496 :       target_percent_c[0] = target_percent;
   12132       116496 :       target_percent_c[1] = target_c;
   12133       116496 :       target_percent_c[2] = '\0';
   12134              : 
   12135       116496 :       target_percent_s[0] = target_percent;
   12136       116496 :       target_percent_s[1] = target_s;
   12137       116496 :       target_percent_s[2] = '\0';
   12138              : 
   12139       116496 :       target_percent_s_newline[0] = target_percent;
   12140       116496 :       target_percent_s_newline[1] = target_s;
   12141       116496 :       target_percent_s_newline[2] = target_newline;
   12142       116496 :       target_percent_s_newline[3] = '\0';
   12143              : 
   12144       116496 :       init = true;
   12145              :     }
   12146              :   return true;
   12147              : }
   12148              : 
   12149              : /* Helper function for do_mpfr_arg*().  Ensure M is a normal number
   12150              :    and no overflow/underflow occurred.  INEXACT is true if M was not
   12151              :    exactly calculated.  TYPE is the tree type for the result.  This
   12152              :    function assumes that you cleared the MPFR flags and then
   12153              :    calculated M to see if anything subsequently set a flag prior to
   12154              :    entering this function.  Return NULL_TREE if any checks fail.  */
   12155              : 
   12156              : static tree
   12157         2979 : do_mpfr_ckconv (mpfr_srcptr m, tree type, int inexact)
   12158              : {
   12159              :   /* Proceed iff we get a normal number, i.e. not NaN or Inf and no
   12160              :      overflow/underflow occurred.  If -frounding-math, proceed iff the
   12161              :      result of calling FUNC was exact.  */
   12162         4832 :   if (mpfr_number_p (m) && !mpfr_overflow_p () && !mpfr_underflow_p ()
   12163         4832 :       && (!flag_rounding_math || !inexact))
   12164              :     {
   12165         1853 :       REAL_VALUE_TYPE rr;
   12166              : 
   12167         1853 :       real_from_mpfr (&rr, m, type, MPFR_RNDN);
   12168              :       /* Proceed iff GCC's REAL_VALUE_TYPE can hold the MPFR value,
   12169              :          check for overflow/underflow.  If the REAL_VALUE_TYPE is zero
   12170              :          but the mpfr_t is not, then we underflowed in the
   12171              :          conversion.  */
   12172         1853 :       if (real_isfinite (&rr)
   12173         1853 :           && (rr.cl == rvc_zero) == (mpfr_zero_p (m) != 0))
   12174              :         {
   12175         1853 :           REAL_VALUE_TYPE rmode;
   12176              : 
   12177         1853 :           real_convert (&rmode, TYPE_MODE (type), &rr);
   12178              :           /* Proceed iff the specified mode can hold the value.  */
   12179         1853 :           if (real_identical (&rmode, &rr))
   12180         1853 :             return build_real (type, rmode);
   12181              :         }
   12182              :     }
   12183              :   return NULL_TREE;
   12184              : }
   12185              : 
   12186              : /* Helper function for do_mpc_arg*().  Ensure M is a normal complex
   12187              :    number and no overflow/underflow occurred.  INEXACT is true if M
   12188              :    was not exactly calculated.  TYPE is the tree type for the result.
   12189              :    This function assumes that you cleared the MPFR flags and then
   12190              :    calculated M to see if anything subsequently set a flag prior to
   12191              :    entering this function.  Return NULL_TREE if any checks fail, if
   12192              :    FORCE_CONVERT is true, then bypass the checks.  */
   12193              : 
   12194              : static tree
   12195         4173 : do_mpc_ckconv (mpc_srcptr m, tree type, int inexact, int force_convert)
   12196              : {
   12197              :   /* Proceed iff we get a normal number, i.e. not NaN or Inf and no
   12198              :      overflow/underflow occurred.  If -frounding-math, proceed iff the
   12199              :      result of calling FUNC was exact.  */
   12200         4173 :   if (force_convert
   12201         4173 :       || (mpfr_number_p (mpc_realref (m)) && mpfr_number_p (mpc_imagref (m))
   12202         3854 :           && !mpfr_overflow_p () && !mpfr_underflow_p ()
   12203         3854 :           && (!flag_rounding_math || !inexact)))
   12204              :     {
   12205         3994 :       REAL_VALUE_TYPE re, im;
   12206              : 
   12207         3994 :       real_from_mpfr (&re, mpc_realref (m), TREE_TYPE (type), MPFR_RNDN);
   12208         3994 :       real_from_mpfr (&im, mpc_imagref (m), TREE_TYPE (type), MPFR_RNDN);
   12209              :       /* Proceed iff GCC's REAL_VALUE_TYPE can hold the MPFR values,
   12210              :          check for overflow/underflow.  If the REAL_VALUE_TYPE is zero
   12211              :          but the mpfr_t is not, then we underflowed in the
   12212              :          conversion.  */
   12213         3994 :       if (force_convert
   12214         3994 :           || (real_isfinite (&re) && real_isfinite (&im)
   12215         3854 :               && (re.cl == rvc_zero) == (mpfr_zero_p (mpc_realref (m)) != 0)
   12216         3854 :               && (im.cl == rvc_zero) == (mpfr_zero_p (mpc_imagref (m)) != 0)))
   12217              :         {
   12218         3994 :           REAL_VALUE_TYPE re_mode, im_mode;
   12219              : 
   12220         3994 :           real_convert (&re_mode, TYPE_MODE (TREE_TYPE (type)), &re);
   12221         3994 :           real_convert (&im_mode, TYPE_MODE (TREE_TYPE (type)), &im);
   12222              :           /* Proceed iff the specified mode can hold the value.  */
   12223         3994 :           if (force_convert
   12224         3994 :               || (real_identical (&re_mode, &re)
   12225         3854 :                   && real_identical (&im_mode, &im)))
   12226         3994 :             return build_complex (type, build_real (TREE_TYPE (type), re_mode),
   12227         7988 :                                   build_real (TREE_TYPE (type), im_mode));
   12228              :         }
   12229              :     }
   12230              :   return NULL_TREE;
   12231              : }
   12232              : 
   12233              : /* If arguments ARG0 and ARG1 are REAL_CSTs, call mpfr_remquo() to set
   12234              :    the pointer *(ARG_QUO) and return the result.  The type is taken
   12235              :    from the type of ARG0 and is used for setting the precision of the
   12236              :    calculation and results.  */
   12237              : 
   12238              : static tree
   12239        99724 : do_mpfr_remquo (tree arg0, tree arg1, tree arg_quo)
   12240              : {
   12241        99724 :   tree const type = TREE_TYPE (arg0);
   12242        99724 :   tree result = NULL_TREE;
   12243              : 
   12244        99724 :   STRIP_NOPS (arg0);
   12245        99724 :   STRIP_NOPS (arg1);
   12246              : 
   12247              :   /* To proceed, MPFR must exactly represent the target floating point
   12248              :      format, which only happens when the target base equals two.  */
   12249        99724 :   if (REAL_MODE_FORMAT (TYPE_MODE (type))->b == 2
   12250        99724 :       && TREE_CODE (arg0) == REAL_CST && !TREE_OVERFLOW (arg0)
   12251       102029 :       && TREE_CODE (arg1) == REAL_CST && !TREE_OVERFLOW (arg1))
   12252              :     {
   12253         2305 :       const REAL_VALUE_TYPE *const ra0 = TREE_REAL_CST_PTR (arg0);
   12254         2305 :       const REAL_VALUE_TYPE *const ra1 = TREE_REAL_CST_PTR (arg1);
   12255              : 
   12256         2305 :       if (real_isfinite (ra0) && real_isfinite (ra1))
   12257              :         {
   12258         2305 :           const struct real_format *fmt = REAL_MODE_FORMAT (TYPE_MODE (type));
   12259         2305 :           const int prec = fmt->p;
   12260         2305 :           const mpfr_rnd_t rnd = fmt->round_towards_zero? MPFR_RNDZ : MPFR_RNDN;
   12261         2305 :           tree result_rem;
   12262         2305 :           long integer_quo;
   12263         2305 :           mpfr_t m0, m1;
   12264              : 
   12265         2305 :           mpfr_inits2 (prec, m0, m1, NULL);
   12266         2305 :           mpfr_from_real (m0, ra0, MPFR_RNDN);
   12267         2305 :           mpfr_from_real (m1, ra1, MPFR_RNDN);
   12268         2305 :           mpfr_clear_flags ();
   12269         2305 :           mpfr_remquo (m0, &integer_quo, m0, m1, rnd);
   12270              :           /* Remquo is independent of the rounding mode, so pass
   12271              :              inexact=0 to do_mpfr_ckconv().  */
   12272         2305 :           result_rem = do_mpfr_ckconv (m0, type, /*inexact=*/ 0);
   12273         2305 :           mpfr_clears (m0, m1, NULL);
   12274         2305 :           if (result_rem)
   12275              :             {
   12276              :               /* MPFR calculates quo in the host's long so it may
   12277              :                  return more bits in quo than the target int can hold
   12278              :                  if sizeof(host long) > sizeof(target int).  This can
   12279              :                  happen even for native compilers in LP64 mode.  In
   12280              :                  these cases, modulo the quo value with the largest
   12281              :                  number that the target int can hold while leaving one
   12282              :                  bit for the sign.  */
   12283         1179 :               if (sizeof (integer_quo) * CHAR_BIT > INT_TYPE_SIZE)
   12284         1179 :                 integer_quo %= (long)(1UL << (INT_TYPE_SIZE - 1));
   12285              : 
   12286              :               /* Dereference the quo pointer argument.  */
   12287         1179 :               arg_quo = build_fold_indirect_ref (arg_quo);
   12288              :               /* Proceed iff a valid pointer type was passed in.  */
   12289         1179 :               if (TYPE_MAIN_VARIANT (TREE_TYPE (arg_quo)) == integer_type_node)
   12290              :                 {
   12291              :                   /* Set the value. */
   12292         1179 :                   tree result_quo
   12293         1179 :                     = fold_build2 (MODIFY_EXPR, TREE_TYPE (arg_quo), arg_quo,
   12294              :                                    build_int_cst (TREE_TYPE (arg_quo),
   12295              :                                                   integer_quo));
   12296         1179 :                   TREE_SIDE_EFFECTS (result_quo) = 1;
   12297              :                   /* Combine the quo assignment with the rem.  */
   12298         1179 :                   result = fold_build2 (COMPOUND_EXPR, type,
   12299              :                                         result_quo, result_rem);
   12300         1179 :                   suppress_warning (result, OPT_Wunused_value);
   12301         1179 :                   result = non_lvalue (result);
   12302              :                 }
   12303              :             }
   12304              :         }
   12305              :     }
   12306        99724 :   return result;
   12307              : }
   12308              : 
   12309              : /* If ARG is a REAL_CST, call mpfr_lgamma() on it and return the
   12310              :    resulting value as a tree with type TYPE.  The mpfr precision is
   12311              :    set to the precision of TYPE.  We assume that this mpfr function
   12312              :    returns zero if the result could be calculated exactly within the
   12313              :    requested precision.  In addition, the integer pointer represented
   12314              :    by ARG_SG will be dereferenced and set to the appropriate signgam
   12315              :    (-1,1) value.  */
   12316              : 
   12317              : static tree
   12318         6504 : do_mpfr_lgamma_r (tree arg, tree arg_sg, tree type)
   12319              : {
   12320         6504 :   tree result = NULL_TREE;
   12321              : 
   12322         6504 :   STRIP_NOPS (arg);
   12323              : 
   12324              :   /* To proceed, MPFR must exactly represent the target floating point
   12325              :      format, which only happens when the target base equals two.  Also
   12326              :      verify ARG is a constant and that ARG_SG is an int pointer.  */
   12327         6504 :   if (REAL_MODE_FORMAT (TYPE_MODE (type))->b == 2
   12328         6504 :       && TREE_CODE (arg) == REAL_CST && !TREE_OVERFLOW (arg)
   12329         6470 :       && TREE_CODE (TREE_TYPE (arg_sg)) == POINTER_TYPE
   12330        12974 :       && TYPE_MAIN_VARIANT (TREE_TYPE (TREE_TYPE (arg_sg))) == integer_type_node)
   12331              :     {
   12332         6470 :       const REAL_VALUE_TYPE *const ra = TREE_REAL_CST_PTR (arg);
   12333              : 
   12334              :       /* In addition to NaN and Inf, the argument cannot be zero or a
   12335              :          negative integer.  */
   12336         6470 :       if (real_isfinite (ra)
   12337         6470 :           && ra->cl != rvc_zero
   12338        12940 :           && !(real_isneg (ra) && real_isinteger (ra, TYPE_MODE (type))))
   12339              :         {
   12340          674 :           const struct real_format *fmt = REAL_MODE_FORMAT (TYPE_MODE (type));
   12341          674 :           const int prec = fmt->p;
   12342          674 :           const mpfr_rnd_t rnd = fmt->round_towards_zero? MPFR_RNDZ : MPFR_RNDN;
   12343          674 :           int inexact, sg;
   12344          674 :           tree result_lg;
   12345              : 
   12346          674 :           auto_mpfr m (prec);
   12347          674 :           mpfr_from_real (m, ra, MPFR_RNDN);
   12348          674 :           mpfr_clear_flags ();
   12349          674 :           inexact = mpfr_lgamma (m, &sg, m, rnd);
   12350          674 :           result_lg = do_mpfr_ckconv (m, type, inexact);
   12351          674 :           if (result_lg)
   12352              :             {
   12353          674 :               tree result_sg;
   12354              : 
   12355              :               /* Dereference the arg_sg pointer argument.  */
   12356          674 :               arg_sg = build_fold_indirect_ref (arg_sg);
   12357              :               /* Assign the signgam value into *arg_sg. */
   12358          674 :               result_sg = fold_build2 (MODIFY_EXPR,
   12359              :                                        TREE_TYPE (arg_sg), arg_sg,
   12360              :                                        build_int_cst (TREE_TYPE (arg_sg), sg));
   12361          674 :               TREE_SIDE_EFFECTS (result_sg) = 1;
   12362              :               /* Combine the signgam assignment with the lgamma result.  */
   12363          674 :               result = non_lvalue (fold_build2 (COMPOUND_EXPR, type,
   12364              :                                                 result_sg, result_lg));
   12365              :             }
   12366          674 :         }
   12367              :     }
   12368              : 
   12369         6504 :   return result;
   12370              : }
   12371              : 
   12372              : /* If arguments ARG0 and ARG1 are a COMPLEX_CST, call the two-argument
   12373              :    mpc function FUNC on it and return the resulting value as a tree
   12374              :    with type TYPE.  The mpfr precision is set to the precision of
   12375              :    TYPE.  We assume that function FUNC returns zero if the result
   12376              :    could be calculated exactly within the requested precision.  If
   12377              :    DO_NONFINITE is true, then fold expressions containing Inf or NaN
   12378              :    in the arguments and/or results.  */
   12379              : 
   12380              : tree
   12381         4525 : do_mpc_arg2 (tree arg0, tree arg1, tree type, int do_nonfinite,
   12382              :              int (*func)(mpc_ptr, mpc_srcptr, mpc_srcptr, mpc_rnd_t))
   12383              : {
   12384         4525 :   tree result = NULL_TREE;
   12385              : 
   12386         4525 :   STRIP_NOPS (arg0);
   12387         4525 :   STRIP_NOPS (arg1);
   12388              : 
   12389              :   /* To proceed, MPFR must exactly represent the target floating point
   12390              :      format, which only happens when the target base equals two.  */
   12391         4525 :   if (TREE_CODE (arg0) == COMPLEX_CST && !TREE_OVERFLOW (arg0)
   12392         4525 :       && SCALAR_FLOAT_TYPE_P (TREE_TYPE (TREE_TYPE (arg0)))
   12393         4525 :       && TREE_CODE (arg1) == COMPLEX_CST && !TREE_OVERFLOW (arg1)
   12394         4525 :       && SCALAR_FLOAT_TYPE_P (TREE_TYPE (TREE_TYPE (arg1)))
   12395         9050 :       && REAL_MODE_FORMAT (TYPE_MODE (TREE_TYPE (TREE_TYPE (arg0))))->b == 2)
   12396              :     {
   12397         4525 :       const REAL_VALUE_TYPE *const re0 = TREE_REAL_CST_PTR (TREE_REALPART (arg0));
   12398         4525 :       const REAL_VALUE_TYPE *const im0 = TREE_REAL_CST_PTR (TREE_IMAGPART (arg0));
   12399         4525 :       const REAL_VALUE_TYPE *const re1 = TREE_REAL_CST_PTR (TREE_REALPART (arg1));
   12400         4525 :       const REAL_VALUE_TYPE *const im1 = TREE_REAL_CST_PTR (TREE_IMAGPART (arg1));
   12401              : 
   12402         4525 :       if (do_nonfinite
   12403         4525 :           || (real_isfinite (re0) && real_isfinite (im0)
   12404         4061 :               && real_isfinite (re1) && real_isfinite (im1)))
   12405              :         {
   12406         4173 :           const struct real_format *const fmt =
   12407         4173 :             REAL_MODE_FORMAT (TYPE_MODE (TREE_TYPE (type)));
   12408         4173 :           const int prec = fmt->p;
   12409         4173 :           const mpfr_rnd_t rnd = fmt->round_towards_zero
   12410         4173 :                                  ? MPFR_RNDZ : MPFR_RNDN;
   12411         4173 :           const mpc_rnd_t crnd = fmt->round_towards_zero ? MPC_RNDZZ : MPC_RNDNN;
   12412         4173 :           int inexact;
   12413         4173 :           mpc_t m0, m1;
   12414              : 
   12415         4173 :           mpc_init2 (m0, prec);
   12416         4173 :           mpc_init2 (m1, prec);
   12417         4173 :           mpfr_from_real (mpc_realref (m0), re0, rnd);
   12418         4173 :           mpfr_from_real (mpc_imagref (m0), im0, rnd);
   12419         4173 :           mpfr_from_real (mpc_realref (m1), re1, rnd);
   12420         4173 :           mpfr_from_real (mpc_imagref (m1), im1, rnd);
   12421         4173 :           mpfr_clear_flags ();
   12422         4173 :           inexact = func (m0, m0, m1, crnd);
   12423         4173 :           result = do_mpc_ckconv (m0, type, inexact, do_nonfinite);
   12424         4173 :           mpc_clear (m0);
   12425         4173 :           mpc_clear (m1);
   12426              :         }
   12427              :     }
   12428              : 
   12429         4525 :   return result;
   12430              : }
   12431              : 
   12432              : /* A wrapper function for builtin folding that prevents warnings for
   12433              :    "statement without effect" and the like, caused by removing the
   12434              :    call node earlier than the warning is generated.  */
   12435              : 
   12436              : tree
   12437      6199858 : fold_call_stmt (gcall *stmt, bool ignore)
   12438              : {
   12439      6199858 :   tree ret = NULL_TREE;
   12440      6199858 :   tree fndecl = gimple_call_fndecl (stmt);
   12441      6199858 :   location_t loc = gimple_location (stmt);
   12442      6199858 :   if (fndecl && fndecl_built_in_p (fndecl)
   12443     12399716 :       && !gimple_call_va_arg_pack_p (stmt))
   12444              :     {
   12445      6199792 :       int nargs = gimple_call_num_args (stmt);
   12446      6199792 :       tree *args = (nargs > 0
   12447      6199792 :                     ? gimple_call_arg_ptr (stmt, 0)
   12448              :                     : &error_mark_node);
   12449              : 
   12450      6199792 :       if (avoid_folding_inline_builtin (fndecl))
   12451              :         return NULL_TREE;
   12452      6199792 :       if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_MD)
   12453              :         {
   12454            0 :           return targetm.fold_builtin (fndecl, nargs, args, ignore);
   12455              :         }
   12456              :       else
   12457              :         {
   12458      6199792 :           ret = fold_builtin_n (loc, NULL_TREE, fndecl, args, nargs, ignore);
   12459      6199792 :           if (ret)
   12460              :             {
   12461              :               /* Propagate location information from original call to
   12462              :                  expansion of builtin.  Otherwise things like
   12463              :                  maybe_emit_chk_warning, that operate on the expansion
   12464              :                  of a builtin, will use the wrong location information.  */
   12465         5131 :               if (gimple_has_location (stmt))
   12466              :                 {
   12467         5130 :                   tree realret = ret;
   12468         5130 :                   if (TREE_CODE (ret) == NOP_EXPR)
   12469         5130 :                     realret = TREE_OPERAND (ret, 0);
   12470         5130 :                   if (CAN_HAVE_LOCATION_P (realret)
   12471         9390 :                       && !EXPR_HAS_LOCATION (realret))
   12472            6 :                     SET_EXPR_LOCATION (realret, loc);
   12473              :                   return realret;
   12474              :                 }
   12475              :               return ret;
   12476              :             }
   12477              :         }
   12478              :     }
   12479              :   return NULL_TREE;
   12480              : }
   12481              : 
   12482              : /* Look up the function in builtin_decl that corresponds to DECL
   12483              :    and set ASMSPEC as its user assembler name.  DECL must be a
   12484              :    function decl that declares a builtin.  */
   12485              : 
   12486              : void
   12487       154042 : set_builtin_user_assembler_name (tree decl, const char *asmspec)
   12488              : {
   12489       154042 :   gcc_assert (fndecl_built_in_p (decl, BUILT_IN_NORMAL)
   12490              :               && asmspec != 0);
   12491              : 
   12492       154042 :   tree builtin = builtin_decl_explicit (DECL_FUNCTION_CODE (decl));
   12493       154042 :   set_user_assembler_name (builtin, asmspec);
   12494              : 
   12495       154042 :   if (DECL_FUNCTION_CODE (decl) == BUILT_IN_FFS
   12496       154042 :       && INT_TYPE_SIZE < BITS_PER_WORD)
   12497              :     {
   12498            1 :       scalar_int_mode mode = int_mode_for_size (INT_TYPE_SIZE, 0).require ();
   12499            1 :       set_user_assembler_libfunc ("ffs", asmspec);
   12500            1 :       set_optab_libfunc (ffs_optab, mode, "ffs");
   12501              :     }
   12502       154042 : }
   12503              : 
   12504              : /* Return true if DECL is a builtin that expands to a constant or similarly
   12505              :    simple code.  */
   12506              : bool
   12507     32724441 : is_simple_builtin (tree decl)
   12508              : {
   12509     32724441 :   if (decl && fndecl_built_in_p (decl, BUILT_IN_NORMAL))
   12510     31401791 :     switch (DECL_FUNCTION_CODE (decl))
   12511              :       {
   12512              :         /* Builtins that expand to constants.  */
   12513              :       case BUILT_IN_CONSTANT_P:
   12514              :       case BUILT_IN_EXPECT:
   12515              :       case BUILT_IN_OBJECT_SIZE:
   12516              :       case BUILT_IN_UNREACHABLE:
   12517              :         /* Simple register moves or loads from stack.  */
   12518              :       case BUILT_IN_ASSUME_ALIGNED:
   12519              :       case BUILT_IN_RETURN_ADDRESS:
   12520              :       case BUILT_IN_EXTRACT_RETURN_ADDR:
   12521              :       case BUILT_IN_FROB_RETURN_ADDR:
   12522              :       case BUILT_IN_RETURN:
   12523              :       case BUILT_IN_AGGREGATE_INCOMING_ADDRESS:
   12524              :       case BUILT_IN_FRAME_ADDRESS:
   12525              :       case BUILT_IN_VA_END:
   12526              :       case BUILT_IN_STACK_SAVE:
   12527              :       case BUILT_IN_STACK_RESTORE:
   12528              :       case BUILT_IN_DWARF_CFA:
   12529              :       case BUILT_IN_CALL_CODE_ADDRESS:
   12530              :       case BUILT_IN_CALL_STATIC_CHAIN:
   12531              :         /* Exception state returns or moves registers around.  */
   12532              :       case BUILT_IN_EH_FILTER:
   12533              :       case BUILT_IN_EH_POINTER:
   12534              :       case BUILT_IN_EH_COPY_VALUES:
   12535              :         return true;
   12536              : 
   12537     28237494 :       default:
   12538     28237494 :         return false;
   12539              :       }
   12540              : 
   12541              :   return false;
   12542              : }
   12543              : 
   12544              : /* Return true if DECL is a builtin that is not expensive, i.e., they are
   12545              :    most probably expanded inline into reasonably simple code.  This is a
   12546              :    superset of is_simple_builtin.  */
   12547              : bool
   12548     19651963 : is_inexpensive_builtin (tree decl)
   12549              : {
   12550     19651963 :   if (!decl)
   12551              :     return false;
   12552     19637462 :   else if (DECL_BUILT_IN_CLASS (decl) == BUILT_IN_MD)
   12553              :     return true;
   12554     18288454 :   else if (DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL)
   12555     15287846 :     switch (DECL_FUNCTION_CODE (decl))
   12556              :       {
   12557              :       case BUILT_IN_ABS:
   12558              :       CASE_BUILT_IN_ALLOCA:
   12559              :       case BUILT_IN_BSWAP16:
   12560              :       case BUILT_IN_BSWAP32:
   12561              :       case BUILT_IN_BSWAP64:
   12562              :       case BUILT_IN_BSWAP128:
   12563              :       case BUILT_IN_BITREVERSE8:
   12564              :       case BUILT_IN_BITREVERSE16:
   12565              :       case BUILT_IN_BITREVERSE32:
   12566              :       case BUILT_IN_BITREVERSE64:
   12567              :       case BUILT_IN_BITREVERSE128:
   12568              :       case BUILT_IN_CLZ:
   12569              :       case BUILT_IN_CLZIMAX:
   12570              :       case BUILT_IN_CLZL:
   12571              :       case BUILT_IN_CLZLL:
   12572              :       case BUILT_IN_CTZ:
   12573              :       case BUILT_IN_CTZIMAX:
   12574              :       case BUILT_IN_CTZL:
   12575              :       case BUILT_IN_CTZLL:
   12576              :       case BUILT_IN_FFS:
   12577              :       case BUILT_IN_FFSIMAX:
   12578              :       case BUILT_IN_FFSL:
   12579              :       case BUILT_IN_FFSLL:
   12580              :       case BUILT_IN_IMAXABS:
   12581              :       case BUILT_IN_FINITE:
   12582              :       case BUILT_IN_FINITEF:
   12583              :       case BUILT_IN_FINITEL:
   12584              :       case BUILT_IN_FINITED32:
   12585              :       case BUILT_IN_FINITED64:
   12586              :       case BUILT_IN_FINITED128:
   12587              :       case BUILT_IN_FPCLASSIFY:
   12588              :       case BUILT_IN_ISFINITE:
   12589              :       case BUILT_IN_ISINF_SIGN:
   12590              :       case BUILT_IN_ISINF:
   12591              :       case BUILT_IN_ISINFF:
   12592              :       case BUILT_IN_ISINFL:
   12593              :       case BUILT_IN_ISINFD32:
   12594              :       case BUILT_IN_ISINFD64:
   12595              :       case BUILT_IN_ISINFD128:
   12596              :       case BUILT_IN_ISNAN:
   12597              :       case BUILT_IN_ISNANF:
   12598              :       case BUILT_IN_ISNANL:
   12599              :       case BUILT_IN_ISNAND32:
   12600              :       case BUILT_IN_ISNAND64:
   12601              :       case BUILT_IN_ISNAND128:
   12602              :       case BUILT_IN_ISNORMAL:
   12603              :       case BUILT_IN_ISGREATER:
   12604              :       case BUILT_IN_ISGREATEREQUAL:
   12605              :       case BUILT_IN_ISLESS:
   12606              :       case BUILT_IN_ISLESSEQUAL:
   12607              :       case BUILT_IN_ISLESSGREATER:
   12608              :       case BUILT_IN_ISUNORDERED:
   12609              :       case BUILT_IN_ISEQSIG:
   12610              :       case BUILT_IN_VA_ARG_PACK:
   12611              :       case BUILT_IN_VA_ARG_PACK_LEN:
   12612              :       case BUILT_IN_VA_COPY:
   12613              :       case BUILT_IN_TRAP:
   12614              :       case BUILT_IN_UNREACHABLE_TRAP:
   12615              :       case BUILT_IN_SAVEREGS:
   12616              :       case BUILT_IN_POPCOUNTL:
   12617              :       case BUILT_IN_POPCOUNTLL:
   12618              :       case BUILT_IN_POPCOUNTIMAX:
   12619              :       case BUILT_IN_POPCOUNT:
   12620              :       case BUILT_IN_PARITYL:
   12621              :       case BUILT_IN_PARITYLL:
   12622              :       case BUILT_IN_PARITYIMAX:
   12623              :       case BUILT_IN_PARITY:
   12624              :       case BUILT_IN_LABS:
   12625              :       case BUILT_IN_LLABS:
   12626              :       case BUILT_IN_PREFETCH:
   12627              :       case BUILT_IN_ACC_ON_DEVICE:
   12628              :         return true;
   12629              : 
   12630     14624768 :       default:
   12631     14624768 :         return is_simple_builtin (decl);
   12632              :       }
   12633              : 
   12634              :   return false;
   12635              : }
   12636              : 
   12637              : /* Return true if T is a constant and the value cast to a target char
   12638              :    can be represented by a host char.
   12639              :    Store the casted char constant in *P if so.  */
   12640              : 
   12641              : bool
   12642         3735 : target_char_cst_p (tree t, char *p)
   12643              : {
   12644         3735 :   if (!tree_fits_uhwi_p (t) || CHAR_TYPE_SIZE != HOST_BITS_PER_CHAR)
   12645              :     return false;
   12646              : 
   12647         2065 :   *p = (char)tree_to_uhwi (t);
   12648         2065 :   return true;
   12649              : }
   12650              : 
   12651              : /* Return true if the builtin DECL is implemented in a standard library.
   12652              :    Otherwise return false which doesn't guarantee it is not (thus the list
   12653              :    of handled builtins below may be incomplete).  */
   12654              : 
   12655              : bool
   12656        43012 : builtin_with_linkage_p (tree decl)
   12657              : {
   12658        43012 :   if (DECL_BUILT_IN_CLASS (decl) == BUILT_IN_NORMAL)
   12659        42728 :     switch (DECL_FUNCTION_CODE (decl))
   12660              :     {
   12661         1334 :       CASE_FLT_FN (BUILT_IN_ACOS):
   12662         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ACOS):
   12663         1334 :       CASE_FLT_FN (BUILT_IN_ACOSH):
   12664         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ACOSH):
   12665         1334 :       CASE_FLT_FN (BUILT_IN_ACOSPI):
   12666         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ACOSPI):
   12667         1334 :       CASE_FLT_FN (BUILT_IN_ASIN):
   12668         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ASIN):
   12669         1334 :       CASE_FLT_FN (BUILT_IN_ASINH):
   12670         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ASINH):
   12671         1334 :       CASE_FLT_FN (BUILT_IN_ASINPI):
   12672         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ASINPI):
   12673         1334 :       CASE_FLT_FN (BUILT_IN_ATAN):
   12674         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ATAN):
   12675         1334 :       CASE_FLT_FN (BUILT_IN_ATAN2):
   12676         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ATAN2):
   12677         1334 :       CASE_FLT_FN (BUILT_IN_ATAN2PI):
   12678         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ATAN2PI):
   12679         1334 :       CASE_FLT_FN (BUILT_IN_ATANH):
   12680         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ATANH):
   12681         1334 :       CASE_FLT_FN (BUILT_IN_ATANPI):
   12682         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ATANPI):
   12683         1334 :       CASE_FLT_FN (BUILT_IN_CBRT):
   12684         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_CBRT):
   12685         1334 :       CASE_FLT_FN (BUILT_IN_CEIL):
   12686         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_CEIL):
   12687         1334 :       CASE_FLT_FN (BUILT_IN_COPYSIGN):
   12688         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_COPYSIGN):
   12689         1334 :       CASE_FLT_FN (BUILT_IN_COS):
   12690         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_COS):
   12691         1334 :       CASE_FLT_FN (BUILT_IN_COSH):
   12692         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_COSH):
   12693         1334 :       CASE_FLT_FN (BUILT_IN_COSPI):
   12694         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_COSPI):
   12695         1334 :       CASE_FLT_FN (BUILT_IN_ERF):
   12696         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ERF):
   12697         1334 :       CASE_FLT_FN (BUILT_IN_ERFC):
   12698         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ERFC):
   12699         1334 :       CASE_FLT_FN (BUILT_IN_EXP):
   12700         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_EXP):
   12701         1334 :       CASE_FLT_FN (BUILT_IN_EXP2):
   12702         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_EXP2):
   12703         1334 :       CASE_FLT_FN (BUILT_IN_EXPM1):
   12704         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_EXPM1):
   12705         1334 :       CASE_FLT_FN (BUILT_IN_FABS):
   12706         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_FABS):
   12707         1334 :       CASE_FLT_FN (BUILT_IN_FDIM):
   12708         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_FDIM):
   12709         1334 :       CASE_FLT_FN (BUILT_IN_FLOOR):
   12710         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_FLOOR):
   12711         1334 :       CASE_FLT_FN (BUILT_IN_FMA):
   12712         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMA):
   12713         1334 :       CASE_FLT_FN (BUILT_IN_FMAX):
   12714         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMAX):
   12715         1334 :       CASE_FLT_FN (BUILT_IN_FMIN):
   12716         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMIN):
   12717         1334 :       CASE_FLT_FN (BUILT_IN_FMOD):
   12718         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_FMOD):
   12719         1334 :       CASE_FLT_FN (BUILT_IN_FREXP):
   12720         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_FREXP):
   12721         1334 :       CASE_FLT_FN (BUILT_IN_HYPOT):
   12722         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_HYPOT):
   12723         1334 :       CASE_FLT_FN (BUILT_IN_ILOGB):
   12724         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ILOGB):
   12725         1334 :       CASE_FLT_FN (BUILT_IN_LDEXP):
   12726         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LDEXP):
   12727         1334 :       CASE_FLT_FN (BUILT_IN_LGAMMA):
   12728         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LGAMMA):
   12729         1334 :       CASE_FLT_FN (BUILT_IN_LLRINT):
   12730         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LLRINT):
   12731         1334 :       CASE_FLT_FN (BUILT_IN_LLROUND):
   12732         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LLROUND):
   12733         1334 :       CASE_FLT_FN (BUILT_IN_LOG):
   12734         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOG):
   12735         1334 :       CASE_FLT_FN (BUILT_IN_LOG10):
   12736         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOG10):
   12737         1334 :       CASE_FLT_FN (BUILT_IN_LOG1P):
   12738         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOG1P):
   12739         1334 :       CASE_FLT_FN (BUILT_IN_LOG2):
   12740         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOG2):
   12741         1334 :       CASE_FLT_FN (BUILT_IN_LOGB):
   12742         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LOGB):
   12743         1334 :       CASE_FLT_FN (BUILT_IN_LRINT):
   12744         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LRINT):
   12745         1334 :       CASE_FLT_FN (BUILT_IN_LROUND):
   12746         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_LROUND):
   12747         1334 :       CASE_FLT_FN (BUILT_IN_MODF):
   12748         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_MODF):
   12749         1334 :       CASE_FLT_FN (BUILT_IN_NAN):
   12750         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_NAN):
   12751         1334 :       CASE_FLT_FN (BUILT_IN_NEARBYINT):
   12752         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_NEARBYINT):
   12753         1334 :       CASE_FLT_FN (BUILT_IN_NEXTAFTER):
   12754         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_NEXTAFTER):
   12755         1334 :       CASE_FLT_FN (BUILT_IN_NEXTTOWARD):
   12756         1334 :       CASE_FLT_FN (BUILT_IN_POW):
   12757         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_POW):
   12758         1334 :       CASE_FLT_FN (BUILT_IN_REMAINDER):
   12759         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_REMAINDER):
   12760         1334 :       CASE_FLT_FN (BUILT_IN_REMQUO):
   12761         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_REMQUO):
   12762         1334 :       CASE_FLT_FN (BUILT_IN_RINT):
   12763         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_RINT):
   12764         1334 :       CASE_FLT_FN (BUILT_IN_ROUND):
   12765         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_ROUND):
   12766         1334 :       CASE_FLT_FN (BUILT_IN_SCALBLN):
   12767         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_SCALBLN):
   12768         1334 :       CASE_FLT_FN (BUILT_IN_SCALBN):
   12769         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_SCALBN):
   12770         1334 :       CASE_FLT_FN (BUILT_IN_SIN):
   12771         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_SIN):
   12772         1334 :       CASE_FLT_FN (BUILT_IN_SINCOS):
   12773         1334 :       CASE_FLT_FN (BUILT_IN_SINH):
   12774         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_SINH):
   12775         1334 :       CASE_FLT_FN (BUILT_IN_SINPI):
   12776         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_SINPI):
   12777         1334 :       CASE_FLT_FN (BUILT_IN_SQRT):
   12778         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_SQRT):
   12779         1334 :       CASE_FLT_FN (BUILT_IN_TAN):
   12780         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_TAN):
   12781         1334 :       CASE_FLT_FN (BUILT_IN_TANH):
   12782         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_TANH):
   12783         1334 :       CASE_FLT_FN (BUILT_IN_TANPI):
   12784         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_TANPI):
   12785         1334 :       CASE_FLT_FN (BUILT_IN_TGAMMA):
   12786         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_TGAMMA):
   12787         1334 :       CASE_FLT_FN (BUILT_IN_TRUNC):
   12788         1334 :       CASE_FLT_FN_FLOATN_NX (BUILT_IN_TRUNC):
   12789         1334 :         return true;
   12790              : 
   12791           16 :       case BUILT_IN_STPCPY:
   12792           16 :       case BUILT_IN_STPNCPY:
   12793              :         /* stpcpy is both referenced in libiberty's pex-win32.c and provided
   12794              :            by libiberty's stpcpy.c for MinGW targets so we need to return true
   12795              :            in order to be able to build libiberty in LTO mode for them.  */
   12796           16 :         return true;
   12797              : 
   12798              :       default:
   12799              :         break;
   12800              :     }
   12801              :   return false;
   12802              : }
   12803              : 
   12804              : /* Return true if OFFRNG is bounded to a subrange of offset values
   12805              :    valid for the largest possible object.  */
   12806              : 
   12807              : bool
   12808          501 : access_ref::offset_bounded () const
   12809              : {
   12810          501 :   tree min = TYPE_MIN_VALUE (ptrdiff_type_node);
   12811          501 :   tree max = TYPE_MAX_VALUE (ptrdiff_type_node);
   12812          975 :   return wi::to_offset (min) <= offrng[0] && offrng[1] <= wi::to_offset (max);
   12813              : }
   12814              : 
   12815              : /* If CALLEE has known side effects, fill in INFO and return true.
   12816              :    See tree-ssa-structalias.cc:find_func_aliases
   12817              :    for the list of builtins we might need to handle here.  */
   12818              : 
   12819              : attr_fnspec
   12820    116477241 : builtin_fnspec (tree callee)
   12821              : {
   12822    116477241 :   built_in_function code = DECL_FUNCTION_CODE (callee);
   12823              : 
   12824    116477241 :   switch (code)
   12825              :     {
   12826              :       /* All the following functions read memory pointed to by
   12827              :          their second argument and write memory pointed to by first
   12828              :          argument.
   12829              :          strcat/strncat additionally reads memory pointed to by the first
   12830              :          argument.  */
   12831       133415 :       case BUILT_IN_STRCAT:
   12832       133415 :       case BUILT_IN_STRCAT_CHK:
   12833       133415 :         return "1cW 1 ";
   12834        42044 :       case BUILT_IN_STRNCAT:
   12835        42044 :       case BUILT_IN_STRNCAT_CHK:
   12836        42044 :         return "1cW 13";
   12837       263446 :       case BUILT_IN_STRCPY:
   12838       263446 :       case BUILT_IN_STRCPY_CHK:
   12839       263446 :         return "1cO 1 ";
   12840        32581 :       case BUILT_IN_STPCPY:
   12841        32581 :       case BUILT_IN_STPCPY_CHK:
   12842        32581 :         return ".cO 1 ";
   12843     19086692 :       case BUILT_IN_STRNCPY:
   12844     19086692 :       case BUILT_IN_MEMCPY:
   12845     19086692 :       case BUILT_IN_MEMMOVE:
   12846     19086692 :       case BUILT_IN_TM_MEMCPY:
   12847     19086692 :       case BUILT_IN_TM_MEMMOVE:
   12848     19086692 :       case BUILT_IN_STRNCPY_CHK:
   12849     19086692 :       case BUILT_IN_MEMCPY_CHK:
   12850     19086692 :       case BUILT_IN_MEMMOVE_CHK:
   12851     19086692 :         return "1cO313";
   12852       143964 :       case BUILT_IN_MEMPCPY:
   12853       143964 :       case BUILT_IN_MEMPCPY_CHK:
   12854       143964 :         return ".cO313";
   12855        58403 :       case BUILT_IN_STPNCPY:
   12856        58403 :       case BUILT_IN_STPNCPY_CHK:
   12857        58403 :         return ".cO313";
   12858            0 :       case BUILT_IN_BCOPY:
   12859            0 :         return ".c23O3";
   12860            0 :       case BUILT_IN_BZERO:
   12861            0 :         return ".cO2";
   12862      9880900 :       case BUILT_IN_MEMCMP:
   12863      9880900 :       case BUILT_IN_MEMCMP_EQ:
   12864      9880900 :       case BUILT_IN_BCMP:
   12865      9880900 :       case BUILT_IN_STRNCMP:
   12866      9880900 :       case BUILT_IN_STRNCMP_EQ:
   12867      9880900 :       case BUILT_IN_STRNCASECMP:
   12868      9880900 :         return ".cR3R3";
   12869              : 
   12870              :       /* The following functions read memory pointed to by their
   12871              :          first argument.  */
   12872          773 :       CASE_BUILT_IN_TM_LOAD (1):
   12873          773 :       CASE_BUILT_IN_TM_LOAD (2):
   12874          773 :       CASE_BUILT_IN_TM_LOAD (4):
   12875          773 :       CASE_BUILT_IN_TM_LOAD (8):
   12876          773 :       CASE_BUILT_IN_TM_LOAD (FLOAT):
   12877          773 :       CASE_BUILT_IN_TM_LOAD (DOUBLE):
   12878          773 :       CASE_BUILT_IN_TM_LOAD (LDOUBLE):
   12879          773 :       CASE_BUILT_IN_TM_LOAD (M64):
   12880          773 :       CASE_BUILT_IN_TM_LOAD (M128):
   12881          773 :       CASE_BUILT_IN_TM_LOAD (M256):
   12882          773 :       case BUILT_IN_TM_LOG:
   12883          773 :       case BUILT_IN_TM_LOG_1:
   12884          773 :       case BUILT_IN_TM_LOG_2:
   12885          773 :       case BUILT_IN_TM_LOG_4:
   12886          773 :       case BUILT_IN_TM_LOG_8:
   12887          773 :       case BUILT_IN_TM_LOG_FLOAT:
   12888          773 :       case BUILT_IN_TM_LOG_DOUBLE:
   12889          773 :       case BUILT_IN_TM_LOG_LDOUBLE:
   12890          773 :       case BUILT_IN_TM_LOG_M64:
   12891          773 :       case BUILT_IN_TM_LOG_M128:
   12892          773 :       case BUILT_IN_TM_LOG_M256:
   12893          773 :         return ".cR ";
   12894              : 
   12895       495780 :       case BUILT_IN_INDEX:
   12896       495780 :       case BUILT_IN_RINDEX:
   12897       495780 :       case BUILT_IN_STRCHR:
   12898       495780 :       case BUILT_IN_STRLEN:
   12899       495780 :       case BUILT_IN_STRRCHR:
   12900       495780 :         return ".cR ";
   12901        59121 :       case BUILT_IN_STRNLEN:
   12902        59121 :         return ".cR2";
   12903              : 
   12904              :       /* These read memory pointed to by the first argument.
   12905              :          Allocating memory does not have any side-effects apart from
   12906              :          being the definition point for the pointer.
   12907              :          Unix98 specifies that errno is set on allocation failure.  */
   12908        18254 :       case BUILT_IN_STRDUP:
   12909        18254 :         return "mCR ";
   12910        12930 :       case BUILT_IN_STRNDUP:
   12911        12930 :         return "mCR2";
   12912              :       /* Allocating memory does not have any side-effects apart from
   12913              :          being the definition point for the pointer.  */
   12914     11568219 :       case BUILT_IN_MALLOC:
   12915     11568219 :       case BUILT_IN_ALIGNED_ALLOC:
   12916     11568219 :       case BUILT_IN_CALLOC:
   12917     11568219 :       case BUILT_IN_GOMP_ALLOC:
   12918     11568219 :         return "mC";
   12919       842871 :       CASE_BUILT_IN_ALLOCA:
   12920       842871 :         return "mc";
   12921              :       /* These read memory pointed to by the first argument with size
   12922              :          in the third argument.  */
   12923       618929 :       case BUILT_IN_MEMCHR:
   12924       618929 :         return ".cR3";
   12925              :       /* These read memory pointed to by the first and second arguments.  */
   12926     12362221 :       case BUILT_IN_STRSTR:
   12927     12362221 :       case BUILT_IN_STRPBRK:
   12928     12362221 :       case BUILT_IN_STRCASECMP:
   12929     12362221 :       case BUILT_IN_STRCSPN:
   12930     12362221 :       case BUILT_IN_STRSPN:
   12931     12362221 :       case BUILT_IN_STRCMP:
   12932     12362221 :       case BUILT_IN_STRCMP_EQ:
   12933     12362221 :         return ".cR R ";
   12934              :       /* Freeing memory kills the pointed-to memory.  More importantly
   12935              :          the call has to serve as a barrier for moving loads and stores
   12936              :          across it.  */
   12937      6705025 :       case BUILT_IN_STACK_RESTORE:
   12938      6705025 :       case BUILT_IN_FREE:
   12939      6705025 :       case BUILT_IN_GOMP_FREE:
   12940      6705025 :         return ".co ";
   12941       108681 :       case BUILT_IN_VA_END:
   12942       108681 :         return ".cO ";
   12943              :       /* Realloc serves both as allocation point and deallocation point.  */
   12944      1106300 :       case BUILT_IN_REALLOC:
   12945      1106300 :       case BUILT_IN_GOMP_REALLOC:
   12946      1106300 :         return ".Cw ";
   12947        15732 :       case BUILT_IN_GAMMA_R:
   12948        15732 :       case BUILT_IN_GAMMAF_R:
   12949        15732 :       case BUILT_IN_GAMMAL_R:
   12950        15732 :       case BUILT_IN_LGAMMA_R:
   12951        15732 :       case BUILT_IN_LGAMMAF_R:
   12952        15732 :       case BUILT_IN_LGAMMAL_R:
   12953        15732 :         return ".C. Ot";
   12954        44448 :       case BUILT_IN_FREXP:
   12955        44448 :       case BUILT_IN_FREXPF:
   12956        44448 :       case BUILT_IN_FREXPL:
   12957        44448 :       case BUILT_IN_MODF:
   12958        44448 :       case BUILT_IN_MODFF:
   12959        44448 :       case BUILT_IN_MODFL:
   12960        44448 :         return ".c. Ot";
   12961         7586 :       case BUILT_IN_REMQUO:
   12962         7586 :       case BUILT_IN_REMQUOF:
   12963         7586 :       case BUILT_IN_REMQUOL:
   12964         7586 :         return ".c. . Ot";
   12965          167 :       case BUILT_IN_SINCOS:
   12966          167 :       case BUILT_IN_SINCOSF:
   12967          167 :       case BUILT_IN_SINCOSL:
   12968          167 :         return ".c. OtOt";
   12969      4365765 :       case BUILT_IN_MEMSET:
   12970      4365765 :       case BUILT_IN_MEMSET_CHK:
   12971      4365765 :       case BUILT_IN_TM_MEMSET:
   12972      4365765 :         return "1cO3";
   12973          366 :       CASE_BUILT_IN_TM_STORE (1):
   12974          366 :       CASE_BUILT_IN_TM_STORE (2):
   12975          366 :       CASE_BUILT_IN_TM_STORE (4):
   12976          366 :       CASE_BUILT_IN_TM_STORE (8):
   12977          366 :       CASE_BUILT_IN_TM_STORE (FLOAT):
   12978          366 :       CASE_BUILT_IN_TM_STORE (DOUBLE):
   12979          366 :       CASE_BUILT_IN_TM_STORE (LDOUBLE):
   12980          366 :       CASE_BUILT_IN_TM_STORE (M64):
   12981          366 :       CASE_BUILT_IN_TM_STORE (M128):
   12982          366 :       CASE_BUILT_IN_TM_STORE (M256):
   12983          366 :         return ".cO ";
   12984      2689482 :       case BUILT_IN_STACK_SAVE:
   12985      2689482 :       case BUILT_IN_RETURN:
   12986      2689482 :       case BUILT_IN_EH_POINTER:
   12987      2689482 :       case BUILT_IN_EH_FILTER:
   12988      2689482 :       case BUILT_IN_UNWIND_RESUME:
   12989      2689482 :       case BUILT_IN_CXA_END_CLEANUP:
   12990      2689482 :       case BUILT_IN_EH_COPY_VALUES:
   12991      2689482 :       case BUILT_IN_FRAME_ADDRESS:
   12992      2689482 :       case BUILT_IN_APPLY_ARGS:
   12993      2689482 :       case BUILT_IN_ASAN_BEFORE_DYNAMIC_INIT:
   12994      2689482 :       case BUILT_IN_ASAN_AFTER_DYNAMIC_INIT:
   12995      2689482 :       case BUILT_IN_PREFETCH:
   12996      2689482 :       case BUILT_IN_DWARF_CFA:
   12997      2689482 :       case BUILT_IN_RETURN_ADDRESS:
   12998      2689482 :         return ".c";
   12999      2351524 :       case BUILT_IN_ASSUME_ALIGNED:
   13000      2351524 :       case BUILT_IN_EXPECT:
   13001      2351524 :       case BUILT_IN_EXPECT_WITH_PROBABILITY:
   13002      2351524 :         return "1cX ";
   13003              :       /* But posix_memalign stores a pointer into the memory pointed to
   13004              :          by its first argument.  */
   13005         6162 :       case BUILT_IN_POSIX_MEMALIGN:
   13006         6162 :         return ".cOt";
   13007         4438 :       case BUILT_IN_OMP_GET_MAPPED_PTR:
   13008         4438 :         return ". R ";
   13009              : 
   13010     43451022 :       default:
   13011     43451022 :         return "";
   13012              :     }
   13013              : }
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.