LCOV - code coverage report
Current view: top level - gcc - asan.cc (source / functions) Coverage Total Hit
Test: gcc.info Lines: 85.1 % 2037 1733
Test Date: 2026-11-14 16:39:50 Functions: 92.9 % 113 105
Legend: Lines:     hit not hit

            Line data    Source code
       1              : /* AddressSanitizer, a fast memory error detector.
       2              :    Copyright (C) 2012-2026 Free Software Foundation, Inc.
       3              :    Contributed by Kostya Serebryany <kcc@google.com>
       4              : 
       5              : This file is part of GCC.
       6              : 
       7              : GCC is free software; you can redistribute it and/or modify it under
       8              : the terms of the GNU General Public License as published by the Free
       9              : Software Foundation; either version 3, or (at your option) any later
      10              : version.
      11              : 
      12              : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
      13              : WARRANTY; without even the implied warranty of MERCHANTABILITY or
      14              : FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
      15              : for more details.
      16              : 
      17              : You should have received a copy of the GNU General Public License
      18              : along with GCC; see the file COPYING3.  If not see
      19              : <http://www.gnu.org/licenses/>.  */
      20              : 
      21              : 
      22              : #include "config.h"
      23              : #include "system.h"
      24              : #include "coretypes.h"
      25              : #include "backend.h"
      26              : #include "target.h"
      27              : #include "rtl.h"
      28              : #include "tree.h"
      29              : #include "gimple.h"
      30              : #include "cfghooks.h"
      31              : #include "alloc-pool.h"
      32              : #include "tree-pass.h"
      33              : #include "memmodel.h"
      34              : #include "tm_p.h"
      35              : #include "ssa.h"
      36              : #include "stringpool.h"
      37              : #include "optabs.h"
      38              : #include "emit-rtl.h"
      39              : #include "cgraph.h"
      40              : #include "gimple-pretty-print.h"
      41              : #include "alias.h"
      42              : #include "fold-const.h"
      43              : #include "cfganal.h"
      44              : #include "gimplify.h"
      45              : #include "gimple-iterator.h"
      46              : #include "varasm.h"
      47              : #include "stor-layout.h"
      48              : #include "tree-iterator.h"
      49              : #include "attribs.h"
      50              : #include "asan.h"
      51              : #include "dojump.h"
      52              : #include "explow.h"
      53              : #include "expr.h"
      54              : #include "output.h"
      55              : #include "langhooks.h"
      56              : #include "cfgloop.h"
      57              : #include "gimple-builder.h"
      58              : #include "gimple-fold.h"
      59              : #include "ubsan.h"
      60              : #include "builtins.h"
      61              : #include "fnmatch.h"
      62              : #include "tree-inline.h"
      63              : #include "tree-ssa.h"
      64              : #include "tree-eh.h"
      65              : #include "diagnostic-core.h"
      66              : 
      67              : /* AddressSanitizer finds out-of-bounds and use-after-free bugs
      68              :    with <2x slowdown on average.
      69              : 
      70              :    The tool consists of two parts:
      71              :    instrumentation module (this file) and a run-time library.
      72              :    The instrumentation module adds a run-time check before every memory insn.
      73              :      For a 8- or 16- byte load accessing address X:
      74              :        ShadowAddr = (X >> 3) + Offset
      75              :        ShadowValue = *(char*)ShadowAddr;  // *(short*) for 16-byte access.
      76              :        if (ShadowValue)
      77              :          __asan_report_load8(X);
      78              :      For a load of N bytes (N=1, 2 or 4) from address X:
      79              :        ShadowAddr = (X >> 3) + Offset
      80              :        ShadowValue = *(char*)ShadowAddr;
      81              :        if (ShadowValue)
      82              :          if ((X & 7) + N - 1 > ShadowValue)
      83              :            __asan_report_loadN(X);
      84              :    Stores are instrumented similarly, but using __asan_report_storeN functions.
      85              :    A call too __asan_init_vN() is inserted to the list of module CTORs.
      86              :    N is the version number of the AddressSanitizer API. The changes between the
      87              :    API versions are listed in libsanitizer/asan/asan_interface_internal.h.
      88              : 
      89              :    The run-time library redefines malloc (so that redzone are inserted around
      90              :    the allocated memory) and free (so that reuse of free-ed memory is delayed),
      91              :    provides __asan_report* and __asan_init_vN functions.
      92              : 
      93              :    Read more:
      94              :    http://code.google.com/p/address-sanitizer/wiki/AddressSanitizerAlgorithm
      95              : 
      96              :    The current implementation supports detection of out-of-bounds and
      97              :    use-after-free in the heap, on the stack and for global variables.
      98              : 
      99              :    [Protection of stack variables]
     100              : 
     101              :    To understand how detection of out-of-bounds and use-after-free works
     102              :    for stack variables, lets look at this example on x86_64 where the
     103              :    stack grows downward:
     104              : 
     105              :      int
     106              :      foo ()
     107              :      {
     108              :        char a[24] = {0};
     109              :        int b[2] = {0};
     110              : 
     111              :        a[5] = 1;
     112              :        b[1] = 2;
     113              : 
     114              :        return a[5] + b[1];
     115              :      }
     116              : 
     117              :    For this function, the stack protected by asan will be organized as
     118              :    follows, from the top of the stack to the bottom:
     119              : 
     120              :    Slot 1/ [red zone of 32 bytes called 'RIGHT RedZone']
     121              : 
     122              :    Slot 2/ [8 bytes of red zone, that adds up to the space of 'a' to make
     123              :            the next slot be 32 bytes aligned; this one is called Partial
     124              :            Redzone; this 32 bytes alignment is an asan constraint]
     125              : 
     126              :    Slot 3/ [24 bytes for variable 'a']
     127              : 
     128              :    Slot 4/ [red zone of 32 bytes called 'Middle RedZone']
     129              : 
     130              :    Slot 5/ [24 bytes of Partial Red Zone (similar to slot 2]
     131              : 
     132              :    Slot 6/ [8 bytes for variable 'b']
     133              : 
     134              :    Slot 7/ [32 bytes of Red Zone at the bottom of the stack, called
     135              :             'LEFT RedZone']
     136              : 
     137              :    The 32 bytes of LEFT red zone at the bottom of the stack can be
     138              :    decomposed as such:
     139              : 
     140              :      1/ The first 8 bytes contain a magical asan number that is always
     141              :      0x41B58AB3.
     142              : 
     143              :      2/ The following 8 bytes contains a pointer to a string (to be
     144              :      parsed at runtime by the runtime asan library), which format is
     145              :      the following:
     146              : 
     147              :       "<function-name> <space> <num-of-variables-on-the-stack>
     148              :       (<32-bytes-aligned-offset-in-bytes-of-variable> <space>
     149              :       <length-of-var-in-bytes> ){n} "
     150              : 
     151              :         where '(...){n}' means the content inside the parenthesis occurs 'n'
     152              :         times, with 'n' being the number of variables on the stack.
     153              : 
     154              :      3/ The following 8 bytes contain the PC of the current function which
     155              :      will be used by the run-time library to print an error message.
     156              : 
     157              :      4/ The following 8 bytes are reserved for internal use by the run-time.
     158              : 
     159              :    The shadow memory for that stack layout is going to look like this:
     160              : 
     161              :      - content of shadow memory 8 bytes for slot 7: 0xF1F1F1F1.
     162              :        The F1 byte pattern is a magic number called
     163              :        ASAN_STACK_MAGIC_LEFT and is a way for the runtime to know that
     164              :        the memory for that shadow byte is part of a the LEFT red zone
     165              :        intended to seat at the bottom of the variables on the stack.
     166              : 
     167              :      - content of shadow memory 8 bytes for slots 6 and 5:
     168              :        0xF4F4F400.  The F4 byte pattern is a magic number
     169              :        called ASAN_STACK_MAGIC_PARTIAL.  It flags the fact that the
     170              :        memory region for this shadow byte is a PARTIAL red zone
     171              :        intended to pad a variable A, so that the slot following
     172              :        {A,padding} is 32 bytes aligned.
     173              : 
     174              :        Note that the fact that the least significant byte of this
     175              :        shadow memory content is 00 means that 8 bytes of its
     176              :        corresponding memory (which corresponds to the memory of
     177              :        variable 'b') is addressable.
     178              : 
     179              :      - content of shadow memory 8 bytes for slot 4: 0xF2F2F2F2.
     180              :        The F2 byte pattern is a magic number called
     181              :        ASAN_STACK_MAGIC_MIDDLE.  It flags the fact that the memory
     182              :        region for this shadow byte is a MIDDLE red zone intended to
     183              :        seat between two 32 aligned slots of {variable,padding}.
     184              : 
     185              :      - content of shadow memory 8 bytes for slot 3 and 2:
     186              :        0xF4000000.  This represents is the concatenation of
     187              :        variable 'a' and the partial red zone following it, like what we
     188              :        had for variable 'b'.  The least significant 3 bytes being 00
     189              :        means that the 3 bytes of variable 'a' are addressable.
     190              : 
     191              :      - content of shadow memory 8 bytes for slot 1: 0xF3F3F3F3.
     192              :        The F3 byte pattern is a magic number called
     193              :        ASAN_STACK_MAGIC_RIGHT.  It flags the fact that the memory
     194              :        region for this shadow byte is a RIGHT red zone intended to seat
     195              :        at the top of the variables of the stack.
     196              : 
     197              :    Note that the real variable layout is done in expand_used_vars in
     198              :    cfgexpand.cc.  As far as Address Sanitizer is concerned, it lays out
     199              :    stack variables as well as the different red zones, emits some
     200              :    prologue code to populate the shadow memory as to poison (mark as
     201              :    non-accessible) the regions of the red zones and mark the regions of
     202              :    stack variables as accessible, and emit some epilogue code to
     203              :    un-poison (mark as accessible) the regions of red zones right before
     204              :    the function exits.
     205              : 
     206              :    [Protection of global variables]
     207              : 
     208              :    The basic idea is to insert a red zone between two global variables
     209              :    and install a constructor function that calls the asan runtime to do
     210              :    the populating of the relevant shadow memory regions at load time.
     211              : 
     212              :    So the global variables are laid out as to insert a red zone between
     213              :    them. The size of the red zones is so that each variable starts on a
     214              :    32 bytes boundary.
     215              : 
     216              :    Then a constructor function is installed so that, for each global
     217              :    variable, it calls the runtime asan library function
     218              :    __asan_register_globals_with an instance of this type:
     219              : 
     220              :      struct __asan_global
     221              :      {
     222              :        // Address of the beginning of the global variable.
     223              :        const void *__beg;
     224              : 
     225              :        // Initial size of the global variable.
     226              :        uptr __size;
     227              : 
     228              :        // Size of the global variable + size of the red zone.  This
     229              :        //   size is 32 bytes aligned.
     230              :        uptr __size_with_redzone;
     231              : 
     232              :        // Name of the global variable.
     233              :        const void *__name;
     234              : 
     235              :        // Name of the module where the global variable is declared.
     236              :        const void *__module_name;
     237              : 
     238              :        // 1 if it has dynamic initialization, 0 otherwise.
     239              :        uptr __has_dynamic_init;
     240              : 
     241              :        // A pointer to struct that contains source location, could be NULL.
     242              :        __asan_global_source_location *__location;
     243              :      }
     244              : 
     245              :    A destructor function that calls the runtime asan library function
     246              :    _asan_unregister_globals is also installed.  */
     247              : 
     248              : static unsigned HOST_WIDE_INT asan_shadow_offset_value;
     249              : static bool asan_shadow_offset_computed;
     250              : static vec<char *> sanitized_sections;
     251              : static tree last_alloca_addr;
     252              : 
     253              : /* Set of variable declarations that are going to be guarded by
     254              :    use-after-scope sanitizer.  */
     255              : 
     256              : hash_set<tree> *asan_handled_variables = NULL;
     257              : 
     258              : hash_set <tree> *asan_used_labels = NULL;
     259              : 
     260              : /* Global variables for HWASAN stack tagging.  */
     261              : /* hwasan_frame_tag_offset records the offset from the frame base tag that the
     262              :    next object should have.  */
     263              : static uint8_t hwasan_frame_tag_offset = 0;
     264              : /* hwasan_frame_base_ptr is a pointer with the same address as
     265              :    `virtual_stack_vars_rtx` for the current frame, and with the frame base tag
     266              :    stored in it.  N.b. this global RTX does not need to be marked GTY, but is
     267              :    done so anyway.  The need is not there since all uses are in just one pass
     268              :    (cfgexpand) and there are no calls to ggc_collect between the uses.  We mark
     269              :    it GTY(()) anyway to allow the use of the variable later on if needed by
     270              :    future features.  */
     271              : static GTY(()) rtx hwasan_frame_base_ptr = NULL_RTX;
     272              : /* hwasan_frame_base_init_seq is the sequence of RTL insns that will initialize
     273              :    the hwasan_frame_base_ptr.  When the hwasan_frame_base_ptr is requested, we
     274              :    generate this sequence but do not emit it.  If the sequence was created it
     275              :    is emitted once the function body has been expanded.
     276              : 
     277              :    This delay is because the frame base pointer may be needed anywhere in the
     278              :    function body, or needed by the expand_used_vars function.  Emitting once in
     279              :    a known place is simpler than requiring the emission of the instructions to
     280              :    be know where it should go depending on the first place the hwasan frame
     281              :    base is needed.  */
     282              : static GTY(()) rtx_insn *hwasan_frame_base_init_seq = NULL;
     283              : 
     284              : /* Structure defining the extent of one object on the stack that HWASAN needs
     285              :    to tag in the corresponding shadow stack space.
     286              : 
     287              :    The range this object spans on the stack is between `untagged_base +
     288              :    nearest_offset` and `untagged_base + farthest_offset`.
     289              :    `tagged_base` is an rtx containing the same value as `untagged_base` but
     290              :    with a random tag stored in the top byte.  We record both `untagged_base`
     291              :    and `tagged_base` so that `hwasan_emit_prologue` can use both without having
     292              :    to emit RTL into the instruction stream to re-calculate one from the other.
     293              :    (`hwasan_emit_prologue` needs to use both bases since the
     294              :    __hwasan_tag_memory call it emits uses an untagged value, and it calculates
     295              :    the tag to store in shadow memory based on the tag_offset plus the tag in
     296              :    tagged_base).  */
     297              : struct hwasan_stack_var
     298              : {
     299              :   rtx untagged_base;
     300              :   rtx tagged_base;
     301              :   poly_int64 nearest_offset;
     302              :   poly_int64 farthest_offset;
     303              :   uint8_t tag_offset;
     304              : };
     305              : 
     306              : /* Variable recording all stack variables that HWASAN needs to tag.
     307              :    Does not need to be marked as GTY(()) since every use is in the cfgexpand
     308              :    pass and gcc_collect is not called in the middle of that pass.  */
     309              : static vec<hwasan_stack_var> hwasan_tagged_stack_vars;
     310              : 
     311              : 
     312              : /* Sets shadow offset to value in string VAL.  */
     313              : 
     314              : bool
     315           11 : set_asan_shadow_offset (const char *val)
     316              : {
     317           11 :   char *endp;
     318              : 
     319           11 :   errno = 0;
     320           11 :   asan_shadow_offset_value = strtoull (val, &endp, 0);
     321           11 :   if (!(*val != '\0' && *endp == '\0' && errno == 0))
     322              :     return false;
     323              : 
     324           11 :   asan_shadow_offset_computed = true;
     325              : 
     326           11 :   return true;
     327              : }
     328              : 
     329              : /* Set list of user-defined sections that need to be sanitized.  */
     330              : 
     331              : void
     332           40 : set_sanitized_sections (const char *sections)
     333              : {
     334           40 :   char *pat;
     335           40 :   unsigned i;
     336           50 :   FOR_EACH_VEC_ELT (sanitized_sections, i, pat)
     337           10 :     free (pat);
     338           40 :   sanitized_sections.truncate (0);
     339              : 
     340          130 :   for (const char *s = sections; *s; )
     341              :     {
     342              :       const char *end;
     343          220 :       for (end = s; *end && *end != ','; ++end);
     344           50 :       size_t len = end - s;
     345           50 :       sanitized_sections.safe_push (xstrndup (s, len));
     346           50 :       s = *end ? end + 1 : end;
     347              :     }
     348           40 : }
     349              : 
     350              : bool
     351       106035 : asan_mark_p (gimple *stmt, enum asan_mark_flags flag)
     352              : {
     353       106035 :   return (gimple_call_internal_p (stmt, IFN_ASAN_MARK)
     354       106035 :           && tree_to_uhwi (gimple_call_arg (stmt, 0)) == flag);
     355              : }
     356              : 
     357              : bool
     358     21207829 : asan_sanitize_stack_p (void)
     359              : {
     360     21207829 :   return (sanitize_flags_p (SANITIZE_ADDRESS) && param_asan_stack);
     361              : }
     362              : 
     363              : bool
     364      1518312 : asan_sanitize_allocas_p (void)
     365              : {
     366      1518312 :   return (asan_sanitize_stack_p () && param_asan_protect_allocas);
     367              : }
     368              : 
     369              : bool
     370        14922 : asan_instrument_reads (void)
     371              : {
     372        14922 :   return (sanitize_flags_p (SANITIZE_ADDRESS) && param_asan_instrument_reads);
     373              : }
     374              : 
     375              : bool
     376        13891 : asan_instrument_writes (void)
     377              : {
     378        13891 :   return (sanitize_flags_p (SANITIZE_ADDRESS) && param_asan_instrument_writes);
     379              : }
     380              : 
     381              : bool
     382         4672 : asan_memintrin (void)
     383              : {
     384         4672 :   return (sanitize_flags_p (SANITIZE_ADDRESS) && param_asan_memintrin);
     385              : }
     386              : 
     387              : 
     388              : /* Support for --param asan-kernel-mem-intrinsic-prefix=1.  */
     389              : static GTY(()) rtx asan_memfn_rtls[3];
     390              : 
     391              : rtx
     392           42 : asan_memfn_rtl (tree fndecl)
     393              : {
     394           42 :   int i;
     395           42 :   const char *f, *p;
     396           42 :   char buf[sizeof ("__hwasan_memmove")];
     397              : 
     398           42 :   switch (DECL_FUNCTION_CODE (fndecl))
     399              :     {
     400              :     case BUILT_IN_MEMCPY: i = 0; f = "memcpy"; break;
     401           14 :     case BUILT_IN_MEMSET: i = 1; f = "memset"; break;
     402           14 :     case BUILT_IN_MEMMOVE: i = 2; f = "memmove"; break;
     403            0 :     default: gcc_unreachable ();
     404              :     }
     405           42 :   if (asan_memfn_rtls[i] == NULL_RTX)
     406              :     {
     407           42 :       tree save_name = DECL_NAME (fndecl);
     408           42 :       tree save_assembler_name = DECL_ASSEMBLER_NAME (fndecl);
     409           42 :       rtx save_rtl = DECL_RTL (fndecl);
     410           42 :       if (flag_sanitize & SANITIZE_KERNEL_HWADDRESS)
     411              :         p = "__hwasan_";
     412              :       else
     413           42 :         p = "__asan_";
     414           42 :       strcpy (buf, p);
     415           42 :       strcat (buf, f);
     416           42 :       DECL_NAME (fndecl) = get_identifier (buf);
     417           42 :       DECL_ASSEMBLER_NAME_RAW (fndecl) = NULL_TREE;
     418           42 :       SET_DECL_RTL (fndecl, NULL_RTX);
     419           42 :       asan_memfn_rtls[i] = DECL_RTL (fndecl);
     420           42 :       DECL_NAME (fndecl) = save_name;
     421           42 :       DECL_ASSEMBLER_NAME_RAW (fndecl) = save_assembler_name;
     422           42 :       SET_DECL_RTL (fndecl, save_rtl);
     423              :     }
     424           42 :   return asan_memfn_rtls[i];
     425              : }
     426              : 
     427              : 
     428              : /* Checks whether section SEC should be sanitized.  */
     429              : 
     430              : static bool
     431          270 : section_sanitized_p (const char *sec)
     432              : {
     433          270 :   char *pat;
     434          270 :   unsigned i;
     435          420 :   FOR_EACH_VEC_ELT (sanitized_sections, i, pat)
     436          330 :     if (fnmatch (pat, sec, FNM_PERIOD) == 0)
     437              :       return true;
     438              :   return false;
     439              : }
     440              : 
     441              : /* Returns Asan shadow offset.  */
     442              : 
     443              : static unsigned HOST_WIDE_INT
     444        25944 : asan_shadow_offset ()
     445              : {
     446        25944 :   if (!asan_shadow_offset_computed)
     447              :     {
     448         1764 :       asan_shadow_offset_computed = true;
     449         1764 :       asan_shadow_offset_value = targetm.asan_shadow_offset ();
     450              :     }
     451        25944 :   return asan_shadow_offset_value;
     452              : }
     453              : 
     454              : static bool
     455        32422 : asan_dynamic_shadow_offset_p ()
     456              : {
     457        32422 :   return (asan_shadow_offset_value == 0)
     458        32422 :          && targetm.asan_dynamic_shadow_offset_p ();
     459              : }
     460              : 
     461              : /* Returns Asan shadow offset has been set.  */
     462              : bool
     463            0 : asan_shadow_offset_set_p ()
     464              : {
     465            0 :   return asan_shadow_offset_computed;
     466              : }
     467              : 
     468              : alias_set_type asan_shadow_set = -1;
     469              : 
     470              : /* Pointer types to 1, 2 or 4 byte integers in shadow memory.  A separate
     471              :    alias set is used for all shadow memory accesses.  */
     472              : static GTY(()) tree shadow_ptr_types[3];
     473              : 
     474              : /* Decl for __asan_option_detect_stack_use_after_return.  */
     475              : static GTY(()) tree asan_detect_stack_use_after_return;
     476              : 
     477              : static GTY (()) tree asan_shadow_memory_dynamic_address;
     478              : 
     479              : /* Local copy for the asan_shadow_memory_dynamic_address within the
     480              :    function.  */
     481              : static GTY (()) tree asan_local_shadow_memory_dynamic_address;
     482              : 
     483              : static tree
     484            0 : get_asan_shadow_memory_dynamic_address_decl ()
     485              : {
     486            0 :   if (asan_shadow_memory_dynamic_address == NULL_TREE)
     487              :     {
     488            0 :       tree id, decl;
     489            0 :       id = get_identifier ("__asan_shadow_memory_dynamic_address");
     490            0 :       decl
     491            0 :         = build_decl (BUILTINS_LOCATION, VAR_DECL, id, pointer_sized_int_node);
     492            0 :       SET_DECL_ASSEMBLER_NAME (decl, id);
     493            0 :       TREE_ADDRESSABLE (decl) = 1;
     494            0 :       DECL_ARTIFICIAL (decl) = 1;
     495            0 :       DECL_IGNORED_P (decl) = 1;
     496            0 :       DECL_EXTERNAL (decl) = 1;
     497            0 :       TREE_STATIC (decl) = 1;
     498            0 :       TREE_PUBLIC (decl) = 1;
     499            0 :       TREE_USED (decl) = 1;
     500            0 :       asan_shadow_memory_dynamic_address = decl;
     501              :     }
     502              : 
     503            0 :   return asan_shadow_memory_dynamic_address;
     504              : }
     505              : 
     506              : void
     507         6478 : asan_maybe_insert_dynamic_shadow_at_function_entry (function *fun)
     508              : {
     509         6478 :   asan_local_shadow_memory_dynamic_address = NULL_TREE;
     510         6478 :   if (!asan_dynamic_shadow_offset_p ())
     511              :     return;
     512              : 
     513            0 :   gimple *g;
     514              : 
     515            0 :   tree lhs = create_tmp_var (pointer_sized_int_node,
     516              :                              "__local_asan_shadow_memory_dynamic_address");
     517              : 
     518            0 :   g = gimple_build_assign (lhs, get_asan_shadow_memory_dynamic_address_decl ());
     519            0 :   gimple_set_location (g, fun->function_start_locus);
     520            0 :   edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
     521            0 :   gsi_insert_on_edge_immediate (e, g);
     522              : 
     523            0 :   asan_local_shadow_memory_dynamic_address = lhs;
     524              : }
     525              : 
     526              : /* Hashtable support for memory references used by gimple
     527              :    statements.  */
     528              : 
     529              : /* This type represents a reference to a memory region.  */
     530              : struct asan_mem_ref
     531              : {
     532              :   /* The expression of the beginning of the memory region.  */
     533              :   tree start;
     534              : 
     535              :   /* The size of the access.  */
     536              :   HOST_WIDE_INT access_size;
     537              : };
     538              : 
     539              : object_allocator <asan_mem_ref> asan_mem_ref_pool ("asan_mem_ref");
     540              : 
     541              : /* Initializes an instance of asan_mem_ref.  */
     542              : 
     543              : static void
     544       174764 : asan_mem_ref_init (asan_mem_ref *ref, tree start, HOST_WIDE_INT access_size)
     545              : {
     546       174835 :   ref->start = start;
     547       174835 :   ref->access_size = access_size;
     548            0 : }
     549              : 
     550              : /* Allocates memory for an instance of asan_mem_ref into the memory
     551              :    pool returned by asan_mem_ref_get_alloc_pool and initialize it.
     552              :    START is the address of (or the expression pointing to) the
     553              :    beginning of memory reference.  ACCESS_SIZE is the size of the
     554              :    access to the referenced memory.  */
     555              : 
     556              : static asan_mem_ref*
     557        41380 : asan_mem_ref_new (tree start, HOST_WIDE_INT access_size)
     558              : {
     559            0 :   asan_mem_ref *ref = asan_mem_ref_pool.allocate ();
     560              : 
     561        41380 :   asan_mem_ref_init (ref, start, access_size);
     562        41380 :   return ref;
     563              : }
     564              : 
     565              : /* This builds and returns a pointer to the end of the memory region
     566              :    that starts at START and of length LEN.  */
     567              : 
     568              : tree
     569            0 : asan_mem_ref_get_end (tree start, tree len)
     570              : {
     571            0 :   if (len == NULL_TREE || integer_zerop (len))
     572              :     return start;
     573              : 
     574            0 :   if (!ptrofftype_p (len))
     575            0 :     len = convert_to_ptrofftype (len);
     576              : 
     577            0 :   return fold_build2 (POINTER_PLUS_EXPR, TREE_TYPE (start), start, len);
     578              : }
     579              : 
     580              : /*  Return a tree expression that represents the end of the referenced
     581              :     memory region.  Beware that this function can actually build a new
     582              :     tree expression.  */
     583              : 
     584              : tree
     585            0 : asan_mem_ref_get_end (const asan_mem_ref *ref, tree len)
     586              : {
     587            0 :   return asan_mem_ref_get_end (ref->start, len);
     588              : }
     589              : 
     590              : struct asan_mem_ref_hasher : nofree_ptr_hash <asan_mem_ref>
     591              : {
     592              :   static inline hashval_t hash (const asan_mem_ref *);
     593              :   static inline bool equal (const asan_mem_ref *, const asan_mem_ref *);
     594              : };
     595              : 
     596              : /* Hash a memory reference.  */
     597              : 
     598              : inline hashval_t
     599       286989 : asan_mem_ref_hasher::hash (const asan_mem_ref *mem_ref)
     600              : {
     601       286989 :   return iterative_hash_expr (mem_ref->start, 0);
     602              : }
     603              : 
     604              : /* Compare two memory references.  We accept the length of either
     605              :    memory references to be NULL_TREE.  */
     606              : 
     607              : inline bool
     608       220136 : asan_mem_ref_hasher::equal (const asan_mem_ref *m1,
     609              :                             const asan_mem_ref *m2)
     610              : {
     611       220136 :   return operand_equal_p (m1->start, m2->start, 0);
     612              : }
     613              : 
     614              : static hash_table<asan_mem_ref_hasher> *asan_mem_ref_ht;
     615              : 
     616              : /* Returns a reference to the hash table containing memory references.
     617              :    This function ensures that the hash table is created.  Note that
     618              :    this hash table is updated by the function
     619              :    update_mem_ref_hash_table.  */
     620              : 
     621              : static hash_table<asan_mem_ref_hasher> *
     622        91399 : get_mem_ref_hash_table ()
     623              : {
     624        91399 :   if (!asan_mem_ref_ht)
     625         3588 :     asan_mem_ref_ht = new hash_table<asan_mem_ref_hasher> (10);
     626              : 
     627        91399 :   return asan_mem_ref_ht;
     628              : }
     629              : 
     630              : /* Clear all entries from the memory references hash table.  */
     631              : 
     632              : static void
     633        38470 : empty_mem_ref_hash_table ()
     634              : {
     635        38470 :   if (asan_mem_ref_ht)
     636        21774 :     asan_mem_ref_ht->empty ();
     637        38470 : }
     638              : 
     639              : /* Free the memory references hash table.  */
     640              : 
     641              : static void
     642         6659 : free_mem_ref_resources ()
     643              : {
     644         6659 :   delete asan_mem_ref_ht;
     645         6659 :   asan_mem_ref_ht = NULL;
     646              : 
     647         6659 :   asan_mem_ref_pool.release ();
     648         6659 : }
     649              : 
     650              : /* Return true iff the memory reference REF has been instrumented.  */
     651              : 
     652              : static bool
     653        50007 : has_mem_ref_been_instrumented (tree ref, HOST_WIDE_INT access_size)
     654              : {
     655        50007 :   asan_mem_ref r;
     656        50007 :   asan_mem_ref_init (&r, ref, access_size);
     657              : 
     658        50007 :   asan_mem_ref *saved_ref = get_mem_ref_hash_table ()->find (&r);
     659        50007 :   return saved_ref && saved_ref->access_size >= access_size;
     660              : }
     661              : 
     662              : /* Return true iff the memory reference REF has been instrumented.  */
     663              : 
     664              : static bool
     665        29129 : has_mem_ref_been_instrumented (const asan_mem_ref *ref)
     666              : {
     667            0 :   return has_mem_ref_been_instrumented (ref->start, ref->access_size);
     668              : }
     669              : 
     670              : /* Return true iff access to memory region starting at REF and of
     671              :    length LEN has been instrumented.  */
     672              : 
     673              : static bool
     674          996 : has_mem_ref_been_instrumented (const asan_mem_ref *ref, tree len)
     675              : {
     676          996 :   HOST_WIDE_INT size_in_bytes
     677          996 :     = tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
     678              : 
     679          251 :   return size_in_bytes != -1
     680          251 :     && has_mem_ref_been_instrumented (ref->start, size_in_bytes);
     681              : }
     682              : 
     683              : /* Set REF to the memory reference present in a gimple assignment
     684              :    ASSIGNMENT.  Return true upon successful completion, false
     685              :    otherwise.  */
     686              : 
     687              : static bool
     688        32653 : get_mem_ref_of_assignment (const gassign *assignment,
     689              :                            asan_mem_ref *ref,
     690              :                            bool *ref_is_store)
     691              : {
     692        32653 :   gcc_assert (gimple_assign_single_p (assignment));
     693              : 
     694        32653 :   if (gimple_store_p (assignment)
     695        32653 :       && !gimple_clobber_p (assignment))
     696              :     {
     697        14321 :       ref->start = gimple_assign_lhs (assignment);
     698        14321 :       *ref_is_store = true;
     699              :     }
     700        18332 :   else if (gimple_assign_load_p (assignment))
     701              :     {
     702        14737 :       ref->start = gimple_assign_rhs1 (assignment);
     703        14737 :       *ref_is_store = false;
     704              :     }
     705              :   else
     706              :     return false;
     707              : 
     708        29058 :   ref->access_size = int_size_in_bytes (TREE_TYPE (ref->start));
     709        29058 :   return true;
     710              : }
     711              : 
     712              : /* Return address of last allocated dynamic alloca.  */
     713              : 
     714              : static tree
     715          400 : get_last_alloca_addr ()
     716              : {
     717          400 :   if (last_alloca_addr)
     718              :     return last_alloca_addr;
     719              : 
     720          187 :   last_alloca_addr = create_tmp_reg (ptr_type_node, "last_alloca_addr");
     721          187 :   gassign *g = gimple_build_assign (last_alloca_addr, null_pointer_node);
     722          187 :   edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
     723          187 :   gsi_insert_on_edge_immediate (e, g);
     724          187 :   return last_alloca_addr;
     725              : }
     726              : 
     727              : /* Insert __asan_allocas_unpoison (top, bottom) call before
     728              :    __builtin_stack_restore (new_sp) call.
     729              :    The pseudocode of this routine should look like this:
     730              :      top = last_alloca_addr;
     731              :      bot = new_sp;
     732              :      __asan_allocas_unpoison (top, bot);
     733              :      last_alloca_addr = new_sp;
     734              :      __builtin_stack_restore (new_sp);
     735              :    In general, we can't use new_sp as bot parameter because on some
     736              :    architectures SP has non zero offset from dynamic stack area.  Moreover, on
     737              :    some architectures this offset (STACK_DYNAMIC_OFFSET) becomes known for each
     738              :    particular function only after all callees were expanded to rtl.
     739              :    The most noticeable example is PowerPC{,64}, see
     740              :    http://refspecs.linuxfoundation.org/ELF/ppc64/PPC-elf64abi.html#DYNAM-STACK.
     741              :    To overcome the issue we use following trick: pass new_sp as a second
     742              :    parameter to __asan_allocas_unpoison and rewrite it during expansion with
     743              :    new_sp + (virtual_dynamic_stack_rtx - sp) later in
     744              :    expand_asan_emit_allocas_unpoison function.
     745              : 
     746              :    HWASAN needs to do very similar, the eventual pseudocode should be:
     747              :       __hwasan_tag_memory (virtual_stack_dynamic_rtx,
     748              :                            0,
     749              :                            new_sp - sp);
     750              :       __builtin_stack_restore (new_sp)
     751              : 
     752              :    Need to use the same trick to handle STACK_DYNAMIC_OFFSET as described
     753              :    above.  */
     754              : 
     755              : static void
     756          410 : handle_builtin_stack_restore (gcall *call, gimple_stmt_iterator *iter)
     757              : {
     758          410 :   if (!iter
     759          412 :       || !(asan_sanitize_allocas_p () || hwasan_sanitize_allocas_p ()
     760            2 :            || memtag_sanitize_allocas_p ()))
     761              :     return;
     762              : 
     763          203 :   tree restored_stack = gimple_call_arg (call, 0);
     764              : 
     765          203 :   gimple *g;
     766              : 
     767          203 :   if (hwasan_sanitize_allocas_p () || memtag_sanitize_allocas_p ())
     768              :     {
     769            0 :       enum internal_fn fn = IFN_HWASAN_ALLOCA_UNPOISON;
     770              :       /* There is only one piece of information `expand_HWASAN_ALLOCA_UNPOISON`
     771              :          needs to work.  This is the length of the area that we're
     772              :          deallocating.  Since the stack pointer is known at expand time, the
     773              :          position of the new stack pointer after deallocation is enough
     774              :          information to calculate this length.  */
     775            0 :       g = gimple_build_call_internal (fn, 1, restored_stack);
     776              :     }
     777              :   else
     778              :     {
     779          203 :       tree last_alloca = get_last_alloca_addr ();
     780          203 :       tree fn = builtin_decl_implicit (BUILT_IN_ASAN_ALLOCAS_UNPOISON);
     781          203 :       g = gimple_build_call (fn, 2, last_alloca, restored_stack);
     782          203 :       gsi_insert_before (iter, g, GSI_SAME_STMT);
     783          203 :       g = gimple_build_assign (last_alloca, restored_stack);
     784              :     }
     785              : 
     786          203 :   gsi_insert_before (iter, g, GSI_SAME_STMT);
     787              : }
     788              : 
     789              : /* Deploy and poison redzones around __builtin_alloca call.  To do this, we
     790              :    should replace this call with another one with changed parameters and
     791              :    replace all its uses with new address, so
     792              :        addr = __builtin_alloca (old_size, align);
     793              :    is replaced by
     794              :        left_redzone_size = max (align, ASAN_RED_ZONE_SIZE);
     795              :    Following two statements are optimized out if we know that
     796              :    old_size & (ASAN_RED_ZONE_SIZE - 1) == 0, i.e. alloca doesn't need partial
     797              :    redzone.
     798              :        misalign = old_size & (ASAN_RED_ZONE_SIZE - 1);
     799              :        partial_redzone_size = ASAN_RED_ZONE_SIZE - misalign;
     800              :        right_redzone_size = ASAN_RED_ZONE_SIZE;
     801              :        additional_size = left_redzone_size + partial_redzone_size +
     802              :                          right_redzone_size;
     803              :        new_size = old_size + additional_size;
     804              :        new_alloca = __builtin_alloca (new_size, max (align, 32))
     805              :        __asan_alloca_poison (new_alloca, old_size)
     806              :        addr = new_alloca + max (align, ASAN_RED_ZONE_SIZE);
     807              :        last_alloca_addr = new_alloca;
     808              :    ADDITIONAL_SIZE is added to make new memory allocation contain not only
     809              :    requested memory, but also left, partial and right redzones as well as some
     810              :    additional space, required by alignment.  */
     811              : 
     812              : static void
     813          398 : handle_builtin_alloca (gcall *call, gimple_stmt_iterator *iter)
     814              : {
     815          398 :   if (!iter
     816          400 :       || !(asan_sanitize_allocas_p () || hwasan_sanitize_allocas_p ()
     817            2 :            || memtag_sanitize_allocas_p ()))
     818          201 :     return;
     819              : 
     820          197 :   gassign *g;
     821          197 :   gcall *gg;
     822          197 :   tree callee = gimple_call_fndecl (call);
     823          197 :   tree lhs = gimple_call_lhs (call);
     824          197 :   tree old_size = gimple_call_arg (call, 0);
     825          197 :   tree ptr_type = lhs ? TREE_TYPE (lhs) : ptr_type_node;
     826          197 :   tree partial_size = NULL_TREE;
     827          197 :   unsigned int align
     828          197 :     = DECL_FUNCTION_CODE (callee) == BUILT_IN_ALLOCA
     829          390 :       ? 0 : tree_to_uhwi (gimple_call_arg (call, 1));
     830              : 
     831          197 :   bool throws = false;
     832          197 :   edge e = NULL;
     833          197 :   if (stmt_can_throw_internal (cfun, call))
     834              :     {
     835            9 :       if (!lhs)
     836              :         return;
     837            9 :       throws = true;
     838            9 :       e = find_fallthru_edge (gsi_bb (*iter)->succs);
     839              :     }
     840              : 
     841          197 :   if (hwasan_sanitize_allocas_p () || memtag_sanitize_allocas_p ())
     842              :     {
     843            0 :       gimple_seq stmts = NULL;
     844            0 :       location_t loc = gimple_location (gsi_stmt (*iter));
     845              :       /* HWASAN and MEMTAG need a different expansion.
     846              : 
     847              :          addr = __builtin_alloca (size, align);
     848              : 
     849              :          in case of HWASAN, should be replaced by
     850              : 
     851              :          new_size = size rounded up to HWASAN_TAG_GRANULE_SIZE byte alignment;
     852              :          untagged_addr = __builtin_alloca (new_size, align);
     853              :          tag = __hwasan_choose_alloca_tag ();
     854              :          addr = ifn_HWASAN_SET_TAG (untagged_addr, tag);
     855              :          __hwasan_tag_memory (untagged_addr, tag, new_size);
     856              : 
     857              :          in case of MEMTAG, should be replaced by
     858              : 
     859              :          new_size = size rounded up to HWASAN_TAG_GRANULE_SIZE byte alignment;
     860              :          untagged_addr = __builtin_alloca (new_size, align);
     861              :          addr = ifn_HWASAN_ALLOCA_POISON (untagged_addr, new_size);
     862              : 
     863              :          where a new tag is chosen and set on untagged_addr when
     864              :          HWASAN_ALLOCA_POISON is expanded.  */
     865              : 
     866              :       /* Ensure alignment at least HWASAN_TAG_GRANULE_SIZE bytes so we start on
     867              :          a tag granule.  */
     868            0 :       align = align > HWASAN_TAG_GRANULE_SIZE ? align : HWASAN_TAG_GRANULE_SIZE;
     869              : 
     870            0 :       tree old_size = gimple_call_arg (call, 0);
     871            0 :       tree new_size = gimple_build_round_up (&stmts, loc, size_type_node,
     872              :                                              old_size,
     873            0 :                                              HWASAN_TAG_GRANULE_SIZE);
     874              : 
     875              :       /* Make the alloca call */
     876            0 :       tree untagged_addr
     877            0 :         = gimple_build (&stmts, loc,
     878              :                         as_combined_fn (BUILT_IN_ALLOCA_WITH_ALIGN), ptr_type,
     879            0 :                         new_size, build_int_cst (size_type_node, align));
     880              : 
     881            0 :       tree addr;
     882              : 
     883            0 :       if (memtag_sanitize_p ())
     884            0 :         addr = gimple_build (&stmts, loc, CFN_HWASAN_ALLOCA_POISON, ptr_type,
     885              :                              untagged_addr, new_size);
     886              :       else
     887              :         {
     888              :           /* Choose the tag.
     889              :              Here we use an internal function so we can choose the tag at expand
     890              :              time.  We need the decision to be made after stack variables have been
     891              :              assigned their tag (i.e. once the hwasan_frame_tag_offset variable has
     892              :              been set to one after the last stack variables tag).  */
     893            0 :           tree tag = gimple_build (&stmts, loc, CFN_HWASAN_CHOOSE_TAG,
     894              :                                    unsigned_char_type_node);
     895              : 
     896              :           /* Add tag to pointer.  */
     897            0 :           addr = gimple_build (&stmts, loc, CFN_HWASAN_SET_TAG, ptr_type,
     898              :                                untagged_addr, tag);
     899              : 
     900              :           /* Tag shadow memory.
     901              :              NOTE: require using `untagged_addr` here for libhwasan API.  */
     902            0 :           gimple_build (&stmts, loc, as_combined_fn (BUILT_IN_HWASAN_TAG_MEM),
     903              :                         void_type_node, untagged_addr, tag, new_size);
     904              :         }
     905              : 
     906              :       /* Insert the built up code sequence into the original instruction stream
     907              :          the iterator points to.  */
     908            0 :       gsi_insert_seq_before (iter, stmts, GSI_SAME_STMT);
     909              : 
     910              :       /* Finally, replace old alloca ptr with NEW_ALLOCA.  */
     911            0 :       replace_call_with_value (iter, addr);
     912            0 :       return;
     913              :     }
     914              : 
     915          197 :   tree last_alloca = get_last_alloca_addr ();
     916          197 :   const HOST_WIDE_INT redzone_mask = ASAN_RED_ZONE_SIZE - 1;
     917              : 
     918              :   /* If ALIGN > ASAN_RED_ZONE_SIZE, we embed left redzone into first ALIGN
     919              :      bytes of allocated space.  Otherwise, align alloca to ASAN_RED_ZONE_SIZE
     920              :      manually.  */
     921          197 :   align = MAX (align, ASAN_RED_ZONE_SIZE * BITS_PER_UNIT);
     922              : 
     923          197 :   tree alloca_rz_mask = build_int_cst (size_type_node, redzone_mask);
     924          197 :   tree redzone_size = build_int_cst (size_type_node, ASAN_RED_ZONE_SIZE);
     925              : 
     926              :   /* Extract lower bits from old_size.  */
     927          197 :   wide_int size_nonzero_bits = get_nonzero_bits (old_size);
     928          197 :   wide_int rz_mask
     929          197 :     = wi::uhwi (redzone_mask, wi::get_precision (size_nonzero_bits));
     930          197 :   wide_int old_size_lower_bits = wi::bit_and (size_nonzero_bits, rz_mask);
     931              : 
     932              :   /* If alloca size is aligned to ASAN_RED_ZONE_SIZE, we don't need partial
     933              :      redzone.  Otherwise, compute its size here.  */
     934          197 :   if (wi::ne_p (old_size_lower_bits, 0))
     935              :     {
     936              :       /* misalign = size & (ASAN_RED_ZONE_SIZE - 1)
     937              :          partial_size = ASAN_RED_ZONE_SIZE - misalign.  */
     938          194 :       g = gimple_build_assign (make_ssa_name (size_type_node, NULL),
     939              :                                BIT_AND_EXPR, old_size, alloca_rz_mask);
     940          194 :       gsi_insert_before (iter, g, GSI_SAME_STMT);
     941          194 :       tree misalign = gimple_assign_lhs (g);
     942          194 :       g = gimple_build_assign (make_ssa_name (size_type_node, NULL), MINUS_EXPR,
     943              :                                redzone_size, misalign);
     944          194 :       gsi_insert_before (iter, g, GSI_SAME_STMT);
     945          194 :       partial_size = gimple_assign_lhs (g);
     946              :     }
     947              : 
     948              :   /* additional_size = align + ASAN_RED_ZONE_SIZE.  */
     949          394 :   tree additional_size = build_int_cst (size_type_node, align / BITS_PER_UNIT
     950          197 :                                                         + ASAN_RED_ZONE_SIZE);
     951              :   /* If alloca has partial redzone, include it to additional_size too.  */
     952          197 :   if (partial_size)
     953              :     {
     954              :       /* additional_size += partial_size.  */
     955          194 :       g = gimple_build_assign (make_ssa_name (size_type_node), PLUS_EXPR,
     956              :                                partial_size, additional_size);
     957          194 :       gsi_insert_before (iter, g, GSI_SAME_STMT);
     958          194 :       additional_size = gimple_assign_lhs (g);
     959              :     }
     960              : 
     961              :   /* new_size = old_size + additional_size.  */
     962          197 :   g = gimple_build_assign (make_ssa_name (size_type_node), PLUS_EXPR, old_size,
     963              :                            additional_size);
     964          197 :   gsi_insert_before (iter, g, GSI_SAME_STMT);
     965          197 :   tree new_size = gimple_assign_lhs (g);
     966              : 
     967              :   /* Build new __builtin_alloca call:
     968              :        new_alloca_with_rz = __builtin_alloca (new_size, align).  */
     969          197 :   tree fn = builtin_decl_implicit (BUILT_IN_ALLOCA_WITH_ALIGN);
     970          197 :   gg = gimple_build_call (fn, 2, new_size,
     971          197 :                           build_int_cst (size_type_node, align));
     972          197 :   tree new_alloca_with_rz = make_ssa_name (ptr_type, gg);
     973          197 :   gimple_call_set_lhs (gg, new_alloca_with_rz);
     974          197 :   if (throws)
     975              :     {
     976            9 :       gimple_call_set_lhs (call, NULL);
     977            9 :       gsi_replace (iter, gg, true);
     978              :     }
     979              :   else
     980          188 :     gsi_insert_before (iter, gg, GSI_SAME_STMT);
     981              : 
     982              :   /* new_alloca = new_alloca_with_rz + align.  */
     983          197 :   g = gimple_build_assign (make_ssa_name (ptr_type), POINTER_PLUS_EXPR,
     984              :                            new_alloca_with_rz,
     985              :                            build_int_cst (size_type_node,
     986          197 :                                           align / BITS_PER_UNIT));
     987          197 :   gimple_stmt_iterator gsi = gsi_none ();
     988          197 :   if (throws)
     989              :     {
     990            9 :       gsi_insert_on_edge_immediate (e, g);
     991            9 :       gsi = gsi_for_stmt (g);
     992              :     }
     993              :   else
     994          188 :     gsi_insert_before (iter, g, GSI_SAME_STMT);
     995          197 :   tree new_alloca = gimple_assign_lhs (g);
     996              : 
     997              :   /* Poison newly created alloca redzones:
     998              :       __asan_alloca_poison (new_alloca, old_size).  */
     999          197 :   fn = builtin_decl_implicit (BUILT_IN_ASAN_ALLOCA_POISON);
    1000          197 :   gg = gimple_build_call (fn, 2, new_alloca, old_size);
    1001          197 :   if (throws)
    1002            9 :     gsi_insert_after (&gsi, gg, GSI_NEW_STMT);
    1003              :   else
    1004          188 :     gsi_insert_before (iter, gg, GSI_SAME_STMT);
    1005              : 
    1006              :   /* Save new_alloca_with_rz value into last_alloca to use it during
    1007              :      allocas unpoisoning.  */
    1008          197 :   g = gimple_build_assign (last_alloca, new_alloca_with_rz);
    1009          197 :   if (throws)
    1010            9 :     gsi_insert_after (&gsi, g, GSI_NEW_STMT);
    1011              :   else
    1012          188 :     gsi_insert_before (iter, g, GSI_SAME_STMT);
    1013              : 
    1014              :   /* Finally, replace old alloca ptr with NEW_ALLOCA.  */
    1015          197 :   if (throws)
    1016              :     {
    1017            9 :       g = gimple_build_assign (lhs, new_alloca);
    1018            9 :       gsi_insert_after (&gsi, g, GSI_NEW_STMT);
    1019              :     }
    1020              :   else
    1021          188 :     replace_call_with_value (iter, new_alloca);
    1022          197 : }
    1023              : 
    1024              : /* Return the memory references contained in a gimple statement
    1025              :    representing a builtin call that has to do with memory access.  */
    1026              : 
    1027              : static bool
    1028         9332 : get_mem_refs_of_builtin_call (gcall *call,
    1029              :                               asan_mem_ref *src0,
    1030              :                               tree *src0_len,
    1031              :                               bool *src0_is_store,
    1032              :                               asan_mem_ref *src1,
    1033              :                               tree *src1_len,
    1034              :                               bool *src1_is_store,
    1035              :                               asan_mem_ref *dst,
    1036              :                               tree *dst_len,
    1037              :                               bool *dst_is_store,
    1038              :                               bool *dest_is_deref,
    1039              :                               bool *intercepted_p,
    1040              :                               gimple_stmt_iterator *iter = NULL)
    1041              : {
    1042         9332 :   gcc_checking_assert (gimple_call_builtin_p (call, BUILT_IN_NORMAL));
    1043              : 
    1044         9332 :   tree callee = gimple_call_fndecl (call);
    1045         9332 :   tree source0 = NULL_TREE, source1 = NULL_TREE,
    1046         9332 :     dest = NULL_TREE, len = NULL_TREE;
    1047         9332 :   bool is_store = true, got_reference_p = false;
    1048         9332 :   HOST_WIDE_INT access_size = 1;
    1049              : 
    1050         9332 :   *intercepted_p = asan_intercepted_p ((DECL_FUNCTION_CODE (callee)));
    1051              : 
    1052         9332 :   switch (DECL_FUNCTION_CODE (callee))
    1053              :     {
    1054              :       /* (s, s, n) style memops.  */
    1055          112 :     case BUILT_IN_BCMP:
    1056          112 :     case BUILT_IN_MEMCMP:
    1057          112 :       source0 = gimple_call_arg (call, 0);
    1058          112 :       source1 = gimple_call_arg (call, 1);
    1059          112 :       len = gimple_call_arg (call, 2);
    1060          112 :       break;
    1061              : 
    1062              :       /* (src, dest, n) style memops.  */
    1063            0 :     case BUILT_IN_BCOPY:
    1064            0 :       source0 = gimple_call_arg (call, 0);
    1065            0 :       dest = gimple_call_arg (call, 1);
    1066            0 :       len = gimple_call_arg (call, 2);
    1067            0 :       break;
    1068              : 
    1069              :       /* (dest, src, n) style memops.  */
    1070         1316 :     case BUILT_IN_MEMCPY:
    1071         1316 :     case BUILT_IN_MEMCPY_CHK:
    1072         1316 :     case BUILT_IN_MEMMOVE:
    1073         1316 :     case BUILT_IN_MEMMOVE_CHK:
    1074         1316 :     case BUILT_IN_MEMPCPY:
    1075         1316 :     case BUILT_IN_MEMPCPY_CHK:
    1076         1316 :       dest = gimple_call_arg (call, 0);
    1077         1316 :       source0 = gimple_call_arg (call, 1);
    1078         1316 :       len = gimple_call_arg (call, 2);
    1079         1316 :       break;
    1080              : 
    1081              :       /* (dest, n) style memops.  */
    1082            0 :     case BUILT_IN_BZERO:
    1083            0 :       dest = gimple_call_arg (call, 0);
    1084            0 :       len = gimple_call_arg (call, 1);
    1085            0 :       break;
    1086              : 
    1087              :       /* (dest, x, n) style memops*/
    1088          388 :     case BUILT_IN_MEMSET:
    1089          388 :     case BUILT_IN_MEMSET_CHK:
    1090          388 :       dest = gimple_call_arg (call, 0);
    1091          388 :       len = gimple_call_arg (call, 2);
    1092          388 :       break;
    1093              : 
    1094          104 :     case BUILT_IN_STRLEN:
    1095              :       /* Special case strlen here since its length is taken from its return
    1096              :          value.
    1097              : 
    1098              :          The approach taken by the sanitizers is to check a memory access
    1099              :          before it's taken.  For ASAN strlen is intercepted by libasan, so no
    1100              :          check is inserted by the compiler.
    1101              : 
    1102              :          This function still returns `true` and provides a length to the rest
    1103              :          of the ASAN pass in order to record what areas have been checked,
    1104              :          avoiding superfluous checks later on.
    1105              : 
    1106              :          HWASAN does not intercept any of these internal functions.
    1107              :          This means that checks for memory accesses must be inserted by the
    1108              :          compiler.
    1109              :          strlen is a special case, because we can tell the length from the
    1110              :          return of the function, but that is not known until after the function
    1111              :          has returned.
    1112              : 
    1113              :          Hence we can't check the memory access before it happens.
    1114              :          We could check the memory access after it has already happened, but
    1115              :          for now we choose to just ignore `strlen` calls.
    1116              :          This decision was simply made because that means the special case is
    1117              :          limited to this one case of this one function.  */
    1118          104 :       if (hwassist_sanitize_p ())
    1119              :         return false;
    1120           72 :       source0 = gimple_call_arg (call, 0);
    1121           72 :       len = gimple_call_lhs (call);
    1122           72 :       break;
    1123              : 
    1124          410 :     case BUILT_IN_STACK_RESTORE:
    1125          410 :       handle_builtin_stack_restore (call, iter);
    1126          410 :       break;
    1127              : 
    1128          398 :     CASE_BUILT_IN_ALLOCA:
    1129          398 :       handle_builtin_alloca (call, iter);
    1130          398 :       break;
    1131              :     /* And now the __atomic* and __sync builtins.
    1132              :        These are handled differently from the classical memory
    1133              :        access builtins above.  */
    1134              : 
    1135            0 :     case BUILT_IN_ATOMIC_LOAD_1:
    1136            0 :       is_store = false;
    1137              :       /* FALLTHRU */
    1138            0 :     case BUILT_IN_SYNC_FETCH_AND_ADD_1:
    1139            0 :     case BUILT_IN_SYNC_FETCH_AND_SUB_1:
    1140            0 :     case BUILT_IN_SYNC_FETCH_AND_OR_1:
    1141            0 :     case BUILT_IN_SYNC_FETCH_AND_AND_1:
    1142            0 :     case BUILT_IN_SYNC_FETCH_AND_XOR_1:
    1143            0 :     case BUILT_IN_SYNC_FETCH_AND_NAND_1:
    1144            0 :     case BUILT_IN_SYNC_ADD_AND_FETCH_1:
    1145            0 :     case BUILT_IN_SYNC_SUB_AND_FETCH_1:
    1146            0 :     case BUILT_IN_SYNC_OR_AND_FETCH_1:
    1147            0 :     case BUILT_IN_SYNC_AND_AND_FETCH_1:
    1148            0 :     case BUILT_IN_SYNC_XOR_AND_FETCH_1:
    1149            0 :     case BUILT_IN_SYNC_NAND_AND_FETCH_1:
    1150            0 :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_1:
    1151            0 :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_1:
    1152            0 :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_1:
    1153            0 :     case BUILT_IN_SYNC_LOCK_RELEASE_1:
    1154            0 :     case BUILT_IN_ATOMIC_EXCHANGE_1:
    1155            0 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_1:
    1156            0 :     case BUILT_IN_ATOMIC_STORE_1:
    1157            0 :     case BUILT_IN_ATOMIC_ADD_FETCH_1:
    1158            0 :     case BUILT_IN_ATOMIC_SUB_FETCH_1:
    1159            0 :     case BUILT_IN_ATOMIC_AND_FETCH_1:
    1160            0 :     case BUILT_IN_ATOMIC_NAND_FETCH_1:
    1161            0 :     case BUILT_IN_ATOMIC_XOR_FETCH_1:
    1162            0 :     case BUILT_IN_ATOMIC_OR_FETCH_1:
    1163            0 :     case BUILT_IN_ATOMIC_FETCH_ADD_1:
    1164            0 :     case BUILT_IN_ATOMIC_FETCH_SUB_1:
    1165            0 :     case BUILT_IN_ATOMIC_FETCH_AND_1:
    1166            0 :     case BUILT_IN_ATOMIC_FETCH_NAND_1:
    1167            0 :     case BUILT_IN_ATOMIC_FETCH_XOR_1:
    1168            0 :     case BUILT_IN_ATOMIC_FETCH_OR_1:
    1169            0 :       access_size = 1;
    1170            0 :       goto do_atomic;
    1171              : 
    1172            0 :     case BUILT_IN_ATOMIC_LOAD_2:
    1173            0 :       is_store = false;
    1174              :       /* FALLTHRU */
    1175            0 :     case BUILT_IN_SYNC_FETCH_AND_ADD_2:
    1176            0 :     case BUILT_IN_SYNC_FETCH_AND_SUB_2:
    1177            0 :     case BUILT_IN_SYNC_FETCH_AND_OR_2:
    1178            0 :     case BUILT_IN_SYNC_FETCH_AND_AND_2:
    1179            0 :     case BUILT_IN_SYNC_FETCH_AND_XOR_2:
    1180            0 :     case BUILT_IN_SYNC_FETCH_AND_NAND_2:
    1181            0 :     case BUILT_IN_SYNC_ADD_AND_FETCH_2:
    1182            0 :     case BUILT_IN_SYNC_SUB_AND_FETCH_2:
    1183            0 :     case BUILT_IN_SYNC_OR_AND_FETCH_2:
    1184            0 :     case BUILT_IN_SYNC_AND_AND_FETCH_2:
    1185            0 :     case BUILT_IN_SYNC_XOR_AND_FETCH_2:
    1186            0 :     case BUILT_IN_SYNC_NAND_AND_FETCH_2:
    1187            0 :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_2:
    1188            0 :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_2:
    1189            0 :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_2:
    1190            0 :     case BUILT_IN_SYNC_LOCK_RELEASE_2:
    1191            0 :     case BUILT_IN_ATOMIC_EXCHANGE_2:
    1192            0 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_2:
    1193            0 :     case BUILT_IN_ATOMIC_STORE_2:
    1194            0 :     case BUILT_IN_ATOMIC_ADD_FETCH_2:
    1195            0 :     case BUILT_IN_ATOMIC_SUB_FETCH_2:
    1196            0 :     case BUILT_IN_ATOMIC_AND_FETCH_2:
    1197            0 :     case BUILT_IN_ATOMIC_NAND_FETCH_2:
    1198            0 :     case BUILT_IN_ATOMIC_XOR_FETCH_2:
    1199            0 :     case BUILT_IN_ATOMIC_OR_FETCH_2:
    1200            0 :     case BUILT_IN_ATOMIC_FETCH_ADD_2:
    1201            0 :     case BUILT_IN_ATOMIC_FETCH_SUB_2:
    1202            0 :     case BUILT_IN_ATOMIC_FETCH_AND_2:
    1203            0 :     case BUILT_IN_ATOMIC_FETCH_NAND_2:
    1204            0 :     case BUILT_IN_ATOMIC_FETCH_XOR_2:
    1205            0 :     case BUILT_IN_ATOMIC_FETCH_OR_2:
    1206            0 :       access_size = 2;
    1207            0 :       goto do_atomic;
    1208              : 
    1209            0 :     case BUILT_IN_ATOMIC_LOAD_4:
    1210            0 :       is_store = false;
    1211              :       /* FALLTHRU */
    1212           56 :     case BUILT_IN_SYNC_FETCH_AND_ADD_4:
    1213           56 :     case BUILT_IN_SYNC_FETCH_AND_SUB_4:
    1214           56 :     case BUILT_IN_SYNC_FETCH_AND_OR_4:
    1215           56 :     case BUILT_IN_SYNC_FETCH_AND_AND_4:
    1216           56 :     case BUILT_IN_SYNC_FETCH_AND_XOR_4:
    1217           56 :     case BUILT_IN_SYNC_FETCH_AND_NAND_4:
    1218           56 :     case BUILT_IN_SYNC_ADD_AND_FETCH_4:
    1219           56 :     case BUILT_IN_SYNC_SUB_AND_FETCH_4:
    1220           56 :     case BUILT_IN_SYNC_OR_AND_FETCH_4:
    1221           56 :     case BUILT_IN_SYNC_AND_AND_FETCH_4:
    1222           56 :     case BUILT_IN_SYNC_XOR_AND_FETCH_4:
    1223           56 :     case BUILT_IN_SYNC_NAND_AND_FETCH_4:
    1224           56 :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_4:
    1225           56 :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_4:
    1226           56 :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_4:
    1227           56 :     case BUILT_IN_SYNC_LOCK_RELEASE_4:
    1228           56 :     case BUILT_IN_ATOMIC_EXCHANGE_4:
    1229           56 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_4:
    1230           56 :     case BUILT_IN_ATOMIC_STORE_4:
    1231           56 :     case BUILT_IN_ATOMIC_ADD_FETCH_4:
    1232           56 :     case BUILT_IN_ATOMIC_SUB_FETCH_4:
    1233           56 :     case BUILT_IN_ATOMIC_AND_FETCH_4:
    1234           56 :     case BUILT_IN_ATOMIC_NAND_FETCH_4:
    1235           56 :     case BUILT_IN_ATOMIC_XOR_FETCH_4:
    1236           56 :     case BUILT_IN_ATOMIC_OR_FETCH_4:
    1237           56 :     case BUILT_IN_ATOMIC_FETCH_ADD_4:
    1238           56 :     case BUILT_IN_ATOMIC_FETCH_SUB_4:
    1239           56 :     case BUILT_IN_ATOMIC_FETCH_AND_4:
    1240           56 :     case BUILT_IN_ATOMIC_FETCH_NAND_4:
    1241           56 :     case BUILT_IN_ATOMIC_FETCH_XOR_4:
    1242           56 :     case BUILT_IN_ATOMIC_FETCH_OR_4:
    1243           56 :       access_size = 4;
    1244           56 :       goto do_atomic;
    1245              : 
    1246            0 :     case BUILT_IN_ATOMIC_LOAD_8:
    1247            0 :       is_store = false;
    1248              :       /* FALLTHRU */
    1249            0 :     case BUILT_IN_SYNC_FETCH_AND_ADD_8:
    1250            0 :     case BUILT_IN_SYNC_FETCH_AND_SUB_8:
    1251            0 :     case BUILT_IN_SYNC_FETCH_AND_OR_8:
    1252            0 :     case BUILT_IN_SYNC_FETCH_AND_AND_8:
    1253            0 :     case BUILT_IN_SYNC_FETCH_AND_XOR_8:
    1254            0 :     case BUILT_IN_SYNC_FETCH_AND_NAND_8:
    1255            0 :     case BUILT_IN_SYNC_ADD_AND_FETCH_8:
    1256            0 :     case BUILT_IN_SYNC_SUB_AND_FETCH_8:
    1257            0 :     case BUILT_IN_SYNC_OR_AND_FETCH_8:
    1258            0 :     case BUILT_IN_SYNC_AND_AND_FETCH_8:
    1259            0 :     case BUILT_IN_SYNC_XOR_AND_FETCH_8:
    1260            0 :     case BUILT_IN_SYNC_NAND_AND_FETCH_8:
    1261            0 :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_8:
    1262            0 :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_8:
    1263            0 :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_8:
    1264            0 :     case BUILT_IN_SYNC_LOCK_RELEASE_8:
    1265            0 :     case BUILT_IN_ATOMIC_EXCHANGE_8:
    1266            0 :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_8:
    1267            0 :     case BUILT_IN_ATOMIC_STORE_8:
    1268            0 :     case BUILT_IN_ATOMIC_ADD_FETCH_8:
    1269            0 :     case BUILT_IN_ATOMIC_SUB_FETCH_8:
    1270            0 :     case BUILT_IN_ATOMIC_AND_FETCH_8:
    1271            0 :     case BUILT_IN_ATOMIC_NAND_FETCH_8:
    1272            0 :     case BUILT_IN_ATOMIC_XOR_FETCH_8:
    1273            0 :     case BUILT_IN_ATOMIC_OR_FETCH_8:
    1274            0 :     case BUILT_IN_ATOMIC_FETCH_ADD_8:
    1275            0 :     case BUILT_IN_ATOMIC_FETCH_SUB_8:
    1276            0 :     case BUILT_IN_ATOMIC_FETCH_AND_8:
    1277            0 :     case BUILT_IN_ATOMIC_FETCH_NAND_8:
    1278            0 :     case BUILT_IN_ATOMIC_FETCH_XOR_8:
    1279            0 :     case BUILT_IN_ATOMIC_FETCH_OR_8:
    1280            0 :       access_size = 8;
    1281            0 :       goto do_atomic;
    1282              : 
    1283            0 :     case BUILT_IN_ATOMIC_LOAD_16:
    1284            0 :       is_store = false;
    1285              :       /* FALLTHRU */
    1286              :     case BUILT_IN_SYNC_FETCH_AND_ADD_16:
    1287              :     case BUILT_IN_SYNC_FETCH_AND_SUB_16:
    1288              :     case BUILT_IN_SYNC_FETCH_AND_OR_16:
    1289              :     case BUILT_IN_SYNC_FETCH_AND_AND_16:
    1290              :     case BUILT_IN_SYNC_FETCH_AND_XOR_16:
    1291              :     case BUILT_IN_SYNC_FETCH_AND_NAND_16:
    1292              :     case BUILT_IN_SYNC_ADD_AND_FETCH_16:
    1293              :     case BUILT_IN_SYNC_SUB_AND_FETCH_16:
    1294              :     case BUILT_IN_SYNC_OR_AND_FETCH_16:
    1295              :     case BUILT_IN_SYNC_AND_AND_FETCH_16:
    1296              :     case BUILT_IN_SYNC_XOR_AND_FETCH_16:
    1297              :     case BUILT_IN_SYNC_NAND_AND_FETCH_16:
    1298              :     case BUILT_IN_SYNC_BOOL_COMPARE_AND_SWAP_16:
    1299              :     case BUILT_IN_SYNC_VAL_COMPARE_AND_SWAP_16:
    1300              :     case BUILT_IN_SYNC_LOCK_TEST_AND_SET_16:
    1301              :     case BUILT_IN_SYNC_LOCK_RELEASE_16:
    1302              :     case BUILT_IN_ATOMIC_EXCHANGE_16:
    1303              :     case BUILT_IN_ATOMIC_COMPARE_EXCHANGE_16:
    1304              :     case BUILT_IN_ATOMIC_STORE_16:
    1305              :     case BUILT_IN_ATOMIC_ADD_FETCH_16:
    1306              :     case BUILT_IN_ATOMIC_SUB_FETCH_16:
    1307              :     case BUILT_IN_ATOMIC_AND_FETCH_16:
    1308              :     case BUILT_IN_ATOMIC_NAND_FETCH_16:
    1309              :     case BUILT_IN_ATOMIC_XOR_FETCH_16:
    1310              :     case BUILT_IN_ATOMIC_OR_FETCH_16:
    1311              :     case BUILT_IN_ATOMIC_FETCH_ADD_16:
    1312              :     case BUILT_IN_ATOMIC_FETCH_SUB_16:
    1313              :     case BUILT_IN_ATOMIC_FETCH_AND_16:
    1314              :     case BUILT_IN_ATOMIC_FETCH_NAND_16:
    1315              :     case BUILT_IN_ATOMIC_FETCH_XOR_16:
    1316              :     case BUILT_IN_ATOMIC_FETCH_OR_16:
    1317              :       access_size = 16;
    1318              :       /* FALLTHRU */
    1319           56 :     do_atomic:
    1320           56 :       {
    1321           56 :         dest = gimple_call_arg (call, 0);
    1322              :         /* DEST represents the address of a memory location.
    1323              :            instrument_derefs wants the memory location, so lets
    1324              :            dereference the address DEST before handing it to
    1325              :            instrument_derefs.  */
    1326          112 :         tree type = build_nonstandard_integer_type (access_size
    1327           56 :                                                     * BITS_PER_UNIT, 1);
    1328           56 :         dest = build2 (MEM_REF, type, dest,
    1329              :                        build_int_cst (build_pointer_type (char_type_node), 0));
    1330           56 :         break;
    1331              :       }
    1332              : 
    1333              :     default:
    1334              :       /* The other builtins memory access are not instrumented in this
    1335              :          function because they either don't have any length parameter,
    1336              :          or their length parameter is just a limit.  */
    1337              :       break;
    1338              :     }
    1339              : 
    1340         2752 :   if (len != NULL_TREE)
    1341              :     {
    1342         1888 :       if (source0 != NULL_TREE)
    1343              :         {
    1344         1500 :           src0->start = source0;
    1345         1500 :           src0->access_size = access_size;
    1346         1500 :           *src0_len = len;
    1347         1500 :           *src0_is_store = false;
    1348              :         }
    1349              : 
    1350         1888 :       if (source1 != NULL_TREE)
    1351              :         {
    1352          112 :           src1->start = source1;
    1353          112 :           src1->access_size = access_size;
    1354          112 :           *src1_len = len;
    1355          112 :           *src1_is_store = false;
    1356              :         }
    1357              : 
    1358         1888 :       if (dest != NULL_TREE)
    1359              :         {
    1360         1704 :           dst->start = dest;
    1361         1704 :           dst->access_size = access_size;
    1362         1704 :           *dst_len = len;
    1363         1704 :           *dst_is_store = true;
    1364              :         }
    1365              : 
    1366              :       got_reference_p = true;
    1367              :     }
    1368         7412 :   else if (dest)
    1369              :     {
    1370           56 :       dst->start = dest;
    1371           56 :       dst->access_size = access_size;
    1372           56 :       *dst_len = NULL_TREE;
    1373           56 :       *dst_is_store = is_store;
    1374           56 :       *dest_is_deref = true;
    1375           56 :       got_reference_p = true;
    1376              :     }
    1377              : 
    1378              :   return got_reference_p;
    1379              : }
    1380              : 
    1381              : /* Return true iff a given gimple statement has been instrumented.
    1382              :    Note that the statement is "defined" by the memory references it
    1383              :    contains.  */
    1384              : 
    1385              : static bool
    1386       176064 : has_stmt_been_instrumented_p (gimple *stmt)
    1387              : {
    1388       176064 :   if (gimple_assign_single_p (stmt))
    1389              :     {
    1390        32653 :       bool r_is_store;
    1391        32653 :       asan_mem_ref r;
    1392        32653 :       asan_mem_ref_init (&r, NULL, 1);
    1393              : 
    1394        32653 :       if (get_mem_ref_of_assignment (as_a <gassign *> (stmt), &r,
    1395              :                                      &r_is_store))
    1396              :         {
    1397        29058 :           if (!has_mem_ref_been_instrumented (&r))
    1398        29058 :             return false;
    1399         1134 :           if (r_is_store && gimple_assign_load_p (stmt))
    1400              :             {
    1401            9 :               asan_mem_ref src;
    1402            9 :               asan_mem_ref_init (&src, NULL, 1);
    1403            9 :               src.start = gimple_assign_rhs1 (stmt);
    1404            9 :               src.access_size = int_size_in_bytes (TREE_TYPE (src.start));
    1405            9 :               if (!has_mem_ref_been_instrumented (&src))
    1406              :                 return false;
    1407              :             }
    1408         1125 :           return true;
    1409              :         }
    1410              :     }
    1411       143411 :   else if (gimple_call_builtin_p (stmt, BUILT_IN_NORMAL))
    1412              :     {
    1413         4684 :       asan_mem_ref src0, src1, dest;
    1414         4684 :       asan_mem_ref_init (&src0, NULL, 1);
    1415         4684 :       asan_mem_ref_init (&src1, NULL, 1);
    1416         4684 :       asan_mem_ref_init (&dest, NULL, 1);
    1417              : 
    1418         4684 :       tree src0_len = NULL_TREE, src1_len = NULL_TREE, dest_len = NULL_TREE;
    1419         4684 :       bool src0_is_store = false, src1_is_store = false,
    1420              :         dest_is_store = false, dest_is_deref = false, intercepted_p = true;
    1421         4684 :       if (get_mem_refs_of_builtin_call (as_a <gcall *> (stmt),
    1422              :                                         &src0, &src0_len, &src0_is_store,
    1423              :                                         &src1, &src1_len, &src1_is_store,
    1424              :                                         &dest, &dest_len, &dest_is_store,
    1425              :                                         &dest_is_deref, &intercepted_p))
    1426              :         {
    1427          984 :           if (src0.start != NULL_TREE
    1428          984 :               && !has_mem_ref_been_instrumented (&src0, src0_len))
    1429          984 :             return false;
    1430              : 
    1431          240 :           if (src1.start != NULL_TREE
    1432          240 :               && !has_mem_ref_been_instrumented (&src1, src1_len))
    1433              :             return false;
    1434              : 
    1435          240 :           if (dest.start != NULL_TREE
    1436          240 :               && !has_mem_ref_been_instrumented (&dest, dest_len))
    1437          228 :             return false;
    1438              : 
    1439              :           return true;
    1440              :         }
    1441              :     }
    1442       138727 :   else if (is_gimple_call (stmt)
    1443        18527 :            && gimple_store_p (stmt)
    1444       139015 :            && (gimple_call_builtin_p (stmt)
    1445          288 :                || gimple_call_internal_p (stmt)
    1446          287 :                || !aggregate_value_p (TREE_TYPE (gimple_call_lhs (stmt)),
    1447          287 :                                       gimple_call_fntype (stmt))))
    1448              :     {
    1449           62 :       asan_mem_ref r;
    1450           62 :       asan_mem_ref_init (&r, NULL, 1);
    1451              : 
    1452           62 :       r.start = gimple_call_lhs (stmt);
    1453           62 :       r.access_size = int_size_in_bytes (TREE_TYPE (r.start));
    1454           62 :       return has_mem_ref_been_instrumented (&r);
    1455              :     }
    1456              : 
    1457              :   return false;
    1458              : }
    1459              : 
    1460              : /*  Insert a memory reference into the hash table.  */
    1461              : 
    1462              : static void
    1463        41392 : update_mem_ref_hash_table (tree ref, HOST_WIDE_INT access_size)
    1464              : {
    1465        41392 :   hash_table<asan_mem_ref_hasher> *ht = get_mem_ref_hash_table ();
    1466              : 
    1467        41392 :   asan_mem_ref r;
    1468        41392 :   asan_mem_ref_init (&r, ref, access_size);
    1469              : 
    1470        41392 :   asan_mem_ref **slot = ht->find_slot (&r, INSERT);
    1471        41392 :   if (*slot == NULL || (*slot)->access_size < access_size)
    1472        41380 :     *slot = asan_mem_ref_new (ref, access_size);
    1473        41392 : }
    1474              : 
    1475              : /* Initialize shadow_ptr_types array.  */
    1476              : 
    1477              : static void
    1478         2494 : asan_init_shadow_ptr_types (void)
    1479              : {
    1480         2494 :   asan_shadow_set = new_alias_set ();
    1481         2494 :   tree types[3] = { signed_char_type_node, short_integer_type_node,
    1482         2494 :                     integer_type_node };
    1483              : 
    1484         9976 :   for (unsigned i = 0; i < 3; i++)
    1485              :     {
    1486         7482 :       shadow_ptr_types[i] = build_distinct_type_copy (types[i]);
    1487         7482 :       TYPE_ALIAS_SET (shadow_ptr_types[i]) = asan_shadow_set;
    1488         7482 :       shadow_ptr_types[i] = build_pointer_type (shadow_ptr_types[i]);
    1489              :     }
    1490              : 
    1491         2494 :   initialize_sanitizer_builtins ();
    1492         2494 : }
    1493              : 
    1494              : /* Create ADDR_EXPR of STRING_CST with the PP pretty printer text.  */
    1495              : 
    1496              : static tree
    1497        12499 : asan_pp_string (pretty_printer *pp)
    1498              : {
    1499        12499 :   const char *buf = pp_formatted_text (pp);
    1500        12499 :   size_t len = strlen (buf);
    1501        12499 :   tree ret = build_string (len + 1, buf);
    1502        24998 :   TREE_TYPE (ret)
    1503        12499 :     = build_array_type (TREE_TYPE (shadow_ptr_types[0]),
    1504        12499 :                         build_index_type (size_int (len)));
    1505        12499 :   TREE_READONLY (ret) = 1;
    1506        12499 :   TREE_STATIC (ret) = 1;
    1507        12499 :   return build1 (ADDR_EXPR, shadow_ptr_types[0], ret);
    1508              : }
    1509              : 
    1510              : /* Clear shadow memory at SHADOW_MEM, LEN bytes.  Can't call a library call here
    1511              :    though.  */
    1512              : 
    1513              : static void
    1514         2477 : asan_clear_shadow (rtx shadow_mem, HOST_WIDE_INT len)
    1515              : {
    1516         2477 :   rtx_insn *insn, *insns, *jump;
    1517         2477 :   rtx_code_label *top_label;
    1518         2477 :   rtx end, addr, tmp;
    1519              : 
    1520         2477 :   gcc_assert ((len & 3) == 0);
    1521         2477 :   start_sequence ();
    1522         2477 :   clear_storage (shadow_mem, GEN_INT (len), BLOCK_OP_NORMAL);
    1523         2477 :   insns = end_sequence ();
    1524         8846 :   for (insn = insns; insn; insn = NEXT_INSN (insn))
    1525         3905 :     if (CALL_P (insn))
    1526              :       break;
    1527         2477 :   if (insn == NULL_RTX)
    1528              :     {
    1529         2464 :       emit_insn (insns);
    1530         2464 :       return;
    1531              :     }
    1532              : 
    1533           13 :   top_label = gen_label_rtx ();
    1534           13 :   addr = copy_to_mode_reg (Pmode, XEXP (shadow_mem, 0));
    1535           13 :   shadow_mem = adjust_automodify_address (shadow_mem, SImode, addr, 0);
    1536           13 :   end = force_reg (Pmode, plus_constant (Pmode, addr, len));
    1537           13 :   emit_label (top_label);
    1538              : 
    1539           13 :   emit_move_insn (shadow_mem, const0_rtx);
    1540           13 :   tmp = expand_simple_binop (Pmode, PLUS, addr, gen_int_mode (4, Pmode), addr,
    1541              :                              true, OPTAB_LIB_WIDEN);
    1542           13 :   if (tmp != addr)
    1543            0 :     emit_move_insn (addr, tmp);
    1544           13 :   emit_cmp_and_jump_insns (addr, end, LT, NULL_RTX, Pmode, true, top_label);
    1545           13 :   jump = get_last_insn ();
    1546           13 :   gcc_assert (JUMP_P (jump));
    1547           13 :   add_reg_br_prob_note (jump,
    1548           26 :                         profile_probability::guessed_always ()
    1549              :                            .apply_scale (80, 100));
    1550              : }
    1551              : 
    1552              : void
    1553         6428 : asan_function_start (void)
    1554              : {
    1555         6428 :   ASM_OUTPUT_DEBUG_LABEL (asm_out_file, "LASANPC", current_function_funcdef_no);
    1556         6428 : }
    1557              : 
    1558              : /* Return number of shadow bytes that are occupied by a local variable
    1559              :    of SIZE bytes.  */
    1560              : 
    1561              : static unsigned HOST_WIDE_INT
    1562         2096 : shadow_mem_size (unsigned HOST_WIDE_INT size)
    1563              : {
    1564              :   /* It must be possible to align stack variables to granularity
    1565              :      of shadow memory.  */
    1566         2096 :   gcc_assert (BITS_PER_UNIT
    1567              :               * ASAN_SHADOW_GRANULARITY <= MAX_SUPPORTED_STACK_ALIGNMENT);
    1568              : 
    1569         2096 :   return ROUND_UP (size, ASAN_SHADOW_GRANULARITY) / ASAN_SHADOW_GRANULARITY;
    1570              : }
    1571              : 
    1572              : /* Always emit 4 bytes at a time.  */
    1573              : #define RZ_BUFFER_SIZE 4
    1574              : 
    1575              : /* ASAN redzone buffer container that handles emission of shadow bytes.  */
    1576         3476 : class asan_redzone_buffer
    1577              : {
    1578              : public:
    1579              :   /* Constructor.  */
    1580         1738 :   asan_redzone_buffer (rtx shadow_mem, HOST_WIDE_INT prev_offset):
    1581         1738 :     m_shadow_mem (shadow_mem), m_prev_offset (prev_offset),
    1582         1738 :     m_original_offset (prev_offset), m_shadow_bytes (RZ_BUFFER_SIZE)
    1583              :   {}
    1584              : 
    1585              :   /* Emit VALUE shadow byte at a given OFFSET.  */
    1586              :   void emit_redzone_byte (HOST_WIDE_INT offset, unsigned char value);
    1587              : 
    1588              :   /* Emit RTX emission of the content of the buffer.  */
    1589              :   void flush_redzone_payload (void);
    1590              : 
    1591              : private:
    1592              :   /* Flush if the content of the buffer is full
    1593              :      (equal to RZ_BUFFER_SIZE).  */
    1594              :   void flush_if_full (void);
    1595              : 
    1596              :   /* Memory where we last emitted a redzone payload.  */
    1597              :   rtx m_shadow_mem;
    1598              : 
    1599              :   /* Relative offset where we last emitted a redzone payload.  */
    1600              :   HOST_WIDE_INT m_prev_offset;
    1601              : 
    1602              :   /* Relative original offset.  Used for checking only.  */
    1603              :   HOST_WIDE_INT m_original_offset;
    1604              : 
    1605              : public:
    1606              :   /* Buffer with redzone payload.  */
    1607              :   auto_vec<unsigned char> m_shadow_bytes;
    1608              : };
    1609              : 
    1610              : /* Emit VALUE shadow byte at a given OFFSET.  */
    1611              : 
    1612              : void
    1613        23207 : asan_redzone_buffer::emit_redzone_byte (HOST_WIDE_INT offset,
    1614              :                                         unsigned char value)
    1615              : {
    1616        23207 :   gcc_assert ((offset & (ASAN_SHADOW_GRANULARITY - 1)) == 0);
    1617        23207 :   gcc_assert (offset >= m_prev_offset);
    1618              : 
    1619        23207 :   HOST_WIDE_INT off
    1620        23207 :     = m_prev_offset + ASAN_SHADOW_GRANULARITY * m_shadow_bytes.length ();
    1621        23207 :   if (off == offset)
    1622              :     /* Consecutive shadow memory byte.  */;
    1623         4840 :   else if (offset < m_prev_offset + (HOST_WIDE_INT) (ASAN_SHADOW_GRANULARITY
    1624              :                                                      * RZ_BUFFER_SIZE)
    1625         4840 :            && !m_shadow_bytes.is_empty ())
    1626              :     {
    1627              :       /* Shadow memory byte with a small gap.  */
    1628           68 :       for (; off < offset; off += ASAN_SHADOW_GRANULARITY)
    1629           34 :         m_shadow_bytes.safe_push (0);
    1630              :     }
    1631              :   else
    1632              :     {
    1633         4806 :       if (!m_shadow_bytes.is_empty ())
    1634          380 :         flush_redzone_payload ();
    1635              : 
    1636              :       /* Maybe start earlier in order to use aligned store.  */
    1637         4806 :       HOST_WIDE_INT align = (offset - m_prev_offset) % ASAN_RED_ZONE_SIZE;
    1638         4806 :       if (align)
    1639              :         {
    1640         1321 :           offset -= align;
    1641         3496 :           for (unsigned i = 0; i < align / BITS_PER_UNIT; i++)
    1642         2175 :             m_shadow_bytes.safe_push (0);
    1643              :         }
    1644              : 
    1645              :       /* Adjust m_prev_offset and m_shadow_mem.  */
    1646         4806 :       HOST_WIDE_INT diff = offset - m_prev_offset;
    1647         4806 :       m_shadow_mem = adjust_address (m_shadow_mem, VOIDmode,
    1648              :                                      diff >> ASAN_SHADOW_SHIFT);
    1649         4806 :       m_prev_offset = offset;
    1650              :     }
    1651        23207 :   m_shadow_bytes.safe_push (value);
    1652        23207 :   flush_if_full ();
    1653        23207 : }
    1654              : 
    1655              : /* Emit RTX emission of the content of the buffer.  */
    1656              : 
    1657              : void
    1658         6544 : asan_redzone_buffer::flush_redzone_payload (void)
    1659              : {
    1660         6544 :   gcc_assert (WORDS_BIG_ENDIAN == BYTES_BIG_ENDIAN);
    1661              : 
    1662         6544 :   if (m_shadow_bytes.is_empty ())
    1663         6544 :     return;
    1664              : 
    1665              :   /* Be sure we always emit to an aligned address.  */
    1666         6544 :   gcc_assert (((m_prev_offset - m_original_offset)
    1667              :               & (ASAN_RED_ZONE_SIZE - 1)) == 0);
    1668              : 
    1669              :   /* Fill it to RZ_BUFFER_SIZE bytes with zeros if needed.  */
    1670              :   unsigned l = m_shadow_bytes.length ();
    1671        13848 :   for (unsigned i = 0; i <= RZ_BUFFER_SIZE - l; i++)
    1672         7304 :     m_shadow_bytes.safe_push (0);
    1673              : 
    1674         6544 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1675            0 :     fprintf (dump_file,
    1676              :              "Flushing rzbuffer at offset %" PRId64 " with: ", m_prev_offset);
    1677              : 
    1678         6544 :   unsigned HOST_WIDE_INT val = 0;
    1679        32720 :   for (unsigned i = 0; i < RZ_BUFFER_SIZE; i++)
    1680              :     {
    1681        26176 :       unsigned char v
    1682        26176 :         = m_shadow_bytes[BYTES_BIG_ENDIAN ? RZ_BUFFER_SIZE - i - 1 : i];
    1683        26176 :       val |= (unsigned HOST_WIDE_INT)v << (BITS_PER_UNIT * i);
    1684        26176 :       if (dump_file && (dump_flags & TDF_DETAILS))
    1685            0 :         fprintf (dump_file, "%02x ", v);
    1686              :     }
    1687              : 
    1688         6544 :   if (dump_file && (dump_flags & TDF_DETAILS))
    1689            0 :     fprintf (dump_file, "\n");
    1690              : 
    1691         6544 :   rtx c = gen_int_mode (val, SImode);
    1692         6544 :   m_shadow_mem = adjust_address (m_shadow_mem, SImode, 0);
    1693         6544 :   emit_move_insn (m_shadow_mem, c);
    1694         6544 :   m_shadow_bytes.truncate (0);
    1695              : }
    1696              : 
    1697              : /* Flush if the content of the buffer is full
    1698              :    (equal to RZ_BUFFER_SIZE).  */
    1699              : 
    1700              : void
    1701        23207 : asan_redzone_buffer::flush_if_full (void)
    1702              : {
    1703        23207 :   if (m_shadow_bytes.length () == RZ_BUFFER_SIZE)
    1704         6164 :     flush_redzone_payload ();
    1705        23207 : }
    1706              : 
    1707              : 
    1708              : /* HWAddressSanitizer (hwasan) is a probabilistic method for detecting
    1709              :    out-of-bounds and use-after-free bugs.
    1710              :    Read more:
    1711              :    http://code.google.com/p/address-sanitizer/
    1712              : 
    1713              :    Similar to AddressSanitizer (asan) it consists of two parts: the
    1714              :    instrumentation module in this file, and a run-time library.
    1715              : 
    1716              :    The instrumentation module adds a run-time check before every memory insn in
    1717              :    the same manner as asan (see the block comment for AddressSanitizer above).
    1718              :    Currently, hwasan only adds out-of-line instrumentation, where each check is
    1719              :    implemented as a function call to the run-time library.  Hence a check for a
    1720              :    load of N bytes from address X would be implemented with a function call to
    1721              :    __hwasan_loadN(X), and checking a store of N bytes from address X would be
    1722              :    implemented with a function call to __hwasan_storeN(X).
    1723              : 
    1724              :    The main difference between hwasan and asan is in the information stored to
    1725              :    help this checking.  Both sanitizers use a shadow memory area which stores
    1726              :    data recording the state of main memory at a corresponding address.
    1727              : 
    1728              :    For hwasan, each 16 byte granule in main memory has a corresponding 1 byte
    1729              :    in shadow memory.  This shadow address can be calculated with equation:
    1730              :      (addr >> log_2(HWASAN_TAG_GRANULE_SIZE))
    1731              :           + __hwasan_shadow_memory_dynamic_address;
    1732              :    The conversion between real and shadow memory for asan is given in the block
    1733              :    comment at the top of this file.
    1734              :    The description of how this shadow memory is laid out for asan is in the
    1735              :    block comment at the top of this file, here we describe how this shadow
    1736              :    memory is used for hwasan.
    1737              : 
    1738              :    For hwasan, each variable is assigned a byte-sized 'tag'.  The extent of
    1739              :    the shadow memory for that variable is filled with the assigned tag, and
    1740              :    every pointer referencing that variable has its top byte set to the same
    1741              :    tag.  The run-time library redefines malloc so that every allocation returns
    1742              :    a tagged pointer and tags the corresponding shadow memory with the same tag.
    1743              : 
    1744              :    On each pointer dereference the tag found in the pointer is compared to the
    1745              :    tag found in the shadow memory corresponding to the accessed memory address.
    1746              :    If these tags are found to differ then this memory access is judged to be
    1747              :    invalid and a report is generated.
    1748              : 
    1749              :    This method of bug detection is not perfect -- it can not catch every bad
    1750              :    access -- but catches them probabilistically instead.  There is always the
    1751              :    possibility that an invalid memory access will happen to access memory
    1752              :    tagged with the same tag as the pointer that this access used.
    1753              :    The chances of this are approx. 0.4% for any two uncorrelated objects.
    1754              : 
    1755              :    Random tag generation can mitigate this problem by decreasing the
    1756              :    probability that an invalid access will be missed in the same manner over
    1757              :    multiple runs.  i.e. if two objects are tagged the same in one run of the
    1758              :    binary they are unlikely to be tagged the same in the next run.
    1759              :    Both heap and stack allocated objects have random tags by default.
    1760              : 
    1761              :    [16 byte granule implications]
    1762              :     Since the shadow memory only has a resolution on real memory of 16 bytes,
    1763              :     invalid accesses that are within the same 16 byte granule as a valid
    1764              :     address will not be caught.
    1765              : 
    1766              :     There is a "short-granule" feature in the runtime library which does catch
    1767              :     such accesses, but this feature is not implemented for stack objects (since
    1768              :     stack objects are allocated and tagged by compiler instrumentation, and
    1769              :     this feature has not yet been implemented in GCC instrumentation).
    1770              : 
    1771              :     Another outcome of this 16 byte resolution is that each tagged object must
    1772              :     be 16 byte aligned.  If two objects were to share any 16 byte granule in
    1773              :     memory, then they both would have to be given the same tag, and invalid
    1774              :     accesses to one using a pointer to the other would be undetectable.
    1775              : 
    1776              :    [Compiler instrumentation]
    1777              :     Compiler instrumentation ensures that two adjacent buffers on the stack are
    1778              :     given different tags, this means an access to one buffer using a pointer
    1779              :     generated from the other (e.g. through buffer overrun) will have mismatched
    1780              :     tags and be caught by hwasan.
    1781              : 
    1782              :     We don't randomly tag every object on the stack, since that would require
    1783              :     keeping many registers to record each tag.  Instead we randomly generate a
    1784              :     tag for each function frame, and each new stack object uses a tag offset
    1785              :     from that frame tag.
    1786              :     i.e. each object is tagged as RFT + offset, where RFT is the "random frame
    1787              :     tag" generated for this frame.
    1788              :     This means that randomisation does not peturb the difference between tags
    1789              :     on tagged stack objects within a frame, but this is mitigated by the fact
    1790              :     that objects with the same tag within a frame are very far apart
    1791              :     (approx. 2^HWASAN_TAG_SIZE objects apart).
    1792              : 
    1793              :     As a demonstration, using the same example program as in the asan block
    1794              :     comment above:
    1795              : 
    1796              :      int
    1797              :      foo ()
    1798              :      {
    1799              :        char a[24] = {0};
    1800              :        int b[2] = {0};
    1801              : 
    1802              :        a[5] = 1;
    1803              :        b[1] = 2;
    1804              : 
    1805              :        return a[5] + b[1];
    1806              :      }
    1807              : 
    1808              :     On AArch64 the stack will be ordered as follows for the above function:
    1809              : 
    1810              :     Slot 1/ [24 bytes for variable 'a']
    1811              :     Slot 2/ [8 bytes padding for alignment]
    1812              :     Slot 3/ [8 bytes for variable 'b']
    1813              :     Slot 4/ [8 bytes padding for alignment]
    1814              : 
    1815              :     (The padding is there to ensure 16 byte alignment as described in the 16
    1816              :      byte granule implications).
    1817              : 
    1818              :     While the shadow memory will be ordered as follows:
    1819              : 
    1820              :     - 2 bytes (representing 32 bytes in real memory) tagged with RFT + 1.
    1821              :     - 1 byte (representing 16 bytes in real memory) tagged with RFT + 2.
    1822              : 
    1823              :     And any pointer to "a" will have the tag RFT + 1, and any pointer to "b"
    1824              :     will have the tag RFT + 2.
    1825              : 
    1826              :    [Top Byte Ignore requirements]
    1827              :     Hwasan requires the ability to store an 8 bit tag in every pointer.  There
    1828              :     is no instrumentation done to remove this tag from pointers before
    1829              :     dereferencing, which means the hardware must ignore this tag during memory
    1830              :     accesses.
    1831              : 
    1832              :     Architectures where this feature is available should indicate this using
    1833              :     the TARGET_MEMTAG_CAN_TAG_ADDRESSES hook.
    1834              : 
    1835              :    [Stack requires cleanup on unwinding]
    1836              :     During normal operation of a hwasan sanitized program more space in the
    1837              :     shadow memory becomes tagged as the stack grows.  As the stack shrinks this
    1838              :     shadow memory space must become untagged.  If it is not untagged then when
    1839              :     the stack grows again (during other function calls later on in the program)
    1840              :     objects on the stack that are usually not tagged (e.g. parameters passed on
    1841              :     the stack) can be placed in memory whose shadow space is tagged with
    1842              :     something else, and accesses can cause false positive reports.
    1843              : 
    1844              :     Hence we place untagging code on every epilogue of functions which tag some
    1845              :     stack objects.
    1846              : 
    1847              :     Moreover, the run-time library intercepts longjmp & setjmp to untag when
    1848              :     the stack is unwound this way.
    1849              : 
    1850              :     C++ exceptions are not yet handled, which means this sanitizer can not
    1851              :     handle C++ code that throws exceptions -- it will give false positives
    1852              :     after an exception has been thrown.  The implementation that the hwasan
    1853              :     library has for handling these relies on the frame pointer being after any
    1854              :     local variables.  This is not generally the case for GCC.  */
    1855              : 
    1856              : 
    1857              : /* Returns whether we are tagging pointers and checking those tags on memory
    1858              :    access.  */
    1859              : bool
    1860     36940664 : hwasan_sanitize_p ()
    1861              : {
    1862     36940664 :   return sanitize_flags_p (SANITIZE_HWADDRESS);
    1863              : }
    1864              : 
    1865              : /* Are we tagging the stack?  */
    1866              : bool
    1867     35051910 : hwasan_sanitize_stack_p ()
    1868              : {
    1869     35051910 :   return (hwasan_sanitize_p () && param_hwasan_instrument_stack);
    1870              : }
    1871              : 
    1872              : /* Are we tagging alloca objects?  */
    1873              : bool
    1874      1518134 : hwasan_sanitize_allocas_p (void)
    1875              : {
    1876      1518134 :   return (hwasan_sanitize_stack_p () && param_hwasan_instrument_allocas);
    1877              : }
    1878              : 
    1879              : /* Should we instrument reads?  */
    1880              : bool
    1881          368 : hwasan_instrument_reads (void)
    1882              : {
    1883          368 :   return (hwasan_sanitize_p () && param_hwasan_instrument_reads);
    1884              : }
    1885              : 
    1886              : /* Should we instrument writes?  */
    1887              : bool
    1888          210 : hwasan_instrument_writes (void)
    1889              : {
    1890          210 :   return (hwasan_sanitize_p () && param_hwasan_instrument_writes);
    1891              : }
    1892              : 
    1893              : /* Should we instrument builtin calls?  */
    1894              : bool
    1895           94 : hwasan_memintrin (void)
    1896              : {
    1897           94 :   return (hwasan_sanitize_p () && param_hwasan_instrument_mem_intrinsics);
    1898              : }
    1899              : 
    1900              : /* MEMoryTAGging sanitizer (MEMTAG) uses a hardware based capability known as
    1901              :    memory tagging to detect memory safety vulnerabilities.  Similar to HWASAN,
    1902              :    it is also a probabilistic method.
    1903              : 
    1904              :    MEMTAG relies on the optional extension in armv8.5a known as MTE (Memory
    1905              :    Tagging Extension).  The extension is available in AArch64 only and
    1906              :    introduces two types of tags:
    1907              :      - Logical Address Tag - bits 56-59 (TARGET_MEMTAG_TAG_BITSIZE) of the
    1908              :        virtual address.
    1909              :      - Allocation Tag - 4 bits for each tag granule (TARGET_MEMTAG_GRANULE_SIZE
    1910              :        set to 16 bytes), stored separately.
    1911              :    Load / store instructions raise an exception if tags differ, thereby
    1912              :    providing a faster way (than HWASAN) to detect memory safety issues.
    1913              :    Further, new instructions are available in MTE to manipulate (generate,
    1914              :    update address with) tags.  Load / store instructions with SP base register
    1915              :    and immediate offset do not check tags.
    1916              : 
    1917              :    PS: Currently, MEMTAG sanitizer is capable of stack (variable / memory)
    1918              :    tagging only.
    1919              : 
    1920              :    In general, detecting stack-related memory bugs requires the compiler to:
    1921              :      - ensure that each tag granule is only used by one variable at a time.
    1922              :        This includes alloca.
    1923              :      - Tag/Color: put tags into each stack variable pointer.
    1924              :      - Untag: the function epilogue will retag the memory.
    1925              : 
    1926              :    MEMTAG sanitizer is based off the HWASAN sanitizer implementation
    1927              :    internally.  Similar to HWASAN:
    1928              :      - Assigning an independently random tag to each variable is carried out by
    1929              :        keeping a tagged base pointer.  A tagged base pointer allows addressing
    1930              :        variables with (addr offset, tag offset).
    1931              :    */
    1932              : 
    1933              : /* Returns whether we are tagging pointers and checking those tags on memory
    1934              :    access.  */
    1935              : bool
    1936      3125878 : memtag_sanitize_p ()
    1937              : {
    1938      3125878 :   return sanitize_flags_p (SANITIZE_MEMTAG);
    1939              : }
    1940              : 
    1941              : /* Are we tagging the stack?  */
    1942              : bool
    1943     33515809 : memtag_sanitize_stack_p ()
    1944              : {
    1945     33515809 :   return (sanitize_flags_p (SANITIZE_MEMTAG_STACK));
    1946              : }
    1947              : 
    1948              : /* Are we tagging alloca objects?  */
    1949              : bool
    1950          404 : memtag_sanitize_allocas_p (void)
    1951              : {
    1952          404 :   return (memtag_sanitize_stack_p () && param_memtag_instrument_allocas);
    1953              : }
    1954              : 
    1955              : /* Are we taggin mem intrinsics?  */
    1956              : bool
    1957           24 : memtag_memintrin (void)
    1958              : {
    1959           24 :   return (memtag_sanitize_p () && param_memtag_instrument_mem_intrinsics);
    1960              : }
    1961              : 
    1962              : /* Returns whether we are tagging pointers and checking those tags on memory
    1963              :    access.  */
    1964              : bool
    1965        98365 : hwassist_sanitize_p ()
    1966              : {
    1967        98365 :   return (hwasan_sanitize_p () || memtag_sanitize_p ());
    1968              : }
    1969              : 
    1970              : /* Are we tagging stack objects for hwasan or memtag?  */
    1971              : bool
    1972     33520055 : hwassist_sanitize_stack_p ()
    1973              : {
    1974     33520055 :   return (hwasan_sanitize_stack_p () || memtag_sanitize_stack_p ());
    1975              : }
    1976              : 
    1977              : /* Insert code to protect stack vars.  The prologue sequence should be emitted
    1978              :    directly, epilogue sequence returned.  BASE is the register holding the
    1979              :    stack base, against which OFFSETS array offsets are relative to, OFFSETS
    1980              :    array contains pairs of offsets in reverse order, always the end offset
    1981              :    of some gap that needs protection followed by starting offset,
    1982              :    and DECLS is an array of representative decls for each var partition.
    1983              :    LENGTH is the length of the OFFSETS array, DECLS array is LENGTH / 2 - 1
    1984              :    elements long (OFFSETS include gap before the first variable as well
    1985              :    as gaps after each stack variable).  PBASE is, if non-NULL, some pseudo
    1986              :    register which stack vars DECL_RTLs are based on.  Either BASE should be
    1987              :    assigned to PBASE, when not doing use after return protection, or
    1988              :    corresponding address based on __asan_stack_malloc* return value.  */
    1989              : 
    1990              : rtx_insn *
    1991         1738 : asan_emit_stack_protection (rtx base, rtx pbase, unsigned int alignb,
    1992              :                             HOST_WIDE_INT *offsets, tree *decls, int length)
    1993              : {
    1994         1738 :   rtx shadow_base, shadow_mem, ret, mem, orig_base;
    1995         1738 :   rtx_code_label *lab;
    1996         1738 :   rtx_insn *insns;
    1997         1738 :   char buf[32];
    1998         1738 :   HOST_WIDE_INT base_offset = offsets[length - 1];
    1999         1738 :   HOST_WIDE_INT base_align_bias = 0, offset, prev_offset;
    2000         1738 :   HOST_WIDE_INT asan_frame_size = offsets[0] - base_offset;
    2001         1738 :   HOST_WIDE_INT last_offset, last_size, last_size_aligned;
    2002         1738 :   int l;
    2003         1738 :   unsigned char cur_shadow_byte = ASAN_STACK_MAGIC_LEFT;
    2004         1738 :   tree str_cst, decl, id;
    2005         1738 :   int use_after_return_class = -1;
    2006              : 
    2007              :   /* Don't emit anything when doing error recovery, the assertions
    2008              :      might fail e.g. if a function had a frame offset overflow.  */
    2009         1738 :   if (seen_error ())
    2010              :     return NULL;
    2011              : 
    2012         1738 :   if (shadow_ptr_types[0] == NULL_TREE)
    2013            0 :     asan_init_shadow_ptr_types ();
    2014              : 
    2015         1738 :   expanded_location cfun_xloc
    2016         1738 :     = expand_location (DECL_SOURCE_LOCATION (current_function_decl));
    2017              : 
    2018              :   /* First of all, prepare the description string.  */
    2019         1738 :   pretty_printer asan_pp;
    2020              : 
    2021         1738 :   pp_decimal_int (&asan_pp, length / 2 - 1);
    2022         1738 :   pp_space (&asan_pp);
    2023         5125 :   for (l = length - 2; l; l -= 2)
    2024              :     {
    2025         3387 :       tree decl = decls[l / 2 - 1];
    2026         3387 :       pp_wide_integer (&asan_pp, offsets[l] - base_offset);
    2027         3387 :       pp_space (&asan_pp);
    2028         3387 :       pp_wide_integer (&asan_pp, offsets[l - 1] - offsets[l]);
    2029         3387 :       pp_space (&asan_pp);
    2030              : 
    2031         3387 :       expanded_location xloc
    2032         3387 :         = expand_location (DECL_SOURCE_LOCATION (decl));
    2033         3387 :       char location[32];
    2034              : 
    2035         3387 :       if (xloc.file == cfun_xloc.file)
    2036         3163 :         sprintf (location, ":%d", xloc.line);
    2037              :       else
    2038          224 :         location[0] = '\0';
    2039              : 
    2040         3387 :       if (DECL_P (decl) && DECL_NAME (decl))
    2041              :         {
    2042         2829 :           unsigned idlen
    2043         2829 :             = IDENTIFIER_LENGTH (DECL_NAME (decl)) + strlen (location);
    2044         2829 :           pp_decimal_int (&asan_pp, idlen);
    2045         2829 :           pp_space (&asan_pp);
    2046         2829 :           pp_tree_identifier (&asan_pp, DECL_NAME (decl));
    2047         2829 :           pp_string (&asan_pp, location);
    2048              :         }
    2049              :       else
    2050          558 :         pp_string (&asan_pp, "9 <unknown>");
    2051              : 
    2052         3387 :       if (l > 2)
    2053         1649 :         pp_space (&asan_pp);
    2054              :     }
    2055         1738 :   str_cst = asan_pp_string (&asan_pp);
    2056              : 
    2057         3415 :   gcc_checking_assert (offsets[0] == (crtl->stack_protect_guard
    2058              :                                       ? -ASAN_RED_ZONE_SIZE : 0));
    2059              :   /* Emit the prologue sequence.  */
    2060         1738 :   if (asan_frame_size > 32 && asan_frame_size <= 65536 && pbase
    2061         1707 :       && param_asan_use_after_return)
    2062              :     {
    2063         1673 :       HOST_WIDE_INT adjusted_frame_size = asan_frame_size;
    2064              :       /* The stack protector guard is allocated at the top of the frame
    2065              :          and cfgexpand.cc then uses align_frame_offset (ASAN_RED_ZONE_SIZE);
    2066              :          while in that case we can still use asan_frame_size, we need to take
    2067              :          that into account when computing base_align_bias.  */
    2068         1673 :       if (alignb > ASAN_RED_ZONE_SIZE && crtl->stack_protect_guard)
    2069           28 :         adjusted_frame_size += ASAN_RED_ZONE_SIZE;
    2070         1673 :       use_after_return_class = floor_log2 (asan_frame_size - 1) - 5;
    2071              :       /* __asan_stack_malloc_N guarantees alignment
    2072              :          N < 6 ? (64 << N) : 4096 bytes.  */
    2073         1673 :       if (alignb > (use_after_return_class < 6
    2074         1673 :                     ? (64U << use_after_return_class) : 4096U))
    2075              :         use_after_return_class = -1;
    2076         1673 :       else if (alignb > ASAN_RED_ZONE_SIZE
    2077           54 :                && (adjusted_frame_size & (alignb - 1)))
    2078              :         {
    2079           28 :           base_align_bias
    2080           28 :             = ((adjusted_frame_size + alignb - 1)
    2081           28 :                & ~(alignb - HOST_WIDE_INT_1)) - adjusted_frame_size;
    2082           28 :           use_after_return_class
    2083           28 :             = floor_log2 (asan_frame_size + base_align_bias - 1) - 5;
    2084           28 :           if (use_after_return_class > 10)
    2085              :             {
    2086           65 :               base_align_bias = 0;
    2087           65 :               use_after_return_class = -1;
    2088              :             }
    2089              :         }
    2090              :     }
    2091              : 
    2092              :   /* Align base if target is STRICT_ALIGNMENT.  */
    2093         1738 :   if (STRICT_ALIGNMENT)
    2094              :     {
    2095              :       const HOST_WIDE_INT align
    2096              :         = (GET_MODE_ALIGNMENT (SImode) / BITS_PER_UNIT) << ASAN_SHADOW_SHIFT;
    2097              :       base = expand_binop (Pmode, and_optab, base, gen_int_mode (-align, Pmode),
    2098              :                            NULL_RTX, 1, OPTAB_DIRECT);
    2099              :     }
    2100              : 
    2101         1738 :   if (use_after_return_class == -1 && pbase)
    2102           65 :     emit_move_insn (pbase, base);
    2103              : 
    2104         1738 :   base = expand_binop (Pmode, add_optab, base,
    2105         1738 :                        gen_int_mode (base_offset - base_align_bias, Pmode),
    2106              :                        NULL_RTX, 1, OPTAB_DIRECT);
    2107         1738 :   orig_base = NULL_RTX;
    2108         1738 :   if (use_after_return_class != -1)
    2109              :     {
    2110         1673 :       if (asan_detect_stack_use_after_return == NULL_TREE)
    2111              :         {
    2112         1222 :           id = get_identifier ("__asan_option_detect_stack_use_after_return");
    2113         1222 :           decl = build_decl (BUILTINS_LOCATION, VAR_DECL, id,
    2114              :                              integer_type_node);
    2115         1222 :           SET_DECL_ASSEMBLER_NAME (decl, id);
    2116         1222 :           TREE_ADDRESSABLE (decl) = 1;
    2117         1222 :           DECL_ARTIFICIAL (decl) = 1;
    2118         1222 :           DECL_IGNORED_P (decl) = 1;
    2119         1222 :           DECL_EXTERNAL (decl) = 1;
    2120         1222 :           TREE_STATIC (decl) = 1;
    2121         1222 :           TREE_PUBLIC (decl) = 1;
    2122         1222 :           TREE_USED (decl) = 1;
    2123         1222 :           asan_detect_stack_use_after_return = decl;
    2124              :         }
    2125         1673 :       orig_base = gen_reg_rtx (Pmode);
    2126         1673 :       emit_move_insn (orig_base, base);
    2127         1673 :       ret = expand_normal (asan_detect_stack_use_after_return);
    2128         1673 :       lab = gen_label_rtx ();
    2129         1673 :       emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX,
    2130              :                                VOIDmode, 0, lab,
    2131              :                                profile_probability::very_likely ());
    2132         1673 :       snprintf (buf, sizeof buf, "__asan_stack_malloc_%d",
    2133              :                 use_after_return_class);
    2134         1673 :       ret = init_one_libfunc (buf);
    2135         1673 :       ret = emit_library_call_value (ret, NULL_RTX, LCT_NORMAL, ptr_mode,
    2136              :                                      GEN_INT (asan_frame_size
    2137              :                                               + base_align_bias),
    2138         1673 :                                      TYPE_MODE (pointer_sized_int_node));
    2139              :       /* __asan_stack_malloc_[n] returns a pointer to fake stack if succeeded
    2140              :          and NULL otherwise.  Check RET value is NULL here and jump over the
    2141              :          BASE reassignment in this case.  Otherwise, reassign BASE to RET.  */
    2142         1673 :       emit_cmp_and_jump_insns (ret, const0_rtx, EQ, NULL_RTX,
    2143              :                                VOIDmode, 0, lab,
    2144              :                                profile_probability:: very_unlikely ());
    2145         1673 :       ret = convert_memory_address (Pmode, ret);
    2146         1673 :       emit_move_insn (base, ret);
    2147         1673 :       emit_label (lab);
    2148         1673 :       emit_move_insn (pbase, expand_binop (Pmode, add_optab, base,
    2149              :                                            gen_int_mode (base_align_bias
    2150         1673 :                                                          - base_offset, Pmode),
    2151              :                                            NULL_RTX, 1, OPTAB_DIRECT));
    2152              :     }
    2153         1738 :   mem = gen_rtx_MEM (ptr_mode, base);
    2154         1738 :   mem = adjust_address (mem, VOIDmode, base_align_bias);
    2155         1738 :   emit_move_insn (mem, gen_int_mode (ASAN_STACK_FRAME_MAGIC, ptr_mode));
    2156         3476 :   mem = adjust_address (mem, VOIDmode, GET_MODE_SIZE (ptr_mode));
    2157         1738 :   emit_move_insn (mem, expand_normal (str_cst));
    2158         3476 :   mem = adjust_address (mem, VOIDmode, GET_MODE_SIZE (ptr_mode));
    2159         1738 :   ASM_GENERATE_INTERNAL_LABEL (buf, "LASANPC", current_function_funcdef_no);
    2160         1738 :   id = get_identifier (buf);
    2161         1738 :   decl = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
    2162              :                     VAR_DECL, id, char_type_node);
    2163         1738 :   SET_DECL_ASSEMBLER_NAME (decl, id);
    2164         1738 :   TREE_ADDRESSABLE (decl) = 1;
    2165         1738 :   TREE_READONLY (decl) = 1;
    2166         1738 :   DECL_ARTIFICIAL (decl) = 1;
    2167         1738 :   DECL_IGNORED_P (decl) = 1;
    2168         1738 :   TREE_STATIC (decl) = 1;
    2169         1738 :   TREE_PUBLIC (decl) = 0;
    2170         1738 :   TREE_USED (decl) = 1;
    2171         1738 :   DECL_INITIAL (decl) = decl;
    2172         1738 :   TREE_ASM_WRITTEN (decl) = 1;
    2173         1738 :   TREE_ASM_WRITTEN (id) = 1;
    2174         1738 :   DECL_ALIGN_RAW (decl) = DECL_ALIGN_RAW (current_function_decl);
    2175         1738 :   emit_move_insn (mem, expand_normal (build_fold_addr_expr (decl)));
    2176         1738 :   shadow_base = expand_binop (Pmode, lshr_optab, base,
    2177         1738 :                               gen_int_shift_amount (Pmode, ASAN_SHADOW_SHIFT),
    2178              :                               NULL_RTX, 1, OPTAB_DIRECT);
    2179         1738 :   if (asan_dynamic_shadow_offset_p ())
    2180              :     {
    2181            0 :       ret = expand_normal (get_asan_shadow_memory_dynamic_address_decl ());
    2182            0 :       shadow_base
    2183            0 :         = expand_simple_binop (Pmode, PLUS, shadow_base, ret, NULL_RTX,
    2184              :                                /* unsignedp = */ 1, OPTAB_WIDEN);
    2185            0 :       shadow_base = plus_constant (Pmode, shadow_base,
    2186            0 :                                    (base_align_bias >> ASAN_SHADOW_SHIFT));
    2187              :     }
    2188              :   else
    2189              :     {
    2190         1738 :       shadow_base = plus_constant (Pmode, shadow_base,
    2191         1738 :                                    asan_shadow_offset ()
    2192         1738 :                                      + (base_align_bias >> ASAN_SHADOW_SHIFT));
    2193              :     }
    2194         1738 :   gcc_assert (asan_shadow_set != -1
    2195              :               && (ASAN_RED_ZONE_SIZE >> ASAN_SHADOW_SHIFT) == 4);
    2196         1738 :   shadow_mem = gen_rtx_MEM (SImode, shadow_base);
    2197         1738 :   set_mem_alias_set (shadow_mem, asan_shadow_set);
    2198         1738 :   if (STRICT_ALIGNMENT)
    2199              :     set_mem_align (shadow_mem, (GET_MODE_ALIGNMENT (SImode)));
    2200         1738 :   prev_offset = base_offset;
    2201              : 
    2202         1738 :   asan_redzone_buffer rz_buffer (shadow_mem, prev_offset);
    2203         8601 :   for (l = length; l; l -= 2)
    2204              :     {
    2205         5125 :       if (l == 2)
    2206         1738 :         cur_shadow_byte = ASAN_STACK_MAGIC_RIGHT;
    2207         5125 :       offset = offsets[l - 1];
    2208              : 
    2209         5125 :       bool extra_byte = (offset - base_offset) & (ASAN_SHADOW_GRANULARITY - 1);
    2210              :       /* If a red-zone is not aligned to ASAN_SHADOW_GRANULARITY then
    2211              :          the previous stack variable has size % ASAN_SHADOW_GRANULARITY != 0.
    2212              :          In that case we have to emit one extra byte that will describe
    2213              :          how many bytes (our of ASAN_SHADOW_GRANULARITY) can be accessed.  */
    2214         5125 :       if (extra_byte)
    2215              :         {
    2216         1512 :           HOST_WIDE_INT aoff
    2217         1512 :             = base_offset + ((offset - base_offset)
    2218         1512 :                              & ~(ASAN_SHADOW_GRANULARITY - HOST_WIDE_INT_1));
    2219         1512 :           rz_buffer.emit_redzone_byte (aoff, offset - aoff);
    2220         1512 :           offset = aoff + ASAN_SHADOW_GRANULARITY;
    2221              :         }
    2222              : 
    2223              :       /* Calculate size of red zone payload.  */
    2224        26820 :       while (offset < offsets[l - 2])
    2225              :         {
    2226        21695 :           rz_buffer.emit_redzone_byte (offset, cur_shadow_byte);
    2227        21695 :           offset += ASAN_SHADOW_GRANULARITY;
    2228              :         }
    2229              : 
    2230         5125 :       cur_shadow_byte = ASAN_STACK_MAGIC_MIDDLE;
    2231              :     }
    2232              : 
    2233              :   /* As the automatic variables are aligned to
    2234              :      ASAN_RED_ZONE_SIZE / ASAN_SHADOW_GRANULARITY, the buffer should be
    2235              :      flushed here.  */
    2236         1738 :   gcc_assert (rz_buffer.m_shadow_bytes.is_empty ());
    2237              : 
    2238         1738 :   do_pending_stack_adjust ();
    2239              : 
    2240              :   /* Construct epilogue sequence.  */
    2241         1738 :   start_sequence ();
    2242              : 
    2243         1738 :   lab = NULL;
    2244         1738 :   if (use_after_return_class != -1)
    2245              :     {
    2246         1673 :       rtx_code_label *lab2 = gen_label_rtx ();
    2247         1673 :       char c = (char) ASAN_STACK_MAGIC_USE_AFTER_RET;
    2248         1673 :       emit_cmp_and_jump_insns (orig_base, base, EQ, NULL_RTX,
    2249              :                                VOIDmode, 0, lab2,
    2250              :                                profile_probability::very_likely ());
    2251         1673 :       shadow_mem = gen_rtx_MEM (BLKmode, shadow_base);
    2252         1673 :       set_mem_alias_set (shadow_mem, asan_shadow_set);
    2253         1673 :       mem = gen_rtx_MEM (ptr_mode, base);
    2254         1673 :       mem = adjust_address (mem, VOIDmode, base_align_bias);
    2255         1673 :       emit_move_insn (mem, gen_int_mode (ASAN_STACK_RETIRED_MAGIC, ptr_mode));
    2256         1673 :       unsigned HOST_WIDE_INT sz = asan_frame_size >> ASAN_SHADOW_SHIFT;
    2257         1673 :       bool asan_stack_free_emitted_p = false;
    2258         1673 :       if (use_after_return_class < 5
    2259         1673 :           && can_store_by_pieces (sz, builtin_memset_read_str, &c,
    2260              :                                   BITS_PER_UNIT, true))
    2261              :         /* Emit memset (ShadowBase, kAsanStackAfterReturnMagic, ShadowSize).  */
    2262         1544 :         store_by_pieces (shadow_mem, sz, builtin_memset_read_str, &c,
    2263              :                          BITS_PER_UNIT, true, RETURN_BEGIN);
    2264          129 :       else if (use_after_return_class >= 5
    2265          147 :                || !set_storage_via_setmem (shadow_mem,
    2266              :                                            GEN_INT (sz),
    2267           18 :                                            gen_int_mode (c, QImode),
    2268              :                                            BITS_PER_UNIT, BITS_PER_UNIT,
    2269              :                                            -1, sz, sz, sz))
    2270              :         {
    2271          111 :           snprintf (buf, sizeof buf, "__asan_stack_free_%d",
    2272              :                     use_after_return_class);
    2273          111 :           ret = init_one_libfunc (buf);
    2274          111 :           rtx addr = convert_memory_address (ptr_mode, base);
    2275          111 :           rtx orig_addr = convert_memory_address (ptr_mode, orig_base);
    2276          222 :           emit_library_call (ret, LCT_NORMAL, ptr_mode, addr, ptr_mode,
    2277              :                              GEN_INT (asan_frame_size + base_align_bias),
    2278          111 :                              TYPE_MODE (pointer_sized_int_node),
    2279              :                              orig_addr, ptr_mode);
    2280          111 :           asan_stack_free_emitted_p = true;
    2281              :         }
    2282         1673 :       if (!asan_stack_free_emitted_p)
    2283              :         {
    2284              :           /* Emit **SavedFlagPtr (FakeStack, class_id) = 0.  */
    2285         1562 :           unsigned HOST_WIDE_INT offset = (1 << (use_after_return_class + 6));
    2286         1562 :           offset -= GET_MODE_SIZE (ptr_mode);
    2287         1562 :           mem = gen_rtx_MEM (ptr_mode, base);
    2288         1562 :           mem = adjust_address (mem, ptr_mode, offset);
    2289         1562 :           rtx addr = gen_reg_rtx (ptr_mode);
    2290         1562 :           emit_move_insn (addr, mem);
    2291         1562 :           addr = convert_memory_address (Pmode, addr);
    2292         1562 :           mem = gen_rtx_MEM (QImode, addr);
    2293         1562 :           emit_move_insn (mem, const0_rtx);
    2294              :         }
    2295         1673 :       lab = gen_label_rtx ();
    2296         1673 :       emit_jump (lab);
    2297         1673 :       emit_label (lab2);
    2298              :     }
    2299              : 
    2300         1738 :   shadow_mem = gen_rtx_MEM (BLKmode, shadow_base);
    2301         1738 :   set_mem_alias_set (shadow_mem, asan_shadow_set);
    2302              : 
    2303         1738 :   if (STRICT_ALIGNMENT)
    2304              :     set_mem_align (shadow_mem, (GET_MODE_ALIGNMENT (SImode)));
    2305              : 
    2306         1738 :   prev_offset = base_offset;
    2307         1738 :   last_offset = base_offset;
    2308         1738 :   last_size = 0;
    2309         1738 :   last_size_aligned = 0;
    2310         8601 :   for (l = length; l; l -= 2)
    2311              :     {
    2312         5125 :       offset = base_offset + ((offsets[l - 1] - base_offset)
    2313         5125 :                               & ~(ASAN_RED_ZONE_SIZE - HOST_WIDE_INT_1));
    2314         5125 :       if (last_offset + last_size_aligned < offset)
    2315              :         {
    2316          739 :           shadow_mem = adjust_address (shadow_mem, VOIDmode,
    2317              :                                        (last_offset - prev_offset)
    2318              :                                        >> ASAN_SHADOW_SHIFT);
    2319          739 :           prev_offset = last_offset;
    2320          739 :           asan_clear_shadow (shadow_mem, last_size_aligned >> ASAN_SHADOW_SHIFT);
    2321          739 :           last_offset = offset;
    2322          739 :           last_size = 0;
    2323              :         }
    2324              :       else
    2325         4386 :         last_size = offset - last_offset;
    2326         5125 :       last_size += base_offset + ((offsets[l - 2] - base_offset)
    2327         5125 :                                   & ~(ASAN_MIN_RED_ZONE_SIZE - HOST_WIDE_INT_1))
    2328         5125 :                    - offset;
    2329              : 
    2330              :       /* Unpoison shadow memory that corresponds to a variable that is
    2331              :          is subject of use-after-return sanitization.  */
    2332         5125 :       if (l > 2)
    2333              :         {
    2334         3387 :           decl = decls[l / 2 - 2];
    2335         3387 :           if (asan_handled_variables != NULL
    2336         3387 :               && asan_handled_variables->contains (decl))
    2337              :             {
    2338          871 :               HOST_WIDE_INT size = offsets[l - 3] - offsets[l - 2];
    2339          871 :               if (dump_file && (dump_flags & TDF_DETAILS))
    2340              :                 {
    2341            0 :                   const char *n = (DECL_NAME (decl)
    2342            0 :                                    ? IDENTIFIER_POINTER (DECL_NAME (decl))
    2343            0 :                                    : "<unknown>");
    2344            0 :                   fprintf (dump_file, "Unpoisoning shadow stack for variable: "
    2345              :                            "%s (%" PRId64 " B)\n", n, size);
    2346              :                 }
    2347              : 
    2348          871 :                 last_size += size & ~(ASAN_MIN_RED_ZONE_SIZE - HOST_WIDE_INT_1);
    2349              :             }
    2350              :         }
    2351         5125 :       last_size_aligned
    2352         5125 :         = ((last_size + (ASAN_RED_ZONE_SIZE - HOST_WIDE_INT_1))
    2353              :            & ~(ASAN_RED_ZONE_SIZE - HOST_WIDE_INT_1));
    2354              :     }
    2355         1738 :   if (last_size_aligned)
    2356              :     {
    2357         1738 :       shadow_mem = adjust_address (shadow_mem, VOIDmode,
    2358              :                                    (last_offset - prev_offset)
    2359              :                                    >> ASAN_SHADOW_SHIFT);
    2360         1738 :       asan_clear_shadow (shadow_mem, last_size_aligned >> ASAN_SHADOW_SHIFT);
    2361              :     }
    2362              : 
    2363              :   /* Clean-up set with instrumented stack variables.  */
    2364         2125 :   delete asan_handled_variables;
    2365         1738 :   asan_handled_variables = NULL;
    2366         1833 :   delete asan_used_labels;
    2367         1738 :   asan_used_labels = NULL;
    2368              : 
    2369         1738 :   do_pending_stack_adjust ();
    2370         1738 :   if (lab)
    2371         1673 :     emit_label (lab);
    2372              : 
    2373         1738 :   insns = end_sequence ();
    2374         1738 :   return insns;
    2375         1738 : }
    2376              : 
    2377              : /* Emit __asan_allocas_unpoison (top, bot) call.  The BASE parameter corresponds
    2378              :    to BOT argument, for TOP virtual_stack_dynamic_rtx is used.  NEW_SEQUENCE
    2379              :    indicates whether we're emitting new instructions sequence or not.  */
    2380              : 
    2381              : rtx_insn *
    2382          178 : asan_emit_allocas_unpoison (rtx top, rtx bot, rtx_insn *before)
    2383              : {
    2384          178 :   if (before)
    2385           38 :     push_to_sequence (before);
    2386              :   else
    2387          140 :     start_sequence ();
    2388          178 :   rtx ret = init_one_libfunc ("__asan_allocas_unpoison");
    2389          178 :   top = convert_memory_address (ptr_mode, top);
    2390          178 :   bot = convert_memory_address (ptr_mode, bot);
    2391          178 :   emit_library_call (ret, LCT_NORMAL, ptr_mode,
    2392              :                      top, ptr_mode, bot, ptr_mode);
    2393              : 
    2394          178 :   do_pending_stack_adjust ();
    2395          178 :   return end_sequence ();
    2396              : }
    2397              : 
    2398              : /* Return true if DECL, a global var, might be overridden and needs
    2399              :    therefore a local alias.  */
    2400              : 
    2401              : static bool
    2402         4218 : asan_needs_local_alias (tree decl)
    2403              : {
    2404         4218 :   return DECL_WEAK (decl) || !targetm.binds_local_p (decl);
    2405              : }
    2406              : 
    2407              : /* Return true if DECL, a global var, is an artificial ODR indicator symbol
    2408              :    therefore doesn't need protection.  */
    2409              : 
    2410              : static bool
    2411         8789 : is_odr_indicator (tree decl)
    2412              : {
    2413         8789 :   return (DECL_ARTIFICIAL (decl)
    2414         8789 :           && lookup_attribute ("asan odr indicator", DECL_ATTRIBUTES (decl)));
    2415              : }
    2416              : 
    2417              : /* Return true if DECL is a VAR_DECL that should be protected
    2418              :    by Address Sanitizer, by appending a red zone with protected
    2419              :    shadow memory after it and aligning it to at least
    2420              :    ASAN_RED_ZONE_SIZE bytes.  */
    2421              : 
    2422              : bool
    2423        23121 : asan_protect_global (tree decl, bool ignore_decl_rtl_set_p)
    2424              : {
    2425        23121 :   if (!param_asan_globals)
    2426              :     return false;
    2427              : 
    2428        22958 :   rtx rtl, symbol;
    2429              : 
    2430        22958 :   if (TREE_CODE (decl) == STRING_CST)
    2431              :     {
    2432              :       /* Instrument all STRING_CSTs except those created
    2433              :          by asan_pp_string here.  */
    2434        13718 :       if (shadow_ptr_types[0] != NULL_TREE
    2435        13684 :           && TREE_CODE (TREE_TYPE (decl)) == ARRAY_TYPE
    2436        27402 :           && TREE_TYPE (TREE_TYPE (decl)) == TREE_TYPE (shadow_ptr_types[0]))
    2437              :         return false;
    2438         6886 :       return true;
    2439              :     }
    2440         9240 :   if (!VAR_P (decl)
    2441              :       /* TLS vars aren't statically protectable.  */
    2442         9240 :       || DECL_THREAD_LOCAL_P (decl)
    2443              :       /* Externs will be protected elsewhere.  */
    2444         9234 :       || DECL_EXTERNAL (decl)
    2445              :       /* PR sanitizer/81697: For architectures that use section anchors first
    2446              :          call to asan_protect_global may occur before DECL_RTL (decl) is set.
    2447              :          We should ignore DECL_RTL_SET_P then, because otherwise the first call
    2448              :          to asan_protect_global will return FALSE and the following calls on the
    2449              :          same decl after setting DECL_RTL (decl) will return TRUE and we'll end
    2450              :          up with inconsistency at runtime.  */
    2451         9234 :       || (!DECL_RTL_SET_P (decl) && !ignore_decl_rtl_set_p)
    2452              :       /* Comdat vars pose an ABI problem, we can't know if
    2453              :          the var that is selected by the linker will have
    2454              :          padding or not.  */
    2455         9222 :       || DECL_ONE_ONLY (decl)
    2456              :       /* Similarly for common vars.  People can use -fno-common.
    2457              :          Note: Linux kernel is built with -fno-common, so we do instrument
    2458              :          globals there even if it is C.  */
    2459         8939 :       || (DECL_COMMON (decl) && TREE_PUBLIC (decl))
    2460              :       /* Don't protect if using user section, often vars placed
    2461              :          into user section from multiple TUs are then assumed
    2462              :          to be an array of such vars, putting padding in there
    2463              :          breaks this assumption.  */
    2464         8939 :       || (DECL_SECTION_NAME (decl) != NULL
    2465          270 :           && !symtab_node::get (decl)->implicit_section
    2466          270 :           && !section_sanitized_p (DECL_SECTION_NAME (decl)))
    2467              :       /* Don't protect variables in non-generic address-space.  */
    2468         8849 :       || !ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (decl)))
    2469         8847 :       || DECL_SIZE (decl) == 0
    2470         8847 :       || ASAN_RED_ZONE_SIZE * BITS_PER_UNIT > MAX_OFILE_ALIGNMENT
    2471         8847 :       || TREE_CODE (DECL_SIZE_UNIT (decl)) != INTEGER_CST
    2472         8847 :       || !valid_constant_size_p (DECL_SIZE_UNIT (decl))
    2473         8847 :       || DECL_ALIGN_UNIT (decl) > 2 * ASAN_RED_ZONE_SIZE
    2474         8789 :       || TREE_TYPE (decl) == ubsan_get_source_location_type ()
    2475        18029 :       || is_odr_indicator (decl))
    2476              :     return false;
    2477              : 
    2478         8789 :   if (!ignore_decl_rtl_set_p || DECL_RTL_SET_P (decl))
    2479              :     {
    2480              : 
    2481         8789 :       rtl = DECL_RTL (decl);
    2482         8789 :       if (!MEM_P (rtl) || GET_CODE (XEXP (rtl, 0)) != SYMBOL_REF)
    2483              :         return false;
    2484         8789 :       symbol = XEXP (rtl, 0);
    2485              : 
    2486         8789 :       if (CONSTANT_POOL_ADDRESS_P (symbol)
    2487         8789 :           || TREE_CONSTANT_POOL_ADDRESS_P (symbol))
    2488              :         return false;
    2489              :     }
    2490              : 
    2491         8789 :   if (lookup_attribute ("weakref", DECL_ATTRIBUTES (decl)))
    2492            0 :     return false;
    2493              : 
    2494              :   if (!TARGET_SUPPORTS_ALIASES && asan_needs_local_alias (decl))
    2495              :     return false;
    2496              : 
    2497              :   return true;
    2498              : }
    2499              : 
    2500              : /* Construct a function tree for __asan_report_{load,store}{1,2,4,8,16,_n}.
    2501              :    IS_STORE is either 1 (for a store) or 0 (for a load).  */
    2502              : 
    2503              : static tree
    2504        21091 : report_error_func (bool is_store, bool recover_p, HOST_WIDE_INT size_in_bytes,
    2505              :                    int *nargs)
    2506              : {
    2507        21091 :   gcc_assert (!hwassist_sanitize_p ());
    2508              : 
    2509        21091 :   static enum built_in_function report[2][2][6]
    2510              :     = { { { BUILT_IN_ASAN_REPORT_LOAD1, BUILT_IN_ASAN_REPORT_LOAD2,
    2511              :             BUILT_IN_ASAN_REPORT_LOAD4, BUILT_IN_ASAN_REPORT_LOAD8,
    2512              :             BUILT_IN_ASAN_REPORT_LOAD16, BUILT_IN_ASAN_REPORT_LOAD_N },
    2513              :           { BUILT_IN_ASAN_REPORT_STORE1, BUILT_IN_ASAN_REPORT_STORE2,
    2514              :             BUILT_IN_ASAN_REPORT_STORE4, BUILT_IN_ASAN_REPORT_STORE8,
    2515              :             BUILT_IN_ASAN_REPORT_STORE16, BUILT_IN_ASAN_REPORT_STORE_N } },
    2516              :         { { BUILT_IN_ASAN_REPORT_LOAD1_NOABORT,
    2517              :             BUILT_IN_ASAN_REPORT_LOAD2_NOABORT,
    2518              :             BUILT_IN_ASAN_REPORT_LOAD4_NOABORT,
    2519              :             BUILT_IN_ASAN_REPORT_LOAD8_NOABORT,
    2520              :             BUILT_IN_ASAN_REPORT_LOAD16_NOABORT,
    2521              :             BUILT_IN_ASAN_REPORT_LOAD_N_NOABORT },
    2522              :           { BUILT_IN_ASAN_REPORT_STORE1_NOABORT,
    2523              :             BUILT_IN_ASAN_REPORT_STORE2_NOABORT,
    2524              :             BUILT_IN_ASAN_REPORT_STORE4_NOABORT,
    2525              :             BUILT_IN_ASAN_REPORT_STORE8_NOABORT,
    2526              :             BUILT_IN_ASAN_REPORT_STORE16_NOABORT,
    2527              :             BUILT_IN_ASAN_REPORT_STORE_N_NOABORT } } };
    2528        21091 :   if (size_in_bytes == -1)
    2529              :     {
    2530         1078 :       *nargs = 2;
    2531         1078 :       return builtin_decl_implicit (report[recover_p][is_store][5]);
    2532              :     }
    2533        20013 :   int size_log2 = exact_log2 (size_in_bytes);
    2534        20013 :   if (size_log2 == -1 || size_log2 >= 5)
    2535              :     {
    2536           28 :       *nargs = 2;
    2537           28 :       return builtin_decl_implicit (report[recover_p][is_store][5]);
    2538              :     }
    2539        19985 :   *nargs = 1;
    2540        19985 :   return builtin_decl_implicit (report[recover_p][is_store][size_log2]);
    2541              : }
    2542              : 
    2543              : /* Construct a function tree for __asan_{load,store}{1,2,4,8,16,_n}.
    2544              :    IS_STORE is either 1 (for a store) or 0 (for a load).  */
    2545              : 
    2546              : static tree
    2547          103 : check_func (bool is_store, bool recover_p, HOST_WIDE_INT size_in_bytes,
    2548              :             int *nargs)
    2549              : {
    2550          103 :   static enum built_in_function check[2][2][6]
    2551              :     = { { { BUILT_IN_ASAN_LOAD1, BUILT_IN_ASAN_LOAD2,
    2552              :             BUILT_IN_ASAN_LOAD4, BUILT_IN_ASAN_LOAD8,
    2553              :             BUILT_IN_ASAN_LOAD16, BUILT_IN_ASAN_LOADN },
    2554              :           { BUILT_IN_ASAN_STORE1, BUILT_IN_ASAN_STORE2,
    2555              :             BUILT_IN_ASAN_STORE4, BUILT_IN_ASAN_STORE8,
    2556              :             BUILT_IN_ASAN_STORE16, BUILT_IN_ASAN_STOREN } },
    2557              :         { { BUILT_IN_ASAN_LOAD1_NOABORT,
    2558              :             BUILT_IN_ASAN_LOAD2_NOABORT,
    2559              :             BUILT_IN_ASAN_LOAD4_NOABORT,
    2560              :             BUILT_IN_ASAN_LOAD8_NOABORT,
    2561              :             BUILT_IN_ASAN_LOAD16_NOABORT,
    2562              :             BUILT_IN_ASAN_LOADN_NOABORT },
    2563              :           { BUILT_IN_ASAN_STORE1_NOABORT,
    2564              :             BUILT_IN_ASAN_STORE2_NOABORT,
    2565              :             BUILT_IN_ASAN_STORE4_NOABORT,
    2566              :             BUILT_IN_ASAN_STORE8_NOABORT,
    2567              :             BUILT_IN_ASAN_STORE16_NOABORT,
    2568              :             BUILT_IN_ASAN_STOREN_NOABORT } } };
    2569          103 :   if (size_in_bytes == -1)
    2570              :     {
    2571           28 :       *nargs = 2;
    2572           28 :       return builtin_decl_implicit (check[recover_p][is_store][5]);
    2573              :     }
    2574           75 :   *nargs = 1;
    2575           75 :   int size_log2 = exact_log2 (size_in_bytes);
    2576           75 :   return builtin_decl_implicit (check[recover_p][is_store][size_log2]);
    2577              : }
    2578              : 
    2579              : /* Split the current basic block and create a condition statement
    2580              :    insertion point right before or after the statement pointed to by
    2581              :    ITER.  Return an iterator to the point at which the caller might
    2582              :    safely insert the condition statement.
    2583              : 
    2584              :    THEN_BLOCK must be set to the address of an uninitialized instance
    2585              :    of basic_block.  The function will then set *THEN_BLOCK to the
    2586              :    'then block' of the condition statement to be inserted by the
    2587              :    caller.
    2588              : 
    2589              :    If CREATE_THEN_FALLTHRU_EDGE is false, no edge will be created from
    2590              :    *THEN_BLOCK to *FALLTHROUGH_BLOCK.
    2591              : 
    2592              :    Similarly, the function will set *FALLTRHOUGH_BLOCK to the 'else
    2593              :    block' of the condition statement to be inserted by the caller.
    2594              : 
    2595              :    Note that *FALLTHROUGH_BLOCK is a new block that contains the
    2596              :    statements starting from *ITER, and *THEN_BLOCK is a new empty
    2597              :    block.
    2598              : 
    2599              :    *ITER is adjusted to point to always point to the first statement
    2600              :     of the basic block * FALLTHROUGH_BLOCK.  That statement is the
    2601              :     same as what ITER was pointing to prior to calling this function,
    2602              :     if BEFORE_P is true; otherwise, it is its following statement.  */
    2603              : 
    2604              : gimple_stmt_iterator
    2605        25287 : create_cond_insert_point (gimple_stmt_iterator *iter,
    2606              :                           bool before_p,
    2607              :                           bool then_more_likely_p,
    2608              :                           bool create_then_fallthru_edge,
    2609              :                           basic_block *then_block,
    2610              :                           basic_block *fallthrough_block)
    2611              : {
    2612        25287 :   gimple_stmt_iterator gsi = *iter;
    2613              : 
    2614        25287 :   if (!gsi_end_p (gsi) && before_p)
    2615         1275 :     gsi_prev (&gsi);
    2616              : 
    2617        25287 :   basic_block cur_bb = gsi_bb (*iter);
    2618              : 
    2619        25287 :   edge e = split_block (cur_bb, gsi_stmt (gsi));
    2620              : 
    2621              :   /* Get a hold on the 'condition block', the 'then block' and the
    2622              :      'else block'.  */
    2623        25287 :   basic_block cond_bb = e->src;
    2624        25287 :   basic_block fallthru_bb = e->dest;
    2625        25287 :   basic_block then_bb = create_empty_bb (cond_bb);
    2626        25287 :   if (current_loops)
    2627              :     {
    2628        25287 :       add_bb_to_loop (then_bb, cond_bb->loop_father);
    2629        25287 :       loops_state_set (LOOPS_NEED_FIXUP);
    2630              :     }
    2631              : 
    2632              :   /* Set up the newly created 'then block'.  */
    2633        25287 :   e = make_edge (cond_bb, then_bb, EDGE_TRUE_VALUE);
    2634        25287 :   profile_probability fallthrough_probability
    2635              :     = then_more_likely_p
    2636        25287 :     ? profile_probability::very_unlikely ()
    2637        25287 :     : profile_probability::very_likely ();
    2638        25287 :   e->probability = fallthrough_probability.invert ();
    2639        25287 :   then_bb->count = e->count ();
    2640        25287 :   if (create_then_fallthru_edge)
    2641         4377 :     make_single_succ_edge (then_bb, fallthru_bb, EDGE_FALLTHRU);
    2642              : 
    2643              :   /* Set up the fallthrough basic block.  */
    2644        25287 :   e = find_edge (cond_bb, fallthru_bb);
    2645        25287 :   e->flags = EDGE_FALSE_VALUE;
    2646        25287 :   e->probability = fallthrough_probability;
    2647              : 
    2648              :   /* Update dominance info for the newly created then_bb; note that
    2649              :      fallthru_bb's dominance info has already been updated by
    2650              :      split_bock.  */
    2651        25287 :   if (dom_info_available_p (CDI_DOMINATORS))
    2652        23840 :     set_immediate_dominator (CDI_DOMINATORS, then_bb, cond_bb);
    2653              : 
    2654        25287 :   *then_block = then_bb;
    2655        25287 :   *fallthrough_block = fallthru_bb;
    2656        25287 :   *iter = gsi_start_bb (fallthru_bb);
    2657              : 
    2658        25287 :   return gsi_last_bb (cond_bb);
    2659              : }
    2660              : 
    2661              : /* Insert an if condition followed by a 'then block' right before the
    2662              :    statement pointed to by ITER.  The fallthrough block -- which is the
    2663              :    else block of the condition as well as the destination of the
    2664              :    outcoming edge of the 'then block' -- starts with the statement
    2665              :    pointed to by ITER.
    2666              : 
    2667              :    COND is the condition of the if.
    2668              : 
    2669              :    If THEN_MORE_LIKELY_P is true, the probability of the edge to the
    2670              :    'then block' is higher than the probability of the edge to the
    2671              :    fallthrough block.
    2672              : 
    2673              :    Upon completion of the function, *THEN_BB is set to the newly
    2674              :    inserted 'then block' and similarly, *FALLTHROUGH_BB is set to the
    2675              :    fallthrough block.
    2676              : 
    2677              :    *ITER is adjusted to still point to the same statement it was
    2678              :    pointing to initially.  */
    2679              : 
    2680              : static void
    2681            0 : insert_if_then_before_iter (gcond *cond,
    2682              :                             gimple_stmt_iterator *iter,
    2683              :                             bool then_more_likely_p,
    2684              :                             basic_block *then_bb,
    2685              :                             basic_block *fallthrough_bb)
    2686              : {
    2687            0 :   gimple_stmt_iterator cond_insert_point =
    2688            0 :     create_cond_insert_point (iter,
    2689              :                               /*before_p=*/true,
    2690              :                               then_more_likely_p,
    2691              :                               /*create_then_fallthru_edge=*/true,
    2692              :                               then_bb,
    2693              :                               fallthrough_bb);
    2694            0 :   gsi_insert_after (&cond_insert_point, cond, GSI_NEW_STMT);
    2695            0 : }
    2696              : 
    2697              : /* Build (base_addr >> ASAN_SHADOW_SHIFT) + asan_shadow_offset ().
    2698              :    If RETURN_ADDRESS is set to true, return memory location instead
    2699              :    of a value in the shadow memory.  */
    2700              : 
    2701              : static tree
    2702        24206 : build_shadow_mem_access (gimple_stmt_iterator *gsi, location_t location,
    2703              :                          tree base_addr, tree shadow_ptr_type,
    2704              :                          bool return_address = false)
    2705              : {
    2706        24206 :   tree t, uintptr_type = TREE_TYPE (base_addr);
    2707        24206 :   tree shadow_type = TREE_TYPE (shadow_ptr_type);
    2708        24206 :   gimple *g;
    2709              : 
    2710        24206 :   t = build_int_cst (uintptr_type, ASAN_SHADOW_SHIFT);
    2711        24206 :   g = gimple_build_assign (make_ssa_name (uintptr_type), RSHIFT_EXPR,
    2712              :                            base_addr, t);
    2713        24206 :   gimple_set_location (g, location);
    2714        24206 :   gsi_insert_after (gsi, g, GSI_NEW_STMT);
    2715              : 
    2716        24206 :   if (asan_dynamic_shadow_offset_p ())
    2717            0 :     t = asan_local_shadow_memory_dynamic_address;
    2718              :   else
    2719        24206 :     t = build_int_cst (uintptr_type, asan_shadow_offset ());
    2720        24206 :   g = gimple_build_assign (make_ssa_name (uintptr_type), PLUS_EXPR,
    2721              :                            gimple_assign_lhs (g), t);
    2722        24206 :   gimple_set_location (g, location);
    2723        24206 :   gsi_insert_after (gsi, g, GSI_NEW_STMT);
    2724              : 
    2725        24206 :   g = gimple_build_assign (make_ssa_name (shadow_ptr_type), NOP_EXPR,
    2726              :                            gimple_assign_lhs (g));
    2727        24206 :   gimple_set_location (g, location);
    2728        24206 :   gsi_insert_after (gsi, g, GSI_NEW_STMT);
    2729              : 
    2730        24206 :   if (!return_address)
    2731              :     {
    2732        22110 :       t = build2 (MEM_REF, shadow_type, gimple_assign_lhs (g),
    2733              :                   build_int_cst (shadow_ptr_type, 0));
    2734        22110 :       g = gimple_build_assign (make_ssa_name (shadow_type), MEM_REF, t);
    2735        22110 :       gimple_set_location (g, location);
    2736        22110 :       gsi_insert_after (gsi, g, GSI_NEW_STMT);
    2737              :     }
    2738              : 
    2739        24206 :   return gimple_assign_lhs (g);
    2740              : }
    2741              : 
    2742              : /* BASE can already be an SSA_NAME; in that case, do not create a
    2743              :    new SSA_NAME for it.  */
    2744              : 
    2745              : static tree
    2746        20503 : maybe_create_ssa_name (location_t loc, tree base, gimple_stmt_iterator *iter,
    2747              :                        bool before_p)
    2748              : {
    2749        20503 :   STRIP_USELESS_TYPE_CONVERSION (base);
    2750        20503 :   if (TREE_CODE (base) == SSA_NAME)
    2751              :     return base;
    2752        18347 :   gimple *g = gimple_build_assign (make_ssa_name (TREE_TYPE (base)), base);
    2753        18347 :   gimple_set_location (g, loc);
    2754        18347 :   if (before_p)
    2755        18347 :     gsi_safe_insert_before (iter, g);
    2756              :   else
    2757            0 :     gsi_insert_after (iter, g, GSI_NEW_STMT);
    2758        18347 :   return gimple_assign_lhs (g);
    2759              : }
    2760              : 
    2761              : /* LEN can already have necessary size and precision;
    2762              :    in that case, do not create a new variable.  */
    2763              : 
    2764              : tree
    2765            0 : maybe_cast_to_ptrmode (location_t loc, tree len, gimple_stmt_iterator *iter,
    2766              :                        bool before_p)
    2767              : {
    2768            0 :   if (ptrofftype_p (len))
    2769              :     return len;
    2770            0 :   gimple *g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
    2771              :                                    NOP_EXPR, len);
    2772            0 :   gimple_set_location (g, loc);
    2773            0 :   if (before_p)
    2774            0 :     gsi_safe_insert_before (iter, g);
    2775              :   else
    2776            0 :     gsi_insert_after (iter, g, GSI_NEW_STMT);
    2777            0 :   return gimple_assign_lhs (g);
    2778              : }
    2779              : 
    2780              : /* Instrument the memory access instruction BASE.  Insert new
    2781              :    statements before or after ITER.
    2782              : 
    2783              :    Note that the memory access represented by BASE can be either an
    2784              :    SSA_NAME, or a non-SSA expression.  LOCATION is the source code
    2785              :    location.  IS_STORE is TRUE for a store, FALSE for a load.
    2786              :    BEFORE_P is TRUE for inserting the instrumentation code before
    2787              :    ITER, FALSE for inserting it after ITER.  IS_SCALAR_ACCESS is TRUE
    2788              :    for a scalar memory access and FALSE for memory region access.
    2789              :    NON_ZERO_P is TRUE if memory region is guaranteed to have non-zero
    2790              :    length.  ALIGN tells alignment of accessed memory object.
    2791              : 
    2792              :    START_INSTRUMENTED and END_INSTRUMENTED are TRUE if start/end of
    2793              :    memory region have already been instrumented.
    2794              : 
    2795              :    If BEFORE_P is TRUE, *ITER is arranged to still point to the
    2796              :    statement it was pointing to prior to calling this function,
    2797              :    otherwise, it points to the statement logically following it.  */
    2798              : 
    2799              : static void
    2800        20503 : build_check_stmt (location_t loc, tree base, tree len,
    2801              :                   HOST_WIDE_INT size_in_bytes, gimple_stmt_iterator *iter,
    2802              :                   bool is_non_zero_len, bool before_p, bool is_store,
    2803              :                   bool is_scalar_access, unsigned int align = 0)
    2804              : {
    2805        20503 :   gimple *g;
    2806              : 
    2807        20503 :   gcc_assert (!(size_in_bytes > 0 && !is_non_zero_len));
    2808        20503 :   gcc_assert (size_in_bytes == -1 || size_in_bytes >= 1);
    2809              : 
    2810        20503 :   base = unshare_expr (base);
    2811        20503 :   base = maybe_create_ssa_name (loc, base, iter, before_p);
    2812              : 
    2813        20503 :   if (len)
    2814              :     {
    2815            0 :       len = unshare_expr (len);
    2816            0 :       len = maybe_cast_to_ptrmode (loc, len, iter, before_p);
    2817              :     }
    2818              :   else
    2819              :     {
    2820        20503 :       gcc_assert (size_in_bytes != -1);
    2821        20503 :       len = build_int_cst (pointer_sized_int_node, size_in_bytes);
    2822              :     }
    2823              : 
    2824        20503 :   if (size_in_bytes > 1)
    2825              :     {
    2826        17563 :       if ((size_in_bytes & (size_in_bytes - 1)) != 0
    2827        17214 :           || size_in_bytes > 16)
    2828              :         is_scalar_access = false;
    2829        16823 :       else if (align && align < size_in_bytes * BITS_PER_UNIT)
    2830              :         {
    2831              :           /* On non-strict alignment targets, if
    2832              :              16-byte access is just 8-byte aligned,
    2833              :              this will result in misaligned shadow
    2834              :              memory 2 byte load, but otherwise can
    2835              :              be handled using one read.  */
    2836          286 :           if (size_in_bytes != 16
    2837              :               || STRICT_ALIGNMENT
    2838          152 :               || align < 8 * BITS_PER_UNIT)
    2839        20503 :             is_scalar_access = false;
    2840              :         }
    2841              :     }
    2842              : 
    2843        20503 :   HOST_WIDE_INT flags = 0;
    2844        20503 :   if (is_store)
    2845        10302 :     flags |= ASAN_CHECK_STORE;
    2846        20503 :   if (is_non_zero_len)
    2847        20503 :     flags |= ASAN_CHECK_NON_ZERO_LEN;
    2848        20503 :   if (is_scalar_access)
    2849        19620 :     flags |= ASAN_CHECK_SCALAR_ACCESS;
    2850              : 
    2851        20503 :   enum internal_fn fn = hwassist_sanitize_p ()
    2852        20503 :     ? IFN_HWASAN_CHECK
    2853        20131 :     : IFN_ASAN_CHECK;
    2854              : 
    2855        20503 :   g = gimple_build_call_internal (fn, 4,
    2856        20503 :                                   build_int_cst (integer_type_node, flags),
    2857              :                                   base, len,
    2858              :                                   build_int_cst (integer_type_node,
    2859        20503 :                                                  align / BITS_PER_UNIT));
    2860        20503 :   gimple_set_location (g, loc);
    2861        20503 :   if (before_p)
    2862        20503 :     gsi_safe_insert_before (iter, g);
    2863              :   else
    2864              :     {
    2865            0 :       gsi_insert_after (iter, g, GSI_NEW_STMT);
    2866            0 :       gsi_next (iter);
    2867              :     }
    2868        20503 : }
    2869              : 
    2870              : /* If T represents a memory access, add instrumentation code before ITER.
    2871              :    LOCATION is source code location.
    2872              :    IS_STORE is either TRUE (for a store) or FALSE (for a load).  */
    2873              : 
    2874              : static void
    2875        28813 : instrument_derefs (gimple_stmt_iterator *iter, tree t,
    2876              :                    location_t location, bool is_store)
    2877              : {
    2878        28813 :   if (is_store && !(asan_instrument_writes () || hwasan_instrument_writes ()))
    2879         8186 :     return;
    2880        28771 :   if (!is_store && !(asan_instrument_reads () || hwasan_instrument_reads ()))
    2881              :     return;
    2882              : 
    2883        28715 :   tree type, base;
    2884        28715 :   HOST_WIDE_INT size_in_bytes;
    2885        28715 :   if (location == UNKNOWN_LOCATION)
    2886          377 :     location = EXPR_LOCATION (t);
    2887              : 
    2888        28715 :   type = TREE_TYPE (t);
    2889        28715 :   switch (TREE_CODE (t))
    2890              :     {
    2891        28523 :     case ARRAY_REF:
    2892        28523 :     case COMPONENT_REF:
    2893        28523 :     case INDIRECT_REF:
    2894        28523 :     case MEM_REF:
    2895        28523 :     case VAR_DECL:
    2896        28523 :     case BIT_FIELD_REF:
    2897        28523 :       break;
    2898              :       /* FALLTHRU */
    2899              :     default:
    2900              :       return;
    2901              :     }
    2902              : 
    2903        28523 :   size_in_bytes = int_size_in_bytes (type);
    2904        28523 :   if (size_in_bytes <= 0)
    2905              :     return;
    2906              : 
    2907        28523 :   poly_int64 bitsize, bitpos;
    2908        28523 :   tree offset;
    2909        28523 :   machine_mode mode;
    2910        28523 :   int unsignedp, reversep, volatilep = 0;
    2911        28523 :   tree inner = get_inner_reference (t, &bitsize, &bitpos, &offset, &mode,
    2912              :                                     &unsignedp, &reversep, &volatilep);
    2913              : 
    2914        28523 :   if (TREE_CODE (t) == COMPONENT_REF
    2915        28523 :       && DECL_BIT_FIELD_REPRESENTATIVE (TREE_OPERAND (t, 1)) != NULL_TREE)
    2916              :     {
    2917           72 :       tree repr = DECL_BIT_FIELD_REPRESENTATIVE (TREE_OPERAND (t, 1));
    2918           72 :       instrument_derefs (iter, build3 (COMPONENT_REF, TREE_TYPE (repr),
    2919           72 :                                        TREE_OPERAND (t, 0), repr,
    2920           72 :                                        TREE_OPERAND (t, 2)),
    2921              :                          location, is_store);
    2922           72 :       return;
    2923              :     }
    2924              : 
    2925        28451 :   if (!multiple_p (bitpos, BITS_PER_UNIT)
    2926        28451 :       || maybe_ne (bitsize, size_in_bytes * BITS_PER_UNIT))
    2927              :     return;
    2928              : 
    2929        28451 :   if (VAR_P (inner) && DECL_HARD_REGISTER (inner))
    2930              :     return;
    2931              : 
    2932              :   /* Accesses to non-generic address-spaces should not be instrumented.  */
    2933        28445 :   if (!ADDR_SPACE_GENERIC_P (TYPE_ADDR_SPACE (TREE_TYPE (inner))))
    2934              :     return;
    2935              : 
    2936        28435 :   poly_int64 decl_size;
    2937        28435 :   if ((VAR_P (inner)
    2938        12216 :        || (TREE_CODE (inner) == RESULT_DECL
    2939          145 :            && !aggregate_value_p (inner, current_function_decl)))
    2940        16219 :       && offset == NULL_TREE
    2941        15600 :       && DECL_SIZE (inner)
    2942        15600 :       && poly_int_tree_p (DECL_SIZE (inner), &decl_size)
    2943        56870 :       && known_subrange_p (bitpos, bitsize, 0, decl_size))
    2944              :     {
    2945        15581 :       if (VAR_P (inner) && DECL_THREAD_LOCAL_P (inner))
    2946              :         return;
    2947              :       /* If we're not sanitizing globals and we can tell statically that this
    2948              :          access is inside a global variable, then there's no point adding
    2949              :          instrumentation to check the access.  N.b. hwasan currently never
    2950              :          sanitizes globals.  */
    2951        30990 :       if ((hwassist_sanitize_p () || !param_asan_globals)
    2952        15659 :           && is_global_var (inner))
    2953              :         return;
    2954        15395 :       if (!TREE_STATIC (inner))
    2955              :         {
    2956              :           /* Automatic vars in the current function will be always
    2957              :              accessible.  */
    2958        10469 :           if (decl_function_context (inner) == current_function_decl
    2959        10469 :               && (!asan_sanitize_use_after_scope ()
    2960         9677 :                   || !TREE_ADDRESSABLE (inner)))
    2961              :             return;
    2962              :         }
    2963              :       /* Always instrument external vars, they might be dynamically
    2964              :          initialized.  */
    2965         4926 :       else if (!DECL_EXTERNAL (inner))
    2966              :         {
    2967              :           /* For static vars if they are known not to be dynamically
    2968              :              initialized, they will be always accessible.  */
    2969         4917 :           varpool_node *vnode = varpool_node::get (inner);
    2970         4917 :           if (vnode && !vnode->dynamically_initialized)
    2971              :             return;
    2972              :         }
    2973              :     }
    2974              : 
    2975        20627 :   if (DECL_P (inner)
    2976         8582 :       && decl_function_context (inner) == current_function_decl
    2977        28515 :       && !TREE_ADDRESSABLE (inner))
    2978          112 :     mark_addressable (inner);
    2979              : 
    2980        20627 :   base = build_fold_addr_expr (t);
    2981        20627 :   if (!has_mem_ref_been_instrumented (base, size_in_bytes))
    2982              :     {
    2983        20503 :       unsigned int align = get_object_alignment (t);
    2984        20503 :       build_check_stmt (location, base, NULL_TREE, size_in_bytes, iter,
    2985              :                         /*is_non_zero_len*/size_in_bytes > 0, /*before_p=*/true,
    2986              :                         is_store, /*is_scalar_access*/true, align);
    2987        20503 :       update_mem_ref_hash_table (base, size_in_bytes);
    2988        20503 :       update_mem_ref_hash_table (t, size_in_bytes);
    2989              :     }
    2990              : 
    2991              : }
    2992              : 
    2993              : /*  Insert a memory reference into the hash table if access length
    2994              :     can be determined in compile time.  */
    2995              : 
    2996              : static void
    2997         1640 : maybe_update_mem_ref_hash_table (tree base, tree len)
    2998              : {
    2999         1706 :   if (!POINTER_TYPE_P (TREE_TYPE (base))
    3000         1706 :       || !INTEGRAL_TYPE_P (TREE_TYPE (len)))
    3001              :     return;
    3002              : 
    3003         1640 :   HOST_WIDE_INT size_in_bytes = tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
    3004              : 
    3005          386 :   if (size_in_bytes != -1)
    3006          386 :     update_mem_ref_hash_table (base, size_in_bytes);
    3007              : }
    3008              : 
    3009              : /* Instrument an access to a contiguous memory region that starts at
    3010              :    the address pointed to by BASE, over a length of LEN (expressed in
    3011              :    the sizeof (*BASE) bytes).  ITER points to the instruction before
    3012              :    which the instrumentation instructions must be inserted.  LOCATION
    3013              :    is the source location that the instrumentation instructions must
    3014              :    have.  If IS_STORE is true, then the memory access is a store;
    3015              :    otherwise, it's a load.  */
    3016              : 
    3017              : static void
    3018            0 : instrument_mem_region_access (tree base, tree len,
    3019              :                               gimple_stmt_iterator *iter,
    3020              :                               location_t location, bool is_store)
    3021              : {
    3022            0 :   if (!POINTER_TYPE_P (TREE_TYPE (base))
    3023            0 :       || !INTEGRAL_TYPE_P (TREE_TYPE (len))
    3024            0 :       || integer_zerop (len))
    3025              :     return;
    3026              : 
    3027            0 :   HOST_WIDE_INT size_in_bytes = tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
    3028              : 
    3029            0 :   if ((size_in_bytes == -1)
    3030            0 :       || !has_mem_ref_been_instrumented (base, size_in_bytes))
    3031              :     {
    3032            0 :       build_check_stmt (location, base, len, size_in_bytes, iter,
    3033              :                         /*is_non_zero_len*/size_in_bytes > 0, /*before_p*/true,
    3034              :                         is_store, /*is_scalar_access*/false, /*align*/0);
    3035              :     }
    3036              : 
    3037            0 :   maybe_update_mem_ref_hash_table (base, len);
    3038            0 :   *iter = gsi_for_stmt (gsi_stmt (*iter));
    3039              : }
    3040              : 
    3041              : /* Instrument the call to a built-in memory access function that is
    3042              :    pointed to by the iterator ITER.
    3043              : 
    3044              :    Upon completion, return TRUE iff *ITER has been advanced to the
    3045              :    statement following the one it was originally pointing to.  */
    3046              : 
    3047              : static bool
    3048         4672 : instrument_builtin_call (gimple_stmt_iterator *iter)
    3049              : {
    3050         4696 :   if (!(asan_memintrin () || hwasan_memintrin ()
    3051           24 :         || memtag_memintrin ()))
    3052              :     return false;
    3053              : 
    3054         4648 :   bool iter_advanced_p = false;
    3055         4648 :   gcall *call = as_a <gcall *> (gsi_stmt (*iter));
    3056              : 
    3057         4648 :   gcc_checking_assert (gimple_call_builtin_p (call, BUILT_IN_NORMAL));
    3058              : 
    3059         4648 :   location_t loc = gimple_location (call);
    3060              : 
    3061         4648 :   asan_mem_ref src0, src1, dest;
    3062         4648 :   asan_mem_ref_init (&src0, NULL, 1);
    3063         4648 :   asan_mem_ref_init (&src1, NULL, 1);
    3064         4648 :   asan_mem_ref_init (&dest, NULL, 1);
    3065              : 
    3066         4648 :   tree src0_len = NULL_TREE, src1_len = NULL_TREE, dest_len = NULL_TREE;
    3067         4648 :   bool src0_is_store = false, src1_is_store = false, dest_is_store = false,
    3068         4648 :     dest_is_deref = false, intercepted_p = true;
    3069              : 
    3070         4648 :   if (get_mem_refs_of_builtin_call (call,
    3071              :                                     &src0, &src0_len, &src0_is_store,
    3072              :                                     &src1, &src1_len, &src1_is_store,
    3073              :                                     &dest, &dest_len, &dest_is_store,
    3074              :                                     &dest_is_deref, &intercepted_p, iter))
    3075              :     {
    3076          960 :       if (dest_is_deref)
    3077              :         {
    3078           28 :           instrument_derefs (iter, dest.start, loc, dest_is_store);
    3079           28 :           gsi_next (iter);
    3080           28 :           iter_advanced_p = true;
    3081              :         }
    3082          932 :       else if (!intercepted_p
    3083            0 :                && (src0_len || src1_len || dest_len))
    3084              :         {
    3085            0 :           if (src0.start != NULL_TREE)
    3086            0 :             instrument_mem_region_access (src0.start, src0_len,
    3087              :                                           iter, loc, /*is_store=*/false);
    3088            0 :           if (src1.start != NULL_TREE)
    3089            0 :             instrument_mem_region_access (src1.start, src1_len,
    3090              :                                           iter, loc, /*is_store=*/false);
    3091            0 :           if (dest.start != NULL_TREE)
    3092            0 :             instrument_mem_region_access (dest.start, dest_len,
    3093              :                                           iter, loc, /*is_store=*/true);
    3094              : 
    3095            0 :           *iter = gsi_for_stmt (call);
    3096            0 :           gsi_next (iter);
    3097            0 :           iter_advanced_p = true;
    3098              :         }
    3099              :       else
    3100              :         {
    3101          932 :           if (src0.start != NULL_TREE)
    3102          744 :             maybe_update_mem_ref_hash_table (src0.start, src0_len);
    3103          932 :           if (src1.start != NULL_TREE)
    3104           50 :             maybe_update_mem_ref_hash_table (src1.start, src1_len);
    3105          932 :           if (dest.start != NULL_TREE)
    3106          846 :             maybe_update_mem_ref_hash_table (dest.start, dest_len);
    3107              :         }
    3108              :     }
    3109              :   return iter_advanced_p;
    3110              : }
    3111              : 
    3112              : /*  Instrument the assignment statement ITER if it is subject to
    3113              :     instrumentation.  Return TRUE iff instrumentation actually
    3114              :     happened.  In that case, the iterator ITER is advanced to the next
    3115              :     logical expression following the one initially pointed to by ITER,
    3116              :     and the relevant memory reference that which access has been
    3117              :     instrumented is added to the memory references hash table.  */
    3118              : 
    3119              : static bool
    3120        30564 : maybe_instrument_assignment (gimple_stmt_iterator *iter)
    3121              : {
    3122        30564 :   gimple *s = gsi_stmt (*iter);
    3123              : 
    3124        30564 :   gcc_assert (gimple_assign_single_p (s));
    3125              : 
    3126        30564 :   tree ref_expr = NULL_TREE;
    3127        30564 :   bool is_store, is_instrumented = false;
    3128              : 
    3129        30564 :   if (gimple_store_p (s))
    3130              :     {
    3131        13794 :       ref_expr = gimple_assign_lhs (s);
    3132        13794 :       is_store = true;
    3133        13794 :       instrument_derefs (iter, ref_expr,
    3134              :                          gimple_location (s),
    3135              :                          is_store);
    3136        13794 :       is_instrumented = true;
    3137              :     }
    3138              : 
    3139        30564 :   if (gimple_assign_load_p (s))
    3140              :     {
    3141        14758 :       ref_expr = gimple_assign_rhs1 (s);
    3142        14758 :       is_store = false;
    3143        14758 :       instrument_derefs (iter, ref_expr,
    3144              :                          gimple_location (s),
    3145              :                          is_store);
    3146        14758 :       is_instrumented = true;
    3147              :     }
    3148              : 
    3149        30564 :   if (is_instrumented)
    3150        27933 :     gsi_next (iter);
    3151              : 
    3152        30564 :   return is_instrumented;
    3153              : }
    3154              : 
    3155              : /* Instrument the function call pointed to by the iterator ITER, if it
    3156              :    is subject to instrumentation.  At the moment, the only function
    3157              :    calls that are instrumented are some built-in functions that access
    3158              :    memory.  Look at instrument_builtin_call to learn more.
    3159              : 
    3160              :    Upon completion return TRUE iff *ITER was advanced to the statement
    3161              :    following the one it was originally pointing to.  */
    3162              : 
    3163              : static bool
    3164        23199 : maybe_instrument_call (gimple_stmt_iterator *iter)
    3165              : {
    3166        23199 :   gimple *stmt = gsi_stmt (*iter);
    3167        23199 :   bool is_builtin = gimple_call_builtin_p (stmt, BUILT_IN_NORMAL);
    3168              : 
    3169        23199 :   if (is_builtin && instrument_builtin_call (iter))
    3170              :     return true;
    3171              : 
    3172        23171 :   if (gimple_call_noreturn_p (stmt))
    3173              :     {
    3174         1195 :       if (is_builtin)
    3175              :         {
    3176          320 :           tree callee = gimple_call_fndecl (stmt);
    3177          320 :           switch (DECL_FUNCTION_CODE (callee))
    3178              :             {
    3179              :             case BUILT_IN_UNREACHABLE:
    3180              :             case BUILT_IN_UNREACHABLE_TRAP:
    3181              :             case BUILT_IN_TRAP:
    3182              :             case BUILT_IN_ASAN_REPORT_LOAD1:
    3183              :             case BUILT_IN_ASAN_REPORT_LOAD2:
    3184              :             case BUILT_IN_ASAN_REPORT_LOAD4:
    3185              :             case BUILT_IN_ASAN_REPORT_LOAD8:
    3186              :             case BUILT_IN_ASAN_REPORT_LOAD16:
    3187              :             case BUILT_IN_ASAN_REPORT_LOAD_N:
    3188              :             case BUILT_IN_ASAN_REPORT_STORE1:
    3189              :             case BUILT_IN_ASAN_REPORT_STORE2:
    3190              :             case BUILT_IN_ASAN_REPORT_STORE4:
    3191              :             case BUILT_IN_ASAN_REPORT_STORE8:
    3192              :             case BUILT_IN_ASAN_REPORT_STORE16:
    3193              :             case BUILT_IN_ASAN_REPORT_STORE_N:
    3194              :               /* Don't instrument these.  */
    3195              :               return false;
    3196              :             default:
    3197              :               break;
    3198              :             }
    3199              :         }
    3200         1015 :       if (gimple_call_internal_p (stmt, IFN_ABNORMAL_DISPATCHER))
    3201              :         /* Don't instrument this.  */
    3202              :         return false;
    3203              :       /* If a function does not return, then we must handle clearing up the
    3204              :          shadow stack accordingly.  For ASAN we can simply set the entire stack
    3205              :          to "valid" for accesses by setting the shadow space to 0 and all
    3206              :          accesses will pass checks.  That means that some bad accesses may be
    3207              :          missed, but we will not report any false positives.
    3208              : 
    3209              :          This is not possible for HWASAN.  Since there is no "always valid" tag
    3210              :          we can not set any space to "always valid".  If we were to clear the
    3211              :          entire shadow stack then code resuming from `longjmp` or a caught
    3212              :          exception would trigger false positives when correctly accessing
    3213              :          variables on the stack.  Hence we need to handle things like
    3214              :          `longjmp`, thread exit, and exceptions in a different way.  These
    3215              :          problems must be handled externally to the compiler, e.g. in the
    3216              :          language runtime.  */
    3217          930 :       if (! hwassist_sanitize_p ())
    3218              :         {
    3219          896 :           tree decl = builtin_decl_implicit (BUILT_IN_ASAN_HANDLE_NO_RETURN);
    3220          896 :           gimple *g = gimple_build_call (decl, 0);
    3221          896 :           gimple_set_location (g, gimple_location (stmt));
    3222          896 :           gsi_safe_insert_before (iter, g);
    3223              :         }
    3224              :     }
    3225              : 
    3226        22906 :   bool instrumented = false;
    3227        22906 :   if (gimple_store_p (stmt)
    3228        22906 :       && (gimple_call_builtin_p (stmt)
    3229          288 :           || gimple_call_internal_p (stmt)
    3230          287 :           || !aggregate_value_p (TREE_TYPE (gimple_call_lhs (stmt)),
    3231          287 :                                  gimple_call_fntype (stmt))))
    3232              :     {
    3233           62 :       tree ref_expr = gimple_call_lhs (stmt);
    3234           62 :       instrument_derefs (iter, ref_expr,
    3235              :                          gimple_location (stmt),
    3236              :                          /*is_store=*/true);
    3237              : 
    3238           62 :       instrumented = true;
    3239              :     }
    3240              : 
    3241              :   /* Walk through gimple_call arguments and check them id needed.  */
    3242        22906 :   unsigned args_num = gimple_call_num_args (stmt);
    3243        94414 :   for (unsigned i = 0; i < args_num; ++i)
    3244              :     {
    3245        48602 :       tree arg = gimple_call_arg (stmt, i);
    3246              :       /* If ARG is not a non-aggregate register variable, compiler in general
    3247              :          creates temporary for it and pass it as argument to gimple call.
    3248              :          But in some cases, e.g. when we pass by value a small structure that
    3249              :          fits to register, compiler can avoid extra overhead by pulling out
    3250              :          these temporaries.  In this case, we should check the argument.  */
    3251        48602 :       if (!is_gimple_reg (arg) && !is_gimple_min_invariant (arg))
    3252              :         {
    3253           99 :           instrument_derefs (iter, arg,
    3254              :                              gimple_location (stmt),
    3255              :                              /*is_store=*/false);
    3256           99 :           instrumented = true;
    3257              :         }
    3258              :     }
    3259        22906 :   if (instrumented)
    3260          155 :     gsi_next (iter);
    3261              :   return instrumented;
    3262              : }
    3263              : 
    3264              : /* Walk each instruction of all basic block and instrument those that
    3265              :    represent memory references: loads, stores, or function calls.
    3266              :    In a given basic block, this function avoids instrumenting memory
    3267              :    references that have already been instrumented.  */
    3268              : 
    3269              : static void
    3270         6659 : transform_statements (void)
    3271              : {
    3272         6659 :   basic_block bb, last_bb = NULL;
    3273         6659 :   gimple_stmt_iterator i;
    3274         6659 :   int saved_last_basic_block = last_basic_block_for_fn (cfun);
    3275              : 
    3276        39512 :   FOR_EACH_BB_FN (bb, cfun)
    3277              :     {
    3278        32853 :       basic_block prev_bb = bb;
    3279              : 
    3280        32853 :       if (bb->index >= saved_last_basic_block) continue;
    3281              : 
    3282              :       /* Flush the mem ref hash table, if current bb doesn't have
    3283              :          exactly one predecessor, or if that predecessor (skipping
    3284              :          over asan created basic blocks) isn't the last processed
    3285              :          basic block.  Thus we effectively flush on extended basic
    3286              :          block boundaries.  */
    3287        32851 :       while (single_pred_p (prev_bb))
    3288              :         {
    3289        26541 :           prev_bb = single_pred (prev_bb);
    3290        26541 :           if (prev_bb->index < saved_last_basic_block)
    3291              :             break;
    3292              :         }
    3293        32845 :       if (prev_bb != last_bb)
    3294        22123 :         empty_mem_ref_hash_table ();
    3295        32845 :       last_bb = bb;
    3296              : 
    3297       241754 :       for (i = gsi_start_bb (bb); !gsi_end_p (i);)
    3298              :         {
    3299       176064 :           gimple *s = gsi_stmt (i);
    3300              : 
    3301       176064 :           if (has_stmt_been_instrumented_p (s))
    3302         1137 :             gsi_next (&i);
    3303       174927 :           else if (gimple_assign_single_p (s)
    3304        31528 :                    && !gimple_clobber_p (s)
    3305       205491 :                    && maybe_instrument_assignment (&i))
    3306              :             /*  Nothing to do as maybe_instrument_assignment advanced
    3307              :                 the iterator I.  */;
    3308       146994 :           else if (is_gimple_call (s) && maybe_instrument_call (&i))
    3309              :             /*  Nothing to do as maybe_instrument_call
    3310              :                 advanced the iterator I.  */;
    3311              :           else
    3312              :             {
    3313              :               /* No instrumentation happened.
    3314              : 
    3315              :                  If the current instruction is a function call that
    3316              :                  might free something, let's forget about the memory
    3317              :                  references that got instrumented.  Otherwise we might
    3318              :                  miss some instrumentation opportunities.  Do the same
    3319              :                  for a ASAN_MARK poisoning internal function.  */
    3320       146811 :               if (is_gimple_call (s)
    3321       146811 :                   && (!nonfreeing_call_p (s)
    3322         6669 :                       || asan_mark_p (s, ASAN_MARK_POISON)))
    3323        16347 :                 empty_mem_ref_hash_table ();
    3324              : 
    3325       146811 :               gsi_next (&i);
    3326              :             }
    3327              :         }
    3328              :     }
    3329         6659 :   free_mem_ref_resources ();
    3330         6659 : }
    3331              : 
    3332              : /* Build
    3333              :    __asan_before_dynamic_init (module_name)
    3334              :    or
    3335              :    __asan_after_dynamic_init ()
    3336              :    call.  */
    3337              : 
    3338              : tree
    3339           42 : asan_dynamic_init_call (bool after_p)
    3340              : {
    3341           42 :   if (shadow_ptr_types[0] == NULL_TREE)
    3342           21 :     asan_init_shadow_ptr_types ();
    3343              : 
    3344           63 :   tree fn = builtin_decl_implicit (after_p
    3345              :                                    ? BUILT_IN_ASAN_AFTER_DYNAMIC_INIT
    3346              :                                    : BUILT_IN_ASAN_BEFORE_DYNAMIC_INIT);
    3347           42 :   tree module_name_cst = NULL_TREE;
    3348           42 :   if (!after_p)
    3349              :     {
    3350           21 :       pretty_printer module_name_pp;
    3351           21 :       pp_string (&module_name_pp, main_input_filename);
    3352              : 
    3353           21 :       module_name_cst = asan_pp_string (&module_name_pp);
    3354           21 :       module_name_cst = fold_convert (const_ptr_type_node,
    3355              :                                       module_name_cst);
    3356           21 :     }
    3357              : 
    3358           42 :   return build_call_expr (fn, after_p ? 0 : 1, module_name_cst);
    3359              : }
    3360              : 
    3361              : /* Build
    3362              :    struct __asan_global
    3363              :    {
    3364              :      const void *__beg;
    3365              :      uptr __size;
    3366              :      uptr __size_with_redzone;
    3367              :      const void *__name;
    3368              :      const void *__module_name;
    3369              :      uptr __has_dynamic_init;
    3370              :      __asan_global_source_location *__location;
    3371              :      char *__odr_indicator;
    3372              :    } type.  */
    3373              : 
    3374              : static tree
    3375         1102 : asan_global_struct (void)
    3376              : {
    3377         1102 :   static const char *field_names[]
    3378              :     = { "__beg", "__size", "__size_with_redzone",
    3379              :         "__name", "__module_name", "__has_dynamic_init", "__location",
    3380              :         "__odr_indicator" };
    3381         1102 :   tree fields[ARRAY_SIZE (field_names)], ret;
    3382         1102 :   unsigned i;
    3383              : 
    3384         1102 :   ret = make_node (RECORD_TYPE);
    3385        11020 :   for (i = 0; i < ARRAY_SIZE (field_names); i++)
    3386              :     {
    3387         8816 :       fields[i]
    3388         8816 :         = build_decl (UNKNOWN_LOCATION, FIELD_DECL,
    3389              :                       get_identifier (field_names[i]),
    3390         8816 :                       (i == 0 || i == 3) ? const_ptr_type_node
    3391              :                       : pointer_sized_int_node);
    3392         8816 :       DECL_CONTEXT (fields[i]) = ret;
    3393         8816 :       if (i)
    3394         7714 :         DECL_CHAIN (fields[i - 1]) = fields[i];
    3395              :     }
    3396         1102 :   tree type_decl = build_decl (input_location, TYPE_DECL,
    3397              :                                get_identifier ("__asan_global"), ret);
    3398         1102 :   DECL_IGNORED_P (type_decl) = 1;
    3399         1102 :   DECL_ARTIFICIAL (type_decl) = 1;
    3400         1102 :   TYPE_FIELDS (ret) = fields[0];
    3401         1102 :   TYPE_NAME (ret) = type_decl;
    3402         1102 :   TYPE_STUB_DECL (ret) = type_decl;
    3403         1102 :   TYPE_ARTIFICIAL (ret) = 1;
    3404         1102 :   layout_type (ret);
    3405         1102 :   return ret;
    3406              : }
    3407              : 
    3408              : /* Create and return odr indicator symbol for DECL.
    3409              :    TYPE is __asan_global struct type as returned by asan_global_struct.  */
    3410              : 
    3411              : static tree
    3412         1264 : create_odr_indicator (tree decl, tree type)
    3413              : {
    3414         1264 :   char *name;
    3415         1264 :   tree uptr = TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (type)));
    3416         1264 :   tree decl_name
    3417         1264 :     = (HAS_DECL_ASSEMBLER_NAME_P (decl) ? DECL_ASSEMBLER_NAME (decl)
    3418            0 :                                         : DECL_NAME (decl));
    3419              :   /* DECL_NAME theoretically might be NULL.  Bail out with 0 in this case.  */
    3420         1264 :   if (decl_name == NULL_TREE)
    3421            0 :     return build_int_cst (uptr, 0);
    3422         1264 :   const char *dname = IDENTIFIER_POINTER (decl_name);
    3423         1264 :   if (HAS_DECL_ASSEMBLER_NAME_P (decl))
    3424         1264 :     dname = targetm.strip_name_encoding (dname);
    3425         1264 :   size_t len = strlen (dname) + sizeof ("__odr_asan_");
    3426         1264 :   name = XALLOCAVEC (char, len);
    3427         1264 :   snprintf (name, len, "__odr_asan_%s", dname);
    3428              : #ifndef NO_DOT_IN_LABEL
    3429         1264 :   name[sizeof ("__odr_asan") - 1] = '.';
    3430              : #elif !defined(NO_DOLLAR_IN_LABEL)
    3431              :   name[sizeof ("__odr_asan") - 1] = '$';
    3432              : #endif
    3433         1264 :   tree var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (name),
    3434              :                          char_type_node);
    3435         1264 :   TREE_ADDRESSABLE (var) = 1;
    3436         1264 :   TREE_READONLY (var) = 0;
    3437         1264 :   TREE_THIS_VOLATILE (var) = 1;
    3438         1264 :   DECL_ARTIFICIAL (var) = 1;
    3439         1264 :   DECL_IGNORED_P (var) = 1;
    3440         1264 :   TREE_STATIC (var) = 1;
    3441         1264 :   TREE_PUBLIC (var) = 1;
    3442         1264 :   DECL_VISIBILITY (var) = DECL_VISIBILITY (decl);
    3443         1264 :   DECL_VISIBILITY_SPECIFIED (var) = DECL_VISIBILITY_SPECIFIED (decl);
    3444              : 
    3445         1264 :   TREE_USED (var) = 1;
    3446         1264 :   tree ctor = build_constructor_va (TREE_TYPE (var), 1, NULL_TREE,
    3447              :                                     build_int_cst (unsigned_type_node, 0));
    3448         1264 :   TREE_CONSTANT (ctor) = 1;
    3449         1264 :   TREE_STATIC (ctor) = 1;
    3450         1264 :   DECL_INITIAL (var) = ctor;
    3451         1264 :   DECL_ATTRIBUTES (var) = tree_cons (get_identifier ("asan odr indicator"),
    3452         1264 :                                      NULL, DECL_ATTRIBUTES (var));
    3453         1264 :   make_decl_rtl (var);
    3454         1264 :   varpool_node::finalize_decl (var);
    3455         1264 :   return fold_convert (uptr, build_fold_addr_expr (var));
    3456              : }
    3457              : 
    3458              : /* Return true if DECL, a global var, might be overridden and needs
    3459              :    an additional odr indicator symbol.  */
    3460              : 
    3461              : static bool
    3462         4218 : asan_needs_odr_indicator_p (tree decl)
    3463              : {
    3464              :   /* Don't emit ODR indicators for kernel because:
    3465              :      a) Kernel is written in C thus doesn't need ODR indicators.
    3466              :      b) Some kernel code may have assumptions about symbols containing specific
    3467              :         patterns in their names.  Since ODR indicators contain original names
    3468              :         of symbols they are emitted for, these assumptions would be broken for
    3469              :         ODR indicator symbols.  */
    3470         4218 :   return (!(flag_sanitize & SANITIZE_KERNEL_ADDRESS)
    3471         4218 :           && !DECL_ARTIFICIAL (decl)
    3472         1676 :           && !DECL_WEAK (decl)
    3473         5894 :           && TREE_PUBLIC (decl));
    3474              : }
    3475              : 
    3476              : /* Append description of a single global DECL into vector V.
    3477              :    TYPE is __asan_global struct type as returned by asan_global_struct.  */
    3478              : 
    3479              : static void
    3480         4218 : asan_add_global (tree decl, tree type, vec<constructor_elt, va_gc> *v)
    3481              : {
    3482         4218 :   tree init, uptr = TREE_TYPE (DECL_CHAIN (TYPE_FIELDS (type)));
    3483         4218 :   unsigned HOST_WIDE_INT size;
    3484         4218 :   tree str_cst, module_name_cst, refdecl = decl;
    3485         4218 :   vec<constructor_elt, va_gc> *vinner = NULL;
    3486              : 
    3487         4218 :   pretty_printer asan_pp, module_name_pp;
    3488              : 
    3489         4218 :   if (DECL_NAME (decl))
    3490         4218 :     pp_tree_identifier (&asan_pp, DECL_NAME (decl));
    3491              :   else
    3492            0 :     pp_string (&asan_pp, "<unknown>");
    3493         4218 :   str_cst = asan_pp_string (&asan_pp);
    3494              : 
    3495         4218 :   if (!in_lto_p)
    3496         3794 :     pp_string (&module_name_pp, main_input_filename);
    3497              :   else
    3498              :     {
    3499          424 :       const_tree tu = get_ultimate_context ((const_tree)decl);
    3500          424 :       if (tu != NULL_TREE)
    3501          291 :         pp_string (&module_name_pp, IDENTIFIER_POINTER (DECL_NAME (tu)));
    3502              :       else
    3503          133 :         pp_string (&module_name_pp, aux_base_name);
    3504              :     }
    3505              : 
    3506         4218 :   module_name_cst = asan_pp_string (&module_name_pp);
    3507              : 
    3508         4218 :   if (asan_needs_local_alias (decl))
    3509              :     {
    3510            0 :       char buf[20];
    3511            0 :       ASM_GENERATE_INTERNAL_LABEL (buf, "LASAN", vec_safe_length (v) + 1);
    3512            0 :       refdecl = build_decl (DECL_SOURCE_LOCATION (decl),
    3513            0 :                             VAR_DECL, get_identifier (buf), TREE_TYPE (decl));
    3514            0 :       TREE_ADDRESSABLE (refdecl) = TREE_ADDRESSABLE (decl);
    3515            0 :       TREE_READONLY (refdecl) = TREE_READONLY (decl);
    3516            0 :       TREE_THIS_VOLATILE (refdecl) = TREE_THIS_VOLATILE (decl);
    3517            0 :       DECL_NOT_GIMPLE_REG_P (refdecl) = DECL_NOT_GIMPLE_REG_P (decl);
    3518            0 :       DECL_ARTIFICIAL (refdecl) = DECL_ARTIFICIAL (decl);
    3519            0 :       DECL_IGNORED_P (refdecl) = DECL_IGNORED_P (decl);
    3520            0 :       TREE_STATIC (refdecl) = 1;
    3521            0 :       TREE_PUBLIC (refdecl) = 0;
    3522            0 :       TREE_USED (refdecl) = 1;
    3523            0 :       assemble_alias (refdecl, DECL_ASSEMBLER_NAME (decl));
    3524              :     }
    3525              : 
    3526         4218 :   tree odr_indicator_ptr
    3527         4218 :     = (asan_needs_odr_indicator_p (decl) ? create_odr_indicator (decl, type)
    3528         2954 :                                          : build_int_cst (uptr, 0));
    3529         4218 :   CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
    3530              :                           fold_convert (const_ptr_type_node,
    3531              :                                         build_fold_addr_expr (refdecl)));
    3532         4218 :   size = tree_to_uhwi (DECL_SIZE_UNIT (decl));
    3533         4218 :   CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, build_int_cst (uptr, size));
    3534         4218 :   size += asan_red_zone_size (size);
    3535         4218 :   CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, build_int_cst (uptr, size));
    3536         4218 :   CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
    3537              :                           fold_convert (const_ptr_type_node, str_cst));
    3538         4218 :   CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
    3539              :                           fold_convert (const_ptr_type_node, module_name_cst));
    3540         4218 :   varpool_node *vnode = varpool_node::get (decl);
    3541         4218 :   int has_dynamic_init = 0;
    3542              :   /* FIXME: Enable initialization order fiasco detection in LTO mode once
    3543              :      proper fix for PR 79061 will be applied.  */
    3544         4218 :   if (!in_lto_p)
    3545         3794 :     has_dynamic_init = vnode ? vnode->dynamically_initialized : 0;
    3546         4218 :   CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE,
    3547              :                           build_int_cst (uptr, has_dynamic_init));
    3548         4218 :   tree locptr = NULL_TREE;
    3549         4218 :   location_t loc = DECL_SOURCE_LOCATION (decl);
    3550         4218 :   expanded_location xloc = expand_location (loc);
    3551         4218 :   if (xloc.file != NULL)
    3552              :     {
    3553         2304 :       static int lasanloccnt = 0;
    3554         2304 :       char buf[25];
    3555         2304 :       ASM_GENERATE_INTERNAL_LABEL (buf, "LASANLOC", ++lasanloccnt);
    3556         2304 :       tree var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (buf),
    3557              :                              ubsan_get_source_location_type ());
    3558         2304 :       TREE_STATIC (var) = 1;
    3559         2304 :       TREE_PUBLIC (var) = 0;
    3560         2304 :       DECL_ARTIFICIAL (var) = 1;
    3561         2304 :       DECL_IGNORED_P (var) = 1;
    3562         2304 :       pretty_printer filename_pp;
    3563         2304 :       pp_string (&filename_pp, xloc.file);
    3564         2304 :       tree str = asan_pp_string (&filename_pp);
    3565         2304 :       tree ctor = build_constructor_va (TREE_TYPE (var), 3,
    3566              :                                         NULL_TREE, str, NULL_TREE,
    3567              :                                         build_int_cst (unsigned_type_node,
    3568         2304 :                                                        xloc.line), NULL_TREE,
    3569              :                                         build_int_cst (unsigned_type_node,
    3570         2304 :                                                        xloc.column));
    3571         2304 :       TREE_CONSTANT (ctor) = 1;
    3572         2304 :       TREE_STATIC (ctor) = 1;
    3573         2304 :       DECL_INITIAL (var) = ctor;
    3574         2304 :       varpool_node::finalize_decl (var);
    3575         2304 :       locptr = fold_convert (uptr, build_fold_addr_expr (var));
    3576         2304 :     }
    3577              :   else
    3578         1914 :     locptr = build_int_cst (uptr, 0);
    3579         4218 :   CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, locptr);
    3580         4218 :   CONSTRUCTOR_APPEND_ELT (vinner, NULL_TREE, odr_indicator_ptr);
    3581         4218 :   init = build_constructor (type, vinner);
    3582         4218 :   CONSTRUCTOR_APPEND_ELT (v, NULL_TREE, init);
    3583         4218 : }
    3584              : 
    3585              : /* Initialize sanitizer.def builtins if the FE hasn't initialized them.  */
    3586              : void
    3587        38276 : initialize_sanitizer_builtins (void)
    3588              : {
    3589        38276 :   tree decl;
    3590              : 
    3591        38276 :   if (builtin_decl_implicit_p (BUILT_IN_ASAN_INIT))
    3592        38172 :     return;
    3593              : 
    3594          104 :   tree ptrmode_type = (*lang_hooks.types.type_for_mode) (ptr_mode, 0);
    3595          104 :   tree BT_FN_VOID = build_function_type_list (void_type_node, NULL_TREE);
    3596          104 :   tree BT_FN_VOID_PTR
    3597          104 :     = build_function_type_list (void_type_node, ptr_type_node, NULL_TREE);
    3598          104 :   tree BT_FN_VOID_CONST_PTR
    3599          104 :     = build_function_type_list (void_type_node, const_ptr_type_node, NULL_TREE);
    3600          104 :   tree BT_FN_VOID_PTR_PTR
    3601          104 :     = build_function_type_list (void_type_node, ptr_type_node,
    3602              :                                 ptr_type_node, NULL_TREE);
    3603          104 :   tree BT_FN_VOID_PTR_PTR_PTR
    3604          104 :     = build_function_type_list (void_type_node, ptr_type_node,
    3605              :                                 ptr_type_node, ptr_type_node, NULL_TREE);
    3606          104 :   tree BT_FN_VOID_PTR_PTRMODE
    3607          104 :     = build_function_type_list (void_type_node, ptr_type_node,
    3608              :                                 ptrmode_type, NULL_TREE);
    3609          104 :   tree BT_FN_VOID_INT
    3610          104 :     = build_function_type_list (void_type_node, integer_type_node, NULL_TREE);
    3611          104 :   tree BT_FN_SIZE_CONST_PTR_INT
    3612          104 :     = build_function_type_list (size_type_node, const_ptr_type_node,
    3613              :                                 integer_type_node, NULL_TREE);
    3614              : 
    3615          104 :   tree BT_FN_VOID_UINT8_UINT8
    3616          104 :     = build_function_type_list (void_type_node, unsigned_char_type_node,
    3617              :                                 unsigned_char_type_node, NULL_TREE);
    3618          104 :   tree BT_FN_VOID_UINT16_UINT16
    3619          104 :     = build_function_type_list (void_type_node, uint16_type_node,
    3620              :                                 uint16_type_node, NULL_TREE);
    3621          104 :   tree BT_FN_VOID_UINT32_UINT32
    3622          104 :     = build_function_type_list (void_type_node, uint32_type_node,
    3623              :                                 uint32_type_node, NULL_TREE);
    3624          104 :   tree BT_FN_VOID_UINT64_UINT64
    3625          104 :     = build_function_type_list (void_type_node, uint64_type_node,
    3626              :                                 uint64_type_node, NULL_TREE);
    3627          104 :   tree BT_FN_VOID_FLOAT_FLOAT
    3628          104 :     = build_function_type_list (void_type_node, float_type_node,
    3629              :                                 float_type_node, NULL_TREE);
    3630          104 :   tree BT_FN_VOID_DOUBLE_DOUBLE
    3631          104 :     = build_function_type_list (void_type_node, double_type_node,
    3632              :                                 double_type_node, NULL_TREE);
    3633          104 :   tree BT_FN_VOID_UINT64_PTR
    3634          104 :     = build_function_type_list (void_type_node, uint64_type_node,
    3635              :                                 ptr_type_node, NULL_TREE);
    3636              : 
    3637          104 :   tree BT_FN_PTR_CONST_PTR_UINT8
    3638          104 :     = build_function_type_list (ptr_type_node, const_ptr_type_node,
    3639              :                                 unsigned_char_type_node, NULL_TREE);
    3640          104 :   tree BT_FN_VOID_PTR_UINT8_PTRMODE
    3641          104 :     = build_function_type_list (void_type_node, ptr_type_node,
    3642              :                                 unsigned_char_type_node,
    3643              :                                 ptrmode_type, NULL_TREE);
    3644              : 
    3645          104 :   tree BT_FN_BOOL_VPTR_PTR_IX_INT_INT[5];
    3646          104 :   tree BT_FN_IX_CONST_VPTR_INT[5];
    3647          104 :   tree BT_FN_IX_VPTR_IX_INT[5];
    3648          104 :   tree BT_FN_VOID_VPTR_IX_INT[5];
    3649          104 :   tree vptr
    3650          104 :     = build_pointer_type (build_qualified_type (void_type_node,
    3651              :                                                 TYPE_QUAL_VOLATILE));
    3652          104 :   tree cvptr
    3653          104 :     = build_pointer_type (build_qualified_type (void_type_node,
    3654              :                                                 TYPE_QUAL_VOLATILE
    3655              :                                                 |TYPE_QUAL_CONST));
    3656          104 :   tree boolt
    3657          104 :     = lang_hooks.types.type_for_size (BOOL_TYPE_SIZE, 1);
    3658          104 :   int i;
    3659          728 :   for (i = 0; i < 5; i++)
    3660              :     {
    3661          520 :       tree ix = build_nonstandard_integer_type (BITS_PER_UNIT * (1 << i), 1);
    3662          520 :       BT_FN_BOOL_VPTR_PTR_IX_INT_INT[i]
    3663          520 :         = build_function_type_list (boolt, vptr, ptr_type_node, ix,
    3664              :                                     integer_type_node, integer_type_node,
    3665              :                                     NULL_TREE);
    3666          520 :       BT_FN_IX_CONST_VPTR_INT[i]
    3667          520 :         = build_function_type_list (ix, cvptr, integer_type_node, NULL_TREE);
    3668          520 :       BT_FN_IX_VPTR_IX_INT[i]
    3669          520 :         = build_function_type_list (ix, vptr, ix, integer_type_node,
    3670              :                                     NULL_TREE);
    3671          520 :       BT_FN_VOID_VPTR_IX_INT[i]
    3672          520 :         = build_function_type_list (void_type_node, vptr, ix,
    3673              :                                     integer_type_node, NULL_TREE);
    3674              :     }
    3675              : #define BT_FN_BOOL_VPTR_PTR_I1_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[0]
    3676              : #define BT_FN_I1_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[0]
    3677              : #define BT_FN_I1_VPTR_I1_INT BT_FN_IX_VPTR_IX_INT[0]
    3678              : #define BT_FN_VOID_VPTR_I1_INT BT_FN_VOID_VPTR_IX_INT[0]
    3679              : #define BT_FN_BOOL_VPTR_PTR_I2_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[1]
    3680              : #define BT_FN_I2_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[1]
    3681              : #define BT_FN_I2_VPTR_I2_INT BT_FN_IX_VPTR_IX_INT[1]
    3682              : #define BT_FN_VOID_VPTR_I2_INT BT_FN_VOID_VPTR_IX_INT[1]
    3683              : #define BT_FN_BOOL_VPTR_PTR_I4_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[2]
    3684              : #define BT_FN_I4_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[2]
    3685              : #define BT_FN_I4_VPTR_I4_INT BT_FN_IX_VPTR_IX_INT[2]
    3686              : #define BT_FN_VOID_VPTR_I4_INT BT_FN_VOID_VPTR_IX_INT[2]
    3687              : #define BT_FN_BOOL_VPTR_PTR_I8_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[3]
    3688              : #define BT_FN_I8_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[3]
    3689              : #define BT_FN_I8_VPTR_I8_INT BT_FN_IX_VPTR_IX_INT[3]
    3690              : #define BT_FN_VOID_VPTR_I8_INT BT_FN_VOID_VPTR_IX_INT[3]
    3691              : #define BT_FN_BOOL_VPTR_PTR_I16_INT_INT BT_FN_BOOL_VPTR_PTR_IX_INT_INT[4]
    3692              : #define BT_FN_I16_CONST_VPTR_INT BT_FN_IX_CONST_VPTR_INT[4]
    3693              : #define BT_FN_I16_VPTR_I16_INT BT_FN_IX_VPTR_IX_INT[4]
    3694              : #define BT_FN_VOID_VPTR_I16_INT BT_FN_VOID_VPTR_IX_INT[4]
    3695              : #undef ATTR_NOTHROW_LIST
    3696              : #define ATTR_NOTHROW_LIST ECF_NOTHROW
    3697              : #undef ATTR_NOTHROW_LEAF_LIST
    3698              : #define ATTR_NOTHROW_LEAF_LIST ECF_NOTHROW | ECF_LEAF
    3699              : #undef ATTR_TMPURE_NOTHROW_LEAF_LIST
    3700              : #define ATTR_TMPURE_NOTHROW_LEAF_LIST ECF_TM_PURE | ATTR_NOTHROW_LEAF_LIST
    3701              : #undef ATTR_NORETURN_NOTHROW_LEAF_LIST
    3702              : #define ATTR_NORETURN_NOTHROW_LEAF_LIST ECF_NORETURN | ATTR_NOTHROW_LEAF_LIST
    3703              : #undef ATTR_CONST_NORETURN_NOTHROW_LEAF_LIST
    3704              : #define ATTR_CONST_NORETURN_NOTHROW_LEAF_LIST \
    3705              :   ECF_CONST | ATTR_NORETURN_NOTHROW_LEAF_LIST
    3706              : #undef ATTR_TMPURE_NORETURN_NOTHROW_LEAF_LIST
    3707              : #define ATTR_TMPURE_NORETURN_NOTHROW_LEAF_LIST \
    3708              :   ECF_TM_PURE | ATTR_NORETURN_NOTHROW_LEAF_LIST
    3709              : #undef ATTR_COLD_NOTHROW_LEAF_LIST
    3710              : #define ATTR_COLD_NOTHROW_LEAF_LIST \
    3711              :   /* ECF_COLD missing */ ATTR_NOTHROW_LEAF_LIST
    3712              : #undef ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST
    3713              : #define ATTR_COLD_NORETURN_NOTHROW_LEAF_LIST \
    3714              :   /* ECF_COLD missing */ ATTR_NORETURN_NOTHROW_LEAF_LIST
    3715              : #undef ATTR_COLD_CONST_NORETURN_NOTHROW_LEAF_LIST
    3716              : #define ATTR_COLD_CONST_NORETURN_NOTHROW_LEAF_LIST \
    3717              :   /* ECF_COLD missing */ ATTR_CONST_NORETURN_NOTHROW_LEAF_LIST
    3718              : #undef ATTR_PURE_NOTHROW_LEAF_LIST
    3719              : #define ATTR_PURE_NOTHROW_LEAF_LIST ECF_PURE | ATTR_NOTHROW_LEAF_LIST
    3720              : #undef DEF_BUILTIN_STUB
    3721              : #define DEF_BUILTIN_STUB(ENUM, NAME)
    3722              : #undef DEF_SANITIZER_BUILTIN_1
    3723              : #define DEF_SANITIZER_BUILTIN_1(ENUM, NAME, TYPE, ATTRS)                \
    3724              :   do {                                                                  \
    3725              :     decl = add_builtin_function ("__builtin_" NAME, TYPE, ENUM,               \
    3726              :                                  BUILT_IN_NORMAL, NAME, NULL_TREE);     \
    3727              :     set_call_expr_flags (decl, ATTRS);                                  \
    3728              :     set_builtin_decl (ENUM, decl, true);                                \
    3729              :   } while (0)
    3730              : #undef DEF_SANITIZER_BUILTIN
    3731              : #define DEF_SANITIZER_BUILTIN(ENUM, NAME, TYPE, ATTRS)  \
    3732              :   DEF_SANITIZER_BUILTIN_1 (ENUM, NAME, TYPE, ATTRS);
    3733              : 
    3734              : #include "sanitizer.def"
    3735              : 
    3736              :   /* -fsanitize=object-size uses __builtin_dynamic_object_size and
    3737              :      __builtin_object_size, but they might not be available for e.g. Fortran at
    3738              :      this point.  We use DEF_SANITIZER_BUILTIN here only as a convenience
    3739              :      macro.  */
    3740          104 :   if (flag_sanitize & SANITIZE_OBJECT_SIZE)
    3741              :     {
    3742           21 :       if (!builtin_decl_implicit_p (BUILT_IN_OBJECT_SIZE))
    3743           15 :         DEF_SANITIZER_BUILTIN_1 (BUILT_IN_OBJECT_SIZE, "object_size",
    3744              :                                  BT_FN_SIZE_CONST_PTR_INT,
    3745              :                                  ATTR_PURE_NOTHROW_LEAF_LIST);
    3746          125 :       if (!builtin_decl_implicit_p (BUILT_IN_DYNAMIC_OBJECT_SIZE))
    3747           15 :         DEF_SANITIZER_BUILTIN_1 (BUILT_IN_DYNAMIC_OBJECT_SIZE,
    3748              :                                  "dynamic_object_size",
    3749              :                                  BT_FN_SIZE_CONST_PTR_INT,
    3750              :                                  ATTR_PURE_NOTHROW_LEAF_LIST);
    3751              :     }
    3752              : 
    3753              : #undef DEF_SANITIZER_BUILTIN_1
    3754              : #undef DEF_SANITIZER_BUILTIN
    3755              : #undef DEF_BUILTIN_STUB
    3756              : }
    3757              : 
    3758              : /* Called via htab_traverse.  Count number of emitted
    3759              :    STRING_CSTs in the constant hash table.  */
    3760              : 
    3761              : int
    3762         3435 : count_string_csts (constant_descriptor_tree **slot,
    3763              :                    unsigned HOST_WIDE_INT *data)
    3764              : {
    3765         3435 :   struct constant_descriptor_tree *desc = *slot;
    3766         3435 :   if (TREE_CODE (desc->value) == STRING_CST
    3767         3401 :       && TREE_ASM_WRITTEN (desc->value)
    3768         6836 :       && asan_protect_global (desc->value))
    3769         1714 :     ++*data;
    3770         3435 :   return 1;
    3771              : }
    3772              : 
    3773              : /* Helper structure to pass two parameters to
    3774              :    add_string_csts.  */
    3775              : 
    3776              : struct asan_add_string_csts_data
    3777              : {
    3778              :   tree type;
    3779              :   vec<constructor_elt, va_gc> *v;
    3780              : };
    3781              : 
    3782              : /* Called via hash_table::traverse.  Call asan_add_global
    3783              :    on emitted STRING_CSTs from the constant hash table.  */
    3784              : 
    3785              : int
    3786         3521 : add_string_csts (constant_descriptor_tree **slot,
    3787              :                  asan_add_string_csts_data *aascd)
    3788              : {
    3789         3521 :   struct constant_descriptor_tree *desc = *slot;
    3790         3521 :   if (TREE_CODE (desc->value) == STRING_CST
    3791         3494 :       && TREE_ASM_WRITTEN (desc->value)
    3792         7015 :       && asan_protect_global (desc->value))
    3793              :     {
    3794         1714 :       asan_add_global (SYMBOL_REF_DECL (XEXP (desc->rtl, 0)),
    3795              :                        aascd->type, aascd->v);
    3796              :     }
    3797         3521 :   return 1;
    3798              : }
    3799              : 
    3800              : /* Needs to be GTY(()), because cgraph_build_static_cdtor may
    3801              :    invoke ggc_collect.  */
    3802              : static GTY(()) tree asan_ctor_statements;
    3803              : 
    3804              : /* Module-level instrumentation.
    3805              :    - Insert __asan_init_vN() into the list of CTORs.
    3806              :    - TODO: insert redzones around globals.
    3807              :  */
    3808              : 
    3809              : void
    3810         2491 : asan_finish_file (void)
    3811              : {
    3812         2491 :   varpool_node *vnode;
    3813         2491 :   unsigned HOST_WIDE_INT gcount = 0;
    3814              : 
    3815         2491 :   if (shadow_ptr_types[0] == NULL_TREE)
    3816          109 :     asan_init_shadow_ptr_types ();
    3817              :   /* Avoid instrumenting code in the asan ctors/dtors.
    3818              :      We don't need to insert padding after the description strings,
    3819              :      nor after .LASAN* array.  */
    3820         2491 :   flag_sanitize &= ~SANITIZE_ADDRESS;
    3821              : 
    3822              :   /* For user-space we want asan constructors to run first.
    3823              :      Linux kernel does not support priorities other than default, and the only
    3824              :      other user of constructors is coverage. So we run with the default
    3825              :      priority.  */
    3826          126 :   int priority = flag_sanitize & SANITIZE_USER_ADDRESS
    3827         2491 :                  ? MAX_RESERVED_INIT_PRIORITY - 1 : DEFAULT_INIT_PRIORITY;
    3828              : 
    3829         2491 :   if (flag_sanitize & SANITIZE_USER_ADDRESS)
    3830              :     {
    3831         2365 :       tree fn = builtin_decl_implicit (BUILT_IN_ASAN_INIT);
    3832         2365 :       append_to_statement_list (build_call_expr (fn, 0), &asan_ctor_statements);
    3833         2365 :       fn = builtin_decl_implicit (BUILT_IN_ASAN_VERSION_MISMATCH_CHECK);
    3834         2365 :       append_to_statement_list (build_call_expr (fn, 0), &asan_ctor_statements);
    3835              :     }
    3836         5214 :   FOR_EACH_DEFINED_VARIABLE (vnode)
    3837         2723 :     if (TREE_ASM_WRITTEN (vnode->decl)
    3838         2723 :         && asan_protect_global (vnode->decl))
    3839         2504 :       ++gcount;
    3840         2491 :   hash_table<tree_descriptor_hasher> *const_desc_htab = constant_pool_htab ();
    3841         2491 :   const_desc_htab->traverse<unsigned HOST_WIDE_INT *, count_string_csts>
    3842         5926 :     (&gcount);
    3843         2491 :   if (gcount)
    3844              :     {
    3845         1102 :       tree type = asan_global_struct (), var, ctor;
    3846         1102 :       tree dtor_statements = NULL_TREE;
    3847         1102 :       vec<constructor_elt, va_gc> *v;
    3848         1102 :       char buf[20];
    3849              : 
    3850         1102 :       type = build_array_type_nelts (type, gcount);
    3851         1102 :       ASM_GENERATE_INTERNAL_LABEL (buf, "LASAN", 0);
    3852         1102 :       var = build_decl (UNKNOWN_LOCATION, VAR_DECL, get_identifier (buf),
    3853              :                         type);
    3854         1102 :       TREE_STATIC (var) = 1;
    3855         1102 :       TREE_PUBLIC (var) = 0;
    3856         1102 :       DECL_ARTIFICIAL (var) = 1;
    3857         1102 :       DECL_IGNORED_P (var) = 1;
    3858         1102 :       vec_alloc (v, gcount);
    3859         3743 :       FOR_EACH_DEFINED_VARIABLE (vnode)
    3860         2641 :         if (TREE_ASM_WRITTEN (vnode->decl)
    3861         2641 :             && asan_protect_global (vnode->decl))
    3862         2504 :           asan_add_global (vnode->decl, TREE_TYPE (type), v);
    3863         1102 :       struct asan_add_string_csts_data aascd;
    3864         1102 :       aascd.type = TREE_TYPE (type);
    3865         1102 :       aascd.v = v;
    3866         1102 :       const_desc_htab->traverse<asan_add_string_csts_data *, add_string_csts>
    3867         4623 :         (&aascd);
    3868         1102 :       ctor = build_constructor (type, v);
    3869         1102 :       TREE_CONSTANT (ctor) = 1;
    3870         1102 :       TREE_STATIC (ctor) = 1;
    3871         1102 :       DECL_INITIAL (var) = ctor;
    3872         1102 :       SET_DECL_ALIGN (var, MAX (DECL_ALIGN (var),
    3873              :                                 ASAN_SHADOW_GRANULARITY * BITS_PER_UNIT));
    3874              : 
    3875         1102 :       varpool_node::finalize_decl (var);
    3876              : 
    3877         1102 :       tree fn = builtin_decl_implicit (BUILT_IN_ASAN_REGISTER_GLOBALS);
    3878         1102 :       tree gcount_tree = build_int_cst (pointer_sized_int_node, gcount);
    3879         1102 :       append_to_statement_list (build_call_expr (fn, 2,
    3880              :                                                  build_fold_addr_expr (var),
    3881              :                                                  gcount_tree),
    3882              :                                 &asan_ctor_statements);
    3883              : 
    3884         1102 :       fn = builtin_decl_implicit (BUILT_IN_ASAN_UNREGISTER_GLOBALS);
    3885         1102 :       append_to_statement_list (build_call_expr (fn, 2,
    3886              :                                                  build_fold_addr_expr (var),
    3887              :                                                  gcount_tree),
    3888              :                                 &dtor_statements);
    3889         1102 :       cgraph_build_static_cdtor ('D', dtor_statements, priority);
    3890              :     }
    3891         2491 :   if (asan_ctor_statements)
    3892         2365 :     cgraph_build_static_cdtor ('I', asan_ctor_statements, priority);
    3893         2491 :   flag_sanitize |= SANITIZE_ADDRESS;
    3894         2491 : }
    3895              : 
    3896              : /* Poison or unpoison (depending on IS_CLOBBER variable) shadow memory based
    3897              :    on SHADOW address.  Newly added statements will be added to ITER with
    3898              :    given location LOC.  We mark SIZE bytes in shadow memory, where
    3899              :    LAST_CHUNK_SIZE is greater than zero in situation where we are at the
    3900              :    end of a variable.  */
    3901              : 
    3902              : static void
    3903         2193 : asan_store_shadow_bytes (gimple_stmt_iterator *iter, location_t loc,
    3904              :                          tree shadow,
    3905              :                          unsigned HOST_WIDE_INT base_addr_offset,
    3906              :                          bool is_clobber, unsigned size,
    3907              :                          unsigned last_chunk_size)
    3908              : {
    3909         2193 :   tree shadow_ptr_type;
    3910              : 
    3911         2193 :   switch (size)
    3912              :     {
    3913         1422 :     case 1:
    3914         1422 :       shadow_ptr_type = shadow_ptr_types[0];
    3915         1422 :       break;
    3916          272 :     case 2:
    3917          272 :       shadow_ptr_type = shadow_ptr_types[1];
    3918          272 :       break;
    3919          499 :     case 4:
    3920          499 :       shadow_ptr_type = shadow_ptr_types[2];
    3921          499 :       break;
    3922            0 :     default:
    3923            0 :       gcc_unreachable ();
    3924              :     }
    3925              : 
    3926         2193 :   unsigned char c = (char) is_clobber ? ASAN_STACK_MAGIC_USE_AFTER_SCOPE : 0;
    3927         2193 :   unsigned HOST_WIDE_INT val = 0;
    3928         2193 :   unsigned last_pos = size;
    3929         2193 :   if (last_chunk_size && !is_clobber)
    3930          322 :     last_pos = BYTES_BIG_ENDIAN ? 0 : size - 1;
    3931         6155 :   for (unsigned i = 0; i < size; ++i)
    3932              :     {
    3933         3962 :       unsigned char shadow_c = c;
    3934         3962 :       if (i == last_pos)
    3935          322 :         shadow_c = last_chunk_size;
    3936         3962 :       val |= (unsigned HOST_WIDE_INT) shadow_c << (BITS_PER_UNIT * i);
    3937              :     }
    3938              : 
    3939              :   /* Handle last chunk in unpoisoning.  */
    3940         2193 :   tree magic = build_int_cst (TREE_TYPE (shadow_ptr_type), val);
    3941              : 
    3942         2193 :   tree dest = build2 (MEM_REF, TREE_TYPE (shadow_ptr_type), shadow,
    3943         2193 :                       build_int_cst (shadow_ptr_type, base_addr_offset));
    3944              : 
    3945         2193 :   gimple *g = gimple_build_assign (dest, magic);
    3946         2193 :   gimple_set_location (g, loc);
    3947         2193 :   gsi_insert_after (iter, g, GSI_NEW_STMT);
    3948         2193 : }
    3949              : 
    3950              : /* Expand the ASAN_MARK builtins.  */
    3951              : 
    3952              : bool
    3953         2149 : asan_expand_mark_ifn (gimple_stmt_iterator *iter)
    3954              : {
    3955         2149 :   gimple *g = gsi_stmt (*iter);
    3956         2149 :   location_t loc = gimple_location (g);
    3957         2149 :   HOST_WIDE_INT flag = tree_to_shwi (gimple_call_arg (g, 0));
    3958         2149 :   bool is_poison = ((asan_mark_flags)flag) == ASAN_MARK_POISON;
    3959              : 
    3960         2149 :   tree base = gimple_call_arg (g, 1);
    3961         2149 :   gcc_checking_assert (TREE_CODE (base) == ADDR_EXPR);
    3962         2149 :   tree decl = TREE_OPERAND (base, 0);
    3963              : 
    3964              :   /* For a nested function, we can have: ASAN_MARK (2, &FRAME.2.fp_input, 4) */
    3965         2149 :   if (TREE_CODE (decl) == COMPONENT_REF
    3966         2149 :       && DECL_NONLOCAL_FRAME (TREE_OPERAND (decl, 0)))
    3967            1 :     decl = TREE_OPERAND (decl, 0);
    3968              : 
    3969         2149 :   gcc_checking_assert (TREE_CODE (decl) == VAR_DECL);
    3970              : 
    3971         2149 :   if (hwassist_sanitize_p ())
    3972              :     {
    3973            0 :       gcc_assert (param_hwasan_instrument_stack);
    3974            0 :       gimple_seq stmts = NULL;
    3975              :       /* Here we swap ASAN_MARK calls for HWASAN_MARK.
    3976              :          This is because we are using the approach of using ASAN_MARK as a
    3977              :          synonym until here.
    3978              :          That approach means we don't yet have to duplicate all the special
    3979              :          cases for ASAN_MARK and ASAN_POISON with the exact same handling but
    3980              :          called HWASAN_MARK etc.
    3981              : 
    3982              :          N.b. __asan_poison_stack_memory (which implements ASAN_MARK for ASAN)
    3983              :          rounds the size up to its shadow memory granularity, while
    3984              :          __hwasan_tag_memory (which implements the same for HWASAN) does not.
    3985              :          Hence we emit HWASAN_MARK with an aligned size unlike ASAN_MARK.  */
    3986            0 :       tree len = gimple_call_arg (g, 2);
    3987            0 :       tree new_len = gimple_build_round_up (&stmts, loc, size_type_node, len,
    3988            0 :                                             HWASAN_TAG_GRANULE_SIZE);
    3989            0 :       gimple_build (&stmts, loc, CFN_HWASAN_MARK,
    3990              :                     void_type_node, gimple_call_arg (g, 0),
    3991              :                     base, new_len);
    3992            0 :       gsi_replace_with_seq (iter, stmts, true);
    3993            0 :       return false;
    3994              :     }
    3995              : 
    3996         2149 :   if (is_poison)
    3997              :     {
    3998         1412 :       if (asan_handled_variables == NULL)
    3999          387 :         asan_handled_variables = new hash_set<tree> (16);
    4000         1412 :       asan_handled_variables->add (decl);
    4001              :     }
    4002         2149 :   tree len = gimple_call_arg (g, 2);
    4003              : 
    4004         2149 :   gcc_assert (poly_int_tree_p (len));
    4005              : 
    4006         2149 :   g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
    4007              :                            NOP_EXPR, base);
    4008         2149 :   gimple_set_location (g, loc);
    4009         2149 :   gsi_replace (iter, g, false);
    4010         2149 :   tree base_addr = gimple_assign_lhs (g);
    4011              : 
    4012              :   /* Generate direct emission if size_in_bytes is small.  */
    4013         2149 :   unsigned threshold = param_use_after_scope_direct_emission_threshold;
    4014         2149 :   if (tree_fits_uhwi_p (len) && tree_to_uhwi (len) <= threshold)
    4015              :     {
    4016         2096 :       unsigned HOST_WIDE_INT size_in_bytes = tree_to_uhwi (len);
    4017         2096 :       const unsigned HOST_WIDE_INT shadow_size
    4018         2096 :         = shadow_mem_size (size_in_bytes);
    4019         2096 :       const unsigned int shadow_align
    4020         2096 :         = (get_pointer_alignment (base) / BITS_PER_UNIT) >> ASAN_SHADOW_SHIFT;
    4021              : 
    4022         2096 :       tree shadow = build_shadow_mem_access (iter, loc, base_addr,
    4023              :                                              shadow_ptr_types[0], true);
    4024              : 
    4025         6385 :       for (unsigned HOST_WIDE_INT offset = 0; offset < shadow_size;)
    4026              :         {
    4027         2193 :           unsigned size = 1;
    4028         2193 :           if (shadow_size - offset >= 4
    4029              :               && (!STRICT_ALIGNMENT || shadow_align >= 4))
    4030              :             size = 4;
    4031         1694 :           else if (shadow_size - offset >= 2
    4032              :                    && (!STRICT_ALIGNMENT || shadow_align >= 2))
    4033          272 :             size = 2;
    4034              : 
    4035         2193 :           unsigned HOST_WIDE_INT last_chunk_size = 0;
    4036         2193 :           unsigned HOST_WIDE_INT s = (offset + size) * ASAN_SHADOW_GRANULARITY;
    4037         2193 :           if (s > size_in_bytes)
    4038         1152 :             last_chunk_size = ASAN_SHADOW_GRANULARITY - (s - size_in_bytes);
    4039              : 
    4040         2193 :           asan_store_shadow_bytes (iter, loc, shadow, offset, is_poison,
    4041              :                                    size, last_chunk_size);
    4042         2193 :           offset += size;
    4043              :         }
    4044              :     }
    4045              :   else
    4046              :     {
    4047           53 :       g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
    4048              :                                NOP_EXPR, len);
    4049           53 :       gimple_set_location (g, loc);
    4050           53 :       gsi_safe_insert_before (iter, g);
    4051           53 :       tree sz_arg = gimple_assign_lhs (g);
    4052              : 
    4053           53 :       tree fun
    4054           71 :         = builtin_decl_implicit (is_poison ? BUILT_IN_ASAN_POISON_STACK_MEMORY
    4055              :                                  : BUILT_IN_ASAN_UNPOISON_STACK_MEMORY);
    4056           53 :       g = gimple_build_call (fun, 2, base_addr, sz_arg);
    4057           53 :       gimple_set_location (g, loc);
    4058           53 :       gsi_insert_after (iter, g, GSI_NEW_STMT);
    4059              :     }
    4060              : 
    4061              :   return false;
    4062              : }
    4063              : 
    4064              : /* Expand the ASAN_{LOAD,STORE} builtins.  */
    4065              : 
    4066              : bool
    4067        21135 : asan_expand_check_ifn (gimple_stmt_iterator *iter, bool use_calls)
    4068              : {
    4069        21135 :   gcc_assert (!hwassist_sanitize_p ());
    4070        21135 :   gimple *g = gsi_stmt (*iter);
    4071        21135 :   location_t loc = gimple_location (g);
    4072        21135 :   bool recover_p;
    4073        21135 :   if (flag_sanitize & SANITIZE_USER_ADDRESS)
    4074        21090 :     recover_p = (flag_sanitize_recover & SANITIZE_USER_ADDRESS) != 0;
    4075              :   else
    4076           45 :     recover_p = (flag_sanitize_recover & SANITIZE_KERNEL_ADDRESS) != 0;
    4077              : 
    4078        21135 :   HOST_WIDE_INT flags = tree_to_shwi (gimple_call_arg (g, 0));
    4079        21135 :   gcc_assert (flags < ASAN_CHECK_LAST);
    4080        21135 :   bool is_scalar_access = (flags & ASAN_CHECK_SCALAR_ACCESS) != 0;
    4081        21135 :   bool is_store = (flags & ASAN_CHECK_STORE) != 0;
    4082        21135 :   bool is_non_zero_len = (flags & ASAN_CHECK_NON_ZERO_LEN) != 0;
    4083              : 
    4084        21135 :   tree base = gimple_call_arg (g, 1);
    4085        21135 :   tree len = gimple_call_arg (g, 2);
    4086        21135 :   HOST_WIDE_INT align = tree_to_shwi (gimple_call_arg (g, 3));
    4087              : 
    4088        41164 :   HOST_WIDE_INT size_in_bytes
    4089        21135 :     = is_scalar_access && tree_fits_shwi_p (len) ? tree_to_shwi (len) : -1;
    4090              : 
    4091        21135 :   if (use_calls)
    4092              :     {
    4093              :       /* Instrument using callbacks.  */
    4094          103 :       gimple *g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
    4095              :                                       NOP_EXPR, base);
    4096          103 :       gimple_set_location (g, loc);
    4097          103 :       gsi_insert_before (iter, g, GSI_SAME_STMT);
    4098          103 :       tree base_addr = gimple_assign_lhs (g);
    4099              : 
    4100          103 :       int nargs;
    4101          103 :       tree fun = check_func (is_store, recover_p, size_in_bytes, &nargs);
    4102          103 :       if (nargs == 1)
    4103           75 :         g = gimple_build_call (fun, 1, base_addr);
    4104              :       else
    4105              :         {
    4106           28 :           gcc_assert (nargs == 2);
    4107           28 :           g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
    4108              :                                    NOP_EXPR, len);
    4109           28 :           gimple_set_location (g, loc);
    4110           28 :           gsi_insert_before (iter, g, GSI_SAME_STMT);
    4111           28 :           tree sz_arg = gimple_assign_lhs (g);
    4112           28 :           g = gimple_build_call (fun, nargs, base_addr, sz_arg);
    4113              :         }
    4114          103 :       gimple_set_location (g, loc);
    4115          103 :       gsi_replace (iter, g, false);
    4116          103 :       return false;
    4117              :     }
    4118              : 
    4119        21032 :   HOST_WIDE_INT real_size_in_bytes = size_in_bytes == -1 ? 1 : size_in_bytes;
    4120              : 
    4121        19954 :   tree shadow_ptr_type = shadow_ptr_types[real_size_in_bytes == 16 ? 1 : 0];
    4122        21032 :   tree shadow_type = TREE_TYPE (shadow_ptr_type);
    4123              : 
    4124        21032 :   gimple_stmt_iterator gsi = *iter;
    4125              : 
    4126        21032 :   if (!is_non_zero_len)
    4127              :     {
    4128              :       /* So, the length of the memory area to asan-protect is
    4129              :          non-constant.  Let's guard the generated instrumentation code
    4130              :          like:
    4131              : 
    4132              :          if (len != 0)
    4133              :            {
    4134              :              //asan instrumentation code goes here.
    4135              :            }
    4136              :          // fallthrough instructions, starting with *ITER.  */
    4137              : 
    4138            0 :       g = gimple_build_cond (NE_EXPR,
    4139              :                             len,
    4140            0 :                             build_int_cst (TREE_TYPE (len), 0),
    4141              :                             NULL_TREE, NULL_TREE);
    4142            0 :       gimple_set_location (g, loc);
    4143              : 
    4144            0 :       basic_block then_bb, fallthrough_bb;
    4145            0 :       insert_if_then_before_iter (as_a <gcond *> (g), iter,
    4146              :                                   /*then_more_likely_p=*/true,
    4147              :                                   &then_bb, &fallthrough_bb);
    4148              :       /* Note that fallthrough_bb starts with the statement that was
    4149              :         pointed to by ITER.  */
    4150              : 
    4151              :       /* The 'then block' of the 'if (len != 0) condition is where
    4152              :         we'll generate the asan instrumentation code now.  */
    4153            0 :       gsi = gsi_last_bb (then_bb);
    4154              :     }
    4155              : 
    4156              :   /* Get an iterator on the point where we can add the condition
    4157              :      statement for the instrumentation.  */
    4158        21032 :   basic_block then_bb, else_bb;
    4159        21032 :   gsi = create_cond_insert_point (&gsi, /*before_p*/false,
    4160              :                                   /*then_more_likely_p=*/false,
    4161              :                                   /*create_then_fallthru_edge*/recover_p,
    4162              :                                   &then_bb,
    4163              :                                   &else_bb);
    4164              : 
    4165        21032 :   g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
    4166              :                            NOP_EXPR, base);
    4167        21032 :   gimple_set_location (g, loc);
    4168        21032 :   gsi_insert_before (&gsi, g, GSI_NEW_STMT);
    4169        21032 :   tree base_addr = gimple_assign_lhs (g);
    4170              : 
    4171        21032 :   tree t = NULL_TREE;
    4172        21032 :   if (real_size_in_bytes >= 8)
    4173              :     {
    4174        13890 :       tree shadow = build_shadow_mem_access (&gsi, loc, base_addr,
    4175              :                                              shadow_ptr_type);
    4176        13890 :       t = shadow;
    4177              :     }
    4178              :   else
    4179              :     {
    4180              :       /* Slow path for 1, 2 and 4 byte accesses.  */
    4181              :       /* Test (shadow != 0)
    4182              :          & ((base_addr & 7) + (real_size_in_bytes - 1)) >= shadow).  */
    4183         7142 :       tree shadow = build_shadow_mem_access (&gsi, loc, base_addr,
    4184              :                                              shadow_ptr_type);
    4185         7142 :       gimple *shadow_test = build_assign (NE_EXPR, shadow, 0);
    4186         7142 :       gimple_seq seq = NULL;
    4187         7142 :       gimple_seq_add_stmt (&seq, shadow_test);
    4188              :       /* Aligned (>= 8 bytes) can test just
    4189              :          (real_size_in_bytes - 1 >= shadow), as base_addr & 7 is known
    4190              :          to be 0.  */
    4191         7142 :       if (align < 8)
    4192              :         {
    4193         4986 :           gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR,
    4194              :                                                    base_addr, 7));
    4195         4986 :           gimple_seq_add_stmt (&seq,
    4196         9972 :                                build_type_cast (shadow_type,
    4197              :                                                 gimple_seq_last (seq)));
    4198         4986 :           if (real_size_in_bytes > 1)
    4199         1994 :             gimple_seq_add_stmt (&seq,
    4200         1994 :                                  build_assign (PLUS_EXPR,
    4201              :                                                gimple_seq_last (seq),
    4202         1994 :                                                real_size_in_bytes - 1));
    4203         9972 :           t = gimple_assign_lhs (gimple_seq_last_stmt (seq));
    4204              :         }
    4205              :       else
    4206         2156 :         t = build_int_cst (shadow_type, real_size_in_bytes - 1);
    4207         7142 :       gimple_seq_add_stmt (&seq, build_assign (GE_EXPR, t, shadow));
    4208        14284 :       gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR, shadow_test,
    4209              :                                                gimple_seq_last (seq)));
    4210        14284 :       t = gimple_assign_lhs (gimple_seq_last (seq));
    4211         7142 :       gimple_seq_set_location (seq, loc);
    4212         7142 :       gsi_insert_seq_after (&gsi, seq, GSI_CONTINUE_LINKING);
    4213              : 
    4214              :       /* For non-constant, misaligned or otherwise weird access sizes,
    4215              :        check first and last byte.  */
    4216         7142 :       if (size_in_bytes == -1)
    4217              :         {
    4218         1078 :           g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
    4219              :                                    MINUS_EXPR, len,
    4220              :                                    build_int_cst (pointer_sized_int_node, 1));
    4221         1078 :           gimple_set_location (g, loc);
    4222         1078 :           gsi_insert_after (&gsi, g, GSI_NEW_STMT);
    4223         1078 :           tree last = gimple_assign_lhs (g);
    4224         1078 :           g = gimple_build_assign (make_ssa_name (pointer_sized_int_node),
    4225              :                                    PLUS_EXPR, base_addr, last);
    4226         1078 :           gimple_set_location (g, loc);
    4227         1078 :           gsi_insert_after (&gsi, g, GSI_NEW_STMT);
    4228         1078 :           tree base_end_addr = gimple_assign_lhs (g);
    4229              : 
    4230         1078 :           tree shadow = build_shadow_mem_access (&gsi, loc, base_end_addr,
    4231              :                                                  shadow_ptr_type);
    4232         1078 :           gimple *shadow_test = build_assign (NE_EXPR, shadow, 0);
    4233         1078 :           gimple_seq seq = NULL;
    4234         1078 :           gimple_seq_add_stmt (&seq, shadow_test);
    4235         1078 :           gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR,
    4236              :                                                    base_end_addr, 7));
    4237         2156 :           gimple_seq_add_stmt (&seq, build_type_cast (shadow_type,
    4238              :                                                       gimple_seq_last (seq)));
    4239         2156 :           gimple_seq_add_stmt (&seq, build_assign (GE_EXPR,
    4240              :                                                    gimple_seq_last (seq),
    4241              :                                                    shadow));
    4242         2156 :           gimple_seq_add_stmt (&seq, build_assign (BIT_AND_EXPR, shadow_test,
    4243              :                                                    gimple_seq_last (seq)));
    4244         2156 :           gimple_seq_add_stmt (&seq, build_assign (BIT_IOR_EXPR, t,
    4245              :                                                    gimple_seq_last (seq)));
    4246         2156 :           t = gimple_assign_lhs (gimple_seq_last (seq));
    4247         1078 :           gimple_seq_set_location (seq, loc);
    4248         1078 :           gsi_insert_seq_after (&gsi, seq, GSI_CONTINUE_LINKING);
    4249              :         }
    4250              :     }
    4251              : 
    4252        21032 :   g = gimple_build_cond (NE_EXPR, t, build_int_cst (TREE_TYPE (t), 0),
    4253              :                          NULL_TREE, NULL_TREE);
    4254        21032 :   gimple_set_location (g, loc);
    4255        21032 :   gsi_insert_after (&gsi, g, GSI_NEW_STMT);
    4256              : 
    4257              :   /* Generate call to the run-time library (e.g. __asan_report_load8).  */
    4258        21032 :   gsi = gsi_start_bb (then_bb);
    4259        21032 :   int nargs;
    4260        21032 :   tree fun = report_error_func (is_store, recover_p, size_in_bytes, &nargs);
    4261        21032 :   g = gimple_build_call (fun, nargs, base_addr, len);
    4262        21032 :   gimple_set_location (g, loc);
    4263        21032 :   gsi_insert_after (&gsi, g, GSI_NEW_STMT);
    4264              : 
    4265        21032 :   gsi_remove (iter, true);
    4266        21032 :   *iter = gsi_start_bb (else_bb);
    4267              : 
    4268        21032 :   return true;
    4269              : }
    4270              : 
    4271              : /* Create ASAN shadow variable for a VAR_DECL which has been rewritten
    4272              :    into SSA.  Already seen VAR_DECLs are stored in SHADOW_VARS_MAPPING.  */
    4273              : 
    4274              : static tree
    4275           59 : create_asan_shadow_var (tree var_decl,
    4276              :                         hash_map<tree, tree> &shadow_vars_mapping)
    4277              : {
    4278           59 :   tree *slot = shadow_vars_mapping.get (var_decl);
    4279           59 :   if (slot == NULL)
    4280              :     {
    4281           59 :       tree shadow_var = copy_node (var_decl);
    4282              : 
    4283           59 :       copy_body_data id;
    4284           59 :       memset (&id, 0, sizeof (copy_body_data));
    4285           59 :       id.src_fn = id.dst_fn = current_function_decl;
    4286           59 :       copy_decl_for_dup_finish (&id, var_decl, shadow_var);
    4287              : 
    4288           59 :       DECL_ARTIFICIAL (shadow_var) = 1;
    4289           59 :       DECL_IGNORED_P (shadow_var) = 1;
    4290           59 :       DECL_SEEN_IN_BIND_EXPR_P (shadow_var) = 0;
    4291           59 :       gimple_add_tmp_var (shadow_var);
    4292              : 
    4293           59 :       shadow_vars_mapping.put (var_decl, shadow_var);
    4294           59 :       return shadow_var;
    4295              :     }
    4296              :   else
    4297            0 :     return *slot;
    4298              : }
    4299              : 
    4300              : /* Expand ASAN_POISON ifn.  */
    4301              : 
    4302              : bool
    4303           64 : asan_expand_poison_ifn (gimple_stmt_iterator *iter,
    4304              :                         bool *need_commit_edge_insert,
    4305              :                         hash_map<tree, tree> &shadow_vars_mapping)
    4306              : {
    4307           64 :   gimple *g = gsi_stmt (*iter);
    4308           64 :   tree poisoned_var = gimple_call_lhs (g);
    4309           64 :   if (!poisoned_var || has_zero_uses (poisoned_var))
    4310              :     {
    4311            5 :       gsi_remove (iter, true);
    4312            5 :       return true;
    4313              :     }
    4314              : 
    4315           59 :   if (SSA_NAME_VAR (poisoned_var) == NULL_TREE)
    4316            0 :     SET_SSA_NAME_VAR_OR_IDENTIFIER (poisoned_var,
    4317              :                                     create_tmp_var (TREE_TYPE (poisoned_var)));
    4318              : 
    4319           59 :   tree shadow_var = create_asan_shadow_var (SSA_NAME_VAR (poisoned_var),
    4320              :                                             shadow_vars_mapping);
    4321              : 
    4322           59 :   bool recover_p;
    4323           59 :   if (flag_sanitize & SANITIZE_USER_ADDRESS)
    4324           59 :     recover_p = (flag_sanitize_recover & SANITIZE_USER_ADDRESS) != 0;
    4325              :   else
    4326            0 :     recover_p = (flag_sanitize_recover & SANITIZE_KERNEL_ADDRESS) != 0;
    4327           59 :   tree size = DECL_SIZE_UNIT (shadow_var);
    4328           59 :   gimple *poison_call
    4329           59 :     = gimple_build_call_internal (IFN_ASAN_MARK, 3,
    4330              :                                   build_int_cst (integer_type_node,
    4331              :                                                  ASAN_MARK_POISON),
    4332              :                                   build_fold_addr_expr (shadow_var), size);
    4333              : 
    4334           59 :   gimple *use;
    4335           59 :   imm_use_iterator imm_iter;
    4336          177 :   FOR_EACH_IMM_USE_STMT (use, imm_iter, poisoned_var)
    4337              :     {
    4338          118 :       if (is_gimple_debug (use))
    4339           59 :         continue;
    4340              : 
    4341           59 :       int nargs;
    4342           59 :       bool store_p = gimple_call_internal_p (use, IFN_ASAN_POISON_USE);
    4343           59 :       gcall *call;
    4344           59 :       if (hwassist_sanitize_p ())
    4345              :         {
    4346            0 :           tree fun = builtin_decl_implicit (BUILT_IN_HWASAN_TAG_MISMATCH4);
    4347              :           /* NOTE: hwasan has no __hwasan_report_* functions like asan does.
    4348              :                 We use __hwasan_tag_mismatch4 with arguments that tell it the
    4349              :                 size of access and load to report all tag mismatches.
    4350              : 
    4351              :                 The arguments to this function are:
    4352              :                   Address of invalid access.
    4353              :                   Bitfield containing information about the access
    4354              :                     (access_info)
    4355              :                   Pointer to a frame of registers
    4356              :                     (for use in printing the contents of registers in a dump)
    4357              :                     Not used yet -- to be used by inline instrumentation.
    4358              :                   Size of access.
    4359              : 
    4360              :                 The access_info bitfield encodes the following pieces of
    4361              :                 information:
    4362              :                   - Is this a store or load?
    4363              :                     access_info & 0x10  =>  store
    4364              :                   - Should the program continue after reporting the error?
    4365              :                     access_info & 0x20  =>  recover
    4366              :                   - What size access is this (not used here since we can always
    4367              :                     pass the size in the last argument)
    4368              : 
    4369              :                     if (access_info & 0xf == 0xf)
    4370              :                       size is taken from last argument.
    4371              :                     else
    4372              :                       size == 1 << (access_info & 0xf)
    4373              : 
    4374              :                 The last argument contains the size of the access iff the
    4375              :                 access_info size indicator is 0xf (we always use this argument
    4376              :                 rather than storing the size in the access_info bitfield).
    4377              : 
    4378              :                 See the function definition `__hwasan_tag_mismatch4` in
    4379              :                 libsanitizer/hwasan for the full definition.
    4380              :                 */
    4381            0 :           unsigned access_info = (0x20 * recover_p)
    4382            0 :             + (0x10 * store_p)
    4383            0 :             + (0xf);
    4384            0 :           call = gimple_build_call (fun, 4,
    4385              :                                     build_fold_addr_expr (shadow_var),
    4386              :                                     build_int_cst (pointer_sized_int_node,
    4387            0 :                                                    access_info),
    4388              :                                     build_int_cst (pointer_sized_int_node, 0),
    4389              :                                     size);
    4390              :         }
    4391              :       else
    4392              :         {
    4393           59 :           tree fun = report_error_func (store_p, recover_p, tree_to_uhwi (size),
    4394              :                                         &nargs);
    4395           59 :           tree ptrmode_type
    4396           59 :             = (nargs == 2 ? (*lang_hooks.types.type_for_mode) (ptr_mode, 0)
    4397              :                : NULL_TREE);
    4398           87 :           call = gimple_build_call (fun, nargs,
    4399              :                                     build_fold_addr_expr (shadow_var),
    4400              :                                     nargs == 2
    4401           28 :                                     ? fold_convert (ptrmode_type, size)
    4402              :                                     : NULL_TREE);
    4403              :         }
    4404           59 :       gimple_set_location (call, gimple_location (use));
    4405           59 :       gimple *call_to_insert = call;
    4406              : 
    4407              :       /* The USE can be a gimple PHI node.  If so, insert the call on
    4408              :          all edges leading to the PHI node.  */
    4409           59 :       if (is_a <gphi *> (use))
    4410              :         {
    4411              :           gphi *phi = dyn_cast<gphi *> (use);
    4412           20 :           for (unsigned i = 0; i < gimple_phi_num_args (phi); ++i)
    4413           15 :             if (gimple_phi_arg_def (phi, i) == poisoned_var)
    4414              :               {
    4415            5 :                 edge e = gimple_phi_arg_edge (phi, i);
    4416              : 
    4417              :                 /* Do not insert on an edge we can't split.  */
    4418            5 :                 if (e->flags & EDGE_ABNORMAL)
    4419            5 :                   continue;
    4420              : 
    4421            0 :                 if (call_to_insert == NULL)
    4422            0 :                   call_to_insert = gimple_copy (call);
    4423              : 
    4424            0 :                 gsi_insert_seq_on_edge (e, call_to_insert);
    4425            0 :                 *need_commit_edge_insert = true;
    4426            0 :                 call_to_insert = NULL;
    4427              :               }
    4428              :         }
    4429              :       else
    4430              :         {
    4431           54 :           gimple_stmt_iterator gsi = gsi_for_stmt (use);
    4432           54 :           if (store_p)
    4433           18 :             gsi_replace (&gsi, call, true);
    4434              :           else
    4435           36 :             gsi_insert_before (&gsi, call, GSI_NEW_STMT);
    4436              :         }
    4437           59 :     }
    4438              : 
    4439           59 :   SSA_NAME_IS_DEFAULT_DEF (poisoned_var) = true;
    4440           59 :   SSA_NAME_DEF_STMT (poisoned_var) = gimple_build_nop ();
    4441           59 :   gsi_replace (iter, poison_call, false);
    4442              : 
    4443           59 :   return true;
    4444              : }
    4445              : 
    4446              : /* Instrument the current function.  */
    4447              : 
    4448              : static unsigned int
    4449         6659 : asan_instrument (void)
    4450              : {
    4451         6659 :   if (hwassist_sanitize_p ())
    4452              :     {
    4453          452 :       initialize_sanitizer_builtins ();
    4454          452 :       transform_statements ();
    4455          452 :       return 0;
    4456              :     }
    4457              : 
    4458         6207 :   if (shadow_ptr_types[0] == NULL_TREE)
    4459         2364 :     asan_init_shadow_ptr_types ();
    4460         6207 :   transform_statements ();
    4461         6207 :   last_alloca_addr = NULL_TREE;
    4462         6207 :   return 0;
    4463              : }
    4464              : 
    4465              : static bool
    4466      1518070 : gate_asan (void)
    4467              : {
    4468       452618 :   return sanitize_flags_p (SANITIZE_ADDRESS);
    4469              : }
    4470              : 
    4471              : namespace {
    4472              : 
    4473              : const pass_data pass_data_asan =
    4474              : {
    4475              :   GIMPLE_PASS, /* type */
    4476              :   "asan", /* name */
    4477              :   OPTGROUP_NONE, /* optinfo_flags */
    4478              :   TV_NONE, /* tv_id */
    4479              :   ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
    4480              :   0, /* properties_provided */
    4481              :   0, /* properties_destroyed */
    4482              :   0, /* todo_flags_start */
    4483              :   TODO_update_ssa, /* todo_flags_finish */
    4484              : };
    4485              : 
    4486              : class pass_asan : public gimple_opt_pass
    4487              : {
    4488              : public:
    4489       594360 :   pass_asan (gcc::context *ctxt)
    4490      1188720 :     : gimple_opt_pass (pass_data_asan, ctxt)
    4491              :   {}
    4492              : 
    4493              :   /* opt_pass methods: */
    4494       297180 :   opt_pass * clone () final override { return new pass_asan (m_ctxt); }
    4495      1065452 :   bool gate (function *) final override
    4496              :   {
    4497      1065452 :     return gate_asan () || gate_hwasan () || gate_memtag ();
    4498              :   }
    4499         5246 :   unsigned int execute (function *) final override
    4500              :   {
    4501         5246 :     return asan_instrument ();
    4502              :   }
    4503              : 
    4504              : }; // class pass_asan
    4505              : 
    4506              : } // anon namespace
    4507              : 
    4508              : gimple_opt_pass *
    4509       297180 : make_pass_asan (gcc::context *ctxt)
    4510              : {
    4511       297180 :   return new pass_asan (ctxt);
    4512              : }
    4513              : 
    4514              : namespace {
    4515              : 
    4516              : const pass_data pass_data_asan_O0 =
    4517              : {
    4518              :   GIMPLE_PASS, /* type */
    4519              :   "asan0", /* name */
    4520              :   OPTGROUP_NONE, /* optinfo_flags */
    4521              :   TV_NONE, /* tv_id */
    4522              :   ( PROP_ssa | PROP_cfg | PROP_gimple_leh ), /* properties_required */
    4523              :   0, /* properties_provided */
    4524              :   0, /* properties_destroyed */
    4525              :   0, /* todo_flags_start */
    4526              :   TODO_update_ssa, /* todo_flags_finish */
    4527              : };
    4528              : 
    4529              : class pass_asan_O0 : public gimple_opt_pass
    4530              : {
    4531              : public:
    4532       297180 :   pass_asan_O0 (gcc::context *ctxt)
    4533       594360 :     : gimple_opt_pass (pass_data_asan_O0, ctxt)
    4534              :   {}
    4535              : 
    4536              :   /* opt_pass methods: */
    4537      1517917 :   bool gate (function *) final override
    4538              :     {
    4539      1970535 :       return !optimize && (gate_asan () || gate_hwasan () || gate_memtag ());
    4540              :     }
    4541         1413 :   unsigned int execute (function *) final override
    4542              :   {
    4543         1413 :     return asan_instrument ();
    4544              :   }
    4545              : 
    4546              : }; // class pass_asan_O0
    4547              : 
    4548              : } // anon namespace
    4549              : 
    4550              : gimple_opt_pass *
    4551       297180 : make_pass_asan_O0 (gcc::context *ctxt)
    4552              : {
    4553       297180 :   return new pass_asan_O0 (ctxt);
    4554              : }
    4555              : 
    4556              : /*  HWASAN  */
    4557              : 
    4558              : /* For stack tagging:
    4559              : 
    4560              :    Return the offset from the frame base tag that the "next" expanded object
    4561              :    should have.  */
    4562              : uint8_t
    4563          172 : hwasan_current_frame_tag ()
    4564              : {
    4565          172 :   return hwasan_frame_tag_offset;
    4566              : }
    4567              : 
    4568              : /* For stack tagging:
    4569              : 
    4570              :    Return the 'base pointer' for this function.  If that base pointer has not
    4571              :    yet been created then we create a register to hold it and record the insns
    4572              :    to initialize the register in `hwasan_frame_base_init_seq` for later
    4573              :    emission.  */
    4574              : rtx
    4575           86 : hwasan_frame_base ()
    4576              : {
    4577           86 :   if (! hwasan_frame_base_ptr)
    4578              :     {
    4579           60 :       start_sequence ();
    4580           60 :       hwasan_frame_base_ptr
    4581           60 :         = force_reg (Pmode,
    4582           60 :                      targetm.memtag.insert_random_tag (virtual_stack_vars_rtx,
    4583              :                                                        NULL_RTX));
    4584           60 :       hwasan_frame_base_init_seq = end_sequence ();
    4585              :     }
    4586              : 
    4587           86 :   return hwasan_frame_base_ptr;
    4588              : }
    4589              : 
    4590              : /* For stack tagging:
    4591              : 
    4592              :    Check whether this RTX is a standard pointer addressing the base of the
    4593              :    stack variables for this frame.  Returns true if the RTX is either
    4594              :    virtual_stack_vars_rtx or hwasan_frame_base_ptr.  */
    4595              : bool
    4596      2036615 : stack_vars_base_reg_p (rtx base)
    4597              : {
    4598      2036615 :   return base == virtual_stack_vars_rtx || base == hwasan_frame_base_ptr;
    4599              : }
    4600              : 
    4601              : /* For stack tagging:
    4602              : 
    4603              :    Emit frame base initialisation.
    4604              :    If hwasan_frame_base has been used before here then
    4605              :    hwasan_frame_base_init_seq contains the sequence of instructions to
    4606              :    initialize it.  This must be put just before the hwasan prologue, so we emit
    4607              :    the insns before parm_birth_insn (which will point to the first instruction
    4608              :    of the hwasan prologue if it exists).
    4609              : 
    4610              :    We update `parm_birth_insn` to point to the start of this initialisation
    4611              :    since that represents the end of the initialisation done by
    4612              :    expand_function_{start,end} functions and we want to maintain that.  */
    4613              : void
    4614          368 : hwasan_maybe_emit_frame_base_init ()
    4615              : {
    4616          368 :   if (! hwasan_frame_base_init_seq)
    4617              :     return;
    4618           16 :   emit_insn_before (hwasan_frame_base_init_seq, parm_birth_insn);
    4619           16 :   parm_birth_insn = hwasan_frame_base_init_seq;
    4620              : }
    4621              : 
    4622              : /* Record a compile-time constant size stack variable that HWASAN will need to
    4623              :    tag.  This record of the range of a stack variable will be used by
    4624              :    `hwasan_emit_prologue` to emit the RTL at the start of each frame which will
    4625              :    set tags in the shadow memory according to the assigned tag for each object.
    4626              : 
    4627              :    The range that the object spans in stack space should be described by the
    4628              :    bounds `untagged_base + nearest_offset` and
    4629              :    `untagged_base + farthest_offset`.
    4630              :    `tagged_base` is the base address which contains the "base frame tag" for
    4631              :    this frame, and from which the value to address this object with will be
    4632              :    calculated.
    4633              : 
    4634              :    We record the `untagged_base` since the functions in the hwasan library we
    4635              :    use to tag memory take pointers without a tag.  */
    4636              : void
    4637           86 : hwasan_record_stack_var (rtx untagged_base, rtx tagged_base,
    4638              :                          poly_int64 nearest_offset, poly_int64 farthest_offset)
    4639              : {
    4640           86 :   hwasan_stack_var cur_var;
    4641           86 :   cur_var.untagged_base = untagged_base;
    4642           86 :   cur_var.tagged_base = tagged_base;
    4643           86 :   cur_var.nearest_offset = nearest_offset;
    4644           86 :   cur_var.farthest_offset = farthest_offset;
    4645           86 :   cur_var.tag_offset = hwasan_current_frame_tag ();
    4646              : 
    4647           86 :   hwasan_tagged_stack_vars.safe_push (cur_var);
    4648           86 : }
    4649              : 
    4650              : /* Return the RTX representing the farthest extent of the statically allocated
    4651              :    stack objects for this frame.  If hwasan_frame_base_ptr has not been
    4652              :    initialized then we are not storing any static variables on the stack in
    4653              :    this frame.  In this case we return NULL_RTX to represent that.
    4654              : 
    4655              :    Otherwise simply return virtual_stack_vars_rtx + frame_offset.  */
    4656              : rtx
    4657          368 : hwasan_get_frame_extent ()
    4658              : {
    4659          368 :   return (hwasan_frame_base_ptr
    4660          368 :           ? plus_constant (Pmode, virtual_stack_vars_rtx, frame_offset)
    4661          368 :           : NULL_RTX);
    4662              : }
    4663              : 
    4664              : /* For stack tagging:
    4665              : 
    4666              :    Increment the frame tag offset modulo the size a tag can represent.  */
    4667              : void
    4668           86 : hwasan_increment_frame_tag ()
    4669              : {
    4670           86 :   uint8_t tag_bits = HWASAN_TAG_SIZE;
    4671           86 :   gcc_assert (HWASAN_TAG_SIZE
    4672              :               <= sizeof (hwasan_frame_tag_offset) * CHAR_BIT);
    4673           86 :   hwasan_frame_tag_offset = (hwasan_frame_tag_offset + 1) % (1 << tag_bits);
    4674              :   /* The "background tag" of the stack is zero by definition.
    4675              :      This is the tag that objects like parameters passed on the stack and
    4676              :      spilled registers are given.  It is handy to avoid this tag for objects
    4677              :      whose tags we decide ourselves, partly to ensure that buffer overruns
    4678              :      can't affect these important variables (e.g. saved link register, saved
    4679              :      stack pointer etc) and partly to make debugging easier (everything with a
    4680              :      tag of zero is space allocated automatically by the compiler).
    4681              : 
    4682              :      This is not feasible when using random frame tags (the default
    4683              :      configuration for hwasan) since the tag for the given frame is randomly
    4684              :      chosen at runtime.  In order to avoid any tags matching the stack
    4685              :      background we would need to decide tag offsets at runtime instead of
    4686              :      compile time (and pay the resulting performance cost).
    4687              : 
    4688              :      When not using random base tags for each frame (i.e. when compiled with
    4689              :      `--param hwasan-random-frame-tag=0`) the base tag for each frame is zero.
    4690              :      This means the tag that each object gets is equal to the
    4691              :      hwasan_frame_tag_offset used in determining it.
    4692              :      When this is the case we *can* ensure no object gets the tag of zero by
    4693              :      simply ensuring no object has the hwasan_frame_tag_offset of zero.
    4694              : 
    4695              :      There is the extra complication that we only record the
    4696              :      hwasan_frame_tag_offset here (which is the offset from the tag stored in
    4697              :      the stack pointer).  In the kernel, the tag in the stack pointer is 0xff
    4698              :      rather than zero.  This does not cause problems since tags of 0xff are
    4699              :      never checked in the kernel.  As mentioned at the beginning of this
    4700              :      comment the background tag of the stack is zero by definition, which means
    4701              :      that for the kernel we should skip offsets of both 0 and 1 from the stack
    4702              :      pointer.  Avoiding the offset of 0 ensures we use a tag which will be
    4703              :      checked, avoiding the offset of 1 ensures we use a tag that is not the
    4704              :      same as the background.  */
    4705           86 :   if (hwasan_frame_tag_offset == 0 && ! param_hwasan_random_frame_tag)
    4706            0 :     hwasan_frame_tag_offset += 1;
    4707           16 :   if (hwasan_frame_tag_offset == 1 && ! param_hwasan_random_frame_tag
    4708           86 :       && sanitize_flags_p (SANITIZE_KERNEL_HWADDRESS))
    4709            0 :     hwasan_frame_tag_offset += 1;
    4710           86 : }
    4711              : 
    4712              : /* Clear internal state for the next function.
    4713              :    This function is called before variables on the stack get expanded, in
    4714              :    `init_vars_expansion`.  */
    4715              : void
    4716         1112 : hwasan_record_frame_init ()
    4717              : {
    4718         1112 :   delete asan_used_labels;
    4719         1112 :   asan_used_labels = NULL;
    4720              : 
    4721              :   /* If this isn't the case then some stack variable was recorded *before*
    4722              :      hwasan_record_frame_init is called, yet *after* the hwasan prologue for
    4723              :      the previous frame was emitted.  Such stack variables would not have
    4724              :      their shadow stack filled in.  */
    4725         1112 :   gcc_assert (hwasan_tagged_stack_vars.is_empty ());
    4726         1112 :   hwasan_frame_base_ptr = NULL_RTX;
    4727         1112 :   hwasan_frame_base_init_seq = NULL;
    4728              : 
    4729              :   /* When not using a random frame tag we can avoid the background stack
    4730              :      color which gives the user a little better debug output upon a crash.
    4731              :      Meanwhile, when using a random frame tag it will be nice to avoid adding
    4732              :      tags for the first object since that is unnecessary extra work.
    4733              :      Hence set the initial hwasan_frame_tag_offset to be 0 if using a random
    4734              :      frame tag and 1 otherwise.
    4735              : 
    4736              :      As described in hwasan_increment_frame_tag, in the kernel the stack
    4737              :      pointer has the tag 0xff.  That means that to avoid 0xff and 0 (the tag
    4738              :      which the kernel does not check and the background tag respectively) we
    4739              :      start with a tag offset of 2.  */
    4740         2074 :   hwasan_frame_tag_offset = param_hwasan_random_frame_tag
    4741              :     ? 0
    4742          962 :     : sanitize_flags_p (SANITIZE_KERNEL_HWADDRESS) ? 2 : 1;
    4743         1112 : }
    4744              : 
    4745              : /* For stack tagging:
    4746              :    (Emits HWASAN equivalent of what is emitted by
    4747              :    `asan_emit_stack_protection`).
    4748              : 
    4749              :    Emits the extra prologue code to set the shadow stack as required for HWASAN
    4750              :    stack instrumentation.
    4751              : 
    4752              :    Uses the vector of recorded stack variables hwasan_tagged_stack_vars.  When
    4753              :    this function has completed hwasan_tagged_stack_vars is empty and all
    4754              :    objects it had pointed to are deallocated.  */
    4755              : void
    4756          368 : hwasan_emit_prologue ()
    4757              : {
    4758              :   /* We need untagged base pointers since libhwasan only accepts untagged
    4759              :     pointers in __hwasan_tag_memory.  We need the tagged base pointer to obtain
    4760              :     the base tag for an offset.  */
    4761              : 
    4762          368 :   if (hwasan_tagged_stack_vars.is_empty ())
    4763          368 :     return;
    4764              : 
    4765           60 :   poly_int64 bot = 0, top = 0;
    4766          146 :   for (hwasan_stack_var &cur : hwasan_tagged_stack_vars)
    4767              :     {
    4768           86 :       poly_int64 nearest = cur.nearest_offset;
    4769           86 :       poly_int64 farthest = cur.farthest_offset;
    4770              : 
    4771           86 :       if (known_ge (nearest, farthest))
    4772              :         {
    4773              :           top = nearest;
    4774              :           bot = farthest;
    4775              :         }
    4776              :       else
    4777              :         {
    4778              :           /* Given how these values are calculated, one must be known greater
    4779              :              than the other.  */
    4780            0 :           gcc_assert (known_le (nearest, farthest));
    4781            0 :           top = farthest;
    4782            0 :           bot = nearest;
    4783              :         }
    4784           86 :       poly_int64 size = (top - bot);
    4785              : 
    4786              :       /* Assert the edge of each variable is aligned to the HWASAN tag granule
    4787              :          size.  */
    4788          172 :       gcc_assert (multiple_p (top, HWASAN_TAG_GRANULE_SIZE));
    4789          172 :       gcc_assert (multiple_p (bot, HWASAN_TAG_GRANULE_SIZE));
    4790          172 :       gcc_assert (multiple_p (size, HWASAN_TAG_GRANULE_SIZE));
    4791              : 
    4792           86 :       rtx base_tag = targetm.memtag.extract_tag (cur.tagged_base, NULL_RTX);
    4793              : 
    4794           86 :       rtx bottom = convert_memory_address (ptr_mode,
    4795              :                                            plus_constant (Pmode,
    4796              :                                                           cur.untagged_base,
    4797              :                                                           bot));
    4798           86 :       if (memtag_sanitize_p ())
    4799              :         {
    4800            0 :           expand_operand ops[3];
    4801            0 :           rtx tagged_addr = gen_reg_rtx (ptr_mode);
    4802              : 
    4803              :           /* Check if the required target instructions are present.  */
    4804            0 :           gcc_assert (targetm.have_compose_tag ());
    4805            0 :           gcc_assert (targetm.have_tag_memory ());
    4806              : 
    4807              :           /* The AArch64 has addg/subg instructions which are working directly
    4808              :              on a tagged pointer.  */
    4809            0 :           create_output_operand (&ops[0], tagged_addr, ptr_mode);
    4810            0 :           create_input_operand (&ops[1], base_tag, ptr_mode);
    4811            0 :           create_integer_operand (&ops[2], cur.tag_offset);
    4812            0 :           expand_insn (targetm.code_for_compose_tag, 3, ops);
    4813              : 
    4814            0 :           emit_insn (targetm.gen_tag_memory (bottom, tagged_addr,
    4815              :                                              gen_int_mode (size, ptr_mode)));
    4816              :         }
    4817              :       else
    4818              :         {
    4819           86 :           rtx fn = init_one_libfunc ("__hwasan_tag_memory");
    4820           86 :           rtx tag = plus_constant (QImode, base_tag, cur.tag_offset);
    4821           86 :           tag = hwasan_truncate_to_tag_size (tag, NULL_RTX);
    4822           86 :           emit_library_call (fn, LCT_NORMAL, VOIDmode,
    4823              :                              bottom, ptr_mode,
    4824              :                              tag, QImode,
    4825              :                              gen_int_mode (size, ptr_mode), ptr_mode);
    4826              :         }
    4827              :     }
    4828              :   /* Clear the stack vars, we've emitted the prologue for them all now.  */
    4829           60 :   hwasan_tagged_stack_vars.truncate (0);
    4830              : }
    4831              : 
    4832              : /* For stack tagging:
    4833              : 
    4834              :    Return RTL insns to clear the tags between DYNAMIC and VARS pointers
    4835              :    into the stack.  These instructions should be emitted at the end of
    4836              :    every function.
    4837              : 
    4838              :    If `dynamic` is NULL_RTX then no insns are returned.  */
    4839              : rtx_insn *
    4840          368 : hwasan_emit_untag_frame (rtx dynamic, rtx vars)
    4841              : {
    4842          368 :   if (! dynamic)
    4843              :     return NULL;
    4844              : 
    4845           60 :   start_sequence ();
    4846              : 
    4847           60 :   dynamic = convert_memory_address (ptr_mode, dynamic);
    4848           60 :   vars = convert_memory_address (ptr_mode, vars);
    4849              : 
    4850           60 :   rtx top_rtx;
    4851           60 :   rtx bot_rtx;
    4852           60 :   if (FRAME_GROWS_DOWNWARD)
    4853              :     {
    4854           60 :       top_rtx = vars;
    4855           60 :       bot_rtx = dynamic;
    4856              :     }
    4857              :   else
    4858              :     {
    4859              :       top_rtx = dynamic;
    4860              :       bot_rtx = vars;
    4861              :     }
    4862              : 
    4863           60 :   rtx size_rtx = simplify_gen_binary (MINUS, ptr_mode, top_rtx, bot_rtx);
    4864           60 :   if (!CONST_INT_P (size_rtx))
    4865            0 :     size_rtx = force_reg (ptr_mode, size_rtx);
    4866              : 
    4867           60 :   if (memtag_sanitize_p ())
    4868            0 :     emit_insn (targetm.gen_tag_memory (bot_rtx, HWASAN_STACK_BACKGROUND,
    4869              :                                        size_rtx));
    4870              :   else
    4871              :     {
    4872           60 :       rtx fn = init_one_libfunc ("__hwasan_tag_memory");
    4873           60 :       emit_library_call (fn, LCT_NORMAL, VOIDmode,
    4874              :                          bot_rtx, ptr_mode,
    4875              :                          HWASAN_STACK_BACKGROUND, QImode,
    4876              :                          size_rtx, ptr_mode);
    4877              :     }
    4878              : 
    4879           60 :   do_pending_stack_adjust ();
    4880           60 :   return end_sequence ();
    4881              : }
    4882              : 
    4883              : /* Needs to be GTY(()), because cgraph_build_static_cdtor may
    4884              :    invoke ggc_collect.  */
    4885              : static GTY(()) tree hwasan_ctor_statements;
    4886              : 
    4887              : /* Insert module initialization into this TU.  This initialization calls the
    4888              :    initialization code for libhwasan.  */
    4889              : void
    4890          254 : hwasan_finish_file (void)
    4891              : {
    4892              :   /* Do not emit constructor initialization for the kernel.
    4893              :      (the kernel has its own initialization already).  */
    4894          254 :   if (flag_sanitize & SANITIZE_KERNEL_HWADDRESS)
    4895              :     return;
    4896              : 
    4897          240 :   initialize_sanitizer_builtins ();
    4898              : 
    4899              :   /* Avoid instrumenting code in the hwasan constructors/destructors.  */
    4900          240 :   flag_sanitize &= ~SANITIZE_HWADDRESS;
    4901          240 :   int priority = MAX_RESERVED_INIT_PRIORITY - 1;
    4902          240 :   tree fn = builtin_decl_implicit (BUILT_IN_HWASAN_INIT);
    4903          240 :   append_to_statement_list (build_call_expr (fn, 0), &hwasan_ctor_statements);
    4904          240 :   cgraph_build_static_cdtor ('I', hwasan_ctor_statements, priority);
    4905          240 :   flag_sanitize |= SANITIZE_HWADDRESS;
    4906              : }
    4907              : 
    4908              : /* For stack tagging:
    4909              : 
    4910              :    Truncate `tag` to the number of bits that a tag uses (i.e. to
    4911              :    HWASAN_TAG_SIZE).  Store the result in `target` if it's convenient.  */
    4912              : rtx
    4913           86 : hwasan_truncate_to_tag_size (rtx tag, rtx target)
    4914              : {
    4915           86 :   gcc_assert (GET_MODE (tag) == QImode);
    4916           86 :   if (HWASAN_TAG_SIZE != GET_MODE_PRECISION (QImode))
    4917              :     {
    4918           86 :       gcc_assert (GET_MODE_PRECISION (QImode) > HWASAN_TAG_SIZE);
    4919           86 :       rtx mask = gen_int_mode ((HOST_WIDE_INT_1U << HWASAN_TAG_SIZE) - 1,
    4920              :                                QImode);
    4921           86 :       tag = expand_simple_binop (QImode, AND, tag, mask, target,
    4922              :                                  /* unsignedp = */1, OPTAB_WIDEN);
    4923           86 :       gcc_assert (tag);
    4924              :     }
    4925           86 :   return tag;
    4926              : }
    4927              : 
    4928              : /* Construct a function tree for __hwasan_{load,store}{1,2,4,8,16,_n}.
    4929              :    IS_STORE is either 1 (for a store) or 0 (for a load).  */
    4930              : static combined_fn
    4931          372 : hwasan_check_func (bool is_store, bool recover_p, HOST_WIDE_INT size_in_bytes,
    4932              :                    int *nargs)
    4933              : {
    4934          372 :   static enum built_in_function check[2][2][6]
    4935              :     = { { { BUILT_IN_HWASAN_LOAD1, BUILT_IN_HWASAN_LOAD2,
    4936              :             BUILT_IN_HWASAN_LOAD4, BUILT_IN_HWASAN_LOAD8,
    4937              :             BUILT_IN_HWASAN_LOAD16, BUILT_IN_HWASAN_LOADN },
    4938              :           { BUILT_IN_HWASAN_STORE1, BUILT_IN_HWASAN_STORE2,
    4939              :             BUILT_IN_HWASAN_STORE4, BUILT_IN_HWASAN_STORE8,
    4940              :             BUILT_IN_HWASAN_STORE16, BUILT_IN_HWASAN_STOREN } },
    4941              :         { { BUILT_IN_HWASAN_LOAD1_NOABORT,
    4942              :             BUILT_IN_HWASAN_LOAD2_NOABORT,
    4943              :             BUILT_IN_HWASAN_LOAD4_NOABORT,
    4944              :             BUILT_IN_HWASAN_LOAD8_NOABORT,
    4945              :             BUILT_IN_HWASAN_LOAD16_NOABORT,
    4946              :             BUILT_IN_HWASAN_LOADN_NOABORT },
    4947              :           { BUILT_IN_HWASAN_STORE1_NOABORT,
    4948              :             BUILT_IN_HWASAN_STORE2_NOABORT,
    4949              :             BUILT_IN_HWASAN_STORE4_NOABORT,
    4950              :             BUILT_IN_HWASAN_STORE8_NOABORT,
    4951              :             BUILT_IN_HWASAN_STORE16_NOABORT,
    4952              :             BUILT_IN_HWASAN_STOREN_NOABORT } } };
    4953          372 :   if (size_in_bytes == -1)
    4954              :     {
    4955            0 :       *nargs = 2;
    4956            0 :       return as_combined_fn (check[recover_p][is_store][5]);
    4957              :     }
    4958          372 :   *nargs = 1;
    4959          372 :   int size_log2 = exact_log2 (size_in_bytes);
    4960          372 :   gcc_assert (size_log2 >= 0 && size_log2 <= 5);
    4961          372 :   return as_combined_fn (check[recover_p][is_store][size_log2]);
    4962              : }
    4963              : 
    4964              : /* Expand the HWASAN_{LOAD,STORE} builtins.  */
    4965              : bool
    4966          372 : hwasan_expand_check_ifn (gimple_stmt_iterator *iter, bool)
    4967              : {
    4968          372 :   gimple *g = gsi_stmt (*iter);
    4969          372 :   location_t loc = gimple_location (g);
    4970          372 :   bool recover_p;
    4971          372 :   if (flag_sanitize & SANITIZE_USER_HWADDRESS)
    4972          312 :     recover_p = (flag_sanitize_recover & SANITIZE_USER_HWADDRESS) != 0;
    4973              :   else
    4974           60 :     recover_p = (flag_sanitize_recover & SANITIZE_KERNEL_HWADDRESS) != 0;
    4975              : 
    4976          372 :   HOST_WIDE_INT flags = tree_to_shwi (gimple_call_arg (g, 0));
    4977          372 :   gcc_assert (flags < ASAN_CHECK_LAST);
    4978          372 :   bool is_scalar_access = (flags & ASAN_CHECK_SCALAR_ACCESS) != 0;
    4979          372 :   bool is_store = (flags & ASAN_CHECK_STORE) != 0;
    4980          372 :   bool is_non_zero_len = (flags & ASAN_CHECK_NON_ZERO_LEN) != 0;
    4981              : 
    4982          372 :   tree base = gimple_call_arg (g, 1);
    4983          372 :   tree len = gimple_call_arg (g, 2);
    4984              : 
    4985              :   /* `align` is unused for HWASAN_CHECK, but we pass the argument anyway
    4986              :      since that way the arguments match ASAN_CHECK.  */
    4987              :   /* HOST_WIDE_INT align = tree_to_shwi (gimple_call_arg (g, 3));  */
    4988              : 
    4989         1116 :   unsigned HOST_WIDE_INT size_in_bytes
    4990          372 :     = is_scalar_access ? tree_to_shwi (len) : -1;
    4991              : 
    4992          372 :   gimple_stmt_iterator gsi = *iter;
    4993              : 
    4994          372 :   if (!is_non_zero_len)
    4995              :     {
    4996              :       /* So, the length of the memory area to hwasan-protect is
    4997              :          non-constant.  Let's guard the generated instrumentation code
    4998              :          like:
    4999              : 
    5000              :          if (len != 0)
    5001              :            {
    5002              :              // hwasan instrumentation code goes here.
    5003              :            }
    5004              :          // fallthrough instructions, starting with *ITER.  */
    5005              : 
    5006            0 :       g = gimple_build_cond (NE_EXPR,
    5007              :                             len,
    5008            0 :                             build_int_cst (TREE_TYPE (len), 0),
    5009              :                             NULL_TREE, NULL_TREE);
    5010            0 :       gimple_set_location (g, loc);
    5011              : 
    5012            0 :       basic_block then_bb, fallthrough_bb;
    5013            0 :       insert_if_then_before_iter (as_a <gcond *> (g), iter,
    5014              :                                   /*then_more_likely_p=*/true,
    5015              :                                   &then_bb, &fallthrough_bb);
    5016              :       /* Note that fallthrough_bb starts with the statement that was
    5017              :         pointed to by ITER.  */
    5018              : 
    5019              :       /* The 'then block' of the 'if (len != 0) condition is where
    5020              :         we'll generate the hwasan instrumentation code now.  */
    5021            0 :       gsi = gsi_last_bb (then_bb);
    5022              :     }
    5023              : 
    5024          372 :   gimple_seq stmts = NULL;
    5025          372 :   tree base_addr = gimple_build (&stmts, loc, NOP_EXPR,
    5026              :                                  pointer_sized_int_node, base);
    5027              : 
    5028          372 :   int nargs = 0;
    5029          372 :   combined_fn fn
    5030          372 :     = hwasan_check_func (is_store, recover_p, size_in_bytes, &nargs);
    5031          372 :   if (nargs == 1)
    5032          372 :     gimple_build (&stmts, loc, fn, void_type_node, base_addr);
    5033              :   else
    5034              :     {
    5035            0 :       gcc_assert (nargs == 2);
    5036            0 :       tree sz_arg = gimple_build (&stmts, loc, NOP_EXPR,
    5037              :                                   pointer_sized_int_node, len);
    5038            0 :       gimple_build (&stmts, loc, fn, void_type_node, base_addr, sz_arg);
    5039              :     }
    5040              : 
    5041          372 :   gsi_insert_seq_after (&gsi, stmts, GSI_NEW_STMT);
    5042          372 :   gsi_remove (iter, true);
    5043          372 :   *iter = gsi;
    5044          372 :   return false;
    5045              : }
    5046              : 
    5047              : /* For stack tagging:
    5048              : 
    5049              :    Dummy: the HWASAN_MARK internal function should only ever be in the code
    5050              :    after the sanopt pass.  */
    5051              : bool
    5052            0 : hwasan_expand_mark_ifn (gimple_stmt_iterator *)
    5053              : {
    5054            0 :   gcc_unreachable ();
    5055              : }
    5056              : 
    5057              : bool
    5058      1751308 : gate_hwasan ()
    5059              : {
    5060      1751308 :   return hwasan_sanitize_p ();
    5061              : }
    5062              : 
    5063              : bool
    5064      1511411 : gate_memtag ()
    5065              : {
    5066      1511411 :   return memtag_sanitize_p ();
    5067              : }
    5068              : 
    5069              : #include "gt-asan.h"
        

Generated by: LCOV version 2.5.0-beta

LCOV profile is generated on x86_64 machine using following configure options: configure --disable-bootstrap --enable-coverage=opt --enable-languages=c,c++,fortran,go,jit,lto,rust,m2 --enable-host-shared. GCC test suite is run with the built compiler.