Line data Source code
1 : /* A pass for lowering trees to RTL.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify
7 : it under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful,
12 : but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : GNU General Public License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "tree-pass.h"
30 : #include "memmodel.h"
31 : #include "tm_p.h"
32 : #include "ssa.h"
33 : #include "optabs.h"
34 : #include "regs.h" /* For reg_renumber. */
35 : #include "emit-rtl.h"
36 : #include "recog.h"
37 : #include "cgraph.h"
38 : #include "diagnostic.h"
39 : #include "fold-const.h"
40 : #include "varasm.h"
41 : #include "stor-layout.h"
42 : #include "stmt.h"
43 : #include "print-tree.h"
44 : #include "cfgrtl.h"
45 : #include "cfganal.h"
46 : #include "cfgbuild.h"
47 : #include "cfgcleanup.h"
48 : #include "dojump.h"
49 : #include "explow.h"
50 : #include "calls.h"
51 : #include "expr.h"
52 : #include "internal-fn.h"
53 : #include "tree-eh.h"
54 : #include "gimple-iterator.h"
55 : #include "gimple-expr.h"
56 : #include "gimple-walk.h"
57 : #include "tree-cfg.h"
58 : #include "tree-dfa.h"
59 : #include "tree-ssa.h"
60 : #include "except.h"
61 : #include "gimple-pretty-print.h"
62 : #include "toplev.h"
63 : #include "debug.h"
64 : #include "tree-inline.h"
65 : #include "value-prof.h"
66 : #include "tree-ssa-live.h"
67 : #include "tree-outof-ssa.h"
68 : #include "cfgloop.h"
69 : #include "insn-attr.h" /* For INSN_SCHEDULING. */
70 : #include "stringpool.h"
71 : #include "attribs.h"
72 : #include "asan.h"
73 : #include "tree-ssa-address.h"
74 : #include "output.h"
75 : #include "builtins.h"
76 : #include "opts.h"
77 : #include "gimple-range.h"
78 : #include "rtl-iter.h"
79 :
80 : /* Some systems use __main in a way incompatible with its use in gcc, in these
81 : cases use the macros NAME__MAIN to give a quoted symbol and SYMBOL__MAIN to
82 : give the same symbol without quotes for an alternative entry point. You
83 : must define both, or neither. */
84 : #ifndef NAME__MAIN
85 : #define NAME__MAIN "__main"
86 : #endif
87 :
88 : /* This variable holds information helping the rewriting of SSA trees
89 : into RTL. */
90 : struct ssaexpand SA;
91 :
92 : /* This variable holds the currently expanded gimple statement for purposes
93 : of communicating the profile info to the builtin expanders. */
94 : gimple *currently_expanding_gimple_stmt;
95 :
96 : static rtx expand_debug_expr (tree);
97 :
98 : static bool defer_stack_allocation (tree, bool);
99 :
100 : static void record_alignment_for_reg_var (unsigned int);
101 :
102 : /* Return an expression tree corresponding to the RHS of GIMPLE
103 : statement STMT. */
104 :
105 : tree
106 3526116 : gimple_assign_rhs_to_tree (gimple *stmt)
107 : {
108 3526116 : tree t;
109 3526116 : switch (gimple_assign_rhs_class (stmt))
110 : {
111 3809 : case GIMPLE_TERNARY_RHS:
112 3809 : t = build3 (gimple_assign_rhs_code (stmt),
113 3809 : TREE_TYPE (gimple_assign_lhs (stmt)),
114 : gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt),
115 : gimple_assign_rhs3 (stmt));
116 3809 : break;
117 1314591 : case GIMPLE_BINARY_RHS:
118 1314591 : t = build2 (gimple_assign_rhs_code (stmt),
119 1314591 : TREE_TYPE (gimple_assign_lhs (stmt)),
120 : gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt));
121 1314591 : break;
122 371756 : case GIMPLE_UNARY_RHS:
123 371756 : t = build1 (gimple_assign_rhs_code (stmt),
124 371756 : TREE_TYPE (gimple_assign_lhs (stmt)),
125 : gimple_assign_rhs1 (stmt));
126 371756 : break;
127 1835960 : case GIMPLE_SINGLE_RHS:
128 1835960 : {
129 1835960 : t = gimple_assign_rhs1 (stmt);
130 : /* Avoid modifying this tree in place below. */
131 3584086 : if ((gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t)
132 1715514 : && gimple_location (stmt) != EXPR_LOCATION (t))
133 2039764 : || (gimple_block (stmt) && currently_expanding_to_rtl
134 37826 : && EXPR_P (t)))
135 1574874 : t = copy_node (t);
136 : break;
137 : }
138 0 : default:
139 0 : gcc_unreachable ();
140 : }
141 :
142 3526116 : if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t))
143 3331823 : SET_EXPR_LOCATION (t, gimple_location (stmt));
144 :
145 3526116 : return t;
146 : }
147 :
148 :
149 : #ifndef STACK_ALIGNMENT_NEEDED
150 : #define STACK_ALIGNMENT_NEEDED 1
151 : #endif
152 :
153 : #define SSAVAR(x) (TREE_CODE (x) == SSA_NAME ? SSA_NAME_VAR (x) : x)
154 :
155 : /* Choose either CUR or NEXT as the leader DECL for a partition.
156 : Prefer ignored decls, to simplify debug dumps and reduce ambiguity
157 : out of the same user variable being in multiple partitions (this is
158 : less likely for compiler-introduced temps). Also used by out-of-SSA
159 : to work out which variable a partition will be given. */
160 :
161 : tree
162 79670955 : expand_leader_merge (tree cur, tree next)
163 : {
164 79670955 : if (cur == NULL || cur == next)
165 : return next;
166 :
167 3868071 : if (DECL_P (cur) && DECL_IGNORED_P (cur))
168 : return cur;
169 :
170 2801470 : if (DECL_P (next) && DECL_IGNORED_P (next))
171 307480 : return next;
172 :
173 : return cur;
174 : }
175 :
176 : /* Associate declaration T with storage space X. If T is no
177 : SSA name this is exactly SET_DECL_RTL, otherwise make the
178 : partition of T associated with X. */
179 : static inline void
180 72479341 : set_rtl (tree t, rtx x)
181 : {
182 72479341 : gcc_checking_assert (!x
183 : || !(TREE_CODE (t) == SSA_NAME || is_gimple_reg (t))
184 : || (use_register_for_decl (t)
185 : ? (REG_P (x)
186 : || (GET_CODE (x) == CONCAT
187 : && (REG_P (XEXP (x, 0))
188 : || SUBREG_P (XEXP (x, 0)))
189 : && (REG_P (XEXP (x, 1))
190 : || SUBREG_P (XEXP (x, 1))))
191 : /* We need to accept PARALLELs for RESUT_DECLs
192 : because of vector types with BLKmode returned
193 : in multiple registers, but they are supposed
194 : to be uncoalesced. */
195 : || (GET_CODE (x) == PARALLEL
196 : && SSAVAR (t)
197 : && TREE_CODE (SSAVAR (t)) == RESULT_DECL
198 : && (GET_MODE (x) == BLKmode
199 : || !flag_tree_coalesce_vars)))
200 : : (MEM_P (x) || x == pc_rtx
201 : || (GET_CODE (x) == CONCAT
202 : && MEM_P (XEXP (x, 0))
203 : && MEM_P (XEXP (x, 1))))));
204 : /* Check that the RTL for SSA_NAMEs and gimple-reg PARM_DECLs and
205 : RESULT_DECLs has the expected mode. For memory, we accept
206 : unpromoted modes, since that's what we're likely to get. For
207 : PARM_DECLs and RESULT_DECLs, we'll have been called by
208 : set_parm_rtl, which will give us the default def, so we don't
209 : have to compute it ourselves. For RESULT_DECLs, we accept mode
210 : mismatches too, as long as we have BLKmode or are not coalescing
211 : across variables, so that we don't reject BLKmode PARALLELs or
212 : unpromoted REGs. */
213 67147045 : gcc_checking_assert (!x || x == pc_rtx || TREE_CODE (t) != SSA_NAME
214 : || (SSAVAR (t)
215 : && TREE_CODE (SSAVAR (t)) == RESULT_DECL
216 : && (promote_ssa_mode (t, NULL) == BLKmode
217 : || !flag_tree_coalesce_vars))
218 : || !use_register_for_decl (t)
219 : || GET_MODE (x) == promote_ssa_mode (t, NULL));
220 :
221 67147045 : if (x)
222 : {
223 67583716 : bool skip = false;
224 67583716 : tree cur = NULL_TREE;
225 : rtx xm = x;
226 :
227 436671 : retry:
228 67583716 : if (MEM_P (xm))
229 4862962 : cur = MEM_EXPR (xm);
230 : else if (REG_P (xm))
231 56448327 : cur = REG_EXPR (xm);
232 : else if (SUBREG_P (xm))
233 : {
234 0 : gcc_assert (subreg_lowpart_p (xm));
235 0 : xm = SUBREG_REG (xm);
236 0 : goto retry;
237 : }
238 : else if (GET_CODE (xm) == CONCAT)
239 : {
240 432620 : xm = XEXP (xm, 0);
241 432620 : goto retry;
242 : }
243 : else if (GET_CODE (xm) == PARALLEL)
244 : {
245 4051 : xm = XVECEXP (xm, 0, 0);
246 4051 : gcc_assert (GET_CODE (xm) == EXPR_LIST);
247 4051 : xm = XEXP (xm, 0);
248 4051 : goto retry;
249 : }
250 5835756 : else if (xm == pc_rtx)
251 : skip = true;
252 : else
253 0 : gcc_unreachable ();
254 :
255 35168334 : tree next
256 61311289 : = skip ? cur : expand_leader_merge (cur, SSAVAR (t) ? SSAVAR (t) : t);
257 :
258 67147045 : if (cur != next)
259 : {
260 28335904 : if (MEM_P (x))
261 4074400 : set_mem_attributes (x,
262 2037200 : next && TREE_CODE (next) == SSA_NAME
263 12617 : ? TREE_TYPE (next)
264 : : next, true);
265 : else
266 26298704 : set_reg_attrs_for_decl_rtl (next, x);
267 : }
268 : }
269 :
270 72479341 : if (TREE_CODE (t) == SSA_NAME)
271 : {
272 58780837 : int part = var_to_partition (SA.map, t);
273 58780837 : if (part != NO_PARTITION)
274 : {
275 58780837 : if (SA.partition_to_pseudo[part])
276 32140800 : gcc_assert (SA.partition_to_pseudo[part] == x);
277 26640037 : else if (x != pc_rtx)
278 26636854 : SA.partition_to_pseudo[part] = x;
279 : }
280 : /* For the benefit of debug information at -O0 (where
281 : vartracking doesn't run) record the place also in the base
282 : DECL. For PARMs and RESULTs, do so only when setting the
283 : default def. */
284 117558491 : if (x && x != pc_rtx && SSA_NAME_VAR (t)
285 76983720 : && (VAR_P (SSA_NAME_VAR (t))
286 7544513 : || SSA_NAME_IS_DEFAULT_DEF (t)))
287 : {
288 18025074 : tree var = SSA_NAME_VAR (t);
289 : /* If we don't yet have something recorded, just record it now. */
290 18025074 : if (!DECL_RTL_SET_P (var))
291 7598875 : SET_DECL_RTL (var, x);
292 : /* If we have it set already to "multiple places" don't
293 : change this. */
294 10426199 : else if (DECL_RTL (var) == pc_rtx)
295 : ;
296 : /* If we have something recorded and it's not the same place
297 : as we want to record now, we have multiple partitions for the
298 : same base variable, with different places. We can't just
299 : randomly chose one, hence we have to say that we don't know.
300 : This only happens with optimization, and there var-tracking
301 : will figure out the right thing. */
302 9278209 : else if (DECL_RTL (var) != x)
303 243637 : SET_DECL_RTL (var, pc_rtx);
304 : }
305 : }
306 : else
307 13698504 : SET_DECL_RTL (t, x);
308 72479341 : }
309 :
310 : /* This structure holds data relevant to one variable that will be
311 : placed in a stack slot. */
312 : class stack_var
313 : {
314 : public:
315 : /* The Variable. */
316 : tree decl;
317 :
318 : /* Initially, the size of the variable. Later, the size of the partition,
319 : if this variable becomes it's partition's representative. */
320 : poly_uint64 size;
321 :
322 : /* The *byte* alignment required for this variable. Or as, with the
323 : size, the alignment for this partition. */
324 : unsigned int alignb;
325 :
326 : /* The partition representative. */
327 : unsigned representative;
328 :
329 : /* The next stack variable in the partition, or EOC. */
330 : unsigned next;
331 :
332 : /* The numbers of conflicting stack variables. */
333 : bitmap conflicts;
334 : };
335 :
336 : #define EOC ((unsigned)-1)
337 :
338 : /* We have an array of such objects while deciding allocation. */
339 : static class stack_var *stack_vars;
340 : static unsigned stack_vars_alloc;
341 : static unsigned stack_vars_num;
342 : static hash_map<tree, unsigned> *decl_to_stack_part;
343 :
344 : #define INVALID_STACK_INDEX ((unsigned)-1)
345 :
346 : /* Conflict bitmaps go on this obstack. This allows us to destroy
347 : all of them in one big sweep. */
348 : static bitmap_obstack stack_var_bitmap_obstack;
349 :
350 : /* An array of indices such that stack_vars[stack_vars_sorted[i]].size
351 : is non-decreasing. */
352 : static unsigned *stack_vars_sorted;
353 :
354 : /* The phase of the stack frame. This is the known misalignment of
355 : virtual_stack_vars_rtx from PREFERRED_STACK_BOUNDARY. That is,
356 : (frame_offset+frame_phase) % PREFERRED_STACK_BOUNDARY == 0. */
357 : static int frame_phase;
358 :
359 : /* Used during expand_used_vars to remember if we saw any decls for
360 : which we'd like to enable stack smashing protection. */
361 : static bool has_protected_decls;
362 :
363 : /* Used during expand_used_vars. Remember if we say a character buffer
364 : smaller than our cutoff threshold. Used for -Wstack-protector. */
365 : static bool has_short_buffer;
366 :
367 : /* Compute the byte alignment to use for DECL. Ignore alignment
368 : we can't do with expected alignment of the stack boundary. */
369 :
370 : static unsigned int
371 6649733 : align_local_variable (tree decl, bool really_expand)
372 : {
373 6649733 : unsigned int align;
374 :
375 6649733 : if (TREE_CODE (decl) == SSA_NAME)
376 : {
377 457923 : tree type = TREE_TYPE (decl);
378 457923 : machine_mode mode = TYPE_MODE (type);
379 :
380 457923 : align = TYPE_ALIGN (type);
381 457923 : if (mode != BLKmode
382 457923 : && align < GET_MODE_ALIGNMENT (mode))
383 460 : align = GET_MODE_ALIGNMENT (mode);
384 : }
385 : else
386 6191810 : align = LOCAL_DECL_ALIGNMENT (decl);
387 :
388 6649733 : if (hwassist_sanitize_stack_p ())
389 268 : align = MAX (align, (unsigned) HWASAN_TAG_GRANULE_SIZE * BITS_PER_UNIT);
390 :
391 6649733 : if (TREE_CODE (decl) != SSA_NAME && really_expand)
392 : /* Don't change DECL_ALIGN when called from estimated_stack_frame_size.
393 : That is done before IPA and could bump alignment based on host
394 : backend even for offloaded code which wants different
395 : LOCAL_DECL_ALIGNMENT. */
396 1570901 : SET_DECL_ALIGN (decl, align);
397 :
398 6649733 : return align / BITS_PER_UNIT;
399 : }
400 :
401 : /* Align given offset BASE with ALIGN. Truncate up if ALIGN_UP is true,
402 : down otherwise. Return truncated BASE value. */
403 :
404 : static inline unsigned HOST_WIDE_INT
405 : align_base (HOST_WIDE_INT base, unsigned HOST_WIDE_INT align, bool align_up)
406 : {
407 : return align_up ? (base + align - 1) & -align : base & -align;
408 : }
409 :
410 : /* Allocate SIZE bytes at byte alignment ALIGN from the stack frame.
411 : Return the frame offset. */
412 :
413 : static poly_int64
414 1757049 : alloc_stack_frame_space (poly_int64 size, unsigned HOST_WIDE_INT align)
415 : {
416 1757049 : poly_int64 offset, new_frame_offset;
417 :
418 1757049 : if (FRAME_GROWS_DOWNWARD)
419 : {
420 1757049 : new_frame_offset
421 1757049 : = aligned_lower_bound (frame_offset - frame_phase - size,
422 1757049 : align) + frame_phase;
423 1757049 : offset = new_frame_offset;
424 : }
425 : else
426 : {
427 : new_frame_offset
428 : = aligned_upper_bound (frame_offset - frame_phase,
429 : align) + frame_phase;
430 : offset = new_frame_offset;
431 : new_frame_offset += size;
432 : }
433 1757049 : frame_offset = new_frame_offset;
434 :
435 1757049 : if (frame_offset_overflow (frame_offset, cfun->decl))
436 0 : frame_offset = offset = 0;
437 :
438 1757049 : return offset;
439 : }
440 :
441 : /* Ensure that the stack is aligned to ALIGN bytes.
442 : Return the new frame offset. */
443 : static poly_int64
444 1897 : align_frame_offset (unsigned HOST_WIDE_INT align)
445 : {
446 0 : return alloc_stack_frame_space (0, align);
447 : }
448 :
449 : /* Accumulate DECL into STACK_VARS. */
450 :
451 : static void
452 5835756 : add_stack_var (tree decl, bool really_expand)
453 : {
454 5835756 : class stack_var *v;
455 :
456 5835756 : if (stack_vars_num >= stack_vars_alloc)
457 : {
458 1358109 : if (stack_vars_alloc)
459 35448 : stack_vars_alloc = stack_vars_alloc * 3 / 2;
460 : else
461 : stack_vars_alloc = 32;
462 1358109 : stack_vars
463 1358109 : = XRESIZEVEC (class stack_var, stack_vars, stack_vars_alloc);
464 : }
465 5835756 : if (!decl_to_stack_part)
466 0 : decl_to_stack_part = new hash_map<tree, unsigned>;
467 :
468 5835756 : v = &stack_vars[stack_vars_num];
469 5835756 : decl_to_stack_part->put (decl, stack_vars_num);
470 :
471 5835756 : v->decl = decl;
472 5835756 : tree size = TREE_CODE (decl) == SSA_NAME
473 5835756 : ? TYPE_SIZE_UNIT (TREE_TYPE (decl))
474 5835756 : : DECL_SIZE_UNIT (decl);
475 5835756 : v->size = tree_to_poly_uint64 (size);
476 : /* Ensure that all variables have size, so that &a != &b for any two
477 : variables that are simultaneously live. */
478 5835756 : if (known_eq (v->size, 0U))
479 24316 : v->size = 1;
480 5835756 : v->alignb = align_local_variable (decl, really_expand);
481 : /* An alignment of zero can mightily confuse us later. */
482 5835756 : gcc_assert (v->alignb != 0);
483 :
484 : /* All variables are initially in their own partition. */
485 5835756 : v->representative = stack_vars_num;
486 5835756 : v->next = EOC;
487 :
488 : /* All variables initially conflict with no other. */
489 5835756 : v->conflicts = NULL;
490 :
491 : /* Ensure that this decl doesn't get put onto the list twice. */
492 5835756 : set_rtl (decl, pc_rtx);
493 :
494 5835756 : stack_vars_num++;
495 5835756 : }
496 :
497 : /* Make the decls associated with luid's X and Y conflict. */
498 :
499 : static void
500 11619912 : add_stack_var_conflict (unsigned x, unsigned y)
501 : {
502 11619912 : class stack_var *a = &stack_vars[x];
503 11619912 : class stack_var *b = &stack_vars[y];
504 11619912 : if (x == y)
505 : return;
506 10539426 : if (!a->conflicts)
507 666500 : a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
508 10539426 : if (!b->conflicts)
509 121731 : b->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
510 10539426 : bitmap_set_bit (a->conflicts, y);
511 10539426 : bitmap_set_bit (b->conflicts, x);
512 : }
513 :
514 : /* Check whether the decls associated with luid's X and Y conflict. */
515 :
516 : static bool
517 11529941 : stack_var_conflict_p (unsigned x, unsigned y)
518 : {
519 11529941 : class stack_var *a = &stack_vars[x];
520 11529941 : class stack_var *b = &stack_vars[y];
521 11529941 : if (x == y)
522 : return false;
523 : /* Partitions containing an SSA name result from gimple registers
524 : with things like unsupported modes. They are top-level and
525 : hence conflict with everything else. */
526 11529941 : if (TREE_CODE (a->decl) == SSA_NAME || TREE_CODE (b->decl) == SSA_NAME)
527 : return true;
528 :
529 11495170 : if (!a->conflicts || !b->conflicts)
530 : return false;
531 11442572 : return bitmap_bit_p (a->conflicts, y);
532 : }
533 :
534 : /* Returns the DECL's index into the stack_vars array.
535 : If the DECL does not exist return INVALID_STACK_INDEX. */
536 : static unsigned
537 47215284 : decl_stack_index (tree decl)
538 : {
539 47215284 : if (!decl)
540 : return INVALID_STACK_INDEX;
541 47215284 : if (!DECL_P (decl))
542 : return INVALID_STACK_INDEX;
543 36237420 : if (DECL_RTL_IF_SET (decl) != pc_rtx)
544 : return INVALID_STACK_INDEX;
545 24407547 : unsigned *v = decl_to_stack_part->get (decl);
546 24407547 : if (!v)
547 : return INVALID_STACK_INDEX;
548 :
549 24407547 : unsigned indx = *v;
550 24407547 : gcc_checking_assert (indx != INVALID_STACK_INDEX);
551 24407547 : gcc_checking_assert (indx < stack_vars_num);
552 : return indx;
553 : }
554 :
555 : /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
556 : enter its partition number into bitmap DATA. */
557 :
558 : static bool
559 29980256 : visit_op (gimple *, tree op, tree, void *data)
560 : {
561 29980256 : bitmap active = (bitmap)data;
562 29980256 : op = get_base_address (op);
563 29980256 : unsigned idx = decl_stack_index (op);
564 29980256 : if (idx != INVALID_STACK_INDEX)
565 14872669 : bitmap_set_bit (active, idx);
566 29980256 : return false;
567 : }
568 :
569 : /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
570 : record conflicts between it and all currently active other partitions
571 : from bitmap DATA. */
572 :
573 : static bool
574 13720256 : visit_conflict (gimple *, tree op, tree, void *data)
575 : {
576 13720256 : bitmap active = (bitmap)data;
577 13720256 : op = get_base_address (op);
578 13720256 : unsigned num = decl_stack_index (op);
579 13720256 : if (num != INVALID_STACK_INDEX
580 13720256 : && bitmap_set_bit (active, num))
581 : {
582 1002737 : bitmap_iterator bi;
583 1002737 : unsigned i;
584 1002737 : gcc_assert (num < stack_vars_num);
585 10871690 : EXECUTE_IF_SET_IN_BITMAP (active, 0, i, bi)
586 9868953 : add_stack_var_conflict (num, i);
587 : }
588 13720256 : return false;
589 : }
590 :
591 : /* A cache for ssa name to address of stack variables.
592 : When taking into account if a ssa name refers to an
593 : address of a stack variable, we need to walk the
594 : expressions backwards to find the addresses. This
595 : cache is there so we don't need to walk the expressions
596 : all the time. */
597 : struct vars_ssa_cache
598 : {
599 : private:
600 : /* Currently an entry is a bitmap of all of the known stack variables
601 : addresses that are referenced by the ssa name.
602 : When the bitmap is the nullptr, then there is no cache.
603 : Currently only empty bitmaps are shared.
604 : The reason for why empty cache is not just a null is so we know the
605 : cache for an entry is filled in. */
606 : struct entry
607 : {
608 : bitmap bmap = nullptr;
609 : };
610 : entry *vars_ssa_caches;
611 : public:
612 :
613 : vars_ssa_cache();
614 : ~vars_ssa_cache();
615 : const_bitmap operator() (tree name);
616 : void dump (FILE *file);
617 :
618 : private:
619 : /* Can't copy. */
620 : vars_ssa_cache(const vars_ssa_cache&) = delete;
621 : vars_ssa_cache(vars_ssa_cache&&) = delete;
622 :
623 : /* The shared empty bitmap. */
624 : bitmap empty;
625 :
626 : /* Unshare the index, currently only need
627 : to unshare if the entry was empty. */
628 1029118 : void unshare(int indx)
629 : {
630 1029118 : if (vars_ssa_caches[indx].bmap == empty)
631 445198 : vars_ssa_caches[indx].bmap = BITMAP_ALLOC (&stack_var_bitmap_obstack);
632 1029118 : }
633 : void create (tree);
634 : bool exists (tree use);
635 : void add_one (tree old_name, unsigned);
636 : bool update (tree old_name, tree use);
637 : };
638 :
639 : /* Constructor of the cache, create the cache array. */
640 148751 : vars_ssa_cache::vars_ssa_cache ()
641 : {
642 33225362 : vars_ssa_caches = new entry[num_ssa_names]{};
643 :
644 : /* Create the shared empty bitmap too. */
645 148751 : empty = BITMAP_ALLOC (&stack_var_bitmap_obstack);
646 148751 : }
647 :
648 : /* Delete the array. The bitmaps will be freed
649 : when stack_var_bitmap_obstack is freed. */
650 148751 : vars_ssa_cache::~vars_ssa_cache ()
651 : {
652 148751 : delete []vars_ssa_caches;
653 148751 : }
654 :
655 : /* Create an empty entry for the USE ssa name. */
656 : void
657 10967232 : vars_ssa_cache::create (tree use)
658 : {
659 10967232 : int num = SSA_NAME_VERSION (use);
660 10967232 : if (vars_ssa_caches[num].bmap)
661 : return;
662 10967232 : vars_ssa_caches[num].bmap = empty;
663 : }
664 :
665 : /* Returns true if the cache for USE exists. */
666 : bool
667 71913663 : vars_ssa_cache::exists (tree use)
668 : {
669 71913663 : int num = SSA_NAME_VERSION (use);
670 71913663 : return vars_ssa_caches[num].bmap != nullptr;
671 : }
672 :
673 : /* Add to USE's bitmap for stack variable IDX. */
674 : void
675 159714 : vars_ssa_cache::add_one (tree use, unsigned idx)
676 : {
677 159714 : gcc_assert (idx != INVALID_STACK_INDEX);
678 159714 : int num = SSA_NAME_VERSION (use);
679 159714 : gcc_assert (vars_ssa_caches[num].bmap);
680 159714 : unshare (num);
681 159714 : bitmap_set_bit (vars_ssa_caches[num].bmap, idx);
682 159714 : }
683 :
684 : /* Update cache of OLD_NAME from the USE's cache. */
685 : bool
686 25325668 : vars_ssa_cache::update (tree old_name, tree use)
687 : {
688 25325668 : if (old_name == use)
689 : return false;
690 16197371 : int num = SSA_NAME_VERSION (use);
691 16197371 : int old_num = SSA_NAME_VERSION (old_name);
692 :
693 : /* If the old name was empty, then there is nothing to be updated. */
694 16197371 : if (vars_ssa_caches[num].bmap == empty)
695 : return false;
696 869404 : unshare (old_num);
697 869404 : return bitmap_ior_into (vars_ssa_caches[old_num].bmap, vars_ssa_caches[num].bmap);
698 : }
699 :
700 : /* Dump out the cache. Note empty and non-filled
701 : in ssa names are not printed out. */
702 : void
703 0 : vars_ssa_cache::dump (FILE *file)
704 : {
705 0 : fprintf (file, "var ssa address cache\n");
706 0 : for (unsigned num = 0; num < num_ssa_names; num++)
707 : {
708 0 : if (!vars_ssa_caches[num].bmap
709 0 : || vars_ssa_caches[num].bmap == empty)
710 0 : continue;
711 0 : fprintf (file, "_%d refers to:\n", num);
712 0 : bitmap_iterator bi;
713 0 : unsigned i;
714 0 : EXECUTE_IF_SET_IN_BITMAP (vars_ssa_caches[num].bmap, 0, i, bi)
715 : {
716 0 : fputc ('\t', file);
717 0 : print_generic_expr (file, stack_vars[i].decl, dump_flags);
718 : }
719 0 : fputc ('\n', file);
720 : }
721 0 : fputc ('\n', file);
722 0 : }
723 :
724 : /* Returns the filled in cache for NAME.
725 : This will fill in the cache if it does not exist already.
726 : Returns an empty for ssa names that can't contain pointers
727 : (only integral types and pointer types will contain pointers). */
728 :
729 : const_bitmap
730 56351774 : vars_ssa_cache::operator() (tree name)
731 : {
732 56351774 : gcc_assert (TREE_CODE (name) == SSA_NAME);
733 :
734 92723970 : if (!POINTER_TYPE_P (TREE_TYPE (name))
735 91549644 : && !ANY_INTEGRAL_TYPE_P (TREE_TYPE (name)))
736 2478115 : return empty;
737 :
738 53873659 : if (exists (name))
739 44788323 : return vars_ssa_caches[SSA_NAME_VERSION (name)].bmap;
740 :
741 9085336 : auto_vec<std::pair<tree,tree>, 4> work_list;
742 9085336 : auto_vec<std::pair<tree,tree>, 4> update_cache_list;
743 :
744 9085336 : work_list.safe_push (std::make_pair (name, name));
745 :
746 33509582 : while (!work_list.is_empty ())
747 : {
748 24424246 : auto item = work_list.pop();
749 24424246 : tree use = item.first;
750 24424246 : tree old_name = item.second;
751 24424246 : if (TREE_CODE (use) == ADDR_EXPR)
752 : {
753 411994 : tree op = TREE_OPERAND (use, 0);
754 411994 : op = get_base_address (op);
755 411994 : unsigned idx = decl_stack_index (op);
756 411994 : if (idx != INVALID_STACK_INDEX)
757 159714 : add_one (old_name, idx);
758 13457014 : continue;
759 411994 : }
760 :
761 24012252 : if (TREE_CODE (use) != SSA_NAME)
762 5893933 : continue;
763 :
764 31554615 : if (!POINTER_TYPE_P (TREE_TYPE (use))
765 31424728 : && !ANY_INTEGRAL_TYPE_P (TREE_TYPE (use)))
766 78315 : continue;
767 :
768 : /* Mark the old ssa name needs to be update from the use. */
769 18040004 : update_cache_list.safe_push (item);
770 :
771 : /* If the cache exists for the use, don't try to recreate it. */
772 18040004 : if (exists (use))
773 : {
774 : /* Update the cache here, this can reduce the number of
775 : times through the update loop below. */
776 7072772 : update (old_name, use);
777 7072772 : continue;
778 : }
779 :
780 : /* Create the cache bitmap for the use and also
781 : so we don't go into an infinite loop for some phi nodes with loops. */
782 10967232 : create (use);
783 :
784 10967232 : gimple *g = SSA_NAME_DEF_STMT (use);
785 :
786 : /* CONSTRUCTOR here is always a vector initialization,
787 : walk each element too. */
788 10967232 : if (gimple_assign_single_p (g)
789 10967232 : && TREE_CODE (gimple_assign_rhs1 (g)) == CONSTRUCTOR)
790 : {
791 : tree ctr = gimple_assign_rhs1 (g);
792 : unsigned i;
793 : tree elm;
794 314399 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (ctr), i, elm)
795 220807 : work_list.safe_push (std::make_pair (elm, use));
796 : }
797 : /* For assignments, walk each operand for possible addresses.
798 : For PHI nodes, walk each argument. */
799 10873640 : else if (gassign *a = dyn_cast <gassign *> (g))
800 : {
801 : /* operand 0 is the lhs. */
802 19421138 : for (unsigned i = 1; i < gimple_num_ops (g); i++)
803 11215824 : work_list.safe_push (std::make_pair (gimple_op (a, i), use));
804 : }
805 12537798 : else if (gphi *p = dyn_cast <gphi *> (g))
806 5472845 : for (unsigned i = 0; i < gimple_phi_num_args (p); ++i)
807 3902279 : work_list.safe_push (std::make_pair (gimple_phi_arg_def (p, i), use));
808 : }
809 :
810 : /* Update the cache. Note a loop is needed as phi nodes could
811 : cause a loop to form. The number of times through this loop
812 : will be small though. */
813 9128241 : bool changed;
814 18256482 : do {
815 9128241 : changed = false;
816 9128241 : unsigned int i;
817 9128241 : std::pair<tree,tree> *e;
818 45594714 : FOR_EACH_VEC_ELT_REVERSE (update_cache_list, i, e)
819 : {
820 18252896 : if (update (e->second, e->first))
821 50236 : changed = true;
822 : }
823 : } while (changed);
824 :
825 9085336 : return vars_ssa_caches[SSA_NAME_VERSION (name)].bmap;
826 9085336 : }
827 :
828 : /* Helper function for add_scope_conflicts_1. For USE on
829 : a stmt, if it is a SSA_NAME and in its defining statement
830 : is known to be based on some ADDR_EXPR, invoke VISIT
831 : on that ADDR_EXPR. */
832 :
833 : static inline void
834 56351774 : add_scope_conflicts_2 (vars_ssa_cache &cache, tree name,
835 : bitmap work, walk_stmt_load_store_addr_fn visit)
836 : {
837 56351774 : gcc_assert (TREE_CODE (name) == SSA_NAME);
838 :
839 : /* Query the cache for the mapping of addresses that are referenced by
840 : ssa name NAME. Querying it will fill in it. */
841 56351774 : bitmap_iterator bi;
842 56351774 : unsigned i;
843 56351774 : const_bitmap bmap = cache (name);
844 : /* Visit each stack variable that is referenced. */
845 58640969 : EXECUTE_IF_SET_IN_BITMAP (bmap, 0, i, bi)
846 2289195 : visit (nullptr, stack_vars[i].decl, nullptr, work);
847 56351774 : }
848 :
849 : /* Helper routine for add_scope_conflicts, calculating the active partitions
850 : at the end of BB, leaving the result in WORK. We're called to generate
851 : conflicts when FOR_CONFLICT is true, otherwise we're just tracking
852 : liveness. */
853 :
854 : static void
855 16456026 : add_scope_conflicts_1 (vars_ssa_cache &cache, basic_block bb, bitmap work, bool for_conflict)
856 : {
857 16456026 : edge e;
858 16456026 : edge_iterator ei;
859 16456026 : gimple_stmt_iterator gsi;
860 16456026 : walk_stmt_load_store_addr_fn visit;
861 16456026 : use_operand_p use_p;
862 16456026 : ssa_op_iter iter;
863 16456026 : bool had_non_clobbers = false;
864 :
865 16456026 : bitmap_clear (work);
866 : /* The copy what was alive out going from the edges. */
867 41554847 : FOR_EACH_EDGE (e, ei, bb->preds)
868 25098821 : bitmap_ior_into (work, (bitmap)e->src->aux);
869 :
870 16456026 : visit = for_conflict ? visit_conflict : visit_op;
871 :
872 : /* Addresses coming into the bb via phis are alive at the entry point. */
873 21579787 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
874 5123761 : add_scope_conflicts_2 (cache, gimple_phi_result (gsi_stmt (gsi)), work, visit_op);
875 :
876 190902258 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
877 : {
878 174446232 : gimple *stmt = gsi_stmt (gsi);
879 :
880 : /* Debug statements are not considered for liveness. */
881 174446232 : if (is_gimple_debug (stmt))
882 110752124 : continue;
883 :
884 : /* If we had `var = {CLOBBER}`, then var is no longer
885 : considered alive after this point but might become
886 : alive later on. */
887 63694108 : if (gimple_clobber_p (stmt))
888 : {
889 3177500 : tree lhs = gimple_assign_lhs (stmt);
890 : /* Handle only plain var clobbers, not partial ones.
891 : Nested functions lowering and C++ front-end inserts clobbers
892 : which are partial clobbers. */
893 3177500 : if (!VAR_P (lhs))
894 74722 : continue;
895 3102778 : unsigned indx = decl_stack_index (lhs);
896 3102778 : if (indx != INVALID_STACK_INDEX)
897 2551667 : bitmap_clear_bit (work, indx);
898 : }
899 : else
900 : {
901 60516608 : if (for_conflict && !had_non_clobbers)
902 : {
903 : /* When we are inheriting live variables from our predecessors
904 : through a CFG merge we might not see an actual mention of
905 : the variables to record the appropriate conflict as defs/uses
906 : might be through indirect stores/loads. For this reason
907 : we have to make sure each live variable conflicts with
908 : each other. When there's just a single predecessor the
909 : set of conflicts is already up-to-date.
910 : We perform this delayed at the first real instruction to
911 : allow clobbers starting this block to remove variables from
912 : the set of live variables. */
913 4971229 : bitmap_iterator bi;
914 4971229 : unsigned i;
915 4971229 : if (EDGE_COUNT (bb->preds) > 1)
916 12613487 : EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
917 : {
918 11161246 : class stack_var *a = &stack_vars[i];
919 11161246 : if (!a->conflicts)
920 255140 : a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
921 11161246 : bitmap_ior_into (a->conflicts, work);
922 : }
923 4971229 : had_non_clobbers = true;
924 : }
925 60516608 : walk_stmt_load_store_addr_ops (stmt, work, visit, visit, visit);
926 111744621 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
927 51228013 : add_scope_conflicts_2 (cache, USE_FROM_PTR (use_p), work, visit);
928 : }
929 : }
930 :
931 : /* When there was no real instruction but there's a CFG merge we need
932 : to add the conflicts now. */
933 16456026 : if (for_conflict && !had_non_clobbers && EDGE_COUNT (bb->preds) > 1)
934 : {
935 128659 : bitmap_iterator bi;
936 128659 : unsigned i;
937 939171 : EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
938 : {
939 810512 : class stack_var *a = &stack_vars[i];
940 810512 : if (!a->conflicts)
941 22558 : a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
942 810512 : bitmap_ior_into (a->conflicts, work);
943 : }
944 : }
945 16456026 : }
946 :
947 : /* Generate stack partition conflicts between all partitions that are
948 : simultaneously live. */
949 :
950 : static void
951 235927 : add_scope_conflicts (void)
952 : {
953 : /* If there is only one variable, there is nothing to be done as
954 : there is only possible partition. */
955 235927 : if (stack_vars_num == 1)
956 87176 : return;
957 :
958 148751 : basic_block bb;
959 148751 : bool changed;
960 148751 : bitmap work = BITMAP_ALLOC (NULL);
961 148751 : int *rpo;
962 148751 : int n_bbs;
963 :
964 148751 : vars_ssa_cache cache;
965 :
966 : /* We approximate the live range of a stack variable by taking the first
967 : mention of its name as starting point(s), and by the end-of-scope
968 : death clobber added by gimplify as ending point(s) of the range.
969 : This overapproximates in the case we for instance moved an address-taken
970 : operation upward, without also moving a dereference to it upwards.
971 : But it's conservatively correct as a variable never can hold values
972 : before its name is mentioned at least once.
973 :
974 : We then do a mostly classical bitmap liveness algorithm. */
975 :
976 5619949 : FOR_ALL_BB_FN (bb, cfun)
977 5471198 : bb->aux = BITMAP_ALLOC (&stack_var_bitmap_obstack);
978 :
979 148751 : rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
980 148751 : n_bbs = pre_and_rev_post_order_compute (NULL, rpo, false);
981 :
982 148751 : changed = true;
983 601221 : while (changed)
984 : {
985 : int i;
986 : changed = false;
987 11586049 : for (i = 0; i < n_bbs; i++)
988 : {
989 11282330 : bitmap active;
990 11282330 : bb = BASIC_BLOCK_FOR_FN (cfun, rpo[i]);
991 11282330 : active = (bitmap)bb->aux;
992 11282330 : add_scope_conflicts_1 (cache, bb, work, false);
993 11282330 : if (bitmap_ior_into (active, work))
994 4649468 : changed = true;
995 : }
996 : }
997 :
998 5322447 : FOR_EACH_BB_FN (bb, cfun)
999 5173696 : add_scope_conflicts_1 (cache, bb, work, true);
1000 :
1001 148751 : if (dump_file && (dump_flags & TDF_DETAILS))
1002 0 : cache.dump (dump_file);
1003 :
1004 148751 : free (rpo);
1005 148751 : BITMAP_FREE (work);
1006 5619949 : FOR_ALL_BB_FN (bb, cfun)
1007 5471198 : BITMAP_FREE (bb->aux);
1008 148751 : }
1009 :
1010 : /* A subroutine of partition_stack_vars. A comparison function for qsort,
1011 : sorting an array of indices by the properties of the object. */
1012 :
1013 : static int
1014 20426942 : stack_var_cmp (const void *a, const void *b)
1015 : {
1016 20426942 : unsigned ia = *(const unsigned *)a;
1017 20426942 : unsigned ib = *(const unsigned *)b;
1018 20426942 : unsigned int aligna = stack_vars[ia].alignb;
1019 20426942 : unsigned int alignb = stack_vars[ib].alignb;
1020 20426942 : poly_int64 sizea = stack_vars[ia].size;
1021 20426942 : poly_int64 sizeb = stack_vars[ib].size;
1022 20426942 : tree decla = stack_vars[ia].decl;
1023 20426942 : tree declb = stack_vars[ib].decl;
1024 20426942 : bool largea, largeb;
1025 20426942 : unsigned int uida, uidb;
1026 :
1027 : /* Primary compare on "large" alignment. Large comes first. */
1028 20426942 : largea = (aligna * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
1029 20426942 : largeb = (alignb * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
1030 20426942 : if (largea != largeb)
1031 0 : return (int)largeb - (int)largea;
1032 :
1033 : /* Secondary compare on size, decreasing */
1034 36827359 : int diff = compare_sizes_for_sort (sizeb, sizea);
1035 4679172 : if (diff != 0)
1036 : return diff;
1037 :
1038 : /* Tertiary compare on true alignment, decreasing. */
1039 11721245 : if (aligna < alignb)
1040 : return -1;
1041 11410195 : if (aligna > alignb)
1042 : return 1;
1043 :
1044 : /* Final compare on ID for sort stability, increasing.
1045 : Two SSA names are compared by their version, SSA names come before
1046 : non-SSA names, and two normal decls are compared by their DECL_UID. */
1047 11155523 : if (TREE_CODE (decla) == SSA_NAME)
1048 : {
1049 37982 : if (TREE_CODE (declb) == SSA_NAME)
1050 33321 : uida = SSA_NAME_VERSION (decla), uidb = SSA_NAME_VERSION (declb);
1051 : else
1052 : return -1;
1053 : }
1054 11117541 : else if (TREE_CODE (declb) == SSA_NAME)
1055 : return 1;
1056 : else
1057 11111886 : uida = DECL_UID (decla), uidb = DECL_UID (declb);
1058 11145207 : if (uida < uidb)
1059 : return 1;
1060 5542930 : if (uida > uidb)
1061 5542930 : return -1;
1062 : return 0;
1063 : }
1064 :
1065 : struct part_traits : unbounded_int_hashmap_traits <unsigned , bitmap> {};
1066 : typedef hash_map<unsigned, bitmap, part_traits> part_hashmap;
1067 :
1068 : /* If the points-to solution *PI points to variables that are in a partition
1069 : together with other variables add all partition members to the pointed-to
1070 : variables bitmap. */
1071 :
1072 : static void
1073 1636362 : add_partitioned_vars_to_ptset (struct pt_solution *pt,
1074 : part_hashmap *decls_to_partitions,
1075 : hash_set<bitmap> *visited, bitmap temp)
1076 : {
1077 1636362 : bitmap_iterator bi;
1078 1636362 : unsigned i;
1079 1636362 : bitmap *part;
1080 :
1081 1636362 : if (pt->anything
1082 1579811 : || pt->vars == NULL
1083 : /* The pointed-to vars bitmap is shared, it is enough to
1084 : visit it once. */
1085 3216173 : || visited->add (pt->vars))
1086 1330253 : return;
1087 :
1088 306109 : bitmap_clear (temp);
1089 :
1090 : /* By using a temporary bitmap to store all members of the partitions
1091 : we have to add we make sure to visit each of the partitions only
1092 : once. */
1093 1337076 : EXECUTE_IF_SET_IN_BITMAP (pt->vars, 0, i, bi)
1094 1030967 : if ((!temp
1095 1030967 : || !bitmap_bit_p (temp, i))
1096 1668815 : && (part = decls_to_partitions->get (i)))
1097 175154 : bitmap_ior_into (temp, *part);
1098 306109 : if (!bitmap_empty_p (temp))
1099 151280 : bitmap_ior_into (pt->vars, temp);
1100 : }
1101 :
1102 : /* Update points-to sets based on partition info, so we can use them on RTL.
1103 : The bitmaps representing stack partitions will be saved until expand,
1104 : where partitioned decls used as bases in memory expressions will be
1105 : rewritten.
1106 :
1107 : It is not necessary to update TBAA info on accesses to the coalesced
1108 : storage since our memory model doesn't allow TBAA to be used for
1109 : WAW or WAR dependences. For RAW when the write is to an old object
1110 : the new object would not have been initialized at the point of the
1111 : read, invoking undefined behavior. */
1112 :
1113 : static void
1114 148751 : update_alias_info_with_stack_vars (void)
1115 : {
1116 148751 : part_hashmap *decls_to_partitions = NULL;
1117 148751 : unsigned i, j;
1118 148751 : tree var = NULL_TREE;
1119 :
1120 1276422 : for (i = 0; i < stack_vars_num; i++)
1121 : {
1122 1127671 : bitmap part = NULL;
1123 1127671 : tree name;
1124 1127671 : struct ptr_info_def *pi;
1125 :
1126 : /* Not interested in partitions with single variable. */
1127 1127671 : if (stack_vars[i].representative != i
1128 852278 : || stack_vars[i].next == EOC)
1129 1057029 : continue;
1130 :
1131 70642 : if (!decls_to_partitions)
1132 : {
1133 42662 : decls_to_partitions = new part_hashmap;
1134 42662 : cfun->gimple_df->decls_to_pointers = new hash_map<tree, tree>;
1135 : }
1136 :
1137 : /* Create an SSA_NAME that points to the partition for use
1138 : as base during alias-oracle queries on RTL for bases that
1139 : have been partitioned. */
1140 70642 : if (var == NULL_TREE)
1141 42662 : var = create_tmp_var (ptr_type_node);
1142 70642 : name = make_ssa_name (var);
1143 :
1144 : /* Create bitmaps representing partitions. They will be used for
1145 : points-to sets later, so use GGC alloc. */
1146 70642 : part = BITMAP_GGC_ALLOC ();
1147 416677 : for (j = i; j != EOC; j = stack_vars[j].next)
1148 : {
1149 346035 : tree decl = stack_vars[j].decl;
1150 346035 : unsigned int uid = DECL_PT_UID (decl);
1151 346035 : bitmap_set_bit (part, uid);
1152 346035 : decls_to_partitions->put (uid, part);
1153 346035 : cfun->gimple_df->decls_to_pointers->put (decl, name);
1154 346035 : if (TREE_ADDRESSABLE (decl))
1155 316750 : TREE_ADDRESSABLE (name) = 1;
1156 : }
1157 :
1158 : /* Make the SSA name point to all partition members. */
1159 70642 : pi = get_ptr_info (name);
1160 70642 : pt_solution_set (&pi->pt, part, false);
1161 : }
1162 :
1163 : /* Make all points-to sets that contain one member of a partition
1164 : contain all members of the partition. */
1165 148751 : if (decls_to_partitions)
1166 : {
1167 42662 : unsigned i;
1168 42662 : tree name;
1169 42662 : hash_set<bitmap> visited;
1170 42662 : bitmap temp = BITMAP_ALLOC (&stack_var_bitmap_obstack);
1171 :
1172 17188608 : FOR_EACH_SSA_NAME (i, name, cfun)
1173 : {
1174 10527817 : struct ptr_info_def *pi;
1175 :
1176 19302726 : if (POINTER_TYPE_P (TREE_TYPE (name))
1177 10615563 : && ((pi = SSA_NAME_PTR_INFO (name)) != NULL))
1178 1551038 : add_partitioned_vars_to_ptset (&pi->pt, decls_to_partitions,
1179 : &visited, temp);
1180 : }
1181 :
1182 42662 : add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped,
1183 : decls_to_partitions, &visited, temp);
1184 42662 : add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped_return,
1185 : decls_to_partitions, &visited, temp);
1186 42662 : delete decls_to_partitions;
1187 42662 : BITMAP_FREE (temp);
1188 42662 : }
1189 148751 : }
1190 :
1191 : /* A subroutine of partition_stack_vars. The UNION portion of a UNION/FIND
1192 : partitioning algorithm. Partitions A and B are known to be non-conflicting.
1193 : Merge them into a single partition A. */
1194 :
1195 : static void
1196 275393 : union_stack_vars (unsigned a, unsigned b)
1197 : {
1198 275393 : class stack_var *vb = &stack_vars[b];
1199 275393 : bitmap_iterator bi;
1200 275393 : unsigned u;
1201 :
1202 275393 : gcc_assert (stack_vars[b].next == EOC);
1203 : /* Add B to A's partition. */
1204 275393 : stack_vars[b].next = stack_vars[a].next;
1205 275393 : stack_vars[b].representative = a;
1206 275393 : stack_vars[a].next = b;
1207 :
1208 : /* Make sure A is big enough to hold B. */
1209 275393 : stack_vars[a].size = upper_bound (stack_vars[a].size, stack_vars[b].size);
1210 :
1211 : /* Update the required alignment of partition A to account for B. */
1212 275393 : if (stack_vars[a].alignb < stack_vars[b].alignb)
1213 1813 : stack_vars[a].alignb = stack_vars[b].alignb;
1214 :
1215 : /* Update the interference graph and merge the conflicts. */
1216 275393 : if (vb->conflicts)
1217 : {
1218 1975274 : EXECUTE_IF_SET_IN_BITMAP (vb->conflicts, 0, u, bi)
1219 1750630 : add_stack_var_conflict (a, stack_vars[u].representative);
1220 224644 : BITMAP_FREE (vb->conflicts);
1221 : }
1222 275393 : }
1223 :
1224 : /* A subroutine of expand_used_vars. Binpack the variables into
1225 : partitions constrained by the interference graph. The overall
1226 : algorithm used is as follows:
1227 :
1228 : Sort the objects by size in descending order.
1229 : For each object A {
1230 : S = size(A)
1231 : O = 0
1232 : loop {
1233 : Look for the largest non-conflicting object B with size <= S.
1234 : UNION (A, B)
1235 : }
1236 : }
1237 : */
1238 :
1239 : static void
1240 235927 : partition_stack_vars (void)
1241 : {
1242 235927 : unsigned si, sj, n = stack_vars_num;
1243 :
1244 235927 : stack_vars_sorted = XNEWVEC (unsigned, stack_vars_num);
1245 1450774 : for (si = 0; si < n; ++si)
1246 1214847 : stack_vars_sorted[si] = si;
1247 :
1248 235927 : if (n == 1)
1249 : return;
1250 :
1251 148751 : qsort (stack_vars_sorted, n, sizeof (unsigned), stack_var_cmp);
1252 :
1253 1425173 : for (si = 0; si < n; ++si)
1254 : {
1255 1127671 : unsigned i = stack_vars_sorted[si];
1256 1127671 : unsigned int ialign = stack_vars[i].alignb;
1257 1127671 : poly_int64 isize = stack_vars[i].size;
1258 :
1259 : /* Ignore objects that aren't partition representatives. If we
1260 : see a var that is not a partition representative, it must
1261 : have been merged earlier. */
1262 1127671 : if (stack_vars[i].representative != i)
1263 275393 : continue;
1264 :
1265 13117731 : for (sj = si + 1; sj < n; ++sj)
1266 : {
1267 12266615 : unsigned j = stack_vars_sorted[sj];
1268 12266615 : unsigned int jalign = stack_vars[j].alignb;
1269 12266615 : poly_int64 jsize = stack_vars[j].size;
1270 :
1271 : /* Ignore objects that aren't partition representatives. */
1272 12266615 : if (stack_vars[j].representative != j)
1273 12265453 : continue;
1274 :
1275 : /* Do not mix objects of "small" (supported) alignment
1276 : and "large" (unsupported) alignment. */
1277 11531103 : if ((ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1278 11531103 : != (jalign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT))
1279 : break;
1280 :
1281 : /* For Address Sanitizer do not mix objects with different
1282 : sizes, as the shorter vars wouldn't be adequately protected.
1283 : Don't do that for "large" (unsupported) alignment objects,
1284 : those aren't protected anyway. */
1285 11531103 : if (asan_sanitize_stack_p ()
1286 6247 : && maybe_ne (isize, jsize)
1287 11532265 : && ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1288 : break;
1289 :
1290 : /* Ignore conflicting objects. */
1291 11529941 : if (stack_var_conflict_p (i, j))
1292 11254548 : continue;
1293 :
1294 : /* UNION the objects, placing J at OFFSET. */
1295 275393 : union_stack_vars (i, j);
1296 : }
1297 : }
1298 :
1299 148751 : update_alias_info_with_stack_vars ();
1300 : }
1301 :
1302 : /* A debugging aid for expand_used_vars. Dump the generated partitions. */
1303 :
1304 : static void
1305 33 : dump_stack_var_partition (void)
1306 : {
1307 33 : unsigned si, i, j, n = stack_vars_num;
1308 :
1309 79 : for (si = 0; si < n; ++si)
1310 : {
1311 46 : i = stack_vars_sorted[si];
1312 :
1313 : /* Skip variables that aren't partition representatives, for now. */
1314 46 : if (stack_vars[i].representative != i)
1315 3 : continue;
1316 :
1317 43 : fprintf (dump_file, "Partition %u: size ", i);
1318 43 : print_dec (stack_vars[i].size, dump_file);
1319 43 : fprintf (dump_file, " align %u\n", stack_vars[i].alignb);
1320 :
1321 132 : for (j = i; j != EOC; j = stack_vars[j].next)
1322 : {
1323 46 : fputc ('\t', dump_file);
1324 46 : print_generic_expr (dump_file, stack_vars[j].decl, dump_flags);
1325 : }
1326 43 : fputc ('\n', dump_file);
1327 : }
1328 33 : }
1329 :
1330 : /* Assign rtl to DECL at BASE + OFFSET. */
1331 :
1332 : static void
1333 2028824 : expand_one_stack_var_at (tree decl, rtx base, unsigned base_align,
1334 : poly_int64 offset)
1335 : {
1336 2028824 : unsigned align;
1337 2028824 : rtx x;
1338 :
1339 : /* If this fails, we've overflowed the stack frame. Error nicely? */
1340 2304624 : gcc_assert (known_eq (offset, trunc_int_for_mode (offset, Pmode)));
1341 :
1342 2028824 : if (hwassist_sanitize_stack_p ())
1343 88 : x = targetm.memtag.add_tag (base, offset,
1344 : hwasan_current_frame_tag ());
1345 : else
1346 2304536 : x = plus_constant (Pmode, base, offset);
1347 :
1348 4057648 : x = gen_rtx_MEM (TREE_CODE (decl) == SSA_NAME
1349 457923 : ? TYPE_MODE (TREE_TYPE (decl))
1350 1570901 : : DECL_MODE (decl), x);
1351 :
1352 : /* Set alignment we actually gave this decl if it isn't an SSA name.
1353 : If it is we generate stack slots only accidentally so it isn't as
1354 : important, we'll simply set the alignment directly on the MEM. */
1355 :
1356 2028824 : if (stack_vars_base_reg_p (base))
1357 2025514 : offset -= frame_phase;
1358 2028824 : align = known_alignment (offset);
1359 2028824 : align *= BITS_PER_UNIT;
1360 2028824 : if (align == 0 || align > base_align)
1361 823511 : align = base_align;
1362 :
1363 2028824 : if (TREE_CODE (decl) != SSA_NAME)
1364 : {
1365 : /* One would think that we could assert that we're not decreasing
1366 : alignment here, but (at least) the i386 port does exactly this
1367 : via the MINIMUM_ALIGNMENT hook. */
1368 :
1369 1570901 : SET_DECL_ALIGN (decl, align);
1370 1570901 : DECL_USER_ALIGN (decl) = 0;
1371 : }
1372 :
1373 2028824 : set_rtl (decl, x);
1374 :
1375 2028824 : set_mem_align (x, align);
1376 2028824 : }
1377 :
1378 235927 : class stack_vars_data
1379 : {
1380 : public:
1381 : /* Vector of offset pairs, always end of some padding followed
1382 : by start of the padding that needs Address Sanitizer protection.
1383 : The vector is in reversed, highest offset pairs come first. */
1384 : auto_vec<HOST_WIDE_INT> asan_vec;
1385 :
1386 : /* Vector of partition representative decls in between the paddings. */
1387 : auto_vec<tree> asan_decl_vec;
1388 :
1389 : /* Base pseudo register for Address Sanitizer protected automatic vars. */
1390 : rtx asan_base;
1391 :
1392 : /* Alignment needed for the Address Sanitizer protected automatic vars. */
1393 : unsigned int asan_alignb;
1394 : };
1395 :
1396 : /* A subroutine of expand_used_vars. Give each partition representative
1397 : a unique location within the stack frame. Update each partition member
1398 : with that location. */
1399 : static void
1400 238113 : expand_stack_vars (bool (*pred) (unsigned), class stack_vars_data *data)
1401 : {
1402 238113 : unsigned si, i, j, n = stack_vars_num;
1403 238113 : poly_uint64 large_size = 0, large_alloc = 0;
1404 238113 : rtx large_base = NULL;
1405 238113 : rtx large_untagged_base = NULL;
1406 238113 : unsigned large_align = 0;
1407 238113 : bool large_allocation_done = false;
1408 238113 : tree decl;
1409 :
1410 : /* Determine if there are any variables requiring "large" alignment.
1411 : Since these are dynamically allocated, we only process these if
1412 : no predicate involved. */
1413 238113 : large_align = stack_vars[stack_vars_sorted[0]].alignb * BITS_PER_UNIT;
1414 238113 : if (pred == NULL && large_align > MAX_SUPPORTED_STACK_ALIGNMENT)
1415 : {
1416 : /* Find the total size of these variables. */
1417 0 : for (si = 0; si < n; ++si)
1418 : {
1419 0 : unsigned alignb;
1420 :
1421 0 : i = stack_vars_sorted[si];
1422 0 : alignb = stack_vars[i].alignb;
1423 :
1424 : /* All "large" alignment decls come before all "small" alignment
1425 : decls, but "large" alignment decls are not sorted based on
1426 : their alignment. Increase large_align to track the largest
1427 : required alignment. */
1428 0 : if ((alignb * BITS_PER_UNIT) > large_align)
1429 : large_align = alignb * BITS_PER_UNIT;
1430 :
1431 : /* Stop when we get to the first decl with "small" alignment. */
1432 0 : if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1433 : break;
1434 :
1435 : /* Skip variables that aren't partition representatives. */
1436 0 : if (stack_vars[i].representative != i)
1437 0 : continue;
1438 :
1439 : /* Skip variables that have already had rtl assigned. See also
1440 : add_stack_var where we perpetrate this pc_rtx hack. */
1441 0 : decl = stack_vars[i].decl;
1442 0 : if (TREE_CODE (decl) == SSA_NAME
1443 0 : ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1444 0 : : DECL_RTL (decl) != pc_rtx)
1445 0 : continue;
1446 :
1447 0 : large_size = aligned_upper_bound (large_size, alignb);
1448 0 : large_size += stack_vars[i].size;
1449 : }
1450 : }
1451 :
1452 1457692 : for (si = 0; si < n; ++si)
1453 : {
1454 1219579 : rtx base;
1455 1219579 : unsigned base_align, alignb;
1456 1219579 : poly_int64 offset = 0;
1457 :
1458 1219579 : i = stack_vars_sorted[si];
1459 :
1460 : /* Skip variables that aren't partition representatives, for now. */
1461 1219579 : if (stack_vars[i].representative != i)
1462 275404 : continue;
1463 :
1464 : /* Skip variables that have already had rtl assigned. See also
1465 : add_stack_var where we perpetrate this pc_rtx hack. */
1466 944175 : decl = stack_vars[i].decl;
1467 944175 : if (TREE_CODE (decl) == SSA_NAME
1468 944175 : ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1469 940302 : : DECL_RTL (decl) != pc_rtx)
1470 3615 : continue;
1471 :
1472 : /* Check the predicate to see whether this variable should be
1473 : allocated in this pass. */
1474 940560 : if (pred && !pred (i))
1475 1106 : continue;
1476 :
1477 939454 : base = (hwassist_sanitize_stack_p ()
1478 939454 : ? hwasan_frame_base ()
1479 939398 : : virtual_stack_vars_rtx);
1480 939454 : alignb = stack_vars[i].alignb;
1481 939454 : if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1482 : {
1483 939454 : poly_int64 hwasan_orig_offset;
1484 939454 : if (hwassist_sanitize_stack_p ())
1485 : {
1486 : /* There must be no tag granule "shared" between different
1487 : objects. This means that no HWASAN_TAG_GRANULE_SIZE byte
1488 : chunk can have more than one object in it.
1489 :
1490 : We ensure this by forcing the end of the last bit of data to
1491 : be aligned to HWASAN_TAG_GRANULE_SIZE bytes here, and setting
1492 : the start of each variable to be aligned to
1493 : HWASAN_TAG_GRANULE_SIZE bytes in `align_local_variable`.
1494 :
1495 : We can't align just one of the start or end, since there are
1496 : untagged things stored on the stack which we do not align to
1497 : HWASAN_TAG_GRANULE_SIZE bytes. If we only aligned the start
1498 : or the end of tagged objects then untagged objects could end
1499 : up sharing the first granule of a tagged object or sharing the
1500 : last granule of a tagged object respectively. */
1501 56 : hwasan_orig_offset = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1502 56 : gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
1503 : }
1504 : /* ASAN description strings don't yet have a syntax for expressing
1505 : polynomial offsets. */
1506 939454 : HOST_WIDE_INT prev_offset;
1507 939454 : if (asan_sanitize_stack_p ()
1508 4110 : && pred
1509 3305 : && frame_offset.is_constant (&prev_offset)
1510 939454 : && stack_vars[i].size.is_constant ())
1511 : {
1512 3305 : if (data->asan_vec.is_empty ())
1513 : {
1514 1721 : align_frame_offset (ASAN_RED_ZONE_SIZE);
1515 1721 : prev_offset = frame_offset.to_constant ();
1516 : }
1517 3305 : prev_offset = align_base (prev_offset,
1518 : ASAN_MIN_RED_ZONE_SIZE,
1519 : !FRAME_GROWS_DOWNWARD);
1520 3305 : tree repr_decl = NULL_TREE;
1521 3305 : unsigned HOST_WIDE_INT size
1522 3305 : = asan_var_and_redzone_size (stack_vars[i].size.to_constant ());
1523 3305 : if (data->asan_vec.is_empty ())
1524 1721 : size = MAX (size, ASAN_RED_ZONE_SIZE);
1525 :
1526 3305 : unsigned HOST_WIDE_INT alignment = MAX (alignb,
1527 : ASAN_MIN_RED_ZONE_SIZE);
1528 3305 : offset = alloc_stack_frame_space (size, alignment);
1529 :
1530 3305 : data->asan_vec.safe_push (prev_offset);
1531 : /* Allocating a constant amount of space from a constant
1532 : starting offset must give a constant result. */
1533 3305 : data->asan_vec.safe_push ((offset + stack_vars[i].size)
1534 3305 : .to_constant ());
1535 : /* Find best representative of the partition.
1536 : Prefer those with DECL_NAME, even better
1537 : satisfying asan_protect_stack_decl predicate. */
1538 3857 : for (j = i; j != EOC; j = stack_vars[j].next)
1539 3309 : if (asan_protect_stack_decl (stack_vars[j].decl)
1540 3309 : && DECL_NAME (stack_vars[j].decl))
1541 : {
1542 2757 : repr_decl = stack_vars[j].decl;
1543 2757 : break;
1544 : }
1545 552 : else if (repr_decl == NULL_TREE
1546 552 : && DECL_P (stack_vars[j].decl)
1547 1104 : && DECL_NAME (stack_vars[j].decl))
1548 0 : repr_decl = stack_vars[j].decl;
1549 3305 : if (repr_decl == NULL_TREE)
1550 548 : repr_decl = stack_vars[i].decl;
1551 3305 : data->asan_decl_vec.safe_push (repr_decl);
1552 :
1553 : /* Make sure a representative is unpoison if another
1554 : variable in the partition is handled by
1555 : use-after-scope sanitization. */
1556 3305 : if (asan_handled_variables != NULL
1557 3305 : && !asan_handled_variables->contains (repr_decl))
1558 : {
1559 440 : for (j = i; j != EOC; j = stack_vars[j].next)
1560 222 : if (asan_handled_variables->contains (stack_vars[j].decl))
1561 : break;
1562 220 : if (j != EOC)
1563 2 : asan_handled_variables->add (repr_decl);
1564 : }
1565 :
1566 3305 : data->asan_alignb = MAX (data->asan_alignb, alignb);
1567 3305 : if (data->asan_base == NULL)
1568 1721 : data->asan_base = gen_reg_rtx (Pmode);
1569 3305 : base = data->asan_base;
1570 :
1571 3305 : if (!STRICT_ALIGNMENT)
1572 3305 : base_align = crtl->max_used_stack_slot_alignment;
1573 : else
1574 : base_align = MAX (crtl->max_used_stack_slot_alignment,
1575 : GET_MODE_ALIGNMENT (SImode)
1576 : << ASAN_SHADOW_SHIFT);
1577 : }
1578 : else
1579 : {
1580 936149 : offset = alloc_stack_frame_space (stack_vars[i].size, alignb);
1581 936149 : base_align = crtl->max_used_stack_slot_alignment;
1582 :
1583 936149 : if (hwassist_sanitize_stack_p ())
1584 : {
1585 : /* Align again since the point of this alignment is to handle
1586 : the "end" of the object (i.e. smallest address after the
1587 : stack object). For FRAME_GROWS_DOWNWARD that requires
1588 : aligning the stack before allocating, but for a frame that
1589 : grows upwards that requires aligning the stack after
1590 : allocation.
1591 :
1592 : Use `frame_offset` to record the offset value rather than
1593 : `offset` since the `frame_offset` describes the extent
1594 : allocated for this particular variable while `offset`
1595 : describes the address that this variable starts at. */
1596 56 : align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1597 56 : hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1598 : hwasan_orig_offset, frame_offset);
1599 : }
1600 : }
1601 : }
1602 : else
1603 : {
1604 : /* Large alignment is only processed in the last pass. */
1605 0 : if (pred)
1606 0 : continue;
1607 :
1608 : /* If there were any variables requiring "large" alignment, allocate
1609 : space. */
1610 0 : if (maybe_ne (large_size, 0U) && ! large_allocation_done)
1611 : {
1612 0 : poly_int64 loffset;
1613 0 : rtx large_allocsize;
1614 :
1615 0 : large_allocsize = gen_int_mode (large_size, Pmode);
1616 0 : get_dynamic_stack_size (&large_allocsize, 0, large_align, NULL);
1617 0 : loffset = alloc_stack_frame_space
1618 0 : (rtx_to_poly_int64 (large_allocsize),
1619 0 : PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT);
1620 0 : large_base = get_dynamic_stack_base (loffset, large_align, base);
1621 0 : large_allocation_done = true;
1622 : }
1623 :
1624 0 : gcc_assert (large_base != NULL);
1625 0 : large_alloc = aligned_upper_bound (large_alloc, alignb);
1626 0 : offset = large_alloc;
1627 0 : large_alloc += stack_vars[i].size;
1628 0 : if (hwassist_sanitize_stack_p ())
1629 : {
1630 : /* An object with a large alignment requirement means that the
1631 : alignment requirement is greater than the required alignment
1632 : for tags. */
1633 0 : if (!large_untagged_base)
1634 0 : large_untagged_base
1635 0 : = targetm.memtag.untagged_pointer (large_base, NULL_RTX);
1636 : /* Ensure the end of the variable is also aligned correctly. */
1637 0 : poly_int64 align_again
1638 0 : = aligned_upper_bound (large_alloc, HWASAN_TAG_GRANULE_SIZE);
1639 : /* For large allocations we always allocate a chunk of space
1640 : (which is addressed by large_untagged_base/large_base) and
1641 : then use positive offsets from that. Hence the farthest
1642 : offset is `align_again` and the nearest offset from the base
1643 : is `offset`. */
1644 0 : hwasan_record_stack_var (large_untagged_base, large_base,
1645 : offset, align_again);
1646 : }
1647 :
1648 : base = large_base;
1649 : base_align = large_align;
1650 : }
1651 :
1652 : /* Create rtl for each variable based on their location within the
1653 : partition. */
1654 2154301 : for (j = i; j != EOC; j = stack_vars[j].next)
1655 : {
1656 1214847 : expand_one_stack_var_at (stack_vars[j].decl,
1657 : base, base_align, offset);
1658 : }
1659 939454 : if (hwassist_sanitize_stack_p ())
1660 56 : hwasan_increment_frame_tag ();
1661 : }
1662 :
1663 238113 : gcc_assert (known_eq (large_alloc, large_size));
1664 238113 : }
1665 :
1666 : /* Take into account all sizes of partitions and reset DECL_RTLs. */
1667 : static poly_uint64
1668 1086734 : account_stack_vars (void)
1669 : {
1670 1086734 : unsigned si, j, i, n = stack_vars_num;
1671 1086734 : poly_uint64 size = 0;
1672 :
1673 5707643 : for (si = 0; si < n; ++si)
1674 : {
1675 4620909 : i = stack_vars_sorted[si];
1676 :
1677 : /* Skip variables that aren't partition representatives, for now. */
1678 4620909 : if (stack_vars[i].representative != i)
1679 0 : continue;
1680 :
1681 9241818 : size += stack_vars[i].size;
1682 9241818 : for (j = i; j != EOC; j = stack_vars[j].next)
1683 4620909 : set_rtl (stack_vars[j].decl, NULL);
1684 : }
1685 1086734 : return size;
1686 : }
1687 :
1688 : /* Record the RTL assignment X for the default def of PARM. */
1689 :
1690 : extern void
1691 4773405 : set_parm_rtl (tree parm, rtx x)
1692 : {
1693 4773405 : gcc_assert (TREE_CODE (parm) == PARM_DECL
1694 : || TREE_CODE (parm) == RESULT_DECL);
1695 :
1696 4773405 : if (x && !MEM_P (x))
1697 : {
1698 3074700 : unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (parm),
1699 : TYPE_MODE (TREE_TYPE (parm)),
1700 : TYPE_ALIGN (TREE_TYPE (parm)));
1701 :
1702 : /* If the variable alignment is very large we'll dynamically
1703 : allocate it, which means that in-frame portion is just a
1704 : pointer. ??? We've got a pseudo for sure here, do we
1705 : actually dynamically allocate its spilling area if needed?
1706 : ??? Isn't it a problem when Pmode alignment also exceeds
1707 : MAX_SUPPORTED_STACK_ALIGNMENT, as can happen on cris and lm32? */
1708 3074700 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1709 0 : align = GET_MODE_ALIGNMENT (Pmode);
1710 :
1711 3074700 : record_alignment_for_reg_var (align);
1712 : }
1713 :
1714 4773405 : tree ssa = ssa_default_def (cfun, parm);
1715 4773405 : if (!ssa)
1716 1090053 : return set_rtl (parm, x);
1717 :
1718 3683352 : int part = var_to_partition (SA.map, ssa);
1719 3683352 : gcc_assert (part != NO_PARTITION);
1720 :
1721 3683352 : bool changed = bitmap_bit_p (SA.partitions_for_parm_default_defs, part);
1722 3683352 : gcc_assert (changed);
1723 :
1724 3683352 : set_rtl (ssa, x);
1725 3683352 : gcc_assert (DECL_RTL (parm) == x);
1726 : }
1727 :
1728 : /* A subroutine of expand_one_var. Called to immediately assign rtl
1729 : to a variable to be allocated in the stack frame. */
1730 :
1731 : static void
1732 813977 : expand_one_stack_var_1 (tree var)
1733 : {
1734 813977 : poly_uint64 size;
1735 813977 : poly_int64 offset;
1736 813977 : unsigned byte_align;
1737 :
1738 813977 : if (TREE_CODE (var) == SSA_NAME)
1739 : {
1740 454740 : tree type = TREE_TYPE (var);
1741 454740 : size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
1742 : }
1743 : else
1744 359237 : size = tree_to_poly_uint64 (DECL_SIZE_UNIT (var));
1745 :
1746 813977 : byte_align = align_local_variable (var, true);
1747 :
1748 : /* We handle highly aligned variables in expand_stack_vars. */
1749 813977 : gcc_assert (byte_align * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT);
1750 :
1751 813977 : rtx base;
1752 813977 : if (hwassist_sanitize_stack_p ())
1753 : {
1754 : /* Allocate zero bytes to align the stack. */
1755 32 : poly_int64 hwasan_orig_offset
1756 32 : = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1757 32 : offset = alloc_stack_frame_space (size, byte_align);
1758 32 : align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1759 32 : base = hwasan_frame_base ();
1760 : /* Use `frame_offset` to automatically account for machines where the
1761 : frame grows upwards.
1762 :
1763 : `offset` will always point to the "start" of the stack object, which
1764 : will be the smallest address, for ! FRAME_GROWS_DOWNWARD this is *not*
1765 : the "furthest" offset from the base delimiting the current stack
1766 : object. `frame_offset` will always delimit the extent that the frame.
1767 : */
1768 32 : hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1769 : hwasan_orig_offset, frame_offset);
1770 : }
1771 : else
1772 : {
1773 813945 : offset = alloc_stack_frame_space (size, byte_align);
1774 813945 : base = virtual_stack_vars_rtx;
1775 : }
1776 :
1777 813977 : expand_one_stack_var_at (var, base,
1778 : crtl->max_used_stack_slot_alignment, offset);
1779 :
1780 813977 : if (hwassist_sanitize_stack_p ())
1781 32 : hwasan_increment_frame_tag ();
1782 813977 : }
1783 :
1784 : /* Wrapper for expand_one_stack_var_1 that checks SSA_NAMEs are
1785 : already assigned some MEM. */
1786 :
1787 : static void
1788 359237 : expand_one_stack_var (tree var)
1789 : {
1790 359237 : if (TREE_CODE (var) == SSA_NAME)
1791 : {
1792 0 : int part = var_to_partition (SA.map, var);
1793 0 : if (part != NO_PARTITION)
1794 : {
1795 0 : rtx x = SA.partition_to_pseudo[part];
1796 0 : gcc_assert (x);
1797 0 : gcc_assert (MEM_P (x));
1798 : return;
1799 : }
1800 : }
1801 :
1802 359237 : return expand_one_stack_var_1 (var);
1803 : }
1804 :
1805 : /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1806 : that will reside in a hard register. */
1807 :
1808 : static void
1809 1085 : expand_one_hard_reg_var (tree var)
1810 : {
1811 0 : rest_of_decl_compilation (var, 0, 0);
1812 0 : }
1813 :
1814 : /* Record the alignment requirements of some variable assigned to a
1815 : pseudo. */
1816 :
1817 : static void
1818 41649225 : record_alignment_for_reg_var (unsigned int align)
1819 : {
1820 41649225 : if (SUPPORTS_STACK_ALIGNMENT
1821 41649225 : && crtl->stack_alignment_estimated < align)
1822 : {
1823 : /* stack_alignment_estimated shouldn't change after stack
1824 : realign decision made */
1825 1744689 : gcc_assert (!crtl->stack_realign_processed);
1826 1744689 : crtl->stack_alignment_estimated = align;
1827 : }
1828 :
1829 : /* stack_alignment_needed > PREFERRED_STACK_BOUNDARY is permitted.
1830 : So here we only make sure stack_alignment_needed >= align. */
1831 41649225 : if (crtl->stack_alignment_needed < align)
1832 562148 : crtl->stack_alignment_needed = align;
1833 41649225 : if (crtl->max_used_stack_slot_alignment < align)
1834 562148 : crtl->max_used_stack_slot_alignment = align;
1835 41649225 : }
1836 :
1837 : /* Create RTL for an SSA partition. */
1838 :
1839 : static void
1840 22953502 : expand_one_ssa_partition (tree var)
1841 : {
1842 22953502 : int part = var_to_partition (SA.map, var);
1843 22953502 : gcc_assert (part != NO_PARTITION);
1844 :
1845 22953502 : if (SA.partition_to_pseudo[part])
1846 : return;
1847 :
1848 22953502 : unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1849 : TYPE_MODE (TREE_TYPE (var)),
1850 : TYPE_ALIGN (TREE_TYPE (var)));
1851 :
1852 : /* If the variable alignment is very large we'll dynamically allocate
1853 : it, which means that in-frame portion is just a pointer. */
1854 22953502 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1855 0 : align = GET_MODE_ALIGNMENT (Pmode);
1856 :
1857 22953502 : record_alignment_for_reg_var (align);
1858 :
1859 22953502 : if (!use_register_for_decl (var))
1860 : {
1861 457923 : if (defer_stack_allocation (var, true))
1862 3183 : add_stack_var (var, true);
1863 : else
1864 454740 : expand_one_stack_var_1 (var);
1865 : return;
1866 : }
1867 :
1868 22495579 : machine_mode reg_mode = promote_ssa_mode (var, NULL);
1869 22495579 : rtx x = gen_reg_rtx (reg_mode);
1870 :
1871 22495579 : set_rtl (var, x);
1872 :
1873 : /* For a promoted variable, X will not be used directly but wrapped in a
1874 : SUBREG with SUBREG_PROMOTED_VAR_P set, which means that the RTL land
1875 : will assume that its upper bits can be inferred from its lower bits.
1876 : Therefore, if X isn't initialized on every path from the entry, then
1877 : we must do it manually in order to fulfill the above assumption. */
1878 22495579 : if (reg_mode != TYPE_MODE (TREE_TYPE (var))
1879 22495579 : && bitmap_bit_p (SA.partitions_for_undefined_values, part))
1880 0 : emit_move_insn (x, CONST0_RTX (reg_mode));
1881 : }
1882 :
1883 : /* Record the association between the RTL generated for partition PART
1884 : and the underlying variable of the SSA_NAME VAR. */
1885 :
1886 : static void
1887 49915066 : adjust_one_expanded_partition_var (tree var)
1888 : {
1889 49915066 : if (!var)
1890 : return;
1891 :
1892 49915066 : tree decl = SSA_NAME_VAR (var);
1893 :
1894 49915066 : int part = var_to_partition (SA.map, var);
1895 49915066 : if (part == NO_PARTITION)
1896 : return;
1897 :
1898 32140800 : rtx x = SA.partition_to_pseudo[part];
1899 :
1900 32140800 : gcc_assert (x);
1901 :
1902 32140800 : set_rtl (var, x);
1903 :
1904 32140800 : if (!REG_P (x))
1905 : return;
1906 :
1907 : /* Note if the object is a user variable. */
1908 30075850 : if (decl && !DECL_ARTIFICIAL (decl))
1909 4190898 : mark_user_reg (x);
1910 :
1911 30075850 : if (POINTER_TYPE_P (decl ? TREE_TYPE (decl) : TREE_TYPE (var)))
1912 7717119 : mark_reg_pointer (x, get_pointer_alignment (var));
1913 : }
1914 :
1915 : /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1916 : that will reside in a pseudo register. */
1917 :
1918 : static void
1919 584068 : expand_one_register_var (tree var)
1920 : {
1921 584068 : if (TREE_CODE (var) == SSA_NAME)
1922 : {
1923 0 : int part = var_to_partition (SA.map, var);
1924 0 : if (part != NO_PARTITION)
1925 : {
1926 0 : rtx x = SA.partition_to_pseudo[part];
1927 0 : gcc_assert (x);
1928 0 : gcc_assert (REG_P (x));
1929 : return;
1930 : }
1931 0 : gcc_unreachable ();
1932 : }
1933 :
1934 584068 : tree decl = var;
1935 584068 : tree type = TREE_TYPE (decl);
1936 584068 : machine_mode reg_mode = promote_decl_mode (decl, NULL);
1937 584068 : rtx x = gen_reg_rtx (reg_mode);
1938 :
1939 584068 : set_rtl (var, x);
1940 :
1941 : /* Note if the object is a user variable. */
1942 584068 : if (!DECL_ARTIFICIAL (decl))
1943 92748 : mark_user_reg (x);
1944 :
1945 584068 : if (POINTER_TYPE_P (type))
1946 100 : mark_reg_pointer (x, get_pointer_alignment (var));
1947 : }
1948 :
1949 : /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL that
1950 : has some associated error, e.g. its type is error-mark. We just need
1951 : to pick something that won't crash the rest of the compiler. */
1952 :
1953 : static void
1954 32 : expand_one_error_var (tree var)
1955 : {
1956 32 : machine_mode mode = DECL_MODE (var);
1957 32 : rtx x;
1958 :
1959 32 : if (mode == BLKmode)
1960 1 : x = gen_rtx_MEM (BLKmode, const0_rtx);
1961 31 : else if (mode == VOIDmode)
1962 0 : x = const0_rtx;
1963 : else
1964 31 : x = gen_reg_rtx (mode);
1965 :
1966 32 : SET_DECL_RTL (var, x);
1967 32 : }
1968 :
1969 : /* A subroutine of expand_one_var. VAR is a variable that will be
1970 : allocated to the local stack frame. Return true if we wish to
1971 : add VAR to STACK_VARS so that it will be coalesced with other
1972 : variables. Return false to allocate VAR immediately.
1973 :
1974 : This function is used to reduce the number of variables considered
1975 : for coalescing, which reduces the size of the quadratic problem. */
1976 :
1977 : static bool
1978 6787042 : defer_stack_allocation (tree var, bool toplevel)
1979 : {
1980 6787042 : tree size_unit = TREE_CODE (var) == SSA_NAME
1981 6787042 : ? TYPE_SIZE_UNIT (TREE_TYPE (var))
1982 6787042 : : DECL_SIZE_UNIT (var);
1983 6787042 : poly_uint64 size;
1984 :
1985 : /* Whether the variable is small enough for immediate allocation not to be
1986 : a problem with regard to the frame size. */
1987 6787042 : bool smallish
1988 6787042 : = (poly_int_tree_p (size_unit, &size)
1989 6787042 : && (estimated_poly_value (size)
1990 6787042 : < param_min_size_for_stack_sharing));
1991 :
1992 : /* If stack protection is enabled, *all* stack variables must be deferred,
1993 : so that we can re-order the strings to the top of the frame.
1994 : Similarly for Address Sanitizer. */
1995 6787042 : if (flag_stack_protect || asan_sanitize_stack_p ())
1996 : return true;
1997 :
1998 6773075 : unsigned int align = TREE_CODE (var) == SSA_NAME
1999 6773075 : ? TYPE_ALIGN (TREE_TYPE (var))
2000 6315845 : : DECL_ALIGN (var);
2001 :
2002 : /* We handle "large" alignment via dynamic allocation. We want to handle
2003 : this extra complication in only one place, so defer them. */
2004 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
2005 : return true;
2006 :
2007 6773075 : bool ignored = TREE_CODE (var) == SSA_NAME
2008 6779514 : ? !SSAVAR (var) || DECL_IGNORED_P (SSA_NAME_VAR (var))
2009 6315845 : : DECL_IGNORED_P (var);
2010 :
2011 : /* When optimization is enabled, DECL_IGNORED_P variables originally scoped
2012 : might be detached from their block and appear at toplevel when we reach
2013 : here. We want to coalesce them with variables from other blocks when
2014 : the immediate contribution to the frame size would be noticeable. */
2015 6773075 : if (toplevel && optimize > 0 && ignored && !smallish)
2016 : return true;
2017 :
2018 : /* Variables declared in the outermost scope automatically conflict
2019 : with every other variable. The only reason to want to defer them
2020 : at all is that, after sorting, we can more efficiently pack
2021 : small variables in the stack frame. Continue to defer at -O2. */
2022 5246071 : if (toplevel && optimize < 2)
2023 : return false;
2024 :
2025 : /* Without optimization, *most* variables are allocated from the
2026 : stack, which makes the quadratic problem large exactly when we
2027 : want compilation to proceed as quickly as possible. On the
2028 : other hand, we don't want the function's stack frame size to
2029 : get completely out of hand. So we avoid adding scalars and
2030 : "small" aggregates to the list at all. */
2031 4588472 : if (optimize == 0 && smallish)
2032 48103 : return false;
2033 :
2034 : return true;
2035 : }
2036 :
2037 : /* A subroutine of expand_used_vars. Expand one variable according to
2038 : its flavor. Variables to be placed on the stack are not actually
2039 : expanded yet, merely recorded.
2040 : When REALLY_EXPAND is false, only add stack values to be allocated.
2041 : Return stack usage this variable is supposed to take.
2042 : */
2043 :
2044 : static poly_uint64
2045 15924251 : expand_one_var (tree var, bool toplevel, bool really_expand,
2046 : bitmap forced_stack_var = NULL)
2047 : {
2048 15924251 : unsigned int align = BITS_PER_UNIT;
2049 15924251 : tree origvar = var;
2050 :
2051 15924251 : var = SSAVAR (var);
2052 :
2053 15924251 : if (TREE_TYPE (var) != error_mark_node && VAR_P (var))
2054 : {
2055 15924251 : if (is_global_var (var))
2056 303228 : return 0;
2057 :
2058 : /* Because we don't know if VAR will be in register or on stack,
2059 : we conservatively assume it will be on stack even if VAR is
2060 : eventually put into register after RA pass. For non-automatic
2061 : variables, which won't be on stack, we collect alignment of
2062 : type and ignore user specified alignment. Similarly for
2063 : SSA_NAMEs for which use_register_for_decl returns true. */
2064 15621023 : if (TREE_STATIC (var)
2065 15621023 : || DECL_EXTERNAL (var)
2066 31242046 : || (TREE_CODE (origvar) == SSA_NAME && use_register_for_decl (var)))
2067 0 : align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
2068 : TYPE_MODE (TREE_TYPE (var)),
2069 : TYPE_ALIGN (TREE_TYPE (var)));
2070 15621023 : else if (DECL_HAS_VALUE_EXPR_P (var)
2071 15621023 : || (DECL_RTL_SET_P (var) && MEM_P (DECL_RTL (var))))
2072 : /* Don't consider debug only variables with DECL_HAS_VALUE_EXPR_P set
2073 : or variables which were assigned a stack slot already by
2074 : expand_one_stack_var_at - in the latter case DECL_ALIGN has been
2075 : changed from the offset chosen to it. */
2076 17062 : align = crtl->stack_alignment_estimated;
2077 : else
2078 15603961 : align = MINIMUM_ALIGNMENT (var, DECL_MODE (var), DECL_ALIGN (var));
2079 :
2080 : /* If the variable alignment is very large we'll dynamically allocate
2081 : it, which means that in-frame portion is just a pointer. */
2082 15621023 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
2083 0 : align = GET_MODE_ALIGNMENT (Pmode);
2084 : }
2085 :
2086 15621023 : record_alignment_for_reg_var (align);
2087 :
2088 15621023 : poly_uint64 size;
2089 15621023 : if (TREE_CODE (origvar) == SSA_NAME)
2090 : {
2091 0 : gcc_assert (!VAR_P (var)
2092 : || (!DECL_EXTERNAL (var)
2093 : && !DECL_HAS_VALUE_EXPR_P (var)
2094 : && !TREE_STATIC (var)
2095 : && TREE_TYPE (var) != error_mark_node
2096 : && !DECL_HARD_REGISTER (var)
2097 : && really_expand));
2098 : }
2099 15621023 : if (!VAR_P (var) && TREE_CODE (origvar) != SSA_NAME)
2100 : ;
2101 15621023 : else if (DECL_EXTERNAL (var))
2102 : ;
2103 15621023 : else if (DECL_HAS_VALUE_EXPR_P (var))
2104 : ;
2105 15604055 : else if (TREE_STATIC (var))
2106 : ;
2107 15604055 : else if (TREE_CODE (origvar) != SSA_NAME && DECL_RTL_SET_P (var))
2108 : ;
2109 15603846 : else if (TREE_TYPE (var) == error_mark_node)
2110 : {
2111 0 : if (really_expand)
2112 0 : expand_one_error_var (var);
2113 : }
2114 15603846 : else if (VAR_P (var) && DECL_HARD_REGISTER (var))
2115 : {
2116 4121 : if (really_expand)
2117 : {
2118 1085 : expand_one_hard_reg_var (var);
2119 1085 : if (!DECL_HARD_REGISTER (var))
2120 : /* Invalid register specification. */
2121 32 : expand_one_error_var (var);
2122 : }
2123 : }
2124 15599725 : else if (use_register_for_decl (var)
2125 15599725 : && (!forced_stack_var
2126 592653 : || !bitmap_bit_p (forced_stack_var, DECL_UID (var))))
2127 : {
2128 9270606 : if (really_expand)
2129 584068 : expand_one_register_var (origvar);
2130 : }
2131 6329119 : else if (!poly_int_tree_p (DECL_SIZE_UNIT (var), &size)
2132 6329119 : || !valid_constant_size_p (DECL_SIZE_UNIT (var)))
2133 : {
2134 : /* Reject variables which cover more than half of the address-space. */
2135 0 : if (really_expand)
2136 : {
2137 0 : if (DECL_NONLOCAL_FRAME (var))
2138 0 : error_at (DECL_SOURCE_LOCATION (current_function_decl),
2139 : "total size of local objects is too large");
2140 : else
2141 0 : error_at (DECL_SOURCE_LOCATION (var),
2142 : "size of variable %q+D is too large", var);
2143 0 : expand_one_error_var (var);
2144 : }
2145 : }
2146 6329119 : else if (defer_stack_allocation (var, toplevel))
2147 5832573 : add_stack_var (origvar, really_expand);
2148 : else
2149 : {
2150 496546 : if (really_expand)
2151 : {
2152 358940 : if (lookup_attribute ("naked",
2153 358940 : DECL_ATTRIBUTES (current_function_decl)))
2154 0 : error ("cannot allocate stack for variable %q+D, naked function",
2155 : var);
2156 :
2157 358940 : expand_one_stack_var (origvar);
2158 : }
2159 496546 : return size;
2160 : }
2161 15124477 : return 0;
2162 : }
2163 :
2164 : /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
2165 : expanding variables. Those variables that can be put into registers
2166 : are allocated pseudos; those that can't are put on the stack.
2167 :
2168 : TOPLEVEL is true if this is the outermost BLOCK. */
2169 :
2170 : static void
2171 17365650 : expand_used_vars_for_block (tree block, bool toplevel, bitmap forced_stack_vars)
2172 : {
2173 17365650 : tree t;
2174 :
2175 : /* Expand all variables at this level. */
2176 37911259 : for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
2177 20545609 : if (TREE_USED (t)
2178 20545609 : && ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
2179 772905 : || !DECL_NONSHAREABLE (t)))
2180 772905 : expand_one_var (t, toplevel, true, forced_stack_vars);
2181 :
2182 : /* Expand all variables at containing levels. */
2183 33215401 : for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
2184 15849751 : expand_used_vars_for_block (t, false, forced_stack_vars);
2185 17365650 : }
2186 :
2187 : /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
2188 : and clear TREE_USED on all local variables. */
2189 :
2190 : static void
2191 17365650 : clear_tree_used (tree block)
2192 : {
2193 17365650 : tree t;
2194 :
2195 37911259 : for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
2196 : /* if (!TREE_STATIC (t) && !DECL_EXTERNAL (t)) */
2197 1129223 : if ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
2198 20545609 : || !DECL_NONSHAREABLE (t))
2199 20545609 : TREE_USED (t) = 0;
2200 :
2201 33215401 : for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
2202 15849751 : clear_tree_used (t);
2203 17365650 : }
2204 :
2205 : /* Examine TYPE and determine a bit mask of the following features. */
2206 :
2207 : #define SPCT_HAS_LARGE_CHAR_ARRAY 1
2208 : #define SPCT_HAS_SMALL_CHAR_ARRAY 2
2209 : #define SPCT_HAS_ARRAY 4
2210 : #define SPCT_HAS_AGGREGATE 8
2211 :
2212 : static unsigned int
2213 2746 : stack_protect_classify_type (tree type)
2214 : {
2215 2746 : unsigned int ret = 0;
2216 2746 : tree t;
2217 :
2218 2746 : switch (TREE_CODE (type))
2219 : {
2220 710 : case ARRAY_TYPE:
2221 710 : t = TYPE_MAIN_VARIANT (TREE_TYPE (type));
2222 710 : if (t == char_type_node
2223 178 : || t == signed_char_type_node
2224 178 : || t == unsigned_char_type_node)
2225 : {
2226 532 : unsigned HOST_WIDE_INT max = param_ssp_buffer_size;
2227 532 : unsigned HOST_WIDE_INT len;
2228 :
2229 532 : if (!TYPE_SIZE_UNIT (type)
2230 532 : || !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)))
2231 : len = max;
2232 : else
2233 532 : len = tree_to_uhwi (TYPE_SIZE_UNIT (type));
2234 :
2235 532 : if (len < max)
2236 : ret = SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_ARRAY;
2237 : else
2238 500 : ret = SPCT_HAS_LARGE_CHAR_ARRAY | SPCT_HAS_ARRAY;
2239 : }
2240 : else
2241 : ret = SPCT_HAS_ARRAY;
2242 : break;
2243 :
2244 1021 : case UNION_TYPE:
2245 1021 : case QUAL_UNION_TYPE:
2246 1021 : case RECORD_TYPE:
2247 1021 : ret = SPCT_HAS_AGGREGATE;
2248 10915 : for (t = TYPE_FIELDS (type); t ; t = TREE_CHAIN (t))
2249 9894 : if (TREE_CODE (t) == FIELD_DECL)
2250 1702 : ret |= stack_protect_classify_type (TREE_TYPE (t));
2251 : break;
2252 :
2253 : default:
2254 : break;
2255 : }
2256 :
2257 2746 : return ret;
2258 : }
2259 :
2260 : /* Return nonzero if DECL should be segregated into the "vulnerable" upper
2261 : part of the local stack frame. Remember if we ever return nonzero for
2262 : any variable in this function. The return value is the phase number in
2263 : which the variable should be allocated. */
2264 :
2265 : static int
2266 1044 : stack_protect_decl_phase (tree decl)
2267 : {
2268 1044 : unsigned int bits = stack_protect_classify_type (TREE_TYPE (decl));
2269 1044 : int ret = 0;
2270 :
2271 1044 : if (bits & SPCT_HAS_SMALL_CHAR_ARRAY)
2272 28 : has_short_buffer = true;
2273 :
2274 1044 : tree attribs = DECL_ATTRIBUTES (current_function_decl);
2275 1044 : if (!lookup_attribute ("no_stack_protector", attribs)
2276 1044 : && (flag_stack_protect == SPCT_FLAG_ALL
2277 1044 : || flag_stack_protect == SPCT_FLAG_STRONG
2278 217 : || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2279 5 : && lookup_attribute ("stack_protect", attribs))))
2280 : {
2281 832 : if ((bits & (SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_LARGE_CHAR_ARRAY))
2282 368 : && !(bits & SPCT_HAS_AGGREGATE))
2283 : ret = 1;
2284 752 : else if (bits & SPCT_HAS_ARRAY)
2285 : ret = 2;
2286 : }
2287 : else
2288 212 : ret = (bits & SPCT_HAS_LARGE_CHAR_ARRAY) != 0;
2289 :
2290 212 : if (ret)
2291 692 : has_protected_decls = true;
2292 :
2293 1044 : return ret;
2294 : }
2295 :
2296 : /* Two helper routines that check for phase 1 and phase 2. These are used
2297 : as callbacks for expand_stack_vars. */
2298 :
2299 : static bool
2300 354 : stack_protect_decl_phase_1 (unsigned i)
2301 : {
2302 354 : return stack_protect_decl_phase (stack_vars[i].decl) == 1;
2303 : }
2304 :
2305 : static bool
2306 217 : stack_protect_decl_phase_2 (unsigned i)
2307 : {
2308 217 : return stack_protect_decl_phase (stack_vars[i].decl) == 2;
2309 : }
2310 :
2311 : /* And helper function that checks for asan phase (with stack protector
2312 : it is phase 3). This is used as callback for expand_stack_vars.
2313 : Returns true if any of the vars in the partition need to be protected. */
2314 :
2315 : static bool
2316 4026 : asan_decl_phase_3 (unsigned i)
2317 : {
2318 4831 : while (i != EOC)
2319 : {
2320 4026 : if (asan_protect_stack_decl (stack_vars[i].decl))
2321 : return true;
2322 805 : i = stack_vars[i].next;
2323 : }
2324 : return false;
2325 : }
2326 :
2327 : /* Ensure that variables in different stack protection phases conflict
2328 : so that they are not merged and share the same stack slot.
2329 : Return true if there are any address taken variables. */
2330 :
2331 : static bool
2332 262 : add_stack_protection_conflicts (void)
2333 : {
2334 262 : unsigned i, j, n = stack_vars_num;
2335 262 : unsigned char *phase;
2336 262 : bool ret = false;
2337 :
2338 262 : phase = XNEWVEC (unsigned char, n);
2339 997 : for (i = 0; i < n; ++i)
2340 : {
2341 473 : phase[i] = stack_protect_decl_phase (stack_vars[i].decl);
2342 473 : if (TREE_ADDRESSABLE (stack_vars[i].decl))
2343 396 : ret = true;
2344 : }
2345 :
2346 735 : for (i = 0; i < n; ++i)
2347 : {
2348 473 : unsigned char ph_i = phase[i];
2349 1189 : for (j = i + 1; j < n; ++j)
2350 716 : if (ph_i != phase[j])
2351 329 : add_stack_var_conflict (i, j);
2352 : }
2353 :
2354 262 : XDELETEVEC (phase);
2355 262 : return ret;
2356 : }
2357 :
2358 : /* Create a decl for the guard at the top of the stack frame. */
2359 :
2360 : static void
2361 297 : create_stack_guard (void)
2362 : {
2363 297 : tree guard = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
2364 : VAR_DECL, NULL, ptr_type_node);
2365 297 : TREE_THIS_VOLATILE (guard) = 1;
2366 297 : TREE_USED (guard) = 1;
2367 297 : expand_one_stack_var (guard);
2368 297 : crtl->stack_protect_guard = guard;
2369 297 : }
2370 :
2371 : /* Prepare for expanding variables. */
2372 : static void
2373 8068218 : init_vars_expansion (void)
2374 : {
2375 : /* Conflict bitmaps, and a few related temporary bitmaps, go here. */
2376 8068218 : bitmap_obstack_initialize (&stack_var_bitmap_obstack);
2377 :
2378 : /* A map from decl to stack partition. */
2379 8068218 : decl_to_stack_part = new hash_map<tree, unsigned>;
2380 :
2381 : /* Initialize local stack smashing state. */
2382 8068218 : has_protected_decls = false;
2383 8068218 : has_short_buffer = false;
2384 8068218 : if (hwassist_sanitize_stack_p ())
2385 1108 : hwasan_record_frame_init ();
2386 8068218 : }
2387 :
2388 : /* Free up stack variable graph data. */
2389 : static void
2390 8068218 : fini_vars_expansion (void)
2391 : {
2392 8068218 : bitmap_obstack_release (&stack_var_bitmap_obstack);
2393 8068218 : if (stack_vars)
2394 1322661 : XDELETEVEC (stack_vars);
2395 8068218 : if (stack_vars_sorted)
2396 1322661 : XDELETEVEC (stack_vars_sorted);
2397 8068218 : stack_vars = NULL;
2398 8068218 : stack_vars_sorted = NULL;
2399 8068218 : stack_vars_alloc = stack_vars_num = 0;
2400 16136436 : delete decl_to_stack_part;
2401 8068218 : decl_to_stack_part = NULL;
2402 8068218 : }
2403 :
2404 : /* Make a fair guess for the size of the stack frame of the function
2405 : in NODE. This doesn't have to be exact, the result is only used in
2406 : the inline heuristics. So we don't want to run the full stack var
2407 : packing algorithm (which is quadratic in the number of stack vars).
2408 : Instead, we calculate the total size of all stack vars. This turns
2409 : out to be a pretty fair estimate -- packing of stack vars doesn't
2410 : happen very often. */
2411 :
2412 : HOST_WIDE_INT
2413 6552319 : estimated_stack_frame_size (struct cgraph_node *node)
2414 : {
2415 6552319 : poly_int64 size = 0;
2416 6552319 : unsigned i;
2417 6552319 : tree var;
2418 6552319 : struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
2419 :
2420 6552319 : push_cfun (fn);
2421 :
2422 6552319 : init_vars_expansion ();
2423 :
2424 25957458 : FOR_EACH_LOCAL_DECL (fn, i, var)
2425 14039443 : if (auto_var_in_fn_p (var, fn->decl))
2426 13448199 : size += expand_one_var (var, true, false);
2427 :
2428 6552319 : if (stack_vars_num > 0)
2429 : {
2430 : /* Fake sorting the stack vars for account_stack_vars (). */
2431 1086734 : stack_vars_sorted = XNEWVEC (unsigned , stack_vars_num);
2432 5707643 : for (i = 0; i < stack_vars_num; ++i)
2433 4620909 : stack_vars_sorted[i] = i;
2434 1086734 : size += account_stack_vars ();
2435 : }
2436 :
2437 6552319 : fini_vars_expansion ();
2438 6552319 : pop_cfun ();
2439 6552319 : return estimated_poly_value (size);
2440 : }
2441 :
2442 : /* Check if the current function has calls that use a return slot. */
2443 :
2444 : static bool
2445 418 : stack_protect_return_slot_p ()
2446 : {
2447 418 : basic_block bb;
2448 :
2449 1876 : FOR_ALL_BB_FN (bb, cfun)
2450 2974 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2451 4028 : !gsi_end_p (gsi); gsi_next (&gsi))
2452 : {
2453 2570 : gimple *stmt = gsi_stmt (gsi);
2454 : /* This assumes that calls to internal-only functions never
2455 : use a return slot. */
2456 2570 : if (is_gimple_call (stmt)
2457 542 : && !gimple_call_internal_p (stmt)
2458 3095 : && aggregate_value_p (TREE_TYPE (gimple_call_fntype (stmt)),
2459 525 : gimple_call_fndecl (stmt)))
2460 418 : return true;
2461 : }
2462 : return false;
2463 : }
2464 :
2465 : /* Verify that partitions which claim to be the same object really are at the
2466 : same address. MEM_EXPR-based disambiguation identifies a location by a
2467 : MEM_EXPR base and an offset from it, so two stack slots carrying one
2468 : MEM_EXPR read as a single object, which lets an access to one be redirected
2469 : to the other. out-of-SSA keeps them apart, see the comment above
2470 : split_overlapping_partition_decls.
2471 :
2472 : Call this once the RTL of every partition is final. The partition of a
2473 : PARM_DECL or RESULT_DECL default definition is given the MEM_EXPR of any
2474 : other variable in it while its names are walked, and only gets the decl
2475 : back when its RTL is restored in pass_expand::execute. */
2476 :
2477 : static void
2478 1515879 : verify_partition_mem_exprs (void)
2479 : {
2480 1515879 : hash_map<tree, rtx> slots;
2481 29668553 : for (unsigned i = 0; i < num_var_partitions (SA.map); i++)
2482 : {
2483 26636795 : rtx x = SA.partition_to_pseudo[i];
2484 27885602 : if (!x || !MEM_P (x) || !MEM_EXPR (x))
2485 25394204 : continue;
2486 1242591 : bool existed;
2487 1242591 : rtx &known = slots.get_or_insert (MEM_EXPR (x), &existed);
2488 1242591 : if (!existed)
2489 1242591 : known = x;
2490 : else
2491 0 : gcc_assert (rtx_equal_p (XEXP (known, 0), XEXP (x, 0)));
2492 : }
2493 1515879 : }
2494 :
2495 : /* Expand all variables used in the function. */
2496 :
2497 : static rtx_insn *
2498 1515899 : expand_used_vars (bitmap forced_stack_vars)
2499 : {
2500 1515899 : tree var, outer_block = DECL_INITIAL (current_function_decl);
2501 1515899 : auto_vec<tree> maybe_local_decls;
2502 1515899 : rtx_insn *var_end_seq = NULL;
2503 1515899 : unsigned i;
2504 1515899 : unsigned len;
2505 1515899 : bool gen_stack_protect_signal = false;
2506 :
2507 : /* Compute the phase of the stack frame for this function. */
2508 1515899 : {
2509 1515899 : int align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2510 1515899 : int off = targetm.starting_frame_offset () % align;
2511 1515899 : frame_phase = off ? align - off : 0;
2512 : }
2513 :
2514 : /* Set TREE_USED on all variables in the local_decls. */
2515 12024682 : FOR_EACH_LOCAL_DECL (cfun, i, var)
2516 7971375 : TREE_USED (var) = 1;
2517 : /* Clear TREE_USED on all variables associated with a block scope. */
2518 1515899 : clear_tree_used (DECL_INITIAL (current_function_decl));
2519 :
2520 1515899 : init_vars_expansion ();
2521 :
2522 1515899 : if (targetm.use_pseudo_pic_reg ())
2523 80957 : pic_offset_table_rtx = gen_reg_rtx (Pmode);
2524 :
2525 28152753 : for (i = 0; i < SA.map->num_partitions; i++)
2526 : {
2527 26636854 : if (bitmap_bit_p (SA.partitions_for_parm_default_defs, i))
2528 3683352 : continue;
2529 :
2530 22953502 : tree var = partition_to_var (SA.map, i);
2531 :
2532 45907004 : gcc_assert (!virtual_operand_p (var));
2533 :
2534 22953502 : expand_one_ssa_partition (var);
2535 : }
2536 :
2537 1515899 : if (flag_stack_protect == SPCT_FLAG_STRONG)
2538 418 : gen_stack_protect_signal = stack_protect_return_slot_p ();
2539 :
2540 : /* At this point all variables on the local_decls with TREE_USED
2541 : set are not associated with any block scope. Lay them out. */
2542 :
2543 1515899 : len = vec_safe_length (cfun->local_decls);
2544 10755978 : FOR_EACH_LOCAL_DECL (cfun, i, var)
2545 : {
2546 7971375 : bool expand_now = false;
2547 :
2548 : /* Expanded above already. */
2549 7971375 : if (is_gimple_reg (var))
2550 : {
2551 5582321 : TREE_USED (var) = 0;
2552 5582321 : goto next;
2553 : }
2554 : /* We didn't set a block for static or extern because it's hard
2555 : to tell the difference between a global variable (re)declared
2556 : in a local scope, and one that's really declared there to
2557 : begin with. And it doesn't really matter much, since we're
2558 : not giving them stack space. Expand them now. */
2559 2389054 : else if (TREE_STATIC (var) || DECL_EXTERNAL (var))
2560 : expand_now = true;
2561 :
2562 : /* Expand variables not associated with any block now. Those created by
2563 : the optimizers could be live anywhere in the function. Those that
2564 : could possibly have been scoped originally and detached from their
2565 : block will have their allocation deferred so we coalesce them with
2566 : others when optimization is enabled. */
2567 2172824 : else if (TREE_USED (var))
2568 1703147 : expand_now = true;
2569 :
2570 : /* Finally, mark all variables on the list as used. We'll use
2571 : this in a moment when we expand those associated with scopes. */
2572 2389054 : TREE_USED (var) = 1;
2573 :
2574 2389054 : if (expand_now)
2575 1703147 : expand_one_var (var, true, true, forced_stack_vars);
2576 :
2577 685907 : next:
2578 7971375 : if (DECL_ARTIFICIAL (var) && !DECL_IGNORED_P (var))
2579 : {
2580 426432 : rtx rtl = DECL_RTL_IF_SET (var);
2581 :
2582 : /* Keep artificial non-ignored vars in cfun->local_decls
2583 : chain until instantiate_decls. */
2584 320385 : if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2585 22198 : add_local_decl (cfun, var);
2586 106047 : else if (rtl == NULL_RTX)
2587 : /* If rtl isn't set yet, which can happen e.g. with
2588 : -fstack-protector, retry before returning from this
2589 : function. */
2590 106047 : maybe_local_decls.safe_push (var);
2591 : }
2592 : }
2593 :
2594 : /* We duplicated some of the decls in CFUN->LOCAL_DECLS.
2595 :
2596 : +-----------------+-----------------+
2597 : | ...processed... | ...duplicates...|
2598 : +-----------------+-----------------+
2599 : ^
2600 : +-- LEN points here.
2601 :
2602 : We just want the duplicates, as those are the artificial
2603 : non-ignored vars that we want to keep until instantiate_decls.
2604 : Move them down and truncate the array. */
2605 1515899 : if (!vec_safe_is_empty (cfun->local_decls))
2606 854473 : cfun->local_decls->block_remove (0, len);
2607 :
2608 : /* At this point, all variables within the block tree with TREE_USED
2609 : set are actually used by the optimized function. Lay them out. */
2610 1515899 : expand_used_vars_for_block (outer_block, true, forced_stack_vars);
2611 :
2612 1515899 : tree attribs = DECL_ATTRIBUTES (current_function_decl);
2613 1515899 : if (stack_vars_num > 0)
2614 : {
2615 235927 : bool has_addressable_vars = false;
2616 :
2617 235927 : add_scope_conflicts ();
2618 :
2619 : /* If stack protection is enabled, we don't share space between
2620 : vulnerable data and non-vulnerable data. */
2621 235927 : if (flag_stack_protect != 0
2622 262 : && !lookup_attribute ("no_stack_protector", attribs)
2623 236189 : && (flag_stack_protect != SPCT_FLAG_EXPLICIT
2624 : || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2625 1 : && lookup_attribute ("stack_protect", attribs))))
2626 262 : has_addressable_vars = add_stack_protection_conflicts ();
2627 :
2628 235927 : if (flag_stack_protect == SPCT_FLAG_STRONG && has_addressable_vars)
2629 235927 : gen_stack_protect_signal = true;
2630 :
2631 : /* Now that we have collected all stack variables, and have computed a
2632 : minimal interference graph, attempt to save some stack space. */
2633 235927 : partition_stack_vars ();
2634 235927 : if (dump_file)
2635 33 : dump_stack_var_partition ();
2636 : }
2637 :
2638 :
2639 1515899 : if (!lookup_attribute ("no_stack_protector", attribs))
2640 1515890 : switch (flag_stack_protect)
2641 : {
2642 51 : case SPCT_FLAG_ALL:
2643 51 : create_stack_guard ();
2644 51 : break;
2645 :
2646 418 : case SPCT_FLAG_STRONG:
2647 418 : if (gen_stack_protect_signal
2648 274 : || cfun->calls_alloca
2649 272 : || has_protected_decls
2650 681 : || lookup_attribute ("stack_protect", attribs))
2651 155 : create_stack_guard ();
2652 : break;
2653 :
2654 165 : case SPCT_FLAG_DEFAULT:
2655 165 : if (cfun->calls_alloca
2656 162 : || has_protected_decls
2657 249 : || lookup_attribute ("stack_protect", attribs))
2658 81 : create_stack_guard ();
2659 : break;
2660 :
2661 14 : case SPCT_FLAG_EXPLICIT:
2662 14 : if (lookup_attribute ("stack_protect", attribs))
2663 10 : create_stack_guard ();
2664 : break;
2665 :
2666 : default:
2667 : break;
2668 : }
2669 :
2670 : /* Assign rtl to each variable based on these partitions. */
2671 1515899 : if (stack_vars_num > 0)
2672 : {
2673 235927 : class stack_vars_data data;
2674 :
2675 235927 : data.asan_base = NULL_RTX;
2676 235927 : data.asan_alignb = 0;
2677 :
2678 : /* Reorder decls to be protected by iterating over the variables
2679 : array multiple times, and allocating out of each phase in turn. */
2680 : /* ??? We could probably integrate this into the qsort we did
2681 : earlier, such that we naturally see these variables first,
2682 : and thus naturally allocate things in the right order. */
2683 235927 : if (has_protected_decls)
2684 : {
2685 : /* Phase 1 contains only character arrays. */
2686 167 : expand_stack_vars (stack_protect_decl_phase_1, &data);
2687 :
2688 : /* Phase 2 contains other kinds of arrays. */
2689 167 : if (!lookup_attribute ("no_stack_protector", attribs)
2690 167 : && (flag_stack_protect == SPCT_FLAG_ALL
2691 167 : || flag_stack_protect == SPCT_FLAG_STRONG
2692 79 : || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2693 1 : && lookup_attribute ("stack_protect", attribs))))
2694 89 : expand_stack_vars (stack_protect_decl_phase_2, &data);
2695 : }
2696 :
2697 235927 : if (asan_sanitize_stack_p ())
2698 : /* Phase 3, any partitions that need asan protection
2699 : in addition to phase 1 and 2. */
2700 1930 : expand_stack_vars (asan_decl_phase_3, &data);
2701 :
2702 : /* ASAN description strings don't yet have a syntax for expressing
2703 : polynomial offsets. */
2704 235927 : HOST_WIDE_INT prev_offset;
2705 237648 : if (!data.asan_vec.is_empty ()
2706 1721 : && frame_offset.is_constant (&prev_offset))
2707 : {
2708 1721 : HOST_WIDE_INT offset, sz, redzonesz;
2709 1721 : redzonesz = ASAN_RED_ZONE_SIZE;
2710 1721 : sz = data.asan_vec[0] - prev_offset;
2711 1721 : data.asan_alignb = MAX (data.asan_alignb,
2712 : crtl->stack_alignment_needed
2713 : / BITS_PER_UNIT);
2714 1721 : if (data.asan_alignb > ASAN_RED_ZONE_SIZE
2715 1721 : && data.asan_alignb <= 4096
2716 54 : && sz + ASAN_RED_ZONE_SIZE >= (int) data.asan_alignb)
2717 54 : redzonesz = ((sz + ASAN_RED_ZONE_SIZE + data.asan_alignb - 1)
2718 54 : & ~(data.asan_alignb - HOST_WIDE_INT_1)) - sz;
2719 : /* Allocating a constant amount of space from a constant
2720 : starting offset must give a constant result. */
2721 1721 : offset = (alloc_stack_frame_space (redzonesz, ASAN_RED_ZONE_SIZE)
2722 1721 : .to_constant ());
2723 1721 : data.asan_vec.safe_push (prev_offset);
2724 1721 : data.asan_vec.safe_push (offset);
2725 : /* Leave space for alignment if STRICT_ALIGNMENT. */
2726 1721 : if (STRICT_ALIGNMENT)
2727 : alloc_stack_frame_space ((GET_MODE_ALIGNMENT (SImode)
2728 : << ASAN_SHADOW_SHIFT)
2729 : / BITS_PER_UNIT, 1);
2730 :
2731 1721 : var_end_seq
2732 3442 : = asan_emit_stack_protection (virtual_stack_vars_rtx,
2733 : data.asan_base,
2734 : data.asan_alignb,
2735 : data.asan_vec.address (),
2736 : data.asan_decl_vec.address (),
2737 1721 : data.asan_vec.length ());
2738 : }
2739 :
2740 235927 : expand_stack_vars (NULL, &data);
2741 235927 : }
2742 :
2743 1515899 : if (hwassist_sanitize_stack_p ())
2744 364 : hwasan_emit_prologue ();
2745 1515899 : if (asan_sanitize_allocas_p () && cfun->calls_alloca)
2746 178 : var_end_seq = asan_emit_allocas_unpoison (virtual_stack_dynamic_rtx,
2747 : virtual_stack_vars_rtx,
2748 : var_end_seq);
2749 3031078 : else if ((hwasan_sanitize_allocas_p () || memtag_sanitize_p ())
2750 1515721 : && cfun->calls_alloca)
2751 : /* When using out-of-line instrumentation we only want to emit one function
2752 : call for clearing the tags in a region of shadow stack. When there are
2753 : alloca calls in this frame we want to emit a call using the
2754 : virtual_stack_dynamic_rtx, but when not we use the hwasan_frame_extent
2755 : rtx we created in expand_stack_vars. */
2756 0 : var_end_seq = hwasan_emit_untag_frame (virtual_stack_dynamic_rtx,
2757 : virtual_stack_vars_rtx);
2758 1515721 : else if (hwassist_sanitize_stack_p ())
2759 : /* If no variables were stored on the stack, `hwasan_get_frame_extent`
2760 : will return NULL_RTX and hence `hwasan_emit_untag_frame` will return
2761 : NULL (i.e. an empty sequence). */
2762 364 : var_end_seq = hwasan_emit_untag_frame (hwasan_get_frame_extent (),
2763 : virtual_stack_vars_rtx);
2764 :
2765 1515899 : fini_vars_expansion ();
2766 :
2767 : /* If there were any artificial non-ignored vars without rtl
2768 : found earlier, see if deferred stack allocation hasn't assigned
2769 : rtl to them. */
2770 1706628 : FOR_EACH_VEC_ELT_REVERSE (maybe_local_decls, i, var)
2771 : {
2772 106047 : rtx rtl = DECL_RTL_IF_SET (var);
2773 :
2774 : /* Keep artificial non-ignored vars in cfun->local_decls
2775 : chain until instantiate_decls. */
2776 5073 : if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2777 4820 : add_local_decl (cfun, var);
2778 : }
2779 :
2780 : /* If the target requires that FRAME_OFFSET be aligned, do it. */
2781 1515899 : if (STACK_ALIGNMENT_NEEDED)
2782 : {
2783 1515899 : HOST_WIDE_INT align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2784 1515899 : if (FRAME_GROWS_DOWNWARD)
2785 1515899 : frame_offset = aligned_lower_bound (frame_offset, align);
2786 : else
2787 : frame_offset = aligned_upper_bound (frame_offset, align);
2788 : }
2789 :
2790 1515899 : return var_end_seq;
2791 1515899 : }
2792 :
2793 :
2794 : /* If we need to produce a detailed dump, print the tree representation
2795 : for STMT to the dump file. SINCE is the last RTX after which the RTL
2796 : generated for STMT should have been appended. */
2797 :
2798 : static void
2799 42833835 : maybe_dump_rtl_for_gimple_stmt (gimple *stmt, rtx_insn *since)
2800 : {
2801 42833835 : if (dump_file && (dump_flags & TDF_DETAILS))
2802 : {
2803 377 : fprintf (dump_file, "\n;; ");
2804 377 : print_gimple_stmt (dump_file, stmt, 0,
2805 : TDF_SLIM | (dump_flags & TDF_LINENO));
2806 377 : fprintf (dump_file, "\n");
2807 :
2808 377 : print_rtl (dump_file, since ? NEXT_INSN (since) : since);
2809 : }
2810 42833835 : }
2811 :
2812 : /* Temporary storage for BB_HEAD and BB_END of bbs until they are converted
2813 : to BB_RTL. */
2814 : static vec<std::pair <rtx_insn *, rtx_insn *>> head_end_for_bb;
2815 :
2816 : /* Maps the blocks that do not contain tree labels to rtx labels. */
2817 :
2818 : static hash_map<basic_block, rtx_code_label *> *lab_rtx_for_bb;
2819 :
2820 : /* Returns the label_rtx expression for a label starting basic block BB. */
2821 :
2822 : static rtx_code_label *
2823 7843356 : label_rtx_for_bb (basic_block bb)
2824 : {
2825 7843356 : if (bb->flags & BB_RTL)
2826 0 : return block_label (bb);
2827 :
2828 7843356 : if ((unsigned) bb->index < head_end_for_bb.length ()
2829 7843356 : && head_end_for_bb[bb->index].first)
2830 : {
2831 1595015 : if (!LABEL_P (head_end_for_bb[bb->index].first))
2832 : {
2833 968154 : head_end_for_bb[bb->index].first
2834 1936308 : = emit_label_before (gen_label_rtx (),
2835 968154 : head_end_for_bb[bb->index].first);
2836 : }
2837 1595015 : return as_a <rtx_code_label *> (head_end_for_bb[bb->index].first);
2838 : }
2839 :
2840 6248341 : rtx_code_label **elt = lab_rtx_for_bb->get (bb);
2841 6248341 : if (elt)
2842 1299922 : return *elt;
2843 :
2844 : /* Find the tree label if it is present. */
2845 4948419 : gimple_stmt_iterator gsi = gsi_start_bb (bb);
2846 4948419 : glabel *lab_stmt;
2847 4948419 : if (!gsi_end_p (gsi)
2848 4802635 : && (lab_stmt = dyn_cast <glabel *> (gsi_stmt (gsi)))
2849 5184912 : && !DECL_NONLOCAL (gimple_label_label (lab_stmt)))
2850 236493 : return jump_target_rtx (gimple_label_label (lab_stmt));
2851 :
2852 4711926 : rtx_code_label *l = gen_label_rtx ();
2853 4711926 : lab_rtx_for_bb->put (bb, l);
2854 4711926 : return l;
2855 : }
2856 :
2857 :
2858 : /* Wrapper around remove_edge during expansion. */
2859 :
2860 : void
2861 131161 : expand_remove_edge (edge e)
2862 : {
2863 131161 : if (current_ir_type () != IR_GIMPLE
2864 131161 : && (e->dest->flags & BB_RTL) == 0
2865 192401 : && !gimple_seq_empty_p (phi_nodes (e->dest)))
2866 9467 : remove_phi_args (e);
2867 131161 : remove_edge (e);
2868 131161 : }
2869 :
2870 : /* Split edge E during expansion and instead of creating a new
2871 : bb on that edge, add there BB. FLAGS should be flags on the
2872 : new edge from BB to former E->dest. */
2873 :
2874 : static void
2875 1918259 : expand_split_edge (edge e, basic_block bb, int flags)
2876 : {
2877 1918259 : unsigned int dest_idx = e->dest_idx;
2878 1918259 : basic_block dest = e->dest;
2879 1918259 : redirect_edge_succ (e, bb);
2880 1918259 : e = make_single_succ_edge (bb, dest, flags);
2881 1918259 : if ((dest->flags & BB_RTL) == 0
2882 1918259 : && phi_nodes (dest)
2883 2055222 : && e->dest_idx != dest_idx)
2884 : {
2885 : /* If there are any PHI nodes on dest, swap the new succ edge
2886 : with the one moved into false_edge's former position, so that
2887 : PHI arguments don't need adjustment. */
2888 85016 : edge e2 = EDGE_PRED (dest, dest_idx);
2889 85016 : std::swap (e->dest_idx, e2->dest_idx);
2890 85016 : std::swap (EDGE_PRED (dest, e->dest_idx),
2891 : EDGE_PRED (dest, e2->dest_idx));
2892 : }
2893 1918259 : }
2894 :
2895 :
2896 : /* A subroutine of expand_gimple_cond. Given E, a fallthrough edge
2897 : of a basic block where we just expanded the conditional at the end,
2898 : possibly clean up the CFG and instruction sequence. LAST is the
2899 : last instruction before the just emitted jump sequence. */
2900 :
2901 : static void
2902 5374540 : maybe_cleanup_end_of_block (edge e, rtx_insn *last)
2903 : {
2904 : /* Special case: when jumpif decides that the condition is
2905 : trivial it emits an unconditional jump (and the necessary
2906 : barrier). But we still have two edges, the fallthru one is
2907 : wrong. purge_dead_edges would clean this up later. Unfortunately
2908 : we have to insert insns (and split edges) before
2909 : find_many_sub_basic_blocks and hence before purge_dead_edges.
2910 : But splitting edges might create new blocks which depend on the
2911 : fact that if there are two edges there's no barrier. So the
2912 : barrier would get lost and verify_flow_info would ICE. Instead
2913 : of auditing all edge splitters to care for the barrier (which
2914 : normally isn't there in a cleaned CFG), fix it here. */
2915 5374540 : if (BARRIER_P (get_last_insn ()))
2916 : {
2917 122 : rtx_insn *insn;
2918 122 : expand_remove_edge (e);
2919 : /* Now, we have a single successor block, if we have insns to
2920 : insert on the remaining edge we potentially will insert
2921 : it at the end of this block (if the dest block isn't feasible)
2922 : in order to avoid splitting the edge. This insertion will take
2923 : place in front of the last jump. But we might have emitted
2924 : multiple jumps (conditional and one unconditional) to the
2925 : same destination. Inserting in front of the last one then
2926 : is a problem. See PR 40021. We fix this by deleting all
2927 : jumps except the last unconditional one. */
2928 122 : insn = PREV_INSN (get_last_insn ());
2929 : /* Make sure we have an unconditional jump. Otherwise we're
2930 : confused. */
2931 122 : gcc_assert (JUMP_P (insn) && !any_condjump_p (insn));
2932 244 : for (insn = PREV_INSN (insn); insn != last;)
2933 : {
2934 0 : insn = PREV_INSN (insn);
2935 0 : if (JUMP_P (NEXT_INSN (insn)))
2936 : {
2937 0 : if (!any_condjump_p (NEXT_INSN (insn)))
2938 : {
2939 0 : gcc_assert (BARRIER_P (NEXT_INSN (NEXT_INSN (insn))));
2940 0 : delete_insn (NEXT_INSN (NEXT_INSN (insn)));
2941 : }
2942 0 : delete_insn (NEXT_INSN (insn));
2943 : }
2944 : }
2945 : }
2946 5374540 : }
2947 :
2948 : /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_COND.
2949 : Returns a new basic block if we've terminated the current basic
2950 : block and created a new one. */
2951 :
2952 : static basic_block
2953 5776900 : expand_gimple_cond (basic_block bb, gcond *stmt)
2954 : {
2955 5776900 : basic_block new_bb, dest;
2956 5776900 : edge true_edge;
2957 5776900 : edge false_edge;
2958 5776900 : rtx_insn *last2, *last;
2959 5776900 : enum tree_code code;
2960 5776900 : tree op0, op1;
2961 :
2962 5776900 : code = gimple_cond_code (stmt);
2963 5776900 : op0 = gimple_cond_lhs (stmt);
2964 5776900 : op1 = gimple_cond_rhs (stmt);
2965 : /* We're sometimes presented with such code:
2966 : D.123_1 = x < y;
2967 : if (D.123_1 != 0)
2968 : ...
2969 : This would expand to two comparisons which then later might
2970 : be cleaned up by combine. But some pattern matchers like if-conversion
2971 : work better when there's only one compare, so make up for this
2972 : here as special exception if TER would have made the same change. */
2973 5776900 : if (SA.values
2974 4229732 : && TREE_CODE (op0) == SSA_NAME
2975 4229021 : && TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE
2976 398422 : && TREE_CODE (op1) == INTEGER_CST
2977 388061 : && ((gimple_cond_code (stmt) == NE_EXPR
2978 388059 : && integer_zerop (op1))
2979 2 : || (gimple_cond_code (stmt) == EQ_EXPR
2980 2 : && integer_onep (op1)))
2981 6164961 : && bitmap_bit_p (SA.values, SSA_NAME_VERSION (op0)))
2982 : {
2983 210064 : gimple *second = SSA_NAME_DEF_STMT (op0);
2984 210064 : if (gimple_code (second) == GIMPLE_ASSIGN)
2985 : {
2986 210045 : enum tree_code code2 = gimple_assign_rhs_code (second);
2987 210045 : if (TREE_CODE_CLASS (code2) == tcc_comparison)
2988 : {
2989 902 : code = code2;
2990 902 : op0 = gimple_assign_rhs1 (second);
2991 902 : op1 = gimple_assign_rhs2 (second);
2992 : }
2993 : /* If jumps are cheap and the target does not support conditional
2994 : compare, turn some more codes into jumpy sequences. */
2995 209143 : else if (BRANCH_COST (optimize_insn_for_speed_p (), false) < 4
2996 209143 : && !targetm.have_ccmp ())
2997 : {
2998 209140 : if ((code2 == BIT_AND_EXPR
2999 68526 : && TYPE_PRECISION (TREE_TYPE (op0)) == 1
3000 68526 : && TREE_CODE (gimple_assign_rhs2 (second)) != INTEGER_CST)
3001 209140 : || code2 == TRUTH_AND_EXPR)
3002 : {
3003 68526 : code = TRUTH_ANDIF_EXPR;
3004 68526 : op0 = gimple_assign_rhs1 (second);
3005 68526 : op1 = gimple_assign_rhs2 (second);
3006 : }
3007 140614 : else if (code2 == BIT_IOR_EXPR || code2 == TRUTH_OR_EXPR)
3008 : {
3009 62738 : code = TRUTH_ORIF_EXPR;
3010 62738 : op0 = gimple_assign_rhs1 (second);
3011 62738 : op1 = gimple_assign_rhs2 (second);
3012 : }
3013 : }
3014 : }
3015 : }
3016 :
3017 : /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
3018 : into (x - C2) * C3 < C4. */
3019 5776900 : if ((code == EQ_EXPR || code == NE_EXPR)
3020 4515585 : && TREE_CODE (op0) == SSA_NAME
3021 4514820 : && TREE_CODE (op1) == INTEGER_CST)
3022 3124575 : code = maybe_optimize_mod_cmp (code, &op0, &op1);
3023 :
3024 : /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
3025 5776900 : if (!TYPE_UNSIGNED (TREE_TYPE (op0))
3026 2317813 : && (code == LT_EXPR || code == LE_EXPR
3027 2317813 : || code == GT_EXPR || code == GE_EXPR)
3028 700477 : && integer_zerop (op1)
3029 5979878 : && TREE_CODE (op0) == SSA_NAME)
3030 202978 : maybe_optimize_sub_cmp_0 (code, &op0, &op1);
3031 :
3032 5776900 : last2 = last = get_last_insn ();
3033 :
3034 5776900 : extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
3035 5776900 : set_curr_insn_location (gimple_location (stmt));
3036 :
3037 : /* We can either have a pure conditional jump with one fallthru edge or
3038 : two-way jump that needs to be decomposed into two basic blocks. */
3039 5776900 : if (false_edge->dest == bb->next_bb)
3040 : {
3041 2310994 : jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
3042 : true_edge->probability);
3043 2310994 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
3044 2310994 : if (true_edge->goto_locus != UNKNOWN_LOCATION)
3045 355129 : set_curr_insn_location (true_edge->goto_locus);
3046 2310994 : false_edge->flags |= EDGE_FALLTHRU;
3047 2310994 : maybe_cleanup_end_of_block (false_edge, last);
3048 2310994 : return NULL;
3049 : }
3050 3465906 : if (true_edge->dest == bb->next_bb)
3051 : {
3052 3063546 : jumpifnot_1 (code, op0, op1, label_rtx_for_bb (false_edge->dest),
3053 : false_edge->probability);
3054 3063546 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
3055 3063546 : if (false_edge->goto_locus != UNKNOWN_LOCATION)
3056 24558 : set_curr_insn_location (false_edge->goto_locus);
3057 3063546 : true_edge->flags |= EDGE_FALLTHRU;
3058 3063546 : maybe_cleanup_end_of_block (true_edge, last);
3059 3063546 : return NULL;
3060 : }
3061 :
3062 402360 : jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
3063 : true_edge->probability);
3064 402360 : last = get_last_insn ();
3065 402360 : if (false_edge->goto_locus != UNKNOWN_LOCATION)
3066 1346 : set_curr_insn_location (false_edge->goto_locus);
3067 402360 : emit_jump (label_rtx_for_bb (false_edge->dest));
3068 :
3069 402360 : head_end_for_bb[bb->index].second = last;
3070 402360 : if (BARRIER_P (head_end_for_bb[bb->index].second))
3071 98 : head_end_for_bb[bb->index].second
3072 49 : = PREV_INSN (head_end_for_bb[bb->index].second);
3073 804720 : update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
3074 402360 : head_end_for_bb[bb->index].second, bb);
3075 :
3076 402360 : new_bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
3077 402360 : dest = false_edge->dest;
3078 402360 : expand_split_edge (false_edge, new_bb, 0);
3079 402360 : false_edge->flags |= EDGE_FALLTHRU;
3080 402360 : new_bb->count = false_edge->count ();
3081 402360 : loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
3082 402360 : add_bb_to_loop (new_bb, loop);
3083 402360 : if (loop->latch == bb
3084 27532 : && loop->header == dest)
3085 27524 : loop->latch = new_bb;
3086 402360 : if (BARRIER_P (BB_END (new_bb)))
3087 402360 : BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
3088 402360 : update_bb_for_insn (new_bb);
3089 :
3090 402360 : maybe_dump_rtl_for_gimple_stmt (stmt, last2);
3091 :
3092 402360 : if (true_edge->goto_locus != UNKNOWN_LOCATION)
3093 : {
3094 45946 : set_curr_insn_location (true_edge->goto_locus);
3095 45946 : true_edge->goto_locus = curr_insn_location ();
3096 : }
3097 :
3098 : return new_bb;
3099 : }
3100 :
3101 : /* Mark all calls that can have a transaction restart. */
3102 :
3103 : static void
3104 6805641 : mark_transaction_restart_calls (gimple *stmt)
3105 : {
3106 6805641 : struct tm_restart_node dummy;
3107 6805641 : tm_restart_node **slot;
3108 :
3109 6805641 : if (!cfun->gimple_df->tm_restart)
3110 6802657 : return;
3111 :
3112 2984 : dummy.stmt = stmt;
3113 2984 : slot = cfun->gimple_df->tm_restart->find_slot (&dummy, NO_INSERT);
3114 2984 : if (slot)
3115 : {
3116 0 : struct tm_restart_node *n = *slot;
3117 0 : tree list = n->label_or_list;
3118 0 : rtx_insn *insn;
3119 :
3120 0 : for (insn = next_real_insn (get_last_insn ());
3121 0 : !CALL_P (insn);
3122 0 : insn = next_real_insn (insn))
3123 0 : continue;
3124 :
3125 0 : if (TREE_CODE (list) == LABEL_DECL)
3126 0 : add_reg_note (insn, REG_TM, label_rtx (list));
3127 : else
3128 0 : for (; list ; list = TREE_CHAIN (list))
3129 0 : add_reg_note (insn, REG_TM, label_rtx (TREE_VALUE (list)));
3130 0 : }
3131 : }
3132 :
3133 : /* A subroutine of expand_gimple_stmt_1, expanding one GIMPLE_CALL
3134 : statement STMT. */
3135 :
3136 : static void
3137 7035961 : expand_call_stmt (gcall *stmt)
3138 : {
3139 7035961 : tree exp, decl, lhs;
3140 7035961 : bool builtin_p;
3141 7035961 : size_t i;
3142 :
3143 7035961 : if (gimple_call_internal_p (stmt))
3144 : {
3145 197026 : expand_internal_call (stmt);
3146 197026 : return;
3147 : }
3148 :
3149 : /* If this is a call to a built-in function and it has no effect other
3150 : than setting the lhs, try to implement it using an internal function
3151 : instead. */
3152 6838935 : decl = gimple_call_fndecl (stmt);
3153 6838935 : if (gimple_call_lhs (stmt)
3154 2403275 : && !gimple_has_side_effects (stmt)
3155 7551079 : && (optimize || (decl && called_as_built_in (decl))))
3156 : {
3157 691566 : internal_fn ifn = replacement_internal_fn (stmt);
3158 691566 : if (ifn != IFN_LAST)
3159 : {
3160 33292 : expand_internal_call (ifn, stmt);
3161 33292 : return;
3162 : }
3163 : }
3164 :
3165 6805643 : exp = build_vl_exp (CALL_EXPR, gimple_call_num_args (stmt) + 3);
3166 :
3167 6805643 : CALL_EXPR_FN (exp) = gimple_call_fn (stmt);
3168 6805643 : builtin_p = decl && fndecl_built_in_p (decl);
3169 :
3170 : /* If this is not a builtin function, the function type through which the
3171 : call is made may be different from the type of the function. */
3172 4794631 : if (!builtin_p)
3173 9589262 : CALL_EXPR_FN (exp)
3174 14383893 : = fold_convert (build_pointer_type (gimple_call_fntype (stmt)),
3175 : CALL_EXPR_FN (exp));
3176 :
3177 6805643 : TREE_TYPE (exp) = gimple_call_return_type (stmt);
3178 6805643 : CALL_EXPR_STATIC_CHAIN (exp) = gimple_call_chain (stmt);
3179 :
3180 20521556 : for (i = 0; i < gimple_call_num_args (stmt); i++)
3181 : {
3182 13715913 : tree arg = gimple_call_arg (stmt, i);
3183 13715913 : gimple *def;
3184 : /* TER addresses into arguments of builtin functions so we have a
3185 : chance to infer more correct alignment information. See PR39954. */
3186 13715913 : if (builtin_p
3187 4185271 : && TREE_CODE (arg) == SSA_NAME
3188 1435583 : && (def = get_gimple_for_ssa_name (arg))
3189 352279 : && is_gimple_assign (def)
3190 14068031 : && gimple_assign_rhs_code (def) == ADDR_EXPR)
3191 14691 : arg = gimple_assign_rhs1 (def);
3192 13715913 : CALL_EXPR_ARG (exp, i) = arg;
3193 : }
3194 :
3195 6805643 : if (gimple_has_side_effects (stmt)
3196 : /* ??? Downstream in expand_expr_real_1 we assume that expressions
3197 : w/o side-effects do not throw so work around this here. */
3198 6805643 : || stmt_could_throw_p (cfun, stmt))
3199 6128762 : TREE_SIDE_EFFECTS (exp) = 1;
3200 :
3201 6805643 : if (gimple_call_nothrow_p (stmt))
3202 3042363 : TREE_NOTHROW (exp) = 1;
3203 :
3204 6805643 : CALL_EXPR_TAILCALL (exp) = gimple_call_tail_p (stmt);
3205 6805643 : CALL_EXPR_MUST_TAIL_CALL (exp) = gimple_call_must_tail_p (stmt);
3206 6805643 : CALL_EXPR_RETURN_SLOT_OPT (exp) = gimple_call_return_slot_opt_p (stmt);
3207 6805643 : if (decl
3208 6618738 : && fndecl_built_in_p (decl, BUILT_IN_NORMAL)
3209 8639531 : && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
3210 27808 : CALL_ALLOCA_FOR_VAR_P (exp) = gimple_call_alloca_for_var_p (stmt);
3211 : else
3212 6777835 : CALL_FROM_THUNK_P (exp) = gimple_call_from_thunk_p (stmt);
3213 6805643 : CALL_EXPR_VA_ARG_PACK (exp) = gimple_call_va_arg_pack_p (stmt);
3214 6805643 : CALL_EXPR_BY_DESCRIPTOR (exp) = gimple_call_by_descriptor_p (stmt);
3215 6805643 : SET_EXPR_LOCATION (exp, gimple_location (stmt));
3216 :
3217 : /* Must come after copying location. */
3218 6805643 : copy_warning (exp, stmt);
3219 :
3220 : /* Ensure RTL is created for debug args. */
3221 13424381 : if (decl && DECL_HAS_DEBUG_ARGS_P (decl))
3222 : {
3223 64753 : vec<tree, va_gc> **debug_args = decl_debug_args_lookup (decl);
3224 64753 : unsigned int ix;
3225 64753 : tree dtemp;
3226 :
3227 64753 : if (debug_args)
3228 142881 : for (ix = 1; (*debug_args)->iterate (ix, &dtemp); ix += 2)
3229 : {
3230 78128 : gcc_assert (TREE_CODE (dtemp) == DEBUG_EXPR_DECL);
3231 78128 : expand_debug_expr (dtemp);
3232 : }
3233 : }
3234 :
3235 6805643 : rtx_insn *before_call = get_last_insn ();
3236 6805643 : lhs = gimple_call_lhs (stmt);
3237 6805643 : if (lhs)
3238 2369983 : expand_assignment (lhs, exp, false);
3239 : else
3240 4435660 : expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
3241 :
3242 : /* If the gimple call is an indirect call and has 'nocf_check'
3243 : attribute find a generated CALL insn to mark it as no
3244 : control-flow verification is needed. */
3245 6805641 : if (gimple_call_nocf_check_p (stmt)
3246 6805662 : && !gimple_call_fndecl (stmt))
3247 : {
3248 20 : rtx_insn *last = get_last_insn ();
3249 20 : while (!CALL_P (last)
3250 34 : && last != before_call)
3251 14 : last = PREV_INSN (last);
3252 :
3253 20 : if (last != before_call)
3254 20 : add_reg_note (last, REG_CALL_NOCF_CHECK, const0_rtx);
3255 : }
3256 :
3257 6805641 : mark_transaction_restart_calls (stmt);
3258 : }
3259 :
3260 :
3261 : /* Generate RTL for an asm statement (explicit assembler code).
3262 : STRING is a STRING_CST node containing the assembler code text,
3263 : or an ADDR_EXPR containing a STRING_CST. VOL nonzero means the
3264 : insn is volatile; don't optimize it. */
3265 :
3266 : static void
3267 2853 : expand_asm_loc (tree string, int vol, location_t locus)
3268 : {
3269 2853 : rtx body;
3270 :
3271 2853 : body = gen_rtx_ASM_INPUT_loc (VOIDmode,
3272 : ggc_strdup (TREE_STRING_POINTER (string)),
3273 : locus);
3274 :
3275 2853 : MEM_VOLATILE_P (body) = vol;
3276 :
3277 : /* Non-empty basic ASM implicitly clobbers memory. */
3278 2853 : if (TREE_STRING_LENGTH (string) != 0)
3279 : {
3280 835 : rtx asm_op, clob;
3281 835 : unsigned i, nclobbers;
3282 835 : auto_vec<rtx> input_rvec, output_rvec;
3283 835 : auto_vec<machine_mode> input_mode;
3284 835 : auto_vec<const char *> constraints;
3285 835 : auto_vec<rtx> use_rvec;
3286 835 : auto_vec<rtx> clobber_rvec;
3287 835 : HARD_REG_SET clobbered_regs;
3288 835 : CLEAR_HARD_REG_SET (clobbered_regs);
3289 :
3290 835 : clob = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3291 835 : clobber_rvec.safe_push (clob);
3292 :
3293 835 : if (targetm.md_asm_adjust)
3294 835 : targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3295 : constraints, use_rvec, clobber_rvec,
3296 : clobbered_regs, locus);
3297 :
3298 835 : asm_op = body;
3299 835 : nclobbers = clobber_rvec.length ();
3300 835 : auto nuses = use_rvec.length ();
3301 835 : body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (1 + nuses + nclobbers));
3302 :
3303 835 : i = 0;
3304 835 : XVECEXP (body, 0, i++) = asm_op;
3305 835 : for (rtx use : use_rvec)
3306 0 : XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
3307 4175 : for (rtx clobber : clobber_rvec)
3308 1670 : XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobber);
3309 835 : }
3310 :
3311 2853 : emit_insn (body);
3312 2853 : }
3313 :
3314 : /* Check for overlap between registers marked in CLOBBERED_REGS and
3315 : anything inappropriate in T. Emit error and return the register
3316 : variable definition for error, NULL_TREE for ok. */
3317 :
3318 : static bool
3319 138568 : tree_conflicts_with_clobbers_p (tree t, HARD_REG_SET *clobbered_regs,
3320 : location_t loc)
3321 : {
3322 : /* Conflicts between asm-declared register variables and the clobber
3323 : list are not allowed. */
3324 138568 : tree overlap = tree_overlaps_hard_reg_set (t, clobbered_regs);
3325 :
3326 138568 : if (overlap)
3327 : {
3328 21 : error_at (loc, "%<asm%> specifier for variable %qE conflicts with "
3329 21 : "%<asm%> clobber list", DECL_NAME (overlap));
3330 :
3331 : /* Reset registerness to stop multiple errors emitted for a single
3332 : variable. */
3333 21 : DECL_REGISTER (overlap) = 0;
3334 21 : return true;
3335 : }
3336 :
3337 : return false;
3338 : }
3339 :
3340 : /* Check that the given REGNO spanning NREGS is a valid
3341 : asm clobber operand. Some HW registers cannot be
3342 : saved/restored, hence they should not be clobbered by
3343 : asm statements. */
3344 : static bool
3345 41543 : asm_clobber_reg_is_valid (int regno, int nregs, const char *regname)
3346 : {
3347 41543 : bool is_valid = true;
3348 41543 : HARD_REG_SET regset;
3349 :
3350 41543 : CLEAR_HARD_REG_SET (regset);
3351 :
3352 41543 : add_range_to_hard_reg_set (®set, regno, nregs);
3353 :
3354 : /* Clobbering the PIC register is an error. */
3355 41543 : if (PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
3356 0 : && overlaps_hard_reg_set_p (regset, Pmode, PIC_OFFSET_TABLE_REGNUM))
3357 : {
3358 : /* ??? Diagnose during gimplification? */
3359 0 : error ("PIC register clobbered by %qs in %<asm%>", regname);
3360 0 : is_valid = false;
3361 : }
3362 41543 : else if (!in_hard_reg_set_p
3363 41543 : (accessible_reg_set, reg_raw_mode[regno], regno))
3364 : {
3365 : /* ??? Diagnose during gimplification? */
3366 0 : error ("the register %qs cannot be clobbered in %<asm%>"
3367 : " for the current target", regname);
3368 0 : is_valid = false;
3369 : }
3370 :
3371 : /* Clobbering the stack pointer register is deprecated. GCC expects
3372 : the value of the stack pointer after an asm statement to be the same
3373 : as it was before, so no asm can validly clobber the stack pointer in
3374 : the usual sense. Adding the stack pointer to the clobber list has
3375 : traditionally had some undocumented and somewhat obscure side-effects. */
3376 41563 : if (overlaps_hard_reg_set_p (regset, Pmode, STACK_POINTER_REGNUM))
3377 : {
3378 1 : crtl->sp_is_clobbered_by_asm = true;
3379 1 : if (warning (OPT_Wdeprecated, "listing the stack pointer register"
3380 : " %qs in a clobber list is deprecated", regname))
3381 1 : inform (input_location, "the value of the stack pointer after"
3382 : " an %<asm%> statement must be the same as it was before"
3383 : " the statement");
3384 : }
3385 :
3386 41543 : return is_valid;
3387 : }
3388 :
3389 : /* Generate RTL for an asm statement with arguments.
3390 : STRING is the instruction template.
3391 : OUTPUTS is a list of output arguments (lvalues); INPUTS a list of inputs.
3392 : Each output or input has an expression in the TREE_VALUE and
3393 : a tree list in TREE_PURPOSE which in turn contains a constraint
3394 : name in TREE_VALUE (or NULL_TREE) and a constraint string
3395 : in TREE_PURPOSE.
3396 : CLOBBERS is a list of STRING_CST nodes each naming a hard register
3397 : that is clobbered by this insn.
3398 :
3399 : LABELS is a list of labels, and if LABELS is non-NULL, FALLTHRU_BB
3400 : should be the fallthru basic block of the asm goto.
3401 :
3402 : Not all kinds of lvalue that may appear in OUTPUTS can be stored directly.
3403 : Some elements of OUTPUTS may be replaced with trees representing temporary
3404 : values. The caller should copy those temporary values to the originally
3405 : specified lvalues.
3406 :
3407 : VOL nonzero means the insn is volatile; don't optimize it. */
3408 :
3409 : static void
3410 96548 : expand_asm_stmt (gasm *stmt)
3411 : {
3412 96548 : class save_input_location
3413 : {
3414 : location_t old;
3415 :
3416 : public:
3417 93695 : explicit save_input_location(location_t where)
3418 : {
3419 93695 : old = input_location;
3420 93695 : input_location = where;
3421 : }
3422 :
3423 93695 : ~save_input_location()
3424 : {
3425 93695 : input_location = old;
3426 5 : }
3427 : };
3428 :
3429 96548 : location_t locus = gimple_location (stmt);
3430 :
3431 96548 : if (gimple_asm_basic_p (stmt))
3432 : {
3433 2853 : const char *s = gimple_asm_string (stmt);
3434 2853 : tree string = build_string (strlen (s), s);
3435 2853 : expand_asm_loc (string, gimple_asm_volatile_p (stmt), locus);
3436 5711 : return;
3437 : }
3438 :
3439 : /* There are some legacy diagnostics in here. */
3440 93695 : save_input_location s_i_l(locus);
3441 :
3442 93695 : unsigned noutputs = gimple_asm_noutputs (stmt);
3443 93695 : unsigned ninputs = gimple_asm_ninputs (stmt);
3444 93695 : unsigned nlabels = gimple_asm_nlabels (stmt);
3445 93695 : unsigned i;
3446 93695 : bool error_seen = false;
3447 :
3448 : /* ??? Diagnose during gimplification? */
3449 93695 : if (ninputs + noutputs + nlabels > MAX_RECOG_OPERANDS)
3450 : {
3451 5 : error_at (locus, "more than %d operands in %<asm%>", MAX_RECOG_OPERANDS);
3452 5 : return;
3453 : }
3454 :
3455 93690 : auto_vec<tree, MAX_RECOG_OPERANDS> output_tvec;
3456 93690 : auto_vec<tree, MAX_RECOG_OPERANDS> input_tvec;
3457 93690 : auto_vec<const char *, MAX_RECOG_OPERANDS> constraints;
3458 :
3459 : /* Copy the gimple vectors into new vectors that we can manipulate. */
3460 :
3461 93690 : output_tvec.safe_grow (noutputs, true);
3462 93690 : input_tvec.safe_grow (ninputs, true);
3463 93690 : constraints.safe_grow (noutputs + ninputs, true);
3464 :
3465 267180 : for (i = 0; i < noutputs; ++i)
3466 : {
3467 79800 : tree t = gimple_asm_output_op (stmt, i);
3468 79800 : output_tvec[i] = TREE_VALUE (t);
3469 79800 : constraints[i] = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3470 : }
3471 152462 : for (i = 0; i < ninputs; i++)
3472 : {
3473 58772 : tree t = gimple_asm_input_op (stmt, i);
3474 58772 : input_tvec[i] = TREE_VALUE (t);
3475 58772 : constraints[i + noutputs]
3476 58772 : = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3477 : }
3478 :
3479 : /* Count the number of meaningful clobbered registers, ignoring what
3480 : we would ignore later. */
3481 93690 : auto_vec<rtx> clobber_rvec;
3482 93690 : HARD_REG_SET clobbered_regs;
3483 93690 : CLEAR_HARD_REG_SET (clobbered_regs);
3484 :
3485 93690 : if (unsigned n = gimple_asm_nclobbers (stmt))
3486 : {
3487 54271 : clobber_rvec.reserve (n);
3488 190370 : for (i = 0; i < n; i++)
3489 : {
3490 81828 : tree t = gimple_asm_clobber_op (stmt, i);
3491 81828 : const char *regname = TREE_STRING_POINTER (TREE_VALUE (t));
3492 81828 : int nregs, j;
3493 :
3494 81828 : j = decode_reg_name_and_count (regname, &nregs);
3495 81828 : if (j < 0)
3496 : {
3497 40285 : if (j == -2)
3498 : {
3499 : /* ??? Diagnose during gimplification? */
3500 15 : error_at (locus, "unknown register name %qs in %<asm%>",
3501 : regname);
3502 15 : error_seen = true;
3503 : }
3504 40270 : else if (j == -4)
3505 : {
3506 39893 : rtx x = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3507 39893 : clobber_rvec.safe_push (x);
3508 : }
3509 377 : else if (j == -5)
3510 : {
3511 2 : if (targetm.redzone_clobber)
3512 2 : if (rtx x = targetm.redzone_clobber ())
3513 2 : clobber_rvec.safe_push (x);
3514 : }
3515 : else
3516 : {
3517 : /* Otherwise we should have -1 == empty string
3518 : or -3 == cc, which is not a register. */
3519 375 : gcc_assert (j == -1 || j == -3);
3520 : }
3521 : }
3522 : else
3523 83086 : for (int reg = j; reg < j + nregs; reg++)
3524 : {
3525 41543 : if (!asm_clobber_reg_is_valid (reg, nregs, regname))
3526 0 : return;
3527 :
3528 41543 : SET_HARD_REG_BIT (clobbered_regs, reg);
3529 41543 : rtx x = gen_rtx_REG (reg_raw_mode[reg], reg);
3530 41543 : clobber_rvec.safe_push (x);
3531 : }
3532 : }
3533 : }
3534 :
3535 : /* First pass over inputs and outputs checks validity and sets
3536 : mark_addressable if needed. */
3537 : /* ??? Diagnose during gimplification? */
3538 :
3539 173490 : for (i = 0; i < noutputs; ++i)
3540 : {
3541 79800 : tree val = output_tvec[i];
3542 79800 : tree type = TREE_TYPE (val);
3543 79800 : const char *constraint;
3544 79800 : bool is_inout;
3545 79800 : bool allows_reg;
3546 79800 : bool allows_mem;
3547 :
3548 : /* Try to parse the output constraint. If that fails, there's
3549 : no point in going further. */
3550 79800 : constraint = constraints[i];
3551 79800 : if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
3552 : &allows_mem, &allows_reg, &is_inout,
3553 : nullptr))
3554 0 : return;
3555 :
3556 : /* If the output is a hard register, verify it doesn't conflict with
3557 : any other operand's possible hard register use. */
3558 79800 : if (DECL_P (val)
3559 7871 : && REG_P (DECL_RTL (val))
3560 80757 : && HARD_REGISTER_P (DECL_RTL (val)))
3561 : {
3562 938 : unsigned j, output_hregno = REGNO (DECL_RTL (val));
3563 938 : bool early_clobber_p = strchr (constraints[i], '&') != NULL;
3564 938 : unsigned long match;
3565 :
3566 : /* Verify the other outputs do not use the same hard register. */
3567 1098 : for (j = i + 1; j < noutputs; ++j)
3568 160 : if (DECL_P (output_tvec[j])
3569 159 : && REG_P (DECL_RTL (output_tvec[j]))
3570 159 : && HARD_REGISTER_P (DECL_RTL (output_tvec[j]))
3571 319 : && output_hregno == REGNO (DECL_RTL (output_tvec[j])))
3572 : {
3573 1 : error_at (locus, "invalid hard register usage between output "
3574 : "operands");
3575 1 : error_seen = true;
3576 : }
3577 :
3578 : /* Verify matching constraint operands use the same hard register
3579 : and that the non-matching constraint operands do not use the same
3580 : hard register if the output is an early clobber operand. */
3581 1849 : for (j = 0; j < ninputs; ++j)
3582 911 : if (DECL_P (input_tvec[j])
3583 772 : && REG_P (DECL_RTL (input_tvec[j]))
3584 1683 : && HARD_REGISTER_P (DECL_RTL (input_tvec[j])))
3585 : {
3586 772 : unsigned input_hregno = REGNO (DECL_RTL (input_tvec[j]));
3587 772 : switch (*constraints[j + noutputs])
3588 : {
3589 761 : case '0': case '1': case '2': case '3': case '4':
3590 761 : case '5': case '6': case '7': case '8': case '9':
3591 761 : match = strtoul (constraints[j + noutputs], NULL, 10);
3592 761 : break;
3593 : default:
3594 : match = ULONG_MAX;
3595 : break;
3596 : }
3597 761 : if (i == match
3598 661 : && output_hregno != input_hregno)
3599 : {
3600 1 : error_at (locus, "invalid hard register usage between "
3601 : "output operand and matching constraint operand");
3602 1 : error_seen = true;
3603 : }
3604 771 : else if (early_clobber_p
3605 4 : && i != match
3606 2 : && output_hregno == input_hregno)
3607 : {
3608 0 : error_at (locus, "invalid hard register usage between "
3609 : "earlyclobber operand and input operand");
3610 0 : error_seen = true;
3611 : }
3612 : }
3613 : }
3614 :
3615 79800 : if (! allows_reg
3616 79800 : && (allows_mem
3617 0 : || is_inout
3618 0 : || (DECL_P (val)
3619 0 : && REG_P (DECL_RTL (val))
3620 0 : && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type))))
3621 8855 : mark_addressable (val);
3622 : }
3623 :
3624 152462 : for (i = 0; i < ninputs; ++i)
3625 : {
3626 58772 : bool allows_reg, allows_mem;
3627 58772 : const char *constraint;
3628 :
3629 58772 : constraint = constraints[i + noutputs];
3630 58772 : if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3631 58772 : constraints.address (), &allows_mem,
3632 : &allows_reg, nullptr))
3633 0 : return;
3634 :
3635 58772 : if (! allows_reg && allows_mem)
3636 9497 : mark_addressable (input_tvec[i]);
3637 : }
3638 :
3639 : /* Second pass evaluates arguments. */
3640 :
3641 : /* Make sure stack is consistent for asm goto. */
3642 93690 : if (nlabels > 0)
3643 622 : do_pending_stack_adjust ();
3644 93690 : int old_generating_concat_p = generating_concat_p;
3645 :
3646 : /* Vector of RTX's of evaluated output operands. */
3647 187380 : auto_vec<rtx, MAX_RECOG_OPERANDS> output_rvec;
3648 93690 : auto_vec<int, MAX_RECOG_OPERANDS> inout_opnum;
3649 93690 : rtx_insn *after_rtl_seq = NULL, *after_rtl_end = NULL;
3650 :
3651 93690 : output_rvec.safe_grow (noutputs, true);
3652 :
3653 267180 : for (i = 0; i < noutputs; ++i)
3654 : {
3655 79800 : tree val = output_tvec[i];
3656 79800 : tree type = TREE_TYPE (val);
3657 79800 : bool is_inout, allows_reg, allows_mem, ok;
3658 79800 : rtx op;
3659 :
3660 79800 : ok = parse_output_constraint (&constraints[i], i, ninputs,
3661 : noutputs, &allows_mem, &allows_reg,
3662 : &is_inout, nullptr);
3663 79800 : gcc_assert (ok);
3664 :
3665 : /* If an output operand is not a decl or indirect ref and our constraint
3666 : allows a register, make a temporary to act as an intermediate.
3667 : Make the asm insn write into that, then we will copy it to
3668 : the real output operand. Likewise for promoted variables. */
3669 :
3670 79800 : generating_concat_p = 0;
3671 :
3672 79800 : gcc_assert (TREE_CODE (val) != INDIRECT_REF);
3673 79800 : if (((TREE_CODE (val) == MEM_REF
3674 4464 : && TREE_CODE (TREE_OPERAND (val, 0)) != ADDR_EXPR)
3675 3867 : && allows_mem)
3676 77637 : || (DECL_P (val)
3677 7871 : && (allows_mem || REG_P (DECL_RTL (val)))
3678 8115 : && ! (REG_P (DECL_RTL (val))
3679 957 : && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type)))
3680 70479 : || ! allows_reg
3681 69780 : || is_inout
3682 69780 : || TREE_ADDRESSABLE (type)
3683 149580 : || (!tree_fits_poly_int64_p (TYPE_SIZE (type))
3684 0 : && !known_size_p (max_int_size_in_bytes (type))))
3685 : {
3686 10020 : op = expand_expr (val, NULL_RTX, VOIDmode,
3687 : !allows_reg ? EXPAND_MEMORY : EXPAND_WRITE);
3688 10020 : if (MEM_P (op))
3689 9063 : op = validize_mem (op);
3690 :
3691 10020 : if (! allows_reg && !MEM_P (op))
3692 : {
3693 0 : error_at (locus, "output number %d not directly addressable", i);
3694 0 : error_seen = true;
3695 : }
3696 10020 : if ((! allows_mem && MEM_P (op) && GET_MODE (op) != BLKmode)
3697 10020 : || GET_CODE (op) == CONCAT)
3698 : {
3699 0 : rtx old_op = op;
3700 0 : op = gen_reg_rtx (GET_MODE (op));
3701 :
3702 0 : generating_concat_p = old_generating_concat_p;
3703 :
3704 0 : if (is_inout)
3705 0 : emit_move_insn (op, old_op);
3706 :
3707 0 : push_to_sequence2 (after_rtl_seq, after_rtl_end);
3708 0 : emit_move_insn (old_op, op);
3709 0 : after_rtl_seq = get_insns ();
3710 0 : after_rtl_end = get_last_insn ();
3711 0 : end_sequence ();
3712 : }
3713 : }
3714 : else
3715 : {
3716 69780 : op = assign_temp (type, 0, 1);
3717 69780 : op = validize_mem (op);
3718 69780 : if (!MEM_P (op) && TREE_CODE (val) == SSA_NAME)
3719 66622 : set_reg_attrs_for_decl_rtl (SSA_NAME_VAR (val), op);
3720 :
3721 69780 : generating_concat_p = old_generating_concat_p;
3722 :
3723 69780 : push_to_sequence2 (after_rtl_seq, after_rtl_end);
3724 69780 : expand_assignment (val, make_tree (type, op), false);
3725 69780 : after_rtl_seq = get_insns ();
3726 69780 : after_rtl_end = get_last_insn ();
3727 69780 : end_sequence ();
3728 : }
3729 79800 : output_rvec[i] = op;
3730 :
3731 79800 : if (is_inout)
3732 0 : inout_opnum.safe_push (i);
3733 : }
3734 :
3735 93690 : const char *str = gimple_asm_string (stmt);
3736 93690 : if (error_seen)
3737 : {
3738 17 : ninputs = 0;
3739 17 : noutputs = 0;
3740 17 : inout_opnum.truncate (0);
3741 17 : output_rvec.truncate (0);
3742 17 : clobber_rvec.truncate (0);
3743 17 : constraints.truncate (0);
3744 17 : CLEAR_HARD_REG_SET (clobbered_regs);
3745 : str = "";
3746 : }
3747 :
3748 187380 : auto_vec<rtx, MAX_RECOG_OPERANDS> input_rvec;
3749 93690 : auto_vec<machine_mode, MAX_RECOG_OPERANDS> input_mode;
3750 :
3751 93690 : input_rvec.safe_grow (ninputs, true);
3752 93690 : input_mode.safe_grow (ninputs, true);
3753 :
3754 93690 : generating_concat_p = 0;
3755 :
3756 152461 : for (i = 0; i < ninputs; ++i)
3757 : {
3758 58771 : tree val = input_tvec[i];
3759 58771 : tree type = TREE_TYPE (val);
3760 58771 : bool allows_reg, allows_mem, ok;
3761 58771 : const char *constraint;
3762 58771 : rtx op;
3763 :
3764 58771 : constraint = constraints[i + noutputs];
3765 58771 : ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3766 58771 : constraints.address (),
3767 : &allows_mem, &allows_reg, nullptr);
3768 58771 : gcc_assert (ok);
3769 :
3770 : /* EXPAND_INITIALIZER will not generate code for valid initializer
3771 : constants, but will still generate code for other types of operand.
3772 : This is the behavior we want for constant constraints. */
3773 58771 : op = expand_expr (val, NULL_RTX, VOIDmode,
3774 : allows_reg ? EXPAND_NORMAL
3775 11210 : : allows_mem ? EXPAND_MEMORY
3776 : : EXPAND_INITIALIZER);
3777 :
3778 : /* Never pass a CONCAT to an ASM. */
3779 58771 : if (GET_CODE (op) == CONCAT)
3780 2 : op = force_reg (GET_MODE (op), op);
3781 58769 : else if (MEM_P (op))
3782 14419 : op = validize_mem (op);
3783 :
3784 58771 : if (asm_operand_ok (op, constraint, NULL) <= 0)
3785 : {
3786 25449 : if (allows_reg && TYPE_MODE (type) != BLKmode)
3787 25428 : op = force_reg (TYPE_MODE (type), op);
3788 21 : else if (!allows_mem)
3789 20 : warning_at (locus, 0, "%<asm%> operand %d probably does not match "
3790 : "constraints", i + noutputs);
3791 1 : else if (MEM_P (op))
3792 : {
3793 : /* We won't recognize either volatile memory or memory
3794 : with a queued address as available a memory_operand
3795 : at this point. Ignore it: clearly this *is* a memory. */
3796 : }
3797 : else
3798 0 : gcc_unreachable ();
3799 : }
3800 58771 : input_rvec[i] = op;
3801 58771 : input_mode[i] = TYPE_MODE (type);
3802 : }
3803 :
3804 : /* For in-out operands, copy output rtx to input rtx. */
3805 93690 : unsigned ninout = inout_opnum.length ();
3806 93690 : for (i = 0; i < ninout; i++)
3807 : {
3808 0 : int j = inout_opnum[i];
3809 0 : rtx o = output_rvec[j];
3810 :
3811 0 : input_rvec.safe_push (o);
3812 0 : input_mode.safe_push (GET_MODE (o));
3813 :
3814 0 : char buffer[16];
3815 0 : sprintf (buffer, "%d", j);
3816 0 : constraints.safe_push (ggc_strdup (buffer));
3817 : }
3818 93690 : ninputs += ninout;
3819 :
3820 : /* Sometimes we wish to automatically clobber registers across an asm.
3821 : Case in point is when the i386 backend moved from cc0 to a hard reg --
3822 : maintaining source-level compatibility means automatically clobbering
3823 : the flags register. */
3824 187380 : auto_vec<rtx> use_rvec;
3825 93690 : if (targetm.md_asm_adjust)
3826 : {
3827 93690 : rtx_insn *after_md_seq
3828 93690 : = targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3829 : constraints, use_rvec, clobber_rvec,
3830 : clobbered_regs, locus);
3831 93690 : if (after_md_seq)
3832 : {
3833 72 : push_to_sequence (after_md_seq);
3834 72 : emit_insn (after_rtl_seq);
3835 72 : after_rtl_seq = get_insns ();
3836 72 : after_rtl_end = get_last_insn ();
3837 72 : end_sequence ();
3838 : }
3839 : }
3840 :
3841 : /* Do not allow the hook to change the output and input count,
3842 : lest it mess up the operand numbering. */
3843 187380 : gcc_assert (output_rvec.length() == noutputs);
3844 187380 : gcc_assert (input_rvec.length() == ninputs);
3845 187380 : gcc_assert (constraints.length() == noutputs + ninputs);
3846 :
3847 : /* But it certainly can adjust the uses and clobbers. */
3848 93690 : unsigned nuses = use_rvec.length ();
3849 93690 : unsigned nclobbers = clobber_rvec.length ();
3850 :
3851 : /* Third pass checks for easy conflicts. */
3852 : /* ??? Why are we doing this on trees instead of rtx. */
3853 :
3854 93690 : bool clobber_conflict_found = 0;
3855 173487 : for (i = 0; i < noutputs; ++i)
3856 79797 : if (tree_conflicts_with_clobbers_p (output_tvec[i], &clobbered_regs, locus))
3857 10 : clobber_conflict_found = 1;
3858 152461 : for (i = 0; i < ninputs - ninout; ++i)
3859 58771 : if (tree_conflicts_with_clobbers_p (input_tvec[i], &clobbered_regs, locus))
3860 11 : clobber_conflict_found = 1;
3861 :
3862 : /* Make vectors for the expression-rtx, constraint strings,
3863 : and named operands. */
3864 :
3865 93690 : rtvec argvec = rtvec_alloc (ninputs);
3866 93690 : rtvec constraintvec = rtvec_alloc (ninputs);
3867 93690 : rtvec labelvec = rtvec_alloc (nlabels);
3868 :
3869 130066 : rtx body = gen_rtx_ASM_OPERANDS ((noutputs == 0 ? VOIDmode
3870 : : GET_MODE (output_rvec[0])),
3871 : ggc_strdup (str),
3872 : "", 0, argvec, constraintvec,
3873 : labelvec, locus);
3874 93690 : MEM_VOLATILE_P (body) = gimple_asm_volatile_p (stmt);
3875 :
3876 152461 : for (i = 0; i < ninputs; ++i)
3877 : {
3878 58771 : ASM_OPERANDS_INPUT (body, i) = input_rvec[i];
3879 117542 : ASM_OPERANDS_INPUT_CONSTRAINT_EXP (body, i)
3880 58771 : = gen_rtx_ASM_INPUT_loc (input_mode[i],
3881 : constraints[i + noutputs],
3882 : locus);
3883 : }
3884 :
3885 : /* Copy labels to the vector. */
3886 93690 : rtx_code_label *fallthru_label = NULL;
3887 93690 : if (nlabels > 0)
3888 : {
3889 622 : basic_block fallthru_bb = NULL;
3890 622 : edge fallthru = find_fallthru_edge (gimple_bb (stmt)->succs);
3891 622 : if (fallthru)
3892 622 : fallthru_bb = fallthru->dest;
3893 :
3894 1503 : for (i = 0; i < nlabels; ++i)
3895 : {
3896 881 : tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
3897 881 : rtx_insn *r;
3898 : /* If asm goto has any labels in the fallthru basic block, use
3899 : a label that we emit immediately after the asm goto. Expansion
3900 : may insert further instructions into the same basic block after
3901 : asm goto and if we don't do this, insertion of instructions on
3902 : the fallthru edge might misbehave. See PR58670. */
3903 881 : if (fallthru_bb && label_to_block (cfun, label) == fallthru_bb)
3904 : {
3905 284 : if (fallthru_label == NULL_RTX)
3906 279 : fallthru_label = gen_label_rtx ();
3907 : r = fallthru_label;
3908 : }
3909 : else
3910 597 : r = label_rtx (label);
3911 881 : ASM_OPERANDS_LABEL (body, i) = gen_rtx_LABEL_REF (Pmode, r);
3912 : }
3913 : }
3914 :
3915 : /* Now, for each output, construct an rtx
3916 : (set OUTPUT (asm_operands INSN OUTPUTCONSTRAINT OUTPUTNUMBER
3917 : ARGVEC CONSTRAINTS OPNAMES))
3918 : If there is more than one, put them inside a PARALLEL. */
3919 :
3920 93690 : if (noutputs == 0 && nuses == 0 && nclobbers == 0)
3921 : {
3922 : /* No output operands: put in a raw ASM_OPERANDS rtx. */
3923 0 : if (nlabels > 0)
3924 0 : emit_jump_insn (body);
3925 : else
3926 0 : emit_insn (body);
3927 : }
3928 93690 : else if (noutputs == 1 && nuses == 0 && nclobbers == 0)
3929 : {
3930 62 : ASM_OPERANDS_OUTPUT_CONSTRAINT (body) = constraints[0];
3931 62 : if (nlabels > 0)
3932 0 : emit_jump_insn (gen_rtx_SET (output_rvec[0], body));
3933 : else
3934 62 : emit_insn (gen_rtx_SET (output_rvec[0], body));
3935 : }
3936 : else
3937 : {
3938 93628 : rtx obody = body;
3939 93628 : int num = noutputs;
3940 :
3941 93628 : if (num == 0)
3942 57314 : num = 1;
3943 :
3944 93628 : body = gen_rtx_PARALLEL (VOIDmode,
3945 : rtvec_alloc (num + nuses + nclobbers));
3946 :
3947 : /* For each output operand, store a SET. */
3948 173363 : for (i = 0; i < noutputs; ++i)
3949 : {
3950 79735 : rtx src, o = output_rvec[i];
3951 79735 : if (i == 0)
3952 : {
3953 36314 : ASM_OPERANDS_OUTPUT_CONSTRAINT (obody) = constraints[0];
3954 36314 : src = obody;
3955 : }
3956 : else
3957 : {
3958 43421 : src = gen_rtx_ASM_OPERANDS (GET_MODE (o),
3959 : ASM_OPERANDS_TEMPLATE (obody),
3960 : constraints[i], i, argvec,
3961 : constraintvec, labelvec, locus);
3962 43421 : MEM_VOLATILE_P (src) = gimple_asm_volatile_p (stmt);
3963 : }
3964 79735 : XVECEXP (body, 0, i) = gen_rtx_SET (o, src);
3965 : }
3966 :
3967 : /* If there are no outputs (but there are some clobbers)
3968 : store the bare ASM_OPERANDS into the PARALLEL. */
3969 93628 : if (i == 0)
3970 57314 : XVECEXP (body, 0, i++) = obody;
3971 :
3972 : /* Add the uses specified by the target hook. No checking should
3973 : be needed since this doesn't come directly from user code. */
3974 93628 : for (rtx use : use_rvec)
3975 0 : XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
3976 :
3977 : /* Store (clobber REG) for each clobbered register specified. */
3978 268681 : for (unsigned j = 0; j < nclobbers; ++j)
3979 : {
3980 175053 : rtx clobbered_reg = clobber_rvec[j];
3981 :
3982 : /* Do sanity check for overlap between clobbers and respectively
3983 : input and outputs that hasn't been handled. Such overlap
3984 : should have been detected and reported above. */
3985 175053 : if (!clobber_conflict_found && REG_P (clobbered_reg))
3986 : {
3987 : /* We test the old body (obody) contents to avoid
3988 : tripping over the under-construction body. */
3989 215810 : for (unsigned k = 0; k < noutputs; ++k)
3990 80682 : if (reg_overlap_mentioned_p (clobbered_reg, output_rvec[k]))
3991 0 : internal_error ("%<asm%> clobber conflict with "
3992 : "output operand");
3993 :
3994 196640 : for (unsigned k = 0; k < ninputs - ninout; ++k)
3995 61512 : if (reg_overlap_mentioned_p (clobbered_reg, input_rvec[k]))
3996 0 : internal_error ("%<asm%> clobber conflict with "
3997 : "input operand");
3998 : }
3999 :
4000 175053 : XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobbered_reg);
4001 : }
4002 :
4003 93628 : if (nlabels > 0)
4004 622 : emit_jump_insn (body);
4005 : else
4006 93006 : emit_insn (body);
4007 : }
4008 :
4009 93690 : generating_concat_p = old_generating_concat_p;
4010 :
4011 93690 : if (fallthru_label)
4012 279 : emit_label (fallthru_label);
4013 :
4014 93690 : if (after_rtl_seq)
4015 : {
4016 29326 : if (nlabels == 0)
4017 29193 : emit_insn (after_rtl_seq);
4018 : else
4019 : {
4020 133 : edge e;
4021 133 : edge_iterator ei;
4022 133 : unsigned int cnt = EDGE_COUNT (gimple_bb (stmt)->succs);
4023 :
4024 475 : FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
4025 : {
4026 342 : rtx_insn *copy;
4027 342 : if (--cnt == 0)
4028 : copy = after_rtl_seq;
4029 : else
4030 : {
4031 209 : start_sequence ();
4032 209 : duplicate_insn_chain (after_rtl_seq, after_rtl_end,
4033 : NULL, NULL);
4034 209 : copy = end_sequence ();
4035 : }
4036 342 : prepend_insn_to_edge (copy, e);
4037 : }
4038 : }
4039 : }
4040 :
4041 93690 : free_temp_slots ();
4042 93690 : crtl->has_asm_statement = 1;
4043 93695 : }
4044 :
4045 : /* Emit code to jump to the address
4046 : specified by the pointer expression EXP. */
4047 :
4048 : static void
4049 409 : expand_computed_goto (tree exp)
4050 : {
4051 409 : rtx x = expand_normal (exp);
4052 :
4053 409 : do_pending_stack_adjust ();
4054 409 : emit_indirect_jump (x);
4055 409 : }
4056 :
4057 : /* Generate RTL code for a `goto' statement with target label LABEL.
4058 : LABEL should be a LABEL_DECL tree node that was or will later be
4059 : defined with `expand_label'. */
4060 :
4061 : static void
4062 0 : expand_goto (tree label)
4063 : {
4064 0 : if (flag_checking)
4065 : {
4066 : /* Check for a nonlocal goto to a containing function. Should have
4067 : gotten translated to __builtin_nonlocal_goto. */
4068 0 : tree context = decl_function_context (label);
4069 0 : gcc_assert (!context || context == current_function_decl);
4070 : }
4071 :
4072 0 : emit_jump (jump_target_rtx (label));
4073 0 : }
4074 :
4075 : /* Output a return with no value. */
4076 :
4077 : static void
4078 841224 : expand_null_return_1 (void)
4079 : {
4080 841224 : clear_pending_stack_adjust ();
4081 841224 : do_pending_stack_adjust ();
4082 841224 : emit_jump (return_label);
4083 841224 : }
4084 :
4085 : /* Generate RTL to return from the current function, with no value.
4086 : (That is, we do not do anything about returning any value.) */
4087 :
4088 : void
4089 70676 : expand_null_return (void)
4090 : {
4091 : /* If this function was declared to return a value, but we
4092 : didn't, clobber the return registers so that they are not
4093 : propagated live to the rest of the function. */
4094 70676 : clobber_return_register ();
4095 :
4096 70676 : expand_null_return_1 ();
4097 70676 : }
4098 :
4099 : /* Generate RTL to return from the current function, with value VAL. */
4100 :
4101 : static void
4102 770548 : expand_value_return (rtx val)
4103 : {
4104 : /* Copy the value to the return location unless it's already there. */
4105 :
4106 770548 : tree decl = DECL_RESULT (current_function_decl);
4107 770548 : rtx return_reg = DECL_RTL (decl);
4108 770548 : if (return_reg != val)
4109 : {
4110 447930 : tree funtype = TREE_TYPE (current_function_decl);
4111 447930 : tree type = TREE_TYPE (decl);
4112 447930 : int unsignedp = TYPE_UNSIGNED (type);
4113 447930 : machine_mode old_mode = DECL_MODE (decl);
4114 447930 : machine_mode mode;
4115 447930 : if (DECL_BY_REFERENCE (decl))
4116 0 : mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 2);
4117 : else
4118 447930 : mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 1);
4119 :
4120 447930 : if (mode != old_mode)
4121 : {
4122 : /* Some ABIs require scalar floating point modes to be returned
4123 : in a wider scalar integer mode. We need to explicitly
4124 : reinterpret to an integer mode of the correct precision
4125 : before extending to the desired result. */
4126 8 : if (SCALAR_INT_MODE_P (mode)
4127 8 : && SCALAR_FLOAT_MODE_P (old_mode)
4128 8 : && known_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (old_mode)))
4129 0 : val = convert_float_to_wider_int (mode, old_mode, val);
4130 : else
4131 8 : val = convert_modes (mode, old_mode, val, unsignedp);
4132 : }
4133 :
4134 447930 : if (GET_CODE (return_reg) == PARALLEL)
4135 3405 : emit_group_load (return_reg, val, type, int_size_in_bytes (type));
4136 : else
4137 444525 : emit_move_insn (return_reg, val);
4138 : }
4139 :
4140 770548 : expand_null_return_1 ();
4141 770548 : }
4142 :
4143 : /* Generate RTL to evaluate the expression RETVAL and return it
4144 : from the current function. */
4145 :
4146 : static void
4147 772912 : expand_return (tree retval)
4148 : {
4149 772912 : rtx result_rtl;
4150 772912 : rtx val = 0;
4151 772912 : tree retval_rhs;
4152 :
4153 : /* If function wants no value, give it none. */
4154 772912 : if (VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
4155 : {
4156 0 : expand_normal (retval);
4157 0 : expand_null_return ();
4158 0 : return;
4159 : }
4160 :
4161 772912 : if (retval == error_mark_node)
4162 : {
4163 : /* Treat this like a return of no value from a function that
4164 : returns a value. */
4165 0 : expand_null_return ();
4166 0 : return;
4167 : }
4168 772912 : else if ((TREE_CODE (retval) == MODIFY_EXPR
4169 772912 : || TREE_CODE (retval) == INIT_EXPR)
4170 772912 : && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
4171 709758 : retval_rhs = TREE_OPERAND (retval, 1);
4172 : else
4173 : retval_rhs = retval;
4174 :
4175 772912 : result_rtl = DECL_RTL (DECL_RESULT (current_function_decl));
4176 :
4177 : /* If we are returning the RESULT_DECL, then the value has already
4178 : been stored into it, so we don't have to do anything special. */
4179 772912 : if (TREE_CODE (retval_rhs) == RESULT_DECL)
4180 63154 : expand_value_return (result_rtl);
4181 :
4182 : /* If the result is an aggregate that is being returned in one (or more)
4183 : registers, load the registers here. */
4184 :
4185 709758 : else if (retval_rhs != 0
4186 709758 : && TYPE_MODE (TREE_TYPE (retval_rhs)) == BLKmode
4187 5082 : && REG_P (result_rtl))
4188 : {
4189 3588 : val = copy_blkmode_to_reg (GET_MODE (result_rtl), retval_rhs);
4190 3588 : if (val)
4191 : {
4192 : /* Use the mode of the result value on the return register. */
4193 1224 : PUT_MODE (result_rtl, GET_MODE (val));
4194 1224 : expand_value_return (val);
4195 : }
4196 : else
4197 2364 : expand_null_return ();
4198 : }
4199 706170 : else if (retval_rhs != 0
4200 706170 : && !VOID_TYPE_P (TREE_TYPE (retval_rhs))
4201 706170 : && (REG_P (result_rtl)
4202 6657 : || (GET_CODE (result_rtl) == PARALLEL)))
4203 : {
4204 : /* Compute the return value into a temporary (usually a pseudo reg). */
4205 702918 : val
4206 702918 : = assign_temp (TREE_TYPE (DECL_RESULT (current_function_decl)), 0, 1);
4207 702918 : val = expand_expr (retval_rhs, val, GET_MODE (val), EXPAND_NORMAL);
4208 702918 : val = force_not_mem (val);
4209 702918 : expand_value_return (val);
4210 : }
4211 : else
4212 : {
4213 : /* No hard reg used; calculate value into hard return reg. */
4214 3252 : expand_expr (retval, const0_rtx, VOIDmode, EXPAND_NORMAL);
4215 3252 : expand_value_return (result_rtl);
4216 : }
4217 : }
4218 :
4219 : /* Expand a clobber of LHS. If LHS is stored it in a multi-part
4220 : register, tell the rtl optimizers that its value is no longer
4221 : needed. */
4222 :
4223 : static void
4224 1397508 : expand_clobber (tree lhs)
4225 : {
4226 1397508 : if (DECL_P (lhs))
4227 : {
4228 1370684 : rtx decl_rtl = DECL_RTL_IF_SET (lhs);
4229 1370593 : if (decl_rtl && REG_P (decl_rtl))
4230 : {
4231 187658 : machine_mode decl_mode = GET_MODE (decl_rtl);
4232 375316 : if (maybe_gt (GET_MODE_SIZE (decl_mode),
4233 : REGMODE_NATURAL_SIZE (decl_mode)))
4234 146890 : emit_clobber (decl_rtl);
4235 : }
4236 : }
4237 1397508 : }
4238 :
4239 : /* A subroutine of expand_gimple_stmt, expanding one gimple statement
4240 : STMT that doesn't require special handling for outgoing edges. That
4241 : is no tailcalls and no GIMPLE_COND. */
4242 :
4243 : static void
4244 32345010 : expand_gimple_stmt_1 (gimple *stmt)
4245 : {
4246 32345010 : tree op0;
4247 :
4248 32345010 : set_curr_insn_location (gimple_location (stmt));
4249 :
4250 32345010 : switch (gimple_code (stmt))
4251 : {
4252 409 : case GIMPLE_GOTO:
4253 409 : op0 = gimple_goto_dest (stmt);
4254 409 : if (TREE_CODE (op0) == LABEL_DECL)
4255 0 : expand_goto (op0);
4256 : else
4257 409 : expand_computed_goto (op0);
4258 : break;
4259 659716 : case GIMPLE_LABEL:
4260 659716 : expand_label (gimple_label_label (as_a <glabel *> (stmt)));
4261 659716 : break;
4262 : case GIMPLE_NOP:
4263 : case GIMPLE_PREDICT:
4264 : break;
4265 7077 : case GIMPLE_SWITCH:
4266 7077 : {
4267 7077 : gswitch *swtch = as_a <gswitch *> (stmt);
4268 7077 : if (gimple_switch_num_labels (swtch) == 1)
4269 0 : expand_goto (CASE_LABEL (gimple_switch_default_label (swtch)));
4270 : else
4271 7077 : expand_case (swtch);
4272 : }
4273 : break;
4274 96548 : case GIMPLE_ASM:
4275 96548 : expand_asm_stmt (as_a <gasm *> (stmt));
4276 96548 : break;
4277 7035961 : case GIMPLE_CALL:
4278 7035961 : expand_call_stmt (as_a <gcall *> (stmt));
4279 7035961 : break;
4280 :
4281 841224 : case GIMPLE_RETURN:
4282 841224 : {
4283 841224 : op0 = gimple_return_retval (as_a <greturn *> (stmt));
4284 :
4285 : /* If a return doesn't have a location, it very likely represents
4286 : multiple user returns so we cannot let it inherit the location
4287 : of the last statement of the previous basic block in RTL. */
4288 841224 : if (!gimple_has_location (stmt))
4289 216236 : set_curr_insn_location (cfun->function_end_locus);
4290 :
4291 841224 : if (op0 && op0 != error_mark_node)
4292 : {
4293 772912 : tree result = DECL_RESULT (current_function_decl);
4294 :
4295 : /* If we are not returning the current function's RESULT_DECL,
4296 : build an assignment to it. */
4297 772912 : if (op0 != result)
4298 : {
4299 : /* I believe that a function's RESULT_DECL is unique. */
4300 709758 : gcc_assert (TREE_CODE (op0) != RESULT_DECL);
4301 :
4302 : /* ??? We'd like to use simply expand_assignment here,
4303 : but this fails if the value is of BLKmode but the return
4304 : decl is a register. expand_return has special handling
4305 : for this combination, which eventually should move
4306 : to common code. See comments there. Until then, let's
4307 : build a modify expression :-/ */
4308 709758 : op0 = build2 (MODIFY_EXPR, TREE_TYPE (result),
4309 : result, op0);
4310 : }
4311 : }
4312 :
4313 772912 : if (!op0)
4314 68312 : expand_null_return ();
4315 : else
4316 772912 : expand_return (op0);
4317 : }
4318 : break;
4319 :
4320 23656765 : case GIMPLE_ASSIGN:
4321 23656765 : {
4322 23656765 : gassign *assign_stmt = as_a <gassign *> (stmt);
4323 23656765 : tree lhs = gimple_assign_lhs (assign_stmt);
4324 :
4325 : /* Tree expand used to fiddle with |= and &= of two bitfield
4326 : COMPONENT_REFs here. This can't happen with gimple, the LHS
4327 : of binary assigns must be a gimple reg. */
4328 :
4329 23656765 : if (TREE_CODE (lhs) != SSA_NAME
4330 37655426 : || gimple_assign_rhs_class (assign_stmt) == GIMPLE_SINGLE_RHS)
4331 : {
4332 17609836 : tree rhs = gimple_assign_rhs1 (assign_stmt);
4333 17609836 : gcc_assert (gimple_assign_rhs_class (assign_stmt)
4334 : == GIMPLE_SINGLE_RHS);
4335 33609279 : if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (rhs)
4336 : /* Do not put locations on possibly shared trees. */
4337 23713772 : && !is_gimple_min_invariant (rhs))
4338 4954554 : SET_EXPR_LOCATION (rhs, gimple_location (stmt));
4339 17609836 : if (TREE_CLOBBER_P (rhs))
4340 : /* This is a clobber to mark the going out of scope for
4341 : this LHS. */
4342 1397508 : expand_clobber (lhs);
4343 : else
4344 16212328 : expand_assignment (lhs, rhs,
4345 : gimple_assign_nontemporal_move_p (
4346 : assign_stmt));
4347 : }
4348 : else
4349 : {
4350 6046929 : rtx target, temp;
4351 6046929 : gcc_assert (!gimple_assign_nontemporal_move_p (assign_stmt));
4352 6046929 : bool promoted = false;
4353 :
4354 6046929 : target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
4355 6046929 : if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
4356 : promoted = true;
4357 :
4358 : /* If we store into a promoted register, don't directly
4359 : expand to target. */
4360 : temp = promoted ? NULL_RTX : target;
4361 12093858 : temp = expand_expr_real_gassign (assign_stmt, temp,
4362 6046929 : GET_MODE (target), EXPAND_NORMAL);
4363 :
4364 6046929 : if (temp == target)
4365 : ;
4366 840104 : else if (promoted)
4367 : {
4368 2 : int unsignedp = SUBREG_PROMOTED_SIGN (target);
4369 : /* If TEMP is a VOIDmode constant, use convert_modes to make
4370 : sure that we properly convert it. */
4371 2 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
4372 : {
4373 0 : temp = convert_modes (GET_MODE (target),
4374 0 : TYPE_MODE (TREE_TYPE (lhs)),
4375 : temp, unsignedp);
4376 0 : temp = convert_modes (GET_MODE (SUBREG_REG (target)),
4377 0 : GET_MODE (target), temp, unsignedp);
4378 : }
4379 :
4380 2 : convert_move (SUBREG_REG (target), temp, unsignedp);
4381 : }
4382 : else
4383 : {
4384 840102 : temp = force_operand (temp, target);
4385 840102 : if (temp == target)
4386 : ;
4387 840102 : else if (GET_MODE (target) != BLKmode)
4388 840101 : emit_move_insn (target, temp);
4389 : else
4390 1 : emit_block_move (target, temp, expr_size (lhs),
4391 : BLOCK_OP_NORMAL);
4392 : }
4393 : }
4394 : }
4395 : break;
4396 :
4397 0 : default:
4398 0 : gcc_unreachable ();
4399 : }
4400 32345008 : }
4401 :
4402 : /* Expand one gimple statement STMT and return the last RTL instruction
4403 : before any of the newly generated ones.
4404 :
4405 : In addition to generating the necessary RTL instructions this also
4406 : sets REG_EH_REGION notes if necessary and sets the current source
4407 : location for diagnostics. */
4408 :
4409 : static rtx_insn *
4410 32345010 : expand_gimple_stmt (gimple *stmt)
4411 : {
4412 32345010 : location_t saved_location = input_location;
4413 32345010 : rtx_insn *last = get_last_insn ();
4414 32345010 : int lp_nr;
4415 :
4416 32345010 : gcc_assert (cfun);
4417 :
4418 : /* We need to save and restore the current source location so that errors
4419 : discovered during expansion are emitted with the right location. But
4420 : it would be better if the diagnostic routines used the source location
4421 : embedded in the tree nodes rather than globals. */
4422 32345010 : if (gimple_has_location (stmt))
4423 28378109 : input_location = gimple_location (stmt);
4424 :
4425 32345010 : expand_gimple_stmt_1 (stmt);
4426 :
4427 : /* Free any temporaries used to evaluate this statement. */
4428 32345008 : free_temp_slots ();
4429 :
4430 32345008 : input_location = saved_location;
4431 :
4432 : /* Mark all insns that may trap. */
4433 32345008 : lp_nr = lookup_stmt_eh_lp (stmt);
4434 32345008 : if (lp_nr)
4435 : {
4436 728897 : rtx_insn *insn;
4437 4178077 : for (insn = next_real_insn (last); insn;
4438 3449180 : insn = next_real_insn (insn))
4439 : {
4440 3449180 : if (! find_reg_note (insn, REG_EH_REGION, NULL_RTX)
4441 : /* If we want exceptions for non-call insns, any
4442 : may_trap_p instruction may throw. */
4443 3449092 : && GET_CODE (PATTERN (insn)) != CLOBBER
4444 3449022 : && GET_CODE (PATTERN (insn)) != USE
4445 6898202 : && insn_could_throw_p (insn))
4446 786182 : make_reg_eh_region_note (insn, 0, lp_nr);
4447 : }
4448 : }
4449 :
4450 32345008 : return last;
4451 : }
4452 :
4453 : /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_CALL
4454 : that has CALL_EXPR_TAILCALL set. Returns non-null if we actually
4455 : generated a tail call (something that might be denied by the ABI
4456 : rules governing the call; see calls.cc).
4457 :
4458 : Sets CAN_FALLTHRU if we generated a *conditional* tail call, and
4459 : can still reach the rest of BB. The case here is __builtin_sqrt,
4460 : where the NaN result goes through the external function (with a
4461 : tailcall) and the normal result happens via a sqrt instruction. */
4462 :
4463 : static basic_block
4464 180694 : expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru,
4465 : rtx_insn *asan_epilog_seq)
4466 : {
4467 180694 : rtx_insn *last2, *last, *first = get_last_insn ();
4468 180694 : edge e;
4469 180694 : edge_iterator ei;
4470 180694 : profile_probability probability;
4471 :
4472 180694 : last2 = last = expand_gimple_stmt (stmt);
4473 :
4474 978756 : for (last = NEXT_INSN (last); last; last = NEXT_INSN (last))
4475 748373 : if (CALL_P (last) && SIBLING_CALL_P (last))
4476 131005 : goto found;
4477 :
4478 49689 : maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4479 :
4480 49689 : *can_fallthru = true;
4481 49689 : return NULL;
4482 :
4483 131005 : found:
4484 :
4485 131005 : if (asan_epilog_seq)
4486 : {
4487 : /* We need to emit a copy of the asan_epilog_seq before
4488 : the insns emitted by expand_gimple_stmt above. The sequence
4489 : can contain labels, which need to be remapped. */
4490 117 : hash_map<rtx, rtx> label_map;
4491 117 : start_sequence ();
4492 117 : emit_note (NOTE_INSN_DELETED);
4493 1767 : for (rtx_insn *insn = asan_epilog_seq; insn; insn = NEXT_INSN (insn))
4494 1533 : switch (GET_CODE (insn))
4495 : {
4496 1182 : case INSN:
4497 1182 : case CALL_INSN:
4498 1182 : case JUMP_INSN:
4499 1182 : emit_copy_of_insn_after (insn, get_last_insn ());
4500 1182 : break;
4501 234 : case CODE_LABEL:
4502 234 : label_map.put ((rtx) insn, (rtx) emit_label (gen_label_rtx ()));
4503 234 : break;
4504 117 : case BARRIER:
4505 117 : emit_barrier ();
4506 117 : break;
4507 0 : default:
4508 0 : gcc_unreachable ();
4509 : }
4510 1767 : for (rtx_insn *insn = get_insns (); insn; insn = NEXT_INSN (insn))
4511 1650 : if (JUMP_P (insn))
4512 : {
4513 234 : subrtx_ptr_iterator::array_type array;
4514 1521 : FOR_EACH_SUBRTX_PTR (iter, array, &PATTERN (insn), ALL)
4515 : {
4516 1287 : rtx *loc = *iter;
4517 1287 : if (LABEL_REF_P (*loc))
4518 : {
4519 234 : rtx *lab = label_map.get ((rtx) label_ref_label (*loc));
4520 234 : gcc_assert (lab);
4521 234 : set_label_ref_label (*loc, as_a <rtx_insn *> (*lab));
4522 : }
4523 : }
4524 234 : if (JUMP_LABEL (insn))
4525 : {
4526 234 : rtx *lab = label_map.get (JUMP_LABEL (insn));
4527 234 : gcc_assert (lab);
4528 234 : JUMP_LABEL (insn) = *lab;
4529 : }
4530 234 : }
4531 117 : asan_epilog_seq = NEXT_INSN (get_insns ());
4532 117 : end_sequence ();
4533 117 : emit_insn_before (asan_epilog_seq, NEXT_INSN (first));
4534 117 : }
4535 :
4536 : /* ??? Wouldn't it be better to just reset any pending stack adjust?
4537 : Any instructions emitted here are about to be deleted. */
4538 131005 : do_pending_stack_adjust ();
4539 :
4540 : /* Remove any non-eh, non-abnormal edges that don't go to exit. */
4541 : /* ??? I.e. the fallthrough edge. HOWEVER! If there were to be
4542 : EH or abnormal edges, we shouldn't have created a tail call in
4543 : the first place. So it seems to me we should just be removing
4544 : all edges here, or redirecting the existing fallthru edge to
4545 : the exit block. */
4546 :
4547 131005 : probability = profile_probability::never ();
4548 :
4549 262004 : for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
4550 : {
4551 130999 : if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
4552 : {
4553 130993 : if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
4554 61072 : e->dest->count -= e->count ();
4555 130993 : probability += e->probability;
4556 130993 : expand_remove_edge (e);
4557 : }
4558 : else
4559 6 : ei_next (&ei);
4560 : }
4561 :
4562 : /* This is somewhat ugly: the call_expr expander often emits instructions
4563 : after the sibcall (to perform the function return). These confuse the
4564 : find_many_sub_basic_blocks code, so we need to get rid of these. */
4565 131005 : last = NEXT_INSN (last);
4566 131005 : gcc_assert (BARRIER_P (last));
4567 :
4568 131005 : *can_fallthru = false;
4569 163177 : while (NEXT_INSN (last))
4570 : {
4571 : /* For instance an sqrt builtin expander expands if with
4572 : sibcall in the then and label for `else`. */
4573 32172 : if (LABEL_P (NEXT_INSN (last)))
4574 : {
4575 0 : *can_fallthru = true;
4576 0 : break;
4577 : }
4578 32172 : delete_insn (NEXT_INSN (last));
4579 : }
4580 :
4581 131005 : e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
4582 : | EDGE_SIBCALL);
4583 131005 : e->probability = probability;
4584 131005 : head_end_for_bb[bb->index].second = last;
4585 262010 : update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
4586 131005 : head_end_for_bb[bb->index].second, bb);
4587 :
4588 131005 : if (NEXT_INSN (last))
4589 : {
4590 0 : bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
4591 :
4592 0 : last = BB_END (bb);
4593 0 : if (BARRIER_P (last))
4594 0 : BB_END (bb) = PREV_INSN (last);
4595 : }
4596 :
4597 131005 : maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4598 :
4599 131005 : return bb;
4600 : }
4601 :
4602 : /* Return the difference between the floor and the truncated result of
4603 : a signed division by OP1 with remainder MOD. */
4604 : static rtx
4605 4 : floor_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4606 : {
4607 : /* (mod != 0 ? (op1 / mod < 0 ? -1 : 0) : 0) */
4608 4 : return gen_rtx_IF_THEN_ELSE
4609 : (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4610 : gen_rtx_IF_THEN_ELSE
4611 : (mode, gen_rtx_LT (BImode,
4612 : gen_rtx_DIV (mode, op1, mod),
4613 : const0_rtx),
4614 : constm1_rtx, const0_rtx),
4615 : const0_rtx);
4616 : }
4617 :
4618 : /* Return the difference between the ceil and the truncated result of
4619 : a signed division by OP1 with remainder MOD. */
4620 : static rtx
4621 3 : ceil_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4622 : {
4623 : /* (mod != 0 ? (op1 / mod > 0 ? 1 : 0) : 0) */
4624 3 : return gen_rtx_IF_THEN_ELSE
4625 : (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4626 : gen_rtx_IF_THEN_ELSE
4627 : (mode, gen_rtx_GT (BImode,
4628 : gen_rtx_DIV (mode, op1, mod),
4629 : const0_rtx),
4630 : const1_rtx, const0_rtx),
4631 : const0_rtx);
4632 : }
4633 :
4634 : /* Return the difference between the ceil and the truncated result of
4635 : an unsigned division by OP1 with remainder MOD. */
4636 : static rtx
4637 0 : ceil_udiv_adjust (machine_mode mode, rtx mod, rtx op1 ATTRIBUTE_UNUSED)
4638 : {
4639 : /* (mod != 0 ? 1 : 0) */
4640 0 : return gen_rtx_IF_THEN_ELSE
4641 : (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4642 : const1_rtx, const0_rtx);
4643 : }
4644 :
4645 : /* Return the difference between the rounded and the truncated result
4646 : of a signed division by OP1 with remainder MOD. Halfway cases are
4647 : rounded away from zero, rather than to the nearest even number. */
4648 : static rtx
4649 0 : round_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4650 : {
4651 : /* (abs (mod) >= abs (op1) - abs (mod)
4652 : ? (op1 / mod > 0 ? 1 : -1)
4653 : : 0) */
4654 0 : return gen_rtx_IF_THEN_ELSE
4655 : (mode, gen_rtx_GE (BImode, gen_rtx_ABS (mode, mod),
4656 : gen_rtx_MINUS (mode,
4657 : gen_rtx_ABS (mode, op1),
4658 : gen_rtx_ABS (mode, mod))),
4659 : gen_rtx_IF_THEN_ELSE
4660 : (mode, gen_rtx_GT (BImode,
4661 : gen_rtx_DIV (mode, op1, mod),
4662 : const0_rtx),
4663 : const1_rtx, constm1_rtx),
4664 : const0_rtx);
4665 : }
4666 :
4667 : /* Return the difference between the rounded and the truncated result
4668 : of a unsigned division by OP1 with remainder MOD. Halfway cases
4669 : are rounded away from zero, rather than to the nearest even
4670 : number. */
4671 : static rtx
4672 0 : round_udiv_adjust (machine_mode mode, rtx mod, rtx op1)
4673 : {
4674 : /* (mod >= op1 - mod ? 1 : 0) */
4675 0 : return gen_rtx_IF_THEN_ELSE
4676 : (mode, gen_rtx_GE (BImode, mod,
4677 : gen_rtx_MINUS (mode, op1, mod)),
4678 : const1_rtx, const0_rtx);
4679 : }
4680 :
4681 : /* Convert X to MODE, that must be Pmode or ptr_mode, without emitting
4682 : any rtl. */
4683 :
4684 : static rtx
4685 7598878 : convert_debug_memory_address (scalar_int_mode mode, rtx x,
4686 : addr_space_t as)
4687 : {
4688 : #ifndef POINTERS_EXTEND_UNSIGNED
4689 : gcc_assert (mode == Pmode
4690 : || mode == targetm.addr_space.address_mode (as));
4691 : gcc_assert (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode);
4692 : #else
4693 7598878 : rtx temp;
4694 :
4695 7598878 : gcc_assert (targetm.addr_space.valid_pointer_mode (mode, as));
4696 :
4697 7598878 : if (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode)
4698 : return x;
4699 :
4700 : /* X must have some form of address mode already. */
4701 0 : scalar_int_mode xmode = as_a <scalar_int_mode> (GET_MODE (x));
4702 0 : if (GET_MODE_PRECISION (mode) < GET_MODE_PRECISION (xmode))
4703 0 : x = lowpart_subreg (mode, x, xmode);
4704 0 : else if (POINTERS_EXTEND_UNSIGNED > 0)
4705 0 : x = gen_rtx_ZERO_EXTEND (mode, x);
4706 : else if (!POINTERS_EXTEND_UNSIGNED)
4707 : x = gen_rtx_SIGN_EXTEND (mode, x);
4708 : else
4709 : {
4710 : switch (GET_CODE (x))
4711 : {
4712 : case SUBREG:
4713 : if ((SUBREG_PROMOTED_VAR_P (x)
4714 : || (REG_P (SUBREG_REG (x)) && REG_POINTER (SUBREG_REG (x)))
4715 : || (GET_CODE (SUBREG_REG (x)) == PLUS
4716 : && REG_P (XEXP (SUBREG_REG (x), 0))
4717 : && REG_POINTER (XEXP (SUBREG_REG (x), 0))
4718 : && CONST_INT_P (XEXP (SUBREG_REG (x), 1))))
4719 : && GET_MODE (SUBREG_REG (x)) == mode)
4720 : return SUBREG_REG (x);
4721 : break;
4722 : case LABEL_REF:
4723 : temp = gen_rtx_LABEL_REF (mode, label_ref_label (x));
4724 : LABEL_REF_NONLOCAL_P (temp) = LABEL_REF_NONLOCAL_P (x);
4725 : return temp;
4726 : case SYMBOL_REF:
4727 : temp = shallow_copy_rtx (x);
4728 : PUT_MODE (temp, mode);
4729 : return temp;
4730 : case CONST:
4731 : temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4732 : if (temp)
4733 : temp = gen_rtx_CONST (mode, temp);
4734 : return temp;
4735 : case PLUS:
4736 : case MINUS:
4737 : if (CONST_INT_P (XEXP (x, 1)))
4738 : {
4739 : temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4740 : if (temp)
4741 : return gen_rtx_fmt_ee (GET_CODE (x), mode, temp, XEXP (x, 1));
4742 : }
4743 : break;
4744 : default:
4745 : break;
4746 : }
4747 : /* Don't know how to express ptr_extend as operation in debug info. */
4748 : return NULL;
4749 : }
4750 : #endif /* POINTERS_EXTEND_UNSIGNED */
4751 :
4752 : return x;
4753 : }
4754 :
4755 : /* Map from SSA_NAMEs to corresponding DEBUG_EXPR_DECLs created
4756 : by avoid_deep_ter_for_debug. */
4757 :
4758 : static hash_map<tree, tree> *deep_ter_debug_map;
4759 :
4760 : /* Split too deep TER chains for debug stmts using debug temporaries. */
4761 :
4762 : static void
4763 43504729 : avoid_deep_ter_for_debug (gimple *stmt, int depth)
4764 : {
4765 43504729 : use_operand_p use_p;
4766 43504729 : ssa_op_iter iter;
4767 53834777 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
4768 : {
4769 10330048 : tree use = USE_FROM_PTR (use_p);
4770 10330048 : if (TREE_CODE (use) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (use))
4771 8608064 : continue;
4772 8951564 : gimple *g = get_gimple_for_ssa_name (use);
4773 8951564 : if (g == NULL)
4774 7225992 : continue;
4775 1725572 : if ((depth > 6 || !is_gimple_assign (g)) && !stmt_ends_bb_p (g))
4776 : {
4777 9219 : if (deep_ter_debug_map == NULL)
4778 860 : deep_ter_debug_map = new hash_map<tree, tree>;
4779 :
4780 9219 : tree &vexpr = deep_ter_debug_map->get_or_insert (use);
4781 9219 : if (vexpr != NULL)
4782 3588 : continue;
4783 5631 : vexpr = build_debug_expr_decl (TREE_TYPE (use));
4784 5631 : gimple *def_temp = gimple_build_debug_bind (vexpr, use, g);
4785 5631 : gimple_stmt_iterator gsi = gsi_for_stmt (g);
4786 5631 : gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
4787 5631 : avoid_deep_ter_for_debug (def_temp, 0);
4788 : }
4789 : else
4790 1716353 : avoid_deep_ter_for_debug (g, depth + 1);
4791 : }
4792 43504729 : }
4793 :
4794 : /* Return an RTX equivalent to the value of the parameter DECL. */
4795 :
4796 : static rtx
4797 192739 : expand_debug_parm_decl (tree decl)
4798 : {
4799 192739 : rtx incoming = DECL_INCOMING_RTL (decl);
4800 :
4801 192739 : if (incoming
4802 19754 : && GET_MODE (incoming) != BLKmode
4803 212489 : && ((REG_P (incoming) && HARD_REGISTER_P (incoming))
4804 965 : || (MEM_P (incoming)
4805 961 : && REG_P (XEXP (incoming, 0))
4806 334 : && HARD_REGISTER_P (XEXP (incoming, 0)))))
4807 : {
4808 18812 : rtx rtl = gen_rtx_ENTRY_VALUE (GET_MODE (incoming));
4809 :
4810 : #ifdef HAVE_window_save
4811 : /* DECL_INCOMING_RTL uses the INCOMING_REGNO of parameter registers.
4812 : If the target machine has an explicit window save instruction, the
4813 : actual entry value is the corresponding OUTGOING_REGNO instead. */
4814 : if (REG_P (incoming)
4815 : && OUTGOING_REGNO (REGNO (incoming)) != REGNO (incoming))
4816 : incoming
4817 : = gen_rtx_REG_offset (incoming, GET_MODE (incoming),
4818 : OUTGOING_REGNO (REGNO (incoming)), 0);
4819 : else if (MEM_P (incoming))
4820 : {
4821 : rtx reg = XEXP (incoming, 0);
4822 : if (OUTGOING_REGNO (REGNO (reg)) != REGNO (reg))
4823 : {
4824 : reg = gen_raw_REG (GET_MODE (reg), OUTGOING_REGNO (REGNO (reg)));
4825 : incoming = replace_equiv_address_nv (incoming, reg);
4826 : }
4827 : else
4828 : incoming = copy_rtx (incoming);
4829 : }
4830 : #endif
4831 :
4832 18812 : ENTRY_VALUE_EXP (rtl) = incoming;
4833 18812 : return rtl;
4834 : }
4835 :
4836 173927 : if (incoming
4837 942 : && GET_MODE (incoming) != BLKmode
4838 938 : && !TREE_ADDRESSABLE (decl)
4839 938 : && MEM_P (incoming)
4840 934 : && (XEXP (incoming, 0) == virtual_incoming_args_rtx
4841 627 : || (GET_CODE (XEXP (incoming, 0)) == PLUS
4842 627 : && XEXP (XEXP (incoming, 0), 0) == virtual_incoming_args_rtx
4843 611 : && CONST_INT_P (XEXP (XEXP (incoming, 0), 1)))))
4844 918 : return copy_rtx (incoming);
4845 :
4846 : return NULL_RTX;
4847 : }
4848 :
4849 : /* Return an RTX equivalent to the value of the tree expression EXP. */
4850 :
4851 : static rtx
4852 44050286 : expand_debug_expr (tree exp)
4853 : {
4854 44050286 : rtx op0 = NULL_RTX, op1 = NULL_RTX, op2 = NULL_RTX;
4855 44050286 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
4856 44050286 : machine_mode inner_mode = VOIDmode;
4857 44050286 : int unsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
4858 44050286 : addr_space_t as;
4859 44050286 : scalar_int_mode op0_mode, op1_mode, addr_mode;
4860 :
4861 44050286 : switch (TREE_CODE_CLASS (TREE_CODE (exp)))
4862 : {
4863 7268078 : case tcc_expression:
4864 7268078 : switch (TREE_CODE (exp))
4865 : {
4866 8585 : case COND_EXPR:
4867 8585 : case DOT_PROD_EXPR:
4868 8585 : case SAD_EXPR:
4869 8585 : case WIDEN_MULT_PLUS_EXPR:
4870 8585 : case WIDEN_MULT_MINUS_EXPR:
4871 8585 : goto ternary;
4872 :
4873 0 : case TRUTH_ANDIF_EXPR:
4874 0 : case TRUTH_ORIF_EXPR:
4875 0 : case TRUTH_AND_EXPR:
4876 0 : case TRUTH_OR_EXPR:
4877 0 : case TRUTH_XOR_EXPR:
4878 0 : goto binary;
4879 :
4880 0 : case TRUTH_NOT_EXPR:
4881 0 : goto unary;
4882 :
4883 : default:
4884 : break;
4885 : }
4886 : break;
4887 :
4888 8585 : ternary:
4889 8585 : op2 = expand_debug_expr (TREE_OPERAND (exp, 2));
4890 8585 : if (!op2)
4891 : return NULL_RTX;
4892 : /* Fall through. */
4893 :
4894 8585 : binary:
4895 1522980 : case tcc_binary:
4896 1522980 : if (mode == BLKmode)
4897 : return NULL_RTX;
4898 1522970 : op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4899 1522970 : if (!op1)
4900 : return NULL_RTX;
4901 1522699 : switch (TREE_CODE (exp))
4902 : {
4903 81677 : case LSHIFT_EXPR:
4904 81677 : case RSHIFT_EXPR:
4905 81677 : case LROTATE_EXPR:
4906 81677 : case RROTATE_EXPR:
4907 81677 : case WIDEN_LSHIFT_EXPR:
4908 : /* Ensure second operand isn't wider than the first one. */
4909 81677 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 1)));
4910 81677 : if (is_a <scalar_int_mode> (inner_mode, &op1_mode)
4911 81669 : && (GET_MODE_UNIT_PRECISION (mode)
4912 81669 : < GET_MODE_PRECISION (op1_mode)))
4913 5574 : op1 = lowpart_subreg (GET_MODE_INNER (mode), op1, op1_mode);
4914 : break;
4915 : default:
4916 : break;
4917 : }
4918 : /* Fall through. */
4919 :
4920 2141397 : unary:
4921 2141397 : case tcc_unary:
4922 2141397 : if (mode == BLKmode)
4923 : return NULL_RTX;
4924 2141395 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4925 2141395 : op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4926 2141395 : if (!op0)
4927 : return NULL_RTX;
4928 : break;
4929 :
4930 73715 : case tcc_comparison:
4931 73715 : unsignedp = TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)));
4932 73715 : goto binary;
4933 :
4934 0 : case tcc_type:
4935 0 : case tcc_statement:
4936 0 : gcc_unreachable ();
4937 :
4938 : case tcc_constant:
4939 : case tcc_exceptional:
4940 : case tcc_declaration:
4941 : case tcc_reference:
4942 : case tcc_vl_exp:
4943 : break;
4944 : }
4945 :
4946 44045696 : switch (TREE_CODE (exp))
4947 : {
4948 302591 : case STRING_CST:
4949 302591 : if (!lookup_constant_def (exp))
4950 : {
4951 118264 : if (strlen (TREE_STRING_POINTER (exp)) + 1
4952 118264 : != (size_t) TREE_STRING_LENGTH (exp))
4953 : return NULL_RTX;
4954 118374 : op0 = gen_rtx_CONST_STRING (Pmode, TREE_STRING_POINTER (exp));
4955 117536 : op0 = gen_rtx_MEM (BLKmode, op0);
4956 117536 : set_mem_attributes (op0, exp, 0);
4957 117536 : return op0;
4958 : }
4959 : /* Fall through. */
4960 :
4961 6267556 : case INTEGER_CST:
4962 12535074 : if (BITINT_TYPE_P (TREE_TYPE (exp))
4963 6267556 : && TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
4964 : return NULL;
4965 : /* FALLTHRU */
4966 6337469 : case REAL_CST:
4967 6337469 : case FIXED_CST:
4968 6337469 : op0 = expand_expr (exp, NULL_RTX, mode, EXPAND_INITIALIZER);
4969 6337469 : return op0;
4970 :
4971 0 : case POLY_INT_CST:
4972 0 : return immed_wide_int_const (poly_int_cst_value (exp), mode);
4973 :
4974 4052 : case COMPLEX_CST:
4975 4052 : gcc_assert (COMPLEX_MODE_P (mode));
4976 4052 : op0 = expand_debug_expr (TREE_REALPART (exp));
4977 4052 : op1 = expand_debug_expr (TREE_IMAGPART (exp));
4978 4052 : return gen_rtx_CONCAT (mode, op0, op1);
4979 :
4980 6328102 : case DEBUG_EXPR_DECL:
4981 6328102 : op0 = DECL_RTL_IF_SET (exp);
4982 :
4983 5103050 : if (op0)
4984 : {
4985 5103050 : if (GET_MODE (op0) != mode)
4986 0 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (exp)));
4987 : else
4988 : return op0;
4989 : }
4990 :
4991 1225052 : op0 = gen_rtx_DEBUG_EXPR (mode);
4992 1225052 : DEBUG_EXPR_TREE_DECL (op0) = exp;
4993 1225052 : SET_DECL_RTL (exp, op0);
4994 :
4995 1225052 : return op0;
4996 :
4997 5745739 : case VAR_DECL:
4998 5745739 : case PARM_DECL:
4999 5745739 : case FUNCTION_DECL:
5000 5745739 : case LABEL_DECL:
5001 5745739 : case CONST_DECL:
5002 5745739 : case RESULT_DECL:
5003 5745739 : op0 = DECL_RTL_IF_SET (exp);
5004 :
5005 : /* This decl was probably optimized away. */
5006 3994093 : if (!op0
5007 : /* At least label RTXen are sometimes replaced by
5008 : NOTE_INSN_DELETED_LABEL. Any notes here are not
5009 : handled by copy_rtx. */
5010 3994093 : || NOTE_P (op0))
5011 : {
5012 1751647 : if (!VAR_P (exp)
5013 1737166 : || DECL_EXTERNAL (exp)
5014 1736543 : || !TREE_STATIC (exp)
5015 133355 : || !DECL_NAME (exp)
5016 133355 : || DECL_HARD_REGISTER (exp)
5017 133355 : || DECL_IN_CONSTANT_POOL (exp)
5018 133355 : || mode == VOIDmode
5019 1885002 : || symtab_node::get (exp) == NULL)
5020 : return NULL;
5021 :
5022 18850 : op0 = make_decl_rtl_for_debug (exp);
5023 18850 : if (!MEM_P (op0)
5024 18850 : || GET_CODE (XEXP (op0, 0)) != SYMBOL_REF
5025 37700 : || SYMBOL_REF_DECL (XEXP (op0, 0)) != exp)
5026 : return NULL;
5027 : }
5028 3994092 : else if (VAR_P (exp)
5029 3873078 : && is_global_var (exp)
5030 4222636 : && symtab_node::get (exp) == NULL)
5031 : return NULL;
5032 : else
5033 3994092 : op0 = copy_rtx (op0);
5034 :
5035 4012942 : if (GET_MODE (op0) == BLKmode
5036 : /* If op0 is not BLKmode, but mode is, adjust_mode
5037 : below would ICE. While it is likely a FE bug,
5038 : try to be robust here. See PR43166. */
5039 532972 : || mode == BLKmode
5040 532972 : || (mode == VOIDmode && GET_MODE (op0) != VOIDmode))
5041 : {
5042 3479970 : gcc_assert (MEM_P (op0));
5043 3479970 : op0 = adjust_address_nv (op0, mode, 0);
5044 3479970 : return op0;
5045 : }
5046 :
5047 : /* Fall through. */
5048 :
5049 532972 : adjust_mode:
5050 10981760 : case PAREN_EXPR:
5051 10981760 : CASE_CONVERT:
5052 10981760 : {
5053 10981760 : inner_mode = GET_MODE (op0);
5054 :
5055 10981760 : if (mode == inner_mode)
5056 : return op0;
5057 :
5058 159453 : if (inner_mode == VOIDmode)
5059 : {
5060 9049 : if (TREE_CODE (exp) == SSA_NAME)
5061 244 : inner_mode = TYPE_MODE (TREE_TYPE (exp));
5062 : else
5063 8805 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
5064 9049 : if (mode == inner_mode)
5065 : return op0;
5066 : }
5067 :
5068 153212 : if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5069 : {
5070 6314 : if (GET_MODE_UNIT_BITSIZE (mode)
5071 12628 : == GET_MODE_UNIT_BITSIZE (inner_mode))
5072 4 : op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
5073 6310 : else if (GET_MODE_UNIT_BITSIZE (mode)
5074 12620 : < GET_MODE_UNIT_BITSIZE (inner_mode))
5075 3585 : op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
5076 : else
5077 2725 : op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
5078 : }
5079 146898 : else if (FLOAT_MODE_P (mode))
5080 : {
5081 0 : gcc_assert (TREE_CODE (exp) != SSA_NAME);
5082 0 : if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5083 0 : op0 = simplify_gen_unary (UNSIGNED_FLOAT, mode, op0, inner_mode);
5084 : else
5085 0 : op0 = simplify_gen_unary (FLOAT, mode, op0, inner_mode);
5086 : }
5087 146898 : else if (FLOAT_MODE_P (inner_mode))
5088 : {
5089 0 : if (unsignedp)
5090 0 : op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
5091 : else
5092 0 : op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
5093 : }
5094 293796 : else if (GET_MODE_UNIT_PRECISION (mode)
5095 146898 : == GET_MODE_UNIT_PRECISION (inner_mode))
5096 0 : op0 = lowpart_subreg (mode, op0, inner_mode);
5097 293796 : else if (GET_MODE_UNIT_PRECISION (mode)
5098 146898 : < GET_MODE_UNIT_PRECISION (inner_mode))
5099 71497 : op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
5100 75401 : else if (UNARY_CLASS_P (exp)
5101 75401 : ? TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)))
5102 0 : : unsignedp)
5103 50582 : op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5104 : else
5105 24819 : op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5106 :
5107 : return op0;
5108 : }
5109 :
5110 2241106 : case MEM_REF:
5111 2241106 : if (!is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
5112 : {
5113 977600 : tree newexp = fold_binary (MEM_REF, TREE_TYPE (exp),
5114 : TREE_OPERAND (exp, 0),
5115 : TREE_OPERAND (exp, 1));
5116 977600 : if (newexp)
5117 0 : return expand_debug_expr (newexp);
5118 : }
5119 : /* FALLTHROUGH */
5120 2241106 : case INDIRECT_REF:
5121 2241106 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
5122 2241106 : op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5123 2241106 : if (!op0)
5124 : return NULL;
5125 :
5126 2234982 : if (TREE_CODE (exp) == MEM_REF)
5127 : {
5128 2234982 : if (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5129 2186437 : || (GET_CODE (op0) == PLUS
5130 292639 : && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR))
5131 : /* (mem (debug_implicit_ptr)) might confuse aliasing.
5132 : Instead just use get_inner_reference. */
5133 48545 : goto component_ref;
5134 :
5135 2186437 : op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
5136 2186437 : poly_int64 offset;
5137 2186437 : if (!op1 || !poly_int_rtx_p (op1, &offset))
5138 0 : return NULL;
5139 :
5140 2186437 : op0 = plus_constant (inner_mode, op0, offset);
5141 : }
5142 :
5143 2186437 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
5144 :
5145 2186437 : op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
5146 : op0, as);
5147 2186437 : if (op0 == NULL_RTX)
5148 : return NULL;
5149 :
5150 2186437 : op0 = gen_rtx_MEM (mode, op0);
5151 2186437 : set_mem_attributes (op0, exp, 0);
5152 2186437 : if (TREE_CODE (exp) == MEM_REF
5153 2186437 : && !is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
5154 977600 : set_mem_expr (op0, NULL_TREE);
5155 2186437 : set_mem_addr_space (op0, as);
5156 :
5157 2186437 : return op0;
5158 :
5159 67338 : case TARGET_MEM_REF:
5160 67338 : if (TREE_CODE (TMR_BASE (exp)) == ADDR_EXPR
5161 67338 : && !DECL_RTL_SET_P (TREE_OPERAND (TMR_BASE (exp), 0)))
5162 : return NULL;
5163 :
5164 67328 : op0 = expand_debug_expr
5165 67328 : (tree_mem_ref_addr (build_pointer_type (TREE_TYPE (exp)), exp));
5166 67328 : if (!op0)
5167 : return NULL;
5168 :
5169 67328 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
5170 67328 : op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
5171 : op0, as);
5172 67328 : if (op0 == NULL_RTX)
5173 : return NULL;
5174 :
5175 67328 : op0 = gen_rtx_MEM (mode, op0);
5176 :
5177 67328 : set_mem_attributes (op0, exp, 0);
5178 67328 : set_mem_addr_space (op0, as);
5179 :
5180 67328 : return op0;
5181 :
5182 48545 : component_ref:
5183 3955531 : case ARRAY_REF:
5184 3955531 : case ARRAY_RANGE_REF:
5185 3955531 : case COMPONENT_REF:
5186 3955531 : case BIT_FIELD_REF:
5187 3955531 : case REALPART_EXPR:
5188 3955531 : case IMAGPART_EXPR:
5189 3955531 : case VIEW_CONVERT_EXPR:
5190 3955531 : {
5191 3955531 : machine_mode mode1;
5192 3955531 : poly_int64 bitsize, bitpos;
5193 3955531 : tree offset;
5194 3955531 : int reversep, volatilep = 0;
5195 3955531 : tree tem
5196 3955531 : = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
5197 : &unsignedp, &reversep, &volatilep);
5198 3955531 : rtx orig_op0;
5199 :
5200 3955531 : if (known_eq (bitsize, 0))
5201 : return NULL;
5202 :
5203 3894617 : orig_op0 = op0 = expand_debug_expr (tem);
5204 :
5205 3894617 : if (!op0)
5206 : return NULL;
5207 :
5208 3546770 : if (offset)
5209 : {
5210 15892 : machine_mode addrmode, offmode;
5211 :
5212 15892 : if (!MEM_P (op0))
5213 : return NULL;
5214 :
5215 15884 : op0 = XEXP (op0, 0);
5216 15884 : addrmode = GET_MODE (op0);
5217 15884 : if (addrmode == VOIDmode)
5218 0 : addrmode = Pmode;
5219 :
5220 15884 : op1 = expand_debug_expr (offset);
5221 15884 : if (!op1)
5222 : return NULL;
5223 :
5224 15882 : offmode = GET_MODE (op1);
5225 15882 : if (offmode == VOIDmode)
5226 6 : offmode = TYPE_MODE (TREE_TYPE (offset));
5227 :
5228 15882 : if (addrmode != offmode)
5229 0 : op1 = lowpart_subreg (addrmode, op1, offmode);
5230 :
5231 : /* Don't use offset_address here, we don't need a
5232 : recognizable address, and we don't want to generate
5233 : code. */
5234 15882 : op0 = gen_rtx_MEM (mode, simplify_gen_binary (PLUS, addrmode,
5235 : op0, op1));
5236 : }
5237 :
5238 3546760 : if (MEM_P (op0))
5239 : {
5240 3406794 : if (mode1 == VOIDmode)
5241 : {
5242 3433 : if (maybe_gt (bitsize, MAX_BITSIZE_MODE_ANY_INT))
5243 3955531 : return NULL;
5244 : /* Bitfield. */
5245 3433 : mode1 = smallest_int_mode_for_size (bitsize).require ();
5246 : }
5247 3406794 : poly_int64 bytepos = bits_to_bytes_round_down (bitpos);
5248 3406794 : if (maybe_ne (bytepos, 0))
5249 : {
5250 1436000 : op0 = adjust_address_nv (op0, mode1, bytepos);
5251 1436000 : bitpos = num_trailing_bits (bitpos);
5252 : }
5253 1970794 : else if (known_eq (bitpos, 0)
5254 3940962 : && known_eq (bitsize, GET_MODE_BITSIZE (mode)))
5255 523551 : op0 = adjust_address_nv (op0, mode, 0);
5256 1447243 : else if (GET_MODE (op0) != mode1)
5257 3041 : op0 = adjust_address_nv (op0, mode1, 0);
5258 : else
5259 1444202 : op0 = copy_rtx (op0);
5260 3406794 : if (op0 == orig_op0)
5261 151948 : op0 = shallow_copy_rtx (op0);
5262 3406794 : if (TREE_CODE (tem) != SSA_NAME)
5263 3404425 : set_mem_attributes (op0, exp, 0);
5264 : }
5265 :
5266 3546760 : if (known_eq (bitpos, 0) && mode == GET_MODE (op0))
5267 : return op0;
5268 :
5269 105396 : if (maybe_lt (bitpos, 0))
5270 : return NULL;
5271 :
5272 105394 : if (GET_MODE (op0) == BLKmode || mode == BLKmode)
5273 : return NULL;
5274 :
5275 105362 : poly_int64 bytepos;
5276 105362 : if (multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
5277 207590 : && known_eq (bitsize, GET_MODE_BITSIZE (mode1)))
5278 : {
5279 102930 : machine_mode opmode = GET_MODE (op0);
5280 :
5281 102930 : if (opmode == VOIDmode)
5282 0 : opmode = TYPE_MODE (TREE_TYPE (tem));
5283 :
5284 : /* This condition may hold if we're expanding the address
5285 : right past the end of an array that turned out not to
5286 : be addressable (i.e., the address was only computed in
5287 : debug stmts). The gen_subreg below would rightfully
5288 : crash, and the address doesn't really exist, so just
5289 : drop it. */
5290 205860 : if (known_ge (bitpos, GET_MODE_BITSIZE (opmode)))
5291 : return NULL;
5292 :
5293 208240 : if (multiple_p (bitpos, GET_MODE_BITSIZE (mode)))
5294 102858 : return simplify_gen_subreg (mode, op0, opmode, bytepos);
5295 : }
5296 :
5297 4956 : return simplify_gen_ternary (SCALAR_INT_MODE_P (GET_MODE (op0))
5298 1829 : && TYPE_UNSIGNED (TREE_TYPE (exp))
5299 : ? SIGN_EXTRACT
5300 : : ZERO_EXTRACT, mode,
5301 2478 : GET_MODE (op0) != VOIDmode
5302 : ? GET_MODE (op0)
5303 0 : : TYPE_MODE (TREE_TYPE (tem)),
5304 : op0, gen_int_mode (bitsize, word_mode),
5305 2478 : gen_int_mode (bitpos, word_mode));
5306 : }
5307 :
5308 4022 : case ABS_EXPR:
5309 4022 : case ABSU_EXPR:
5310 4022 : return simplify_gen_unary (ABS, mode, op0, mode);
5311 :
5312 14498 : case NEGATE_EXPR:
5313 14498 : return simplify_gen_unary (NEG, mode, op0, mode);
5314 :
5315 20902 : case BIT_NOT_EXPR:
5316 20902 : return simplify_gen_unary (NOT, mode, op0, mode);
5317 :
5318 2578 : case FLOAT_EXPR:
5319 2578 : return simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5320 : 0)))
5321 : ? UNSIGNED_FLOAT : FLOAT, mode, op0,
5322 2578 : inner_mode);
5323 :
5324 436 : case FIX_TRUNC_EXPR:
5325 436 : return simplify_gen_unary (unsignedp ? UNSIGNED_FIX : FIX, mode, op0,
5326 436 : inner_mode);
5327 :
5328 259515 : case POINTER_PLUS_EXPR:
5329 : /* For the rare target where pointers are not the same size as
5330 : size_t, we need to check for mis-matched modes and correct
5331 : the addend. */
5332 259515 : if (op0 && op1
5333 259515 : && is_a <scalar_int_mode> (GET_MODE (op0), &op0_mode)
5334 259423 : && is_a <scalar_int_mode> (GET_MODE (op1), &op1_mode)
5335 407321 : && op0_mode != op1_mode)
5336 : {
5337 0 : if (GET_MODE_BITSIZE (op0_mode) < GET_MODE_BITSIZE (op1_mode)
5338 : /* If OP0 is a partial mode, then we must truncate, even
5339 : if it has the same bitsize as OP1 as GCC's
5340 : representation of partial modes is opaque. */
5341 0 : || (GET_MODE_CLASS (op0_mode) == MODE_PARTIAL_INT
5342 0 : && (GET_MODE_BITSIZE (op0_mode)
5343 0 : == GET_MODE_BITSIZE (op1_mode))))
5344 0 : op1 = simplify_gen_unary (TRUNCATE, op0_mode, op1, op1_mode);
5345 : else
5346 : /* We always sign-extend, regardless of the signedness of
5347 : the operand, because the operand is always unsigned
5348 : here even if the original C expression is signed. */
5349 0 : op1 = simplify_gen_unary (SIGN_EXTEND, op0_mode, op1, op1_mode);
5350 : }
5351 : /* Fall through. */
5352 791608 : case PLUS_EXPR:
5353 791608 : return simplify_gen_binary (PLUS, mode, op0, op1);
5354 :
5355 110507 : case MINUS_EXPR:
5356 110507 : case POINTER_DIFF_EXPR:
5357 110507 : return simplify_gen_binary (MINUS, mode, op0, op1);
5358 :
5359 156396 : case MULT_EXPR:
5360 156396 : return simplify_gen_binary (MULT, mode, op0, op1);
5361 :
5362 4071 : case RDIV_EXPR:
5363 4071 : gcc_assert (FLOAT_MODE_P (mode)
5364 : || ALL_FIXED_POINT_MODE_P (mode));
5365 : /* Fall through. */
5366 105099 : case TRUNC_DIV_EXPR:
5367 105099 : case EXACT_DIV_EXPR:
5368 105099 : if (unsignedp)
5369 10931 : return simplify_gen_binary (UDIV, mode, op0, op1);
5370 : else
5371 94168 : return simplify_gen_binary (DIV, mode, op0, op1);
5372 :
5373 4121 : case TRUNC_MOD_EXPR:
5374 8242 : return simplify_gen_binary (unsignedp ? UMOD : MOD, mode, op0, op1);
5375 :
5376 3 : case FLOOR_DIV_EXPR:
5377 3 : if (unsignedp)
5378 0 : return simplify_gen_binary (UDIV, mode, op0, op1);
5379 : else
5380 : {
5381 3 : rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5382 3 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5383 3 : rtx adj = floor_sdiv_adjust (mode, mod, op1);
5384 3 : return simplify_gen_binary (PLUS, mode, div, adj);
5385 : }
5386 :
5387 1 : case FLOOR_MOD_EXPR:
5388 1 : if (unsignedp)
5389 0 : return simplify_gen_binary (UMOD, mode, op0, op1);
5390 : else
5391 : {
5392 1 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5393 1 : rtx adj = floor_sdiv_adjust (mode, mod, op1);
5394 1 : adj = simplify_gen_unary (NEG, mode,
5395 : simplify_gen_binary (MULT, mode, adj, op1),
5396 : mode);
5397 1 : return simplify_gen_binary (PLUS, mode, mod, adj);
5398 : }
5399 :
5400 3 : case CEIL_DIV_EXPR:
5401 3 : if (unsignedp)
5402 : {
5403 0 : rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
5404 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5405 0 : rtx adj = ceil_udiv_adjust (mode, mod, op1);
5406 0 : return simplify_gen_binary (PLUS, mode, div, adj);
5407 : }
5408 : else
5409 : {
5410 3 : rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5411 3 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5412 3 : rtx adj = ceil_sdiv_adjust (mode, mod, op1);
5413 3 : return simplify_gen_binary (PLUS, mode, div, adj);
5414 : }
5415 :
5416 0 : case CEIL_MOD_EXPR:
5417 0 : if (unsignedp)
5418 : {
5419 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5420 0 : rtx adj = ceil_udiv_adjust (mode, mod, op1);
5421 0 : adj = simplify_gen_unary (NEG, mode,
5422 : simplify_gen_binary (MULT, mode, adj, op1),
5423 : mode);
5424 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5425 : }
5426 : else
5427 : {
5428 0 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5429 0 : rtx adj = ceil_sdiv_adjust (mode, mod, op1);
5430 0 : adj = simplify_gen_unary (NEG, mode,
5431 : simplify_gen_binary (MULT, mode, adj, op1),
5432 : mode);
5433 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5434 : }
5435 :
5436 0 : case ROUND_DIV_EXPR:
5437 0 : if (unsignedp)
5438 : {
5439 0 : rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
5440 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5441 0 : rtx adj = round_udiv_adjust (mode, mod, op1);
5442 0 : return simplify_gen_binary (PLUS, mode, div, adj);
5443 : }
5444 : else
5445 : {
5446 0 : rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5447 0 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5448 0 : rtx adj = round_sdiv_adjust (mode, mod, op1);
5449 0 : return simplify_gen_binary (PLUS, mode, div, adj);
5450 : }
5451 :
5452 0 : case ROUND_MOD_EXPR:
5453 0 : if (unsignedp)
5454 : {
5455 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5456 0 : rtx adj = round_udiv_adjust (mode, mod, op1);
5457 0 : adj = simplify_gen_unary (NEG, mode,
5458 : simplify_gen_binary (MULT, mode, adj, op1),
5459 : mode);
5460 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5461 : }
5462 : else
5463 : {
5464 0 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5465 0 : rtx adj = round_sdiv_adjust (mode, mod, op1);
5466 0 : adj = simplify_gen_unary (NEG, mode,
5467 : simplify_gen_binary (MULT, mode, adj, op1),
5468 : mode);
5469 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5470 : }
5471 :
5472 15086 : case LSHIFT_EXPR:
5473 15086 : return simplify_gen_binary (ASHIFT, mode, op0, op1);
5474 :
5475 65654 : case RSHIFT_EXPR:
5476 65654 : if (unsignedp)
5477 61824 : return simplify_gen_binary (LSHIFTRT, mode, op0, op1);
5478 : else
5479 3830 : return simplify_gen_binary (ASHIFTRT, mode, op0, op1);
5480 :
5481 2 : case LROTATE_EXPR:
5482 2 : return simplify_gen_binary (ROTATE, mode, op0, op1);
5483 :
5484 891 : case RROTATE_EXPR:
5485 891 : return simplify_gen_binary (ROTATERT, mode, op0, op1);
5486 :
5487 10016 : case MIN_EXPR:
5488 20032 : return simplify_gen_binary (unsignedp ? UMIN : SMIN, mode, op0, op1);
5489 :
5490 15249 : case MAX_EXPR:
5491 30498 : return simplify_gen_binary (unsignedp ? UMAX : SMAX, mode, op0, op1);
5492 :
5493 92843 : case BIT_AND_EXPR:
5494 92843 : case TRUTH_AND_EXPR:
5495 92843 : return simplify_gen_binary (AND, mode, op0, op1);
5496 :
5497 43507 : case BIT_IOR_EXPR:
5498 43507 : case TRUTH_OR_EXPR:
5499 43507 : return simplify_gen_binary (IOR, mode, op0, op1);
5500 :
5501 9932 : case BIT_XOR_EXPR:
5502 9932 : case TRUTH_XOR_EXPR:
5503 9932 : return simplify_gen_binary (XOR, mode, op0, op1);
5504 :
5505 0 : case TRUTH_ANDIF_EXPR:
5506 0 : return gen_rtx_IF_THEN_ELSE (mode, op0, op1, const0_rtx);
5507 :
5508 0 : case TRUTH_ORIF_EXPR:
5509 0 : return gen_rtx_IF_THEN_ELSE (mode, op0, const_true_rtx, op1);
5510 :
5511 0 : case TRUTH_NOT_EXPR:
5512 0 : return simplify_gen_relational (EQ, mode, inner_mode, op0, const0_rtx);
5513 :
5514 5591 : case LT_EXPR:
5515 5591 : return simplify_gen_relational (unsignedp ? LTU : LT, mode, inner_mode,
5516 5591 : op0, op1);
5517 :
5518 6086 : case LE_EXPR:
5519 6086 : return simplify_gen_relational (unsignedp ? LEU : LE, mode, inner_mode,
5520 6086 : op0, op1);
5521 :
5522 9956 : case GT_EXPR:
5523 9956 : return simplify_gen_relational (unsignedp ? GTU : GT, mode, inner_mode,
5524 9956 : op0, op1);
5525 :
5526 3056 : case GE_EXPR:
5527 3056 : return simplify_gen_relational (unsignedp ? GEU : GE, mode, inner_mode,
5528 3056 : op0, op1);
5529 :
5530 30680 : case EQ_EXPR:
5531 30680 : return simplify_gen_relational (EQ, mode, inner_mode, op0, op1);
5532 :
5533 15150 : case NE_EXPR:
5534 15150 : return simplify_gen_relational (NE, mode, inner_mode, op0, op1);
5535 :
5536 1586 : case UNORDERED_EXPR:
5537 1586 : return simplify_gen_relational (UNORDERED, mode, inner_mode, op0, op1);
5538 :
5539 0 : case ORDERED_EXPR:
5540 0 : return simplify_gen_relational (ORDERED, mode, inner_mode, op0, op1);
5541 :
5542 0 : case UNLT_EXPR:
5543 0 : return simplify_gen_relational (UNLT, mode, inner_mode, op0, op1);
5544 :
5545 1576 : case UNLE_EXPR:
5546 1576 : return simplify_gen_relational (UNLE, mode, inner_mode, op0, op1);
5547 :
5548 0 : case UNGT_EXPR:
5549 0 : return simplify_gen_relational (UNGT, mode, inner_mode, op0, op1);
5550 :
5551 0 : case UNGE_EXPR:
5552 0 : return simplify_gen_relational (UNGE, mode, inner_mode, op0, op1);
5553 :
5554 0 : case UNEQ_EXPR:
5555 0 : return simplify_gen_relational (UNEQ, mode, inner_mode, op0, op1);
5556 :
5557 0 : case LTGT_EXPR:
5558 0 : return simplify_gen_relational (LTGT, mode, inner_mode, op0, op1);
5559 :
5560 8585 : case COND_EXPR:
5561 8585 : return gen_rtx_IF_THEN_ELSE (mode, op0, op1, op2);
5562 :
5563 10500 : case COMPLEX_EXPR:
5564 10500 : gcc_assert (COMPLEX_MODE_P (mode));
5565 10500 : if (GET_MODE (op0) == VOIDmode)
5566 0 : op0 = gen_rtx_CONST (GET_MODE_INNER (mode), op0);
5567 10500 : if (GET_MODE (op1) == VOIDmode)
5568 0 : op1 = gen_rtx_CONST (GET_MODE_INNER (mode), op1);
5569 10500 : return gen_rtx_CONCAT (mode, op0, op1);
5570 :
5571 0 : case CONJ_EXPR:
5572 0 : if (GET_CODE (op0) == CONCAT)
5573 0 : return gen_rtx_CONCAT (mode, XEXP (op0, 0),
5574 : simplify_gen_unary (NEG, GET_MODE_INNER (mode),
5575 : XEXP (op0, 1),
5576 : GET_MODE_INNER (mode)));
5577 : else
5578 : {
5579 0 : scalar_mode imode = GET_MODE_INNER (mode);
5580 0 : rtx re, im;
5581 :
5582 0 : if (MEM_P (op0))
5583 : {
5584 0 : re = adjust_address_nv (op0, imode, 0);
5585 0 : im = adjust_address_nv (op0, imode, GET_MODE_SIZE (imode));
5586 : }
5587 : else
5588 : {
5589 0 : scalar_int_mode ifmode;
5590 0 : scalar_int_mode ihmode;
5591 0 : rtx halfsize;
5592 0 : if (!int_mode_for_mode (mode).exists (&ifmode)
5593 0 : || !int_mode_for_mode (imode).exists (&ihmode))
5594 44050286 : return NULL;
5595 0 : halfsize = GEN_INT (GET_MODE_BITSIZE (ihmode));
5596 0 : re = op0;
5597 0 : if (mode != ifmode)
5598 0 : re = gen_rtx_SUBREG (ifmode, re, 0);
5599 0 : re = gen_rtx_ZERO_EXTRACT (ihmode, re, halfsize, const0_rtx);
5600 0 : if (imode != ihmode)
5601 0 : re = gen_rtx_SUBREG (imode, re, 0);
5602 0 : im = copy_rtx (op0);
5603 0 : if (mode != ifmode)
5604 0 : im = gen_rtx_SUBREG (ifmode, im, 0);
5605 0 : im = gen_rtx_ZERO_EXTRACT (ihmode, im, halfsize, halfsize);
5606 0 : if (imode != ihmode)
5607 0 : im = gen_rtx_SUBREG (imode, im, 0);
5608 : }
5609 0 : im = gen_rtx_NEG (imode, im);
5610 0 : return gen_rtx_CONCAT (mode, re, im);
5611 : }
5612 :
5613 7251554 : case ADDR_EXPR:
5614 7251554 : op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5615 7251554 : if (!op0 || !MEM_P (op0))
5616 : {
5617 1906441 : if ((TREE_CODE (TREE_OPERAND (exp, 0)) == VAR_DECL
5618 438634 : || TREE_CODE (TREE_OPERAND (exp, 0)) == PARM_DECL
5619 390445 : || TREE_CODE (TREE_OPERAND (exp, 0)) == RESULT_DECL)
5620 1954630 : && (!TREE_ADDRESSABLE (TREE_OPERAND (exp, 0))
5621 185209 : || target_for_debug_bind (TREE_OPERAND (exp, 0))))
5622 1341462 : return gen_rtx_DEBUG_IMPLICIT_PTR (mode, TREE_OPERAND (exp, 0));
5623 :
5624 564979 : if (handled_component_p (TREE_OPERAND (exp, 0)))
5625 : {
5626 362055 : poly_int64 bitoffset, bitsize, maxsize, byteoffset;
5627 362055 : bool reverse;
5628 362055 : tree decl
5629 362055 : = get_ref_base_and_extent (TREE_OPERAND (exp, 0), &bitoffset,
5630 : &bitsize, &maxsize, &reverse);
5631 362055 : if ((VAR_P (decl)
5632 : || TREE_CODE (decl) == PARM_DECL
5633 : || TREE_CODE (decl) == RESULT_DECL)
5634 329448 : && (!TREE_ADDRESSABLE (decl)
5635 27300 : || target_for_debug_bind (decl))
5636 77464 : && multiple_p (bitoffset, BITS_PER_UNIT, &byteoffset)
5637 302148 : && known_gt (bitsize, 0)
5638 284594 : && known_eq (bitsize, maxsize))
5639 : {
5640 284591 : rtx base = gen_rtx_DEBUG_IMPLICIT_PTR (mode, decl);
5641 284591 : return plus_constant (mode, base, byteoffset);
5642 : }
5643 : }
5644 :
5645 280388 : if (TREE_CODE (TREE_OPERAND (exp, 0)) == MEM_REF
5646 280388 : && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
5647 : == ADDR_EXPR)
5648 : {
5649 13333 : op0 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5650 : 0));
5651 13333 : if (op0 != NULL
5652 7919 : && (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5653 0 : || (GET_CODE (op0) == PLUS
5654 0 : && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR
5655 0 : && CONST_INT_P (XEXP (op0, 1)))))
5656 : {
5657 7919 : op1 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5658 : 1));
5659 7919 : poly_int64 offset;
5660 15838 : if (!op1 || !poly_int_rtx_p (op1, &offset))
5661 : return NULL;
5662 :
5663 7919 : return plus_constant (mode, op0, offset);
5664 : }
5665 : }
5666 :
5667 : return NULL;
5668 : }
5669 :
5670 5345113 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
5671 5345113 : addr_mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
5672 5345113 : op0 = convert_debug_memory_address (addr_mode, XEXP (op0, 0), as);
5673 :
5674 5345113 : return op0;
5675 :
5676 11819 : case VECTOR_CST:
5677 11819 : {
5678 11819 : unsigned HOST_WIDE_INT i, nelts;
5679 :
5680 11819 : if (!VECTOR_CST_NELTS (exp).is_constant (&nelts))
5681 : return NULL;
5682 :
5683 11819 : op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5684 :
5685 65439 : for (i = 0; i < nelts; ++i)
5686 : {
5687 53620 : op1 = expand_debug_expr (VECTOR_CST_ELT (exp, i));
5688 53620 : if (!op1)
5689 : return NULL;
5690 53620 : XVECEXP (op0, 0, i) = op1;
5691 : }
5692 :
5693 : return op0;
5694 : }
5695 :
5696 1084 : case CONSTRUCTOR:
5697 1084 : if (TREE_CLOBBER_P (exp))
5698 : return NULL;
5699 1084 : else if (TREE_CODE (TREE_TYPE (exp)) == VECTOR_TYPE)
5700 : {
5701 1084 : unsigned i;
5702 1084 : poly_uint64 elts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (exp));
5703 1084 : unsigned HOST_WIDE_INT nelts = constant_lower_bound (elts);
5704 1084 : tree val;
5705 :
5706 1084 : op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5707 :
5708 4825 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), i, val)
5709 : {
5710 3874 : op1 = expand_debug_expr (val);
5711 3874 : if (!op1)
5712 : return NULL;
5713 3741 : XVECEXP (op0, 0, i) = op1;
5714 : }
5715 :
5716 951 : if (i < nelts)
5717 : {
5718 116 : op1 = expand_debug_expr
5719 116 : (build_zero_cst (TREE_TYPE (TREE_TYPE (exp))));
5720 :
5721 116 : if (!op1)
5722 : return NULL;
5723 :
5724 377 : for (; i < nelts; i++)
5725 261 : XVECEXP (op0, 0, i) = op1;
5726 : }
5727 :
5728 : return op0;
5729 : }
5730 : else
5731 0 : goto flag_unsupported;
5732 :
5733 : case CALL_EXPR:
5734 : /* ??? Maybe handle some builtins? */
5735 : return NULL;
5736 :
5737 9887134 : case SSA_NAME:
5738 9887134 : {
5739 9887134 : gimple *g = get_gimple_for_ssa_name (exp);
5740 9887134 : if (g)
5741 : {
5742 1519539 : tree t = NULL_TREE;
5743 1519539 : if (deep_ter_debug_map)
5744 : {
5745 130276 : tree *slot = deep_ter_debug_map->get (exp);
5746 130276 : if (slot)
5747 19726 : t = *slot;
5748 : }
5749 19726 : if (t == NULL_TREE)
5750 : {
5751 1505444 : if (is_gimple_assign (g))
5752 1504876 : t = gimple_assign_rhs_to_tree (g);
5753 : else
5754 : /* expand_debug_expr doesn't handle CALL_EXPR right now. */
5755 : return NULL;
5756 : }
5757 1518971 : op0 = expand_debug_expr (t);
5758 1518971 : if (!op0)
5759 : return NULL;
5760 : }
5761 : else
5762 : {
5763 : /* If this is a reference to an incoming value of
5764 : parameter that is never used in the code or where the
5765 : incoming value is never used in the code, use
5766 : PARM_DECL's DECL_RTL if set. */
5767 8367595 : if (SSA_NAME_IS_DEFAULT_DEF (exp)
5768 1348567 : && SSA_NAME_VAR (exp)
5769 1348567 : && TREE_CODE (SSA_NAME_VAR (exp)) == PARM_DECL
5770 9651095 : && has_zero_uses (exp))
5771 : {
5772 18750 : op0 = expand_debug_parm_decl (SSA_NAME_VAR (exp));
5773 18750 : if (op0)
5774 18742 : goto adjust_mode;
5775 8 : op0 = expand_debug_expr (SSA_NAME_VAR (exp));
5776 8 : if (op0)
5777 8 : goto adjust_mode;
5778 : }
5779 :
5780 8348845 : int part = var_to_partition (SA.map, exp);
5781 :
5782 8348845 : if (part == NO_PARTITION)
5783 : return NULL;
5784 :
5785 8344238 : gcc_assert (part >= 0 && (unsigned)part < SA.map->num_partitions);
5786 :
5787 8344238 : op0 = copy_rtx (SA.partition_to_pseudo[part]);
5788 : }
5789 9856649 : goto adjust_mode;
5790 : }
5791 :
5792 : case ERROR_MARK:
5793 : return NULL;
5794 :
5795 : /* Vector stuff. For most of the codes we don't have rtl codes. */
5796 : case REALIGN_LOAD_EXPR:
5797 : case VEC_COND_EXPR:
5798 : case VEC_PACK_FIX_TRUNC_EXPR:
5799 : case VEC_PACK_FLOAT_EXPR:
5800 : case VEC_PACK_SAT_EXPR:
5801 : case VEC_PACK_TRUNC_EXPR:
5802 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
5803 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
5804 : case VEC_UNPACK_FLOAT_HI_EXPR:
5805 : case VEC_UNPACK_FLOAT_LO_EXPR:
5806 : case VEC_UNPACK_HI_EXPR:
5807 : case VEC_UNPACK_LO_EXPR:
5808 : case VEC_WIDEN_MULT_HI_EXPR:
5809 : case VEC_WIDEN_MULT_LO_EXPR:
5810 : case VEC_WIDEN_MULT_EVEN_EXPR:
5811 : case VEC_WIDEN_MULT_ODD_EXPR:
5812 : case VEC_WIDEN_LSHIFT_HI_EXPR:
5813 : case VEC_WIDEN_LSHIFT_LO_EXPR:
5814 : case VEC_PERM_EXPR:
5815 : case VEC_DUPLICATE_EXPR:
5816 : case VEC_SERIES_EXPR:
5817 : case SAD_EXPR:
5818 : return NULL;
5819 :
5820 : /* Misc codes. */
5821 : case ADDR_SPACE_CONVERT_EXPR:
5822 : case FIXED_CONVERT_EXPR:
5823 : case OBJ_TYPE_REF:
5824 : case WITH_SIZE_EXPR:
5825 : case BIT_INSERT_EXPR:
5826 : return NULL;
5827 :
5828 0 : case DOT_PROD_EXPR:
5829 0 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
5830 0 : && SCALAR_INT_MODE_P (mode))
5831 : {
5832 0 : op0
5833 0 : = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5834 : 0)))
5835 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5836 : inner_mode);
5837 0 : op1
5838 0 : = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5839 : 1)))
5840 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op1,
5841 : inner_mode);
5842 0 : op0 = simplify_gen_binary (MULT, mode, op0, op1);
5843 0 : return simplify_gen_binary (PLUS, mode, op0, op2);
5844 : }
5845 : return NULL;
5846 :
5847 6145 : case WIDEN_MULT_EXPR:
5848 6145 : case WIDEN_MULT_PLUS_EXPR:
5849 6145 : case WIDEN_MULT_MINUS_EXPR:
5850 6145 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
5851 6145 : && SCALAR_INT_MODE_P (mode))
5852 : {
5853 6145 : inner_mode = GET_MODE (op0);
5854 6145 : if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5855 5529 : op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5856 : else
5857 616 : op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5858 6145 : if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 1))))
5859 5529 : op1 = simplify_gen_unary (ZERO_EXTEND, mode, op1, inner_mode);
5860 : else
5861 616 : op1 = simplify_gen_unary (SIGN_EXTEND, mode, op1, inner_mode);
5862 6145 : op0 = simplify_gen_binary (MULT, mode, op0, op1);
5863 6145 : if (TREE_CODE (exp) == WIDEN_MULT_EXPR)
5864 : return op0;
5865 0 : else if (TREE_CODE (exp) == WIDEN_MULT_PLUS_EXPR)
5866 0 : return simplify_gen_binary (PLUS, mode, op0, op2);
5867 : else
5868 0 : return simplify_gen_binary (MINUS, mode, op2, op0);
5869 : }
5870 : return NULL;
5871 :
5872 : case MULT_HIGHPART_EXPR:
5873 : /* ??? Similar to the above. */
5874 : return NULL;
5875 :
5876 0 : case WIDEN_SUM_EXPR:
5877 0 : case WIDEN_LSHIFT_EXPR:
5878 0 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
5879 0 : && SCALAR_INT_MODE_P (mode))
5880 : {
5881 0 : op0
5882 0 : = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5883 : 0)))
5884 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5885 : inner_mode);
5886 0 : return simplify_gen_binary (TREE_CODE (exp) == WIDEN_LSHIFT_EXPR
5887 0 : ? ASHIFT : PLUS, mode, op0, op1);
5888 : }
5889 : return NULL;
5890 :
5891 0 : default:
5892 0 : flag_unsupported:
5893 0 : if (flag_checking)
5894 : {
5895 0 : debug_tree (exp);
5896 0 : gcc_unreachable ();
5897 : }
5898 : return NULL;
5899 : }
5900 : }
5901 :
5902 : /* Return an RTX equivalent to the source bind value of the tree expression
5903 : EXP. */
5904 :
5905 : static rtx
5906 180829 : expand_debug_source_expr (tree exp)
5907 : {
5908 336749 : rtx op0 = NULL_RTX;
5909 336749 : machine_mode mode = VOIDmode, inner_mode;
5910 :
5911 336749 : switch (TREE_CODE (exp))
5912 : {
5913 155920 : case VAR_DECL:
5914 155920 : if (DECL_ABSTRACT_ORIGIN (exp))
5915 155920 : return expand_debug_source_expr (DECL_ABSTRACT_ORIGIN (exp));
5916 : break;
5917 173989 : case PARM_DECL:
5918 173989 : {
5919 173989 : mode = DECL_MODE (exp);
5920 173989 : op0 = expand_debug_parm_decl (exp);
5921 173989 : if (op0)
5922 : break;
5923 : /* See if this isn't an argument that has been completely
5924 : optimized out. */
5925 173001 : if (!DECL_RTL_SET_P (exp)
5926 172985 : && !DECL_INCOMING_RTL (exp)
5927 345986 : && DECL_ABSTRACT_ORIGIN (current_function_decl))
5928 : {
5929 58901 : tree aexp = DECL_ORIGIN (exp);
5930 58901 : if (DECL_CONTEXT (aexp)
5931 58901 : == DECL_ABSTRACT_ORIGIN (current_function_decl))
5932 : {
5933 20982 : vec<tree, va_gc> **debug_args;
5934 20982 : unsigned int ix;
5935 20982 : tree ddecl;
5936 20982 : debug_args = decl_debug_args_lookup (current_function_decl);
5937 20982 : if (debug_args != NULL)
5938 : {
5939 25902 : for (ix = 0; vec_safe_iterate (*debug_args, ix, &ddecl);
5940 4924 : ix += 2)
5941 25894 : if (ddecl == aexp)
5942 20970 : return gen_rtx_DEBUG_PARAMETER_REF (mode, aexp);
5943 : }
5944 : }
5945 : }
5946 : break;
5947 : }
5948 : default:
5949 : break;
5950 : }
5951 :
5952 : if (op0 == NULL_RTX)
5953 : return NULL_RTX;
5954 :
5955 988 : inner_mode = GET_MODE (op0);
5956 988 : if (mode == inner_mode)
5957 : return op0;
5958 :
5959 21 : if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5960 : {
5961 7 : if (GET_MODE_UNIT_BITSIZE (mode)
5962 14 : == GET_MODE_UNIT_BITSIZE (inner_mode))
5963 0 : op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
5964 7 : else if (GET_MODE_UNIT_BITSIZE (mode)
5965 14 : < GET_MODE_UNIT_BITSIZE (inner_mode))
5966 7 : op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
5967 : else
5968 0 : op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
5969 : }
5970 14 : else if (FLOAT_MODE_P (mode))
5971 0 : gcc_unreachable ();
5972 14 : else if (FLOAT_MODE_P (inner_mode))
5973 : {
5974 0 : if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5975 0 : op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
5976 : else
5977 0 : op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
5978 : }
5979 28 : else if (GET_MODE_UNIT_PRECISION (mode)
5980 14 : == GET_MODE_UNIT_PRECISION (inner_mode))
5981 0 : op0 = lowpart_subreg (mode, op0, inner_mode);
5982 28 : else if (GET_MODE_UNIT_PRECISION (mode)
5983 14 : < GET_MODE_UNIT_PRECISION (inner_mode))
5984 14 : op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
5985 0 : else if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5986 0 : op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5987 : else
5988 0 : op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5989 :
5990 : return op0;
5991 : }
5992 :
5993 : /* Ensure INSN_VAR_LOCATION_LOC (insn) doesn't have unbound complexity.
5994 : Allow 4 levels of rtl nesting for most rtl codes, and if we see anything
5995 : deeper than that, create DEBUG_EXPRs and emit DEBUG_INSNs before INSN. */
5996 :
5997 : static void
5998 54401805 : avoid_complex_debug_insns (rtx_insn *insn, rtx *exp_p, int depth)
5999 : {
6000 54401805 : rtx exp = *exp_p;
6001 :
6002 54401805 : if (exp == NULL_RTX)
6003 : return;
6004 :
6005 54401805 : if ((OBJECT_P (exp) && !MEM_P (exp)) || GET_CODE (exp) == CLOBBER)
6006 : return;
6007 :
6008 7078414 : if (depth == 4)
6009 : {
6010 : /* Create DEBUG_EXPR (and DEBUG_EXPR_DECL). */
6011 26693 : rtx dval = make_debug_expr_from_rtl (exp);
6012 :
6013 : /* Emit a debug bind insn before INSN. */
6014 26693 : rtx bind = gen_rtx_VAR_LOCATION (GET_MODE (exp),
6015 : DEBUG_EXPR_TREE_DECL (dval), exp,
6016 : VAR_INIT_STATUS_INITIALIZED);
6017 :
6018 26693 : emit_debug_insn_before (bind, insn);
6019 26693 : *exp_p = dval;
6020 26693 : return;
6021 : }
6022 :
6023 7051721 : const char *format_ptr = GET_RTX_FORMAT (GET_CODE (exp));
6024 7051721 : int i, j;
6025 21037101 : for (i = 0; i < GET_RTX_LENGTH (GET_CODE (exp)); i++)
6026 13985380 : switch (*format_ptr++)
6027 : {
6028 12851158 : case 'e':
6029 12851158 : avoid_complex_debug_insns (insn, &XEXP (exp, i), depth + 1);
6030 12851158 : break;
6031 :
6032 : case 'E':
6033 : case 'V':
6034 0 : for (j = 0; j < XVECLEN (exp, i); j++)
6035 0 : avoid_complex_debug_insns (insn, &XVECEXP (exp, i, j), depth + 1);
6036 : break;
6037 :
6038 : default:
6039 : break;
6040 : }
6041 : }
6042 :
6043 : /* Expand the _LOCs in debug insns. We run this after expanding all
6044 : regular insns, so that any variables referenced in the function
6045 : will have their DECL_RTLs set. */
6046 :
6047 : static void
6048 509911 : expand_debug_locations (void)
6049 : {
6050 509911 : rtx_insn *insn;
6051 509911 : rtx_insn *last = get_last_insn ();
6052 509911 : int save_strict_alias = flag_strict_aliasing;
6053 :
6054 : /* New alias sets while setting up memory attributes cause
6055 : -fcompare-debug failures, even though it doesn't bring about any
6056 : codegen changes. */
6057 509911 : flag_strict_aliasing = 0;
6058 :
6059 110243935 : for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
6060 109734024 : if (DEBUG_BIND_INSN_P (insn))
6061 : {
6062 41523954 : tree value = (tree)INSN_VAR_LOCATION_LOC (insn);
6063 41523954 : rtx val;
6064 41523954 : rtx_insn *prev_insn, *insn2;
6065 41523954 : machine_mode mode;
6066 :
6067 41523954 : if (value == NULL_TREE)
6068 : val = NULL_RTX;
6069 : else
6070 : {
6071 23217166 : if (INSN_VAR_LOCATION_STATUS (insn)
6072 : == VAR_INIT_STATUS_UNINITIALIZED)
6073 180829 : val = expand_debug_source_expr (value);
6074 : /* The avoid_deep_ter_for_debug function inserts
6075 : debug bind stmts after SSA_NAME definition, with the
6076 : SSA_NAME as the whole bind location. Disable temporarily
6077 : expansion of that SSA_NAME into the DEBUG_EXPR_DECL
6078 : being defined in this DEBUG_INSN. */
6079 23036337 : else if (deep_ter_debug_map && TREE_CODE (value) == SSA_NAME)
6080 : {
6081 187645 : tree *slot = deep_ter_debug_map->get (value);
6082 187645 : if (slot)
6083 : {
6084 6387 : if (*slot == INSN_VAR_LOCATION_DECL (insn))
6085 5631 : *slot = NULL_TREE;
6086 : else
6087 : slot = NULL;
6088 : }
6089 187645 : val = expand_debug_expr (value);
6090 187645 : if (slot)
6091 5631 : *slot = INSN_VAR_LOCATION_DECL (insn);
6092 : }
6093 : else
6094 22848692 : val = expand_debug_expr (value);
6095 23217166 : gcc_assert (last == get_last_insn ());
6096 : }
6097 :
6098 23217166 : if (!val)
6099 18800525 : val = gen_rtx_UNKNOWN_VAR_LOC ();
6100 : else
6101 : {
6102 22723429 : mode = GET_MODE (INSN_VAR_LOCATION (insn));
6103 :
6104 22723429 : gcc_assert (mode == GET_MODE (val)
6105 : || (GET_MODE (val) == VOIDmode
6106 : && (CONST_SCALAR_INT_P (val)
6107 : || GET_CODE (val) == CONST_FIXED
6108 : || GET_CODE (val) == LABEL_REF)));
6109 : }
6110 :
6111 41523954 : INSN_VAR_LOCATION_LOC (insn) = val;
6112 41523954 : prev_insn = PREV_INSN (insn);
6113 124598555 : for (insn2 = insn; insn2 != prev_insn; insn2 = PREV_INSN (insn2))
6114 41550647 : avoid_complex_debug_insns (insn2, &INSN_VAR_LOCATION_LOC (insn2), 0);
6115 : }
6116 :
6117 509911 : flag_strict_aliasing = save_strict_alias;
6118 509911 : }
6119 :
6120 : /* Performs swapping operands of commutative operations to expand
6121 : the expensive one first. */
6122 :
6123 : static void
6124 9522873 : reorder_operands (basic_block bb)
6125 : {
6126 9522873 : unsigned int *lattice; /* Hold cost of each statement. */
6127 9522873 : unsigned int i = 0, n = 0;
6128 9522873 : gimple_stmt_iterator gsi;
6129 9522873 : gimple_seq stmts;
6130 9522873 : gimple *stmt;
6131 9522873 : bool swap;
6132 9522873 : tree op0, op1;
6133 9522873 : ssa_op_iter iter;
6134 9522873 : use_operand_p use_p;
6135 9522873 : gimple *def0, *def1;
6136 :
6137 : /* Compute cost of each statement using estimate_num_insns. */
6138 9522873 : stmts = bb_seq (bb);
6139 98296226 : for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
6140 : {
6141 88773353 : stmt = gsi_stmt (gsi);
6142 88773353 : if (!is_gimple_debug (stmt))
6143 34859644 : gimple_set_uid (stmt, n++);
6144 : }
6145 9522873 : lattice = XNEWVEC (unsigned int, n);
6146 107819099 : for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
6147 : {
6148 88773353 : unsigned cost;
6149 88773353 : stmt = gsi_stmt (gsi);
6150 88773353 : if (is_gimple_debug (stmt))
6151 53913709 : continue;
6152 34859644 : cost = estimate_num_insns (stmt, &eni_size_weights);
6153 34859644 : lattice[i] = cost;
6154 66399576 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
6155 : {
6156 31539932 : tree use = USE_FROM_PTR (use_p);
6157 31539932 : gimple *def_stmt;
6158 31539932 : if (TREE_CODE (use) != SSA_NAME)
6159 0 : continue;
6160 31539932 : def_stmt = get_gimple_for_ssa_name (use);
6161 31539932 : if (!def_stmt)
6162 22436283 : continue;
6163 9103649 : lattice[i] += lattice[gimple_uid (def_stmt)];
6164 : }
6165 34859644 : i++;
6166 34859644 : if (!is_gimple_assign (stmt)
6167 34859644 : || !commutative_tree_code (gimple_assign_rhs_code (stmt)))
6168 30577302 : continue;
6169 4282342 : op0 = gimple_op (stmt, 1);
6170 4282342 : op1 = gimple_op (stmt, 2);
6171 4282342 : if (TREE_CODE (op0) != SSA_NAME
6172 4282090 : || TREE_CODE (op1) != SSA_NAME)
6173 2706077 : continue;
6174 : /* Swap operands if the second one is more expensive. */
6175 1576265 : def0 = get_gimple_for_ssa_name (op0);
6176 1576265 : def1 = get_gimple_for_ssa_name (op1);
6177 1576265 : if (!def1)
6178 844239 : continue;
6179 732026 : swap = false;
6180 732026 : if (!def0 || lattice[gimple_uid (def1)] > lattice[gimple_uid (def0)])
6181 350383 : swap = true;
6182 350383 : if (swap)
6183 : {
6184 350383 : if (dump_file && (dump_flags & TDF_DETAILS))
6185 : {
6186 2 : fprintf (dump_file, "Swap operands in stmt:\n");
6187 2 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
6188 2 : fprintf (dump_file, "Cost left opnd=%d, right opnd=%d\n",
6189 1 : def0 ? lattice[gimple_uid (def0)] : 0,
6190 2 : lattice[gimple_uid (def1)]);
6191 : }
6192 350383 : swap_ssa_operands (stmt, gimple_assign_rhs1_ptr (stmt),
6193 : gimple_assign_rhs2_ptr (stmt));
6194 : }
6195 : }
6196 9522873 : XDELETE (lattice);
6197 9522873 : }
6198 :
6199 : /* Expand basic block BB from GIMPLE trees to RTL. */
6200 :
6201 : static basic_block
6202 12904155 : expand_gimple_basic_block (basic_block bb, rtx_insn *asan_epilog_seq)
6203 : {
6204 12904155 : gimple_stmt_iterator gsi;
6205 12904155 : gimple *stmt = NULL;
6206 12904155 : rtx_note *note = NULL;
6207 12904155 : rtx_insn *last;
6208 12904155 : edge e;
6209 12904155 : edge_iterator ei;
6210 12904155 : bool nondebug_stmt_seen = false;
6211 :
6212 12904155 : if (dump_file)
6213 787 : fprintf (dump_file, "\n;; Generating RTL for gimple basic block %d\n",
6214 : bb->index);
6215 :
6216 : /* Note that since we are now transitioning from GIMPLE to RTL, we
6217 : cannot use the gsi_*_bb() routines because they expect the basic
6218 : block to be in GIMPLE, instead of RTL. Therefore, we need to
6219 : access the BB sequence directly. */
6220 12904155 : if (optimize)
6221 9522873 : reorder_operands (bb);
6222 12904155 : rtl_profile_for_bb (bb);
6223 :
6224 : /* Remove the RETURN_EXPR if we may fall though to the exit
6225 : instead. */
6226 12904155 : gsi = gsi_last_bb (bb);
6227 12904155 : if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) == GIMPLE_RETURN)
6228 : {
6229 1483182 : greturn *ret_stmt = as_a <greturn *> (gsi_stmt (gsi));
6230 :
6231 1483182 : gcc_assert (single_succ_p (bb));
6232 1483182 : gcc_assert (single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun));
6233 :
6234 1483182 : if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
6235 1483182 : && !gimple_return_retval (ret_stmt))
6236 : {
6237 624972 : gsi_remove (&gsi, false);
6238 624972 : single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
6239 : }
6240 : }
6241 :
6242 12904155 : gsi = gsi_start_bb (bb);
6243 12904155 : if (!gsi_end_p (gsi))
6244 : {
6245 12357486 : stmt = gsi_stmt (gsi);
6246 12357486 : if (gimple_code (stmt) != GIMPLE_LABEL)
6247 12261284 : stmt = NULL;
6248 : }
6249 :
6250 12904155 : rtx_code_label **elt = lab_rtx_for_bb->get (bb);
6251 12904155 : if ((unsigned) bb->index >= head_end_for_bb.length ())
6252 10912073 : head_end_for_bb.safe_grow_cleared (bb->index + 1);
6253 :
6254 12904155 : if (stmt || elt)
6255 : {
6256 5354797 : gcc_checking_assert (!note);
6257 5354797 : last = get_last_insn ();
6258 :
6259 5354797 : if (stmt)
6260 : {
6261 642871 : expand_gimple_stmt (stmt);
6262 642871 : gsi_next (&gsi);
6263 : }
6264 :
6265 5354797 : if (elt)
6266 4711926 : emit_label (*elt);
6267 :
6268 5354797 : head_end_for_bb[bb->index].first = NEXT_INSN (last);
6269 5354797 : if (NOTE_P (head_end_for_bb[bb->index].first))
6270 0 : head_end_for_bb[bb->index].first
6271 0 : = NEXT_INSN (head_end_for_bb[bb->index].first);
6272 5354797 : gcc_assert (LABEL_P (head_end_for_bb[bb->index].first));
6273 10709594 : note = emit_note_after (NOTE_INSN_BASIC_BLOCK,
6274 5354797 : head_end_for_bb[bb->index].first);
6275 :
6276 5354797 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
6277 : }
6278 : else
6279 7549358 : head_end_for_bb[bb->index].first = note = emit_note (NOTE_INSN_BASIC_BLOCK);
6280 :
6281 12904155 : if (note)
6282 12904155 : NOTE_BASIC_BLOCK (note) = bb;
6283 :
6284 67398358 : for (; !gsi_end_p (gsi); gsi_next (&gsi))
6285 : {
6286 55027570 : basic_block new_bb;
6287 :
6288 55027570 : stmt = gsi_stmt (gsi);
6289 55027570 : if (!is_gimple_debug (stmt))
6290 46582567 : nondebug_stmt_seen = true;
6291 :
6292 : /* If this statement is a non-debug one, and we generate debug
6293 : insns, then this one might be the last real use of a TERed
6294 : SSA_NAME, but where there are still some debug uses further
6295 : down. Expanding the current SSA name in such further debug
6296 : uses by their RHS might lead to wrong debug info, as coalescing
6297 : might make the operands of such RHS be placed into the same
6298 : pseudo as something else. Like so:
6299 : a_1 = a_0 + 1; // Assume a_1 is TERed and a_0 is dead
6300 : use(a_1);
6301 : a_2 = ...
6302 : #DEBUG ... => a_1
6303 : As a_0 and a_2 don't overlap in lifetime, assume they are coalesced.
6304 : If we now would expand a_1 by it's RHS (a_0 + 1) in the debug use,
6305 : the write to a_2 would actually have clobbered the place which
6306 : formerly held a_0.
6307 :
6308 : So, instead of that, we recognize the situation, and generate
6309 : debug temporaries at the last real use of TERed SSA names:
6310 : a_1 = a_0 + 1;
6311 : #DEBUG #D1 => a_1
6312 : use(a_1);
6313 : a_2 = ...
6314 : #DEBUG ... => #D1
6315 : */
6316 55027570 : if (MAY_HAVE_DEBUG_BIND_INSNS
6317 29892062 : && SA.values
6318 83693277 : && !is_gimple_debug (stmt))
6319 : {
6320 20609745 : ssa_op_iter iter;
6321 20609745 : tree op;
6322 20609745 : gimple *def;
6323 :
6324 20609745 : location_t sloc = curr_insn_location ();
6325 :
6326 : /* Look for SSA names that have their last use here (TERed
6327 : names always have only one real use). */
6328 39576675 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
6329 18966930 : if ((def = get_gimple_for_ssa_name (op))
6330 18966930 : && is_gimple_assign (def))
6331 : {
6332 5146177 : imm_use_iterator imm_iter;
6333 5146177 : use_operand_p use_p;
6334 5146177 : bool have_debug_uses = false;
6335 :
6336 10268317 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, op)
6337 : {
6338 5169914 : if (gimple_debug_bind_p (USE_STMT (use_p)))
6339 : {
6340 : have_debug_uses = true;
6341 : break;
6342 : }
6343 5146177 : }
6344 :
6345 5146177 : if (have_debug_uses)
6346 : {
6347 : /* OP is a TERed SSA name, with DEF its defining
6348 : statement, and where OP is used in further debug
6349 : instructions. Generate a debug temporary, and
6350 : replace all uses of OP in debug insns with that
6351 : temporary. */
6352 47774 : gimple *debugstmt;
6353 47774 : tree value = gimple_assign_rhs_to_tree (def);
6354 47774 : tree vexpr = build_debug_expr_decl (TREE_TYPE (value));
6355 47774 : rtx val;
6356 47774 : machine_mode mode;
6357 :
6358 47774 : set_curr_insn_location (gimple_location (def));
6359 :
6360 47774 : if (DECL_P (value))
6361 365 : mode = DECL_MODE (value);
6362 : else
6363 47409 : mode = TYPE_MODE (TREE_TYPE (value));
6364 : /* FIXME: Is setting the mode really necessary? */
6365 47774 : SET_DECL_MODE (vexpr, mode);
6366 :
6367 47774 : val = gen_rtx_VAR_LOCATION
6368 47774 : (mode, vexpr, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
6369 :
6370 47774 : emit_debug_insn (val);
6371 :
6372 172567 : FOR_EACH_IMM_USE_STMT (debugstmt, imm_iter, op)
6373 : {
6374 124793 : if (!gimple_debug_bind_p (debugstmt))
6375 47774 : continue;
6376 :
6377 155792 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
6378 77896 : SET_USE (use_p, vexpr);
6379 :
6380 77019 : update_stmt (debugstmt);
6381 47774 : }
6382 : }
6383 : }
6384 20609745 : set_curr_insn_location (sloc);
6385 : }
6386 :
6387 55027570 : currently_expanding_gimple_stmt = stmt;
6388 :
6389 : /* Expand this statement, then evaluate the resulting RTL and
6390 : fixup the CFG accordingly. */
6391 55027570 : if (gimple_code (stmt) == GIMPLE_COND)
6392 : {
6393 5776900 : new_bb = expand_gimple_cond (bb, as_a <gcond *> (stmt));
6394 5776900 : if (new_bb)
6395 : {
6396 402360 : currently_expanding_gimple_stmt = NULL;
6397 402360 : return new_bb;
6398 : }
6399 : }
6400 49250670 : else if (is_gimple_debug (stmt))
6401 : {
6402 8445003 : location_t sloc = curr_insn_location ();
6403 8445003 : gimple_stmt_iterator nsi = gsi;
6404 :
6405 53786193 : for (;;)
6406 : {
6407 53786193 : tree var;
6408 53786193 : tree value = NULL_TREE;
6409 53786193 : rtx val = NULL_RTX;
6410 53786193 : machine_mode mode;
6411 :
6412 53786193 : if (!gimple_debug_nonbind_marker_p (stmt))
6413 : {
6414 41854826 : if (gimple_debug_bind_p (stmt))
6415 : {
6416 41673997 : var = gimple_debug_bind_get_var (stmt);
6417 :
6418 41673997 : if (TREE_CODE (var) != DEBUG_EXPR_DECL
6419 38624147 : && TREE_CODE (var) != LABEL_DECL
6420 80288161 : && !target_for_debug_bind (var))
6421 378646 : goto delink_debug_stmt;
6422 :
6423 41295351 : if (DECL_P (var) && !VECTOR_TYPE_P (TREE_TYPE (var)))
6424 41209326 : mode = DECL_MODE (var);
6425 : else
6426 86025 : mode = TYPE_MODE (TREE_TYPE (var));
6427 :
6428 41295351 : if (gimple_debug_bind_has_value_p (stmt))
6429 22988563 : value = gimple_debug_bind_get_value (stmt);
6430 :
6431 41295351 : val = gen_rtx_VAR_LOCATION
6432 41295351 : (mode, var, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
6433 : }
6434 180829 : else if (gimple_debug_source_bind_p (stmt))
6435 : {
6436 180829 : var = gimple_debug_source_bind_get_var (stmt);
6437 :
6438 180829 : value = gimple_debug_source_bind_get_value (stmt);
6439 :
6440 180829 : if (!VECTOR_TYPE_P (TREE_TYPE (var)))
6441 180829 : mode = DECL_MODE (var);
6442 : else
6443 0 : mode = TYPE_MODE (TREE_TYPE (var));
6444 :
6445 180829 : val = gen_rtx_VAR_LOCATION (mode, var, (rtx)value,
6446 : VAR_INIT_STATUS_UNINITIALIZED);
6447 : }
6448 : else
6449 0 : gcc_unreachable ();
6450 : }
6451 : /* If this function was first compiled with markers
6452 : enabled, but they're now disable (e.g. LTO), drop
6453 : them on the floor. */
6454 11931367 : else if (gimple_debug_nonbind_marker_p (stmt)
6455 11931367 : && !MAY_HAVE_DEBUG_MARKER_INSNS)
6456 0 : goto delink_debug_stmt;
6457 11931367 : else if (gimple_debug_begin_stmt_p (stmt))
6458 4123920 : val = GEN_RTX_DEBUG_MARKER_BEGIN_STMT_PAT ();
6459 7807447 : else if (gimple_debug_inline_entry_p (stmt))
6460 7807447 : val = GEN_RTX_DEBUG_MARKER_INLINE_ENTRY_PAT ();
6461 : else
6462 : gcc_unreachable ();
6463 :
6464 53407547 : last = get_last_insn ();
6465 :
6466 53407547 : set_curr_insn_location (gimple_location (stmt));
6467 :
6468 53407547 : emit_debug_insn (val);
6469 :
6470 53407547 : if (dump_file && (dump_flags & TDF_DETAILS))
6471 : {
6472 : /* We can't dump the insn with a TREE where an RTX
6473 : is expected. */
6474 1 : if (GET_CODE (val) == VAR_LOCATION)
6475 : {
6476 0 : gcc_checking_assert (PAT_VAR_LOCATION_LOC (val) == (rtx)value);
6477 0 : PAT_VAR_LOCATION_LOC (val) = const0_rtx;
6478 : }
6479 1 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
6480 1 : if (GET_CODE (val) == VAR_LOCATION)
6481 0 : PAT_VAR_LOCATION_LOC (val) = (rtx)value;
6482 : }
6483 :
6484 53786193 : delink_debug_stmt:
6485 : /* In order not to generate too many debug temporaries,
6486 : we delink all uses of debug statements we already expanded.
6487 : Therefore debug statements between definition and real
6488 : use of TERed SSA names will continue to use the SSA name,
6489 : and not be replaced with debug temps. */
6490 53786193 : delink_stmt_imm_use (stmt);
6491 :
6492 53786193 : gsi = nsi;
6493 53786193 : gsi_next (&nsi);
6494 53786193 : if (gsi_end_p (nsi))
6495 : break;
6496 53099890 : stmt = gsi_stmt (nsi);
6497 53099890 : if (!is_gimple_debug (stmt))
6498 : break;
6499 : }
6500 :
6501 8445003 : set_curr_insn_location (sloc);
6502 : }
6503 : else
6504 : {
6505 40805667 : gcall *call_stmt = dyn_cast <gcall *> (stmt);
6506 40805667 : if (call_stmt
6507 40805667 : && asan_epilog_seq
6508 23414 : && gimple_call_tail_p (call_stmt)
6509 40805813 : && !gimple_call_must_tail_p (call_stmt))
6510 29 : gimple_call_set_tail (call_stmt, false);
6511 :
6512 40805667 : if (call_stmt && gimple_call_tail_p (call_stmt))
6513 : {
6514 180694 : bool can_fallthru;
6515 180694 : new_bb = expand_gimple_tailcall (bb, call_stmt, &can_fallthru,
6516 : asan_epilog_seq);
6517 180694 : if (new_bb)
6518 : {
6519 131005 : if (can_fallthru)
6520 0 : bb = new_bb;
6521 : else
6522 : {
6523 131005 : currently_expanding_gimple_stmt = NULL;
6524 131005 : return new_bb;
6525 : }
6526 : }
6527 : }
6528 : else
6529 : {
6530 40624973 : def_operand_p def_p;
6531 40624973 : def_p = SINGLE_SSA_DEF_OPERAND (stmt, SSA_OP_DEF);
6532 :
6533 40624973 : if (def_p != NULL)
6534 : {
6535 : /* Ignore this stmt if it is in the list of
6536 : replaceable expressions. */
6537 34399283 : if (SA.values
6538 42692522 : && bitmap_bit_p (SA.values,
6539 17396767 : SSA_NAME_VERSION (DEF_FROM_PTR (def_p))))
6540 9103528 : continue;
6541 : }
6542 31521445 : last = expand_gimple_stmt (stmt);
6543 31521443 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
6544 : }
6545 : }
6546 : }
6547 :
6548 12370788 : currently_expanding_gimple_stmt = NULL;
6549 :
6550 : /* Expand implicit goto and convert goto_locus. */
6551 30036178 : FOR_EACH_EDGE (e, ei, bb->succs)
6552 : {
6553 17665390 : if (e->goto_locus != UNKNOWN_LOCATION || !nondebug_stmt_seen)
6554 3510840 : set_curr_insn_location (e->goto_locus);
6555 17665390 : if ((e->flags & EDGE_FALLTHRU) && e->dest != bb->next_bb)
6556 : {
6557 1664096 : emit_jump (label_rtx_for_bb (e->dest));
6558 1664096 : e->flags &= ~EDGE_FALLTHRU;
6559 : }
6560 : }
6561 :
6562 : /* Expanded RTL can create a jump in the last instruction of block.
6563 : This later might be assumed to be a jump to successor and break edge insertion.
6564 : We need to insert dummy move to prevent this. PR41440. */
6565 12370788 : if (single_succ_p (bb)
6566 5588155 : && (single_succ_edge (bb)->flags & EDGE_FALLTHRU)
6567 3162400 : && (last = get_last_insn ())
6568 15347884 : && (JUMP_P (last)
6569 3162352 : || (DEBUG_INSN_P (last)
6570 385729 : && JUMP_P (prev_nondebug_insn (last)))))
6571 : {
6572 59 : rtx dummy = gen_reg_rtx (SImode);
6573 59 : emit_insn_after_noloc (gen_move_insn (dummy, dummy), last, NULL);
6574 : }
6575 :
6576 : /* A __builtin_unreachable () will insert a barrier that should end
6577 : the basic block. In gimple, any code after it will have already
6578 : deleted, even without optimization. If we emit additional code
6579 : here, as we would to adjust the stack after a call, it should be
6580 : eventually deleted, but it confuses internal checkers (PR118006)
6581 : and optimizers before it does, because we don't expect to find
6582 : barriers inside basic blocks. */
6583 12370788 : if (!BARRIER_P (get_last_insn ()))
6584 9022363 : do_pending_stack_adjust ();
6585 : else
6586 3348425 : discard_pending_stack_adjust ();
6587 :
6588 : /* Find the block tail. The last insn in the block is the insn
6589 : before a barrier and/or table jump insn. */
6590 12370788 : last = get_last_insn ();
6591 12370788 : if (BARRIER_P (last))
6592 3348425 : last = PREV_INSN (last);
6593 12370788 : if (JUMP_TABLE_DATA_P (last))
6594 7077 : last = PREV_INSN (PREV_INSN (last));
6595 12370788 : if (BARRIER_P (last))
6596 5900 : last = PREV_INSN (last);
6597 12370788 : head_end_for_bb[bb->index].second = last;
6598 :
6599 24741576 : update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
6600 12370788 : head_end_for_bb[bb->index].second, bb);
6601 :
6602 12370788 : return bb;
6603 : }
6604 :
6605 :
6606 : /* Create a basic block for initialization code. */
6607 :
6608 : static basic_block
6609 1515899 : construct_init_block (void)
6610 : {
6611 1515899 : basic_block init_block;
6612 1515899 : edge e = NULL;
6613 1515899 : int flags;
6614 :
6615 : /* Multiple entry points not supported yet. */
6616 1515899 : gcc_assert (EDGE_COUNT (ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs) == 1);
6617 1515899 : init_rtl_bb_info (ENTRY_BLOCK_PTR_FOR_FN (cfun));
6618 1515899 : init_rtl_bb_info (EXIT_BLOCK_PTR_FOR_FN (cfun));
6619 1515899 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6620 1515899 : EXIT_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6621 :
6622 1515899 : e = EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun), 0);
6623 :
6624 : /* When entry edge points to first basic block, we don't need jump,
6625 : otherwise we have to jump into proper target. */
6626 1515899 : if (e && e->dest != ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb)
6627 : {
6628 0 : tree label = gimple_block_label (e->dest);
6629 :
6630 0 : emit_jump (jump_target_rtx (label));
6631 0 : flags = 0;
6632 0 : }
6633 : else
6634 : flags = EDGE_FALLTHRU;
6635 :
6636 1515899 : init_block = create_basic_block (NEXT_INSN (get_insns ()),
6637 : get_last_insn (),
6638 1515899 : ENTRY_BLOCK_PTR_FOR_FN (cfun));
6639 1515899 : init_block->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6640 1515899 : add_bb_to_loop (init_block, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6641 1515899 : if (e)
6642 1515899 : expand_split_edge (e, init_block, flags);
6643 : else
6644 0 : make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6645 : EDGE_FALLTHRU);
6646 :
6647 1515899 : update_bb_for_insn (init_block);
6648 1515899 : return init_block;
6649 : }
6650 :
6651 : /* For each lexical block, set BLOCK_NUMBER to the depth at which it is
6652 : found in the block tree. */
6653 :
6654 : static void
6655 18881545 : set_block_levels (tree block, int level)
6656 : {
6657 36247193 : while (block)
6658 : {
6659 17365648 : BLOCK_NUMBER (block) = level;
6660 17365648 : set_block_levels (BLOCK_SUBBLOCKS (block), level + 1);
6661 17365648 : block = BLOCK_CHAIN (block);
6662 : }
6663 18881545 : }
6664 :
6665 : /* Create a block containing landing pads and similar stuff. */
6666 :
6667 : static void
6668 1515897 : construct_exit_block (void)
6669 : {
6670 1515897 : rtx_insn *head = get_last_insn ();
6671 1515897 : rtx_insn *end;
6672 1515897 : basic_block exit_block;
6673 1515897 : edge e, e2;
6674 1515897 : unsigned ix;
6675 1515897 : edge_iterator ei;
6676 1515897 : basic_block prev_bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
6677 1515897 : rtx_insn *orig_end = BB_END (prev_bb);
6678 :
6679 1515897 : rtl_profile_for_bb (EXIT_BLOCK_PTR_FOR_FN (cfun));
6680 :
6681 : /* Make sure the locus is set to the end of the function, so that
6682 : epilogue line numbers and warnings are set properly. */
6683 1515897 : if (LOCATION_LOCUS (cfun->function_end_locus) != UNKNOWN_LOCATION)
6684 1196325 : input_location = cfun->function_end_locus;
6685 :
6686 : /* Generate rtl for function exit. */
6687 1515897 : expand_function_end ();
6688 :
6689 1515897 : end = get_last_insn ();
6690 1515897 : if (head == end)
6691 0 : return;
6692 : /* While emitting the function end we could move end of the last basic
6693 : block. */
6694 1515897 : BB_END (prev_bb) = orig_end;
6695 1515897 : while (NEXT_INSN (head) && NOTE_P (NEXT_INSN (head)))
6696 : head = NEXT_INSN (head);
6697 : /* But make sure exit_block starts with RETURN_LABEL, otherwise the
6698 : bb count counting will be confused. Any instructions before that
6699 : label are emitted for the case where PREV_BB falls through into the
6700 : exit block, so append those instructions to prev_bb in that case. */
6701 1515897 : if (NEXT_INSN (head) != return_label)
6702 : {
6703 14955 : while (NEXT_INSN (head) != return_label)
6704 : {
6705 9929 : if (!NOTE_P (NEXT_INSN (head)))
6706 9929 : BB_END (prev_bb) = NEXT_INSN (head);
6707 9929 : head = NEXT_INSN (head);
6708 : }
6709 : }
6710 1515897 : exit_block = create_basic_block (NEXT_INSN (head), end, prev_bb);
6711 1515897 : exit_block->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
6712 1515897 : add_bb_to_loop (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6713 :
6714 1515897 : ix = 0;
6715 4576439 : while (ix < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
6716 : {
6717 1544645 : e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), ix);
6718 1544645 : if (!(e->flags & EDGE_ABNORMAL))
6719 1413640 : redirect_edge_succ (e, exit_block);
6720 : else
6721 131005 : ix++;
6722 : }
6723 :
6724 1515897 : e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6725 : EDGE_FALLTHRU);
6726 3162799 : FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
6727 1646902 : if (e2 != e)
6728 : {
6729 131005 : exit_block->count -= e2->count ();
6730 : }
6731 1515897 : update_bb_for_insn (exit_block);
6732 : }
6733 :
6734 : /* Helper function for discover_nonconstant_array_refs.
6735 : Look for ARRAY_REF nodes with non-constant indexes and mark them
6736 : addressable. */
6737 :
6738 : static tree
6739 167602424 : discover_nonconstant_array_refs_r (tree * tp, int *walk_subtrees,
6740 : void *data)
6741 : {
6742 167602424 : tree t = *tp;
6743 167602424 : bitmap forced_stack_vars = (bitmap)((walk_stmt_info *)data)->info;
6744 :
6745 167602424 : if (IS_TYPE_OR_DECL_P (t))
6746 34370416 : *walk_subtrees = 0;
6747 133232008 : else if (REFERENCE_CLASS_P (t) && TREE_THIS_VOLATILE (t))
6748 : {
6749 71263 : t = get_base_address (t);
6750 71263 : if (t && DECL_P (t)
6751 58701 : && DECL_MODE (t) != BLKmode
6752 108027 : && !TREE_ADDRESSABLE (t))
6753 32718 : bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6754 71263 : *walk_subtrees = 0;
6755 : }
6756 133160745 : else if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6757 : {
6758 6453189 : while (((TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6759 3082794 : && is_gimple_min_invariant (TREE_OPERAND (t, 1))
6760 2606403 : && (!TREE_OPERAND (t, 2)
6761 37302 : || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6762 3880955 : || (TREE_CODE (t) == COMPONENT_REF
6763 958980 : && (!TREE_OPERAND (t,2)
6764 0 : || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6765 2921975 : || TREE_CODE (t) == BIT_FIELD_REF
6766 : || TREE_CODE (t) == REALPART_EXPR
6767 : || TREE_CODE (t) == IMAGPART_EXPR
6768 : || TREE_CODE (t) == VIEW_CONVERT_EXPR
6769 6453189 : || CONVERT_EXPR_P (t))
6770 3541272 : t = TREE_OPERAND (t, 0);
6771 :
6772 2911917 : if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6773 : {
6774 510560 : t = get_base_address (t);
6775 510560 : if (t && DECL_P (t)
6776 409673 : && DECL_MODE (t) != BLKmode
6777 532737 : && !TREE_ADDRESSABLE (t))
6778 12950 : bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6779 : }
6780 :
6781 2911917 : *walk_subtrees = 0;
6782 : }
6783 : /* References of size POLY_INT_CST to a fixed-size object must go
6784 : through memory. It's more efficient to force that here than
6785 : to create temporary slots on the fly.
6786 : RTL expansion expects TARGET_MEM_REF to always address actual memory.
6787 : Also, force to stack non-BLKmode vars accessed through VIEW_CONVERT_EXPR
6788 : to BLKmode type. */
6789 130248828 : else if (TREE_CODE (t) == TARGET_MEM_REF
6790 : || (TREE_CODE (t) == MEM_REF
6791 129392851 : && TYPE_SIZE (TREE_TYPE (t))
6792 : && POLY_INT_CST_P (TYPE_SIZE (TREE_TYPE (t))))
6793 259641679 : || (TREE_CODE (t) == VIEW_CONVERT_EXPR
6794 335612 : && TYPE_MODE (TREE_TYPE (t)) == BLKmode))
6795 : {
6796 934500 : tree base = get_base_address (t);
6797 934500 : if (base
6798 934500 : && DECL_P (base)
6799 266934 : && !TREE_ADDRESSABLE (base)
6800 122628 : && DECL_MODE (base) != BLKmode
6801 940052 : && GET_MODE_SIZE (DECL_MODE (base)).is_constant ())
6802 5552 : bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6803 934500 : *walk_subtrees = 0;
6804 : }
6805 :
6806 167602424 : return NULL_TREE;
6807 : }
6808 :
6809 : /* If there's a chance to get a pseudo for t then if it would be of float mode
6810 : and the actual access is via an integer mode (lowered memcpy or similar
6811 : access) then avoid the register expansion if the mode likely is not storage
6812 : suitable for raw bits processing (like XFmode on i?86). */
6813 :
6814 : static void
6815 6763004 : avoid_type_punning_on_regs (tree t, bitmap forced_stack_vars)
6816 : {
6817 6763004 : machine_mode access_mode = TYPE_MODE (TREE_TYPE (t));
6818 6763004 : if (access_mode != BLKmode
6819 6592257 : && !SCALAR_INT_MODE_P (access_mode))
6820 : return;
6821 5768544 : tree base = get_base_address (t);
6822 5768544 : if (DECL_P (base)
6823 4246624 : && !TREE_ADDRESSABLE (base)
6824 1250447 : && FLOAT_MODE_P (DECL_MODE (base))
6825 80 : && maybe_lt (GET_MODE_PRECISION (DECL_MODE (base)),
6826 240 : GET_MODE_BITSIZE (GET_MODE_INNER (DECL_MODE (base))))
6827 : /* Double check in the expensive way we really would get a pseudo. */
6828 5768549 : && use_register_for_decl (base))
6829 5 : bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6830 : }
6831 :
6832 : /* RTL expansion is not able to compile array references with variable
6833 : offsets for arrays stored in single register. Discover such
6834 : expressions and mark variables as addressable to avoid this
6835 : scenario. */
6836 :
6837 : static void
6838 1515899 : discover_nonconstant_array_refs (bitmap forced_stack_vars)
6839 : {
6840 1515899 : basic_block bb;
6841 1515899 : gimple_stmt_iterator gsi;
6842 :
6843 1515899 : walk_stmt_info wi = {};
6844 1515899 : wi.info = forced_stack_vars;
6845 14416832 : FOR_EACH_BB_FN (bb, cfun)
6846 127585930 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
6847 : {
6848 101784064 : gimple *stmt = gsi_stmt (gsi);
6849 101784064 : if (!is_gimple_debug (stmt))
6850 : {
6851 47870339 : walk_gimple_op (stmt, discover_nonconstant_array_refs_r, &wi);
6852 47870339 : gcall *call = dyn_cast <gcall *> (stmt);
6853 7073391 : if (call && gimple_call_internal_p (call))
6854 : {
6855 234445 : tree cand = NULL_TREE;
6856 234445 : switch (gimple_call_internal_fn (call))
6857 : {
6858 0 : case IFN_LOAD_LANES:
6859 : /* The source must be a MEM. */
6860 0 : cand = gimple_call_arg (call, 0);
6861 0 : break;
6862 0 : case IFN_STORE_LANES:
6863 : /* The destination must be a MEM. */
6864 0 : cand = gimple_call_lhs (call);
6865 0 : break;
6866 : default:
6867 : break;
6868 : }
6869 0 : if (cand)
6870 0 : cand = get_base_address (cand);
6871 0 : if (cand
6872 0 : && DECL_P (cand)
6873 0 : && use_register_for_decl (cand))
6874 0 : bitmap_set_bit (forced_stack_vars, DECL_UID (cand));
6875 : }
6876 159068831 : if (gimple_vdef (stmt))
6877 : {
6878 15855304 : tree t = gimple_get_lhs (stmt);
6879 15855304 : if (t && REFERENCE_CLASS_P (t))
6880 6763004 : avoid_type_punning_on_regs (t, forced_stack_vars);
6881 : }
6882 : }
6883 : }
6884 1515899 : }
6885 :
6886 : /* This function sets crtl->args.internal_arg_pointer to a virtual
6887 : register if DRAP is needed. Local register allocator will replace
6888 : virtual_incoming_args_rtx with the virtual register. */
6889 :
6890 : static void
6891 1515897 : expand_stack_alignment (void)
6892 : {
6893 1515897 : rtx drap_rtx;
6894 1515897 : unsigned int preferred_stack_boundary;
6895 :
6896 1515897 : if (! SUPPORTS_STACK_ALIGNMENT)
6897 : return;
6898 :
6899 1515897 : if (cfun->calls_alloca
6900 1500395 : || cfun->has_nonlocal_label
6901 1499592 : || crtl->has_nonlocal_goto)
6902 16708 : crtl->need_drap = true;
6903 :
6904 : /* Call update_stack_boundary here again to update incoming stack
6905 : boundary. It may set incoming stack alignment to a different
6906 : value after RTL expansion. TARGET_FUNCTION_OK_FOR_SIBCALL may
6907 : use the minimum incoming stack alignment to check if it is OK
6908 : to perform sibcall optimization since sibcall optimization will
6909 : only align the outgoing stack to incoming stack boundary. */
6910 1515897 : if (targetm.calls.update_stack_boundary)
6911 1515897 : targetm.calls.update_stack_boundary ();
6912 :
6913 : /* The incoming stack frame has to be aligned at least at
6914 : parm_stack_boundary. */
6915 1515897 : gcc_assert (crtl->parm_stack_boundary <= INCOMING_STACK_BOUNDARY);
6916 :
6917 : /* Update crtl->stack_alignment_estimated and use it later to align
6918 : stack. We check PREFERRED_STACK_BOUNDARY if there may be non-call
6919 : exceptions since callgraph doesn't collect incoming stack alignment
6920 : in this case. */
6921 1515897 : if (cfun->can_throw_non_call_exceptions
6922 262936 : && PREFERRED_STACK_BOUNDARY > crtl->preferred_stack_boundary)
6923 : preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
6924 : else
6925 1487932 : preferred_stack_boundary = crtl->preferred_stack_boundary;
6926 1515897 : if (preferred_stack_boundary > crtl->stack_alignment_estimated)
6927 730094 : crtl->stack_alignment_estimated = preferred_stack_boundary;
6928 1515897 : if (preferred_stack_boundary > crtl->stack_alignment_needed)
6929 681788 : crtl->stack_alignment_needed = preferred_stack_boundary;
6930 :
6931 1515897 : gcc_assert (crtl->stack_alignment_needed
6932 : <= crtl->stack_alignment_estimated);
6933 :
6934 1515897 : crtl->stack_realign_needed
6935 1515897 : = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
6936 1515897 : crtl->stack_realign_tried = crtl->stack_realign_needed;
6937 :
6938 1515897 : crtl->stack_realign_processed = true;
6939 :
6940 : /* Target has to redefine TARGET_GET_DRAP_RTX to support stack
6941 : alignment. */
6942 1515897 : gcc_assert (targetm.calls.get_drap_rtx != NULL);
6943 1515897 : drap_rtx = targetm.calls.get_drap_rtx ();
6944 :
6945 : /* stack_realign_drap and drap_rtx must match. */
6946 1515897 : gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
6947 :
6948 : /* Do nothing if NULL is returned, which means DRAP is not needed. */
6949 1515897 : if (drap_rtx != NULL)
6950 : {
6951 7240 : crtl->args.internal_arg_pointer = drap_rtx;
6952 :
6953 : /* Call fixup_tail_calls to clean up REG_EQUIV note if DRAP is
6954 : needed. */
6955 7240 : fixup_tail_calls ();
6956 : }
6957 : }
6958 :
6959 :
6960 : static void
6961 0 : expand_main_function (void)
6962 : {
6963 : #if (defined(INVOKE__main) \
6964 : || (!defined(HAS_INIT_SECTION) \
6965 : && !defined(INIT_SECTION_ASM_OP) \
6966 : && !defined(INIT_ARRAY_SECTION_ASM_OP)))
6967 : emit_library_call (init_one_libfunc (NAME__MAIN), LCT_NORMAL, VOIDmode);
6968 : #endif
6969 0 : }
6970 :
6971 :
6972 : /* Expand code to initialize the stack_protect_guard. This is invoked at
6973 : the beginning of a function to be protected. */
6974 :
6975 : static void
6976 296 : stack_protect_prologue (void)
6977 : {
6978 296 : tree guard_decl = targetm.stack_protect_guard ();
6979 296 : rtx x, y;
6980 :
6981 296 : crtl->stack_protect_guard_decl = guard_decl;
6982 296 : x = expand_normal (crtl->stack_protect_guard);
6983 :
6984 296 : if (targetm.have_stack_protect_combined_set () && guard_decl)
6985 : {
6986 0 : gcc_assert (DECL_P (guard_decl));
6987 0 : y = DECL_RTL (guard_decl);
6988 :
6989 : /* Allow the target to compute address of Y and copy it to X without
6990 : leaking Y into a register. This combined address + copy pattern
6991 : allows the target to prevent spilling of any intermediate results by
6992 : splitting it after register allocator. */
6993 0 : if (rtx_insn *insn = targetm.gen_stack_protect_combined_set (x, y))
6994 : {
6995 0 : emit_insn (insn);
6996 0 : return;
6997 : }
6998 : }
6999 :
7000 296 : if (guard_decl)
7001 296 : y = expand_normal (guard_decl);
7002 : else
7003 0 : y = const0_rtx;
7004 :
7005 : /* Allow the target to copy from Y to X without leaking Y into a
7006 : register. */
7007 296 : if (targetm.have_stack_protect_set ())
7008 296 : if (rtx_insn *insn = targetm.gen_stack_protect_set (x, y))
7009 : {
7010 296 : emit_insn (insn);
7011 296 : return;
7012 : }
7013 :
7014 : /* Otherwise do a straight move. */
7015 0 : emit_move_insn (x, y);
7016 : }
7017 :
7018 : /* Translate the intermediate representation contained in the CFG
7019 : from GIMPLE trees to RTL.
7020 :
7021 : We do conversion per basic block and preserve/update the tree CFG.
7022 : This implies we have to do some magic as the CFG can simultaneously
7023 : consist of basic blocks containing RTL and GIMPLE trees. This can
7024 : confuse the CFG hooks, so be careful to not manipulate CFG during
7025 : the expansion. */
7026 :
7027 : namespace {
7028 :
7029 : const pass_data pass_data_expand =
7030 : {
7031 : RTL_PASS, /* type */
7032 : "expand", /* name */
7033 : OPTGROUP_NONE, /* optinfo_flags */
7034 : TV_EXPAND, /* tv_id */
7035 : ( PROP_ssa | PROP_gimple_leh | PROP_cfg
7036 : | PROP_gimple_lcx
7037 : | PROP_gimple_lvec
7038 : | PROP_gimple_lva), /* properties_required */
7039 : PROP_rtl, /* properties_provided */
7040 : ( PROP_ssa | PROP_gimple ), /* properties_destroyed */
7041 : 0, /* todo_flags_start */
7042 : 0, /* todo_flags_finish */
7043 : };
7044 :
7045 : class pass_expand : public rtl_opt_pass
7046 : {
7047 : public:
7048 294200 : pass_expand (gcc::context *ctxt)
7049 588400 : : rtl_opt_pass (pass_data_expand, ctxt)
7050 : {}
7051 :
7052 : /* opt_pass methods: */
7053 : unsigned int execute (function *) final override;
7054 :
7055 : }; // class pass_expand
7056 :
7057 : unsigned int
7058 1515899 : pass_expand::execute (function *fun)
7059 : {
7060 1515899 : basic_block bb, init_block;
7061 1515899 : edge_iterator ei;
7062 1515899 : edge e;
7063 1515899 : rtx_insn *var_seq, *var_ret_seq;
7064 1515899 : unsigned i;
7065 :
7066 1515899 : timevar_push (TV_OUT_OF_SSA);
7067 1515899 : rewrite_out_of_ssa (&SA);
7068 1515899 : timevar_pop (TV_OUT_OF_SSA);
7069 1515899 : SA.partition_to_pseudo = XCNEWVEC (rtx, SA.map->num_partitions);
7070 :
7071 1515899 : if (MAY_HAVE_DEBUG_BIND_STMTS && flag_tree_ter)
7072 : {
7073 509886 : gimple_stmt_iterator gsi;
7074 6942937 : FOR_EACH_BB_FN (bb, cfun)
7075 88677969 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7076 117594612 : if (gimple_debug_bind_p (gsi_stmt (gsi)))
7077 41782745 : avoid_deep_ter_for_debug (gsi_stmt (gsi), 0);
7078 : }
7079 :
7080 : /* Mark arrays indexed with non-constant indices with TREE_ADDRESSABLE. */
7081 1515899 : auto_bitmap forced_stack_vars;
7082 1515899 : discover_nonconstant_array_refs (forced_stack_vars);
7083 :
7084 : /* Make sure all values used by the optimization passes have sane
7085 : defaults. */
7086 1515899 : reg_renumber = 0;
7087 :
7088 : /* Some backends want to know that we are expanding to RTL. */
7089 1515899 : currently_expanding_to_rtl = 1;
7090 : /* Dominators are not kept up-to-date as we may create new basic-blocks. */
7091 1515899 : free_dominance_info (CDI_DOMINATORS);
7092 :
7093 1515899 : rtl_profile_for_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
7094 :
7095 1515899 : insn_locations_init ();
7096 1515899 : if (!DECL_IS_UNDECLARED_BUILTIN (current_function_decl))
7097 : {
7098 : /* Eventually, all FEs should explicitly set function_start_locus. */
7099 1391895 : if (LOCATION_LOCUS (fun->function_start_locus) == UNKNOWN_LOCATION)
7100 658207 : set_curr_insn_location
7101 658207 : (DECL_SOURCE_LOCATION (current_function_decl));
7102 : else
7103 733688 : set_curr_insn_location (fun->function_start_locus);
7104 : }
7105 : else
7106 124004 : set_curr_insn_location (UNKNOWN_LOCATION);
7107 1515899 : prologue_location = curr_insn_location ();
7108 :
7109 : #ifdef INSN_SCHEDULING
7110 1515899 : init_sched_attrs ();
7111 : #endif
7112 :
7113 : /* Make sure first insn is a note even if we don't want linenums.
7114 : This makes sure the first insn will never be deleted.
7115 : Also, final expects a note to appear there. */
7116 1515899 : emit_note (NOTE_INSN_DELETED);
7117 :
7118 1515899 : targetm.expand_to_rtl_hook ();
7119 1515899 : crtl->init_stack_alignment ();
7120 1515899 : fun->cfg->max_jumptable_ents = 0;
7121 :
7122 : /* Resolve the function section. Some targets, like ARM EABI rely on knowledge
7123 : of the function section at expansion time to predict distance of calls. */
7124 1515899 : resolve_unique_section (current_function_decl, 0, flag_function_sections);
7125 :
7126 : /* Expand the variables recorded during gimple lowering. */
7127 1515899 : timevar_push (TV_VAR_EXPAND);
7128 1515899 : start_sequence ();
7129 :
7130 1515899 : var_ret_seq = expand_used_vars (forced_stack_vars);
7131 :
7132 1515899 : var_seq = end_sequence ();
7133 1515899 : timevar_pop (TV_VAR_EXPAND);
7134 :
7135 : /* Honor stack protection warnings. */
7136 1515899 : if (warn_stack_protect)
7137 : {
7138 0 : if (fun->calls_alloca)
7139 0 : warning (OPT_Wstack_protector,
7140 : "stack protector not protecting local variables: "
7141 : "variable length buffer");
7142 0 : if (has_short_buffer && !crtl->stack_protect_guard)
7143 0 : warning (OPT_Wstack_protector,
7144 : "stack protector not protecting function: "
7145 : "all local arrays are less than %d bytes long",
7146 0 : (int) param_ssp_buffer_size);
7147 : }
7148 :
7149 : /* Temporarily mark PARM_DECLs and RESULT_DECLs we need to expand to
7150 : memory addressable so expand_function_start can emit the required
7151 : copies. */
7152 1515899 : auto_vec<tree, 16> marked_parms;
7153 4683776 : for (tree parm = DECL_ARGUMENTS (current_function_decl); parm;
7154 3167877 : parm = DECL_CHAIN (parm))
7155 3167877 : if (!TREE_ADDRESSABLE (parm)
7156 3167877 : && bitmap_bit_p (forced_stack_vars, DECL_UID (parm)))
7157 : {
7158 121 : TREE_ADDRESSABLE (parm) = 1;
7159 121 : marked_parms.safe_push (parm);
7160 : }
7161 1515899 : if (DECL_RESULT (current_function_decl)
7162 1515899 : && !TREE_ADDRESSABLE (DECL_RESULT (current_function_decl))
7163 3024809 : && bitmap_bit_p (forced_stack_vars,
7164 1508910 : DECL_UID (DECL_RESULT (current_function_decl))))
7165 : {
7166 0 : TREE_ADDRESSABLE (DECL_RESULT (current_function_decl)) = 1;
7167 0 : marked_parms.safe_push (DECL_RESULT (current_function_decl));
7168 : }
7169 :
7170 : /* Set up parameters and prepare for return, for the function. */
7171 1515899 : expand_function_start (current_function_decl);
7172 :
7173 : /* Clear TREE_ADDRESSABLE again. */
7174 3031919 : while (!marked_parms.is_empty ())
7175 121 : TREE_ADDRESSABLE (marked_parms.pop ()) = 0;
7176 :
7177 : /* If we emitted any instructions for setting up the variables,
7178 : emit them before the FUNCTION_START note. */
7179 1515899 : if (var_seq)
7180 : {
7181 1783 : emit_insn_before (var_seq, parm_birth_insn);
7182 :
7183 : /* In expand_function_end we'll insert the alloca save/restore
7184 : before parm_birth_insn. We've just insertted an alloca call.
7185 : Adjust the pointer to match. */
7186 1783 : parm_birth_insn = var_seq;
7187 : }
7188 :
7189 : /* Now propagate the RTL assignment of each partition to the
7190 : underlying var of each SSA_NAME. */
7191 1515899 : tree name;
7192 :
7193 75125206 : FOR_EACH_SSA_NAME (i, name, cfun)
7194 : {
7195 : /* We might have generated new SSA names in
7196 : update_alias_info_with_stack_vars. They will have a NULL
7197 : defining statements, and won't be part of the partitioning,
7198 : so ignore those. */
7199 49985708 : if (!SSA_NAME_DEF_STMT (name))
7200 70642 : continue;
7201 :
7202 49915066 : adjust_one_expanded_partition_var (name);
7203 : }
7204 :
7205 : /* Clean up RTL of variables that straddle across multiple
7206 : partitions, and check that the rtl of any PARM_DECLs that are not
7207 : cleaned up is that of their default defs. */
7208 75125206 : FOR_EACH_SSA_NAME (i, name, cfun)
7209 : {
7210 49985708 : int part;
7211 :
7212 : /* We might have generated new SSA names in
7213 : update_alias_info_with_stack_vars. They will have a NULL
7214 : defining statements, and won't be part of the partitioning,
7215 : so ignore those. */
7216 49985708 : if (!SSA_NAME_DEF_STMT (name))
7217 70642 : continue;
7218 49915066 : part = var_to_partition (SA.map, name);
7219 49915066 : if (part == NO_PARTITION)
7220 17774266 : continue;
7221 :
7222 : /* If this decl was marked as living in multiple places, reset
7223 : this now to NULL. */
7224 32140800 : tree var = SSA_NAME_VAR (name);
7225 10615841 : if (var && DECL_RTL_IF_SET (var) == pc_rtx)
7226 243637 : SET_DECL_RTL (var, NULL);
7227 : /* Check that the pseudos chosen by assign_parms are those of
7228 : the corresponding default defs. */
7229 31897163 : else if (SSA_NAME_IS_DEFAULT_DEF (name)
7230 31897163 : && (TREE_CODE (var) == PARM_DECL
7231 3745318 : || TREE_CODE (var) == RESULT_DECL))
7232 : {
7233 3683352 : rtx in = DECL_RTL_IF_SET (var);
7234 3683352 : gcc_assert (in);
7235 3683352 : rtx out = SA.partition_to_pseudo[part];
7236 3683352 : gcc_assert (in == out);
7237 :
7238 : /* Now reset VAR's RTL to IN, so that the _EXPR attrs match
7239 : those expected by debug backends for each parm and for
7240 : the result. This is particularly important for stabs,
7241 : whose register elimination from parm's DECL_RTL may cause
7242 : -fcompare-debug differences as SET_DECL_RTL changes reg's
7243 : attrs. So, make sure the RTL already has the parm as the
7244 : EXPR, so that it won't change. */
7245 3683352 : SET_DECL_RTL (var, NULL_RTX);
7246 3683352 : if (MEM_P (in))
7247 790888 : set_mem_attributes (in, var, true);
7248 3683352 : SET_DECL_RTL (var, in);
7249 : }
7250 : }
7251 :
7252 1515899 : if (flag_checking)
7253 1515879 : verify_partition_mem_exprs ();
7254 :
7255 : /* If this function is `main', emit a call to `__main'
7256 : to run global initializers, etc. */
7257 1515899 : if (DECL_NAME (current_function_decl)
7258 1515899 : && MAIN_NAME_P (DECL_NAME (current_function_decl))
7259 1629429 : && DECL_FILE_SCOPE_P (current_function_decl))
7260 1515899 : expand_main_function ();
7261 :
7262 : /* Initialize the stack_protect_guard field. This must happen after the
7263 : call to __main (if any) so that the external decl is initialized. */
7264 1515899 : if (crtl->stack_protect_guard && targetm.stack_protect_runtime_enabled_p ())
7265 296 : stack_protect_prologue ();
7266 :
7267 1515899 : expand_phi_nodes (&SA);
7268 :
7269 : /* Release any stale SSA redirection data. */
7270 1515899 : redirect_edge_var_map_empty ();
7271 :
7272 : /* Register rtl specific functions for cfg. */
7273 1515899 : rtl_register_cfg_hooks ();
7274 :
7275 1515899 : init_block = construct_init_block ();
7276 :
7277 : /* Clear EDGE_EXECUTABLE on the entry edge(s). It is cleaned from the
7278 : remaining edges later. */
7279 3031798 : FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs)
7280 1515899 : e->flags &= ~EDGE_EXECUTABLE;
7281 :
7282 : /* If the function has too many markers, drop them while expanding. */
7283 1515899 : if (cfun->debug_marker_count
7284 1515899 : >= param_max_debug_marker_count)
7285 1 : cfun->debug_nonbind_markers = false;
7286 :
7287 1515899 : if (optimize)
7288 1062676 : enable_ranger (fun);
7289 1515899 : lab_rtx_for_bb = new hash_map<basic_block, rtx_code_label *>;
7290 1515899 : head_end_for_bb.create (last_basic_block_for_fn (fun));
7291 14420052 : FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
7292 : next_bb)
7293 12904155 : bb = expand_gimple_basic_block (bb, var_ret_seq);
7294 1515897 : if (optimize)
7295 1062676 : disable_ranger (fun);
7296 14822402 : FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
7297 : next_bb)
7298 : {
7299 13306505 : if ((bb->flags & BB_RTL) == 0)
7300 : {
7301 12904145 : bb->il.gimple.seq = NULL;
7302 12904145 : bb->il.gimple.phi_nodes = NULL;
7303 12904145 : init_rtl_bb_info (bb);
7304 12904145 : bb->flags |= BB_RTL;
7305 12904145 : BB_HEAD (bb) = head_end_for_bb[bb->index].first;
7306 12904145 : BB_END (bb) = head_end_for_bb[bb->index].second;
7307 : }
7308 : /* These flags have no purpose in RTL land. */
7309 13306505 : if (EDGE_COUNT (bb->succs) == 2)
7310 : {
7311 6400497 : EDGE_SUCC (bb, 0)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
7312 6400497 : EDGE_SUCC (bb, 1)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
7313 : }
7314 19428015 : else if (single_succ_p (bb))
7315 6121510 : single_succ_edge (bb)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
7316 : }
7317 1515897 : head_end_for_bb.release ();
7318 :
7319 1515897 : if (MAY_HAVE_DEBUG_BIND_INSNS)
7320 509911 : expand_debug_locations ();
7321 :
7322 1515897 : if (deep_ter_debug_map)
7323 : {
7324 860 : delete deep_ter_debug_map;
7325 860 : deep_ter_debug_map = NULL;
7326 : }
7327 :
7328 : /* Free stuff we no longer need after GIMPLE optimizations. */
7329 1515897 : free_dominance_info (CDI_DOMINATORS);
7330 1515897 : free_dominance_info (CDI_POST_DOMINATORS);
7331 1515897 : delete_tree_cfg_annotations (fun);
7332 :
7333 1515897 : timevar_push (TV_OUT_OF_SSA);
7334 1515897 : finish_out_of_ssa (&SA);
7335 1515897 : timevar_pop (TV_OUT_OF_SSA);
7336 :
7337 1515897 : timevar_push (TV_POST_EXPAND);
7338 : /* We are no longer in SSA form. */
7339 1515897 : fun->gimple_df->in_ssa_p = false;
7340 1515897 : loops_state_clear (LOOP_CLOSED_SSA);
7341 :
7342 : /* Expansion is used by optimization passes too, set maybe_hot_insn_p
7343 : conservatively to true until they are all profile aware. */
7344 3031794 : delete lab_rtx_for_bb;
7345 1515897 : free_histograms (fun);
7346 :
7347 1515897 : construct_exit_block ();
7348 1515897 : insn_locations_finalize ();
7349 :
7350 1515897 : if (var_ret_seq)
7351 : {
7352 1923 : rtx_insn *after = return_label;
7353 1923 : rtx_insn *next = NEXT_INSN (after);
7354 1923 : if (next && NOTE_INSN_BASIC_BLOCK_P (next))
7355 1923 : after = next;
7356 1923 : emit_insn_after (var_ret_seq, after);
7357 : }
7358 :
7359 1515897 : if (hwassist_sanitize_stack_p ())
7360 364 : hwasan_maybe_emit_frame_base_init ();
7361 :
7362 : /* Zap the tree EH table. */
7363 1515897 : set_eh_throw_stmt_table (fun, NULL);
7364 :
7365 : /* We need JUMP_LABEL be set in order to redirect jumps, and hence
7366 : split edges which edge insertions might do. */
7367 1515897 : rebuild_jump_labels (get_insns ());
7368 :
7369 : /* If we have a single successor to the entry block, put the pending insns
7370 : after parm birth, but before NOTE_INSNS_FUNCTION_BEG. */
7371 1515897 : if (single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (fun)))
7372 : {
7373 1515897 : edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
7374 1515897 : if (e->insns.r)
7375 : {
7376 11 : rtx_insn *insns = e->insns.r;
7377 11 : e->insns.r = NULL;
7378 11 : rebuild_jump_labels_chain (insns);
7379 11 : if (NOTE_P (parm_birth_insn)
7380 11 : && NOTE_KIND (parm_birth_insn) == NOTE_INSN_FUNCTION_BEG)
7381 11 : emit_insn_before_noloc (insns, parm_birth_insn, e->dest);
7382 : else
7383 0 : emit_insn_after_noloc (insns, parm_birth_insn, e->dest);
7384 : }
7385 : }
7386 :
7387 : /* Otherwise, as well as for other edges, take the usual way. */
7388 1515897 : commit_edge_insertions ();
7389 :
7390 : /* We're done expanding trees to RTL. */
7391 1515897 : currently_expanding_to_rtl = 0;
7392 :
7393 1515897 : flush_mark_addressable_queue ();
7394 :
7395 18462228 : FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun)->next_bb,
7396 : EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
7397 : {
7398 16946331 : edge e;
7399 16946331 : edge_iterator ei;
7400 39589626 : for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
7401 : {
7402 : /* Clear EDGE_EXECUTABLE. This flag is never used in the backend. */
7403 22643295 : e->flags &= ~EDGE_EXECUTABLE;
7404 :
7405 : /* At the moment not all abnormal edges match the RTL
7406 : representation. It is safe to remove them here as
7407 : find_many_sub_basic_blocks will rediscover them.
7408 : In the future we should get this fixed properly. */
7409 22643295 : if ((e->flags & EDGE_ABNORMAL)
7410 141322 : && !(e->flags & EDGE_SIBCALL))
7411 10317 : remove_edge (e);
7412 : else
7413 22632978 : ei_next (&ei);
7414 : }
7415 : }
7416 :
7417 1515897 : auto_sbitmap blocks (last_basic_block_for_fn (fun));
7418 1515897 : bitmap_ones (blocks);
7419 1515897 : find_many_sub_basic_blocks (blocks);
7420 1515897 : purge_all_dead_edges ();
7421 :
7422 : /* After initial rtl generation, call back to finish generating
7423 : exception support code. We need to do this before cleaning up
7424 : the CFG as the code does not expect dead landing pads. */
7425 1515897 : if (fun->eh->region_tree != NULL)
7426 64611 : finish_eh_generation ();
7427 :
7428 : /* Call expand_stack_alignment after finishing all
7429 : updates to crtl->preferred_stack_boundary. */
7430 1515897 : expand_stack_alignment ();
7431 :
7432 : /* Fixup REG_EQUIV notes in the prologue if there are tailcalls in this
7433 : function. */
7434 1515897 : if (crtl->tail_call_emit)
7435 118353 : fixup_tail_calls ();
7436 :
7437 1515897 : HOST_WIDE_INT patch_area_size, patch_area_entry;
7438 1515897 : parse_and_check_patch_area (flag_patchable_function_entry, false,
7439 : &patch_area_size, &patch_area_entry);
7440 :
7441 1515897 : tree patchable_function_entry_attr
7442 1515897 : = lookup_attribute ("patchable_function_entry",
7443 1515897 : DECL_ATTRIBUTES (cfun->decl));
7444 1515897 : if (patchable_function_entry_attr)
7445 : {
7446 16 : tree pp_val = TREE_VALUE (patchable_function_entry_attr);
7447 16 : tree patchable_function_entry_value1 = TREE_VALUE (pp_val);
7448 :
7449 16 : patch_area_size = tree_to_uhwi (patchable_function_entry_value1);
7450 16 : patch_area_entry = 0;
7451 16 : if (TREE_CHAIN (pp_val) != NULL_TREE)
7452 : {
7453 8 : tree patchable_function_entry_value2
7454 8 : = TREE_VALUE (TREE_CHAIN (pp_val));
7455 8 : patch_area_entry = tree_to_uhwi (patchable_function_entry_value2);
7456 : }
7457 : }
7458 :
7459 1515897 : if (patch_area_entry > patch_area_size)
7460 : {
7461 0 : if (patch_area_size > 0)
7462 0 : warning (OPT_Wattributes,
7463 : "patchable function entry %wu exceeds size %wu",
7464 : patch_area_entry, patch_area_size);
7465 0 : patch_area_entry = 0;
7466 : }
7467 :
7468 1515897 : crtl->patch_area_size = patch_area_size;
7469 1515897 : crtl->patch_area_entry = patch_area_entry;
7470 :
7471 : /* BB subdivision may have created basic blocks that are only reachable
7472 : from unlikely bbs but not marked as such in the profile. */
7473 1515897 : if (optimize)
7474 1062676 : propagate_unlikely_bbs_forward ();
7475 :
7476 : /* Remove unreachable blocks, otherwise we cannot compute dominators
7477 : which are needed for loop state verification. As a side-effect
7478 : this also compacts blocks.
7479 : ??? We cannot remove trivially dead insns here as for example
7480 : the DRAP reg on i?86 is not magically live at this point.
7481 : gcc.c-torture/execute/ipa-sra-2.c execution, -Os -m32 fails otherwise. */
7482 1515897 : cleanup_cfg (CLEANUP_NO_INSN_DEL);
7483 :
7484 1515897 : checking_verify_flow_info ();
7485 :
7486 : /* Initialize pseudos allocated for hard registers. */
7487 1515897 : emit_initial_value_sets ();
7488 :
7489 : /* And finally unshare all RTL. */
7490 1515897 : unshare_all_rtl ();
7491 :
7492 : /* There's no need to defer outputting this function any more; we
7493 : know we want to output it. */
7494 1515897 : DECL_DEFER_OUTPUT (current_function_decl) = 0;
7495 :
7496 : /* Now that we're done expanding trees to RTL, we shouldn't have any
7497 : more CONCATs anywhere. */
7498 1515897 : generating_concat_p = 0;
7499 :
7500 1515897 : if (dump_file)
7501 : {
7502 374 : fprintf (dump_file,
7503 : "\n\n;;\n;; Full RTL generated for this function:\n;;\n");
7504 : /* And the pass manager will dump RTL for us. */
7505 : }
7506 :
7507 : /* If we're emitting a nested function, make sure its parent gets
7508 : emitted as well. Doing otherwise confuses debug info. */
7509 1515897 : {
7510 1515897 : tree parent;
7511 1515897 : for (parent = DECL_CONTEXT (current_function_decl);
7512 3268710 : parent != NULL_TREE;
7513 1752813 : parent = get_containing_scope (parent))
7514 1752813 : if (TREE_CODE (parent) == FUNCTION_DECL)
7515 76829 : TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (parent)) = 1;
7516 : }
7517 :
7518 1515897 : TREE_ASM_WRITTEN (current_function_decl) = 1;
7519 :
7520 : /* After expanding, the return labels are no longer needed. */
7521 1515897 : return_label = NULL;
7522 1515897 : naked_return_label = NULL;
7523 :
7524 : /* After expanding, the tm_restart map is no longer needed. */
7525 1515897 : if (fun->gimple_df->tm_restart)
7526 265 : fun->gimple_df->tm_restart = NULL;
7527 :
7528 : /* Tag the blocks with a depth number so that change_scope can find
7529 : the common parent easily. */
7530 1515897 : set_block_levels (DECL_INITIAL (fun->decl), 0);
7531 1515897 : default_rtl_profile ();
7532 :
7533 : /* For -dx discard loops now, otherwise IL verify in clean_state will
7534 : ICE. */
7535 1515897 : if (rtl_dump_and_exit)
7536 : {
7537 79 : cfun->curr_properties &= ~PROP_loops;
7538 79 : loop_optimizer_finalize ();
7539 : }
7540 :
7541 1515897 : timevar_pop (TV_POST_EXPAND);
7542 :
7543 1515897 : return 0;
7544 1515897 : }
7545 :
7546 : } // anon namespace
7547 :
7548 : rtl_opt_pass *
7549 294200 : make_pass_expand (gcc::context *ctxt)
7550 : {
7551 294200 : return new pass_expand (ctxt);
7552 : }
|
