Line data Source code
1 : /* A pass for lowering trees to RTL.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify
7 : it under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful,
12 : but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : GNU General Public License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "tree-pass.h"
30 : #include "memmodel.h"
31 : #include "tm_p.h"
32 : #include "ssa.h"
33 : #include "optabs.h"
34 : #include "regs.h" /* For reg_renumber. */
35 : #include "emit-rtl.h"
36 : #include "recog.h"
37 : #include "cgraph.h"
38 : #include "diagnostic.h"
39 : #include "fold-const.h"
40 : #include "varasm.h"
41 : #include "stor-layout.h"
42 : #include "stmt.h"
43 : #include "print-tree.h"
44 : #include "cfgrtl.h"
45 : #include "cfganal.h"
46 : #include "cfgbuild.h"
47 : #include "cfgcleanup.h"
48 : #include "dojump.h"
49 : #include "explow.h"
50 : #include "calls.h"
51 : #include "expr.h"
52 : #include "internal-fn.h"
53 : #include "tree-eh.h"
54 : #include "gimple-iterator.h"
55 : #include "gimple-expr.h"
56 : #include "gimple-walk.h"
57 : #include "tree-cfg.h"
58 : #include "tree-dfa.h"
59 : #include "tree-ssa.h"
60 : #include "except.h"
61 : #include "gimple-pretty-print.h"
62 : #include "toplev.h"
63 : #include "debug.h"
64 : #include "tree-inline.h"
65 : #include "value-prof.h"
66 : #include "tree-ssa-live.h"
67 : #include "tree-outof-ssa.h"
68 : #include "cfgloop.h"
69 : #include "insn-attr.h" /* For INSN_SCHEDULING. */
70 : #include "stringpool.h"
71 : #include "attribs.h"
72 : #include "asan.h"
73 : #include "tree-ssa-address.h"
74 : #include "output.h"
75 : #include "builtins.h"
76 : #include "opts.h"
77 : #include "gimple-range.h"
78 : #include "rtl-iter.h"
79 :
80 : /* Some systems use __main in a way incompatible with its use in gcc, in these
81 : cases use the macros NAME__MAIN to give a quoted symbol and SYMBOL__MAIN to
82 : give the same symbol without quotes for an alternative entry point. You
83 : must define both, or neither. */
84 : #ifndef NAME__MAIN
85 : #define NAME__MAIN "__main"
86 : #endif
87 :
88 : /* This variable holds information helping the rewriting of SSA trees
89 : into RTL. */
90 : struct ssaexpand SA;
91 :
92 : /* This variable holds the currently expanded gimple statement for purposes
93 : of communicating the profile info to the builtin expanders. */
94 : gimple *currently_expanding_gimple_stmt;
95 :
96 : static rtx expand_debug_expr (tree);
97 :
98 : static bool defer_stack_allocation (tree, bool);
99 :
100 : static void record_alignment_for_reg_var (unsigned int);
101 :
102 : /* Return an expression tree corresponding to the RHS of GIMPLE
103 : statement STMT. */
104 :
105 : tree
106 3510805 : gimple_assign_rhs_to_tree (gimple *stmt)
107 : {
108 3510805 : tree t;
109 3510805 : switch (gimple_assign_rhs_class (stmt))
110 : {
111 3839 : case GIMPLE_TERNARY_RHS:
112 3839 : t = build3 (gimple_assign_rhs_code (stmt),
113 3839 : TREE_TYPE (gimple_assign_lhs (stmt)),
114 : gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt),
115 : gimple_assign_rhs3 (stmt));
116 3839 : break;
117 1357574 : case GIMPLE_BINARY_RHS:
118 1357574 : t = build2 (gimple_assign_rhs_code (stmt),
119 1357574 : TREE_TYPE (gimple_assign_lhs (stmt)),
120 : gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt));
121 1357574 : break;
122 370472 : case GIMPLE_UNARY_RHS:
123 370472 : t = build1 (gimple_assign_rhs_code (stmt),
124 370472 : TREE_TYPE (gimple_assign_lhs (stmt)),
125 : gimple_assign_rhs1 (stmt));
126 370472 : break;
127 1778920 : case GIMPLE_SINGLE_RHS:
128 1778920 : {
129 1778920 : t = gimple_assign_rhs1 (stmt);
130 : /* Avoid modifying this tree in place below. */
131 3472038 : if ((gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t)
132 1661588 : && gimple_location (stmt) != EXPR_LOCATION (t))
133 1969814 : || (gimple_block (stmt) && currently_expanding_to_rtl
134 37274 : && EXPR_P (t)))
135 1531969 : t = copy_node (t);
136 : break;
137 : }
138 0 : default:
139 0 : gcc_unreachable ();
140 : }
141 :
142 3510805 : if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t))
143 3264782 : SET_EXPR_LOCATION (t, gimple_location (stmt));
144 :
145 3510805 : return t;
146 : }
147 :
148 :
149 : #ifndef STACK_ALIGNMENT_NEEDED
150 : #define STACK_ALIGNMENT_NEEDED 1
151 : #endif
152 :
153 : #define SSAVAR(x) (TREE_CODE (x) == SSA_NAME ? SSA_NAME_VAR (x) : x)
154 :
155 : /* Choose either CUR or NEXT as the leader DECL for a partition.
156 : Prefer ignored decls, to simplify debug dumps and reduce ambiguity
157 : out of the same user variable being in multiple partitions (this is
158 : less likely for compiler-introduced temps). Also used by out-of-SSA
159 : to work out which variable a partition will be given. */
160 :
161 : tree
162 80012754 : expand_leader_merge (tree cur, tree next)
163 : {
164 80012754 : if (cur == NULL || cur == next)
165 : return next;
166 :
167 3867619 : if (DECL_P (cur) && DECL_IGNORED_P (cur))
168 : return cur;
169 :
170 2806314 : if (DECL_P (next) && DECL_IGNORED_P (next))
171 319354 : return next;
172 :
173 : return cur;
174 : }
175 :
176 : /* Associate declaration T with storage space X. If T is no
177 : SSA name this is exactly SET_DECL_RTL, otherwise make the
178 : partition of T associated with X. */
179 : static inline void
180 72695814 : set_rtl (tree t, rtx x)
181 : {
182 72695814 : gcc_checking_assert (!x
183 : || !(TREE_CODE (t) == SSA_NAME || is_gimple_reg (t))
184 : || (use_register_for_decl (t)
185 : ? (REG_P (x)
186 : || (GET_CODE (x) == CONCAT
187 : && (REG_P (XEXP (x, 0))
188 : || SUBREG_P (XEXP (x, 0)))
189 : && (REG_P (XEXP (x, 1))
190 : || SUBREG_P (XEXP (x, 1))))
191 : /* We need to accept PARALLELs for RESUT_DECLs
192 : because of vector types with BLKmode returned
193 : in multiple registers, but they are supposed
194 : to be uncoalesced. */
195 : || (GET_CODE (x) == PARALLEL
196 : && SSAVAR (t)
197 : && TREE_CODE (SSAVAR (t)) == RESULT_DECL
198 : && (GET_MODE (x) == BLKmode
199 : || !flag_tree_coalesce_vars)))
200 : : (MEM_P (x) || x == pc_rtx
201 : || (GET_CODE (x) == CONCAT
202 : && MEM_P (XEXP (x, 0))
203 : && MEM_P (XEXP (x, 1))))));
204 : /* Check that the RTL for SSA_NAMEs and gimple-reg PARM_DECLs and
205 : RESULT_DECLs has the expected mode. For memory, we accept
206 : unpromoted modes, since that's what we're likely to get. For
207 : PARM_DECLs and RESULT_DECLs, we'll have been called by
208 : set_parm_rtl, which will give us the default def, so we don't
209 : have to compute it ourselves. For RESULT_DECLs, we accept mode
210 : mismatches too, as long as we have BLKmode or are not coalescing
211 : across variables, so that we don't reject BLKmode PARALLELs or
212 : unpromoted REGs. */
213 67353365 : gcc_checking_assert (!x || x == pc_rtx || TREE_CODE (t) != SSA_NAME
214 : || (SSAVAR (t)
215 : && TREE_CODE (SSAVAR (t)) == RESULT_DECL
216 : && (promote_ssa_mode (t, NULL) == BLKmode
217 : || !flag_tree_coalesce_vars))
218 : || !use_register_for_decl (t)
219 : || GET_MODE (x) == promote_ssa_mode (t, NULL));
220 :
221 67353365 : if (x)
222 : {
223 67789354 : bool skip = false;
224 67789354 : tree cur = NULL_TREE;
225 : rtx xm = x;
226 :
227 435989 : retry:
228 67789354 : if (MEM_P (xm))
229 4865580 : cur = MEM_EXPR (xm);
230 : else if (REG_P (xm))
231 56635833 : cur = REG_EXPR (xm);
232 : else if (SUBREG_P (xm))
233 : {
234 0 : gcc_assert (subreg_lowpart_p (xm));
235 0 : xm = SUBREG_REG (xm);
236 0 : goto retry;
237 : }
238 : else if (GET_CODE (xm) == CONCAT)
239 : {
240 432151 : xm = XEXP (xm, 0);
241 432151 : goto retry;
242 : }
243 : else if (GET_CODE (xm) == PARALLEL)
244 : {
245 3838 : xm = XVECEXP (xm, 0, 0);
246 3838 : gcc_assert (GET_CODE (xm) == EXPR_LIST);
247 3838 : xm = XEXP (xm, 0);
248 3838 : goto retry;
249 : }
250 5851952 : else if (xm == pc_rtx)
251 : skip = true;
252 : else
253 0 : gcc_unreachable ();
254 :
255 35280120 : tree next
256 61501413 : = skip ? cur : expand_leader_merge (cur, SSAVAR (t) ? SSAVAR (t) : t);
257 :
258 67353365 : if (cur != next)
259 : {
260 28426298 : if (MEM_P (x))
261 4089916 : set_mem_attributes (x,
262 2044958 : next && TREE_CODE (next) == SSA_NAME
263 12556 : ? TREE_TYPE (next)
264 : : next, true);
265 : else
266 26381340 : set_reg_attrs_for_decl_rtl (next, x);
267 : }
268 : }
269 :
270 72695814 : if (TREE_CODE (t) == SSA_NAME)
271 : {
272 58962413 : int part = var_to_partition (SA.map, t);
273 58962413 : if (part != NO_PARTITION)
274 : {
275 58962413 : if (SA.partition_to_pseudo[part])
276 32247359 : gcc_assert (SA.partition_to_pseudo[part] == x);
277 26715054 : else if (x != pc_rtx)
278 26711816 : SA.partition_to_pseudo[part] = x;
279 : }
280 : /* For the benefit of debug information at -O0 (where
281 : vartracking doesn't run) record the place also in the base
282 : DECL. For PARMs and RESULTs, do so only when setting the
283 : default def. */
284 117921588 : if (x && x != pc_rtx && SSA_NAME_VAR (t)
285 77315838 : && (VAR_P (SSA_NAME_VAR (t))
286 7554118 : || SSA_NAME_IS_DEFAULT_DEF (t)))
287 : {
288 18173665 : tree var = SSA_NAME_VAR (t);
289 : /* If we don't yet have something recorded, just record it now. */
290 18173665 : if (!DECL_RTL_SET_P (var))
291 7649064 : SET_DECL_RTL (var, x);
292 : /* If we have it set already to "multiple places" don't
293 : change this. */
294 10524601 : else if (DECL_RTL (var) == pc_rtx)
295 : ;
296 : /* If we have something recorded and it's not the same place
297 : as we want to record now, we have multiple partitions for the
298 : same base variable, with different places. We can't just
299 : randomly chose one, hence we have to say that we don't know.
300 : This only happens with optimization, and there var-tracking
301 : will figure out the right thing. */
302 9358248 : else if (DECL_RTL (var) != x)
303 248538 : SET_DECL_RTL (var, pc_rtx);
304 : }
305 : }
306 : else
307 13733401 : SET_DECL_RTL (t, x);
308 72695814 : }
309 :
310 : /* This structure holds data relevant to one variable that will be
311 : placed in a stack slot. */
312 : class stack_var
313 : {
314 : public:
315 : /* The Variable. */
316 : tree decl;
317 :
318 : /* Initially, the size of the variable. Later, the size of the partition,
319 : if this variable becomes it's partition's representative. */
320 : poly_uint64 size;
321 :
322 : /* The *byte* alignment required for this variable. Or as, with the
323 : size, the alignment for this partition. */
324 : unsigned int alignb;
325 :
326 : /* The partition representative. */
327 : unsigned representative;
328 :
329 : /* The next stack variable in the partition, or EOC. */
330 : unsigned next;
331 :
332 : /* The numbers of conflicting stack variables. */
333 : bitmap conflicts;
334 : };
335 :
336 : #define EOC ((unsigned)-1)
337 :
338 : /* We have an array of such objects while deciding allocation. */
339 : static class stack_var *stack_vars;
340 : static unsigned stack_vars_alloc;
341 : static unsigned stack_vars_num;
342 : static hash_map<tree, unsigned> *decl_to_stack_part;
343 :
344 : #define INVALID_STACK_INDEX ((unsigned)-1)
345 :
346 : /* Conflict bitmaps go on this obstack. This allows us to destroy
347 : all of them in one big sweep. */
348 : static bitmap_obstack stack_var_bitmap_obstack;
349 :
350 : /* An array of indices such that stack_vars[stack_vars_sorted[i]].size
351 : is non-decreasing. */
352 : static unsigned *stack_vars_sorted;
353 :
354 : /* The phase of the stack frame. This is the known misalignment of
355 : virtual_stack_vars_rtx from PREFERRED_STACK_BOUNDARY. That is,
356 : (frame_offset+frame_phase) % PREFERRED_STACK_BOUNDARY == 0. */
357 : static int frame_phase;
358 :
359 : /* Used during expand_used_vars to remember if we saw any decls for
360 : which we'd like to enable stack smashing protection. */
361 : static bool has_protected_decls;
362 :
363 : /* Used during expand_used_vars. Remember if we say a character buffer
364 : smaller than our cutoff threshold. Used for -Wstack-protector. */
365 : static bool has_short_buffer;
366 :
367 : /* Compute the byte alignment to use for DECL. Ignore alignment
368 : we can't do with expected alignment of the stack boundary. */
369 :
370 : static unsigned int
371 13464934 : align_local_variable (tree decl, bool really_expand)
372 : {
373 13464934 : unsigned int align;
374 :
375 13464934 : if (TREE_CODE (decl) == SSA_NAME)
376 : {
377 918907 : tree type = TREE_TYPE (decl);
378 918907 : machine_mode mode = TYPE_MODE (type);
379 :
380 918907 : align = TYPE_ALIGN (type);
381 918907 : if (mode != BLKmode
382 918907 : && align < GET_MODE_ALIGNMENT (mode))
383 924 : align = GET_MODE_ALIGNMENT (mode);
384 : }
385 : else
386 12546027 : align = LOCAL_DECL_ALIGNMENT (decl);
387 :
388 13464934 : if (hwassist_sanitize_stack_p ())
389 568 : align = MAX (align, (unsigned) HWASAN_TAG_GRANULE_SIZE * BITS_PER_UNIT);
390 :
391 13464934 : if (TREE_CODE (decl) != SSA_NAME && really_expand)
392 : /* Don't change DECL_ALIGN when called from estimated_stack_frame_size.
393 : That is done before IPA and could bump alignment based on host
394 : backend even for offloaded code which wants different
395 : LOCAL_DECL_ALIGNMENT. */
396 1576798 : SET_DECL_ALIGN (decl, align);
397 :
398 13464934 : return align / BITS_PER_UNIT;
399 : }
400 :
401 : /* Align given offset BASE with ALIGN. Truncate up if ALIGN_UP is true,
402 : down otherwise. Return truncated BASE value. */
403 :
404 : static inline unsigned HOST_WIDE_INT
405 : align_base (HOST_WIDE_INT base, unsigned HOST_WIDE_INT align, bool align_up)
406 : {
407 : return align_up ? (base + align - 1) & -align : base & -align;
408 : }
409 :
410 : /* Allocate SIZE bytes at byte alignment ALIGN from the stack frame.
411 : Return the frame offset. */
412 :
413 : static poly_int64
414 1766024 : alloc_stack_frame_space (poly_int64 size, unsigned HOST_WIDE_INT align)
415 : {
416 1766024 : poly_int64 offset, new_frame_offset;
417 :
418 1766024 : if (FRAME_GROWS_DOWNWARD)
419 : {
420 1766024 : new_frame_offset
421 1766024 : = aligned_lower_bound (frame_offset - frame_phase - size,
422 1766024 : align) + frame_phase;
423 1766024 : offset = new_frame_offset;
424 : }
425 : else
426 : {
427 : new_frame_offset
428 : = aligned_upper_bound (frame_offset - frame_phase,
429 : align) + frame_phase;
430 : offset = new_frame_offset;
431 : new_frame_offset += size;
432 : }
433 1766024 : frame_offset = new_frame_offset;
434 :
435 1766024 : if (frame_offset_overflow (frame_offset, cfun->decl))
436 0 : frame_offset = offset = 0;
437 :
438 1766024 : return offset;
439 : }
440 :
441 : /* Ensure that the stack is aligned to ALIGN bytes.
442 : Return the new frame offset. */
443 : static poly_int64
444 1910 : align_frame_offset (unsigned HOST_WIDE_INT align)
445 : {
446 0 : return alloc_stack_frame_space (0, align);
447 : }
448 :
449 : /* Accumulate DECL into STACK_VARS. */
450 :
451 : static void
452 5851952 : add_stack_var (tree decl, bool really_expand)
453 : {
454 5851952 : class stack_var *v;
455 :
456 5851952 : if (stack_vars_num >= stack_vars_alloc)
457 : {
458 1362096 : if (stack_vars_alloc)
459 34871 : stack_vars_alloc = stack_vars_alloc * 3 / 2;
460 : else
461 : stack_vars_alloc = 32;
462 1362096 : stack_vars
463 1362096 : = XRESIZEVEC (class stack_var, stack_vars, stack_vars_alloc);
464 : }
465 5851952 : if (!decl_to_stack_part)
466 0 : decl_to_stack_part = new hash_map<tree, unsigned>;
467 :
468 5851952 : v = &stack_vars[stack_vars_num];
469 5851952 : decl_to_stack_part->put (decl, stack_vars_num);
470 :
471 5851952 : v->decl = decl;
472 5851952 : tree size = TREE_CODE (decl) == SSA_NAME
473 5851952 : ? TYPE_SIZE_UNIT (TREE_TYPE (decl))
474 5851952 : : DECL_SIZE_UNIT (decl);
475 5851952 : v->size = tree_to_poly_uint64 (size);
476 : /* Ensure that all variables have size, so that &a != &b for any two
477 : variables that are simultaneously live. */
478 5851952 : if (known_eq (v->size, 0U))
479 24515 : v->size = 1;
480 5851952 : v->alignb = align_local_variable (decl, really_expand);
481 : /* An alignment of zero can mightily confuse us later. */
482 5851952 : gcc_assert (v->alignb != 0);
483 :
484 : /* All variables are initially in their own partition. */
485 5851952 : v->representative = stack_vars_num;
486 5851952 : v->next = EOC;
487 :
488 : /* All variables initially conflict with no other. */
489 5851952 : v->conflicts = NULL;
490 :
491 : /* Ensure that this decl doesn't get put onto the list twice. */
492 5851952 : set_rtl (decl, pc_rtx);
493 :
494 5851952 : stack_vars_num++;
495 5851952 : }
496 :
497 : /* Make the decls associated with luid's X and Y conflict. */
498 :
499 : static void
500 11710625 : add_stack_var_conflict (unsigned x, unsigned y)
501 : {
502 11710625 : class stack_var *a = &stack_vars[x];
503 11710625 : class stack_var *b = &stack_vars[y];
504 11710625 : if (x == y)
505 : return;
506 10621131 : if (!a->conflicts)
507 673063 : a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
508 10621131 : if (!b->conflicts)
509 121301 : b->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
510 10621131 : bitmap_set_bit (a->conflicts, y);
511 10621131 : bitmap_set_bit (b->conflicts, x);
512 : }
513 :
514 : /* Check whether the decls associated with luid's X and Y conflict. */
515 :
516 : static bool
517 11558681 : stack_var_conflict_p (unsigned x, unsigned y)
518 : {
519 11558681 : class stack_var *a = &stack_vars[x];
520 11558681 : class stack_var *b = &stack_vars[y];
521 11558681 : if (x == y)
522 : return false;
523 : /* Partitions containing an SSA name result from gimple registers
524 : with things like unsupported modes. They are top-level and
525 : hence conflict with everything else. */
526 11558681 : if (TREE_CODE (a->decl) == SSA_NAME || TREE_CODE (b->decl) == SSA_NAME)
527 : return true;
528 :
529 11523343 : if (!a->conflicts || !b->conflicts)
530 : return false;
531 11472267 : return bitmap_bit_p (a->conflicts, y);
532 : }
533 :
534 : /* Returns the DECL's index into the stack_vars array.
535 : If the DECL does not exist return INVALID_STACK_INDEX. */
536 : static unsigned
537 47612605 : decl_stack_index (tree decl)
538 : {
539 47612605 : if (!decl)
540 : return INVALID_STACK_INDEX;
541 47612605 : if (!DECL_P (decl))
542 : return INVALID_STACK_INDEX;
543 36557704 : if (DECL_RTL_IF_SET (decl) != pc_rtx)
544 : return INVALID_STACK_INDEX;
545 24599383 : unsigned *v = decl_to_stack_part->get (decl);
546 24599383 : if (!v)
547 : return INVALID_STACK_INDEX;
548 :
549 24599383 : unsigned indx = *v;
550 24599383 : gcc_checking_assert (indx != INVALID_STACK_INDEX);
551 24599383 : gcc_checking_assert (indx < stack_vars_num);
552 : return indx;
553 : }
554 :
555 : /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
556 : enter its partition number into bitmap DATA. */
557 :
558 : static bool
559 30103076 : visit_op (gimple *, tree op, tree, void *data)
560 : {
561 30103076 : bitmap active = (bitmap)data;
562 30103076 : op = get_base_address (op);
563 30103076 : unsigned idx = decl_stack_index (op);
564 30103076 : if (idx != INVALID_STACK_INDEX)
565 14856550 : bitmap_set_bit (active, idx);
566 30103076 : return false;
567 : }
568 :
569 : /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
570 : record conflicts between it and all currently active other partitions
571 : from bitmap DATA. */
572 :
573 : static bool
574 13944119 : visit_conflict (gimple *, tree op, tree, void *data)
575 : {
576 13944119 : bitmap active = (bitmap)data;
577 13944119 : op = get_base_address (op);
578 13944119 : unsigned num = decl_stack_index (op);
579 13944119 : if (num != INVALID_STACK_INDEX
580 13944119 : && bitmap_set_bit (active, num))
581 : {
582 1009003 : bitmap_iterator bi;
583 1009003 : unsigned i;
584 1009003 : gcc_assert (num < stack_vars_num);
585 10929059 : EXECUTE_IF_SET_IN_BITMAP (active, 0, i, bi)
586 9920056 : add_stack_var_conflict (num, i);
587 : }
588 13944119 : return false;
589 : }
590 :
591 : /* A cache for ssa name to address of stack variables.
592 : When taking into account if a ssa name refers to an
593 : address of a stack variable, we need to walk the
594 : expressions backwards to find the addresses. This
595 : cache is there so we don't need to walk the expressions
596 : all the time. */
597 : struct vars_ssa_cache
598 : {
599 : private:
600 : /* Currently an entry is a bitmap of all of the known stack variables
601 : addresses that are referenced by the ssa name.
602 : When the bitmap is the nullptr, then there is no cache.
603 : Currently only empty bitmaps are shared.
604 : The reason for why empty cache is not just a null is so we know the
605 : cache for an entry is filled in. */
606 : struct entry
607 : {
608 : bitmap bmap = nullptr;
609 : };
610 : entry *vars_ssa_caches;
611 : public:
612 :
613 : vars_ssa_cache();
614 : ~vars_ssa_cache();
615 : const_bitmap operator() (tree name);
616 : void dump (FILE *file);
617 :
618 : private:
619 : /* Can't copy. */
620 : vars_ssa_cache(const vars_ssa_cache&) = delete;
621 : vars_ssa_cache(vars_ssa_cache&&) = delete;
622 :
623 : /* The shared empty bitmap. */
624 : bitmap empty;
625 :
626 : /* Unshare the index, currently only need
627 : to unshare if the entry was empty. */
628 1107831 : void unshare(int indx)
629 : {
630 1107831 : if (vars_ssa_caches[indx].bmap == empty)
631 475796 : vars_ssa_caches[indx].bmap = BITMAP_ALLOC (&stack_var_bitmap_obstack);
632 1107831 : }
633 : void create (tree);
634 : bool exists (tree use);
635 : void add_one (tree old_name, unsigned);
636 : bool update (tree old_name, tree use);
637 : };
638 :
639 : /* Constructor of the cache, create the cache array. */
640 149743 : vars_ssa_cache::vars_ssa_cache ()
641 : {
642 33249107 : vars_ssa_caches = new entry[num_ssa_names]{};
643 :
644 : /* Create the shared empty bitmap too. */
645 149743 : empty = BITMAP_ALLOC (&stack_var_bitmap_obstack);
646 149743 : }
647 :
648 : /* Delete the array. The bitmaps will be freed
649 : when stack_var_bitmap_obstack is freed. */
650 149743 : vars_ssa_cache::~vars_ssa_cache ()
651 : {
652 149743 : delete []vars_ssa_caches;
653 149743 : }
654 :
655 : /* Create an empty entry for the USE ssa name. */
656 : void
657 10990337 : vars_ssa_cache::create (tree use)
658 : {
659 10990337 : int num = SSA_NAME_VERSION (use);
660 10990337 : if (vars_ssa_caches[num].bmap)
661 : return;
662 10990337 : vars_ssa_caches[num].bmap = empty;
663 : }
664 :
665 : /* Returns true if the cache for USE exists. */
666 : bool
667 71898247 : vars_ssa_cache::exists (tree use)
668 : {
669 71898247 : int num = SSA_NAME_VERSION (use);
670 71898247 : return vars_ssa_caches[num].bmap != nullptr;
671 : }
672 :
673 : /* Add to USE's bitmap for stack variable IDX. */
674 : void
675 164410 : vars_ssa_cache::add_one (tree use, unsigned idx)
676 : {
677 164410 : gcc_assert (idx != INVALID_STACK_INDEX);
678 164410 : int num = SSA_NAME_VERSION (use);
679 164410 : gcc_assert (vars_ssa_caches[num].bmap);
680 164410 : unshare (num);
681 164410 : bitmap_set_bit (vars_ssa_caches[num].bmap, idx);
682 164410 : }
683 :
684 : /* Update cache of OLD_NAME from the USE's cache. */
685 : bool
686 25352441 : vars_ssa_cache::update (tree old_name, tree use)
687 : {
688 25352441 : if (old_name == use)
689 : return false;
690 16194167 : int num = SSA_NAME_VERSION (use);
691 16194167 : int old_num = SSA_NAME_VERSION (old_name);
692 :
693 : /* If the old name was empty, then there is nothing to be updated. */
694 16194167 : if (vars_ssa_caches[num].bmap == empty)
695 : return false;
696 943421 : unshare (old_num);
697 943421 : return bitmap_ior_into (vars_ssa_caches[old_num].bmap, vars_ssa_caches[num].bmap);
698 : }
699 :
700 : /* Dump out the cache. Note empty and non-filled
701 : in ssa names are not printed out. */
702 : void
703 0 : vars_ssa_cache::dump (FILE *file)
704 : {
705 0 : fprintf (file, "var ssa address cache\n");
706 0 : for (unsigned num = 0; num < num_ssa_names; num++)
707 : {
708 0 : if (!vars_ssa_caches[num].bmap
709 0 : || vars_ssa_caches[num].bmap == empty)
710 0 : continue;
711 0 : fprintf (file, "_%d refers to:\n", num);
712 0 : bitmap_iterator bi;
713 0 : unsigned i;
714 0 : EXECUTE_IF_SET_IN_BITMAP (vars_ssa_caches[num].bmap, 0, i, bi)
715 : {
716 0 : fputc ('\t', file);
717 0 : print_generic_expr (file, stack_vars[i].decl, dump_flags);
718 : }
719 0 : fputc ('\n', file);
720 : }
721 0 : fputc ('\n', file);
722 0 : }
723 :
724 : /* Returns the filled in cache for NAME.
725 : This will fill in the cache if it does not exist already.
726 : Returns an empty for ssa names that can't contain pointers
727 : (only integral types and pointer types will contain pointers). */
728 :
729 : const_bitmap
730 56356229 : vars_ssa_cache::operator() (tree name)
731 : {
732 56356229 : gcc_assert (TREE_CODE (name) == SSA_NAME);
733 :
734 92728590 : if (!POINTER_TYPE_P (TREE_TYPE (name))
735 91558559 : && !ANY_INTEGRAL_TYPE_P (TREE_TYPE (name)))
736 2502103 : return empty;
737 :
738 53854126 : if (exists (name))
739 44749098 : return vars_ssa_caches[SSA_NAME_VERSION (name)].bmap;
740 :
741 9105028 : auto_vec<std::pair<tree,tree>, 4> work_list;
742 9105028 : auto_vec<std::pair<tree,tree>, 4> update_cache_list;
743 :
744 9105028 : work_list.safe_push (std::make_pair (name, name));
745 :
746 33542002 : while (!work_list.is_empty ())
747 : {
748 24436974 : auto item = work_list.pop();
749 24436974 : tree use = item.first;
750 24436974 : tree old_name = item.second;
751 24436974 : if (TREE_CODE (use) == ADDR_EXPR)
752 : {
753 417031 : tree op = TREE_OPERAND (use, 0);
754 417031 : op = get_base_address (op);
755 417031 : unsigned idx = decl_stack_index (op);
756 417031 : if (idx != INVALID_STACK_INDEX)
757 164410 : add_one (old_name, idx);
758 13446637 : continue;
759 417031 : }
760 :
761 24019943 : if (TREE_CODE (use) != SSA_NAME)
762 5897334 : continue;
763 :
764 31601204 : if (!POINTER_TYPE_P (TREE_TYPE (use))
765 31467383 : && !ANY_INTEGRAL_TYPE_P (TREE_TYPE (use)))
766 78488 : continue;
767 :
768 : /* Mark the old ssa name needs to be update from the use. */
769 18044121 : update_cache_list.safe_push (item);
770 :
771 : /* If the cache exists for the use, don't try to recreate it. */
772 18044121 : if (exists (use))
773 : {
774 : /* Update the cache here, this can reduce the number of
775 : times through the update loop below. */
776 7053784 : update (old_name, use);
777 7053784 : continue;
778 : }
779 :
780 : /* Create the cache bitmap for the use and also
781 : so we don't go into an infinite loop for some phi nodes with loops. */
782 10990337 : create (use);
783 :
784 10990337 : gimple *g = SSA_NAME_DEF_STMT (use);
785 :
786 : /* CONSTRUCTOR here is always a vector initialization,
787 : walk each element too. */
788 10990337 : if (gimple_assign_single_p (g)
789 10990337 : && TREE_CODE (gimple_assign_rhs1 (g)) == CONSTRUCTOR)
790 : {
791 : tree ctr = gimple_assign_rhs1 (g);
792 : unsigned i;
793 : tree elm;
794 314745 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (ctr), i, elm)
795 221191 : work_list.safe_push (std::make_pair (elm, use));
796 : }
797 : /* For assignments, walk each operand for possible addresses.
798 : For PHI nodes, walk each argument. */
799 10896783 : else if (gassign *a = dyn_cast <gassign *> (g))
800 : {
801 : /* operand 0 is the lhs. */
802 19443359 : for (unsigned i = 1; i < gimple_num_ops (g); i++)
803 11229319 : work_list.safe_push (std::make_pair (gimple_op (a, i), use));
804 : }
805 12564532 : else if (gphi *p = dyn_cast <gphi *> (g))
806 5455631 : for (unsigned i = 0; i < gimple_phi_num_args (p); ++i)
807 3881436 : work_list.safe_push (std::make_pair (gimple_phi_arg_def (p, i), use));
808 : }
809 :
810 : /* Update the cache. Note a loop is needed as phi nodes could
811 : cause a loop to form. The number of times through this loop
812 : will be small though. */
813 9158218 : bool changed;
814 18316436 : do {
815 9158218 : changed = false;
816 9158218 : unsigned int i;
817 9158218 : std::pair<tree,tree> *e;
818 45720121 : FOR_EACH_VEC_ELT_REVERSE (update_cache_list, i, e)
819 : {
820 18298657 : if (update (e->second, e->first))
821 59685 : changed = true;
822 : }
823 : } while (changed);
824 :
825 9105028 : return vars_ssa_caches[SSA_NAME_VERSION (name)].bmap;
826 9105028 : }
827 :
828 : /* Helper function for add_scope_conflicts_1. For USE on
829 : a stmt, if it is a SSA_NAME and in its defining statement
830 : is known to be based on some ADDR_EXPR, invoke VISIT
831 : on that ADDR_EXPR. */
832 :
833 : static inline void
834 56356229 : add_scope_conflicts_2 (vars_ssa_cache &cache, tree name,
835 : bitmap work, walk_stmt_load_store_addr_fn visit)
836 : {
837 56356229 : gcc_assert (TREE_CODE (name) == SSA_NAME);
838 :
839 : /* Query the cache for the mapping of addresses that are referenced by
840 : ssa name NAME. Querying it will fill in it. */
841 56356229 : bitmap_iterator bi;
842 56356229 : unsigned i;
843 56356229 : const_bitmap bmap = cache (name);
844 : /* Visit each stack variable that is referenced. */
845 58767850 : EXECUTE_IF_SET_IN_BITMAP (bmap, 0, i, bi)
846 2411621 : visit (nullptr, stack_vars[i].decl, nullptr, work);
847 56356229 : }
848 :
849 : /* Helper routine for add_scope_conflicts, calculating the active partitions
850 : at the end of BB, leaving the result in WORK. We're called to generate
851 : conflicts when FOR_CONFLICT is true, otherwise we're just tracking
852 : liveness. */
853 :
854 : static void
855 16463613 : add_scope_conflicts_1 (vars_ssa_cache &cache, basic_block bb, bitmap work, bool for_conflict)
856 : {
857 16463613 : edge e;
858 16463613 : edge_iterator ei;
859 16463613 : gimple_stmt_iterator gsi;
860 16463613 : walk_stmt_load_store_addr_fn visit;
861 16463613 : use_operand_p use_p;
862 16463613 : ssa_op_iter iter;
863 16463613 : bool had_non_clobbers = false;
864 :
865 16463613 : bitmap_clear (work);
866 : /* The copy what was alive out going from the edges. */
867 41602893 : FOR_EACH_EDGE (e, ei, bb->preds)
868 25139280 : bitmap_ior_into (work, (bitmap)e->src->aux);
869 :
870 16463613 : visit = for_conflict ? visit_conflict : visit_op;
871 :
872 : /* Addresses coming into the bb via phis are alive at the entry point. */
873 21591835 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
874 5128222 : add_scope_conflicts_2 (cache, gimple_phi_result (gsi_stmt (gsi)), work, visit);
875 :
876 189120182 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
877 : {
878 172656569 : gimple *stmt = gsi_stmt (gsi);
879 :
880 : /* Debug statements are not considered for liveness. */
881 172656569 : if (is_gimple_debug (stmt))
882 108730146 : continue;
883 :
884 : /* If we had `var = {CLOBBER}`, then var is no longer
885 : considered alive after this point but might become
886 : alive later on. */
887 63926423 : if (gimple_clobber_p (stmt))
888 : {
889 3227961 : tree lhs = gimple_assign_lhs (stmt);
890 : /* Handle only plain var clobbers, not partial ones.
891 : Nested functions lowering and C++ front-end inserts clobbers
892 : which are partial clobbers. */
893 3227961 : if (!VAR_P (lhs))
894 79582 : continue;
895 3148379 : unsigned indx = decl_stack_index (lhs);
896 3148379 : if (indx != INVALID_STACK_INDEX)
897 2590029 : bitmap_clear_bit (work, indx);
898 : }
899 : else
900 : {
901 60698462 : if (for_conflict && !had_non_clobbers)
902 : {
903 : /* When we are inheriting live variables from our predecessors
904 : through a CFG merge we might not see an actual mention of
905 : the variables to record the appropriate conflict as defs/uses
906 : might be through indirect stores/loads. For this reason
907 : we have to make sure each live variable conflicts with
908 : each other. When there's just a single predecessor the
909 : set of conflicts is already up-to-date.
910 : We perform this delayed at the first real instruction to
911 : allow clobbers starting this block to remove variables from
912 : the set of live variables. */
913 4969938 : bitmap_iterator bi;
914 4969938 : unsigned i;
915 4969938 : if (EDGE_COUNT (bb->preds) > 1)
916 12605495 : EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
917 : {
918 11157942 : class stack_var *a = &stack_vars[i];
919 11157942 : if (!a->conflicts)
920 254049 : a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
921 11157942 : bitmap_ior_into (a->conflicts, work);
922 : }
923 4969938 : had_non_clobbers = true;
924 : }
925 60698462 : walk_stmt_load_store_addr_ops (stmt, work, visit, visit, visit);
926 111926469 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
927 51228007 : add_scope_conflicts_2 (cache, USE_FROM_PTR (use_p), work, visit);
928 : }
929 : }
930 :
931 : /* When there was no real instruction but there's a CFG merge we need
932 : to add the conflicts now. */
933 16463613 : if (for_conflict && !had_non_clobbers && EDGE_COUNT (bb->preds) > 1)
934 : {
935 129824 : bitmap_iterator bi;
936 129824 : unsigned i;
937 943343 : EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
938 : {
939 813519 : class stack_var *a = &stack_vars[i];
940 813519 : if (!a->conflicts)
941 22506 : a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
942 813519 : bitmap_ior_into (a->conflicts, work);
943 : }
944 : }
945 16463613 : }
946 :
947 : /* Generate stack partition conflicts between all partitions that are
948 : simultaneously live. */
949 :
950 : static void
951 236836 : add_scope_conflicts (void)
952 : {
953 : /* If there is only one variable, there is nothing to be done as
954 : there is only possible partition. */
955 236836 : if (stack_vars_num == 1)
956 87093 : return;
957 :
958 149743 : basic_block bb;
959 149743 : bool changed;
960 149743 : bitmap work = BITMAP_ALLOC (NULL);
961 149743 : int *rpo;
962 149743 : int n_bbs;
963 :
964 149743 : vars_ssa_cache cache;
965 :
966 : /* We approximate the live range of a stack variable by taking the first
967 : mention of its name as starting point(s), and by the end-of-scope
968 : death clobber added by gimplify as ending point(s) of the range.
969 : This overapproximates in the case we for instance moved an address-taken
970 : operation upward, without also moving a dereference to it upwards.
971 : But it's conservatively correct as a variable never can hold values
972 : before its name is mentioned at least once.
973 :
974 : We then do a mostly classical bitmap liveness algorithm. */
975 :
976 5624003 : FOR_ALL_BB_FN (bb, cfun)
977 5474260 : bb->aux = BITMAP_ALLOC (&stack_var_bitmap_obstack);
978 :
979 149743 : rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
980 149743 : n_bbs = pre_and_rev_post_order_compute (NULL, rpo, false);
981 :
982 149743 : changed = true;
983 605295 : while (changed)
984 : {
985 : int i;
986 : changed = false;
987 11594648 : for (i = 0; i < n_bbs; i++)
988 : {
989 11288839 : bitmap active;
990 11288839 : bb = BASIC_BLOCK_FOR_FN (cfun, rpo[i]);
991 11288839 : active = (bitmap)bb->aux;
992 11288839 : add_scope_conflicts_1 (cache, bb, work, false);
993 11288839 : if (bitmap_ior_into (active, work))
994 4647572 : changed = true;
995 : }
996 : }
997 :
998 5324517 : FOR_EACH_BB_FN (bb, cfun)
999 5174774 : add_scope_conflicts_1 (cache, bb, work, true);
1000 :
1001 149743 : if (dump_file && (dump_flags & TDF_DETAILS))
1002 0 : cache.dump (dump_file);
1003 :
1004 149743 : free (rpo);
1005 149743 : BITMAP_FREE (work);
1006 5624003 : FOR_ALL_BB_FN (bb, cfun)
1007 5474260 : BITMAP_FREE (bb->aux);
1008 149743 : }
1009 :
1010 : /* A subroutine of partition_stack_vars. A comparison function for qsort,
1011 : sorting an array of indices by the properties of the object. */
1012 :
1013 : static int
1014 20406680 : stack_var_cmp (const void *a, const void *b)
1015 : {
1016 20406680 : unsigned ia = *(const unsigned *)a;
1017 20406680 : unsigned ib = *(const unsigned *)b;
1018 20406680 : unsigned int aligna = stack_vars[ia].alignb;
1019 20406680 : unsigned int alignb = stack_vars[ib].alignb;
1020 20406680 : poly_int64 sizea = stack_vars[ia].size;
1021 20406680 : poly_int64 sizeb = stack_vars[ib].size;
1022 20406680 : tree decla = stack_vars[ia].decl;
1023 20406680 : tree declb = stack_vars[ib].decl;
1024 20406680 : bool largea, largeb;
1025 20406680 : unsigned int uida, uidb;
1026 :
1027 : /* Primary compare on "large" alignment. Large comes first. */
1028 20406680 : largea = (aligna * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
1029 20406680 : largeb = (alignb * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
1030 20406680 : if (largea != largeb)
1031 0 : return (int)largeb - (int)largea;
1032 :
1033 : /* Secondary compare on size, decreasing */
1034 36752293 : int diff = compare_sizes_for_sort (sizeb, sizea);
1035 4715073 : if (diff != 0)
1036 : return diff;
1037 :
1038 : /* Tertiary compare on true alignment, decreasing. */
1039 11630540 : if (aligna < alignb)
1040 : return -1;
1041 11325039 : if (aligna > alignb)
1042 : return 1;
1043 :
1044 : /* Final compare on ID for sort stability, increasing.
1045 : Two SSA names are compared by their version, SSA names come before
1046 : non-SSA names, and two normal decls are compared by their DECL_UID. */
1047 11075940 : if (TREE_CODE (decla) == SSA_NAME)
1048 : {
1049 37938 : if (TREE_CODE (declb) == SSA_NAME)
1050 33214 : uida = SSA_NAME_VERSION (decla), uidb = SSA_NAME_VERSION (declb);
1051 : else
1052 : return -1;
1053 : }
1054 11038002 : else if (TREE_CODE (declb) == SSA_NAME)
1055 : return 1;
1056 : else
1057 11032265 : uida = DECL_UID (decla), uidb = DECL_UID (declb);
1058 11065479 : if (uida < uidb)
1059 : return 1;
1060 5507928 : if (uida > uidb)
1061 5507928 : return -1;
1062 : return 0;
1063 : }
1064 :
1065 : struct part_traits : unbounded_int_hashmap_traits <unsigned , bitmap> {};
1066 : typedef hash_map<unsigned, bitmap, part_traits> part_hashmap;
1067 :
1068 : /* If the points-to solution *PI points to variables that are in a partition
1069 : together with other variables add all partition members to the pointed-to
1070 : variables bitmap. */
1071 :
1072 : static void
1073 1626949 : add_partitioned_vars_to_ptset (const pt_solution *pt,
1074 : part_hashmap *decls_to_partitions,
1075 : hash_set<bitmap> *visited, bitmap temp)
1076 : {
1077 1626949 : bitmap_iterator bi;
1078 1626949 : unsigned i;
1079 1626949 : bitmap *part;
1080 :
1081 1626949 : if (pt->anything
1082 1582549 : || pt->vars == NULL
1083 : /* The pointed-to vars bitmap is shared, it is enough to
1084 : visit it once. */
1085 3209498 : || visited->add (pt->vars))
1086 1314696 : return;
1087 :
1088 312253 : bitmap_clear (temp);
1089 :
1090 : /* By using a temporary bitmap to store all members of the partitions
1091 : we have to add we make sure to visit each of the partitions only
1092 : once. */
1093 1343529 : EXECUTE_IF_SET_IN_BITMAP (pt->vars, 0, i, bi)
1094 1031276 : if ((!temp
1095 1031276 : || !bitmap_bit_p (temp, i))
1096 1671903 : && (part = decls_to_partitions->get (i)))
1097 176670 : bitmap_ior_into (temp, *part);
1098 312253 : if (!bitmap_empty_p (temp))
1099 149536 : bitmap_ior_into (pt->vars, temp);
1100 : }
1101 :
1102 : /* Update points-to sets based on partition info, so we can use them on RTL.
1103 : The bitmaps representing stack partitions will be saved until expand,
1104 : where partitioned decls used as bases in memory expressions will be
1105 : rewritten.
1106 :
1107 : It is not necessary to update TBAA info on accesses to the coalesced
1108 : storage since our memory model doesn't allow TBAA to be used for
1109 : WAW or WAR dependences. For RAW when the write is to an old object
1110 : the new object would not have been initialized at the point of the
1111 : read, invoking undefined behavior. */
1112 :
1113 : static void
1114 149743 : update_alias_info_with_stack_vars (void)
1115 : {
1116 149743 : part_hashmap *decls_to_partitions = NULL;
1117 149743 : unsigned i, j;
1118 149743 : tree var = NULL_TREE;
1119 :
1120 1280937 : for (i = 0; i < stack_vars_num; i++)
1121 : {
1122 1131194 : bitmap part = NULL;
1123 1131194 : tree name;
1124 :
1125 : /* Not interested in partitions with single variable. */
1126 1131194 : if (stack_vars[i].representative != i
1127 856955 : || stack_vars[i].next == EOC)
1128 1058814 : continue;
1129 :
1130 72380 : if (!decls_to_partitions)
1131 : {
1132 43169 : decls_to_partitions = new part_hashmap;
1133 43169 : cfun->gimple_df->decls_to_pointers = new hash_map<tree, tree>;
1134 : }
1135 :
1136 : /* Create an SSA_NAME that points to the partition for use
1137 : as base during alias-oracle queries on RTL for bases that
1138 : have been partitioned. */
1139 72380 : if (var == NULL_TREE)
1140 43169 : var = create_tmp_var (ptr_type_node);
1141 72380 : name = make_ssa_name (var);
1142 :
1143 : /* Create bitmaps representing partitions. They will be used for
1144 : points-to sets later, so use GGC alloc. */
1145 72380 : part = BITMAP_GGC_ALLOC ();
1146 418999 : for (j = i; j != EOC; j = stack_vars[j].next)
1147 : {
1148 346619 : tree decl = stack_vars[j].decl;
1149 346619 : unsigned int uid = DECL_PT_UID (decl);
1150 346619 : bitmap_set_bit (part, uid);
1151 346619 : decls_to_partitions->put (uid, part);
1152 346619 : cfun->gimple_df->decls_to_pointers->put (decl, name);
1153 346619 : if (TREE_ADDRESSABLE (decl))
1154 316884 : TREE_ADDRESSABLE (name) = 1;
1155 : }
1156 :
1157 : /* Make the SSA name point to all partition members. */
1158 72380 : pt_solution pt;
1159 72380 : pt_solution_set (&pt, part, false);
1160 72380 : set_ssa_name_pt_solution (name, pt);
1161 : }
1162 :
1163 : /* Make all points-to sets that contain one member of a partition
1164 : contain all members of the partition. */
1165 149743 : if (decls_to_partitions)
1166 : {
1167 43169 : unsigned i;
1168 43169 : tree name;
1169 43169 : hash_set<bitmap> visited;
1170 43169 : bitmap temp = BITMAP_ALLOC (&stack_var_bitmap_obstack);
1171 :
1172 17079070 : FOR_EACH_SSA_NAME (i, name, cfun)
1173 : {
1174 27516528 : const pt_solution *pt = ssa_name_pt_solution (name);
1175 :
1176 1549678 : if (pt)
1177 1540611 : add_partitioned_vars_to_ptset (pt, decls_to_partitions,
1178 : &visited, temp);
1179 : }
1180 :
1181 43169 : add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped,
1182 : decls_to_partitions, &visited, temp);
1183 43169 : add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped_return,
1184 : decls_to_partitions, &visited, temp);
1185 43169 : delete decls_to_partitions;
1186 43169 : BITMAP_FREE (temp);
1187 43169 : }
1188 149743 : }
1189 :
1190 : /* A subroutine of partition_stack_vars. The UNION portion of a UNION/FIND
1191 : partitioning algorithm. Partitions A and B are known to be non-conflicting.
1192 : Merge them into a single partition A. */
1193 :
1194 : static void
1195 274239 : union_stack_vars (unsigned a, unsigned b)
1196 : {
1197 274239 : class stack_var *vb = &stack_vars[b];
1198 274239 : bitmap_iterator bi;
1199 274239 : unsigned u;
1200 :
1201 274239 : gcc_assert (stack_vars[b].next == EOC);
1202 : /* Add B to A's partition. */
1203 274239 : stack_vars[b].next = stack_vars[a].next;
1204 274239 : stack_vars[b].representative = a;
1205 274239 : stack_vars[a].next = b;
1206 :
1207 : /* Make sure A is big enough to hold B. */
1208 274239 : stack_vars[a].size = upper_bound (stack_vars[a].size, stack_vars[b].size);
1209 :
1210 : /* Update the required alignment of partition A to account for B. */
1211 274239 : if (stack_vars[a].alignb < stack_vars[b].alignb)
1212 1813 : stack_vars[a].alignb = stack_vars[b].alignb;
1213 :
1214 : /* Update the interference graph and merge the conflicts. */
1215 274239 : if (vb->conflicts)
1216 : {
1217 2015232 : EXECUTE_IF_SET_IN_BITMAP (vb->conflicts, 0, u, bi)
1218 1790240 : add_stack_var_conflict (a, stack_vars[u].representative);
1219 224992 : BITMAP_FREE (vb->conflicts);
1220 : }
1221 274239 : }
1222 :
1223 : /* A subroutine of expand_used_vars. Binpack the variables into
1224 : partitions constrained by the interference graph. The overall
1225 : algorithm used is as follows:
1226 :
1227 : Sort the objects by size in descending order.
1228 : For each object A {
1229 : S = size(A)
1230 : O = 0
1231 : loop {
1232 : Look for the largest non-conflicting object B with size <= S.
1233 : UNION (A, B)
1234 : }
1235 : }
1236 : */
1237 :
1238 : static void
1239 236836 : partition_stack_vars (void)
1240 : {
1241 236836 : unsigned si, sj, n = stack_vars_num;
1242 :
1243 236836 : stack_vars_sorted = XNEWVEC (unsigned, stack_vars_num);
1244 1455123 : for (si = 0; si < n; ++si)
1245 1218287 : stack_vars_sorted[si] = si;
1246 :
1247 236836 : if (n == 1)
1248 : return;
1249 :
1250 149743 : qsort (stack_vars_sorted, n, sizeof (unsigned), stack_var_cmp);
1251 :
1252 1430680 : for (si = 0; si < n; ++si)
1253 : {
1254 1131194 : unsigned i = stack_vars_sorted[si];
1255 1131194 : unsigned int ialign = stack_vars[i].alignb;
1256 1131194 : poly_int64 isize = stack_vars[i].size;
1257 :
1258 : /* Ignore objects that aren't partition representatives. If we
1259 : see a var that is not a partition representative, it must
1260 : have been merged earlier. */
1261 1131194 : if (stack_vars[i].representative != i)
1262 274239 : continue;
1263 :
1264 13169656 : for (sj = si + 1; sj < n; ++sj)
1265 : {
1266 12313948 : unsigned j = stack_vars_sorted[sj];
1267 12313948 : unsigned int jalign = stack_vars[j].alignb;
1268 12313948 : poly_int64 jsize = stack_vars[j].size;
1269 :
1270 : /* Ignore objects that aren't partition representatives. */
1271 12313948 : if (stack_vars[j].representative != j)
1272 12312701 : continue;
1273 :
1274 : /* Do not mix objects of "small" (supported) alignment
1275 : and "large" (unsupported) alignment. */
1276 11559928 : if ((ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1277 11559928 : != (jalign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT))
1278 : break;
1279 :
1280 : /* For Address Sanitizer do not mix objects with different
1281 : sizes, as the shorter vars wouldn't be adequately protected.
1282 : Don't do that for "large" (unsupported) alignment objects,
1283 : those aren't protected anyway. */
1284 11559928 : if (asan_sanitize_stack_p ()
1285 6753 : && maybe_ne (isize, jsize)
1286 11561175 : && ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1287 : break;
1288 :
1289 : /* Ignore conflicting objects. */
1290 11558681 : if (stack_var_conflict_p (i, j))
1291 11284442 : continue;
1292 :
1293 : /* UNION the objects, placing J at OFFSET. */
1294 274239 : union_stack_vars (i, j);
1295 : }
1296 : }
1297 :
1298 149743 : update_alias_info_with_stack_vars ();
1299 : }
1300 :
1301 : /* A debugging aid for expand_used_vars. Dump the generated partitions. */
1302 :
1303 : static void
1304 41 : dump_stack_var_partition (void)
1305 : {
1306 41 : unsigned si, i, j, n = stack_vars_num;
1307 :
1308 95 : for (si = 0; si < n; ++si)
1309 : {
1310 54 : i = stack_vars_sorted[si];
1311 :
1312 : /* Skip variables that aren't partition representatives, for now. */
1313 54 : if (stack_vars[i].representative != i)
1314 3 : continue;
1315 :
1316 51 : fprintf (dump_file, "Partition %u: size ", i);
1317 51 : print_dec (stack_vars[i].size, dump_file);
1318 51 : fprintf (dump_file, " align %u\n", stack_vars[i].alignb);
1319 :
1320 156 : for (j = i; j != EOC; j = stack_vars[j].next)
1321 : {
1322 54 : fputc ('\t', dump_file);
1323 54 : print_generic_expr (dump_file, stack_vars[j].decl, dump_flags);
1324 : }
1325 51 : fputc ('\n', dump_file);
1326 : }
1327 41 : }
1328 :
1329 : /* Assign rtl to DECL at BASE + OFFSET. */
1330 :
1331 : static void
1332 2036615 : expand_one_stack_var_at (tree decl, rtx base, unsigned base_align,
1333 : poly_int64 offset)
1334 : {
1335 2036615 : unsigned align;
1336 2036615 : rtx x;
1337 :
1338 : /* If this fails, we've overflowed the stack frame. Error nicely? */
1339 2312415 : gcc_assert (known_eq (offset, trunc_int_for_mode (offset, Pmode)));
1340 :
1341 2036615 : if (hwassist_sanitize_stack_p ())
1342 86 : x = targetm.memtag.add_tag (base, offset,
1343 : hwasan_current_frame_tag ());
1344 : else
1345 2312329 : x = plus_constant (Pmode, base, offset);
1346 :
1347 4073230 : x = gen_rtx_MEM (TREE_CODE (decl) == SSA_NAME
1348 459817 : ? TYPE_MODE (TREE_TYPE (decl))
1349 1576798 : : DECL_MODE (decl), x);
1350 :
1351 : /* Set alignment we actually gave this decl if it isn't an SSA name.
1352 : If it is we generate stack slots only accidentally so it isn't as
1353 : important, we'll simply set the alignment directly on the MEM. */
1354 :
1355 2036615 : if (stack_vars_base_reg_p (base))
1356 2033223 : offset -= frame_phase;
1357 2036615 : align = known_alignment (offset);
1358 2036615 : align *= BITS_PER_UNIT;
1359 2036615 : if (align == 0 || align > base_align)
1360 827457 : align = base_align;
1361 :
1362 2036615 : if (TREE_CODE (decl) != SSA_NAME)
1363 : {
1364 : /* One would think that we could assert that we're not decreasing
1365 : alignment here, but (at least) the i386 port does exactly this
1366 : via the MINIMUM_ALIGNMENT hook. */
1367 :
1368 1576798 : SET_DECL_ALIGN (decl, align);
1369 1576798 : DECL_USER_ALIGN (decl) = 0;
1370 : }
1371 :
1372 2036615 : set_rtl (decl, x);
1373 :
1374 2036615 : set_mem_align (x, align);
1375 2036615 : }
1376 :
1377 236836 : class stack_vars_data
1378 : {
1379 : public:
1380 : /* Vector of offset pairs, always end of some padding followed
1381 : by start of the padding that needs Address Sanitizer protection.
1382 : The vector is in reversed, highest offset pairs come first. */
1383 : auto_vec<HOST_WIDE_INT> asan_vec;
1384 :
1385 : /* Vector of partition representative decls in between the paddings. */
1386 : auto_vec<tree> asan_decl_vec;
1387 :
1388 : /* Base pseudo register for Address Sanitizer protected automatic vars. */
1389 : rtx asan_base;
1390 :
1391 : /* Alignment needed for the Address Sanitizer protected automatic vars. */
1392 : unsigned int asan_alignb;
1393 : };
1394 :
1395 : /* A subroutine of expand_used_vars. Give each partition representative
1396 : a unique location within the stack frame. Update each partition member
1397 : with that location. */
1398 : static void
1399 239043 : expand_stack_vars (bool (*pred) (unsigned), class stack_vars_data *data)
1400 : {
1401 239043 : unsigned si, i, j, n = stack_vars_num;
1402 239043 : poly_uint64 large_size = 0, large_alloc = 0;
1403 239043 : rtx large_base = NULL;
1404 239043 : rtx large_untagged_base = NULL;
1405 239043 : unsigned large_align = 0;
1406 239043 : bool large_allocation_done = false;
1407 239043 : tree decl;
1408 :
1409 : /* Determine if there are any variables requiring "large" alignment.
1410 : Since these are dynamically allocated, we only process these if
1411 : no predicate involved. */
1412 239043 : large_align = stack_vars[stack_vars_sorted[0]].alignb * BITS_PER_UNIT;
1413 239043 : if (pred == NULL && large_align > MAX_SUPPORTED_STACK_ALIGNMENT)
1414 : {
1415 : /* Find the total size of these variables. */
1416 0 : for (si = 0; si < n; ++si)
1417 : {
1418 0 : unsigned alignb;
1419 :
1420 0 : i = stack_vars_sorted[si];
1421 0 : alignb = stack_vars[i].alignb;
1422 :
1423 : /* All "large" alignment decls come before all "small" alignment
1424 : decls, but "large" alignment decls are not sorted based on
1425 : their alignment. Increase large_align to track the largest
1426 : required alignment. */
1427 0 : if ((alignb * BITS_PER_UNIT) > large_align)
1428 : large_align = alignb * BITS_PER_UNIT;
1429 :
1430 : /* Stop when we get to the first decl with "small" alignment. */
1431 0 : if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1432 : break;
1433 :
1434 : /* Skip variables that aren't partition representatives. */
1435 0 : if (stack_vars[i].representative != i)
1436 0 : continue;
1437 :
1438 : /* Skip variables that have already had rtl assigned. See also
1439 : add_stack_var where we perpetrate this pc_rtx hack. */
1440 0 : decl = stack_vars[i].decl;
1441 0 : if (TREE_CODE (decl) == SSA_NAME
1442 0 : ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1443 0 : : DECL_RTL (decl) != pc_rtx)
1444 0 : continue;
1445 :
1446 0 : large_size = aligned_upper_bound (large_size, alignb);
1447 0 : large_size += stack_vars[i].size;
1448 : }
1449 : }
1450 :
1451 1462214 : for (si = 0; si < n; ++si)
1452 : {
1453 1223171 : rtx base;
1454 1223171 : unsigned base_align, alignb;
1455 1223171 : poly_int64 offset = 0;
1456 :
1457 1223171 : i = stack_vars_sorted[si];
1458 :
1459 : /* Skip variables that aren't partition representatives, for now. */
1460 1223171 : if (stack_vars[i].representative != i)
1461 274250 : continue;
1462 :
1463 : /* Skip variables that have already had rtl assigned. See also
1464 : add_stack_var where we perpetrate this pc_rtx hack. */
1465 948921 : decl = stack_vars[i].decl;
1466 948921 : if (TREE_CODE (decl) == SSA_NAME
1467 948921 : ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1468 944959 : : DECL_RTL (decl) != pc_rtx)
1469 3696 : continue;
1470 :
1471 : /* Check the predicate to see whether this variable should be
1472 : allocated in this pass. */
1473 945225 : if (pred && !pred (i))
1474 1177 : continue;
1475 :
1476 944048 : base = (hwassist_sanitize_stack_p ()
1477 944048 : ? hwasan_frame_base ()
1478 943992 : : virtual_stack_vars_rtx);
1479 944048 : alignb = stack_vars[i].alignb;
1480 944048 : if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1481 : {
1482 944048 : poly_int64 hwasan_orig_offset;
1483 944048 : if (hwassist_sanitize_stack_p ())
1484 : {
1485 : /* There must be no tag granule "shared" between different
1486 : objects. This means that no HWASAN_TAG_GRANULE_SIZE byte
1487 : chunk can have more than one object in it.
1488 :
1489 : We ensure this by forcing the end of the last bit of data to
1490 : be aligned to HWASAN_TAG_GRANULE_SIZE bytes here, and setting
1491 : the start of each variable to be aligned to
1492 : HWASAN_TAG_GRANULE_SIZE bytes in `align_local_variable`.
1493 :
1494 : We can't align just one of the start or end, since there are
1495 : untagged things stored on the stack which we do not align to
1496 : HWASAN_TAG_GRANULE_SIZE bytes. If we only aligned the start
1497 : or the end of tagged objects then untagged objects could end
1498 : up sharing the first granule of a tagged object or sharing the
1499 : last granule of a tagged object respectively. */
1500 56 : hwasan_orig_offset = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1501 56 : gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
1502 : }
1503 : /* ASAN description strings don't yet have a syntax for expressing
1504 : polynomial offsets. */
1505 944048 : HOST_WIDE_INT prev_offset;
1506 944048 : if (asan_sanitize_stack_p ()
1507 4263 : && pred
1508 3387 : && frame_offset.is_constant (&prev_offset)
1509 944048 : && stack_vars[i].size.is_constant ())
1510 : {
1511 3387 : if (data->asan_vec.is_empty ())
1512 : {
1513 1738 : align_frame_offset (ASAN_RED_ZONE_SIZE);
1514 1738 : prev_offset = frame_offset.to_constant ();
1515 : }
1516 3387 : prev_offset = align_base (prev_offset,
1517 : ASAN_MIN_RED_ZONE_SIZE,
1518 : !FRAME_GROWS_DOWNWARD);
1519 3387 : tree repr_decl = NULL_TREE;
1520 3387 : unsigned HOST_WIDE_INT size
1521 3387 : = asan_var_and_redzone_size (stack_vars[i].size.to_constant ());
1522 3387 : if (data->asan_vec.is_empty ())
1523 1738 : size = MAX (size, ASAN_RED_ZONE_SIZE);
1524 :
1525 3387 : unsigned HOST_WIDE_INT alignment = MAX (alignb,
1526 : ASAN_MIN_RED_ZONE_SIZE);
1527 3387 : offset = alloc_stack_frame_space (size, alignment);
1528 :
1529 3387 : data->asan_vec.safe_push (prev_offset);
1530 : /* Allocating a constant amount of space from a constant
1531 : starting offset must give a constant result. */
1532 3387 : data->asan_vec.safe_push ((offset + stack_vars[i].size)
1533 3387 : .to_constant ());
1534 : /* Find best representative of the partition.
1535 : Prefer those with DECL_NAME, even better
1536 : satisfying asan_protect_stack_decl predicate. */
1537 3949 : for (j = i; j != EOC; j = stack_vars[j].next)
1538 3391 : if (asan_protect_stack_decl (stack_vars[j].decl)
1539 3391 : && DECL_NAME (stack_vars[j].decl))
1540 : {
1541 2829 : repr_decl = stack_vars[j].decl;
1542 2829 : break;
1543 : }
1544 562 : else if (repr_decl == NULL_TREE
1545 562 : && DECL_P (stack_vars[j].decl)
1546 1124 : && DECL_NAME (stack_vars[j].decl))
1547 0 : repr_decl = stack_vars[j].decl;
1548 3387 : if (repr_decl == NULL_TREE)
1549 558 : repr_decl = stack_vars[i].decl;
1550 3387 : data->asan_decl_vec.safe_push (repr_decl);
1551 :
1552 : /* Make sure a representative is unpoison if another
1553 : variable in the partition is handled by
1554 : use-after-scope sanitization. */
1555 3387 : if (asan_handled_variables != NULL
1556 3387 : && !asan_handled_variables->contains (repr_decl))
1557 : {
1558 440 : for (j = i; j != EOC; j = stack_vars[j].next)
1559 222 : if (asan_handled_variables->contains (stack_vars[j].decl))
1560 : break;
1561 220 : if (j != EOC)
1562 2 : asan_handled_variables->add (repr_decl);
1563 : }
1564 :
1565 3387 : data->asan_alignb = MAX (data->asan_alignb, alignb);
1566 3387 : if (data->asan_base == NULL)
1567 1738 : data->asan_base = gen_reg_rtx (Pmode);
1568 3387 : base = data->asan_base;
1569 :
1570 3387 : if (!STRICT_ALIGNMENT)
1571 3387 : base_align = crtl->max_used_stack_slot_alignment;
1572 : else
1573 : base_align = MAX (crtl->max_used_stack_slot_alignment,
1574 : GET_MODE_ALIGNMENT (SImode)
1575 : << ASAN_SHADOW_SHIFT);
1576 : }
1577 : else
1578 : {
1579 940661 : offset = alloc_stack_frame_space (stack_vars[i].size, alignb);
1580 940661 : base_align = crtl->max_used_stack_slot_alignment;
1581 :
1582 940661 : if (hwassist_sanitize_stack_p ())
1583 : {
1584 : /* Align again since the point of this alignment is to handle
1585 : the "end" of the object (i.e. smallest address after the
1586 : stack object). For FRAME_GROWS_DOWNWARD that requires
1587 : aligning the stack before allocating, but for a frame that
1588 : grows upwards that requires aligning the stack after
1589 : allocation.
1590 :
1591 : Use `frame_offset` to record the offset value rather than
1592 : `offset` since the `frame_offset` describes the extent
1593 : allocated for this particular variable while `offset`
1594 : describes the address that this variable starts at. */
1595 56 : align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1596 56 : hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1597 : hwasan_orig_offset, frame_offset);
1598 : }
1599 : }
1600 : }
1601 : else
1602 : {
1603 : /* Large alignment is only processed in the last pass. */
1604 0 : if (pred)
1605 0 : continue;
1606 :
1607 : /* If there were any variables requiring "large" alignment, allocate
1608 : space. */
1609 0 : if (maybe_ne (large_size, 0U) && ! large_allocation_done)
1610 : {
1611 0 : poly_int64 loffset;
1612 0 : rtx large_allocsize;
1613 :
1614 0 : large_allocsize = gen_int_mode (large_size, Pmode);
1615 0 : get_dynamic_stack_size (&large_allocsize, 0, large_align, NULL);
1616 0 : loffset = alloc_stack_frame_space
1617 0 : (rtx_to_poly_int64 (large_allocsize),
1618 0 : PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT);
1619 0 : large_base = get_dynamic_stack_base (loffset, large_align, base);
1620 0 : large_allocation_done = true;
1621 : }
1622 :
1623 0 : gcc_assert (large_base != NULL);
1624 0 : large_alloc = aligned_upper_bound (large_alloc, alignb);
1625 0 : offset = large_alloc;
1626 0 : large_alloc += stack_vars[i].size;
1627 0 : if (hwassist_sanitize_stack_p ())
1628 : {
1629 : /* An object with a large alignment requirement means that the
1630 : alignment requirement is greater than the required alignment
1631 : for tags. */
1632 0 : if (!large_untagged_base)
1633 0 : large_untagged_base
1634 0 : = targetm.memtag.untagged_pointer (large_base, NULL_RTX);
1635 : /* Ensure the end of the variable is also aligned correctly. */
1636 0 : poly_int64 align_again
1637 0 : = aligned_upper_bound (large_alloc, HWASAN_TAG_GRANULE_SIZE);
1638 : /* For large allocations we always allocate a chunk of space
1639 : (which is addressed by large_untagged_base/large_base) and
1640 : then use positive offsets from that. Hence the farthest
1641 : offset is `align_again` and the nearest offset from the base
1642 : is `offset`. */
1643 0 : hwasan_record_stack_var (large_untagged_base, large_base,
1644 : offset, align_again);
1645 : }
1646 :
1647 : base = large_base;
1648 : base_align = large_align;
1649 : }
1650 :
1651 : /* Create rtl for each variable based on their location within the
1652 : partition. */
1653 2162335 : for (j = i; j != EOC; j = stack_vars[j].next)
1654 : {
1655 1218287 : expand_one_stack_var_at (stack_vars[j].decl,
1656 : base, base_align, offset);
1657 : }
1658 944048 : if (hwassist_sanitize_stack_p ())
1659 56 : hwasan_increment_frame_tag ();
1660 : }
1661 :
1662 239043 : gcc_assert (known_eq (large_alloc, large_size));
1663 239043 : }
1664 :
1665 : /* Take into account all sizes of partitions and reset DECL_RTLs. */
1666 : static poly_uint64
1667 1090389 : account_stack_vars (void)
1668 : {
1669 1090389 : unsigned si, j, i, n = stack_vars_num;
1670 1090389 : poly_uint64 size = 0;
1671 :
1672 5724054 : for (si = 0; si < n; ++si)
1673 : {
1674 4633665 : i = stack_vars_sorted[si];
1675 :
1676 : /* Skip variables that aren't partition representatives, for now. */
1677 4633665 : if (stack_vars[i].representative != i)
1678 0 : continue;
1679 :
1680 9267330 : size += stack_vars[i].size;
1681 9267330 : for (j = i; j != EOC; j = stack_vars[j].next)
1682 4633665 : set_rtl (stack_vars[j].decl, NULL);
1683 : }
1684 1090389 : return size;
1685 : }
1686 :
1687 : /* Record the RTL assignment X for the default def of PARM. */
1688 :
1689 : extern void
1690 4774793 : set_parm_rtl (tree parm, rtx x)
1691 : {
1692 4774793 : gcc_assert (TREE_CODE (parm) == PARM_DECL
1693 : || TREE_CODE (parm) == RESULT_DECL);
1694 :
1695 4774793 : if (x && !MEM_P (x))
1696 : {
1697 3081114 : unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (parm),
1698 : TYPE_MODE (TREE_TYPE (parm)),
1699 : TYPE_ALIGN (TREE_TYPE (parm)));
1700 :
1701 : /* If the variable alignment is very large we'll dynamically
1702 : allocate it, which means that in-frame portion is just a
1703 : pointer. ??? We've got a pseudo for sure here, do we
1704 : actually dynamically allocate its spilling area if needed?
1705 : ??? Isn't it a problem when Pmode alignment also exceeds
1706 : MAX_SUPPORTED_STACK_ALIGNMENT, as can happen on cris and lm32? */
1707 3081114 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1708 0 : align = GET_MODE_ALIGNMENT (Pmode);
1709 :
1710 3081114 : record_alignment_for_reg_var (align);
1711 : }
1712 :
1713 4774793 : tree ssa = ssa_default_def (cfun, parm);
1714 4774793 : if (!ssa)
1715 1087614 : return set_rtl (parm, x);
1716 :
1717 3687179 : int part = var_to_partition (SA.map, ssa);
1718 3687179 : gcc_assert (part != NO_PARTITION);
1719 :
1720 3687179 : bool changed = bitmap_bit_p (SA.partitions_for_parm_default_defs, part);
1721 3687179 : gcc_assert (changed);
1722 :
1723 3687179 : set_rtl (ssa, x);
1724 3687179 : gcc_assert (DECL_RTL (parm) == x);
1725 : }
1726 :
1727 : /* A subroutine of expand_one_var. Called to immediately assign rtl
1728 : to a variable to be allocated in the stack frame. */
1729 :
1730 : static void
1731 818328 : expand_one_stack_var_1 (tree var)
1732 : {
1733 818328 : poly_uint64 size;
1734 818328 : poly_int64 offset;
1735 818328 : unsigned byte_align;
1736 :
1737 818328 : if (TREE_CODE (var) == SSA_NAME)
1738 : {
1739 456579 : tree type = TREE_TYPE (var);
1740 456579 : size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
1741 : }
1742 : else
1743 361749 : size = tree_to_poly_uint64 (DECL_SIZE_UNIT (var));
1744 :
1745 818328 : byte_align = align_local_variable (var, true);
1746 :
1747 : /* We handle highly aligned variables in expand_stack_vars. */
1748 818328 : gcc_assert (byte_align * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT);
1749 :
1750 818328 : rtx base;
1751 818328 : if (hwassist_sanitize_stack_p ())
1752 : {
1753 : /* Allocate zero bytes to align the stack. */
1754 30 : poly_int64 hwasan_orig_offset
1755 30 : = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1756 30 : offset = alloc_stack_frame_space (size, byte_align);
1757 30 : align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1758 30 : base = hwasan_frame_base ();
1759 : /* Use `frame_offset` to automatically account for machines where the
1760 : frame grows upwards.
1761 :
1762 : `offset` will always point to the "start" of the stack object, which
1763 : will be the smallest address, for ! FRAME_GROWS_DOWNWARD this is *not*
1764 : the "furthest" offset from the base delimiting the current stack
1765 : object. `frame_offset` will always delimit the extent that the frame.
1766 : */
1767 30 : hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1768 : hwasan_orig_offset, frame_offset);
1769 : }
1770 : else
1771 : {
1772 818298 : offset = alloc_stack_frame_space (size, byte_align);
1773 818298 : base = virtual_stack_vars_rtx;
1774 : }
1775 :
1776 818328 : expand_one_stack_var_at (var, base,
1777 : crtl->max_used_stack_slot_alignment, offset);
1778 :
1779 818328 : if (hwassist_sanitize_stack_p ())
1780 30 : hwasan_increment_frame_tag ();
1781 818328 : }
1782 :
1783 : /* Wrapper for expand_one_stack_var_1 that checks SSA_NAMEs are
1784 : already assigned some MEM. */
1785 :
1786 : static void
1787 361749 : expand_one_stack_var (tree var)
1788 : {
1789 361749 : if (TREE_CODE (var) == SSA_NAME)
1790 : {
1791 0 : int part = var_to_partition (SA.map, var);
1792 0 : if (part != NO_PARTITION)
1793 : {
1794 0 : rtx x = SA.partition_to_pseudo[part];
1795 0 : gcc_assert (x);
1796 0 : gcc_assert (MEM_P (x));
1797 : return;
1798 : }
1799 : }
1800 :
1801 361749 : return expand_one_stack_var_1 (var);
1802 : }
1803 :
1804 : /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1805 : that will reside in a hard register. */
1806 :
1807 : static void
1808 1110 : expand_one_hard_reg_var (tree var)
1809 : {
1810 0 : rest_of_decl_compilation (var, 0, 0);
1811 0 : }
1812 :
1813 : /* Record the alignment requirements of some variable assigned to a
1814 : pseudo. */
1815 :
1816 : static void
1817 41806985 : record_alignment_for_reg_var (unsigned int align)
1818 : {
1819 41806985 : if (SUPPORTS_STACK_ALIGNMENT
1820 41806985 : && crtl->stack_alignment_estimated < align)
1821 : {
1822 : /* stack_alignment_estimated shouldn't change after stack
1823 : realign decision made */
1824 1749830 : gcc_assert (!crtl->stack_realign_processed);
1825 1749830 : crtl->stack_alignment_estimated = align;
1826 : }
1827 :
1828 : /* stack_alignment_needed > PREFERRED_STACK_BOUNDARY is permitted.
1829 : So here we only make sure stack_alignment_needed >= align. */
1830 41806985 : if (crtl->stack_alignment_needed < align)
1831 565665 : crtl->stack_alignment_needed = align;
1832 41806985 : if (crtl->max_used_stack_slot_alignment < align)
1833 565665 : crtl->max_used_stack_slot_alignment = align;
1834 41806985 : }
1835 :
1836 : /* Create RTL for an SSA partition. */
1837 :
1838 : static void
1839 23024637 : expand_one_ssa_partition (tree var)
1840 : {
1841 23024637 : int part = var_to_partition (SA.map, var);
1842 23024637 : gcc_assert (part != NO_PARTITION);
1843 :
1844 23024637 : if (SA.partition_to_pseudo[part])
1845 : return;
1846 :
1847 23024637 : unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1848 : TYPE_MODE (TREE_TYPE (var)),
1849 : TYPE_ALIGN (TREE_TYPE (var)));
1850 :
1851 : /* If the variable alignment is very large we'll dynamically allocate
1852 : it, which means that in-frame portion is just a pointer. */
1853 23024637 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1854 0 : align = GET_MODE_ALIGNMENT (Pmode);
1855 :
1856 23024637 : record_alignment_for_reg_var (align);
1857 :
1858 23024637 : if (!use_register_for_decl (var))
1859 : {
1860 459817 : if (defer_stack_allocation (var, true))
1861 3238 : add_stack_var (var, true);
1862 : else
1863 456579 : expand_one_stack_var_1 (var);
1864 : return;
1865 : }
1866 :
1867 22564820 : machine_mode reg_mode = promote_ssa_mode (var, NULL);
1868 22564820 : rtx x = gen_reg_rtx (reg_mode);
1869 :
1870 22564820 : set_rtl (var, x);
1871 :
1872 : /* For a promoted variable, X will not be used directly but wrapped in a
1873 : SUBREG with SUBREG_PROMOTED_VAR_P set, which means that the RTL land
1874 : will assume that its upper bits can be inferred from its lower bits.
1875 : Therefore, if X isn't initialized on every path from the entry, then
1876 : we must do it manually in order to fulfill the above assumption. */
1877 22564820 : if (reg_mode != TYPE_MODE (TREE_TYPE (var))
1878 22564820 : && bitmap_bit_p (SA.partitions_for_undefined_values, part))
1879 0 : emit_move_insn (x, CONST0_RTX (reg_mode));
1880 : }
1881 :
1882 : /* Record the association between the RTL generated for partition PART
1883 : and the underlying variable of the SSA_NAME VAR. */
1884 :
1885 : static void
1886 50203415 : adjust_one_expanded_partition_var (tree var)
1887 : {
1888 50203415 : if (!var)
1889 : return;
1890 :
1891 50203415 : tree decl = SSA_NAME_VAR (var);
1892 :
1893 50203415 : int part = var_to_partition (SA.map, var);
1894 50203415 : if (part == NO_PARTITION)
1895 : return;
1896 :
1897 32247359 : rtx x = SA.partition_to_pseudo[part];
1898 :
1899 32247359 : gcc_assert (x);
1900 :
1901 32247359 : set_rtl (var, x);
1902 :
1903 32247359 : if (!REG_P (x))
1904 : return;
1905 :
1906 : /* Note if the object is a user variable. */
1907 30185590 : if (decl && !DECL_ARTIFICIAL (decl))
1908 4209148 : mark_user_reg (x);
1909 :
1910 30185590 : if (POINTER_TYPE_P (decl ? TREE_TYPE (decl) : TREE_TYPE (var)))
1911 7714390 : mark_reg_pointer (x, get_pointer_alignment (var));
1912 : }
1913 :
1914 : /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1915 : that will reside in a pseudo register. */
1916 :
1917 : static void
1918 586610 : expand_one_register_var (tree var)
1919 : {
1920 586610 : if (TREE_CODE (var) == SSA_NAME)
1921 : {
1922 0 : int part = var_to_partition (SA.map, var);
1923 0 : if (part != NO_PARTITION)
1924 : {
1925 0 : rtx x = SA.partition_to_pseudo[part];
1926 0 : gcc_assert (x);
1927 0 : gcc_assert (REG_P (x));
1928 : return;
1929 : }
1930 0 : gcc_unreachable ();
1931 : }
1932 :
1933 586610 : tree decl = var;
1934 586610 : tree type = TREE_TYPE (decl);
1935 586610 : machine_mode reg_mode = promote_decl_mode (decl, NULL);
1936 586610 : rtx x = gen_reg_rtx (reg_mode);
1937 :
1938 586610 : set_rtl (var, x);
1939 :
1940 : /* Note if the object is a user variable. */
1941 586610 : if (!DECL_ARTIFICIAL (decl))
1942 94577 : mark_user_reg (x);
1943 :
1944 586610 : if (POINTER_TYPE_P (type))
1945 100 : mark_reg_pointer (x, get_pointer_alignment (var));
1946 : }
1947 :
1948 : /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL that
1949 : has some associated error, e.g. its type is error-mark. We just need
1950 : to pick something that won't crash the rest of the compiler. */
1951 :
1952 : static void
1953 33 : expand_one_error_var (tree var)
1954 : {
1955 33 : machine_mode mode = DECL_MODE (var);
1956 33 : rtx x;
1957 :
1958 33 : if (mode == BLKmode)
1959 1 : x = gen_rtx_MEM (BLKmode, const0_rtx);
1960 32 : else if (mode == VOIDmode)
1961 0 : x = const0_rtx;
1962 : else
1963 32 : x = gen_reg_rtx (mode);
1964 :
1965 33 : SET_DECL_RTL (var, x);
1966 33 : }
1967 :
1968 : /* A subroutine of expand_one_var. VAR is a variable that will be
1969 : allocated to the local stack frame. Return true if we wish to
1970 : add VAR to STACK_VARS so that it will be coalesced with other
1971 : variables. Return false to allocate VAR immediately.
1972 :
1973 : This function is used to reduce the number of variables considered
1974 : for coalescing, which reduces the size of the quadratic problem. */
1975 :
1976 : static bool
1977 6809219 : defer_stack_allocation (tree var, bool toplevel)
1978 : {
1979 6809219 : tree size_unit = TREE_CODE (var) == SSA_NAME
1980 6809219 : ? TYPE_SIZE_UNIT (TREE_TYPE (var))
1981 6809219 : : DECL_SIZE_UNIT (var);
1982 6809219 : poly_uint64 size;
1983 :
1984 : /* Whether the variable is small enough for immediate allocation not to be
1985 : a problem with regard to the frame size. */
1986 6809219 : bool smallish
1987 6809219 : = (poly_int_tree_p (size_unit, &size)
1988 6809219 : && (estimated_poly_value (size)
1989 6809219 : < param_min_size_for_stack_sharing));
1990 :
1991 : /* If stack protection is enabled, *all* stack variables must be deferred,
1992 : so that we can re-order the strings to the top of the frame.
1993 : Similarly for Address Sanitizer. */
1994 6809219 : if (flag_stack_protect || asan_sanitize_stack_p ())
1995 : return true;
1996 :
1997 : /* Use the same effective alignment that expand_one_stack_var_1 would use.
1998 : If that alignment exceeds MAX_SUPPORTED_STACK_ALIGNMENT, defer
1999 : the variable so that expand_stack_vars handles its large alignment,
2000 : rather than calling expand_one_stack_var_1 and failing its assertion. */
2001 6794654 : unsigned int align = align_local_variable (var, false) * BITS_PER_UNIT;
2002 :
2003 : /* We handle "large" alignment via dynamic allocation. We want to handle
2004 : this extra complication in only one place, so defer them. */
2005 6794654 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
2006 : return true;
2007 :
2008 6794654 : bool ignored = TREE_CODE (var) == SSA_NAME
2009 6801124 : ? !SSAVAR (var) || DECL_IGNORED_P (SSA_NAME_VAR (var))
2010 6335564 : : DECL_IGNORED_P (var);
2011 :
2012 : /* When optimization is enabled, DECL_IGNORED_P variables originally scoped
2013 : might be detached from their block and appear at toplevel when we reach
2014 : here. We want to coalesce them with variables from other blocks when
2015 : the immediate contribution to the frame size would be noticeable. */
2016 6794654 : if (toplevel && optimize > 0 && ignored && !smallish)
2017 : return true;
2018 :
2019 : /* Variables declared in the outermost scope automatically conflict
2020 : with every other variable. The only reason to want to defer them
2021 : at all is that, after sorting, we can more efficiently pack
2022 : small variables in the stack frame. Continue to defer at -O2. */
2023 5255871 : if (toplevel && optimize < 2)
2024 : return false;
2025 :
2026 : /* Without optimization, *most* variables are allocated from the
2027 : stack, which makes the quadratic problem large exactly when we
2028 : want compilation to proceed as quickly as possible. On the
2029 : other hand, we don't want the function's stack frame size to
2030 : get completely out of hand. So we avoid adding scalars and
2031 : "small" aggregates to the list at all. */
2032 4597808 : if (optimize == 0 && smallish)
2033 49566 : return false;
2034 :
2035 : return true;
2036 : }
2037 :
2038 : /* A subroutine of expand_used_vars. Expand one variable according to
2039 : its flavor. Variables to be placed on the stack are not actually
2040 : expanded yet, merely recorded.
2041 : When REALLY_EXPAND is false, only add stack values to be allocated.
2042 : Return stack usage this variable is supposed to take.
2043 : */
2044 :
2045 : static poly_uint64
2046 16007763 : expand_one_var (tree var, bool toplevel, bool really_expand,
2047 : bitmap forced_stack_var = NULL)
2048 : {
2049 16007763 : unsigned int align = BITS_PER_UNIT;
2050 16007763 : tree origvar = var;
2051 :
2052 16007763 : var = SSAVAR (var);
2053 :
2054 16007763 : if (TREE_TYPE (var) != error_mark_node && VAR_P (var))
2055 : {
2056 16007763 : if (is_global_var (var))
2057 306529 : return 0;
2058 :
2059 : /* Because we don't know if VAR will be in register or on stack,
2060 : we conservatively assume it will be on stack even if VAR is
2061 : eventually put into register after RA pass. For non-automatic
2062 : variables, which won't be on stack, we collect alignment of
2063 : type and ignore user specified alignment. Similarly for
2064 : SSA_NAMEs for which use_register_for_decl returns true. */
2065 15701234 : if (TREE_STATIC (var)
2066 15701234 : || DECL_EXTERNAL (var)
2067 31402468 : || (TREE_CODE (origvar) == SSA_NAME && use_register_for_decl (var)))
2068 0 : align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
2069 : TYPE_MODE (TREE_TYPE (var)),
2070 : TYPE_ALIGN (TREE_TYPE (var)));
2071 15701234 : else if (DECL_HAS_VALUE_EXPR_P (var)
2072 15701234 : || (DECL_RTL_SET_P (var) && MEM_P (DECL_RTL (var))))
2073 : /* Don't consider debug only variables with DECL_HAS_VALUE_EXPR_P set
2074 : or variables which were assigned a stack slot already by
2075 : expand_one_stack_var_at - in the latter case DECL_ALIGN has been
2076 : changed from the offset chosen to it. */
2077 17565 : align = crtl->stack_alignment_estimated;
2078 : else
2079 15683669 : align = MINIMUM_ALIGNMENT (var, DECL_MODE (var), DECL_ALIGN (var));
2080 :
2081 : /* If the variable alignment is very large we'll dynamically allocate
2082 : it, which means that in-frame portion is just a pointer. */
2083 15701234 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
2084 0 : align = GET_MODE_ALIGNMENT (Pmode);
2085 : }
2086 :
2087 15701234 : record_alignment_for_reg_var (align);
2088 :
2089 15701234 : poly_uint64 size;
2090 15701234 : if (TREE_CODE (origvar) == SSA_NAME)
2091 : {
2092 0 : gcc_assert (!VAR_P (var)
2093 : || (!DECL_EXTERNAL (var)
2094 : && !DECL_HAS_VALUE_EXPR_P (var)
2095 : && !TREE_STATIC (var)
2096 : && TREE_TYPE (var) != error_mark_node
2097 : && !DECL_HARD_REGISTER (var)
2098 : && really_expand));
2099 : }
2100 15701234 : if (!VAR_P (var) && TREE_CODE (origvar) != SSA_NAME)
2101 : ;
2102 15701234 : else if (DECL_EXTERNAL (var))
2103 : ;
2104 15701234 : else if (DECL_HAS_VALUE_EXPR_P (var))
2105 : ;
2106 15683674 : else if (TREE_STATIC (var))
2107 : ;
2108 15683674 : else if (TREE_CODE (origvar) != SSA_NAME && DECL_RTL_SET_P (var))
2109 : ;
2110 15683669 : else if (TREE_TYPE (var) == error_mark_node)
2111 : {
2112 0 : if (really_expand)
2113 0 : expand_one_error_var (var);
2114 : }
2115 15683669 : else if (VAR_P (var) && DECL_HARD_REGISTER (var))
2116 : {
2117 4158 : if (really_expand)
2118 : {
2119 1110 : expand_one_hard_reg_var (var);
2120 1110 : if (!DECL_HARD_REGISTER (var))
2121 : /* Invalid register specification. */
2122 33 : expand_one_error_var (var);
2123 : }
2124 : }
2125 15679511 : else if (use_register_for_decl (var)
2126 15679511 : && (!forced_stack_var
2127 595494 : || !bitmap_bit_p (forced_stack_var, DECL_UID (var))))
2128 : {
2129 9330109 : if (really_expand)
2130 586610 : expand_one_register_var (origvar);
2131 : }
2132 6349402 : else if (!poly_int_tree_p (DECL_SIZE_UNIT (var), &size)
2133 6349402 : || !valid_constant_size_p (DECL_SIZE_UNIT (var)))
2134 : {
2135 : /* Reject variables which cover more than half of the address-space. */
2136 0 : if (really_expand)
2137 : {
2138 0 : if (DECL_NONLOCAL_FRAME (var))
2139 0 : error_at (DECL_SOURCE_LOCATION (current_function_decl),
2140 : "total size of local objects is too large");
2141 : else
2142 0 : error_at (DECL_SOURCE_LOCATION (var),
2143 : "size of variable %q+D is too large", var);
2144 0 : expand_one_error_var (var);
2145 : }
2146 : }
2147 6349402 : else if (defer_stack_allocation (var, toplevel))
2148 5848714 : add_stack_var (origvar, really_expand);
2149 : else
2150 : {
2151 500688 : if (really_expand)
2152 : {
2153 361453 : if (lookup_attribute ("naked",
2154 361453 : DECL_ATTRIBUTES (current_function_decl)))
2155 0 : error ("cannot allocate stack for variable %q+D, naked function",
2156 : var);
2157 :
2158 361453 : expand_one_stack_var (origvar);
2159 : }
2160 500688 : return size;
2161 : }
2162 15200546 : return 0;
2163 : }
2164 :
2165 : /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
2166 : expanding variables. Those variables that can be put into registers
2167 : are allocated pseudos; those that can't are put on the stack.
2168 :
2169 : TOPLEVEL is true if this is the outermost BLOCK. */
2170 :
2171 : static void
2172 17239902 : expand_used_vars_for_block (tree block, bool toplevel, bitmap forced_stack_vars)
2173 : {
2174 17239902 : tree t;
2175 :
2176 : /* Expand all variables at this level. */
2177 37563527 : for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
2178 20323625 : if (TREE_USED (t)
2179 20323625 : && ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
2180 779333 : || !DECL_NONSHAREABLE (t)))
2181 779333 : expand_one_var (t, toplevel, true, forced_stack_vars);
2182 :
2183 : /* Expand all variables at containing levels. */
2184 32961896 : for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
2185 15721994 : expand_used_vars_for_block (t, false, forced_stack_vars);
2186 17239902 : }
2187 :
2188 : /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
2189 : and clear TREE_USED on all local variables. */
2190 :
2191 : static void
2192 17239902 : clear_tree_used (tree block)
2193 : {
2194 17239902 : tree t;
2195 :
2196 37563527 : for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
2197 : /* if (!TREE_STATIC (t) && !DECL_EXTERNAL (t)) */
2198 1122202 : if ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
2199 20323625 : || !DECL_NONSHAREABLE (t))
2200 20323625 : TREE_USED (t) = 0;
2201 :
2202 32961896 : for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
2203 15721994 : clear_tree_used (t);
2204 17239902 : }
2205 :
2206 : /* Examine TYPE and determine a bit mask of the following features. */
2207 :
2208 : #define SPCT_HAS_LARGE_CHAR_ARRAY 1
2209 : #define SPCT_HAS_SMALL_CHAR_ARRAY 2
2210 : #define SPCT_HAS_ARRAY 4
2211 : #define SPCT_HAS_AGGREGATE 8
2212 :
2213 : static unsigned int
2214 2744 : stack_protect_classify_type (tree type)
2215 : {
2216 2744 : unsigned int ret = 0;
2217 2744 : tree t;
2218 :
2219 2744 : switch (TREE_CODE (type))
2220 : {
2221 708 : case ARRAY_TYPE:
2222 708 : t = TYPE_MAIN_VARIANT (TREE_TYPE (type));
2223 708 : if (t == char_type_node
2224 178 : || t == signed_char_type_node
2225 178 : || t == unsigned_char_type_node)
2226 : {
2227 530 : unsigned HOST_WIDE_INT max = param_ssp_buffer_size;
2228 530 : unsigned HOST_WIDE_INT len;
2229 :
2230 530 : if (!TYPE_SIZE_UNIT (type)
2231 530 : || !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)))
2232 : len = max;
2233 : else
2234 530 : len = tree_to_uhwi (TYPE_SIZE_UNIT (type));
2235 :
2236 530 : if (len < max)
2237 : ret = SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_ARRAY;
2238 : else
2239 498 : ret = SPCT_HAS_LARGE_CHAR_ARRAY | SPCT_HAS_ARRAY;
2240 : }
2241 : else
2242 : ret = SPCT_HAS_ARRAY;
2243 : break;
2244 :
2245 1021 : case UNION_TYPE:
2246 1021 : case QUAL_UNION_TYPE:
2247 1021 : case RECORD_TYPE:
2248 1021 : ret = SPCT_HAS_AGGREGATE;
2249 10915 : for (t = TYPE_FIELDS (type); t ; t = TREE_CHAIN (t))
2250 9894 : if (TREE_CODE (t) == FIELD_DECL)
2251 1702 : ret |= stack_protect_classify_type (TREE_TYPE (t));
2252 : break;
2253 :
2254 : default:
2255 : break;
2256 : }
2257 :
2258 2744 : return ret;
2259 : }
2260 :
2261 : /* Return nonzero if DECL should be segregated into the "vulnerable" upper
2262 : part of the local stack frame. Remember if we ever return nonzero for
2263 : any variable in this function. The return value is the phase number in
2264 : which the variable should be allocated. */
2265 :
2266 : static int
2267 1042 : stack_protect_decl_phase (tree decl)
2268 : {
2269 1042 : unsigned int bits = stack_protect_classify_type (TREE_TYPE (decl));
2270 1042 : int ret = 0;
2271 :
2272 1042 : if (bits & SPCT_HAS_SMALL_CHAR_ARRAY)
2273 28 : has_short_buffer = true;
2274 :
2275 1042 : tree attribs = DECL_ATTRIBUTES (current_function_decl);
2276 1042 : if (!lookup_attribute ("no_stack_protector", attribs)
2277 1042 : && (flag_stack_protect == SPCT_FLAG_ALL
2278 1042 : || flag_stack_protect == SPCT_FLAG_STRONG
2279 215 : || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2280 5 : && lookup_attribute ("stack_protect", attribs))))
2281 : {
2282 832 : if ((bits & (SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_LARGE_CHAR_ARRAY))
2283 368 : && !(bits & SPCT_HAS_AGGREGATE))
2284 : ret = 1;
2285 752 : else if (bits & SPCT_HAS_ARRAY)
2286 : ret = 2;
2287 : }
2288 : else
2289 210 : ret = (bits & SPCT_HAS_LARGE_CHAR_ARRAY) != 0;
2290 :
2291 210 : if (ret)
2292 690 : has_protected_decls = true;
2293 :
2294 1042 : return ret;
2295 : }
2296 :
2297 : /* Two helper routines that check for phase 1 and phase 2. These are used
2298 : as callbacks for expand_stack_vars. */
2299 :
2300 : static bool
2301 353 : stack_protect_decl_phase_1 (unsigned i)
2302 : {
2303 353 : return stack_protect_decl_phase (stack_vars[i].decl) == 1;
2304 : }
2305 :
2306 : static bool
2307 217 : stack_protect_decl_phase_2 (unsigned i)
2308 : {
2309 217 : return stack_protect_decl_phase (stack_vars[i].decl) == 2;
2310 : }
2311 :
2312 : /* And helper function that checks for asan phase (with stack protector
2313 : it is phase 3). This is used as callback for expand_stack_vars.
2314 : Returns true if any of the vars in the partition need to be protected. */
2315 :
2316 : static bool
2317 4178 : asan_decl_phase_3 (unsigned i)
2318 : {
2319 5054 : while (i != EOC)
2320 : {
2321 4178 : if (asan_protect_stack_decl (stack_vars[i].decl))
2322 : return true;
2323 876 : i = stack_vars[i].next;
2324 : }
2325 : return false;
2326 : }
2327 :
2328 : /* Ensure that variables in different stack protection phases conflict
2329 : so that they are not merged and share the same stack slot.
2330 : Return true if there are any address taken variables. */
2331 :
2332 : static bool
2333 261 : add_stack_protection_conflicts (void)
2334 : {
2335 261 : unsigned i, j, n = stack_vars_num;
2336 261 : unsigned char *phase;
2337 261 : bool ret = false;
2338 :
2339 261 : phase = XNEWVEC (unsigned char, n);
2340 994 : for (i = 0; i < n; ++i)
2341 : {
2342 472 : phase[i] = stack_protect_decl_phase (stack_vars[i].decl);
2343 472 : if (TREE_ADDRESSABLE (stack_vars[i].decl))
2344 395 : ret = true;
2345 : }
2346 :
2347 733 : for (i = 0; i < n; ++i)
2348 : {
2349 472 : unsigned char ph_i = phase[i];
2350 1188 : for (j = i + 1; j < n; ++j)
2351 716 : if (ph_i != phase[j])
2352 329 : add_stack_var_conflict (i, j);
2353 : }
2354 :
2355 261 : XDELETEVEC (phase);
2356 261 : return ret;
2357 : }
2358 :
2359 : /* Create a decl for the guard at the top of the stack frame. */
2360 :
2361 : static void
2362 296 : create_stack_guard (void)
2363 : {
2364 296 : tree guard = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
2365 : VAR_DECL, NULL, ptr_type_node);
2366 296 : TREE_THIS_VOLATILE (guard) = 1;
2367 296 : TREE_USED (guard) = 1;
2368 296 : expand_one_stack_var (guard);
2369 296 : crtl->stack_protect_guard = guard;
2370 296 : }
2371 :
2372 : /* Prepare for expanding variables. */
2373 : static void
2374 8035674 : init_vars_expansion (void)
2375 : {
2376 : /* Conflict bitmaps, and a few related temporary bitmaps, go here. */
2377 8035674 : bitmap_obstack_initialize (&stack_var_bitmap_obstack);
2378 :
2379 : /* A map from decl to stack partition. */
2380 8035674 : decl_to_stack_part = new hash_map<tree, unsigned>;
2381 :
2382 : /* Initialize local stack smashing state. */
2383 8035674 : has_protected_decls = false;
2384 8035674 : has_short_buffer = false;
2385 8035674 : if (hwassist_sanitize_stack_p ())
2386 1112 : hwasan_record_frame_init ();
2387 8035674 : }
2388 :
2389 : /* Free up stack variable graph data. */
2390 : static void
2391 8035674 : fini_vars_expansion (void)
2392 : {
2393 8035674 : bitmap_obstack_release (&stack_var_bitmap_obstack);
2394 8035674 : if (stack_vars)
2395 1327225 : XDELETEVEC (stack_vars);
2396 8035674 : if (stack_vars_sorted)
2397 1327225 : XDELETEVEC (stack_vars_sorted);
2398 8035674 : stack_vars = NULL;
2399 8035674 : stack_vars_sorted = NULL;
2400 8035674 : stack_vars_alloc = stack_vars_num = 0;
2401 16071348 : delete decl_to_stack_part;
2402 8035674 : decl_to_stack_part = NULL;
2403 8035674 : }
2404 :
2405 : /* Make a fair guess for the size of the stack frame of the function
2406 : in NODE. This doesn't have to be exact, the result is only used in
2407 : the inline heuristics. So we don't want to run the full stack var
2408 : packing algorithm (which is quadratic in the number of stack vars).
2409 : Instead, we calculate the total size of all stack vars. This turns
2410 : out to be a pretty fair estimate -- packing of stack vars doesn't
2411 : happen very often. */
2412 :
2413 : HOST_WIDE_INT
2414 6517766 : estimated_stack_frame_size (struct cgraph_node *node)
2415 : {
2416 6517766 : poly_int64 size = 0;
2417 6517766 : unsigned i;
2418 6517766 : tree var;
2419 6517766 : struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
2420 :
2421 6517766 : push_cfun (fn);
2422 :
2423 6517766 : init_vars_expansion ();
2424 :
2425 25985493 : FOR_EACH_LOCAL_DECL (fn, i, var)
2426 14119442 : if (auto_var_in_fn_p (var, fn->decl))
2427 13519448 : size += expand_one_var (var, true, false);
2428 :
2429 6517766 : if (stack_vars_num > 0)
2430 : {
2431 : /* Fake sorting the stack vars for account_stack_vars (). */
2432 1090389 : stack_vars_sorted = XNEWVEC (unsigned , stack_vars_num);
2433 5724054 : for (i = 0; i < stack_vars_num; ++i)
2434 4633665 : stack_vars_sorted[i] = i;
2435 1090389 : size += account_stack_vars ();
2436 : }
2437 :
2438 6517766 : fini_vars_expansion ();
2439 6517766 : pop_cfun ();
2440 6517766 : return estimated_poly_value (size);
2441 : }
2442 :
2443 : /* Check if the current function has calls that use a return slot. */
2444 :
2445 : static bool
2446 418 : stack_protect_return_slot_p ()
2447 : {
2448 418 : basic_block bb;
2449 :
2450 1876 : FOR_ALL_BB_FN (bb, cfun)
2451 2974 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2452 4028 : !gsi_end_p (gsi); gsi_next (&gsi))
2453 : {
2454 2570 : gimple *stmt = gsi_stmt (gsi);
2455 : /* This assumes that calls to internal-only functions never
2456 : use a return slot. */
2457 2570 : if (is_gimple_call (stmt)
2458 542 : && !gimple_call_internal_p (stmt)
2459 3095 : && aggregate_value_p (TREE_TYPE (gimple_call_fntype (stmt)),
2460 525 : gimple_call_fndecl (stmt)))
2461 418 : return true;
2462 : }
2463 : return false;
2464 : }
2465 :
2466 : /* Verify that partitions which claim to be the same object really are at the
2467 : same address. MEM_EXPR-based disambiguation identifies a location by a
2468 : MEM_EXPR base and an offset from it, so two stack slots carrying one
2469 : MEM_EXPR read as a single object, which lets an access to one be redirected
2470 : to the other. out-of-SSA keeps them apart, see the comment above
2471 : split_overlapping_partition_decls.
2472 :
2473 : Call this once the RTL of every partition is final. The partition of a
2474 : PARM_DECL or RESULT_DECL default definition is given the MEM_EXPR of any
2475 : other variable in it while its names are walked, and only gets the decl
2476 : back when its RTL is restored in pass_expand::execute. */
2477 :
2478 : static void
2479 1517888 : verify_partition_mem_exprs (void)
2480 : {
2481 1517888 : hash_map<tree, rtx> slots;
2482 29747533 : for (unsigned i = 0; i < num_var_partitions (SA.map); i++)
2483 : {
2484 26711757 : rtx x = SA.partition_to_pseudo[i];
2485 27959741 : if (!x || !MEM_P (x) || !MEM_EXPR (x))
2486 25469973 : continue;
2487 1241784 : bool existed;
2488 1241784 : rtx &known = slots.get_or_insert (MEM_EXPR (x), &existed);
2489 1241784 : if (!existed)
2490 1241784 : known = x;
2491 : else
2492 0 : gcc_assert (rtx_equal_p (XEXP (known, 0), XEXP (x, 0)));
2493 : }
2494 1517888 : }
2495 :
2496 : /* Expand all variables used in the function. */
2497 :
2498 : static rtx_insn *
2499 1517908 : expand_used_vars (bitmap forced_stack_vars)
2500 : {
2501 1517908 : tree var, outer_block = DECL_INITIAL (current_function_decl);
2502 1517908 : auto_vec<tree> maybe_local_decls;
2503 1517908 : rtx_insn *var_end_seq = NULL;
2504 1517908 : unsigned i;
2505 1517908 : unsigned len;
2506 1517908 : bool gen_stack_protect_signal = false;
2507 :
2508 : /* Compute the phase of the stack frame for this function. */
2509 1517908 : {
2510 1517908 : int align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2511 1517908 : int off = targetm.starting_frame_offset () % align;
2512 1517908 : frame_phase = off ? align - off : 0;
2513 : }
2514 :
2515 : /* Set TREE_USED on all variables in the local_decls. */
2516 12081913 : FOR_EACH_LOCAL_DECL (cfun, i, var)
2517 8019405 : TREE_USED (var) = 1;
2518 : /* Clear TREE_USED on all variables associated with a block scope. */
2519 1517908 : clear_tree_used (DECL_INITIAL (current_function_decl));
2520 :
2521 1517908 : init_vars_expansion ();
2522 :
2523 1517908 : if (targetm.use_pseudo_pic_reg ())
2524 81059 : pic_offset_table_rtx = gen_reg_rtx (Pmode);
2525 :
2526 28229724 : for (i = 0; i < SA.map->num_partitions; i++)
2527 : {
2528 26711816 : if (bitmap_bit_p (SA.partitions_for_parm_default_defs, i))
2529 3687179 : continue;
2530 :
2531 23024637 : tree var = partition_to_var (SA.map, i);
2532 :
2533 46049274 : gcc_assert (!virtual_operand_p (var));
2534 :
2535 23024637 : expand_one_ssa_partition (var);
2536 : }
2537 :
2538 1517908 : if (flag_stack_protect == SPCT_FLAG_STRONG)
2539 418 : gen_stack_protect_signal = stack_protect_return_slot_p ();
2540 :
2541 : /* At this point all variables on the local_decls with TREE_USED
2542 : set are not associated with any block scope. Lay them out. */
2543 :
2544 1517908 : len = vec_safe_length (cfun->local_decls);
2545 10809613 : FOR_EACH_LOCAL_DECL (cfun, i, var)
2546 : {
2547 8019405 : bool expand_now = false;
2548 :
2549 : /* Expanded above already. */
2550 8019405 : if (is_gimple_reg (var))
2551 : {
2552 5618872 : TREE_USED (var) = 0;
2553 5618872 : goto next;
2554 : }
2555 : /* We didn't set a block for static or extern because it's hard
2556 : to tell the difference between a global variable (re)declared
2557 : in a local scope, and one that's really declared there to
2558 : begin with. And it doesn't really matter much, since we're
2559 : not giving them stack space. Expand them now. */
2560 2400533 : else if (TREE_STATIC (var) || DECL_EXTERNAL (var))
2561 : expand_now = true;
2562 :
2563 : /* Expand variables not associated with any block now. Those created by
2564 : the optimizers could be live anywhere in the function. Those that
2565 : could possibly have been scoped originally and detached from their
2566 : block will have their allocation deferred so we coalesce them with
2567 : others when optimization is enabled. */
2568 2181786 : else if (TREE_USED (var))
2569 1708982 : expand_now = true;
2570 :
2571 : /* Finally, mark all variables on the list as used. We'll use
2572 : this in a moment when we expand those associated with scopes. */
2573 2400533 : TREE_USED (var) = 1;
2574 :
2575 2400533 : if (expand_now)
2576 1708982 : expand_one_var (var, true, true, forced_stack_vars);
2577 :
2578 691551 : next:
2579 8019405 : if (DECL_ARTIFICIAL (var) && !DECL_IGNORED_P (var))
2580 : {
2581 445045 : rtx rtl = DECL_RTL_IF_SET (var);
2582 :
2583 : /* Keep artificial non-ignored vars in cfun->local_decls
2584 : chain until instantiate_decls. */
2585 336660 : if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2586 23280 : add_local_decl (cfun, var);
2587 108385 : else if (rtl == NULL_RTX)
2588 : /* If rtl isn't set yet, which can happen e.g. with
2589 : -fstack-protector, retry before returning from this
2590 : function. */
2591 108385 : maybe_local_decls.safe_push (var);
2592 : }
2593 : }
2594 :
2595 : /* We duplicated some of the decls in CFUN->LOCAL_DECLS.
2596 :
2597 : +-----------------+-----------------+
2598 : | ...processed... | ...duplicates...|
2599 : +-----------------+-----------------+
2600 : ^
2601 : +-- LEN points here.
2602 :
2603 : We just want the duplicates, as those are the artificial
2604 : non-ignored vars that we want to keep until instantiate_decls.
2605 : Move them down and truncate the array. */
2606 1517908 : if (!vec_safe_is_empty (cfun->local_decls))
2607 857825 : cfun->local_decls->block_remove (0, len);
2608 :
2609 : /* At this point, all variables within the block tree with TREE_USED
2610 : set are actually used by the optimized function. Lay them out. */
2611 1517908 : expand_used_vars_for_block (outer_block, true, forced_stack_vars);
2612 :
2613 1517908 : tree attribs = DECL_ATTRIBUTES (current_function_decl);
2614 1517908 : if (stack_vars_num > 0)
2615 : {
2616 236836 : bool has_addressable_vars = false;
2617 :
2618 236836 : add_scope_conflicts ();
2619 :
2620 : /* If stack protection is enabled, we don't share space between
2621 : vulnerable data and non-vulnerable data. */
2622 236836 : if (flag_stack_protect != 0
2623 261 : && !lookup_attribute ("no_stack_protector", attribs)
2624 237097 : && (flag_stack_protect != SPCT_FLAG_EXPLICIT
2625 : || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2626 1 : && lookup_attribute ("stack_protect", attribs))))
2627 261 : has_addressable_vars = add_stack_protection_conflicts ();
2628 :
2629 236836 : if (flag_stack_protect == SPCT_FLAG_STRONG && has_addressable_vars)
2630 236836 : gen_stack_protect_signal = true;
2631 :
2632 : /* Now that we have collected all stack variables, and have computed a
2633 : minimal interference graph, attempt to save some stack space. */
2634 236836 : partition_stack_vars ();
2635 236836 : if (dump_file)
2636 41 : dump_stack_var_partition ();
2637 : }
2638 :
2639 :
2640 1517908 : if (!lookup_attribute ("no_stack_protector", attribs))
2641 1517899 : switch (flag_stack_protect)
2642 : {
2643 51 : case SPCT_FLAG_ALL:
2644 51 : create_stack_guard ();
2645 51 : break;
2646 :
2647 418 : case SPCT_FLAG_STRONG:
2648 418 : if (gen_stack_protect_signal
2649 274 : || cfun->calls_alloca
2650 272 : || has_protected_decls
2651 681 : || lookup_attribute ("stack_protect", attribs))
2652 155 : create_stack_guard ();
2653 : break;
2654 :
2655 169 : case SPCT_FLAG_DEFAULT:
2656 169 : if (cfun->calls_alloca
2657 166 : || has_protected_decls
2658 258 : || lookup_attribute ("stack_protect", attribs))
2659 80 : create_stack_guard ();
2660 : break;
2661 :
2662 14 : case SPCT_FLAG_EXPLICIT:
2663 14 : if (lookup_attribute ("stack_protect", attribs))
2664 10 : create_stack_guard ();
2665 : break;
2666 :
2667 : default:
2668 : break;
2669 : }
2670 :
2671 : /* Assign rtl to each variable based on these partitions. */
2672 1517908 : if (stack_vars_num > 0)
2673 : {
2674 236836 : class stack_vars_data data;
2675 :
2676 236836 : data.asan_base = NULL_RTX;
2677 236836 : data.asan_alignb = 0;
2678 :
2679 : /* Reorder decls to be protected by iterating over the variables
2680 : array multiple times, and allocating out of each phase in turn. */
2681 : /* ??? We could probably integrate this into the qsort we did
2682 : earlier, such that we naturally see these variables first,
2683 : and thus naturally allocate things in the right order. */
2684 236836 : if (has_protected_decls)
2685 : {
2686 : /* Phase 1 contains only character arrays. */
2687 166 : expand_stack_vars (stack_protect_decl_phase_1, &data);
2688 :
2689 : /* Phase 2 contains other kinds of arrays. */
2690 166 : if (!lookup_attribute ("no_stack_protector", attribs)
2691 166 : && (flag_stack_protect == SPCT_FLAG_ALL
2692 166 : || flag_stack_protect == SPCT_FLAG_STRONG
2693 78 : || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2694 1 : && lookup_attribute ("stack_protect", attribs))))
2695 89 : expand_stack_vars (stack_protect_decl_phase_2, &data);
2696 : }
2697 :
2698 236836 : if (asan_sanitize_stack_p ())
2699 : /* Phase 3, any partitions that need asan protection
2700 : in addition to phase 1 and 2. */
2701 1952 : expand_stack_vars (asan_decl_phase_3, &data);
2702 :
2703 : /* ASAN description strings don't yet have a syntax for expressing
2704 : polynomial offsets. */
2705 236836 : HOST_WIDE_INT prev_offset;
2706 238574 : if (!data.asan_vec.is_empty ()
2707 1738 : && frame_offset.is_constant (&prev_offset))
2708 : {
2709 1738 : HOST_WIDE_INT offset, sz, redzonesz;
2710 1738 : redzonesz = ASAN_RED_ZONE_SIZE;
2711 1738 : sz = data.asan_vec[0] - prev_offset;
2712 1738 : data.asan_alignb = MAX (data.asan_alignb,
2713 : crtl->stack_alignment_needed
2714 : / BITS_PER_UNIT);
2715 1738 : if (data.asan_alignb > ASAN_RED_ZONE_SIZE
2716 1738 : && data.asan_alignb <= 4096
2717 54 : && sz + ASAN_RED_ZONE_SIZE >= (int) data.asan_alignb)
2718 54 : redzonesz = ((sz + ASAN_RED_ZONE_SIZE + data.asan_alignb - 1)
2719 54 : & ~(data.asan_alignb - HOST_WIDE_INT_1)) - sz;
2720 : /* Allocating a constant amount of space from a constant
2721 : starting offset must give a constant result. */
2722 1738 : offset = (alloc_stack_frame_space (redzonesz, ASAN_RED_ZONE_SIZE)
2723 1738 : .to_constant ());
2724 1738 : data.asan_vec.safe_push (prev_offset);
2725 1738 : data.asan_vec.safe_push (offset);
2726 : /* Leave space for alignment if STRICT_ALIGNMENT. */
2727 1738 : if (STRICT_ALIGNMENT)
2728 : alloc_stack_frame_space ((GET_MODE_ALIGNMENT (SImode)
2729 : << ASAN_SHADOW_SHIFT)
2730 : / BITS_PER_UNIT, 1);
2731 :
2732 1738 : var_end_seq
2733 3476 : = asan_emit_stack_protection (virtual_stack_vars_rtx,
2734 : data.asan_base,
2735 : data.asan_alignb,
2736 : data.asan_vec.address (),
2737 : data.asan_decl_vec.address (),
2738 1738 : data.asan_vec.length ());
2739 : }
2740 :
2741 236836 : expand_stack_vars (NULL, &data);
2742 236836 : }
2743 :
2744 1517908 : if (hwassist_sanitize_stack_p ())
2745 368 : hwasan_emit_prologue ();
2746 1517908 : if (asan_sanitize_allocas_p () && cfun->calls_alloca)
2747 178 : var_end_seq = asan_emit_allocas_unpoison (virtual_stack_dynamic_rtx,
2748 : virtual_stack_vars_rtx,
2749 : var_end_seq);
2750 3035092 : else if ((hwasan_sanitize_allocas_p () || memtag_sanitize_p ())
2751 1517730 : && cfun->calls_alloca)
2752 : /* When using out-of-line instrumentation we only want to emit one function
2753 : call for clearing the tags in a region of shadow stack. When there are
2754 : alloca calls in this frame we want to emit a call using the
2755 : virtual_stack_dynamic_rtx, but when not we use the hwasan_frame_extent
2756 : rtx we created in expand_stack_vars. */
2757 0 : var_end_seq = hwasan_emit_untag_frame (virtual_stack_dynamic_rtx,
2758 : virtual_stack_vars_rtx);
2759 1517730 : else if (hwassist_sanitize_stack_p ())
2760 : /* If no variables were stored on the stack, `hwasan_get_frame_extent`
2761 : will return NULL_RTX and hence `hwasan_emit_untag_frame` will return
2762 : NULL (i.e. an empty sequence). */
2763 368 : var_end_seq = hwasan_emit_untag_frame (hwasan_get_frame_extent (),
2764 : virtual_stack_vars_rtx);
2765 :
2766 1517908 : fini_vars_expansion ();
2767 :
2768 : /* If there were any artificial non-ignored vars without rtl
2769 : found earlier, see if deferred stack allocation hasn't assigned
2770 : rtl to them. */
2771 1715571 : FOR_EACH_VEC_ELT_REVERSE (maybe_local_decls, i, var)
2772 : {
2773 108385 : rtx rtl = DECL_RTL_IF_SET (var);
2774 :
2775 : /* Keep artificial non-ignored vars in cfun->local_decls
2776 : chain until instantiate_decls. */
2777 5145 : if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2778 4888 : add_local_decl (cfun, var);
2779 : }
2780 :
2781 : /* If the target requires that FRAME_OFFSET be aligned, do it. */
2782 1517908 : if (STACK_ALIGNMENT_NEEDED)
2783 : {
2784 1517908 : HOST_WIDE_INT align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2785 1517908 : if (FRAME_GROWS_DOWNWARD)
2786 1517908 : frame_offset = aligned_lower_bound (frame_offset, align);
2787 : else
2788 : frame_offset = aligned_upper_bound (frame_offset, align);
2789 : }
2790 :
2791 1517908 : return var_end_seq;
2792 1517908 : }
2793 :
2794 :
2795 : /* If we need to produce a detailed dump, print the tree representation
2796 : for STMT to the dump file. SINCE is the last RTX after which the RTL
2797 : generated for STMT should have been appended. */
2798 :
2799 : static void
2800 43076344 : maybe_dump_rtl_for_gimple_stmt (gimple *stmt, rtx_insn *since)
2801 : {
2802 43076344 : if (dump_file && (dump_flags & TDF_DETAILS))
2803 : {
2804 407 : fprintf (dump_file, "\n;; ");
2805 407 : print_gimple_stmt (dump_file, stmt, 0,
2806 : TDF_SLIM | (dump_flags & TDF_LINENO));
2807 407 : fprintf (dump_file, "\n");
2808 :
2809 407 : print_rtl (dump_file, since ? NEXT_INSN (since) : since);
2810 : }
2811 43076344 : }
2812 :
2813 : /* Temporary storage for BB_HEAD and BB_END of bbs until they are converted
2814 : to BB_RTL. */
2815 : static vec<std::pair <rtx_insn *, rtx_insn *>> head_end_for_bb;
2816 :
2817 : /* Maps the blocks that do not contain tree labels to rtx labels. */
2818 :
2819 : static hash_map<basic_block, rtx_code_label *> *lab_rtx_for_bb;
2820 :
2821 : /* Returns the label_rtx expression for a label starting basic block BB. */
2822 :
2823 : static rtx_code_label *
2824 7846191 : label_rtx_for_bb (basic_block bb)
2825 : {
2826 7846191 : if (bb->flags & BB_RTL)
2827 0 : return block_label (bb);
2828 :
2829 7846191 : if ((unsigned) bb->index < head_end_for_bb.length ()
2830 7846191 : && head_end_for_bb[bb->index].first)
2831 : {
2832 1598741 : if (!LABEL_P (head_end_for_bb[bb->index].first))
2833 : {
2834 972157 : head_end_for_bb[bb->index].first
2835 1944314 : = emit_label_before (gen_label_rtx (),
2836 972157 : head_end_for_bb[bb->index].first);
2837 : }
2838 1598741 : return as_a <rtx_code_label *> (head_end_for_bb[bb->index].first);
2839 : }
2840 :
2841 6247450 : rtx_code_label **elt = lab_rtx_for_bb->get (bb);
2842 6247450 : if (elt)
2843 1303808 : return *elt;
2844 :
2845 : /* Find the tree label if it is present. */
2846 4943642 : gimple_stmt_iterator gsi = gsi_start_bb (bb);
2847 4943642 : glabel *lab_stmt;
2848 4943642 : if (!gsi_end_p (gsi)
2849 4796700 : && (lab_stmt = dyn_cast <glabel *> (gsi_stmt (gsi)))
2850 5180973 : && !DECL_NONLOCAL (gimple_label_label (lab_stmt)))
2851 237331 : return jump_target_rtx (gimple_label_label (lab_stmt));
2852 :
2853 4706311 : rtx_code_label *l = gen_label_rtx ();
2854 4706311 : lab_rtx_for_bb->put (bb, l);
2855 4706311 : return l;
2856 : }
2857 :
2858 :
2859 : /* Wrapper around remove_edge during expansion. */
2860 :
2861 : void
2862 128613 : expand_remove_edge (edge e)
2863 : {
2864 128613 : if (current_ir_type () != IR_GIMPLE
2865 128613 : && (e->dest->flags & BB_RTL) == 0
2866 189321 : && !gimple_seq_empty_p (phi_nodes (e->dest)))
2867 9169 : remove_phi_args (e);
2868 128613 : remove_edge (e);
2869 128613 : }
2870 :
2871 : /* Split edge E during expansion and instead of creating a new
2872 : bb on that edge, add there BB. FLAGS should be flags on the
2873 : new edge from BB to former E->dest. */
2874 :
2875 : static void
2876 1920278 : expand_split_edge (edge e, basic_block bb, int flags)
2877 : {
2878 1920278 : unsigned int dest_idx = e->dest_idx;
2879 1920278 : basic_block dest = e->dest;
2880 1920278 : redirect_edge_succ (e, bb);
2881 1920278 : e = make_single_succ_edge (bb, dest, flags);
2882 1920278 : if ((dest->flags & BB_RTL) == 0
2883 1920278 : && phi_nodes (dest)
2884 2059514 : && e->dest_idx != dest_idx)
2885 : {
2886 : /* If there are any PHI nodes on dest, swap the new succ edge
2887 : with the one moved into false_edge's former position, so that
2888 : PHI arguments don't need adjustment. */
2889 86305 : edge e2 = EDGE_PRED (dest, dest_idx);
2890 86305 : std::swap (e->dest_idx, e2->dest_idx);
2891 86305 : std::swap (EDGE_PRED (dest, e->dest_idx),
2892 : EDGE_PRED (dest, e2->dest_idx));
2893 : }
2894 1920278 : }
2895 :
2896 :
2897 : /* A subroutine of expand_gimple_cond. Given E, a fallthrough edge
2898 : of a basic block where we just expanded the conditional at the end,
2899 : possibly clean up the CFG and instruction sequence. LAST is the
2900 : last instruction before the just emitted jump sequence. */
2901 :
2902 : static void
2903 5378338 : maybe_cleanup_end_of_block (edge e, rtx_insn *last)
2904 : {
2905 : /* Special case: when jumpif decides that the condition is
2906 : trivial it emits an unconditional jump (and the necessary
2907 : barrier). But we still have two edges, the fallthru one is
2908 : wrong. purge_dead_edges would clean this up later. Unfortunately
2909 : we have to insert insns (and split edges) before
2910 : find_many_sub_basic_blocks and hence before purge_dead_edges.
2911 : But splitting edges might create new blocks which depend on the
2912 : fact that if there are two edges there's no barrier. So the
2913 : barrier would get lost and verify_flow_info would ICE. Instead
2914 : of auditing all edge splitters to care for the barrier (which
2915 : normally isn't there in a cleaned CFG), fix it here. */
2916 5378338 : if (BARRIER_P (get_last_insn ()))
2917 : {
2918 115 : rtx_insn *insn;
2919 115 : expand_remove_edge (e);
2920 : /* Now, we have a single successor block, if we have insns to
2921 : insert on the remaining edge we potentially will insert
2922 : it at the end of this block (if the dest block isn't feasible)
2923 : in order to avoid splitting the edge. This insertion will take
2924 : place in front of the last jump. But we might have emitted
2925 : multiple jumps (conditional and one unconditional) to the
2926 : same destination. Inserting in front of the last one then
2927 : is a problem. See PR 40021. We fix this by deleting all
2928 : jumps except the last unconditional one. */
2929 115 : insn = PREV_INSN (get_last_insn ());
2930 : /* Make sure we have an unconditional jump. Otherwise we're
2931 : confused. */
2932 115 : gcc_assert (JUMP_P (insn) && !any_condjump_p (insn));
2933 230 : for (insn = PREV_INSN (insn); insn != last;)
2934 : {
2935 0 : insn = PREV_INSN (insn);
2936 0 : if (JUMP_P (NEXT_INSN (insn)))
2937 : {
2938 0 : if (!any_condjump_p (NEXT_INSN (insn)))
2939 : {
2940 0 : gcc_assert (BARRIER_P (NEXT_INSN (NEXT_INSN (insn))));
2941 0 : delete_insn (NEXT_INSN (NEXT_INSN (insn)));
2942 : }
2943 0 : delete_insn (NEXT_INSN (insn));
2944 : }
2945 : }
2946 : }
2947 5378338 : }
2948 :
2949 : /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_COND.
2950 : Returns a new basic block if we've terminated the current basic
2951 : block and created a new one. */
2952 :
2953 : static basic_block
2954 5780708 : expand_gimple_cond (basic_block bb, gcond *stmt)
2955 : {
2956 5780708 : basic_block new_bb, dest;
2957 5780708 : edge true_edge;
2958 5780708 : edge false_edge;
2959 5780708 : rtx_insn *last2, *last;
2960 5780708 : enum tree_code code;
2961 5780708 : tree op0, op1;
2962 :
2963 5780708 : code = gimple_cond_code (stmt);
2964 5780708 : op0 = gimple_cond_lhs (stmt);
2965 5780708 : op1 = gimple_cond_rhs (stmt);
2966 : /* We're sometimes presented with such code:
2967 : D.123_1 = x < y;
2968 : if (D.123_1 != 0)
2969 : ...
2970 : This would expand to two comparisons which then later might
2971 : be cleaned up by combine. But some pattern matchers like if-conversion
2972 : work better when there's only one compare, so make up for this
2973 : here as special exception if TER would have made the same change. */
2974 5780708 : if (SA.values
2975 4233799 : && TREE_CODE (op0) == SSA_NAME
2976 4233093 : && TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE
2977 399836 : && TREE_CODE (op1) == INTEGER_CST
2978 389063 : && ((gimple_cond_code (stmt) == NE_EXPR
2979 389061 : && integer_zerop (op1))
2980 2 : || (gimple_cond_code (stmt) == EQ_EXPR
2981 2 : && integer_onep (op1)))
2982 6169771 : && bitmap_bit_p (SA.values, SSA_NAME_VERSION (op0)))
2983 : {
2984 212452 : gimple *second = SSA_NAME_DEF_STMT (op0);
2985 212452 : if (gimple_code (second) == GIMPLE_ASSIGN)
2986 : {
2987 212433 : enum tree_code code2 = gimple_assign_rhs_code (second);
2988 212433 : if (TREE_CODE_CLASS (code2) == tcc_comparison)
2989 : {
2990 1220 : code = code2;
2991 1220 : op0 = gimple_assign_rhs1 (second);
2992 1220 : op1 = gimple_assign_rhs2 (second);
2993 : }
2994 : /* If jumps are cheap and the target does not support conditional
2995 : compare, turn some more codes into jumpy sequences. */
2996 211213 : else if (BRANCH_COST (optimize_insn_for_speed_p (), false) < 4
2997 211213 : && !targetm.have_ccmp ())
2998 : {
2999 211210 : if ((code2 == BIT_AND_EXPR
3000 69766 : && TYPE_PRECISION (TREE_TYPE (op0)) == 1
3001 69766 : && TREE_CODE (gimple_assign_rhs2 (second)) != INTEGER_CST)
3002 211210 : || code2 == TRUTH_AND_EXPR)
3003 : {
3004 69766 : code = TRUTH_ANDIF_EXPR;
3005 69766 : op0 = gimple_assign_rhs1 (second);
3006 69766 : op1 = gimple_assign_rhs2 (second);
3007 : }
3008 141444 : else if (code2 == BIT_IOR_EXPR || code2 == TRUTH_OR_EXPR)
3009 : {
3010 62491 : code = TRUTH_ORIF_EXPR;
3011 62491 : op0 = gimple_assign_rhs1 (second);
3012 62491 : op1 = gimple_assign_rhs2 (second);
3013 : }
3014 : }
3015 : }
3016 : }
3017 :
3018 : /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
3019 : into (x - C2) * C3 < C4. */
3020 5780708 : if ((code == EQ_EXPR || code == NE_EXPR)
3021 4529881 : && TREE_CODE (op0) == SSA_NAME
3022 4529122 : && TREE_CODE (op1) == INTEGER_CST)
3023 3136410 : code = maybe_optimize_mod_cmp (code, &op0, &op1);
3024 :
3025 : /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
3026 5780708 : if (!TYPE_UNSIGNED (TREE_TYPE (op0))
3027 2329183 : && (code == LT_EXPR || code == LE_EXPR
3028 2329183 : || code == GT_EXPR || code == GE_EXPR)
3029 695830 : && integer_zerop (op1)
3030 5985776 : && TREE_CODE (op0) == SSA_NAME)
3031 205068 : maybe_optimize_sub_cmp_0 (code, &op0, &op1);
3032 :
3033 5780708 : last2 = last = get_last_insn ();
3034 :
3035 5780708 : extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
3036 5780708 : set_curr_insn_location (gimple_location (stmt));
3037 :
3038 : /* We can either have a pure conditional jump with one fallthru edge or
3039 : two-way jump that needs to be decomposed into two basic blocks. */
3040 5780708 : if (false_edge->dest == bb->next_bb)
3041 : {
3042 2311682 : jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
3043 : true_edge->probability);
3044 2311682 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
3045 2311682 : if (true_edge->goto_locus != UNKNOWN_LOCATION)
3046 341511 : set_curr_insn_location (true_edge->goto_locus);
3047 2311682 : false_edge->flags |= EDGE_FALLTHRU;
3048 2311682 : maybe_cleanup_end_of_block (false_edge, last);
3049 2311682 : return NULL;
3050 : }
3051 3469026 : if (true_edge->dest == bb->next_bb)
3052 : {
3053 3066656 : jumpifnot_1 (code, op0, op1, label_rtx_for_bb (false_edge->dest),
3054 : false_edge->probability);
3055 3066656 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
3056 3066656 : if (false_edge->goto_locus != UNKNOWN_LOCATION)
3057 24956 : set_curr_insn_location (false_edge->goto_locus);
3058 3066656 : true_edge->flags |= EDGE_FALLTHRU;
3059 3066656 : maybe_cleanup_end_of_block (true_edge, last);
3060 3066656 : return NULL;
3061 : }
3062 :
3063 402370 : jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
3064 : true_edge->probability);
3065 402370 : last = get_last_insn ();
3066 402370 : if (false_edge->goto_locus != UNKNOWN_LOCATION)
3067 1337 : set_curr_insn_location (false_edge->goto_locus);
3068 402370 : emit_jump (label_rtx_for_bb (false_edge->dest));
3069 :
3070 402370 : head_end_for_bb[bb->index].second = last;
3071 402370 : if (BARRIER_P (head_end_for_bb[bb->index].second))
3072 102 : head_end_for_bb[bb->index].second
3073 51 : = PREV_INSN (head_end_for_bb[bb->index].second);
3074 804740 : update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
3075 402370 : head_end_for_bb[bb->index].second, bb);
3076 :
3077 402370 : new_bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
3078 402370 : dest = false_edge->dest;
3079 402370 : expand_split_edge (false_edge, new_bb, 0);
3080 402370 : false_edge->flags |= EDGE_FALLTHRU;
3081 402370 : new_bb->count = false_edge->count ();
3082 402370 : loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
3083 402370 : add_bb_to_loop (new_bb, loop);
3084 402370 : if (loop->latch == bb
3085 28144 : && loop->header == dest)
3086 28135 : loop->latch = new_bb;
3087 402370 : if (BARRIER_P (BB_END (new_bb)))
3088 402370 : BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
3089 402370 : update_bb_for_insn (new_bb);
3090 :
3091 402370 : maybe_dump_rtl_for_gimple_stmt (stmt, last2);
3092 :
3093 402370 : if (true_edge->goto_locus != UNKNOWN_LOCATION)
3094 : {
3095 45671 : set_curr_insn_location (true_edge->goto_locus);
3096 45671 : true_edge->goto_locus = curr_insn_location ();
3097 : }
3098 :
3099 : return new_bb;
3100 : }
3101 :
3102 : /* Mark all calls that can have a transaction restart. */
3103 :
3104 : static void
3105 6840444 : mark_transaction_restart_calls (gimple *stmt)
3106 : {
3107 6840444 : struct tm_restart_node dummy;
3108 6840444 : tm_restart_node **slot;
3109 :
3110 6840444 : if (!cfun->gimple_df->tm_restart)
3111 6837460 : return;
3112 :
3113 2984 : dummy.stmt = stmt;
3114 2984 : slot = cfun->gimple_df->tm_restart->find_slot (&dummy, NO_INSERT);
3115 2984 : if (slot)
3116 : {
3117 0 : struct tm_restart_node *n = *slot;
3118 0 : tree list = n->label_or_list;
3119 0 : rtx_insn *insn;
3120 :
3121 0 : for (insn = next_real_insn (get_last_insn ());
3122 0 : !CALL_P (insn);
3123 0 : insn = next_real_insn (insn))
3124 0 : continue;
3125 :
3126 0 : if (TREE_CODE (list) == LABEL_DECL)
3127 0 : add_reg_note (insn, REG_TM, label_rtx (list));
3128 : else
3129 0 : for (; list ; list = TREE_CHAIN (list))
3130 0 : add_reg_note (insn, REG_TM, label_rtx (TREE_VALUE (list)));
3131 0 : }
3132 : }
3133 :
3134 : /* A subroutine of expand_gimple_stmt_1, expanding one GIMPLE_CALL
3135 : statement STMT. */
3136 :
3137 : static void
3138 7072400 : expand_call_stmt (gcall *stmt)
3139 : {
3140 7072400 : tree exp, decl, lhs;
3141 7072400 : bool builtin_p;
3142 7072400 : size_t i;
3143 :
3144 7072400 : if (gimple_call_internal_p (stmt))
3145 : {
3146 198367 : expand_internal_call (stmt);
3147 198367 : return;
3148 : }
3149 :
3150 : /* If this is a call to a built-in function and it has no effect other
3151 : than setting the lhs, try to implement it using an internal function
3152 : instead. */
3153 6874033 : decl = gimple_call_fndecl (stmt);
3154 6874033 : if (gimple_call_lhs (stmt)
3155 2420565 : && !gimple_has_side_effects (stmt)
3156 7585733 : && (optimize || (decl && called_as_built_in (decl))))
3157 : {
3158 690973 : internal_fn ifn = replacement_internal_fn (stmt);
3159 690973 : if (ifn != IFN_LAST)
3160 : {
3161 33587 : expand_internal_call (ifn, stmt);
3162 33587 : return;
3163 : }
3164 : }
3165 :
3166 6840446 : exp = build_vl_exp (CALL_EXPR, gimple_call_num_args (stmt) + 3);
3167 :
3168 6840446 : CALL_EXPR_FN (exp) = gimple_call_fn (stmt);
3169 6840446 : builtin_p = decl && fndecl_built_in_p (decl);
3170 :
3171 : /* If this is not a builtin function, the function type through which the
3172 : call is made may be different from the type of the function. */
3173 4810930 : if (!builtin_p)
3174 9621860 : CALL_EXPR_FN (exp)
3175 14432790 : = fold_convert (build_pointer_type (gimple_call_fntype (stmt)),
3176 : CALL_EXPR_FN (exp));
3177 :
3178 6840446 : TREE_TYPE (exp) = gimple_call_return_type (stmt);
3179 6840446 : CALL_EXPR_STATIC_CHAIN (exp) = gimple_call_chain (stmt);
3180 :
3181 20612895 : for (i = 0; i < gimple_call_num_args (stmt); i++)
3182 : {
3183 13772449 : tree arg = gimple_call_arg (stmt, i);
3184 13772449 : gimple *def;
3185 : /* TER addresses into arguments of builtin functions so we have a
3186 : chance to infer more correct alignment information. See PR39954. */
3187 13772449 : if (builtin_p
3188 4216415 : && TREE_CODE (arg) == SSA_NAME
3189 1439033 : && (def = get_gimple_for_ssa_name (arg))
3190 354609 : && is_gimple_assign (def)
3191 14126897 : && gimple_assign_rhs_code (def) == ADDR_EXPR)
3192 14882 : arg = gimple_assign_rhs1 (def);
3193 13772449 : CALL_EXPR_ARG (exp, i) = arg;
3194 : }
3195 :
3196 6840446 : if (gimple_has_side_effects (stmt)
3197 : /* ??? Downstream in expand_expr_real_1 we assume that expressions
3198 : w/o side-effects do not throw so work around this here. */
3199 6840446 : || stmt_could_throw_p (cfun, stmt))
3200 6164306 : TREE_SIDE_EFFECTS (exp) = 1;
3201 :
3202 6840446 : if (gimple_call_nothrow_p (stmt))
3203 3062653 : TREE_NOTHROW (exp) = 1;
3204 :
3205 6840446 : CALL_EXPR_TAILCALL (exp) = gimple_call_tail_p (stmt);
3206 6840446 : CALL_EXPR_MUST_TAIL_CALL (exp) = gimple_call_must_tail_p (stmt);
3207 6840446 : CALL_EXPR_RETURN_SLOT_OPT (exp) = gimple_call_return_slot_opt_p (stmt);
3208 6840446 : if (decl
3209 6653440 : && fndecl_built_in_p (decl, BUILT_IN_NORMAL)
3210 8692282 : && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
3211 27680 : CALL_ALLOCA_FOR_VAR_P (exp) = gimple_call_alloca_for_var_p (stmt);
3212 : else
3213 6812766 : CALL_FROM_THUNK_P (exp) = gimple_call_from_thunk_p (stmt);
3214 6840446 : CALL_EXPR_VA_ARG_PACK (exp) = gimple_call_va_arg_pack_p (stmt);
3215 6840446 : CALL_EXPR_BY_DESCRIPTOR (exp) = gimple_call_by_descriptor_p (stmt);
3216 6840446 : SET_EXPR_LOCATION (exp, gimple_location (stmt));
3217 :
3218 : /* Must come after copying location. */
3219 6840446 : copy_warning (exp, stmt);
3220 :
3221 : /* Ensure RTL is created for debug args. */
3222 13493886 : if (decl && DECL_HAS_DEBUG_ARGS_P (decl))
3223 : {
3224 68666 : vec<tree, va_gc> **debug_args = decl_debug_args_lookup (decl);
3225 68666 : unsigned int ix;
3226 68666 : tree dtemp;
3227 :
3228 68666 : if (debug_args)
3229 150761 : for (ix = 1; (*debug_args)->iterate (ix, &dtemp); ix += 2)
3230 : {
3231 82095 : gcc_assert (TREE_CODE (dtemp) == DEBUG_EXPR_DECL);
3232 82095 : expand_debug_expr (dtemp);
3233 : }
3234 : }
3235 :
3236 6840446 : rtx_insn *before_call = get_last_insn ();
3237 6840446 : lhs = gimple_call_lhs (stmt);
3238 6840446 : if (lhs)
3239 2386978 : expand_assignment (lhs, exp, false);
3240 : else
3241 4453468 : expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
3242 :
3243 : /* If the gimple call is an indirect call and has 'nocf_check'
3244 : attribute find a generated CALL insn to mark it as no
3245 : control-flow verification is needed. */
3246 6840444 : if (gimple_call_nocf_check_p (stmt)
3247 6840465 : && !gimple_call_fndecl (stmt))
3248 : {
3249 20 : rtx_insn *last = get_last_insn ();
3250 20 : while (!CALL_P (last)
3251 34 : && last != before_call)
3252 14 : last = PREV_INSN (last);
3253 :
3254 20 : if (last != before_call)
3255 20 : add_reg_note (last, REG_CALL_NOCF_CHECK, const0_rtx);
3256 : }
3257 :
3258 6840444 : mark_transaction_restart_calls (stmt);
3259 : }
3260 :
3261 :
3262 : /* Generate RTL for an asm statement (explicit assembler code).
3263 : STRING is a STRING_CST node containing the assembler code text,
3264 : or an ADDR_EXPR containing a STRING_CST. VOL nonzero means the
3265 : insn is volatile; don't optimize it. */
3266 :
3267 : static void
3268 2876 : expand_asm_loc (tree string, int vol, location_t locus)
3269 : {
3270 2876 : rtx body;
3271 :
3272 2876 : body = gen_rtx_ASM_INPUT_loc (VOIDmode,
3273 : ggc_strdup (TREE_STRING_POINTER (string)),
3274 : locus);
3275 :
3276 2876 : MEM_VOLATILE_P (body) = vol;
3277 :
3278 : /* Non-empty basic ASM implicitly clobbers memory. */
3279 2876 : if (TREE_STRING_LENGTH (string) != 0)
3280 : {
3281 858 : rtx asm_op, clob;
3282 858 : unsigned i, nclobbers;
3283 858 : auto_vec<rtx> input_rvec, output_rvec;
3284 858 : auto_vec<machine_mode> input_mode;
3285 858 : auto_vec<const char *> constraints;
3286 858 : auto_vec<rtx> use_rvec;
3287 858 : auto_vec<rtx> clobber_rvec;
3288 858 : HARD_REG_SET clobbered_regs;
3289 858 : CLEAR_HARD_REG_SET (clobbered_regs);
3290 :
3291 858 : clob = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3292 858 : clobber_rvec.safe_push (clob);
3293 :
3294 858 : if (targetm.md_asm_adjust)
3295 858 : targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3296 : constraints, use_rvec, clobber_rvec,
3297 : clobbered_regs, locus);
3298 :
3299 858 : asm_op = body;
3300 858 : nclobbers = clobber_rvec.length ();
3301 858 : auto nuses = use_rvec.length ();
3302 858 : body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (1 + nuses + nclobbers));
3303 :
3304 858 : i = 0;
3305 858 : XVECEXP (body, 0, i++) = asm_op;
3306 858 : for (rtx use : use_rvec)
3307 0 : XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
3308 4290 : for (rtx clobber : clobber_rvec)
3309 1716 : XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobber);
3310 858 : }
3311 :
3312 2876 : emit_insn (body);
3313 2876 : }
3314 :
3315 : /* Check for overlap between registers marked in CLOBBERED_REGS and
3316 : anything inappropriate in T. Emit error and return the register
3317 : variable definition for error, NULL_TREE for ok. */
3318 :
3319 : static bool
3320 138513 : tree_conflicts_with_clobbers_p (tree t, HARD_REG_SET *clobbered_regs,
3321 : location_t loc)
3322 : {
3323 : /* Conflicts between asm-declared register variables and the clobber
3324 : list are not allowed. */
3325 138513 : tree overlap = tree_overlaps_hard_reg_set (t, clobbered_regs);
3326 :
3327 138513 : if (overlap)
3328 : {
3329 21 : error_at (loc, "%<asm%> specifier for variable %qE conflicts with "
3330 21 : "%<asm%> clobber list", DECL_NAME (overlap));
3331 :
3332 : /* Reset registerness to stop multiple errors emitted for a single
3333 : variable. */
3334 21 : DECL_REGISTER (overlap) = 0;
3335 21 : return true;
3336 : }
3337 :
3338 : return false;
3339 : }
3340 :
3341 : /* Check that the given REGNO spanning NREGS is a valid
3342 : asm clobber operand. Some HW registers cannot be
3343 : saved/restored, hence they should not be clobbered by
3344 : asm statements. */
3345 : static bool
3346 41660 : asm_clobber_reg_is_valid (int regno, int nregs, const char *regname)
3347 : {
3348 41660 : bool is_valid = true;
3349 41660 : HARD_REG_SET regset;
3350 :
3351 41660 : CLEAR_HARD_REG_SET (regset);
3352 :
3353 41660 : add_range_to_hard_reg_set (®set, regno, nregs);
3354 :
3355 : /* Clobbering the PIC register is an error. */
3356 41660 : if (PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
3357 0 : && overlaps_hard_reg_set_p (regset, Pmode, PIC_OFFSET_TABLE_REGNUM))
3358 : {
3359 : /* ??? Diagnose during gimplification? */
3360 0 : error ("PIC register clobbered by %qs in %<asm%>", regname);
3361 0 : is_valid = false;
3362 : }
3363 41660 : else if (!in_hard_reg_set_p
3364 41660 : (accessible_reg_set, reg_raw_mode[regno], regno))
3365 : {
3366 : /* ??? Diagnose during gimplification? */
3367 0 : error ("the register %qs cannot be clobbered in %<asm%>"
3368 : " for the current target", regname);
3369 0 : is_valid = false;
3370 : }
3371 :
3372 : /* Clobbering the stack pointer register is deprecated. GCC expects
3373 : the value of the stack pointer after an asm statement to be the same
3374 : as it was before, so no asm can validly clobber the stack pointer in
3375 : the usual sense. Adding the stack pointer to the clobber list has
3376 : traditionally had some undocumented and somewhat obscure side-effects. */
3377 41680 : if (overlaps_hard_reg_set_p (regset, Pmode, STACK_POINTER_REGNUM))
3378 : {
3379 1 : crtl->sp_is_clobbered_by_asm = true;
3380 1 : if (warning (OPT_Wdeprecated, "listing the stack pointer register"
3381 : " %qs in a clobber list is deprecated", regname))
3382 1 : inform (input_location, "the value of the stack pointer after"
3383 : " an %<asm%> statement must be the same as it was before"
3384 : " the statement");
3385 : }
3386 :
3387 41660 : return is_valid;
3388 : }
3389 :
3390 : /* Generate RTL for an asm statement with arguments.
3391 : STRING is the instruction template.
3392 : OUTPUTS is a list of output arguments (lvalues); INPUTS a list of inputs.
3393 : Each output or input has an expression in the TREE_VALUE and
3394 : a tree list in TREE_PURPOSE which in turn contains a constraint
3395 : name in TREE_VALUE (or NULL_TREE) and a constraint string
3396 : in TREE_PURPOSE.
3397 : CLOBBERS is a list of STRING_CST nodes each naming a hard register
3398 : that is clobbered by this insn.
3399 :
3400 : LABELS is a list of labels, and if LABELS is non-NULL, FALLTHRU_BB
3401 : should be the fallthru basic block of the asm goto.
3402 :
3403 : Not all kinds of lvalue that may appear in OUTPUTS can be stored directly.
3404 : Some elements of OUTPUTS may be replaced with trees representing temporary
3405 : values. The caller should copy those temporary values to the originally
3406 : specified lvalues.
3407 :
3408 : VOL nonzero means the insn is volatile; don't optimize it. */
3409 :
3410 : static void
3411 96681 : expand_asm_stmt (gasm *stmt)
3412 : {
3413 96681 : class save_input_location
3414 : {
3415 : location_t old;
3416 :
3417 : public:
3418 93805 : explicit save_input_location(location_t where)
3419 : {
3420 93805 : old = input_location;
3421 93805 : input_location = where;
3422 : }
3423 :
3424 93805 : ~save_input_location()
3425 : {
3426 93805 : input_location = old;
3427 5 : }
3428 : };
3429 :
3430 96681 : location_t locus = gimple_location (stmt);
3431 :
3432 96681 : if (gimple_asm_basic_p (stmt))
3433 : {
3434 2876 : const char *s = gimple_asm_string (stmt);
3435 2876 : tree string = build_string (strlen (s), s);
3436 2876 : expand_asm_loc (string, gimple_asm_volatile_p (stmt), locus);
3437 5757 : return;
3438 : }
3439 :
3440 : /* There are some legacy diagnostics in here. */
3441 93805 : save_input_location s_i_l(locus);
3442 :
3443 93805 : unsigned noutputs = gimple_asm_noutputs (stmt);
3444 93805 : unsigned ninputs = gimple_asm_ninputs (stmt);
3445 93805 : unsigned nlabels = gimple_asm_nlabels (stmt);
3446 93805 : unsigned i;
3447 93805 : bool error_seen = false;
3448 :
3449 : /* ??? Diagnose during gimplification? */
3450 93805 : if (ninputs + noutputs + nlabels > MAX_RECOG_OPERANDS)
3451 : {
3452 5 : error_at (locus, "more than %d operands in %<asm%>", MAX_RECOG_OPERANDS);
3453 5 : return;
3454 : }
3455 :
3456 93800 : auto_vec<tree, MAX_RECOG_OPERANDS> output_tvec;
3457 93800 : auto_vec<tree, MAX_RECOG_OPERANDS> input_tvec;
3458 93800 : auto_vec<const char *, MAX_RECOG_OPERANDS> constraints;
3459 :
3460 : /* Copy the gimple vectors into new vectors that we can manipulate. */
3461 :
3462 93800 : output_tvec.safe_grow (noutputs, true);
3463 93800 : input_tvec.safe_grow (ninputs, true);
3464 93800 : constraints.safe_grow (noutputs + ninputs, true);
3465 :
3466 267438 : for (i = 0; i < noutputs; ++i)
3467 : {
3468 79838 : tree t = gimple_asm_output_op (stmt, i);
3469 79838 : output_tvec[i] = TREE_VALUE (t);
3470 79838 : constraints[i] = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3471 : }
3472 152479 : for (i = 0; i < ninputs; i++)
3473 : {
3474 58679 : tree t = gimple_asm_input_op (stmt, i);
3475 58679 : input_tvec[i] = TREE_VALUE (t);
3476 58679 : constraints[i + noutputs]
3477 58679 : = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3478 : }
3479 :
3480 : /* Count the number of meaningful clobbered registers, ignoring what
3481 : we would ignore later. */
3482 93800 : auto_vec<rtx> clobber_rvec;
3483 93800 : HARD_REG_SET clobbered_regs;
3484 93800 : CLEAR_HARD_REG_SET (clobbered_regs);
3485 :
3486 93800 : if (unsigned n = gimple_asm_nclobbers (stmt))
3487 : {
3488 54574 : clobber_rvec.reserve (n);
3489 191475 : for (i = 0; i < n; i++)
3490 : {
3491 82327 : tree t = gimple_asm_clobber_op (stmt, i);
3492 82327 : const char *regname = TREE_STRING_POINTER (TREE_VALUE (t));
3493 82327 : int nregs, j;
3494 :
3495 82327 : j = decode_reg_name_and_count (regname, &nregs);
3496 82327 : if (j < 0)
3497 : {
3498 40667 : if (j == -2)
3499 : {
3500 : /* ??? Diagnose during gimplification? */
3501 15 : error_at (locus, "unknown register name %qs in %<asm%>",
3502 : regname);
3503 15 : error_seen = true;
3504 : }
3505 40652 : else if (j == -4)
3506 : {
3507 40273 : rtx x = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3508 40273 : clobber_rvec.safe_push (x);
3509 : }
3510 379 : else if (j == -5)
3511 : {
3512 2 : if (targetm.redzone_clobber)
3513 2 : if (rtx x = targetm.redzone_clobber ())
3514 2 : clobber_rvec.safe_push (x);
3515 : }
3516 : else
3517 : {
3518 : /* Otherwise we should have -1 == empty string
3519 : or -3 == cc, which is not a register. */
3520 377 : gcc_assert (j == -1 || j == -3);
3521 : }
3522 : }
3523 : else
3524 83320 : for (int reg = j; reg < j + nregs; reg++)
3525 : {
3526 41660 : if (!asm_clobber_reg_is_valid (reg, nregs, regname))
3527 0 : return;
3528 :
3529 41660 : SET_HARD_REG_BIT (clobbered_regs, reg);
3530 41660 : rtx x = gen_rtx_REG (reg_raw_mode[reg], reg);
3531 41660 : clobber_rvec.safe_push (x);
3532 : }
3533 : }
3534 : }
3535 :
3536 : /* First pass over inputs and outputs checks validity and sets
3537 : mark_addressable if needed. */
3538 : /* ??? Diagnose during gimplification? */
3539 :
3540 173638 : for (i = 0; i < noutputs; ++i)
3541 : {
3542 79838 : tree val = output_tvec[i];
3543 79838 : tree type = TREE_TYPE (val);
3544 79838 : const char *constraint;
3545 79838 : bool is_inout;
3546 79838 : bool allows_reg;
3547 79838 : bool allows_mem;
3548 :
3549 : /* Try to parse the output constraint. If that fails, there's
3550 : no point in going further. */
3551 79838 : constraint = constraints[i];
3552 79838 : if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
3553 : &allows_mem, &allows_reg, &is_inout,
3554 : nullptr))
3555 0 : return;
3556 :
3557 : /* If the output is a hard register, verify it doesn't conflict with
3558 : any other operand's possible hard register use. */
3559 79838 : if (DECL_P (val)
3560 7883 : && REG_P (DECL_RTL (val))
3561 80800 : && HARD_REGISTER_P (DECL_RTL (val)))
3562 : {
3563 942 : unsigned j, output_hregno = REGNO (DECL_RTL (val));
3564 942 : bool early_clobber_p = strchr (constraints[i], '&') != NULL;
3565 942 : unsigned long match;
3566 :
3567 : /* Verify the other outputs do not use the same hard register. */
3568 1102 : for (j = i + 1; j < noutputs; ++j)
3569 160 : if (DECL_P (output_tvec[j])
3570 159 : && REG_P (DECL_RTL (output_tvec[j]))
3571 159 : && HARD_REGISTER_P (DECL_RTL (output_tvec[j]))
3572 319 : && output_hregno == REGNO (DECL_RTL (output_tvec[j])))
3573 : {
3574 1 : error_at (locus, "invalid hard register usage between output "
3575 : "operands");
3576 1 : error_seen = true;
3577 : }
3578 :
3579 : /* Verify matching constraint operands use the same hard register
3580 : and that the non-matching constraint operands do not use the same
3581 : hard register if the output is an early clobber operand. */
3582 1857 : for (j = 0; j < ninputs; ++j)
3583 915 : if (DECL_P (input_tvec[j])
3584 776 : && REG_P (DECL_RTL (input_tvec[j]))
3585 1691 : && HARD_REGISTER_P (DECL_RTL (input_tvec[j])))
3586 : {
3587 776 : unsigned input_hregno = REGNO (DECL_RTL (input_tvec[j]));
3588 776 : switch (*constraints[j + noutputs])
3589 : {
3590 765 : case '0': case '1': case '2': case '3': case '4':
3591 765 : case '5': case '6': case '7': case '8': case '9':
3592 765 : match = strtoul (constraints[j + noutputs], NULL, 10);
3593 765 : break;
3594 : default:
3595 : match = ULONG_MAX;
3596 : break;
3597 : }
3598 765 : if (i == match
3599 665 : && output_hregno != input_hregno)
3600 : {
3601 1 : error_at (locus, "invalid hard register usage between "
3602 : "output operand and matching constraint operand");
3603 1 : error_seen = true;
3604 : }
3605 775 : else if (early_clobber_p
3606 4 : && i != match
3607 2 : && output_hregno == input_hregno)
3608 : {
3609 0 : error_at (locus, "invalid hard register usage between "
3610 : "earlyclobber operand and input operand");
3611 0 : error_seen = true;
3612 : }
3613 : }
3614 : }
3615 :
3616 79838 : if (! allows_reg
3617 79838 : && (allows_mem
3618 0 : || is_inout
3619 0 : || (DECL_P (val)
3620 0 : && REG_P (DECL_RTL (val))
3621 0 : && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type))))
3622 8347 : mark_addressable (val);
3623 : }
3624 :
3625 152479 : for (i = 0; i < ninputs; ++i)
3626 : {
3627 58679 : bool allows_reg, allows_mem;
3628 58679 : const char *constraint;
3629 :
3630 58679 : constraint = constraints[i + noutputs];
3631 58679 : if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3632 58679 : constraints.address (), &allows_mem,
3633 : &allows_reg, nullptr))
3634 0 : return;
3635 :
3636 58679 : if (! allows_reg && allows_mem)
3637 8994 : mark_addressable (input_tvec[i]);
3638 : }
3639 :
3640 : /* Second pass evaluates arguments. */
3641 :
3642 : /* Make sure stack is consistent for asm goto. */
3643 93800 : if (nlabels > 0)
3644 633 : do_pending_stack_adjust ();
3645 93800 : int old_generating_concat_p = generating_concat_p;
3646 :
3647 : /* Vector of RTX's of evaluated output operands. */
3648 187600 : auto_vec<rtx, MAX_RECOG_OPERANDS> output_rvec;
3649 93800 : auto_vec<int, MAX_RECOG_OPERANDS> inout_opnum;
3650 93800 : rtx_insn *after_rtl_seq = NULL, *after_rtl_end = NULL;
3651 :
3652 93800 : output_rvec.safe_grow (noutputs, true);
3653 :
3654 267438 : for (i = 0; i < noutputs; ++i)
3655 : {
3656 79838 : tree val = output_tvec[i];
3657 79838 : tree type = TREE_TYPE (val);
3658 79838 : bool is_inout, allows_reg, allows_mem, ok;
3659 79838 : rtx op;
3660 :
3661 79838 : ok = parse_output_constraint (&constraints[i], i, ninputs,
3662 : noutputs, &allows_mem, &allows_reg,
3663 : &is_inout, nullptr);
3664 79838 : gcc_assert (ok);
3665 :
3666 : /* If an output operand is not a decl or indirect ref and our constraint
3667 : allows a register, make a temporary to act as an intermediate.
3668 : Make the asm insn write into that, then we will copy it to
3669 : the real output operand. Likewise for promoted variables. */
3670 :
3671 79838 : generating_concat_p = 0;
3672 :
3673 79838 : gcc_assert (TREE_CODE (val) != INDIRECT_REF);
3674 79838 : if (((TREE_CODE (val) == MEM_REF
3675 3990 : && TREE_CODE (TREE_OPERAND (val, 0)) != ADDR_EXPR)
3676 3873 : && allows_mem)
3677 77701 : || (DECL_P (val)
3678 7883 : && (allows_mem || REG_P (DECL_RTL (val)))
3679 8131 : && ! (REG_P (DECL_RTL (val))
3680 962 : && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type)))
3681 70532 : || ! allows_reg
3682 70321 : || is_inout
3683 70321 : || TREE_ADDRESSABLE (type)
3684 150159 : || (!tree_fits_poly_int64_p (TYPE_SIZE (type))
3685 0 : && !known_size_p (max_int_size_in_bytes (type))))
3686 : {
3687 9517 : op = expand_expr (val, NULL_RTX, VOIDmode,
3688 : !allows_reg ? EXPAND_MEMORY : EXPAND_WRITE);
3689 9517 : if (MEM_P (op))
3690 8555 : op = validize_mem (op);
3691 :
3692 9517 : if (! allows_reg && !MEM_P (op))
3693 : {
3694 0 : error_at (locus, "output number %d not directly addressable", i);
3695 0 : error_seen = true;
3696 : }
3697 9517 : if ((! allows_mem && MEM_P (op) && GET_MODE (op) != BLKmode)
3698 9517 : || GET_CODE (op) == CONCAT)
3699 : {
3700 0 : rtx old_op = op;
3701 0 : op = gen_reg_rtx (GET_MODE (op));
3702 :
3703 0 : generating_concat_p = old_generating_concat_p;
3704 :
3705 0 : if (is_inout)
3706 0 : emit_move_insn (op, old_op);
3707 :
3708 0 : push_to_sequence2 (after_rtl_seq, after_rtl_end);
3709 0 : emit_move_insn (old_op, op);
3710 0 : after_rtl_seq = get_insns ();
3711 0 : after_rtl_end = get_last_insn ();
3712 0 : end_sequence ();
3713 : }
3714 : }
3715 : else
3716 : {
3717 70321 : op = assign_temp (type, 0, 1);
3718 70321 : op = validize_mem (op);
3719 70321 : if (!MEM_P (op) && TREE_CODE (val) == SSA_NAME)
3720 67122 : set_reg_attrs_for_decl_rtl (SSA_NAME_VAR (val), op);
3721 :
3722 70321 : generating_concat_p = old_generating_concat_p;
3723 :
3724 70321 : push_to_sequence2 (after_rtl_seq, after_rtl_end);
3725 70321 : expand_assignment (val, make_tree (type, op), false);
3726 70321 : after_rtl_seq = get_insns ();
3727 70321 : after_rtl_end = get_last_insn ();
3728 70321 : end_sequence ();
3729 : }
3730 79838 : output_rvec[i] = op;
3731 :
3732 79838 : if (is_inout)
3733 0 : inout_opnum.safe_push (i);
3734 : }
3735 :
3736 93800 : const char *str = gimple_asm_string (stmt);
3737 93800 : if (error_seen)
3738 : {
3739 17 : ninputs = 0;
3740 17 : noutputs = 0;
3741 17 : inout_opnum.truncate (0);
3742 17 : output_rvec.truncate (0);
3743 17 : clobber_rvec.truncate (0);
3744 17 : constraints.truncate (0);
3745 17 : CLEAR_HARD_REG_SET (clobbered_regs);
3746 : str = "";
3747 : }
3748 :
3749 187600 : auto_vec<rtx, MAX_RECOG_OPERANDS> input_rvec;
3750 93800 : auto_vec<machine_mode, MAX_RECOG_OPERANDS> input_mode;
3751 :
3752 93800 : input_rvec.safe_grow (ninputs, true);
3753 93800 : input_mode.safe_grow (ninputs, true);
3754 :
3755 93800 : generating_concat_p = 0;
3756 :
3757 152478 : for (i = 0; i < ninputs; ++i)
3758 : {
3759 58678 : tree val = input_tvec[i];
3760 58678 : tree type = TREE_TYPE (val);
3761 58678 : bool allows_reg, allows_mem, ok;
3762 58678 : const char *constraint;
3763 58678 : rtx op;
3764 :
3765 58678 : constraint = constraints[i + noutputs];
3766 58678 : ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3767 58678 : constraints.address (),
3768 : &allows_mem, &allows_reg, nullptr);
3769 58678 : gcc_assert (ok);
3770 :
3771 : /* EXPAND_INITIALIZER will not generate code for valid initializer
3772 : constants, but will still generate code for other types of operand.
3773 : This is the behavior we want for constant constraints. */
3774 58678 : op = expand_expr (val, NULL_RTX, VOIDmode,
3775 : allows_reg ? EXPAND_NORMAL
3776 10707 : : allows_mem ? EXPAND_MEMORY
3777 : : EXPAND_INITIALIZER);
3778 :
3779 : /* Never pass a CONCAT to an ASM. */
3780 58678 : if (GET_CODE (op) == CONCAT)
3781 2 : op = force_reg (GET_MODE (op), op);
3782 58676 : else if (MEM_P (op))
3783 : {
3784 13991 : op = validize_mem (op);
3785 13991 : MEM_NOTRAP_P (op) = !tree_could_trap_p (val);
3786 : }
3787 :
3788 58678 : if (asm_operand_ok (op, constraint, NULL) <= 0)
3789 : {
3790 26059 : if (allows_reg && TYPE_MODE (type) != BLKmode)
3791 26038 : op = force_reg (TYPE_MODE (type), op);
3792 21 : else if (!allows_mem)
3793 20 : warning_at (locus, 0, "%<asm%> operand %d probably does not match "
3794 : "constraints", i + noutputs);
3795 1 : else if (MEM_P (op))
3796 : {
3797 : /* We won't recognize either volatile memory or memory
3798 : with a queued address as available a memory_operand
3799 : at this point. Ignore it: clearly this *is* a memory. */
3800 : }
3801 : else
3802 0 : gcc_unreachable ();
3803 : }
3804 58678 : input_rvec[i] = op;
3805 58678 : input_mode[i] = TYPE_MODE (type);
3806 : }
3807 :
3808 : /* For in-out operands, copy output rtx to input rtx. */
3809 93800 : unsigned ninout = inout_opnum.length ();
3810 93800 : for (i = 0; i < ninout; i++)
3811 : {
3812 0 : int j = inout_opnum[i];
3813 0 : rtx o = output_rvec[j];
3814 :
3815 0 : input_rvec.safe_push (o);
3816 0 : input_mode.safe_push (GET_MODE (o));
3817 :
3818 0 : char buffer[16];
3819 0 : sprintf (buffer, "%d", j);
3820 0 : constraints.safe_push (ggc_strdup (buffer));
3821 : }
3822 93800 : ninputs += ninout;
3823 :
3824 : /* Sometimes we wish to automatically clobber registers across an asm.
3825 : Case in point is when the i386 backend moved from cc0 to a hard reg --
3826 : maintaining source-level compatibility means automatically clobbering
3827 : the flags register. */
3828 187600 : auto_vec<rtx> use_rvec;
3829 93800 : if (targetm.md_asm_adjust)
3830 : {
3831 93800 : rtx_insn *after_md_seq
3832 93800 : = targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3833 : constraints, use_rvec, clobber_rvec,
3834 : clobbered_regs, locus);
3835 93800 : if (after_md_seq)
3836 : {
3837 72 : push_to_sequence (after_md_seq);
3838 72 : emit_insn (after_rtl_seq);
3839 72 : after_rtl_seq = get_insns ();
3840 72 : after_rtl_end = get_last_insn ();
3841 72 : end_sequence ();
3842 : }
3843 : }
3844 :
3845 : /* Do not allow the hook to change the output and input count,
3846 : lest it mess up the operand numbering. */
3847 187600 : gcc_assert (output_rvec.length() == noutputs);
3848 187600 : gcc_assert (input_rvec.length() == ninputs);
3849 187600 : gcc_assert (constraints.length() == noutputs + ninputs);
3850 :
3851 : /* But it certainly can adjust the uses and clobbers. */
3852 93800 : unsigned nuses = use_rvec.length ();
3853 93800 : unsigned nclobbers = clobber_rvec.length ();
3854 :
3855 : /* Third pass checks for easy conflicts. */
3856 : /* ??? Why are we doing this on trees instead of rtx. */
3857 :
3858 93800 : bool clobber_conflict_found = 0;
3859 173635 : for (i = 0; i < noutputs; ++i)
3860 79835 : if (tree_conflicts_with_clobbers_p (output_tvec[i], &clobbered_regs, locus))
3861 10 : clobber_conflict_found = 1;
3862 152478 : for (i = 0; i < ninputs - ninout; ++i)
3863 58678 : if (tree_conflicts_with_clobbers_p (input_tvec[i], &clobbered_regs, locus))
3864 11 : clobber_conflict_found = 1;
3865 :
3866 : /* Make vectors for the expression-rtx, constraint strings,
3867 : and named operands. */
3868 :
3869 93800 : rtvec argvec = rtvec_alloc (ninputs);
3870 93800 : rtvec constraintvec = rtvec_alloc (ninputs);
3871 93800 : rtvec labelvec = rtvec_alloc (nlabels);
3872 :
3873 130072 : rtx body = gen_rtx_ASM_OPERANDS ((noutputs == 0 ? VOIDmode
3874 : : GET_MODE (output_rvec[0])),
3875 : ggc_strdup (str),
3876 : "", 0, argvec, constraintvec,
3877 : labelvec, locus);
3878 93800 : MEM_VOLATILE_P (body) = gimple_asm_volatile_p (stmt);
3879 :
3880 152478 : for (i = 0; i < ninputs; ++i)
3881 : {
3882 58678 : ASM_OPERANDS_INPUT (body, i) = input_rvec[i];
3883 117356 : ASM_OPERANDS_INPUT_CONSTRAINT_EXP (body, i)
3884 58678 : = gen_rtx_ASM_INPUT_loc (input_mode[i],
3885 : constraints[i + noutputs],
3886 : locus);
3887 : }
3888 :
3889 : /* Copy labels to the vector. */
3890 93800 : rtx_code_label *fallthru_label = NULL;
3891 93800 : if (nlabels > 0)
3892 : {
3893 633 : basic_block fallthru_bb = NULL;
3894 633 : edge fallthru = find_fallthru_edge (gimple_bb (stmt)->succs);
3895 633 : if (fallthru)
3896 633 : fallthru_bb = fallthru->dest;
3897 :
3898 1525 : for (i = 0; i < nlabels; ++i)
3899 : {
3900 892 : tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
3901 892 : rtx_insn *r;
3902 : /* If asm goto has any labels in the fallthru basic block, use
3903 : a label that we emit immediately after the asm goto. Expansion
3904 : may insert further instructions into the same basic block after
3905 : asm goto and if we don't do this, insertion of instructions on
3906 : the fallthru edge might misbehave. See PR58670. */
3907 892 : if (fallthru_bb && label_to_block (cfun, label) == fallthru_bb)
3908 : {
3909 294 : if (fallthru_label == NULL_RTX)
3910 289 : fallthru_label = gen_label_rtx ();
3911 : r = fallthru_label;
3912 : }
3913 : else
3914 598 : r = label_rtx (label);
3915 892 : ASM_OPERANDS_LABEL (body, i) = gen_rtx_LABEL_REF (Pmode, r);
3916 : }
3917 : }
3918 :
3919 : /* Now, for each output, construct an rtx
3920 : (set OUTPUT (asm_operands INSN OUTPUTCONSTRAINT OUTPUTNUMBER
3921 : ARGVEC CONSTRAINTS OPNAMES))
3922 : If there is more than one, put them inside a PARALLEL. */
3923 :
3924 93800 : if (noutputs == 0 && nuses == 0 && nclobbers == 0)
3925 : {
3926 : /* No output operands: put in a raw ASM_OPERANDS rtx. */
3927 0 : if (nlabels > 0)
3928 0 : emit_jump_insn (body);
3929 : else
3930 0 : emit_insn (body);
3931 : }
3932 93800 : else if (noutputs == 1 && nuses == 0 && nclobbers == 0)
3933 : {
3934 62 : ASM_OPERANDS_OUTPUT_CONSTRAINT (body) = constraints[0];
3935 62 : if (nlabels > 0)
3936 0 : emit_jump_insn (gen_rtx_SET (output_rvec[0], body));
3937 : else
3938 62 : emit_insn (gen_rtx_SET (output_rvec[0], body));
3939 : }
3940 : else
3941 : {
3942 93738 : rtx obody = body;
3943 93738 : int num = noutputs;
3944 :
3945 93738 : if (num == 0)
3946 57528 : num = 1;
3947 :
3948 93738 : body = gen_rtx_PARALLEL (VOIDmode,
3949 : rtvec_alloc (num + nuses + nclobbers));
3950 :
3951 : /* For each output operand, store a SET. */
3952 173511 : for (i = 0; i < noutputs; ++i)
3953 : {
3954 79773 : rtx src, o = output_rvec[i];
3955 79773 : if (i == 0)
3956 : {
3957 36210 : ASM_OPERANDS_OUTPUT_CONSTRAINT (obody) = constraints[0];
3958 36210 : src = obody;
3959 : }
3960 : else
3961 : {
3962 43563 : src = gen_rtx_ASM_OPERANDS (GET_MODE (o),
3963 : ASM_OPERANDS_TEMPLATE (obody),
3964 : constraints[i], i, argvec,
3965 : constraintvec, labelvec, locus);
3966 43563 : MEM_VOLATILE_P (src) = gimple_asm_volatile_p (stmt);
3967 : }
3968 79773 : XVECEXP (body, 0, i) = gen_rtx_SET (o, src);
3969 : }
3970 :
3971 : /* If there are no outputs (but there are some clobbers)
3972 : store the bare ASM_OPERANDS into the PARALLEL. */
3973 93738 : if (i == 0)
3974 57528 : XVECEXP (body, 0, i++) = obody;
3975 :
3976 : /* Add the uses specified by the target hook. No checking should
3977 : be needed since this doesn't come directly from user code. */
3978 93738 : for (rtx use : use_rvec)
3979 0 : XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
3980 :
3981 : /* Store (clobber REG) for each clobbered register specified. */
3982 269398 : for (unsigned j = 0; j < nclobbers; ++j)
3983 : {
3984 175660 : rtx clobbered_reg = clobber_rvec[j];
3985 :
3986 : /* Do sanity check for overlap between clobbers and respectively
3987 : input and outputs that hasn't been handled. Such overlap
3988 : should have been detected and reported above. */
3989 175660 : if (!clobber_conflict_found && REG_P (clobbered_reg))
3990 : {
3991 : /* We test the old body (obody) contents to avoid
3992 : tripping over the under-construction body. */
3993 216187 : for (unsigned k = 0; k < noutputs; ++k)
3994 80832 : if (reg_overlap_mentioned_p (clobbered_reg, output_rvec[k]))
3995 0 : internal_error ("%<asm%> clobber conflict with "
3996 : "output operand");
3997 :
3998 197262 : for (unsigned k = 0; k < ninputs - ninout; ++k)
3999 61907 : if (reg_overlap_mentioned_p (clobbered_reg, input_rvec[k]))
4000 0 : internal_error ("%<asm%> clobber conflict with "
4001 : "input operand");
4002 : }
4003 :
4004 175660 : XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobbered_reg);
4005 : }
4006 :
4007 93738 : if (nlabels > 0)
4008 633 : emit_jump_insn (body);
4009 : else
4010 93105 : emit_insn (body);
4011 : }
4012 :
4013 93800 : generating_concat_p = old_generating_concat_p;
4014 :
4015 93800 : if (fallthru_label)
4016 289 : emit_label (fallthru_label);
4017 :
4018 93800 : if (after_rtl_seq)
4019 : {
4020 29211 : if (nlabels == 0)
4021 29076 : emit_insn (after_rtl_seq);
4022 : else
4023 : {
4024 135 : edge e;
4025 135 : edge_iterator ei;
4026 135 : unsigned int cnt = EDGE_COUNT (gimple_bb (stmt)->succs);
4027 :
4028 480 : FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
4029 : {
4030 345 : rtx_insn *copy;
4031 345 : if (--cnt == 0)
4032 : copy = after_rtl_seq;
4033 : else
4034 : {
4035 210 : start_sequence ();
4036 210 : duplicate_insn_chain (after_rtl_seq, after_rtl_end,
4037 : NULL, NULL);
4038 210 : copy = end_sequence ();
4039 : }
4040 345 : prepend_insn_to_edge (copy, e);
4041 : }
4042 : }
4043 : }
4044 :
4045 93800 : free_temp_slots ();
4046 93800 : crtl->has_asm_statement = 1;
4047 93805 : }
4048 :
4049 : /* Emit code to jump to the address
4050 : specified by the pointer expression EXP. */
4051 :
4052 : static void
4053 422 : expand_computed_goto (tree exp)
4054 : {
4055 422 : rtx x = expand_normal (exp);
4056 :
4057 422 : do_pending_stack_adjust ();
4058 422 : emit_indirect_jump (x);
4059 422 : }
4060 :
4061 : /* Generate RTL code for a `goto' statement with target label LABEL.
4062 : LABEL should be a LABEL_DECL tree node that was or will later be
4063 : defined with `expand_label'. */
4064 :
4065 : static void
4066 0 : expand_goto (tree label)
4067 : {
4068 0 : if (flag_checking)
4069 : {
4070 : /* Check for a nonlocal goto to a containing function. Should have
4071 : gotten translated to __builtin_nonlocal_goto. */
4072 0 : tree context = decl_function_context (label);
4073 0 : gcc_assert (!context || context == current_function_decl);
4074 : }
4075 :
4076 0 : emit_jump (jump_target_rtx (label));
4077 0 : }
4078 :
4079 : /* Output a return with no value. */
4080 :
4081 : static void
4082 846661 : expand_null_return_1 (void)
4083 : {
4084 846661 : clear_pending_stack_adjust ();
4085 846661 : do_pending_stack_adjust ();
4086 846661 : emit_jump (return_label);
4087 846661 : }
4088 :
4089 : /* Generate RTL to return from the current function, with no value.
4090 : (That is, we do not do anything about returning any value.) */
4091 :
4092 : void
4093 70993 : expand_null_return (void)
4094 : {
4095 : /* If this function was declared to return a value, but we
4096 : didn't, clobber the return registers so that they are not
4097 : propagated live to the rest of the function. */
4098 70993 : clobber_return_register ();
4099 :
4100 70993 : expand_null_return_1 ();
4101 70993 : }
4102 :
4103 : /* Generate RTL to return from the current function, with value VAL. */
4104 :
4105 : static void
4106 775668 : expand_value_return (rtx val)
4107 : {
4108 : /* Copy the value to the return location unless it's already there. */
4109 :
4110 775668 : tree decl = DECL_RESULT (current_function_decl);
4111 775668 : rtx return_reg = DECL_RTL (decl);
4112 775668 : if (return_reg != val)
4113 : {
4114 450168 : tree funtype = TREE_TYPE (current_function_decl);
4115 450168 : tree type = TREE_TYPE (decl);
4116 450168 : int unsignedp = TYPE_UNSIGNED (type);
4117 450168 : machine_mode old_mode = DECL_MODE (decl);
4118 450168 : machine_mode mode;
4119 450168 : if (DECL_BY_REFERENCE (decl))
4120 0 : mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 2);
4121 : else
4122 450168 : mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 1);
4123 :
4124 450168 : if (mode != old_mode)
4125 : {
4126 : /* Some ABIs require scalar floating point modes to be returned
4127 : in a wider scalar integer mode. We need to explicitly
4128 : reinterpret to an integer mode of the correct precision
4129 : before extending to the desired result. */
4130 8 : if (SCALAR_INT_MODE_P (mode)
4131 8 : && SCALAR_FLOAT_MODE_P (old_mode)
4132 8 : && known_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (old_mode)))
4133 0 : val = convert_float_to_wider_int (mode, old_mode, val);
4134 : else
4135 8 : val = convert_modes (mode, old_mode, val, unsignedp);
4136 : }
4137 :
4138 450168 : if (GET_CODE (return_reg) == PARALLEL)
4139 3309 : emit_group_load (return_reg, val, type, int_size_in_bytes (type));
4140 : else
4141 446859 : emit_move_insn (return_reg, val);
4142 : }
4143 :
4144 775668 : expand_null_return_1 ();
4145 775668 : }
4146 :
4147 : /* Generate RTL to evaluate the expression RETVAL and return it
4148 : from the current function. */
4149 :
4150 : static void
4151 778209 : expand_return (tree retval)
4152 : {
4153 778209 : rtx result_rtl;
4154 778209 : rtx val = 0;
4155 778209 : tree retval_rhs;
4156 :
4157 : /* If function wants no value, give it none. */
4158 778209 : if (VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
4159 : {
4160 0 : expand_normal (retval);
4161 0 : expand_null_return ();
4162 0 : return;
4163 : }
4164 :
4165 778209 : if (retval == error_mark_node)
4166 : {
4167 : /* Treat this like a return of no value from a function that
4168 : returns a value. */
4169 0 : expand_null_return ();
4170 0 : return;
4171 : }
4172 778209 : else if ((TREE_CODE (retval) == MODIFY_EXPR
4173 778209 : || TREE_CODE (retval) == INIT_EXPR)
4174 778209 : && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
4175 714917 : retval_rhs = TREE_OPERAND (retval, 1);
4176 : else
4177 : retval_rhs = retval;
4178 :
4179 778209 : result_rtl = DECL_RTL (DECL_RESULT (current_function_decl));
4180 :
4181 : /* If we are returning the RESULT_DECL, then the value has already
4182 : been stored into it, so we don't have to do anything special. */
4183 778209 : if (TREE_CODE (retval_rhs) == RESULT_DECL)
4184 63292 : expand_value_return (result_rtl);
4185 :
4186 : /* If the result is an aggregate that is being returned in one (or more)
4187 : registers, load the registers here. */
4188 :
4189 714917 : else if (retval_rhs != 0
4190 714917 : && TYPE_MODE (TREE_TYPE (retval_rhs)) == BLKmode
4191 5259 : && REG_P (result_rtl))
4192 : {
4193 3765 : val = copy_blkmode_to_reg (GET_MODE (result_rtl), retval_rhs);
4194 3765 : if (val)
4195 : {
4196 : /* Use the mode of the result value on the return register. */
4197 1224 : PUT_MODE (result_rtl, GET_MODE (val));
4198 1224 : expand_value_return (val);
4199 : }
4200 : else
4201 2541 : expand_null_return ();
4202 : }
4203 711152 : else if (retval_rhs != 0
4204 711152 : && !VOID_TYPE_P (TREE_TYPE (retval_rhs))
4205 711152 : && (REG_P (result_rtl)
4206 6451 : || (GET_CODE (result_rtl) == PARALLEL)))
4207 : {
4208 : /* Compute the return value into a temporary (usually a pseudo reg). */
4209 708010 : val
4210 708010 : = assign_temp (TREE_TYPE (DECL_RESULT (current_function_decl)), 0, 1);
4211 708010 : val = expand_expr (retval_rhs, val, GET_MODE (val), EXPAND_NORMAL);
4212 708010 : val = force_not_mem (val);
4213 708010 : expand_value_return (val);
4214 : }
4215 : else
4216 : {
4217 : /* No hard reg used; calculate value into hard return reg. */
4218 3142 : expand_expr (retval, const0_rtx, VOIDmode, EXPAND_NORMAL);
4219 3142 : expand_value_return (result_rtl);
4220 : }
4221 : }
4222 :
4223 : /* Expand a clobber of LHS. If LHS is stored it in a multi-part
4224 : register, tell the rtl optimizers that its value is no longer
4225 : needed. */
4226 :
4227 : static void
4228 1412366 : expand_clobber (tree lhs)
4229 : {
4230 1412366 : if (DECL_P (lhs))
4231 : {
4232 1383964 : rtx decl_rtl = DECL_RTL_IF_SET (lhs);
4233 1383873 : if (decl_rtl && REG_P (decl_rtl))
4234 : {
4235 189499 : machine_mode decl_mode = GET_MODE (decl_rtl);
4236 378998 : if (maybe_gt (GET_MODE_SIZE (decl_mode),
4237 : REGMODE_NATURAL_SIZE (decl_mode)))
4238 146929 : emit_clobber (decl_rtl);
4239 : }
4240 : }
4241 1412366 : }
4242 :
4243 : /* A subroutine of expand_gimple_stmt, expanding one gimple statement
4244 : STMT that doesn't require special handling for outgoing edges. That
4245 : is no tailcalls and no GIMPLE_COND. */
4246 :
4247 : static void
4248 32589326 : expand_gimple_stmt_1 (gimple *stmt)
4249 : {
4250 32589326 : tree op0;
4251 :
4252 32589326 : set_curr_insn_location (gimple_location (stmt));
4253 :
4254 32589326 : switch (gimple_code (stmt))
4255 : {
4256 422 : case GIMPLE_GOTO:
4257 422 : op0 = gimple_goto_dest (stmt);
4258 422 : if (TREE_CODE (op0) == LABEL_DECL)
4259 0 : expand_goto (op0);
4260 : else
4261 422 : expand_computed_goto (op0);
4262 : break;
4263 666412 : case GIMPLE_LABEL:
4264 666412 : expand_label (gimple_label_label (as_a <glabel *> (stmt)));
4265 666412 : break;
4266 : case GIMPLE_NOP:
4267 : case GIMPLE_PREDICT:
4268 : break;
4269 7659 : case GIMPLE_SWITCH:
4270 7659 : {
4271 7659 : gswitch *swtch = as_a <gswitch *> (stmt);
4272 7659 : if (gimple_switch_num_labels (swtch) == 1)
4273 0 : expand_goto (CASE_LABEL (gimple_switch_default_label (swtch)));
4274 : else
4275 7659 : expand_case (swtch);
4276 : }
4277 : break;
4278 96681 : case GIMPLE_ASM:
4279 96681 : expand_asm_stmt (as_a <gasm *> (stmt));
4280 96681 : break;
4281 7072400 : case GIMPLE_CALL:
4282 7072400 : expand_call_stmt (as_a <gcall *> (stmt));
4283 7072400 : break;
4284 :
4285 846661 : case GIMPLE_RETURN:
4286 846661 : {
4287 846661 : op0 = gimple_return_retval (as_a <greturn *> (stmt));
4288 :
4289 : /* If a return doesn't have a location, it very likely represents
4290 : multiple user returns so we cannot let it inherit the location
4291 : of the last statement of the previous basic block in RTL. */
4292 846661 : if (!gimple_has_location (stmt))
4293 218724 : set_curr_insn_location (cfun->function_end_locus);
4294 :
4295 846661 : if (op0 && op0 != error_mark_node)
4296 : {
4297 778209 : tree result = DECL_RESULT (current_function_decl);
4298 :
4299 : /* If we are not returning the current function's RESULT_DECL,
4300 : build an assignment to it. */
4301 778209 : if (op0 != result)
4302 : {
4303 : /* I believe that a function's RESULT_DECL is unique. */
4304 714917 : gcc_assert (TREE_CODE (op0) != RESULT_DECL);
4305 :
4306 : /* ??? We'd like to use simply expand_assignment here,
4307 : but this fails if the value is of BLKmode but the return
4308 : decl is a register. expand_return has special handling
4309 : for this combination, which eventually should move
4310 : to common code. See comments there. Until then, let's
4311 : build a modify expression :-/ */
4312 714917 : op0 = build2 (MODIFY_EXPR, TREE_TYPE (result),
4313 : result, op0);
4314 : }
4315 : }
4316 :
4317 778209 : if (!op0)
4318 68452 : expand_null_return ();
4319 : else
4320 778209 : expand_return (op0);
4321 : }
4322 : break;
4323 :
4324 23849744 : case GIMPLE_ASSIGN:
4325 23849744 : {
4326 23849744 : gassign *assign_stmt = as_a <gassign *> (stmt);
4327 23849744 : tree lhs = gimple_assign_lhs (assign_stmt);
4328 :
4329 : /* Tree expand used to fiddle with |= and &= of two bitfield
4330 : COMPONENT_REFs here. This can't happen with gimple, the LHS
4331 : of binary assigns must be a gimple reg. */
4332 :
4333 23849744 : if (TREE_CODE (lhs) != SSA_NAME
4334 37906155 : || gimple_assign_rhs_class (assign_stmt) == GIMPLE_SINGLE_RHS)
4335 : {
4336 17752918 : tree rhs = gimple_assign_rhs1 (assign_stmt);
4337 17752918 : gcc_assert (gimple_assign_rhs_class (assign_stmt)
4338 : == GIMPLE_SINGLE_RHS);
4339 33873807 : if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (rhs)
4340 : /* Do not put locations on possibly shared trees. */
4341 23860878 : && !is_gimple_min_invariant (rhs))
4342 4964554 : SET_EXPR_LOCATION (rhs, gimple_location (stmt));
4343 17752918 : if (TREE_CLOBBER_P (rhs))
4344 : /* This is a clobber to mark the going out of scope for
4345 : this LHS. */
4346 1412366 : expand_clobber (lhs);
4347 : else
4348 16340552 : expand_assignment (lhs, rhs,
4349 : gimple_assign_nontemporal_move_p (
4350 : assign_stmt));
4351 : }
4352 : else
4353 : {
4354 6096826 : rtx target, temp;
4355 6096826 : gcc_assert (!gimple_assign_nontemporal_move_p (assign_stmt));
4356 6096826 : bool promoted = false;
4357 :
4358 6096826 : target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
4359 6096826 : if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
4360 : promoted = true;
4361 :
4362 : /* If we store into a promoted register, don't directly
4363 : expand to target. */
4364 : temp = promoted ? NULL_RTX : target;
4365 12193652 : temp = expand_expr_real_gassign (assign_stmt, temp,
4366 6096826 : GET_MODE (target), EXPAND_NORMAL);
4367 :
4368 6096826 : if (temp == target)
4369 : ;
4370 850827 : else if (promoted)
4371 : {
4372 2 : int unsignedp = SUBREG_PROMOTED_SIGN (target);
4373 : /* If TEMP is a VOIDmode constant, use convert_modes to make
4374 : sure that we properly convert it. */
4375 2 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
4376 : {
4377 0 : temp = convert_modes (GET_MODE (target),
4378 0 : TYPE_MODE (TREE_TYPE (lhs)),
4379 : temp, unsignedp);
4380 0 : temp = convert_modes (GET_MODE (SUBREG_REG (target)),
4381 0 : GET_MODE (target), temp, unsignedp);
4382 : }
4383 :
4384 2 : convert_move (SUBREG_REG (target), temp, unsignedp);
4385 : }
4386 : else
4387 : {
4388 850825 : temp = force_operand (temp, target);
4389 850825 : if (temp == target)
4390 : ;
4391 850825 : else if (GET_MODE (target) != BLKmode)
4392 850824 : emit_move_insn (target, temp);
4393 : else
4394 1 : emit_block_move (target, temp, expr_size (lhs),
4395 : BLOCK_OP_NORMAL);
4396 : }
4397 : }
4398 : }
4399 : break;
4400 :
4401 0 : default:
4402 0 : gcc_unreachable ();
4403 : }
4404 32589324 : }
4405 :
4406 : /* Expand one gimple statement STMT and return the last RTL instruction
4407 : before any of the newly generated ones.
4408 :
4409 : In addition to generating the necessary RTL instructions this also
4410 : sets REG_EH_REGION notes if necessary and sets the current source
4411 : location for diagnostics. */
4412 :
4413 : static rtx_insn *
4414 32589326 : expand_gimple_stmt (gimple *stmt)
4415 : {
4416 32589326 : location_t saved_location = input_location;
4417 32589326 : rtx_insn *last = get_last_insn ();
4418 32589326 : int lp_nr;
4419 :
4420 32589326 : gcc_assert (cfun);
4421 :
4422 : /* We need to save and restore the current source location so that errors
4423 : discovered during expansion are emitted with the right location. But
4424 : it would be better if the diagnostic routines used the source location
4425 : embedded in the tree nodes rather than globals. */
4426 32589326 : if (gimple_has_location (stmt))
4427 28580372 : input_location = gimple_location (stmt);
4428 :
4429 32589326 : expand_gimple_stmt_1 (stmt);
4430 :
4431 : /* Free any temporaries used to evaluate this statement. */
4432 32589324 : free_temp_slots ();
4433 :
4434 32589324 : input_location = saved_location;
4435 :
4436 : /* Mark all insns that may trap. */
4437 32589324 : lp_nr = lookup_stmt_eh_lp (stmt);
4438 32589324 : if (lp_nr)
4439 : {
4440 745802 : rtx_insn *insn;
4441 4271369 : for (insn = next_real_insn (last); insn;
4442 3525567 : insn = next_real_insn (insn))
4443 : {
4444 3525567 : if (! find_reg_note (insn, REG_EH_REGION, NULL_RTX)
4445 : /* If we want exceptions for non-call insns, any
4446 : may_trap_p instruction may throw. */
4447 3525473 : && GET_CODE (PATTERN (insn)) != CLOBBER
4448 3525403 : && GET_CODE (PATTERN (insn)) != USE
4449 7050970 : && insn_could_throw_p (insn))
4450 803085 : make_reg_eh_region_note (insn, 0, lp_nr);
4451 : }
4452 : }
4453 :
4454 32589324 : return last;
4455 : }
4456 :
4457 : /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_CALL
4458 : that has CALL_EXPR_TAILCALL set. Returns non-null if we actually
4459 : generated a tail call (something that might be denied by the ABI
4460 : rules governing the call; see calls.cc).
4461 :
4462 : Sets CAN_FALLTHRU if we generated a *conditional* tail call, and
4463 : can still reach the rest of BB. The case here is __builtin_sqrt,
4464 : where the NaN result goes through the external function (with a
4465 : tailcall) and the normal result happens via a sqrt instruction. */
4466 :
4467 : static basic_block
4468 178659 : expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru,
4469 : rtx_insn *asan_epilog_seq)
4470 : {
4471 178659 : rtx_insn *last2, *last, *first = get_last_insn ();
4472 178659 : edge e;
4473 178659 : edge_iterator ei;
4474 178659 : profile_probability probability;
4475 :
4476 178659 : last2 = last = expand_gimple_stmt (stmt);
4477 :
4478 975745 : for (last = NEXT_INSN (last); last; last = NEXT_INSN (last))
4479 746861 : if (CALL_P (last) && SIBLING_CALL_P (last))
4480 128434 : goto found;
4481 :
4482 50225 : maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4483 :
4484 50225 : *can_fallthru = true;
4485 50225 : return NULL;
4486 :
4487 128434 : found:
4488 :
4489 128434 : if (asan_epilog_seq)
4490 : {
4491 : /* We need to emit a copy of the asan_epilog_seq before
4492 : the insns emitted by expand_gimple_stmt above. The sequence
4493 : can contain labels, which need to be remapped. */
4494 117 : hash_map<rtx, rtx_code_label *> label_map;
4495 117 : start_sequence ();
4496 117 : emit_note (NOTE_INSN_DELETED);
4497 1767 : for (rtx_insn *insn = asan_epilog_seq; insn; insn = NEXT_INSN (insn))
4498 1533 : switch (GET_CODE (insn))
4499 : {
4500 1182 : case INSN:
4501 1182 : case CALL_INSN:
4502 1182 : case JUMP_INSN:
4503 1182 : emit_copy_of_insn_after (insn, get_last_insn ());
4504 1182 : break;
4505 234 : case CODE_LABEL:
4506 234 : label_map.put ((rtx) insn, emit_label (gen_label_rtx ()));
4507 234 : break;
4508 117 : case BARRIER:
4509 117 : emit_barrier ();
4510 117 : break;
4511 0 : default:
4512 0 : gcc_unreachable ();
4513 : }
4514 1767 : for (rtx_insn *insn = get_insns (); insn; insn = NEXT_INSN (insn))
4515 1650 : if (JUMP_P (insn))
4516 : {
4517 234 : subrtx_ptr_iterator::array_type array;
4518 1521 : FOR_EACH_SUBRTX_PTR (iter, array, &PATTERN (insn), ALL)
4519 : {
4520 1287 : rtx *loc = *iter;
4521 1287 : if (LABEL_REF_P (*loc))
4522 : {
4523 234 : rtx_code_label **lab
4524 234 : = label_map.get ((rtx) label_ref_label (*loc));
4525 234 : gcc_assert (lab);
4526 234 : set_label_ref_label (*loc, *lab);
4527 : }
4528 : }
4529 234 : if (JUMP_LABEL (insn))
4530 : {
4531 234 : rtx_code_label **lab = label_map.get (JUMP_LABEL (insn));
4532 234 : gcc_assert (lab);
4533 234 : JUMP_LABEL (insn) = *lab;
4534 : }
4535 234 : }
4536 117 : asan_epilog_seq = NEXT_INSN (get_insns ());
4537 117 : end_sequence ();
4538 117 : emit_insn_before (asan_epilog_seq, NEXT_INSN (first));
4539 117 : }
4540 :
4541 : /* ??? Wouldn't it be better to just reset any pending stack adjust?
4542 : Any instructions emitted here are about to be deleted. */
4543 128434 : do_pending_stack_adjust ();
4544 :
4545 : /* Remove any non-eh, non-abnormal edges that don't go to exit. */
4546 : /* ??? I.e. the fallthrough edge. HOWEVER! If there were to be
4547 : EH or abnormal edges, we shouldn't have created a tail call in
4548 : the first place. So it seems to me we should just be removing
4549 : all edges here, or redirecting the existing fallthru edge to
4550 : the exit block. */
4551 :
4552 128434 : probability = profile_probability::never ();
4553 :
4554 256862 : for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
4555 : {
4556 128428 : if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
4557 : {
4558 128422 : if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
4559 60517 : e->dest->count -= e->count ();
4560 128422 : probability += e->probability;
4561 128422 : expand_remove_edge (e);
4562 : }
4563 : else
4564 6 : ei_next (&ei);
4565 : }
4566 :
4567 : /* This is somewhat ugly: the call_expr expander often emits instructions
4568 : after the sibcall (to perform the function return). These confuse the
4569 : find_many_sub_basic_blocks code, so we need to get rid of these. */
4570 128434 : last = NEXT_INSN (last);
4571 128434 : gcc_assert (BARRIER_P (last));
4572 :
4573 128434 : *can_fallthru = false;
4574 158874 : while (NEXT_INSN (last))
4575 : {
4576 : /* For instance an sqrt builtin expander expands if with
4577 : sibcall in the then and label for `else`. */
4578 30440 : if (LABEL_P (NEXT_INSN (last)))
4579 : {
4580 0 : *can_fallthru = true;
4581 0 : break;
4582 : }
4583 30440 : delete_insn (NEXT_INSN (last));
4584 : }
4585 :
4586 128434 : e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
4587 : | EDGE_SIBCALL);
4588 128434 : e->probability = probability;
4589 128434 : head_end_for_bb[bb->index].second = last;
4590 256868 : update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
4591 128434 : head_end_for_bb[bb->index].second, bb);
4592 :
4593 128434 : if (NEXT_INSN (last))
4594 : {
4595 0 : bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
4596 :
4597 0 : last = BB_END (bb);
4598 0 : if (BARRIER_P (last))
4599 0 : BB_END (bb) = PREV_INSN (last);
4600 : }
4601 :
4602 128434 : maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4603 :
4604 128434 : return bb;
4605 : }
4606 :
4607 : /* Return the difference between the floor and the truncated result of
4608 : a signed division by OP1 with remainder MOD. */
4609 : static rtx
4610 4 : floor_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4611 : {
4612 : /* (mod != 0 ? (op1 / mod < 0 ? -1 : 0) : 0) */
4613 4 : return gen_rtx_IF_THEN_ELSE
4614 : (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4615 : gen_rtx_IF_THEN_ELSE
4616 : (mode, gen_rtx_LT (BImode,
4617 : gen_rtx_DIV (mode, op1, mod),
4618 : const0_rtx),
4619 : constm1_rtx, const0_rtx),
4620 : const0_rtx);
4621 : }
4622 :
4623 : /* Return the difference between the ceil and the truncated result of
4624 : a signed division by OP1 with remainder MOD. */
4625 : static rtx
4626 3 : ceil_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4627 : {
4628 : /* (mod != 0 ? (op1 / mod > 0 ? 1 : 0) : 0) */
4629 3 : return gen_rtx_IF_THEN_ELSE
4630 : (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4631 : gen_rtx_IF_THEN_ELSE
4632 : (mode, gen_rtx_GT (BImode,
4633 : gen_rtx_DIV (mode, op1, mod),
4634 : const0_rtx),
4635 : const1_rtx, const0_rtx),
4636 : const0_rtx);
4637 : }
4638 :
4639 : /* Return the difference between the ceil and the truncated result of
4640 : an unsigned division by OP1 with remainder MOD. */
4641 : static rtx
4642 0 : ceil_udiv_adjust (machine_mode mode, rtx mod, rtx op1 ATTRIBUTE_UNUSED)
4643 : {
4644 : /* (mod != 0 ? 1 : 0) */
4645 0 : return gen_rtx_IF_THEN_ELSE
4646 : (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4647 : const1_rtx, const0_rtx);
4648 : }
4649 :
4650 : /* Return the difference between the rounded and the truncated result
4651 : of a signed division by OP1 with remainder MOD. Halfway cases are
4652 : rounded away from zero, rather than to the nearest even number. */
4653 : static rtx
4654 0 : round_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4655 : {
4656 : /* (abs (mod) >= abs (op1) - abs (mod)
4657 : ? (op1 / mod > 0 ? 1 : -1)
4658 : : 0) */
4659 0 : return gen_rtx_IF_THEN_ELSE
4660 : (mode, gen_rtx_GE (BImode, gen_rtx_ABS (mode, mod),
4661 : gen_rtx_MINUS (mode,
4662 : gen_rtx_ABS (mode, op1),
4663 : gen_rtx_ABS (mode, mod))),
4664 : gen_rtx_IF_THEN_ELSE
4665 : (mode, gen_rtx_GT (BImode,
4666 : gen_rtx_DIV (mode, op1, mod),
4667 : const0_rtx),
4668 : const1_rtx, constm1_rtx),
4669 : const0_rtx);
4670 : }
4671 :
4672 : /* Return the difference between the rounded and the truncated result
4673 : of a unsigned division by OP1 with remainder MOD. Halfway cases
4674 : are rounded away from zero, rather than to the nearest even
4675 : number. */
4676 : static rtx
4677 0 : round_udiv_adjust (machine_mode mode, rtx mod, rtx op1)
4678 : {
4679 : /* (mod >= op1 - mod ? 1 : 0) */
4680 0 : return gen_rtx_IF_THEN_ELSE
4681 : (mode, gen_rtx_GE (BImode, mod,
4682 : gen_rtx_MINUS (mode, op1, mod)),
4683 : const1_rtx, const0_rtx);
4684 : }
4685 :
4686 : /* Convert X to MODE, that must be Pmode or ptr_mode, without emitting
4687 : any rtl. */
4688 :
4689 : static rtx
4690 7510597 : convert_debug_memory_address (scalar_int_mode mode, rtx x,
4691 : addr_space_t as)
4692 : {
4693 : #ifndef POINTERS_EXTEND_UNSIGNED
4694 : gcc_assert (mode == Pmode
4695 : || mode == targetm.addr_space.address_mode (as));
4696 : gcc_assert (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode);
4697 : #else
4698 7510597 : rtx temp;
4699 :
4700 7510597 : gcc_assert (targetm.addr_space.valid_pointer_mode (mode, as));
4701 :
4702 7510597 : if (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode)
4703 : return x;
4704 :
4705 : /* X must have some form of address mode already. */
4706 0 : scalar_int_mode xmode = as_a <scalar_int_mode> (GET_MODE (x));
4707 0 : if (GET_MODE_PRECISION (mode) < GET_MODE_PRECISION (xmode))
4708 0 : x = lowpart_subreg (mode, x, xmode);
4709 0 : else if (POINTERS_EXTEND_UNSIGNED > 0)
4710 0 : x = gen_rtx_ZERO_EXTEND (mode, x);
4711 : else if (!POINTERS_EXTEND_UNSIGNED)
4712 : x = gen_rtx_SIGN_EXTEND (mode, x);
4713 : else
4714 : {
4715 : switch (GET_CODE (x))
4716 : {
4717 : case SUBREG:
4718 : if ((SUBREG_PROMOTED_VAR_P (x)
4719 : || (REG_P (SUBREG_REG (x)) && REG_POINTER (SUBREG_REG (x)))
4720 : || (GET_CODE (SUBREG_REG (x)) == PLUS
4721 : && REG_P (XEXP (SUBREG_REG (x), 0))
4722 : && REG_POINTER (XEXP (SUBREG_REG (x), 0))
4723 : && CONST_INT_P (XEXP (SUBREG_REG (x), 1))))
4724 : && GET_MODE (SUBREG_REG (x)) == mode)
4725 : return SUBREG_REG (x);
4726 : break;
4727 : case LABEL_REF:
4728 : temp = gen_rtx_LABEL_REF (mode, label_ref_label (x));
4729 : LABEL_REF_NONLOCAL_P (temp) = LABEL_REF_NONLOCAL_P (x);
4730 : return temp;
4731 : case SYMBOL_REF:
4732 : temp = shallow_copy_rtx (x);
4733 : PUT_MODE (temp, mode);
4734 : return temp;
4735 : case CONST:
4736 : temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4737 : if (temp)
4738 : temp = gen_rtx_CONST (mode, temp);
4739 : return temp;
4740 : case PLUS:
4741 : case MINUS:
4742 : if (CONST_INT_P (XEXP (x, 1)))
4743 : {
4744 : temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4745 : if (temp)
4746 : return gen_rtx_fmt_ee (GET_CODE (x), mode, temp, XEXP (x, 1));
4747 : }
4748 : break;
4749 : default:
4750 : break;
4751 : }
4752 : /* Don't know how to express ptr_extend as operation in debug info. */
4753 : return NULL;
4754 : }
4755 : #endif /* POINTERS_EXTEND_UNSIGNED */
4756 :
4757 : return x;
4758 : }
4759 :
4760 : /* Map from SSA_NAMEs to corresponding DEBUG_EXPR_DECLs created
4761 : by avoid_deep_ter_for_debug. */
4762 :
4763 : static hash_map<tree, tree> *deep_ter_debug_map;
4764 :
4765 : /* Split too deep TER chains for debug stmts using debug temporaries. */
4766 :
4767 : static void
4768 42870112 : avoid_deep_ter_for_debug (gimple *stmt, int depth)
4769 : {
4770 42870112 : use_operand_p use_p;
4771 42870112 : ssa_op_iter iter;
4772 53147358 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
4773 : {
4774 10277246 : tree use = USE_FROM_PTR (use_p);
4775 10277246 : if (TREE_CODE (use) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (use))
4776 8533279 : continue;
4777 8904324 : gimple *g = get_gimple_for_ssa_name (use);
4778 8904324 : if (g == NULL)
4779 7156901 : continue;
4780 1747423 : if ((depth > 6 || !is_gimple_assign (g)) && !stmt_ends_bb_p (g))
4781 : {
4782 8867 : if (deep_ter_debug_map == NULL)
4783 729 : deep_ter_debug_map = new hash_map<tree, tree>;
4784 :
4785 8867 : tree &vexpr = deep_ter_debug_map->get_or_insert (use);
4786 8867 : if (vexpr != NULL)
4787 3456 : continue;
4788 5411 : vexpr = build_debug_expr_decl (TREE_TYPE (use));
4789 5411 : gimple *def_temp = gimple_build_debug_bind (vexpr, use, g);
4790 5411 : gimple_stmt_iterator gsi = gsi_for_stmt (g);
4791 5411 : gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
4792 5411 : avoid_deep_ter_for_debug (def_temp, 0);
4793 : }
4794 : else
4795 1738556 : avoid_deep_ter_for_debug (g, depth + 1);
4796 : }
4797 42870112 : }
4798 :
4799 : /* Return an RTX equivalent to the value of the parameter DECL. */
4800 :
4801 : static rtx
4802 189605 : expand_debug_parm_decl (tree decl)
4803 : {
4804 189605 : rtx incoming = DECL_INCOMING_RTL (decl);
4805 :
4806 189605 : if (incoming
4807 19661 : && GET_MODE (incoming) != BLKmode
4808 209262 : && ((REG_P (incoming) && HARD_REGISTER_P (incoming))
4809 962 : || (MEM_P (incoming)
4810 958 : && REG_P (XEXP (incoming, 0))
4811 343 : && HARD_REGISTER_P (XEXP (incoming, 0)))))
4812 : {
4813 18722 : rtx rtl = gen_rtx_ENTRY_VALUE (GET_MODE (incoming));
4814 :
4815 : #ifdef HAVE_window_save
4816 : /* DECL_INCOMING_RTL uses the INCOMING_REGNO of parameter registers.
4817 : If the target machine has an explicit window save instruction, the
4818 : actual entry value is the corresponding OUTGOING_REGNO instead. */
4819 : if (REG_P (incoming)
4820 : && OUTGOING_REGNO (REGNO (incoming)) != REGNO (incoming))
4821 : incoming
4822 : = gen_rtx_REG_offset (incoming, GET_MODE (incoming),
4823 : OUTGOING_REGNO (REGNO (incoming)), 0);
4824 : else if (MEM_P (incoming))
4825 : {
4826 : rtx reg = XEXP (incoming, 0);
4827 : if (OUTGOING_REGNO (REGNO (reg)) != REGNO (reg))
4828 : {
4829 : reg = gen_raw_REG (GET_MODE (reg), OUTGOING_REGNO (REGNO (reg)));
4830 : incoming = replace_equiv_address_nv (incoming, reg);
4831 : }
4832 : else
4833 : incoming = copy_rtx (incoming);
4834 : }
4835 : #endif
4836 :
4837 18722 : ENTRY_VALUE_EXP (rtl) = incoming;
4838 18722 : return rtl;
4839 : }
4840 :
4841 170883 : if (incoming
4842 939 : && GET_MODE (incoming) != BLKmode
4843 935 : && !TREE_ADDRESSABLE (decl)
4844 935 : && MEM_P (incoming)
4845 931 : && (XEXP (incoming, 0) == virtual_incoming_args_rtx
4846 615 : || (GET_CODE (XEXP (incoming, 0)) == PLUS
4847 615 : && XEXP (XEXP (incoming, 0), 0) == virtual_incoming_args_rtx
4848 599 : && CONST_INT_P (XEXP (XEXP (incoming, 0), 1)))))
4849 915 : return copy_rtx (incoming);
4850 :
4851 : return NULL_RTX;
4852 : }
4853 :
4854 : /* Return an RTX equivalent to the value of the tree expression EXP. */
4855 :
4856 : static rtx
4857 43621833 : expand_debug_expr (tree exp)
4858 : {
4859 43621833 : rtx op0 = NULL_RTX, op1 = NULL_RTX, op2 = NULL_RTX;
4860 43621833 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
4861 43621833 : machine_mode inner_mode = VOIDmode;
4862 43621833 : int unsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
4863 43621833 : addr_space_t as;
4864 43621833 : scalar_int_mode op0_mode, op1_mode, addr_mode;
4865 :
4866 43621833 : switch (TREE_CODE_CLASS (TREE_CODE (exp)))
4867 : {
4868 7180801 : case tcc_expression:
4869 7180801 : switch (TREE_CODE (exp))
4870 : {
4871 8331 : case COND_EXPR:
4872 8331 : case DOT_PROD_EXPR:
4873 8331 : case SAD_EXPR:
4874 8331 : case WIDEN_MULT_PLUS_EXPR:
4875 8331 : case WIDEN_MULT_MINUS_EXPR:
4876 8331 : goto ternary;
4877 :
4878 0 : case TRUTH_ANDIF_EXPR:
4879 0 : case TRUTH_ORIF_EXPR:
4880 0 : case TRUTH_AND_EXPR:
4881 0 : case TRUTH_OR_EXPR:
4882 0 : case TRUTH_XOR_EXPR:
4883 0 : goto binary;
4884 :
4885 0 : case TRUTH_NOT_EXPR:
4886 0 : goto unary;
4887 :
4888 : default:
4889 : break;
4890 : }
4891 : break;
4892 :
4893 8331 : ternary:
4894 8331 : op2 = expand_debug_expr (TREE_OPERAND (exp, 2));
4895 8331 : if (!op2)
4896 : return NULL_RTX;
4897 : /* Fall through. */
4898 :
4899 8331 : binary:
4900 1550158 : case tcc_binary:
4901 1550158 : if (mode == BLKmode)
4902 : return NULL_RTX;
4903 1550148 : op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4904 1550148 : if (!op1)
4905 : return NULL_RTX;
4906 1549881 : switch (TREE_CODE (exp))
4907 : {
4908 83378 : case LSHIFT_EXPR:
4909 83378 : case RSHIFT_EXPR:
4910 83378 : case LROTATE_EXPR:
4911 83378 : case RROTATE_EXPR:
4912 83378 : case WIDEN_LSHIFT_EXPR:
4913 : /* Ensure second operand isn't wider than the first one. */
4914 83378 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 1)));
4915 83378 : if (is_a <scalar_int_mode> (inner_mode, &op1_mode)
4916 83370 : && (GET_MODE_UNIT_PRECISION (mode)
4917 83370 : < GET_MODE_PRECISION (op1_mode)))
4918 5564 : op1 = lowpart_subreg (GET_MODE_INNER (mode), op1, op1_mode);
4919 : break;
4920 : default:
4921 : break;
4922 : }
4923 : /* Fall through. */
4924 :
4925 2146665 : unary:
4926 2146665 : case tcc_unary:
4927 2146665 : if (mode == BLKmode)
4928 : return NULL_RTX;
4929 2146663 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4930 2146663 : op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4931 2146663 : if (!op0)
4932 : return NULL_RTX;
4933 : break;
4934 :
4935 73870 : case tcc_comparison:
4936 73870 : unsignedp = TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)));
4937 73870 : goto binary;
4938 :
4939 0 : case tcc_type:
4940 0 : case tcc_statement:
4941 0 : gcc_unreachable ();
4942 :
4943 : case tcc_constant:
4944 : case tcc_exceptional:
4945 : case tcc_declaration:
4946 : case tcc_reference:
4947 : case tcc_vl_exp:
4948 : break;
4949 : }
4950 :
4951 43617272 : switch (TREE_CODE (exp))
4952 : {
4953 276156 : case STRING_CST:
4954 276156 : if (!lookup_constant_def (exp))
4955 : {
4956 98746 : if (strlen (TREE_STRING_POINTER (exp)) + 1
4957 98746 : != (size_t) TREE_STRING_LENGTH (exp))
4958 : return NULL_RTX;
4959 98847 : op0 = gen_rtx_CONST_STRING (Pmode, TREE_STRING_POINTER (exp));
4960 98027 : op0 = gen_rtx_MEM (BLKmode, op0);
4961 98027 : set_mem_attributes (op0, exp, 0);
4962 98027 : return op0;
4963 : }
4964 : /* Fall through. */
4965 :
4966 6252588 : case INTEGER_CST:
4967 12505134 : if (BITINT_TYPE_P (TREE_TYPE (exp))
4968 6252588 : && TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
4969 : return NULL;
4970 : /* FALLTHRU */
4971 6325867 : case REAL_CST:
4972 6325867 : case FIXED_CST:
4973 6325867 : op0 = expand_expr (exp, NULL_RTX, mode, EXPAND_INITIALIZER);
4974 6325867 : return op0;
4975 :
4976 0 : case POLY_INT_CST:
4977 0 : return immed_wide_int_const (poly_int_cst_value (exp), mode);
4978 :
4979 4063 : case COMPLEX_CST:
4980 4063 : gcc_assert (COMPLEX_MODE_P (mode));
4981 4063 : op0 = expand_debug_expr (TREE_REALPART (exp));
4982 4063 : op1 = expand_debug_expr (TREE_IMAGPART (exp));
4983 4063 : return gen_rtx_CONCAT (mode, op0, op1);
4984 :
4985 6268160 : case DEBUG_EXPR_DECL:
4986 6268160 : op0 = DECL_RTL_IF_SET (exp);
4987 :
4988 5054229 : if (op0)
4989 : {
4990 5054229 : if (GET_MODE (op0) != mode)
4991 0 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (exp)));
4992 : else
4993 : return op0;
4994 : }
4995 :
4996 1213931 : op0 = gen_rtx_DEBUG_EXPR (mode);
4997 1213931 : DEBUG_EXPR_TREE_DECL (op0) = exp;
4998 1213931 : SET_DECL_RTL (exp, op0);
4999 :
5000 1213931 : return op0;
5001 :
5002 5641778 : case VAR_DECL:
5003 5641778 : case PARM_DECL:
5004 5641778 : case FUNCTION_DECL:
5005 5641778 : case LABEL_DECL:
5006 5641778 : case CONST_DECL:
5007 5641778 : case RESULT_DECL:
5008 5641778 : op0 = DECL_RTL_IF_SET (exp);
5009 :
5010 : /* This decl was probably optimized away. */
5011 3899295 : if (!op0
5012 : /* At least label RTXen are sometimes replaced by
5013 : NOTE_INSN_DELETED_LABEL. Any notes here are not
5014 : handled by copy_rtx. */
5015 3899295 : || NOTE_P (op0))
5016 : {
5017 1742484 : if (!VAR_P (exp)
5018 1729946 : || DECL_EXTERNAL (exp)
5019 1729323 : || !TREE_STATIC (exp)
5020 135300 : || !DECL_NAME (exp)
5021 135300 : || DECL_HARD_REGISTER (exp)
5022 135300 : || DECL_IN_CONSTANT_POOL (exp)
5023 135300 : || mode == VOIDmode
5024 1877784 : || symtab_node::get (exp) == NULL)
5025 : return NULL;
5026 :
5027 19596 : op0 = make_decl_rtl_for_debug (exp);
5028 19596 : if (!MEM_P (op0)
5029 19596 : || GET_CODE (XEXP (op0, 0)) != SYMBOL_REF
5030 39192 : || SYMBOL_REF_DECL (XEXP (op0, 0)) != exp)
5031 : return NULL;
5032 : }
5033 3899294 : else if (VAR_P (exp)
5034 3770550 : && is_global_var (exp)
5035 4127376 : && symtab_node::get (exp) == NULL)
5036 : return NULL;
5037 : else
5038 3899294 : op0 = copy_rtx (op0);
5039 :
5040 3918890 : if (GET_MODE (op0) == BLKmode
5041 : /* If op0 is not BLKmode, but mode is, adjust_mode
5042 : below would ICE. While it is likely a FE bug,
5043 : try to be robust here. See PR43166. */
5044 543488 : || mode == BLKmode
5045 543488 : || (mode == VOIDmode && GET_MODE (op0) != VOIDmode))
5046 : {
5047 3375402 : gcc_assert (MEM_P (op0));
5048 3375402 : op0 = adjust_address_nv (op0, mode, 0);
5049 3375402 : return op0;
5050 : }
5051 :
5052 : /* Fall through. */
5053 :
5054 543488 : adjust_mode:
5055 10871139 : case PAREN_EXPR:
5056 10871139 : CASE_CONVERT:
5057 10871139 : {
5058 10871139 : inner_mode = GET_MODE (op0);
5059 :
5060 10871139 : if (mode == inner_mode)
5061 : return op0;
5062 :
5063 159463 : if (inner_mode == VOIDmode)
5064 : {
5065 8300 : if (TREE_CODE (exp) == SSA_NAME)
5066 37 : inner_mode = TYPE_MODE (TREE_TYPE (exp));
5067 : else
5068 8263 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
5069 8300 : if (mode == inner_mode)
5070 : return op0;
5071 : }
5072 :
5073 153884 : if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5074 : {
5075 6302 : if (GET_MODE_UNIT_BITSIZE (mode)
5076 12604 : == GET_MODE_UNIT_BITSIZE (inner_mode))
5077 4 : op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
5078 6298 : else if (GET_MODE_UNIT_BITSIZE (mode)
5079 12596 : < GET_MODE_UNIT_BITSIZE (inner_mode))
5080 3579 : op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
5081 : else
5082 2719 : op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
5083 : }
5084 147582 : else if (FLOAT_MODE_P (mode))
5085 : {
5086 0 : gcc_assert (TREE_CODE (exp) != SSA_NAME);
5087 0 : if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5088 0 : op0 = simplify_gen_unary (UNSIGNED_FLOAT, mode, op0, inner_mode);
5089 : else
5090 0 : op0 = simplify_gen_unary (FLOAT, mode, op0, inner_mode);
5091 : }
5092 147582 : else if (FLOAT_MODE_P (inner_mode))
5093 : {
5094 0 : if (unsignedp)
5095 0 : op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
5096 : else
5097 0 : op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
5098 : }
5099 295164 : else if (GET_MODE_UNIT_PRECISION (mode)
5100 147582 : == GET_MODE_UNIT_PRECISION (inner_mode))
5101 0 : op0 = lowpart_subreg (mode, op0, inner_mode);
5102 295164 : else if (GET_MODE_UNIT_PRECISION (mode)
5103 147582 : < GET_MODE_UNIT_PRECISION (inner_mode))
5104 71998 : op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
5105 75584 : else if (UNARY_CLASS_P (exp)
5106 75584 : ? TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)))
5107 0 : : unsignedp)
5108 50733 : op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5109 : else
5110 24851 : op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5111 :
5112 : return op0;
5113 : }
5114 :
5115 2240705 : case MEM_REF:
5116 2240705 : if (!is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
5117 : {
5118 990155 : tree newexp = fold_binary (MEM_REF, TREE_TYPE (exp),
5119 : TREE_OPERAND (exp, 0),
5120 : TREE_OPERAND (exp, 1));
5121 990155 : if (newexp)
5122 0 : return expand_debug_expr (newexp);
5123 : }
5124 : /* FALLTHROUGH */
5125 2240705 : case INDIRECT_REF:
5126 2240705 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
5127 2240705 : op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5128 2240705 : if (!op0)
5129 : return NULL;
5130 :
5131 2234096 : if (TREE_CODE (exp) == MEM_REF)
5132 : {
5133 2234096 : if (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5134 2187269 : || (GET_CODE (op0) == PLUS
5135 286288 : && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR))
5136 : /* (mem (debug_implicit_ptr)) might confuse aliasing.
5137 : Instead just use get_inner_reference. */
5138 46827 : goto component_ref;
5139 :
5140 2187269 : op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
5141 2187269 : poly_int64 offset;
5142 2187269 : if (!op1 || !poly_int_rtx_p (op1, &offset))
5143 0 : return NULL;
5144 :
5145 2187269 : op0 = plus_constant (inner_mode, op0, offset);
5146 : }
5147 :
5148 2187269 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
5149 :
5150 2187269 : op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
5151 : op0, as);
5152 2187269 : if (op0 == NULL_RTX)
5153 : return NULL;
5154 :
5155 2187269 : op0 = gen_rtx_MEM (mode, op0);
5156 2187269 : set_mem_attributes (op0, exp, 0);
5157 2187269 : if (TREE_CODE (exp) == MEM_REF
5158 2187269 : && !is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
5159 990155 : set_mem_expr (op0, NULL_TREE);
5160 2187269 : set_mem_addr_space (op0, as);
5161 :
5162 2187269 : return op0;
5163 :
5164 56210 : case TARGET_MEM_REF:
5165 56210 : if (TREE_CODE (TMR_BASE (exp)) == ADDR_EXPR
5166 56210 : && !DECL_RTL_SET_P (TREE_OPERAND (TMR_BASE (exp), 0)))
5167 : return NULL;
5168 :
5169 56202 : op0 = expand_debug_expr
5170 56202 : (tree_mem_ref_addr (build_pointer_type (TREE_TYPE (exp)), exp));
5171 56202 : if (!op0)
5172 : return NULL;
5173 :
5174 56202 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
5175 56202 : op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
5176 : op0, as);
5177 56202 : if (op0 == NULL_RTX)
5178 : return NULL;
5179 :
5180 56202 : op0 = gen_rtx_MEM (mode, op0);
5181 :
5182 56202 : set_mem_attributes (op0, exp, 0);
5183 56202 : set_mem_addr_space (op0, as);
5184 :
5185 56202 : return op0;
5186 :
5187 46827 : component_ref:
5188 3912734 : case ARRAY_REF:
5189 3912734 : case ARRAY_RANGE_REF:
5190 3912734 : case COMPONENT_REF:
5191 3912734 : case BIT_FIELD_REF:
5192 3912734 : case REALPART_EXPR:
5193 3912734 : case IMAGPART_EXPR:
5194 3912734 : case VIEW_CONVERT_EXPR:
5195 3912734 : {
5196 3912734 : machine_mode mode1;
5197 3912734 : poly_int64 bitsize, bitpos;
5198 3912734 : tree offset;
5199 3912734 : int reversep, volatilep = 0;
5200 3912734 : tree tem
5201 3912734 : = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
5202 : &unsignedp, &reversep, &volatilep);
5203 3912734 : rtx orig_op0;
5204 :
5205 3912734 : if (known_eq (bitsize, 0))
5206 : return NULL;
5207 :
5208 3853115 : orig_op0 = op0 = expand_debug_expr (tem);
5209 :
5210 3853115 : if (!op0)
5211 : return NULL;
5212 :
5213 3503536 : if (offset)
5214 : {
5215 12169 : machine_mode addrmode, offmode;
5216 :
5217 12169 : if (!MEM_P (op0))
5218 : return NULL;
5219 :
5220 12161 : op0 = XEXP (op0, 0);
5221 12161 : addrmode = GET_MODE (op0);
5222 12161 : if (addrmode == VOIDmode)
5223 0 : addrmode = Pmode;
5224 :
5225 12161 : op1 = expand_debug_expr (offset);
5226 12161 : if (!op1)
5227 : return NULL;
5228 :
5229 12159 : offmode = GET_MODE (op1);
5230 12159 : if (offmode == VOIDmode)
5231 6 : offmode = TYPE_MODE (TREE_TYPE (offset));
5232 :
5233 12159 : if (addrmode != offmode)
5234 0 : op1 = lowpart_subreg (addrmode, op1, offmode);
5235 :
5236 : /* Don't use offset_address here, we don't need a
5237 : recognizable address, and we don't want to generate
5238 : code. */
5239 12159 : op0 = gen_rtx_MEM (mode, simplify_gen_binary (PLUS, addrmode,
5240 : op0, op1));
5241 : }
5242 :
5243 3503526 : if (MEM_P (op0))
5244 : {
5245 3365542 : if (mode1 == VOIDmode)
5246 : {
5247 3437 : if (maybe_gt (bitsize, MAX_BITSIZE_MODE_ANY_INT))
5248 3912734 : return NULL;
5249 : /* Bitfield. */
5250 3437 : mode1 = smallest_int_mode_for_size (bitsize).require ();
5251 : }
5252 3365542 : poly_int64 bytepos = bits_to_bytes_round_down (bitpos);
5253 3365542 : if (maybe_ne (bytepos, 0))
5254 : {
5255 1385138 : op0 = adjust_address_nv (op0, mode1, bytepos);
5256 1385138 : bitpos = num_trailing_bits (bitpos);
5257 : }
5258 1980404 : else if (known_eq (bitpos, 0)
5259 3960180 : && known_eq (bitsize, GET_MODE_BITSIZE (mode)))
5260 515316 : op0 = adjust_address_nv (op0, mode, 0);
5261 1465088 : else if (GET_MODE (op0) != mode1)
5262 3111 : op0 = adjust_address_nv (op0, mode1, 0);
5263 : else
5264 1461977 : op0 = copy_rtx (op0);
5265 3365542 : if (op0 == orig_op0)
5266 152152 : op0 = shallow_copy_rtx (op0);
5267 3365542 : if (TREE_CODE (tem) != SSA_NAME)
5268 3363157 : set_mem_attributes (op0, exp, 0);
5269 : }
5270 :
5271 3503526 : if (known_eq (bitpos, 0) && mode == GET_MODE (op0))
5272 : return op0;
5273 :
5274 103726 : if (maybe_lt (bitpos, 0))
5275 : return NULL;
5276 :
5277 103724 : if (GET_MODE (op0) == BLKmode || mode == BLKmode)
5278 : return NULL;
5279 :
5280 103687 : poly_int64 bytepos;
5281 103687 : if (multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
5282 204196 : && known_eq (bitsize, GET_MODE_BITSIZE (mode1)))
5283 : {
5284 101258 : machine_mode opmode = GET_MODE (op0);
5285 :
5286 101258 : if (opmode == VOIDmode)
5287 0 : opmode = TYPE_MODE (TREE_TYPE (tem));
5288 :
5289 : /* This condition may hold if we're expanding the address
5290 : right past the end of an array that turned out not to
5291 : be addressable (i.e., the address was only computed in
5292 : debug stmts). The gen_subreg below would rightfully
5293 : crash, and the address doesn't really exist, so just
5294 : drop it. */
5295 202516 : if (known_ge (bitpos, GET_MODE_BITSIZE (opmode)))
5296 : return NULL;
5297 :
5298 204893 : if (multiple_p (bitpos, GET_MODE_BITSIZE (mode)))
5299 101186 : return simplify_gen_subreg (mode, op0, opmode, bytepos);
5300 : }
5301 :
5302 4950 : return simplify_gen_ternary (SCALAR_INT_MODE_P (GET_MODE (op0))
5303 1852 : && TYPE_UNSIGNED (TREE_TYPE (exp))
5304 : ? SIGN_EXTRACT
5305 : : ZERO_EXTRACT, mode,
5306 2475 : GET_MODE (op0) != VOIDmode
5307 : ? GET_MODE (op0)
5308 0 : : TYPE_MODE (TREE_TYPE (tem)),
5309 : op0, gen_int_mode (bitsize, word_mode),
5310 2475 : gen_int_mode (bitpos, word_mode));
5311 : }
5312 :
5313 3997 : case ABS_EXPR:
5314 3997 : case ABSU_EXPR:
5315 3997 : return simplify_gen_unary (ABS, mode, op0, mode);
5316 :
5317 14270 : case NEGATE_EXPR:
5318 14270 : return simplify_gen_unary (NEG, mode, op0, mode);
5319 :
5320 20945 : case BIT_NOT_EXPR:
5321 20945 : return simplify_gen_unary (NOT, mode, op0, mode);
5322 :
5323 3330 : case FLOAT_EXPR:
5324 3330 : return simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5325 : 0)))
5326 : ? UNSIGNED_FLOAT : FLOAT, mode, op0,
5327 3330 : inner_mode);
5328 :
5329 456 : case FIX_TRUNC_EXPR:
5330 456 : return simplify_gen_unary (unsignedp ? UNSIGNED_FIX : FIX, mode, op0,
5331 456 : inner_mode);
5332 :
5333 239884 : case POINTER_PLUS_EXPR:
5334 : /* For the rare target where pointers are not the same size as
5335 : size_t, we need to check for mis-matched modes and correct
5336 : the addend. */
5337 239884 : if (op0 && op1
5338 239884 : && is_a <scalar_int_mode> (GET_MODE (op0), &op0_mode)
5339 239782 : && is_a <scalar_int_mode> (GET_MODE (op1), &op1_mode)
5340 381679 : && op0_mode != op1_mode)
5341 : {
5342 0 : if (GET_MODE_BITSIZE (op0_mode) < GET_MODE_BITSIZE (op1_mode)
5343 : /* If OP0 is a partial mode, then we must truncate, even
5344 : if it has the same bitsize as OP1 as GCC's
5345 : representation of partial modes is opaque. */
5346 0 : || (GET_MODE_CLASS (op0_mode) == MODE_PARTIAL_INT
5347 0 : && (GET_MODE_BITSIZE (op0_mode)
5348 0 : == GET_MODE_BITSIZE (op1_mode))))
5349 0 : op1 = simplify_gen_unary (TRUNCATE, op0_mode, op1, op1_mode);
5350 : else
5351 : /* We always sign-extend, regardless of the signedness of
5352 : the operand, because the operand is always unsigned
5353 : here even if the original C expression is signed. */
5354 0 : op1 = simplify_gen_unary (SIGN_EXTEND, op0_mode, op1, op1_mode);
5355 : }
5356 : /* Fall through. */
5357 737893 : case PLUS_EXPR:
5358 737893 : return simplify_gen_binary (PLUS, mode, op0, op1);
5359 :
5360 114655 : case MINUS_EXPR:
5361 114655 : case POINTER_DIFF_EXPR:
5362 114655 : return simplify_gen_binary (MINUS, mode, op0, op1);
5363 :
5364 179548 : case MULT_EXPR:
5365 179548 : return simplify_gen_binary (MULT, mode, op0, op1);
5366 :
5367 4145 : case RDIV_EXPR:
5368 4145 : gcc_assert (FLOAT_MODE_P (mode)
5369 : || ALL_FIXED_POINT_MODE_P (mode));
5370 : /* Fall through. */
5371 100982 : case TRUNC_DIV_EXPR:
5372 100982 : case EXACT_DIV_EXPR:
5373 100982 : if (unsignedp)
5374 10501 : return simplify_gen_binary (UDIV, mode, op0, op1);
5375 : else
5376 90481 : return simplify_gen_binary (DIV, mode, op0, op1);
5377 :
5378 4173 : case TRUNC_MOD_EXPR:
5379 8346 : return simplify_gen_binary (unsignedp ? UMOD : MOD, mode, op0, op1);
5380 :
5381 3 : case FLOOR_DIV_EXPR:
5382 3 : if (unsignedp)
5383 0 : return simplify_gen_binary (UDIV, mode, op0, op1);
5384 : else
5385 : {
5386 3 : rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5387 3 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5388 3 : rtx adj = floor_sdiv_adjust (mode, mod, op1);
5389 3 : return simplify_gen_binary (PLUS, mode, div, adj);
5390 : }
5391 :
5392 1 : case FLOOR_MOD_EXPR:
5393 1 : if (unsignedp)
5394 0 : return simplify_gen_binary (UMOD, mode, op0, op1);
5395 : else
5396 : {
5397 1 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5398 1 : rtx adj = floor_sdiv_adjust (mode, mod, op1);
5399 1 : adj = simplify_gen_unary (NEG, mode,
5400 : simplify_gen_binary (MULT, mode, adj, op1),
5401 : mode);
5402 1 : return simplify_gen_binary (PLUS, mode, mod, adj);
5403 : }
5404 :
5405 3 : case CEIL_DIV_EXPR:
5406 3 : if (unsignedp)
5407 : {
5408 0 : rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
5409 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5410 0 : rtx adj = ceil_udiv_adjust (mode, mod, op1);
5411 0 : return simplify_gen_binary (PLUS, mode, div, adj);
5412 : }
5413 : else
5414 : {
5415 3 : rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5416 3 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5417 3 : rtx adj = ceil_sdiv_adjust (mode, mod, op1);
5418 3 : return simplify_gen_binary (PLUS, mode, div, adj);
5419 : }
5420 :
5421 0 : case CEIL_MOD_EXPR:
5422 0 : if (unsignedp)
5423 : {
5424 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5425 0 : rtx adj = ceil_udiv_adjust (mode, mod, op1);
5426 0 : adj = simplify_gen_unary (NEG, mode,
5427 : simplify_gen_binary (MULT, mode, adj, op1),
5428 : mode);
5429 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5430 : }
5431 : else
5432 : {
5433 0 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5434 0 : rtx adj = ceil_sdiv_adjust (mode, mod, op1);
5435 0 : adj = simplify_gen_unary (NEG, mode,
5436 : simplify_gen_binary (MULT, mode, adj, op1),
5437 : mode);
5438 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5439 : }
5440 :
5441 0 : case ROUND_DIV_EXPR:
5442 0 : if (unsignedp)
5443 : {
5444 0 : rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
5445 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5446 0 : rtx adj = round_udiv_adjust (mode, mod, op1);
5447 0 : return simplify_gen_binary (PLUS, mode, div, adj);
5448 : }
5449 : else
5450 : {
5451 0 : rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5452 0 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5453 0 : rtx adj = round_sdiv_adjust (mode, mod, op1);
5454 0 : return simplify_gen_binary (PLUS, mode, div, adj);
5455 : }
5456 :
5457 0 : case ROUND_MOD_EXPR:
5458 0 : if (unsignedp)
5459 : {
5460 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5461 0 : rtx adj = round_udiv_adjust (mode, mod, op1);
5462 0 : adj = simplify_gen_unary (NEG, mode,
5463 : simplify_gen_binary (MULT, mode, adj, op1),
5464 : mode);
5465 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5466 : }
5467 : else
5468 : {
5469 0 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5470 0 : rtx adj = round_sdiv_adjust (mode, mod, op1);
5471 0 : adj = simplify_gen_unary (NEG, mode,
5472 : simplify_gen_binary (MULT, mode, adj, op1),
5473 : mode);
5474 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5475 : }
5476 :
5477 14865 : case LSHIFT_EXPR:
5478 14865 : return simplify_gen_binary (ASHIFT, mode, op0, op1);
5479 :
5480 67584 : case RSHIFT_EXPR:
5481 67584 : if (unsignedp)
5482 61295 : return simplify_gen_binary (LSHIFTRT, mode, op0, op1);
5483 : else
5484 6289 : return simplify_gen_binary (ASHIFTRT, mode, op0, op1);
5485 :
5486 2 : case LROTATE_EXPR:
5487 2 : return simplify_gen_binary (ROTATE, mode, op0, op1);
5488 :
5489 883 : case RROTATE_EXPR:
5490 883 : return simplify_gen_binary (ROTATERT, mode, op0, op1);
5491 :
5492 40034 : case MIN_EXPR:
5493 80068 : return simplify_gen_binary (unsignedp ? UMIN : SMIN, mode, op0, op1);
5494 :
5495 39656 : case MAX_EXPR:
5496 79312 : return simplify_gen_binary (unsignedp ? UMAX : SMAX, mode, op0, op1);
5497 :
5498 93454 : case BIT_AND_EXPR:
5499 93454 : case TRUTH_AND_EXPR:
5500 93454 : return simplify_gen_binary (AND, mode, op0, op1);
5501 :
5502 43158 : case BIT_IOR_EXPR:
5503 43158 : case TRUTH_OR_EXPR:
5504 43158 : return simplify_gen_binary (IOR, mode, op0, op1);
5505 :
5506 11273 : case BIT_XOR_EXPR:
5507 11273 : case TRUTH_XOR_EXPR:
5508 11273 : return simplify_gen_binary (XOR, mode, op0, op1);
5509 :
5510 0 : case TRUTH_ANDIF_EXPR:
5511 0 : return gen_rtx_IF_THEN_ELSE (mode, op0, op1, const0_rtx);
5512 :
5513 0 : case TRUTH_ORIF_EXPR:
5514 0 : return gen_rtx_IF_THEN_ELSE (mode, op0, const_true_rtx, op1);
5515 :
5516 0 : case TRUTH_NOT_EXPR:
5517 0 : return simplify_gen_relational (EQ, mode, inner_mode, op0, const0_rtx);
5518 :
5519 5229 : case LT_EXPR:
5520 5229 : return simplify_gen_relational (unsignedp ? LTU : LT, mode, inner_mode,
5521 5229 : op0, op1);
5522 :
5523 6269 : case LE_EXPR:
5524 6269 : return simplify_gen_relational (unsignedp ? LEU : LE, mode, inner_mode,
5525 6269 : op0, op1);
5526 :
5527 9974 : case GT_EXPR:
5528 9974 : return simplify_gen_relational (unsignedp ? GTU : GT, mode, inner_mode,
5529 9974 : op0, op1);
5530 :
5531 3096 : case GE_EXPR:
5532 3096 : return simplify_gen_relational (unsignedp ? GEU : GE, mode, inner_mode,
5533 3096 : op0, op1);
5534 :
5535 30737 : case EQ_EXPR:
5536 30737 : return simplify_gen_relational (EQ, mode, inner_mode, op0, op1);
5537 :
5538 15368 : case NE_EXPR:
5539 15368 : return simplify_gen_relational (NE, mode, inner_mode, op0, op1);
5540 :
5541 1587 : case UNORDERED_EXPR:
5542 1587 : return simplify_gen_relational (UNORDERED, mode, inner_mode, op0, op1);
5543 :
5544 1 : case ORDERED_EXPR:
5545 1 : return simplify_gen_relational (ORDERED, mode, inner_mode, op0, op1);
5546 :
5547 0 : case UNLT_EXPR:
5548 0 : return simplify_gen_relational (UNLT, mode, inner_mode, op0, op1);
5549 :
5550 1576 : case UNLE_EXPR:
5551 1576 : return simplify_gen_relational (UNLE, mode, inner_mode, op0, op1);
5552 :
5553 0 : case UNGT_EXPR:
5554 0 : return simplify_gen_relational (UNGT, mode, inner_mode, op0, op1);
5555 :
5556 0 : case UNGE_EXPR:
5557 0 : return simplify_gen_relational (UNGE, mode, inner_mode, op0, op1);
5558 :
5559 0 : case UNEQ_EXPR:
5560 0 : return simplify_gen_relational (UNEQ, mode, inner_mode, op0, op1);
5561 :
5562 0 : case LTGT_EXPR:
5563 0 : return simplify_gen_relational (LTGT, mode, inner_mode, op0, op1);
5564 :
5565 8331 : case COND_EXPR:
5566 8331 : return gen_rtx_IF_THEN_ELSE (mode, op0, op1, op2);
5567 :
5568 10745 : case COMPLEX_EXPR:
5569 10745 : gcc_assert (COMPLEX_MODE_P (mode));
5570 10745 : if (GET_MODE (op0) == VOIDmode)
5571 4 : op0 = gen_rtx_CONST (GET_MODE_INNER (mode), op0);
5572 10745 : if (GET_MODE (op1) == VOIDmode)
5573 0 : op1 = gen_rtx_CONST (GET_MODE_INNER (mode), op1);
5574 10745 : return gen_rtx_CONCAT (mode, op0, op1);
5575 :
5576 0 : case CONJ_EXPR:
5577 0 : if (GET_CODE (op0) == CONCAT)
5578 0 : return gen_rtx_CONCAT (mode, XEXP (op0, 0),
5579 : simplify_gen_unary (NEG, GET_MODE_INNER (mode),
5580 : XEXP (op0, 1),
5581 : GET_MODE_INNER (mode)));
5582 : else
5583 : {
5584 0 : scalar_mode imode = GET_MODE_INNER (mode);
5585 0 : rtx re, im;
5586 :
5587 0 : if (MEM_P (op0))
5588 : {
5589 0 : re = adjust_address_nv (op0, imode, 0);
5590 0 : im = adjust_address_nv (op0, imode, GET_MODE_SIZE (imode));
5591 : }
5592 : else
5593 : {
5594 0 : scalar_int_mode ifmode;
5595 0 : scalar_int_mode ihmode;
5596 0 : rtx halfsize;
5597 0 : if (!int_mode_for_mode (mode).exists (&ifmode)
5598 0 : || !int_mode_for_mode (imode).exists (&ihmode))
5599 43621833 : return NULL;
5600 0 : halfsize = GEN_INT (GET_MODE_BITSIZE (ihmode));
5601 0 : re = op0;
5602 0 : if (mode != ifmode)
5603 0 : re = gen_rtx_SUBREG (ifmode, re, 0);
5604 0 : re = gen_rtx_ZERO_EXTRACT (ihmode, re, halfsize, const0_rtx);
5605 0 : if (imode != ihmode)
5606 0 : re = gen_rtx_SUBREG (imode, re, 0);
5607 0 : im = copy_rtx (op0);
5608 0 : if (mode != ifmode)
5609 0 : im = gen_rtx_SUBREG (ifmode, im, 0);
5610 0 : im = gen_rtx_ZERO_EXTRACT (ihmode, im, halfsize, halfsize);
5611 0 : if (imode != ihmode)
5612 0 : im = gen_rtx_SUBREG (imode, im, 0);
5613 : }
5614 0 : im = gen_rtx_NEG (imode, im);
5615 0 : return gen_rtx_CONCAT (mode, re, im);
5616 : }
5617 :
5618 7164396 : case ADDR_EXPR:
5619 7164396 : op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5620 7164396 : if (!op0 || !MEM_P (op0))
5621 : {
5622 1897270 : if ((TREE_CODE (TREE_OPERAND (exp, 0)) == VAR_DECL
5623 437895 : || TREE_CODE (TREE_OPERAND (exp, 0)) == PARM_DECL
5624 388945 : || TREE_CODE (TREE_OPERAND (exp, 0)) == RESULT_DECL)
5625 1946220 : && (!TREE_ADDRESSABLE (TREE_OPERAND (exp, 0))
5626 190234 : || target_for_debug_bind (TREE_OPERAND (exp, 0))))
5627 1329481 : return gen_rtx_DEBUG_IMPLICIT_PTR (mode, TREE_OPERAND (exp, 0));
5628 :
5629 567789 : if (handled_component_p (TREE_OPERAND (exp, 0)))
5630 : {
5631 363698 : poly_int64 bitoffset, bitsize, maxsize, byteoffset;
5632 363698 : bool reverse;
5633 363698 : tree decl
5634 363698 : = get_ref_base_and_extent (TREE_OPERAND (exp, 0), &bitoffset,
5635 : &bitsize, &maxsize, &reverse);
5636 363698 : if ((VAR_P (decl)
5637 : || TREE_CODE (decl) == PARM_DECL
5638 : || TREE_CODE (decl) == RESULT_DECL)
5639 332044 : && (!TREE_ADDRESSABLE (decl)
5640 30216 : || target_for_debug_bind (decl))
5641 78963 : && multiple_p (bitoffset, BITS_PER_UNIT, &byteoffset)
5642 301828 : && known_gt (bitsize, 0)
5643 284738 : && known_eq (bitsize, maxsize))
5644 : {
5645 284735 : rtx base = gen_rtx_DEBUG_IMPLICIT_PTR (mode, decl);
5646 284735 : return plus_constant (mode, base, byteoffset);
5647 : }
5648 : }
5649 :
5650 283054 : if (TREE_CODE (TREE_OPERAND (exp, 0)) == MEM_REF
5651 283054 : && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
5652 : == ADDR_EXPR)
5653 : {
5654 12144 : op0 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5655 : 0));
5656 12144 : if (op0 != NULL
5657 6360 : && (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5658 0 : || (GET_CODE (op0) == PLUS
5659 0 : && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR
5660 0 : && CONST_INT_P (XEXP (op0, 1)))))
5661 : {
5662 6360 : op1 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5663 : 1));
5664 6360 : poly_int64 offset;
5665 12720 : if (!op1 || !poly_int_rtx_p (op1, &offset))
5666 : return NULL;
5667 :
5668 6360 : return plus_constant (mode, op0, offset);
5669 : }
5670 : }
5671 :
5672 : return NULL;
5673 : }
5674 :
5675 5267126 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
5676 5267126 : addr_mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
5677 5267126 : op0 = convert_debug_memory_address (addr_mode, XEXP (op0, 0), as);
5678 :
5679 5267126 : return op0;
5680 :
5681 11837 : case VECTOR_CST:
5682 11837 : {
5683 11837 : unsigned HOST_WIDE_INT i, nelts;
5684 :
5685 11837 : if (!VECTOR_CST_NELTS (exp).is_constant (&nelts))
5686 : return NULL;
5687 :
5688 11837 : op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5689 :
5690 65873 : for (i = 0; i < nelts; ++i)
5691 : {
5692 54036 : op1 = expand_debug_expr (VECTOR_CST_ELT (exp, i));
5693 54036 : if (!op1)
5694 : return NULL;
5695 54036 : XVECEXP (op0, 0, i) = op1;
5696 : }
5697 :
5698 : return op0;
5699 : }
5700 :
5701 1128 : case CONSTRUCTOR:
5702 1128 : if (TREE_CLOBBER_P (exp))
5703 : return NULL;
5704 1128 : else if (TREE_CODE (TREE_TYPE (exp)) == VECTOR_TYPE)
5705 : {
5706 1128 : unsigned i;
5707 1128 : poly_uint64 elts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (exp));
5708 1128 : unsigned HOST_WIDE_INT nelts = constant_lower_bound (elts);
5709 1128 : tree val;
5710 :
5711 1128 : op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5712 :
5713 5215 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), i, val)
5714 : {
5715 4220 : op1 = expand_debug_expr (val);
5716 4220 : if (!op1)
5717 : return NULL;
5718 4087 : XVECEXP (op0, 0, i) = op1;
5719 : }
5720 :
5721 995 : if (i < nelts)
5722 : {
5723 117 : op1 = expand_debug_expr
5724 117 : (build_zero_cst (TREE_TYPE (TREE_TYPE (exp))));
5725 :
5726 117 : if (!op1)
5727 : return NULL;
5728 :
5729 384 : for (; i < nelts; i++)
5730 267 : XVECEXP (op0, 0, i) = op1;
5731 : }
5732 :
5733 : return op0;
5734 : }
5735 : else
5736 0 : goto flag_unsupported;
5737 :
5738 : case CALL_EXPR:
5739 : /* ??? Maybe handle some builtins? */
5740 : return NULL;
5741 :
5742 9787995 : case SSA_NAME:
5743 9787995 : {
5744 9787995 : gimple *g = get_gimple_for_ssa_name (exp);
5745 9787995 : if (g)
5746 : {
5747 1510015 : tree t = NULL_TREE;
5748 1510015 : if (deep_ter_debug_map)
5749 : {
5750 72607 : tree *slot = deep_ter_debug_map->get (exp);
5751 72607 : if (slot)
5752 19332 : t = *slot;
5753 : }
5754 19332 : if (t == NULL_TREE)
5755 : {
5756 1496094 : if (is_gimple_assign (g))
5757 1495649 : t = gimple_assign_rhs_to_tree (g);
5758 : else
5759 : /* expand_debug_expr doesn't handle CALL_EXPR right now. */
5760 : return NULL;
5761 : }
5762 1509570 : op0 = expand_debug_expr (t);
5763 1509570 : if (!op0)
5764 : return NULL;
5765 : }
5766 : else
5767 : {
5768 : /* If this is a reference to an incoming value of
5769 : parameter that is never used in the code or where the
5770 : incoming value is never used in the code, use
5771 : PARM_DECL's DECL_RTL if set. */
5772 8277980 : if (SSA_NAME_IS_DEFAULT_DEF (exp)
5773 1343655 : && SSA_NAME_VAR (exp)
5774 1343655 : && TREE_CODE (SSA_NAME_VAR (exp)) == PARM_DECL
5775 9560679 : && has_zero_uses (exp))
5776 : {
5777 18675 : op0 = expand_debug_parm_decl (SSA_NAME_VAR (exp));
5778 18675 : if (op0)
5779 18667 : goto adjust_mode;
5780 8 : op0 = expand_debug_expr (SSA_NAME_VAR (exp));
5781 8 : if (op0)
5782 8 : goto adjust_mode;
5783 : }
5784 :
5785 8259305 : int part = var_to_partition (SA.map, exp);
5786 :
5787 8259305 : if (part == NO_PARTITION)
5788 : return NULL;
5789 :
5790 8254988 : gcc_assert (part >= 0 && (unsigned)part < SA.map->num_partitions);
5791 :
5792 8254988 : op0 = copy_rtx (SA.partition_to_pseudo[part]);
5793 : }
5794 9758073 : goto adjust_mode;
5795 : }
5796 :
5797 : case ERROR_MARK:
5798 : return NULL;
5799 :
5800 : /* Vector stuff. For most of the codes we don't have rtl codes. */
5801 : case REALIGN_LOAD_EXPR:
5802 : case VEC_COND_EXPR:
5803 : case VEC_PACK_FIX_TRUNC_EXPR:
5804 : case VEC_PACK_FLOAT_EXPR:
5805 : case VEC_PACK_SAT_EXPR:
5806 : case VEC_PACK_TRUNC_EXPR:
5807 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
5808 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
5809 : case VEC_UNPACK_FLOAT_HI_EXPR:
5810 : case VEC_UNPACK_FLOAT_LO_EXPR:
5811 : case VEC_UNPACK_HI_EXPR:
5812 : case VEC_UNPACK_LO_EXPR:
5813 : case VEC_WIDEN_MULT_HI_EXPR:
5814 : case VEC_WIDEN_MULT_LO_EXPR:
5815 : case VEC_WIDEN_MULT_EVEN_EXPR:
5816 : case VEC_WIDEN_MULT_ODD_EXPR:
5817 : case VEC_WIDEN_LSHIFT_HI_EXPR:
5818 : case VEC_WIDEN_LSHIFT_LO_EXPR:
5819 : case VEC_PERM_EXPR:
5820 : case VEC_DUPLICATE_EXPR:
5821 : case VEC_SERIES_EXPR:
5822 : case SAD_EXPR:
5823 : return NULL;
5824 :
5825 : /* Misc codes. */
5826 : case ADDR_SPACE_CONVERT_EXPR:
5827 : case FIXED_CONVERT_EXPR:
5828 : case OBJ_TYPE_REF:
5829 : case WITH_SIZE_EXPR:
5830 : case BIT_INSERT_EXPR:
5831 : return NULL;
5832 :
5833 0 : case DOT_PROD_EXPR:
5834 0 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
5835 0 : && SCALAR_INT_MODE_P (mode))
5836 : {
5837 0 : op0
5838 0 : = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5839 : 0)))
5840 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5841 : inner_mode);
5842 0 : op1
5843 0 : = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5844 : 1)))
5845 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op1,
5846 : inner_mode);
5847 0 : op0 = simplify_gen_binary (MULT, mode, op0, op1);
5848 0 : return simplify_gen_binary (PLUS, mode, op0, op2);
5849 : }
5850 : return NULL;
5851 :
5852 5954 : case WIDEN_MULT_EXPR:
5853 5954 : case WIDEN_MULT_PLUS_EXPR:
5854 5954 : case WIDEN_MULT_MINUS_EXPR:
5855 5954 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
5856 5954 : && SCALAR_INT_MODE_P (mode))
5857 : {
5858 5954 : inner_mode = GET_MODE (op0);
5859 5954 : if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5860 5338 : op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5861 : else
5862 616 : op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5863 5954 : if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 1))))
5864 5338 : op1 = simplify_gen_unary (ZERO_EXTEND, mode, op1, inner_mode);
5865 : else
5866 616 : op1 = simplify_gen_unary (SIGN_EXTEND, mode, op1, inner_mode);
5867 5954 : op0 = simplify_gen_binary (MULT, mode, op0, op1);
5868 5954 : if (TREE_CODE (exp) == WIDEN_MULT_EXPR)
5869 : return op0;
5870 0 : else if (TREE_CODE (exp) == WIDEN_MULT_PLUS_EXPR)
5871 0 : return simplify_gen_binary (PLUS, mode, op0, op2);
5872 : else
5873 0 : return simplify_gen_binary (MINUS, mode, op2, op0);
5874 : }
5875 : return NULL;
5876 :
5877 : case MULT_HIGHPART_EXPR:
5878 : /* ??? Similar to the above. */
5879 : return NULL;
5880 :
5881 0 : case WIDEN_SUM_EXPR:
5882 0 : case WIDEN_LSHIFT_EXPR:
5883 0 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
5884 0 : && SCALAR_INT_MODE_P (mode))
5885 : {
5886 0 : op0
5887 0 : = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5888 : 0)))
5889 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5890 : inner_mode);
5891 0 : return simplify_gen_binary (TREE_CODE (exp) == WIDEN_LSHIFT_EXPR
5892 0 : ? ASHIFT : PLUS, mode, op0, op1);
5893 : }
5894 : return NULL;
5895 :
5896 0 : default:
5897 0 : flag_unsupported:
5898 0 : if (flag_checking)
5899 : {
5900 0 : debug_tree (exp);
5901 0 : gcc_unreachable ();
5902 : }
5903 : return NULL;
5904 : }
5905 : }
5906 :
5907 : /* Return an RTX equivalent to the source bind value of the tree expression
5908 : EXP. */
5909 :
5910 : static rtx
5911 176022 : expand_debug_source_expr (tree exp)
5912 : {
5913 330589 : rtx op0 = NULL_RTX;
5914 330589 : machine_mode mode = VOIDmode, inner_mode;
5915 :
5916 330589 : switch (TREE_CODE (exp))
5917 : {
5918 154567 : case VAR_DECL:
5919 154567 : if (DECL_ABSTRACT_ORIGIN (exp))
5920 154567 : return expand_debug_source_expr (DECL_ABSTRACT_ORIGIN (exp));
5921 : break;
5922 170930 : case PARM_DECL:
5923 170930 : {
5924 170930 : mode = DECL_MODE (exp);
5925 170930 : op0 = expand_debug_parm_decl (exp);
5926 170930 : if (op0)
5927 : break;
5928 : /* See if this isn't an argument that has been completely
5929 : optimized out. */
5930 169960 : if (!DECL_RTL_SET_P (exp)
5931 169944 : && !DECL_INCOMING_RTL (exp)
5932 339904 : && DECL_ABSTRACT_ORIGIN (current_function_decl))
5933 : {
5934 58482 : tree aexp = DECL_ORIGIN (exp);
5935 58482 : if (DECL_CONTEXT (aexp)
5936 58482 : == DECL_ABSTRACT_ORIGIN (current_function_decl))
5937 : {
5938 19407 : vec<tree, va_gc> **debug_args;
5939 19407 : unsigned int ix;
5940 19407 : tree ddecl;
5941 19407 : debug_args = decl_debug_args_lookup (current_function_decl);
5942 19407 : if (debug_args != NULL)
5943 : {
5944 23943 : for (ix = 0; vec_safe_iterate (*debug_args, ix, &ddecl);
5945 4540 : ix += 2)
5946 23935 : if (ddecl == aexp)
5947 19395 : return gen_rtx_DEBUG_PARAMETER_REF (mode, aexp);
5948 : }
5949 : }
5950 : }
5951 : break;
5952 : }
5953 : default:
5954 : break;
5955 : }
5956 :
5957 : if (op0 == NULL_RTX)
5958 : return NULL_RTX;
5959 :
5960 970 : inner_mode = GET_MODE (op0);
5961 970 : if (mode == inner_mode)
5962 : return op0;
5963 :
5964 21 : if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5965 : {
5966 7 : if (GET_MODE_UNIT_BITSIZE (mode)
5967 14 : == GET_MODE_UNIT_BITSIZE (inner_mode))
5968 0 : op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
5969 7 : else if (GET_MODE_UNIT_BITSIZE (mode)
5970 14 : < GET_MODE_UNIT_BITSIZE (inner_mode))
5971 7 : op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
5972 : else
5973 0 : op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
5974 : }
5975 14 : else if (FLOAT_MODE_P (mode))
5976 0 : gcc_unreachable ();
5977 14 : else if (FLOAT_MODE_P (inner_mode))
5978 : {
5979 0 : if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5980 0 : op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
5981 : else
5982 0 : op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
5983 : }
5984 28 : else if (GET_MODE_UNIT_PRECISION (mode)
5985 14 : == GET_MODE_UNIT_PRECISION (inner_mode))
5986 0 : op0 = lowpart_subreg (mode, op0, inner_mode);
5987 28 : else if (GET_MODE_UNIT_PRECISION (mode)
5988 14 : < GET_MODE_UNIT_PRECISION (inner_mode))
5989 14 : op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
5990 0 : else if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5991 0 : op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5992 : else
5993 0 : op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5994 :
5995 : return op0;
5996 : }
5997 :
5998 : /* Ensure INSN_VAR_LOCATION_LOC (insn) doesn't have unbound complexity.
5999 : Allow 4 levels of rtl nesting for most rtl codes, and if we see anything
6000 : deeper than that, create DEBUG_EXPRs and emit DEBUG_INSNs before INSN. */
6001 :
6002 : static void
6003 53520680 : avoid_complex_debug_insns (rtx_insn *insn, rtx *exp_p, int depth)
6004 : {
6005 53520680 : rtx exp = *exp_p;
6006 :
6007 53520680 : if (exp == NULL_RTX)
6008 : return;
6009 :
6010 53520680 : if ((OBJECT_P (exp) && !MEM_P (exp)) || GET_CODE (exp) == CLOBBER)
6011 : return;
6012 :
6013 6942587 : if (depth == 4)
6014 : {
6015 : /* Create DEBUG_EXPR (and DEBUG_EXPR_DECL). */
6016 25984 : rtx dval = make_debug_expr_from_rtl (exp);
6017 :
6018 : /* Emit a debug bind insn before INSN. */
6019 25984 : rtx bind = gen_rtx_VAR_LOCATION (GET_MODE (exp),
6020 : DEBUG_EXPR_TREE_DECL (dval), exp,
6021 : VAR_INIT_STATUS_INITIALIZED);
6022 :
6023 25984 : emit_debug_insn_before (bind, insn);
6024 25984 : *exp_p = dval;
6025 25984 : return;
6026 : }
6027 :
6028 6916603 : const char *format_ptr = GET_RTX_FORMAT (GET_CODE (exp));
6029 6916603 : int i, j;
6030 20630892 : for (i = 0; i < GET_RTX_LENGTH (GET_CODE (exp)); i++)
6031 13714289 : switch (*format_ptr++)
6032 : {
6033 12635001 : case 'e':
6034 12635001 : avoid_complex_debug_insns (insn, &XEXP (exp, i), depth + 1);
6035 12635001 : break;
6036 :
6037 : case 'E':
6038 : case 'V':
6039 0 : for (j = 0; j < XVECLEN (exp, i); j++)
6040 0 : avoid_complex_debug_insns (insn, &XVECEXP (exp, i, j), depth + 1);
6041 : break;
6042 :
6043 : default:
6044 : break;
6045 : }
6046 : }
6047 :
6048 : /* Expand the _LOCs in debug insns. We run this after expanding all
6049 : regular insns, so that any variables referenced in the function
6050 : will have their DECL_RTLs set. */
6051 :
6052 : static void
6053 503732 : expand_debug_locations (void)
6054 : {
6055 503732 : rtx_insn *insn;
6056 503732 : rtx_insn *last = get_last_insn ();
6057 503732 : int save_strict_alias = flag_strict_aliasing;
6058 :
6059 : /* New alias sets while setting up memory attributes cause
6060 : -fcompare-debug failures, even though it doesn't bring about any
6061 : codegen changes. */
6062 503732 : flag_strict_aliasing = 0;
6063 :
6064 109334613 : for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
6065 161487042 : if (DEBUG_BIND_INSN_P (insn))
6066 : {
6067 40859695 : tree value = (tree)INSN_VAR_LOCATION_LOC (insn);
6068 40859695 : rtx val;
6069 40859695 : rtx_insn *prev_insn, *insn2;
6070 40859695 : machine_mode mode;
6071 :
6072 40859695 : if (value == NULL_TREE)
6073 : val = NULL_RTX;
6074 : else
6075 : {
6076 22902189 : if (INSN_VAR_LOCATION_STATUS (insn)
6077 : == VAR_INIT_STATUS_UNINITIALIZED)
6078 176022 : val = expand_debug_source_expr (value);
6079 : /* The avoid_deep_ter_for_debug function inserts
6080 : debug bind stmts after SSA_NAME definition, with the
6081 : SSA_NAME as the whole bind location. Disable temporarily
6082 : expansion of that SSA_NAME into the DEBUG_EXPR_DECL
6083 : being defined in this DEBUG_INSN. */
6084 22726167 : else if (deep_ter_debug_map && TREE_CODE (value) == SSA_NAME)
6085 : {
6086 75869 : tree *slot = deep_ter_debug_map->get (value);
6087 75869 : if (slot)
6088 : {
6089 6173 : if (*slot == INSN_VAR_LOCATION_DECL (insn))
6090 5411 : *slot = NULL_TREE;
6091 : else
6092 : slot = NULL;
6093 : }
6094 75869 : val = expand_debug_expr (value);
6095 75869 : if (slot)
6096 5411 : *slot = INSN_VAR_LOCATION_DECL (insn);
6097 : }
6098 : else
6099 22650298 : val = expand_debug_expr (value);
6100 22902189 : gcc_assert (last == get_last_insn ());
6101 : }
6102 :
6103 22902189 : if (!val)
6104 18451085 : val = gen_rtx_UNKNOWN_VAR_LOC ();
6105 : else
6106 : {
6107 22408610 : mode = GET_MODE (INSN_VAR_LOCATION (insn));
6108 :
6109 22408610 : gcc_assert (mode == GET_MODE (val)
6110 : || (GET_MODE (val) == VOIDmode
6111 : && (CONST_SCALAR_INT_P (val)
6112 : || GET_CODE (val) == CONST_FIXED
6113 : || GET_CODE (val) == LABEL_REF)));
6114 : }
6115 :
6116 40859695 : INSN_VAR_LOCATION_LOC (insn) = val;
6117 40859695 : prev_insn = PREV_INSN (insn);
6118 122605069 : for (insn2 = insn; insn2 != prev_insn; insn2 = PREV_INSN (insn2))
6119 40885679 : avoid_complex_debug_insns (insn2, &INSN_VAR_LOCATION_LOC (insn2), 0);
6120 : }
6121 :
6122 503732 : flag_strict_aliasing = save_strict_alias;
6123 503732 : }
6124 :
6125 : /* Performs swapping operands of commutative operations to expand
6126 : the expensive one first. */
6127 :
6128 : static void
6129 9540118 : reorder_operands (basic_block bb)
6130 : {
6131 9540118 : unsigned int *lattice; /* Hold cost of each statement. */
6132 9540118 : unsigned int i = 0, n = 0;
6133 9540118 : gimple_stmt_iterator gsi;
6134 9540118 : gimple_seq stmts;
6135 9540118 : gimple *stmt;
6136 9540118 : bool swap;
6137 9540118 : tree op0, op1;
6138 9540118 : ssa_op_iter iter;
6139 9540118 : use_operand_p use_p;
6140 9540118 : gimple *def0, *def1;
6141 :
6142 : /* Compute cost of each statement using estimate_num_insns. */
6143 9540118 : stmts = bb_seq (bb);
6144 97732251 : for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
6145 : {
6146 88192133 : stmt = gsi_stmt (gsi);
6147 88192133 : if (!is_gimple_debug (stmt))
6148 35074596 : gimple_set_uid (stmt, n++);
6149 : }
6150 9540118 : lattice = XNEWVEC (unsigned int, n);
6151 107272369 : for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
6152 : {
6153 88192133 : unsigned cost;
6154 88192133 : stmt = gsi_stmt (gsi);
6155 88192133 : if (is_gimple_debug (stmt))
6156 53117537 : continue;
6157 35074596 : cost = estimate_num_insns (stmt, &eni_size_weights);
6158 35074596 : lattice[i] = cost;
6159 66672726 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
6160 : {
6161 31598130 : tree use = USE_FROM_PTR (use_p);
6162 31598130 : gimple *def_stmt;
6163 31598130 : if (TREE_CODE (use) != SSA_NAME)
6164 0 : continue;
6165 31598130 : def_stmt = get_gimple_for_ssa_name (use);
6166 31598130 : if (!def_stmt)
6167 22492850 : continue;
6168 9105280 : lattice[i] += lattice[gimple_uid (def_stmt)];
6169 : }
6170 35074596 : i++;
6171 35074596 : if (!is_gimple_assign (stmt)
6172 35074596 : || !commutative_tree_code (gimple_assign_rhs_code (stmt)))
6173 30750904 : continue;
6174 4323692 : op0 = gimple_op (stmt, 1);
6175 4323692 : op1 = gimple_op (stmt, 2);
6176 4323692 : if (TREE_CODE (op0) != SSA_NAME
6177 4323427 : || TREE_CODE (op1) != SSA_NAME)
6178 2745280 : continue;
6179 : /* Swap operands if the second one is more expensive. */
6180 1578412 : def0 = get_gimple_for_ssa_name (op0);
6181 1578412 : def1 = get_gimple_for_ssa_name (op1);
6182 1578412 : if (!def1)
6183 872828 : continue;
6184 705584 : swap = false;
6185 705584 : if (!def0 || lattice[gimple_uid (def1)] > lattice[gimple_uid (def0)])
6186 330540 : swap = true;
6187 330540 : if (swap)
6188 : {
6189 330540 : if (dump_file && (dump_flags & TDF_DETAILS))
6190 : {
6191 2 : fprintf (dump_file, "Swap operands in stmt:\n");
6192 2 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
6193 2 : fprintf (dump_file, "Cost left opnd=%d, right opnd=%d\n",
6194 1 : def0 ? lattice[gimple_uid (def0)] : 0,
6195 2 : lattice[gimple_uid (def1)]);
6196 : }
6197 330540 : swap_ssa_operands (stmt, gimple_assign_rhs1_ptr (stmt),
6198 : gimple_assign_rhs2_ptr (stmt));
6199 : }
6200 : }
6201 9540118 : XDELETE (lattice);
6202 9540118 : }
6203 :
6204 : /* Expand basic block BB from GIMPLE trees to RTL. */
6205 :
6206 : static basic_block
6207 12929173 : expand_gimple_basic_block (basic_block bb, rtx_insn *asan_epilog_seq)
6208 : {
6209 12929173 : gimple_stmt_iterator gsi;
6210 12929173 : gimple *stmt = NULL;
6211 12929173 : rtx_note *note = NULL;
6212 12929173 : rtx_insn *last;
6213 12929173 : edge e;
6214 12929173 : edge_iterator ei;
6215 12929173 : bool nondebug_stmt_seen = false;
6216 :
6217 12929173 : if (dump_file)
6218 826 : fprintf (dump_file, "\n;; Generating RTL for gimple basic block %d\n",
6219 : bb->index);
6220 :
6221 : /* Note that since we are now transitioning from GIMPLE to RTL, we
6222 : cannot use the gsi_*_bb() routines because they expect the basic
6223 : block to be in GIMPLE, instead of RTL. Therefore, we need to
6224 : access the BB sequence directly. */
6225 12929173 : if (optimize)
6226 9540118 : reorder_operands (bb);
6227 12929173 : rtl_profile_for_bb (bb);
6228 :
6229 : /* Remove the RETURN_EXPR if we may fall though to the exit
6230 : instead. */
6231 12929173 : gsi = gsi_last_bb (bb);
6232 12929173 : if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) == GIMPLE_RETURN)
6233 : {
6234 1485203 : greturn *ret_stmt = as_a <greturn *> (gsi_stmt (gsi));
6235 :
6236 1485203 : gcc_assert (single_succ_p (bb));
6237 1485203 : gcc_assert (single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun));
6238 :
6239 1485203 : if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
6240 1485203 : && !gimple_return_retval (ret_stmt))
6241 : {
6242 622200 : gsi_remove (&gsi, false);
6243 622200 : single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
6244 : }
6245 : }
6246 :
6247 12929173 : gsi = gsi_start_bb (bb);
6248 12929173 : if (!gsi_end_p (gsi))
6249 : {
6250 12381747 : stmt = gsi_stmt (gsi);
6251 12381747 : if (gimple_code (stmt) != GIMPLE_LABEL)
6252 12279562 : stmt = NULL;
6253 : }
6254 :
6255 12929173 : rtx_code_label **elt = lab_rtx_for_bb->get (bb);
6256 12929173 : if ((unsigned) bb->index >= head_end_for_bb.length ())
6257 10934849 : head_end_for_bb.safe_grow_cleared (bb->index + 1);
6258 :
6259 12929173 : if (stmt || elt)
6260 : {
6261 5355922 : gcc_checking_assert (!note);
6262 5355922 : last = get_last_insn ();
6263 :
6264 5355922 : if (stmt)
6265 : {
6266 649611 : expand_gimple_stmt (stmt);
6267 649611 : gsi_next (&gsi);
6268 : }
6269 :
6270 5355922 : if (elt)
6271 4706311 : emit_label (*elt);
6272 :
6273 5355922 : head_end_for_bb[bb->index].first = NEXT_INSN (last);
6274 5355922 : if (NOTE_P (head_end_for_bb[bb->index].first))
6275 0 : head_end_for_bb[bb->index].first
6276 0 : = NEXT_INSN (head_end_for_bb[bb->index].first);
6277 5355922 : gcc_assert (LABEL_P (head_end_for_bb[bb->index].first));
6278 10711844 : note = emit_note_after (NOTE_INSN_BASIC_BLOCK,
6279 5355922 : head_end_for_bb[bb->index].first);
6280 :
6281 5355922 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
6282 : }
6283 : else
6284 7573251 : head_end_for_bb[bb->index].first = note = emit_note (NOTE_INSN_BASIC_BLOCK);
6285 :
6286 12929173 : if (note)
6287 12929173 : NOTE_BASIC_BLOCK (note) = bb;
6288 :
6289 67620576 : for (; !gsi_end_p (gsi); gsi_next (&gsi))
6290 : {
6291 55222209 : basic_block new_bb;
6292 :
6293 55222209 : stmt = gsi_stmt (gsi);
6294 55222209 : if (!is_gimple_debug (stmt))
6295 46825570 : nondebug_stmt_seen = true;
6296 :
6297 : /* If this statement is a non-debug one, and we generate debug
6298 : insns, then this one might be the last real use of a TERed
6299 : SSA_NAME, but where there are still some debug uses further
6300 : down. Expanding the current SSA name in such further debug
6301 : uses by their RHS might lead to wrong debug info, as coalescing
6302 : might make the operands of such RHS be placed into the same
6303 : pseudo as something else. Like so:
6304 : a_1 = a_0 + 1; // Assume a_1 is TERed and a_0 is dead
6305 : use(a_1);
6306 : a_2 = ...
6307 : #DEBUG ... => a_1
6308 : As a_0 and a_2 don't overlap in lifetime, assume they are coalesced.
6309 : If we now would expand a_1 by it's RHS (a_0 + 1) in the debug use,
6310 : the write to a_2 would actually have clobbered the place which
6311 : formerly held a_0.
6312 :
6313 : So, instead of that, we recognize the situation, and generate
6314 : debug temporaries at the last real use of TERed SSA names:
6315 : a_1 = a_0 + 1;
6316 : #DEBUG #D1 => a_1
6317 : use(a_1);
6318 : a_2 = ...
6319 : #DEBUG ... => #D1
6320 : */
6321 55222209 : if (MAY_HAVE_DEBUG_BIND_INSNS
6322 29840996 : && SA.values
6323 83844093 : && !is_gimple_debug (stmt))
6324 : {
6325 20611675 : ssa_op_iter iter;
6326 20611675 : tree op;
6327 20611675 : gimple *def;
6328 :
6329 20611675 : location_t sloc = curr_insn_location ();
6330 :
6331 : /* Look for SSA names that have their last use here (TERed
6332 : names always have only one real use). */
6333 39514652 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
6334 18902977 : if ((def = get_gimple_for_ssa_name (op))
6335 18902977 : && is_gimple_assign (def))
6336 : {
6337 5129431 : imm_use_iterator imm_iter;
6338 5129431 : use_operand_p use_p;
6339 5129431 : bool have_debug_uses = false;
6340 :
6341 10231068 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, op)
6342 : {
6343 5152718 : if (gimple_debug_bind_p (USE_STMT (use_p)))
6344 : {
6345 : have_debug_uses = true;
6346 : break;
6347 : }
6348 5129431 : }
6349 :
6350 5129431 : if (have_debug_uses)
6351 : {
6352 : /* OP is a TERed SSA name, with DEF its defining
6353 : statement, and where OP is used in further debug
6354 : instructions. Generate a debug temporary, and
6355 : replace all uses of OP in debug insns with that
6356 : temporary. */
6357 51081 : gimple *debugstmt;
6358 51081 : tree value = gimple_assign_rhs_to_tree (def);
6359 51081 : tree vexpr = build_debug_expr_decl (TREE_TYPE (value));
6360 51081 : rtx val;
6361 51081 : machine_mode mode;
6362 :
6363 51081 : set_curr_insn_location (gimple_location (def));
6364 :
6365 51081 : if (DECL_P (value))
6366 364 : mode = DECL_MODE (value);
6367 : else
6368 50717 : mode = TYPE_MODE (TREE_TYPE (value));
6369 : /* FIXME: Is setting the mode really necessary? */
6370 51081 : SET_DECL_MODE (vexpr, mode);
6371 :
6372 51081 : val = gen_rtx_VAR_LOCATION
6373 51081 : (mode, vexpr, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
6374 :
6375 51081 : emit_debug_insn (val);
6376 :
6377 191714 : FOR_EACH_IMM_USE_STMT (debugstmt, imm_iter, op)
6378 : {
6379 140633 : if (!gimple_debug_bind_p (debugstmt))
6380 51081 : continue;
6381 :
6382 180906 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
6383 90453 : SET_USE (use_p, vexpr);
6384 :
6385 89552 : update_stmt (debugstmt);
6386 51081 : }
6387 : }
6388 : }
6389 20611675 : set_curr_insn_location (sloc);
6390 : }
6391 :
6392 55222209 : currently_expanding_gimple_stmt = stmt;
6393 :
6394 : /* Expand this statement, then evaluate the resulting RTL and
6395 : fixup the CFG accordingly. */
6396 55222209 : if (gimple_code (stmt) == GIMPLE_COND)
6397 : {
6398 5780708 : new_bb = expand_gimple_cond (bb, as_a <gcond *> (stmt));
6399 5780708 : if (new_bb)
6400 : {
6401 402370 : currently_expanding_gimple_stmt = NULL;
6402 402370 : return new_bb;
6403 : }
6404 : }
6405 49441501 : else if (is_gimple_debug (stmt))
6406 : {
6407 8396639 : location_t sloc = curr_insn_location ();
6408 8396639 : gimple_stmt_iterator nsi = gsi;
6409 :
6410 52990328 : for (;;)
6411 : {
6412 52990328 : tree var;
6413 52990328 : tree value = NULL_TREE;
6414 52990328 : rtx val = NULL_RTX;
6415 52990328 : machine_mode mode;
6416 :
6417 52990328 : if (!gimple_debug_nonbind_marker_p (stmt))
6418 : {
6419 41193720 : if (gimple_debug_bind_p (stmt))
6420 : {
6421 41017698 : var = gimple_debug_bind_get_var (stmt);
6422 :
6423 41017698 : if (TREE_CODE (var) != DEBUG_EXPR_DECL
6424 37971998 : && TREE_CODE (var) != LABEL_DECL
6425 78979659 : && !target_for_debug_bind (var))
6426 385106 : goto delink_debug_stmt;
6427 :
6428 40632592 : if (DECL_P (var) && !VECTOR_TYPE_P (TREE_TYPE (var)))
6429 40545457 : mode = DECL_MODE (var);
6430 : else
6431 87135 : mode = TYPE_MODE (TREE_TYPE (var));
6432 :
6433 40632592 : if (gimple_debug_bind_has_value_p (stmt))
6434 22675086 : value = gimple_debug_bind_get_value (stmt);
6435 :
6436 40632592 : val = gen_rtx_VAR_LOCATION
6437 40632592 : (mode, var, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
6438 : }
6439 176022 : else if (gimple_debug_source_bind_p (stmt))
6440 : {
6441 176022 : var = gimple_debug_source_bind_get_var (stmt);
6442 :
6443 176022 : value = gimple_debug_source_bind_get_value (stmt);
6444 :
6445 176022 : if (!VECTOR_TYPE_P (TREE_TYPE (var)))
6446 176022 : mode = DECL_MODE (var);
6447 : else
6448 0 : mode = TYPE_MODE (TREE_TYPE (var));
6449 :
6450 176022 : val = gen_rtx_VAR_LOCATION (mode, var, (rtx)value,
6451 : VAR_INIT_STATUS_UNINITIALIZED);
6452 : }
6453 : else
6454 0 : gcc_unreachable ();
6455 : }
6456 : /* If this function was first compiled with markers
6457 : enabled, but they're now disable (e.g. LTO), drop
6458 : them on the floor. */
6459 11796608 : else if (gimple_debug_nonbind_marker_p (stmt)
6460 11796608 : && !MAY_HAVE_DEBUG_MARKER_INSNS)
6461 0 : goto delink_debug_stmt;
6462 11796608 : else if (gimple_debug_begin_stmt_p (stmt))
6463 4103506 : val = GEN_RTX_DEBUG_MARKER_BEGIN_STMT_PAT ();
6464 7693102 : else if (gimple_debug_inline_entry_p (stmt))
6465 7693102 : val = GEN_RTX_DEBUG_MARKER_INLINE_ENTRY_PAT ();
6466 : else
6467 : gcc_unreachable ();
6468 :
6469 52605222 : last = get_last_insn ();
6470 :
6471 52605222 : set_curr_insn_location (gimple_location (stmt));
6472 :
6473 52605222 : emit_debug_insn (val);
6474 :
6475 52605222 : if (dump_file && (dump_flags & TDF_DETAILS))
6476 : {
6477 : /* We can't dump the insn with a TREE where an RTX
6478 : is expected. */
6479 1 : if (GET_CODE (val) == VAR_LOCATION)
6480 : {
6481 0 : gcc_checking_assert (PAT_VAR_LOCATION_LOC (val) == (rtx)value);
6482 0 : PAT_VAR_LOCATION_LOC (val) = const0_rtx;
6483 : }
6484 1 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
6485 1 : if (GET_CODE (val) == VAR_LOCATION)
6486 0 : PAT_VAR_LOCATION_LOC (val) = (rtx)value;
6487 : }
6488 :
6489 52990328 : delink_debug_stmt:
6490 : /* In order not to generate too many debug temporaries,
6491 : we delink all uses of debug statements we already expanded.
6492 : Therefore debug statements between definition and real
6493 : use of TERed SSA names will continue to use the SSA name,
6494 : and not be replaced with debug temps. */
6495 52990328 : delink_stmt_imm_use (stmt);
6496 :
6497 52990328 : gsi = nsi;
6498 52990328 : gsi_next (&nsi);
6499 52990328 : if (gsi_end_p (nsi))
6500 : break;
6501 52310795 : stmt = gsi_stmt (nsi);
6502 52310795 : if (!is_gimple_debug (stmt))
6503 : break;
6504 : }
6505 :
6506 8396639 : set_curr_insn_location (sloc);
6507 : }
6508 : else
6509 : {
6510 41044862 : gcall *call_stmt = dyn_cast <gcall *> (stmt);
6511 41044862 : if (call_stmt
6512 41044862 : && asan_epilog_seq
6513 25284 : && gimple_call_tail_p (call_stmt)
6514 41045012 : && !gimple_call_must_tail_p (call_stmt))
6515 33 : gimple_call_set_tail (call_stmt, false);
6516 :
6517 41044862 : if (call_stmt && gimple_call_tail_p (call_stmt))
6518 : {
6519 178659 : bool can_fallthru;
6520 178659 : new_bb = expand_gimple_tailcall (bb, call_stmt, &can_fallthru,
6521 : asan_epilog_seq);
6522 178659 : if (new_bb)
6523 : {
6524 128434 : if (can_fallthru)
6525 0 : bb = new_bb;
6526 : else
6527 : {
6528 128434 : currently_expanding_gimple_stmt = NULL;
6529 128434 : return new_bb;
6530 : }
6531 : }
6532 : }
6533 : else
6534 : {
6535 40866203 : def_operand_p def_p;
6536 40866203 : def_p = SINGLE_SSA_DEF_OPERAND (stmt, SSA_OP_DEF);
6537 :
6538 40866203 : if (def_p != NULL)
6539 : {
6540 : /* Ignore this stmt if it is in the list of
6541 : replaceable expressions. */
6542 34477799 : if (SA.values
6543 42832426 : && bitmap_bit_p (SA.values,
6544 17459774 : SSA_NAME_VERSION (DEF_FROM_PTR (def_p))))
6545 9105147 : continue;
6546 : }
6547 31761056 : last = expand_gimple_stmt (stmt);
6548 31761054 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
6549 : }
6550 : }
6551 : }
6552 :
6553 12398367 : currently_expanding_gimple_stmt = NULL;
6554 :
6555 : /* Expand implicit goto and convert goto_locus. */
6556 30112926 : FOR_EACH_EDGE (e, ei, bb->succs)
6557 : {
6558 17714559 : if (e->goto_locus != UNKNOWN_LOCATION || !nondebug_stmt_seen)
6559 3488995 : set_curr_insn_location (e->goto_locus);
6560 17714559 : if ((e->flags & EDGE_FALLTHRU) && e->dest != bb->next_bb)
6561 : {
6562 1663113 : emit_jump (label_rtx_for_bb (e->dest));
6563 1663113 : e->flags &= ~EDGE_FALLTHRU;
6564 : }
6565 : }
6566 :
6567 : /* Expanded RTL can create a jump in the last instruction of block.
6568 : This later might be assumed to be a jump to successor and break edge insertion.
6569 : We need to insert dummy move to prevent this. PR41440. */
6570 12398367 : if (single_succ_p (bb)
6571 5590981 : && (single_succ_edge (bb)->flags & EDGE_FALLTHRU)
6572 3162038 : && (last = get_last_insn ())
6573 15370448 : && (JUMP_P (last)
6574 3161967 : || (DEBUG_INSN_P (last)
6575 378430 : && JUMP_P (prev_nondebug_insn (last)))))
6576 : {
6577 84 : rtx dummy = gen_reg_rtx (SImode);
6578 84 : emit_insn_after_noloc (gen_move_insn (dummy, dummy), last, NULL);
6579 : }
6580 :
6581 : /* A __builtin_unreachable () will insert a barrier that should end
6582 : the basic block. In gimple, any code after it will have already
6583 : deleted, even without optimization. If we emit additional code
6584 : here, as we would to adjust the stack after a call, it should be
6585 : eventually deleted, but it confuses internal checkers (PR118006)
6586 : and optimizers before it does, because we don't expect to find
6587 : barriers inside basic blocks. */
6588 12398367 : if (!BARRIER_P (get_last_insn ()))
6589 9041933 : do_pending_stack_adjust ();
6590 : else
6591 3356434 : discard_pending_stack_adjust ();
6592 :
6593 : /* Find the block tail. The last insn in the block is the insn
6594 : before a barrier and/or table jump insn. */
6595 12398367 : last = get_last_insn ();
6596 12398367 : if (BARRIER_P (last))
6597 3356434 : last = PREV_INSN (last);
6598 12398367 : if (JUMP_TABLE_DATA_P (last))
6599 7659 : last = PREV_INSN (PREV_INSN (last));
6600 12398367 : if (BARRIER_P (last))
6601 6449 : last = PREV_INSN (last);
6602 12398367 : head_end_for_bb[bb->index].second = last;
6603 :
6604 24796734 : update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
6605 12398367 : head_end_for_bb[bb->index].second, bb);
6606 :
6607 12398367 : return bb;
6608 : }
6609 :
6610 :
6611 : /* Create a basic block for initialization code. */
6612 :
6613 : static basic_block
6614 1517908 : construct_init_block (void)
6615 : {
6616 1517908 : basic_block init_block;
6617 1517908 : edge e = NULL;
6618 1517908 : int flags;
6619 :
6620 : /* Multiple entry points not supported yet. */
6621 1517908 : gcc_assert (EDGE_COUNT (ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs) == 1);
6622 1517908 : init_rtl_bb_info (ENTRY_BLOCK_PTR_FOR_FN (cfun));
6623 1517908 : init_rtl_bb_info (EXIT_BLOCK_PTR_FOR_FN (cfun));
6624 1517908 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6625 1517908 : EXIT_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6626 :
6627 1517908 : e = EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun), 0);
6628 :
6629 : /* When entry edge points to first basic block, we don't need jump,
6630 : otherwise we have to jump into proper target. */
6631 1517908 : if (e && e->dest != ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb)
6632 : {
6633 0 : tree label = gimple_block_label (e->dest);
6634 :
6635 0 : emit_jump (jump_target_rtx (label));
6636 0 : flags = 0;
6637 0 : }
6638 : else
6639 : flags = EDGE_FALLTHRU;
6640 :
6641 1517908 : init_block = create_basic_block (NEXT_INSN (get_insns ()),
6642 : get_last_insn (),
6643 1517908 : ENTRY_BLOCK_PTR_FOR_FN (cfun));
6644 1517908 : init_block->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6645 1517908 : add_bb_to_loop (init_block, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6646 1517908 : if (e)
6647 1517908 : expand_split_edge (e, init_block, flags);
6648 : else
6649 0 : make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6650 : EDGE_FALLTHRU);
6651 :
6652 1517908 : update_bb_for_insn (init_block);
6653 1517908 : return init_block;
6654 : }
6655 :
6656 : /* For each lexical block, set BLOCK_NUMBER to the depth at which it is
6657 : found in the block tree. */
6658 :
6659 : static void
6660 18757806 : set_block_levels (tree block, int level)
6661 : {
6662 35997706 : while (block)
6663 : {
6664 17239900 : BLOCK_NUMBER (block) = level;
6665 17239900 : set_block_levels (BLOCK_SUBBLOCKS (block), level + 1);
6666 17239900 : block = BLOCK_CHAIN (block);
6667 : }
6668 18757806 : }
6669 :
6670 : /* Create a block containing landing pads and similar stuff. */
6671 :
6672 : static void
6673 1517906 : construct_exit_block (void)
6674 : {
6675 1517906 : rtx_insn *head = get_last_insn ();
6676 1517906 : rtx_insn *end;
6677 1517906 : basic_block exit_block;
6678 1517906 : edge e, e2;
6679 1517906 : unsigned ix;
6680 1517906 : edge_iterator ei;
6681 1517906 : basic_block prev_bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
6682 1517906 : rtx_insn *orig_end = BB_END (prev_bb);
6683 :
6684 1517906 : rtl_profile_for_bb (EXIT_BLOCK_PTR_FOR_FN (cfun));
6685 :
6686 : /* Make sure the locus is set to the end of the function, so that
6687 : epilogue line numbers and warnings are set properly. */
6688 1517906 : if (LOCATION_LOCUS (cfun->function_end_locus) != UNKNOWN_LOCATION)
6689 1196186 : input_location = cfun->function_end_locus;
6690 :
6691 : /* Generate rtl for function exit. */
6692 1517906 : expand_function_end ();
6693 :
6694 1517906 : end = get_last_insn ();
6695 1517906 : if (head == end)
6696 0 : return;
6697 : /* While emitting the function end we could move end of the last basic
6698 : block. */
6699 1517906 : BB_END (prev_bb) = orig_end;
6700 1517906 : while (NEXT_INSN (head) && NOTE_P (NEXT_INSN (head)))
6701 : head = NEXT_INSN (head);
6702 : /* But make sure exit_block starts with RETURN_LABEL, otherwise the
6703 : bb count counting will be confused. Any instructions before that
6704 : label are emitted for the case where PREV_BB falls through into the
6705 : exit block, so append those instructions to prev_bb in that case. */
6706 1517906 : if (NEXT_INSN (head) != return_label)
6707 : {
6708 14993 : while (NEXT_INSN (head) != return_label)
6709 : {
6710 9947 : if (!NOTE_P (NEXT_INSN (head)))
6711 9947 : BB_END (prev_bb) = NEXT_INSN (head);
6712 9947 : head = NEXT_INSN (head);
6713 : }
6714 : }
6715 1517906 : exit_block = create_basic_block (NEXT_INSN (head), end, prev_bb);
6716 1517906 : exit_block->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
6717 1517906 : add_bb_to_loop (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6718 :
6719 1517906 : ix = 0;
6720 4581923 : while (ix < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
6721 : {
6722 1546111 : e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), ix);
6723 1546111 : if (!(e->flags & EDGE_ABNORMAL))
6724 1417677 : redirect_edge_succ (e, exit_block);
6725 : else
6726 128434 : ix++;
6727 : }
6728 :
6729 1517906 : e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6730 : EDGE_FALLTHRU);
6731 3164246 : FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
6732 1646340 : if (e2 != e)
6733 : {
6734 128434 : exit_block->count -= e2->count ();
6735 : }
6736 1517906 : update_bb_for_insn (exit_block);
6737 : }
6738 :
6739 : /* Helper function for discover_nonconstant_array_refs.
6740 : Look for ARRAY_REF nodes with non-constant indexes and mark them
6741 : addressable. */
6742 :
6743 : static tree
6744 168368762 : discover_nonconstant_array_refs_r (tree * tp, int *walk_subtrees,
6745 : void *data)
6746 : {
6747 168368762 : tree t = *tp;
6748 168368762 : bitmap forced_stack_vars = (bitmap)((walk_stmt_info *)data)->info;
6749 :
6750 168368762 : if (IS_TYPE_OR_DECL_P (t))
6751 34549104 : *walk_subtrees = 0;
6752 133819658 : else if (REFERENCE_CLASS_P (t) && TREE_THIS_VOLATILE (t))
6753 : {
6754 138560 : t = get_base_address (t);
6755 138560 : if (t && DECL_P (t)
6756 125441 : && DECL_MODE (t) != BLKmode
6757 176194 : && !TREE_ADDRESSABLE (t))
6758 32735 : bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6759 138560 : *walk_subtrees = 0;
6760 : }
6761 133681098 : else if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6762 : {
6763 6507672 : while (((TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6764 3107353 : && is_gimple_min_invariant (TREE_OPERAND (t, 1))
6765 2626823 : && (!TREE_OPERAND (t, 2)
6766 37161 : || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6767 3914871 : || (TREE_CODE (t) == COMPONENT_REF
6768 968364 : && (!TREE_OPERAND (t,2)
6769 0 : || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6770 2946507 : || TREE_CODE (t) == BIT_FIELD_REF
6771 : || TREE_CODE (t) == REALPART_EXPR
6772 : || TREE_CODE (t) == IMAGPART_EXPR
6773 : || TREE_CODE (t) == VIEW_CONVERT_EXPR
6774 6507672 : || CONVERT_EXPR_P (t))
6775 3571362 : t = TREE_OPERAND (t, 0);
6776 :
6777 2936310 : if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6778 : {
6779 514552 : t = get_base_address (t);
6780 514552 : if (t && DECL_P (t)
6781 412547 : && DECL_MODE (t) != BLKmode
6782 537101 : && !TREE_ADDRESSABLE (t))
6783 13048 : bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6784 : }
6785 :
6786 2936310 : *walk_subtrees = 0;
6787 : }
6788 : /* References of size POLY_INT_CST to a fixed-size object must go
6789 : through memory. It's more efficient to force that here than
6790 : to create temporary slots on the fly.
6791 : RTL expansion expects TARGET_MEM_REF to always address actual memory.
6792 : Also, force to stack non-BLKmode vars accessed through VIEW_CONVERT_EXPR
6793 : to BLKmode type. */
6794 130744788 : else if (TREE_CODE (t) == TARGET_MEM_REF
6795 : || (TREE_CODE (t) == MEM_REF
6796 129886432 : && TYPE_SIZE (TREE_TYPE (t))
6797 : && POLY_INT_CST_P (TYPE_SIZE (TREE_TYPE (t))))
6798 260631220 : || (TREE_CODE (t) == VIEW_CONVERT_EXPR
6799 332530 : && TYPE_MODE (TREE_TYPE (t)) == BLKmode))
6800 : {
6801 936980 : tree base = get_base_address (t);
6802 936980 : if (base
6803 936980 : && DECL_P (base)
6804 260572 : && !TREE_ADDRESSABLE (base)
6805 119675 : && DECL_MODE (base) != BLKmode
6806 942993 : && GET_MODE_SIZE (DECL_MODE (base)).is_constant ())
6807 6013 : bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6808 936980 : *walk_subtrees = 0;
6809 : }
6810 :
6811 168368762 : return NULL_TREE;
6812 : }
6813 :
6814 : /* If there's a chance to get a pseudo for t then if it would be of float mode
6815 : and the actual access is via an integer mode (lowered memcpy or similar
6816 : access) then avoid the register expansion if the mode likely is not storage
6817 : suitable for raw bits processing (like XFmode on i?86). */
6818 :
6819 : static void
6820 6877315 : avoid_type_punning_on_regs (tree t, bitmap forced_stack_vars)
6821 : {
6822 6877315 : machine_mode access_mode = TYPE_MODE (TREE_TYPE (t));
6823 6877315 : if (access_mode != BLKmode
6824 6699910 : && !SCALAR_INT_MODE_P (access_mode))
6825 : return;
6826 5884767 : tree base = get_base_address (t);
6827 5884767 : if (DECL_P (base)
6828 4351946 : && !TREE_ADDRESSABLE (base)
6829 1301907 : && FLOAT_MODE_P (DECL_MODE (base))
6830 80 : && maybe_lt (GET_MODE_PRECISION (DECL_MODE (base)),
6831 240 : GET_MODE_BITSIZE (GET_MODE_INNER (DECL_MODE (base))))
6832 : /* Double check in the expensive way we really would get a pseudo. */
6833 5884772 : && use_register_for_decl (base))
6834 5 : bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6835 : }
6836 :
6837 : /* RTL expansion is not able to compile array references with variable
6838 : offsets for arrays stored in single register. Discover such
6839 : expressions and mark variables as addressable to avoid this
6840 : scenario. */
6841 :
6842 : static void
6843 1517908 : discover_nonconstant_array_refs (bitmap forced_stack_vars)
6844 : {
6845 1517908 : basic_block bb;
6846 1517908 : gimple_stmt_iterator gsi;
6847 :
6848 1517908 : walk_stmt_info wi = {};
6849 1517908 : wi.info = forced_stack_vars;
6850 14443998 : FOR_EACH_BB_FN (bb, cfun)
6851 127086899 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
6852 : {
6853 101234719 : gimple *stmt = gsi_stmt (gsi);
6854 101234719 : if (!is_gimple_debug (stmt))
6855 : {
6856 48117166 : walk_gimple_op (stmt, discover_nonconstant_array_refs_r, &wi);
6857 48117166 : gcall *call = dyn_cast <gcall *> (stmt);
6858 7106994 : if (call && gimple_call_internal_p (call))
6859 : {
6860 232950 : tree cand = NULL_TREE;
6861 232950 : switch (gimple_call_internal_fn (call))
6862 : {
6863 0 : case IFN_LOAD_LANES:
6864 : /* The source must be a MEM. */
6865 0 : cand = gimple_call_arg (call, 0);
6866 0 : break;
6867 0 : case IFN_STORE_LANES:
6868 : /* The destination must be a MEM. */
6869 0 : cand = gimple_call_lhs (call);
6870 0 : break;
6871 : default:
6872 : break;
6873 : }
6874 0 : if (cand)
6875 0 : cand = get_base_address (cand);
6876 0 : if (cand
6877 0 : && DECL_P (cand)
6878 0 : && use_register_for_decl (cand))
6879 0 : bitmap_set_bit (forced_stack_vars, DECL_UID (cand));
6880 : }
6881 158925246 : if (gimple_vdef (stmt))
6882 : {
6883 16025475 : tree t = gimple_get_lhs (stmt);
6884 16025475 : if (t && REFERENCE_CLASS_P (t))
6885 6877315 : avoid_type_punning_on_regs (t, forced_stack_vars);
6886 : }
6887 : }
6888 : }
6889 1517908 : }
6890 :
6891 : /* This function sets crtl->args.internal_arg_pointer to a virtual
6892 : register if DRAP is needed. Local register allocator will replace
6893 : virtual_incoming_args_rtx with the virtual register. */
6894 :
6895 : static void
6896 1517906 : expand_stack_alignment (void)
6897 : {
6898 1517906 : rtx drap_rtx;
6899 1517906 : unsigned int preferred_stack_boundary;
6900 :
6901 1517906 : if (! SUPPORTS_STACK_ALIGNMENT)
6902 : return;
6903 :
6904 1517906 : if (cfun->calls_alloca
6905 1502392 : || cfun->has_nonlocal_label
6906 1501576 : || crtl->has_nonlocal_goto)
6907 16745 : crtl->need_drap = true;
6908 :
6909 : /* Call update_stack_boundary here again to update incoming stack
6910 : boundary. It may set incoming stack alignment to a different
6911 : value after RTL expansion. TARGET_FUNCTION_OK_FOR_SIBCALL may
6912 : use the minimum incoming stack alignment to check if it is OK
6913 : to perform sibcall optimization since sibcall optimization will
6914 : only align the outgoing stack to incoming stack boundary. */
6915 1517906 : if (targetm.calls.update_stack_boundary)
6916 1517906 : targetm.calls.update_stack_boundary ();
6917 :
6918 : /* The incoming stack frame has to be aligned at least at
6919 : parm_stack_boundary. */
6920 1517906 : gcc_assert (crtl->parm_stack_boundary <= INCOMING_STACK_BOUNDARY);
6921 :
6922 : /* Update crtl->stack_alignment_estimated and use it later to align
6923 : stack. We check PREFERRED_STACK_BOUNDARY if there may be non-call
6924 : exceptions since callgraph doesn't collect incoming stack alignment
6925 : in this case. */
6926 1517906 : if (cfun->can_throw_non_call_exceptions
6927 263025 : && PREFERRED_STACK_BOUNDARY > crtl->preferred_stack_boundary)
6928 : preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
6929 : else
6930 1489928 : preferred_stack_boundary = crtl->preferred_stack_boundary;
6931 1517906 : if (preferred_stack_boundary > crtl->stack_alignment_estimated)
6932 729861 : crtl->stack_alignment_estimated = preferred_stack_boundary;
6933 1517906 : if (preferred_stack_boundary > crtl->stack_alignment_needed)
6934 680468 : crtl->stack_alignment_needed = preferred_stack_boundary;
6935 :
6936 1517906 : gcc_assert (crtl->stack_alignment_needed
6937 : <= crtl->stack_alignment_estimated);
6938 :
6939 1517906 : crtl->stack_realign_needed
6940 1517906 : = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
6941 1517906 : crtl->stack_realign_tried = crtl->stack_realign_needed;
6942 :
6943 1517906 : crtl->stack_realign_processed = true;
6944 :
6945 : /* Target has to redefine TARGET_GET_DRAP_RTX to support stack
6946 : alignment. */
6947 1517906 : gcc_assert (targetm.calls.get_drap_rtx != NULL);
6948 1517906 : drap_rtx = targetm.calls.get_drap_rtx ();
6949 :
6950 : /* stack_realign_drap and drap_rtx must match. */
6951 1517906 : gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
6952 :
6953 : /* Do nothing if NULL is returned, which means DRAP is not needed. */
6954 1517906 : if (drap_rtx != NULL)
6955 : {
6956 7406 : crtl->args.internal_arg_pointer = drap_rtx;
6957 :
6958 : /* Call fixup_tail_calls to clean up REG_EQUIV note if DRAP is
6959 : needed. */
6960 7406 : fixup_tail_calls ();
6961 : }
6962 : }
6963 :
6964 :
6965 : static void
6966 0 : expand_main_function (void)
6967 : {
6968 : #if (defined(INVOKE__main) \
6969 : || (!defined(HAS_INIT_SECTION) \
6970 : && !defined(INIT_SECTION_ASM_OP) \
6971 : && !defined(INIT_ARRAY_SECTION_ASM_OP)))
6972 : emit_library_call (init_one_libfunc (NAME__MAIN), LCT_NORMAL, VOIDmode);
6973 : #endif
6974 0 : }
6975 :
6976 :
6977 : /* Expand code to initialize the stack_protect_guard. This is invoked at
6978 : the beginning of a function to be protected. */
6979 :
6980 : static void
6981 295 : stack_protect_prologue (void)
6982 : {
6983 295 : tree guard_decl = targetm.stack_protect_guard ();
6984 295 : rtx x, y;
6985 :
6986 295 : crtl->stack_protect_guard_decl = guard_decl;
6987 295 : x = expand_normal (crtl->stack_protect_guard);
6988 :
6989 295 : if (targetm.have_stack_protect_combined_set () && guard_decl)
6990 : {
6991 0 : gcc_assert (DECL_P (guard_decl));
6992 0 : y = DECL_RTL (guard_decl);
6993 :
6994 : /* Allow the target to compute address of Y and copy it to X without
6995 : leaking Y into a register. This combined address + copy pattern
6996 : allows the target to prevent spilling of any intermediate results by
6997 : splitting it after register allocator. */
6998 0 : if (rtx_insn *insn = targetm.gen_stack_protect_combined_set (x, y))
6999 : {
7000 0 : emit_insn (insn);
7001 0 : return;
7002 : }
7003 : }
7004 :
7005 295 : if (guard_decl)
7006 295 : y = expand_normal (guard_decl);
7007 : else
7008 0 : y = const0_rtx;
7009 :
7010 : /* Allow the target to copy from Y to X without leaking Y into a
7011 : register. */
7012 295 : if (targetm.have_stack_protect_set ())
7013 295 : if (rtx_insn *insn = targetm.gen_stack_protect_set (x, y))
7014 : {
7015 295 : emit_insn (insn);
7016 295 : return;
7017 : }
7018 :
7019 : /* Otherwise do a straight move. */
7020 0 : emit_move_insn (x, y);
7021 : }
7022 :
7023 : /* Translate the intermediate representation contained in the CFG
7024 : from GIMPLE trees to RTL.
7025 :
7026 : We do conversion per basic block and preserve/update the tree CFG.
7027 : This implies we have to do some magic as the CFG can simultaneously
7028 : consist of basic blocks containing RTL and GIMPLE trees. This can
7029 : confuse the CFG hooks, so be careful to not manipulate CFG during
7030 : the expansion. */
7031 :
7032 : namespace {
7033 :
7034 : const pass_data pass_data_expand =
7035 : {
7036 : RTL_PASS, /* type */
7037 : "expand", /* name */
7038 : OPTGROUP_NONE, /* optinfo_flags */
7039 : TV_EXPAND, /* tv_id */
7040 : ( PROP_ssa | PROP_gimple_leh | PROP_cfg
7041 : | PROP_gimple_lcx
7042 : | PROP_gimple_lvec
7043 : | PROP_gimple_lva), /* properties_required */
7044 : PROP_rtl, /* properties_provided */
7045 : ( PROP_ssa | PROP_gimple ), /* properties_destroyed */
7046 : 0, /* todo_flags_start */
7047 : 0, /* todo_flags_finish */
7048 : };
7049 :
7050 : class pass_expand : public rtl_opt_pass
7051 : {
7052 : public:
7053 297184 : pass_expand (gcc::context *ctxt)
7054 594368 : : rtl_opt_pass (pass_data_expand, ctxt)
7055 : {}
7056 :
7057 : /* opt_pass methods: */
7058 : unsigned int execute (function *) final override;
7059 :
7060 : }; // class pass_expand
7061 :
7062 : unsigned int
7063 1517908 : pass_expand::execute (function *fun)
7064 : {
7065 1517908 : basic_block bb, init_block;
7066 1517908 : edge_iterator ei;
7067 1517908 : edge e;
7068 1517908 : rtx_insn *var_seq, *var_ret_seq;
7069 1517908 : unsigned i;
7070 :
7071 1517908 : timevar_push (TV_OUT_OF_SSA);
7072 1517908 : rewrite_out_of_ssa (&SA);
7073 1517908 : timevar_pop (TV_OUT_OF_SSA);
7074 1517908 : SA.partition_to_pseudo = XCNEWVEC (rtx, SA.map->num_partitions);
7075 :
7076 1517908 : if (MAY_HAVE_DEBUG_BIND_STMTS && flag_tree_ter)
7077 : {
7078 503707 : gimple_stmt_iterator gsi;
7079 6913843 : FOR_EACH_BB_FN (bb, cfun)
7080 87836256 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7081 116142129 : if (gimple_debug_bind_p (gsi_stmt (gsi)))
7082 41126145 : avoid_deep_ter_for_debug (gsi_stmt (gsi), 0);
7083 : }
7084 :
7085 : /* Mark arrays indexed with non-constant indices with TREE_ADDRESSABLE. */
7086 1517908 : auto_bitmap forced_stack_vars;
7087 1517908 : discover_nonconstant_array_refs (forced_stack_vars);
7088 :
7089 : /* Make sure all values used by the optimization passes have sane
7090 : defaults. */
7091 1517908 : reg_renumber = 0;
7092 :
7093 : /* Some backends want to know that we are expanding to RTL. */
7094 1517908 : currently_expanding_to_rtl = 1;
7095 : /* Dominators are not kept up-to-date as we may create new basic-blocks. */
7096 1517908 : free_dominance_info (CDI_DOMINATORS);
7097 :
7098 1517908 : rtl_profile_for_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
7099 :
7100 1517908 : insn_locations_init ();
7101 1517908 : if (!DECL_IS_UNDECLARED_BUILTIN (current_function_decl))
7102 : {
7103 : /* Eventually, all FEs should explicitly set function_start_locus. */
7104 1395529 : if (LOCATION_LOCUS (fun->function_start_locus) == UNKNOWN_LOCATION)
7105 656700 : set_curr_insn_location
7106 656700 : (DECL_SOURCE_LOCATION (current_function_decl));
7107 : else
7108 738829 : set_curr_insn_location (fun->function_start_locus);
7109 : }
7110 : else
7111 122379 : set_curr_insn_location (UNKNOWN_LOCATION);
7112 1517908 : prologue_location = curr_insn_location ();
7113 :
7114 : #ifdef INSN_SCHEDULING
7115 1517908 : init_sched_attrs ();
7116 : #endif
7117 :
7118 : /* Make sure first insn is a note even if we don't want linenums.
7119 : This makes sure the first insn will never be deleted.
7120 : Also, final expects a note to appear there. */
7121 1517908 : emit_note (NOTE_INSN_DELETED);
7122 :
7123 1517908 : targetm.expand_to_rtl_hook ();
7124 1517908 : crtl->init_stack_alignment ();
7125 1517908 : fun->cfg->max_jumptable_ents = 0;
7126 :
7127 : /* Resolve the function section. Some targets, like ARM EABI rely on knowledge
7128 : of the function section at expansion time to predict distance of calls. */
7129 1517908 : resolve_unique_section (current_function_decl, 0, flag_function_sections);
7130 :
7131 : /* Expand the variables recorded during gimple lowering. */
7132 1517908 : timevar_push (TV_VAR_EXPAND);
7133 1517908 : start_sequence ();
7134 :
7135 1517908 : var_ret_seq = expand_used_vars (forced_stack_vars);
7136 :
7137 1517908 : var_seq = end_sequence ();
7138 1517908 : timevar_pop (TV_VAR_EXPAND);
7139 :
7140 : /* Honor stack protection warnings. */
7141 1517908 : if (warn_stack_protect)
7142 : {
7143 0 : if (fun->calls_alloca)
7144 0 : warning (OPT_Wstack_protector,
7145 : "stack protector not protecting local variables: "
7146 : "variable length buffer");
7147 0 : if (has_short_buffer && !crtl->stack_protect_guard)
7148 0 : warning (OPT_Wstack_protector,
7149 : "stack protector not protecting function: "
7150 : "all local arrays are less than %d bytes long",
7151 0 : (int) param_ssp_buffer_size);
7152 : }
7153 :
7154 : /* Temporarily mark PARM_DECLs and RESULT_DECLs we need to expand to
7155 : memory addressable so expand_function_start can emit the required
7156 : copies. */
7157 1517908 : auto_vec<tree, 16> marked_parms;
7158 4685110 : for (tree parm = DECL_ARGUMENTS (current_function_decl); parm;
7159 3167202 : parm = DECL_CHAIN (parm))
7160 3167202 : if (!TREE_ADDRESSABLE (parm)
7161 3167202 : && bitmap_bit_p (forced_stack_vars, DECL_UID (parm)))
7162 : {
7163 189 : TREE_ADDRESSABLE (parm) = 1;
7164 189 : marked_parms.safe_push (parm);
7165 : }
7166 1517908 : if (DECL_RESULT (current_function_decl)
7167 1517908 : && !TREE_ADDRESSABLE (DECL_RESULT (current_function_decl))
7168 3028755 : && bitmap_bit_p (forced_stack_vars,
7169 1510847 : DECL_UID (DECL_RESULT (current_function_decl))))
7170 : {
7171 0 : TREE_ADDRESSABLE (DECL_RESULT (current_function_decl)) = 1;
7172 0 : marked_parms.safe_push (DECL_RESULT (current_function_decl));
7173 : }
7174 :
7175 : /* Set up parameters and prepare for return, for the function. */
7176 1517908 : expand_function_start (current_function_decl);
7177 :
7178 : /* Clear TREE_ADDRESSABLE again. */
7179 3036005 : while (!marked_parms.is_empty ())
7180 189 : TREE_ADDRESSABLE (marked_parms.pop ()) = 0;
7181 :
7182 : /* If we emitted any instructions for setting up the variables,
7183 : emit them before the FUNCTION_START note. */
7184 1517908 : if (var_seq)
7185 : {
7186 1798 : emit_insn_before (var_seq, parm_birth_insn);
7187 :
7188 : /* In expand_function_end we'll insert the alloca save/restore
7189 : before parm_birth_insn. We've just insertted an alloca call.
7190 : Adjust the pointer to match. */
7191 1798 : parm_birth_insn = var_seq;
7192 : }
7193 :
7194 : /* Now propagate the RTL assignment of each partition to the
7195 : underlying var of each SSA_NAME. */
7196 1517908 : tree name;
7197 :
7198 75416766 : FOR_EACH_SSA_NAME (i, name, cfun)
7199 : {
7200 : /* We might have generated new SSA names in
7201 : update_alias_info_with_stack_vars. They will have a NULL
7202 : defining statements, and won't be part of the partitioning,
7203 : so ignore those. */
7204 50275795 : if (!SSA_NAME_DEF_STMT (name))
7205 72380 : continue;
7206 :
7207 50203415 : adjust_one_expanded_partition_var (name);
7208 : }
7209 :
7210 : /* Clean up RTL of variables that straddle across multiple
7211 : partitions, and check that the rtl of any PARM_DECLs that are not
7212 : cleaned up is that of their default defs. */
7213 75416766 : FOR_EACH_SSA_NAME (i, name, cfun)
7214 : {
7215 50275795 : int part;
7216 :
7217 : /* We might have generated new SSA names in
7218 : update_alias_info_with_stack_vars. They will have a NULL
7219 : defining statements, and won't be part of the partitioning,
7220 : so ignore those. */
7221 50275795 : if (!SSA_NAME_DEF_STMT (name))
7222 72380 : continue;
7223 50203415 : part = var_to_partition (SA.map, name);
7224 50203415 : if (part == NO_PARTITION)
7225 17956056 : continue;
7226 :
7227 : /* If this decl was marked as living in multiple places, reset
7228 : this now to NULL. */
7229 32247359 : tree var = SSA_NAME_VAR (name);
7230 10713814 : if (var && DECL_RTL_IF_SET (var) == pc_rtx)
7231 248538 : SET_DECL_RTL (var, NULL);
7232 : /* Check that the pseudos chosen by assign_parms are those of
7233 : the corresponding default defs. */
7234 31998821 : else if (SSA_NAME_IS_DEFAULT_DEF (name)
7235 31998821 : && (TREE_CODE (var) == PARM_DECL
7236 3749895 : || TREE_CODE (var) == RESULT_DECL))
7237 : {
7238 3687179 : rtx in = DECL_RTL_IF_SET (var);
7239 3687179 : gcc_assert (in);
7240 3687179 : rtx out = SA.partition_to_pseudo[part];
7241 3687179 : gcc_assert (in == out);
7242 :
7243 : /* Now reset VAR's RTL to IN, so that the _EXPR attrs match
7244 : those expected by debug backends for each parm and for
7245 : the result. This is particularly important for stabs,
7246 : whose register elimination from parm's DECL_RTL may cause
7247 : -fcompare-debug differences as SET_DECL_RTL changes reg's
7248 : attrs. So, make sure the RTL already has the parm as the
7249 : EXPR, so that it won't change. */
7250 3687179 : SET_DECL_RTL (var, NULL_RTX);
7251 3687179 : if (MEM_P (in))
7252 788171 : set_mem_attributes (in, var, true);
7253 3687179 : SET_DECL_RTL (var, in);
7254 : }
7255 : }
7256 :
7257 1517908 : if (flag_checking)
7258 1517888 : verify_partition_mem_exprs ();
7259 :
7260 : /* If this function is `main', emit a call to `__main'
7261 : to run global initializers, etc. */
7262 1517908 : if (DECL_NAME (current_function_decl)
7263 1517908 : && MAIN_NAME_P (DECL_NAME (current_function_decl))
7264 1633636 : && DECL_FILE_SCOPE_P (current_function_decl))
7265 1517908 : expand_main_function ();
7266 :
7267 : /* Initialize the stack_protect_guard field. This must happen after the
7268 : call to __main (if any) so that the external decl is initialized. */
7269 1517908 : if (crtl->stack_protect_guard && targetm.stack_protect_runtime_enabled_p ())
7270 295 : stack_protect_prologue ();
7271 :
7272 1517908 : expand_phi_nodes (&SA);
7273 :
7274 : /* Release any stale SSA redirection data. */
7275 1517908 : redirect_edge_var_map_empty ();
7276 :
7277 : /* Register rtl specific functions for cfg. */
7278 1517908 : rtl_register_cfg_hooks ();
7279 :
7280 1517908 : init_block = construct_init_block ();
7281 :
7282 : /* Clear EDGE_EXECUTABLE on the entry edge(s). It is cleaned from the
7283 : remaining edges later. */
7284 3035816 : FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs)
7285 1517908 : e->flags &= ~EDGE_EXECUTABLE;
7286 :
7287 : /* If the function has too many markers, drop them while expanding. */
7288 1517908 : if (cfun->debug_marker_count
7289 1517908 : >= param_max_debug_marker_count)
7290 1 : cfun->debug_nonbind_markers = false;
7291 :
7292 1517908 : if (optimize)
7293 1065298 : enable_ranger (fun);
7294 1517908 : lab_rtx_for_bb = new hash_map<basic_block, rtx_code_label *>;
7295 1517908 : head_end_for_bb.create (last_basic_block_for_fn (fun));
7296 14447079 : FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
7297 : next_bb)
7298 12929173 : bb = expand_gimple_basic_block (bb, var_ret_seq);
7299 1517906 : if (optimize)
7300 1065298 : disable_ranger (fun);
7301 14849439 : FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
7302 : next_bb)
7303 : {
7304 13331533 : if ((bb->flags & BB_RTL) == 0)
7305 : {
7306 12929163 : bb->il.gimple.seq = NULL;
7307 12929163 : bb->il.gimple.phi_nodes = NULL;
7308 12929163 : init_rtl_bb_info (bb);
7309 12929163 : bb->flags |= BB_RTL;
7310 12929163 : BB_HEAD (bb) = head_end_for_bb[bb->index].first;
7311 12929163 : BB_END (bb) = head_end_for_bb[bb->index].second;
7312 : }
7313 : /* These flags have no purpose in RTL land. */
7314 13331533 : if (EDGE_COUNT (bb->succs) == 2)
7315 : {
7316 6420612 : EDGE_SUCC (bb, 0)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
7317 6420612 : EDGE_SUCC (bb, 1)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
7318 : }
7319 19453308 : else if (single_succ_p (bb))
7320 6121775 : single_succ_edge (bb)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
7321 : }
7322 1517906 : head_end_for_bb.release ();
7323 :
7324 1517906 : if (MAY_HAVE_DEBUG_BIND_INSNS)
7325 503732 : expand_debug_locations ();
7326 :
7327 1517906 : if (deep_ter_debug_map)
7328 : {
7329 729 : delete deep_ter_debug_map;
7330 729 : deep_ter_debug_map = NULL;
7331 : }
7332 :
7333 : /* Free stuff we no longer need after GIMPLE optimizations. */
7334 1517906 : free_dominance_info (CDI_DOMINATORS);
7335 1517906 : free_dominance_info (CDI_POST_DOMINATORS);
7336 1517906 : delete_tree_cfg_annotations (fun);
7337 :
7338 1517906 : timevar_push (TV_OUT_OF_SSA);
7339 1517906 : finish_out_of_ssa (&SA);
7340 1517906 : timevar_pop (TV_OUT_OF_SSA);
7341 :
7342 1517906 : timevar_push (TV_POST_EXPAND);
7343 : /* We are no longer in SSA form. */
7344 1517906 : fun->gimple_df->in_ssa_p = false;
7345 1517906 : loops_state_clear (LOOP_CLOSED_SSA);
7346 :
7347 : /* Expansion is used by optimization passes too, set maybe_hot_insn_p
7348 : conservatively to true until they are all profile aware. */
7349 3035812 : delete lab_rtx_for_bb;
7350 1517906 : free_histograms (fun);
7351 :
7352 1517906 : construct_exit_block ();
7353 1517906 : insn_locations_finalize ();
7354 :
7355 1517906 : if (var_ret_seq)
7356 : {
7357 1938 : rtx_insn *after = return_label;
7358 1938 : rtx_insn *next = NEXT_INSN (after);
7359 1938 : if (next && NOTE_INSN_BASIC_BLOCK_P (next))
7360 : after = next;
7361 1938 : emit_insn_after (var_ret_seq, after);
7362 : }
7363 :
7364 1517906 : if (hwassist_sanitize_stack_p ())
7365 368 : hwasan_maybe_emit_frame_base_init ();
7366 :
7367 : /* Zap the tree EH table. */
7368 1517906 : set_eh_throw_stmt_table (fun, NULL);
7369 :
7370 : /* We need JUMP_LABEL be set in order to redirect jumps, and hence
7371 : split edges which edge insertions might do. */
7372 1517906 : rebuild_jump_labels (get_insns ());
7373 :
7374 : /* If we have a single successor to the entry block, put the pending insns
7375 : after parm birth, but before NOTE_INSNS_FUNCTION_BEG. */
7376 1517906 : if (single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (fun)))
7377 : {
7378 1517906 : edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
7379 1517906 : if (e->insns.r)
7380 : {
7381 11 : rtx_insn *insns = e->insns.r;
7382 11 : e->insns.r = NULL;
7383 11 : rebuild_jump_labels_chain (insns);
7384 11 : if (NOTE_P (parm_birth_insn)
7385 11 : && NOTE_KIND (parm_birth_insn) == NOTE_INSN_FUNCTION_BEG)
7386 11 : emit_insn_before_noloc (insns, parm_birth_insn, e->dest);
7387 : else
7388 0 : emit_insn_after_noloc (insns, parm_birth_insn, e->dest);
7389 : }
7390 : }
7391 :
7392 : /* Otherwise, as well as for other edges, take the usual way. */
7393 1517906 : commit_edge_insertions ();
7394 :
7395 : /* We're done expanding trees to RTL. */
7396 1517906 : currently_expanding_to_rtl = 0;
7397 :
7398 1517906 : flush_mark_addressable_queue ();
7399 :
7400 18498289 : FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun)->next_bb,
7401 : EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
7402 : {
7403 16980383 : edge e;
7404 16980383 : edge_iterator ei;
7405 39679330 : for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
7406 : {
7407 : /* Clear EDGE_EXECUTABLE. This flag is never used in the backend. */
7408 22698947 : e->flags &= ~EDGE_EXECUTABLE;
7409 :
7410 : /* At the moment not all abnormal edges match the RTL
7411 : representation. It is safe to remove them here as
7412 : find_many_sub_basic_blocks will rediscover them.
7413 : In the future we should get this fixed properly. */
7414 22698947 : if ((e->flags & EDGE_ABNORMAL)
7415 139066 : && !(e->flags & EDGE_SIBCALL))
7416 10632 : remove_edge (e);
7417 : else
7418 22688315 : ei_next (&ei);
7419 : }
7420 : }
7421 :
7422 1517906 : auto_sbitmap blocks (last_basic_block_for_fn (fun));
7423 1517906 : bitmap_ones (blocks);
7424 1517906 : find_many_sub_basic_blocks (blocks);
7425 1517906 : purge_all_dead_edges ();
7426 :
7427 : /* After initial rtl generation, call back to finish generating
7428 : exception support code. We need to do this before cleaning up
7429 : the CFG as the code does not expect dead landing pads. */
7430 1517906 : if (fun->eh->region_tree != NULL)
7431 66023 : finish_eh_generation ();
7432 :
7433 : /* Call expand_stack_alignment after finishing all
7434 : updates to crtl->preferred_stack_boundary. */
7435 1517906 : expand_stack_alignment ();
7436 :
7437 : /* Fixup REG_EQUIV notes in the prologue if there are tailcalls in this
7438 : function. */
7439 1517906 : if (crtl->tail_call_emit)
7440 115810 : fixup_tail_calls ();
7441 :
7442 1517906 : HOST_WIDE_INT patch_area_size, patch_area_entry;
7443 1517906 : parse_and_check_patch_area (flag_patchable_function_entry, false,
7444 : &patch_area_size, &patch_area_entry);
7445 :
7446 1517906 : tree patchable_function_entry_attr
7447 1517906 : = lookup_attribute ("patchable_function_entry",
7448 1517906 : DECL_ATTRIBUTES (cfun->decl));
7449 1517906 : if (patchable_function_entry_attr)
7450 : {
7451 16 : tree pp_val = TREE_VALUE (patchable_function_entry_attr);
7452 16 : tree patchable_function_entry_value1 = TREE_VALUE (pp_val);
7453 :
7454 16 : patch_area_size = tree_to_uhwi (patchable_function_entry_value1);
7455 16 : patch_area_entry = 0;
7456 16 : if (TREE_CHAIN (pp_val) != NULL_TREE)
7457 : {
7458 8 : tree patchable_function_entry_value2
7459 8 : = TREE_VALUE (TREE_CHAIN (pp_val));
7460 8 : patch_area_entry = tree_to_uhwi (patchable_function_entry_value2);
7461 : }
7462 : }
7463 :
7464 1517906 : if (patch_area_entry > patch_area_size)
7465 : {
7466 0 : if (patch_area_size > 0)
7467 0 : warning (OPT_Wattributes,
7468 : "patchable function entry %wu exceeds size %wu",
7469 : patch_area_entry, patch_area_size);
7470 0 : patch_area_entry = 0;
7471 : }
7472 :
7473 1517906 : crtl->patch_area_size = patch_area_size;
7474 1517906 : crtl->patch_area_entry = patch_area_entry;
7475 :
7476 : /* BB subdivision may have created basic blocks that are only reachable
7477 : from unlikely bbs but not marked as such in the profile. */
7478 1517906 : if (optimize)
7479 1065298 : propagate_unlikely_bbs_forward ();
7480 :
7481 : /* Remove unreachable blocks, otherwise we cannot compute dominators
7482 : which are needed for loop state verification. As a side-effect
7483 : this also compacts blocks.
7484 : ??? We cannot remove trivially dead insns here as for example
7485 : the DRAP reg on i?86 is not magically live at this point.
7486 : gcc.c-torture/execute/ipa-sra-2.c execution, -Os -m32 fails otherwise. */
7487 1517906 : cleanup_cfg (CLEANUP_NO_INSN_DEL);
7488 :
7489 1517906 : checking_verify_flow_info ();
7490 :
7491 : /* Initialize pseudos allocated for hard registers. */
7492 1517906 : emit_initial_value_sets ();
7493 :
7494 : /* And finally unshare all RTL. */
7495 1517906 : unshare_all_rtl ();
7496 :
7497 : /* There's no need to defer outputting this function any more; we
7498 : know we want to output it. */
7499 1517906 : DECL_DEFER_OUTPUT (current_function_decl) = 0;
7500 :
7501 : /* Now that we're done expanding trees to RTL, we shouldn't have any
7502 : more CONCATs anywhere. */
7503 1517906 : generating_concat_p = 0;
7504 :
7505 1517906 : if (dump_file)
7506 : {
7507 404 : fprintf (dump_file,
7508 : "\n\n;;\n;; Full RTL generated for this function:\n;;\n");
7509 : /* And the pass manager will dump RTL for us. */
7510 : }
7511 :
7512 : /* If we're emitting a nested function, make sure its parent gets
7513 : emitted as well. Doing otherwise confuses debug info. */
7514 1517906 : {
7515 1517906 : tree parent;
7516 1517906 : for (parent = DECL_CONTEXT (current_function_decl);
7517 3263571 : parent != NULL_TREE;
7518 1745665 : parent = get_containing_scope (parent))
7519 1745665 : if (TREE_CODE (parent) == FUNCTION_DECL)
7520 78086 : TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (parent)) = 1;
7521 : }
7522 :
7523 1517906 : TREE_ASM_WRITTEN (current_function_decl) = 1;
7524 :
7525 : /* After expanding, the return labels are no longer needed. */
7526 1517906 : return_label = NULL;
7527 1517906 : naked_return_label = NULL;
7528 :
7529 : /* After expanding, the tm_restart map is no longer needed. */
7530 1517906 : if (fun->gimple_df->tm_restart)
7531 265 : fun->gimple_df->tm_restart = NULL;
7532 :
7533 : /* Tag the blocks with a depth number so that change_scope can find
7534 : the common parent easily. */
7535 1517906 : set_block_levels (DECL_INITIAL (fun->decl), 0);
7536 1517906 : default_rtl_profile ();
7537 :
7538 : /* For -dx discard loops now, otherwise IL verify in clean_state will
7539 : ICE. */
7540 1517906 : if (rtl_dump_and_exit)
7541 : {
7542 79 : cfun->curr_properties &= ~PROP_loops;
7543 79 : loop_optimizer_finalize ();
7544 : }
7545 :
7546 1517906 : timevar_pop (TV_POST_EXPAND);
7547 :
7548 1517906 : return 0;
7549 1517906 : }
7550 :
7551 : } // anon namespace
7552 :
7553 : rtl_opt_pass *
7554 297184 : make_pass_expand (gcc::context *ctxt)
7555 : {
7556 297184 : return new pass_expand (ctxt);
7557 : }
|