Line data Source code
1 : /* A pass for lowering trees to RTL.
2 : Copyright (C) 2004-2026 Free Software Foundation, Inc.
3 :
4 : This file is part of GCC.
5 :
6 : GCC is free software; you can redistribute it and/or modify
7 : it under the terms of the GNU General Public License as published by
8 : the Free Software Foundation; either version 3, or (at your option)
9 : any later version.
10 :
11 : GCC is distributed in the hope that it will be useful,
12 : but WITHOUT ANY WARRANTY; without even the implied warranty of
13 : MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 : GNU General Public License for more details.
15 :
16 : You should have received a copy of the GNU General Public License
17 : along with GCC; see the file COPYING3. If not see
18 : <http://www.gnu.org/licenses/>. */
19 :
20 : #include "config.h"
21 : #include "system.h"
22 : #include "coretypes.h"
23 : #include "backend.h"
24 : #include "target.h"
25 : #include "rtl.h"
26 : #include "tree.h"
27 : #include "gimple.h"
28 : #include "cfghooks.h"
29 : #include "tree-pass.h"
30 : #include "memmodel.h"
31 : #include "tm_p.h"
32 : #include "ssa.h"
33 : #include "optabs.h"
34 : #include "regs.h" /* For reg_renumber. */
35 : #include "emit-rtl.h"
36 : #include "recog.h"
37 : #include "cgraph.h"
38 : #include "diagnostic.h"
39 : #include "fold-const.h"
40 : #include "varasm.h"
41 : #include "stor-layout.h"
42 : #include "stmt.h"
43 : #include "print-tree.h"
44 : #include "cfgrtl.h"
45 : #include "cfganal.h"
46 : #include "cfgbuild.h"
47 : #include "cfgcleanup.h"
48 : #include "dojump.h"
49 : #include "explow.h"
50 : #include "calls.h"
51 : #include "expr.h"
52 : #include "internal-fn.h"
53 : #include "tree-eh.h"
54 : #include "gimple-iterator.h"
55 : #include "gimple-expr.h"
56 : #include "gimple-walk.h"
57 : #include "tree-cfg.h"
58 : #include "tree-dfa.h"
59 : #include "tree-ssa.h"
60 : #include "except.h"
61 : #include "gimple-pretty-print.h"
62 : #include "toplev.h"
63 : #include "debug.h"
64 : #include "tree-inline.h"
65 : #include "value-prof.h"
66 : #include "tree-ssa-live.h"
67 : #include "tree-outof-ssa.h"
68 : #include "cfgloop.h"
69 : #include "insn-attr.h" /* For INSN_SCHEDULING. */
70 : #include "stringpool.h"
71 : #include "attribs.h"
72 : #include "asan.h"
73 : #include "tree-ssa-address.h"
74 : #include "output.h"
75 : #include "builtins.h"
76 : #include "opts.h"
77 : #include "gimple-range.h"
78 : #include "rtl-iter.h"
79 :
80 : /* Some systems use __main in a way incompatible with its use in gcc, in these
81 : cases use the macros NAME__MAIN to give a quoted symbol and SYMBOL__MAIN to
82 : give the same symbol without quotes for an alternative entry point. You
83 : must define both, or neither. */
84 : #ifndef NAME__MAIN
85 : #define NAME__MAIN "__main"
86 : #endif
87 :
88 : /* This variable holds information helping the rewriting of SSA trees
89 : into RTL. */
90 : struct ssaexpand SA;
91 :
92 : /* This variable holds the currently expanded gimple statement for purposes
93 : of communicating the profile info to the builtin expanders. */
94 : gimple *currently_expanding_gimple_stmt;
95 :
96 : static rtx expand_debug_expr (tree);
97 :
98 : static bool defer_stack_allocation (tree, bool);
99 :
100 : static void record_alignment_for_reg_var (unsigned int);
101 :
102 : /* Return an expression tree corresponding to the RHS of GIMPLE
103 : statement STMT. */
104 :
105 : tree
106 3190210 : gimple_assign_rhs_to_tree (gimple *stmt)
107 : {
108 3190210 : tree t;
109 3190210 : switch (gimple_assign_rhs_class (stmt))
110 : {
111 1714 : case GIMPLE_TERNARY_RHS:
112 1714 : t = build3 (gimple_assign_rhs_code (stmt),
113 1714 : TREE_TYPE (gimple_assign_lhs (stmt)),
114 : gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt),
115 : gimple_assign_rhs3 (stmt));
116 1714 : break;
117 1170375 : case GIMPLE_BINARY_RHS:
118 1170375 : t = build2 (gimple_assign_rhs_code (stmt),
119 1170375 : TREE_TYPE (gimple_assign_lhs (stmt)),
120 : gimple_assign_rhs1 (stmt), gimple_assign_rhs2 (stmt));
121 1170375 : break;
122 339432 : case GIMPLE_UNARY_RHS:
123 339432 : t = build1 (gimple_assign_rhs_code (stmt),
124 339432 : TREE_TYPE (gimple_assign_lhs (stmt)),
125 : gimple_assign_rhs1 (stmt));
126 339432 : break;
127 1678689 : case GIMPLE_SINGLE_RHS:
128 1678689 : {
129 1678689 : t = gimple_assign_rhs1 (stmt);
130 : /* Avoid modifying this tree in place below. */
131 3280123 : if ((gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t)
132 1572547 : && gimple_location (stmt) != EXPR_LOCATION (t))
133 1878087 : || (gimple_block (stmt) && currently_expanding_to_rtl
134 36690 : && EXPR_P (t)))
135 1432034 : t = copy_node (t);
136 : break;
137 : }
138 0 : default:
139 0 : gcc_unreachable ();
140 : }
141 :
142 3190210 : if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (t))
143 3021074 : SET_EXPR_LOCATION (t, gimple_location (stmt));
144 :
145 3190210 : return t;
146 : }
147 :
148 :
149 : #ifndef STACK_ALIGNMENT_NEEDED
150 : #define STACK_ALIGNMENT_NEEDED 1
151 : #endif
152 :
153 : #define SSAVAR(x) (TREE_CODE (x) == SSA_NAME ? SSA_NAME_VAR (x) : x)
154 :
155 : /* Choose either CUR or NEXT as the leader DECL for a partition.
156 : Prefer ignored decls, to simplify debug dumps and reduce ambiguity
157 : out of the same user variable being in multiple partitions (this is
158 : less likely for compiler-introduced temps). */
159 :
160 : static tree
161 59326682 : leader_merge (tree cur, tree next)
162 : {
163 59326682 : if (cur == NULL || cur == next)
164 : return next;
165 :
166 3133029 : if (DECL_P (cur) && DECL_IGNORED_P (cur))
167 : return cur;
168 :
169 2326982 : if (DECL_P (next) && DECL_IGNORED_P (next))
170 155275 : return next;
171 :
172 : return cur;
173 : }
174 :
175 : /* Associate declaration T with storage space X. If T is no
176 : SSA name this is exactly SET_DECL_RTL, otherwise make the
177 : partition of T associated with X. */
178 : static inline void
179 70097899 : set_rtl (tree t, rtx x)
180 : {
181 70097899 : gcc_checking_assert (!x
182 : || !(TREE_CODE (t) == SSA_NAME || is_gimple_reg (t))
183 : || (use_register_for_decl (t)
184 : ? (REG_P (x)
185 : || (GET_CODE (x) == CONCAT
186 : && (REG_P (XEXP (x, 0))
187 : || SUBREG_P (XEXP (x, 0)))
188 : && (REG_P (XEXP (x, 1))
189 : || SUBREG_P (XEXP (x, 1))))
190 : /* We need to accept PARALLELs for RESUT_DECLs
191 : because of vector types with BLKmode returned
192 : in multiple registers, but they are supposed
193 : to be uncoalesced. */
194 : || (GET_CODE (x) == PARALLEL
195 : && SSAVAR (t)
196 : && TREE_CODE (SSAVAR (t)) == RESULT_DECL
197 : && (GET_MODE (x) == BLKmode
198 : || !flag_tree_coalesce_vars)))
199 : : (MEM_P (x) || x == pc_rtx
200 : || (GET_CODE (x) == CONCAT
201 : && MEM_P (XEXP (x, 0))
202 : && MEM_P (XEXP (x, 1))))));
203 : /* Check that the RTL for SSA_NAMEs and gimple-reg PARM_DECLs and
204 : RESULT_DECLs has the expected mode. For memory, we accept
205 : unpromoted modes, since that's what we're likely to get. For
206 : PARM_DECLs and RESULT_DECLs, we'll have been called by
207 : set_parm_rtl, which will give us the default def, so we don't
208 : have to compute it ourselves. For RESULT_DECLs, we accept mode
209 : mismatches too, as long as we have BLKmode or are not coalescing
210 : across variables, so that we don't reject BLKmode PARALLELs or
211 : unpromoted REGs. */
212 64949061 : gcc_checking_assert (!x || x == pc_rtx || TREE_CODE (t) != SSA_NAME
213 : || (SSAVAR (t)
214 : && TREE_CODE (SSAVAR (t)) == RESULT_DECL
215 : && (promote_ssa_mode (t, NULL) == BLKmode
216 : || !flag_tree_coalesce_vars))
217 : || !use_register_for_decl (t)
218 : || GET_MODE (x) == promote_ssa_mode (t, NULL));
219 :
220 : if (x)
221 : {
222 65345894 : bool skip = false;
223 65345894 : tree cur = NULL_TREE;
224 : rtx xm = x;
225 :
226 65345894 : retry:
227 65345894 : if (MEM_P (xm))
228 4760190 : cur = MEM_EXPR (xm);
229 : else if (REG_P (xm))
230 54566492 : cur = REG_EXPR (xm);
231 : else if (SUBREG_P (xm))
232 : {
233 0 : gcc_assert (subreg_lowpart_p (xm));
234 0 : xm = SUBREG_REG (xm);
235 0 : goto retry;
236 : }
237 : else if (GET_CODE (xm) == CONCAT)
238 : {
239 393509 : xm = XEXP (xm, 0);
240 393509 : goto retry;
241 : }
242 : else if (GET_CODE (xm) == PARALLEL)
243 : {
244 3324 : xm = XVECEXP (xm, 0, 0);
245 3324 : gcc_assert (GET_CODE (xm) == EXPR_LIST);
246 3324 : xm = XEXP (xm, 0);
247 3324 : goto retry;
248 : }
249 5622379 : else if (xm == pc_rtx)
250 : skip = true;
251 : else
252 0 : gcc_unreachable ();
253 :
254 59326682 : tree next = skip ? cur : leader_merge (cur, SSAVAR (t) ? SSAVAR (t) : t);
255 :
256 64949061 : if (cur != next)
257 : {
258 27434294 : if (MEM_P (x))
259 3967468 : set_mem_attributes (x,
260 1983734 : next && TREE_CODE (next) == SSA_NAME
261 12432 : ? TREE_TYPE (next)
262 : : next, true);
263 : else
264 25450560 : set_reg_attrs_for_decl_rtl (next, x);
265 : }
266 : }
267 :
268 70097899 : if (TREE_CODE (t) == SSA_NAME)
269 : {
270 56867144 : int part = var_to_partition (SA.map, t);
271 56867144 : if (part != NO_PARTITION)
272 : {
273 56867144 : if (SA.partition_to_pseudo[part])
274 31069984 : gcc_assert (SA.partition_to_pseudo[part] == x);
275 25797160 : else if (x != pc_rtx)
276 25794186 : SA.partition_to_pseudo[part] = x;
277 : }
278 : /* For the benefit of debug information at -O0 (where
279 : vartracking doesn't run) record the place also in the base
280 : DECL. For PARMs and RESULTs, do so only when setting the
281 : default def. */
282 113731314 : if (x && x != pc_rtx && SSA_NAME_VAR (t)
283 74693659 : && (VAR_P (SSA_NAME_VAR (t))
284 7433567 : || SSA_NAME_IS_DEFAULT_DEF (t)))
285 : {
286 17650258 : tree var = SSA_NAME_VAR (t);
287 : /* If we don't yet have something recorded, just record it now. */
288 17650258 : if (!DECL_RTL_SET_P (var))
289 7474407 : SET_DECL_RTL (var, x);
290 : /* If we have it set already to "multiple places" don't
291 : change this. */
292 10175851 : else if (DECL_RTL (var) == pc_rtx)
293 : ;
294 : /* If we have something recorded and it's not the same place
295 : as we want to record now, we have multiple partitions for the
296 : same base variable, with different places. We can't just
297 : randomly chose one, hence we have to say that we don't know.
298 : This only happens with optimization, and there var-tracking
299 : will figure out the right thing. */
300 9080829 : else if (DECL_RTL (var) != x)
301 235396 : SET_DECL_RTL (var, pc_rtx);
302 : }
303 : }
304 : else
305 13230755 : SET_DECL_RTL (t, x);
306 70097899 : }
307 :
308 : /* This structure holds data relevant to one variable that will be
309 : placed in a stack slot. */
310 : class stack_var
311 : {
312 : public:
313 : /* The Variable. */
314 : tree decl;
315 :
316 : /* Initially, the size of the variable. Later, the size of the partition,
317 : if this variable becomes it's partition's representative. */
318 : poly_uint64 size;
319 :
320 : /* The *byte* alignment required for this variable. Or as, with the
321 : size, the alignment for this partition. */
322 : unsigned int alignb;
323 :
324 : /* The partition representative. */
325 : unsigned representative;
326 :
327 : /* The next stack variable in the partition, or EOC. */
328 : unsigned next;
329 :
330 : /* The numbers of conflicting stack variables. */
331 : bitmap conflicts;
332 : };
333 :
334 : #define EOC ((unsigned)-1)
335 :
336 : /* We have an array of such objects while deciding allocation. */
337 : static class stack_var *stack_vars;
338 : static unsigned stack_vars_alloc;
339 : static unsigned stack_vars_num;
340 : static hash_map<tree, unsigned> *decl_to_stack_part;
341 :
342 : #define INVALID_STACK_INDEX ((unsigned)-1)
343 :
344 : /* Conflict bitmaps go on this obstack. This allows us to destroy
345 : all of them in one big sweep. */
346 : static bitmap_obstack stack_var_bitmap_obstack;
347 :
348 : /* An array of indices such that stack_vars[stack_vars_sorted[i]].size
349 : is non-decreasing. */
350 : static unsigned *stack_vars_sorted;
351 :
352 : /* The phase of the stack frame. This is the known misalignment of
353 : virtual_stack_vars_rtx from PREFERRED_STACK_BOUNDARY. That is,
354 : (frame_offset+frame_phase) % PREFERRED_STACK_BOUNDARY == 0. */
355 : static int frame_phase;
356 :
357 : /* Used during expand_used_vars to remember if we saw any decls for
358 : which we'd like to enable stack smashing protection. */
359 : static bool has_protected_decls;
360 :
361 : /* Used during expand_used_vars. Remember if we say a character buffer
362 : smaller than our cutoff threshold. Used for -Wstack-protector. */
363 : static bool has_short_buffer;
364 :
365 : /* Compute the byte alignment to use for DECL. Ignore alignment
366 : we can't do with expected alignment of the stack boundary. */
367 :
368 : static unsigned int
369 6428434 : align_local_variable (tree decl, bool really_expand)
370 : {
371 6428434 : unsigned int align;
372 :
373 6428434 : if (TREE_CODE (decl) == SSA_NAME)
374 : {
375 452443 : tree type = TREE_TYPE (decl);
376 452443 : machine_mode mode = TYPE_MODE (type);
377 :
378 452443 : align = TYPE_ALIGN (type);
379 452443 : if (mode != BLKmode
380 452443 : && align < GET_MODE_ALIGNMENT (mode))
381 460 : align = GET_MODE_ALIGNMENT (mode);
382 : }
383 : else
384 5975991 : align = LOCAL_DECL_ALIGNMENT (decl);
385 :
386 6428434 : if (hwassist_sanitize_stack_p ())
387 270 : align = MAX (align, (unsigned) HWASAN_TAG_GRANULE_SIZE * BITS_PER_UNIT);
388 :
389 6428434 : if (TREE_CODE (decl) != SSA_NAME && really_expand)
390 : /* Don't change DECL_ALIGN when called from estimated_stack_frame_size.
391 : That is done before IPA and could bump alignment based on host
392 : backend even for offloaded code which wants different
393 : LOCAL_DECL_ALIGNMENT. */
394 1523052 : SET_DECL_ALIGN (decl, align);
395 :
396 6428434 : return align / BITS_PER_UNIT;
397 : }
398 :
399 : /* Align given offset BASE with ALIGN. Truncate up if ALIGN_UP is true,
400 : down otherwise. Return truncated BASE value. */
401 :
402 : static inline unsigned HOST_WIDE_INT
403 : align_base (HOST_WIDE_INT base, unsigned HOST_WIDE_INT align, bool align_up)
404 : {
405 : return align_up ? (base + align - 1) & -align : base & -align;
406 : }
407 :
408 : /* Allocate SIZE bytes at byte alignment ALIGN from the stack frame.
409 : Return the frame offset. */
410 :
411 : static poly_int64
412 1728686 : alloc_stack_frame_space (poly_int64 size, unsigned HOST_WIDE_INT align)
413 : {
414 1728686 : poly_int64 offset, new_frame_offset;
415 :
416 1728686 : if (FRAME_GROWS_DOWNWARD)
417 : {
418 1728686 : new_frame_offset
419 1728686 : = aligned_lower_bound (frame_offset - frame_phase - size,
420 1728686 : align) + frame_phase;
421 1728686 : offset = new_frame_offset;
422 : }
423 : else
424 : {
425 : new_frame_offset
426 : = aligned_upper_bound (frame_offset - frame_phase,
427 : align) + frame_phase;
428 : offset = new_frame_offset;
429 : new_frame_offset += size;
430 : }
431 1728686 : frame_offset = new_frame_offset;
432 :
433 1728686 : if (frame_offset_overflow (frame_offset, cfun->decl))
434 0 : frame_offset = offset = 0;
435 :
436 1728686 : return offset;
437 : }
438 :
439 : /* Ensure that the stack is aligned to ALIGN bytes.
440 : Return the new frame offset. */
441 : static poly_int64
442 1847 : align_frame_offset (unsigned HOST_WIDE_INT align)
443 : {
444 0 : return alloc_stack_frame_space (0, align);
445 : }
446 :
447 : /* Accumulate DECL into STACK_VARS. */
448 :
449 : static void
450 5622379 : add_stack_var (tree decl, bool really_expand)
451 : {
452 5622379 : class stack_var *v;
453 :
454 5622379 : if (stack_vars_num >= stack_vars_alloc)
455 : {
456 1305104 : if (stack_vars_alloc)
457 34069 : stack_vars_alloc = stack_vars_alloc * 3 / 2;
458 : else
459 1271035 : stack_vars_alloc = 32;
460 1305104 : stack_vars
461 1305104 : = XRESIZEVEC (class stack_var, stack_vars, stack_vars_alloc);
462 : }
463 5622379 : if (!decl_to_stack_part)
464 0 : decl_to_stack_part = new hash_map<tree, unsigned>;
465 :
466 5622379 : v = &stack_vars[stack_vars_num];
467 5622379 : decl_to_stack_part->put (decl, stack_vars_num);
468 :
469 5622379 : v->decl = decl;
470 5622379 : tree size = TREE_CODE (decl) == SSA_NAME
471 5622379 : ? TYPE_SIZE_UNIT (TREE_TYPE (decl))
472 5622379 : : DECL_SIZE_UNIT (decl);
473 5622379 : v->size = tree_to_poly_uint64 (size);
474 : /* Ensure that all variables have size, so that &a != &b for any two
475 : variables that are simultaneously live. */
476 5622379 : if (known_eq (v->size, 0U))
477 24156 : v->size = 1;
478 5622379 : v->alignb = align_local_variable (decl, really_expand);
479 : /* An alignment of zero can mightily confuse us later. */
480 5622379 : gcc_assert (v->alignb != 0);
481 :
482 : /* All variables are initially in their own partition. */
483 5622379 : v->representative = stack_vars_num;
484 5622379 : v->next = EOC;
485 :
486 : /* All variables initially conflict with no other. */
487 5622379 : v->conflicts = NULL;
488 :
489 : /* Ensure that this decl doesn't get put onto the list twice. */
490 5622379 : set_rtl (decl, pc_rtx);
491 :
492 5622379 : stack_vars_num++;
493 5622379 : }
494 :
495 : /* Make the decls associated with luid's X and Y conflict. */
496 :
497 : static void
498 11151506 : add_stack_var_conflict (unsigned x, unsigned y)
499 : {
500 11151506 : class stack_var *a = &stack_vars[x];
501 11151506 : class stack_var *b = &stack_vars[y];
502 11151506 : if (x == y)
503 : return;
504 10121599 : if (!a->conflicts)
505 634805 : a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
506 10121599 : if (!b->conflicts)
507 110487 : b->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
508 10121599 : bitmap_set_bit (a->conflicts, y);
509 10121599 : bitmap_set_bit (b->conflicts, x);
510 : }
511 :
512 : /* Check whether the decls associated with luid's X and Y conflict. */
513 :
514 : static bool
515 11265404 : stack_var_conflict_p (unsigned x, unsigned y)
516 : {
517 11265404 : class stack_var *a = &stack_vars[x];
518 11265404 : class stack_var *b = &stack_vars[y];
519 11265404 : if (x == y)
520 : return false;
521 : /* Partitions containing an SSA name result from gimple registers
522 : with things like unsupported modes. They are top-level and
523 : hence conflict with everything else. */
524 11265404 : if (TREE_CODE (a->decl) == SSA_NAME || TREE_CODE (b->decl) == SSA_NAME)
525 : return true;
526 :
527 11230968 : if (!a->conflicts || !b->conflicts)
528 : return false;
529 11177121 : return bitmap_bit_p (a->conflicts, y);
530 : }
531 :
532 : /* Returns the DECL's index into the stack_vars array.
533 : If the DECL does not exist return INVALID_STACK_INDEX. */
534 : static unsigned
535 45129140 : decl_stack_index (tree decl)
536 : {
537 45129140 : if (!decl)
538 : return INVALID_STACK_INDEX;
539 45129140 : if (!DECL_P (decl))
540 : return INVALID_STACK_INDEX;
541 34690855 : if (DECL_RTL_IF_SET (decl) != pc_rtx)
542 : return INVALID_STACK_INDEX;
543 23211731 : unsigned *v = decl_to_stack_part->get (decl);
544 23211731 : if (!v)
545 : return INVALID_STACK_INDEX;
546 :
547 23211731 : unsigned indx = *v;
548 23211731 : gcc_checking_assert (indx != INVALID_STACK_INDEX);
549 23211731 : gcc_checking_assert (indx < stack_vars_num);
550 : return indx;
551 : }
552 :
553 : /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
554 : enter its partition number into bitmap DATA. */
555 :
556 : static bool
557 28777161 : visit_op (gimple *, tree op, tree, void *data)
558 : {
559 28777161 : bitmap active = (bitmap)data;
560 28777161 : op = get_base_address (op);
561 28777161 : unsigned idx = decl_stack_index (op);
562 28777161 : if (idx != INVALID_STACK_INDEX)
563 14232538 : bitmap_set_bit (active, idx);
564 28777161 : return false;
565 : }
566 :
567 : /* Callback for walk_stmt_ops. If OP is a decl touched by add_stack_var
568 : record conflicts between it and all currently active other partitions
569 : from bitmap DATA. */
570 :
571 : static bool
572 13141278 : visit_conflict (gimple *, tree op, tree, void *data)
573 : {
574 13141278 : bitmap active = (bitmap)data;
575 13141278 : op = get_base_address (op);
576 13141278 : unsigned num = decl_stack_index (op);
577 13141278 : if (num != INVALID_STACK_INDEX
578 13141278 : && bitmap_set_bit (active, num))
579 : {
580 957687 : bitmap_iterator bi;
581 957687 : unsigned i;
582 957687 : gcc_assert (num < stack_vars_num);
583 10673911 : EXECUTE_IF_SET_IN_BITMAP (active, 0, i, bi)
584 9716224 : add_stack_var_conflict (num, i);
585 : }
586 13141278 : return false;
587 : }
588 :
589 : /* A cache for ssa name to address of stack variables.
590 : When taking into account if a ssa name refers to an
591 : address of a stack variable, we need to walk the
592 : expressions backwards to find the addresses. This
593 : cache is there so we don't need to walk the expressions
594 : all the time. */
595 : struct vars_ssa_cache
596 : {
597 : private:
598 : /* Currently an entry is a bitmap of all of the known stack variables
599 : addresses that are referenced by the ssa name.
600 : When the bitmap is the nullptr, then there is no cache.
601 : Currently only empty bitmaps are shared.
602 : The reason for why empty cache is not just a null is so we know the
603 : cache for an entry is filled in. */
604 : struct entry
605 : {
606 : bitmap bmap = nullptr;
607 : };
608 : entry *vars_ssa_caches;
609 : public:
610 :
611 : vars_ssa_cache();
612 : ~vars_ssa_cache();
613 : const_bitmap operator() (tree name);
614 : void dump (FILE *file);
615 :
616 : private:
617 : /* Can't copy. */
618 : vars_ssa_cache(const vars_ssa_cache&) = delete;
619 : vars_ssa_cache(vars_ssa_cache&&) = delete;
620 :
621 : /* The shared empty bitmap. */
622 : bitmap empty;
623 :
624 : /* Unshare the index, currently only need
625 : to unshare if the entry was empty. */
626 894209 : void unshare(int indx)
627 : {
628 894209 : if (vars_ssa_caches[indx].bmap == empty)
629 400004 : vars_ssa_caches[indx].bmap = BITMAP_ALLOC (&stack_var_bitmap_obstack);
630 894209 : }
631 : void create (tree);
632 : bool exists (tree use);
633 : void add_one (tree old_name, unsigned);
634 : bool update (tree old_name, tree use);
635 : };
636 :
637 : /* Constructor of the cache, create the cache array. */
638 144415 : vars_ssa_cache::vars_ssa_cache ()
639 : {
640 31440421 : vars_ssa_caches = new entry[num_ssa_names]{};
641 :
642 : /* Create the shared empty bitmap too. */
643 144415 : empty = BITMAP_ALLOC (&stack_var_bitmap_obstack);
644 144415 : }
645 :
646 : /* Delete the array. The bitmaps will be freed
647 : when stack_var_bitmap_obstack is freed. */
648 144415 : vars_ssa_cache::~vars_ssa_cache ()
649 : {
650 144415 : delete []vars_ssa_caches;
651 144415 : }
652 :
653 : /* Create an empty entry for the USE ssa name. */
654 : void
655 10434105 : vars_ssa_cache::create (tree use)
656 : {
657 10434105 : int num = SSA_NAME_VERSION (use);
658 10434105 : if (vars_ssa_caches[num].bmap)
659 : return;
660 10434105 : vars_ssa_caches[num].bmap = empty;
661 : }
662 :
663 : /* Returns true if the cache for USE exists. */
664 : bool
665 68485867 : vars_ssa_cache::exists (tree use)
666 : {
667 68485867 : int num = SSA_NAME_VERSION (use);
668 68485867 : return vars_ssa_caches[num].bmap != nullptr;
669 : }
670 :
671 : /* Add to USE's bitmap for stack variable IDX. */
672 : void
673 138376 : vars_ssa_cache::add_one (tree use, unsigned idx)
674 : {
675 138376 : gcc_assert (idx != INVALID_STACK_INDEX);
676 138376 : int num = SSA_NAME_VERSION (use);
677 138376 : gcc_assert (vars_ssa_caches[num].bmap);
678 138376 : unshare (num);
679 138376 : bitmap_set_bit (vars_ssa_caches[num].bmap, idx);
680 138376 : }
681 :
682 : /* Update cache of OLD_NAME from the USE's cache. */
683 : bool
684 23926098 : vars_ssa_cache::update (tree old_name, tree use)
685 : {
686 23926098 : if (old_name == use)
687 : return false;
688 15244422 : int num = SSA_NAME_VERSION (use);
689 15244422 : int old_num = SSA_NAME_VERSION (old_name);
690 :
691 : /* If the old name was empty, then there is nothing to be updated. */
692 15244422 : if (vars_ssa_caches[num].bmap == empty)
693 : return false;
694 755833 : unshare (old_num);
695 755833 : return bitmap_ior_into (vars_ssa_caches[old_num].bmap, vars_ssa_caches[num].bmap);
696 : }
697 :
698 : /* Dump out the cache. Note empty and non-filled
699 : in ssa names are not printed out. */
700 : void
701 0 : vars_ssa_cache::dump (FILE *file)
702 : {
703 0 : fprintf (file, "var ssa address cache\n");
704 0 : for (unsigned num = 0; num < num_ssa_names; num++)
705 : {
706 0 : if (!vars_ssa_caches[num].bmap
707 0 : || vars_ssa_caches[num].bmap == empty)
708 0 : continue;
709 0 : fprintf (file, "_%d refers to:\n", num);
710 0 : bitmap_iterator bi;
711 0 : unsigned i;
712 0 : EXECUTE_IF_SET_IN_BITMAP (vars_ssa_caches[num].bmap, 0, i, bi)
713 : {
714 0 : fputc ('\t', file);
715 0 : print_generic_expr (file, stack_vars[i].decl, dump_flags);
716 : }
717 0 : fputc ('\n', file);
718 : }
719 0 : fputc ('\n', file);
720 0 : }
721 :
722 : /* Returns the filled in cache for NAME.
723 : This will fill in the cache if it does not exist already.
724 : Returns an empty for ssa names that can't contain pointers
725 : (only integral types and pointer types will contain pointers). */
726 :
727 : const_bitmap
728 53805625 : vars_ssa_cache::operator() (tree name)
729 : {
730 53805625 : gcc_assert (TREE_CODE (name) == SSA_NAME);
731 :
732 88658062 : if (!POINTER_TYPE_P (TREE_TYPE (name))
733 87639452 : && !ANY_INTEGRAL_TYPE_P (TREE_TYPE (name)))
734 2410253 : return empty;
735 :
736 51395372 : if (exists (name))
737 42751238 : return vars_ssa_caches[SSA_NAME_VERSION (name)].bmap;
738 :
739 8644134 : auto_vec<std::pair<tree,tree>, 4> work_list;
740 8644134 : auto_vec<std::pair<tree,tree>, 4> update_cache_list;
741 :
742 8644134 : work_list.safe_push (std::make_pair (name, name));
743 :
744 31786994 : while (!work_list.is_empty ())
745 : {
746 23142860 : auto item = work_list.pop();
747 23142860 : tree use = item.first;
748 23142860 : tree old_name = item.second;
749 23142860 : if (TREE_CODE (use) == ADDR_EXPR)
750 : {
751 378678 : tree op = TREE_OPERAND (use, 0);
752 378678 : op = get_base_address (op);
753 378678 : unsigned idx = decl_stack_index (op);
754 378678 : if (idx != INVALID_STACK_INDEX)
755 138376 : add_one (old_name, idx);
756 12708755 : continue;
757 378678 : }
758 :
759 22764182 : if (TREE_CODE (use) != SSA_NAME)
760 5598139 : continue;
761 :
762 29960397 : if (!POINTER_TYPE_P (TREE_TYPE (use))
763 29851981 : && !ANY_INTEGRAL_TYPE_P (TREE_TYPE (use)))
764 75548 : continue;
765 :
766 : /* Mark the old ssa name needs to be update from the use. */
767 17090495 : update_cache_list.safe_push (item);
768 :
769 : /* If the cache exists for the use, don't try to recreate it. */
770 17090495 : if (exists (use))
771 : {
772 : /* Update the cache here, this can reduce the number of
773 : times through the update loop below. */
774 6656390 : update (old_name, use);
775 6656390 : continue;
776 : }
777 :
778 : /* Create the cache bitmap for the use and also
779 : so we don't go into an infinite loop for some phi nodes with loops. */
780 10434105 : create (use);
781 :
782 10434105 : gimple *g = SSA_NAME_DEF_STMT (use);
783 :
784 : /* CONSTRUCTOR here is always a vector initialization,
785 : walk each element too. */
786 10434105 : if (gimple_assign_single_p (g)
787 10434105 : && TREE_CODE (gimple_assign_rhs1 (g)) == CONSTRUCTOR)
788 : {
789 : tree ctr = gimple_assign_rhs1 (g);
790 : unsigned i;
791 : tree elm;
792 302258 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (ctr), i, elm)
793 211791 : work_list.safe_push (std::make_pair (elm, use));
794 : }
795 : /* For assignments, walk each operand for possible addresses.
796 : For PHI nodes, walk each argument. */
797 10343638 : else if (gassign *a = dyn_cast <gassign *> (g))
798 : {
799 : /* operand 0 is the lhs. */
800 18530196 : for (unsigned i = 1; i < gimple_num_ops (g); i++)
801 10705414 : work_list.safe_push (std::make_pair (gimple_op (a, i), use));
802 : }
803 11903287 : else if (gphi *p = dyn_cast <gphi *> (g))
804 5050703 : for (unsigned i = 0; i < gimple_phi_num_args (p); ++i)
805 3581521 : work_list.safe_push (std::make_pair (gimple_phi_arg_def (p, i), use));
806 : }
807 :
808 : /* Update the cache. Note a loop is needed as phi nodes could
809 : cause a loop to form. The number of times through this loop
810 : will be small though. */
811 8681642 : bool changed;
812 17363284 : do {
813 8681642 : changed = false;
814 8681642 : unsigned int i;
815 8681642 : std::pair<tree,tree> *e;
816 43277126 : FOR_EACH_VEC_ELT_REVERSE (update_cache_list, i, e)
817 : {
818 17269708 : if (update (e->second, e->first))
819 44145 : changed = true;
820 : }
821 : } while (changed);
822 :
823 8644134 : return vars_ssa_caches[SSA_NAME_VERSION (name)].bmap;
824 8644134 : }
825 :
826 : /* Helper function for add_scope_conflicts_1. For USE on
827 : a stmt, if it is a SSA_NAME and in its defining statement
828 : is known to be based on some ADDR_EXPR, invoke VISIT
829 : on that ADDR_EXPR. */
830 :
831 : static inline void
832 53805625 : add_scope_conflicts_2 (vars_ssa_cache &cache, tree name,
833 : bitmap work, walk_stmt_load_store_addr_fn visit)
834 : {
835 53805625 : gcc_assert (TREE_CODE (name) == SSA_NAME);
836 :
837 : /* Query the cache for the mapping of addresses that are referenced by
838 : ssa name NAME. Querying it will fill in it. */
839 53805625 : bitmap_iterator bi;
840 53805625 : unsigned i;
841 53805625 : const_bitmap bmap = cache (name);
842 : /* Visit each stack variable that is referenced. */
843 55903680 : EXECUTE_IF_SET_IN_BITMAP (bmap, 0, i, bi)
844 2098055 : visit (nullptr, stack_vars[i].decl, nullptr, work);
845 53805625 : }
846 :
847 : /* Helper routine for add_scope_conflicts, calculating the active partitions
848 : at the end of BB, leaving the result in WORK. We're called to generate
849 : conflicts when FOR_CONFLICT is true, otherwise we're just tracking
850 : liveness. */
851 :
852 : static void
853 15660147 : add_scope_conflicts_1 (vars_ssa_cache &cache, basic_block bb, bitmap work, bool for_conflict)
854 : {
855 15660147 : edge e;
856 15660147 : edge_iterator ei;
857 15660147 : gimple_stmt_iterator gsi;
858 15660147 : walk_stmt_load_store_addr_fn visit;
859 15660147 : use_operand_p use_p;
860 15660147 : ssa_op_iter iter;
861 15660147 : bool had_non_clobbers = false;
862 :
863 15660147 : bitmap_clear (work);
864 : /* The copy what was alive out going from the edges. */
865 39553793 : FOR_EACH_EDGE (e, ei, bb->preds)
866 23893646 : bitmap_ior_into (work, (bitmap)e->src->aux);
867 :
868 15660147 : visit = for_conflict ? visit_conflict : visit_op;
869 :
870 : /* Addresses coming into the bb via phis are alive at the entry point. */
871 20457496 : for (gsi = gsi_start_phis (bb); !gsi_end_p (gsi); gsi_next (&gsi))
872 4797349 : add_scope_conflicts_2 (cache, gimple_phi_result (gsi_stmt (gsi)), work, visit_op);
873 :
874 171995948 : for (gsi = gsi_after_labels (bb); !gsi_end_p (gsi); gsi_next (&gsi))
875 : {
876 156335801 : gimple *stmt = gsi_stmt (gsi);
877 :
878 : /* Debug statements are not considered for liveness. */
879 156335801 : if (is_gimple_debug (stmt))
880 95402675 : continue;
881 :
882 : /* If we had `var = {CLOBBER}`, then var is no longer
883 : considered alive after this point but might become
884 : alive later on. */
885 60933126 : if (gimple_clobber_p (stmt))
886 : {
887 2901320 : tree lhs = gimple_assign_lhs (stmt);
888 : /* Handle only plain var clobbers, not partial ones.
889 : Nested functions lowering and C++ front-end inserts clobbers
890 : which are partial clobbers. */
891 2901320 : if (!VAR_P (lhs))
892 69297 : continue;
893 2832023 : unsigned indx = decl_stack_index (lhs);
894 2832023 : if (indx != INVALID_STACK_INDEX)
895 2320371 : bitmap_clear_bit (work, indx);
896 : }
897 : else
898 : {
899 58031806 : if (for_conflict && !had_non_clobbers)
900 : {
901 : /* When we are inheriting live variables from our predecessors
902 : through a CFG merge we might not see an actual mention of
903 : the variables to record the appropriate conflict as defs/uses
904 : might be through indirect stores/loads. For this reason
905 : we have to make sure each live variable conflicts with
906 : each other. When there's just a single predecessor the
907 : set of conflicts is already up-to-date.
908 : We perform this delayed at the first real instruction to
909 : allow clobbers starting this block to remove variables from
910 : the set of live variables. */
911 4732189 : bitmap_iterator bi;
912 4732189 : unsigned i;
913 4732189 : if (EDGE_COUNT (bb->preds) > 1)
914 12352043 : EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
915 : {
916 10964817 : class stack_var *a = &stack_vars[i];
917 10964817 : if (!a->conflicts)
918 254551 : a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
919 10964817 : bitmap_ior_into (a->conflicts, work);
920 : }
921 4732189 : had_non_clobbers = true;
922 : }
923 58031806 : walk_stmt_load_store_addr_ops (stmt, work, visit, visit, visit);
924 107040082 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
925 49008276 : add_scope_conflicts_2 (cache, USE_FROM_PTR (use_p), work, visit);
926 : }
927 : }
928 :
929 : /* When there was no real instruction but there's a CFG merge we need
930 : to add the conflicts now. */
931 15660147 : if (for_conflict && !had_non_clobbers && EDGE_COUNT (bb->preds) > 1)
932 : {
933 120480 : bitmap_iterator bi;
934 120480 : unsigned i;
935 899390 : EXECUTE_IF_SET_IN_BITMAP (work, 0, i, bi)
936 : {
937 778910 : class stack_var *a = &stack_vars[i];
938 778910 : if (!a->conflicts)
939 21852 : a->conflicts = BITMAP_ALLOC (&stack_var_bitmap_obstack);
940 778910 : bitmap_ior_into (a->conflicts, work);
941 : }
942 : }
943 15660147 : }
944 :
945 : /* Generate stack partition conflicts between all partitions that are
946 : simultaneously live. */
947 :
948 : static void
949 229873 : add_scope_conflicts (void)
950 : {
951 : /* If there is only one variable, there is nothing to be done as
952 : there is only possible partition. */
953 229873 : if (stack_vars_num == 1)
954 85458 : return;
955 :
956 144415 : basic_block bb;
957 144415 : bool changed;
958 144415 : bitmap work = BITMAP_ALLOC (NULL);
959 144415 : int *rpo;
960 144415 : int n_bbs;
961 :
962 144415 : vars_ssa_cache cache;
963 :
964 : /* We approximate the live range of a stack variable by taking the first
965 : mention of its name as starting point(s), and by the end-of-scope
966 : death clobber added by gimplify as ending point(s) of the range.
967 : This overapproximates in the case we for instance moved an address-taken
968 : operation upward, without also moving a dereference to it upwards.
969 : But it's conservatively correct as a variable never can hold values
970 : before its name is mentioned at least once.
971 :
972 : We then do a mostly classical bitmap liveness algorithm. */
973 :
974 5352266 : FOR_ALL_BB_FN (bb, cfun)
975 5207851 : bb->aux = BITMAP_ALLOC (&stack_var_bitmap_obstack);
976 :
977 144415 : rpo = XNEWVEC (int, last_basic_block_for_fn (cfun));
978 144415 : n_bbs = pre_and_rev_post_order_compute (NULL, rpo, false);
979 :
980 144415 : changed = true;
981 585252 : while (changed)
982 : {
983 : int i;
984 : changed = false;
985 11037548 : for (i = 0; i < n_bbs; i++)
986 : {
987 10741126 : bitmap active;
988 10741126 : bb = BASIC_BLOCK_FOR_FN (cfun, rpo[i]);
989 10741126 : active = (bitmap)bb->aux;
990 10741126 : add_scope_conflicts_1 (cache, bb, work, false);
991 10741126 : if (bitmap_ior_into (active, work))
992 4412578 : changed = true;
993 : }
994 : }
995 :
996 5063436 : FOR_EACH_BB_FN (bb, cfun)
997 4919021 : add_scope_conflicts_1 (cache, bb, work, true);
998 :
999 144415 : if (dump_file && (dump_flags & TDF_DETAILS))
1000 0 : cache.dump (dump_file);
1001 :
1002 144415 : free (rpo);
1003 144415 : BITMAP_FREE (work);
1004 5352266 : FOR_ALL_BB_FN (bb, cfun)
1005 5207851 : BITMAP_FREE (bb->aux);
1006 144415 : }
1007 :
1008 : /* A subroutine of partition_stack_vars. A comparison function for qsort,
1009 : sorting an array of indices by the properties of the object. */
1010 :
1011 : static int
1012 19383062 : stack_var_cmp (const void *a, const void *b)
1013 : {
1014 19383062 : unsigned ia = *(const unsigned *)a;
1015 19383062 : unsigned ib = *(const unsigned *)b;
1016 19383062 : unsigned int aligna = stack_vars[ia].alignb;
1017 19383062 : unsigned int alignb = stack_vars[ib].alignb;
1018 19383062 : poly_int64 sizea = stack_vars[ia].size;
1019 19383062 : poly_int64 sizeb = stack_vars[ib].size;
1020 19383062 : tree decla = stack_vars[ia].decl;
1021 19383062 : tree declb = stack_vars[ib].decl;
1022 19383062 : bool largea, largeb;
1023 19383062 : unsigned int uida, uidb;
1024 :
1025 : /* Primary compare on "large" alignment. Large comes first. */
1026 19383062 : largea = (aligna * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
1027 19383062 : largeb = (alignb * BITS_PER_UNIT > MAX_SUPPORTED_STACK_ALIGNMENT);
1028 19383062 : if (largea != largeb)
1029 0 : return (int)largeb - (int)largea;
1030 :
1031 : /* Secondary compare on size, decreasing */
1032 19383062 : int diff = compare_sizes_for_sort (sizeb, sizea);
1033 19383062 : if (diff != 0)
1034 8339072 : return diff;
1035 :
1036 : /* Tertiary compare on true alignment, decreasing. */
1037 11043990 : if (aligna < alignb)
1038 : return -1;
1039 10706512 : if (aligna > alignb)
1040 : return 1;
1041 :
1042 : /* Final compare on ID for sort stability, increasing.
1043 : Two SSA names are compared by their version, SSA names come before
1044 : non-SSA names, and two normal decls are compared by their DECL_UID. */
1045 10427415 : if (TREE_CODE (decla) == SSA_NAME)
1046 : {
1047 37236 : if (TREE_CODE (declb) == SSA_NAME)
1048 32736 : uida = SSA_NAME_VERSION (decla), uidb = SSA_NAME_VERSION (declb);
1049 : else
1050 : return -1;
1051 : }
1052 10390179 : else if (TREE_CODE (declb) == SSA_NAME)
1053 : return 1;
1054 : else
1055 10384737 : uida = DECL_UID (decla), uidb = DECL_UID (declb);
1056 10417473 : if (uida < uidb)
1057 : return 1;
1058 5188330 : if (uida > uidb)
1059 5188330 : return -1;
1060 : return 0;
1061 : }
1062 :
1063 : struct part_traits : unbounded_int_hashmap_traits <unsigned , bitmap> {};
1064 : typedef hash_map<unsigned, bitmap, part_traits> part_hashmap;
1065 :
1066 : /* If the points-to solution *PI points to variables that are in a partition
1067 : together with other variables add all partition members to the pointed-to
1068 : variables bitmap. */
1069 :
1070 : static void
1071 1492892 : add_partitioned_vars_to_ptset (struct pt_solution *pt,
1072 : part_hashmap *decls_to_partitions,
1073 : hash_set<bitmap> *visited, bitmap temp)
1074 : {
1075 1492892 : bitmap_iterator bi;
1076 1492892 : unsigned i;
1077 1492892 : bitmap *part;
1078 :
1079 1492892 : if (pt->anything
1080 1442315 : || pt->vars == NULL
1081 : /* The pointed-to vars bitmap is shared, it is enough to
1082 : visit it once. */
1083 2935207 : || visited->add (pt->vars))
1084 1212596 : return;
1085 :
1086 280296 : bitmap_clear (temp);
1087 :
1088 : /* By using a temporary bitmap to store all members of the partitions
1089 : we have to add we make sure to visit each of the partitions only
1090 : once. */
1091 1229325 : EXECUTE_IF_SET_IN_BITMAP (pt->vars, 0, i, bi)
1092 949029 : if ((!temp
1093 949029 : || !bitmap_bit_p (temp, i))
1094 1540335 : && (part = decls_to_partitions->get (i)))
1095 155992 : bitmap_ior_into (temp, *part);
1096 280296 : if (!bitmap_empty_p (temp))
1097 135604 : bitmap_ior_into (pt->vars, temp);
1098 : }
1099 :
1100 : /* Update points-to sets based on partition info, so we can use them on RTL.
1101 : The bitmaps representing stack partitions will be saved until expand,
1102 : where partitioned decls used as bases in memory expressions will be
1103 : rewritten.
1104 :
1105 : It is not necessary to update TBAA info on accesses to the coalesced
1106 : storage since our memory model doesn't allow TBAA to be used for
1107 : WAW or WAR dependences. For RAW when the write is to an old object
1108 : the new object would not have been initialized at the point of the
1109 : read, invoking undefined behavior. */
1110 :
1111 : static void
1112 144415 : update_alias_info_with_stack_vars (void)
1113 : {
1114 144415 : part_hashmap *decls_to_partitions = NULL;
1115 144415 : unsigned i, j;
1116 144415 : tree var = NULL_TREE;
1117 :
1118 1228397 : for (i = 0; i < stack_vars_num; i++)
1119 : {
1120 1083982 : bitmap part = NULL;
1121 1083982 : tree name;
1122 1083982 : struct ptr_info_def *pi;
1123 :
1124 : /* Not interested in partitions with single variable. */
1125 1083982 : if (stack_vars[i].representative != i
1126 833659 : || stack_vars[i].next == EOC)
1127 1019130 : continue;
1128 :
1129 64852 : if (!decls_to_partitions)
1130 : {
1131 40188 : decls_to_partitions = new part_hashmap;
1132 40188 : cfun->gimple_df->decls_to_pointers = new hash_map<tree, tree>;
1133 : }
1134 :
1135 : /* Create an SSA_NAME that points to the partition for use
1136 : as base during alias-oracle queries on RTL for bases that
1137 : have been partitioned. */
1138 64852 : if (var == NULL_TREE)
1139 40188 : var = create_tmp_var (ptr_type_node);
1140 64852 : name = make_ssa_name (var);
1141 :
1142 : /* Create bitmaps representing partitions. They will be used for
1143 : points-to sets later, so use GGC alloc. */
1144 64852 : part = BITMAP_GGC_ALLOC ();
1145 380027 : for (j = i; j != EOC; j = stack_vars[j].next)
1146 : {
1147 315175 : tree decl = stack_vars[j].decl;
1148 315175 : unsigned int uid = DECL_PT_UID (decl);
1149 315175 : bitmap_set_bit (part, uid);
1150 315175 : decls_to_partitions->put (uid, part);
1151 315175 : cfun->gimple_df->decls_to_pointers->put (decl, name);
1152 315175 : if (TREE_ADDRESSABLE (decl))
1153 286369 : TREE_ADDRESSABLE (name) = 1;
1154 : }
1155 :
1156 : /* Make the SSA name point to all partition members. */
1157 64852 : pi = get_ptr_info (name);
1158 64852 : pt_solution_set (&pi->pt, part, false);
1159 : }
1160 :
1161 : /* Make all points-to sets that contain one member of a partition
1162 : contain all members of the partition. */
1163 144415 : if (decls_to_partitions)
1164 : {
1165 40188 : unsigned i;
1166 40188 : tree name;
1167 40188 : hash_set<bitmap> visited;
1168 40188 : bitmap temp = BITMAP_ALLOC (&stack_var_bitmap_obstack);
1169 :
1170 15733348 : FOR_EACH_SSA_NAME (i, name, cfun)
1171 : {
1172 9791280 : struct ptr_info_def *pi;
1173 :
1174 17973927 : if (POINTER_TYPE_P (TREE_TYPE (name))
1175 9863161 : && ((pi = SSA_NAME_PTR_INFO (name)) != NULL))
1176 1412516 : add_partitioned_vars_to_ptset (&pi->pt, decls_to_partitions,
1177 : &visited, temp);
1178 : }
1179 :
1180 40188 : add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped,
1181 : decls_to_partitions, &visited, temp);
1182 40188 : add_partitioned_vars_to_ptset (&cfun->gimple_df->escaped_return,
1183 : decls_to_partitions, &visited, temp);
1184 40188 : delete decls_to_partitions;
1185 40188 : BITMAP_FREE (temp);
1186 40188 : }
1187 144415 : }
1188 :
1189 : /* A subroutine of partition_stack_vars. The UNION portion of a UNION/FIND
1190 : partitioning algorithm. Partitions A and B are known to be non-conflicting.
1191 : Merge them into a single partition A. */
1192 :
1193 : static void
1194 250323 : union_stack_vars (unsigned a, unsigned b)
1195 : {
1196 250323 : class stack_var *vb = &stack_vars[b];
1197 250323 : bitmap_iterator bi;
1198 250323 : unsigned u;
1199 :
1200 250323 : gcc_assert (stack_vars[b].next == EOC);
1201 : /* Add B to A's partition. */
1202 250323 : stack_vars[b].next = stack_vars[a].next;
1203 250323 : stack_vars[b].representative = a;
1204 250323 : stack_vars[a].next = b;
1205 :
1206 : /* Make sure A is big enough to hold B. */
1207 250323 : stack_vars[a].size = upper_bound (stack_vars[a].size, stack_vars[b].size);
1208 :
1209 : /* Update the required alignment of partition A to account for B. */
1210 250323 : if (stack_vars[a].alignb < stack_vars[b].alignb)
1211 1967 : stack_vars[a].alignb = stack_vars[b].alignb;
1212 :
1213 : /* Update the interference graph and merge the conflicts. */
1214 250323 : if (vb->conflicts)
1215 : {
1216 1633247 : EXECUTE_IF_SET_IN_BITMAP (vb->conflicts, 0, u, bi)
1217 1434953 : add_stack_var_conflict (a, stack_vars[u].representative);
1218 198294 : BITMAP_FREE (vb->conflicts);
1219 : }
1220 250323 : }
1221 :
1222 : /* A subroutine of expand_used_vars. Binpack the variables into
1223 : partitions constrained by the interference graph. The overall
1224 : algorithm used is as follows:
1225 :
1226 : Sort the objects by size in descending order.
1227 : For each object A {
1228 : S = size(A)
1229 : O = 0
1230 : loop {
1231 : Look for the largest non-conflicting object B with size <= S.
1232 : UNION (A, B)
1233 : }
1234 : }
1235 : */
1236 :
1237 : static void
1238 229873 : partition_stack_vars (void)
1239 : {
1240 229873 : unsigned si, sj, n = stack_vars_num;
1241 :
1242 229873 : stack_vars_sorted = XNEWVEC (unsigned, stack_vars_num);
1243 1399313 : for (si = 0; si < n; ++si)
1244 1169440 : stack_vars_sorted[si] = si;
1245 :
1246 229873 : if (n == 1)
1247 : return;
1248 :
1249 144415 : qsort (stack_vars_sorted, n, sizeof (unsigned), stack_var_cmp);
1250 :
1251 1228397 : for (si = 0; si < n; ++si)
1252 : {
1253 1083982 : unsigned i = stack_vars_sorted[si];
1254 1083982 : unsigned int ialign = stack_vars[i].alignb;
1255 1083982 : poly_int64 isize = stack_vars[i].size;
1256 :
1257 : /* Ignore objects that aren't partition representatives. If we
1258 : see a var that is not a partition representative, it must
1259 : have been merged earlier. */
1260 1083982 : if (stack_vars[i].representative != i)
1261 250323 : continue;
1262 :
1263 12666539 : for (sj = si + 1; sj < n; ++sj)
1264 : {
1265 11833954 : unsigned j = stack_vars_sorted[sj];
1266 11833954 : unsigned int jalign = stack_vars[j].alignb;
1267 11833954 : poly_int64 jsize = stack_vars[j].size;
1268 :
1269 : /* Ignore objects that aren't partition representatives. */
1270 11833954 : if (stack_vars[j].representative != j)
1271 11832880 : continue;
1272 :
1273 : /* Do not mix objects of "small" (supported) alignment
1274 : and "large" (unsupported) alignment. */
1275 11266478 : if ((ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1276 11266478 : != (jalign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT))
1277 : break;
1278 :
1279 : /* For Address Sanitizer do not mix objects with different
1280 : sizes, as the shorter vars wouldn't be adequately protected.
1281 : Don't do that for "large" (unsupported) alignment objects,
1282 : those aren't protected anyway. */
1283 11266478 : if (asan_sanitize_stack_p ()
1284 5825 : && maybe_ne (isize, jsize)
1285 11267552 : && ialign * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1286 : break;
1287 :
1288 : /* Ignore conflicting objects. */
1289 11265404 : if (stack_var_conflict_p (i, j))
1290 11015081 : continue;
1291 :
1292 : /* UNION the objects, placing J at OFFSET. */
1293 250323 : union_stack_vars (i, j);
1294 : }
1295 : }
1296 :
1297 144415 : update_alias_info_with_stack_vars ();
1298 : }
1299 :
1300 : /* A debugging aid for expand_used_vars. Dump the generated partitions. */
1301 :
1302 : static void
1303 29 : dump_stack_var_partition (void)
1304 : {
1305 29 : unsigned si, i, j, n = stack_vars_num;
1306 :
1307 65 : for (si = 0; si < n; ++si)
1308 : {
1309 36 : i = stack_vars_sorted[si];
1310 :
1311 : /* Skip variables that aren't partition representatives, for now. */
1312 36 : if (stack_vars[i].representative != i)
1313 3 : continue;
1314 :
1315 33 : fprintf (dump_file, "Partition %u: size ", i);
1316 33 : print_dec (stack_vars[i].size, dump_file);
1317 33 : fprintf (dump_file, " align %u\n", stack_vars[i].alignb);
1318 :
1319 102 : for (j = i; j != EOC; j = stack_vars[j].next)
1320 : {
1321 36 : fputc ('\t', dump_file);
1322 36 : print_generic_expr (dump_file, stack_vars[j].decl, dump_flags);
1323 : }
1324 33 : fputc ('\n', dump_file);
1325 : }
1326 29 : }
1327 :
1328 : /* Assign rtl to DECL at BASE + OFFSET. */
1329 :
1330 : static void
1331 1975495 : expand_one_stack_var_at (tree decl, rtx base, unsigned base_align,
1332 : poly_int64 offset)
1333 : {
1334 1975495 : unsigned align;
1335 1975495 : rtx x;
1336 :
1337 : /* If this fails, we've overflowed the stack frame. Error nicely? */
1338 2251659 : gcc_assert (known_eq (offset, trunc_int_for_mode (offset, Pmode)));
1339 :
1340 1975495 : if (hwassist_sanitize_stack_p ())
1341 90 : x = targetm.memtag.add_tag (base, offset,
1342 90 : hwasan_current_frame_tag ());
1343 : else
1344 2251569 : x = plus_constant (Pmode, base, offset);
1345 :
1346 3950990 : x = gen_rtx_MEM (TREE_CODE (decl) == SSA_NAME
1347 452443 : ? TYPE_MODE (TREE_TYPE (decl))
1348 1523052 : : DECL_MODE (decl), x);
1349 :
1350 : /* Set alignment we actually gave this decl if it isn't an SSA name.
1351 : If it is we generate stack slots only accidentally so it isn't as
1352 : important, we'll simply set the alignment directly on the MEM. */
1353 :
1354 1975495 : if (stack_vars_base_reg_p (base))
1355 1972284 : offset -= frame_phase;
1356 1975495 : align = known_alignment (offset);
1357 1975495 : align *= BITS_PER_UNIT;
1358 1975495 : if (align == 0 || align > base_align)
1359 803138 : align = base_align;
1360 :
1361 1975495 : if (TREE_CODE (decl) != SSA_NAME)
1362 : {
1363 : /* One would think that we could assert that we're not decreasing
1364 : alignment here, but (at least) the i386 port does exactly this
1365 : via the MINIMUM_ALIGNMENT hook. */
1366 :
1367 1523052 : SET_DECL_ALIGN (decl, align);
1368 1523052 : DECL_USER_ALIGN (decl) = 0;
1369 : }
1370 :
1371 1975495 : set_rtl (decl, x);
1372 :
1373 1975495 : set_mem_align (x, align);
1374 1975495 : }
1375 :
1376 229873 : class stack_vars_data
1377 : {
1378 : public:
1379 : /* Vector of offset pairs, always end of some padding followed
1380 : by start of the padding that needs Address Sanitizer protection.
1381 : The vector is in reversed, highest offset pairs come first. */
1382 : auto_vec<HOST_WIDE_INT> asan_vec;
1383 :
1384 : /* Vector of partition representative decls in between the paddings. */
1385 : auto_vec<tree> asan_decl_vec;
1386 :
1387 : /* Base pseudo register for Address Sanitizer protected automatic vars. */
1388 : rtx asan_base;
1389 :
1390 : /* Alignment needed for the Address Sanitizer protected automatic vars. */
1391 : unsigned int asan_alignb;
1392 : };
1393 :
1394 : /* A subroutine of expand_used_vars. Give each partition representative
1395 : a unique location within the stack frame. Update each partition member
1396 : with that location. */
1397 : static void
1398 232002 : expand_stack_vars (bool (*pred) (unsigned), class stack_vars_data *data)
1399 : {
1400 232002 : unsigned si, i, j, n = stack_vars_num;
1401 232002 : poly_uint64 large_size = 0, large_alloc = 0;
1402 232002 : rtx large_base = NULL;
1403 232002 : rtx large_untagged_base = NULL;
1404 232002 : unsigned large_align = 0;
1405 232002 : bool large_allocation_done = false;
1406 232002 : tree decl;
1407 :
1408 : /* Determine if there are any variables requiring "large" alignment.
1409 : Since these are dynamically allocated, we only process these if
1410 : no predicate involved. */
1411 232002 : large_align = stack_vars[stack_vars_sorted[0]].alignb * BITS_PER_UNIT;
1412 232002 : if (pred == NULL && large_align > MAX_SUPPORTED_STACK_ALIGNMENT)
1413 : {
1414 : /* Find the total size of these variables. */
1415 0 : for (si = 0; si < n; ++si)
1416 : {
1417 0 : unsigned alignb;
1418 :
1419 0 : i = stack_vars_sorted[si];
1420 0 : alignb = stack_vars[i].alignb;
1421 :
1422 : /* All "large" alignment decls come before all "small" alignment
1423 : decls, but "large" alignment decls are not sorted based on
1424 : their alignment. Increase large_align to track the largest
1425 : required alignment. */
1426 0 : if ((alignb * BITS_PER_UNIT) > large_align)
1427 : large_align = alignb * BITS_PER_UNIT;
1428 :
1429 : /* Stop when we get to the first decl with "small" alignment. */
1430 0 : if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1431 : break;
1432 :
1433 : /* Skip variables that aren't partition representatives. */
1434 0 : if (stack_vars[i].representative != i)
1435 0 : continue;
1436 :
1437 : /* Skip variables that have already had rtl assigned. See also
1438 : add_stack_var where we perpetrate this pc_rtx hack. */
1439 0 : decl = stack_vars[i].decl;
1440 0 : if (TREE_CODE (decl) == SSA_NAME
1441 0 : ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1442 0 : : DECL_RTL (decl) != pc_rtx)
1443 0 : continue;
1444 :
1445 0 : large_size = aligned_upper_bound (large_size, alignb);
1446 0 : large_size += stack_vars[i].size;
1447 : }
1448 : }
1449 :
1450 1406000 : for (si = 0; si < n; ++si)
1451 : {
1452 1173998 : rtx base;
1453 1173998 : unsigned base_align, alignb;
1454 1173998 : poly_int64 offset = 0;
1455 :
1456 1173998 : i = stack_vars_sorted[si];
1457 :
1458 : /* Skip variables that aren't partition representatives, for now. */
1459 1173998 : if (stack_vars[i].representative != i)
1460 1424332 : continue;
1461 :
1462 : /* Skip variables that have already had rtl assigned. See also
1463 : add_stack_var where we perpetrate this pc_rtx hack. */
1464 923664 : decl = stack_vars[i].decl;
1465 923664 : if (TREE_CODE (decl) == SSA_NAME
1466 923664 : ? SA.partition_to_pseudo[var_to_partition (SA.map, decl)] != NULL_RTX
1467 920051 : : DECL_RTL (decl) != pc_rtx)
1468 3521 : continue;
1469 :
1470 : /* Check the predicate to see whether this variable should be
1471 : allocated in this pass. */
1472 920143 : if (pred && !pred (i))
1473 1026 : continue;
1474 :
1475 919117 : base = (hwassist_sanitize_stack_p ()
1476 919117 : ? hwasan_frame_base ()
1477 919061 : : virtual_stack_vars_rtx);
1478 919117 : alignb = stack_vars[i].alignb;
1479 919117 : if (alignb * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT)
1480 : {
1481 919117 : poly_int64 hwasan_orig_offset;
1482 919117 : if (hwassist_sanitize_stack_p ())
1483 : {
1484 : /* There must be no tag granule "shared" between different
1485 : objects. This means that no HWASAN_TAG_GRANULE_SIZE byte
1486 : chunk can have more than one object in it.
1487 :
1488 : We ensure this by forcing the end of the last bit of data to
1489 : be aligned to HWASAN_TAG_GRANULE_SIZE bytes here, and setting
1490 : the start of each variable to be aligned to
1491 : HWASAN_TAG_GRANULE_SIZE bytes in `align_local_variable`.
1492 :
1493 : We can't align just one of the start or end, since there are
1494 : untagged things stored on the stack which we do not align to
1495 : HWASAN_TAG_GRANULE_SIZE bytes. If we only aligned the start
1496 : or the end of tagged objects then untagged objects could end
1497 : up sharing the first granule of a tagged object or sharing the
1498 : last granule of a tagged object respectively. */
1499 56 : hwasan_orig_offset = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1500 56 : gcc_assert (stack_vars[i].alignb >= HWASAN_TAG_GRANULE_SIZE);
1501 : }
1502 : /* ASAN description strings don't yet have a syntax for expressing
1503 : polynomial offsets. */
1504 919117 : HOST_WIDE_INT prev_offset;
1505 919117 : if (asan_sanitize_stack_p ()
1506 3931 : && pred
1507 3206 : && frame_offset.is_constant (&prev_offset)
1508 919117 : && stack_vars[i].size.is_constant ())
1509 : {
1510 3206 : if (data->asan_vec.is_empty ())
1511 : {
1512 1667 : align_frame_offset (ASAN_RED_ZONE_SIZE);
1513 1667 : prev_offset = frame_offset.to_constant ();
1514 : }
1515 3206 : prev_offset = align_base (prev_offset,
1516 : ASAN_MIN_RED_ZONE_SIZE,
1517 : !FRAME_GROWS_DOWNWARD);
1518 3206 : tree repr_decl = NULL_TREE;
1519 3206 : unsigned HOST_WIDE_INT size
1520 3206 : = asan_var_and_redzone_size (stack_vars[i].size.to_constant ());
1521 3206 : if (data->asan_vec.is_empty ())
1522 1667 : size = MAX (size, ASAN_RED_ZONE_SIZE);
1523 :
1524 3206 : unsigned HOST_WIDE_INT alignment = MAX (alignb,
1525 : ASAN_MIN_RED_ZONE_SIZE);
1526 3206 : offset = alloc_stack_frame_space (size, alignment);
1527 :
1528 3206 : data->asan_vec.safe_push (prev_offset);
1529 : /* Allocating a constant amount of space from a constant
1530 : starting offset must give a constant result. */
1531 3206 : data->asan_vec.safe_push ((offset + stack_vars[i].size)
1532 3206 : .to_constant ());
1533 : /* Find best representative of the partition.
1534 : Prefer those with DECL_NAME, even better
1535 : satisfying asan_protect_stack_decl predicate. */
1536 3751 : for (j = i; j != EOC; j = stack_vars[j].next)
1537 3210 : if (asan_protect_stack_decl (stack_vars[j].decl)
1538 3210 : && DECL_NAME (stack_vars[j].decl))
1539 : {
1540 2665 : repr_decl = stack_vars[j].decl;
1541 2665 : break;
1542 : }
1543 545 : else if (repr_decl == NULL_TREE
1544 545 : && DECL_P (stack_vars[j].decl)
1545 1090 : && DECL_NAME (stack_vars[j].decl))
1546 0 : repr_decl = stack_vars[j].decl;
1547 3206 : if (repr_decl == NULL_TREE)
1548 541 : repr_decl = stack_vars[i].decl;
1549 3206 : data->asan_decl_vec.safe_push (repr_decl);
1550 :
1551 : /* Make sure a representative is unpoison if another
1552 : variable in the partition is handled by
1553 : use-after-scope sanitization. */
1554 3206 : if (asan_handled_variables != NULL
1555 3206 : && !asan_handled_variables->contains (repr_decl))
1556 : {
1557 448 : for (j = i; j != EOC; j = stack_vars[j].next)
1558 226 : if (asan_handled_variables->contains (stack_vars[j].decl))
1559 : break;
1560 224 : if (j != EOC)
1561 2 : asan_handled_variables->add (repr_decl);
1562 : }
1563 :
1564 3206 : data->asan_alignb = MAX (data->asan_alignb, alignb);
1565 3206 : if (data->asan_base == NULL)
1566 1667 : data->asan_base = gen_reg_rtx (Pmode);
1567 3206 : base = data->asan_base;
1568 :
1569 3206 : if (!STRICT_ALIGNMENT)
1570 3206 : base_align = crtl->max_used_stack_slot_alignment;
1571 : else
1572 : base_align = MAX (crtl->max_used_stack_slot_alignment,
1573 : GET_MODE_ALIGNMENT (SImode)
1574 : << ASAN_SHADOW_SHIFT);
1575 : }
1576 : else
1577 : {
1578 915911 : offset = alloc_stack_frame_space (stack_vars[i].size, alignb);
1579 915911 : base_align = crtl->max_used_stack_slot_alignment;
1580 :
1581 915911 : if (hwassist_sanitize_stack_p ())
1582 : {
1583 : /* Align again since the point of this alignment is to handle
1584 : the "end" of the object (i.e. smallest address after the
1585 : stack object). For FRAME_GROWS_DOWNWARD that requires
1586 : aligning the stack before allocating, but for a frame that
1587 : grows upwards that requires aligning the stack after
1588 : allocation.
1589 :
1590 : Use `frame_offset` to record the offset value rather than
1591 : `offset` since the `frame_offset` describes the extent
1592 : allocated for this particular variable while `offset`
1593 : describes the address that this variable starts at. */
1594 56 : align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1595 56 : hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1596 : hwasan_orig_offset, frame_offset);
1597 : }
1598 : }
1599 : }
1600 : else
1601 : {
1602 : /* Large alignment is only processed in the last pass. */
1603 0 : if (pred)
1604 0 : continue;
1605 :
1606 : /* If there were any variables requiring "large" alignment, allocate
1607 : space. */
1608 0 : if (maybe_ne (large_size, 0U) && ! large_allocation_done)
1609 : {
1610 0 : poly_int64 loffset;
1611 0 : rtx large_allocsize;
1612 :
1613 0 : large_allocsize = gen_int_mode (large_size, Pmode);
1614 0 : get_dynamic_stack_size (&large_allocsize, 0, large_align, NULL);
1615 0 : loffset = alloc_stack_frame_space
1616 0 : (rtx_to_poly_int64 (large_allocsize),
1617 0 : PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT);
1618 0 : large_base = get_dynamic_stack_base (loffset, large_align, base);
1619 0 : large_allocation_done = true;
1620 : }
1621 :
1622 0 : gcc_assert (large_base != NULL);
1623 0 : large_alloc = aligned_upper_bound (large_alloc, alignb);
1624 0 : offset = large_alloc;
1625 0 : large_alloc += stack_vars[i].size;
1626 0 : if (hwassist_sanitize_stack_p ())
1627 : {
1628 : /* An object with a large alignment requirement means that the
1629 : alignment requirement is greater than the required alignment
1630 : for tags. */
1631 0 : if (!large_untagged_base)
1632 0 : large_untagged_base
1633 0 : = targetm.memtag.untagged_pointer (large_base, NULL_RTX);
1634 : /* Ensure the end of the variable is also aligned correctly. */
1635 0 : poly_int64 align_again
1636 0 : = aligned_upper_bound (large_alloc, HWASAN_TAG_GRANULE_SIZE);
1637 : /* For large allocations we always allocate a chunk of space
1638 : (which is addressed by large_untagged_base/large_base) and
1639 : then use positive offsets from that. Hence the farthest
1640 : offset is `align_again` and the nearest offset from the base
1641 : is `offset`. */
1642 0 : hwasan_record_stack_var (large_untagged_base, large_base,
1643 : offset, align_again);
1644 : }
1645 :
1646 : base = large_base;
1647 : base_align = large_align;
1648 : }
1649 :
1650 : /* Create rtl for each variable based on their location within the
1651 : partition. */
1652 2088557 : for (j = i; j != EOC; j = stack_vars[j].next)
1653 : {
1654 1169440 : expand_one_stack_var_at (stack_vars[j].decl,
1655 : base, base_align, offset);
1656 : }
1657 919117 : if (hwassist_sanitize_stack_p ())
1658 56 : hwasan_increment_frame_tag ();
1659 : }
1660 :
1661 232002 : gcc_assert (known_eq (large_alloc, large_size));
1662 232002 : }
1663 :
1664 : /* Take into account all sizes of partitions and reset DECL_RTLs. */
1665 : static poly_uint64
1666 1041162 : account_stack_vars (void)
1667 : {
1668 1041162 : unsigned si, j, i, n = stack_vars_num;
1669 1041162 : poly_uint64 size = 0;
1670 :
1671 5494101 : for (si = 0; si < n; ++si)
1672 : {
1673 4452939 : i = stack_vars_sorted[si];
1674 :
1675 : /* Skip variables that aren't partition representatives, for now. */
1676 4452939 : if (stack_vars[i].representative != i)
1677 0 : continue;
1678 :
1679 8905878 : size += stack_vars[i].size;
1680 8905878 : for (j = i; j != EOC; j = stack_vars[j].next)
1681 4452939 : set_rtl (stack_vars[j].decl, NULL);
1682 : }
1683 1041162 : return size;
1684 : }
1685 :
1686 : /* Record the RTL assignment X for the default def of PARM. */
1687 :
1688 : extern void
1689 4697276 : set_parm_rtl (tree parm, rtx x)
1690 : {
1691 4697276 : gcc_assert (TREE_CODE (parm) == PARM_DECL
1692 : || TREE_CODE (parm) == RESULT_DECL);
1693 :
1694 4697276 : if (x && !MEM_P (x))
1695 : {
1696 3028382 : unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (parm),
1697 : TYPE_MODE (TREE_TYPE (parm)),
1698 : TYPE_ALIGN (TREE_TYPE (parm)));
1699 :
1700 : /* If the variable alignment is very large we'll dynamically
1701 : allocate it, which means that in-frame portion is just a
1702 : pointer. ??? We've got a pseudo for sure here, do we
1703 : actually dynamically allocate its spilling area if needed?
1704 : ??? Isn't it a problem when Pmode alignment also exceeds
1705 : MAX_SUPPORTED_STACK_ALIGNMENT, as can happen on cris and lm32? */
1706 3028382 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1707 0 : align = GET_MODE_ALIGNMENT (Pmode);
1708 :
1709 3028382 : record_alignment_for_reg_var (align);
1710 : }
1711 :
1712 4697276 : tree ssa = ssa_default_def (cfun, parm);
1713 4697276 : if (!ssa)
1714 1068621 : return set_rtl (parm, x);
1715 :
1716 3628655 : int part = var_to_partition (SA.map, ssa);
1717 3628655 : gcc_assert (part != NO_PARTITION);
1718 :
1719 3628655 : bool changed = bitmap_bit_p (SA.partitions_for_parm_default_defs, part);
1720 3628655 : gcc_assert (changed);
1721 :
1722 3628655 : set_rtl (ssa, x);
1723 3628655 : gcc_assert (DECL_RTL (parm) == x);
1724 : }
1725 :
1726 : /* A subroutine of expand_one_var. Called to immediately assign rtl
1727 : to a variable to be allocated in the stack frame. */
1728 :
1729 : static void
1730 806055 : expand_one_stack_var_1 (tree var)
1731 : {
1732 806055 : poly_uint64 size;
1733 806055 : poly_int64 offset;
1734 806055 : unsigned byte_align;
1735 :
1736 806055 : if (TREE_CODE (var) == SSA_NAME)
1737 : {
1738 449469 : tree type = TREE_TYPE (var);
1739 449469 : size = tree_to_poly_uint64 (TYPE_SIZE_UNIT (type));
1740 : }
1741 : else
1742 356586 : size = tree_to_poly_uint64 (DECL_SIZE_UNIT (var));
1743 :
1744 806055 : byte_align = align_local_variable (var, true);
1745 :
1746 : /* We handle highly aligned variables in expand_stack_vars. */
1747 806055 : gcc_assert (byte_align * BITS_PER_UNIT <= MAX_SUPPORTED_STACK_ALIGNMENT);
1748 :
1749 806055 : rtx base;
1750 806055 : if (hwassist_sanitize_stack_p ())
1751 : {
1752 : /* Allocate zero bytes to align the stack. */
1753 34 : poly_int64 hwasan_orig_offset
1754 34 : = align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1755 34 : offset = alloc_stack_frame_space (size, byte_align);
1756 34 : align_frame_offset (HWASAN_TAG_GRANULE_SIZE);
1757 34 : base = hwasan_frame_base ();
1758 : /* Use `frame_offset` to automatically account for machines where the
1759 : frame grows upwards.
1760 :
1761 : `offset` will always point to the "start" of the stack object, which
1762 : will be the smallest address, for ! FRAME_GROWS_DOWNWARD this is *not*
1763 : the "furthest" offset from the base delimiting the current stack
1764 : object. `frame_offset` will always delimit the extent that the frame.
1765 : */
1766 34 : hwasan_record_stack_var (virtual_stack_vars_rtx, base,
1767 : hwasan_orig_offset, frame_offset);
1768 : }
1769 : else
1770 : {
1771 806021 : offset = alloc_stack_frame_space (size, byte_align);
1772 806021 : base = virtual_stack_vars_rtx;
1773 : }
1774 :
1775 806055 : expand_one_stack_var_at (var, base,
1776 : crtl->max_used_stack_slot_alignment, offset);
1777 :
1778 806055 : if (hwassist_sanitize_stack_p ())
1779 34 : hwasan_increment_frame_tag ();
1780 806055 : }
1781 :
1782 : /* Wrapper for expand_one_stack_var_1 that checks SSA_NAMEs are
1783 : already assigned some MEM. */
1784 :
1785 : static void
1786 356586 : expand_one_stack_var (tree var)
1787 : {
1788 356586 : if (TREE_CODE (var) == SSA_NAME)
1789 : {
1790 0 : int part = var_to_partition (SA.map, var);
1791 0 : if (part != NO_PARTITION)
1792 : {
1793 0 : rtx x = SA.partition_to_pseudo[part];
1794 0 : gcc_assert (x);
1795 0 : gcc_assert (MEM_P (x));
1796 : return;
1797 : }
1798 : }
1799 :
1800 356586 : return expand_one_stack_var_1 (var);
1801 : }
1802 :
1803 : /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1804 : that will reside in a hard register. */
1805 :
1806 : static void
1807 1085 : expand_one_hard_reg_var (tree var)
1808 : {
1809 0 : rest_of_decl_compilation (var, 0, 0);
1810 0 : }
1811 :
1812 : /* Record the alignment requirements of some variable assigned to a
1813 : pseudo. */
1814 :
1815 : static void
1816 40276427 : record_alignment_for_reg_var (unsigned int align)
1817 : {
1818 40276427 : if (SUPPORTS_STACK_ALIGNMENT
1819 40276427 : && crtl->stack_alignment_estimated < align)
1820 : {
1821 : /* stack_alignment_estimated shouldn't change after stack
1822 : realign decision made */
1823 1716849 : gcc_assert (!crtl->stack_realign_processed);
1824 1716849 : crtl->stack_alignment_estimated = align;
1825 : }
1826 :
1827 : /* stack_alignment_needed > PREFERRED_STACK_BOUNDARY is permitted.
1828 : So here we only make sure stack_alignment_needed >= align. */
1829 40276427 : if (crtl->stack_alignment_needed < align)
1830 559017 : crtl->stack_alignment_needed = align;
1831 40276427 : if (crtl->max_used_stack_slot_alignment < align)
1832 559017 : crtl->max_used_stack_slot_alignment = align;
1833 40276427 : }
1834 :
1835 : /* Create RTL for an SSA partition. */
1836 :
1837 : static void
1838 22165531 : expand_one_ssa_partition (tree var)
1839 : {
1840 22165531 : int part = var_to_partition (SA.map, var);
1841 22165531 : gcc_assert (part != NO_PARTITION);
1842 :
1843 22165531 : if (SA.partition_to_pseudo[part])
1844 : return;
1845 :
1846 22165531 : unsigned int align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
1847 : TYPE_MODE (TREE_TYPE (var)),
1848 : TYPE_ALIGN (TREE_TYPE (var)));
1849 :
1850 : /* If the variable alignment is very large we'll dynamically allocate
1851 : it, which means that in-frame portion is just a pointer. */
1852 22165531 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
1853 0 : align = GET_MODE_ALIGNMENT (Pmode);
1854 :
1855 22165531 : record_alignment_for_reg_var (align);
1856 :
1857 22165531 : if (!use_register_for_decl (var))
1858 : {
1859 452443 : if (defer_stack_allocation (var, true))
1860 2974 : add_stack_var (var, true);
1861 : else
1862 449469 : expand_one_stack_var_1 (var);
1863 452443 : return;
1864 : }
1865 :
1866 21713088 : machine_mode reg_mode = promote_ssa_mode (var, NULL);
1867 21713088 : rtx x = gen_reg_rtx (reg_mode);
1868 :
1869 21713088 : set_rtl (var, x);
1870 :
1871 : /* For a promoted variable, X will not be used directly but wrapped in a
1872 : SUBREG with SUBREG_PROMOTED_VAR_P set, which means that the RTL land
1873 : will assume that its upper bits can be inferred from its lower bits.
1874 : Therefore, if X isn't initialized on every path from the entry, then
1875 : we must do it manually in order to fulfill the above assumption. */
1876 21713088 : if (reg_mode != TYPE_MODE (TREE_TYPE (var))
1877 21713088 : && bitmap_bit_p (SA.partitions_for_undefined_values, part))
1878 0 : emit_move_insn (x, CONST0_RTX (reg_mode));
1879 : }
1880 :
1881 : /* Record the association between the RTL generated for partition PART
1882 : and the underlying variable of the SSA_NAME VAR. */
1883 :
1884 : static void
1885 48295347 : adjust_one_expanded_partition_var (tree var)
1886 : {
1887 48295347 : if (!var)
1888 : return;
1889 :
1890 48295347 : tree decl = SSA_NAME_VAR (var);
1891 :
1892 48295347 : int part = var_to_partition (SA.map, var);
1893 48295347 : if (part == NO_PARTITION)
1894 : return;
1895 :
1896 31069984 : rtx x = SA.partition_to_pseudo[part];
1897 :
1898 31069984 : gcc_assert (x);
1899 :
1900 31069984 : set_rtl (var, x);
1901 :
1902 31069984 : if (!REG_P (x))
1903 : return;
1904 :
1905 : /* Note if the object is a user variable. */
1906 29060013 : if (decl && !DECL_ARTIFICIAL (decl))
1907 4084321 : mark_user_reg (x);
1908 :
1909 29060013 : if (POINTER_TYPE_P (decl ? TREE_TYPE (decl) : TREE_TYPE (var)))
1910 7418877 : mark_reg_pointer (x, get_pointer_alignment (var));
1911 : }
1912 :
1913 : /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL
1914 : that will reside in a pseudo register. */
1915 :
1916 : static void
1917 566738 : expand_one_register_var (tree var)
1918 : {
1919 566738 : if (TREE_CODE (var) == SSA_NAME)
1920 : {
1921 0 : int part = var_to_partition (SA.map, var);
1922 0 : if (part != NO_PARTITION)
1923 : {
1924 0 : rtx x = SA.partition_to_pseudo[part];
1925 0 : gcc_assert (x);
1926 0 : gcc_assert (REG_P (x));
1927 : return;
1928 : }
1929 0 : gcc_unreachable ();
1930 : }
1931 :
1932 566738 : tree decl = var;
1933 566738 : tree type = TREE_TYPE (decl);
1934 566738 : machine_mode reg_mode = promote_decl_mode (decl, NULL);
1935 566738 : rtx x = gen_reg_rtx (reg_mode);
1936 :
1937 566738 : set_rtl (var, x);
1938 :
1939 : /* Note if the object is a user variable. */
1940 566738 : if (!DECL_ARTIFICIAL (decl))
1941 87176 : mark_user_reg (x);
1942 :
1943 566738 : if (POINTER_TYPE_P (type))
1944 100 : mark_reg_pointer (x, get_pointer_alignment (var));
1945 : }
1946 :
1947 : /* A subroutine of expand_one_var. Called to assign rtl to a VAR_DECL that
1948 : has some associated error, e.g. its type is error-mark. We just need
1949 : to pick something that won't crash the rest of the compiler. */
1950 :
1951 : static void
1952 32 : expand_one_error_var (tree var)
1953 : {
1954 32 : machine_mode mode = DECL_MODE (var);
1955 32 : rtx x;
1956 :
1957 32 : if (mode == BLKmode)
1958 1 : x = gen_rtx_MEM (BLKmode, const0_rtx);
1959 31 : else if (mode == VOIDmode)
1960 0 : x = const0_rtx;
1961 : else
1962 31 : x = gen_reg_rtx (mode);
1963 :
1964 32 : SET_DECL_RTL (var, x);
1965 32 : }
1966 :
1967 : /* A subroutine of expand_one_var. VAR is a variable that will be
1968 : allocated to the local stack frame. Return true if we wish to
1969 : add VAR to STACK_VARS so that it will be coalesced with other
1970 : variables. Return false to allocate VAR immediately.
1971 :
1972 : This function is used to reduce the number of variables considered
1973 : for coalescing, which reduces the size of the quadratic problem. */
1974 :
1975 : static bool
1976 6564731 : defer_stack_allocation (tree var, bool toplevel)
1977 : {
1978 6564731 : tree size_unit = TREE_CODE (var) == SSA_NAME
1979 6564731 : ? TYPE_SIZE_UNIT (TREE_TYPE (var))
1980 6564731 : : DECL_SIZE_UNIT (var);
1981 6564731 : poly_uint64 size;
1982 :
1983 : /* Whether the variable is small enough for immediate allocation not to be
1984 : a problem with regard to the frame size. */
1985 6564731 : bool smallish
1986 6564731 : = (poly_int_tree_p (size_unit, &size)
1987 6564731 : && (estimated_poly_value (size)
1988 6564731 : < param_min_size_for_stack_sharing));
1989 :
1990 : /* If stack protection is enabled, *all* stack variables must be deferred,
1991 : so that we can re-order the strings to the top of the frame.
1992 : Similarly for Address Sanitizer. */
1993 6564731 : if (flag_stack_protect || asan_sanitize_stack_p ())
1994 13437 : return true;
1995 :
1996 6551294 : unsigned int align = TREE_CODE (var) == SSA_NAME
1997 6551294 : ? TYPE_ALIGN (TREE_TYPE (var))
1998 6099493 : : DECL_ALIGN (var);
1999 :
2000 : /* We handle "large" alignment via dynamic allocation. We want to handle
2001 : this extra complication in only one place, so defer them. */
2002 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
2003 : return true;
2004 :
2005 6551294 : bool ignored = TREE_CODE (var) == SSA_NAME
2006 6557565 : ? !SSAVAR (var) || DECL_IGNORED_P (SSA_NAME_VAR (var))
2007 6099493 : : DECL_IGNORED_P (var);
2008 :
2009 : /* When optimization is enabled, DECL_IGNORED_P variables originally scoped
2010 : might be detached from their block and appear at toplevel when we reach
2011 : here. We want to coalesce them with variables from other blocks when
2012 : the immediate contribution to the frame size would be noticeable. */
2013 6551294 : if (toplevel && optimize > 0 && ignored && !smallish)
2014 : return true;
2015 :
2016 : /* Variables declared in the outermost scope automatically conflict
2017 : with every other variable. The only reason to want to defer them
2018 : at all is that, after sorting, we can more efficiently pack
2019 : small variables in the stack frame. Continue to defer at -O2. */
2020 5072758 : if (toplevel && optimize < 2)
2021 : return false;
2022 :
2023 : /* Without optimization, *most* variables are allocated from the
2024 : stack, which makes the quadratic problem large exactly when we
2025 : want compilation to proceed as quickly as possible. On the
2026 : other hand, we don't want the function's stack frame size to
2027 : get completely out of hand. So we avoid adding scalars and
2028 : "small" aggregates to the list at all. */
2029 4411447 : if (optimize == 0 && smallish)
2030 : return false;
2031 :
2032 : return true;
2033 : }
2034 :
2035 : /* A subroutine of expand_used_vars. Expand one variable according to
2036 : its flavor. Variables to be placed on the stack are not actually
2037 : expanded yet, merely recorded.
2038 : When REALLY_EXPAND is false, only add stack values to be allocated.
2039 : Return stack usage this variable is supposed to take.
2040 : */
2041 :
2042 : static poly_uint64
2043 15383610 : expand_one_var (tree var, bool toplevel, bool really_expand,
2044 : bitmap forced_stack_var = NULL)
2045 : {
2046 15383610 : unsigned int align = BITS_PER_UNIT;
2047 15383610 : tree origvar = var;
2048 :
2049 15383610 : var = SSAVAR (var);
2050 :
2051 15383610 : if (TREE_TYPE (var) != error_mark_node && VAR_P (var))
2052 : {
2053 15383610 : if (is_global_var (var))
2054 301096 : return 0;
2055 :
2056 : /* Because we don't know if VAR will be in register or on stack,
2057 : we conservatively assume it will be on stack even if VAR is
2058 : eventually put into register after RA pass. For non-automatic
2059 : variables, which won't be on stack, we collect alignment of
2060 : type and ignore user specified alignment. Similarly for
2061 : SSA_NAMEs for which use_register_for_decl returns true. */
2062 15082514 : if (TREE_STATIC (var)
2063 15082514 : || DECL_EXTERNAL (var)
2064 30165028 : || (TREE_CODE (origvar) == SSA_NAME && use_register_for_decl (var)))
2065 0 : align = MINIMUM_ALIGNMENT (TREE_TYPE (var),
2066 : TYPE_MODE (TREE_TYPE (var)),
2067 : TYPE_ALIGN (TREE_TYPE (var)));
2068 15082514 : else if (DECL_HAS_VALUE_EXPR_P (var)
2069 15082514 : || (DECL_RTL_SET_P (var) && MEM_P (DECL_RTL (var))))
2070 : /* Don't consider debug only variables with DECL_HAS_VALUE_EXPR_P set
2071 : or variables which were assigned a stack slot already by
2072 : expand_one_stack_var_at - in the latter case DECL_ALIGN has been
2073 : changed from the offset chosen to it. */
2074 16924 : align = crtl->stack_alignment_estimated;
2075 : else
2076 15065590 : align = MINIMUM_ALIGNMENT (var, DECL_MODE (var), DECL_ALIGN (var));
2077 :
2078 : /* If the variable alignment is very large we'll dynamically allocate
2079 : it, which means that in-frame portion is just a pointer. */
2080 15082514 : if (align > MAX_SUPPORTED_STACK_ALIGNMENT)
2081 0 : align = GET_MODE_ALIGNMENT (Pmode);
2082 : }
2083 :
2084 15082514 : record_alignment_for_reg_var (align);
2085 :
2086 15082514 : poly_uint64 size;
2087 15082514 : if (TREE_CODE (origvar) == SSA_NAME)
2088 : {
2089 0 : gcc_assert (!VAR_P (var)
2090 : || (!DECL_EXTERNAL (var)
2091 : && !DECL_HAS_VALUE_EXPR_P (var)
2092 : && !TREE_STATIC (var)
2093 : && TREE_TYPE (var) != error_mark_node
2094 : && !DECL_HARD_REGISTER (var)
2095 : && really_expand));
2096 : }
2097 15082514 : if (!VAR_P (var) && TREE_CODE (origvar) != SSA_NAME)
2098 : ;
2099 15082514 : else if (DECL_EXTERNAL (var))
2100 : ;
2101 15082514 : else if (DECL_HAS_VALUE_EXPR_P (var))
2102 : ;
2103 15065683 : else if (TREE_STATIC (var))
2104 : ;
2105 15065683 : else if (TREE_CODE (origvar) != SSA_NAME && DECL_RTL_SET_P (var))
2106 : ;
2107 15065447 : else if (TREE_TYPE (var) == error_mark_node)
2108 : {
2109 0 : if (really_expand)
2110 0 : expand_one_error_var (var);
2111 : }
2112 15065447 : else if (VAR_P (var) && DECL_HARD_REGISTER (var))
2113 : {
2114 4121 : if (really_expand)
2115 : {
2116 1085 : expand_one_hard_reg_var (var);
2117 1085 : if (!DECL_HARD_REGISTER (var))
2118 : /* Invalid register specification. */
2119 32 : expand_one_error_var (var);
2120 : }
2121 : }
2122 15061326 : else if (use_register_for_decl (var)
2123 15061326 : && (!forced_stack_var
2124 575339 : || !bitmap_bit_p (forced_stack_var, DECL_UID (var))))
2125 : {
2126 8949038 : if (really_expand)
2127 566738 : expand_one_register_var (origvar);
2128 : }
2129 6112288 : else if (!poly_int_tree_p (DECL_SIZE_UNIT (var), &size)
2130 6112288 : || !valid_constant_size_p (DECL_SIZE_UNIT (var)))
2131 : {
2132 : /* Reject variables which cover more than half of the address-space. */
2133 0 : if (really_expand)
2134 : {
2135 0 : if (DECL_NONLOCAL_FRAME (var))
2136 0 : error_at (DECL_SOURCE_LOCATION (current_function_decl),
2137 : "total size of local objects is too large");
2138 : else
2139 0 : error_at (DECL_SOURCE_LOCATION (var),
2140 : "size of variable %q+D is too large", var);
2141 0 : expand_one_error_var (var);
2142 : }
2143 : }
2144 6112288 : else if (defer_stack_allocation (var, toplevel))
2145 5619405 : add_stack_var (origvar, really_expand);
2146 : else
2147 : {
2148 492883 : if (really_expand)
2149 : {
2150 356284 : if (lookup_attribute ("naked",
2151 356284 : DECL_ATTRIBUTES (current_function_decl)))
2152 0 : error ("cannot allocate stack for variable %q+D, naked function",
2153 : var);
2154 :
2155 356284 : expand_one_stack_var (origvar);
2156 : }
2157 492883 : return size;
2158 : }
2159 14589631 : return 0;
2160 : }
2161 :
2162 : /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
2163 : expanding variables. Those variables that can be put into registers
2164 : are allocated pseudos; those that can't are put on the stack.
2165 :
2166 : TOPLEVEL is true if this is the outermost BLOCK. */
2167 :
2168 : static void
2169 15979797 : expand_used_vars_for_block (tree block, bool toplevel, bitmap forced_stack_vars)
2170 : {
2171 15979797 : tree t;
2172 :
2173 : /* Expand all variables at this level. */
2174 34564433 : for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
2175 18584636 : if (TREE_USED (t)
2176 18584636 : && ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
2177 748826 : || !DECL_NONSHAREABLE (t)))
2178 748826 : expand_one_var (t, toplevel, true, forced_stack_vars);
2179 :
2180 : /* Expand all variables at containing levels. */
2181 30470444 : for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
2182 14490647 : expand_used_vars_for_block (t, false, forced_stack_vars);
2183 15979797 : }
2184 :
2185 : /* A subroutine of expand_used_vars. Walk down through the BLOCK tree
2186 : and clear TREE_USED on all local variables. */
2187 :
2188 : static void
2189 15979797 : clear_tree_used (tree block)
2190 : {
2191 15979797 : tree t;
2192 :
2193 34564433 : for (t = BLOCK_VARS (block); t ; t = DECL_CHAIN (t))
2194 : /* if (!TREE_STATIC (t) && !DECL_EXTERNAL (t)) */
2195 1035471 : if ((!VAR_P (t) && TREE_CODE (t) != RESULT_DECL)
2196 18584636 : || !DECL_NONSHAREABLE (t))
2197 18584636 : TREE_USED (t) = 0;
2198 :
2199 30470444 : for (t = BLOCK_SUBBLOCKS (block); t ; t = BLOCK_CHAIN (t))
2200 14490647 : clear_tree_used (t);
2201 15979797 : }
2202 :
2203 : /* Examine TYPE and determine a bit mask of the following features. */
2204 :
2205 : #define SPCT_HAS_LARGE_CHAR_ARRAY 1
2206 : #define SPCT_HAS_SMALL_CHAR_ARRAY 2
2207 : #define SPCT_HAS_ARRAY 4
2208 : #define SPCT_HAS_AGGREGATE 8
2209 :
2210 : static unsigned int
2211 2756 : stack_protect_classify_type (tree type)
2212 : {
2213 2756 : unsigned int ret = 0;
2214 2756 : tree t;
2215 :
2216 2756 : switch (TREE_CODE (type))
2217 : {
2218 720 : case ARRAY_TYPE:
2219 720 : t = TYPE_MAIN_VARIANT (TREE_TYPE (type));
2220 720 : if (t == char_type_node
2221 178 : || t == signed_char_type_node
2222 178 : || t == unsigned_char_type_node)
2223 : {
2224 542 : unsigned HOST_WIDE_INT max = param_ssp_buffer_size;
2225 542 : unsigned HOST_WIDE_INT len;
2226 :
2227 542 : if (!TYPE_SIZE_UNIT (type)
2228 542 : || !tree_fits_uhwi_p (TYPE_SIZE_UNIT (type)))
2229 : len = max;
2230 : else
2231 542 : len = tree_to_uhwi (TYPE_SIZE_UNIT (type));
2232 :
2233 542 : if (len < max)
2234 : ret = SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_ARRAY;
2235 : else
2236 510 : ret = SPCT_HAS_LARGE_CHAR_ARRAY | SPCT_HAS_ARRAY;
2237 : }
2238 : else
2239 : ret = SPCT_HAS_ARRAY;
2240 : break;
2241 :
2242 1021 : case UNION_TYPE:
2243 1021 : case QUAL_UNION_TYPE:
2244 1021 : case RECORD_TYPE:
2245 1021 : ret = SPCT_HAS_AGGREGATE;
2246 10915 : for (t = TYPE_FIELDS (type); t ; t = TREE_CHAIN (t))
2247 9894 : if (TREE_CODE (t) == FIELD_DECL)
2248 1702 : ret |= stack_protect_classify_type (TREE_TYPE (t));
2249 : break;
2250 :
2251 : default:
2252 : break;
2253 : }
2254 :
2255 2756 : return ret;
2256 : }
2257 :
2258 : /* Return nonzero if DECL should be segregated into the "vulnerable" upper
2259 : part of the local stack frame. Remember if we ever return nonzero for
2260 : any variable in this function. The return value is the phase number in
2261 : which the variable should be allocated. */
2262 :
2263 : static int
2264 1054 : stack_protect_decl_phase (tree decl)
2265 : {
2266 1054 : unsigned int bits = stack_protect_classify_type (TREE_TYPE (decl));
2267 1054 : int ret = 0;
2268 :
2269 1054 : if (bits & SPCT_HAS_SMALL_CHAR_ARRAY)
2270 28 : has_short_buffer = true;
2271 :
2272 1054 : tree attribs = DECL_ATTRIBUTES (current_function_decl);
2273 1054 : if (!lookup_attribute ("no_stack_protector", attribs)
2274 1054 : && (flag_stack_protect == SPCT_FLAG_ALL
2275 1054 : || flag_stack_protect == SPCT_FLAG_STRONG
2276 227 : || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2277 5 : && lookup_attribute ("stack_protect", attribs))))
2278 : {
2279 832 : if ((bits & (SPCT_HAS_SMALL_CHAR_ARRAY | SPCT_HAS_LARGE_CHAR_ARRAY))
2280 368 : && !(bits & SPCT_HAS_AGGREGATE))
2281 : ret = 1;
2282 752 : else if (bits & SPCT_HAS_ARRAY)
2283 : ret = 2;
2284 : }
2285 : else
2286 222 : ret = (bits & SPCT_HAS_LARGE_CHAR_ARRAY) != 0;
2287 :
2288 222 : if (ret)
2289 702 : has_protected_decls = true;
2290 :
2291 1054 : return ret;
2292 : }
2293 :
2294 : /* Two helper routines that check for phase 1 and phase 2. These are used
2295 : as callbacks for expand_stack_vars. */
2296 :
2297 : static bool
2298 359 : stack_protect_decl_phase_1 (unsigned i)
2299 : {
2300 359 : return stack_protect_decl_phase (stack_vars[i].decl) == 1;
2301 : }
2302 :
2303 : static bool
2304 217 : stack_protect_decl_phase_2 (unsigned i)
2305 : {
2306 217 : return stack_protect_decl_phase (stack_vars[i].decl) == 2;
2307 : }
2308 :
2309 : /* And helper function that checks for asan phase (with stack protector
2310 : it is phase 3). This is used as callback for expand_stack_vars.
2311 : Returns true if any of the vars in the partition need to be protected. */
2312 :
2313 : static bool
2314 3846 : asan_decl_phase_3 (unsigned i)
2315 : {
2316 4571 : while (i != EOC)
2317 : {
2318 3846 : if (asan_protect_stack_decl (stack_vars[i].decl))
2319 : return true;
2320 725 : i = stack_vars[i].next;
2321 : }
2322 : return false;
2323 : }
2324 :
2325 : /* Ensure that variables in different stack protection phases conflict
2326 : so that they are not merged and share the same stack slot.
2327 : Return true if there are any address taken variables. */
2328 :
2329 : static bool
2330 267 : add_stack_protection_conflicts (void)
2331 : {
2332 267 : unsigned i, j, n = stack_vars_num;
2333 267 : unsigned char *phase;
2334 267 : bool ret = false;
2335 :
2336 267 : phase = XNEWVEC (unsigned char, n);
2337 1012 : for (i = 0; i < n; ++i)
2338 : {
2339 478 : phase[i] = stack_protect_decl_phase (stack_vars[i].decl);
2340 478 : if (TREE_ADDRESSABLE (stack_vars[i].decl))
2341 401 : ret = true;
2342 : }
2343 :
2344 745 : for (i = 0; i < n; ++i)
2345 : {
2346 478 : unsigned char ph_i = phase[i];
2347 1194 : for (j = i + 1; j < n; ++j)
2348 716 : if (ph_i != phase[j])
2349 329 : add_stack_var_conflict (i, j);
2350 : }
2351 :
2352 267 : XDELETEVEC (phase);
2353 267 : return ret;
2354 : }
2355 :
2356 : /* Create a decl for the guard at the top of the stack frame. */
2357 :
2358 : static void
2359 302 : create_stack_guard (void)
2360 : {
2361 302 : tree guard = build_decl (DECL_SOURCE_LOCATION (current_function_decl),
2362 : VAR_DECL, NULL, ptr_type_node);
2363 302 : TREE_THIS_VOLATILE (guard) = 1;
2364 302 : TREE_USED (guard) = 1;
2365 302 : expand_one_stack_var (guard);
2366 302 : crtl->stack_protect_guard = guard;
2367 302 : }
2368 :
2369 : /* Prepare for expanding variables. */
2370 : static void
2371 7716549 : init_vars_expansion (void)
2372 : {
2373 : /* Conflict bitmaps, and a few related temporary bitmaps, go here. */
2374 7716549 : bitmap_obstack_initialize (&stack_var_bitmap_obstack);
2375 :
2376 : /* A map from decl to stack partition. */
2377 7716549 : decl_to_stack_part = new hash_map<tree, unsigned>;
2378 :
2379 : /* Initialize local stack smashing state. */
2380 7716549 : has_protected_decls = false;
2381 7716549 : has_short_buffer = false;
2382 7716549 : if (hwassist_sanitize_stack_p ())
2383 1208 : hwasan_record_frame_init ();
2384 7716549 : }
2385 :
2386 : /* Free up stack variable graph data. */
2387 : static void
2388 7716549 : fini_vars_expansion (void)
2389 : {
2390 7716549 : bitmap_obstack_release (&stack_var_bitmap_obstack);
2391 7716549 : if (stack_vars)
2392 1271035 : XDELETEVEC (stack_vars);
2393 7716549 : if (stack_vars_sorted)
2394 1271035 : XDELETEVEC (stack_vars_sorted);
2395 7716549 : stack_vars = NULL;
2396 7716549 : stack_vars_sorted = NULL;
2397 7716549 : stack_vars_alloc = stack_vars_num = 0;
2398 15433098 : delete decl_to_stack_part;
2399 7716549 : decl_to_stack_part = NULL;
2400 7716549 : }
2401 :
2402 : /* Make a fair guess for the size of the stack frame of the function
2403 : in NODE. This doesn't have to be exact, the result is only used in
2404 : the inline heuristics. So we don't want to run the full stack var
2405 : packing algorithm (which is quadratic in the number of stack vars).
2406 : Instead, we calculate the total size of all stack vars. This turns
2407 : out to be a pretty fair estimate -- packing of stack vars doesn't
2408 : happen very often. */
2409 :
2410 : HOST_WIDE_INT
2411 6227399 : estimated_stack_frame_size (struct cgraph_node *node)
2412 : {
2413 6227399 : poly_int64 size = 0;
2414 6227399 : unsigned i;
2415 6227399 : tree var;
2416 6227399 : struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
2417 :
2418 6227399 : push_cfun (fn);
2419 :
2420 6227399 : init_vars_expansion ();
2421 :
2422 24889347 : FOR_EACH_LOCAL_DECL (fn, i, var)
2423 13561042 : if (auto_var_in_fn_p (var, fn->decl))
2424 12975000 : size += expand_one_var (var, true, false);
2425 :
2426 6227399 : if (stack_vars_num > 0)
2427 : {
2428 : /* Fake sorting the stack vars for account_stack_vars (). */
2429 1041162 : stack_vars_sorted = XNEWVEC (unsigned , stack_vars_num);
2430 5494101 : for (i = 0; i < stack_vars_num; ++i)
2431 4452939 : stack_vars_sorted[i] = i;
2432 1041162 : size += account_stack_vars ();
2433 : }
2434 :
2435 6227399 : fini_vars_expansion ();
2436 6227399 : pop_cfun ();
2437 6227399 : return estimated_poly_value (size);
2438 : }
2439 :
2440 : /* Check if the current function has calls that use a return slot. */
2441 :
2442 : static bool
2443 418 : stack_protect_return_slot_p ()
2444 : {
2445 418 : basic_block bb;
2446 :
2447 1876 : FOR_ALL_BB_FN (bb, cfun)
2448 2974 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2449 4028 : !gsi_end_p (gsi); gsi_next (&gsi))
2450 : {
2451 2570 : gimple *stmt = gsi_stmt (gsi);
2452 : /* This assumes that calls to internal-only functions never
2453 : use a return slot. */
2454 2570 : if (is_gimple_call (stmt)
2455 542 : && !gimple_call_internal_p (stmt)
2456 3095 : && aggregate_value_p (TREE_TYPE (gimple_call_fntype (stmt)),
2457 525 : gimple_call_fndecl (stmt)))
2458 418 : return true;
2459 : }
2460 : return false;
2461 : }
2462 :
2463 : /* Expand all variables used in the function. */
2464 :
2465 : static rtx_insn *
2466 1489150 : expand_used_vars (bitmap forced_stack_vars)
2467 : {
2468 1489150 : tree var, outer_block = DECL_INITIAL (current_function_decl);
2469 1489150 : auto_vec<tree> maybe_local_decls;
2470 1489150 : rtx_insn *var_end_seq = NULL;
2471 1489150 : unsigned i;
2472 1489150 : unsigned len;
2473 1489150 : bool gen_stack_protect_signal = false;
2474 :
2475 : /* Compute the phase of the stack frame for this function. */
2476 1489150 : {
2477 1489150 : int align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2478 1489150 : int off = targetm.starting_frame_offset () % align;
2479 1489150 : frame_phase = off ? align - off : 0;
2480 : }
2481 :
2482 : /* Set TREE_USED on all variables in the local_decls. */
2483 10524625 : FOR_EACH_LOCAL_DECL (cfun, i, var)
2484 7789382 : TREE_USED (var) = 1;
2485 : /* Clear TREE_USED on all variables associated with a block scope. */
2486 1489150 : clear_tree_used (DECL_INITIAL (current_function_decl));
2487 :
2488 1489150 : init_vars_expansion ();
2489 :
2490 1489150 : if (targetm.use_pseudo_pic_reg ())
2491 80871 : pic_offset_table_rtx = gen_reg_rtx (Pmode);
2492 :
2493 27283336 : for (i = 0; i < SA.map->num_partitions; i++)
2494 : {
2495 25794186 : if (bitmap_bit_p (SA.partitions_for_parm_default_defs, i))
2496 3628655 : continue;
2497 :
2498 22165531 : tree var = partition_to_var (SA.map, i);
2499 :
2500 44331062 : gcc_assert (!virtual_operand_p (var));
2501 :
2502 22165531 : expand_one_ssa_partition (var);
2503 : }
2504 :
2505 1489150 : if (flag_stack_protect == SPCT_FLAG_STRONG)
2506 418 : gen_stack_protect_signal = stack_protect_return_slot_p ();
2507 :
2508 : /* At this point all variables on the local_decls with TREE_USED
2509 : set are not associated with any block scope. Lay them out. */
2510 :
2511 1489150 : len = vec_safe_length (cfun->local_decls);
2512 9278532 : FOR_EACH_LOCAL_DECL (cfun, i, var)
2513 : {
2514 7789382 : bool expand_now = false;
2515 :
2516 : /* Expanded above already. */
2517 7789382 : if (is_gimple_reg (var))
2518 : {
2519 5467219 : TREE_USED (var) = 0;
2520 5467219 : goto next;
2521 : }
2522 : /* We didn't set a block for static or extern because it's hard
2523 : to tell the difference between a global variable (re)declared
2524 : in a local scope, and one that's really declared there to
2525 : begin with. And it doesn't really matter much, since we're
2526 : not giving them stack space. Expand them now. */
2527 2322163 : else if (TREE_STATIC (var) || DECL_EXTERNAL (var))
2528 : expand_now = true;
2529 :
2530 : /* Expand variables not associated with any block now. Those created by
2531 : the optimizers could be live anywhere in the function. Those that
2532 : could possibly have been scoped originally and detached from their
2533 : block will have their allocation deferred so we coalesce them with
2534 : others when optimization is enabled. */
2535 2107514 : else if (TREE_USED (var))
2536 1659784 : expand_now = true;
2537 :
2538 : /* Finally, mark all variables on the list as used. We'll use
2539 : this in a moment when we expand those associated with scopes. */
2540 2322163 : TREE_USED (var) = 1;
2541 :
2542 2322163 : if (expand_now)
2543 1659784 : expand_one_var (var, true, true, forced_stack_vars);
2544 :
2545 662379 : next:
2546 7789382 : if (DECL_ARTIFICIAL (var) && !DECL_IGNORED_P (var))
2547 : {
2548 406580 : rtx rtl = DECL_RTL_IF_SET (var);
2549 :
2550 : /* Keep artificial non-ignored vars in cfun->local_decls
2551 : chain until instantiate_decls. */
2552 303061 : if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2553 20569 : add_local_decl (cfun, var);
2554 103519 : else if (rtl == NULL_RTX)
2555 : /* If rtl isn't set yet, which can happen e.g. with
2556 : -fstack-protector, retry before returning from this
2557 : function. */
2558 103519 : maybe_local_decls.safe_push (var);
2559 : }
2560 : }
2561 :
2562 : /* We duplicated some of the decls in CFUN->LOCAL_DECLS.
2563 :
2564 : +-----------------+-----------------+
2565 : | ...processed... | ...duplicates...|
2566 : +-----------------+-----------------+
2567 : ^
2568 : +-- LEN points here.
2569 :
2570 : We just want the duplicates, as those are the artificial
2571 : non-ignored vars that we want to keep until instantiate_decls.
2572 : Move them down and truncate the array. */
2573 1489150 : if (!vec_safe_is_empty (cfun->local_decls))
2574 839173 : cfun->local_decls->block_remove (0, len);
2575 :
2576 : /* At this point, all variables within the block tree with TREE_USED
2577 : set are actually used by the optimized function. Lay them out. */
2578 1489150 : expand_used_vars_for_block (outer_block, true, forced_stack_vars);
2579 :
2580 1489150 : tree attribs = DECL_ATTRIBUTES (current_function_decl);
2581 1489150 : if (stack_vars_num > 0)
2582 : {
2583 229873 : bool has_addressable_vars = false;
2584 :
2585 229873 : add_scope_conflicts ();
2586 :
2587 : /* If stack protection is enabled, we don't share space between
2588 : vulnerable data and non-vulnerable data. */
2589 229873 : if (flag_stack_protect != 0
2590 267 : && !lookup_attribute ("no_stack_protector", attribs)
2591 230140 : && (flag_stack_protect != SPCT_FLAG_EXPLICIT
2592 : || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2593 1 : && lookup_attribute ("stack_protect", attribs))))
2594 267 : has_addressable_vars = add_stack_protection_conflicts ();
2595 :
2596 229873 : if (flag_stack_protect == SPCT_FLAG_STRONG && has_addressable_vars)
2597 229873 : gen_stack_protect_signal = true;
2598 :
2599 : /* Now that we have collected all stack variables, and have computed a
2600 : minimal interference graph, attempt to save some stack space. */
2601 229873 : partition_stack_vars ();
2602 229873 : if (dump_file)
2603 29 : dump_stack_var_partition ();
2604 : }
2605 :
2606 :
2607 1489150 : if (!lookup_attribute ("no_stack_protector", attribs))
2608 1489141 : switch (flag_stack_protect)
2609 : {
2610 51 : case SPCT_FLAG_ALL:
2611 51 : create_stack_guard ();
2612 51 : break;
2613 :
2614 418 : case SPCT_FLAG_STRONG:
2615 418 : if (gen_stack_protect_signal
2616 274 : || cfun->calls_alloca
2617 272 : || has_protected_decls
2618 681 : || lookup_attribute ("stack_protect", attribs))
2619 155 : create_stack_guard ();
2620 : break;
2621 :
2622 171 : case SPCT_FLAG_DEFAULT:
2623 171 : if (cfun->calls_alloca
2624 168 : || has_protected_decls
2625 256 : || lookup_attribute ("stack_protect", attribs))
2626 86 : create_stack_guard ();
2627 : break;
2628 :
2629 14 : case SPCT_FLAG_EXPLICIT:
2630 14 : if (lookup_attribute ("stack_protect", attribs))
2631 10 : create_stack_guard ();
2632 : break;
2633 :
2634 : default:
2635 : break;
2636 : }
2637 :
2638 : /* Assign rtl to each variable based on these partitions. */
2639 1489150 : if (stack_vars_num > 0)
2640 : {
2641 229873 : class stack_vars_data data;
2642 :
2643 229873 : data.asan_base = NULL_RTX;
2644 229873 : data.asan_alignb = 0;
2645 :
2646 : /* Reorder decls to be protected by iterating over the variables
2647 : array multiple times, and allocating out of each phase in turn. */
2648 : /* ??? We could probably integrate this into the qsort we did
2649 : earlier, such that we naturally see these variables first,
2650 : and thus naturally allocate things in the right order. */
2651 229873 : if (has_protected_decls)
2652 : {
2653 : /* Phase 1 contains only character arrays. */
2654 172 : expand_stack_vars (stack_protect_decl_phase_1, &data);
2655 :
2656 : /* Phase 2 contains other kinds of arrays. */
2657 172 : if (!lookup_attribute ("no_stack_protector", attribs)
2658 172 : && (flag_stack_protect == SPCT_FLAG_ALL
2659 172 : || flag_stack_protect == SPCT_FLAG_STRONG
2660 84 : || (flag_stack_protect == SPCT_FLAG_EXPLICIT
2661 1 : && lookup_attribute ("stack_protect", attribs))))
2662 89 : expand_stack_vars (stack_protect_decl_phase_2, &data);
2663 : }
2664 :
2665 229873 : if (asan_sanitize_stack_p ())
2666 : /* Phase 3, any partitions that need asan protection
2667 : in addition to phase 1 and 2. */
2668 1868 : expand_stack_vars (asan_decl_phase_3, &data);
2669 :
2670 : /* ASAN description strings don't yet have a syntax for expressing
2671 : polynomial offsets. */
2672 229873 : HOST_WIDE_INT prev_offset;
2673 231540 : if (!data.asan_vec.is_empty ()
2674 1667 : && frame_offset.is_constant (&prev_offset))
2675 : {
2676 1667 : HOST_WIDE_INT offset, sz, redzonesz;
2677 1667 : redzonesz = ASAN_RED_ZONE_SIZE;
2678 1667 : sz = data.asan_vec[0] - prev_offset;
2679 1667 : if (data.asan_alignb > ASAN_RED_ZONE_SIZE
2680 1667 : && data.asan_alignb <= 4096
2681 45 : && sz + ASAN_RED_ZONE_SIZE >= (int) data.asan_alignb)
2682 45 : redzonesz = ((sz + ASAN_RED_ZONE_SIZE + data.asan_alignb - 1)
2683 45 : & ~(data.asan_alignb - HOST_WIDE_INT_1)) - sz;
2684 : /* Allocating a constant amount of space from a constant
2685 : starting offset must give a constant result. */
2686 1667 : offset = (alloc_stack_frame_space (redzonesz, ASAN_RED_ZONE_SIZE)
2687 1667 : .to_constant ());
2688 1667 : data.asan_vec.safe_push (prev_offset);
2689 1667 : data.asan_vec.safe_push (offset);
2690 : /* Leave space for alignment if STRICT_ALIGNMENT. */
2691 1667 : if (STRICT_ALIGNMENT)
2692 : alloc_stack_frame_space ((GET_MODE_ALIGNMENT (SImode)
2693 : << ASAN_SHADOW_SHIFT)
2694 : / BITS_PER_UNIT, 1);
2695 :
2696 1667 : var_end_seq
2697 3334 : = asan_emit_stack_protection (virtual_stack_vars_rtx,
2698 : data.asan_base,
2699 : data.asan_alignb,
2700 : data.asan_vec.address (),
2701 : data.asan_decl_vec.address (),
2702 1667 : data.asan_vec.length ());
2703 : }
2704 :
2705 229873 : expand_stack_vars (NULL, &data);
2706 229873 : }
2707 :
2708 1489150 : if (hwassist_sanitize_stack_p ())
2709 464 : hwasan_emit_prologue ();
2710 1489150 : if (asan_sanitize_allocas_p () && cfun->calls_alloca)
2711 178 : var_end_seq = asan_emit_allocas_unpoison (virtual_stack_dynamic_rtx,
2712 : virtual_stack_vars_rtx,
2713 : var_end_seq);
2714 2977480 : else if ((hwasan_sanitize_allocas_p () || memtag_sanitize_p ())
2715 1488972 : && cfun->calls_alloca)
2716 : /* When using out-of-line instrumentation we only want to emit one function
2717 : call for clearing the tags in a region of shadow stack. When there are
2718 : alloca calls in this frame we want to emit a call using the
2719 : virtual_stack_dynamic_rtx, but when not we use the hwasan_frame_extent
2720 : rtx we created in expand_stack_vars. */
2721 0 : var_end_seq = hwasan_emit_untag_frame (virtual_stack_dynamic_rtx,
2722 : virtual_stack_vars_rtx);
2723 1488972 : else if (hwassist_sanitize_stack_p ())
2724 : /* If no variables were stored on the stack, `hwasan_get_frame_extent`
2725 : will return NULL_RTX and hence `hwasan_emit_untag_frame` will return
2726 : NULL (i.e. an empty sequence). */
2727 464 : var_end_seq = hwasan_emit_untag_frame (hwasan_get_frame_extent (),
2728 : virtual_stack_vars_rtx);
2729 :
2730 1489150 : fini_vars_expansion ();
2731 :
2732 : /* If there were any artificial non-ignored vars without rtl
2733 : found earlier, see if deferred stack allocation hasn't assigned
2734 : rtl to them. */
2735 1633840 : FOR_EACH_VEC_ELT_REVERSE (maybe_local_decls, i, var)
2736 : {
2737 103519 : rtx rtl = DECL_RTL_IF_SET (var);
2738 :
2739 : /* Keep artificial non-ignored vars in cfun->local_decls
2740 : chain until instantiate_decls. */
2741 4926 : if (rtl && (MEM_P (rtl) || GET_CODE (rtl) == CONCAT))
2742 4717 : add_local_decl (cfun, var);
2743 : }
2744 :
2745 : /* If the target requires that FRAME_OFFSET be aligned, do it. */
2746 1489150 : if (STACK_ALIGNMENT_NEEDED)
2747 : {
2748 1489150 : HOST_WIDE_INT align = PREFERRED_STACK_BOUNDARY / BITS_PER_UNIT;
2749 1489150 : if (FRAME_GROWS_DOWNWARD)
2750 1489150 : frame_offset = aligned_lower_bound (frame_offset, align);
2751 : else
2752 : frame_offset = aligned_upper_bound (frame_offset, align);
2753 : }
2754 :
2755 1489150 : return var_end_seq;
2756 1489150 : }
2757 :
2758 :
2759 : /* If we need to produce a detailed dump, print the tree representation
2760 : for STMT to the dump file. SINCE is the last RTX after which the RTL
2761 : generated for STMT should have been appended. */
2762 :
2763 : static void
2764 41610841 : maybe_dump_rtl_for_gimple_stmt (gimple *stmt, rtx_insn *since)
2765 : {
2766 41610841 : if (dump_file && (dump_flags & TDF_DETAILS))
2767 : {
2768 396 : fprintf (dump_file, "\n;; ");
2769 396 : print_gimple_stmt (dump_file, stmt, 0,
2770 : TDF_SLIM | (dump_flags & TDF_LINENO));
2771 396 : fprintf (dump_file, "\n");
2772 :
2773 396 : print_rtl (dump_file, since ? NEXT_INSN (since) : since);
2774 : }
2775 41610841 : }
2776 :
2777 : /* Temporary storage for BB_HEAD and BB_END of bbs until they are converted
2778 : to BB_RTL. */
2779 : static vec<std::pair <rtx_insn *, rtx_insn *>> head_end_for_bb;
2780 :
2781 : /* Maps the blocks that do not contain tree labels to rtx labels. */
2782 :
2783 : static hash_map<basic_block, rtx_code_label *> *lab_rtx_for_bb;
2784 :
2785 : /* Returns the label_rtx expression for a label starting basic block BB. */
2786 :
2787 : static rtx_code_label *
2788 7585012 : label_rtx_for_bb (basic_block bb)
2789 : {
2790 7585012 : if (bb->flags & BB_RTL)
2791 0 : return block_label (bb);
2792 :
2793 7585012 : if ((unsigned) bb->index < head_end_for_bb.length ()
2794 7585012 : && head_end_for_bb[bb->index].first)
2795 : {
2796 1560564 : if (!LABEL_P (head_end_for_bb[bb->index].first))
2797 : {
2798 950624 : head_end_for_bb[bb->index].first
2799 1901248 : = emit_label_before (gen_label_rtx (),
2800 950624 : head_end_for_bb[bb->index].first);
2801 : }
2802 1560564 : return as_a <rtx_code_label *> (head_end_for_bb[bb->index].first);
2803 : }
2804 :
2805 6024448 : rtx_code_label **elt = lab_rtx_for_bb->get (bb);
2806 6024448 : if (elt)
2807 1246294 : return *elt;
2808 :
2809 : /* Find the tree label if it is present. */
2810 4778154 : gimple_stmt_iterator gsi = gsi_start_bb (bb);
2811 4778154 : glabel *lab_stmt;
2812 4778154 : if (!gsi_end_p (gsi)
2813 4638040 : && (lab_stmt = dyn_cast <glabel *> (gsi_stmt (gsi)))
2814 5000549 : && !DECL_NONLOCAL (gimple_label_label (lab_stmt)))
2815 222395 : return jump_target_rtx (gimple_label_label (lab_stmt));
2816 :
2817 4555759 : rtx_code_label *l = gen_label_rtx ();
2818 4555759 : lab_rtx_for_bb->put (bb, l);
2819 4555759 : return l;
2820 : }
2821 :
2822 :
2823 : /* Wrapper around remove_edge during expansion. */
2824 :
2825 : void
2826 127630 : expand_remove_edge (edge e)
2827 : {
2828 127630 : if (current_ir_type () != IR_GIMPLE
2829 127630 : && (e->dest->flags & BB_RTL) == 0
2830 188837 : && !gimple_seq_empty_p (phi_nodes (e->dest)))
2831 9069 : remove_phi_args (e);
2832 127630 : remove_edge (e);
2833 127630 : }
2834 :
2835 : /* Split edge E during expansion and instead of creating a new
2836 : bb on that edge, add there BB. FLAGS should be flags on the
2837 : new edge from BB to former E->dest. */
2838 :
2839 : static void
2840 1883871 : expand_split_edge (edge e, basic_block bb, int flags)
2841 : {
2842 1883871 : unsigned int dest_idx = e->dest_idx;
2843 1883871 : basic_block dest = e->dest;
2844 1883871 : redirect_edge_succ (e, bb);
2845 1883871 : e = make_single_succ_edge (bb, dest, flags);
2846 1883871 : if ((dest->flags & BB_RTL) == 0
2847 1883871 : && phi_nodes (dest)
2848 2016042 : && e->dest_idx != dest_idx)
2849 : {
2850 : /* If there are any PHI nodes on dest, swap the new succ edge
2851 : with the one moved into false_edge's former position, so that
2852 : PHI arguments don't need adjustment. */
2853 82269 : edge e2 = EDGE_PRED (dest, dest_idx);
2854 82269 : std::swap (e->dest_idx, e2->dest_idx);
2855 82269 : std::swap (EDGE_PRED (dest, e->dest_idx),
2856 : EDGE_PRED (dest, e2->dest_idx));
2857 : }
2858 1883871 : }
2859 :
2860 :
2861 : /* A subroutine of expand_gimple_cond. Given E, a fallthrough edge
2862 : of a basic block where we just expanded the conditional at the end,
2863 : possibly clean up the CFG and instruction sequence. LAST is the
2864 : last instruction before the just emitted jump sequence. */
2865 :
2866 : static void
2867 5218143 : maybe_cleanup_end_of_block (edge e, rtx_insn *last)
2868 : {
2869 : /* Special case: when jumpif decides that the condition is
2870 : trivial it emits an unconditional jump (and the necessary
2871 : barrier). But we still have two edges, the fallthru one is
2872 : wrong. purge_dead_edges would clean this up later. Unfortunately
2873 : we have to insert insns (and split edges) before
2874 : find_many_sub_basic_blocks and hence before purge_dead_edges.
2875 : But splitting edges might create new blocks which depend on the
2876 : fact that if there are two edges there's no barrier. So the
2877 : barrier would get lost and verify_flow_info would ICE. Instead
2878 : of auditing all edge splitters to care for the barrier (which
2879 : normally isn't there in a cleaned CFG), fix it here. */
2880 5218143 : if (BARRIER_P (get_last_insn ()))
2881 : {
2882 126 : rtx_insn *insn;
2883 126 : expand_remove_edge (e);
2884 : /* Now, we have a single successor block, if we have insns to
2885 : insert on the remaining edge we potentially will insert
2886 : it at the end of this block (if the dest block isn't feasible)
2887 : in order to avoid splitting the edge. This insertion will take
2888 : place in front of the last jump. But we might have emitted
2889 : multiple jumps (conditional and one unconditional) to the
2890 : same destination. Inserting in front of the last one then
2891 : is a problem. See PR 40021. We fix this by deleting all
2892 : jumps except the last unconditional one. */
2893 126 : insn = PREV_INSN (get_last_insn ());
2894 : /* Make sure we have an unconditional jump. Otherwise we're
2895 : confused. */
2896 126 : gcc_assert (JUMP_P (insn) && !any_condjump_p (insn));
2897 252 : for (insn = PREV_INSN (insn); insn != last;)
2898 : {
2899 0 : insn = PREV_INSN (insn);
2900 0 : if (JUMP_P (NEXT_INSN (insn)))
2901 : {
2902 0 : if (!any_condjump_p (NEXT_INSN (insn)))
2903 : {
2904 0 : gcc_assert (BARRIER_P (NEXT_INSN (NEXT_INSN (insn))));
2905 0 : delete_insn (NEXT_INSN (NEXT_INSN (insn)));
2906 : }
2907 0 : delete_insn (NEXT_INSN (insn));
2908 : }
2909 : }
2910 : }
2911 5218143 : }
2912 :
2913 : /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_COND.
2914 : Returns a new basic block if we've terminated the current basic
2915 : block and created a new one. */
2916 :
2917 : static basic_block
2918 5612864 : expand_gimple_cond (basic_block bb, gcond *stmt)
2919 : {
2920 5612864 : basic_block new_bb, dest;
2921 5612864 : edge true_edge;
2922 5612864 : edge false_edge;
2923 5612864 : rtx_insn *last2, *last;
2924 5612864 : enum tree_code code;
2925 5612864 : tree op0, op1;
2926 :
2927 5612864 : code = gimple_cond_code (stmt);
2928 5612864 : op0 = gimple_cond_lhs (stmt);
2929 5612864 : op1 = gimple_cond_rhs (stmt);
2930 : /* We're sometimes presented with such code:
2931 : D.123_1 = x < y;
2932 : if (D.123_1 != 0)
2933 : ...
2934 : This would expand to two comparisons which then later might
2935 : be cleaned up by combine. But some pattern matchers like if-conversion
2936 : work better when there's only one compare, so make up for this
2937 : here as special exception if TER would have made the same change. */
2938 5612864 : if (SA.values
2939 4079403 : && TREE_CODE (op0) == SSA_NAME
2940 4078676 : && TREE_CODE (TREE_TYPE (op0)) == BOOLEAN_TYPE
2941 371426 : && TREE_CODE (op1) == INTEGER_CST
2942 361134 : && ((gimple_cond_code (stmt) == NE_EXPR
2943 361132 : && integer_zerop (op1))
2944 2 : || (gimple_cond_code (stmt) == EQ_EXPR
2945 2 : && integer_onep (op1)))
2946 5973998 : && bitmap_bit_p (SA.values, SSA_NAME_VERSION (op0)))
2947 : {
2948 190451 : gimple *second = SSA_NAME_DEF_STMT (op0);
2949 190451 : if (gimple_code (second) == GIMPLE_ASSIGN)
2950 : {
2951 190432 : enum tree_code code2 = gimple_assign_rhs_code (second);
2952 190432 : if (TREE_CODE_CLASS (code2) == tcc_comparison)
2953 : {
2954 816 : code = code2;
2955 816 : op0 = gimple_assign_rhs1 (second);
2956 816 : op1 = gimple_assign_rhs2 (second);
2957 : }
2958 : /* If jumps are cheap and the target does not support conditional
2959 : compare, turn some more codes into jumpy sequences. */
2960 189616 : else if (BRANCH_COST (optimize_insn_for_speed_p (), false) < 4
2961 189616 : && !targetm.have_ccmp ())
2962 : {
2963 189613 : if ((code2 == BIT_AND_EXPR
2964 54660 : && TYPE_PRECISION (TREE_TYPE (op0)) == 1
2965 54660 : && TREE_CODE (gimple_assign_rhs2 (second)) != INTEGER_CST)
2966 189613 : || code2 == TRUTH_AND_EXPR)
2967 : {
2968 54660 : code = TRUTH_ANDIF_EXPR;
2969 54660 : op0 = gimple_assign_rhs1 (second);
2970 54660 : op1 = gimple_assign_rhs2 (second);
2971 : }
2972 134953 : else if (code2 == BIT_IOR_EXPR || code2 == TRUTH_OR_EXPR)
2973 : {
2974 58602 : code = TRUTH_ORIF_EXPR;
2975 58602 : op0 = gimple_assign_rhs1 (second);
2976 58602 : op1 = gimple_assign_rhs2 (second);
2977 : }
2978 : }
2979 : }
2980 : }
2981 :
2982 : /* Optimize (x % C1) == C2 or (x % C1) != C2 if it is beneficial
2983 : into (x - C2) * C3 < C4. */
2984 5612864 : if ((code == EQ_EXPR || code == NE_EXPR)
2985 4398160 : && TREE_CODE (op0) == SSA_NAME
2986 4397371 : && TREE_CODE (op1) == INTEGER_CST)
2987 3039101 : code = maybe_optimize_mod_cmp (code, &op0, &op1);
2988 :
2989 : /* Optimize (x - y) < 0 into x < y if x - y has undefined overflow. */
2990 5612864 : if (!TYPE_UNSIGNED (TREE_TYPE (op0))
2991 2272262 : && (code == LT_EXPR || code == LE_EXPR
2992 2272262 : || code == GT_EXPR || code == GE_EXPR)
2993 694111 : && integer_zerop (op1)
2994 5814846 : && TREE_CODE (op0) == SSA_NAME)
2995 201982 : maybe_optimize_sub_cmp_0 (code, &op0, &op1);
2996 :
2997 5612864 : last2 = last = get_last_insn ();
2998 :
2999 5612864 : extract_true_false_edges_from_block (bb, &true_edge, &false_edge);
3000 5612864 : set_curr_insn_location (gimple_location (stmt));
3001 :
3002 : /* We can either have a pure conditional jump with one fallthru edge or
3003 : two-way jump that needs to be decomposed into two basic blocks. */
3004 5612864 : if (false_edge->dest == bb->next_bb)
3005 : {
3006 2234757 : jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
3007 : true_edge->probability);
3008 2234757 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
3009 2234757 : if (true_edge->goto_locus != UNKNOWN_LOCATION)
3010 332793 : set_curr_insn_location (true_edge->goto_locus);
3011 2234757 : false_edge->flags |= EDGE_FALLTHRU;
3012 2234757 : maybe_cleanup_end_of_block (false_edge, last);
3013 2234757 : return NULL;
3014 : }
3015 3378107 : if (true_edge->dest == bb->next_bb)
3016 : {
3017 2983386 : jumpifnot_1 (code, op0, op1, label_rtx_for_bb (false_edge->dest),
3018 : false_edge->probability);
3019 2983386 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
3020 2983386 : if (false_edge->goto_locus != UNKNOWN_LOCATION)
3021 24097 : set_curr_insn_location (false_edge->goto_locus);
3022 2983386 : true_edge->flags |= EDGE_FALLTHRU;
3023 2983386 : maybe_cleanup_end_of_block (true_edge, last);
3024 2983386 : return NULL;
3025 : }
3026 :
3027 394721 : jumpif_1 (code, op0, op1, label_rtx_for_bb (true_edge->dest),
3028 : true_edge->probability);
3029 394721 : last = get_last_insn ();
3030 394721 : if (false_edge->goto_locus != UNKNOWN_LOCATION)
3031 1286 : set_curr_insn_location (false_edge->goto_locus);
3032 394721 : emit_jump (label_rtx_for_bb (false_edge->dest));
3033 :
3034 394721 : head_end_for_bb[bb->index].second = last;
3035 394721 : if (BARRIER_P (head_end_for_bb[bb->index].second))
3036 98 : head_end_for_bb[bb->index].second
3037 49 : = PREV_INSN (head_end_for_bb[bb->index].second);
3038 789442 : update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
3039 394721 : head_end_for_bb[bb->index].second, bb);
3040 :
3041 394721 : new_bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
3042 394721 : dest = false_edge->dest;
3043 394721 : expand_split_edge (false_edge, new_bb, 0);
3044 394721 : false_edge->flags |= EDGE_FALLTHRU;
3045 394721 : new_bb->count = false_edge->count ();
3046 394721 : loop_p loop = find_common_loop (bb->loop_father, dest->loop_father);
3047 394721 : add_bb_to_loop (new_bb, loop);
3048 394721 : if (loop->latch == bb
3049 26608 : && loop->header == dest)
3050 26608 : loop->latch = new_bb;
3051 394721 : if (BARRIER_P (BB_END (new_bb)))
3052 394721 : BB_END (new_bb) = PREV_INSN (BB_END (new_bb));
3053 394721 : update_bb_for_insn (new_bb);
3054 :
3055 394721 : maybe_dump_rtl_for_gimple_stmt (stmt, last2);
3056 :
3057 394721 : if (true_edge->goto_locus != UNKNOWN_LOCATION)
3058 : {
3059 44999 : set_curr_insn_location (true_edge->goto_locus);
3060 44999 : true_edge->goto_locus = curr_insn_location ();
3061 : }
3062 :
3063 : return new_bb;
3064 : }
3065 :
3066 : /* Mark all calls that can have a transaction restart. */
3067 :
3068 : static void
3069 6603669 : mark_transaction_restart_calls (gimple *stmt)
3070 : {
3071 6603669 : struct tm_restart_node dummy;
3072 6603669 : tm_restart_node **slot;
3073 :
3074 6603669 : if (!cfun->gimple_df->tm_restart)
3075 6600681 : return;
3076 :
3077 2988 : dummy.stmt = stmt;
3078 2988 : slot = cfun->gimple_df->tm_restart->find_slot (&dummy, NO_INSERT);
3079 2988 : if (slot)
3080 : {
3081 0 : struct tm_restart_node *n = *slot;
3082 0 : tree list = n->label_or_list;
3083 0 : rtx_insn *insn;
3084 :
3085 0 : for (insn = next_real_insn (get_last_insn ());
3086 0 : !CALL_P (insn);
3087 0 : insn = next_real_insn (insn))
3088 0 : continue;
3089 :
3090 0 : if (TREE_CODE (list) == LABEL_DECL)
3091 0 : add_reg_note (insn, REG_TM, label_rtx (list));
3092 : else
3093 0 : for (; list ; list = TREE_CHAIN (list))
3094 0 : add_reg_note (insn, REG_TM, label_rtx (TREE_VALUE (list)));
3095 0 : }
3096 : }
3097 :
3098 : /* A subroutine of expand_gimple_stmt_1, expanding one GIMPLE_CALL
3099 : statement STMT. */
3100 :
3101 : static void
3102 6825106 : expand_call_stmt (gcall *stmt)
3103 : {
3104 6825106 : tree exp, decl, lhs;
3105 6825106 : bool builtin_p;
3106 6825106 : size_t i;
3107 :
3108 6825106 : if (gimple_call_internal_p (stmt))
3109 : {
3110 188883 : expand_internal_call (stmt);
3111 188883 : return;
3112 : }
3113 :
3114 : /* If this is a call to a built-in function and it has no effect other
3115 : than setting the lhs, try to implement it using an internal function
3116 : instead. */
3117 6636223 : decl = gimple_call_fndecl (stmt);
3118 6636223 : if (gimple_call_lhs (stmt)
3119 2351664 : && !gimple_has_side_effects (stmt)
3120 7338927 : && (optimize || (decl && called_as_built_in (decl))))
3121 : {
3122 682088 : internal_fn ifn = replacement_internal_fn (stmt);
3123 682088 : if (ifn != IFN_LAST)
3124 : {
3125 32552 : expand_internal_call (ifn, stmt);
3126 32552 : return;
3127 : }
3128 : }
3129 :
3130 6603671 : exp = build_vl_exp (CALL_EXPR, gimple_call_num_args (stmt) + 3);
3131 :
3132 6603671 : CALL_EXPR_FN (exp) = gimple_call_fn (stmt);
3133 6603671 : builtin_p = decl && fndecl_built_in_p (decl);
3134 :
3135 : /* If this is not a builtin function, the function type through which the
3136 : call is made may be different from the type of the function. */
3137 4637488 : if (!builtin_p)
3138 9274976 : CALL_EXPR_FN (exp)
3139 13912464 : = fold_convert (build_pointer_type (gimple_call_fntype (stmt)),
3140 : CALL_EXPR_FN (exp));
3141 :
3142 6603671 : TREE_TYPE (exp) = gimple_call_return_type (stmt);
3143 6603671 : CALL_EXPR_STATIC_CHAIN (exp) = gimple_call_chain (stmt);
3144 :
3145 19862349 : for (i = 0; i < gimple_call_num_args (stmt); i++)
3146 : {
3147 13258678 : tree arg = gimple_call_arg (stmt, i);
3148 13258678 : gimple *def;
3149 : /* TER addresses into arguments of builtin functions so we have a
3150 : chance to infer more correct alignment information. See PR39954. */
3151 13258678 : if (builtin_p
3152 4035342 : && TREE_CODE (arg) == SSA_NAME
3153 1377355 : && (def = get_gimple_for_ssa_name (arg))
3154 334943 : && is_gimple_assign (def)
3155 13593460 : && gimple_assign_rhs_code (def) == ADDR_EXPR)
3156 14589 : arg = gimple_assign_rhs1 (def);
3157 13258678 : CALL_EXPR_ARG (exp, i) = arg;
3158 : }
3159 :
3160 6603671 : if (gimple_has_side_effects (stmt)
3161 : /* ??? Downstream in expand_expr_real_1 we assume that expressions
3162 : w/o side-effects do not throw so work around this here. */
3163 6603671 : || stmt_could_throw_p (cfun, stmt))
3164 5935760 : TREE_SIDE_EFFECTS (exp) = 1;
3165 :
3166 6603671 : if (gimple_call_nothrow_p (stmt))
3167 2945739 : TREE_NOTHROW (exp) = 1;
3168 :
3169 6603671 : CALL_EXPR_TAILCALL (exp) = gimple_call_tail_p (stmt);
3170 6603671 : CALL_EXPR_MUST_TAIL_CALL (exp) = gimple_call_must_tail_p (stmt);
3171 6603671 : CALL_EXPR_RETURN_SLOT_OPT (exp) = gimple_call_return_slot_opt_p (stmt);
3172 6603671 : if (decl
3173 6421252 : && fndecl_built_in_p (decl, BUILT_IN_NORMAL)
3174 8395031 : && ALLOCA_FUNCTION_CODE_P (DECL_FUNCTION_CODE (decl)))
3175 27738 : CALL_ALLOCA_FOR_VAR_P (exp) = gimple_call_alloca_for_var_p (stmt);
3176 : else
3177 6575933 : CALL_FROM_THUNK_P (exp) = gimple_call_from_thunk_p (stmt);
3178 6603671 : CALL_EXPR_VA_ARG_PACK (exp) = gimple_call_va_arg_pack_p (stmt);
3179 6603671 : CALL_EXPR_BY_DESCRIPTOR (exp) = gimple_call_by_descriptor_p (stmt);
3180 6603671 : SET_EXPR_LOCATION (exp, gimple_location (stmt));
3181 :
3182 : /* Must come after copying location. */
3183 6603671 : copy_warning (exp, stmt);
3184 :
3185 : /* Ensure RTL is created for debug args. */
3186 13024923 : if (decl && DECL_HAS_DEBUG_ARGS_P (decl))
3187 : {
3188 66881 : vec<tree, va_gc> **debug_args = decl_debug_args_lookup (decl);
3189 66881 : unsigned int ix;
3190 66881 : tree dtemp;
3191 :
3192 66881 : if (debug_args)
3193 146026 : for (ix = 1; (*debug_args)->iterate (ix, &dtemp); ix += 2)
3194 : {
3195 79145 : gcc_assert (TREE_CODE (dtemp) == DEBUG_EXPR_DECL);
3196 79145 : expand_debug_expr (dtemp);
3197 : }
3198 : }
3199 :
3200 6603671 : rtx_insn *before_call = get_last_insn ();
3201 6603671 : lhs = gimple_call_lhs (stmt);
3202 6603671 : if (lhs)
3203 2319112 : expand_assignment (lhs, exp, false);
3204 : else
3205 4284559 : expand_expr (exp, const0_rtx, VOIDmode, EXPAND_NORMAL);
3206 :
3207 : /* If the gimple call is an indirect call and has 'nocf_check'
3208 : attribute find a generated CALL insn to mark it as no
3209 : control-flow verification is needed. */
3210 6603669 : if (gimple_call_nocf_check_p (stmt)
3211 6603690 : && !gimple_call_fndecl (stmt))
3212 : {
3213 20 : rtx_insn *last = get_last_insn ();
3214 20 : while (!CALL_P (last)
3215 34 : && last != before_call)
3216 14 : last = PREV_INSN (last);
3217 :
3218 20 : if (last != before_call)
3219 20 : add_reg_note (last, REG_CALL_NOCF_CHECK, const0_rtx);
3220 : }
3221 :
3222 6603669 : mark_transaction_restart_calls (stmt);
3223 : }
3224 :
3225 :
3226 : /* Generate RTL for an asm statement (explicit assembler code).
3227 : STRING is a STRING_CST node containing the assembler code text,
3228 : or an ADDR_EXPR containing a STRING_CST. VOL nonzero means the
3229 : insn is volatile; don't optimize it. */
3230 :
3231 : static void
3232 2926 : expand_asm_loc (tree string, int vol, location_t locus)
3233 : {
3234 2926 : rtx body;
3235 :
3236 2926 : body = gen_rtx_ASM_INPUT_loc (VOIDmode,
3237 : ggc_strdup (TREE_STRING_POINTER (string)),
3238 : locus);
3239 :
3240 2926 : MEM_VOLATILE_P (body) = vol;
3241 :
3242 : /* Non-empty basic ASM implicitly clobbers memory. */
3243 2926 : if (TREE_STRING_LENGTH (string) != 0)
3244 : {
3245 1036 : rtx asm_op, clob;
3246 1036 : unsigned i, nclobbers;
3247 1036 : auto_vec<rtx> input_rvec, output_rvec;
3248 1036 : auto_vec<machine_mode> input_mode;
3249 1036 : auto_vec<const char *> constraints;
3250 1036 : auto_vec<rtx> use_rvec;
3251 1036 : auto_vec<rtx> clobber_rvec;
3252 1036 : HARD_REG_SET clobbered_regs;
3253 1036 : CLEAR_HARD_REG_SET (clobbered_regs);
3254 :
3255 1036 : clob = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3256 1036 : clobber_rvec.safe_push (clob);
3257 :
3258 1036 : if (targetm.md_asm_adjust)
3259 1036 : targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3260 : constraints, use_rvec, clobber_rvec,
3261 : clobbered_regs, locus);
3262 :
3263 1036 : asm_op = body;
3264 1036 : nclobbers = clobber_rvec.length ();
3265 1036 : auto nuses = use_rvec.length ();
3266 1036 : body = gen_rtx_PARALLEL (VOIDmode, rtvec_alloc (1 + nuses + nclobbers));
3267 :
3268 1036 : i = 0;
3269 1036 : XVECEXP (body, 0, i++) = asm_op;
3270 1036 : for (rtx use : use_rvec)
3271 0 : XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
3272 5180 : for (rtx clobber : clobber_rvec)
3273 2072 : XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobber);
3274 1036 : }
3275 :
3276 2926 : emit_insn (body);
3277 2926 : }
3278 :
3279 : /* Check for overlap between registers marked in CLOBBERED_REGS and
3280 : anything inappropriate in T. Emit error and return the register
3281 : variable definition for error, NULL_TREE for ok. */
3282 :
3283 : static bool
3284 133971 : tree_conflicts_with_clobbers_p (tree t, HARD_REG_SET *clobbered_regs,
3285 : location_t loc)
3286 : {
3287 : /* Conflicts between asm-declared register variables and the clobber
3288 : list are not allowed. */
3289 133971 : tree overlap = tree_overlaps_hard_reg_set (t, clobbered_regs);
3290 :
3291 133971 : if (overlap)
3292 : {
3293 21 : error_at (loc, "%<asm%> specifier for variable %qE conflicts with "
3294 21 : "%<asm%> clobber list", DECL_NAME (overlap));
3295 :
3296 : /* Reset registerness to stop multiple errors emitted for a single
3297 : variable. */
3298 21 : DECL_REGISTER (overlap) = 0;
3299 21 : return true;
3300 : }
3301 :
3302 : return false;
3303 : }
3304 :
3305 : /* Check that the given REGNO spanning NREGS is a valid
3306 : asm clobber operand. Some HW registers cannot be
3307 : saved/restored, hence they should not be clobbered by
3308 : asm statements. */
3309 : static bool
3310 41543 : asm_clobber_reg_is_valid (int regno, int nregs, const char *regname)
3311 : {
3312 41543 : bool is_valid = true;
3313 41543 : HARD_REG_SET regset;
3314 :
3315 41543 : CLEAR_HARD_REG_SET (regset);
3316 :
3317 41543 : add_range_to_hard_reg_set (®set, regno, nregs);
3318 :
3319 : /* Clobbering the PIC register is an error. */
3320 41543 : if (PIC_OFFSET_TABLE_REGNUM != INVALID_REGNUM
3321 0 : && overlaps_hard_reg_set_p (regset, Pmode, PIC_OFFSET_TABLE_REGNUM))
3322 : {
3323 : /* ??? Diagnose during gimplification? */
3324 0 : error ("PIC register clobbered by %qs in %<asm%>", regname);
3325 0 : is_valid = false;
3326 : }
3327 41543 : else if (!in_hard_reg_set_p
3328 41543 : (accessible_reg_set, reg_raw_mode[regno], regno))
3329 : {
3330 : /* ??? Diagnose during gimplification? */
3331 0 : error ("the register %qs cannot be clobbered in %<asm%>"
3332 : " for the current target", regname);
3333 0 : is_valid = false;
3334 : }
3335 :
3336 : /* Clobbering the stack pointer register is deprecated. GCC expects
3337 : the value of the stack pointer after an asm statement to be the same
3338 : as it was before, so no asm can validly clobber the stack pointer in
3339 : the usual sense. Adding the stack pointer to the clobber list has
3340 : traditionally had some undocumented and somewhat obscure side-effects. */
3341 41563 : if (overlaps_hard_reg_set_p (regset, Pmode, STACK_POINTER_REGNUM))
3342 : {
3343 1 : crtl->sp_is_clobbered_by_asm = true;
3344 1 : if (warning (OPT_Wdeprecated, "listing the stack pointer register"
3345 : " %qs in a clobber list is deprecated", regname))
3346 1 : inform (input_location, "the value of the stack pointer after"
3347 : " an %<asm%> statement must be the same as it was before"
3348 : " the statement");
3349 : }
3350 :
3351 41543 : return is_valid;
3352 : }
3353 :
3354 : /* Generate RTL for an asm statement with arguments.
3355 : STRING is the instruction template.
3356 : OUTPUTS is a list of output arguments (lvalues); INPUTS a list of inputs.
3357 : Each output or input has an expression in the TREE_VALUE and
3358 : a tree list in TREE_PURPOSE which in turn contains a constraint
3359 : name in TREE_VALUE (or NULL_TREE) and a constraint string
3360 : in TREE_PURPOSE.
3361 : CLOBBERS is a list of STRING_CST nodes each naming a hard register
3362 : that is clobbered by this insn.
3363 :
3364 : LABELS is a list of labels, and if LABELS is non-NULL, FALLTHRU_BB
3365 : should be the fallthru basic block of the asm goto.
3366 :
3367 : Not all kinds of lvalue that may appear in OUTPUTS can be stored directly.
3368 : Some elements of OUTPUTS may be replaced with trees representing temporary
3369 : values. The caller should copy those temporary values to the originally
3370 : specified lvalues.
3371 :
3372 : VOL nonzero means the insn is volatile; don't optimize it. */
3373 :
3374 : static void
3375 110669 : expand_asm_stmt (gasm *stmt)
3376 : {
3377 110669 : class save_input_location
3378 : {
3379 : location_t old;
3380 :
3381 : public:
3382 107743 : explicit save_input_location(location_t where)
3383 107743 : {
3384 107743 : old = input_location;
3385 107743 : input_location = where;
3386 : }
3387 :
3388 107743 : ~save_input_location()
3389 : {
3390 107743 : input_location = old;
3391 5 : }
3392 : };
3393 :
3394 110669 : location_t locus = gimple_location (stmt);
3395 :
3396 110669 : if (gimple_asm_basic_p (stmt))
3397 : {
3398 2926 : const char *s = gimple_asm_string (stmt);
3399 2926 : tree string = build_string (strlen (s), s);
3400 2926 : expand_asm_loc (string, gimple_asm_volatile_p (stmt), locus);
3401 5857 : return;
3402 : }
3403 :
3404 : /* There are some legacy diagnostics in here. */
3405 107743 : save_input_location s_i_l(locus);
3406 :
3407 107743 : unsigned noutputs = gimple_asm_noutputs (stmt);
3408 107743 : unsigned ninputs = gimple_asm_ninputs (stmt);
3409 107743 : unsigned nlabels = gimple_asm_nlabels (stmt);
3410 107743 : unsigned i;
3411 107743 : bool error_seen = false;
3412 :
3413 : /* ??? Diagnose during gimplification? */
3414 107743 : if (ninputs + noutputs + nlabels > MAX_RECOG_OPERANDS)
3415 : {
3416 5 : error_at (locus, "more than %d operands in %<asm%>", MAX_RECOG_OPERANDS);
3417 5 : return;
3418 : }
3419 :
3420 107738 : auto_vec<tree, MAX_RECOG_OPERANDS> output_tvec;
3421 107738 : auto_vec<tree, MAX_RECOG_OPERANDS> input_tvec;
3422 107738 : auto_vec<const char *, MAX_RECOG_OPERANDS> constraints;
3423 :
3424 : /* Copy the gimple vectors into new vectors that we can manipulate. */
3425 :
3426 107738 : output_tvec.safe_grow (noutputs, true);
3427 107738 : input_tvec.safe_grow (ninputs, true);
3428 107738 : constraints.safe_grow (noutputs + ninputs, true);
3429 :
3430 185488 : for (i = 0; i < noutputs; ++i)
3431 : {
3432 77750 : tree t = gimple_asm_output_op (stmt, i);
3433 77750 : output_tvec[i] = TREE_VALUE (t);
3434 77750 : constraints[i] = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3435 : }
3436 163963 : for (i = 0; i < ninputs; i++)
3437 : {
3438 56225 : tree t = gimple_asm_input_op (stmt, i);
3439 56225 : input_tvec[i] = TREE_VALUE (t);
3440 56225 : constraints[i + noutputs]
3441 56225 : = TREE_STRING_POINTER (TREE_VALUE (TREE_PURPOSE (t)));
3442 : }
3443 :
3444 : /* Count the number of meaningful clobbered registers, ignoring what
3445 : we would ignore later. */
3446 107738 : auto_vec<rtx> clobber_rvec;
3447 107738 : HARD_REG_SET clobbered_regs;
3448 107738 : CLEAR_HARD_REG_SET (clobbered_regs);
3449 :
3450 107738 : if (unsigned n = gimple_asm_nclobbers (stmt))
3451 : {
3452 69523 : clobber_rvec.reserve (n);
3453 166603 : for (i = 0; i < n; i++)
3454 : {
3455 97080 : tree t = gimple_asm_clobber_op (stmt, i);
3456 97080 : const char *regname = TREE_STRING_POINTER (TREE_VALUE (t));
3457 97080 : int nregs, j;
3458 :
3459 97080 : j = decode_reg_name_and_count (regname, &nregs);
3460 97080 : if (j < 0)
3461 : {
3462 55537 : if (j == -2)
3463 : {
3464 : /* ??? Diagnose during gimplification? */
3465 15 : error_at (locus, "unknown register name %qs in %<asm%>",
3466 : regname);
3467 15 : error_seen = true;
3468 : }
3469 55522 : else if (j == -4)
3470 : {
3471 39785 : rtx x = gen_rtx_MEM (BLKmode, gen_rtx_SCRATCH (VOIDmode));
3472 39785 : clobber_rvec.safe_push (x);
3473 : }
3474 15737 : else if (j == -5)
3475 : {
3476 2 : if (targetm.redzone_clobber)
3477 2 : if (rtx x = targetm.redzone_clobber ())
3478 2 : clobber_rvec.safe_push (x);
3479 : }
3480 : else
3481 : {
3482 : /* Otherwise we should have -1 == empty string
3483 : or -3 == cc, which is not a register. */
3484 15735 : gcc_assert (j == -1 || j == -3);
3485 : }
3486 : }
3487 : else
3488 83086 : for (int reg = j; reg < j + nregs; reg++)
3489 : {
3490 41543 : if (!asm_clobber_reg_is_valid (reg, nregs, regname))
3491 0 : return;
3492 :
3493 41543 : SET_HARD_REG_BIT (clobbered_regs, reg);
3494 41543 : rtx x = gen_rtx_REG (reg_raw_mode[reg], reg);
3495 41543 : clobber_rvec.safe_push (x);
3496 : }
3497 : }
3498 : }
3499 :
3500 : /* First pass over inputs and outputs checks validity and sets
3501 : mark_addressable if needed. */
3502 : /* ??? Diagnose during gimplification? */
3503 :
3504 185488 : for (i = 0; i < noutputs; ++i)
3505 : {
3506 77750 : tree val = output_tvec[i];
3507 77750 : tree type = TREE_TYPE (val);
3508 77750 : const char *constraint;
3509 77750 : bool is_inout;
3510 77750 : bool allows_reg;
3511 77750 : bool allows_mem;
3512 :
3513 : /* Try to parse the output constraint. If that fails, there's
3514 : no point in going further. */
3515 77750 : constraint = constraints[i];
3516 77750 : if (!parse_output_constraint (&constraint, i, ninputs, noutputs,
3517 : &allows_mem, &allows_reg, &is_inout,
3518 : nullptr))
3519 0 : return;
3520 :
3521 : /* If the output is a hard register, verify it doesn't conflict with
3522 : any other operand's possible hard register use. */
3523 77750 : if (DECL_P (val)
3524 7893 : && REG_P (DECL_RTL (val))
3525 78707 : && HARD_REGISTER_P (DECL_RTL (val)))
3526 : {
3527 938 : unsigned j, output_hregno = REGNO (DECL_RTL (val));
3528 938 : bool early_clobber_p = strchr (constraints[i], '&') != NULL;
3529 938 : unsigned long match;
3530 :
3531 : /* Verify the other outputs do not use the same hard register. */
3532 1098 : for (j = i + 1; j < noutputs; ++j)
3533 160 : if (DECL_P (output_tvec[j])
3534 159 : && REG_P (DECL_RTL (output_tvec[j]))
3535 159 : && HARD_REGISTER_P (DECL_RTL (output_tvec[j]))
3536 319 : && output_hregno == REGNO (DECL_RTL (output_tvec[j])))
3537 : {
3538 1 : error_at (locus, "invalid hard register usage between output "
3539 : "operands");
3540 1 : error_seen = true;
3541 : }
3542 :
3543 : /* Verify matching constraint operands use the same hard register
3544 : and that the non-matching constraint operands do not use the same
3545 : hard register if the output is an early clobber operand. */
3546 1849 : for (j = 0; j < ninputs; ++j)
3547 911 : if (DECL_P (input_tvec[j])
3548 772 : && REG_P (DECL_RTL (input_tvec[j]))
3549 1683 : && HARD_REGISTER_P (DECL_RTL (input_tvec[j])))
3550 : {
3551 772 : unsigned input_hregno = REGNO (DECL_RTL (input_tvec[j]));
3552 772 : switch (*constraints[j + noutputs])
3553 : {
3554 761 : case '0': case '1': case '2': case '3': case '4':
3555 761 : case '5': case '6': case '7': case '8': case '9':
3556 761 : match = strtoul (constraints[j + noutputs], NULL, 10);
3557 761 : break;
3558 : default:
3559 : match = ULONG_MAX;
3560 : break;
3561 : }
3562 761 : if (i == match
3563 661 : && output_hregno != input_hregno)
3564 : {
3565 1 : error_at (locus, "invalid hard register usage between "
3566 : "output operand and matching constraint operand");
3567 1 : error_seen = true;
3568 : }
3569 771 : else if (early_clobber_p
3570 4 : && i != match
3571 2 : && output_hregno == input_hregno)
3572 : {
3573 0 : error_at (locus, "invalid hard register usage between "
3574 : "earlyclobber operand and input operand");
3575 0 : error_seen = true;
3576 : }
3577 : }
3578 : }
3579 :
3580 77750 : if (! allows_reg
3581 77750 : && (allows_mem
3582 0 : || is_inout
3583 0 : || (DECL_P (val)
3584 0 : && REG_P (DECL_RTL (val))
3585 0 : && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type))))
3586 7482 : mark_addressable (val);
3587 : }
3588 :
3589 163963 : for (i = 0; i < ninputs; ++i)
3590 : {
3591 56225 : bool allows_reg, allows_mem;
3592 56225 : const char *constraint;
3593 :
3594 56225 : constraint = constraints[i + noutputs];
3595 56225 : if (! parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3596 56225 : constraints.address (), &allows_mem,
3597 : &allows_reg, nullptr))
3598 0 : return;
3599 :
3600 56225 : if (! allows_reg && allows_mem)
3601 8149 : mark_addressable (input_tvec[i]);
3602 : }
3603 :
3604 : /* Second pass evaluates arguments. */
3605 :
3606 : /* Make sure stack is consistent for asm goto. */
3607 107738 : if (nlabels > 0)
3608 565 : do_pending_stack_adjust ();
3609 107738 : int old_generating_concat_p = generating_concat_p;
3610 :
3611 : /* Vector of RTX's of evaluated output operands. */
3612 215476 : auto_vec<rtx, MAX_RECOG_OPERANDS> output_rvec;
3613 107738 : auto_vec<int, MAX_RECOG_OPERANDS> inout_opnum;
3614 107738 : rtx_insn *after_rtl_seq = NULL, *after_rtl_end = NULL;
3615 :
3616 107738 : output_rvec.safe_grow (noutputs, true);
3617 :
3618 185488 : for (i = 0; i < noutputs; ++i)
3619 : {
3620 77750 : tree val = output_tvec[i];
3621 77750 : tree type = TREE_TYPE (val);
3622 77750 : bool is_inout, allows_reg, allows_mem, ok;
3623 77750 : rtx op;
3624 :
3625 77750 : ok = parse_output_constraint (&constraints[i], i, ninputs,
3626 : noutputs, &allows_mem, &allows_reg,
3627 : &is_inout, nullptr);
3628 77750 : gcc_assert (ok);
3629 :
3630 : /* If an output operand is not a decl or indirect ref and our constraint
3631 : allows a register, make a temporary to act as an intermediate.
3632 : Make the asm insn write into that, then we will copy it to
3633 : the real output operand. Likewise for promoted variables. */
3634 :
3635 77750 : generating_concat_p = 0;
3636 :
3637 77750 : gcc_assert (TREE_CODE (val) != INDIRECT_REF);
3638 77750 : if (((TREE_CODE (val) == MEM_REF
3639 3730 : && TREE_CODE (TREE_OPERAND (val, 0)) != ADDR_EXPR)
3640 3377 : && allows_mem)
3641 76741 : || (DECL_P (val)
3642 7893 : && (allows_mem || REG_P (DECL_RTL (val)))
3643 8138 : && ! (REG_P (DECL_RTL (val))
3644 957 : && GET_MODE (DECL_RTL (val)) != TYPE_MODE (type)))
3645 69560 : || ! allows_reg
3646 69103 : || is_inout
3647 69103 : || TREE_ADDRESSABLE (type)
3648 146853 : || (!tree_fits_poly_int64_p (TYPE_SIZE (type))
3649 0 : && !known_size_p (max_int_size_in_bytes (type))))
3650 : {
3651 8647 : op = expand_expr (val, NULL_RTX, VOIDmode,
3652 : !allows_reg ? EXPAND_MEMORY : EXPAND_WRITE);
3653 8647 : if (MEM_P (op))
3654 7690 : op = validize_mem (op);
3655 :
3656 8647 : if (! allows_reg && !MEM_P (op))
3657 : {
3658 0 : error_at (locus, "output number %d not directly addressable", i);
3659 0 : error_seen = true;
3660 : }
3661 8647 : if ((! allows_mem && MEM_P (op) && GET_MODE (op) != BLKmode)
3662 8647 : || GET_CODE (op) == CONCAT)
3663 : {
3664 0 : rtx old_op = op;
3665 0 : op = gen_reg_rtx (GET_MODE (op));
3666 :
3667 0 : generating_concat_p = old_generating_concat_p;
3668 :
3669 0 : if (is_inout)
3670 0 : emit_move_insn (op, old_op);
3671 :
3672 0 : push_to_sequence2 (after_rtl_seq, after_rtl_end);
3673 0 : emit_move_insn (old_op, op);
3674 0 : after_rtl_seq = get_insns ();
3675 0 : after_rtl_end = get_last_insn ();
3676 0 : end_sequence ();
3677 : }
3678 : }
3679 : else
3680 : {
3681 69103 : op = assign_temp (type, 0, 1);
3682 69103 : op = validize_mem (op);
3683 69103 : if (!MEM_P (op) && TREE_CODE (val) == SSA_NAME)
3684 65274 : set_reg_attrs_for_decl_rtl (SSA_NAME_VAR (val), op);
3685 :
3686 69103 : generating_concat_p = old_generating_concat_p;
3687 :
3688 69103 : push_to_sequence2 (after_rtl_seq, after_rtl_end);
3689 69103 : expand_assignment (val, make_tree (type, op), false);
3690 69103 : after_rtl_seq = get_insns ();
3691 69103 : after_rtl_end = get_last_insn ();
3692 69103 : end_sequence ();
3693 : }
3694 77750 : output_rvec[i] = op;
3695 :
3696 77750 : if (is_inout)
3697 0 : inout_opnum.safe_push (i);
3698 : }
3699 :
3700 107738 : const char *str = gimple_asm_string (stmt);
3701 107738 : if (error_seen)
3702 : {
3703 17 : ninputs = 0;
3704 17 : noutputs = 0;
3705 17 : inout_opnum.truncate (0);
3706 17 : output_rvec.truncate (0);
3707 17 : clobber_rvec.truncate (0);
3708 17 : constraints.truncate (0);
3709 17 : CLEAR_HARD_REG_SET (clobbered_regs);
3710 : str = "";
3711 : }
3712 :
3713 215476 : auto_vec<rtx, MAX_RECOG_OPERANDS> input_rvec;
3714 107738 : auto_vec<machine_mode, MAX_RECOG_OPERANDS> input_mode;
3715 :
3716 107738 : input_rvec.safe_grow (ninputs, true);
3717 107738 : input_mode.safe_grow (ninputs, true);
3718 :
3719 107738 : generating_concat_p = 0;
3720 :
3721 163962 : for (i = 0; i < ninputs; ++i)
3722 : {
3723 56224 : tree val = input_tvec[i];
3724 56224 : tree type = TREE_TYPE (val);
3725 56224 : bool allows_reg, allows_mem, ok;
3726 56224 : const char *constraint;
3727 56224 : rtx op;
3728 :
3729 56224 : constraint = constraints[i + noutputs];
3730 56224 : ok = parse_input_constraint (&constraint, i, ninputs, noutputs, 0,
3731 56224 : constraints.address (),
3732 : &allows_mem, &allows_reg, nullptr);
3733 56224 : gcc_assert (ok);
3734 :
3735 : /* EXPAND_INITIALIZER will not generate code for valid initializer
3736 : constants, but will still generate code for other types of operand.
3737 : This is the behavior we want for constant constraints. */
3738 56224 : op = expand_expr (val, NULL_RTX, VOIDmode,
3739 : allows_reg ? EXPAND_NORMAL
3740 9883 : : allows_mem ? EXPAND_MEMORY
3741 : : EXPAND_INITIALIZER);
3742 :
3743 : /* Never pass a CONCAT to an ASM. */
3744 56224 : if (GET_CODE (op) == CONCAT)
3745 2 : op = force_reg (GET_MODE (op), op);
3746 56222 : else if (MEM_P (op))
3747 13550 : op = validize_mem (op);
3748 :
3749 56224 : if (asm_operand_ok (op, constraint, NULL) <= 0)
3750 : {
3751 25645 : if (allows_reg && TYPE_MODE (type) != BLKmode)
3752 25624 : op = force_reg (TYPE_MODE (type), op);
3753 21 : else if (!allows_mem)
3754 20 : warning_at (locus, 0, "%<asm%> operand %d probably does not match "
3755 : "constraints", i + noutputs);
3756 1 : else if (MEM_P (op))
3757 : {
3758 : /* We won't recognize either volatile memory or memory
3759 : with a queued address as available a memory_operand
3760 : at this point. Ignore it: clearly this *is* a memory. */
3761 : }
3762 : else
3763 0 : gcc_unreachable ();
3764 : }
3765 56224 : input_rvec[i] = op;
3766 56224 : input_mode[i] = TYPE_MODE (type);
3767 : }
3768 :
3769 : /* For in-out operands, copy output rtx to input rtx. */
3770 107738 : unsigned ninout = inout_opnum.length ();
3771 107738 : for (i = 0; i < ninout; i++)
3772 : {
3773 0 : int j = inout_opnum[i];
3774 0 : rtx o = output_rvec[j];
3775 :
3776 0 : input_rvec.safe_push (o);
3777 0 : input_mode.safe_push (GET_MODE (o));
3778 :
3779 0 : char buffer[16];
3780 0 : sprintf (buffer, "%d", j);
3781 0 : constraints.safe_push (ggc_strdup (buffer));
3782 : }
3783 107738 : ninputs += ninout;
3784 :
3785 : /* Sometimes we wish to automatically clobber registers across an asm.
3786 : Case in point is when the i386 backend moved from cc0 to a hard reg --
3787 : maintaining source-level compatibility means automatically clobbering
3788 : the flags register. */
3789 215476 : auto_vec<rtx> use_rvec;
3790 107738 : if (targetm.md_asm_adjust)
3791 : {
3792 107738 : rtx_insn *after_md_seq
3793 107738 : = targetm.md_asm_adjust (output_rvec, input_rvec, input_mode,
3794 : constraints, use_rvec, clobber_rvec,
3795 : clobbered_regs, locus);
3796 107738 : if (after_md_seq)
3797 : {
3798 72 : push_to_sequence (after_md_seq);
3799 72 : emit_insn (after_rtl_seq);
3800 72 : after_rtl_seq = get_insns ();
3801 72 : after_rtl_end = get_last_insn ();
3802 72 : end_sequence ();
3803 : }
3804 : }
3805 :
3806 : /* Do not allow the hook to change the output and input count,
3807 : lest it mess up the operand numbering. */
3808 215476 : gcc_assert (output_rvec.length() == noutputs);
3809 215476 : gcc_assert (input_rvec.length() == ninputs);
3810 215476 : gcc_assert (constraints.length() == noutputs + ninputs);
3811 :
3812 : /* But it certainly can adjust the uses and clobbers. */
3813 107738 : unsigned nuses = use_rvec.length ();
3814 107738 : unsigned nclobbers = clobber_rvec.length ();
3815 :
3816 : /* Third pass checks for easy conflicts. */
3817 : /* ??? Why are we doing this on trees instead of rtx. */
3818 :
3819 107738 : bool clobber_conflict_found = 0;
3820 185485 : for (i = 0; i < noutputs; ++i)
3821 77747 : if (tree_conflicts_with_clobbers_p (output_tvec[i], &clobbered_regs, locus))
3822 10 : clobber_conflict_found = 1;
3823 163962 : for (i = 0; i < ninputs - ninout; ++i)
3824 56224 : if (tree_conflicts_with_clobbers_p (input_tvec[i], &clobbered_regs, locus))
3825 11 : clobber_conflict_found = 1;
3826 :
3827 : /* Make vectors for the expression-rtx, constraint strings,
3828 : and named operands. */
3829 :
3830 107738 : rtvec argvec = rtvec_alloc (ninputs);
3831 107738 : rtvec constraintvec = rtvec_alloc (ninputs);
3832 107738 : rtvec labelvec = rtvec_alloc (nlabels);
3833 :
3834 142941 : rtx body = gen_rtx_ASM_OPERANDS ((noutputs == 0 ? VOIDmode
3835 : : GET_MODE (output_rvec[0])),
3836 : ggc_strdup (str),
3837 : "", 0, argvec, constraintvec,
3838 : labelvec, locus);
3839 107738 : MEM_VOLATILE_P (body) = gimple_asm_volatile_p (stmt);
3840 :
3841 163962 : for (i = 0; i < ninputs; ++i)
3842 : {
3843 56224 : ASM_OPERANDS_INPUT (body, i) = input_rvec[i];
3844 112448 : ASM_OPERANDS_INPUT_CONSTRAINT_EXP (body, i)
3845 56224 : = gen_rtx_ASM_INPUT_loc (input_mode[i],
3846 : constraints[i + noutputs],
3847 : locus);
3848 : }
3849 :
3850 : /* Copy labels to the vector. */
3851 107738 : rtx_code_label *fallthru_label = NULL;
3852 107738 : if (nlabels > 0)
3853 : {
3854 565 : basic_block fallthru_bb = NULL;
3855 565 : edge fallthru = find_fallthru_edge (gimple_bb (stmt)->succs);
3856 565 : if (fallthru)
3857 565 : fallthru_bb = fallthru->dest;
3858 :
3859 1389 : for (i = 0; i < nlabels; ++i)
3860 : {
3861 824 : tree label = TREE_VALUE (gimple_asm_label_op (stmt, i));
3862 824 : rtx_insn *r;
3863 : /* If asm goto has any labels in the fallthru basic block, use
3864 : a label that we emit immediately after the asm goto. Expansion
3865 : may insert further instructions into the same basic block after
3866 : asm goto and if we don't do this, insertion of instructions on
3867 : the fallthru edge might misbehave. See PR58670. */
3868 824 : if (fallthru_bb && label_to_block (cfun, label) == fallthru_bb)
3869 : {
3870 283 : if (fallthru_label == NULL_RTX)
3871 278 : fallthru_label = gen_label_rtx ();
3872 : r = fallthru_label;
3873 : }
3874 : else
3875 541 : r = label_rtx (label);
3876 824 : ASM_OPERANDS_LABEL (body, i) = gen_rtx_LABEL_REF (Pmode, r);
3877 : }
3878 : }
3879 :
3880 : /* Now, for each output, construct an rtx
3881 : (set OUTPUT (asm_operands INSN OUTPUTCONSTRAINT OUTPUTNUMBER
3882 : ARGVEC CONSTRAINTS OPNAMES))
3883 : If there is more than one, put them inside a PARALLEL. */
3884 :
3885 107738 : if (noutputs == 0 && nuses == 0 && nclobbers == 0)
3886 : {
3887 : /* No output operands: put in a raw ASM_OPERANDS rtx. */
3888 0 : if (nlabels > 0)
3889 0 : emit_jump_insn (body);
3890 : else
3891 0 : emit_insn (body);
3892 : }
3893 107738 : else if (noutputs == 1 && nuses == 0 && nclobbers == 0)
3894 : {
3895 62 : ASM_OPERANDS_OUTPUT_CONSTRAINT (body) = constraints[0];
3896 62 : if (nlabels > 0)
3897 0 : emit_jump_insn (gen_rtx_SET (output_rvec[0], body));
3898 : else
3899 62 : emit_insn (gen_rtx_SET (output_rvec[0], body));
3900 : }
3901 : else
3902 : {
3903 107676 : rtx obody = body;
3904 107676 : int num = noutputs;
3905 :
3906 107676 : if (num == 0)
3907 72535 : num = 1;
3908 :
3909 107676 : body = gen_rtx_PARALLEL (VOIDmode,
3910 : rtvec_alloc (num + nuses + nclobbers));
3911 :
3912 : /* For each output operand, store a SET. */
3913 185361 : for (i = 0; i < noutputs; ++i)
3914 : {
3915 77685 : rtx src, o = output_rvec[i];
3916 77685 : if (i == 0)
3917 : {
3918 35141 : ASM_OPERANDS_OUTPUT_CONSTRAINT (obody) = constraints[0];
3919 35141 : src = obody;
3920 : }
3921 : else
3922 : {
3923 42544 : src = gen_rtx_ASM_OPERANDS (GET_MODE (o),
3924 : ASM_OPERANDS_TEMPLATE (obody),
3925 : constraints[i], i, argvec,
3926 : constraintvec, labelvec, locus);
3927 42544 : MEM_VOLATILE_P (src) = gimple_asm_volatile_p (stmt);
3928 : }
3929 77685 : XVECEXP (body, 0, i) = gen_rtx_SET (o, src);
3930 : }
3931 :
3932 : /* If there are no outputs (but there are some clobbers)
3933 : store the bare ASM_OPERANDS into the PARALLEL. */
3934 107676 : if (i == 0)
3935 72535 : XVECEXP (body, 0, i++) = obody;
3936 :
3937 : /* Add the uses specified by the target hook. No checking should
3938 : be needed since this doesn't come directly from user code. */
3939 107676 : for (rtx use : use_rvec)
3940 0 : XVECEXP (body, 0, i++) = gen_rtx_USE (VOIDmode, use);
3941 :
3942 : /* Store (clobber REG) for each clobbered register specified. */
3943 296669 : for (unsigned j = 0; j < nclobbers; ++j)
3944 : {
3945 188993 : rtx clobbered_reg = clobber_rvec[j];
3946 :
3947 : /* Do sanity check for overlap between clobbers and respectively
3948 : input and outputs that hasn't been handled. Such overlap
3949 : should have been detected and reported above. */
3950 188993 : if (!clobber_conflict_found && REG_P (clobbered_reg))
3951 : {
3952 : /* We test the old body (obody) contents to avoid
3953 : tripping over the under-construction body. */
3954 227808 : for (unsigned k = 0; k < noutputs; ++k)
3955 78632 : if (reg_overlap_mentioned_p (clobbered_reg, output_rvec[k]))
3956 0 : internal_error ("%<asm%> clobber conflict with "
3957 : "output operand");
3958 :
3959 208141 : for (unsigned k = 0; k < ninputs - ninout; ++k)
3960 58965 : if (reg_overlap_mentioned_p (clobbered_reg, input_rvec[k]))
3961 0 : internal_error ("%<asm%> clobber conflict with "
3962 : "input operand");
3963 : }
3964 :
3965 188993 : XVECEXP (body, 0, i++) = gen_rtx_CLOBBER (VOIDmode, clobbered_reg);
3966 : }
3967 :
3968 107676 : if (nlabels > 0)
3969 565 : emit_jump_insn (body);
3970 : else
3971 107111 : emit_insn (body);
3972 : }
3973 :
3974 107738 : generating_concat_p = old_generating_concat_p;
3975 :
3976 107738 : if (fallthru_label)
3977 278 : emit_label (fallthru_label);
3978 :
3979 107738 : if (after_rtl_seq)
3980 : {
3981 28121 : if (nlabels == 0)
3982 27989 : emit_insn (after_rtl_seq);
3983 : else
3984 : {
3985 132 : edge e;
3986 132 : edge_iterator ei;
3987 132 : unsigned int cnt = EDGE_COUNT (gimple_bb (stmt)->succs);
3988 :
3989 473 : FOR_EACH_EDGE (e, ei, gimple_bb (stmt)->succs)
3990 : {
3991 341 : rtx_insn *copy;
3992 341 : if (--cnt == 0)
3993 : copy = after_rtl_seq;
3994 : else
3995 : {
3996 209 : start_sequence ();
3997 209 : duplicate_insn_chain (after_rtl_seq, after_rtl_end,
3998 : NULL, NULL);
3999 209 : copy = end_sequence ();
4000 : }
4001 341 : prepend_insn_to_edge (copy, e);
4002 : }
4003 : }
4004 : }
4005 :
4006 107738 : free_temp_slots ();
4007 107738 : crtl->has_asm_statement = 1;
4008 107743 : }
4009 :
4010 : /* Emit code to jump to the address
4011 : specified by the pointer expression EXP. */
4012 :
4013 : static void
4014 413 : expand_computed_goto (tree exp)
4015 : {
4016 413 : rtx x = expand_normal (exp);
4017 :
4018 413 : do_pending_stack_adjust ();
4019 413 : emit_indirect_jump (x);
4020 413 : }
4021 :
4022 : /* Generate RTL code for a `goto' statement with target label LABEL.
4023 : LABEL should be a LABEL_DECL tree node that was or will later be
4024 : defined with `expand_label'. */
4025 :
4026 : static void
4027 0 : expand_goto (tree label)
4028 : {
4029 0 : if (flag_checking)
4030 : {
4031 : /* Check for a nonlocal goto to a containing function. Should have
4032 : gotten translated to __builtin_nonlocal_goto. */
4033 0 : tree context = decl_function_context (label);
4034 0 : gcc_assert (!context || context == current_function_decl);
4035 : }
4036 :
4037 0 : emit_jump (jump_target_rtx (label));
4038 0 : }
4039 :
4040 : /* Output a return with no value. */
4041 :
4042 : static void
4043 828887 : expand_null_return_1 (void)
4044 : {
4045 828887 : clear_pending_stack_adjust ();
4046 828887 : do_pending_stack_adjust ();
4047 828887 : emit_jump (return_label);
4048 828887 : }
4049 :
4050 : /* Generate RTL to return from the current function, with no value.
4051 : (That is, we do not do anything about returning any value.) */
4052 :
4053 : void
4054 68858 : expand_null_return (void)
4055 : {
4056 : /* If this function was declared to return a value, but we
4057 : didn't, clobber the return registers so that they are not
4058 : propagated live to the rest of the function. */
4059 68858 : clobber_return_register ();
4060 :
4061 68858 : expand_null_return_1 ();
4062 68858 : }
4063 :
4064 : /* Generate RTL to return from the current function, with value VAL. */
4065 :
4066 : static void
4067 760029 : expand_value_return (rtx val)
4068 : {
4069 : /* Copy the value to the return location unless it's already there. */
4070 :
4071 760029 : tree decl = DECL_RESULT (current_function_decl);
4072 760029 : rtx return_reg = DECL_RTL (decl);
4073 760029 : if (return_reg != val)
4074 : {
4075 439489 : tree funtype = TREE_TYPE (current_function_decl);
4076 439489 : tree type = TREE_TYPE (decl);
4077 439489 : int unsignedp = TYPE_UNSIGNED (type);
4078 439489 : machine_mode old_mode = DECL_MODE (decl);
4079 439489 : machine_mode mode;
4080 439489 : if (DECL_BY_REFERENCE (decl))
4081 0 : mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 2);
4082 : else
4083 439489 : mode = promote_function_mode (type, old_mode, &unsignedp, funtype, 1);
4084 :
4085 439489 : if (mode != old_mode)
4086 : {
4087 : /* Some ABIs require scalar floating point modes to be returned
4088 : in a wider scalar integer mode. We need to explicitly
4089 : reinterpret to an integer mode of the correct precision
4090 : before extending to the desired result. */
4091 8 : if (SCALAR_INT_MODE_P (mode)
4092 8 : && SCALAR_FLOAT_MODE_P (old_mode)
4093 8 : && known_gt (GET_MODE_SIZE (mode), GET_MODE_SIZE (old_mode)))
4094 0 : val = convert_float_to_wider_int (mode, old_mode, val);
4095 : else
4096 8 : val = convert_modes (mode, old_mode, val, unsignedp);
4097 : }
4098 :
4099 439489 : if (GET_CODE (return_reg) == PARALLEL)
4100 3038 : emit_group_load (return_reg, val, type, int_size_in_bytes (type));
4101 : else
4102 436451 : emit_move_insn (return_reg, val);
4103 : }
4104 :
4105 760029 : expand_null_return_1 ();
4106 760029 : }
4107 :
4108 : /* Generate RTL to evaluate the expression RETVAL and return it
4109 : from the current function. */
4110 :
4111 : static void
4112 762273 : expand_return (tree retval)
4113 : {
4114 762273 : rtx result_rtl;
4115 762273 : rtx val = 0;
4116 762273 : tree retval_rhs;
4117 :
4118 : /* If function wants no value, give it none. */
4119 762273 : if (VOID_TYPE_P (TREE_TYPE (TREE_TYPE (current_function_decl))))
4120 : {
4121 0 : expand_normal (retval);
4122 0 : expand_null_return ();
4123 0 : return;
4124 : }
4125 :
4126 762273 : if (retval == error_mark_node)
4127 : {
4128 : /* Treat this like a return of no value from a function that
4129 : returns a value. */
4130 0 : expand_null_return ();
4131 0 : return;
4132 : }
4133 762273 : else if ((TREE_CODE (retval) == MODIFY_EXPR
4134 762273 : || TREE_CODE (retval) == INIT_EXPR)
4135 762273 : && TREE_CODE (TREE_OPERAND (retval, 0)) == RESULT_DECL)
4136 699362 : retval_rhs = TREE_OPERAND (retval, 1);
4137 : else
4138 : retval_rhs = retval;
4139 :
4140 762273 : result_rtl = DECL_RTL (DECL_RESULT (current_function_decl));
4141 :
4142 : /* If we are returning the RESULT_DECL, then the value has already
4143 : been stored into it, so we don't have to do anything special. */
4144 762273 : if (TREE_CODE (retval_rhs) == RESULT_DECL)
4145 62911 : expand_value_return (result_rtl);
4146 :
4147 : /* If the result is an aggregate that is being returned in one (or more)
4148 : registers, load the registers here. */
4149 :
4150 699362 : else if (retval_rhs != 0
4151 699362 : && TYPE_MODE (TREE_TYPE (retval_rhs)) == BLKmode
4152 4951 : && REG_P (result_rtl))
4153 : {
4154 3468 : val = copy_blkmode_to_reg (GET_MODE (result_rtl), retval_rhs);
4155 3468 : if (val)
4156 : {
4157 : /* Use the mode of the result value on the return register. */
4158 1224 : PUT_MODE (result_rtl, GET_MODE (val));
4159 1224 : expand_value_return (val);
4160 : }
4161 : else
4162 2244 : expand_null_return ();
4163 : }
4164 695894 : else if (retval_rhs != 0
4165 695894 : && !VOID_TYPE_P (TREE_TYPE (retval_rhs))
4166 695894 : && (REG_P (result_rtl)
4167 6080 : || (GET_CODE (result_rtl) == PARALLEL)))
4168 : {
4169 : /* Compute the return value into a temporary (usually a pseudo reg). */
4170 692852 : val
4171 692852 : = assign_temp (TREE_TYPE (DECL_RESULT (current_function_decl)), 0, 1);
4172 692852 : val = expand_expr (retval_rhs, val, GET_MODE (val), EXPAND_NORMAL);
4173 692852 : val = force_not_mem (val);
4174 692852 : expand_value_return (val);
4175 : }
4176 : else
4177 : {
4178 : /* No hard reg used; calculate value into hard return reg. */
4179 3042 : expand_expr (retval, const0_rtx, VOIDmode, EXPAND_NORMAL);
4180 3042 : expand_value_return (result_rtl);
4181 : }
4182 : }
4183 :
4184 : /* Expand a clobber of LHS. If LHS is stored it in a multi-part
4185 : register, tell the rtl optimizers that its value is no longer
4186 : needed. */
4187 :
4188 : static void
4189 1301104 : expand_clobber (tree lhs)
4190 : {
4191 1301104 : if (DECL_P (lhs))
4192 : {
4193 1276101 : rtx decl_rtl = DECL_RTL_IF_SET (lhs);
4194 1276010 : if (decl_rtl && REG_P (decl_rtl))
4195 : {
4196 173414 : machine_mode decl_mode = GET_MODE (decl_rtl);
4197 346828 : if (maybe_gt (GET_MODE_SIZE (decl_mode),
4198 : REGMODE_NATURAL_SIZE (decl_mode)))
4199 138540 : emit_clobber (decl_rtl);
4200 : }
4201 : }
4202 1301104 : }
4203 :
4204 : /* A subroutine of expand_gimple_stmt, expanding one gimple statement
4205 : STMT that doesn't require special handling for outgoing edges. That
4206 : is no tailcalls and no GIMPLE_COND. */
4207 :
4208 : static void
4209 31442219 : expand_gimple_stmt_1 (gimple *stmt)
4210 : {
4211 31442219 : tree op0;
4212 :
4213 31442219 : set_curr_insn_location (gimple_location (stmt));
4214 :
4215 31442219 : switch (gimple_code (stmt))
4216 : {
4217 413 : case GIMPLE_GOTO:
4218 413 : op0 = gimple_goto_dest (stmt);
4219 413 : if (TREE_CODE (op0) == LABEL_DECL)
4220 0 : expand_goto (op0);
4221 : else
4222 413 : expand_computed_goto (op0);
4223 : break;
4224 631931 : case GIMPLE_LABEL:
4225 631931 : expand_label (gimple_label_label (as_a <glabel *> (stmt)));
4226 631931 : break;
4227 : case GIMPLE_NOP:
4228 : case GIMPLE_PREDICT:
4229 : break;
4230 7133 : case GIMPLE_SWITCH:
4231 7133 : {
4232 7133 : gswitch *swtch = as_a <gswitch *> (stmt);
4233 7133 : if (gimple_switch_num_labels (swtch) == 1)
4234 0 : expand_goto (CASE_LABEL (gimple_switch_default_label (swtch)));
4235 : else
4236 7133 : expand_case (swtch);
4237 : }
4238 : break;
4239 110669 : case GIMPLE_ASM:
4240 110669 : expand_asm_stmt (as_a <gasm *> (stmt));
4241 110669 : break;
4242 6825106 : case GIMPLE_CALL:
4243 6825106 : expand_call_stmt (as_a <gcall *> (stmt));
4244 6825106 : break;
4245 :
4246 828887 : case GIMPLE_RETURN:
4247 828887 : {
4248 828887 : op0 = gimple_return_retval (as_a <greturn *> (stmt));
4249 :
4250 : /* If a return doesn't have a location, it very likely represents
4251 : multiple user returns so we cannot let it inherit the location
4252 : of the last statement of the previous basic block in RTL. */
4253 828887 : if (!gimple_has_location (stmt))
4254 217067 : set_curr_insn_location (cfun->function_end_locus);
4255 :
4256 828887 : if (op0 && op0 != error_mark_node)
4257 : {
4258 762273 : tree result = DECL_RESULT (current_function_decl);
4259 :
4260 : /* If we are not returning the current function's RESULT_DECL,
4261 : build an assignment to it. */
4262 762273 : if (op0 != result)
4263 : {
4264 : /* I believe that a function's RESULT_DECL is unique. */
4265 699362 : gcc_assert (TREE_CODE (op0) != RESULT_DECL);
4266 :
4267 : /* ??? We'd like to use simply expand_assignment here,
4268 : but this fails if the value is of BLKmode but the return
4269 : decl is a register. expand_return has special handling
4270 : for this combination, which eventually should move
4271 : to common code. See comments there. Until then, let's
4272 : build a modify expression :-/ */
4273 699362 : op0 = build2 (MODIFY_EXPR, TREE_TYPE (result),
4274 : result, op0);
4275 : }
4276 : }
4277 :
4278 762273 : if (!op0)
4279 66614 : expand_null_return ();
4280 : else
4281 762273 : expand_return (op0);
4282 : }
4283 : break;
4284 :
4285 22991215 : case GIMPLE_ASSIGN:
4286 22991215 : {
4287 22991215 : gassign *assign_stmt = as_a <gassign *> (stmt);
4288 22991215 : tree lhs = gimple_assign_lhs (assign_stmt);
4289 :
4290 : /* Tree expand used to fiddle with |= and &= of two bitfield
4291 : COMPONENT_REFs here. This can't happen with gimple, the LHS
4292 : of binary assigns must be a gimple reg. */
4293 :
4294 22991215 : if (TREE_CODE (lhs) != SSA_NAME
4295 36628561 : || gimple_assign_rhs_class (assign_stmt) == GIMPLE_SINGLE_RHS)
4296 : {
4297 17114484 : tree rhs = gimple_assign_rhs1 (assign_stmt);
4298 17114484 : gcc_assert (gimple_assign_rhs_class (assign_stmt)
4299 : == GIMPLE_SINGLE_RHS);
4300 32690613 : if (gimple_has_location (stmt) && CAN_HAVE_LOCATION_P (rhs)
4301 : /* Do not put locations on possibly shared trees. */
4302 23066428 : && !is_gimple_min_invariant (rhs))
4303 4830777 : SET_EXPR_LOCATION (rhs, gimple_location (stmt));
4304 17114484 : if (TREE_CLOBBER_P (rhs))
4305 : /* This is a clobber to mark the going out of scope for
4306 : this LHS. */
4307 1301104 : expand_clobber (lhs);
4308 : else
4309 15813380 : expand_assignment (lhs, rhs,
4310 15813380 : gimple_assign_nontemporal_move_p (
4311 : assign_stmt));
4312 : }
4313 : else
4314 : {
4315 5876731 : rtx target, temp;
4316 5876731 : gcc_assert (!gimple_assign_nontemporal_move_p (assign_stmt));
4317 5876731 : bool promoted = false;
4318 :
4319 5876731 : target = expand_expr (lhs, NULL_RTX, VOIDmode, EXPAND_WRITE);
4320 5876731 : if (GET_CODE (target) == SUBREG && SUBREG_PROMOTED_VAR_P (target))
4321 : promoted = true;
4322 :
4323 : /* If we store into a promoted register, don't directly
4324 : expand to target. */
4325 : temp = promoted ? NULL_RTX : target;
4326 11753462 : temp = expand_expr_real_gassign (assign_stmt, temp,
4327 5876731 : GET_MODE (target), EXPAND_NORMAL);
4328 :
4329 5876731 : if (temp == target)
4330 : ;
4331 818302 : else if (promoted)
4332 : {
4333 3 : int unsignedp = SUBREG_PROMOTED_SIGN (target);
4334 : /* If TEMP is a VOIDmode constant, use convert_modes to make
4335 : sure that we properly convert it. */
4336 3 : if (CONSTANT_P (temp) && GET_MODE (temp) == VOIDmode)
4337 : {
4338 0 : temp = convert_modes (GET_MODE (target),
4339 0 : TYPE_MODE (TREE_TYPE (lhs)),
4340 : temp, unsignedp);
4341 0 : temp = convert_modes (GET_MODE (SUBREG_REG (target)),
4342 0 : GET_MODE (target), temp, unsignedp);
4343 : }
4344 :
4345 3 : convert_move (SUBREG_REG (target), temp, unsignedp);
4346 : }
4347 : else
4348 : {
4349 818299 : temp = force_operand (temp, target);
4350 818299 : if (temp == target)
4351 : ;
4352 818299 : else if (GET_MODE (target) != BLKmode)
4353 818298 : emit_move_insn (target, temp);
4354 : else
4355 1 : emit_block_move (target, temp, expr_size (lhs),
4356 : BLOCK_OP_NORMAL);
4357 : }
4358 : }
4359 : }
4360 : break;
4361 :
4362 0 : default:
4363 0 : gcc_unreachable ();
4364 : }
4365 31442217 : }
4366 :
4367 : /* Expand one gimple statement STMT and return the last RTL instruction
4368 : before any of the newly generated ones.
4369 :
4370 : In addition to generating the necessary RTL instructions this also
4371 : sets REG_EH_REGION notes if necessary and sets the current source
4372 : location for diagnostics. */
4373 :
4374 : static rtx_insn *
4375 31442219 : expand_gimple_stmt (gimple *stmt)
4376 : {
4377 31442219 : location_t saved_location = input_location;
4378 31442219 : rtx_insn *last = get_last_insn ();
4379 31442219 : int lp_nr;
4380 :
4381 31442219 : gcc_assert (cfun);
4382 :
4383 : /* We need to save and restore the current source location so that errors
4384 : discovered during expansion are emitted with the right location. But
4385 : it would be better if the diagnostic routines used the source location
4386 : embedded in the tree nodes rather than globals. */
4387 31442219 : if (gimple_has_location (stmt))
4388 27605272 : input_location = gimple_location (stmt);
4389 :
4390 31442219 : expand_gimple_stmt_1 (stmt);
4391 :
4392 : /* Free any temporaries used to evaluate this statement. */
4393 31442217 : free_temp_slots ();
4394 :
4395 31442217 : input_location = saved_location;
4396 :
4397 : /* Mark all insns that may trap. */
4398 31442217 : lp_nr = lookup_stmt_eh_lp (stmt);
4399 31442217 : if (lp_nr)
4400 : {
4401 686556 : rtx_insn *insn;
4402 3923691 : for (insn = next_real_insn (last); insn;
4403 3237135 : insn = next_real_insn (insn))
4404 : {
4405 3237135 : if (! find_reg_note (insn, REG_EH_REGION, NULL_RTX)
4406 : /* If we want exceptions for non-call insns, any
4407 : may_trap_p instruction may throw. */
4408 3237075 : && GET_CODE (PATTERN (insn)) != CLOBBER
4409 3237005 : && GET_CODE (PATTERN (insn)) != USE
4410 6474140 : && insn_could_throw_p (insn))
4411 743868 : make_reg_eh_region_note (insn, 0, lp_nr);
4412 : }
4413 : }
4414 :
4415 31442217 : return last;
4416 : }
4417 :
4418 : /* A subroutine of expand_gimple_basic_block. Expand one GIMPLE_CALL
4419 : that has CALL_EXPR_TAILCALL set. Returns non-null if we actually
4420 : generated a tail call (something that might be denied by the ABI
4421 : rules governing the call; see calls.cc).
4422 :
4423 : Sets CAN_FALLTHRU if we generated a *conditional* tail call, and
4424 : can still reach the rest of BB. The case here is __builtin_sqrt,
4425 : where the NaN result goes through the external function (with a
4426 : tailcall) and the normal result happens via a sqrt instruction. */
4427 :
4428 : static basic_block
4429 175542 : expand_gimple_tailcall (basic_block bb, gcall *stmt, bool *can_fallthru,
4430 : rtx_insn *asan_epilog_seq)
4431 : {
4432 175542 : rtx_insn *last2, *last, *first = get_last_insn ();
4433 175542 : edge e;
4434 175542 : edge_iterator ei;
4435 175542 : profile_probability probability;
4436 :
4437 175542 : last2 = last = expand_gimple_stmt (stmt);
4438 :
4439 745397 : for (last = NEXT_INSN (last); last; last = NEXT_INSN (last))
4440 695557 : if (CALL_P (last) && SIBLING_CALL_P (last))
4441 125702 : goto found;
4442 :
4443 49840 : maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4444 :
4445 49840 : *can_fallthru = true;
4446 49840 : return NULL;
4447 :
4448 125702 : found:
4449 :
4450 125702 : if (asan_epilog_seq)
4451 : {
4452 : /* We need to emit a copy of the asan_epilog_seq before
4453 : the insns emitted by expand_gimple_stmt above. The sequence
4454 : can contain labels, which need to be remapped. */
4455 117 : hash_map<rtx, rtx> label_map;
4456 117 : start_sequence ();
4457 117 : emit_note (NOTE_INSN_DELETED);
4458 1650 : for (rtx_insn *insn = asan_epilog_seq; insn; insn = NEXT_INSN (insn))
4459 1533 : switch (GET_CODE (insn))
4460 : {
4461 1182 : case INSN:
4462 1182 : case CALL_INSN:
4463 1182 : case JUMP_INSN:
4464 1182 : emit_copy_of_insn_after (insn, get_last_insn ());
4465 1182 : break;
4466 234 : case CODE_LABEL:
4467 234 : label_map.put ((rtx) insn, (rtx) emit_label (gen_label_rtx ()));
4468 234 : break;
4469 117 : case BARRIER:
4470 117 : emit_barrier ();
4471 117 : break;
4472 0 : default:
4473 0 : gcc_unreachable ();
4474 : }
4475 1767 : for (rtx_insn *insn = get_insns (); insn; insn = NEXT_INSN (insn))
4476 1650 : if (JUMP_P (insn))
4477 : {
4478 234 : subrtx_ptr_iterator::array_type array;
4479 1521 : FOR_EACH_SUBRTX_PTR (iter, array, &PATTERN (insn), ALL)
4480 : {
4481 1287 : rtx *loc = *iter;
4482 1287 : if (LABEL_REF_P (*loc))
4483 : {
4484 234 : rtx *lab = label_map.get ((rtx) label_ref_label (*loc));
4485 234 : gcc_assert (lab);
4486 234 : set_label_ref_label (*loc, as_a <rtx_insn *> (*lab));
4487 : }
4488 : }
4489 234 : if (JUMP_LABEL (insn))
4490 : {
4491 234 : rtx *lab = label_map.get (JUMP_LABEL (insn));
4492 234 : gcc_assert (lab);
4493 234 : JUMP_LABEL (insn) = *lab;
4494 : }
4495 234 : }
4496 117 : asan_epilog_seq = NEXT_INSN (get_insns ());
4497 117 : end_sequence ();
4498 117 : emit_insn_before (asan_epilog_seq, NEXT_INSN (first));
4499 117 : }
4500 :
4501 : /* ??? Wouldn't it be better to just reset any pending stack adjust?
4502 : Any instructions emitted here are about to be deleted. */
4503 125702 : do_pending_stack_adjust ();
4504 :
4505 : /* Remove any non-eh, non-abnormal edges that don't go to exit. */
4506 : /* ??? I.e. the fallthrough edge. HOWEVER! If there were to be
4507 : EH or abnormal edges, we shouldn't have created a tail call in
4508 : the first place. So it seems to me we should just be removing
4509 : all edges here, or redirecting the existing fallthru edge to
4510 : the exit block. */
4511 :
4512 125702 : probability = profile_probability::never ();
4513 :
4514 251398 : for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
4515 : {
4516 125696 : if (!(e->flags & (EDGE_ABNORMAL | EDGE_EH)))
4517 : {
4518 125690 : if (e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
4519 59267 : e->dest->count -= e->count ();
4520 125690 : probability += e->probability;
4521 125690 : expand_remove_edge (e);
4522 : }
4523 : else
4524 6 : ei_next (&ei);
4525 : }
4526 :
4527 : /* This is somewhat ugly: the call_expr expander often emits instructions
4528 : after the sibcall (to perform the function return). These confuse the
4529 : find_many_sub_basic_blocks code, so we need to get rid of these. */
4530 125702 : last = NEXT_INSN (last);
4531 125702 : gcc_assert (BARRIER_P (last));
4532 :
4533 125702 : *can_fallthru = false;
4534 156467 : while (NEXT_INSN (last))
4535 : {
4536 : /* For instance an sqrt builtin expander expands if with
4537 : sibcall in the then and label for `else`. */
4538 30765 : if (LABEL_P (NEXT_INSN (last)))
4539 : {
4540 0 : *can_fallthru = true;
4541 0 : break;
4542 : }
4543 30765 : delete_insn (NEXT_INSN (last));
4544 : }
4545 :
4546 125702 : e = make_edge (bb, EXIT_BLOCK_PTR_FOR_FN (cfun), EDGE_ABNORMAL
4547 : | EDGE_SIBCALL);
4548 125702 : e->probability = probability;
4549 125702 : head_end_for_bb[bb->index].second = last;
4550 251404 : update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
4551 125702 : head_end_for_bb[bb->index].second, bb);
4552 :
4553 125702 : if (NEXT_INSN (last))
4554 : {
4555 0 : bb = create_basic_block (NEXT_INSN (last), get_last_insn (), bb);
4556 :
4557 0 : last = BB_END (bb);
4558 0 : if (BARRIER_P (last))
4559 0 : BB_END (bb) = PREV_INSN (last);
4560 : }
4561 :
4562 125702 : maybe_dump_rtl_for_gimple_stmt (stmt, last2);
4563 :
4564 125702 : return bb;
4565 : }
4566 :
4567 : /* Return the difference between the floor and the truncated result of
4568 : a signed division by OP1 with remainder MOD. */
4569 : static rtx
4570 4 : floor_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4571 : {
4572 : /* (mod != 0 ? (op1 / mod < 0 ? -1 : 0) : 0) */
4573 4 : return gen_rtx_IF_THEN_ELSE
4574 : (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4575 : gen_rtx_IF_THEN_ELSE
4576 : (mode, gen_rtx_LT (BImode,
4577 : gen_rtx_DIV (mode, op1, mod),
4578 : const0_rtx),
4579 : constm1_rtx, const0_rtx),
4580 : const0_rtx);
4581 : }
4582 :
4583 : /* Return the difference between the ceil and the truncated result of
4584 : a signed division by OP1 with remainder MOD. */
4585 : static rtx
4586 3 : ceil_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4587 : {
4588 : /* (mod != 0 ? (op1 / mod > 0 ? 1 : 0) : 0) */
4589 3 : return gen_rtx_IF_THEN_ELSE
4590 : (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4591 : gen_rtx_IF_THEN_ELSE
4592 : (mode, gen_rtx_GT (BImode,
4593 : gen_rtx_DIV (mode, op1, mod),
4594 : const0_rtx),
4595 : const1_rtx, const0_rtx),
4596 : const0_rtx);
4597 : }
4598 :
4599 : /* Return the difference between the ceil and the truncated result of
4600 : an unsigned division by OP1 with remainder MOD. */
4601 : static rtx
4602 0 : ceil_udiv_adjust (machine_mode mode, rtx mod, rtx op1 ATTRIBUTE_UNUSED)
4603 : {
4604 : /* (mod != 0 ? 1 : 0) */
4605 0 : return gen_rtx_IF_THEN_ELSE
4606 : (mode, gen_rtx_NE (BImode, mod, const0_rtx),
4607 : const1_rtx, const0_rtx);
4608 : }
4609 :
4610 : /* Return the difference between the rounded and the truncated result
4611 : of a signed division by OP1 with remainder MOD. Halfway cases are
4612 : rounded away from zero, rather than to the nearest even number. */
4613 : static rtx
4614 0 : round_sdiv_adjust (machine_mode mode, rtx mod, rtx op1)
4615 : {
4616 : /* (abs (mod) >= abs (op1) - abs (mod)
4617 : ? (op1 / mod > 0 ? 1 : -1)
4618 : : 0) */
4619 0 : return gen_rtx_IF_THEN_ELSE
4620 : (mode, gen_rtx_GE (BImode, gen_rtx_ABS (mode, mod),
4621 : gen_rtx_MINUS (mode,
4622 : gen_rtx_ABS (mode, op1),
4623 : gen_rtx_ABS (mode, mod))),
4624 : gen_rtx_IF_THEN_ELSE
4625 : (mode, gen_rtx_GT (BImode,
4626 : gen_rtx_DIV (mode, op1, mod),
4627 : const0_rtx),
4628 : const1_rtx, constm1_rtx),
4629 : const0_rtx);
4630 : }
4631 :
4632 : /* Return the difference between the rounded and the truncated result
4633 : of a unsigned division by OP1 with remainder MOD. Halfway cases
4634 : are rounded away from zero, rather than to the nearest even
4635 : number. */
4636 : static rtx
4637 0 : round_udiv_adjust (machine_mode mode, rtx mod, rtx op1)
4638 : {
4639 : /* (mod >= op1 - mod ? 1 : 0) */
4640 0 : return gen_rtx_IF_THEN_ELSE
4641 : (mode, gen_rtx_GE (BImode, mod,
4642 : gen_rtx_MINUS (mode, op1, mod)),
4643 : const1_rtx, const0_rtx);
4644 : }
4645 :
4646 : /* Convert X to MODE, that must be Pmode or ptr_mode, without emitting
4647 : any rtl. */
4648 :
4649 : static rtx
4650 6708672 : convert_debug_memory_address (scalar_int_mode mode, rtx x,
4651 : addr_space_t as)
4652 : {
4653 : #ifndef POINTERS_EXTEND_UNSIGNED
4654 : gcc_assert (mode == Pmode
4655 : || mode == targetm.addr_space.address_mode (as));
4656 : gcc_assert (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode);
4657 : #else
4658 6708672 : rtx temp;
4659 :
4660 6708672 : gcc_assert (targetm.addr_space.valid_pointer_mode (mode, as));
4661 :
4662 6708672 : if (GET_MODE (x) == mode || GET_MODE (x) == VOIDmode)
4663 : return x;
4664 :
4665 : /* X must have some form of address mode already. */
4666 0 : scalar_int_mode xmode = as_a <scalar_int_mode> (GET_MODE (x));
4667 0 : if (GET_MODE_PRECISION (mode) < GET_MODE_PRECISION (xmode))
4668 0 : x = lowpart_subreg (mode, x, xmode);
4669 0 : else if (POINTERS_EXTEND_UNSIGNED > 0)
4670 0 : x = gen_rtx_ZERO_EXTEND (mode, x);
4671 : else if (!POINTERS_EXTEND_UNSIGNED)
4672 : x = gen_rtx_SIGN_EXTEND (mode, x);
4673 : else
4674 : {
4675 : switch (GET_CODE (x))
4676 : {
4677 : case SUBREG:
4678 : if ((SUBREG_PROMOTED_VAR_P (x)
4679 : || (REG_P (SUBREG_REG (x)) && REG_POINTER (SUBREG_REG (x)))
4680 : || (GET_CODE (SUBREG_REG (x)) == PLUS
4681 : && REG_P (XEXP (SUBREG_REG (x), 0))
4682 : && REG_POINTER (XEXP (SUBREG_REG (x), 0))
4683 : && CONST_INT_P (XEXP (SUBREG_REG (x), 1))))
4684 : && GET_MODE (SUBREG_REG (x)) == mode)
4685 : return SUBREG_REG (x);
4686 : break;
4687 : case LABEL_REF:
4688 : temp = gen_rtx_LABEL_REF (mode, label_ref_label (x));
4689 : LABEL_REF_NONLOCAL_P (temp) = LABEL_REF_NONLOCAL_P (x);
4690 : return temp;
4691 : case SYMBOL_REF:
4692 : temp = shallow_copy_rtx (x);
4693 : PUT_MODE (temp, mode);
4694 : return temp;
4695 : case CONST:
4696 : temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4697 : if (temp)
4698 : temp = gen_rtx_CONST (mode, temp);
4699 : return temp;
4700 : case PLUS:
4701 : case MINUS:
4702 : if (CONST_INT_P (XEXP (x, 1)))
4703 : {
4704 : temp = convert_debug_memory_address (mode, XEXP (x, 0), as);
4705 : if (temp)
4706 : return gen_rtx_fmt_ee (GET_CODE (x), mode, temp, XEXP (x, 1));
4707 : }
4708 : break;
4709 : default:
4710 : break;
4711 : }
4712 : /* Don't know how to express ptr_extend as operation in debug info. */
4713 : return NULL;
4714 : }
4715 : #endif /* POINTERS_EXTEND_UNSIGNED */
4716 :
4717 : return x;
4718 : }
4719 :
4720 : /* Map from SSA_NAMEs to corresponding DEBUG_EXPR_DECLs created
4721 : by avoid_deep_ter_for_debug. */
4722 :
4723 : static hash_map<tree, tree> *deep_ter_debug_map;
4724 :
4725 : /* Split too deep TER chains for debug stmts using debug temporaries. */
4726 :
4727 : static void
4728 38159731 : avoid_deep_ter_for_debug (gimple *stmt, int depth)
4729 : {
4730 38159731 : use_operand_p use_p;
4731 38159731 : ssa_op_iter iter;
4732 47467317 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
4733 : {
4734 9307586 : tree use = USE_FROM_PTR (use_p);
4735 9307586 : if (TREE_CODE (use) != SSA_NAME || SSA_NAME_IS_DEFAULT_DEF (use))
4736 7869342 : continue;
4737 8027783 : gimple *g = get_gimple_for_ssa_name (use);
4738 8027783 : if (g == NULL)
4739 6588646 : continue;
4740 1439137 : if ((depth > 6 || !is_gimple_assign (g)) && !stmt_ends_bb_p (g))
4741 : {
4742 3724 : if (deep_ter_debug_map == NULL)
4743 672 : deep_ter_debug_map = new hash_map<tree, tree>;
4744 :
4745 3724 : tree &vexpr = deep_ter_debug_map->get_or_insert (use);
4746 3724 : if (vexpr != NULL)
4747 893 : continue;
4748 2831 : vexpr = build_debug_expr_decl (TREE_TYPE (use));
4749 2831 : gimple *def_temp = gimple_build_debug_bind (vexpr, use, g);
4750 2831 : gimple_stmt_iterator gsi = gsi_for_stmt (g);
4751 2831 : gsi_insert_after (&gsi, def_temp, GSI_NEW_STMT);
4752 2831 : avoid_deep_ter_for_debug (def_temp, 0);
4753 : }
4754 : else
4755 1435413 : avoid_deep_ter_for_debug (g, depth + 1);
4756 : }
4757 38159731 : }
4758 :
4759 : /* Return an RTX equivalent to the value of the parameter DECL. */
4760 :
4761 : static rtx
4762 174110 : expand_debug_parm_decl (tree decl)
4763 : {
4764 174110 : rtx incoming = DECL_INCOMING_RTL (decl);
4765 :
4766 174110 : if (incoming
4767 19228 : && GET_MODE (incoming) != BLKmode
4768 193334 : && ((REG_P (incoming) && HARD_REGISTER_P (incoming))
4769 1701 : || (MEM_P (incoming)
4770 1615 : && REG_P (XEXP (incoming, 0))
4771 683 : && HARD_REGISTER_P (XEXP (incoming, 0)))))
4772 : {
4773 17551 : rtx rtl = gen_rtx_ENTRY_VALUE (GET_MODE (incoming));
4774 :
4775 : #ifdef HAVE_window_save
4776 : /* DECL_INCOMING_RTL uses the INCOMING_REGNO of parameter registers.
4777 : If the target machine has an explicit window save instruction, the
4778 : actual entry value is the corresponding OUTGOING_REGNO instead. */
4779 : if (REG_P (incoming)
4780 : && OUTGOING_REGNO (REGNO (incoming)) != REGNO (incoming))
4781 : incoming
4782 : = gen_rtx_REG_offset (incoming, GET_MODE (incoming),
4783 : OUTGOING_REGNO (REGNO (incoming)), 0);
4784 : else if (MEM_P (incoming))
4785 : {
4786 : rtx reg = XEXP (incoming, 0);
4787 : if (OUTGOING_REGNO (REGNO (reg)) != REGNO (reg))
4788 : {
4789 : reg = gen_raw_REG (GET_MODE (reg), OUTGOING_REGNO (REGNO (reg)));
4790 : incoming = replace_equiv_address_nv (incoming, reg);
4791 : }
4792 : else
4793 : incoming = copy_rtx (incoming);
4794 : }
4795 : #endif
4796 :
4797 17551 : ENTRY_VALUE_EXP (rtl) = incoming;
4798 17551 : return rtl;
4799 : }
4800 :
4801 156559 : if (incoming
4802 1677 : && GET_MODE (incoming) != BLKmode
4803 1673 : && !TREE_ADDRESSABLE (decl)
4804 1673 : && MEM_P (incoming)
4805 1587 : && (XEXP (incoming, 0) == virtual_incoming_args_rtx
4806 932 : || (GET_CODE (XEXP (incoming, 0)) == PLUS
4807 932 : && XEXP (XEXP (incoming, 0), 0) == virtual_incoming_args_rtx
4808 915 : && CONST_INT_P (XEXP (XEXP (incoming, 0), 1)))))
4809 1570 : return copy_rtx (incoming);
4810 :
4811 : return NULL_RTX;
4812 : }
4813 :
4814 : /* Return an RTX equivalent to the value of the tree expression EXP. */
4815 :
4816 : static rtx
4817 38673157 : expand_debug_expr (tree exp)
4818 : {
4819 38673157 : rtx op0 = NULL_RTX, op1 = NULL_RTX, op2 = NULL_RTX;
4820 38673157 : machine_mode mode = TYPE_MODE (TREE_TYPE (exp));
4821 38673157 : machine_mode inner_mode = VOIDmode;
4822 38673157 : int unsignedp = TYPE_UNSIGNED (TREE_TYPE (exp));
4823 38673157 : addr_space_t as;
4824 38673157 : scalar_int_mode op0_mode, op1_mode, addr_mode;
4825 :
4826 38673157 : switch (TREE_CODE_CLASS (TREE_CODE (exp)))
4827 : {
4828 6384277 : case tcc_expression:
4829 6384277 : switch (TREE_CODE (exp))
4830 : {
4831 8377 : case COND_EXPR:
4832 8377 : case DOT_PROD_EXPR:
4833 8377 : case SAD_EXPR:
4834 8377 : case WIDEN_MULT_PLUS_EXPR:
4835 8377 : case WIDEN_MULT_MINUS_EXPR:
4836 8377 : goto ternary;
4837 :
4838 0 : case TRUTH_ANDIF_EXPR:
4839 0 : case TRUTH_ORIF_EXPR:
4840 0 : case TRUTH_AND_EXPR:
4841 0 : case TRUTH_OR_EXPR:
4842 0 : case TRUTH_XOR_EXPR:
4843 0 : goto binary;
4844 :
4845 0 : case TRUTH_NOT_EXPR:
4846 0 : goto unary;
4847 :
4848 : default:
4849 : break;
4850 : }
4851 : break;
4852 :
4853 8377 : ternary:
4854 8377 : op2 = expand_debug_expr (TREE_OPERAND (exp, 2));
4855 8377 : if (!op2)
4856 : return NULL_RTX;
4857 : /* Fall through. */
4858 :
4859 8377 : binary:
4860 1290250 : case tcc_binary:
4861 1290250 : if (mode == BLKmode)
4862 : return NULL_RTX;
4863 1290240 : op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
4864 1290240 : if (!op1)
4865 : return NULL_RTX;
4866 1289671 : switch (TREE_CODE (exp))
4867 : {
4868 56368 : case LSHIFT_EXPR:
4869 56368 : case RSHIFT_EXPR:
4870 56368 : case LROTATE_EXPR:
4871 56368 : case RROTATE_EXPR:
4872 56368 : case WIDEN_LSHIFT_EXPR:
4873 : /* Ensure second operand isn't wider than the first one. */
4874 56368 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 1)));
4875 56368 : if (is_a <scalar_int_mode> (inner_mode, &op1_mode)
4876 56360 : && (GET_MODE_UNIT_PRECISION (mode)
4877 56360 : < GET_MODE_PRECISION (op1_mode)))
4878 5220 : op1 = lowpart_subreg (GET_MODE_INNER (mode), op1, op1_mode);
4879 : break;
4880 : default:
4881 : break;
4882 : }
4883 : /* Fall through. */
4884 :
4885 1847430 : unary:
4886 1847430 : case tcc_unary:
4887 1847430 : if (mode == BLKmode)
4888 : return NULL_RTX;
4889 1847428 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
4890 1847428 : op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
4891 1847428 : if (!op0)
4892 : return NULL_RTX;
4893 : break;
4894 :
4895 57664 : case tcc_comparison:
4896 57664 : unsignedp = TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)));
4897 57664 : goto binary;
4898 :
4899 0 : case tcc_type:
4900 0 : case tcc_statement:
4901 0 : gcc_unreachable ();
4902 :
4903 : case tcc_constant:
4904 : case tcc_exceptional:
4905 : case tcc_declaration:
4906 : case tcc_reference:
4907 : case tcc_vl_exp:
4908 : break;
4909 : }
4910 :
4911 38670337 : switch (TREE_CODE (exp))
4912 : {
4913 260061 : case STRING_CST:
4914 260061 : if (!lookup_constant_def (exp))
4915 : {
4916 98340 : if (strlen (TREE_STRING_POINTER (exp)) + 1
4917 98340 : != (size_t) TREE_STRING_LENGTH (exp))
4918 : return NULL_RTX;
4919 98340 : op0 = gen_rtx_CONST_STRING (Pmode, TREE_STRING_POINTER (exp));
4920 97618 : op0 = gen_rtx_MEM (BLKmode, op0);
4921 97618 : set_mem_attributes (op0, exp, 0);
4922 97618 : return op0;
4923 : }
4924 : /* Fall through. */
4925 :
4926 5127390 : case INTEGER_CST:
4927 10254745 : if (BITINT_TYPE_P (TREE_TYPE (exp))
4928 5127390 : && TYPE_MODE (TREE_TYPE (exp)) == BLKmode)
4929 : return NULL;
4930 : /* FALLTHRU */
4931 5189628 : case REAL_CST:
4932 5189628 : case FIXED_CST:
4933 5189628 : op0 = expand_expr (exp, NULL_RTX, mode, EXPAND_INITIALIZER);
4934 5189628 : return op0;
4935 :
4936 0 : case POLY_INT_CST:
4937 0 : return immed_wide_int_const (poly_int_cst_value (exp), mode);
4938 :
4939 3946 : case COMPLEX_CST:
4940 3946 : gcc_assert (COMPLEX_MODE_P (mode));
4941 3946 : op0 = expand_debug_expr (TREE_REALPART (exp));
4942 3946 : op1 = expand_debug_expr (TREE_IMAGPART (exp));
4943 3946 : return gen_rtx_CONCAT (mode, op0, op1);
4944 :
4945 5605641 : case DEBUG_EXPR_DECL:
4946 5605641 : op0 = DECL_RTL_IF_SET (exp);
4947 :
4948 4461604 : if (op0)
4949 : {
4950 4461604 : if (GET_MODE (op0) != mode)
4951 0 : gcc_assert (VECTOR_TYPE_P (TREE_TYPE (exp)));
4952 : else
4953 : return op0;
4954 : }
4955 :
4956 1144037 : op0 = gen_rtx_DEBUG_EXPR (mode);
4957 1144037 : DEBUG_EXPR_TREE_DECL (op0) = exp;
4958 1144037 : SET_DECL_RTL (exp, op0);
4959 :
4960 1144037 : return op0;
4961 :
4962 5023798 : case VAR_DECL:
4963 5023798 : case PARM_DECL:
4964 5023798 : case FUNCTION_DECL:
4965 5023798 : case LABEL_DECL:
4966 5023798 : case CONST_DECL:
4967 5023798 : case RESULT_DECL:
4968 5023798 : op0 = DECL_RTL_IF_SET (exp);
4969 :
4970 : /* This decl was probably optimized away. */
4971 3461164 : if (!op0
4972 : /* At least label RTXen are sometimes replaced by
4973 : NOTE_INSN_DELETED_LABEL. Any notes here are not
4974 : handled by copy_rtx. */
4975 3461164 : || NOTE_P (op0))
4976 : {
4977 1562635 : if (!VAR_P (exp)
4978 1551907 : || DECL_EXTERNAL (exp)
4979 1551348 : || !TREE_STATIC (exp)
4980 118451 : || !DECL_NAME (exp)
4981 118451 : || DECL_HARD_REGISTER (exp)
4982 118451 : || DECL_IN_CONSTANT_POOL (exp)
4983 118451 : || mode == VOIDmode
4984 1681086 : || symtab_node::get (exp) == NULL)
4985 : return NULL;
4986 :
4987 17136 : op0 = make_decl_rtl_for_debug (exp);
4988 17136 : if (!MEM_P (op0)
4989 17136 : || GET_CODE (XEXP (op0, 0)) != SYMBOL_REF
4990 34272 : || SYMBOL_REF_DECL (XEXP (op0, 0)) != exp)
4991 : return NULL;
4992 : }
4993 3461163 : else if (VAR_P (exp)
4994 3356266 : && is_global_var (exp)
4995 3658247 : && symtab_node::get (exp) == NULL)
4996 : return NULL;
4997 : else
4998 3461163 : op0 = copy_rtx (op0);
4999 :
5000 3478299 : if (GET_MODE (op0) == BLKmode
5001 : /* If op0 is not BLKmode, but mode is, adjust_mode
5002 : below would ICE. While it is likely a FE bug,
5003 : try to be robust here. See PR43166. */
5004 456668 : || mode == BLKmode
5005 456668 : || (mode == VOIDmode && GET_MODE (op0) != VOIDmode))
5006 : {
5007 3021631 : gcc_assert (MEM_P (op0));
5008 3021631 : op0 = adjust_address_nv (op0, mode, 0);
5009 3021631 : return op0;
5010 : }
5011 :
5012 : /* Fall through. */
5013 :
5014 456668 : adjust_mode:
5015 9947688 : case PAREN_EXPR:
5016 9947688 : CASE_CONVERT:
5017 9947688 : {
5018 9947688 : inner_mode = GET_MODE (op0);
5019 :
5020 9947688 : if (mode == inner_mode)
5021 : return op0;
5022 :
5023 142904 : if (inner_mode == VOIDmode)
5024 : {
5025 5607 : if (TREE_CODE (exp) == SSA_NAME)
5026 31 : inner_mode = TYPE_MODE (TREE_TYPE (exp));
5027 : else
5028 5576 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
5029 5607 : if (mode == inner_mode)
5030 : return op0;
5031 : }
5032 :
5033 139843 : if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5034 : {
5035 6255 : if (GET_MODE_UNIT_BITSIZE (mode)
5036 12510 : == GET_MODE_UNIT_BITSIZE (inner_mode))
5037 4 : op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
5038 6251 : else if (GET_MODE_UNIT_BITSIZE (mode)
5039 12502 : < GET_MODE_UNIT_BITSIZE (inner_mode))
5040 3568 : op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
5041 : else
5042 2683 : op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
5043 : }
5044 133588 : else if (FLOAT_MODE_P (mode))
5045 : {
5046 0 : gcc_assert (TREE_CODE (exp) != SSA_NAME);
5047 0 : if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5048 0 : op0 = simplify_gen_unary (UNSIGNED_FLOAT, mode, op0, inner_mode);
5049 : else
5050 0 : op0 = simplify_gen_unary (FLOAT, mode, op0, inner_mode);
5051 : }
5052 133588 : else if (FLOAT_MODE_P (inner_mode))
5053 : {
5054 0 : if (unsignedp)
5055 0 : op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
5056 : else
5057 0 : op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
5058 : }
5059 267176 : else if (GET_MODE_UNIT_PRECISION (mode)
5060 133588 : == GET_MODE_UNIT_PRECISION (inner_mode))
5061 0 : op0 = lowpart_subreg (mode, op0, inner_mode);
5062 267176 : else if (GET_MODE_UNIT_PRECISION (mode)
5063 133588 : < GET_MODE_UNIT_PRECISION (inner_mode))
5064 64334 : op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
5065 69254 : else if (UNARY_CLASS_P (exp)
5066 69254 : ? TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0)))
5067 0 : : unsignedp)
5068 46409 : op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5069 : else
5070 22845 : op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5071 :
5072 : return op0;
5073 : }
5074 :
5075 2028366 : case MEM_REF:
5076 2028366 : if (!is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
5077 : {
5078 868301 : tree newexp = fold_binary (MEM_REF, TREE_TYPE (exp),
5079 : TREE_OPERAND (exp, 0),
5080 : TREE_OPERAND (exp, 1));
5081 868301 : if (newexp)
5082 0 : return expand_debug_expr (newexp);
5083 : }
5084 : /* FALLTHROUGH */
5085 2028366 : case INDIRECT_REF:
5086 2028366 : inner_mode = TYPE_MODE (TREE_TYPE (TREE_OPERAND (exp, 0)));
5087 2028366 : op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5088 2028366 : if (!op0)
5089 : return NULL;
5090 :
5091 2023175 : if (TREE_CODE (exp) == MEM_REF)
5092 : {
5093 2023175 : if (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5094 1977942 : || (GET_CODE (op0) == PLUS
5095 271946 : && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR))
5096 : /* (mem (debug_implicit_ptr)) might confuse aliasing.
5097 : Instead just use get_inner_reference. */
5098 45233 : goto component_ref;
5099 :
5100 1977942 : op1 = expand_debug_expr (TREE_OPERAND (exp, 1));
5101 1977942 : poly_int64 offset;
5102 1977942 : if (!op1 || !poly_int_rtx_p (op1, &offset))
5103 0 : return NULL;
5104 :
5105 1977942 : op0 = plus_constant (inner_mode, op0, offset);
5106 : }
5107 :
5108 1977942 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
5109 :
5110 1977942 : op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
5111 : op0, as);
5112 1977942 : if (op0 == NULL_RTX)
5113 : return NULL;
5114 :
5115 1977942 : op0 = gen_rtx_MEM (mode, op0);
5116 1977942 : set_mem_attributes (op0, exp, 0);
5117 1977942 : if (TREE_CODE (exp) == MEM_REF
5118 1977942 : && !is_gimple_mem_ref_addr (TREE_OPERAND (exp, 0)))
5119 868301 : set_mem_expr (op0, NULL_TREE);
5120 1977942 : set_mem_addr_space (op0, as);
5121 :
5122 1977942 : return op0;
5123 :
5124 56467 : case TARGET_MEM_REF:
5125 56467 : if (TREE_CODE (TMR_BASE (exp)) == ADDR_EXPR
5126 56467 : && !DECL_RTL_SET_P (TREE_OPERAND (TMR_BASE (exp), 0)))
5127 : return NULL;
5128 :
5129 56457 : op0 = expand_debug_expr
5130 56457 : (tree_mem_ref_addr (build_pointer_type (TREE_TYPE (exp)), exp));
5131 56457 : if (!op0)
5132 : return NULL;
5133 :
5134 56457 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (TREE_OPERAND (exp, 0))));
5135 56457 : op0 = convert_debug_memory_address (targetm.addr_space.address_mode (as),
5136 : op0, as);
5137 56457 : if (op0 == NULL_RTX)
5138 : return NULL;
5139 :
5140 56457 : op0 = gen_rtx_MEM (mode, op0);
5141 :
5142 56457 : set_mem_attributes (op0, exp, 0);
5143 56457 : set_mem_addr_space (op0, as);
5144 :
5145 56457 : return op0;
5146 :
5147 3503870 : component_ref:
5148 3503870 : case ARRAY_REF:
5149 3503870 : case ARRAY_RANGE_REF:
5150 3503870 : case COMPONENT_REF:
5151 3503870 : case BIT_FIELD_REF:
5152 3503870 : case REALPART_EXPR:
5153 3503870 : case IMAGPART_EXPR:
5154 3503870 : case VIEW_CONVERT_EXPR:
5155 3503870 : {
5156 3503870 : machine_mode mode1;
5157 3503870 : poly_int64 bitsize, bitpos;
5158 3503870 : tree offset;
5159 3503870 : int reversep, volatilep = 0;
5160 3503870 : tree tem
5161 3503870 : = get_inner_reference (exp, &bitsize, &bitpos, &offset, &mode1,
5162 : &unsignedp, &reversep, &volatilep);
5163 3503870 : rtx orig_op0;
5164 :
5165 3503870 : if (known_eq (bitsize, 0))
5166 : return NULL;
5167 :
5168 3449456 : orig_op0 = op0 = expand_debug_expr (tem);
5169 :
5170 3449456 : if (!op0)
5171 : return NULL;
5172 :
5173 3132963 : if (offset)
5174 : {
5175 12357 : machine_mode addrmode, offmode;
5176 :
5177 12357 : if (!MEM_P (op0))
5178 : return NULL;
5179 :
5180 12357 : op0 = XEXP (op0, 0);
5181 12357 : addrmode = GET_MODE (op0);
5182 12357 : if (addrmode == VOIDmode)
5183 0 : addrmode = Pmode;
5184 :
5185 12357 : op1 = expand_debug_expr (offset);
5186 12357 : if (!op1)
5187 : return NULL;
5188 :
5189 12355 : offmode = GET_MODE (op1);
5190 12355 : if (offmode == VOIDmode)
5191 6 : offmode = TYPE_MODE (TREE_TYPE (offset));
5192 :
5193 12355 : if (addrmode != offmode)
5194 0 : op1 = lowpart_subreg (addrmode, op1, offmode);
5195 :
5196 : /* Don't use offset_address here, we don't need a
5197 : recognizable address, and we don't want to generate
5198 : code. */
5199 12355 : op0 = gen_rtx_MEM (mode, simplify_gen_binary (PLUS, addrmode,
5200 : op0, op1));
5201 : }
5202 :
5203 3132961 : if (MEM_P (op0))
5204 : {
5205 3007527 : if (mode1 == VOIDmode)
5206 : {
5207 3143 : if (maybe_gt (bitsize, MAX_BITSIZE_MODE_ANY_INT))
5208 370953 : return NULL;
5209 : /* Bitfield. */
5210 3143 : mode1 = smallest_int_mode_for_size (bitsize).require ();
5211 : }
5212 3007527 : poly_int64 bytepos = bits_to_bytes_round_down (bitpos);
5213 3007527 : if (maybe_ne (bytepos, 0))
5214 : {
5215 1262490 : op0 = adjust_address_nv (op0, mode1, bytepos);
5216 1262490 : bitpos = num_trailing_bits (bitpos);
5217 : }
5218 1745037 : else if (known_eq (bitpos, 0)
5219 3489494 : && known_eq (bitsize, GET_MODE_BITSIZE (mode)))
5220 477709 : op0 = adjust_address_nv (op0, mode, 0);
5221 1267328 : else if (GET_MODE (op0) != mode1)
5222 2846 : op0 = adjust_address_nv (op0, mode1, 0);
5223 : else
5224 1264482 : op0 = copy_rtx (op0);
5225 3007527 : if (op0 == orig_op0)
5226 143583 : op0 = shallow_copy_rtx (op0);
5227 3007527 : if (TREE_CODE (tem) != SSA_NAME)
5228 3005165 : set_mem_attributes (op0, exp, 0);
5229 : }
5230 :
5231 3132961 : if (known_eq (bitpos, 0) && mode == GET_MODE (op0))
5232 : return op0;
5233 :
5234 98698 : if (maybe_lt (bitpos, 0))
5235 : return NULL;
5236 :
5237 98696 : if (GET_MODE (op0) == BLKmode || mode == BLKmode)
5238 : return NULL;
5239 :
5240 98680 : poly_int64 bytepos;
5241 99407 : if (multiple_p (bitpos, BITS_PER_UNIT, &bytepos)
5242 194384 : && known_eq (bitsize, GET_MODE_BITSIZE (mode1)))
5243 : {
5244 96487 : machine_mode opmode = GET_MODE (op0);
5245 :
5246 96487 : if (opmode == VOIDmode)
5247 0 : opmode = TYPE_MODE (TREE_TYPE (tem));
5248 :
5249 : /* This condition may hold if we're expanding the address
5250 : right past the end of an array that turned out not to
5251 : be addressable (i.e., the address was only computed in
5252 : debug stmts). The gen_subreg below would rightfully
5253 : crash, and the address doesn't really exist, so just
5254 : drop it. */
5255 192974 : if (known_ge (bitpos, GET_MODE_BITSIZE (opmode)))
5256 : return NULL;
5257 :
5258 192922 : if (multiple_p (bitpos, GET_MODE_BITSIZE (mode)))
5259 96439 : return simplify_gen_subreg (mode, op0, opmode, bytepos);
5260 : }
5261 :
5262 4430 : return simplify_gen_ternary (SCALAR_INT_MODE_P (GET_MODE (op0))
5263 1730 : && TYPE_UNSIGNED (TREE_TYPE (exp))
5264 : ? SIGN_EXTRACT
5265 : : ZERO_EXTRACT, mode,
5266 2215 : GET_MODE (op0) != VOIDmode
5267 : ? GET_MODE (op0)
5268 0 : : TYPE_MODE (TREE_TYPE (tem)),
5269 : op0, gen_int_mode (bitsize, word_mode),
5270 2215 : gen_int_mode (bitpos, word_mode));
5271 : }
5272 :
5273 2371 : case ABS_EXPR:
5274 2371 : case ABSU_EXPR:
5275 2371 : return simplify_gen_unary (ABS, mode, op0, mode);
5276 :
5277 13766 : case NEGATE_EXPR:
5278 13766 : return simplify_gen_unary (NEG, mode, op0, mode);
5279 :
5280 17573 : case BIT_NOT_EXPR:
5281 17573 : return simplify_gen_unary (NOT, mode, op0, mode);
5282 :
5283 2356 : case FLOAT_EXPR:
5284 2356 : return simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5285 : 0)))
5286 : ? UNSIGNED_FLOAT : FLOAT, mode, op0,
5287 2356 : inner_mode);
5288 :
5289 401 : case FIX_TRUNC_EXPR:
5290 401 : return simplify_gen_unary (unsignedp ? UNSIGNED_FIX : FIX, mode, op0,
5291 401 : inner_mode);
5292 :
5293 226616 : case POINTER_PLUS_EXPR:
5294 : /* For the rare target where pointers are not the same size as
5295 : size_t, we need to check for mis-matched modes and correct
5296 : the addend. */
5297 226616 : if (op0 && op1
5298 226616 : && is_a <scalar_int_mode> (GET_MODE (op0), &op0_mode)
5299 226525 : && is_a <scalar_int_mode> (GET_MODE (op1), &op1_mode)
5300 356799 : && op0_mode != op1_mode)
5301 : {
5302 0 : if (GET_MODE_BITSIZE (op0_mode) < GET_MODE_BITSIZE (op1_mode)
5303 : /* If OP0 is a partial mode, then we must truncate, even
5304 : if it has the same bitsize as OP1 as GCC's
5305 : representation of partial modes is opaque. */
5306 0 : || (GET_MODE_CLASS (op0_mode) == MODE_PARTIAL_INT
5307 0 : && (GET_MODE_BITSIZE (op0_mode)
5308 0 : == GET_MODE_BITSIZE (op1_mode))))
5309 0 : op1 = simplify_gen_unary (TRUNCATE, op0_mode, op1, op1_mode);
5310 : else
5311 : /* We always sign-extend, regardless of the signedness of
5312 : the operand, because the operand is always unsigned
5313 : here even if the original C expression is signed. */
5314 0 : op1 = simplify_gen_unary (SIGN_EXTEND, op0_mode, op1, op1_mode);
5315 : }
5316 : /* Fall through. */
5317 684329 : case PLUS_EXPR:
5318 684329 : return simplify_gen_binary (PLUS, mode, op0, op1);
5319 :
5320 110329 : case MINUS_EXPR:
5321 110329 : case POINTER_DIFF_EXPR:
5322 110329 : return simplify_gen_binary (MINUS, mode, op0, op1);
5323 :
5324 129710 : case MULT_EXPR:
5325 129710 : return simplify_gen_binary (MULT, mode, op0, op1);
5326 :
5327 3177 : case RDIV_EXPR:
5328 3177 : gcc_assert (FLOAT_MODE_P (mode)
5329 : || ALL_FIXED_POINT_MODE_P (mode));
5330 : /* Fall through. */
5331 78565 : case TRUNC_DIV_EXPR:
5332 78565 : case EXACT_DIV_EXPR:
5333 78565 : if (unsignedp)
5334 10098 : return simplify_gen_binary (UDIV, mode, op0, op1);
5335 : else
5336 68467 : return simplify_gen_binary (DIV, mode, op0, op1);
5337 :
5338 3522 : case TRUNC_MOD_EXPR:
5339 7044 : return simplify_gen_binary (unsignedp ? UMOD : MOD, mode, op0, op1);
5340 :
5341 3 : case FLOOR_DIV_EXPR:
5342 3 : if (unsignedp)
5343 0 : return simplify_gen_binary (UDIV, mode, op0, op1);
5344 : else
5345 : {
5346 3 : rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5347 3 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5348 3 : rtx adj = floor_sdiv_adjust (mode, mod, op1);
5349 3 : return simplify_gen_binary (PLUS, mode, div, adj);
5350 : }
5351 :
5352 1 : case FLOOR_MOD_EXPR:
5353 1 : if (unsignedp)
5354 0 : return simplify_gen_binary (UMOD, mode, op0, op1);
5355 : else
5356 : {
5357 1 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5358 1 : rtx adj = floor_sdiv_adjust (mode, mod, op1);
5359 1 : adj = simplify_gen_unary (NEG, mode,
5360 : simplify_gen_binary (MULT, mode, adj, op1),
5361 : mode);
5362 1 : return simplify_gen_binary (PLUS, mode, mod, adj);
5363 : }
5364 :
5365 3 : case CEIL_DIV_EXPR:
5366 3 : if (unsignedp)
5367 : {
5368 0 : rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
5369 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5370 0 : rtx adj = ceil_udiv_adjust (mode, mod, op1);
5371 0 : return simplify_gen_binary (PLUS, mode, div, adj);
5372 : }
5373 : else
5374 : {
5375 3 : rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5376 3 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5377 3 : rtx adj = ceil_sdiv_adjust (mode, mod, op1);
5378 3 : return simplify_gen_binary (PLUS, mode, div, adj);
5379 : }
5380 :
5381 0 : case CEIL_MOD_EXPR:
5382 0 : if (unsignedp)
5383 : {
5384 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5385 0 : rtx adj = ceil_udiv_adjust (mode, mod, op1);
5386 0 : adj = simplify_gen_unary (NEG, mode,
5387 : simplify_gen_binary (MULT, mode, adj, op1),
5388 : mode);
5389 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5390 : }
5391 : else
5392 : {
5393 0 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5394 0 : rtx adj = ceil_sdiv_adjust (mode, mod, op1);
5395 0 : adj = simplify_gen_unary (NEG, mode,
5396 : simplify_gen_binary (MULT, mode, adj, op1),
5397 : mode);
5398 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5399 : }
5400 :
5401 0 : case ROUND_DIV_EXPR:
5402 0 : if (unsignedp)
5403 : {
5404 0 : rtx div = simplify_gen_binary (UDIV, mode, op0, op1);
5405 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5406 0 : rtx adj = round_udiv_adjust (mode, mod, op1);
5407 0 : return simplify_gen_binary (PLUS, mode, div, adj);
5408 : }
5409 : else
5410 : {
5411 0 : rtx div = simplify_gen_binary (DIV, mode, op0, op1);
5412 0 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5413 0 : rtx adj = round_sdiv_adjust (mode, mod, op1);
5414 0 : return simplify_gen_binary (PLUS, mode, div, adj);
5415 : }
5416 :
5417 0 : case ROUND_MOD_EXPR:
5418 0 : if (unsignedp)
5419 : {
5420 0 : rtx mod = simplify_gen_binary (UMOD, mode, op0, op1);
5421 0 : rtx adj = round_udiv_adjust (mode, mod, op1);
5422 0 : adj = simplify_gen_unary (NEG, mode,
5423 : simplify_gen_binary (MULT, mode, adj, op1),
5424 : mode);
5425 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5426 : }
5427 : else
5428 : {
5429 0 : rtx mod = simplify_gen_binary (MOD, mode, op0, op1);
5430 0 : rtx adj = round_sdiv_adjust (mode, mod, op1);
5431 0 : adj = simplify_gen_unary (NEG, mode,
5432 : simplify_gen_binary (MULT, mode, adj, op1),
5433 : mode);
5434 0 : return simplify_gen_binary (PLUS, mode, mod, adj);
5435 : }
5436 :
5437 13327 : case LSHIFT_EXPR:
5438 13327 : return simplify_gen_binary (ASHIFT, mode, op0, op1);
5439 :
5440 42069 : case RSHIFT_EXPR:
5441 42069 : if (unsignedp)
5442 38946 : return simplify_gen_binary (LSHIFTRT, mode, op0, op1);
5443 : else
5444 3123 : return simplify_gen_binary (ASHIFTRT, mode, op0, op1);
5445 :
5446 2 : case LROTATE_EXPR:
5447 2 : return simplify_gen_binary (ROTATE, mode, op0, op1);
5448 :
5449 701 : case RROTATE_EXPR:
5450 701 : return simplify_gen_binary (ROTATERT, mode, op0, op1);
5451 :
5452 9324 : case MIN_EXPR:
5453 18648 : return simplify_gen_binary (unsignedp ? UMIN : SMIN, mode, op0, op1);
5454 :
5455 13972 : case MAX_EXPR:
5456 27944 : return simplify_gen_binary (unsignedp ? UMAX : SMAX, mode, op0, op1);
5457 :
5458 73553 : case BIT_AND_EXPR:
5459 73553 : case TRUTH_AND_EXPR:
5460 73553 : return simplify_gen_binary (AND, mode, op0, op1);
5461 :
5462 39800 : case BIT_IOR_EXPR:
5463 39800 : case TRUTH_OR_EXPR:
5464 39800 : return simplify_gen_binary (IOR, mode, op0, op1);
5465 :
5466 7907 : case BIT_XOR_EXPR:
5467 7907 : case TRUTH_XOR_EXPR:
5468 7907 : return simplify_gen_binary (XOR, mode, op0, op1);
5469 :
5470 0 : case TRUTH_ANDIF_EXPR:
5471 0 : return gen_rtx_IF_THEN_ELSE (mode, op0, op1, const0_rtx);
5472 :
5473 0 : case TRUTH_ORIF_EXPR:
5474 0 : return gen_rtx_IF_THEN_ELSE (mode, op0, const_true_rtx, op1);
5475 :
5476 0 : case TRUTH_NOT_EXPR:
5477 0 : return simplify_gen_relational (EQ, mode, inner_mode, op0, const0_rtx);
5478 :
5479 4237 : case LT_EXPR:
5480 4237 : return simplify_gen_relational (unsignedp ? LTU : LT, mode, inner_mode,
5481 4237 : op0, op1);
5482 :
5483 4846 : case LE_EXPR:
5484 4846 : return simplify_gen_relational (unsignedp ? LEU : LE, mode, inner_mode,
5485 4846 : op0, op1);
5486 :
5487 9444 : case GT_EXPR:
5488 9444 : return simplify_gen_relational (unsignedp ? GTU : GT, mode, inner_mode,
5489 9444 : op0, op1);
5490 :
5491 2074 : case GE_EXPR:
5492 2074 : return simplify_gen_relational (unsignedp ? GEU : GE, mode, inner_mode,
5493 2074 : op0, op1);
5494 :
5495 22743 : case EQ_EXPR:
5496 22743 : return simplify_gen_relational (EQ, mode, inner_mode, op0, op1);
5497 :
5498 14295 : case NE_EXPR:
5499 14295 : return simplify_gen_relational (NE, mode, inner_mode, op0, op1);
5500 :
5501 2 : case UNORDERED_EXPR:
5502 2 : return simplify_gen_relational (UNORDERED, mode, inner_mode, op0, op1);
5503 :
5504 0 : case ORDERED_EXPR:
5505 0 : return simplify_gen_relational (ORDERED, mode, inner_mode, op0, op1);
5506 :
5507 0 : case UNLT_EXPR:
5508 0 : return simplify_gen_relational (UNLT, mode, inner_mode, op0, op1);
5509 :
5510 1 : case UNLE_EXPR:
5511 1 : return simplify_gen_relational (UNLE, mode, inner_mode, op0, op1);
5512 :
5513 0 : case UNGT_EXPR:
5514 0 : return simplify_gen_relational (UNGT, mode, inner_mode, op0, op1);
5515 :
5516 0 : case UNGE_EXPR:
5517 0 : return simplify_gen_relational (UNGE, mode, inner_mode, op0, op1);
5518 :
5519 0 : case UNEQ_EXPR:
5520 0 : return simplify_gen_relational (UNEQ, mode, inner_mode, op0, op1);
5521 :
5522 0 : case LTGT_EXPR:
5523 0 : return simplify_gen_relational (LTGT, mode, inner_mode, op0, op1);
5524 :
5525 8377 : case COND_EXPR:
5526 8377 : return gen_rtx_IF_THEN_ELSE (mode, op0, op1, op2);
5527 :
5528 10400 : case COMPLEX_EXPR:
5529 10400 : gcc_assert (COMPLEX_MODE_P (mode));
5530 10400 : if (GET_MODE (op0) == VOIDmode)
5531 0 : op0 = gen_rtx_CONST (GET_MODE_INNER (mode), op0);
5532 10400 : if (GET_MODE (op1) == VOIDmode)
5533 0 : op1 = gen_rtx_CONST (GET_MODE_INNER (mode), op1);
5534 10400 : return gen_rtx_CONCAT (mode, op0, op1);
5535 :
5536 0 : case CONJ_EXPR:
5537 0 : if (GET_CODE (op0) == CONCAT)
5538 0 : return gen_rtx_CONCAT (mode, XEXP (op0, 0),
5539 : simplify_gen_unary (NEG, GET_MODE_INNER (mode),
5540 : XEXP (op0, 1),
5541 : GET_MODE_INNER (mode)));
5542 : else
5543 : {
5544 0 : scalar_mode imode = GET_MODE_INNER (mode);
5545 0 : rtx re, im;
5546 :
5547 0 : if (MEM_P (op0))
5548 : {
5549 0 : re = adjust_address_nv (op0, imode, 0);
5550 0 : im = adjust_address_nv (op0, imode, GET_MODE_SIZE (imode));
5551 : }
5552 : else
5553 : {
5554 0 : scalar_int_mode ifmode;
5555 0 : scalar_int_mode ihmode;
5556 0 : rtx halfsize;
5557 0 : if (!int_mode_for_mode (mode).exists (&ifmode)
5558 0 : || !int_mode_for_mode (imode).exists (&ihmode))
5559 0 : return NULL;
5560 0 : halfsize = GEN_INT (GET_MODE_BITSIZE (ihmode));
5561 0 : re = op0;
5562 0 : if (mode != ifmode)
5563 0 : re = gen_rtx_SUBREG (ifmode, re, 0);
5564 0 : re = gen_rtx_ZERO_EXTRACT (ihmode, re, halfsize, const0_rtx);
5565 0 : if (imode != ihmode)
5566 0 : re = gen_rtx_SUBREG (imode, re, 0);
5567 0 : im = copy_rtx (op0);
5568 0 : if (mode != ifmode)
5569 0 : im = gen_rtx_SUBREG (ifmode, im, 0);
5570 0 : im = gen_rtx_ZERO_EXTRACT (ihmode, im, halfsize, halfsize);
5571 0 : if (imode != ihmode)
5572 0 : im = gen_rtx_SUBREG (imode, im, 0);
5573 : }
5574 0 : im = gen_rtx_NEG (imode, im);
5575 0 : return gen_rtx_CONCAT (mode, re, im);
5576 : }
5577 :
5578 6372714 : case ADDR_EXPR:
5579 6372714 : op0 = expand_debug_expr (TREE_OPERAND (exp, 0));
5580 6372714 : if (!op0 || !MEM_P (op0))
5581 : {
5582 1698441 : if ((TREE_CODE (TREE_OPERAND (exp, 0)) == VAR_DECL
5583 394326 : || TREE_CODE (TREE_OPERAND (exp, 0)) == PARM_DECL
5584 352116 : || TREE_CODE (TREE_OPERAND (exp, 0)) == RESULT_DECL)
5585 1740651 : && (!TREE_ADDRESSABLE (TREE_OPERAND (exp, 0))
5586 161474 : || target_for_debug_bind (TREE_OPERAND (exp, 0))))
5587 1192986 : return gen_rtx_DEBUG_IMPLICIT_PTR (mode, TREE_OPERAND (exp, 0));
5588 :
5589 505455 : if (handled_component_p (TREE_OPERAND (exp, 0)))
5590 : {
5591 329159 : poly_int64 bitoffset, bitsize, maxsize, byteoffset;
5592 329159 : bool reverse;
5593 329159 : tree decl
5594 329159 : = get_ref_base_and_extent (TREE_OPERAND (exp, 0), &bitoffset,
5595 : &bitsize, &maxsize, &reverse);
5596 329159 : if ((VAR_P (decl)
5597 : || TREE_CODE (decl) == PARM_DECL
5598 : || TREE_CODE (decl) == RESULT_DECL)
5599 298971 : && (!TREE_ADDRESSABLE (decl)
5600 23523 : || target_for_debug_bind (decl))
5601 69542 : && multiple_p (bitoffset, BITS_PER_UNIT, &byteoffset)
5602 275448 : && known_gt (bitsize, 0)
5603 259620 : && known_eq (bitsize, maxsize))
5604 : {
5605 259617 : rtx base = gen_rtx_DEBUG_IMPLICIT_PTR (mode, decl);
5606 259617 : return plus_constant (mode, base, byteoffset);
5607 : }
5608 : }
5609 :
5610 245838 : if (TREE_CODE (TREE_OPERAND (exp, 0)) == MEM_REF
5611 245838 : && TREE_CODE (TREE_OPERAND (TREE_OPERAND (exp, 0), 0))
5612 : == ADDR_EXPR)
5613 : {
5614 11675 : op0 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5615 : 0));
5616 11675 : if (op0 != NULL
5617 6768 : && (GET_CODE (op0) == DEBUG_IMPLICIT_PTR
5618 0 : || (GET_CODE (op0) == PLUS
5619 0 : && GET_CODE (XEXP (op0, 0)) == DEBUG_IMPLICIT_PTR
5620 0 : && CONST_INT_P (XEXP (op0, 1)))))
5621 : {
5622 6768 : op1 = expand_debug_expr (TREE_OPERAND (TREE_OPERAND (exp, 0),
5623 : 1));
5624 6768 : poly_int64 offset;
5625 13536 : if (!op1 || !poly_int_rtx_p (op1, &offset))
5626 : return NULL;
5627 :
5628 6768 : return plus_constant (mode, op0, offset);
5629 : }
5630 : }
5631 :
5632 239070 : return NULL;
5633 : }
5634 :
5635 4674273 : as = TYPE_ADDR_SPACE (TREE_TYPE (TREE_TYPE (exp)));
5636 4674273 : addr_mode = SCALAR_INT_TYPE_MODE (TREE_TYPE (exp));
5637 4674273 : op0 = convert_debug_memory_address (addr_mode, XEXP (op0, 0), as);
5638 :
5639 4674273 : return op0;
5640 :
5641 5312 : case VECTOR_CST:
5642 5312 : {
5643 5312 : unsigned HOST_WIDE_INT i, nelts;
5644 :
5645 5312 : if (!VECTOR_CST_NELTS (exp).is_constant (&nelts))
5646 : return NULL;
5647 :
5648 5312 : op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5649 :
5650 30462 : for (i = 0; i < nelts; ++i)
5651 : {
5652 25150 : op1 = expand_debug_expr (VECTOR_CST_ELT (exp, i));
5653 25150 : if (!op1)
5654 : return NULL;
5655 25150 : XVECEXP (op0, 0, i) = op1;
5656 : }
5657 :
5658 : return op0;
5659 : }
5660 :
5661 552 : case CONSTRUCTOR:
5662 552 : if (TREE_CLOBBER_P (exp))
5663 : return NULL;
5664 552 : else if (TREE_CODE (TREE_TYPE (exp)) == VECTOR_TYPE)
5665 : {
5666 552 : unsigned i;
5667 552 : poly_uint64 elts = TYPE_VECTOR_SUBPARTS (TREE_TYPE (exp));
5668 552 : unsigned HOST_WIDE_INT nelts = constant_lower_bound (elts);
5669 552 : tree val;
5670 :
5671 552 : op0 = gen_rtx_CONCATN (mode, rtvec_alloc (nelts));
5672 :
5673 2441 : FOR_EACH_CONSTRUCTOR_VALUE (CONSTRUCTOR_ELTS (exp), i, val)
5674 : {
5675 1960 : op1 = expand_debug_expr (val);
5676 1960 : if (!op1)
5677 : return NULL;
5678 1889 : XVECEXP (op0, 0, i) = op1;
5679 : }
5680 :
5681 481 : if (i < nelts)
5682 : {
5683 63 : op1 = expand_debug_expr
5684 63 : (build_zero_cst (TREE_TYPE (TREE_TYPE (exp))));
5685 :
5686 63 : if (!op1)
5687 : return NULL;
5688 :
5689 264 : for (; i < nelts; i++)
5690 201 : XVECEXP (op0, 0, i) = op1;
5691 : }
5692 :
5693 481 : return op0;
5694 : }
5695 : else
5696 0 : goto flag_unsupported;
5697 :
5698 : case CALL_EXPR:
5699 : /* ??? Maybe handle some builtins? */
5700 : return NULL;
5701 :
5702 8978542 : case SSA_NAME:
5703 8978542 : {
5704 8978542 : gimple *g = get_gimple_for_ssa_name (exp);
5705 8978542 : if (g)
5706 : {
5707 1296257 : tree t = NULL_TREE;
5708 1296257 : if (deep_ter_debug_map)
5709 : {
5710 50877 : tree *slot = deep_ter_debug_map->get (exp);
5711 50877 : if (slot)
5712 7232 : t = *slot;
5713 : }
5714 7232 : if (t == NULL_TREE)
5715 : {
5716 1291856 : if (is_gimple_assign (g))
5717 1291457 : t = gimple_assign_rhs_to_tree (g);
5718 : else
5719 : /* expand_debug_expr doesn't handle CALL_EXPR right now. */
5720 : return NULL;
5721 : }
5722 1295858 : op0 = expand_debug_expr (t);
5723 1295858 : if (!op0)
5724 : return NULL;
5725 : }
5726 : else
5727 : {
5728 : /* If this is a reference to an incoming value of
5729 : parameter that is never used in the code or where the
5730 : incoming value is never used in the code, use
5731 : PARM_DECL's DECL_RTL if set. */
5732 7682285 : if (SSA_NAME_IS_DEFAULT_DEF (exp)
5733 1251421 : && SSA_NAME_VAR (exp)
5734 1251421 : && TREE_CODE (SSA_NAME_VAR (exp)) == PARM_DECL
5735 8876474 : && has_zero_uses (exp))
5736 : {
5737 18337 : op0 = expand_debug_parm_decl (SSA_NAME_VAR (exp));
5738 18337 : if (op0)
5739 18247 : goto adjust_mode;
5740 90 : op0 = expand_debug_expr (SSA_NAME_VAR (exp));
5741 90 : if (op0)
5742 90 : goto adjust_mode;
5743 : }
5744 :
5745 7663948 : int part = var_to_partition (SA.map, exp);
5746 :
5747 7663948 : if (part == NO_PARTITION)
5748 : return NULL;
5749 :
5750 7659727 : gcc_assert (part >= 0 && (unsigned)part < SA.map->num_partitions);
5751 :
5752 7659727 : op0 = copy_rtx (SA.partition_to_pseudo[part]);
5753 : }
5754 8952162 : goto adjust_mode;
5755 : }
5756 :
5757 : case ERROR_MARK:
5758 : return NULL;
5759 :
5760 : /* Vector stuff. For most of the codes we don't have rtl codes. */
5761 : case REALIGN_LOAD_EXPR:
5762 : case VEC_COND_EXPR:
5763 : case VEC_PACK_FIX_TRUNC_EXPR:
5764 : case VEC_PACK_FLOAT_EXPR:
5765 : case VEC_PACK_SAT_EXPR:
5766 : case VEC_PACK_TRUNC_EXPR:
5767 : case VEC_UNPACK_FIX_TRUNC_HI_EXPR:
5768 : case VEC_UNPACK_FIX_TRUNC_LO_EXPR:
5769 : case VEC_UNPACK_FLOAT_HI_EXPR:
5770 : case VEC_UNPACK_FLOAT_LO_EXPR:
5771 : case VEC_UNPACK_HI_EXPR:
5772 : case VEC_UNPACK_LO_EXPR:
5773 : case VEC_WIDEN_MULT_HI_EXPR:
5774 : case VEC_WIDEN_MULT_LO_EXPR:
5775 : case VEC_WIDEN_MULT_EVEN_EXPR:
5776 : case VEC_WIDEN_MULT_ODD_EXPR:
5777 : case VEC_WIDEN_LSHIFT_HI_EXPR:
5778 : case VEC_WIDEN_LSHIFT_LO_EXPR:
5779 : case VEC_PERM_EXPR:
5780 : case VEC_DUPLICATE_EXPR:
5781 : case VEC_SERIES_EXPR:
5782 : case SAD_EXPR:
5783 : return NULL;
5784 :
5785 : /* Misc codes. */
5786 : case ADDR_SPACE_CONVERT_EXPR:
5787 : case FIXED_CONVERT_EXPR:
5788 : case OBJ_TYPE_REF:
5789 : case WITH_SIZE_EXPR:
5790 : case BIT_INSERT_EXPR:
5791 : return NULL;
5792 :
5793 0 : case DOT_PROD_EXPR:
5794 0 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
5795 0 : && SCALAR_INT_MODE_P (mode))
5796 : {
5797 0 : op0
5798 0 : = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5799 : 0)))
5800 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5801 : inner_mode);
5802 0 : op1
5803 0 : = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5804 : 1)))
5805 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op1,
5806 : inner_mode);
5807 0 : op0 = simplify_gen_binary (MULT, mode, op0, op1);
5808 0 : return simplify_gen_binary (PLUS, mode, op0, op2);
5809 : }
5810 : return NULL;
5811 :
5812 4101 : case WIDEN_MULT_EXPR:
5813 4101 : case WIDEN_MULT_PLUS_EXPR:
5814 4101 : case WIDEN_MULT_MINUS_EXPR:
5815 4101 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
5816 4101 : && SCALAR_INT_MODE_P (mode))
5817 : {
5818 4101 : inner_mode = GET_MODE (op0);
5819 4101 : if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 0))))
5820 3485 : op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5821 : else
5822 616 : op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5823 4101 : if (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp, 1))))
5824 3485 : op1 = simplify_gen_unary (ZERO_EXTEND, mode, op1, inner_mode);
5825 : else
5826 616 : op1 = simplify_gen_unary (SIGN_EXTEND, mode, op1, inner_mode);
5827 4101 : op0 = simplify_gen_binary (MULT, mode, op0, op1);
5828 4101 : if (TREE_CODE (exp) == WIDEN_MULT_EXPR)
5829 : return op0;
5830 0 : else if (TREE_CODE (exp) == WIDEN_MULT_PLUS_EXPR)
5831 0 : return simplify_gen_binary (PLUS, mode, op0, op2);
5832 : else
5833 0 : return simplify_gen_binary (MINUS, mode, op2, op0);
5834 : }
5835 : return NULL;
5836 :
5837 : case MULT_HIGHPART_EXPR:
5838 : /* ??? Similar to the above. */
5839 : return NULL;
5840 :
5841 0 : case WIDEN_SUM_EXPR:
5842 0 : case WIDEN_LSHIFT_EXPR:
5843 0 : if (SCALAR_INT_MODE_P (GET_MODE (op0))
5844 0 : && SCALAR_INT_MODE_P (mode))
5845 : {
5846 0 : op0
5847 0 : = simplify_gen_unary (TYPE_UNSIGNED (TREE_TYPE (TREE_OPERAND (exp,
5848 : 0)))
5849 : ? ZERO_EXTEND : SIGN_EXTEND, mode, op0,
5850 : inner_mode);
5851 0 : return simplify_gen_binary (TREE_CODE (exp) == WIDEN_LSHIFT_EXPR
5852 0 : ? ASHIFT : PLUS, mode, op0, op1);
5853 : }
5854 : return NULL;
5855 :
5856 0 : default:
5857 0 : flag_unsupported:
5858 0 : if (flag_checking)
5859 : {
5860 0 : debug_tree (exp);
5861 0 : gcc_unreachable ();
5862 : }
5863 : return NULL;
5864 : }
5865 : }
5866 :
5867 : /* Return an RTX equivalent to the source bind value of the tree expression
5868 : EXP. */
5869 :
5870 : static rtx
5871 156331 : expand_debug_source_expr (tree exp)
5872 : {
5873 296200 : rtx op0 = NULL_RTX;
5874 296200 : machine_mode mode = VOIDmode, inner_mode;
5875 :
5876 296200 : switch (TREE_CODE (exp))
5877 : {
5878 139869 : case VAR_DECL:
5879 139869 : if (DECL_ABSTRACT_ORIGIN (exp))
5880 139869 : return expand_debug_source_expr (DECL_ABSTRACT_ORIGIN (exp));
5881 : break;
5882 155773 : case PARM_DECL:
5883 155773 : {
5884 155773 : mode = DECL_MODE (exp);
5885 155773 : op0 = expand_debug_parm_decl (exp);
5886 155773 : if (op0)
5887 : break;
5888 : /* See if this isn't an argument that has been completely
5889 : optimized out. */
5890 154899 : if (!DECL_RTL_SET_P (exp)
5891 154882 : && !DECL_INCOMING_RTL (exp)
5892 309781 : && DECL_ABSTRACT_ORIGIN (current_function_decl))
5893 : {
5894 51319 : tree aexp = DECL_ORIGIN (exp);
5895 51319 : if (DECL_CONTEXT (aexp)
5896 51319 : == DECL_ABSTRACT_ORIGIN (current_function_decl))
5897 : {
5898 19003 : vec<tree, va_gc> **debug_args;
5899 19003 : unsigned int ix;
5900 19003 : tree ddecl;
5901 19003 : debug_args = decl_debug_args_lookup (current_function_decl);
5902 19003 : if (debug_args != NULL)
5903 : {
5904 23543 : for (ix = 0; vec_safe_iterate (*debug_args, ix, &ddecl);
5905 4544 : ix += 2)
5906 23535 : if (ddecl == aexp)
5907 18991 : return gen_rtx_DEBUG_PARAMETER_REF (mode, aexp);
5908 : }
5909 : }
5910 : }
5911 : break;
5912 : }
5913 : default:
5914 : break;
5915 : }
5916 :
5917 874 : if (op0 == NULL_RTX)
5918 136466 : return NULL_RTX;
5919 :
5920 874 : inner_mode = GET_MODE (op0);
5921 874 : if (mode == inner_mode)
5922 : return op0;
5923 :
5924 21 : if (FLOAT_MODE_P (mode) && FLOAT_MODE_P (inner_mode))
5925 : {
5926 7 : if (GET_MODE_UNIT_BITSIZE (mode)
5927 14 : == GET_MODE_UNIT_BITSIZE (inner_mode))
5928 0 : op0 = simplify_gen_subreg (mode, op0, inner_mode, 0);
5929 7 : else if (GET_MODE_UNIT_BITSIZE (mode)
5930 14 : < GET_MODE_UNIT_BITSIZE (inner_mode))
5931 7 : op0 = simplify_gen_unary (FLOAT_TRUNCATE, mode, op0, inner_mode);
5932 : else
5933 0 : op0 = simplify_gen_unary (FLOAT_EXTEND, mode, op0, inner_mode);
5934 : }
5935 14 : else if (FLOAT_MODE_P (mode))
5936 0 : gcc_unreachable ();
5937 14 : else if (FLOAT_MODE_P (inner_mode))
5938 : {
5939 0 : if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5940 0 : op0 = simplify_gen_unary (UNSIGNED_FIX, mode, op0, inner_mode);
5941 : else
5942 0 : op0 = simplify_gen_unary (FIX, mode, op0, inner_mode);
5943 : }
5944 28 : else if (GET_MODE_UNIT_PRECISION (mode)
5945 14 : == GET_MODE_UNIT_PRECISION (inner_mode))
5946 0 : op0 = lowpart_subreg (mode, op0, inner_mode);
5947 28 : else if (GET_MODE_UNIT_PRECISION (mode)
5948 14 : < GET_MODE_UNIT_PRECISION (inner_mode))
5949 14 : op0 = simplify_gen_unary (TRUNCATE, mode, op0, inner_mode);
5950 0 : else if (TYPE_UNSIGNED (TREE_TYPE (exp)))
5951 0 : op0 = simplify_gen_unary (ZERO_EXTEND, mode, op0, inner_mode);
5952 : else
5953 0 : op0 = simplify_gen_unary (SIGN_EXTEND, mode, op0, inner_mode);
5954 :
5955 : return op0;
5956 : }
5957 :
5958 : /* Ensure INSN_VAR_LOCATION_LOC (insn) doesn't have unbound complexity.
5959 : Allow 4 levels of rtl nesting for most rtl codes, and if we see anything
5960 : deeper than that, create DEBUG_EXPRs and emit DEBUG_INSNs before INSN. */
5961 :
5962 : static void
5963 47688865 : avoid_complex_debug_insns (rtx_insn *insn, rtx *exp_p, int depth)
5964 : {
5965 47688865 : rtx exp = *exp_p;
5966 :
5967 47688865 : if (exp == NULL_RTX)
5968 : return;
5969 :
5970 47688865 : if ((OBJECT_P (exp) && !MEM_P (exp)) || GET_CODE (exp) == CLOBBER)
5971 : return;
5972 :
5973 6170721 : if (depth == 4)
5974 : {
5975 : /* Create DEBUG_EXPR (and DEBUG_EXPR_DECL). */
5976 20135 : rtx dval = make_debug_expr_from_rtl (exp);
5977 :
5978 : /* Emit a debug bind insn before INSN. */
5979 20135 : rtx bind = gen_rtx_VAR_LOCATION (GET_MODE (exp),
5980 : DEBUG_EXPR_TREE_DECL (dval), exp,
5981 : VAR_INIT_STATUS_INITIALIZED);
5982 :
5983 20135 : emit_debug_insn_before (bind, insn);
5984 20135 : *exp_p = dval;
5985 20135 : return;
5986 : }
5987 :
5988 6150586 : const char *format_ptr = GET_RTX_FORMAT (GET_CODE (exp));
5989 6150586 : int i, j;
5990 18347757 : for (i = 0; i < GET_RTX_LENGTH (GET_CODE (exp)); i++)
5991 12197171 : switch (*format_ptr++)
5992 : {
5993 11178234 : case 'e':
5994 11178234 : avoid_complex_debug_insns (insn, &XEXP (exp, i), depth + 1);
5995 11178234 : break;
5996 :
5997 : case 'E':
5998 : case 'V':
5999 0 : for (j = 0; j < XVECLEN (exp, i); j++)
6000 0 : avoid_complex_debug_insns (insn, &XVECEXP (exp, i, j), depth + 1);
6001 : break;
6002 :
6003 : default:
6004 : break;
6005 : }
6006 : }
6007 :
6008 : /* Expand the _LOCs in debug insns. We run this after expanding all
6009 : regular insns, so that any variables referenced in the function
6010 : will have their DECL_RTLs set. */
6011 :
6012 : static void
6013 493440 : expand_debug_locations (void)
6014 : {
6015 493440 : rtx_insn *insn;
6016 493440 : rtx_insn *last = get_last_insn ();
6017 493440 : int save_strict_alias = flag_strict_aliasing;
6018 :
6019 : /* New alias sets while setting up memory attributes cause
6020 : -fcompare-debug failures, even though it doesn't bring about any
6021 : codegen changes. */
6022 493440 : flag_strict_aliasing = 0;
6023 :
6024 101494595 : for (insn = get_insns (); insn; insn = NEXT_INSN (insn))
6025 101001155 : if (DEBUG_BIND_INSN_P (insn))
6026 : {
6027 36490496 : tree value = (tree)INSN_VAR_LOCATION_LOC (insn);
6028 36490496 : rtx val;
6029 36490496 : rtx_insn *prev_insn, *insn2;
6030 36490496 : machine_mode mode;
6031 :
6032 36490496 : if (value == NULL_TREE)
6033 : val = NULL_RTX;
6034 : else
6035 : {
6036 20357550 : if (INSN_VAR_LOCATION_STATUS (insn)
6037 : == VAR_INIT_STATUS_UNINITIALIZED)
6038 156331 : val = expand_debug_source_expr (value);
6039 : /* The avoid_deep_ter_for_debug function inserts
6040 : debug bind stmts after SSA_NAME definition, with the
6041 : SSA_NAME as the whole bind location. Disable temporarily
6042 : expansion of that SSA_NAME into the DEBUG_EXPR_DECL
6043 : being defined in this DEBUG_INSN. */
6044 20201219 : else if (deep_ter_debug_map && TREE_CODE (value) == SSA_NAME)
6045 : {
6046 125922 : tree *slot = deep_ter_debug_map->get (value);
6047 125922 : if (slot)
6048 : {
6049 3265 : if (*slot == INSN_VAR_LOCATION_DECL (insn))
6050 2831 : *slot = NULL_TREE;
6051 : else
6052 : slot = NULL;
6053 : }
6054 125922 : val = expand_debug_expr (value);
6055 125922 : if (slot)
6056 2831 : *slot = INSN_VAR_LOCATION_DECL (insn);
6057 : }
6058 : else
6059 20075297 : val = expand_debug_expr (value);
6060 20357550 : gcc_assert (last == get_last_insn ());
6061 : }
6062 :
6063 20357550 : if (!val)
6064 16560061 : val = gen_rtx_UNKNOWN_VAR_LOC ();
6065 : else
6066 : {
6067 19930435 : mode = GET_MODE (INSN_VAR_LOCATION (insn));
6068 :
6069 19930435 : gcc_assert (mode == GET_MODE (val)
6070 : || (GET_MODE (val) == VOIDmode
6071 : && (CONST_SCALAR_INT_P (val)
6072 : || GET_CODE (val) == CONST_FIXED
6073 : || GET_CODE (val) == LABEL_REF)));
6074 : }
6075 :
6076 36490496 : INSN_VAR_LOCATION_LOC (insn) = val;
6077 36490496 : prev_insn = PREV_INSN (insn);
6078 73001127 : for (insn2 = insn; insn2 != prev_insn; insn2 = PREV_INSN (insn2))
6079 36510631 : avoid_complex_debug_insns (insn2, &INSN_VAR_LOCATION_LOC (insn2), 0);
6080 : }
6081 :
6082 493440 : flag_strict_aliasing = save_strict_alias;
6083 493440 : }
6084 :
6085 : /* Performs swapping operands of commutative operations to expand
6086 : the expensive one first. */
6087 :
6088 : static void
6089 9164932 : reorder_operands (basic_block bb)
6090 : {
6091 9164932 : unsigned int *lattice; /* Hold cost of each statement. */
6092 9164932 : unsigned int i = 0, n = 0;
6093 9164932 : gimple_stmt_iterator gsi;
6094 9164932 : gimple_seq stmts;
6095 9164932 : gimple *stmt;
6096 9164932 : bool swap;
6097 9164932 : tree op0, op1;
6098 9164932 : ssa_op_iter iter;
6099 9164932 : use_operand_p use_p;
6100 9164932 : gimple *def0, *def1;
6101 :
6102 : /* Compute cost of each statement using estimate_num_insns. */
6103 9164932 : stmts = bb_seq (bb);
6104 90341660 : for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
6105 : {
6106 81176728 : stmt = gsi_stmt (gsi);
6107 81176728 : if (!is_gimple_debug (stmt))
6108 33464356 : gimple_set_uid (stmt, n++);
6109 : }
6110 9164932 : lattice = XNEWVEC (unsigned int, n);
6111 90341660 : for (gsi = gsi_start (stmts); !gsi_end_p (gsi); gsi_next (&gsi))
6112 : {
6113 81176728 : unsigned cost;
6114 81176728 : stmt = gsi_stmt (gsi);
6115 81176728 : if (is_gimple_debug (stmt))
6116 47712372 : continue;
6117 33464356 : cost = estimate_num_insns (stmt, &eni_size_weights);
6118 33464356 : lattice[i] = cost;
6119 63665397 : FOR_EACH_SSA_USE_OPERAND (use_p, stmt, iter, SSA_OP_USE)
6120 : {
6121 30201041 : tree use = USE_FROM_PTR (use_p);
6122 30201041 : gimple *def_stmt;
6123 30201041 : if (TREE_CODE (use) != SSA_NAME)
6124 0 : continue;
6125 30201041 : def_stmt = get_gimple_for_ssa_name (use);
6126 30201041 : if (!def_stmt)
6127 21563432 : continue;
6128 8637609 : lattice[i] += lattice[gimple_uid (def_stmt)];
6129 : }
6130 33464356 : i++;
6131 33464356 : if (!is_gimple_assign (stmt)
6132 33464356 : || !commutative_tree_code (gimple_assign_rhs_code (stmt)))
6133 29383119 : continue;
6134 4081237 : op0 = gimple_op (stmt, 1);
6135 4081237 : op1 = gimple_op (stmt, 2);
6136 4081237 : if (TREE_CODE (op0) != SSA_NAME
6137 4081011 : || TREE_CODE (op1) != SSA_NAME)
6138 2590970 : continue;
6139 : /* Swap operands if the second one is more expensive. */
6140 1490267 : def0 = get_gimple_for_ssa_name (op0);
6141 1490267 : def1 = get_gimple_for_ssa_name (op1);
6142 1490267 : if (!def1)
6143 823560 : continue;
6144 666707 : swap = false;
6145 666707 : if (!def0 || lattice[gimple_uid (def1)] > lattice[gimple_uid (def0)])
6146 318585 : swap = true;
6147 318585 : if (swap)
6148 : {
6149 318585 : if (dump_file && (dump_flags & TDF_DETAILS))
6150 : {
6151 2 : fprintf (dump_file, "Swap operands in stmt:\n");
6152 2 : print_gimple_stmt (dump_file, stmt, 0, TDF_SLIM);
6153 2 : fprintf (dump_file, "Cost left opnd=%d, right opnd=%d\n",
6154 1 : def0 ? lattice[gimple_uid (def0)] : 0,
6155 2 : lattice[gimple_uid (def1)]);
6156 : }
6157 318585 : swap_ssa_operands (stmt, gimple_assign_rhs1_ptr (stmt),
6158 : gimple_assign_rhs2_ptr (stmt));
6159 : }
6160 : }
6161 9164932 : XDELETE (lattice);
6162 9164932 : }
6163 :
6164 : /* Expand basic block BB from GIMPLE trees to RTL. */
6165 :
6166 : static basic_block
6167 12504323 : expand_gimple_basic_block (basic_block bb, rtx_insn *asan_epilog_seq)
6168 : {
6169 12504323 : gimple_stmt_iterator gsi;
6170 12504323 : gimple *stmt = NULL;
6171 12504323 : rtx_note *note = NULL;
6172 12504323 : rtx_insn *last;
6173 12504323 : edge e;
6174 12504323 : edge_iterator ei;
6175 12504323 : bool nondebug_stmt_seen = false;
6176 :
6177 12504323 : if (dump_file)
6178 789 : fprintf (dump_file, "\n;; Generating RTL for gimple basic block %d\n",
6179 : bb->index);
6180 :
6181 : /* Note that since we are now transitioning from GIMPLE to RTL, we
6182 : cannot use the gsi_*_bb() routines because they expect the basic
6183 : block to be in GIMPLE, instead of RTL. Therefore, we need to
6184 : access the BB sequence directly. */
6185 12504323 : if (optimize)
6186 9164932 : reorder_operands (bb);
6187 12504323 : rtl_profile_for_bb (bb);
6188 :
6189 : /* Remove the RETURN_EXPR if we may fall though to the exit
6190 : instead. */
6191 12504323 : gsi = gsi_last_bb (bb);
6192 12504323 : if (!gsi_end_p (gsi) && gimple_code (gsi_stmt (gsi)) == GIMPLE_RETURN)
6193 : {
6194 1456723 : greturn *ret_stmt = as_a <greturn *> (gsi_stmt (gsi));
6195 :
6196 1456723 : gcc_assert (single_succ_p (bb));
6197 1456723 : gcc_assert (single_succ (bb) == EXIT_BLOCK_PTR_FOR_FN (cfun));
6198 :
6199 1456723 : if (bb->next_bb == EXIT_BLOCK_PTR_FOR_FN (cfun)
6200 1456723 : && !gimple_return_retval (ret_stmt))
6201 : {
6202 611408 : gsi_remove (&gsi, false);
6203 611408 : single_succ_edge (bb)->flags |= EDGE_FALLTHRU;
6204 : }
6205 : }
6206 :
6207 12504323 : gsi = gsi_start_bb (bb);
6208 12504323 : if (!gsi_end_p (gsi))
6209 : {
6210 11972308 : stmt = gsi_stmt (gsi);
6211 11972308 : if (gimple_code (stmt) != GIMPLE_LABEL)
6212 11889408 : stmt = NULL;
6213 : }
6214 :
6215 12504323 : rtx_code_label **elt = lab_rtx_for_bb->get (bb);
6216 12504323 : if ((unsigned) bb->index >= head_end_for_bb.length ())
6217 10630597 : head_end_for_bb.safe_grow_cleared (bb->index + 1);
6218 :
6219 12504323 : if (stmt || elt)
6220 : {
6221 5170674 : gcc_checking_assert (!note);
6222 5170674 : last = get_last_insn ();
6223 :
6224 5170674 : if (stmt)
6225 : {
6226 614915 : expand_gimple_stmt (stmt);
6227 614915 : gsi_next (&gsi);
6228 : }
6229 :
6230 5170674 : if (elt)
6231 4555759 : emit_label (*elt);
6232 :
6233 5170674 : head_end_for_bb[bb->index].first = NEXT_INSN (last);
6234 5170674 : if (NOTE_P (head_end_for_bb[bb->index].first))
6235 0 : head_end_for_bb[bb->index].first
6236 0 : = NEXT_INSN (head_end_for_bb[bb->index].first);
6237 5170674 : gcc_assert (LABEL_P (head_end_for_bb[bb->index].first));
6238 10341348 : note = emit_note_after (NOTE_INSN_BASIC_BLOCK,
6239 5170674 : head_end_for_bb[bb->index].first);
6240 :
6241 5170674 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
6242 : }
6243 : else
6244 7333649 : head_end_for_bb[bb->index].first = note = emit_note (NOTE_INSN_BASIC_BLOCK);
6245 :
6246 12504323 : if (note)
6247 12504323 : NOTE_BASIC_BLOCK (note) = bb;
6248 :
6249 64851639 : for (; !gsi_end_p (gsi); gsi_next (&gsi))
6250 : {
6251 52867741 : basic_block new_bb;
6252 :
6253 52867741 : stmt = gsi_stmt (gsi);
6254 52867741 : if (!is_gimple_debug (stmt))
6255 45077664 : nondebug_stmt_seen = true;
6256 :
6257 : /* If this statement is a non-debug one, and we generate debug
6258 : insns, then this one might be the last real use of a TERed
6259 : SSA_NAME, but where there are still some debug uses further
6260 : down. Expanding the current SSA name in such further debug
6261 : uses by their RHS might lead to wrong debug info, as coalescing
6262 : might make the operands of such RHS be placed into the same
6263 : pseudo as something else. Like so:
6264 : a_1 = a_0 + 1; // Assume a_1 is TERed and a_0 is dead
6265 : use(a_1);
6266 : a_2 = ...
6267 : #DEBUG ... => a_1
6268 : As a_0 and a_2 don't overlap in lifetime, assume they are coalesced.
6269 : If we now would expand a_1 by it's RHS (a_0 + 1) in the debug use,
6270 : the write to a_2 would actually have clobbered the place which
6271 : formerly held a_0.
6272 :
6273 : So, instead of that, we recognize the situation, and generate
6274 : debug temporaries at the last real use of TERed SSA names:
6275 : a_1 = a_0 + 1;
6276 : #DEBUG #D1 => a_1
6277 : use(a_1);
6278 : a_2 = ...
6279 : #DEBUG ... => #D1
6280 : */
6281 52867741 : if (MAY_HAVE_DEBUG_BIND_INSNS
6282 28096316 : && SA.values
6283 79762222 : && !is_gimple_debug (stmt))
6284 : {
6285 19484515 : ssa_op_iter iter;
6286 19484515 : tree op;
6287 19484515 : gimple *def;
6288 :
6289 19484515 : location_t sloc = curr_insn_location ();
6290 :
6291 : /* Look for SSA names that have their last use here (TERed
6292 : names always have only one real use). */
6293 37406177 : FOR_EACH_SSA_TREE_OPERAND (op, stmt, iter, SSA_OP_USE)
6294 17921662 : if ((def = get_gimple_for_ssa_name (op))
6295 17921662 : && is_gimple_assign (def))
6296 : {
6297 4798318 : imm_use_iterator imm_iter;
6298 4798318 : use_operand_p use_p;
6299 4798318 : bool have_debug_uses = false;
6300 :
6301 14373935 : FOR_EACH_IMM_USE_FAST (use_p, imm_iter, op)
6302 : {
6303 4818987 : if (gimple_debug_bind_p (USE_STMT (use_p)))
6304 : {
6305 : have_debug_uses = true;
6306 : break;
6307 : }
6308 4798318 : }
6309 :
6310 4798318 : if (have_debug_uses)
6311 : {
6312 : /* OP is a TERed SSA name, with DEF its defining
6313 : statement, and where OP is used in further debug
6314 : instructions. Generate a debug temporary, and
6315 : replace all uses of OP in debug insns with that
6316 : temporary. */
6317 41688 : gimple *debugstmt;
6318 41688 : tree value = gimple_assign_rhs_to_tree (def);
6319 41688 : tree vexpr = build_debug_expr_decl (TREE_TYPE (value));
6320 41688 : rtx val;
6321 41688 : machine_mode mode;
6322 :
6323 41688 : set_curr_insn_location (gimple_location (def));
6324 :
6325 41688 : if (DECL_P (value))
6326 366 : mode = DECL_MODE (value);
6327 : else
6328 41322 : mode = TYPE_MODE (TREE_TYPE (value));
6329 : /* FIXME: Is setting the mode really necessary? */
6330 41688 : SET_DECL_MODE (vexpr, mode);
6331 :
6332 41688 : val = gen_rtx_VAR_LOCATION
6333 41688 : (mode, vexpr, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
6334 :
6335 41688 : emit_debug_insn (val);
6336 :
6337 190035 : FOR_EACH_IMM_USE_STMT (debugstmt, imm_iter, op)
6338 : {
6339 106659 : if (!gimple_debug_bind_p (debugstmt))
6340 41688 : continue;
6341 :
6342 196547 : FOR_EACH_IMM_USE_ON_STMT (use_p, imm_iter)
6343 65788 : SET_USE (use_p, vexpr);
6344 :
6345 64971 : update_stmt (debugstmt);
6346 41688 : }
6347 : }
6348 : }
6349 19484515 : set_curr_insn_location (sloc);
6350 : }
6351 :
6352 52867741 : currently_expanding_gimple_stmt = stmt;
6353 :
6354 : /* Expand this statement, then evaluate the resulting RTL and
6355 : fixup the CFG accordingly. */
6356 52867741 : if (gimple_code (stmt) == GIMPLE_COND)
6357 : {
6358 5612864 : new_bb = expand_gimple_cond (bb, as_a <gcond *> (stmt));
6359 5612864 : if (new_bb)
6360 : {
6361 394721 : currently_expanding_gimple_stmt = NULL;
6362 394721 : return new_bb;
6363 : }
6364 : }
6365 47254877 : else if (is_gimple_debug (stmt))
6366 : {
6367 7790077 : location_t sloc = curr_insn_location ();
6368 7790077 : gimple_stmt_iterator nsi = gsi;
6369 :
6370 47596364 : for (;;)
6371 : {
6372 47596364 : tree var;
6373 47596364 : tree value = NULL_TREE;
6374 47596364 : rtx val = NULL_RTX;
6375 47596364 : machine_mode mode;
6376 :
6377 47596364 : if (!gimple_debug_nonbind_marker_p (stmt))
6378 : {
6379 36777332 : if (gimple_debug_bind_p (stmt))
6380 : {
6381 36621001 : var = gimple_debug_bind_get_var (stmt);
6382 :
6383 36621001 : if (TREE_CODE (var) != DEBUG_EXPR_DECL
6384 33949716 : && TREE_CODE (var) != LABEL_DECL
6385 70560754 : && !target_for_debug_bind (var))
6386 328524 : goto delink_debug_stmt;
6387 :
6388 36292477 : if (DECL_P (var) && !VECTOR_TYPE_P (TREE_TYPE (var)))
6389 36251684 : mode = DECL_MODE (var);
6390 : else
6391 40793 : mode = TYPE_MODE (TREE_TYPE (var));
6392 :
6393 36292477 : if (gimple_debug_bind_has_value_p (stmt))
6394 20159531 : value = gimple_debug_bind_get_value (stmt);
6395 :
6396 36292477 : val = gen_rtx_VAR_LOCATION
6397 36292477 : (mode, var, (rtx)value, VAR_INIT_STATUS_INITIALIZED);
6398 : }
6399 156331 : else if (gimple_debug_source_bind_p (stmt))
6400 : {
6401 156331 : var = gimple_debug_source_bind_get_var (stmt);
6402 :
6403 156331 : value = gimple_debug_source_bind_get_value (stmt);
6404 :
6405 156331 : if (!VECTOR_TYPE_P (TREE_TYPE (var)))
6406 156331 : mode = DECL_MODE (var);
6407 : else
6408 0 : mode = TYPE_MODE (TREE_TYPE (var));
6409 :
6410 156331 : val = gen_rtx_VAR_LOCATION (mode, var, (rtx)value,
6411 : VAR_INIT_STATUS_UNINITIALIZED);
6412 : }
6413 : else
6414 0 : gcc_unreachable ();
6415 : }
6416 : /* If this function was first compiled with markers
6417 : enabled, but they're now disable (e.g. LTO), drop
6418 : them on the floor. */
6419 10819032 : else if (gimple_debug_nonbind_marker_p (stmt)
6420 10819032 : && !MAY_HAVE_DEBUG_MARKER_INSNS)
6421 0 : goto delink_debug_stmt;
6422 10819032 : else if (gimple_debug_begin_stmt_p (stmt))
6423 3909007 : val = GEN_RTX_DEBUG_MARKER_BEGIN_STMT_PAT ();
6424 6910025 : else if (gimple_debug_inline_entry_p (stmt))
6425 6910025 : val = GEN_RTX_DEBUG_MARKER_INLINE_ENTRY_PAT ();
6426 : else
6427 : gcc_unreachable ();
6428 :
6429 47267840 : last = get_last_insn ();
6430 :
6431 47267840 : set_curr_insn_location (gimple_location (stmt));
6432 :
6433 47267840 : emit_debug_insn (val);
6434 :
6435 47267840 : if (dump_file && (dump_flags & TDF_DETAILS))
6436 : {
6437 : /* We can't dump the insn with a TREE where an RTX
6438 : is expected. */
6439 1 : if (GET_CODE (val) == VAR_LOCATION)
6440 : {
6441 0 : gcc_checking_assert (PAT_VAR_LOCATION_LOC (val) == (rtx)value);
6442 0 : PAT_VAR_LOCATION_LOC (val) = const0_rtx;
6443 : }
6444 1 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
6445 1 : if (GET_CODE (val) == VAR_LOCATION)
6446 0 : PAT_VAR_LOCATION_LOC (val) = (rtx)value;
6447 : }
6448 :
6449 47596364 : delink_debug_stmt:
6450 : /* In order not to generate too many debug temporaries,
6451 : we delink all uses of debug statements we already expanded.
6452 : Therefore debug statements between definition and real
6453 : use of TERed SSA names will continue to use the SSA name,
6454 : and not be replaced with debug temps. */
6455 47596364 : delink_stmt_imm_use (stmt);
6456 :
6457 47596364 : gsi = nsi;
6458 47596364 : gsi_next (&nsi);
6459 47596364 : if (gsi_end_p (nsi))
6460 : break;
6461 46976882 : stmt = gsi_stmt (nsi);
6462 46976882 : if (!is_gimple_debug (stmt))
6463 : break;
6464 : }
6465 :
6466 7790077 : set_curr_insn_location (sloc);
6467 : }
6468 : else
6469 : {
6470 39464800 : gcall *call_stmt = dyn_cast <gcall *> (stmt);
6471 39464800 : if (call_stmt
6472 39464800 : && asan_epilog_seq
6473 21482 : && gimple_call_tail_p (call_stmt)
6474 39464937 : && !gimple_call_must_tail_p (call_stmt))
6475 20 : gimple_call_set_tail (call_stmt, false);
6476 :
6477 39464800 : if (call_stmt && gimple_call_tail_p (call_stmt))
6478 : {
6479 175542 : bool can_fallthru;
6480 175542 : new_bb = expand_gimple_tailcall (bb, call_stmt, &can_fallthru,
6481 : asan_epilog_seq);
6482 175542 : if (new_bb)
6483 : {
6484 125702 : if (can_fallthru)
6485 0 : bb = new_bb;
6486 : else
6487 : {
6488 125702 : currently_expanding_gimple_stmt = NULL;
6489 125702 : return new_bb;
6490 : }
6491 : }
6492 : }
6493 : else
6494 : {
6495 39289258 : def_operand_p def_p;
6496 39289258 : def_p = SINGLE_SSA_DEF_OPERAND (stmt, SSA_OP_DEF);
6497 :
6498 39289258 : if (def_p != NULL)
6499 : {
6500 : /* Ignore this stmt if it is in the list of
6501 : replaceable expressions. */
6502 33059754 : if (SA.values
6503 41043930 : && bitmap_bit_p (SA.values,
6504 16621672 : SSA_NAME_VERSION (DEF_FROM_PTR (def_p))))
6505 8637496 : continue;
6506 : }
6507 30651762 : last = expand_gimple_stmt (stmt);
6508 30651760 : maybe_dump_rtl_for_gimple_stmt (stmt, last);
6509 : }
6510 : }
6511 : }
6512 :
6513 11983898 : currently_expanding_gimple_stmt = NULL;
6514 :
6515 : /* Expand implicit goto and convert goto_locus. */
6516 29085783 : FOR_EACH_EDGE (e, ei, bb->succs)
6517 : {
6518 17101885 : if (e->goto_locus != UNKNOWN_LOCATION || !nondebug_stmt_seen)
6519 3378414 : set_curr_insn_location (e->goto_locus);
6520 17101885 : if ((e->flags & EDGE_FALLTHRU) && e->dest != bb->next_bb)
6521 : {
6522 1577427 : emit_jump (label_rtx_for_bb (e->dest));
6523 1577427 : e->flags &= ~EDGE_FALLTHRU;
6524 : }
6525 : }
6526 :
6527 : /* Expanded RTL can create a jump in the last instruction of block.
6528 : This later might be assumed to be a jump to successor and break edge insertion.
6529 : We need to insert dummy move to prevent this. PR41440. */
6530 11983898 : if (single_succ_p (bb)
6531 5411556 : && (single_succ_edge (bb)->flags & EDGE_FALLTHRU)
6532 3081664 : && (last = get_last_insn ())
6533 14886164 : && (JUMP_P (last)
6534 3081616 : || (DEBUG_INSN_P (last)
6535 354391 : && JUMP_P (prev_nondebug_insn (last)))))
6536 : {
6537 59 : rtx dummy = gen_reg_rtx (SImode);
6538 59 : emit_insn_after_noloc (gen_move_insn (dummy, dummy), last, NULL);
6539 : }
6540 :
6541 : /* A __builtin_unreachable () will insert a barrier that should end
6542 : the basic block. In gimple, any code after it will have already
6543 : deleted, even without optimization. If we emit additional code
6544 : here, as we would to adjust the stack after a call, it should be
6545 : eventually deleted, but it confuses internal checkers (PR118006)
6546 : and optimizers before it does, because we don't expect to find
6547 : barriers inside basic blocks. */
6548 11983898 : if (!BARRIER_P (get_last_insn ()))
6549 8759003 : do_pending_stack_adjust ();
6550 : else
6551 3224895 : discard_pending_stack_adjust ();
6552 :
6553 : /* Find the block tail. The last insn in the block is the insn
6554 : before a barrier and/or table jump insn. */
6555 11983898 : last = get_last_insn ();
6556 11983898 : if (BARRIER_P (last))
6557 3224895 : last = PREV_INSN (last);
6558 11983898 : if (JUMP_TABLE_DATA_P (last))
6559 7133 : last = PREV_INSN (PREV_INSN (last));
6560 11983898 : if (BARRIER_P (last))
6561 5918 : last = PREV_INSN (last);
6562 11983898 : head_end_for_bb[bb->index].second = last;
6563 :
6564 23967796 : update_bb_for_insn_chain (head_end_for_bb[bb->index].first,
6565 11983898 : head_end_for_bb[bb->index].second, bb);
6566 :
6567 11983898 : return bb;
6568 : }
6569 :
6570 :
6571 : /* Create a basic block for initialization code. */
6572 :
6573 : static basic_block
6574 1489150 : construct_init_block (void)
6575 : {
6576 1489150 : basic_block init_block;
6577 1489150 : edge e = NULL;
6578 1489150 : int flags;
6579 :
6580 : /* Multiple entry points not supported yet. */
6581 1489150 : gcc_assert (EDGE_COUNT (ENTRY_BLOCK_PTR_FOR_FN (cfun)->succs) == 1);
6582 1489150 : init_rtl_bb_info (ENTRY_BLOCK_PTR_FOR_FN (cfun));
6583 1489150 : init_rtl_bb_info (EXIT_BLOCK_PTR_FOR_FN (cfun));
6584 1489150 : ENTRY_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6585 1489150 : EXIT_BLOCK_PTR_FOR_FN (cfun)->flags |= BB_RTL;
6586 :
6587 1489150 : e = EDGE_SUCC (ENTRY_BLOCK_PTR_FOR_FN (cfun), 0);
6588 :
6589 : /* When entry edge points to first basic block, we don't need jump,
6590 : otherwise we have to jump into proper target. */
6591 1489150 : if (e && e->dest != ENTRY_BLOCK_PTR_FOR_FN (cfun)->next_bb)
6592 : {
6593 0 : tree label = gimple_block_label (e->dest);
6594 :
6595 0 : emit_jump (jump_target_rtx (label));
6596 0 : flags = 0;
6597 0 : }
6598 : else
6599 : flags = EDGE_FALLTHRU;
6600 :
6601 1489150 : init_block = create_basic_block (NEXT_INSN (get_insns ()),
6602 : get_last_insn (),
6603 1489150 : ENTRY_BLOCK_PTR_FOR_FN (cfun));
6604 1489150 : init_block->count = ENTRY_BLOCK_PTR_FOR_FN (cfun)->count;
6605 1489150 : add_bb_to_loop (init_block, ENTRY_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6606 1489150 : if (e)
6607 1489150 : expand_split_edge (e, init_block, flags);
6608 : else
6609 0 : make_single_succ_edge (init_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6610 : EDGE_FALLTHRU);
6611 :
6612 1489150 : update_bb_for_insn (init_block);
6613 1489150 : return init_block;
6614 : }
6615 :
6616 : /* For each lexical block, set BLOCK_NUMBER to the depth at which it is
6617 : found in the block tree. */
6618 :
6619 : static void
6620 17468943 : set_block_levels (tree block, int level)
6621 : {
6622 33448738 : while (block)
6623 : {
6624 15979795 : BLOCK_NUMBER (block) = level;
6625 15979795 : set_block_levels (BLOCK_SUBBLOCKS (block), level + 1);
6626 15979795 : block = BLOCK_CHAIN (block);
6627 : }
6628 17468943 : }
6629 :
6630 : /* Create a block containing landing pads and similar stuff. */
6631 :
6632 : static void
6633 1489148 : construct_exit_block (void)
6634 : {
6635 1489148 : rtx_insn *head = get_last_insn ();
6636 1489148 : rtx_insn *end;
6637 1489148 : basic_block exit_block;
6638 1489148 : edge e, e2;
6639 1489148 : unsigned ix;
6640 1489148 : edge_iterator ei;
6641 1489148 : basic_block prev_bb = EXIT_BLOCK_PTR_FOR_FN (cfun)->prev_bb;
6642 1489148 : rtx_insn *orig_end = BB_END (prev_bb);
6643 :
6644 1489148 : rtl_profile_for_bb (EXIT_BLOCK_PTR_FOR_FN (cfun));
6645 :
6646 : /* Make sure the locus is set to the end of the function, so that
6647 : epilogue line numbers and warnings are set properly. */
6648 1489148 : if (LOCATION_LOCUS (cfun->function_end_locus) != UNKNOWN_LOCATION)
6649 1170362 : input_location = cfun->function_end_locus;
6650 :
6651 : /* Generate rtl for function exit. */
6652 1489148 : expand_function_end ();
6653 :
6654 1489148 : end = get_last_insn ();
6655 1489148 : if (head == end)
6656 0 : return;
6657 : /* While emitting the function end we could move end of the last basic
6658 : block. */
6659 1489148 : BB_END (prev_bb) = orig_end;
6660 1489148 : while (NEXT_INSN (head) && NOTE_P (NEXT_INSN (head)))
6661 : head = NEXT_INSN (head);
6662 : /* But make sure exit_block starts with RETURN_LABEL, otherwise the
6663 : bb count counting will be confused. Any instructions before that
6664 : label are emitted for the case where PREV_BB falls through into the
6665 : exit block, so append those instructions to prev_bb in that case. */
6666 1489148 : if (NEXT_INSN (head) != return_label)
6667 : {
6668 14854 : while (NEXT_INSN (head) != return_label)
6669 : {
6670 9862 : if (!NOTE_P (NEXT_INSN (head)))
6671 9862 : BB_END (prev_bb) = NEXT_INSN (head);
6672 9862 : head = NEXT_INSN (head);
6673 : }
6674 : }
6675 1489148 : exit_block = create_basic_block (NEXT_INSN (head), end, prev_bb);
6676 1489148 : exit_block->count = EXIT_BLOCK_PTR_FOR_FN (cfun)->count;
6677 1489148 : add_bb_to_loop (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun)->loop_father);
6678 :
6679 1489148 : ix = 0;
6680 4494677 : while (ix < EDGE_COUNT (EXIT_BLOCK_PTR_FOR_FN (cfun)->preds))
6681 : {
6682 1516381 : e = EDGE_PRED (EXIT_BLOCK_PTR_FOR_FN (cfun), ix);
6683 1516381 : if (!(e->flags & EDGE_ABNORMAL))
6684 1390679 : redirect_edge_succ (e, exit_block);
6685 : else
6686 125702 : ix++;
6687 : }
6688 :
6689 1489148 : e = make_single_succ_edge (exit_block, EXIT_BLOCK_PTR_FOR_FN (cfun),
6690 : EDGE_FALLTHRU);
6691 3103998 : FOR_EACH_EDGE (e2, ei, EXIT_BLOCK_PTR_FOR_FN (cfun)->preds)
6692 1614850 : if (e2 != e)
6693 : {
6694 125702 : exit_block->count -= e2->count ();
6695 : }
6696 1489148 : update_bb_for_insn (exit_block);
6697 : }
6698 :
6699 : /* Helper function for discover_nonconstant_array_refs.
6700 : Look for ARRAY_REF nodes with non-constant indexes and mark them
6701 : addressable. */
6702 :
6703 : static tree
6704 161681906 : discover_nonconstant_array_refs_r (tree * tp, int *walk_subtrees,
6705 : void *data)
6706 : {
6707 161681906 : tree t = *tp;
6708 161681906 : bitmap forced_stack_vars = (bitmap)((walk_stmt_info *)data)->info;
6709 :
6710 161681906 : if (IS_TYPE_OR_DECL_P (t))
6711 33188606 : *walk_subtrees = 0;
6712 128493300 : else if (REFERENCE_CLASS_P (t) && TREE_THIS_VOLATILE (t))
6713 : {
6714 71107 : t = get_base_address (t);
6715 71107 : if (t && DECL_P (t)
6716 58592 : && DECL_MODE (t) != BLKmode
6717 107830 : && !TREE_ADDRESSABLE (t))
6718 32680 : bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6719 71107 : *walk_subtrees = 0;
6720 : }
6721 128422193 : else if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6722 : {
6723 6214454 : while (((TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6724 2980544 : && is_gimple_min_invariant (TREE_OPERAND (t, 1))
6725 2510873 : && (!TREE_OPERAND (t, 2)
6726 37196 : || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6727 3737644 : || (TREE_CODE (t) == COMPONENT_REF
6728 918001 : && (!TREE_OPERAND (t,2)
6729 0 : || is_gimple_min_invariant (TREE_OPERAND (t, 2))))
6730 2819643 : || TREE_CODE (t) == BIT_FIELD_REF
6731 : || TREE_CODE (t) == REALPART_EXPR
6732 : || TREE_CODE (t) == IMAGPART_EXPR
6733 : || TREE_CODE (t) == VIEW_CONVERT_EXPR
6734 6214454 : || CONVERT_EXPR_P (t))
6735 3404789 : t = TREE_OPERAND (t, 0);
6736 :
6737 2809665 : if (TREE_CODE (t) == ARRAY_REF || TREE_CODE (t) == ARRAY_RANGE_REF)
6738 : {
6739 503734 : t = get_base_address (t);
6740 503734 : if (t && DECL_P (t)
6741 405115 : && DECL_MODE (t) != BLKmode
6742 525244 : && !TREE_ADDRESSABLE (t))
6743 12414 : bitmap_set_bit (forced_stack_vars, DECL_UID (t));
6744 : }
6745 :
6746 2809665 : *walk_subtrees = 0;
6747 : }
6748 : /* References of size POLY_INT_CST to a fixed-size object must go
6749 : through memory. It's more efficient to force that here than
6750 : to create temporary slots on the fly.
6751 : RTL expansion expects TARGET_MEM_REF to always address actual memory.
6752 : Also, force to stack non-BLKmode vars accessed through VIEW_CONVERT_EXPR
6753 : to BLKmode type. */
6754 125612528 : else if (TREE_CODE (t) == TARGET_MEM_REF
6755 : || (TREE_CODE (t) == MEM_REF
6756 124766525 : && TYPE_SIZE (TREE_TYPE (t))
6757 : && POLY_INT_CST_P (TYPE_SIZE (TREE_TYPE (t))))
6758 250379053 : || (TREE_CODE (t) == VIEW_CONVERT_EXPR
6759 325910 : && TYPE_MODE (TREE_TYPE (t)) == BLKmode))
6760 : {
6761 924572 : tree base = get_base_address (t);
6762 924572 : if (base
6763 924572 : && DECL_P (base)
6764 272830 : && !TREE_ADDRESSABLE (base)
6765 124671 : && DECL_MODE (base) != BLKmode
6766 930289 : && GET_MODE_SIZE (DECL_MODE (base)).is_constant ())
6767 5717 : bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6768 924572 : *walk_subtrees = 0;
6769 : }
6770 :
6771 161681906 : return NULL_TREE;
6772 : }
6773 :
6774 : /* If there's a chance to get a pseudo for t then if it would be of float mode
6775 : and the actual access is via an integer mode (lowered memcpy or similar
6776 : access) then avoid the register expansion if the mode likely is not storage
6777 : suitable for raw bits processing (like XFmode on i?86). */
6778 :
6779 : static void
6780 6571228 : avoid_type_punning_on_regs (tree t, bitmap forced_stack_vars)
6781 : {
6782 6571228 : machine_mode access_mode = TYPE_MODE (TREE_TYPE (t));
6783 6571228 : if (access_mode != BLKmode
6784 6403199 : && !SCALAR_INT_MODE_P (access_mode))
6785 : return;
6786 5596706 : tree base = get_base_address (t);
6787 5596706 : if (DECL_P (base)
6788 4147103 : && !TREE_ADDRESSABLE (base)
6789 1227887 : && FLOAT_MODE_P (DECL_MODE (base))
6790 71 : && maybe_lt (GET_MODE_PRECISION (DECL_MODE (base)),
6791 213 : GET_MODE_BITSIZE (GET_MODE_INNER (DECL_MODE (base))))
6792 : /* Double check in the expensive way we really would get a pseudo. */
6793 5596711 : && use_register_for_decl (base))
6794 5 : bitmap_set_bit (forced_stack_vars, DECL_UID (base));
6795 : }
6796 :
6797 : /* RTL expansion is not able to compile array references with variable
6798 : offsets for arrays stored in single register. Discover such
6799 : expressions and mark variables as addressable to avoid this
6800 : scenario. */
6801 :
6802 : static void
6803 1489150 : discover_nonconstant_array_refs (bitmap forced_stack_vars)
6804 : {
6805 1489150 : basic_block bb;
6806 1489150 : gimple_stmt_iterator gsi;
6807 :
6808 1489150 : walk_stmt_info wi = {};
6809 1489150 : wi.info = forced_stack_vars;
6810 13990996 : FOR_EACH_BB_FN (bb, cfun)
6811 119039718 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
6812 : {
6813 94036026 : gimple *stmt = gsi_stmt (gsi);
6814 94036026 : if (!is_gimple_debug (stmt))
6815 : {
6816 46323638 : walk_gimple_op (stmt, discover_nonconstant_array_refs_r, &wi);
6817 46323638 : gcall *call = dyn_cast <gcall *> (stmt);
6818 6861341 : if (call && gimple_call_internal_p (call))
6819 : {
6820 225107 : tree cand = NULL_TREE;
6821 225107 : switch (gimple_call_internal_fn (call))
6822 : {
6823 0 : case IFN_LOAD_LANES:
6824 : /* The source must be a MEM. */
6825 0 : cand = gimple_call_arg (call, 0);
6826 0 : break;
6827 0 : case IFN_STORE_LANES:
6828 : /* The destination must be a MEM. */
6829 0 : cand = gimple_call_lhs (call);
6830 0 : break;
6831 : default:
6832 : break;
6833 : }
6834 0 : if (cand)
6835 0 : cand = get_base_address (cand);
6836 0 : if (cand
6837 0 : && DECL_P (cand)
6838 0 : && use_register_for_decl (cand))
6839 0 : bitmap_set_bit (forced_stack_vars, DECL_UID (cand));
6840 : }
6841 149457712 : if (gimple_vdef (stmt))
6842 : {
6843 15347908 : tree t = gimple_get_lhs (stmt);
6844 15347908 : if (t && REFERENCE_CLASS_P (t))
6845 6571228 : avoid_type_punning_on_regs (t, forced_stack_vars);
6846 : }
6847 : }
6848 : }
6849 1489150 : }
6850 :
6851 : /* This function sets crtl->args.internal_arg_pointer to a virtual
6852 : register if DRAP is needed. Local register allocator will replace
6853 : virtual_incoming_args_rtx with the virtual register. */
6854 :
6855 : static void
6856 1489148 : expand_stack_alignment (void)
6857 : {
6858 1489148 : rtx drap_rtx;
6859 1489148 : unsigned int preferred_stack_boundary;
6860 :
6861 1489148 : if (! SUPPORTS_STACK_ALIGNMENT)
6862 : return;
6863 :
6864 1489148 : if (cfun->calls_alloca
6865 1473642 : || cfun->has_nonlocal_label
6866 1472840 : || crtl->has_nonlocal_goto)
6867 16711 : crtl->need_drap = true;
6868 :
6869 : /* Call update_stack_boundary here again to update incoming stack
6870 : boundary. It may set incoming stack alignment to a different
6871 : value after RTL expansion. TARGET_FUNCTION_OK_FOR_SIBCALL may
6872 : use the minimum incoming stack alignment to check if it is OK
6873 : to perform sibcall optimization since sibcall optimization will
6874 : only align the outgoing stack to incoming stack boundary. */
6875 1489148 : if (targetm.calls.update_stack_boundary)
6876 1489148 : targetm.calls.update_stack_boundary ();
6877 :
6878 : /* The incoming stack frame has to be aligned at least at
6879 : parm_stack_boundary. */
6880 1489148 : gcc_assert (crtl->parm_stack_boundary <= INCOMING_STACK_BOUNDARY);
6881 :
6882 : /* Update crtl->stack_alignment_estimated and use it later to align
6883 : stack. We check PREFERRED_STACK_BOUNDARY if there may be non-call
6884 : exceptions since callgraph doesn't collect incoming stack alignment
6885 : in this case. */
6886 1489148 : if (cfun->can_throw_non_call_exceptions
6887 262983 : && PREFERRED_STACK_BOUNDARY > crtl->preferred_stack_boundary)
6888 : preferred_stack_boundary = PREFERRED_STACK_BOUNDARY;
6889 : else
6890 1461167 : preferred_stack_boundary = crtl->preferred_stack_boundary;
6891 1489148 : if (preferred_stack_boundary > crtl->stack_alignment_estimated)
6892 718703 : crtl->stack_alignment_estimated = preferred_stack_boundary;
6893 1489148 : if (preferred_stack_boundary > crtl->stack_alignment_needed)
6894 667363 : crtl->stack_alignment_needed = preferred_stack_boundary;
6895 :
6896 1489148 : gcc_assert (crtl->stack_alignment_needed
6897 : <= crtl->stack_alignment_estimated);
6898 :
6899 1489148 : crtl->stack_realign_needed
6900 1489148 : = INCOMING_STACK_BOUNDARY < crtl->stack_alignment_estimated;
6901 1489148 : crtl->stack_realign_tried = crtl->stack_realign_needed;
6902 :
6903 1489148 : crtl->stack_realign_processed = true;
6904 :
6905 : /* Target has to redefine TARGET_GET_DRAP_RTX to support stack
6906 : alignment. */
6907 1489148 : gcc_assert (targetm.calls.get_drap_rtx != NULL);
6908 1489148 : drap_rtx = targetm.calls.get_drap_rtx ();
6909 :
6910 : /* stack_realign_drap and drap_rtx must match. */
6911 1489148 : gcc_assert ((stack_realign_drap != 0) == (drap_rtx != NULL));
6912 :
6913 : /* Do nothing if NULL is returned, which means DRAP is not needed. */
6914 1489148 : if (drap_rtx != NULL)
6915 : {
6916 7295 : crtl->args.internal_arg_pointer = drap_rtx;
6917 :
6918 : /* Call fixup_tail_calls to clean up REG_EQUIV note if DRAP is
6919 : needed. */
6920 7295 : fixup_tail_calls ();
6921 : }
6922 : }
6923 :
6924 :
6925 : static void
6926 0 : expand_main_function (void)
6927 : {
6928 : #if (defined(INVOKE__main) \
6929 : || (!defined(HAS_INIT_SECTION) \
6930 : && !defined(INIT_SECTION_ASM_OP) \
6931 : && !defined(INIT_ARRAY_SECTION_ASM_OP)))
6932 : emit_library_call (init_one_libfunc (NAME__MAIN), LCT_NORMAL, VOIDmode);
6933 : #endif
6934 0 : }
6935 :
6936 :
6937 : /* Expand code to initialize the stack_protect_guard. This is invoked at
6938 : the beginning of a function to be protected. */
6939 :
6940 : static void
6941 301 : stack_protect_prologue (void)
6942 : {
6943 301 : tree guard_decl = targetm.stack_protect_guard ();
6944 301 : rtx x, y;
6945 :
6946 301 : crtl->stack_protect_guard_decl = guard_decl;
6947 301 : x = expand_normal (crtl->stack_protect_guard);
6948 :
6949 301 : if (targetm.have_stack_protect_combined_set () && guard_decl)
6950 : {
6951 0 : gcc_assert (DECL_P (guard_decl));
6952 0 : y = DECL_RTL (guard_decl);
6953 :
6954 : /* Allow the target to compute address of Y and copy it to X without
6955 : leaking Y into a register. This combined address + copy pattern
6956 : allows the target to prevent spilling of any intermediate results by
6957 : splitting it after register allocator. */
6958 0 : if (rtx_insn *insn = targetm.gen_stack_protect_combined_set (x, y))
6959 : {
6960 0 : emit_insn (insn);
6961 0 : return;
6962 : }
6963 : }
6964 :
6965 301 : if (guard_decl)
6966 301 : y = expand_normal (guard_decl);
6967 : else
6968 0 : y = const0_rtx;
6969 :
6970 : /* Allow the target to copy from Y to X without leaking Y into a
6971 : register. */
6972 301 : if (targetm.have_stack_protect_set ())
6973 301 : if (rtx_insn *insn = targetm.gen_stack_protect_set (x, y))
6974 : {
6975 301 : emit_insn (insn);
6976 301 : return;
6977 : }
6978 :
6979 : /* Otherwise do a straight move. */
6980 0 : emit_move_insn (x, y);
6981 : }
6982 :
6983 : /* Translate the intermediate representation contained in the CFG
6984 : from GIMPLE trees to RTL.
6985 :
6986 : We do conversion per basic block and preserve/update the tree CFG.
6987 : This implies we have to do some magic as the CFG can simultaneously
6988 : consist of basic blocks containing RTL and GIMPLE trees. This can
6989 : confuse the CFG hooks, so be careful to not manipulate CFG during
6990 : the expansion. */
6991 :
6992 : namespace {
6993 :
6994 : const pass_data pass_data_expand =
6995 : {
6996 : RTL_PASS, /* type */
6997 : "expand", /* name */
6998 : OPTGROUP_NONE, /* optinfo_flags */
6999 : TV_EXPAND, /* tv_id */
7000 : ( PROP_ssa | PROP_gimple_leh | PROP_cfg
7001 : | PROP_gimple_lcx
7002 : | PROP_gimple_lvec
7003 : | PROP_gimple_lva), /* properties_required */
7004 : PROP_rtl, /* properties_provided */
7005 : ( PROP_ssa | PROP_gimple ), /* properties_destroyed */
7006 : 0, /* todo_flags_start */
7007 : 0, /* todo_flags_finish */
7008 : };
7009 :
7010 : class pass_expand : public rtl_opt_pass
7011 : {
7012 : public:
7013 298832 : pass_expand (gcc::context *ctxt)
7014 597664 : : rtl_opt_pass (pass_data_expand, ctxt)
7015 : {}
7016 :
7017 : /* opt_pass methods: */
7018 : unsigned int execute (function *) final override;
7019 :
7020 : }; // class pass_expand
7021 :
7022 : unsigned int
7023 1489150 : pass_expand::execute (function *fun)
7024 : {
7025 1489150 : basic_block bb, init_block;
7026 1489150 : edge_iterator ei;
7027 1489150 : edge e;
7028 1489150 : rtx_insn *var_seq, *var_ret_seq;
7029 1489150 : unsigned i;
7030 :
7031 1489150 : timevar_push (TV_OUT_OF_SSA);
7032 1489150 : rewrite_out_of_ssa (&SA);
7033 1489150 : timevar_pop (TV_OUT_OF_SSA);
7034 1489150 : SA.partition_to_pseudo = XCNEWVEC (rtx, SA.map->num_partitions);
7035 :
7036 1489150 : if (MAY_HAVE_DEBUG_BIND_STMTS && flag_tree_ter)
7037 : {
7038 493415 : gimple_stmt_iterator gsi;
7039 6605139 : FOR_EACH_BB_FN (bb, cfun)
7040 80667315 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
7041 105165354 : if (gimple_debug_bind_p (gsi_stmt (gsi)))
7042 36721487 : avoid_deep_ter_for_debug (gsi_stmt (gsi), 0);
7043 : }
7044 :
7045 : /* Mark arrays indexed with non-constant indices with TREE_ADDRESSABLE. */
7046 1489150 : auto_bitmap forced_stack_vars;
7047 1489150 : discover_nonconstant_array_refs (forced_stack_vars);
7048 :
7049 : /* Make sure all values used by the optimization passes have sane
7050 : defaults. */
7051 1489150 : reg_renumber = 0;
7052 :
7053 : /* Some backends want to know that we are expanding to RTL. */
7054 1489150 : currently_expanding_to_rtl = 1;
7055 : /* Dominators are not kept up-to-date as we may create new basic-blocks. */
7056 1489150 : free_dominance_info (CDI_DOMINATORS);
7057 :
7058 1489150 : rtl_profile_for_bb (ENTRY_BLOCK_PTR_FOR_FN (fun));
7059 :
7060 1489150 : insn_locations_init ();
7061 1489150 : if (!DECL_IS_UNDECLARED_BUILTIN (current_function_decl))
7062 : {
7063 : /* Eventually, all FEs should explicitly set function_start_locus. */
7064 1366302 : if (LOCATION_LOCUS (fun->function_start_locus) == UNKNOWN_LOCATION)
7065 637504 : set_curr_insn_location
7066 637504 : (DECL_SOURCE_LOCATION (current_function_decl));
7067 : else
7068 728798 : set_curr_insn_location (fun->function_start_locus);
7069 : }
7070 : else
7071 122848 : set_curr_insn_location (UNKNOWN_LOCATION);
7072 1489150 : prologue_location = curr_insn_location ();
7073 :
7074 : #ifdef INSN_SCHEDULING
7075 1489150 : init_sched_attrs ();
7076 : #endif
7077 :
7078 : /* Make sure first insn is a note even if we don't want linenums.
7079 : This makes sure the first insn will never be deleted.
7080 : Also, final expects a note to appear there. */
7081 1489150 : emit_note (NOTE_INSN_DELETED);
7082 :
7083 1489150 : targetm.expand_to_rtl_hook ();
7084 1489150 : crtl->init_stack_alignment ();
7085 1489150 : fun->cfg->max_jumptable_ents = 0;
7086 :
7087 : /* Resolve the function section. Some targets, like ARM EABI rely on knowledge
7088 : of the function section at expansion time to predict distance of calls. */
7089 1489150 : resolve_unique_section (current_function_decl, 0, flag_function_sections);
7090 :
7091 : /* Expand the variables recorded during gimple lowering. */
7092 1489150 : timevar_push (TV_VAR_EXPAND);
7093 1489150 : start_sequence ();
7094 :
7095 1489150 : var_ret_seq = expand_used_vars (forced_stack_vars);
7096 :
7097 1489150 : var_seq = end_sequence ();
7098 1489150 : timevar_pop (TV_VAR_EXPAND);
7099 :
7100 : /* Honor stack protection warnings. */
7101 1489150 : if (warn_stack_protect)
7102 : {
7103 0 : if (fun->calls_alloca)
7104 0 : warning (OPT_Wstack_protector,
7105 : "stack protector not protecting local variables: "
7106 : "variable length buffer");
7107 0 : if (has_short_buffer && !crtl->stack_protect_guard)
7108 0 : warning (OPT_Wstack_protector,
7109 : "stack protector not protecting function: "
7110 : "all local arrays are less than %d bytes long",
7111 0 : (int) param_ssp_buffer_size);
7112 : }
7113 :
7114 : /* Temporarily mark PARM_DECLs and RESULT_DECLs we need to expand to
7115 : memory addressable so expand_function_start can emit the required
7116 : copies. */
7117 1489150 : auto_vec<tree, 16> marked_parms;
7118 4608370 : for (tree parm = DECL_ARGUMENTS (current_function_decl); parm;
7119 3119220 : parm = DECL_CHAIN (parm))
7120 3119220 : if (!TREE_ADDRESSABLE (parm)
7121 3119220 : && bitmap_bit_p (forced_stack_vars, DECL_UID (parm)))
7122 : {
7123 108 : TREE_ADDRESSABLE (parm) = 1;
7124 108 : marked_parms.safe_push (parm);
7125 : }
7126 1489150 : if (DECL_RESULT (current_function_decl)
7127 1489150 : && !TREE_ADDRESSABLE (DECL_RESULT (current_function_decl))
7128 2971428 : && bitmap_bit_p (forced_stack_vars,
7129 1482278 : DECL_UID (DECL_RESULT (current_function_decl))))
7130 : {
7131 0 : TREE_ADDRESSABLE (DECL_RESULT (current_function_decl)) = 1;
7132 0 : marked_parms.safe_push (DECL_RESULT (current_function_decl));
7133 : }
7134 :
7135 : /* Set up parameters and prepare for return, for the function. */
7136 1489150 : expand_function_start (current_function_decl);
7137 :
7138 : /* Clear TREE_ADDRESSABLE again. */
7139 2978408 : while (!marked_parms.is_empty ())
7140 108 : TREE_ADDRESSABLE (marked_parms.pop ()) = 0;
7141 :
7142 : /* If we emitted any instructions for setting up the variables,
7143 : emit them before the FUNCTION_START note. */
7144 1489150 : if (var_seq)
7145 : {
7146 1731 : emit_insn_before (var_seq, parm_birth_insn);
7147 :
7148 : /* In expand_function_end we'll insert the alloca save/restore
7149 : before parm_birth_insn. We've just insertted an alloca call.
7150 : Adjust the pointer to match. */
7151 1731 : parm_birth_insn = var_seq;
7152 : }
7153 :
7154 : /* Now propagate the RTL assignment of each partition to the
7155 : underlying var of each SSA_NAME. */
7156 : tree name;
7157 :
7158 72378647 : FOR_EACH_SSA_NAME (i, name, cfun)
7159 : {
7160 : /* We might have generated new SSA names in
7161 : update_alias_info_with_stack_vars. They will have a NULL
7162 : defining statements, and won't be part of the partitioning,
7163 : so ignore those. */
7164 48360199 : if (!SSA_NAME_DEF_STMT (name))
7165 64852 : continue;
7166 :
7167 48295347 : adjust_one_expanded_partition_var (name);
7168 : }
7169 :
7170 : /* Clean up RTL of variables that straddle across multiple
7171 : partitions, and check that the rtl of any PARM_DECLs that are not
7172 : cleaned up is that of their default defs. */
7173 72378647 : FOR_EACH_SSA_NAME (i, name, cfun)
7174 : {
7175 48360199 : int part;
7176 :
7177 : /* We might have generated new SSA names in
7178 : update_alias_info_with_stack_vars. They will have a NULL
7179 : defining statements, and won't be part of the partitioning,
7180 : so ignore those. */
7181 48360199 : if (!SSA_NAME_DEF_STMT (name))
7182 64852 : continue;
7183 48295347 : part = var_to_partition (SA.map, name);
7184 48295347 : if (part == NO_PARTITION)
7185 17225363 : continue;
7186 :
7187 : /* If this decl was marked as living in multiple places, reset
7188 : this now to NULL. */
7189 31069984 : tree var = SSA_NAME_VAR (name);
7190 10371529 : if (var && DECL_RTL_IF_SET (var) == pc_rtx)
7191 235396 : SET_DECL_RTL (var, NULL);
7192 : /* Check that the pseudos chosen by assign_parms are those of
7193 : the corresponding default defs. */
7194 30834588 : else if (SSA_NAME_IS_DEFAULT_DEF (name)
7195 30834588 : && (TREE_CODE (var) == PARM_DECL
7196 3687249 : || TREE_CODE (var) == RESULT_DECL))
7197 : {
7198 3628655 : rtx in = DECL_RTL_IF_SET (var);
7199 3628655 : gcc_assert (in);
7200 3628655 : rtx out = SA.partition_to_pseudo[part];
7201 3628655 : gcc_assert (in == out);
7202 :
7203 : /* Now reset VAR's RTL to IN, so that the _EXPR attrs match
7204 : those expected by debug backends for each parm and for
7205 : the result. This is particularly important for stabs,
7206 : whose register elimination from parm's DECL_RTL may cause
7207 : -fcompare-debug differences as SET_DECL_RTL changes reg's
7208 : attrs. So, make sure the RTL already has the parm as the
7209 : EXPR, so that it won't change. */
7210 3628655 : SET_DECL_RTL (var, NULL_RTX);
7211 3628655 : if (MEM_P (in))
7212 777886 : set_mem_attributes (in, var, true);
7213 3628655 : SET_DECL_RTL (var, in);
7214 : }
7215 : }
7216 :
7217 : /* If this function is `main', emit a call to `__main'
7218 : to run global initializers, etc. */
7219 1489150 : if (DECL_NAME (current_function_decl)
7220 1489150 : && MAIN_NAME_P (DECL_NAME (current_function_decl))
7221 1604071 : && DECL_FILE_SCOPE_P (current_function_decl))
7222 1489150 : expand_main_function ();
7223 :
7224 : /* Initialize the stack_protect_guard field. This must happen after the
7225 : call to __main (if any) so that the external decl is initialized. */
7226 1489150 : if (crtl->stack_protect_guard && targetm.stack_protect_runtime_enabled_p ())
7227 301 : stack_protect_prologue ();
7228 :
7229 1489150 : expand_phi_nodes (&SA);
7230 :
7231 : /* Release any stale SSA redirection data. */
7232 1489150 : redirect_edge_var_map_empty ();
7233 :
7234 : /* Register rtl specific functions for cfg. */
7235 1489150 : rtl_register_cfg_hooks ();
7236 :
7237 1489150 : init_block = construct_init_block ();
7238 :
7239 : /* Clear EDGE_EXECUTABLE on the entry edge(s). It is cleaned from the
7240 : remaining edges later. */
7241 2978300 : FOR_EACH_EDGE (e, ei, ENTRY_BLOCK_PTR_FOR_FN (fun)->succs)
7242 1489150 : e->flags &= ~EDGE_EXECUTABLE;
7243 :
7244 : /* If the function has too many markers, drop them while expanding. */
7245 1489150 : if (cfun->debug_marker_count
7246 1489150 : >= param_max_debug_marker_count)
7247 1 : cfun->debug_nonbind_markers = false;
7248 :
7249 1489150 : enable_ranger (fun);
7250 1489150 : lab_rtx_for_bb = new hash_map<basic_block, rtx_code_label *>;
7251 1489150 : head_end_for_bb.create (last_basic_block_for_fn (fun));
7252 13993471 : FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
7253 : next_bb)
7254 12504323 : bb = expand_gimple_basic_block (bb, var_ret_seq);
7255 1489148 : disable_ranger (fun);
7256 14388182 : FOR_BB_BETWEEN (bb, init_block->next_bb, EXIT_BLOCK_PTR_FOR_FN (fun),
7257 : next_bb)
7258 : {
7259 12899034 : if ((bb->flags & BB_RTL) == 0)
7260 : {
7261 12504313 : bb->il.gimple.seq = NULL;
7262 12504313 : bb->il.gimple.phi_nodes = NULL;
7263 12504313 : init_rtl_bb_info (bb);
7264 12504313 : bb->flags |= BB_RTL;
7265 12504313 : BB_HEAD (bb) = head_end_for_bb[bb->index].first;
7266 12504313 : BB_END (bb) = head_end_for_bb[bb->index].second;
7267 : }
7268 : /* These flags have no purpose in RTL land. */
7269 12899034 : if (EDGE_COUNT (bb->succs) == 2)
7270 : {
7271 6202250 : EDGE_SUCC (bb, 0)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
7272 6202250 : EDGE_SUCC (bb, 1)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
7273 : }
7274 18831003 : else if (single_succ_p (bb))
7275 5931969 : single_succ_edge (bb)->flags &= ~(EDGE_TRUE_VALUE | EDGE_FALSE_VALUE);
7276 : }
7277 1489148 : head_end_for_bb.release ();
7278 :
7279 1489148 : if (MAY_HAVE_DEBUG_BIND_INSNS)
7280 493440 : expand_debug_locations ();
7281 :
7282 1489148 : if (deep_ter_debug_map)
7283 : {
7284 672 : delete deep_ter_debug_map;
7285 672 : deep_ter_debug_map = NULL;
7286 : }
7287 :
7288 : /* Free stuff we no longer need after GIMPLE optimizations. */
7289 1489148 : free_dominance_info (CDI_DOMINATORS);
7290 1489148 : free_dominance_info (CDI_POST_DOMINATORS);
7291 1489148 : delete_tree_cfg_annotations (fun);
7292 :
7293 1489148 : timevar_push (TV_OUT_OF_SSA);
7294 1489148 : finish_out_of_ssa (&SA);
7295 1489148 : timevar_pop (TV_OUT_OF_SSA);
7296 :
7297 1489148 : timevar_push (TV_POST_EXPAND);
7298 : /* We are no longer in SSA form. */
7299 1489148 : fun->gimple_df->in_ssa_p = false;
7300 1489148 : loops_state_clear (LOOP_CLOSED_SSA);
7301 :
7302 : /* Expansion is used by optimization passes too, set maybe_hot_insn_p
7303 : conservatively to true until they are all profile aware. */
7304 2978296 : delete lab_rtx_for_bb;
7305 1489148 : free_histograms (fun);
7306 :
7307 1489148 : construct_exit_block ();
7308 1489148 : insn_locations_finalize ();
7309 :
7310 1489148 : if (var_ret_seq)
7311 : {
7312 1871 : rtx_insn *after = return_label;
7313 1871 : rtx_insn *next = NEXT_INSN (after);
7314 1871 : if (next && NOTE_INSN_BASIC_BLOCK_P (next))
7315 1871 : after = next;
7316 1871 : emit_insn_after (var_ret_seq, after);
7317 : }
7318 :
7319 1489148 : if (hwassist_sanitize_stack_p ())
7320 464 : hwasan_maybe_emit_frame_base_init ();
7321 :
7322 : /* Zap the tree EH table. */
7323 1489148 : set_eh_throw_stmt_table (fun, NULL);
7324 :
7325 : /* We need JUMP_LABEL be set in order to redirect jumps, and hence
7326 : split edges which edge insertions might do. */
7327 1489148 : rebuild_jump_labels (get_insns ());
7328 :
7329 : /* If we have a single successor to the entry block, put the pending insns
7330 : after parm birth, but before NOTE_INSNS_FUNCTION_BEG. */
7331 1489148 : if (single_succ_p (ENTRY_BLOCK_PTR_FOR_FN (fun)))
7332 : {
7333 1489148 : edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (fun));
7334 1489148 : if (e->insns.r)
7335 : {
7336 11 : rtx_insn *insns = e->insns.r;
7337 11 : e->insns.r = NULL;
7338 11 : rebuild_jump_labels_chain (insns);
7339 11 : if (NOTE_P (parm_birth_insn)
7340 11 : && NOTE_KIND (parm_birth_insn) == NOTE_INSN_FUNCTION_BEG)
7341 11 : emit_insn_before_noloc (insns, parm_birth_insn, e->dest);
7342 : else
7343 0 : emit_insn_after_noloc (insns, parm_birth_insn, e->dest);
7344 : }
7345 : }
7346 :
7347 : /* Otherwise, as well as for other edges, take the usual way. */
7348 1489148 : commit_edge_insertions ();
7349 :
7350 : /* We're done expanding trees to RTL. */
7351 1489148 : currently_expanding_to_rtl = 0;
7352 :
7353 1489148 : flush_mark_addressable_queue ();
7354 :
7355 17954729 : FOR_BB_BETWEEN (bb, ENTRY_BLOCK_PTR_FOR_FN (fun)->next_bb,
7356 : EXIT_BLOCK_PTR_FOR_FN (fun), next_bb)
7357 : {
7358 16465581 : edge e;
7359 16465581 : edge_iterator ei;
7360 38443872 : for (ei = ei_start (bb->succs); (e = ei_safe_edge (ei)); )
7361 : {
7362 : /* Clear EDGE_EXECUTABLE. This flag is never used in the backend. */
7363 21978291 : e->flags &= ~EDGE_EXECUTABLE;
7364 :
7365 : /* At the moment not all abnormal edges match the RTL
7366 : representation. It is safe to remove them here as
7367 : find_many_sub_basic_blocks will rediscover them.
7368 : In the future we should get this fixed properly. */
7369 21978291 : if ((e->flags & EDGE_ABNORMAL)
7370 136022 : && !(e->flags & EDGE_SIBCALL))
7371 10320 : remove_edge (e);
7372 : else
7373 21967971 : ei_next (&ei);
7374 : }
7375 : }
7376 :
7377 1489148 : auto_sbitmap blocks (last_basic_block_for_fn (fun));
7378 1489148 : bitmap_ones (blocks);
7379 1489148 : find_many_sub_basic_blocks (blocks);
7380 1489148 : purge_all_dead_edges ();
7381 :
7382 : /* After initial rtl generation, call back to finish generating
7383 : exception support code. We need to do this before cleaning up
7384 : the CFG as the code does not expect dead landing pads. */
7385 1489148 : if (fun->eh->region_tree != NULL)
7386 61791 : finish_eh_generation ();
7387 :
7388 : /* Call expand_stack_alignment after finishing all
7389 : updates to crtl->preferred_stack_boundary. */
7390 1489148 : expand_stack_alignment ();
7391 :
7392 : /* Fixup REG_EQUIV notes in the prologue if there are tailcalls in this
7393 : function. */
7394 1489148 : if (crtl->tail_call_emit)
7395 113268 : fixup_tail_calls ();
7396 :
7397 1489148 : HOST_WIDE_INT patch_area_size, patch_area_entry;
7398 1489148 : parse_and_check_patch_area (flag_patchable_function_entry, false,
7399 : &patch_area_size, &patch_area_entry);
7400 :
7401 1489148 : tree patchable_function_entry_attr
7402 1489148 : = lookup_attribute ("patchable_function_entry",
7403 1489148 : DECL_ATTRIBUTES (cfun->decl));
7404 1489148 : if (patchable_function_entry_attr)
7405 : {
7406 16 : tree pp_val = TREE_VALUE (patchable_function_entry_attr);
7407 16 : tree patchable_function_entry_value1 = TREE_VALUE (pp_val);
7408 :
7409 16 : patch_area_size = tree_to_uhwi (patchable_function_entry_value1);
7410 16 : patch_area_entry = 0;
7411 16 : if (TREE_CHAIN (pp_val) != NULL_TREE)
7412 : {
7413 8 : tree patchable_function_entry_value2
7414 8 : = TREE_VALUE (TREE_CHAIN (pp_val));
7415 8 : patch_area_entry = tree_to_uhwi (patchable_function_entry_value2);
7416 : }
7417 : }
7418 :
7419 1489148 : if (patch_area_entry > patch_area_size)
7420 : {
7421 0 : if (patch_area_size > 0)
7422 0 : warning (OPT_Wattributes,
7423 : "patchable function entry %wu exceeds size %wu",
7424 : patch_area_entry, patch_area_size);
7425 0 : patch_area_entry = 0;
7426 : }
7427 :
7428 1489148 : crtl->patch_area_size = patch_area_size;
7429 1489148 : crtl->patch_area_entry = patch_area_entry;
7430 :
7431 : /* BB subdivision may have created basic blocks that are only reachable
7432 : from unlikely bbs but not marked as such in the profile. */
7433 1489148 : if (optimize)
7434 1042106 : propagate_unlikely_bbs_forward ();
7435 :
7436 : /* Remove unreachable blocks, otherwise we cannot compute dominators
7437 : which are needed for loop state verification. As a side-effect
7438 : this also compacts blocks.
7439 : ??? We cannot remove trivially dead insns here as for example
7440 : the DRAP reg on i?86 is not magically live at this point.
7441 : gcc.c-torture/execute/ipa-sra-2.c execution, -Os -m32 fails otherwise. */
7442 1489148 : cleanup_cfg (CLEANUP_NO_INSN_DEL);
7443 :
7444 1489148 : checking_verify_flow_info ();
7445 :
7446 : /* Initialize pseudos allocated for hard registers. */
7447 1489148 : emit_initial_value_sets ();
7448 :
7449 : /* And finally unshare all RTL. */
7450 1489148 : unshare_all_rtl ();
7451 :
7452 : /* There's no need to defer outputting this function any more; we
7453 : know we want to output it. */
7454 1489148 : DECL_DEFER_OUTPUT (current_function_decl) = 0;
7455 :
7456 : /* Now that we're done expanding trees to RTL, we shouldn't have any
7457 : more CONCATs anywhere. */
7458 1489148 : generating_concat_p = 0;
7459 :
7460 1489148 : if (dump_file)
7461 : {
7462 374 : fprintf (dump_file,
7463 : "\n\n;;\n;; Full RTL generated for this function:\n;;\n");
7464 : /* And the pass manager will dump RTL for us. */
7465 : }
7466 :
7467 : /* If we're emitting a nested function, make sure its parent gets
7468 : emitted as well. Doing otherwise confuses debug info. */
7469 1489148 : {
7470 1489148 : tree parent;
7471 1489148 : for (parent = DECL_CONTEXT (current_function_decl);
7472 3181114 : parent != NULL_TREE;
7473 1691966 : parent = get_containing_scope (parent))
7474 1691966 : if (TREE_CODE (parent) == FUNCTION_DECL)
7475 75307 : TREE_SYMBOL_REFERENCED (DECL_ASSEMBLER_NAME (parent)) = 1;
7476 : }
7477 :
7478 1489148 : TREE_ASM_WRITTEN (current_function_decl) = 1;
7479 :
7480 : /* After expanding, the return labels are no longer needed. */
7481 1489148 : return_label = NULL;
7482 1489148 : naked_return_label = NULL;
7483 :
7484 : /* After expanding, the tm_restart map is no longer needed. */
7485 1489148 : if (fun->gimple_df->tm_restart)
7486 265 : fun->gimple_df->tm_restart = NULL;
7487 :
7488 : /* Tag the blocks with a depth number so that change_scope can find
7489 : the common parent easily. */
7490 1489148 : set_block_levels (DECL_INITIAL (fun->decl), 0);
7491 1489148 : default_rtl_profile ();
7492 :
7493 : /* For -dx discard loops now, otherwise IL verify in clean_state will
7494 : ICE. */
7495 1489148 : if (rtl_dump_and_exit)
7496 : {
7497 79 : cfun->curr_properties &= ~PROP_loops;
7498 79 : loop_optimizer_finalize ();
7499 : }
7500 :
7501 1489148 : timevar_pop (TV_POST_EXPAND);
7502 :
7503 1489148 : return 0;
7504 1489148 : }
7505 :
7506 : } // anon namespace
7507 :
7508 : rtl_opt_pass *
7509 298832 : make_pass_expand (gcc::context *ctxt)
7510 : {
7511 298832 : return new pass_expand (ctxt);
7512 : }
|
