Branch data Line data Source code
1 : : /* strub (stack scrubbing) support.
2 : : Copyright (C) 2021-2024 Free Software Foundation, Inc.
3 : : Contributed by Alexandre Oliva <oliva@adacore.com>.
4 : :
5 : : This file is part of GCC.
6 : :
7 : : GCC is free software; you can redistribute it and/or modify it under
8 : : the terms of the GNU General Public License as published by the Free
9 : : Software Foundation; either version 3, or (at your option) any later
10 : : version.
11 : :
12 : : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
13 : : WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 : : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 : : for more details.
16 : :
17 : : You should have received a copy of the GNU General Public License
18 : : along with GCC; see the file COPYING3. If not see
19 : : <http://www.gnu.org/licenses/>. */
20 : :
21 : : #include "config.h"
22 : : #include "system.h"
23 : : #include "coretypes.h"
24 : : #include "backend.h"
25 : : #include "tree.h"
26 : : #include "gimple.h"
27 : : #include "gimplify.h"
28 : : #include "tree-pass.h"
29 : : #include "ssa.h"
30 : : #include "gimple-iterator.h"
31 : : #include "gimplify-me.h"
32 : : #include "tree-into-ssa.h"
33 : : #include "tree-ssa.h"
34 : : #include "tree-cfg.h"
35 : : #include "cfghooks.h"
36 : : #include "cfgloop.h"
37 : : #include "cfgcleanup.h"
38 : : #include "tree-eh.h"
39 : : #include "except.h"
40 : : #include "builtins.h"
41 : : #include "attribs.h"
42 : : #include "tree-inline.h"
43 : : #include "cgraph.h"
44 : : #include "alloc-pool.h"
45 : : #include "symbol-summary.h"
46 : : #include "sreal.h"
47 : : #include "ipa-cp.h"
48 : : #include "ipa-prop.h"
49 : : #include "ipa-fnsummary.h"
50 : : #include "gimple-fold.h"
51 : : #include "fold-const.h"
52 : : #include "gimple-walk.h"
53 : : #include "tree-dfa.h"
54 : : #include "langhooks.h"
55 : : #include "calls.h"
56 : : #include "vec.h"
57 : : #include "stor-layout.h"
58 : : #include "varasm.h"
59 : : #include "alias.h"
60 : : #include "diagnostic.h"
61 : : #include "intl.h"
62 : : #include "ipa-strub.h"
63 : : #include "symtab-thunks.h"
64 : : #include "attr-fnspec.h"
65 : : #include "target.h"
66 : : #include "gcc-urlifier.h"
67 : :
68 : : /* This file introduces two passes that, together, implement
69 : : machine-independent stack scrubbing, strub for short. It arranges
70 : : for stack frames that have strub enabled to be zeroed-out after
71 : : relinquishing control to a caller, whether by returning or by
72 : : propagating an exception. This admittedly unusual design decision
73 : : was driven by exception support (one needs a stack frame to be
74 : : active to propagate exceptions out of it), and it enabled an
75 : : implementation that is entirely machine-independent (no custom
76 : : epilogue code is required).
77 : :
78 : : Strub modes can be selected for stack frames by attaching attribute
79 : : strub to functions or to variables (to their types, actually).
80 : : Different strub modes, with different implementation details, are
81 : : available, and they can be selected by an argument to the strub
82 : : attribute. When enabled by strub-enabled variables, whether by
83 : : accessing (as in reading from) statically-allocated ones, or by
84 : : introducing (as in declaring) automatically-allocated ones, a
85 : : suitable mode is selected automatically.
86 : :
87 : : At-calls mode modifies the interface of a function, adding a stack
88 : : watermark argument, that callers use to clean up the stack frame of
89 : : the called function. Because of the interface change, it can only
90 : : be used when explicitly selected, or when a function is internal to
91 : : a translation unit. Strub-at-calls function types are distinct
92 : : from their original types (they're not modified in-place), and they
93 : : are not interchangeable with other function types.
94 : :
95 : : Internal mode, in turn, does not modify the type or the interface
96 : : of a function. It is currently implemented by turning the function
97 : : into a wrapper, moving the function body to a separate wrapped
98 : : function, and scrubbing the wrapped body's stack in the wrapper.
99 : : Internal-strub function types are mostly interface-compatible with
100 : : other strub modes, namely callable (from strub functions, though
101 : : not strub-enabled) and disabled (not callable from strub
102 : : functions).
103 : :
104 : : Always_inline functions can be strub functions, but they can only
105 : : be called from other strub functions, because strub functions must
106 : : never be inlined into non-strub functions. Internal and at-calls
107 : : modes are indistinguishable when it comes to always_inline
108 : : functions: they will necessarily be inlined into another strub
109 : : function, and will thus be integrated into the caller's stack
110 : : frame, whatever the mode. (Contrast with non-always_inline strub
111 : : functions: an at-calls function can be called from other strub
112 : : functions, ensuring no discontinuity in stack erasing, whereas an
113 : : internal-strub function can only be called from other strub
114 : : functions if it happens to be inlined, or if -fstrub=relaxed mode
115 : : is in effect (that's the default). In -fstrub=strict mode,
116 : : internal-strub functions are not callable from strub functions,
117 : : because the wrapper itself is not strubbed.
118 : :
119 : : The implementation involves two simple-IPA passes. The earliest
120 : : one, strub-mode, assigns strub modes to functions. It needs to run
121 : : before any inlining, so that we can prevent inlining of strub
122 : : functions into non-strub functions. It notes explicit strub mode
123 : : requests, enables strub in response to strub variables and testing
124 : : options, and flags unsatisfiable requests.
125 : :
126 : : Three possibilities of unsatisfiable requests come to mind: (a)
127 : : when a strub mode is explicitly selected, but the function uses
128 : : features that make it ineligible for that mode (e.g. at-calls rules
129 : : out calling __builtin_apply_args, because of the interface changes,
130 : : and internal mode rules out noclone or otherwise non-versionable
131 : : functions, non-default varargs, non-local or forced labels, and
132 : : functions with far too many arguments); (b) when some strub mode
133 : : must be enabled because of a strub variable, but the function is
134 : : not eligible or not viable for any mode; and (c) when
135 : : -fstrub=strict is enabled, and calls are found in strub functions
136 : : to functions that are not callable from strub contexts.
137 : : compute_strub_mode implements (a) and (b), and verify_strub
138 : : implements (c).
139 : :
140 : : The second IPA pass modifies interfaces of at-calls-strub functions
141 : : and types, introduces strub calls in and around them. and splits
142 : : internal-strub functions. It is placed after early inlining, so
143 : : that even internal-strub functions get a chance of being inlined
144 : : into other strub functions, but before non-early inlining, so that
145 : : internal-strub wrapper functions still get a chance of inlining
146 : : after splitting.
147 : :
148 : : Wrappers avoid duplicating the copying of large arguments again by
149 : : passing them by reference to the wrapped bodies. This involves
150 : : occasional SSA rewriting of address computations, because of the
151 : : additional indirection. Besides these changes, and the
152 : : introduction of the stack watermark parameter, wrappers and wrapped
153 : : functions cooperate to handle variable argument lists (performing
154 : : va_start in the wrapper, passing the list as an argument, and
155 : : replacing va_start calls in the wrapped body with va_copy), and
156 : : __builtin_apply_args (also called in the wrapper and passed to the
157 : : wrapped body as an argument).
158 : :
159 : : Strub bodies (both internal-mode wrapped bodies, and at-calls
160 : : functions) always start by adjusting the watermark parameter, by
161 : : calling __builtin___strub_update. The compiler inserts them in the
162 : : main strub pass. Allocations of additional stack space for the
163 : : frame (__builtin_alloca) are also followed by watermark updates.
164 : : Stack space temporarily allocated to pass arguments to other
165 : : functions, released right after the call, is not regarded as part
166 : : of the frame. Around calls to them, i.e., in internal-mode
167 : : wrappers and at-calls callers (even calls through pointers), calls
168 : : to __builtin___strub_enter and __builtin___strub_leave are
169 : : inserted, the latter as a __finally block, so that it runs at
170 : : regular and exceptional exit paths. strub_enter only initializes
171 : : the stack watermark, and strub_leave is where the scrubbing takes
172 : : place, overwriting with zeros the stack space from the top of the
173 : : stack to the watermark.
174 : :
175 : : These calls can be optimized in various cases. In
176 : : pass_ipa_strub::adjust_at_calls_call, for example, we enable
177 : : tail-calling and other optimized calls from one strub body to
178 : : another by passing on the watermark parameter. The builtins
179 : : themselves may undergo inline substitution during expansion,
180 : : dependign on optimization levels. This involves dealing with stack
181 : : red zones (when the builtins are called out-of-line, the red zone
182 : : cannot be used) and other ugly details related with inlining strub
183 : : bodies into other strub bodies (see expand_builtin_strub_update).
184 : : expand_builtin_strub_leave may even perform partial inline
185 : : substitution. */
186 : :
187 : : /* Const and pure functions that gain a watermark parameter for strub purposes
188 : : are still regarded as such, which may cause the inline expansions of the
189 : : __strub builtins to malfunction. Ideally, attribute "fn spec" would enable
190 : : us to inform the backend about requirements and side effects of the call, but
191 : : call_fusage building in calls.c:expand_call does not even look at
192 : : attr_fnspec, so we resort to asm loads and updates to attain an equivalent
193 : : effect. Once expand_call gains the ability to issue extra memory uses and
194 : : clobbers based on pure/const function's fnspec, we can define this to 1. */
195 : : #define ATTR_FNSPEC_DECONST_WATERMARK 0
196 : :
197 : : enum strub_mode {
198 : : /* This mode denotes a regular function, that does not require stack
199 : : scrubbing (strubbing). It may call any other functions, but if
200 : : it calls AT_CALLS (or WRAPPED) ones, strubbing logic is
201 : : automatically introduced around those calls (the latter, by
202 : : inlining INTERNAL wrappers). */
203 : : STRUB_DISABLED = 0,
204 : :
205 : : /* This denotes a function whose signature is (to be) modified to
206 : : take an extra parameter, for stack use annotation, and its
207 : : callers must initialize and pass that argument, and perform the
208 : : strubbing. Functions that are explicitly marked with attribute
209 : : strub must have the mark visible wherever the function is,
210 : : including aliases, and overriders and overriding methods.
211 : : Functions that are implicitly marked for strubbing, for accessing
212 : : variables explicitly marked as such, will only select this
213 : : strubbing method if they are internal to a translation unit. It
214 : : can only be inlined into other strubbing functions, i.e.,
215 : : STRUB_AT_CALLS or STRUB_WRAPPED. */
216 : : STRUB_AT_CALLS = 1,
217 : :
218 : : /* This denotes a function that is to perform strubbing internally,
219 : : without any changes to its interface (the function is turned into
220 : : a strubbing wrapper, and its original body is moved to a separate
221 : : STRUB_WRAPPED function, with a modified interface). Functions
222 : : may be explicitly marked with attribute strub(2), and the
223 : : attribute must be visible at the point of definition. Functions
224 : : that are explicitly marked for strubbing, for accessing variables
225 : : explicitly marked as such, may select this strubbing mode if
226 : : their interface cannot change, e.g. because its interface is
227 : : visible to other translation units, directly, by indirection
228 : : (having its address taken), inheritance, etc. Functions that use
229 : : this method must not have the noclone attribute, nor the noipa
230 : : one. Functions marked as always_inline may select this mode, but
231 : : they are NOT wrapped, they remain unchanged, and are only inlined
232 : : into strubbed contexts. Once non-always_inline functions are
233 : : wrapped, the wrapper becomes STRUB_WRAPPER, and the wrapped becomes
234 : : STRUB_WRAPPED. */
235 : : STRUB_INTERNAL = 2,
236 : :
237 : : /* This denotes a function whose stack is not strubbed, but that is
238 : : nevertheless explicitly or implicitly marked as callable from strubbing
239 : : functions. Normally, only STRUB_AT_CALLS (and STRUB_INTERNAL ->
240 : : STRUB_WRAPPED) functions can be called from strubbing contexts (bodies of
241 : : STRUB_AT_CALLS, STRUB_INTERNAL and STRUB_WRAPPED functions), but attribute
242 : : strub(3) enables other functions to be (indirectly) called from these
243 : : contexts. Some builtins and internal functions may be implicitly marked as
244 : : STRUB_CALLABLE. */
245 : : STRUB_CALLABLE = 3,
246 : :
247 : : /* This denotes the function that took over the body of a
248 : : STRUB_INTERNAL function. At first, it's only called by its
249 : : wrapper, but the wrapper may be inlined. The wrapped function,
250 : : in turn, can only be inlined into other functions whose stack
251 : : frames are strubbed, i.e., that are STRUB_WRAPPED or
252 : : STRUB_AT_CALLS. */
253 : : STRUB_WRAPPED = -1,
254 : :
255 : : /* This denotes the wrapper function that replaced the STRUB_INTERNAL
256 : : function. This mode overrides the STRUB_INTERNAL mode at the time the
257 : : internal to-be-wrapped function becomes a wrapper, so that inlining logic
258 : : can tell one from the other. */
259 : : STRUB_WRAPPER = -2,
260 : :
261 : : /* This denotes an always_inline function that requires strubbing. It can
262 : : only be called from, and inlined into, other strubbing contexts. */
263 : : STRUB_INLINABLE = -3,
264 : :
265 : : /* This denotes a function that accesses strub variables, so it would call for
266 : : internal strubbing (whether or not it's eligible for that), but since
267 : : at-calls strubbing is viable, that's selected as an optimization. This
268 : : mode addresses the inconvenience that such functions may have different
269 : : modes selected depending on optimization flags, and get a different
270 : : callable status depending on that choice: if we assigned them
271 : : STRUB_AT_CALLS mode, they would be callable when optimizing, whereas
272 : : STRUB_INTERNAL would not be callable. */
273 : : STRUB_AT_CALLS_OPT = -4,
274 : :
275 : : };
276 : :
277 : : /* Look up a strub attribute in TYPE, and return it. */
278 : :
279 : : static tree
280 : 12170362 : get_strub_attr_from_type (tree type)
281 : : {
282 : 12170362 : return lookup_attribute ("strub", TYPE_ATTRIBUTES (type));
283 : : }
284 : :
285 : : /* Look up a strub attribute in DECL or in its type, and return it. */
286 : :
287 : : static tree
288 : 9255906 : get_strub_attr_from_decl (tree decl)
289 : : {
290 : 9255906 : tree ret = lookup_attribute ("strub", DECL_ATTRIBUTES (decl));
291 : 9255906 : if (ret)
292 : : return ret;
293 : 9240253 : return get_strub_attr_from_type (TREE_TYPE (decl));
294 : : }
295 : :
296 : : #define STRUB_ID_COUNT 8
297 : : #define STRUB_IDENT_COUNT 3
298 : : #define STRUB_TYPE_COUNT 5
299 : :
300 : : #define STRUB_ID_BASE 0
301 : : #define STRUB_IDENT_BASE (STRUB_ID_BASE + STRUB_ID_COUNT)
302 : : #define STRUB_TYPE_BASE (STRUB_IDENT_BASE + STRUB_IDENT_COUNT)
303 : : #define STRUB_CACHE_SIZE (STRUB_TYPE_BASE + STRUB_TYPE_COUNT)
304 : :
305 : : /* Keep the strub mode and temp identifiers and types from being GC'd. */
306 : : static GTY((deletable)) tree strub_cache[STRUB_CACHE_SIZE];
307 : :
308 : : /* Define a function to cache identifier ID, to be used as a strub attribute
309 : : parameter for a strub mode named after NAME. */
310 : : #define DEF_STRUB_IDS(IDX, NAME, ID) \
311 : : static inline tree get_strub_mode_id_ ## NAME () { \
312 : : int idx = STRUB_ID_BASE + IDX; \
313 : : tree identifier = strub_cache[idx]; \
314 : : if (!identifier) \
315 : : strub_cache[idx] = identifier = get_identifier (ID); \
316 : : return identifier; \
317 : : }
318 : : /* Same as DEF_STRUB_IDS, but use the string expansion of NAME as ID. */
319 : : #define DEF_STRUB_ID(IDX, NAME) \
320 : : DEF_STRUB_IDS (IDX, NAME, #NAME)
321 : :
322 : : /* Define functions for each of the strub mode identifiers.
323 : : Expose dashes rather than underscores. */
324 : 10141 : DEF_STRUB_ID (0, disabled)
325 : 18577 : DEF_STRUB_IDS (1, at_calls, "at-calls")
326 : 15368 : DEF_STRUB_ID (2, internal)
327 : 24004 : DEF_STRUB_ID (3, callable)
328 : 2198 : DEF_STRUB_ID (4, wrapped)
329 : 2352 : DEF_STRUB_ID (5, wrapper)
330 : 3488 : DEF_STRUB_ID (6, inlinable)
331 : 40 : DEF_STRUB_IDS (7, at_calls_opt, "at-calls-opt")
332 : :
333 : : /* Release the temporary macro names. */
334 : : #undef DEF_STRUB_IDS
335 : : #undef DEF_STRUB_ID
336 : :
337 : : /* Return the identifier corresponding to strub MODE. */
338 : :
339 : : static tree
340 : 76168 : get_strub_mode_attr_parm (enum strub_mode mode)
341 : : {
342 : 76168 : switch (mode)
343 : : {
344 : 10141 : case STRUB_DISABLED:
345 : 10141 : return get_strub_mode_id_disabled ();
346 : :
347 : 18577 : case STRUB_AT_CALLS:
348 : 18577 : return get_strub_mode_id_at_calls ();
349 : :
350 : 15368 : case STRUB_INTERNAL:
351 : 15368 : return get_strub_mode_id_internal ();
352 : :
353 : 24004 : case STRUB_CALLABLE:
354 : 24004 : return get_strub_mode_id_callable ();
355 : :
356 : 2198 : case STRUB_WRAPPED:
357 : 2198 : return get_strub_mode_id_wrapped ();
358 : :
359 : 2352 : case STRUB_WRAPPER:
360 : 2352 : return get_strub_mode_id_wrapper ();
361 : :
362 : 3488 : case STRUB_INLINABLE:
363 : 3488 : return get_strub_mode_id_inlinable ();
364 : :
365 : 40 : case STRUB_AT_CALLS_OPT:
366 : 40 : return get_strub_mode_id_at_calls_opt ();
367 : :
368 : 0 : default:
369 : 0 : gcc_unreachable ();
370 : : }
371 : : }
372 : :
373 : : /* Return the parmeters (TREE_VALUE) for a strub attribute of MODE.
374 : : We know we use a single parameter, so we bypass the creation of a
375 : : tree list. */
376 : :
377 : : static tree
378 : 2603 : get_strub_mode_attr_value (enum strub_mode mode)
379 : : {
380 : 0 : return get_strub_mode_attr_parm (mode);
381 : : }
382 : :
383 : : /* Determine whether ID is a well-formed strub mode-specifying attribute
384 : : parameter for a function (type). Only user-visible modes are accepted, and
385 : : ID must be non-NULL.
386 : :
387 : : For unacceptable parms, return 0, otherwise a nonzero value as below.
388 : :
389 : : If the parm enables strub, return positive, otherwise negative.
390 : :
391 : : If the affected type must be a distinct, incompatible type,return an integer
392 : : of absolute value 2, otherwise 1. */
393 : :
394 : : int
395 : 2171 : strub_validate_fn_attr_parm (tree id)
396 : : {
397 : 2171 : int ret;
398 : 2171 : const char *s = NULL;
399 : 2171 : size_t len = 0;
400 : :
401 : : /* do NOT test for NULL. This is only to be called with non-NULL arguments.
402 : : We assume that the strub parameter applies to a function, because only
403 : : functions accept an explicit argument. If we accepted NULL, and we
404 : : happened to be called to verify the argument for a variable, our return
405 : : values would be wrong. */
406 : 2171 : if (TREE_CODE (id) == STRING_CST)
407 : : {
408 : 2171 : s = TREE_STRING_POINTER (id);
409 : 2171 : len = TREE_STRING_LENGTH (id) - 1;
410 : : }
411 : 0 : else if (TREE_CODE (id) == IDENTIFIER_NODE)
412 : : {
413 : 0 : s = IDENTIFIER_POINTER (id);
414 : 0 : len = IDENTIFIER_LENGTH (id);
415 : : }
416 : : else
417 : : return 0;
418 : :
419 : 2171 : enum strub_mode mode;
420 : :
421 : 2171 : if (len != 8)
422 : : return 0;
423 : :
424 : 2171 : switch (s[0])
425 : : {
426 : : case 'd':
427 : : mode = STRUB_DISABLED;
428 : : ret = -1;
429 : : break;
430 : :
431 : 578 : case 'a':
432 : 578 : mode = STRUB_AT_CALLS;
433 : 578 : ret = 2;
434 : 578 : break;
435 : :
436 : 559 : case 'i':
437 : 559 : mode = STRUB_INTERNAL;
438 : 559 : ret = 1;
439 : 559 : break;
440 : :
441 : 580 : case 'c':
442 : 580 : mode = STRUB_CALLABLE;
443 : 580 : ret = -2;
444 : 580 : break;
445 : :
446 : : default:
447 : : /* Other parms are for internal use only. */
448 : : return 0;
449 : : }
450 : :
451 : 2171 : tree mode_id = get_strub_mode_attr_parm (mode);
452 : :
453 : 2171 : if (TREE_CODE (id) == IDENTIFIER_NODE
454 : 2171 : ? id != mode_id
455 : 2171 : : strncmp (s, IDENTIFIER_POINTER (mode_id), len) != 0)
456 : : return 0;
457 : :
458 : : return ret;
459 : : }
460 : :
461 : : /* Return the strub mode from STRUB_ATTR. VAR_P should be TRUE if the attribute
462 : : is taken from a variable, rather than from a function, or a type thereof. */
463 : :
464 : : static enum strub_mode
465 : 9287260 : get_strub_mode_from_attr (tree strub_attr, bool var_p = false)
466 : : {
467 : 9287260 : enum strub_mode mode = STRUB_DISABLED;
468 : :
469 : 9287260 : if (strub_attr)
470 : : {
471 : 74839 : if (!TREE_VALUE (strub_attr))
472 : 3469 : mode = !var_p ? STRUB_AT_CALLS : STRUB_INTERNAL;
473 : : else
474 : : {
475 : 71370 : gcc_checking_assert (!var_p);
476 : 71370 : tree id = TREE_VALUE (strub_attr);
477 : 71370 : if (TREE_CODE (id) == TREE_LIST)
478 : 55717 : id = TREE_VALUE (id);
479 : 71370 : const char *s = (TREE_CODE (id) == STRING_CST
480 : 71370 : ? TREE_STRING_POINTER (id)
481 : 15653 : : IDENTIFIER_POINTER (id));
482 : 71370 : size_t len = (TREE_CODE (id) == STRING_CST
483 : 71370 : ? TREE_STRING_LENGTH (id) - 1
484 : 15653 : : IDENTIFIER_LENGTH (id));
485 : :
486 : 71370 : switch (len)
487 : : {
488 : 3618 : case 7:
489 : 3618 : switch (s[6])
490 : : {
491 : : case 'r':
492 : : mode = STRUB_WRAPPER;
493 : : break;
494 : :
495 : 1732 : case 'd':
496 : 1732 : mode = STRUB_WRAPPED;
497 : 1732 : break;
498 : :
499 : 0 : default:
500 : 0 : gcc_unreachable ();
501 : : }
502 : : break;
503 : :
504 : 64610 : case 8:
505 : 64610 : switch (s[0])
506 : : {
507 : : case 'd':
508 : : mode = STRUB_DISABLED;
509 : : break;
510 : :
511 : 17923 : case 'a':
512 : 17923 : mode = STRUB_AT_CALLS;
513 : 17923 : break;
514 : :
515 : 14307 : case 'i':
516 : 14307 : mode = STRUB_INTERNAL;
517 : 14307 : break;
518 : :
519 : 22717 : case 'c':
520 : 22717 : mode = STRUB_CALLABLE;
521 : 22717 : break;
522 : :
523 : 0 : default:
524 : 0 : gcc_unreachable ();
525 : : }
526 : : break;
527 : :
528 : : case 9:
529 : : mode = STRUB_INLINABLE;
530 : : break;
531 : :
532 : 32 : case 12:
533 : 32 : mode = STRUB_AT_CALLS_OPT;
534 : 32 : break;
535 : :
536 : 0 : default:
537 : 0 : gcc_unreachable ();
538 : : }
539 : :
540 : 71370 : gcc_checking_assert (TREE_CODE (id) == IDENTIFIER_NODE
541 : : ? id == get_strub_mode_attr_parm (mode)
542 : : : strncmp (IDENTIFIER_POINTER
543 : : (get_strub_mode_attr_parm (mode)),
544 : : s, len) == 0);
545 : : }
546 : : }
547 : :
548 : 9287260 : return mode;
549 : : }
550 : :
551 : : /* Look up, decode and return the strub mode associated with FNDECL. */
552 : :
553 : : static enum strub_mode
554 : 9245683 : get_strub_mode_from_fndecl (tree fndecl)
555 : : {
556 : 9245683 : return get_strub_mode_from_attr (get_strub_attr_from_decl (fndecl));
557 : : }
558 : :
559 : : /* Look up, decode and return the strub mode associated with NODE. */
560 : :
561 : : static enum strub_mode
562 : 9225955 : get_strub_mode (cgraph_node *node)
563 : : {
564 : 0 : return get_strub_mode_from_fndecl (node->decl);
565 : : }
566 : :
567 : : /* Look up, decode and return the strub mode associated with TYPE. */
568 : :
569 : : static enum strub_mode
570 : 2930109 : get_strub_mode_from_type (tree type)
571 : : {
572 : 2930109 : bool var_p = !FUNC_OR_METHOD_TYPE_P (type);
573 : 2930109 : tree attr = get_strub_attr_from_type (type);
574 : :
575 : 2930109 : if (attr)
576 : 32033 : return get_strub_mode_from_attr (attr, var_p);
577 : :
578 : 2898076 : if (flag_strub >= -1 && !var_p)
579 : 7922 : return STRUB_CALLABLE;
580 : :
581 : : return STRUB_DISABLED;
582 : : }
583 : :
584 : :
585 : : /* Return TRUE iff NODE calls builtin va_start. */
586 : :
587 : : static bool
588 : 466 : calls_builtin_va_start_p (cgraph_node *node)
589 : : {
590 : 466 : bool result = false;
591 : :
592 : 3066 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
593 : : {
594 : 2608 : tree cdecl = e->callee->decl;
595 : 2608 : if (fndecl_built_in_p (cdecl, BUILT_IN_VA_START))
596 : : return true;
597 : : }
598 : :
599 : : return result;
600 : : }
601 : :
602 : : /* Return TRUE iff NODE calls builtin apply_args, and optionally REPORT it. */
603 : :
604 : : static bool
605 : 2820 : calls_builtin_apply_args_p (cgraph_node *node, bool report = false)
606 : : {
607 : 2820 : bool result = false;
608 : :
609 : 10118 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
610 : : {
611 : 7326 : tree cdecl = e->callee->decl;
612 : 7326 : if (!fndecl_built_in_p (cdecl, BUILT_IN_APPLY_ARGS))
613 : 7294 : continue;
614 : :
615 : 32 : result = true;
616 : :
617 : 32 : if (!report)
618 : : break;
619 : :
620 : 8 : sorry_at (e->call_stmt
621 : 4 : ? gimple_location (e->call_stmt)
622 : 0 : : DECL_SOURCE_LOCATION (node->decl),
623 : : "at-calls %<strub%> does not support call to %qD",
624 : : cdecl);
625 : : }
626 : :
627 : 2820 : return result;
628 : : }
629 : :
630 : : /* Return TRUE iff NODE carries the always_inline attribute. */
631 : :
632 : : static inline bool
633 : 7641 : strub_always_inline_p (cgraph_node *node)
634 : : {
635 : 7641 : return lookup_attribute ("always_inline", DECL_ATTRIBUTES (node->decl));
636 : : }
637 : :
638 : : /* Return TRUE iff the target has strub support for T, a function
639 : : decl, or a type used in an indirect call, and optionally REPORT the
640 : : reasons for ineligibility. If T is a type and error REPORTing is
641 : : enabled, the LOCation (of the indirect call) should be provided. */
642 : : static inline bool
643 : 4218 : strub_target_support_p (tree t, bool report = false,
644 : : location_t loc = UNKNOWN_LOCATION)
645 : : {
646 : 4218 : bool result = true;
647 : :
648 : 4218 : if (!targetm.have_strub_support_for (t))
649 : : {
650 : 0 : result = false;
651 : :
652 : 0 : if (!report)
653 : : return result;
654 : :
655 : 0 : if (DECL_P (t))
656 : 0 : sorry_at (DECL_SOURCE_LOCATION (t),
657 : : "%qD is not eligible for %<strub%>"
658 : : " on the target system", t);
659 : : else
660 : 0 : sorry_at (loc,
661 : : "unsupported %<strub%> call"
662 : : " on the target system");
663 : : }
664 : :
665 : : return result;
666 : : }
667 : :
668 : : /* Return TRUE iff NODE is potentially eligible for any strub-enabled mode, and
669 : : optionally REPORT the reasons for ineligibility. */
670 : :
671 : : static inline bool
672 : 2295 : can_strub_p (cgraph_node *node, bool report = false)
673 : : {
674 : 2295 : bool result = strub_target_support_p (node->decl, report);
675 : :
676 : 2295 : if (!report && (!result || strub_always_inline_p (node)))
677 : 0 : return result;
678 : :
679 : 2295 : auto_urlify_attributes sentinel;
680 : :
681 : 2295 : if (flag_split_stack)
682 : : {
683 : 16 : result = false;
684 : :
685 : 16 : if (!report)
686 : : return result;
687 : :
688 : 8 : sorry_at (DECL_SOURCE_LOCATION (node->decl),
689 : : "%qD is not eligible for %<strub%>"
690 : : " because %<-fsplit-stack%> is enabled",
691 : : node->decl);
692 : : }
693 : :
694 : 2287 : if (lookup_attribute ("noipa", DECL_ATTRIBUTES (node->decl)))
695 : : {
696 : 0 : result = false;
697 : :
698 : 0 : if (!report)
699 : : return result;
700 : :
701 : 0 : sorry_at (DECL_SOURCE_LOCATION (node->decl),
702 : : "%qD is not eligible for %<strub%>"
703 : : " because of attribute %<noipa%>",
704 : : node->decl);
705 : : }
706 : :
707 : : /* We can't, and don't want to vectorize the watermark and other
708 : : strub-introduced parms. */
709 : 2287 : if (lookup_attribute ("simd", DECL_ATTRIBUTES (node->decl)))
710 : : {
711 : 0 : result = false;
712 : :
713 : 0 : if (!report)
714 : : return result;
715 : :
716 : 0 : sorry_at (DECL_SOURCE_LOCATION (node->decl),
717 : : "%qD is not eligible for %<strub%>"
718 : : " because of attribute %<simd%>",
719 : : node->decl);
720 : : }
721 : :
722 : : return result;
723 : 2295 : }
724 : :
725 : : /* Return TRUE iff NODE is eligible for at-calls strub, and optionally REPORT
726 : : the reasons for ineligibility. Besides general non-eligibility for
727 : : strub-enabled modes, at-calls rules out calling builtin apply_args. */
728 : :
729 : : static bool
730 : 2354 : can_strub_at_calls_p (cgraph_node *node, bool report = false)
731 : : {
732 : 2354 : bool result = !report || can_strub_p (node, report);
733 : :
734 : : if (!result && !report)
735 : : return result;
736 : :
737 : 2354 : return !calls_builtin_apply_args_p (node, report);
738 : : }
739 : :
740 : : /* Return TRUE iff the called function (pointer or, if available,
741 : : decl) undergoes a significant type conversion for the call. Strub
742 : : mode changes between function types, and other non-useless type
743 : : conversions, are regarded as significant. When the function type
744 : : is overridden, the effective strub mode for the call is that of the
745 : : call fntype, rather than that of the pointer or of the decl.
746 : : Functions called with type overrides cannot undergo type changes;
747 : : it's as if their address was taken, so they're considered
748 : : non-viable for implicit at-calls strub mode. */
749 : :
750 : : static inline bool
751 : 19495 : strub_call_fntype_override_p (const gcall *gs)
752 : : {
753 : 19495 : if (gimple_call_internal_p (gs))
754 : : return false;
755 : 19495 : tree fn_type = TREE_TYPE (TREE_TYPE (gimple_call_fn (gs)));
756 : 19495 : if (tree decl = gimple_call_fndecl (gs))
757 : 19223 : fn_type = TREE_TYPE (decl);
758 : :
759 : : /* We do NOT want to take the mode from the decl here. This
760 : : function is used to tell whether we can change the strub mode of
761 : : a function, and whether the effective mode for the call is to be
762 : : taken from the decl or from an overrider type. When the strub
763 : : mode is explicitly declared, or overridden with a type cast, the
764 : : difference will be noticed in function types. However, if the
765 : : strub mode is implicit due to e.g. strub variables or -fstrub=*
766 : : command-line flags, we will adjust call types along with function
767 : : types. In either case, the presence of type or strub mode
768 : : overriders in calls will prevent a function from having its strub
769 : : modes changed in ways that would imply type changes, but taking
770 : : strub modes from decls would defeat this, since we set strub
771 : : modes and then call this function to tell whether the original
772 : : type was overridden to decide whether to adjust the call. We
773 : : need the answer to be about the type, not the decl. */
774 : 19495 : enum strub_mode mode = get_strub_mode_from_type (fn_type);
775 : 19495 : return (get_strub_mode_from_type (gs->u.fntype) != mode
776 : 19495 : || !useless_type_conversion_p (gs->u.fntype, fn_type));
777 : : }
778 : :
779 : : /* Return TRUE iff NODE is called directly with a type override. */
780 : :
781 : : static bool
782 : 448 : called_directly_with_type_override_p (cgraph_node *node, void *)
783 : : {
784 : 1272 : for (cgraph_edge *e = node->callers; e; e = e->next_caller)
785 : 824 : if (e->call_stmt && strub_call_fntype_override_p (e->call_stmt))
786 : : return true;
787 : :
788 : : return false;
789 : : }
790 : :
791 : : /* Return TRUE iff NODE or any other nodes aliased to it are called
792 : : with type overrides. We can't safely change the type of such
793 : : functions. */
794 : :
795 : : static bool
796 : 448 : called_with_type_override_p (cgraph_node *node)
797 : : {
798 : 448 : return (node->call_for_symbol_thunks_and_aliases
799 : 448 : (called_directly_with_type_override_p, NULL, true, true));
800 : : }
801 : :
802 : : /* Symbolic macro for the max number of arguments that internal strub may add to
803 : : a function. */
804 : :
805 : : #define STRUB_INTERNAL_MAX_EXTRA_ARGS 3
806 : :
807 : : /* We can't perform internal strubbing if the function body involves certain
808 : : features:
809 : :
810 : : - a non-default __builtin_va_start (e.g. x86's __builtin_ms_va_start) is
811 : : currently unsupported because we can't discover the corresponding va_copy and
812 : : va_end decls in the wrapper, and we don't convey the alternate variable
813 : : arguments ABI to the modified wrapped function. The default
814 : : __builtin_va_start is supported by calling va_start/va_end at the wrapper,
815 : : that takes variable arguments, passing a pointer to the va_list object to the
816 : : wrapped function, that runs va_copy from it where the original function ran
817 : : va_start.
818 : :
819 : : __builtin_next_arg is currently unsupported because the wrapped function
820 : : won't be a variable argument function. We could process it in the wrapper,
821 : : that remains a variable argument function, and replace calls in the wrapped
822 : : body, but we currently don't.
823 : :
824 : : __builtin_return_address is rejected because it's generally used when the
825 : : actual caller matters, and introducing a wrapper breaks such uses as those in
826 : : the unwinder. */
827 : :
828 : : static bool
829 : 1881 : can_strub_internally_p (cgraph_node *node, bool report = false)
830 : : {
831 : 1881 : bool result = !report || can_strub_p (node, report);
832 : :
833 : : if (!result && !report)
834 : : return result;
835 : :
836 : 1881 : if (!report && strub_always_inline_p (node))
837 : : return result;
838 : :
839 : : /* Since we're not changing the function identity proper, just
840 : : moving its full implementation, we *could* disable
841 : : fun->cannot_be_copied_reason and/or temporarily drop a noclone
842 : : attribute, but we'd have to prevent remapping of the labels. */
843 : 1881 : if (lookup_attribute ("noclone", DECL_ATTRIBUTES (node->decl)))
844 : : {
845 : 48 : result = false;
846 : :
847 : 48 : if (!report)
848 : : return result;
849 : :
850 : 0 : sorry_at (DECL_SOURCE_LOCATION (node->decl),
851 : : "%qD is not eligible for internal %<strub%>"
852 : : " because of attribute %<noclone%>",
853 : : node->decl);
854 : : }
855 : :
856 : 1833 : if (node->has_gimple_body_p ())
857 : : {
858 : 3763 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
859 : : {
860 : 2905 : tree cdecl = e->callee->decl;
861 : 5810 : if (!((fndecl_built_in_p (cdecl, BUILT_IN_VA_START)
862 : 8 : && cdecl != builtin_decl_explicit (BUILT_IN_VA_START))
863 : 2905 : || fndecl_built_in_p (cdecl, BUILT_IN_NEXT_ARG)
864 : 2905 : || fndecl_built_in_p (cdecl, BUILT_IN_RETURN_ADDRESS)))
865 : 2905 : continue;
866 : :
867 : 0 : result = false;
868 : :
869 : 0 : if (!report)
870 : : return result;
871 : :
872 : 0 : sorry_at (e->call_stmt
873 : 0 : ? gimple_location (e->call_stmt)
874 : 0 : : DECL_SOURCE_LOCATION (node->decl),
875 : : "%qD is not eligible for internal %<strub%> "
876 : : "because it calls %qD",
877 : : node->decl, cdecl);
878 : : }
879 : :
880 : 858 : struct function *fun = DECL_STRUCT_FUNCTION (node->decl);
881 : 858 : if (fun->has_nonlocal_label)
882 : : {
883 : 0 : result = false;
884 : :
885 : 0 : if (!report)
886 : : return result;
887 : :
888 : 0 : sorry_at (DECL_SOURCE_LOCATION (node->decl),
889 : : "%qD is not eligible for internal %<strub%> "
890 : : "because it contains a non-local goto target",
891 : : node->decl);
892 : : }
893 : :
894 : 858 : if (fun->has_forced_label_in_static)
895 : : {
896 : 0 : result = false;
897 : :
898 : 0 : if (!report)
899 : : return result;
900 : :
901 : 0 : sorry_at (DECL_SOURCE_LOCATION (node->decl),
902 : : "%qD is not eligible for internal %<strub%> "
903 : : "because the address of a local label escapes",
904 : : node->decl);
905 : : }
906 : :
907 : : /* Catch any other case that would prevent versioning/cloning
908 : : so as to also have it covered above. */
909 : 858 : gcc_checking_assert (!result /* || !node->has_gimple_body_p () */
910 : : || tree_versionable_function_p (node->decl));
911 : :
912 : :
913 : : /* Label values references are not preserved when copying. If referenced
914 : : in nested functions, as in 920415-1.c and 920721-4.c their decls get
915 : : remapped independently. The exclusion below might be too broad, in
916 : : that we might be able to support correctly cases in which the labels
917 : : are only used internally in a function, but disconnecting forced labels
918 : : from their original declarations is undesirable in general. */
919 : 858 : basic_block bb;
920 : 2970 : FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
921 : 4224 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
922 : 2251 : !gsi_end_p (gsi); gsi_next (&gsi))
923 : : {
924 : 2251 : glabel *label_stmt = dyn_cast <glabel *> (gsi_stmt (gsi));
925 : 139 : tree target;
926 : :
927 : 139 : if (!label_stmt)
928 : : break;
929 : :
930 : 139 : target = gimple_label_label (label_stmt);
931 : :
932 : 139 : if (!FORCED_LABEL (target))
933 : 139 : continue;
934 : :
935 : 0 : result = false;
936 : :
937 : 0 : if (!report)
938 : 48 : return result;
939 : :
940 : 0 : sorry_at (gimple_location (label_stmt),
941 : : "internal %<strub%> does not support forced labels");
942 : : }
943 : : }
944 : :
945 : 1833 : if (list_length (TYPE_ARG_TYPES (TREE_TYPE (node->decl)))
946 : : >= ((HOST_WIDE_INT_1 << IPA_PARAM_MAX_INDEX_BITS)
947 : : - STRUB_INTERNAL_MAX_EXTRA_ARGS))
948 : : {
949 : 0 : result = false;
950 : :
951 : 0 : if (!report)
952 : : return result;
953 : :
954 : 0 : sorry_at (DECL_SOURCE_LOCATION (node->decl),
955 : : "%qD has too many arguments for internal %<strub%>",
956 : : node->decl);
957 : : }
958 : :
959 : : return result;
960 : : }
961 : :
962 : : /* Return TRUE iff NODE has any strub-requiring local variable, or accesses (as
963 : : in reading) any variable through a strub-requiring type. */
964 : :
965 : : static bool
966 : 1880 : strub_from_body_p (cgraph_node *node)
967 : : {
968 : 1880 : if (!node->has_gimple_body_p ())
969 : : return false;
970 : :
971 : : /* If any local variable is marked for strub... */
972 : 1128 : unsigned i;
973 : 1128 : tree var;
974 : 9509 : FOR_EACH_LOCAL_DECL (DECL_STRUCT_FUNCTION (node->decl),
975 : : i, var)
976 : 7435 : if (get_strub_mode_from_type (TREE_TYPE (var))
977 : : != STRUB_DISABLED)
978 : : return true;
979 : :
980 : : /* Now scan the body for loads with strub-requiring types.
981 : : ??? Compound types don't propagate the strub requirement to
982 : : component types. */
983 : 1114 : basic_block bb;
984 : 4132 : FOR_EACH_BB_FN (bb, DECL_STRUCT_FUNCTION (node->decl))
985 : 6600 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
986 : 16985 : !gsi_end_p (gsi); gsi_next (&gsi))
987 : : {
988 : 13967 : gimple *stmt = gsi_stmt (gsi);
989 : :
990 : 13967 : if (!gimple_assign_load_p (stmt))
991 : 11412 : continue;
992 : :
993 : 2555 : tree rhs = gimple_assign_rhs1 (stmt);
994 : 2555 : if (get_strub_mode_from_type (TREE_TYPE (rhs))
995 : : != STRUB_DISABLED)
996 : 296 : return true;
997 : : }
998 : :
999 : : return false;
1000 : : }
1001 : :
1002 : : /* Return TRUE iff node is associated with a builtin that should be callable
1003 : : from strub contexts. */
1004 : :
1005 : : static inline bool
1006 : 1175 : strub_callable_builtin_p (cgraph_node *node)
1007 : : {
1008 : 1175 : if (DECL_BUILT_IN_CLASS (node->decl) != BUILT_IN_NORMAL)
1009 : : return false;
1010 : :
1011 : 489 : enum built_in_function fcode = DECL_FUNCTION_CODE (node->decl);
1012 : :
1013 : 489 : switch (fcode)
1014 : : {
1015 : 0 : case BUILT_IN_NONE:
1016 : 0 : gcc_unreachable ();
1017 : :
1018 : : /* This temporarily allocates stack for the call, and we can't reasonably
1019 : : update the watermark for that. Besides, we don't check the actual call
1020 : : target, nor its signature, and it seems to be overkill to as much as
1021 : : try to do so. */
1022 : : case BUILT_IN_APPLY:
1023 : : return false;
1024 : :
1025 : : /* Conversely, this shouldn't be called from within strub contexts, since
1026 : : the caller may have had its signature modified. STRUB_INTERNAL is ok,
1027 : : the call will remain in the STRUB_WRAPPER, and removed from the
1028 : : STRUB_WRAPPED clone. */
1029 : : case BUILT_IN_APPLY_ARGS:
1030 : : return false;
1031 : :
1032 : : /* ??? Make all other builtins callable. We wish to make any builtin call
1033 : : the compiler might introduce on its own callable. Anything that is
1034 : : predictable enough as to be known not to allow stack data that should
1035 : : be strubbed to unintentionally escape to non-strub contexts can be
1036 : : allowed, and pretty much every builtin appears to fit this description.
1037 : : The exceptions to this rule seem to be rare, and only available as
1038 : : explicit __builtin calls, so let's keep it simple and allow all of
1039 : : them... */
1040 : : default:
1041 : : return true;
1042 : : }
1043 : : }
1044 : :
1045 : : /* Compute the strub mode to be used for NODE. STRUB_ATTR should be the strub
1046 : : attribute,found for NODE, if any. */
1047 : :
1048 : : static enum strub_mode
1049 : 3481 : compute_strub_mode (cgraph_node *node, tree strub_attr)
1050 : : {
1051 : 3481 : enum strub_mode req_mode = get_strub_mode_from_attr (strub_attr);
1052 : :
1053 : 3481 : gcc_checking_assert (flag_strub >= -2 && flag_strub <= 3);
1054 : :
1055 : : /* Symbolic encodings of the -fstrub-* flags. */
1056 : : /* Enable strub when explicitly requested through attributes to functions or
1057 : : variables, reporting errors if the requests cannot be satisfied. */
1058 : 3481 : const bool strub_flag_auto = flag_strub < 0;
1059 : : /* strub_flag_auto with strub call verification; without this, functions are
1060 : : implicitly callable. */
1061 : 3481 : const bool strub_flag_strict = flag_strub < -1;
1062 : : /* Disable strub altogether, ignore attributes entirely. */
1063 : 3481 : const bool strub_flag_disabled = flag_strub == 0;
1064 : : /* On top of _auto, also enable strub implicitly for functions that can
1065 : : safely undergo at-calls strubbing. Internal mode will still be used in
1066 : : functions that request it explicitly with attribute strub(2), or when the
1067 : : function body requires strubbing and at-calls strubbing is not viable. */
1068 : 3481 : const bool strub_flag_at_calls = flag_strub == 1;
1069 : : /* On top of default, also enable strub implicitly for functions that can
1070 : : safely undergo internal strubbing. At-calls mode will still be used in
1071 : : functions that requiest it explicitly with attribute strub() or strub(1),
1072 : : or when the function body requires strubbing and internal strubbing is not
1073 : : viable. */
1074 : 3481 : const bool strub_flag_internal = flag_strub == 2;
1075 : : /* On top of default, also enable strub implicitly for functions that can
1076 : : safely undergo strubbing in either mode. When both modes are viable,
1077 : : at-calls is preferred. */
1078 : 3481 : const bool strub_flag_either = flag_strub == 3;
1079 : : /* Besides the default behavior, enable strub implicitly for all viable
1080 : : functions. */
1081 : 3481 : const bool strub_flag_viable = flag_strub > 0;
1082 : :
1083 : : /* The consider_* variables should be TRUE if selecting the corresponding
1084 : : strub modes would be consistent with requests from attributes and command
1085 : : line flags. Attributes associated with functions pretty much mandate a
1086 : : selection, and should report an error if not satisfied; strub_flag_auto
1087 : : implicitly enables some viable strub mode if that's required by references
1088 : : to variables marked for strub; strub_flag_viable enables strub if viable
1089 : : (even when favoring one mode, body-requested strub can still be satisfied
1090 : : by either mode), and falls back to callable, silently unless variables
1091 : : require strubbing. */
1092 : :
1093 : 10443 : const bool consider_at_calls
1094 : : = (!strub_flag_disabled
1095 : 3481 : && (strub_attr
1096 : 3481 : ? req_mode == STRUB_AT_CALLS
1097 : : : true));
1098 : 6962 : const bool consider_internal
1099 : : = (!strub_flag_disabled
1100 : : && (strub_attr
1101 : 3481 : ? req_mode == STRUB_INTERNAL
1102 : : : true));
1103 : :
1104 : 3481 : const bool consider_callable
1105 : : = (!strub_flag_disabled
1106 : 3481 : && (strub_attr
1107 : 1840 : ? req_mode == STRUB_CALLABLE
1108 : : : (!strub_flag_strict
1109 : 1175 : || strub_callable_builtin_p (node))));
1110 : :
1111 : : /* This is a shorthand for either strub-enabled mode. */
1112 : 3481 : const bool consider_strub
1113 : : = (consider_at_calls || consider_internal);
1114 : :
1115 : : /* We can cope with always_inline functions even with noipa and noclone,
1116 : : because we just leave them alone. */
1117 : 3481 : const bool is_always_inline
1118 : 3481 : = strub_always_inline_p (node);
1119 : :
1120 : : /* Strubbing in general, and each specific strub mode, may have its own set of
1121 : : requirements. We require noipa for strubbing, either because of cloning
1122 : : required for internal strub, or because of caller enumeration required for
1123 : : at-calls strub. We don't consider the at-calls mode eligible if it's not
1124 : : even considered, it has no further requirements. Internal mode requires
1125 : : cloning and the absence of certain features in the body and, like at-calls,
1126 : : it's not eligible if it's not even under consideration.
1127 : :
1128 : : ??? Do we need target hooks for further constraints? E.g., x86's
1129 : : "interrupt" attribute breaks internal strubbing because the wrapped clone
1130 : : carries the attribute and thus isn't callable; in this case, we could use a
1131 : : target hook to adjust the clone instead. */
1132 : 3481 : const bool strub_eligible
1133 : : = (consider_strub
1134 : 3481 : && (is_always_inline || can_strub_p (node)));
1135 : 2815 : const bool at_calls_eligible
1136 : : = (consider_at_calls && strub_eligible
1137 : 2815 : && can_strub_at_calls_p (node));
1138 : 3481 : const bool internal_eligible
1139 : 3481 : = (consider_internal && strub_eligible
1140 : 3481 : && (is_always_inline
1141 : 1877 : || can_strub_internally_p (node)));
1142 : :
1143 : : /* In addition to the strict eligibility requirements, some additional
1144 : : constraints are placed on implicit selection of certain modes. These do
1145 : : not prevent the selection of a mode if explicitly specified as part of a
1146 : : function interface (the strub attribute), but they may prevent modes from
1147 : : being selected by the command line or by function bodies. The only actual
1148 : : constraint is on at-calls mode: since we change the function's exposed
1149 : : signature, we won't do it implicitly if the function can possibly be used
1150 : : in ways that do not expect the signature change, e.g., if the function is
1151 : : available to or interposable by other units, if its address is taken,
1152 : : etc. */
1153 : 3481 : const bool at_calls_viable
1154 : : = (at_calls_eligible
1155 : 3481 : && (strub_attr
1156 : 1832 : || (node->has_gimple_body_p ()
1157 : 929 : && (!node->externally_visible
1158 : 447 : || (node->binds_to_current_def_p ()
1159 : 435 : && node->can_be_local_p ()))
1160 : 482 : && node->only_called_directly_p ()
1161 : 448 : && !called_with_type_override_p (node))));
1162 : 4620 : const bool internal_viable
1163 : : = (internal_eligible);
1164 : :
1165 : : /* Shorthand. */
1166 : 4620 : const bool strub_viable
1167 : : = (at_calls_viable || internal_viable);
1168 : :
1169 : : /* We wish to analyze the body, to look for implicit requests for strub, both
1170 : : to implicitly enable it when the body calls for it, and to report errors if
1171 : : the body calls for it but neither mode is viable (even if that follows from
1172 : : non-eligibility because of the explicit specification of some non-strubbing
1173 : : mode). We can refrain from scanning the body only in rare circumstances:
1174 : : when strub is enabled by a function attribute (scanning might be redundant
1175 : : in telling us to also enable it), and when we are enabling strub implicitly
1176 : : but there are non-viable modes: we want to know whether strubbing is
1177 : : required, to fallback to another mode, even if we're only enabling a
1178 : : certain mode, or, when either mode would do, to report an error if neither
1179 : : happens to be viable. */
1180 : 6460 : const bool analyze_body
1181 : : = (strub_attr
1182 : 3481 : ? !consider_strub
1183 : : : (strub_flag_auto
1184 : 618 : || (strub_flag_viable && (!at_calls_viable && !internal_viable))
1185 : 2458 : || (strub_flag_either && !strub_viable)));
1186 : :
1187 : : /* Cases in which strubbing is enabled or disabled by strub_flag_auto.
1188 : : Unsatisfiable requests ought to be reported. */
1189 : 3481 : const bool strub_required
1190 : 3481 : = ((strub_attr && consider_strub)
1191 : 3481 : || (analyze_body && strub_from_body_p (node)));
1192 : :
1193 : : /* Besides the required cases, we want to abide by the requests to enabling on
1194 : : an if-viable basis. */
1195 : 6983 : const bool strub_enable
1196 : : = (strub_required
1197 : 2202 : || (strub_flag_at_calls && at_calls_viable)
1198 : 2138 : || (strub_flag_internal && internal_viable)
1199 : 1927 : || (strub_flag_either && strub_viable));
1200 : :
1201 : : /* And now we're finally ready to select a mode that abides by the viability
1202 : : and eligibility constraints, and that satisfies the strubbing requirements
1203 : : and requests, subject to the constraints. If both modes are viable and
1204 : : strub is to be enabled, pick STRUB_AT_CALLS unless STRUB_INTERNAL was named
1205 : : as preferred. */
1206 : 7871 : const enum strub_mode mode
1207 : : = ((strub_enable && is_always_inline)
1208 : 1778 : ? (strub_required ? STRUB_INLINABLE : STRUB_CALLABLE)
1209 : : : (strub_enable && internal_viable
1210 : 894 : && (strub_flag_internal || !at_calls_viable))
1211 : 3003 : ? STRUB_INTERNAL
1212 : 2185 : : (strub_enable && at_calls_viable)
1213 : 2185 : ? (strub_required && !strub_attr
1214 : 470 : ? STRUB_AT_CALLS_OPT
1215 : : : STRUB_AT_CALLS)
1216 : : : consider_callable
1217 : 1715 : ? STRUB_CALLABLE
1218 : : : STRUB_DISABLED);
1219 : :
1220 : 8 : switch (mode)
1221 : : {
1222 : 1077 : case STRUB_CALLABLE:
1223 : 1077 : if (is_always_inline)
1224 : : break;
1225 : : /* Fall through. */
1226 : :
1227 : 1527 : case STRUB_DISABLED:
1228 : 1527 : if (strub_enable && !strub_attr)
1229 : : {
1230 : 0 : gcc_checking_assert (analyze_body);
1231 : 0 : error_at (DECL_SOURCE_LOCATION (node->decl),
1232 : : "%qD requires %<strub%>,"
1233 : : " but no viable %<strub%> mode was found",
1234 : : node->decl);
1235 : 0 : break;
1236 : : }
1237 : : /* Fall through. */
1238 : :
1239 : : case STRUB_AT_CALLS:
1240 : : case STRUB_INTERNAL:
1241 : : case STRUB_INLINABLE:
1242 : : /* Differences from an mode requested through a function attribute are
1243 : : reported in set_strub_mode_to. */
1244 : : break;
1245 : :
1246 : 8 : case STRUB_AT_CALLS_OPT:
1247 : : /* Functions that select this mode do so because of references to strub
1248 : : variables. Even if we choose at-calls as an optimization, the
1249 : : requirements for internal strub must still be satisfied. Optimization
1250 : : options may render implicit at-calls strub not viable (-O0 sets
1251 : : force_output for static non-inline functions), and it would not be good
1252 : : if changing optimization options turned a well-formed into an
1253 : : ill-formed one. */
1254 : 8 : if (!internal_viable)
1255 : 0 : can_strub_internally_p (node, true);
1256 : : break;
1257 : :
1258 : : case STRUB_WRAPPED:
1259 : : case STRUB_WRAPPER:
1260 : : default:
1261 : : gcc_unreachable ();
1262 : : }
1263 : :
1264 : 3481 : return mode;
1265 : : }
1266 : :
1267 : : /* Set FNDT's strub mode to MODE; FNDT may be a function decl or
1268 : : function type. If OVERRIDE, do not check whether a mode is already
1269 : : set. */
1270 : :
1271 : : static void
1272 : 2603 : strub_set_fndt_mode_to (tree fndt, enum strub_mode mode, bool override)
1273 : : {
1274 : 2603 : gcc_checking_assert (override
1275 : : || !(DECL_P (fndt)
1276 : : ? get_strub_attr_from_decl (fndt)
1277 : : : get_strub_attr_from_type (fndt)));
1278 : :
1279 : 2603 : tree attr = tree_cons (get_identifier ("strub"),
1280 : : get_strub_mode_attr_value (mode),
1281 : : NULL_TREE);
1282 : 2603 : tree *attrp = NULL;
1283 : 2603 : if (DECL_P (fndt))
1284 : : {
1285 : 2603 : gcc_checking_assert (FUNC_OR_METHOD_TYPE_P (TREE_TYPE (fndt)));
1286 : 2603 : attrp = &DECL_ATTRIBUTES (fndt);
1287 : : }
1288 : 0 : else if (FUNC_OR_METHOD_TYPE_P (fndt))
1289 : 0 : attrp = &TYPE_ATTRIBUTES (fndt);
1290 : : else
1291 : 0 : gcc_unreachable ();
1292 : :
1293 : 2603 : TREE_CHAIN (attr) = *attrp;
1294 : 2603 : *attrp = attr;
1295 : 2603 : }
1296 : :
1297 : : /* Set FNDT's strub mode to callable.
1298 : : FNDT may be a function decl or a function type. */
1299 : :
1300 : : void
1301 : 0 : strub_make_callable (tree fndt)
1302 : : {
1303 : 0 : strub_set_fndt_mode_to (fndt, STRUB_CALLABLE, false);
1304 : 0 : }
1305 : :
1306 : : /* Set NODE to strub MODE. Report incompatibilities between MODE and the mode
1307 : : requested through explicit attributes, and cases of non-eligibility. */
1308 : :
1309 : : static void
1310 : 4422 : set_strub_mode_to (cgraph_node *node, enum strub_mode mode)
1311 : : {
1312 : 4422 : tree attr = get_strub_attr_from_decl (node->decl);
1313 : 4422 : enum strub_mode req_mode = get_strub_mode_from_attr (attr);
1314 : :
1315 : 4422 : if (attr)
1316 : : {
1317 : : /* Check for and report incompatible mode changes. */
1318 : 2573 : if (mode != req_mode
1319 : 1192 : && !(req_mode == STRUB_INTERNAL
1320 : : && (mode == STRUB_WRAPPED
1321 : 1076 : || mode == STRUB_WRAPPER))
1322 : 260 : && !((req_mode == STRUB_INTERNAL
1323 : : || req_mode == STRUB_AT_CALLS
1324 : 260 : || req_mode == STRUB_CALLABLE)
1325 : : && mode == STRUB_INLINABLE))
1326 : : {
1327 : 12 : error_at (DECL_SOURCE_LOCATION (node->decl),
1328 : : "%<strub%> mode %qE selected for %qD, when %qE was requested",
1329 : : get_strub_mode_attr_parm (mode),
1330 : : node->decl,
1331 : : get_strub_mode_attr_parm (req_mode));
1332 : 12 : if (node->alias)
1333 : : {
1334 : 0 : cgraph_node *target = node->ultimate_alias_target ();
1335 : 0 : if (target != node)
1336 : 0 : error_at (DECL_SOURCE_LOCATION (target->decl),
1337 : : "the incompatible selection was determined"
1338 : : " by ultimate alias target %qD",
1339 : : target->decl);
1340 : : }
1341 : :
1342 : : /* Report any incompatibilities with explicitly-requested strub. */
1343 : 12 : switch (req_mode)
1344 : : {
1345 : 8 : case STRUB_AT_CALLS:
1346 : 8 : can_strub_at_calls_p (node, true);
1347 : 8 : break;
1348 : :
1349 : 4 : case STRUB_INTERNAL:
1350 : 4 : can_strub_internally_p (node, true);
1351 : 4 : break;
1352 : :
1353 : : default:
1354 : : break;
1355 : : }
1356 : : }
1357 : :
1358 : : /* Drop any incompatible strub attributes leading the decl attribute
1359 : : chain. Return if we find one with the mode we need. */
1360 : 2573 : for (;;)
1361 : : {
1362 : 2573 : if (mode == req_mode)
1363 : : return;
1364 : :
1365 : 1192 : if (DECL_ATTRIBUTES (node->decl) != attr)
1366 : : break;
1367 : :
1368 : 450 : DECL_ATTRIBUTES (node->decl) = TREE_CHAIN (attr);
1369 : 450 : attr = get_strub_attr_from_decl (node->decl);
1370 : 450 : if (!attr)
1371 : : break;
1372 : :
1373 : 0 : req_mode = get_strub_mode_from_attr (attr);
1374 : : }
1375 : : }
1376 : 1849 : else if (mode == req_mode)
1377 : : return;
1378 : :
1379 : 2603 : strub_set_fndt_mode_to (node->decl, mode, attr);
1380 : : }
1381 : :
1382 : : /* Compute and set NODE's strub mode. */
1383 : :
1384 : : static void
1385 : 3490 : set_strub_mode (cgraph_node *node)
1386 : : {
1387 : 3490 : tree attr = get_strub_attr_from_decl (node->decl);
1388 : :
1389 : 3490 : if (attr)
1390 : 1641 : switch (get_strub_mode_from_attr (attr))
1391 : : {
1392 : : /* These can't have been requested through user attributes, so we must
1393 : : have already gone through them. */
1394 : : case STRUB_WRAPPER:
1395 : : case STRUB_WRAPPED:
1396 : : case STRUB_INLINABLE:
1397 : : case STRUB_AT_CALLS_OPT:
1398 : : return;
1399 : :
1400 : : case STRUB_DISABLED:
1401 : : case STRUB_AT_CALLS:
1402 : : case STRUB_INTERNAL:
1403 : : case STRUB_CALLABLE:
1404 : : break;
1405 : :
1406 : : default:
1407 : : gcc_unreachable ();
1408 : : }
1409 : :
1410 : 3490 : cgraph_node *xnode = node;
1411 : 3490 : if (node->alias)
1412 : 9 : xnode = node->ultimate_alias_target ();
1413 : : /* Weakrefs may remain unresolved (the above will return node) if
1414 : : their targets are not defined, so make sure we compute a strub
1415 : : mode for them, instead of defaulting to STRUB_DISABLED and
1416 : : rendering them uncallable. */
1417 : 9 : enum strub_mode mode = (xnode != node && !xnode->alias
1418 : 3499 : ? get_strub_mode (xnode)
1419 : 3481 : : compute_strub_mode (node, attr));
1420 : :
1421 : 3490 : set_strub_mode_to (node, mode);
1422 : : }
1423 : :
1424 : :
1425 : : /* Non-strub functions shouldn't be called from within strub contexts,
1426 : : except through callable ones. Always inline strub functions can
1427 : : only be called from strub functions. */
1428 : :
1429 : : static bool
1430 : 9144 : strub_callable_from_p (strub_mode caller_mode, strub_mode callee_mode)
1431 : : {
1432 : 9144 : switch (caller_mode)
1433 : : {
1434 : 4831 : case STRUB_WRAPPED:
1435 : 4831 : case STRUB_AT_CALLS_OPT:
1436 : 4831 : case STRUB_AT_CALLS:
1437 : 4831 : case STRUB_INTERNAL:
1438 : 4831 : case STRUB_INLINABLE:
1439 : 4831 : break;
1440 : :
1441 : 4313 : case STRUB_WRAPPER:
1442 : 4313 : case STRUB_DISABLED:
1443 : 4313 : case STRUB_CALLABLE:
1444 : 4313 : return callee_mode != STRUB_INLINABLE;
1445 : :
1446 : 0 : default:
1447 : 0 : gcc_unreachable ();
1448 : : }
1449 : :
1450 : 4831 : switch (callee_mode)
1451 : : {
1452 : : case STRUB_WRAPPED:
1453 : : case STRUB_AT_CALLS:
1454 : : case STRUB_INLINABLE:
1455 : : break;
1456 : :
1457 : 860 : case STRUB_AT_CALLS_OPT:
1458 : 860 : case STRUB_INTERNAL:
1459 : 860 : case STRUB_WRAPPER:
1460 : 860 : return (flag_strub >= -1);
1461 : :
1462 : : case STRUB_DISABLED:
1463 : : return false;
1464 : :
1465 : : case STRUB_CALLABLE:
1466 : : break;
1467 : :
1468 : 0 : default:
1469 : 0 : gcc_unreachable ();
1470 : : }
1471 : :
1472 : 3635 : return true;
1473 : : }
1474 : :
1475 : : /* Return TRUE iff CALLEE can be inlined into CALLER. We wish to avoid inlining
1476 : : WRAPPED functions back into their WRAPPERs. More generally, we wish to avoid
1477 : : inlining strubbed functions into non-strubbed ones. CALLER doesn't have to
1478 : : be an immediate caller of CALLEE: the immediate caller may have already been
1479 : : cloned for inlining, and then CALLER may be further up the original call
1480 : : chain. ??? It would be nice if our own caller would retry inlining callee
1481 : : if caller gets inlined. */
1482 : :
1483 : : bool
1484 : 8077666 : strub_inlinable_to_p (cgraph_node *callee, cgraph_node *caller)
1485 : : {
1486 : 8077666 : strub_mode callee_mode = get_strub_mode (callee);
1487 : :
1488 : 8077666 : switch (callee_mode)
1489 : : {
1490 : 3855 : case STRUB_WRAPPED:
1491 : 3855 : case STRUB_AT_CALLS:
1492 : 3855 : case STRUB_INTERNAL:
1493 : 3855 : case STRUB_INLINABLE:
1494 : 3855 : case STRUB_AT_CALLS_OPT:
1495 : 3855 : break;
1496 : :
1497 : : case STRUB_WRAPPER:
1498 : : case STRUB_DISABLED:
1499 : : case STRUB_CALLABLE:
1500 : : /* When we consider inlining, we've already verified callability, so we
1501 : : can even inline callable and then disabled into a strub context. That
1502 : : will get strubbed along with the context, so it's hopefully not a
1503 : : problem. */
1504 : : return true;
1505 : :
1506 : : default:
1507 : : gcc_unreachable ();
1508 : : }
1509 : :
1510 : 3855 : strub_mode caller_mode = get_strub_mode (caller);
1511 : :
1512 : 3855 : switch (caller_mode)
1513 : : {
1514 : : case STRUB_WRAPPED:
1515 : : case STRUB_AT_CALLS:
1516 : : case STRUB_INTERNAL:
1517 : : case STRUB_INLINABLE:
1518 : : case STRUB_AT_CALLS_OPT:
1519 : : return true;
1520 : :
1521 : : case STRUB_WRAPPER:
1522 : : case STRUB_DISABLED:
1523 : : case STRUB_CALLABLE:
1524 : : break;
1525 : :
1526 : : default:
1527 : : gcc_unreachable ();
1528 : : }
1529 : :
1530 : : return false;
1531 : : }
1532 : :
1533 : : /* Check that types T1 and T2 are strub-compatible. Return 1 if the strub modes
1534 : : are the same, 2 if they are interchangeable, and 0 otherwise. */
1535 : :
1536 : : int
1537 : 1439054 : strub_comptypes (tree t1, tree t2)
1538 : : {
1539 : 1439054 : if (TREE_CODE (t1) != TREE_CODE (t2))
1540 : : return 0;
1541 : :
1542 : 1439054 : enum strub_mode m1 = get_strub_mode_from_type (t1);
1543 : 1439054 : enum strub_mode m2 = get_strub_mode_from_type (t2);
1544 : :
1545 : 1439054 : if (m1 == m2)
1546 : : return 1;
1547 : :
1548 : : /* We're dealing with types, so only strub modes that can be selected by
1549 : : attributes in the front end matter. If either mode is at-calls (for
1550 : : functions) or internal (for variables), the conversion is not
1551 : : compatible. */
1552 : 4351 : bool var_p = !FUNC_OR_METHOD_TYPE_P (t1);
1553 : 4351 : enum strub_mode mr = var_p ? STRUB_INTERNAL : STRUB_AT_CALLS;
1554 : 4351 : if (m1 == mr || m2 == mr)
1555 : : return 0;
1556 : :
1557 : : return 2;
1558 : : }
1559 : :
1560 : : /* Return the effective strub mode used for CALL, and set *TYPEP to
1561 : : the effective type used for the call. The effective type and mode
1562 : : are those of the callee, unless the call involves a typecast. */
1563 : :
1564 : : static enum strub_mode
1565 : 18011 : effective_strub_mode_for_call (gcall *call, tree *typep)
1566 : : {
1567 : 18011 : tree type;
1568 : 18011 : enum strub_mode mode;
1569 : :
1570 : 18011 : if (strub_call_fntype_override_p (call))
1571 : : {
1572 : 0 : type = gimple_call_fntype (call);
1573 : 0 : mode = get_strub_mode_from_type (type);
1574 : : }
1575 : : else
1576 : : {
1577 : 18011 : type = TREE_TYPE (TREE_TYPE (gimple_call_fn (call)));
1578 : 18011 : tree decl = gimple_call_fndecl (call);
1579 : 18011 : if (decl)
1580 : 17739 : mode = get_strub_mode_from_fndecl (decl);
1581 : : else
1582 : 272 : mode = get_strub_mode_from_type (type);
1583 : : }
1584 : :
1585 : 18011 : if (typep)
1586 : 17849 : *typep = type;
1587 : :
1588 : 18011 : return mode;
1589 : : }
1590 : :
1591 : : /* Create a distinct copy of the type of NODE's function, and change
1592 : : the fntype of all calls to it with the same main type to the new
1593 : : type. */
1594 : :
1595 : : static void
1596 : 72 : distinctify_node_type (cgraph_node *node)
1597 : : {
1598 : 72 : tree old_type = TREE_TYPE (node->decl);
1599 : 72 : tree new_type = build_distinct_type_copy (old_type);
1600 : 72 : tree new_ptr_type = NULL_TREE;
1601 : :
1602 : : /* Remap any calls to node->decl that use old_type, or a variant
1603 : : thereof, to new_type as well. We don't look for aliases, their
1604 : : declarations will have their types changed independently, and
1605 : : we'll adjust their fntypes then. */
1606 : 144 : for (cgraph_edge *e = node->callers; e; e = e->next_caller)
1607 : : {
1608 : 72 : if (!e->call_stmt)
1609 : 0 : continue;
1610 : 72 : tree fnaddr = gimple_call_fn (e->call_stmt);
1611 : 72 : gcc_checking_assert (TREE_CODE (fnaddr) == ADDR_EXPR
1612 : : && TREE_OPERAND (fnaddr, 0) == node->decl);
1613 : 72 : if (strub_call_fntype_override_p (e->call_stmt))
1614 : 0 : continue;
1615 : 72 : if (!new_ptr_type)
1616 : 72 : new_ptr_type = build_pointer_type (new_type);
1617 : 72 : TREE_TYPE (fnaddr) = new_ptr_type;
1618 : 72 : gimple_call_set_fntype (e->call_stmt, new_type);
1619 : : }
1620 : :
1621 : 72 : TREE_TYPE (node->decl) = new_type;
1622 : 72 : }
1623 : :
1624 : : /* Return TRUE iff TYPE and any variants have the same strub mode. */
1625 : :
1626 : : static bool
1627 : 2307 : same_strub_mode_in_variants_p (tree type)
1628 : : {
1629 : 2307 : enum strub_mode mode = get_strub_mode_from_type (type);
1630 : :
1631 : 2307 : for (tree other = TYPE_MAIN_VARIANT (type);
1632 : 4672 : other != NULL_TREE; other = TYPE_NEXT_VARIANT (other))
1633 : 2365 : if (type != other && mode != get_strub_mode_from_type (other))
1634 : : return false;
1635 : :
1636 : : /* Check that the canonical type, if set, either is in the same
1637 : : variant chain, or has the same strub mode as type. Also check
1638 : : the variants of the canonical type. */
1639 : 2307 : if (TYPE_CANONICAL (type)
1640 : 2307 : && (TYPE_MAIN_VARIANT (TYPE_CANONICAL (type))
1641 : 144 : != TYPE_MAIN_VARIANT (type)))
1642 : : {
1643 : 0 : if (mode != get_strub_mode_from_type (TYPE_CANONICAL (type)))
1644 : : return false;
1645 : : else
1646 : 0 : return same_strub_mode_in_variants_p (TYPE_CANONICAL (type));
1647 : : }
1648 : :
1649 : : return true;
1650 : : }
1651 : :
1652 : : /* Check that strub functions don't call non-strub functions, and that
1653 : : always_inline strub functions are only called by strub
1654 : : functions. */
1655 : :
1656 : : static void
1657 : 608 : verify_strub ()
1658 : : {
1659 : 608 : cgraph_node *node;
1660 : :
1661 : : /* It's expected that check strub-wise pointer type compatibility of variables
1662 : : and of functions is already taken care of by front-ends, on account of the
1663 : : attribute's being marked as affecting type identity and of the creation of
1664 : : distinct types. */
1665 : :
1666 : : /* Check that call targets in strub contexts have strub-callable types. */
1667 : :
1668 : 2825 : FOR_EACH_FUNCTION_WITH_GIMPLE_BODY (node)
1669 : : {
1670 : 2217 : enum strub_mode caller_mode = get_strub_mode (node);
1671 : :
1672 : 2379 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
1673 : : {
1674 : 162 : gcc_checking_assert (e->indirect_unknown_callee);
1675 : :
1676 : 162 : if (!e->call_stmt)
1677 : 0 : continue;
1678 : :
1679 : 162 : enum strub_mode callee_mode
1680 : 162 : = effective_strub_mode_for_call (e->call_stmt, NULL);
1681 : :
1682 : 162 : if (!strub_callable_from_p (caller_mode, callee_mode))
1683 : 28 : error_at (gimple_location (e->call_stmt),
1684 : : "indirect non-%<strub%> call in %<strub%> context %qD",
1685 : : node->decl);
1686 : : }
1687 : :
1688 : 11199 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
1689 : : {
1690 : 8982 : gcc_checking_assert (!e->indirect_unknown_callee);
1691 : :
1692 : 8982 : if (!e->call_stmt)
1693 : 16 : continue;
1694 : :
1695 : 8982 : tree callee_fntype;
1696 : 8982 : enum strub_mode callee_mode
1697 : 8982 : = effective_strub_mode_for_call (e->call_stmt, &callee_fntype);
1698 : :
1699 : 8982 : if (!strub_callable_from_p (caller_mode, callee_mode))
1700 : : {
1701 : 814 : if (callee_mode == STRUB_INLINABLE)
1702 : 210 : error_at (gimple_location (e->call_stmt),
1703 : : "calling %<always_inline%> %<strub%> %qD"
1704 : : " in non-%<strub%> context %qD",
1705 : 210 : e->callee->decl, node->decl);
1706 : 604 : else if (fndecl_built_in_p (e->callee->decl, BUILT_IN_APPLY_ARGS)
1707 : 604 : && caller_mode == STRUB_INTERNAL)
1708 : : /* This is ok, it will be kept in the STRUB_WRAPPER, and removed
1709 : : from the STRUB_WRAPPED's strub context. */
1710 : 16 : continue;
1711 : 588 : else if (!strub_call_fntype_override_p (e->call_stmt))
1712 : 588 : error_at (gimple_location (e->call_stmt),
1713 : : "calling non-%<strub%> %qD in %<strub%> context %qD",
1714 : 588 : e->callee->decl, node->decl);
1715 : : else
1716 : 0 : error_at (gimple_location (e->call_stmt),
1717 : : "calling %qD using non-%<strub%> type %qT"
1718 : : " in %<strub%> context %qD",
1719 : 0 : e->callee->decl, callee_fntype, node->decl);
1720 : : }
1721 : : }
1722 : : }
1723 : 608 : }
1724 : :
1725 : : namespace {
1726 : :
1727 : : /* Define a pass to compute strub modes. */
1728 : : const pass_data pass_data_ipa_strub_mode = {
1729 : : SIMPLE_IPA_PASS,
1730 : : "strubm",
1731 : : OPTGROUP_NONE,
1732 : : TV_NONE,
1733 : : PROP_cfg, // properties_required
1734 : : 0, // properties_provided
1735 : : 0, // properties_destroyed
1736 : : 0, // properties_start
1737 : : 0, // properties_finish
1738 : : };
1739 : :
1740 : : class pass_ipa_strub_mode : public simple_ipa_opt_pass
1741 : : {
1742 : : public:
1743 : 280114 : pass_ipa_strub_mode (gcc::context *ctxt)
1744 : 560228 : : simple_ipa_opt_pass (pass_data_ipa_strub_mode, ctxt)
1745 : : {}
1746 : 0 : opt_pass *clone () { return new pass_ipa_strub_mode (m_ctxt); }
1747 : 224613 : virtual bool gate (function *) {
1748 : : /* In relaxed (-3) and strict (-4) settings, that only enable strub at a
1749 : : function or variable attribute's request, the attribute handler changes
1750 : : flag_strub to -1 or -2, respectively, if any strub-enabling occurence of
1751 : : the attribute is found. Therefore, if it remains at -3 or -4, nothing
1752 : : that would enable strub was found, so we can disable it and avoid the
1753 : : overhead. */
1754 : 224613 : if (flag_strub < -2)
1755 : 224000 : flag_strub = 0;
1756 : 224613 : return flag_strub;
1757 : : }
1758 : : virtual unsigned int execute (function *);
1759 : : };
1760 : :
1761 : : /* Define a pass to introduce strub transformations. */
1762 : : const pass_data pass_data_ipa_strub = {
1763 : : SIMPLE_IPA_PASS,
1764 : : "strub",
1765 : : OPTGROUP_NONE,
1766 : : TV_NONE,
1767 : : PROP_cfg | PROP_ssa, // properties_required
1768 : : 0, // properties_provided
1769 : : 0, // properties_destroyed
1770 : : 0, // properties_start
1771 : : TODO_update_ssa
1772 : : | TODO_cleanup_cfg
1773 : : | TODO_rebuild_cgraph_edges
1774 : : | TODO_verify_il, // properties_finish
1775 : : };
1776 : :
1777 : : class pass_ipa_strub : public simple_ipa_opt_pass
1778 : : {
1779 : : public:
1780 : 280114 : pass_ipa_strub (gcc::context *ctxt)
1781 : 560228 : : simple_ipa_opt_pass (pass_data_ipa_strub, ctxt)
1782 : : {}
1783 : 0 : opt_pass *clone () { return new pass_ipa_strub (m_ctxt); }
1784 : 224613 : virtual bool gate (function *) { return flag_strub && !seen_error (); }
1785 : : virtual unsigned int execute (function *);
1786 : :
1787 : : /* Define on demand and cache some types we use often. */
1788 : : #define DEF_TYPE(IDX, NAME, INIT) \
1789 : : static inline tree get_ ## NAME () { \
1790 : : int idx = STRUB_TYPE_BASE + IDX; \
1791 : : static tree type = strub_cache[idx]; \
1792 : : if (!type) \
1793 : : strub_cache[idx] = type = (INIT); \
1794 : : return type; \
1795 : : }
1796 : :
1797 : : /* Use a distinct ptr_type_node to denote the watermark, so that we can
1798 : : recognize it in arg lists and avoid modifying types twice. */
1799 : 2509 : DEF_TYPE (0, wmt, build_variant_type_copy (ptr_type_node))
1800 : :
1801 : 2523 : DEF_TYPE (1, pwmt, build_reference_type (get_wmt ()))
1802 : :
1803 : 6236 : DEF_TYPE (2, qpwmt,
1804 : : build_qualified_type (get_pwmt (),
1805 : : TYPE_QUAL_RESTRICT
1806 : : /* | TYPE_QUAL_CONST */))
1807 : :
1808 : 16 : DEF_TYPE (3, qptr,
1809 : : build_qualified_type (ptr_type_node,
1810 : : TYPE_QUAL_RESTRICT
1811 : : | TYPE_QUAL_CONST))
1812 : :
1813 : 8 : DEF_TYPE (4, qpvalst,
1814 : : build_qualified_type (build_reference_type
1815 : : (va_list_type_node),
1816 : : TYPE_QUAL_RESTRICT
1817 : : /* | TYPE_QUAL_CONST */))
1818 : :
1819 : : #undef DEF_TYPE
1820 : :
1821 : : /* Define non-strub builtins on demand. */
1822 : : #define DEF_NM_BUILTIN(NAME, CODE, FNTYPELIST) \
1823 : : static tree get_ ## NAME () { \
1824 : : tree decl = builtin_decl_explicit (CODE); \
1825 : : if (!decl) \
1826 : : { \
1827 : : tree type = build_function_type_list FNTYPELIST; \
1828 : : decl = add_builtin_function \
1829 : : ("__builtin_" #NAME, \
1830 : : type, CODE, BUILT_IN_NORMAL, \
1831 : : NULL, NULL); \
1832 : : TREE_NOTHROW (decl) = true; \
1833 : : set_builtin_decl ((CODE), decl, true); \
1834 : : } \
1835 : : return decl; \
1836 : : }
1837 : :
1838 : : DEF_NM_BUILTIN (stack_address,
1839 : : BUILT_IN_STACK_ADDRESS,
1840 : : (ptr_type_node, NULL))
1841 : :
1842 : : #undef DEF_NM_BUILTIN
1843 : :
1844 : : /* Define strub builtins on demand. */
1845 : : #define DEF_SS_BUILTIN(NAME, FNSPEC, CODE, FNTYPELIST) \
1846 : : static tree get_ ## NAME () { \
1847 : : tree decl = builtin_decl_explicit (CODE); \
1848 : : if (!decl) \
1849 : : { \
1850 : : tree type = build_function_type_list FNTYPELIST; \
1851 : : tree attrs = NULL; \
1852 : : if (FNSPEC) \
1853 : : attrs = tree_cons (get_identifier ("fn spec"), \
1854 : : build_tree_list \
1855 : : (NULL_TREE, \
1856 : : build_string (strlen (FNSPEC), \
1857 : : (FNSPEC))), \
1858 : : attrs); \
1859 : : decl = add_builtin_function_ext_scope \
1860 : : ("__builtin___strub_" #NAME, \
1861 : : type, CODE, BUILT_IN_NORMAL, \
1862 : : "__strub_" #NAME, attrs); \
1863 : : TREE_NOTHROW (decl) = true; \
1864 : : set_builtin_decl ((CODE), decl, true); \
1865 : : } \
1866 : : return decl; \
1867 : : }
1868 : :
1869 : 1985 : DEF_SS_BUILTIN (enter, ". Ot",
1870 : : BUILT_IN___STRUB_ENTER,
1871 : : (void_type_node, get_qpwmt (), NULL))
1872 : 1025 : DEF_SS_BUILTIN (update, ". Wt",
1873 : : BUILT_IN___STRUB_UPDATE,
1874 : : (void_type_node, get_qpwmt (), NULL))
1875 : 1985 : DEF_SS_BUILTIN (leave, ". w ",
1876 : : BUILT_IN___STRUB_LEAVE,
1877 : : (void_type_node, get_qpwmt (), NULL))
1878 : :
1879 : : #undef DEF_SS_BUILTIN
1880 : :
1881 : : /* Define strub identifiers on demand. */
1882 : : #define DEF_IDENT(IDX, NAME) \
1883 : : static inline tree get_ ## NAME () { \
1884 : : int idx = STRUB_IDENT_BASE + IDX; \
1885 : : tree identifier = strub_cache[idx]; \
1886 : : if (!identifier) \
1887 : : strub_cache[idx] = identifier = get_identifier (".strub." #NAME); \
1888 : : return identifier; \
1889 : : }
1890 : :
1891 : 831 : DEF_IDENT (0, watermark_ptr)
1892 : 8 : DEF_IDENT (1, va_list_ptr)
1893 : 16 : DEF_IDENT (2, apply_args)
1894 : :
1895 : : #undef DEF_IDENT
1896 : :
1897 : : static inline int adjust_at_calls_type (tree);
1898 : : static inline void adjust_at_calls_call (cgraph_edge *, int, tree);
1899 : : static inline void adjust_at_calls_calls (cgraph_node *);
1900 : :
1901 : : /* Add to SEQ a call to the strub watermark update builtin, taking NODE's
1902 : : location if given. Optionally add the corresponding edge from NODE, with
1903 : : execution frequency COUNT. Return the modified SEQ. */
1904 : :
1905 : : static inline gimple_seq
1906 : 1025 : call_update_watermark (tree wmptr, cgraph_node *node, profile_count count,
1907 : : gimple_seq seq = NULL)
1908 : : {
1909 : 1025 : tree uwm = get_update ();
1910 : 1025 : gcall *update = gimple_build_call (uwm, 1, wmptr);
1911 : 1025 : if (node)
1912 : 787 : gimple_set_location (update, DECL_SOURCE_LOCATION (node->decl));
1913 : 1025 : gimple_seq_add_stmt (&seq, update);
1914 : 1025 : if (node)
1915 : 787 : node->create_edge (cgraph_node::get_create (uwm), update, count, false);
1916 : 1025 : return seq;
1917 : : }
1918 : :
1919 : : };
1920 : :
1921 : : } // anon namespace
1922 : :
1923 : : /* Gather with this type a collection of parameters that we're turning into
1924 : : explicit references. */
1925 : :
1926 : : typedef hash_set<tree> indirect_parms_t;
1927 : :
1928 : : /* Dereference OP's incoming turned-into-reference parm if it's an
1929 : : INDIRECT_PARMS or an ADDR_EXPR thereof. Set *REC and return according to
1930 : : gimple-walking expectations. */
1931 : :
1932 : : static tree
1933 : 131 : maybe_make_indirect (indirect_parms_t &indirect_parms, tree op, int *rec)
1934 : : {
1935 : 131 : if (DECL_P (op))
1936 : : {
1937 : 19 : *rec = 0;
1938 : 19 : if (indirect_parms.contains (op))
1939 : : {
1940 : 0 : tree ret = gimple_fold_indirect_ref (op);
1941 : 0 : if (!ret)
1942 : 0 : ret = build2 (MEM_REF,
1943 : 0 : TREE_TYPE (TREE_TYPE (op)),
1944 : : op,
1945 : 0 : build_int_cst (TREE_TYPE (op), 0));
1946 : 0 : if (TYPE_VOLATILE (TREE_TYPE (TREE_TYPE (op)))
1947 : 0 : && !TREE_THIS_VOLATILE (ret))
1948 : 0 : TREE_SIDE_EFFECTS (ret) = TREE_THIS_VOLATILE (ret) = 1;
1949 : 0 : return ret;
1950 : : }
1951 : : }
1952 : 112 : else if (TREE_CODE (op) == ADDR_EXPR
1953 : 112 : && DECL_P (TREE_OPERAND (op, 0)))
1954 : : {
1955 : 11 : *rec = 0;
1956 : 11 : if (indirect_parms.contains (TREE_OPERAND (op, 0)))
1957 : : {
1958 : 0 : op = TREE_OPERAND (op, 0);
1959 : 0 : return op;
1960 : : }
1961 : : }
1962 : :
1963 : : return NULL_TREE;
1964 : : }
1965 : :
1966 : : /* A gimple-walking function that adds dereferencing to indirect parms. */
1967 : :
1968 : : static tree
1969 : 131 : walk_make_indirect (tree *op, int *rec, void *arg)
1970 : : {
1971 : 131 : walk_stmt_info *wi = (walk_stmt_info *)arg;
1972 : 131 : indirect_parms_t &indirect_parms = *(indirect_parms_t *)wi->info;
1973 : :
1974 : 131 : if (!*op || TYPE_P (*op))
1975 : : {
1976 : 0 : *rec = 0;
1977 : 0 : return NULL_TREE;
1978 : : }
1979 : :
1980 : 131 : if (tree repl = maybe_make_indirect (indirect_parms, *op, rec))
1981 : : {
1982 : 0 : *op = repl;
1983 : 0 : wi->changed = true;
1984 : : }
1985 : :
1986 : : return NULL_TREE;
1987 : : }
1988 : :
1989 : : /* A gimple-walking function that turns any non-gimple-val ADDR_EXPRs into a
1990 : : separate SSA. Though addresses of e.g. parameters, and of members thereof,
1991 : : are gimple vals, turning parameters into references, with an extra layer of
1992 : : indirection and thus explicit dereferencing, need to be regimplified. */
1993 : :
1994 : : static tree
1995 : 0 : walk_regimplify_addr_expr (tree *op, int *rec, void *arg)
1996 : : {
1997 : 0 : walk_stmt_info *wi = (walk_stmt_info *)arg;
1998 : 0 : gimple_stmt_iterator &gsi = *(gimple_stmt_iterator *)wi->info;
1999 : :
2000 : 0 : *rec = 0;
2001 : :
2002 : 0 : if (!*op || TREE_CODE (*op) != ADDR_EXPR)
2003 : : return NULL_TREE;
2004 : :
2005 : 0 : if (!is_gimple_val (*op))
2006 : : {
2007 : 0 : tree ret = force_gimple_operand_gsi (&gsi, *op, true,
2008 : : NULL_TREE, true, GSI_SAME_STMT);
2009 : 0 : gcc_assert (ret != *op);
2010 : 0 : *op = ret;
2011 : 0 : wi->changed = true;
2012 : : }
2013 : :
2014 : : return NULL_TREE;
2015 : : }
2016 : :
2017 : : /* Turn STMT's PHI arg defs into separate SSA defs if they've become
2018 : : non-gimple_val. Return TRUE if any edge insertions need to be committed. */
2019 : :
2020 : : static bool
2021 : 0 : walk_regimplify_phi (gphi *stmt)
2022 : : {
2023 : 0 : bool needs_commit = false;
2024 : :
2025 : 0 : for (unsigned i = 0, n = gimple_phi_num_args (stmt); i < n; i++)
2026 : : {
2027 : 0 : tree op = gimple_phi_arg_def (stmt, i);
2028 : 0 : if ((TREE_CODE (op) == ADDR_EXPR
2029 : 0 : && !is_gimple_val (op))
2030 : : /* ??? A PARM_DECL that was addressable in the original function and
2031 : : had its address in PHI nodes, but that became a reference in the
2032 : : wrapped clone would NOT be updated by update_ssa in PHI nodes.
2033 : : Alas, if we were to create a default def for it now, update_ssa
2034 : : would complain that the symbol that needed rewriting already has
2035 : : SSA names associated with it. OTOH, leaving the PARM_DECL alone,
2036 : : it eventually causes errors because it remains unchanged in PHI
2037 : : nodes, but it gets rewritten as expected if it appears in other
2038 : : stmts. So we cheat a little here, and force the PARM_DECL out of
2039 : : the PHI node and into an assignment. It's a little expensive,
2040 : : because we insert it at the edge, which introduces a basic block
2041 : : that's entirely unnecessary, but it works, and the block will be
2042 : : removed as the default def gets propagated back into the PHI node,
2043 : : so the final optimized code looks just as expected. */
2044 : 0 : || (TREE_CODE (op) == PARM_DECL
2045 : 0 : && !TREE_ADDRESSABLE (op)))
2046 : : {
2047 : 0 : tree temp = make_ssa_name (TREE_TYPE (op), stmt);
2048 : 0 : if (TREE_CODE (op) == PARM_DECL)
2049 : 0 : SET_SSA_NAME_VAR_OR_IDENTIFIER (temp, DECL_NAME (op));
2050 : 0 : SET_PHI_ARG_DEF (stmt, i, temp);
2051 : :
2052 : 0 : gimple *assign = gimple_build_assign (temp, op);
2053 : 0 : if (gimple_phi_arg_has_location (stmt, i))
2054 : 0 : gimple_set_location (assign, gimple_phi_arg_location (stmt, i));
2055 : 0 : gsi_insert_on_edge (gimple_phi_arg_edge (stmt, i), assign);
2056 : 0 : needs_commit = true;
2057 : : }
2058 : : }
2059 : :
2060 : 0 : return needs_commit;
2061 : : }
2062 : :
2063 : : /* Create a reference type to use for PARM when turning it into a
2064 : : reference. */
2065 : :
2066 : : static tree
2067 : 11 : build_ref_type_for (tree parm)
2068 : : {
2069 : 11 : gcc_checking_assert (TREE_CODE (parm) == PARM_DECL);
2070 : :
2071 : 11 : tree ref_type = build_reference_type (TREE_TYPE (parm));
2072 : :
2073 : 11 : return ref_type;
2074 : : }
2075 : :
2076 : : /* Add cgraph edges from current_function_decl to callees in SEQ with frequency
2077 : : COUNT, assuming all calls in SEQ are direct. */
2078 : :
2079 : : static void
2080 : 2854 : add_call_edges_for_seq (gimple_seq seq, profile_count count)
2081 : : {
2082 : 2854 : cgraph_node *node = cgraph_node::get_create (current_function_decl);
2083 : :
2084 : 2854 : for (gimple_stmt_iterator gsi = gsi_start (seq);
2085 : 9134 : !gsi_end_p (gsi); gsi_next (&gsi))
2086 : : {
2087 : 6280 : gimple *stmt = gsi_stmt (gsi);
2088 : :
2089 : 6280 : gcall *call = dyn_cast <gcall *> (stmt);
2090 : 6280 : if (!call)
2091 : 3418 : continue;
2092 : :
2093 : 2862 : tree callee = gimple_call_fndecl (call);
2094 : 2862 : gcc_checking_assert (callee);
2095 : 2862 : node->create_edge (cgraph_node::get_create (callee), call, count, false);
2096 : : }
2097 : 2854 : }
2098 : :
2099 : : /* Insert SEQ after the call at GSI, as if the call was in a try block with SEQ
2100 : : as finally, i.e., SEQ will run after the call whether it returns or
2101 : : propagates an exception. This handles block splitting, EH edge and block
2102 : : creation, noreturn and nothrow optimizations, and even throwing calls without
2103 : : preexisting local handlers. */
2104 : :
2105 : : static void
2106 : 2627 : gsi_insert_finally_seq_after_call (gimple_stmt_iterator gsi, gimple_seq seq)
2107 : : {
2108 : 2627 : if (!seq)
2109 : : return;
2110 : :
2111 : 2223 : gimple *stmt = gsi_stmt (gsi);
2112 : :
2113 : 2223 : if (gimple_has_location (stmt))
2114 : 1757 : annotate_all_with_location (seq, gimple_location (stmt));
2115 : :
2116 : 2223 : gcall *call = dyn_cast <gcall *> (stmt);
2117 : 2223 : bool noreturn_p = call && gimple_call_noreturn_p (call);
2118 : 2223 : int eh_lp = lookup_stmt_eh_lp (stmt);
2119 : 2223 : bool must_not_throw_p = eh_lp < 0;
2120 : 2223 : bool nothrow_p = (must_not_throw_p
2121 : 4446 : || (call && gimple_call_nothrow_p (call))
2122 : 3473 : || (eh_lp <= 0
2123 : 1240 : && (TREE_NOTHROW (cfun->decl)
2124 : 903 : || !opt_for_fn (cfun->decl, flag_exceptions))));
2125 : :
2126 : 1592 : if (noreturn_p && nothrow_p)
2127 : : return;
2128 : :
2129 : : /* Don't expect an EH edge if we're not to throw, or if we're not in an EH
2130 : : region yet. */
2131 : 2223 : bool no_eh_edge_p = (nothrow_p || !eh_lp);
2132 : 2223 : bool must_end_bb = stmt_ends_bb_p (stmt);
2133 : :
2134 : 2223 : edge eft = NULL, eeh = NULL;
2135 : 2223 : if (must_end_bb && !(noreturn_p && no_eh_edge_p))
2136 : : {
2137 : 10 : gcc_checking_assert (gsi_one_before_end_p (gsi));
2138 : :
2139 : 10 : edge e;
2140 : 10 : edge_iterator ei;
2141 : 30 : FOR_EACH_EDGE (e, ei, gsi_bb (gsi)->succs)
2142 : : {
2143 : 20 : if ((e->flags & EDGE_EH))
2144 : : {
2145 : 10 : gcc_checking_assert (!eeh);
2146 : : eeh = e;
2147 : : #if !CHECKING_P
2148 : : if (eft || noreturn_p)
2149 : : break;
2150 : : #endif
2151 : : }
2152 : 20 : if ((e->flags & EDGE_FALLTHRU))
2153 : : {
2154 : 10 : gcc_checking_assert (!eft);
2155 : : eft = e;
2156 : : #if !CHECKING_P
2157 : : if (eeh || no_eh_edge_p)
2158 : : break;
2159 : : #endif
2160 : : }
2161 : : }
2162 : :
2163 : 10 : gcc_checking_assert (!(eft && (eft->flags & EDGE_FALLTHRU))
2164 : : == noreturn_p);
2165 : 10 : gcc_checking_assert (!(eeh && (eeh->flags & EDGE_EH))
2166 : : == no_eh_edge_p);
2167 : 10 : gcc_checking_assert (eft != eeh);
2168 : : }
2169 : :
2170 : 2223 : if (!noreturn_p)
2171 : : {
2172 : 2223 : gimple_seq nseq = nothrow_p ? seq : gimple_seq_copy (seq);
2173 : :
2174 : 2223 : if (must_end_bb)
2175 : : {
2176 : 10 : gcc_checking_assert (gsi_one_before_end_p (gsi));
2177 : 10 : add_call_edges_for_seq (nseq, eft->count ());
2178 : 10 : gsi_insert_seq_on_edge_immediate (eft, nseq);
2179 : : }
2180 : : else
2181 : : {
2182 : 2213 : add_call_edges_for_seq (nseq, gsi_bb (gsi)->count);
2183 : 2213 : gsi_insert_seq_after (&gsi, nseq, GSI_SAME_STMT);
2184 : : }
2185 : : }
2186 : :
2187 : 2223 : if (nothrow_p)
2188 : : return;
2189 : :
2190 : 631 : if (eh_lp)
2191 : : {
2192 : 10 : add_call_edges_for_seq (seq, eeh->count ());
2193 : 10 : gsi_insert_seq_on_edge_immediate (eeh, seq);
2194 : 10 : return;
2195 : : }
2196 : :
2197 : : /* A throwing call may appear within a basic block in a function that doesn't
2198 : : have any EH regions. We're going to add a cleanup if so, therefore the
2199 : : block will have to be split. */
2200 : 621 : basic_block bb = gsi_bb (gsi);
2201 : 621 : if (!gsi_one_before_end_p (gsi))
2202 : 621 : split_block (bb, stmt);
2203 : :
2204 : : /* Create a new block for the EH cleanup. */
2205 : 621 : basic_block bb_eh_cleanup = create_empty_bb (bb);
2206 : 621 : if (dom_info_available_p (CDI_DOMINATORS))
2207 : 0 : set_immediate_dominator (CDI_DOMINATORS, bb_eh_cleanup, bb);
2208 : 621 : if (current_loops)
2209 : 621 : add_bb_to_loop (bb_eh_cleanup, current_loops->tree_root);
2210 : :
2211 : : /* Make the new block an EH cleanup for the call. */
2212 : 621 : eh_region new_r = gen_eh_region_cleanup (NULL);
2213 : 621 : eh_landing_pad lp = gen_eh_landing_pad (new_r);
2214 : 621 : tree label = gimple_block_label (bb_eh_cleanup);
2215 : 621 : lp->post_landing_pad = label;
2216 : 621 : EH_LANDING_PAD_NR (label) = lp->index;
2217 : 621 : add_stmt_to_eh_lp (stmt, lp->index);
2218 : :
2219 : : /* Add the cleanup code to the EH cleanup block. */
2220 : 621 : gsi = gsi_after_labels (bb_eh_cleanup);
2221 : 621 : gsi_insert_seq_before (&gsi, seq, GSI_SAME_STMT);
2222 : :
2223 : : /* And then propagate the exception further. */
2224 : 621 : gresx *resx = gimple_build_resx (new_r->index);
2225 : 621 : if (gimple_has_location (stmt))
2226 : 511 : gimple_set_location (resx, gimple_location (stmt));
2227 : 621 : gsi_insert_before (&gsi, resx, GSI_SAME_STMT);
2228 : :
2229 : : /* Finally, wire the EH cleanup block into the CFG. */
2230 : 621 : edge neeh = make_eh_edge (stmt);
2231 : 621 : neeh->probability = profile_probability::never ();
2232 : 621 : gcc_checking_assert (neeh->dest == bb_eh_cleanup);
2233 : 621 : gcc_checking_assert (!neeh->dest->count.initialized_p ());
2234 : 621 : neeh->dest->count = neeh->count ();
2235 : 621 : add_call_edges_for_seq (seq, neeh->dest->count);
2236 : : }
2237 : :
2238 : : /* Copy the attribute list at *ATTRS, minus any NAME attributes, leaving
2239 : : shareable trailing nodes alone. */
2240 : :
2241 : : static inline void
2242 : 0 : remove_named_attribute_unsharing (const char *name, tree *attrs)
2243 : : {
2244 : 0 : while (tree found = lookup_attribute (name, *attrs))
2245 : : {
2246 : : /* Copy nodes up to the next NAME attribute. */
2247 : 0 : while (*attrs != found)
2248 : : {
2249 : 0 : *attrs = tree_cons (TREE_PURPOSE (*attrs),
2250 : 0 : TREE_VALUE (*attrs),
2251 : 0 : TREE_CHAIN (*attrs));
2252 : 0 : attrs = &TREE_CHAIN (*attrs);
2253 : : }
2254 : : /* Then drop it. */
2255 : 0 : gcc_checking_assert (*attrs == found);
2256 : 0 : *attrs = TREE_CHAIN (*attrs);
2257 : 0 : }
2258 : 0 : }
2259 : :
2260 : : /* Record the uid of the last cgraph entry whose mode we've already set, so
2261 : : that we can perform mode setting incrementally without duplication. */
2262 : : static int last_cgraph_uid;
2263 : :
2264 : : /* Set strub modes for functions introduced since the last call. */
2265 : :
2266 : : static void
2267 : 1148 : ipa_strub_set_mode_for_new_functions ()
2268 : : {
2269 : 1148 : if (symtab->cgraph_max_uid == last_cgraph_uid)
2270 : : return;
2271 : :
2272 : : cgraph_node *node;
2273 : :
2274 : : /* Go through the functions twice, once over non-aliases, and then over
2275 : : aliases, so that aliases can reuse the mode computation of their ultimate
2276 : : targets. */
2277 : 2130 : for (int aliases = 0; aliases <= 1; aliases++)
2278 : 20396 : FOR_EACH_FUNCTION (node)
2279 : : {
2280 : 8778 : if (!node->alias != !aliases)
2281 : 4389 : continue;
2282 : :
2283 : : /* Already done. */
2284 : 4389 : if (node->get_uid () < last_cgraph_uid)
2285 : 899 : continue;
2286 : :
2287 : 3490 : set_strub_mode (node);
2288 : : }
2289 : :
2290 : 710 : last_cgraph_uid = symtab->cgraph_max_uid;
2291 : : }
2292 : :
2293 : : /* Return FALSE if NODE is a strub context, and TRUE otherwise. */
2294 : :
2295 : : bool
2296 : 1138693 : strub_splittable_p (cgraph_node *node)
2297 : : {
2298 : 1138693 : switch (get_strub_mode (node))
2299 : : {
2300 : : case STRUB_WRAPPED:
2301 : : case STRUB_AT_CALLS:
2302 : : case STRUB_AT_CALLS_OPT:
2303 : : case STRUB_INLINABLE:
2304 : : case STRUB_INTERNAL:
2305 : : case STRUB_WRAPPER:
2306 : : return false;
2307 : :
2308 : 1138559 : case STRUB_CALLABLE:
2309 : 1138559 : case STRUB_DISABLED:
2310 : 1138559 : break;
2311 : :
2312 : : default:
2313 : : gcc_unreachable ();
2314 : : }
2315 : :
2316 : 1138559 : return true;
2317 : : }
2318 : :
2319 : : /* Return the PARM_DECL of the incoming watermark pointer, if there is one. */
2320 : :
2321 : : tree
2322 : 1989 : strub_watermark_parm (tree fndecl)
2323 : : {
2324 : 1989 : switch (get_strub_mode_from_fndecl (fndecl))
2325 : : {
2326 : 1068 : case STRUB_WRAPPED:
2327 : 1068 : case STRUB_AT_CALLS:
2328 : 1068 : case STRUB_AT_CALLS_OPT:
2329 : 1068 : break;
2330 : :
2331 : : case STRUB_INTERNAL:
2332 : : case STRUB_WRAPPER:
2333 : : case STRUB_CALLABLE:
2334 : : case STRUB_DISABLED:
2335 : : case STRUB_INLINABLE:
2336 : : return NULL_TREE;
2337 : :
2338 : : default:
2339 : : gcc_unreachable ();
2340 : : }
2341 : :
2342 : 1068 : for (tree parm = DECL_ARGUMENTS (fndecl); parm; parm = DECL_CHAIN (parm))
2343 : : /* The type (variant) compare finds the parameter even in a just-created
2344 : : clone, before we set its name, but the type-based compare doesn't work
2345 : : during builtin expansion within the lto compiler, because we'll have
2346 : : created a separate variant in that run. */
2347 : 1068 : if (TREE_TYPE (parm) == pass_ipa_strub::get_qpwmt ()
2348 : 1068 : || DECL_NAME (parm) == pass_ipa_strub::get_watermark_ptr ())
2349 : 1068 : return parm;
2350 : :
2351 : 0 : gcc_unreachable ();
2352 : : }
2353 : :
2354 : : /* Adjust a STRUB_AT_CALLS function TYPE, adding a watermark pointer if it
2355 : : hasn't been added yet. Return the named argument count. */
2356 : :
2357 : : int
2358 : 2307 : pass_ipa_strub::adjust_at_calls_type (tree type)
2359 : : {
2360 : 2307 : int named_args = 0;
2361 : :
2362 : 2307 : gcc_checking_assert (same_strub_mode_in_variants_p (type));
2363 : :
2364 : 2307 : if (!TYPE_ARG_TYPES (type))
2365 : : return named_args;
2366 : :
2367 : 2307 : tree *tlist = &TYPE_ARG_TYPES (type);
2368 : 2307 : tree qpwmptrt = get_qpwmt ();
2369 : 4726 : while (*tlist && TREE_VALUE (*tlist) != void_type_node)
2370 : : {
2371 : : /* The type has already been adjusted. */
2372 : 2057 : if (TREE_VALUE (*tlist) == qpwmptrt)
2373 : : return named_args;
2374 : 112 : named_args++;
2375 : 336 : *tlist = tree_cons (TREE_PURPOSE (*tlist),
2376 : 112 : TREE_VALUE (*tlist),
2377 : 112 : TREE_CHAIN (*tlist));
2378 : 112 : tlist = &TREE_CHAIN (*tlist);
2379 : : }
2380 : :
2381 : : /* Add the new argument after all named arguments, so as to not mess with
2382 : : attributes that reference parameters. */
2383 : 362 : *tlist = tree_cons (NULL_TREE, get_qpwmt (), *tlist);
2384 : :
2385 : : #if ATTR_FNSPEC_DECONST_WATERMARK
2386 : : if (!type_already_adjusted)
2387 : : {
2388 : : int flags = flags_from_decl_or_type (type);
2389 : : tree fnspec = lookup_attribute ("fn spec", type);
2390 : :
2391 : : if ((flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS)) || fnspec)
2392 : : {
2393 : : size_t xargs = 1;
2394 : : size_t curlen = 0, tgtlen = 2 + 2 * (named_args + xargs);
2395 : : auto_vec<char> nspecv (tgtlen);
2396 : : char *nspec = &nspecv[0]; /* It will *not* be NUL-terminated! */
2397 : : if (fnspec)
2398 : : {
2399 : : tree fnspecstr = TREE_VALUE (TREE_VALUE (fnspec));
2400 : : curlen = TREE_STRING_LENGTH (fnspecstr);
2401 : : memcpy (nspec, TREE_STRING_POINTER (fnspecstr), curlen);
2402 : : }
2403 : : if (!curlen)
2404 : : {
2405 : : nspec[curlen++] = '.';
2406 : : nspec[curlen++] = ((flags & ECF_CONST)
2407 : : ? 'c'
2408 : : : (flags & ECF_PURE)
2409 : : ? 'p'
2410 : : : ' ');
2411 : : }
2412 : : while (curlen < tgtlen - 2 * xargs)
2413 : : {
2414 : : nspec[curlen++] = '.';
2415 : : nspec[curlen++] = ' ';
2416 : : }
2417 : : nspec[curlen++] = 'W';
2418 : : nspec[curlen++] = 't';
2419 : :
2420 : : /* The type has already been copied, if needed, before adding
2421 : : parameters. */
2422 : : TYPE_ATTRIBUTES (type)
2423 : : = tree_cons (get_identifier ("fn spec"),
2424 : : build_tree_list (NULL_TREE,
2425 : : build_string (tgtlen, nspec)),
2426 : : TYPE_ATTRIBUTES (type));
2427 : : }
2428 : : }
2429 : : #endif
2430 : :
2431 : 362 : return named_args;
2432 : : }
2433 : :
2434 : : /* Adjust a call to an at-calls call target. Create a watermark local variable
2435 : : if needed, initialize it before, pass it to the callee according to the
2436 : : modified at-calls interface, and release the callee's stack space after the
2437 : : call, if not deferred. If the call is const or pure, arrange for the
2438 : : watermark to not be assumed unused or unchanged. */
2439 : :
2440 : : void
2441 : 1923 : pass_ipa_strub::adjust_at_calls_call (cgraph_edge *e, int named_args,
2442 : : tree callee_fntype)
2443 : : {
2444 : 1923 : gcc_checking_assert (e->call_stmt);
2445 : 1923 : gcall *ocall = e->call_stmt;
2446 : 1923 : gimple_stmt_iterator gsi = gsi_for_stmt (ocall);
2447 : :
2448 : : /* Make sure we haven't modified this call yet. */
2449 : 1923 : gcc_checking_assert (!(int (gimple_call_num_args (ocall)) > named_args
2450 : : && (TREE_TYPE (gimple_call_arg (ocall, named_args))
2451 : : == get_pwmt ())));
2452 : :
2453 : 1923 : tree tsup;
2454 : 1923 : if (!(tsup = gimple_call_fndecl (ocall)))
2455 : 46 : tsup = TREE_TYPE (TREE_TYPE (gimple_call_fn (ocall)));
2456 : 1923 : if (!strub_target_support_p (tsup, true, gimple_location (ocall)))
2457 : 0 : return;
2458 : :
2459 : : /* If we're already within a strub context, pass on the incoming watermark
2460 : : pointer, and omit the enter and leave calls around the modified call, as an
2461 : : optimization, or as a means to satisfy a tail-call requirement. */
2462 : 1923 : tree swmp = ((opt_for_fn (e->caller->decl, optimize_size)
2463 : 1624 : || opt_for_fn (e->caller->decl, optimize) > 2
2464 : 1315 : || gimple_call_must_tail_p (ocall)
2465 : 1315 : || (opt_for_fn (e->caller->decl, optimize) == 2
2466 : 537 : && gimple_call_tail_p (ocall)))
2467 : 2232 : ? strub_watermark_parm (e->caller->decl)
2468 : 2127 : : NULL_TREE);
2469 : 2127 : bool omit_own_watermark = swmp;
2470 : 2127 : tree swm = NULL_TREE;
2471 : 608 : if (!omit_own_watermark)
2472 : : {
2473 : 1519 : swm = create_tmp_var (get_wmt (), ".strub.watermark");
2474 : 1519 : TREE_ADDRESSABLE (swm) = true;
2475 : 1519 : swmp = build1 (ADDR_EXPR, get_pwmt (), swm);
2476 : :
2477 : : /* Initialize the watermark before the call. */
2478 : 1519 : tree enter = get_enter ();
2479 : 1519 : gcall *stptr = gimple_build_call (enter, 1,
2480 : : unshare_expr (swmp));
2481 : 1519 : if (gimple_has_location (ocall))
2482 : 1519 : gimple_set_location (stptr, gimple_location (ocall));
2483 : 1519 : gsi_insert_before (&gsi, stptr, GSI_SAME_STMT);
2484 : 1519 : e->caller->create_edge (cgraph_node::get_create (enter),
2485 : 1519 : stptr, gsi_bb (gsi)->count, false);
2486 : : }
2487 : :
2488 : :
2489 : : /* Replace the call with one that passes the swmp argument first. */
2490 : 1923 : gcall *wrcall;
2491 : 1923 : { gcall *stmt = ocall;
2492 : : // Mostly copied from gimple_call_copy_skip_args.
2493 : 1923 : int i = 0;
2494 : 1923 : int nargs = gimple_call_num_args (stmt);
2495 : 1923 : auto_vec<tree> vargs (MAX (nargs, named_args) + 1);
2496 : 1923 : gcall *new_stmt;
2497 : :
2498 : : /* pr71109.c calls a prototypeless function, then defines it with
2499 : : additional arguments. It's ill-formed, but after it's inlined,
2500 : : it somehow works out. */
2501 : 1987 : for (; i < named_args && i < nargs; i++)
2502 : 64 : vargs.quick_push (gimple_call_arg (stmt, i));
2503 : 1923 : for (; i < named_args; i++)
2504 : 0 : vargs.quick_push (null_pointer_node);
2505 : :
2506 : 1923 : vargs.quick_push (unshare_expr (swmp));
2507 : :
2508 : 1951 : for (; i < nargs; i++)
2509 : 28 : vargs.quick_push (gimple_call_arg (stmt, i));
2510 : :
2511 : 1923 : if (gimple_call_internal_p (stmt))
2512 : 0 : gcc_unreachable ();
2513 : : else
2514 : 1923 : new_stmt = gimple_build_call_vec (gimple_call_fn (stmt), vargs);
2515 : 1923 : gimple_call_set_fntype (new_stmt, callee_fntype);
2516 : :
2517 : 1923 : if (gimple_call_lhs (stmt))
2518 : 1861 : gimple_call_set_lhs (new_stmt, gimple_call_lhs (stmt));
2519 : :
2520 : 1923 : gimple_move_vops (new_stmt, stmt);
2521 : :
2522 : 1923 : if (gimple_has_location (stmt))
2523 : 1923 : gimple_set_location (new_stmt, gimple_location (stmt));
2524 : 1923 : gimple_call_copy_flags (new_stmt, stmt);
2525 : 1923 : gimple_call_set_chain (new_stmt, gimple_call_chain (stmt));
2526 : :
2527 : 1923 : gimple_set_modified (new_stmt, true);
2528 : :
2529 : 1923 : wrcall = new_stmt;
2530 : 1923 : }
2531 : :
2532 : 1923 : update_stmt (wrcall);
2533 : 1923 : gsi_replace (&gsi, wrcall, true);
2534 : 1923 : cgraph_edge::set_call_stmt (e, wrcall, false);
2535 : :
2536 : : /* Insert the strub code after the call. */
2537 : 1923 : gimple_seq seq = NULL;
2538 : :
2539 : : #if !ATTR_FNSPEC_DECONST_WATERMARK
2540 : : /* If the call will be assumed to not modify or even read the
2541 : : watermark, make it read and modified ourselves. */
2542 : 1923 : if ((gimple_call_flags (wrcall)
2543 : 1923 : & (ECF_CONST | ECF_PURE | ECF_NOVOPS)))
2544 : : {
2545 : 60 : if (!swm)
2546 : 0 : swm = build2 (MEM_REF,
2547 : 0 : TREE_TYPE (TREE_TYPE (swmp)),
2548 : : swmp,
2549 : 0 : build_int_cst (TREE_TYPE (swmp), 0));
2550 : :
2551 : 60 : vec<tree, va_gc> *inputs = NULL;
2552 : 60 : vec<tree, va_gc> *outputs = NULL;
2553 : 120 : vec_safe_push (outputs,
2554 : : build_tree_list
2555 : 60 : (build_tree_list
2556 : : (NULL_TREE, build_string (2, "=m")),
2557 : : unshare_expr (swm)));
2558 : 120 : vec_safe_push (inputs,
2559 : : build_tree_list
2560 : 60 : (build_tree_list
2561 : : (NULL_TREE, build_string (1, "m")),
2562 : : unshare_expr (swm)));
2563 : 60 : gasm *forcemod = gimple_build_asm_vec ("", inputs, outputs,
2564 : : NULL, NULL);
2565 : 60 : gimple_seq_add_stmt (&seq, forcemod);
2566 : :
2567 : : /* If the call will be assumed to not even read the watermark,
2568 : : make sure it is already in memory before the call. */
2569 : 60 : if ((gimple_call_flags (wrcall) & ECF_CONST))
2570 : : {
2571 : 32 : vec<tree, va_gc> *inputs = NULL;
2572 : 64 : vec_safe_push (inputs,
2573 : : build_tree_list
2574 : 32 : (build_tree_list
2575 : : (NULL_TREE, build_string (1, "m")),
2576 : : unshare_expr (swm)));
2577 : 32 : gasm *force_store = gimple_build_asm_vec ("", inputs, NULL,
2578 : : NULL, NULL);
2579 : 32 : if (gimple_has_location (wrcall))
2580 : 32 : gimple_set_location (force_store, gimple_location (wrcall));
2581 : 32 : gsi_insert_before (&gsi, force_store, GSI_SAME_STMT);
2582 : : }
2583 : : }
2584 : : #endif
2585 : :
2586 : 1923 : if (!omit_own_watermark)
2587 : : {
2588 : 1519 : gcall *sleave = gimple_build_call (get_leave (), 1,
2589 : : unshare_expr (swmp));
2590 : 1519 : gimple_seq_add_stmt (&seq, sleave);
2591 : :
2592 : 1519 : gassign *clobber = gimple_build_assign (swm,
2593 : : build_clobber
2594 : 1519 : (TREE_TYPE (swm)));
2595 : 1519 : gimple_seq_add_stmt (&seq, clobber);
2596 : : }
2597 : :
2598 : 1923 : gsi_insert_finally_seq_after_call (gsi, seq);
2599 : : }
2600 : :
2601 : : /* Adjust all at-calls calls in NODE. */
2602 : :
2603 : : void
2604 : 1219 : pass_ipa_strub::adjust_at_calls_calls (cgraph_node *node)
2605 : : {
2606 : : /* Adjust unknown-callee indirect calls with STRUB_AT_CALLS types within
2607 : : onode. */
2608 : 1219 : if (node->indirect_calls)
2609 : : {
2610 : 110 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
2611 : 220 : for (cgraph_edge *e = node->indirect_calls; e; e = e->next_callee)
2612 : : {
2613 : 110 : gcc_checking_assert (e->indirect_unknown_callee);
2614 : :
2615 : 110 : if (!e->call_stmt)
2616 : 64 : continue;
2617 : :
2618 : 110 : tree callee_fntype;
2619 : 110 : enum strub_mode callee_mode
2620 : 110 : = effective_strub_mode_for_call (e->call_stmt, &callee_fntype);
2621 : :
2622 : 174 : if (callee_mode != STRUB_AT_CALLS
2623 : 110 : && callee_mode != STRUB_AT_CALLS_OPT)
2624 : 64 : continue;
2625 : :
2626 : 46 : int named_args = adjust_at_calls_type (callee_fntype);
2627 : :
2628 : 46 : adjust_at_calls_call (e, named_args, callee_fntype);
2629 : : }
2630 : 110 : pop_cfun ();
2631 : : }
2632 : :
2633 : 1219 : if (node->callees)
2634 : : {
2635 : 1038 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
2636 : 9795 : for (cgraph_edge *e = node->callees; e; e = e->next_callee)
2637 : : {
2638 : 8757 : gcc_checking_assert (!e->indirect_unknown_callee);
2639 : :
2640 : 8757 : if (!e->call_stmt)
2641 : 6880 : continue;
2642 : :
2643 : 8757 : tree callee_fntype;
2644 : 8757 : enum strub_mode callee_mode
2645 : 8757 : = effective_strub_mode_for_call (e->call_stmt, &callee_fntype);
2646 : :
2647 : 15637 : if (callee_mode != STRUB_AT_CALLS
2648 : 8757 : && callee_mode != STRUB_AT_CALLS_OPT)
2649 : 6880 : continue;
2650 : :
2651 : 1877 : int named_args = adjust_at_calls_type (callee_fntype);
2652 : :
2653 : 1877 : adjust_at_calls_call (e, named_args, callee_fntype);
2654 : : }
2655 : 1038 : pop_cfun ();
2656 : : }
2657 : 1219 : }
2658 : :
2659 : : /* The strubm (strub mode) pass computes a strub mode for each function in the
2660 : : call graph, and checks, before any inlining, that strub callability
2661 : : requirements in effect are satisfied. */
2662 : :
2663 : : unsigned int
2664 : 608 : pass_ipa_strub_mode::execute (function *)
2665 : : {
2666 : 608 : last_cgraph_uid = 0;
2667 : 608 : ipa_strub_set_mode_for_new_functions ();
2668 : :
2669 : : /* Verify before any inlining or other transformations. */
2670 : 608 : verify_strub ();
2671 : :
2672 : 608 : return 0;
2673 : : }
2674 : :
2675 : : /* Create a strub mode pass. */
2676 : :
2677 : : simple_ipa_opt_pass *
2678 : 280114 : make_pass_ipa_strub_mode (gcc::context *ctxt)
2679 : : {
2680 : 280114 : return new pass_ipa_strub_mode (ctxt);
2681 : : }
2682 : :
2683 : : /* The strub pass proper adjusts types, signatures, and at-calls calls, and
2684 : : splits internal-strub functions. */
2685 : :
2686 : : unsigned int
2687 : 540 : pass_ipa_strub::execute (function *)
2688 : : {
2689 : 540 : cgraph_node *onode;
2690 : :
2691 : 540 : ipa_strub_set_mode_for_new_functions ();
2692 : :
2693 : : /* First, adjust the signature of at-calls functions. We adjust types of
2694 : : at-calls functions first, so that we don't modify types in place unless
2695 : : strub is explicitly requested. */
2696 : 5672 : FOR_EACH_FUNCTION (onode)
2697 : : {
2698 : 2296 : enum strub_mode mode = get_strub_mode (onode);
2699 : :
2700 : 2296 : if (mode == STRUB_AT_CALLS
2701 : 2296 : || mode == STRUB_AT_CALLS_OPT)
2702 : : {
2703 : : /* Create a type variant if strubbing was not explicitly requested in
2704 : : the function type. */
2705 : 384 : if (get_strub_mode_from_type (TREE_TYPE (onode->decl)) != mode)
2706 : 72 : distinctify_node_type (onode);
2707 : :
2708 : 384 : int named_args = adjust_at_calls_type (TREE_TYPE (onode->decl));
2709 : :
2710 : : /* An external function explicitly declared with strub won't have a
2711 : : body. Even with implicit at-calls strub, a function may have had its
2712 : : body removed after we selected the mode, and then we have nothing
2713 : : further to do. */
2714 : 384 : if (!onode->has_gimple_body_p ())
2715 : 63 : continue;
2716 : :
2717 : 321 : tree *pargs = &DECL_ARGUMENTS (onode->decl);
2718 : :
2719 : : /* A noninterposable_alias reuses the same parm decl chain, don't add
2720 : : the parm twice. */
2721 : 0 : bool aliased_parms = (onode->alias && *pargs
2722 : 321 : && DECL_CONTEXT (*pargs) != onode->decl);
2723 : :
2724 : 0 : if (aliased_parms)
2725 : 0 : continue;
2726 : :
2727 : 369 : for (int i = 0; i < named_args; i++)
2728 : 48 : pargs = &DECL_CHAIN (*pargs);
2729 : :
2730 : 321 : tree wmptr = build_decl (DECL_SOURCE_LOCATION (onode->decl),
2731 : : PARM_DECL,
2732 : : get_watermark_ptr (),
2733 : : get_qpwmt ());
2734 : 321 : DECL_ARTIFICIAL (wmptr) = 1;
2735 : 321 : DECL_ARG_TYPE (wmptr) = get_qpwmt ();
2736 : 321 : DECL_CONTEXT (wmptr) = onode->decl;
2737 : 321 : TREE_USED (wmptr) = 1;
2738 : 321 : DECL_CHAIN (wmptr) = *pargs;
2739 : 321 : *pargs = wmptr;
2740 : :
2741 : 321 : if (onode->alias)
2742 : 0 : continue;
2743 : :
2744 : 321 : cgraph_node *nnode = onode;
2745 : 321 : push_cfun (DECL_STRUCT_FUNCTION (nnode->decl));
2746 : :
2747 : 321 : {
2748 : 321 : edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
2749 : 321 : gimple_seq seq = call_update_watermark (wmptr, nnode, e->src->count);
2750 : 321 : gsi_insert_seq_on_edge_immediate (e, seq);
2751 : : }
2752 : :
2753 : 321 : if (DECL_STRUCT_FUNCTION (nnode->decl)->calls_alloca)
2754 : : {
2755 : 120 : basic_block bb;
2756 : 657 : FOR_EACH_BB_FN (bb, cfun)
2757 : 1074 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
2758 : 4057 : !gsi_end_p (gsi); gsi_next (&gsi))
2759 : : {
2760 : 3520 : gimple *stmt = gsi_stmt (gsi);
2761 : :
2762 : 3520 : gcall *call = dyn_cast <gcall *> (stmt);
2763 : :
2764 : 3520 : if (!call)
2765 : 2222 : continue;
2766 : :
2767 : 1298 : if (gimple_alloca_call_p (call))
2768 : : {
2769 : : /* Capture stack growth. */
2770 : 328 : gimple_seq seq = call_update_watermark (wmptr, NULL,
2771 : 164 : gsi_bb (gsi)
2772 : : ->count);
2773 : 164 : gsi_insert_finally_seq_after_call (gsi, seq);
2774 : : }
2775 : : }
2776 : : }
2777 : :
2778 : 321 : pop_cfun ();
2779 : : }
2780 : : }
2781 : :
2782 : 6178 : FOR_EACH_FUNCTION (onode)
2783 : : {
2784 : 2549 : if (!onode->has_gimple_body_p ())
2785 : 2083 : continue;
2786 : :
2787 : 1219 : enum strub_mode mode = get_strub_mode (onode);
2788 : :
2789 : 1219 : if (mode != STRUB_INTERNAL)
2790 : : {
2791 : 753 : adjust_at_calls_calls (onode);
2792 : 753 : continue;
2793 : : }
2794 : :
2795 : 466 : bool is_stdarg = calls_builtin_va_start_p (onode);;
2796 : 466 : bool apply_args = calls_builtin_apply_args_p (onode);
2797 : :
2798 : 466 : vec<ipa_adjusted_param, va_gc> *nparms = NULL;
2799 : 466 : unsigned j = 0;
2800 : 466 : {
2801 : : // The following loop copied from ipa-split.c:split_function.
2802 : 466 : for (tree parm = DECL_ARGUMENTS (onode->decl);
2803 : 603 : parm; parm = DECL_CHAIN (parm), j++)
2804 : : {
2805 : 137 : ipa_adjusted_param adj = {};
2806 : 137 : adj.op = IPA_PARAM_OP_COPY;
2807 : 137 : adj.base_index = j;
2808 : 137 : adj.prev_clone_index = j;
2809 : 137 : vec_safe_push (nparms, adj);
2810 : : }
2811 : :
2812 : 466 : if (apply_args)
2813 : : {
2814 : 16 : ipa_adjusted_param aaadj = {};
2815 : 16 : aaadj.op = IPA_PARAM_OP_NEW;
2816 : 16 : aaadj.type = get_qptr ();
2817 : 16 : vec_safe_push (nparms, aaadj);
2818 : : }
2819 : :
2820 : 466 : if (is_stdarg)
2821 : : {
2822 : 8 : ipa_adjusted_param vladj = {};
2823 : 8 : vladj.op = IPA_PARAM_OP_NEW;
2824 : 8 : vladj.type = get_qpvalst ();
2825 : 8 : vec_safe_push (nparms, vladj);
2826 : : }
2827 : :
2828 : 466 : ipa_adjusted_param wmadj = {};
2829 : 466 : wmadj.op = IPA_PARAM_OP_NEW;
2830 : 466 : wmadj.type = get_qpwmt ();
2831 : 466 : vec_safe_push (nparms, wmadj);
2832 : : }
2833 : 466 : ipa_param_adjustments adj (nparms, -1, false);
2834 : :
2835 : 466 : cgraph_node *nnode = onode->create_version_clone_with_body
2836 : 466 : (auto_vec<cgraph_edge *> (0),
2837 : : NULL, &adj, NULL, NULL, "strub", NULL);
2838 : :
2839 : 466 : if (!nnode)
2840 : : {
2841 : 0 : error_at (DECL_SOURCE_LOCATION (onode->decl),
2842 : : "failed to split %qD for %<strub%>",
2843 : : onode->decl);
2844 : 0 : continue;
2845 : : }
2846 : :
2847 : 466 : onode->split_part = true;
2848 : 466 : if (onode->calls_comdat_local)
2849 : 0 : nnode->add_to_same_comdat_group (onode);
2850 : :
2851 : 466 : set_strub_mode_to (onode, STRUB_WRAPPER);
2852 : 466 : set_strub_mode_to (nnode, STRUB_WRAPPED);
2853 : :
2854 : 466 : adjust_at_calls_calls (nnode);
2855 : :
2856 : : /* Decide which of the wrapped function's parms we want to turn into
2857 : : references to the argument passed to the wrapper. In general, we want to
2858 : : copy small arguments, and avoid copying large ones. Variable-sized array
2859 : : lengths given by other arguments, as in 20020210-1.c, would lead to
2860 : : problems if passed by value, after resetting the original function and
2861 : : dropping the length computation; passing them by reference works.
2862 : : DECL_BY_REFERENCE is *not* a substitute for this: it involves copying
2863 : : anyway, but performed at the caller. */
2864 : 466 : indirect_parms_t indirect_nparms (3, false);
2865 : 466 : unsigned adjust_ftype = 0;
2866 : 466 : unsigned named_args = 0;
2867 : 466 : for (tree parm = DECL_ARGUMENTS (onode->decl),
2868 : 466 : nparm = DECL_ARGUMENTS (nnode->decl),
2869 : 466 : nparmt = TYPE_ARG_TYPES (TREE_TYPE (nnode->decl));
2870 : 603 : parm;
2871 : : named_args++,
2872 : 137 : parm = DECL_CHAIN (parm),
2873 : 274 : nparm = DECL_CHAIN (nparm),
2874 : 137 : nparmt = nparmt ? TREE_CHAIN (nparmt) : NULL_TREE)
2875 : 137 : if (TREE_THIS_VOLATILE (parm)
2876 : 272 : || !(0 /* DECL_BY_REFERENCE (narg) */
2877 : 135 : || is_gimple_reg_type (TREE_TYPE (nparm))
2878 : 9 : || VECTOR_TYPE_P (TREE_TYPE (nparm))
2879 : 9 : || TREE_CODE (TREE_TYPE (nparm)) == COMPLEX_TYPE
2880 : 9 : || (tree_fits_uhwi_p (TYPE_SIZE_UNIT (TREE_TYPE (nparm)))
2881 : 16 : && (tree_to_uhwi (TYPE_SIZE_UNIT (TREE_TYPE (nparm)))
2882 : 8 : <= 4 * UNITS_PER_WORD))))
2883 : : {
2884 : : /* No point in indirecting pointer types, unless they're
2885 : : volatile. Presumably they won't ever pass the size-based
2886 : : test above, but check the assumption here, because
2887 : : getting this wrong would mess with attribute access and
2888 : : possibly others. We deal with fn spec below. */
2889 : 11 : gcc_checking_assert (!POINTER_TYPE_P (TREE_TYPE (nparm))
2890 : : || TREE_THIS_VOLATILE (parm));
2891 : :
2892 : 11 : indirect_nparms.add (nparm);
2893 : :
2894 : : /* ??? Is there any case in which it is not safe to suggest the parms
2895 : : turned indirect don't alias anything else? They are distinct,
2896 : : unaliased memory in the wrapper, and the wrapped can't possibly
2897 : : take pointers into them because none of the pointers passed to the
2898 : : wrapper can alias other incoming parameters passed by value, even
2899 : : if with transparent reference, and the wrapper doesn't take any
2900 : : extra parms that could point into wrapper's parms. So we can
2901 : : probably drop the TREE_ADDRESSABLE and keep the TRUE. */
2902 : 11 : tree ref_type = build_ref_type_for (nparm);
2903 : :
2904 : 11 : if (TREE_THIS_VOLATILE (nparm)
2905 : 2 : && TYPE_VOLATILE (TREE_TYPE (nparm))
2906 : 13 : && !TYPE_VOLATILE (ref_type))
2907 : 2 : TREE_SIDE_EFFECTS (nparm) = TREE_THIS_VOLATILE (nparm) = 0;
2908 : 11 : DECL_ARG_TYPE (nparm) = TREE_TYPE (nparm) = ref_type;
2909 : 11 : relayout_decl (nparm);
2910 : 11 : TREE_ADDRESSABLE (nparm) = 0;
2911 : 11 : DECL_BY_REFERENCE (nparm) = 0;
2912 : 11 : DECL_NOT_GIMPLE_REG_P (nparm) = 0;
2913 : : /* ??? This avoids mismatches in debug info bind stmts in
2914 : : e.g. a-chahan . */
2915 : 11 : DECL_ABSTRACT_ORIGIN (nparm) = NULL;
2916 : :
2917 : 11 : if (nparmt)
2918 : 11 : adjust_ftype++;
2919 : : }
2920 : :
2921 : : /* Also adjust the wrapped function type, if needed. */
2922 : 466 : if (adjust_ftype)
2923 : : {
2924 : 11 : tree nftype = TREE_TYPE (nnode->decl);
2925 : :
2926 : : /* We always add at least one argument at the end of the signature, when
2927 : : cloning the function, so we don't expect to need to duplicate the
2928 : : type here. */
2929 : 11 : gcc_checking_assert (TYPE_ARG_TYPES (nftype)
2930 : : != TYPE_ARG_TYPES (TREE_TYPE (onode->decl)));
2931 : :
2932 : : /* Check that fnspec still works for the modified function signature,
2933 : : and drop it otherwise. */
2934 : 11 : bool drop_fnspec = false;
2935 : 11 : tree fnspec = lookup_attribute ("fn spec", TYPE_ATTRIBUTES (nftype));
2936 : 11 : attr_fnspec spec = fnspec ? attr_fnspec (fnspec) : attr_fnspec ("");
2937 : :
2938 : 11 : unsigned retcopy;
2939 : 11 : if (!(fnspec && spec.returns_arg (&retcopy)))
2940 : : retcopy = (unsigned) -1;
2941 : :
2942 : 11 : unsigned i = 0;
2943 : 11 : for (tree nparm = DECL_ARGUMENTS (nnode->decl),
2944 : 11 : nparmt = TYPE_ARG_TYPES (nftype);
2945 : 23 : adjust_ftype > 0;
2946 : 12 : i++, nparm = DECL_CHAIN (nparm), nparmt = TREE_CHAIN (nparmt))
2947 : 12 : if (indirect_nparms.contains (nparm))
2948 : : {
2949 : 11 : TREE_VALUE (nparmt) = TREE_TYPE (nparm);
2950 : 11 : adjust_ftype--;
2951 : :
2952 : 11 : if (fnspec && !drop_fnspec)
2953 : : {
2954 : 0 : if (i == retcopy)
2955 : : drop_fnspec = true;
2956 : 0 : else if (spec.arg_specified_p (i))
2957 : : {
2958 : : /* Properties that apply to pointers only must not be
2959 : : present, because we don't make pointers further
2960 : : indirect. */
2961 : 0 : gcc_checking_assert
2962 : : (!spec.arg_max_access_size_given_by_arg_p (i, NULL));
2963 : 0 : gcc_checking_assert (!spec.arg_copied_to_arg_p (i, NULL));
2964 : :
2965 : : /* Any claim of direct access only is invalidated by
2966 : : adding an indirection level. */
2967 : 0 : if (spec.arg_direct_p (i))
2968 : : drop_fnspec = true;
2969 : :
2970 : : /* If there's a claim the argument is not read from, the
2971 : : added indirection invalidates it: if the argument is
2972 : : used at all, then the pointer will necessarily be
2973 : : read. */
2974 : 0 : if (!spec.arg_maybe_read_p (i)
2975 : 0 : && spec.arg_used_p (i))
2976 : : drop_fnspec = true;
2977 : : }
2978 : : }
2979 : : }
2980 : :
2981 : : /* ??? Maybe we could adjust it instead. Note we don't need
2982 : : to mess with attribute access: pointer-typed parameters are
2983 : : not modified, so they can remain unchanged. */
2984 : 11 : if (drop_fnspec)
2985 : 0 : remove_named_attribute_unsharing ("fn spec",
2986 : 0 : &TYPE_ATTRIBUTES (nftype));
2987 : :
2988 : 11 : TREE_TYPE (nnode->decl) = nftype;
2989 : : }
2990 : :
2991 : : #if ATTR_FNSPEC_DECONST_WATERMARK
2992 : : {
2993 : : int flags = flags_from_decl_or_type (nnode->decl);
2994 : : tree fnspec = lookup_attribute ("fn spec", TREE_TYPE (nnode->decl));
2995 : :
2996 : : if ((flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS)) || fnspec)
2997 : : {
2998 : : size_t xargs = 1 + int (is_stdarg) + int (apply_args);
2999 : : size_t curlen = 0, tgtlen = 2 + 2 * (named_args + xargs);
3000 : : auto_vec<char> nspecv (tgtlen);
3001 : : char *nspec = &nspecv[0]; /* It will *not* be NUL-terminated! */
3002 : : bool no_writes_p = true;
3003 : : if (fnspec)
3004 : : {
3005 : : tree fnspecstr = TREE_VALUE (TREE_VALUE (fnspec));
3006 : : curlen = TREE_STRING_LENGTH (fnspecstr);
3007 : : memcpy (nspec, TREE_STRING_POINTER (fnspecstr), curlen);
3008 : : if (!(flags & (ECF_CONST | ECF_PURE | ECF_NOVOPS))
3009 : : && curlen >= 2
3010 : : && nspec[1] != 'c' && nspec[1] != 'C'
3011 : : && nspec[1] != 'p' && nspec[1] != 'P')
3012 : : no_writes_p = false;
3013 : : }
3014 : : if (!curlen)
3015 : : {
3016 : : nspec[curlen++] = '.';
3017 : : nspec[curlen++] = ((flags & ECF_CONST)
3018 : : ? 'c'
3019 : : : (flags & ECF_PURE)
3020 : : ? 'p'
3021 : : : ' ');
3022 : : }
3023 : : while (curlen < tgtlen - 2 * xargs)
3024 : : {
3025 : : nspec[curlen++] = '.';
3026 : : nspec[curlen++] = ' ';
3027 : : }
3028 : :
3029 : : /* These extra args are unlikely to be present in const or pure
3030 : : functions. It's conceivable that a function that takes variable
3031 : : arguments, or that passes its arguments on to another function,
3032 : : could be const or pure, but it would not modify the arguments, and,
3033 : : being pure or const, it couldn't possibly modify or even access
3034 : : memory referenced by them. But it can read from these internal
3035 : : data structures created by the wrapper, and from any
3036 : : argument-passing memory referenced by them, so we denote the
3037 : : possibility of reading from multiple levels of indirection, but
3038 : : only of reading because const/pure. */
3039 : : if (apply_args)
3040 : : {
3041 : : nspec[curlen++] = 'r';
3042 : : nspec[curlen++] = ' ';
3043 : : }
3044 : : if (is_stdarg)
3045 : : {
3046 : : nspec[curlen++] = (no_writes_p ? 'r' : '.');
3047 : : nspec[curlen++] = (no_writes_p ? 't' : ' ');
3048 : : }
3049 : :
3050 : : nspec[curlen++] = 'W';
3051 : : nspec[curlen++] = 't';
3052 : :
3053 : : /* The type has already been copied before adding parameters. */
3054 : : gcc_checking_assert (TYPE_ARG_TYPES (TREE_TYPE (nnode->decl))
3055 : : != TYPE_ARG_TYPES (TREE_TYPE (onode->decl)));
3056 : : TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl))
3057 : : = tree_cons (get_identifier ("fn spec"),
3058 : : build_tree_list (NULL_TREE,
3059 : : build_string (tgtlen, nspec)),
3060 : : TYPE_ATTRIBUTES (TREE_TYPE (nnode->decl)));
3061 : : }
3062 : : }
3063 : : #endif
3064 : :
3065 : 466 : {
3066 : 466 : tree decl = onode->decl;
3067 : 466 : cgraph_node *target = nnode;
3068 : :
3069 : 466 : { // copied from create_wrapper
3070 : :
3071 : : /* Preserve DECL_RESULT so we get right by reference flag. */
3072 : 466 : tree decl_result = DECL_RESULT (decl);
3073 : :
3074 : : /* Remove the function's body but keep arguments to be reused
3075 : : for thunk. */
3076 : 466 : onode->release_body (true);
3077 : 466 : onode->reset (/* unlike create_wrapper: preserve_comdat_group = */true);
3078 : :
3079 : 466 : DECL_UNINLINABLE (decl) = false;
3080 : 466 : DECL_RESULT (decl) = decl_result;
3081 : 466 : DECL_INITIAL (decl) = NULL;
3082 : 466 : allocate_struct_function (decl, false);
3083 : 466 : set_cfun (NULL);
3084 : :
3085 : : /* Turn alias into thunk and expand it into GIMPLE representation. */
3086 : 466 : onode->definition = true;
3087 : :
3088 : 466 : thunk_info::get_create (onode);
3089 : 466 : onode->thunk = true;
3090 : 466 : onode->create_edge (target, NULL, onode->count);
3091 : 466 : onode->callees->can_throw_external = !TREE_NOTHROW (target->decl);
3092 : :
3093 : 466 : tree arguments = DECL_ARGUMENTS (decl);
3094 : :
3095 : 603 : while (arguments)
3096 : : {
3097 : 137 : TREE_ADDRESSABLE (arguments) = false;
3098 : 137 : arguments = TREE_CHAIN (arguments);
3099 : : }
3100 : :
3101 : 466 : {
3102 : 466 : tree alias = onode->callees->callee->decl;
3103 : 466 : tree thunk_fndecl = decl;
3104 : 466 : tree a;
3105 : :
3106 : 466 : int nxargs = 1 + is_stdarg + apply_args;
3107 : :
3108 : 466 : { // Simplified from expand_thunk.
3109 : 466 : tree restype;
3110 : 466 : basic_block bb, then_bb, else_bb, return_bb;
3111 : 466 : gimple_stmt_iterator bsi;
3112 : 466 : int nargs = 0;
3113 : 466 : tree arg;
3114 : 466 : int i;
3115 : 466 : tree resdecl;
3116 : 466 : tree restmp = NULL;
3117 : :
3118 : 466 : gcall *call;
3119 : 466 : greturn *ret;
3120 : 466 : bool alias_is_noreturn = TREE_THIS_VOLATILE (alias);
3121 : :
3122 : 466 : a = DECL_ARGUMENTS (thunk_fndecl);
3123 : :
3124 : 466 : current_function_decl = thunk_fndecl;
3125 : :
3126 : : /* Ensure thunks are emitted in their correct sections. */
3127 : 466 : resolve_unique_section (thunk_fndecl, 0,
3128 : : flag_function_sections);
3129 : :
3130 : 466 : bitmap_obstack_initialize (NULL);
3131 : :
3132 : : /* Build the return declaration for the function. */
3133 : 466 : restype = TREE_TYPE (TREE_TYPE (thunk_fndecl));
3134 : 466 : if (DECL_RESULT (thunk_fndecl) == NULL_TREE)
3135 : : {
3136 : 0 : resdecl = build_decl (input_location, RESULT_DECL, 0, restype);
3137 : 0 : DECL_ARTIFICIAL (resdecl) = 1;
3138 : 0 : DECL_IGNORED_P (resdecl) = 1;
3139 : 0 : DECL_CONTEXT (resdecl) = thunk_fndecl;
3140 : 0 : DECL_RESULT (thunk_fndecl) = resdecl;
3141 : : }
3142 : : else
3143 : : resdecl = DECL_RESULT (thunk_fndecl);
3144 : :
3145 : 466 : profile_count cfg_count = onode->count;
3146 : 466 : if (!cfg_count.initialized_p ())
3147 : 161 : cfg_count = profile_count::from_gcov_type (BB_FREQ_MAX).guessed_local ();
3148 : :
3149 : 932 : bb = then_bb = else_bb = return_bb
3150 : 466 : = init_lowered_empty_function (thunk_fndecl, true, cfg_count);
3151 : :
3152 : 466 : bsi = gsi_start_bb (bb);
3153 : :
3154 : : /* Build call to the function being thunked. */
3155 : 466 : if (!VOID_TYPE_P (restype)
3156 : 466 : && (!alias_is_noreturn
3157 : 0 : || TREE_ADDRESSABLE (restype)
3158 : 0 : || TREE_CODE (TYPE_SIZE_UNIT (restype)) != INTEGER_CST))
3159 : : {
3160 : 353 : if (DECL_BY_REFERENCE (resdecl))
3161 : : {
3162 : 6 : restmp = gimple_fold_indirect_ref (resdecl);
3163 : 6 : if (!restmp)
3164 : 12 : restmp = build2 (MEM_REF,
3165 : 6 : TREE_TYPE (TREE_TYPE (resdecl)),
3166 : : resdecl,
3167 : 6 : build_int_cst (TREE_TYPE (resdecl), 0));
3168 : : }
3169 : 347 : else if (aggregate_value_p (resdecl, TREE_TYPE (thunk_fndecl)))
3170 : : {
3171 : 1 : restmp = resdecl;
3172 : :
3173 : 1 : if (VAR_P (restmp))
3174 : : {
3175 : 0 : add_local_decl (cfun, restmp);
3176 : 0 : BLOCK_VARS (DECL_INITIAL (current_function_decl))
3177 : 0 : = restmp;
3178 : : }
3179 : : }
3180 : : else
3181 : 346 : restmp = create_tmp_reg (restype, "retval");
3182 : : }
3183 : :
3184 : 603 : for (arg = a; arg; arg = DECL_CHAIN (arg))
3185 : 137 : nargs++;
3186 : 466 : auto_vec<tree> vargs (nargs + nxargs);
3187 : 466 : i = 0;
3188 : 466 : arg = a;
3189 : :
3190 : 466 : if (nargs)
3191 : 61 : for (tree nparm = DECL_ARGUMENTS (nnode->decl);
3192 : 198 : i < nargs;
3193 : 137 : i++, arg = DECL_CHAIN (arg), nparm = DECL_CHAIN (nparm))
3194 : : {
3195 : 137 : tree save_arg = arg;
3196 : :
3197 : : /* Arrange to pass indirectly the parms, if we decided to do
3198 : : so, and revert its type in the wrapper. */
3199 : 137 : if (indirect_nparms.contains (nparm))
3200 : : {
3201 : 11 : tree ref_type = TREE_TYPE (nparm);
3202 : 11 : TREE_ADDRESSABLE (arg) = true;
3203 : 11 : arg = build1 (ADDR_EXPR, ref_type, arg);
3204 : : }
3205 : 126 : else if (!TREE_THIS_VOLATILE (arg))
3206 : 126 : DECL_NOT_GIMPLE_REG_P (arg) = 0;
3207 : :
3208 : : /* Convert the argument back to the type used by the calling
3209 : : conventions, e.g. a non-prototyped float type is passed as
3210 : : double, as in 930603-1.c, and needs to be converted back to
3211 : : double to be passed on unchanged to the wrapped
3212 : : function. */
3213 : 137 : if (TREE_TYPE (nparm) != DECL_ARG_TYPE (nparm))
3214 : : {
3215 : 8 : tree tmp = arg;
3216 : : /* If ARG is e.g. volatile, we must copy and
3217 : : convert in separate statements. */
3218 : 8 : if (!is_gimple_val (arg))
3219 : : {
3220 : 0 : tmp = create_tmp_reg (TYPE_MAIN_VARIANT
3221 : : (TREE_TYPE (arg)), "arg");
3222 : 0 : gimple *stmt = gimple_build_assign (tmp, arg);
3223 : 0 : gsi_insert_after (&bsi, stmt, GSI_NEW_STMT);
3224 : : }
3225 : 8 : arg = fold_convert (DECL_ARG_TYPE (nparm), tmp);
3226 : : }
3227 : :
3228 : 137 : if (!is_gimple_val (arg))
3229 : : {
3230 : 8 : tree tmp = create_tmp_reg (TYPE_MAIN_VARIANT
3231 : : (TREE_TYPE (arg)), "arg");
3232 : 8 : gimple *stmt = gimple_build_assign (tmp, arg);
3233 : 8 : gsi_insert_after (&bsi, stmt, GSI_NEW_STMT);
3234 : 8 : arg = tmp;
3235 : : }
3236 : 137 : vargs.quick_push (arg);
3237 : 137 : arg = save_arg;
3238 : : }
3239 : : /* These strub arguments are adjusted later. */
3240 : 466 : if (apply_args)
3241 : 16 : vargs.quick_push (null_pointer_node);
3242 : 466 : if (is_stdarg)
3243 : 8 : vargs.quick_push (null_pointer_node);
3244 : 466 : vargs.quick_push (null_pointer_node);
3245 : 466 : call = gimple_build_call_vec (build_fold_addr_expr_loc (0, alias),
3246 : : vargs);
3247 : 466 : onode->callees->call_stmt = call;
3248 : : // gimple_call_set_from_thunk (call, true);
3249 : 466 : if (DECL_STATIC_CHAIN (alias))
3250 : : {
3251 : 0 : tree p = DECL_STRUCT_FUNCTION (alias)->static_chain_decl;
3252 : 0 : tree type = TREE_TYPE (p);
3253 : 0 : tree decl = build_decl (DECL_SOURCE_LOCATION (thunk_fndecl),
3254 : : PARM_DECL, create_tmp_var_name ("CHAIN"),
3255 : : type);
3256 : 0 : DECL_ARTIFICIAL (decl) = 1;
3257 : 0 : DECL_IGNORED_P (decl) = 1;
3258 : 0 : TREE_USED (decl) = 1;
3259 : 0 : DECL_CONTEXT (decl) = thunk_fndecl;
3260 : 0 : DECL_ARG_TYPE (decl) = type;
3261 : 0 : TREE_READONLY (decl) = 1;
3262 : :
3263 : 0 : struct function *sf = DECL_STRUCT_FUNCTION (thunk_fndecl);
3264 : 0 : sf->static_chain_decl = decl;
3265 : :
3266 : 0 : gimple_call_set_chain (call, decl);
3267 : : }
3268 : :
3269 : : /* Return slot optimization is always possible and in fact required to
3270 : : return values with DECL_BY_REFERENCE. */
3271 : 466 : if (aggregate_value_p (resdecl, TREE_TYPE (thunk_fndecl))
3272 : 466 : && (!is_gimple_reg_type (TREE_TYPE (resdecl))
3273 : 7 : || DECL_BY_REFERENCE (resdecl)))
3274 : 6 : gimple_call_set_return_slot_opt (call, true);
3275 : :
3276 : 466 : if (restmp)
3277 : : {
3278 : 353 : gimple_call_set_lhs (call, restmp);
3279 : 353 : gcc_assert (useless_type_conversion_p (TREE_TYPE (restmp),
3280 : : TREE_TYPE (TREE_TYPE (alias))));
3281 : : }
3282 : 466 : gsi_insert_after (&bsi, call, GSI_NEW_STMT);
3283 : 466 : if (!alias_is_noreturn)
3284 : : {
3285 : : /* Build return value. */
3286 : 466 : if (!DECL_BY_REFERENCE (resdecl))
3287 : 460 : ret = gimple_build_return (restmp);
3288 : : else
3289 : 6 : ret = gimple_build_return (resdecl);
3290 : :
3291 : 466 : gsi_insert_after (&bsi, ret, GSI_NEW_STMT);
3292 : : }
3293 : : else
3294 : : {
3295 : 0 : remove_edge (single_succ_edge (bb));
3296 : : }
3297 : :
3298 : 466 : cfun->gimple_df->in_ssa_p = true;
3299 : 466 : update_max_bb_count ();
3300 : 466 : profile_status_for_fn (cfun)
3301 : 932 : = cfg_count.initialized_p () && cfg_count.ipa_p ()
3302 : 466 : ? PROFILE_READ : PROFILE_GUESSED;
3303 : : /* FIXME: C++ FE should stop setting TREE_ASM_WRITTEN on thunks. */
3304 : : // TREE_ASM_WRITTEN (thunk_fndecl) = false;
3305 : 466 : delete_unreachable_blocks ();
3306 : 466 : update_ssa (TODO_update_ssa);
3307 : 466 : checking_verify_flow_info ();
3308 : 466 : free_dominance_info (CDI_DOMINATORS);
3309 : :
3310 : : /* Since we want to emit the thunk, we explicitly mark its name as
3311 : : referenced. */
3312 : 466 : onode->thunk = false;
3313 : 466 : onode->lowered = true;
3314 : 466 : bitmap_obstack_release (NULL);
3315 : 466 : }
3316 : 466 : current_function_decl = NULL;
3317 : 466 : set_cfun (NULL);
3318 : : }
3319 : :
3320 : 466 : thunk_info::remove (onode);
3321 : :
3322 : : // some more of create_wrapper at the end of the next block.
3323 : : }
3324 : : }
3325 : :
3326 : 466 : {
3327 : 466 : tree aaval = NULL_TREE;
3328 : 466 : tree vaptr = NULL_TREE;
3329 : 466 : tree wmptr = NULL_TREE;
3330 : 1093 : for (tree arg = DECL_ARGUMENTS (nnode->decl); arg; arg = DECL_CHAIN (arg))
3331 : : {
3332 : 627 : aaval = vaptr;
3333 : 627 : vaptr = wmptr;
3334 : 627 : wmptr = arg;
3335 : : }
3336 : :
3337 : 466 : if (!apply_args)
3338 : : aaval = NULL_TREE;
3339 : : /* The trailing args are [apply_args], [va_list_ptr], and
3340 : : watermark. If we don't have a va_list_ptr, the penultimate
3341 : : argument is apply_args.
3342 : : */
3343 : 16 : else if (!is_stdarg)
3344 : : aaval = vaptr;
3345 : :
3346 : 450 : if (!is_stdarg)
3347 : : vaptr = NULL_TREE;
3348 : :
3349 : 466 : DECL_NAME (wmptr) = get_watermark_ptr ();
3350 : 466 : DECL_ARTIFICIAL (wmptr) = 1;
3351 : 466 : DECL_IGNORED_P (wmptr) = 1;
3352 : 466 : TREE_USED (wmptr) = 1;
3353 : :
3354 : 466 : if (is_stdarg)
3355 : : {
3356 : 8 : DECL_NAME (vaptr) = get_va_list_ptr ();
3357 : 8 : DECL_ARTIFICIAL (vaptr) = 1;
3358 : 8 : DECL_IGNORED_P (vaptr) = 1;
3359 : 8 : TREE_USED (vaptr) = 1;
3360 : : }
3361 : :
3362 : 466 : if (apply_args)
3363 : : {
3364 : 16 : DECL_NAME (aaval) = get_apply_args ();
3365 : 16 : DECL_ARTIFICIAL (aaval) = 1;
3366 : 16 : DECL_IGNORED_P (aaval) = 1;
3367 : 16 : TREE_USED (aaval) = 1;
3368 : : }
3369 : :
3370 : 466 : push_cfun (DECL_STRUCT_FUNCTION (nnode->decl));
3371 : :
3372 : 466 : {
3373 : 466 : edge e = single_succ_edge (ENTRY_BLOCK_PTR_FOR_FN (cfun));
3374 : 466 : gimple_seq seq = call_update_watermark (wmptr, nnode, e->src->count);
3375 : 466 : gsi_insert_seq_on_edge_immediate (e, seq);
3376 : : }
3377 : :
3378 : 466 : bool any_indirect = !indirect_nparms.is_empty ();
3379 : :
3380 : 466 : if (any_indirect)
3381 : : {
3382 : 11 : basic_block bb;
3383 : 11 : bool needs_commit = false;
3384 : 33 : FOR_EACH_BB_FN (bb, cfun)
3385 : : {
3386 : 22 : for (gphi_iterator gsi = gsi_start_nonvirtual_phis (bb);
3387 : 22 : !gsi_end_p (gsi);
3388 : 0 : gsi_next_nonvirtual_phi (&gsi))
3389 : : {
3390 : 0 : gphi *stmt = gsi.phi ();
3391 : :
3392 : 0 : walk_stmt_info wi = {};
3393 : 0 : wi.info = &indirect_nparms;
3394 : 0 : walk_gimple_op (stmt, walk_make_indirect, &wi);
3395 : 0 : if (wi.changed && !is_gimple_debug (gsi_stmt (gsi)))
3396 : 0 : if (walk_regimplify_phi (stmt))
3397 : 0 : needs_commit = true;
3398 : : }
3399 : :
3400 : 44 : for (gimple_stmt_iterator gsi = gsi_start_bb (bb);
3401 : 91 : !gsi_end_p (gsi); gsi_next (&gsi))
3402 : : {
3403 : 69 : gimple *stmt = gsi_stmt (gsi);
3404 : :
3405 : 69 : walk_stmt_info wi = {};
3406 : 69 : wi.info = &indirect_nparms;
3407 : 69 : walk_gimple_op (stmt, walk_make_indirect, &wi);
3408 : 69 : if (wi.changed)
3409 : : {
3410 : 0 : if (!is_gimple_debug (stmt))
3411 : : {
3412 : 0 : wi.info = &gsi;
3413 : 0 : walk_gimple_op (stmt, walk_regimplify_addr_expr,
3414 : : &wi);
3415 : : }
3416 : 0 : update_stmt (stmt);
3417 : : }
3418 : : }
3419 : : }
3420 : 11 : if (needs_commit)
3421 : 0 : gsi_commit_edge_inserts ();
3422 : : }
3423 : :
3424 : 466 : if (DECL_STRUCT_FUNCTION (nnode->decl)->calls_alloca
3425 : 466 : || is_stdarg || apply_args)
3426 : 618 : for (cgraph_edge *e = nnode->callees, *enext; e; e = enext)
3427 : : {
3428 : 534 : if (!e->call_stmt)
3429 : 0 : continue;
3430 : :
3431 : 534 : gcall *call = e->call_stmt;
3432 : 534 : gimple_stmt_iterator gsi = gsi_for_stmt (call);
3433 : 534 : tree fndecl = e->callee->decl;
3434 : :
3435 : 534 : enext = e->next_callee;
3436 : :
3437 : 534 : if (gimple_alloca_call_p (call))
3438 : : {
3439 : 74 : gimple_seq seq = call_update_watermark (wmptr, NULL,
3440 : 74 : gsi_bb (gsi)->count);
3441 : 74 : gsi_insert_finally_seq_after_call (gsi, seq);
3442 : : }
3443 : 460 : else if (fndecl && is_stdarg
3444 : 460 : && fndecl_built_in_p (fndecl, BUILT_IN_VA_START))
3445 : : {
3446 : : /* Using a non-default stdarg ABI makes the function ineligible
3447 : : for internal strub. */
3448 : 8 : gcc_checking_assert (builtin_decl_explicit (BUILT_IN_VA_START)
3449 : : == fndecl);
3450 : 8 : tree bvacopy = builtin_decl_explicit (BUILT_IN_VA_COPY);
3451 : 8 : gimple_call_set_fndecl (call, bvacopy);
3452 : 8 : tree arg = vaptr;
3453 : : /* The va_copy source must be dereferenced, unless it's an array
3454 : : type, that would have decayed to a pointer. */
3455 : 8 : if (TREE_CODE (TREE_TYPE (TREE_TYPE (vaptr))) != ARRAY_TYPE)
3456 : : {
3457 : 0 : arg = gimple_fold_indirect_ref (vaptr);
3458 : 0 : if (!arg)
3459 : 0 : arg = build2 (MEM_REF,
3460 : 0 : TREE_TYPE (TREE_TYPE (vaptr)),
3461 : : vaptr,
3462 : 0 : build_int_cst (TREE_TYPE (vaptr), 0));
3463 : 0 : if (!is_gimple_val (arg))
3464 : 0 : arg = force_gimple_operand_gsi (&gsi, arg, true,
3465 : : NULL_TREE, true, GSI_SAME_STMT);
3466 : : }
3467 : 8 : gimple_call_set_arg (call, 1, arg);
3468 : 8 : update_stmt (call);
3469 : 8 : e->redirect_callee (cgraph_node::get_create (bvacopy));
3470 : : }
3471 : 452 : else if (fndecl && apply_args
3472 : 452 : && fndecl_built_in_p (fndecl, BUILT_IN_APPLY_ARGS))
3473 : : {
3474 : 16 : tree lhs = gimple_call_lhs (call);
3475 : 16 : gimple *assign = (lhs
3476 : 16 : ? gimple_build_assign (lhs, aaval)
3477 : 12 : : gimple_build_nop ());
3478 : 16 : gsi_replace (&gsi, assign, true);
3479 : 16 : cgraph_edge::remove (e);
3480 : : }
3481 : : }
3482 : :
3483 : 466 : { // a little more copied from create_wrapper
3484 : :
3485 : : /* Inline summary set-up. */
3486 : 466 : nnode->analyze ();
3487 : : // inline_analyze_function (nnode);
3488 : : }
3489 : :
3490 : 466 : pop_cfun ();
3491 : : }
3492 : :
3493 : 466 : {
3494 : 466 : push_cfun (DECL_STRUCT_FUNCTION (onode->decl));
3495 : 466 : gimple_stmt_iterator gsi
3496 : 466 : = gsi_after_labels (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
3497 : :
3498 : 466 : gcall *wrcall;
3499 : 474 : while (!(wrcall = dyn_cast <gcall *> (gsi_stmt (gsi))))
3500 : 8 : gsi_next (&gsi);
3501 : :
3502 : 466 : tree swm = create_tmp_var (get_wmt (), ".strub.watermark");
3503 : 466 : TREE_ADDRESSABLE (swm) = true;
3504 : 466 : tree swmp = build1 (ADDR_EXPR, get_pwmt (), swm);
3505 : :
3506 : 466 : tree enter = get_enter ();
3507 : 466 : gcall *stptr = gimple_build_call (enter, 1, unshare_expr (swmp));
3508 : 466 : gimple_set_location (stptr, gimple_location (wrcall));
3509 : 466 : gsi_insert_before (&gsi, stptr, GSI_SAME_STMT);
3510 : 466 : onode->create_edge (cgraph_node::get_create (enter),
3511 : 466 : stptr, gsi_bb (gsi)->count, false);
3512 : :
3513 : 466 : int nargs = gimple_call_num_args (wrcall);
3514 : :
3515 : 466 : gimple_seq seq = NULL;
3516 : :
3517 : 466 : if (apply_args)
3518 : : {
3519 : 16 : tree aalst = create_tmp_var (ptr_type_node, ".strub.apply_args");
3520 : 16 : tree bappargs = builtin_decl_explicit (BUILT_IN_APPLY_ARGS);
3521 : 16 : gcall *appargs = gimple_build_call (bappargs, 0);
3522 : 16 : gimple_call_set_lhs (appargs, aalst);
3523 : 16 : gimple_set_location (appargs, gimple_location (wrcall));
3524 : 16 : gsi_insert_before (&gsi, appargs, GSI_SAME_STMT);
3525 : 16 : gimple_call_set_arg (wrcall, nargs - 2 - is_stdarg, aalst);
3526 : 16 : onode->create_edge (cgraph_node::get_create (bappargs),
3527 : 16 : appargs, gsi_bb (gsi)->count, false);
3528 : : }
3529 : :
3530 : 466 : if (is_stdarg)
3531 : : {
3532 : 8 : tree valst = create_tmp_var (va_list_type_node, ".strub.va_list");
3533 : 8 : TREE_ADDRESSABLE (valst) = true;
3534 : 8 : tree vaptr = build1 (ADDR_EXPR,
3535 : : build_pointer_type (va_list_type_node),
3536 : : valst);
3537 : 8 : gimple_call_set_arg (wrcall, nargs - 2, unshare_expr (vaptr));
3538 : :
3539 : 8 : tree bvastart = builtin_decl_explicit (BUILT_IN_VA_START);
3540 : 8 : gcall *vastart = gimple_build_call (bvastart, 2,
3541 : : unshare_expr (vaptr),
3542 : : integer_zero_node);
3543 : 8 : gimple_set_location (vastart, gimple_location (wrcall));
3544 : 8 : gsi_insert_before (&gsi, vastart, GSI_SAME_STMT);
3545 : 8 : onode->create_edge (cgraph_node::get_create (bvastart),
3546 : 8 : vastart, gsi_bb (gsi)->count, false);
3547 : :
3548 : 8 : tree bvaend = builtin_decl_explicit (BUILT_IN_VA_END);
3549 : 8 : gcall *vaend = gimple_build_call (bvaend, 1, unshare_expr (vaptr));
3550 : 8 : gimple_set_location (vaend, gimple_location (wrcall));
3551 : 8 : gimple_seq_add_stmt (&seq, vaend);
3552 : : }
3553 : :
3554 : 466 : gimple_call_set_arg (wrcall, nargs - 1, unshare_expr (swmp));
3555 : : // gimple_call_set_tail (wrcall, false);
3556 : 466 : update_stmt (wrcall);
3557 : :
3558 : 466 : {
3559 : : #if !ATTR_FNSPEC_DECONST_WATERMARK
3560 : : /* If the call will be assumed to not modify or even read the
3561 : : watermark, make it read and modified ourselves. */
3562 : 466 : if ((gimple_call_flags (wrcall)
3563 : 466 : & (ECF_CONST | ECF_PURE | ECF_NOVOPS)))
3564 : : {
3565 : 121 : vec<tree, va_gc> *inputs = NULL;
3566 : 121 : vec<tree, va_gc> *outputs = NULL;
3567 : 242 : vec_safe_push (outputs,
3568 : : build_tree_list
3569 : 121 : (build_tree_list
3570 : : (NULL_TREE, build_string (2, "=m")),
3571 : : swm));
3572 : 242 : vec_safe_push (inputs,
3573 : : build_tree_list
3574 : 121 : (build_tree_list
3575 : : (NULL_TREE, build_string (1, "m")),
3576 : : swm));
3577 : 121 : gasm *forcemod = gimple_build_asm_vec ("", inputs, outputs,
3578 : : NULL, NULL);
3579 : 121 : gimple_seq_add_stmt (&seq, forcemod);
3580 : :
3581 : : /* If the call will be assumed to not even read the watermark,
3582 : : make sure it is already in memory before the call. */
3583 : 121 : if ((gimple_call_flags (wrcall) & ECF_CONST))
3584 : : {
3585 : 41 : vec<tree, va_gc> *inputs = NULL;
3586 : 82 : vec_safe_push (inputs,
3587 : : build_tree_list
3588 : 41 : (build_tree_list
3589 : : (NULL_TREE, build_string (1, "m")),
3590 : : swm));
3591 : 41 : gasm *force_store = gimple_build_asm_vec ("", inputs, NULL,
3592 : : NULL, NULL);
3593 : 41 : gimple_set_location (force_store, gimple_location (wrcall));
3594 : 41 : gsi_insert_before (&gsi, force_store, GSI_SAME_STMT);
3595 : : }
3596 : : }
3597 : : #endif
3598 : :
3599 : 466 : gcall *sleave = gimple_build_call (get_leave (), 1,
3600 : : unshare_expr (swmp));
3601 : 466 : gimple_seq_add_stmt (&seq, sleave);
3602 : :
3603 : 466 : gassign *clobber = gimple_build_assign (swm,
3604 : : build_clobber
3605 : 466 : (TREE_TYPE (swm)));
3606 : 466 : gimple_seq_add_stmt (&seq, clobber);
3607 : : }
3608 : :
3609 : 466 : gsi_insert_finally_seq_after_call (gsi, seq);
3610 : :
3611 : : /* For nnode, we don't rebuild edges because we wish to retain
3612 : : any redirections copied to it from earlier passes, so we add
3613 : : call graph edges explicitly there, but for onode, we create a
3614 : : fresh function, so we may as well just issue the calls and
3615 : : then rebuild all cgraph edges. */
3616 : : // cgraph_edge::rebuild_edges ();
3617 : 466 : onode->analyze ();
3618 : : // inline_analyze_function (onode);
3619 : :
3620 : 466 : pop_cfun ();
3621 : : }
3622 : 466 : }
3623 : :
3624 : 540 : return 0;
3625 : : }
3626 : :
3627 : : simple_ipa_opt_pass *
3628 : 280114 : make_pass_ipa_strub (gcc::context *ctxt)
3629 : : {
3630 : 280114 : return new pass_ipa_strub (ctxt);
3631 : : }
3632 : :
3633 : : #include "gt-ipa-strub.h"
|
