Line data Source code
1 : /* Passes for transactional memory support.
2 : Copyright (C) 2008-2026 Free Software Foundation, Inc.
3 : Contributed by Richard Henderson <rth@redhat.com>
4 : and Aldy Hernandez <aldyh@redhat.com>.
5 :
6 : This file is part of GCC.
7 :
8 : GCC is free software; you can redistribute it and/or modify it under
9 : the terms of the GNU General Public License as published by the Free
10 : Software Foundation; either version 3, or (at your option) any later
11 : version.
12 :
13 : GCC is distributed in the hope that it will be useful, but WITHOUT ANY
14 : WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 : FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 : for more details.
17 :
18 : You should have received a copy of the GNU General Public License
19 : along with GCC; see the file COPYING3. If not see
20 : <http://www.gnu.org/licenses/>. */
21 :
22 : #include "config.h"
23 : #include "system.h"
24 : #include "coretypes.h"
25 : #include "backend.h"
26 : #include "target.h"
27 : #include "rtl.h"
28 : #include "tree.h"
29 : #include "gimple.h"
30 : #include "cfghooks.h"
31 : #include "tree-pass.h"
32 : #include "ssa.h"
33 : #include "cgraph.h"
34 : #include "gimple-pretty-print.h"
35 : #include "diagnostic-core.h"
36 : #include "fold-const.h"
37 : #include "tree-eh.h"
38 : #include "calls.h"
39 : #include "gimplify.h"
40 : #include "gimple-iterator.h"
41 : #include "gimplify-me.h"
42 : #include "gimple-walk.h"
43 : #include "tree-cfg.h"
44 : #include "tree-into-ssa.h"
45 : #include "tree-inline.h"
46 : #include "demangle.h"
47 : #include "output.h"
48 : #include "trans-mem.h"
49 : #include "langhooks.h"
50 : #include "cfgloop.h"
51 : #include "tree-ssa-address.h"
52 : #include "stringpool.h"
53 : #include "attribs.h"
54 : #include "alloc-pool.h"
55 : #include "symbol-summary.h"
56 : #include "symtab-thunks.h"
57 :
58 : #define A_RUNINSTRUMENTEDCODE 0x0001
59 : #define A_RUNUNINSTRUMENTEDCODE 0x0002
60 : #define A_SAVELIVEVARIABLES 0x0004
61 : #define A_RESTORELIVEVARIABLES 0x0008
62 : #define A_ABORTTRANSACTION 0x0010
63 :
64 : #define AR_USERABORT 0x0001
65 : #define AR_USERRETRY 0x0002
66 : #define AR_TMCONFLICT 0x0004
67 : #define AR_EXCEPTIONBLOCKABORT 0x0008
68 : #define AR_OUTERABORT 0x0010
69 :
70 : #define MODE_SERIALIRREVOCABLE 0x0000
71 :
72 :
73 : /* The representation of a transaction changes several times during the
74 : lowering process. In the beginning, in the front-end we have the
75 : GENERIC tree TRANSACTION_EXPR. For example,
76 :
77 : __transaction {
78 : local++;
79 : if (++global == 10)
80 : __tm_abort;
81 : }
82 :
83 : During initial gimplification (gimplify.cc) the TRANSACTION_EXPR node is
84 : trivially replaced with a GIMPLE_TRANSACTION node.
85 :
86 : During pass_lower_tm, we examine the body of transactions looking
87 : for aborts. Transactions that do not contain an abort may be
88 : merged into an outer transaction. We also add a TRY-FINALLY node
89 : to arrange for the transaction to be committed on any exit.
90 :
91 : [??? Think about how this arrangement affects throw-with-commit
92 : and throw-with-abort operations. In this case we want the TRY to
93 : handle gotos, but not to catch any exceptions because the transaction
94 : will already be closed.]
95 :
96 : GIMPLE_TRANSACTION [label=NULL] {
97 : try {
98 : local = local + 1;
99 : t0 = global;
100 : t1 = t0 + 1;
101 : global = t1;
102 : if (t1 == 10)
103 : __builtin___tm_abort ();
104 : } finally {
105 : __builtin___tm_commit ();
106 : }
107 : }
108 :
109 : During pass_lower_eh, we create EH regions for the transactions,
110 : intermixed with the regular EH stuff. This gives us a nice persistent
111 : mapping (all the way through rtl) from transactional memory operation
112 : back to the transaction, which allows us to get the abnormal edges
113 : correct to model transaction aborts and restarts:
114 :
115 : GIMPLE_TRANSACTION [label=over]
116 : local = local + 1;
117 : t0 = global;
118 : t1 = t0 + 1;
119 : global = t1;
120 : if (t1 == 10)
121 : __builtin___tm_abort ();
122 : __builtin___tm_commit ();
123 : over:
124 :
125 : This is the end of all_lowering_passes, and so is what is present
126 : during the IPA passes, and through all of the optimization passes.
127 :
128 : During pass_ipa_tm, we examine all GIMPLE_TRANSACTION blocks in all
129 : functions and mark functions for cloning.
130 :
131 : At the end of gimple optimization, before exiting SSA form,
132 : pass_tm_edges replaces statements that perform transactional
133 : memory operations with the appropriate TM builtins, and swap
134 : out function calls with their transactional clones. At this
135 : point we introduce the abnormal transaction restart edges and
136 : complete lowering of the GIMPLE_TRANSACTION node.
137 :
138 : x = __builtin___tm_start (MAY_ABORT);
139 : eh_label:
140 : if (x & abort_transaction)
141 : goto over;
142 : local = local + 1;
143 : t0 = __builtin___tm_load (global);
144 : t1 = t0 + 1;
145 : __builtin___tm_store (&global, t1);
146 : if (t1 == 10)
147 : __builtin___tm_abort ();
148 : __builtin___tm_commit ();
149 : over:
150 : */
151 :
152 : static void *expand_regions (struct tm_region *,
153 : void *(*callback)(struct tm_region *, void *),
154 : void *, bool);
155 :
156 :
157 : /* Return the attributes we want to examine for X, or NULL if it's not
158 : something we examine. We look at function types, but allow pointers
159 : to function types and function decls and peek through. */
160 :
161 : static tree
162 11823 : get_attrs_for (const_tree x)
163 : {
164 11823 : if (x == NULL_TREE)
165 : return NULL_TREE;
166 :
167 11823 : switch (TREE_CODE (x))
168 : {
169 7895 : case FUNCTION_DECL:
170 7895 : return TYPE_ATTRIBUTES (TREE_TYPE (x));
171 :
172 3904 : default:
173 3904 : if (TYPE_P (x))
174 : return NULL_TREE;
175 3904 : x = TREE_TYPE (x);
176 3904 : if (TREE_CODE (x) != POINTER_TYPE)
177 : return NULL_TREE;
178 : /* FALLTHRU */
179 :
180 3928 : case POINTER_TYPE:
181 3928 : x = TREE_TYPE (x);
182 3928 : if (TREE_CODE (x) != FUNCTION_TYPE && TREE_CODE (x) != METHOD_TYPE)
183 : return NULL_TREE;
184 : /* FALLTHRU */
185 :
186 3928 : case FUNCTION_TYPE:
187 3928 : case METHOD_TYPE:
188 3928 : return TYPE_ATTRIBUTES (x);
189 : }
190 : }
191 :
192 : /* Return true if X has been marked TM_PURE. */
193 :
194 : bool
195 9567394 : is_tm_pure (const_tree x)
196 : {
197 9567394 : unsigned flags;
198 :
199 9567394 : switch (TREE_CODE (x))
200 : {
201 : case FUNCTION_DECL:
202 : case FUNCTION_TYPE:
203 : case METHOD_TYPE:
204 : break;
205 :
206 238 : default:
207 238 : if (TYPE_P (x))
208 : return false;
209 238 : x = TREE_TYPE (x);
210 238 : if (TREE_CODE (x) != POINTER_TYPE)
211 : return false;
212 : /* FALLTHRU */
213 :
214 238 : case POINTER_TYPE:
215 238 : x = TREE_TYPE (x);
216 238 : if (TREE_CODE (x) != FUNCTION_TYPE && TREE_CODE (x) != METHOD_TYPE)
217 : return false;
218 : break;
219 : }
220 :
221 9567394 : flags = flags_from_decl_or_type (x);
222 9567394 : return (flags & ECF_TM_PURE) != 0;
223 : }
224 :
225 : /* Return true if X has been marked TM_IRREVOCABLE. */
226 :
227 : static bool
228 2005 : is_tm_irrevocable (tree x)
229 : {
230 2005 : tree attrs = get_attrs_for (x);
231 :
232 2005 : if (attrs && lookup_attribute ("transaction_unsafe", attrs))
233 : return true;
234 :
235 : /* A call to the irrevocable builtin is by definition,
236 : irrevocable. */
237 1969 : if (TREE_CODE (x) == ADDR_EXPR)
238 1161 : x = TREE_OPERAND (x, 0);
239 1969 : if (TREE_CODE (x) == FUNCTION_DECL
240 1969 : && fndecl_built_in_p (x, BUILT_IN_TM_IRREVOCABLE))
241 : return true;
242 :
243 : return false;
244 : }
245 :
246 : /* Return true if X has been marked TM_SAFE. */
247 :
248 : bool
249 4054 : is_tm_safe (const_tree x)
250 : {
251 4054 : if (flag_tm)
252 : {
253 4054 : tree attrs = get_attrs_for (x);
254 4054 : if (attrs)
255 : {
256 1341 : if (lookup_attribute ("transaction_safe", attrs))
257 : return true;
258 515 : if (lookup_attribute ("transaction_may_cancel_outer", attrs))
259 : return true;
260 : }
261 : }
262 : return false;
263 : }
264 :
265 : /* Return true if CALL is const, or tm_pure. */
266 :
267 : static bool
268 5404 : is_tm_pure_call (gimple *call)
269 : {
270 1588 : return (gimple_call_flags (call) & (ECF_CONST | ECF_TM_PURE)) != 0;
271 : }
272 :
273 : /* Return true if X has been marked TM_CALLABLE. */
274 :
275 : static bool
276 2452 : is_tm_callable (tree x)
277 : {
278 2452 : tree attrs = get_attrs_for (x);
279 2452 : if (attrs)
280 : {
281 704 : if (lookup_attribute ("transaction_callable", attrs))
282 : return true;
283 573 : if (lookup_attribute ("transaction_safe", attrs))
284 : return true;
285 105 : if (lookup_attribute ("transaction_may_cancel_outer", attrs))
286 : return true;
287 : }
288 : return false;
289 : }
290 :
291 : /* Return true if X has been marked TRANSACTION_MAY_CANCEL_OUTER. */
292 :
293 : bool
294 3312 : is_tm_may_cancel_outer (tree x)
295 : {
296 3312 : tree attrs = get_attrs_for (x);
297 3312 : if (attrs)
298 866 : return lookup_attribute ("transaction_may_cancel_outer", attrs) != NULL;
299 : return false;
300 : }
301 :
302 : /* Return true for built in functions that "end" a transaction. */
303 :
304 : bool
305 433321 : is_tm_ending_fndecl (tree fndecl)
306 : {
307 433321 : if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
308 332294 : switch (DECL_FUNCTION_CODE (fndecl))
309 : {
310 2746 : case BUILT_IN_TM_COMMIT:
311 2746 : case BUILT_IN_TM_COMMIT_EH:
312 2746 : case BUILT_IN_TM_ABORT:
313 2746 : case BUILT_IN_TM_IRREVOCABLE:
314 2746 : return true;
315 : default:
316 : break;
317 : }
318 :
319 : return false;
320 : }
321 :
322 : /* Return true if STMT is a built in function call that "ends" a
323 : transaction. */
324 :
325 : bool
326 487082 : is_tm_ending (gimple *stmt)
327 : {
328 487082 : tree fndecl;
329 :
330 487082 : if (gimple_code (stmt) != GIMPLE_CALL)
331 : return false;
332 :
333 428943 : fndecl = gimple_call_fndecl (stmt);
334 428943 : return (fndecl != NULL_TREE
335 428943 : && is_tm_ending_fndecl (fndecl));
336 : }
337 :
338 : /* Return true if STMT is a TM load. */
339 :
340 : static bool
341 2531 : is_tm_load (gimple *stmt)
342 : {
343 2531 : tree fndecl;
344 :
345 2531 : if (gimple_code (stmt) != GIMPLE_CALL)
346 : return false;
347 :
348 1331 : fndecl = gimple_call_fndecl (stmt);
349 1331 : return (fndecl
350 1319 : && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL)
351 2555 : && BUILTIN_TM_LOAD_P (DECL_FUNCTION_CODE (fndecl)));
352 : }
353 :
354 : /* Same as above, but for simple TM loads, that is, not the
355 : after-write, after-read, etc optimized variants. */
356 :
357 : static bool
358 1967 : is_tm_simple_load (gimple *stmt)
359 : {
360 1967 : tree fndecl;
361 :
362 1967 : if (gimple_code (stmt) != GIMPLE_CALL)
363 : return false;
364 :
365 767 : fndecl = gimple_call_fndecl (stmt);
366 767 : if (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
367 : {
368 660 : enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
369 660 : return (fcode == BUILT_IN_TM_LOAD_1
370 660 : || fcode == BUILT_IN_TM_LOAD_2
371 : || fcode == BUILT_IN_TM_LOAD_4
372 : || fcode == BUILT_IN_TM_LOAD_8
373 : || fcode == BUILT_IN_TM_LOAD_FLOAT
374 : || fcode == BUILT_IN_TM_LOAD_DOUBLE
375 : || fcode == BUILT_IN_TM_LOAD_LDOUBLE
376 : || fcode == BUILT_IN_TM_LOAD_M64
377 443 : || fcode == BUILT_IN_TM_LOAD_M128
378 660 : || fcode == BUILT_IN_TM_LOAD_M256);
379 : }
380 : return false;
381 : }
382 :
383 : /* Return true if STMT is a TM store. */
384 :
385 : static bool
386 2189 : is_tm_store (gimple *stmt)
387 : {
388 2189 : tree fndecl;
389 :
390 2189 : if (gimple_code (stmt) != GIMPLE_CALL)
391 : return false;
392 :
393 989 : fndecl = gimple_call_fndecl (stmt);
394 989 : return (fndecl
395 977 : && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL)
396 1871 : && BUILTIN_TM_STORE_P (DECL_FUNCTION_CODE (fndecl)));
397 : }
398 :
399 : /* Same as above, but for simple TM stores, that is, not the
400 : after-write, after-read, etc optimized variants. */
401 :
402 : static bool
403 1746 : is_tm_simple_store (gimple *stmt)
404 : {
405 1746 : tree fndecl;
406 :
407 1746 : if (gimple_code (stmt) != GIMPLE_CALL)
408 : return false;
409 :
410 546 : fndecl = gimple_call_fndecl (stmt);
411 546 : if (fndecl
412 546 : && fndecl_built_in_p (fndecl, BUILT_IN_NORMAL))
413 : {
414 439 : enum built_in_function fcode = DECL_FUNCTION_CODE (fndecl);
415 439 : return (fcode == BUILT_IN_TM_STORE_1
416 439 : || fcode == BUILT_IN_TM_STORE_2
417 : || fcode == BUILT_IN_TM_STORE_4
418 : || fcode == BUILT_IN_TM_STORE_8
419 : || fcode == BUILT_IN_TM_STORE_FLOAT
420 : || fcode == BUILT_IN_TM_STORE_DOUBLE
421 : || fcode == BUILT_IN_TM_STORE_LDOUBLE
422 : || fcode == BUILT_IN_TM_STORE_M64
423 329 : || fcode == BUILT_IN_TM_STORE_M128
424 439 : || fcode == BUILT_IN_TM_STORE_M256);
425 : }
426 : return false;
427 : }
428 :
429 : /* Return true if FNDECL is BUILT_IN_TM_ABORT. */
430 :
431 : static bool
432 1099 : is_tm_abort (tree fndecl)
433 : {
434 467 : return (fndecl && fndecl_built_in_p (fndecl, BUILT_IN_TM_ABORT));
435 : }
436 :
437 : /* Build a GENERIC tree for a user abort. This is called by front ends
438 : while transforming the __tm_abort statement. */
439 :
440 : tree
441 32 : build_tm_abort_call (location_t loc, bool is_outer)
442 : {
443 32 : return build_call_expr_loc (loc, builtin_decl_explicit (BUILT_IN_TM_ABORT), 1,
444 : build_int_cst (integer_type_node,
445 : AR_USERABORT
446 59 : | (is_outer ? AR_OUTERABORT : 0)));
447 : }
448 :
449 : /* Map for arbitrary function replacement under TM, as created
450 : by the tm_wrap attribute. */
451 :
452 : struct tm_wrapper_hasher : ggc_cache_ptr_hash<tree_map>
453 : {
454 378 : static inline hashval_t hash (tree_map *m) { return m->hash; }
455 : static inline bool
456 954 : equal (tree_map *a, tree_map *b)
457 : {
458 954 : return a->base.from == b->base.from;
459 : }
460 :
461 : static int
462 10 : keep_cache_entry (tree_map *&m)
463 : {
464 10 : return ggc_marked_p (m->base.from);
465 : }
466 : };
467 :
468 : static GTY((cache)) hash_table<tm_wrapper_hasher> *tm_wrap_map;
469 :
470 : void
471 116 : record_tm_replacement (tree from, tree to)
472 : {
473 116 : struct tree_map **slot, *h;
474 :
475 : /* Do not inline wrapper functions that will get replaced in the TM
476 : pass.
477 :
478 : Suppose you have foo() that will get replaced into tmfoo(). Make
479 : sure the inliner doesn't try to outsmart us and inline foo()
480 : before we get a chance to do the TM replacement. */
481 116 : DECL_UNINLINABLE (from) = 1;
482 :
483 116 : if (tm_wrap_map == NULL)
484 50 : tm_wrap_map = hash_table<tm_wrapper_hasher>::create_ggc (32);
485 :
486 116 : h = ggc_alloc<tree_map> ();
487 116 : h->hash = htab_hash_pointer (from);
488 116 : h->base.from = from;
489 116 : h->to = to;
490 :
491 116 : slot = tm_wrap_map->find_slot_with_hash (h, h->hash, INSERT);
492 116 : *slot = h;
493 116 : }
494 :
495 : /* Return a TM-aware replacement function for DECL. */
496 :
497 : static tree
498 5010 : find_tm_replacement_function (tree fndecl)
499 : {
500 5010 : if (tm_wrap_map)
501 : {
502 1494 : struct tree_map *h, in;
503 :
504 1494 : in.base.from = fndecl;
505 1494 : in.hash = htab_hash_pointer (fndecl);
506 1494 : h = tm_wrap_map->find_with_hash (&in, in.hash);
507 1494 : if (h)
508 555 : return h->to;
509 : }
510 :
511 : /* ??? We may well want TM versions of most of the common <string.h>
512 : functions. For now, we've already these two defined. */
513 : /* Adjust expand_call_tm() attributes as necessary for the cases
514 : handled here: */
515 4455 : if (DECL_BUILT_IN_CLASS (fndecl) == BUILT_IN_NORMAL)
516 914 : switch (DECL_FUNCTION_CODE (fndecl))
517 : {
518 53 : case BUILT_IN_MEMCPY:
519 53 : return builtin_decl_explicit (BUILT_IN_TM_MEMCPY);
520 33 : case BUILT_IN_MEMMOVE:
521 33 : return builtin_decl_explicit (BUILT_IN_TM_MEMMOVE);
522 22 : case BUILT_IN_MEMSET:
523 22 : return builtin_decl_explicit (BUILT_IN_TM_MEMSET);
524 : default:
525 : return NULL;
526 : }
527 :
528 : return NULL;
529 : }
530 :
531 : /* When appropriate, record TM replacement for memory allocation functions.
532 :
533 : FROM is the FNDECL to wrap. */
534 : void
535 3982 : tm_malloc_replacement (tree from)
536 : {
537 3982 : const char *str;
538 3982 : tree to;
539 :
540 3982 : if (TREE_CODE (from) != FUNCTION_DECL)
541 : return;
542 :
543 : /* If we have a previous replacement, the user must be explicitly
544 : wrapping malloc/calloc/free. They better know what they're
545 : doing... */
546 2402 : if (find_tm_replacement_function (from))
547 : return;
548 :
549 2299 : str = IDENTIFIER_POINTER (DECL_NAME (from));
550 :
551 2299 : if (!strcmp (str, "malloc"))
552 16 : to = builtin_decl_explicit (BUILT_IN_TM_MALLOC);
553 2283 : else if (!strcmp (str, "calloc"))
554 4 : to = builtin_decl_explicit (BUILT_IN_TM_CALLOC);
555 2279 : else if (!strcmp (str, "free"))
556 4 : to = builtin_decl_explicit (BUILT_IN_TM_FREE);
557 : else
558 : return;
559 :
560 24 : TREE_NOTHROW (to) = 0;
561 :
562 24 : record_tm_replacement (from, to);
563 : }
564 :
565 : /* Diagnostics for tm_safe functions/regions. Called by the front end
566 : once we've lowered the function to high-gimple. */
567 :
568 : /* Subroutine of diagnose_tm_safe_errors, called through walk_gimple_seq.
569 : Process exactly one statement. WI->INFO is set to non-null when in
570 : the context of a tm_safe function, and null for a __transaction block. */
571 :
572 : #define DIAG_TM_OUTER 1
573 : #define DIAG_TM_SAFE 2
574 : #define DIAG_TM_RELAXED 4
575 :
576 : struct diagnose_tm
577 : {
578 : unsigned int summary_flags : 8;
579 : unsigned int block_flags : 8;
580 : unsigned int func_flags : 8;
581 : unsigned int saw_volatile : 1;
582 : gimple *stmt;
583 : };
584 :
585 : /* Return true if T is a volatile lvalue of some kind. */
586 :
587 : static bool
588 25288 : volatile_lvalue_p (tree t)
589 : {
590 25288 : return ((SSA_VAR_P (t) || REFERENCE_CLASS_P (t))
591 13466 : && TREE_THIS_VOLATILE (TREE_TYPE (t)));
592 : }
593 :
594 : /* Tree callback function for diagnose_tm pass. */
595 :
596 : static tree
597 22447 : diagnose_tm_1_op (tree *tp, int *walk_subtrees, void *data)
598 : {
599 22447 : struct walk_stmt_info *wi = (struct walk_stmt_info *) data;
600 22447 : struct diagnose_tm *d = (struct diagnose_tm *) wi->info;
601 :
602 22447 : if (TYPE_P (*tp))
603 36 : *walk_subtrees = false;
604 22411 : else if (volatile_lvalue_p (*tp)
605 22411 : && !d->saw_volatile)
606 : {
607 22 : d->saw_volatile = 1;
608 22 : if (d->block_flags & DIAG_TM_SAFE)
609 2 : error_at (gimple_location (d->stmt),
610 : "invalid use of volatile lvalue inside transaction");
611 20 : else if (d->func_flags & DIAG_TM_SAFE)
612 4 : error_at (gimple_location (d->stmt),
613 : "invalid use of volatile lvalue inside %<transaction_safe%> "
614 : "function");
615 : }
616 :
617 22447 : return NULL_TREE;
618 : }
619 :
620 : static inline bool
621 1267 : is_tm_safe_or_pure (const_tree x)
622 : {
623 1267 : return is_tm_safe (x) || is_tm_pure (x);
624 : }
625 :
626 : static tree
627 11356 : diagnose_tm_1 (gimple_stmt_iterator *gsi, bool *handled_ops_p,
628 : struct walk_stmt_info *wi)
629 : {
630 11356 : gimple *stmt = gsi_stmt (*gsi);
631 11356 : struct diagnose_tm *d = (struct diagnose_tm *) wi->info;
632 :
633 : /* Save stmt for use in leaf analysis. */
634 11356 : d->stmt = stmt;
635 :
636 11356 : switch (gimple_code (stmt))
637 : {
638 2209 : case GIMPLE_CALL:
639 2209 : {
640 2209 : if (gimple_call_internal_p (stmt))
641 : break;
642 :
643 2165 : tree fn = gimple_call_fn (stmt);
644 :
645 2165 : if ((d->summary_flags & DIAG_TM_OUTER) == 0
646 2165 : && is_tm_may_cancel_outer (fn))
647 24 : error_at (gimple_location (stmt),
648 : "%<transaction_may_cancel_outer%> function call not within"
649 : " outer transaction or %<transaction_may_cancel_outer%>");
650 :
651 2165 : if (d->summary_flags & DIAG_TM_SAFE)
652 : {
653 624 : bool is_safe, direct_call_p;
654 624 : tree replacement;
655 :
656 624 : if (TREE_CODE (fn) == ADDR_EXPR
657 624 : && TREE_CODE (TREE_OPERAND (fn, 0)) == FUNCTION_DECL)
658 : {
659 570 : direct_call_p = true;
660 570 : replacement = TREE_OPERAND (fn, 0);
661 570 : replacement = find_tm_replacement_function (replacement);
662 570 : if (replacement)
663 158 : fn = replacement;
664 : }
665 : else
666 : {
667 : direct_call_p = false;
668 : replacement = NULL_TREE;
669 : }
670 :
671 624 : if (is_tm_safe_or_pure (fn))
672 : is_safe = true;
673 190 : else if (is_tm_callable (fn) || is_tm_irrevocable (fn))
674 : {
675 : /* A function explicitly marked transaction_callable as
676 : opposed to transaction_safe is being defined to be
677 : unsafe as part of its ABI, regardless of its contents. */
678 : is_safe = false;
679 : }
680 157 : else if (direct_call_p)
681 : {
682 144 : if (IS_TYPE_OR_DECL_P (fn)
683 144 : && flags_from_decl_or_type (fn) & ECF_TM_BUILTIN)
684 : is_safe = true;
685 138 : else if (replacement)
686 : {
687 : /* ??? At present we've been considering replacements
688 : merely transaction_callable, and therefore might
689 : enter irrevocable. The tm_wrap attribute has not
690 : yet made it into the new language spec. */
691 : is_safe = false;
692 : }
693 : else
694 : {
695 : /* ??? Diagnostics for unmarked direct calls moved into
696 : the IPA pass. Section 3.2 of the spec details how
697 : functions not marked should be considered "implicitly
698 : safe" based on having examined the function body. */
699 : is_safe = true;
700 : }
701 : }
702 : else
703 : {
704 : /* An unmarked indirect call. Consider it unsafe even
705 : though optimization may yet figure out how to inline. */
706 : is_safe = false;
707 : }
708 :
709 : if (!is_safe)
710 : {
711 46 : if (TREE_CODE (fn) == ADDR_EXPR)
712 17 : fn = TREE_OPERAND (fn, 0);
713 46 : if (d->block_flags & DIAG_TM_SAFE)
714 : {
715 1 : if (direct_call_p)
716 1 : error_at (gimple_location (stmt),
717 : "unsafe function call %qD within "
718 : "atomic transaction", fn);
719 : else
720 : {
721 0 : if ((!DECL_P (fn) || DECL_NAME (fn))
722 0 : && TREE_CODE (fn) != SSA_NAME)
723 0 : error_at (gimple_location (stmt),
724 : "unsafe function call %qE within "
725 : "atomic transaction", fn);
726 : else
727 0 : error_at (gimple_location (stmt),
728 : "unsafe indirect function call within "
729 : "atomic transaction");
730 : }
731 : }
732 : else
733 : {
734 45 : if (direct_call_p)
735 16 : error_at (gimple_location (stmt),
736 : "unsafe function call %qD within "
737 : "%<transaction_safe%> function", fn);
738 : else
739 : {
740 0 : if ((!DECL_P (fn) || DECL_NAME (fn))
741 29 : && TREE_CODE (fn) != SSA_NAME)
742 3 : error_at (gimple_location (stmt),
743 : "unsafe function call %qE within "
744 : "%<transaction_safe%> function", fn);
745 : else
746 26 : error_at (gimple_location (stmt),
747 : "unsafe indirect function call within "
748 : "%<transaction_safe%> function");
749 : }
750 : }
751 : }
752 : }
753 : }
754 : break;
755 :
756 52 : case GIMPLE_ASM:
757 : /* ??? We ought to come up with a way to add attributes to
758 : asm statements, and then add "transaction_safe" to it.
759 : Either that or get the language spec to resurrect __tm_waiver. */
760 52 : if (d->block_flags & DIAG_TM_SAFE)
761 0 : error_at (gimple_location (stmt),
762 : "%<asm%> not allowed in atomic transaction");
763 52 : else if (d->func_flags & DIAG_TM_SAFE)
764 16 : error_at (gimple_location (stmt),
765 : "%<asm%> not allowed in %<transaction_safe%> function");
766 : break;
767 :
768 474 : case GIMPLE_TRANSACTION:
769 474 : {
770 474 : gtransaction *trans_stmt = as_a <gtransaction *> (stmt);
771 474 : unsigned char inner_flags = DIAG_TM_SAFE;
772 :
773 474 : if (gimple_transaction_subcode (trans_stmt) & GTMA_IS_RELAXED)
774 : {
775 79 : if (d->block_flags & DIAG_TM_SAFE)
776 0 : error_at (gimple_location (stmt),
777 : "relaxed transaction in atomic transaction");
778 79 : else if (d->func_flags & DIAG_TM_SAFE)
779 0 : error_at (gimple_location (stmt),
780 : "relaxed transaction in %<transaction_safe%> function");
781 : inner_flags = DIAG_TM_RELAXED;
782 : }
783 395 : else if (gimple_transaction_subcode (trans_stmt) & GTMA_IS_OUTER)
784 : {
785 31 : if (d->block_flags)
786 0 : error_at (gimple_location (stmt),
787 : "outer transaction in transaction");
788 31 : else if (d->func_flags & DIAG_TM_OUTER)
789 4 : error_at (gimple_location (stmt),
790 : "outer transaction in "
791 : "%<transaction_may_cancel_outer%> function");
792 27 : else if (d->func_flags & DIAG_TM_SAFE)
793 0 : error_at (gimple_location (stmt),
794 : "outer transaction in %<transaction_safe%> function");
795 : inner_flags |= DIAG_TM_OUTER;
796 : }
797 :
798 474 : *handled_ops_p = true;
799 474 : if (gimple_transaction_body (trans_stmt))
800 : {
801 474 : struct walk_stmt_info wi_inner;
802 474 : struct diagnose_tm d_inner;
803 :
804 474 : memset (&d_inner, 0, sizeof (d_inner));
805 474 : d_inner.func_flags = d->func_flags;
806 474 : d_inner.block_flags = d->block_flags | inner_flags;
807 474 : d_inner.summary_flags = d_inner.func_flags | d_inner.block_flags;
808 :
809 474 : memset (&wi_inner, 0, sizeof (wi_inner));
810 474 : wi_inner.info = &d_inner;
811 :
812 474 : walk_gimple_seq (gimple_transaction_body (trans_stmt),
813 : diagnose_tm_1, diagnose_tm_1_op, &wi_inner);
814 : }
815 : }
816 : break;
817 :
818 : default:
819 : break;
820 : }
821 :
822 11356 : return NULL_TREE;
823 : }
824 :
825 : static unsigned int
826 1288 : diagnose_tm_blocks (void)
827 : {
828 1288 : struct walk_stmt_info wi;
829 1288 : struct diagnose_tm d;
830 :
831 1288 : memset (&d, 0, sizeof (d));
832 1288 : if (is_tm_may_cancel_outer (current_function_decl))
833 19 : d.func_flags = DIAG_TM_OUTER | DIAG_TM_SAFE;
834 1269 : else if (is_tm_safe (current_function_decl))
835 155 : d.func_flags = DIAG_TM_SAFE;
836 1288 : d.summary_flags = d.func_flags;
837 :
838 1288 : memset (&wi, 0, sizeof (wi));
839 1288 : wi.info = &d;
840 :
841 1288 : walk_gimple_seq (gimple_body (current_function_decl),
842 : diagnose_tm_1, diagnose_tm_1_op, &wi);
843 :
844 1288 : return 0;
845 : }
846 :
847 : namespace {
848 :
849 : const pass_data pass_data_diagnose_tm_blocks =
850 : {
851 : GIMPLE_PASS, /* type */
852 : "*diagnose_tm_blocks", /* name */
853 : OPTGROUP_NONE, /* optinfo_flags */
854 : TV_TRANS_MEM, /* tv_id */
855 : PROP_gimple_any, /* properties_required */
856 : 0, /* properties_provided */
857 : 0, /* properties_destroyed */
858 : 0, /* todo_flags_start */
859 : 0, /* todo_flags_finish */
860 : };
861 :
862 : class pass_diagnose_tm_blocks : public gimple_opt_pass
863 : {
864 : public:
865 285722 : pass_diagnose_tm_blocks (gcc::context *ctxt)
866 571444 : : gimple_opt_pass (pass_data_diagnose_tm_blocks, ctxt)
867 : {}
868 :
869 : /* opt_pass methods: */
870 2869218 : bool gate (function *) final override { return flag_tm; }
871 1288 : unsigned int execute (function *) final override
872 : {
873 1288 : return diagnose_tm_blocks ();
874 : }
875 :
876 : }; // class pass_diagnose_tm_blocks
877 :
878 : } // anon namespace
879 :
880 : gimple_opt_pass *
881 285722 : make_pass_diagnose_tm_blocks (gcc::context *ctxt)
882 : {
883 285722 : return new pass_diagnose_tm_blocks (ctxt);
884 : }
885 :
886 : /* Instead of instrumenting thread private memory, we save the
887 : addresses in a log which we later use to save/restore the addresses
888 : upon transaction start/restart.
889 :
890 : The log is keyed by address, where each element contains individual
891 : statements among different code paths that perform the store.
892 :
893 : This log is later used to generate either plain save/restore of the
894 : addresses upon transaction start/restart, or calls to the ITM_L*
895 : logging functions.
896 :
897 : So for something like:
898 :
899 : struct large { int x[1000]; };
900 : struct large lala = { 0 };
901 : __transaction {
902 : lala.x[i] = 123;
903 : ...
904 : }
905 :
906 : We can either save/restore:
907 :
908 : lala = { 0 };
909 : trxn = _ITM_startTransaction ();
910 : if (trxn & a_saveLiveVariables)
911 : tmp_lala1 = lala.x[i];
912 : else if (a & a_restoreLiveVariables)
913 : lala.x[i] = tmp_lala1;
914 :
915 : or use the logging functions:
916 :
917 : lala = { 0 };
918 : trxn = _ITM_startTransaction ();
919 : _ITM_LU4 (&lala.x[i]);
920 :
921 : Obviously, if we use _ITM_L* to log, we prefer to call _ITM_L* as
922 : far up the dominator tree to shadow all of the writes to a given
923 : location (thus reducing the total number of logging calls), but not
924 : so high as to be called on a path that does not perform a
925 : write. */
926 :
927 : /* One individual log entry. We may have multiple statements for the
928 : same location if neither dominate each other (on different
929 : execution paths). */
930 : struct tm_log_entry
931 : {
932 : /* Address to save. */
933 : tree addr;
934 : /* Entry block for the transaction this address occurs in. */
935 : basic_block entry_block;
936 : /* Dominating statements the store occurs in. */
937 : vec<gimple *> stmts;
938 : /* Initially, while we are building the log, we place a nonzero
939 : value here to mean that this address *will* be saved with a
940 : save/restore sequence. Later, when generating the save sequence
941 : we place the SSA temp generated here. */
942 : tree save_var;
943 : };
944 :
945 :
946 : /* Log entry hashtable helpers. */
947 :
948 : struct log_entry_hasher : pointer_hash <tm_log_entry>
949 : {
950 : static inline hashval_t hash (const tm_log_entry *);
951 : static inline bool equal (const tm_log_entry *, const tm_log_entry *);
952 : static inline void remove (tm_log_entry *);
953 : };
954 :
955 : /* Htab support. Return hash value for a `tm_log_entry'. */
956 : inline hashval_t
957 85 : log_entry_hasher::hash (const tm_log_entry *log)
958 : {
959 85 : return iterative_hash_expr (log->addr, 0);
960 : }
961 :
962 : /* Htab support. Return true if two log entries are the same. */
963 : inline bool
964 37 : log_entry_hasher::equal (const tm_log_entry *log1, const tm_log_entry *log2)
965 : {
966 : /* FIXME:
967 :
968 : rth: I suggest that we get rid of the component refs etc.
969 : I.e. resolve the reference to base + offset.
970 :
971 : We may need to actually finish a merge with mainline for this,
972 : since we'd like to be presented with Richi's MEM_REF_EXPRs more
973 : often than not. But in the meantime your tm_log_entry could save
974 : the results of get_inner_reference.
975 :
976 : See: g++.dg/tm/pr46653.C
977 : */
978 :
979 : /* Special case plain equality because operand_equal_p() below will
980 : return FALSE if the addresses are equal but they have
981 : side-effects (e.g. a volatile address). */
982 37 : if (log1->addr == log2->addr)
983 : return true;
984 :
985 1 : return operand_equal_p (log1->addr, log2->addr, 0);
986 : }
987 :
988 : /* Htab support. Free one tm_log_entry. */
989 : inline void
990 24 : log_entry_hasher::remove (tm_log_entry *lp)
991 : {
992 24 : lp->stmts.release ();
993 24 : free (lp);
994 24 : }
995 :
996 :
997 : /* The actual log. */
998 : static hash_table<log_entry_hasher> *tm_log;
999 :
1000 : /* Addresses to log with a save/restore sequence. These should be in
1001 : dominator order. */
1002 : static vec<tree> tm_log_save_addresses;
1003 :
1004 : enum thread_memory_type
1005 : {
1006 : mem_non_local = 0,
1007 : mem_thread_local,
1008 : mem_transaction_local,
1009 : mem_max
1010 : };
1011 :
1012 : struct tm_new_mem_map
1013 : {
1014 : /* SSA_NAME being dereferenced. */
1015 : tree val;
1016 : enum thread_memory_type local_new_memory;
1017 : };
1018 :
1019 : /* Hashtable helpers. */
1020 :
1021 : struct tm_mem_map_hasher : free_ptr_hash <tm_new_mem_map>
1022 : {
1023 : static inline hashval_t hash (const tm_new_mem_map *);
1024 : static inline bool equal (const tm_new_mem_map *, const tm_new_mem_map *);
1025 : };
1026 :
1027 : inline hashval_t
1028 287 : tm_mem_map_hasher::hash (const tm_new_mem_map *v)
1029 : {
1030 287 : return (intptr_t)v->val >> 4;
1031 : }
1032 :
1033 : inline bool
1034 140 : tm_mem_map_hasher::equal (const tm_new_mem_map *v, const tm_new_mem_map *c)
1035 : {
1036 140 : return v->val == c->val;
1037 : }
1038 :
1039 : /* Map for an SSA_NAME originally pointing to a non aliased new piece
1040 : of memory (malloc, alloc, etc). */
1041 : static hash_table<tm_mem_map_hasher> *tm_new_mem_hash;
1042 :
1043 : /* Initialize logging data structures. */
1044 : static void
1045 474 : tm_log_init (void)
1046 : {
1047 474 : tm_log = new hash_table<log_entry_hasher> (10);
1048 474 : tm_new_mem_hash = new hash_table<tm_mem_map_hasher> (5);
1049 474 : tm_log_save_addresses.create (5);
1050 474 : }
1051 :
1052 : /* Free logging data structures. */
1053 : static void
1054 474 : tm_log_delete (void)
1055 : {
1056 474 : delete tm_log;
1057 474 : tm_log = NULL;
1058 474 : delete tm_new_mem_hash;
1059 474 : tm_new_mem_hash = NULL;
1060 474 : tm_log_save_addresses.release ();
1061 474 : }
1062 :
1063 : /* Return true if MEM is a transaction invariant memory for the TM
1064 : region starting at REGION_ENTRY_BLOCK. */
1065 : static bool
1066 24 : transaction_invariant_address_p (const_tree mem, basic_block region_entry_block)
1067 : {
1068 24 : if ((INDIRECT_REF_P (mem) || TREE_CODE (mem) == MEM_REF)
1069 31 : && TREE_CODE (TREE_OPERAND (mem, 0)) == SSA_NAME)
1070 : {
1071 7 : basic_block def_bb;
1072 :
1073 7 : def_bb = gimple_bb (SSA_NAME_DEF_STMT (TREE_OPERAND (mem, 0)));
1074 7 : return def_bb != region_entry_block
1075 7 : && dominated_by_p (CDI_DOMINATORS, region_entry_block, def_bb);
1076 : }
1077 :
1078 17 : mem = strip_invariant_refs (mem);
1079 17 : return mem && (CONSTANT_CLASS_P (mem) || decl_address_invariant_p (mem));
1080 : }
1081 :
1082 : /* Given an address ADDR in STMT, find it in the memory log or add it,
1083 : making sure to keep only the addresses highest in the dominator
1084 : tree.
1085 :
1086 : ENTRY_BLOCK is the entry_block for the transaction.
1087 :
1088 : If we find the address in the log, make sure it's either the same
1089 : address, or an equivalent one that dominates ADDR.
1090 :
1091 : If we find the address, but neither ADDR dominates the found
1092 : address, nor the found one dominates ADDR, we're on different
1093 : execution paths. Add it.
1094 :
1095 : If known, ENTRY_BLOCK is the entry block for the region, otherwise
1096 : NULL. */
1097 : static void
1098 24 : tm_log_add (basic_block entry_block, tree addr, gimple *stmt)
1099 : {
1100 24 : tm_log_entry **slot;
1101 24 : struct tm_log_entry l, *lp;
1102 :
1103 24 : l.addr = addr;
1104 24 : slot = tm_log->find_slot (&l, INSERT);
1105 24 : if (!*slot)
1106 : {
1107 24 : tree type = TREE_TYPE (addr);
1108 :
1109 24 : lp = XNEW (struct tm_log_entry);
1110 24 : lp->addr = addr;
1111 24 : *slot = lp;
1112 :
1113 : /* Small invariant addresses can be handled as save/restores. */
1114 24 : if (entry_block
1115 24 : && transaction_invariant_address_p (lp->addr, entry_block)
1116 21 : && TYPE_SIZE_UNIT (type) != NULL
1117 21 : && tree_fits_uhwi_p (TYPE_SIZE_UNIT (type))
1118 21 : && ((HOST_WIDE_INT) tree_to_uhwi (TYPE_SIZE_UNIT (type))
1119 21 : < param_tm_max_aggregate_size)
1120 : /* We must be able to copy this type normally. I.e., no
1121 : special constructors and the like. */
1122 43 : && !TREE_ADDRESSABLE (type))
1123 : {
1124 19 : lp->save_var = create_tmp_reg (TREE_TYPE (lp->addr), "tm_save");
1125 19 : lp->stmts.create (0);
1126 19 : lp->entry_block = entry_block;
1127 : /* Save addresses separately in dominator order so we don't
1128 : get confused by overlapping addresses in the save/restore
1129 : sequence. */
1130 19 : tm_log_save_addresses.safe_push (lp->addr);
1131 : }
1132 : else
1133 : {
1134 : /* Use the logging functions. */
1135 5 : lp->stmts.create (5);
1136 5 : lp->stmts.quick_push (stmt);
1137 5 : lp->save_var = NULL;
1138 : }
1139 : }
1140 : else
1141 : {
1142 0 : size_t i;
1143 0 : gimple *oldstmt;
1144 :
1145 0 : lp = *slot;
1146 :
1147 : /* If we're generating a save/restore sequence, we don't care
1148 : about statements. */
1149 0 : if (lp->save_var)
1150 0 : return;
1151 :
1152 0 : for (i = 0; lp->stmts.iterate (i, &oldstmt); ++i)
1153 : {
1154 0 : if (stmt == oldstmt)
1155 : return;
1156 : /* We already have a store to the same address, higher up the
1157 : dominator tree. Nothing to do. */
1158 0 : if (dominated_by_p (CDI_DOMINATORS,
1159 0 : gimple_bb (stmt), gimple_bb (oldstmt)))
1160 : return;
1161 : /* We should be processing blocks in dominator tree order. */
1162 0 : gcc_assert (!dominated_by_p (CDI_DOMINATORS,
1163 : gimple_bb (oldstmt), gimple_bb (stmt)));
1164 : }
1165 : /* Store is on a different code path. */
1166 0 : lp->stmts.safe_push (stmt);
1167 : }
1168 : }
1169 :
1170 : /* Gimplify the address of a TARGET_MEM_REF. Return the SSA_NAME
1171 : result, insert the new statements before GSI. */
1172 :
1173 : static tree
1174 662 : gimplify_addr (gimple_stmt_iterator *gsi, tree x)
1175 : {
1176 662 : if (TREE_CODE (x) == TARGET_MEM_REF)
1177 12 : x = tree_mem_ref_addr (build_pointer_type (TREE_TYPE (x)), x);
1178 : else
1179 650 : x = build_fold_addr_expr (x);
1180 662 : return force_gimple_operand_gsi (gsi, x, true, NULL, true, GSI_SAME_STMT);
1181 : }
1182 :
1183 : /* Instrument one address with the logging functions.
1184 : ADDR is the address to save.
1185 : STMT is the statement before which to place it. */
1186 : static void
1187 5 : tm_log_emit_stmt (tree addr, gimple *stmt)
1188 : {
1189 5 : tree type = TREE_TYPE (addr);
1190 5 : gimple_stmt_iterator gsi = gsi_for_stmt (stmt);
1191 5 : gimple *log;
1192 5 : enum built_in_function code = BUILT_IN_TM_LOG;
1193 :
1194 5 : if (type == float_type_node)
1195 : code = BUILT_IN_TM_LOG_FLOAT;
1196 5 : else if (type == double_type_node)
1197 : code = BUILT_IN_TM_LOG_DOUBLE;
1198 5 : else if (type == long_double_type_node)
1199 : code = BUILT_IN_TM_LOG_LDOUBLE;
1200 5 : else if (TYPE_SIZE (type) != NULL
1201 5 : && tree_fits_uhwi_p (TYPE_SIZE (type)))
1202 : {
1203 5 : unsigned HOST_WIDE_INT type_size = tree_to_uhwi (TYPE_SIZE (type));
1204 :
1205 5 : if (TREE_CODE (type) == VECTOR_TYPE)
1206 : {
1207 1 : switch (type_size)
1208 : {
1209 : case 64:
1210 : code = BUILT_IN_TM_LOG_M64;
1211 : break;
1212 1 : case 128:
1213 1 : code = BUILT_IN_TM_LOG_M128;
1214 1 : break;
1215 0 : case 256:
1216 0 : code = BUILT_IN_TM_LOG_M256;
1217 0 : break;
1218 0 : default:
1219 0 : goto unhandled_vec;
1220 : }
1221 1 : if (!builtin_decl_explicit_p (code))
1222 0 : goto unhandled_vec;
1223 : }
1224 : else
1225 : {
1226 4 : unhandled_vec:
1227 4 : switch (type_size)
1228 : {
1229 : case 8:
1230 : code = BUILT_IN_TM_LOG_1;
1231 : break;
1232 : case 16:
1233 : code = BUILT_IN_TM_LOG_2;
1234 : break;
1235 : case 32:
1236 : code = BUILT_IN_TM_LOG_4;
1237 : break;
1238 : case 64:
1239 : code = BUILT_IN_TM_LOG_8;
1240 : break;
1241 : }
1242 : }
1243 : }
1244 :
1245 5 : if (code != BUILT_IN_TM_LOG && !builtin_decl_explicit_p (code))
1246 : code = BUILT_IN_TM_LOG;
1247 5 : tree decl = builtin_decl_explicit (code);
1248 :
1249 5 : addr = gimplify_addr (&gsi, addr);
1250 5 : if (code == BUILT_IN_TM_LOG)
1251 0 : log = gimple_build_call (decl, 2, addr, TYPE_SIZE_UNIT (type));
1252 : else
1253 5 : log = gimple_build_call (decl, 1, addr);
1254 5 : gsi_insert_before (&gsi, log, GSI_SAME_STMT);
1255 5 : }
1256 :
1257 : /* Go through the log and instrument address that must be instrumented
1258 : with the logging functions. Leave the save/restore addresses for
1259 : later. */
1260 : static void
1261 474 : tm_log_emit (void)
1262 : {
1263 474 : hash_table<log_entry_hasher>::iterator hi;
1264 474 : struct tm_log_entry *lp;
1265 :
1266 498 : FOR_EACH_HASH_TABLE_ELEMENT (*tm_log, lp, tm_log_entry_t, hi)
1267 : {
1268 24 : size_t i;
1269 24 : gimple *stmt;
1270 :
1271 24 : if (dump_file)
1272 : {
1273 7 : fprintf (dump_file, "TM thread private mem logging: ");
1274 7 : print_generic_expr (dump_file, lp->addr);
1275 7 : fprintf (dump_file, "\n");
1276 : }
1277 :
1278 24 : if (lp->save_var)
1279 : {
1280 19 : if (dump_file)
1281 4 : fprintf (dump_file, "DUMPING to variable\n");
1282 19 : continue;
1283 : }
1284 : else
1285 : {
1286 5 : if (dump_file)
1287 3 : fprintf (dump_file, "DUMPING with logging functions\n");
1288 34 : for (i = 0; lp->stmts.iterate (i, &stmt); ++i)
1289 5 : tm_log_emit_stmt (lp->addr, stmt);
1290 : }
1291 : }
1292 474 : }
1293 :
1294 : /* Emit the save sequence for the corresponding addresses in the log.
1295 : ENTRY_BLOCK is the entry block for the transaction.
1296 : BB is the basic block to insert the code in. */
1297 : static void
1298 18 : tm_log_emit_saves (basic_block entry_block, basic_block bb)
1299 : {
1300 18 : size_t i;
1301 18 : gimple_stmt_iterator gsi = gsi_last_bb (bb);
1302 18 : gimple *stmt;
1303 18 : struct tm_log_entry l, *lp;
1304 :
1305 36 : for (i = 0; i < tm_log_save_addresses.length (); ++i)
1306 : {
1307 18 : l.addr = tm_log_save_addresses[i];
1308 18 : lp = *(tm_log->find_slot (&l, NO_INSERT));
1309 18 : gcc_assert (lp->save_var != NULL);
1310 :
1311 : /* We only care about variables in the current transaction. */
1312 18 : if (lp->entry_block != entry_block)
1313 6 : continue;
1314 :
1315 12 : stmt = gimple_build_assign (lp->save_var, unshare_expr (lp->addr));
1316 :
1317 : /* Make sure we can create an SSA_NAME for this type. For
1318 : instance, aggregates aren't allowed, in which case the system
1319 : will create a VOP for us and everything will just work. */
1320 12 : if (is_gimple_reg_type (TREE_TYPE (lp->save_var)))
1321 : {
1322 11 : lp->save_var = make_ssa_name (lp->save_var, stmt);
1323 11 : gimple_assign_set_lhs (stmt, lp->save_var);
1324 : }
1325 :
1326 12 : gsi_insert_before (&gsi, stmt, GSI_SAME_STMT);
1327 : }
1328 18 : }
1329 :
1330 : /* Emit the restore sequence for the corresponding addresses in the log.
1331 : ENTRY_BLOCK is the entry block for the transaction.
1332 : BB is the basic block to insert the code in. */
1333 : static void
1334 18 : tm_log_emit_restores (basic_block entry_block, basic_block bb)
1335 : {
1336 18 : int i;
1337 18 : struct tm_log_entry l, *lp;
1338 18 : gimple_stmt_iterator gsi;
1339 18 : gimple *stmt;
1340 :
1341 54 : for (i = tm_log_save_addresses.length () - 1; i >= 0; i--)
1342 : {
1343 18 : l.addr = tm_log_save_addresses[i];
1344 18 : lp = *(tm_log->find_slot (&l, NO_INSERT));
1345 18 : gcc_assert (lp->save_var != NULL);
1346 :
1347 : /* We only care about variables in the current transaction. */
1348 18 : if (lp->entry_block != entry_block)
1349 6 : continue;
1350 :
1351 : /* Restores are in LIFO order from the saves in case we have
1352 : overlaps. */
1353 12 : gsi = gsi_start_bb (bb);
1354 :
1355 12 : stmt = gimple_build_assign (unshare_expr (lp->addr), lp->save_var);
1356 12 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
1357 : }
1358 18 : }
1359 :
1360 :
1361 : static tree lower_sequence_tm (gimple_stmt_iterator *, bool *,
1362 : struct walk_stmt_info *);
1363 : static tree lower_sequence_no_tm (gimple_stmt_iterator *, bool *,
1364 : struct walk_stmt_info *);
1365 :
1366 : /* Evaluate an address X being dereferenced and determine if it
1367 : originally points to a non aliased new chunk of memory (malloc,
1368 : alloca, etc).
1369 :
1370 : Return MEM_THREAD_LOCAL if it points to a thread-local address.
1371 : Return MEM_TRANSACTION_LOCAL if it points to a transaction-local address.
1372 : Return MEM_NON_LOCAL otherwise.
1373 :
1374 : ENTRY_BLOCK is the entry block to the transaction containing the
1375 : dereference of X. */
1376 : static enum thread_memory_type
1377 483 : thread_private_new_memory (basic_block entry_block, tree x)
1378 : {
1379 483 : gimple *stmt = NULL;
1380 483 : enum tree_code code;
1381 483 : tm_new_mem_map **slot;
1382 483 : tm_new_mem_map elt, *elt_p;
1383 483 : tree val = x;
1384 483 : enum thread_memory_type retval = mem_transaction_local;
1385 :
1386 483 : if (!entry_block
1387 337 : || TREE_CODE (x) != SSA_NAME
1388 : /* Possible uninitialized use, or a function argument. In
1389 : either case, we don't care. */
1390 817 : || SSA_NAME_IS_DEFAULT_DEF (x))
1391 : return mem_non_local;
1392 :
1393 : /* Look in cache first. */
1394 161 : elt.val = x;
1395 161 : slot = tm_new_mem_hash->find_slot (&elt, INSERT);
1396 161 : elt_p = *slot;
1397 161 : if (elt_p)
1398 40 : return elt_p->local_new_memory;
1399 :
1400 : /* Optimistically assume the memory is transaction local during
1401 : processing. This catches recursion into this variable. */
1402 121 : *slot = elt_p = XNEW (tm_new_mem_map);
1403 121 : elt_p->val = val;
1404 121 : elt_p->local_new_memory = mem_transaction_local;
1405 :
1406 : /* Search DEF chain to find the original definition of this address. */
1407 122 : do
1408 : {
1409 122 : if (ptr_deref_may_alias_global_p (x, true))
1410 : {
1411 : /* Address escapes. This is not thread-private. */
1412 108 : retval = mem_non_local;
1413 108 : goto new_memory_ret;
1414 : }
1415 :
1416 14 : stmt = SSA_NAME_DEF_STMT (x);
1417 :
1418 : /* If the malloc call is outside the transaction, this is
1419 : thread-local. */
1420 14 : if (retval != mem_thread_local
1421 14 : && !dominated_by_p (CDI_DOMINATORS, gimple_bb (stmt), entry_block))
1422 : retval = mem_thread_local;
1423 :
1424 14 : if (is_gimple_assign (stmt))
1425 : {
1426 2 : code = gimple_assign_rhs_code (stmt);
1427 : /* x = foo ==> foo */
1428 2 : if (code == SSA_NAME)
1429 0 : x = gimple_assign_rhs1 (stmt);
1430 : /* x = foo + n ==> foo */
1431 2 : else if (code == POINTER_PLUS_EXPR)
1432 1 : x = gimple_assign_rhs1 (stmt);
1433 : /* x = (cast*) foo ==> foo */
1434 1 : else if (code == VIEW_CONVERT_EXPR || CONVERT_EXPR_CODE_P (code))
1435 0 : x = gimple_assign_rhs1 (stmt);
1436 : /* x = c ? op1 : op2 == > op1 or op2 just like a PHI */
1437 1 : else if (code == COND_EXPR)
1438 : {
1439 1 : tree op1 = gimple_assign_rhs2 (stmt);
1440 1 : tree op2 = gimple_assign_rhs3 (stmt);
1441 1 : enum thread_memory_type mem;
1442 1 : retval = thread_private_new_memory (entry_block, op1);
1443 1 : if (retval == mem_non_local)
1444 0 : goto new_memory_ret;
1445 1 : mem = thread_private_new_memory (entry_block, op2);
1446 1 : retval = MIN (retval, mem);
1447 1 : goto new_memory_ret;
1448 : }
1449 : else
1450 : {
1451 0 : retval = mem_non_local;
1452 0 : goto new_memory_ret;
1453 : }
1454 : }
1455 : else
1456 : {
1457 12 : if (gimple_code (stmt) == GIMPLE_PHI)
1458 : {
1459 4 : unsigned int i;
1460 4 : enum thread_memory_type mem;
1461 4 : tree phi_result = gimple_phi_result (stmt);
1462 :
1463 : /* If any of the ancestors are non-local, we are sure to
1464 : be non-local. Otherwise we can avoid doing anything
1465 : and inherit what has already been generated. */
1466 4 : retval = mem_max;
1467 7 : for (i = 0; i < gimple_phi_num_args (stmt); ++i)
1468 : {
1469 6 : tree op = PHI_ARG_DEF (stmt, i);
1470 :
1471 : /* Exclude self-assignment. */
1472 6 : if (phi_result == op)
1473 0 : continue;
1474 :
1475 6 : mem = thread_private_new_memory (entry_block, op);
1476 6 : if (mem == mem_non_local)
1477 : {
1478 3 : retval = mem;
1479 3 : goto new_memory_ret;
1480 : }
1481 4 : retval = MIN (retval, mem);
1482 : }
1483 1 : goto new_memory_ret;
1484 : }
1485 : break;
1486 : }
1487 : }
1488 1 : while (TREE_CODE (x) == SSA_NAME);
1489 :
1490 8 : if (stmt && is_gimple_call (stmt) && gimple_call_flags (stmt) & ECF_MALLOC)
1491 : /* Thread-local or transaction-local. */
1492 : ;
1493 : else
1494 : retval = mem_non_local;
1495 :
1496 121 : new_memory_ret:
1497 121 : elt_p->local_new_memory = retval;
1498 121 : return retval;
1499 : }
1500 :
1501 : /* Determine whether X has to be instrumented using a read
1502 : or write barrier.
1503 :
1504 : ENTRY_BLOCK is the entry block for the region where stmt resides
1505 : in. NULL if unknown.
1506 :
1507 : STMT is the statement in which X occurs in. It is used for thread
1508 : private memory instrumentation. If no TPM instrumentation is
1509 : desired, STMT should be null. */
1510 : static bool
1511 4009 : requires_barrier (basic_block entry_block, tree x, gimple *stmt)
1512 : {
1513 4009 : tree orig = x;
1514 4338 : while (handled_component_p (x))
1515 329 : x = TREE_OPERAND (x, 0);
1516 :
1517 4009 : switch (TREE_CODE (x))
1518 : {
1519 475 : case INDIRECT_REF:
1520 475 : case MEM_REF:
1521 475 : {
1522 475 : enum thread_memory_type ret;
1523 :
1524 475 : ret = thread_private_new_memory (entry_block, TREE_OPERAND (x, 0));
1525 475 : if (ret == mem_non_local)
1526 : return true;
1527 20 : if (stmt && ret == mem_thread_local)
1528 : /* ?? Should we pass `orig', or the INDIRECT_REF X. ?? */
1529 7 : tm_log_add (entry_block, orig, stmt);
1530 :
1531 : /* Transaction-locals require nothing at all. For malloc, a
1532 : transaction restart frees the memory and we reallocate.
1533 : For alloca, the stack pointer gets reset by the retry and
1534 : we reallocate. */
1535 : return false;
1536 : }
1537 :
1538 12 : case TARGET_MEM_REF:
1539 12 : if (TREE_CODE (TMR_BASE (x)) != ADDR_EXPR)
1540 : return true;
1541 1 : x = TREE_OPERAND (TMR_BASE (x), 0);
1542 1 : if (TREE_CODE (x) == PARM_DECL)
1543 : return false;
1544 1 : gcc_assert (VAR_P (x));
1545 : /* FALLTHRU */
1546 :
1547 1567 : case PARM_DECL:
1548 1567 : case RESULT_DECL:
1549 1567 : case VAR_DECL:
1550 1567 : if (DECL_BY_REFERENCE (x))
1551 : {
1552 : /* ??? This value is a pointer, but aggregate_value_p has been
1553 : jigged to return true which confuses needs_to_live_in_memory.
1554 : This ought to be cleaned up generically.
1555 :
1556 : FIXME: Verify this still happens after the next mainline
1557 : merge. Testcase ie g++.dg/tm/pr47554.C.
1558 : */
1559 : return false;
1560 : }
1561 :
1562 1567 : if (is_global_var (x))
1563 655 : return !TREE_READONLY (x);
1564 912 : if (/* FIXME: This condition should actually go below in the
1565 : tm_log_add() call, however is_call_clobbered() depends on
1566 : aliasing info which is not available during
1567 : gimplification. Since requires_barrier() gets called
1568 : during lower_sequence_tm/gimplification, leave the call
1569 : to needs_to_live_in_memory until we eliminate
1570 : lower_sequence_tm altogether. */
1571 912 : needs_to_live_in_memory (x))
1572 : return true;
1573 : else
1574 : {
1575 : /* For local memory that doesn't escape (aka thread private
1576 : memory), we can either save the value at the beginning of
1577 : the transaction and restore on restart, or call a tm
1578 : function to dynamically save and restore on restart
1579 : (ITM_L*). */
1580 807 : if (stmt)
1581 17 : tm_log_add (entry_block, orig, stmt);
1582 807 : return false;
1583 : }
1584 :
1585 : default:
1586 : return false;
1587 : }
1588 : }
1589 :
1590 : /* Mark the GIMPLE_ASSIGN statement as appropriate for being inside
1591 : a transaction region. */
1592 :
1593 : static void
1594 767 : examine_assign_tm (unsigned *state, gimple_stmt_iterator *gsi)
1595 : {
1596 767 : gimple *stmt = gsi_stmt (*gsi);
1597 :
1598 767 : if (requires_barrier (/*entry_block=*/NULL, gimple_assign_rhs1 (stmt), NULL))
1599 303 : *state |= GTMA_HAVE_LOAD;
1600 767 : if (requires_barrier (/*entry_block=*/NULL, gimple_assign_lhs (stmt), NULL))
1601 258 : *state |= GTMA_HAVE_STORE;
1602 767 : }
1603 :
1604 : /* Mark a GIMPLE_CALL as appropriate for being inside a transaction. */
1605 :
1606 : static void
1607 519 : examine_call_tm (unsigned *state, gimple_stmt_iterator *gsi)
1608 : {
1609 519 : gimple *stmt = gsi_stmt (*gsi);
1610 519 : tree fn;
1611 :
1612 519 : if (is_tm_pure_call (stmt))
1613 : return;
1614 :
1615 : /* Check if this call is a transaction abort. */
1616 474 : fn = gimple_call_fndecl (stmt);
1617 941 : if (is_tm_abort (fn))
1618 29 : *state |= GTMA_HAVE_ABORT;
1619 :
1620 : /* Note that something may happen. */
1621 474 : *state |= GTMA_HAVE_LOAD | GTMA_HAVE_STORE;
1622 : }
1623 :
1624 : /* Iterate through the statements in the sequence, moving labels
1625 : (and thus edges) of transactions from "label_norm" to "label_uninst". */
1626 :
1627 : static tree
1628 3699 : make_tm_uninst (gimple_stmt_iterator *gsi, bool *handled_ops_p,
1629 : struct walk_stmt_info *)
1630 : {
1631 3699 : gimple *stmt = gsi_stmt (*gsi);
1632 :
1633 3699 : if (gtransaction *txn = dyn_cast <gtransaction *> (stmt))
1634 : {
1635 6 : *handled_ops_p = true;
1636 6 : txn->label_uninst = txn->label_norm;
1637 6 : txn->label_norm = NULL;
1638 : }
1639 : else
1640 3693 : *handled_ops_p = !gimple_has_substatements (stmt);
1641 :
1642 3699 : return NULL_TREE;
1643 : }
1644 :
1645 : /* Lower a GIMPLE_TRANSACTION statement. */
1646 :
1647 : static void
1648 474 : lower_transaction (gimple_stmt_iterator *gsi, struct walk_stmt_info *wi)
1649 : {
1650 474 : gimple *g;
1651 474 : gtransaction *stmt = as_a <gtransaction *> (gsi_stmt (*gsi));
1652 474 : unsigned int *outer_state = (unsigned int *) wi->info;
1653 474 : unsigned int this_state = 0;
1654 474 : struct walk_stmt_info this_wi;
1655 :
1656 : /* First, lower the body. The scanning that we do inside gives
1657 : us some idea of what we're dealing with. */
1658 474 : memset (&this_wi, 0, sizeof (this_wi));
1659 474 : this_wi.info = (void *) &this_state;
1660 474 : walk_gimple_seq_mod (gimple_transaction_body_ptr (stmt),
1661 : lower_sequence_tm, NULL, &this_wi);
1662 :
1663 : /* If there was absolutely nothing transaction related inside the
1664 : transaction, we may elide it. Likewise if this is a nested
1665 : transaction and does not contain an abort. */
1666 474 : if (this_state == 0
1667 430 : || (!(this_state & GTMA_HAVE_ABORT) && outer_state != NULL))
1668 : {
1669 44 : if (outer_state)
1670 29 : *outer_state |= this_state;
1671 :
1672 73 : gsi_insert_seq_before (gsi, gimple_transaction_body (stmt),
1673 : GSI_SAME_STMT);
1674 73 : gimple_transaction_set_body (stmt, NULL);
1675 :
1676 73 : gsi_remove (gsi, true);
1677 73 : wi->removed_stmt = true;
1678 73 : return;
1679 : }
1680 :
1681 : /* Wrap the body of the transaction in a try-finally node so that
1682 : the commit call is always properly called. */
1683 401 : g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT), 0);
1684 401 : if (flag_exceptions)
1685 : {
1686 276 : tree ptr;
1687 276 : gimple_seq n_seq, e_seq;
1688 :
1689 276 : n_seq = gimple_seq_alloc_with_stmt (g);
1690 276 : e_seq = NULL;
1691 :
1692 276 : g = gimple_build_call (builtin_decl_explicit (BUILT_IN_EH_POINTER),
1693 : 1, integer_zero_node);
1694 276 : ptr = create_tmp_var (ptr_type_node);
1695 276 : gimple_call_set_lhs (g, ptr);
1696 276 : gimple_seq_add_stmt (&e_seq, g);
1697 :
1698 276 : g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_COMMIT_EH),
1699 : 1, ptr);
1700 276 : gimple_seq_add_stmt (&e_seq, g);
1701 :
1702 276 : g = gimple_build_eh_else (n_seq, e_seq);
1703 : }
1704 :
1705 401 : g = gimple_build_try (gimple_transaction_body (stmt),
1706 : gimple_seq_alloc_with_stmt (g), GIMPLE_TRY_FINALLY);
1707 :
1708 : /* For a (potentially) outer transaction, create two paths. */
1709 401 : gimple_seq uninst = NULL;
1710 401 : if (outer_state == NULL)
1711 : {
1712 391 : uninst = copy_gimple_seq_and_replace_locals (g);
1713 : /* In the uninstrumented copy, reset inner transactions to have only
1714 : an uninstrumented code path. */
1715 391 : memset (&this_wi, 0, sizeof (this_wi));
1716 391 : walk_gimple_seq (uninst, make_tm_uninst, NULL, &this_wi);
1717 : }
1718 :
1719 401 : tree label1 = create_artificial_label (UNKNOWN_LOCATION);
1720 401 : gsi_insert_after (gsi, gimple_build_label (label1), GSI_CONTINUE_LINKING);
1721 401 : gsi_insert_after (gsi, g, GSI_CONTINUE_LINKING);
1722 401 : gimple_transaction_set_label_norm (stmt, label1);
1723 :
1724 : /* If the transaction calls abort or if this is an outer transaction,
1725 : add an "over" label afterwards. */
1726 401 : tree label3 = NULL;
1727 401 : if ((this_state & GTMA_HAVE_ABORT)
1728 372 : || outer_state == NULL
1729 401 : || (gimple_transaction_subcode (stmt) & GTMA_IS_OUTER))
1730 : {
1731 401 : label3 = create_artificial_label (UNKNOWN_LOCATION);
1732 401 : gimple_transaction_set_label_over (stmt, label3);
1733 : }
1734 :
1735 401 : if (uninst != NULL)
1736 : {
1737 391 : gsi_insert_after (gsi, gimple_build_goto (label3), GSI_CONTINUE_LINKING);
1738 :
1739 391 : tree label2 = create_artificial_label (UNKNOWN_LOCATION);
1740 391 : gsi_insert_after (gsi, gimple_build_label (label2), GSI_CONTINUE_LINKING);
1741 391 : gsi_insert_seq_after (gsi, uninst, GSI_CONTINUE_LINKING);
1742 391 : gimple_transaction_set_label_uninst (stmt, label2);
1743 : }
1744 :
1745 401 : if (label3 != NULL)
1746 401 : gsi_insert_after (gsi, gimple_build_label (label3), GSI_CONTINUE_LINKING);
1747 :
1748 401 : gimple_transaction_set_body (stmt, NULL);
1749 :
1750 : /* Record the set of operations found for use later. */
1751 401 : this_state |= gimple_transaction_subcode (stmt) & GTMA_DECLARATION_MASK;
1752 401 : gimple_transaction_set_subcode (stmt, this_state);
1753 : }
1754 :
1755 : /* Iterate through the statements in the sequence, lowering them all
1756 : as appropriate for being in a transaction. */
1757 :
1758 : static tree
1759 2204 : lower_sequence_tm (gimple_stmt_iterator *gsi, bool *handled_ops_p,
1760 : struct walk_stmt_info *wi)
1761 : {
1762 2204 : unsigned int *state = (unsigned int *) wi->info;
1763 2204 : gimple *stmt = gsi_stmt (*gsi);
1764 :
1765 2204 : *handled_ops_p = true;
1766 2204 : switch (gimple_code (stmt))
1767 : {
1768 1015 : case GIMPLE_ASSIGN:
1769 : /* Only memory reads/writes need to be instrumented. */
1770 1015 : if (gimple_assign_single_p (stmt))
1771 767 : examine_assign_tm (state, gsi);
1772 : break;
1773 :
1774 519 : case GIMPLE_CALL:
1775 519 : examine_call_tm (state, gsi);
1776 519 : break;
1777 :
1778 2 : case GIMPLE_ASM:
1779 2 : *state |= GTMA_MAY_ENTER_IRREVOCABLE;
1780 2 : break;
1781 :
1782 39 : case GIMPLE_TRANSACTION:
1783 39 : lower_transaction (gsi, wi);
1784 39 : break;
1785 :
1786 629 : default:
1787 629 : *handled_ops_p = !gimple_has_substatements (stmt);
1788 629 : break;
1789 : }
1790 :
1791 2204 : return NULL_TREE;
1792 : }
1793 :
1794 : /* Iterate through the statements in the sequence, lowering them all
1795 : as appropriate for being outside of a transaction. */
1796 :
1797 : static tree
1798 8769 : lower_sequence_no_tm (gimple_stmt_iterator *gsi, bool *handled_ops_p,
1799 : struct walk_stmt_info * wi)
1800 : {
1801 8769 : gimple *stmt = gsi_stmt (*gsi);
1802 :
1803 8769 : if (gimple_code (stmt) == GIMPLE_TRANSACTION)
1804 : {
1805 435 : *handled_ops_p = true;
1806 435 : lower_transaction (gsi, wi);
1807 : }
1808 : else
1809 8334 : *handled_ops_p = !gimple_has_substatements (stmt);
1810 :
1811 8769 : return NULL_TREE;
1812 : }
1813 :
1814 : /* Main entry point for flattening GIMPLE_TRANSACTION constructs. After
1815 : this, GIMPLE_TRANSACTION nodes still exist, but the nested body has
1816 : been moved out, and all the data required for constructing a proper
1817 : CFG has been recorded. */
1818 :
1819 : static unsigned int
1820 1288 : execute_lower_tm (void)
1821 : {
1822 1288 : struct walk_stmt_info wi;
1823 1288 : gimple_seq body;
1824 :
1825 : /* Transactional clones aren't created until a later pass. */
1826 2576 : gcc_assert (!decl_is_tm_clone (current_function_decl));
1827 :
1828 1288 : body = gimple_body (current_function_decl);
1829 1288 : memset (&wi, 0, sizeof (wi));
1830 1288 : walk_gimple_seq_mod (&body, lower_sequence_no_tm, NULL, &wi);
1831 1288 : gimple_set_body (current_function_decl, body);
1832 :
1833 1288 : return 0;
1834 : }
1835 :
1836 : namespace {
1837 :
1838 : const pass_data pass_data_lower_tm =
1839 : {
1840 : GIMPLE_PASS, /* type */
1841 : "tmlower", /* name */
1842 : OPTGROUP_NONE, /* optinfo_flags */
1843 : TV_TRANS_MEM, /* tv_id */
1844 : PROP_gimple_lcf, /* properties_required */
1845 : 0, /* properties_provided */
1846 : 0, /* properties_destroyed */
1847 : 0, /* todo_flags_start */
1848 : 0, /* todo_flags_finish */
1849 : };
1850 :
1851 : class pass_lower_tm : public gimple_opt_pass
1852 : {
1853 : public:
1854 285722 : pass_lower_tm (gcc::context *ctxt)
1855 571444 : : gimple_opt_pass (pass_data_lower_tm, ctxt)
1856 : {}
1857 :
1858 : /* opt_pass methods: */
1859 2869205 : bool gate (function *) final override { return flag_tm; }
1860 1288 : unsigned int execute (function *) final override
1861 : {
1862 1288 : return execute_lower_tm ();
1863 : }
1864 :
1865 : }; // class pass_lower_tm
1866 :
1867 : } // anon namespace
1868 :
1869 : gimple_opt_pass *
1870 285722 : make_pass_lower_tm (gcc::context *ctxt)
1871 : {
1872 285722 : return new pass_lower_tm (ctxt);
1873 : }
1874 :
1875 : /* Collect region information for each transaction. */
1876 :
1877 : struct tm_region
1878 : {
1879 : public:
1880 :
1881 : /* The field "transaction_stmt" is initially a gtransaction *,
1882 : but eventually gets lowered to a gcall *(to BUILT_IN_TM_START).
1883 :
1884 : Helper method to get it as a gtransaction *, with code-checking
1885 : in a checked-build. */
1886 :
1887 : gtransaction *
1888 2502 : get_transaction_stmt () const
1889 : {
1890 2502 : return as_a <gtransaction *> (transaction_stmt);
1891 : }
1892 :
1893 : public:
1894 :
1895 : /* Link to the next unnested transaction. */
1896 : struct tm_region *next;
1897 :
1898 : /* Link to the next inner transaction. */
1899 : struct tm_region *inner;
1900 :
1901 : /* Link to the next outer transaction. */
1902 : struct tm_region *outer;
1903 :
1904 : /* The GIMPLE_TRANSACTION statement beginning this transaction.
1905 : After TM_MARK, this gets replaced by a call to
1906 : BUILT_IN_TM_START.
1907 : Hence this will be either a gtransaction *or a gcall *. */
1908 : gimple *transaction_stmt;
1909 :
1910 : /* After TM_MARK expands the GIMPLE_TRANSACTION into a call to
1911 : BUILT_IN_TM_START, this field is true if the transaction is an
1912 : outer transaction. */
1913 : bool original_transaction_was_outer;
1914 :
1915 : /* Return value from BUILT_IN_TM_START. */
1916 : tree tm_state;
1917 :
1918 : /* The entry block to this region. This will always be the first
1919 : block of the body of the transaction. */
1920 : basic_block entry_block;
1921 :
1922 : /* The first block after an expanded call to _ITM_beginTransaction. */
1923 : basic_block restart_block;
1924 :
1925 : /* The set of all blocks that end the region; NULL if only EXIT_BLOCK.
1926 : These blocks are still a part of the region (i.e., the border is
1927 : inclusive). Note that this set is only complete for paths in the CFG
1928 : starting at ENTRY_BLOCK, and that there is no exit block recorded for
1929 : the edge to the "over" label. */
1930 : bitmap exit_blocks;
1931 :
1932 : /* The set of all blocks that have an TM_IRREVOCABLE call. */
1933 : bitmap irr_blocks;
1934 : };
1935 :
1936 : /* True if there are pending edge statements to be committed for the
1937 : current function being scanned in the tmmark pass. */
1938 : bool pending_edge_inserts_p;
1939 :
1940 : static struct tm_region *all_tm_regions;
1941 : static bitmap_obstack tm_obstack;
1942 :
1943 :
1944 : /* A subroutine of tm_region_init. Record the existence of the
1945 : GIMPLE_TRANSACTION statement in a tree of tm_region elements. */
1946 :
1947 : static struct tm_region *
1948 763 : tm_region_init_0 (struct tm_region *outer, basic_block bb,
1949 : gtransaction *stmt)
1950 : {
1951 763 : struct tm_region *region;
1952 :
1953 1526 : region = (struct tm_region *)
1954 763 : obstack_alloc (&tm_obstack.obstack, sizeof (struct tm_region));
1955 :
1956 763 : if (outer)
1957 : {
1958 35 : region->next = outer->inner;
1959 35 : outer->inner = region;
1960 : }
1961 : else
1962 : {
1963 728 : region->next = all_tm_regions;
1964 728 : all_tm_regions = region;
1965 : }
1966 763 : region->inner = NULL;
1967 763 : region->outer = outer;
1968 :
1969 763 : region->transaction_stmt = stmt;
1970 763 : region->original_transaction_was_outer = false;
1971 763 : region->tm_state = NULL;
1972 :
1973 : /* There are either one or two edges out of the block containing
1974 : the GIMPLE_TRANSACTION, one to the actual region and one to the
1975 : "over" label if the region contains an abort. The former will
1976 : always be the one marked FALLTHRU. */
1977 763 : region->entry_block = FALLTHRU_EDGE (bb)->dest;
1978 :
1979 763 : region->exit_blocks = BITMAP_ALLOC (&tm_obstack);
1980 763 : region->irr_blocks = BITMAP_ALLOC (&tm_obstack);
1981 :
1982 763 : return region;
1983 : }
1984 :
1985 : /* A subroutine of tm_region_init. Record all the exit and
1986 : irrevocable blocks in BB into the region's exit_blocks and
1987 : irr_blocks bitmaps. Returns the new region being scanned. */
1988 :
1989 : static struct tm_region *
1990 11681 : tm_region_init_1 (struct tm_region *region, basic_block bb)
1991 : {
1992 11681 : gimple_stmt_iterator gsi;
1993 11681 : gimple *g;
1994 :
1995 11681 : if (!region
1996 2775 : || (!region->irr_blocks && !region->exit_blocks))
1997 : return region;
1998 :
1999 : /* Check to see if this is the end of a region by seeing if it
2000 : contains a call to __builtin_tm_commit{,_eh}. Note that the
2001 : outermost region for DECL_IS_TM_CLONE need not collect this. */
2002 13226 : for (gsi = gsi_last_bb (bb); !gsi_end_p (gsi); gsi_prev (&gsi))
2003 : {
2004 4703 : g = gsi_stmt (gsi);
2005 4703 : if (gimple_code (g) == GIMPLE_CALL)
2006 : {
2007 1737 : tree fn = gimple_call_fndecl (g);
2008 1737 : if (fn && fndecl_built_in_p (fn, BUILT_IN_NORMAL))
2009 : {
2010 1088 : if ((DECL_FUNCTION_CODE (fn) == BUILT_IN_TM_COMMIT
2011 397 : || DECL_FUNCTION_CODE (fn) == BUILT_IN_TM_COMMIT_EH)
2012 1287 : && region->exit_blocks)
2013 : {
2014 865 : bitmap_set_bit (region->exit_blocks, bb->index);
2015 865 : region = region->outer;
2016 865 : break;
2017 : }
2018 223 : if (DECL_FUNCTION_CODE (fn) == BUILT_IN_TM_IRREVOCABLE)
2019 38 : bitmap_set_bit (region->irr_blocks, bb->index);
2020 : }
2021 : }
2022 : }
2023 : return region;
2024 : }
2025 :
2026 : /* Collect all of the transaction regions within the current function
2027 : and record them in ALL_TM_REGIONS. The REGION parameter may specify
2028 : an "outermost" region for use by tm clones. */
2029 :
2030 : static void
2031 2093 : tm_region_init (struct tm_region *region)
2032 : {
2033 2093 : gimple *g;
2034 2093 : edge_iterator ei;
2035 2093 : edge e;
2036 2093 : basic_block bb;
2037 2093 : auto_vec<basic_block> queue;
2038 2093 : bitmap visited_blocks = BITMAP_ALLOC (NULL);
2039 2093 : struct tm_region *old_region;
2040 2093 : auto_vec<tm_region *> bb_regions;
2041 :
2042 : /* We could store this information in bb->aux, but we may get called
2043 : through get_all_tm_blocks() from another pass that may be already
2044 : using bb->aux. */
2045 2093 : bb_regions.safe_grow_cleared (last_basic_block_for_fn (cfun), true);
2046 :
2047 2093 : all_tm_regions = region;
2048 2093 : bb = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
2049 2093 : queue.safe_push (bb);
2050 2093 : bitmap_set_bit (visited_blocks, bb->index);
2051 2093 : bb_regions[bb->index] = region;
2052 :
2053 11681 : do
2054 : {
2055 11681 : bb = queue.pop ();
2056 11681 : region = bb_regions[bb->index];
2057 11681 : bb_regions[bb->index] = NULL;
2058 :
2059 : /* Record exit and irrevocable blocks. */
2060 11681 : region = tm_region_init_1 (region, bb);
2061 :
2062 : /* Check for the last statement in the block beginning a new region. */
2063 11681 : g = last_nondebug_stmt (bb);
2064 11681 : old_region = region;
2065 11681 : if (g)
2066 8702 : if (gtransaction *trans_stmt = dyn_cast <gtransaction *> (g))
2067 763 : region = tm_region_init_0 (region, bb, trans_stmt);
2068 :
2069 : /* Process subsequent blocks. */
2070 23395 : FOR_EACH_EDGE (e, ei, bb->succs)
2071 11714 : if (!bitmap_bit_p (visited_blocks, e->dest->index))
2072 : {
2073 9588 : bitmap_set_bit (visited_blocks, e->dest->index);
2074 9588 : queue.safe_push (e->dest);
2075 :
2076 : /* If the current block started a new region, make sure that only
2077 : the entry block of the new region is associated with this region.
2078 : Other successors are still part of the old region. */
2079 9588 : if (old_region != region && e->dest != region->entry_block)
2080 827 : bb_regions[e->dest->index] = old_region;
2081 : else
2082 8761 : bb_regions[e->dest->index] = region;
2083 : }
2084 : }
2085 23362 : while (!queue.is_empty ());
2086 2093 : BITMAP_FREE (visited_blocks);
2087 2093 : }
2088 :
2089 : /* The "gate" function for all transactional memory expansion and optimization
2090 : passes. We collect region information for each top-level transaction, and
2091 : if we don't find any, we skip all of the TM passes. Each region will have
2092 : all of the exit blocks recorded, and the originating statement. */
2093 :
2094 : static bool
2095 1472254 : gate_tm_init (void)
2096 : {
2097 1472254 : if (!flag_tm)
2098 : return false;
2099 :
2100 1162 : calculate_dominance_info (CDI_DOMINATORS);
2101 1162 : bitmap_obstack_initialize (&tm_obstack);
2102 :
2103 : /* If the function is a TM_CLONE, then the entire function is the region. */
2104 2324 : if (decl_is_tm_clone (current_function_decl))
2105 : {
2106 224 : struct tm_region *region = (struct tm_region *)
2107 224 : obstack_alloc (&tm_obstack.obstack, sizeof (struct tm_region));
2108 224 : memset (region, 0, sizeof (*region));
2109 224 : region->entry_block = single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun));
2110 : /* For a clone, the entire function is the region. But even if
2111 : we don't need to record any exit blocks, we may need to
2112 : record irrevocable blocks. */
2113 224 : region->irr_blocks = BITMAP_ALLOC (&tm_obstack);
2114 :
2115 224 : tm_region_init (region);
2116 : }
2117 : else
2118 : {
2119 938 : tm_region_init (NULL);
2120 :
2121 : /* If we didn't find any regions, cleanup and skip the whole tree
2122 : of tm-related optimizations. */
2123 938 : if (all_tm_regions == NULL)
2124 : {
2125 632 : bitmap_obstack_release (&tm_obstack);
2126 632 : return false;
2127 : }
2128 : }
2129 :
2130 : return true;
2131 : }
2132 :
2133 : namespace {
2134 :
2135 : const pass_data pass_data_tm_init =
2136 : {
2137 : GIMPLE_PASS, /* type */
2138 : "*tminit", /* name */
2139 : OPTGROUP_NONE, /* optinfo_flags */
2140 : TV_TRANS_MEM, /* tv_id */
2141 : ( PROP_ssa | PROP_cfg ), /* properties_required */
2142 : 0, /* properties_provided */
2143 : 0, /* properties_destroyed */
2144 : 0, /* todo_flags_start */
2145 : 0, /* todo_flags_finish */
2146 : };
2147 :
2148 : class pass_tm_init : public gimple_opt_pass
2149 : {
2150 : public:
2151 285722 : pass_tm_init (gcc::context *ctxt)
2152 571444 : : gimple_opt_pass (pass_data_tm_init, ctxt)
2153 : {}
2154 :
2155 : /* opt_pass methods: */
2156 1472150 : bool gate (function *) final override { return gate_tm_init (); }
2157 :
2158 : }; // class pass_tm_init
2159 :
2160 : } // anon namespace
2161 :
2162 : gimple_opt_pass *
2163 285722 : make_pass_tm_init (gcc::context *ctxt)
2164 : {
2165 285722 : return new pass_tm_init (ctxt);
2166 : }
2167 :
2168 : /* Add FLAGS to the GIMPLE_TRANSACTION subcode for the transaction region
2169 : represented by STATE. */
2170 :
2171 : static inline void
2172 1086 : transaction_subcode_ior (struct tm_region *region, unsigned flags)
2173 : {
2174 1086 : if (region && region->transaction_stmt)
2175 : {
2176 666 : gtransaction *transaction_stmt = region->get_transaction_stmt ();
2177 666 : flags |= gimple_transaction_subcode (transaction_stmt);
2178 666 : gimple_transaction_set_subcode (transaction_stmt, flags);
2179 : }
2180 1086 : }
2181 :
2182 : /* Construct a memory load in a transactional context. Return the
2183 : gimple statement performing the load, or NULL if there is no
2184 : TM_LOAD builtin of the appropriate size to do the load.
2185 :
2186 : LOC is the location to use for the new statement(s). */
2187 :
2188 : static gcall *
2189 378 : build_tm_load (location_t loc, tree lhs, tree rhs, gimple_stmt_iterator *gsi)
2190 : {
2191 378 : tree t, type = TREE_TYPE (rhs);
2192 378 : gcall *gcall;
2193 :
2194 378 : built_in_function code;
2195 378 : if (type == float_type_node)
2196 : code = BUILT_IN_TM_LOAD_FLOAT;
2197 377 : else if (type == double_type_node)
2198 : code = BUILT_IN_TM_LOAD_DOUBLE;
2199 375 : else if (type == long_double_type_node)
2200 : code = BUILT_IN_TM_LOAD_LDOUBLE;
2201 : else
2202 : {
2203 374 : if (TYPE_SIZE (type) == NULL || !tree_fits_uhwi_p (TYPE_SIZE (type)))
2204 : return NULL;
2205 374 : unsigned HOST_WIDE_INT type_size = tree_to_uhwi (TYPE_SIZE (type));
2206 :
2207 374 : if (TREE_CODE (type) == VECTOR_TYPE)
2208 : {
2209 5 : switch (type_size)
2210 : {
2211 : case 64:
2212 : code = BUILT_IN_TM_LOAD_M64;
2213 : break;
2214 5 : case 128:
2215 5 : code = BUILT_IN_TM_LOAD_M128;
2216 5 : break;
2217 0 : case 256:
2218 0 : code = BUILT_IN_TM_LOAD_M256;
2219 0 : break;
2220 0 : default:
2221 0 : goto unhandled_vec;
2222 : }
2223 5 : if (!builtin_decl_explicit_p (code))
2224 0 : goto unhandled_vec;
2225 : }
2226 : else
2227 : {
2228 369 : unhandled_vec:
2229 369 : switch (type_size)
2230 : {
2231 : case 8:
2232 : code = BUILT_IN_TM_LOAD_1;
2233 : break;
2234 4 : case 16:
2235 4 : code = BUILT_IN_TM_LOAD_2;
2236 4 : break;
2237 183 : case 32:
2238 183 : code = BUILT_IN_TM_LOAD_4;
2239 183 : break;
2240 172 : case 64:
2241 172 : code = BUILT_IN_TM_LOAD_8;
2242 172 : break;
2243 : default:
2244 : return NULL;
2245 : }
2246 : }
2247 : }
2248 :
2249 376 : tree decl = builtin_decl_explicit (code);
2250 376 : gcc_assert (decl);
2251 :
2252 376 : t = gimplify_addr (gsi, rhs);
2253 376 : gcall = gimple_build_call (decl, 1, t);
2254 376 : gimple_set_location (gcall, loc);
2255 :
2256 376 : t = TREE_TYPE (TREE_TYPE (decl));
2257 376 : if (useless_type_conversion_p (type, t))
2258 : {
2259 14 : gimple_call_set_lhs (gcall, lhs);
2260 14 : gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2261 : }
2262 : else
2263 : {
2264 362 : gimple *g;
2265 362 : tree temp;
2266 :
2267 362 : temp = create_tmp_reg (t);
2268 362 : gimple_call_set_lhs (gcall, temp);
2269 362 : gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2270 :
2271 362 : t = fold_build1 (VIEW_CONVERT_EXPR, type, temp);
2272 362 : g = gimple_build_assign (lhs, t);
2273 362 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
2274 : }
2275 :
2276 : return gcall;
2277 : }
2278 :
2279 :
2280 : /* Similarly for storing TYPE in a transactional context. */
2281 :
2282 : static gcall *
2283 247 : build_tm_store (location_t loc, tree lhs, tree rhs, gimple_stmt_iterator *gsi)
2284 : {
2285 247 : tree t, fn, type = TREE_TYPE (rhs), simple_type;
2286 247 : gcall *gcall;
2287 :
2288 247 : built_in_function code;
2289 247 : if (type == float_type_node)
2290 : code = BUILT_IN_TM_STORE_FLOAT;
2291 245 : else if (type == double_type_node)
2292 : code = BUILT_IN_TM_STORE_DOUBLE;
2293 242 : else if (type == long_double_type_node)
2294 : code = BUILT_IN_TM_STORE_LDOUBLE;
2295 : else
2296 : {
2297 240 : if (TYPE_SIZE (type) == NULL || !tree_fits_uhwi_p (TYPE_SIZE (type)))
2298 : return NULL;
2299 240 : unsigned HOST_WIDE_INT type_size = tree_to_uhwi (TYPE_SIZE (type));
2300 :
2301 240 : if (TREE_CODE (type) == VECTOR_TYPE)
2302 : {
2303 2 : switch (type_size)
2304 : {
2305 : case 64:
2306 : code = BUILT_IN_TM_STORE_M64;
2307 : break;
2308 2 : case 128:
2309 2 : code = BUILT_IN_TM_STORE_M128;
2310 2 : break;
2311 0 : case 256:
2312 0 : code = BUILT_IN_TM_STORE_M256;
2313 0 : break;
2314 0 : default:
2315 0 : goto unhandled_vec;
2316 : }
2317 2 : if (!builtin_decl_explicit_p (code))
2318 0 : goto unhandled_vec;
2319 : }
2320 : else
2321 : {
2322 238 : unhandled_vec:
2323 238 : switch (type_size)
2324 : {
2325 : case 8:
2326 : code = BUILT_IN_TM_STORE_1;
2327 : break;
2328 4 : case 16:
2329 4 : code = BUILT_IN_TM_STORE_2;
2330 4 : break;
2331 137 : case 32:
2332 137 : code = BUILT_IN_TM_STORE_4;
2333 137 : break;
2334 82 : case 64:
2335 82 : code = BUILT_IN_TM_STORE_8;
2336 82 : break;
2337 : default:
2338 : return NULL;
2339 : }
2340 : }
2341 : }
2342 :
2343 240 : fn = builtin_decl_explicit (code);
2344 240 : gcc_assert (fn);
2345 :
2346 240 : simple_type = TREE_VALUE (TREE_CHAIN (TYPE_ARG_TYPES (TREE_TYPE (fn))));
2347 :
2348 240 : if (TREE_CODE (rhs) == CONSTRUCTOR)
2349 : {
2350 : /* Handle the easy initialization to zero. */
2351 0 : if (!CONSTRUCTOR_ELTS (rhs))
2352 0 : rhs = build_int_cst (simple_type, 0);
2353 : else
2354 : {
2355 : /* ...otherwise punt to the caller and probably use
2356 : BUILT_IN_TM_MEMMOVE, because we can't wrap a
2357 : VIEW_CONVERT_EXPR around a CONSTRUCTOR (below) and produce
2358 : valid gimple. */
2359 : return NULL;
2360 : }
2361 : }
2362 240 : else if (!useless_type_conversion_p (simple_type, type))
2363 : {
2364 223 : gimple *g;
2365 223 : tree temp;
2366 :
2367 223 : temp = create_tmp_reg (simple_type);
2368 223 : t = fold_build1 (VIEW_CONVERT_EXPR, simple_type, rhs);
2369 223 : g = gimple_build_assign (temp, t);
2370 223 : gimple_set_location (g, loc);
2371 223 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
2372 :
2373 223 : rhs = temp;
2374 : }
2375 :
2376 240 : t = gimplify_addr (gsi, lhs);
2377 240 : gcall = gimple_build_call (fn, 2, t, rhs);
2378 240 : gimple_set_location (gcall, loc);
2379 240 : gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2380 :
2381 240 : return gcall;
2382 : }
2383 :
2384 :
2385 : /* Expand an assignment statement into transactional builtins. */
2386 :
2387 : static void
2388 883 : expand_assign_tm (struct tm_region *region, gimple_stmt_iterator *gsi)
2389 : {
2390 883 : gimple *stmt = gsi_stmt (*gsi);
2391 883 : location_t loc = gimple_location (stmt);
2392 883 : tree lhs = gimple_assign_lhs (stmt);
2393 883 : tree rhs = gimple_assign_rhs1 (stmt);
2394 883 : bool store_p = requires_barrier (region->entry_block, lhs, NULL);
2395 883 : bool load_p = requires_barrier (region->entry_block, rhs, NULL);
2396 883 : gimple *gcall = NULL;
2397 :
2398 883 : if (!load_p && !store_p)
2399 : {
2400 : /* Add thread private addresses to log if applicable. */
2401 243 : requires_barrier (region->entry_block, lhs, stmt);
2402 243 : gsi_next (gsi);
2403 243 : return;
2404 : }
2405 :
2406 640 : if (load_p)
2407 393 : transaction_subcode_ior (region, GTMA_HAVE_LOAD);
2408 640 : if (store_p)
2409 262 : transaction_subcode_ior (region, GTMA_HAVE_STORE);
2410 :
2411 : // Remove original load/store statement.
2412 640 : gsi_remove (gsi, true);
2413 :
2414 : // Attempt to use a simple load/store helper function.
2415 640 : if (load_p && !store_p)
2416 378 : gcall = build_tm_load (loc, lhs, rhs, gsi);
2417 262 : else if (store_p && !load_p)
2418 247 : gcall = build_tm_store (loc, lhs, rhs, gsi);
2419 :
2420 : // If gcall has not been set, then we do not have a simple helper
2421 : // function available for the type. This may be true of larger
2422 : // structures, vectors, and non-standard float types.
2423 625 : if (!gcall)
2424 : {
2425 24 : tree lhs_addr, rhs_addr, ltmp = NULL, copy_fn;
2426 :
2427 : // If this is a type that we couldn't handle above, but it's
2428 : // in a register, we must spill it to memory for the copy.
2429 24 : if (is_gimple_reg (lhs))
2430 : {
2431 1 : ltmp = create_tmp_var (TREE_TYPE (lhs));
2432 1 : lhs_addr = build_fold_addr_expr (ltmp);
2433 : }
2434 : else
2435 23 : lhs_addr = gimplify_addr (gsi, lhs);
2436 24 : if (is_gimple_reg (rhs))
2437 : {
2438 0 : tree rtmp = create_tmp_var (TREE_TYPE (rhs));
2439 0 : TREE_ADDRESSABLE (rtmp) = 1;
2440 0 : rhs_addr = build_fold_addr_expr (rtmp);
2441 0 : gcall = gimple_build_assign (rtmp, rhs);
2442 0 : gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2443 : }
2444 24 : else if (TREE_CODE (rhs) == CONSTRUCTOR
2445 24 : && CONSTRUCTOR_NELTS (rhs) == 0)
2446 : {
2447 : /* Don't take address of an empty CONSTRUCTOR, it might not
2448 : work for C++ non-POD constructors at all and otherwise
2449 : would be inefficient. Use tm memset to clear lhs. */
2450 6 : gcc_assert (!load_p && store_p);
2451 6 : rhs_addr = integer_zero_node;
2452 : }
2453 : else
2454 18 : rhs_addr = gimplify_addr (gsi, rhs);
2455 :
2456 : // Choose the appropriate memory transfer function.
2457 6 : if (store_p
2458 22 : && TREE_CODE (rhs) == CONSTRUCTOR
2459 24 : && CONSTRUCTOR_NELTS (rhs) == 0)
2460 6 : copy_fn = builtin_decl_explicit (BUILT_IN_TM_MEMSET);
2461 18 : else if (load_p && store_p)
2462 : // ??? Figure out if there's any possible overlap between
2463 : // the LHS and the RHS and if not, use MEMCPY.
2464 15 : copy_fn = builtin_decl_explicit (BUILT_IN_TM_MEMMOVE);
2465 3 : else if (load_p)
2466 : // Note that the store is non-transactional and cannot overlap.
2467 2 : copy_fn = builtin_decl_explicit (BUILT_IN_TM_MEMCPY_RTWN);
2468 : else
2469 : // Note that the load is non-transactional and cannot overlap.
2470 1 : copy_fn = builtin_decl_explicit (BUILT_IN_TM_MEMCPY_RNWT);
2471 :
2472 24 : gcall = gimple_build_call (copy_fn, 3, lhs_addr, rhs_addr,
2473 24 : TYPE_SIZE_UNIT (TREE_TYPE (lhs)));
2474 24 : gimple_set_location (gcall, loc);
2475 24 : gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2476 :
2477 24 : if (ltmp)
2478 : {
2479 1 : gcall = gimple_build_assign (lhs, ltmp);
2480 1 : gsi_insert_before (gsi, gcall, GSI_SAME_STMT);
2481 : }
2482 : }
2483 :
2484 : // Now that we have the load/store in its instrumented form, add
2485 : // thread private addresses to the log if applicable.
2486 640 : if (!store_p)
2487 378 : requires_barrier (region->entry_block, lhs, gcall);
2488 : }
2489 :
2490 :
2491 : /* Expand a call statement as appropriate for a transaction. That is,
2492 : either verify that the call does not affect the transaction, or
2493 : redirect the call to a clone that handles transactions, or change
2494 : the transaction state to IRREVOCABLE. Return true if the call is
2495 : one of the builtins that end a transaction. */
2496 :
2497 : static bool
2498 899 : expand_call_tm (struct tm_region *region,
2499 : gimple_stmt_iterator *gsi)
2500 : {
2501 899 : gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
2502 899 : tree lhs = gimple_call_lhs (stmt);
2503 899 : tree fn_decl;
2504 899 : struct cgraph_node *node;
2505 899 : bool retval = false;
2506 :
2507 899 : fn_decl = gimple_call_fndecl (stmt);
2508 :
2509 899 : if (fn_decl == builtin_decl_explicit (BUILT_IN_TM_MEMCPY)
2510 899 : || fn_decl == builtin_decl_explicit (BUILT_IN_TM_MEMMOVE))
2511 4 : transaction_subcode_ior (region, GTMA_HAVE_STORE | GTMA_HAVE_LOAD);
2512 899 : if (fn_decl == builtin_decl_explicit (BUILT_IN_TM_MEMSET))
2513 2 : transaction_subcode_ior (region, GTMA_HAVE_STORE);
2514 :
2515 899 : if (is_tm_pure_call (stmt))
2516 : return false;
2517 :
2518 643 : if (fn_decl)
2519 625 : retval = is_tm_ending_fndecl (fn_decl);
2520 625 : if (!retval)
2521 : {
2522 : /* Assume all non-const/pure calls write to memory, except
2523 : transaction ending builtins. */
2524 236 : transaction_subcode_ior (region, GTMA_HAVE_STORE);
2525 : }
2526 :
2527 : /* For indirect calls, we already generated a call into the runtime. */
2528 643 : if (!fn_decl)
2529 : {
2530 18 : tree fn = gimple_call_fn (stmt);
2531 :
2532 : /* We are guaranteed never to go irrevocable on a safe or pure
2533 : call, and the pure call was handled above. */
2534 18 : if (is_tm_safe (fn))
2535 : return false;
2536 : else
2537 17 : transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE);
2538 :
2539 17 : return false;
2540 : }
2541 :
2542 625 : node = cgraph_node::get (fn_decl);
2543 : /* All calls should have cgraph here. */
2544 625 : if (!node)
2545 : {
2546 : /* We can have a nodeless call here if some pass after IPA-tm
2547 : added uninstrumented calls. For example, loop distribution
2548 : can transform certain loop constructs into __builtin_mem*
2549 : calls. In this case, see if we have a suitable TM
2550 : replacement and fill in the gaps. */
2551 0 : gcc_assert (DECL_BUILT_IN_CLASS (fn_decl) == BUILT_IN_NORMAL);
2552 0 : enum built_in_function code = DECL_FUNCTION_CODE (fn_decl);
2553 0 : gcc_assert (code == BUILT_IN_MEMCPY
2554 : || code == BUILT_IN_MEMMOVE
2555 : || code == BUILT_IN_MEMSET);
2556 :
2557 0 : tree repl = find_tm_replacement_function (fn_decl);
2558 0 : if (repl)
2559 : {
2560 0 : gimple_call_set_fndecl (stmt, repl);
2561 0 : update_stmt (stmt);
2562 0 : node = cgraph_node::create (repl);
2563 0 : node->tm_may_enter_irr = false;
2564 0 : return expand_call_tm (region, gsi);
2565 : }
2566 0 : gcc_unreachable ();
2567 : }
2568 625 : if (node->tm_may_enter_irr)
2569 13 : transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE);
2570 :
2571 625 : if (is_tm_abort (fn_decl))
2572 : {
2573 38 : transaction_subcode_ior (region, GTMA_HAVE_ABORT);
2574 38 : return true;
2575 : }
2576 :
2577 : /* Instrument the store if needed.
2578 :
2579 : If the assignment happens inside the function call (return slot
2580 : optimization), there is no instrumentation to be done, since
2581 : the callee should have done the right thing. */
2582 88 : if (lhs && requires_barrier (region->entry_block, lhs, stmt)
2583 596 : && !gimple_call_return_slot_opt_p (stmt))
2584 : {
2585 8 : tree tmp = create_tmp_reg (TREE_TYPE (lhs));
2586 8 : location_t loc = gimple_location (stmt);
2587 8 : edge fallthru_edge = NULL;
2588 8 : gassign *assign_stmt;
2589 :
2590 : /* Remember if the call was going to throw. */
2591 8 : if (stmt_can_throw_internal (cfun, stmt))
2592 : {
2593 7 : edge_iterator ei;
2594 7 : edge e;
2595 7 : basic_block bb = gimple_bb (stmt);
2596 :
2597 8 : FOR_EACH_EDGE (e, ei, bb->succs)
2598 8 : if (e->flags & EDGE_FALLTHRU)
2599 : {
2600 : fallthru_edge = e;
2601 : break;
2602 : }
2603 : }
2604 :
2605 8 : gimple_call_set_lhs (stmt, tmp);
2606 8 : update_stmt (stmt);
2607 8 : assign_stmt = gimple_build_assign (lhs, tmp);
2608 8 : gimple_set_location (assign_stmt, loc);
2609 :
2610 : /* We cannot throw in the middle of a BB. If the call was going
2611 : to throw, place the instrumentation on the fallthru edge, so
2612 : the call remains the last statement in the block. */
2613 8 : if (fallthru_edge)
2614 : {
2615 7 : gimple_seq fallthru_seq = gimple_seq_alloc_with_stmt (assign_stmt);
2616 7 : gimple_stmt_iterator fallthru_gsi = gsi_start (fallthru_seq);
2617 7 : expand_assign_tm (region, &fallthru_gsi);
2618 7 : gsi_insert_seq_on_edge (fallthru_edge, fallthru_seq);
2619 7 : pending_edge_inserts_p = true;
2620 : }
2621 : else
2622 : {
2623 1 : gsi_insert_after (gsi, assign_stmt, GSI_CONTINUE_LINKING);
2624 1 : expand_assign_tm (region, gsi);
2625 : }
2626 :
2627 8 : transaction_subcode_ior (region, GTMA_HAVE_STORE);
2628 : }
2629 :
2630 : return retval;
2631 : }
2632 :
2633 :
2634 : /* Expand all statements in BB as appropriate for being inside
2635 : a transaction. */
2636 :
2637 : static void
2638 1467 : expand_block_tm (struct tm_region *region, basic_block bb)
2639 : {
2640 1467 : gimple_stmt_iterator gsi;
2641 :
2642 5610 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); )
2643 : {
2644 3083 : gimple *stmt = gsi_stmt (gsi);
2645 3083 : switch (gimple_code (stmt))
2646 : {
2647 1285 : case GIMPLE_ASSIGN:
2648 : /* Only memory reads/writes need to be instrumented. */
2649 1285 : if (gimple_assign_single_p (stmt)
2650 1285 : && !gimple_clobber_p (stmt))
2651 : {
2652 875 : expand_assign_tm (region, &gsi);
2653 875 : continue;
2654 : }
2655 : break;
2656 :
2657 899 : case GIMPLE_CALL:
2658 899 : if (expand_call_tm (region, &gsi))
2659 407 : return;
2660 : break;
2661 :
2662 0 : case GIMPLE_ASM:
2663 0 : gcc_unreachable ();
2664 :
2665 : default:
2666 : break;
2667 : }
2668 1801 : if (!gsi_end_p (gsi))
2669 1801 : gsi_next (&gsi);
2670 : }
2671 : }
2672 :
2673 : /* Return the list of basic-blocks in REGION.
2674 :
2675 : STOP_AT_IRREVOCABLE_P is true if caller is uninterested in blocks
2676 : following a TM_IRREVOCABLE call.
2677 :
2678 : INCLUDE_UNINSTRUMENTED_P is TRUE if we should include the
2679 : uninstrumented code path blocks in the list of basic blocks
2680 : returned, false otherwise. */
2681 :
2682 : static vec<basic_block>
2683 1950 : get_tm_region_blocks (basic_block entry_block,
2684 : bitmap exit_blocks,
2685 : bitmap irr_blocks,
2686 : bitmap all_region_blocks,
2687 : bool stop_at_irrevocable_p,
2688 : bool include_uninstrumented_p = true)
2689 : {
2690 1950 : vec<basic_block> bbs = vNULL;
2691 1950 : unsigned i;
2692 1950 : edge e;
2693 1950 : edge_iterator ei;
2694 1950 : bitmap visited_blocks = BITMAP_ALLOC (NULL);
2695 :
2696 1950 : i = 0;
2697 1950 : bbs.safe_push (entry_block);
2698 1950 : bitmap_set_bit (visited_blocks, entry_block->index);
2699 :
2700 5120 : do
2701 : {
2702 5120 : basic_block bb = bbs[i++];
2703 :
2704 8921 : if (exit_blocks &&
2705 3801 : bitmap_bit_p (exit_blocks, bb->index))
2706 1821 : continue;
2707 :
2708 3342 : if (stop_at_irrevocable_p
2709 3299 : && irr_blocks
2710 3299 : && bitmap_bit_p (irr_blocks, bb->index))
2711 43 : continue;
2712 :
2713 7099 : FOR_EACH_EDGE (e, ei, bb->succs)
2714 3843 : if ((include_uninstrumented_p
2715 1743 : || !(e->flags & EDGE_TM_UNINSTRUMENTED))
2716 5563 : && !bitmap_bit_p (visited_blocks, e->dest->index))
2717 : {
2718 3170 : bitmap_set_bit (visited_blocks, e->dest->index);
2719 3170 : bbs.safe_push (e->dest);
2720 : }
2721 : }
2722 10240 : while (i < bbs.length ());
2723 :
2724 1950 : if (all_region_blocks)
2725 474 : bitmap_ior_into (all_region_blocks, visited_blocks);
2726 :
2727 1950 : BITMAP_FREE (visited_blocks);
2728 1950 : return bbs;
2729 : }
2730 :
2731 : // Callback data for collect_bb2reg.
2732 : struct bb2reg_stuff
2733 : {
2734 : vec<tm_region *> *bb2reg;
2735 : bool include_uninstrumented_p;
2736 : };
2737 :
2738 : // Callback for expand_regions, collect innermost region data for each bb.
2739 : static void *
2740 922 : collect_bb2reg (struct tm_region *region, void *data)
2741 : {
2742 922 : struct bb2reg_stuff *stuff = (struct bb2reg_stuff *)data;
2743 922 : vec<tm_region *> *bb2reg = stuff->bb2reg;
2744 922 : vec<basic_block> queue;
2745 922 : unsigned int i;
2746 922 : basic_block bb;
2747 :
2748 922 : queue = get_tm_region_blocks (region->entry_block,
2749 : region->exit_blocks,
2750 : region->irr_blocks,
2751 : NULL,
2752 : /*stop_at_irr_p=*/true,
2753 922 : stuff->include_uninstrumented_p);
2754 :
2755 : // We expect expand_region to perform a post-order traversal of the region
2756 : // tree. Therefore the last region seen for any bb is the innermost.
2757 4291 : FOR_EACH_VEC_ELT (queue, i, bb)
2758 2447 : (*bb2reg)[bb->index] = region;
2759 :
2760 922 : queue.release ();
2761 922 : return NULL;
2762 : }
2763 :
2764 : // Returns a vector, indexed by BB->INDEX, of the innermost tm_region to
2765 : // which a basic block belongs. Note that we only consider the instrumented
2766 : // code paths for the region; the uninstrumented code paths are ignored if
2767 : // INCLUDE_UNINSTRUMENTED_P is false.
2768 : //
2769 : // ??? This data is very similar to the bb_regions array that is collected
2770 : // during tm_region_init. Or, rather, this data is similar to what could
2771 : // be used within tm_region_init. The actual computation in tm_region_init
2772 : // begins and ends with bb_regions entirely full of NULL pointers, due to
2773 : // the way in which pointers are swapped in and out of the array.
2774 : //
2775 : // ??? Our callers expect that blocks are not shared between transactions.
2776 : // When the optimizers get too smart, and blocks are shared, then during
2777 : // the tm_mark phase we'll add log entries to only one of the two transactions,
2778 : // and in the tm_edge phase we'll add edges to the CFG that create invalid
2779 : // cycles. The symptom being SSA defs that do not dominate their uses.
2780 : // Note that the optimizers were locally correct with their transformation,
2781 : // as we have no info within the program that suggests that the blocks cannot
2782 : // be shared.
2783 : //
2784 : // ??? There is currently a hack inside tree-ssa-pre.cc to work around the
2785 : // only known instance of this block sharing.
2786 :
2787 : static vec<tm_region *>
2788 948 : get_bb_regions_instrumented (bool traverse_clones,
2789 : bool include_uninstrumented_p)
2790 : {
2791 948 : unsigned n = last_basic_block_for_fn (cfun);
2792 948 : struct bb2reg_stuff stuff;
2793 948 : vec<tm_region *> ret;
2794 :
2795 948 : ret.create (n);
2796 948 : ret.safe_grow_cleared (n, true);
2797 948 : stuff.bb2reg = &ret;
2798 948 : stuff.include_uninstrumented_p = include_uninstrumented_p;
2799 948 : expand_regions (all_tm_regions, collect_bb2reg, &stuff, traverse_clones);
2800 :
2801 948 : return ret;
2802 : }
2803 :
2804 : /* Set the IN_TRANSACTION for all gimple statements that appear in a
2805 : transaction. */
2806 :
2807 : void
2808 104 : compute_transaction_bits (void)
2809 : {
2810 104 : struct tm_region *region;
2811 104 : vec<basic_block> queue;
2812 104 : unsigned int i;
2813 104 : basic_block bb;
2814 :
2815 : /* ?? Perhaps we need to abstract gate_tm_init further, because we
2816 : certainly don't need it to calculate CDI_DOMINATOR info. */
2817 104 : gate_tm_init ();
2818 :
2819 1349 : FOR_EACH_BB_FN (bb, cfun)
2820 1245 : bb->flags &= ~BB_IN_TRANSACTION;
2821 :
2822 168 : for (region = all_tm_regions; region; region = region->next)
2823 : {
2824 64 : queue = get_tm_region_blocks (region->entry_block,
2825 : region->exit_blocks,
2826 : region->irr_blocks,
2827 : NULL,
2828 : /*stop_at_irr_p=*/true);
2829 489 : for (i = 0; queue.iterate (i, &bb); ++i)
2830 361 : bb->flags |= BB_IN_TRANSACTION;
2831 64 : queue.release ();
2832 : }
2833 :
2834 104 : if (all_tm_regions)
2835 56 : bitmap_obstack_release (&tm_obstack);
2836 104 : }
2837 :
2838 : /* Replace the GIMPLE_TRANSACTION in this region with the corresponding
2839 : call to BUILT_IN_TM_START. */
2840 :
2841 : static void *
2842 357 : expand_transaction (struct tm_region *region, void *data ATTRIBUTE_UNUSED)
2843 : {
2844 357 : tree tm_start = builtin_decl_explicit (BUILT_IN_TM_START);
2845 357 : basic_block transaction_bb = gimple_bb (region->transaction_stmt);
2846 357 : tree tm_state = region->tm_state;
2847 357 : tree tm_state_type = TREE_TYPE (tm_state);
2848 357 : edge abort_edge = NULL;
2849 357 : edge inst_edge = NULL;
2850 357 : edge uninst_edge = NULL;
2851 357 : edge fallthru_edge = NULL;
2852 :
2853 : // Identify the various successors of the transaction start.
2854 357 : {
2855 357 : edge_iterator i;
2856 357 : edge e;
2857 1100 : FOR_EACH_EDGE (e, i, transaction_bb->succs)
2858 : {
2859 743 : if (e->flags & EDGE_TM_ABORT)
2860 : abort_edge = e;
2861 698 : else if (e->flags & EDGE_TM_UNINSTRUMENTED)
2862 : uninst_edge = e;
2863 : else
2864 351 : inst_edge = e;
2865 743 : if (e->flags & EDGE_FALLTHRU)
2866 357 : fallthru_edge = e;
2867 : }
2868 : }
2869 :
2870 : /* ??? There are plenty of bits here we're not computing. */
2871 357 : {
2872 357 : int subcode = gimple_transaction_subcode (region->get_transaction_stmt ());
2873 357 : int flags = 0;
2874 357 : if (subcode & GTMA_DOES_GO_IRREVOCABLE)
2875 39 : flags |= PR_DOESGOIRREVOCABLE;
2876 357 : if ((subcode & GTMA_MAY_ENTER_IRREVOCABLE) == 0)
2877 302 : flags |= PR_HASNOIRREVOCABLE;
2878 : /* If the transaction does not have an abort in lexical scope and is not
2879 : marked as an outer transaction, then it will never abort. */
2880 357 : if ((subcode & GTMA_HAVE_ABORT) == 0 && (subcode & GTMA_IS_OUTER) == 0)
2881 311 : flags |= PR_HASNOABORT;
2882 357 : if ((subcode & GTMA_HAVE_STORE) == 0)
2883 153 : flags |= PR_READONLY;
2884 357 : if (inst_edge && !(subcode & GTMA_HAS_NO_INSTRUMENTATION))
2885 312 : flags |= PR_INSTRUMENTEDCODE;
2886 357 : if (uninst_edge)
2887 347 : flags |= PR_UNINSTRUMENTEDCODE;
2888 357 : if (subcode & GTMA_IS_OUTER)
2889 10 : region->original_transaction_was_outer = true;
2890 357 : tree t = build_int_cst (tm_state_type, flags);
2891 357 : gcall *call = gimple_build_call (tm_start, 1, t);
2892 357 : gimple_call_set_lhs (call, tm_state);
2893 357 : gimple_set_location (call, gimple_location (region->transaction_stmt));
2894 :
2895 : // Replace the GIMPLE_TRANSACTION with the call to BUILT_IN_TM_START.
2896 357 : gimple_stmt_iterator gsi = gsi_last_bb (transaction_bb);
2897 357 : gcc_assert (gsi_stmt (gsi) == region->transaction_stmt);
2898 357 : gsi_insert_before (&gsi, call, GSI_SAME_STMT);
2899 357 : gsi_remove (&gsi, true);
2900 357 : region->transaction_stmt = call;
2901 : }
2902 :
2903 : // Generate log saves.
2904 357 : if (!tm_log_save_addresses.is_empty ())
2905 18 : tm_log_emit_saves (region->entry_block, transaction_bb);
2906 :
2907 : // In the beginning, we've no tests to perform on transaction restart.
2908 : // Note that after this point, transaction_bb becomes the "most recent
2909 : // block containing tests for the transaction".
2910 357 : region->restart_block = region->entry_block;
2911 :
2912 : // Generate log restores.
2913 357 : if (!tm_log_save_addresses.is_empty ())
2914 : {
2915 18 : basic_block test_bb = create_empty_bb (transaction_bb);
2916 18 : basic_block code_bb = create_empty_bb (test_bb);
2917 18 : basic_block join_bb = create_empty_bb (code_bb);
2918 18 : add_bb_to_loop (test_bb, transaction_bb->loop_father);
2919 18 : add_bb_to_loop (code_bb, transaction_bb->loop_father);
2920 18 : add_bb_to_loop (join_bb, transaction_bb->loop_father);
2921 18 : if (region->restart_block == region->entry_block)
2922 18 : region->restart_block = test_bb;
2923 :
2924 18 : tree t1 = create_tmp_reg (tm_state_type);
2925 18 : tree t2 = build_int_cst (tm_state_type, A_RESTORELIVEVARIABLES);
2926 18 : gimple *stmt = gimple_build_assign (t1, BIT_AND_EXPR, tm_state, t2);
2927 18 : gimple_stmt_iterator gsi = gsi_last_bb (test_bb);
2928 18 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
2929 :
2930 18 : t2 = build_int_cst (tm_state_type, 0);
2931 18 : stmt = gimple_build_cond (NE_EXPR, t1, t2, NULL, NULL);
2932 18 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
2933 :
2934 18 : tm_log_emit_restores (region->entry_block, code_bb);
2935 :
2936 18 : edge ei = make_edge (transaction_bb, test_bb, EDGE_FALLTHRU);
2937 18 : edge et = make_edge (test_bb, code_bb, EDGE_TRUE_VALUE);
2938 18 : edge ef = make_edge (test_bb, join_bb, EDGE_FALSE_VALUE);
2939 18 : redirect_edge_pred (fallthru_edge, join_bb);
2940 :
2941 18 : join_bb->count = test_bb->count = transaction_bb->count;
2942 :
2943 18 : ei->probability = profile_probability::always ();
2944 18 : et->probability = profile_probability::likely ();
2945 18 : ef->probability = profile_probability::unlikely ();
2946 :
2947 18 : code_bb->count = et->count ();
2948 :
2949 18 : transaction_bb = join_bb;
2950 : }
2951 :
2952 : // If we have an ABORT edge, create a test to perform the abort.
2953 357 : if (abort_edge)
2954 : {
2955 45 : basic_block test_bb = create_empty_bb (transaction_bb);
2956 45 : add_bb_to_loop (test_bb, transaction_bb->loop_father);
2957 45 : if (region->restart_block == region->entry_block)
2958 42 : region->restart_block = test_bb;
2959 :
2960 45 : tree t1 = create_tmp_reg (tm_state_type);
2961 45 : tree t2 = build_int_cst (tm_state_type, A_ABORTTRANSACTION);
2962 45 : gimple *stmt = gimple_build_assign (t1, BIT_AND_EXPR, tm_state, t2);
2963 45 : gimple_stmt_iterator gsi = gsi_last_bb (test_bb);
2964 45 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
2965 :
2966 45 : t2 = build_int_cst (tm_state_type, 0);
2967 45 : stmt = gimple_build_cond (NE_EXPR, t1, t2, NULL, NULL);
2968 45 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
2969 :
2970 45 : edge ei = make_edge (transaction_bb, test_bb, EDGE_FALLTHRU);
2971 45 : test_bb->count = transaction_bb->count;
2972 45 : ei->probability = profile_probability::always ();
2973 :
2974 : // Not abort edge. If both are live, chose one at random as we'll
2975 : // we'll be fixing that up below.
2976 45 : redirect_edge_pred (fallthru_edge, test_bb);
2977 45 : fallthru_edge->flags = EDGE_FALSE_VALUE;
2978 45 : fallthru_edge->probability = profile_probability::very_likely ();
2979 :
2980 : // Abort/over edge.
2981 45 : redirect_edge_pred (abort_edge, test_bb);
2982 45 : abort_edge->flags = EDGE_TRUE_VALUE;
2983 45 : abort_edge->probability = profile_probability::unlikely ();
2984 :
2985 45 : transaction_bb = test_bb;
2986 : }
2987 :
2988 : // If we have both instrumented and uninstrumented code paths, select one.
2989 357 : if (inst_edge && uninst_edge)
2990 : {
2991 341 : basic_block test_bb = create_empty_bb (transaction_bb);
2992 341 : add_bb_to_loop (test_bb, transaction_bb->loop_father);
2993 341 : if (region->restart_block == region->entry_block)
2994 297 : region->restart_block = test_bb;
2995 :
2996 341 : tree t1 = create_tmp_reg (tm_state_type);
2997 341 : tree t2 = build_int_cst (tm_state_type, A_RUNUNINSTRUMENTEDCODE);
2998 :
2999 341 : gimple *stmt = gimple_build_assign (t1, BIT_AND_EXPR, tm_state, t2);
3000 341 : gimple_stmt_iterator gsi = gsi_last_bb (test_bb);
3001 341 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3002 :
3003 341 : t2 = build_int_cst (tm_state_type, 0);
3004 341 : stmt = gimple_build_cond (NE_EXPR, t1, t2, NULL, NULL);
3005 341 : gsi_insert_after (&gsi, stmt, GSI_CONTINUE_LINKING);
3006 :
3007 : // Create the edge into test_bb first, as we want to copy values
3008 : // out of the fallthru edge.
3009 341 : edge e = make_edge (transaction_bb, test_bb, fallthru_edge->flags);
3010 341 : e->probability = fallthru_edge->probability;
3011 341 : test_bb->count = fallthru_edge->count ();
3012 :
3013 : // Now update the edges to the inst/uninist implementations.
3014 : // For now assume that the paths are equally likely. When using HTM,
3015 : // we'll try the uninst path first and fallback to inst path if htm
3016 : // buffers are exceeded. Without HTM we start with the inst path and
3017 : // use the uninst path when falling back to serial mode.
3018 341 : redirect_edge_pred (inst_edge, test_bb);
3019 341 : inst_edge->flags = EDGE_FALSE_VALUE;
3020 341 : inst_edge->probability = profile_probability::even ();
3021 :
3022 341 : redirect_edge_pred (uninst_edge, test_bb);
3023 341 : uninst_edge->flags = EDGE_TRUE_VALUE;
3024 341 : uninst_edge->probability = profile_probability::even ();
3025 : }
3026 :
3027 : // If we have no previous special cases, and we have PHIs at the beginning
3028 : // of the atomic region, this means we have a loop at the beginning of the
3029 : // atomic region that shares the first block. This can cause problems with
3030 : // the transaction restart abnormal edges to be added in the tm_edges pass.
3031 : // Solve this by adding a new empty block to receive the abnormal edges.
3032 357 : if (region->restart_block == region->entry_block
3033 357 : && phi_nodes (region->entry_block))
3034 : {
3035 0 : basic_block empty_bb = create_empty_bb (transaction_bb);
3036 0 : region->restart_block = empty_bb;
3037 0 : add_bb_to_loop (empty_bb, transaction_bb->loop_father);
3038 :
3039 0 : redirect_edge_pred (fallthru_edge, empty_bb);
3040 0 : make_edge (transaction_bb, empty_bb, EDGE_FALLTHRU);
3041 : }
3042 :
3043 357 : return NULL;
3044 : }
3045 :
3046 : /* Generate the temporary to be used for the return value of
3047 : BUILT_IN_TM_START. */
3048 :
3049 : static void *
3050 565 : generate_tm_state (struct tm_region *region, void *data ATTRIBUTE_UNUSED)
3051 : {
3052 565 : tree tm_start = builtin_decl_explicit (BUILT_IN_TM_START);
3053 1130 : region->tm_state =
3054 565 : create_tmp_reg (TREE_TYPE (TREE_TYPE (tm_start)), "tm_state");
3055 :
3056 : // Reset the subcode, post optimizations. We'll fill this in
3057 : // again as we process blocks.
3058 565 : if (region->exit_blocks)
3059 : {
3060 357 : gtransaction *transaction_stmt = region->get_transaction_stmt ();
3061 357 : unsigned int subcode = gimple_transaction_subcode (transaction_stmt);
3062 :
3063 357 : if (subcode & GTMA_DOES_GO_IRREVOCABLE)
3064 39 : subcode &= (GTMA_DECLARATION_MASK | GTMA_DOES_GO_IRREVOCABLE
3065 : | GTMA_MAY_ENTER_IRREVOCABLE
3066 : | GTMA_HAS_NO_INSTRUMENTATION);
3067 : else
3068 318 : subcode &= GTMA_DECLARATION_MASK;
3069 357 : gimple_transaction_set_subcode (transaction_stmt, subcode);
3070 : }
3071 :
3072 565 : return NULL;
3073 : }
3074 :
3075 : // Propagate flags from inner transactions outwards.
3076 : static void
3077 1039 : propagate_tm_flags_out (struct tm_region *region)
3078 : {
3079 1604 : if (region == NULL)
3080 1039 : return;
3081 565 : propagate_tm_flags_out (region->inner);
3082 :
3083 565 : if (region->outer && region->outer->transaction_stmt)
3084 : {
3085 8 : unsigned s
3086 8 : = gimple_transaction_subcode (region->get_transaction_stmt ());
3087 8 : s &= (GTMA_HAVE_ABORT | GTMA_HAVE_LOAD | GTMA_HAVE_STORE
3088 : | GTMA_MAY_ENTER_IRREVOCABLE);
3089 8 : s |= gimple_transaction_subcode (region->outer->get_transaction_stmt ());
3090 8 : gimple_transaction_set_subcode (region->outer->get_transaction_stmt (),
3091 : s);
3092 : }
3093 :
3094 565 : propagate_tm_flags_out (region->next);
3095 : }
3096 :
3097 : /* Entry point to the MARK phase of TM expansion. Here we replace
3098 : transactional memory statements with calls to builtins, and function
3099 : calls with their transactional clones (if available). But we don't
3100 : yet lower GIMPLE_TRANSACTION or add the transaction restart back-edges. */
3101 :
3102 : static unsigned int
3103 474 : execute_tm_mark (void)
3104 : {
3105 474 : pending_edge_inserts_p = false;
3106 :
3107 474 : expand_regions (all_tm_regions, generate_tm_state, NULL,
3108 : /*traverse_clones=*/true);
3109 :
3110 474 : tm_log_init ();
3111 :
3112 474 : vec<tm_region *> bb_regions
3113 474 : = get_bb_regions_instrumented (/*traverse_clones=*/true,
3114 : /*include_uninstrumented_p=*/false);
3115 474 : struct tm_region *r;
3116 474 : unsigned i;
3117 :
3118 : // Expand memory operations into calls into the runtime.
3119 : // This collects log entries as well.
3120 4540 : FOR_EACH_VEC_ELT (bb_regions, i, r)
3121 : {
3122 4066 : if (r != NULL)
3123 : {
3124 1575 : if (r->transaction_stmt)
3125 : {
3126 771 : unsigned sub
3127 771 : = gimple_transaction_subcode (r->get_transaction_stmt ());
3128 :
3129 : /* If we're sure to go irrevocable, there won't be
3130 : anything to expand, since the run-time will go
3131 : irrevocable right away. */
3132 771 : if (sub & GTMA_DOES_GO_IRREVOCABLE
3133 108 : && sub & GTMA_MAY_ENTER_IRREVOCABLE)
3134 108 : continue;
3135 : }
3136 1467 : expand_block_tm (r, BASIC_BLOCK_FOR_FN (cfun, i));
3137 : }
3138 : }
3139 :
3140 474 : bb_regions.release ();
3141 :
3142 : // Propagate flags from inner transactions outwards.
3143 474 : propagate_tm_flags_out (all_tm_regions);
3144 :
3145 : // Expand GIMPLE_TRANSACTIONs into calls into the runtime.
3146 474 : expand_regions (all_tm_regions, expand_transaction, NULL,
3147 : /*traverse_clones=*/false);
3148 :
3149 474 : tm_log_emit ();
3150 474 : tm_log_delete ();
3151 :
3152 474 : if (pending_edge_inserts_p)
3153 4 : gsi_commit_edge_inserts ();
3154 474 : free_dominance_info (CDI_DOMINATORS);
3155 474 : return 0;
3156 : }
3157 :
3158 : namespace {
3159 :
3160 : const pass_data pass_data_tm_mark =
3161 : {
3162 : GIMPLE_PASS, /* type */
3163 : "tmmark", /* name */
3164 : OPTGROUP_NONE, /* optinfo_flags */
3165 : TV_TRANS_MEM, /* tv_id */
3166 : ( PROP_ssa | PROP_cfg ), /* properties_required */
3167 : 0, /* properties_provided */
3168 : 0, /* properties_destroyed */
3169 : 0, /* todo_flags_start */
3170 : TODO_update_ssa, /* todo_flags_finish */
3171 : };
3172 :
3173 : class pass_tm_mark : public gimple_opt_pass
3174 : {
3175 : public:
3176 285722 : pass_tm_mark (gcc::context *ctxt)
3177 571444 : : gimple_opt_pass (pass_data_tm_mark, ctxt)
3178 : {}
3179 :
3180 : /* opt_pass methods: */
3181 474 : unsigned int execute (function *) final override
3182 : {
3183 474 : return execute_tm_mark ();
3184 : }
3185 :
3186 : }; // class pass_tm_mark
3187 :
3188 : } // anon namespace
3189 :
3190 : gimple_opt_pass *
3191 285722 : make_pass_tm_mark (gcc::context *ctxt)
3192 : {
3193 285722 : return new pass_tm_mark (ctxt);
3194 : }
3195 :
3196 :
3197 : /* Create an abnormal edge from STMT at iter, splitting the block
3198 : as necessary. Adjust *PNEXT as needed for the split block. */
3199 :
3200 : static inline void
3201 925 : split_bb_make_tm_edge (gimple *stmt, basic_block dest_bb,
3202 : gimple_stmt_iterator iter, gimple_stmt_iterator *pnext)
3203 : {
3204 925 : basic_block bb = gimple_bb (stmt);
3205 925 : if (!gsi_one_before_end_p (iter))
3206 : {
3207 526 : edge e = split_block (bb, stmt);
3208 1052 : *pnext = gsi_start_bb (e->dest);
3209 : }
3210 925 : edge e = make_edge (bb, dest_bb, EDGE_ABNORMAL);
3211 925 : if (e)
3212 900 : e->probability = profile_probability::guessed_never ();
3213 :
3214 : // Record the need for the edge for the benefit of the rtl passes.
3215 925 : if (cfun->gimple_df->tm_restart == NULL)
3216 8 : cfun->gimple_df->tm_restart
3217 8 : = hash_table<tm_restart_hasher>::create_ggc (31);
3218 :
3219 925 : struct tm_restart_node dummy;
3220 925 : dummy.stmt = stmt;
3221 925 : dummy.label_or_list = gimple_block_label (dest_bb);
3222 :
3223 925 : tm_restart_node **slot = cfun->gimple_df->tm_restart->find_slot (&dummy,
3224 : INSERT);
3225 925 : struct tm_restart_node *n = *slot;
3226 925 : if (n == NULL)
3227 : {
3228 925 : *slot = n = ggc_alloc<tm_restart_node> ();
3229 925 : *n = dummy;
3230 : }
3231 : else
3232 : {
3233 0 : tree old = n->label_or_list;
3234 0 : if (TREE_CODE (old) == LABEL_DECL)
3235 0 : old = tree_cons (NULL, old, NULL);
3236 0 : n->label_or_list = tree_cons (NULL, dummy.label_or_list, old);
3237 : }
3238 925 : }
3239 :
3240 : /* Split block BB as necessary for every builtin function we added, and
3241 : wire up the abnormal back edges implied by the transaction restart. */
3242 :
3243 : static void
3244 779 : expand_block_edges (struct tm_region *const region, basic_block bb)
3245 : {
3246 779 : gimple_stmt_iterator gsi, next_gsi;
3247 :
3248 4538 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi = next_gsi)
3249 : {
3250 2980 : gimple *stmt = gsi_stmt (gsi);
3251 2980 : gcall *call_stmt;
3252 :
3253 2980 : next_gsi = gsi;
3254 2980 : gsi_next (&next_gsi);
3255 :
3256 : // ??? Shouldn't we split for any non-pure, non-irrevocable function?
3257 2980 : call_stmt = dyn_cast <gcall *> (stmt);
3258 3302 : if ((!call_stmt)
3259 1280 : || (gimple_call_flags (call_stmt) & ECF_TM_BUILTIN) == 0)
3260 2022 : continue;
3261 :
3262 958 : if (gimple_call_builtin_p (call_stmt, BUILT_IN_TM_ABORT))
3263 : {
3264 : // If we have a ``_transaction_cancel [[outer]]'', there is only
3265 : // one abnormal edge: to the transaction marked OUTER.
3266 : // All compiler-generated instances of BUILT_IN_TM_ABORT have a
3267 : // constant argument, which we can examine here. Users invoking
3268 : // TM_ABORT directly get what they deserve.
3269 35 : tree arg = gimple_call_arg (call_stmt, 0);
3270 35 : if (TREE_CODE (arg) == INTEGER_CST
3271 35 : && (TREE_INT_CST_LOW (arg) & AR_OUTERABORT) != 0
3272 43 : && !decl_is_tm_clone (current_function_decl))
3273 : {
3274 : // Find the GTMA_IS_OUTER transaction.
3275 8 : for (struct tm_region *o = region; o; o = o->outer)
3276 6 : if (o->original_transaction_was_outer)
3277 : {
3278 2 : split_bb_make_tm_edge (call_stmt, o->restart_block,
3279 : gsi, &next_gsi);
3280 2 : break;
3281 : }
3282 :
3283 : // Otherwise, the front-end should have semantically checked
3284 : // outer aborts, but in either case the target region is not
3285 : // within this function.
3286 4 : continue;
3287 4 : }
3288 :
3289 : // Non-outer, TM aborts have an abnormal edge to the inner-most
3290 : // transaction, the one being aborted;
3291 31 : split_bb_make_tm_edge (call_stmt, region->restart_block, gsi,
3292 : &next_gsi);
3293 : }
3294 :
3295 : // All TM builtins have an abnormal edge to the outer-most transaction.
3296 : // We never restart inner transactions. For tm clones, we know a-priori
3297 : // that the outer-most transaction is outside the function.
3298 1908 : if (decl_is_tm_clone (current_function_decl))
3299 62 : continue;
3300 :
3301 892 : if (cfun->gimple_df->tm_restart == NULL)
3302 257 : cfun->gimple_df->tm_restart
3303 257 : = hash_table<tm_restart_hasher>::create_ggc (31);
3304 :
3305 : // All TM builtins have an abnormal edge to the outer-most transaction.
3306 : // We never restart inner transactions.
3307 928 : for (struct tm_region *o = region; o; o = o->outer)
3308 928 : if (!o->outer)
3309 : {
3310 892 : split_bb_make_tm_edge (call_stmt, o->restart_block, gsi, &next_gsi);
3311 892 : break;
3312 : }
3313 :
3314 : // Delete any tail-call annotation that may have been added.
3315 : // The tail-call pass may have mis-identified the commit as being
3316 : // a candidate because we had not yet added this restart edge.
3317 892 : gimple_call_set_tail (call_stmt, false);
3318 : }
3319 779 : }
3320 :
3321 : /* Entry point to the final expansion of transactional nodes. */
3322 :
3323 : namespace {
3324 :
3325 : const pass_data pass_data_tm_edges =
3326 : {
3327 : GIMPLE_PASS, /* type */
3328 : "tmedge", /* name */
3329 : OPTGROUP_NONE, /* optinfo_flags */
3330 : TV_TRANS_MEM, /* tv_id */
3331 : ( PROP_ssa | PROP_cfg ), /* properties_required */
3332 : 0, /* properties_provided */
3333 : 0, /* properties_destroyed */
3334 : 0, /* todo_flags_start */
3335 : TODO_update_ssa, /* todo_flags_finish */
3336 : };
3337 :
3338 : class pass_tm_edges : public gimple_opt_pass
3339 : {
3340 : public:
3341 285722 : pass_tm_edges (gcc::context *ctxt)
3342 571444 : : gimple_opt_pass (pass_data_tm_edges, ctxt)
3343 : {}
3344 :
3345 : /* opt_pass methods: */
3346 : unsigned int execute (function *) final override;
3347 :
3348 : }; // class pass_tm_edges
3349 :
3350 : unsigned int
3351 474 : pass_tm_edges::execute (function *fun)
3352 : {
3353 474 : vec<tm_region *> bb_regions
3354 474 : = get_bb_regions_instrumented (/*traverse_clones=*/false,
3355 : /*include_uninstrumented_p=*/true);
3356 474 : struct tm_region *r;
3357 474 : unsigned i;
3358 :
3359 5454 : FOR_EACH_VEC_ELT (bb_regions, i, r)
3360 4506 : if (r != NULL)
3361 779 : expand_block_edges (r, BASIC_BLOCK_FOR_FN (fun, i));
3362 :
3363 474 : bb_regions.release ();
3364 :
3365 : /* We've got to release the dominance info now, to indicate that it
3366 : must be rebuilt completely. Otherwise we'll crash trying to update
3367 : the SSA web in the TODO section following this pass. */
3368 474 : free_dominance_info (CDI_DOMINATORS);
3369 : /* We'ge also wrecked loops badly with inserting of abnormal edges. */
3370 474 : loops_state_set (LOOPS_NEED_FIXUP);
3371 474 : bitmap_obstack_release (&tm_obstack);
3372 474 : all_tm_regions = NULL;
3373 :
3374 474 : return 0;
3375 : }
3376 :
3377 : } // anon namespace
3378 :
3379 : gimple_opt_pass *
3380 285722 : make_pass_tm_edges (gcc::context *ctxt)
3381 : {
3382 285722 : return new pass_tm_edges (ctxt);
3383 : }
3384 :
3385 : /* Helper function for expand_regions. Expand REGION and recurse to
3386 : the inner region. Call CALLBACK on each region. CALLBACK returns
3387 : NULL to continue the traversal, otherwise a non-null value which
3388 : this function will return as well. TRAVERSE_CLONES is true if we
3389 : should traverse transactional clones. */
3390 :
3391 : static void *
3392 2260 : expand_regions_1 (struct tm_region *region,
3393 : void *(*callback)(struct tm_region *, void *),
3394 : void *data,
3395 : bool traverse_clones)
3396 : {
3397 2260 : void *retval = NULL;
3398 2260 : if (region->exit_blocks
3399 2676 : || (traverse_clones && decl_is_tm_clone (current_function_decl)))
3400 : {
3401 1844 : retval = callback (region, data);
3402 1844 : if (retval)
3403 : return retval;
3404 : }
3405 2260 : if (region->inner)
3406 : {
3407 84 : retval = expand_regions (region->inner, callback, data, traverse_clones);
3408 84 : if (retval)
3409 : return retval;
3410 : }
3411 : return retval;
3412 : }
3413 :
3414 : /* Traverse the regions enclosed and including REGION. Execute
3415 : CALLBACK for each region, passing DATA. CALLBACK returns NULL to
3416 : continue the traversal, otherwise a non-null value which this
3417 : function will return as well. TRAVERSE_CLONES is true if we should
3418 : traverse transactional clones. */
3419 :
3420 : static void *
3421 1980 : expand_regions (struct tm_region *region,
3422 : void *(*callback)(struct tm_region *, void *),
3423 : void *data,
3424 : bool traverse_clones)
3425 : {
3426 1980 : void *retval = NULL;
3427 4240 : while (region)
3428 : {
3429 2260 : retval = expand_regions_1 (region, callback, data, traverse_clones);
3430 2260 : if (retval)
3431 : return retval;
3432 2260 : region = region->next;
3433 : }
3434 : return retval;
3435 : }
3436 :
3437 :
3438 : /* A unique TM memory operation. */
3439 : struct tm_memop
3440 : {
3441 : /* Unique ID that all memory operations to the same location have. */
3442 : unsigned int value_id;
3443 : /* Address of load/store. */
3444 : tree addr;
3445 : };
3446 :
3447 : /* TM memory operation hashtable helpers. */
3448 :
3449 : struct tm_memop_hasher : free_ptr_hash <tm_memop>
3450 : {
3451 : static inline hashval_t hash (const tm_memop *);
3452 : static inline bool equal (const tm_memop *, const tm_memop *);
3453 : };
3454 :
3455 : /* Htab support. Return a hash value for a `tm_memop'. */
3456 : inline hashval_t
3457 1830 : tm_memop_hasher::hash (const tm_memop *mem)
3458 : {
3459 1830 : tree addr = mem->addr;
3460 : /* We drill down to the SSA_NAME/DECL for the hash, but equality is
3461 : actually done with operand_equal_p (see tm_memop_eq). */
3462 1830 : if (TREE_CODE (addr) == ADDR_EXPR)
3463 754 : addr = TREE_OPERAND (addr, 0);
3464 1830 : return iterative_hash_expr (addr, 0);
3465 : }
3466 :
3467 : /* Htab support. Return true if two tm_memop's are the same. */
3468 : inline bool
3469 1292 : tm_memop_hasher::equal (const tm_memop *mem1, const tm_memop *mem2)
3470 : {
3471 1292 : return operand_equal_p (mem1->addr, mem2->addr, 0);
3472 : }
3473 :
3474 : /* Sets for solving data flow equations in the memory optimization pass. */
3475 : struct tm_memopt_bitmaps
3476 : {
3477 : /* Stores available to this BB upon entry. Basically, stores that
3478 : dominate this BB. */
3479 : bitmap store_avail_in;
3480 : /* Stores available at the end of this BB. */
3481 : bitmap store_avail_out;
3482 : bitmap store_antic_in;
3483 : bitmap store_antic_out;
3484 : /* Reads available to this BB upon entry. Basically, reads that
3485 : dominate this BB. */
3486 : bitmap read_avail_in;
3487 : /* Reads available at the end of this BB. */
3488 : bitmap read_avail_out;
3489 : /* Reads performed in this BB. */
3490 : bitmap read_local;
3491 : /* Writes performed in this BB. */
3492 : bitmap store_local;
3493 :
3494 : /* Temporary storage for pass. */
3495 : /* Is the current BB in the worklist? */
3496 : bool avail_in_worklist_p;
3497 : /* Have we visited this BB? */
3498 : bool visited_p;
3499 : };
3500 :
3501 : static bitmap_obstack tm_memopt_obstack;
3502 :
3503 : /* Unique counter for TM loads and stores. Loads and stores of the
3504 : same address get the same ID. */
3505 : static unsigned int tm_memopt_value_id;
3506 : static hash_table<tm_memop_hasher> *tm_memopt_value_numbers;
3507 :
3508 : #define STORE_AVAIL_IN(BB) \
3509 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_in
3510 : #define STORE_AVAIL_OUT(BB) \
3511 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_avail_out
3512 : #define STORE_ANTIC_IN(BB) \
3513 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_in
3514 : #define STORE_ANTIC_OUT(BB) \
3515 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_antic_out
3516 : #define READ_AVAIL_IN(BB) \
3517 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_in
3518 : #define READ_AVAIL_OUT(BB) \
3519 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_avail_out
3520 : #define READ_LOCAL(BB) \
3521 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->read_local
3522 : #define STORE_LOCAL(BB) \
3523 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->store_local
3524 : #define AVAIL_IN_WORKLIST_P(BB) \
3525 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->avail_in_worklist_p
3526 : #define BB_VISITED_P(BB) \
3527 : ((struct tm_memopt_bitmaps *) ((BB)->aux))->visited_p
3528 :
3529 : /* Given a TM load/store in STMT, return the value number for the address
3530 : it accesses. */
3531 :
3532 : static unsigned int
3533 664 : tm_memopt_value_number (gimple *stmt, enum insert_option op)
3534 : {
3535 664 : struct tm_memop tmpmem, *mem;
3536 664 : tm_memop **slot;
3537 :
3538 664 : gcc_assert (is_tm_load (stmt) || is_tm_store (stmt));
3539 664 : tmpmem.addr = gimple_call_arg (stmt, 0);
3540 664 : slot = tm_memopt_value_numbers->find_slot (&tmpmem, op);
3541 664 : if (*slot)
3542 : mem = *slot;
3543 284 : else if (op == INSERT)
3544 : {
3545 284 : mem = XNEW (struct tm_memop);
3546 284 : *slot = mem;
3547 284 : mem->value_id = tm_memopt_value_id++;
3548 284 : mem->addr = tmpmem.addr;
3549 : }
3550 : else
3551 0 : gcc_unreachable ();
3552 664 : return mem->value_id;
3553 : }
3554 :
3555 : /* Accumulate TM memory operations in BB into STORE_LOCAL and READ_LOCAL. */
3556 :
3557 : static void
3558 621 : tm_memopt_accumulate_memops (basic_block bb)
3559 : {
3560 621 : gimple_stmt_iterator gsi;
3561 :
3562 3209 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
3563 : {
3564 1967 : gimple *stmt = gsi_stmt (gsi);
3565 1967 : bitmap bits;
3566 1967 : unsigned int loc;
3567 :
3568 1967 : if (is_tm_store (stmt))
3569 111 : bits = STORE_LOCAL (bb);
3570 1856 : else if (is_tm_load (stmt))
3571 221 : bits = READ_LOCAL (bb);
3572 : else
3573 1635 : continue;
3574 :
3575 332 : loc = tm_memopt_value_number (stmt, INSERT);
3576 332 : bitmap_set_bit (bits, loc);
3577 332 : if (dump_file)
3578 : {
3579 17 : fprintf (dump_file, "TM memopt (%s): value num=%d, BB=%d, addr=",
3580 11 : is_tm_load (stmt) ? "LOAD" : "STORE", loc,
3581 11 : gimple_bb (stmt)->index);
3582 11 : print_generic_expr (dump_file, gimple_call_arg (stmt, 0));
3583 11 : fprintf (dump_file, "\n");
3584 : }
3585 : }
3586 621 : }
3587 :
3588 : /* Prettily dump one of the memopt sets. BITS is the bitmap to dump. */
3589 :
3590 : static void
3591 96 : dump_tm_memopt_set (const char *set_name, bitmap bits)
3592 : {
3593 96 : unsigned i;
3594 96 : bitmap_iterator bi;
3595 96 : const char *comma = "";
3596 :
3597 96 : fprintf (dump_file, "TM memopt: %s: [", set_name);
3598 228 : EXECUTE_IF_SET_IN_BITMAP (bits, 0, i, bi)
3599 : {
3600 132 : hash_table<tm_memop_hasher>::iterator hi;
3601 132 : struct tm_memop *mem = NULL;
3602 :
3603 : /* Yeah, yeah, yeah. Whatever. This is just for debugging. */
3604 556 : FOR_EACH_HASH_TABLE_ELEMENT (*tm_memopt_value_numbers, mem, tm_memop_t, hi)
3605 344 : if (mem->value_id == i)
3606 : break;
3607 132 : gcc_assert (mem->value_id == i);
3608 132 : fprintf (dump_file, "%s", comma);
3609 132 : comma = ", ";
3610 132 : print_generic_expr (dump_file, mem->addr);
3611 : }
3612 96 : fprintf (dump_file, "]\n");
3613 96 : }
3614 :
3615 : /* Prettily dump all of the memopt sets in BLOCKS. */
3616 :
3617 : static void
3618 4 : dump_tm_memopt_sets (vec<basic_block> blocks)
3619 : {
3620 4 : size_t i;
3621 4 : basic_block bb;
3622 :
3623 20 : for (i = 0; blocks.iterate (i, &bb); ++i)
3624 : {
3625 16 : fprintf (dump_file, "------------BB %d---------\n", bb->index);
3626 16 : dump_tm_memopt_set ("STORE_LOCAL", STORE_LOCAL (bb));
3627 16 : dump_tm_memopt_set ("READ_LOCAL", READ_LOCAL (bb));
3628 16 : dump_tm_memopt_set ("STORE_AVAIL_IN", STORE_AVAIL_IN (bb));
3629 16 : dump_tm_memopt_set ("STORE_AVAIL_OUT", STORE_AVAIL_OUT (bb));
3630 16 : dump_tm_memopt_set ("READ_AVAIL_IN", READ_AVAIL_IN (bb));
3631 16 : dump_tm_memopt_set ("READ_AVAIL_OUT", READ_AVAIL_OUT (bb));
3632 : }
3633 4 : }
3634 :
3635 : /* Compute {STORE,READ}_AVAIL_IN for the basic block BB. */
3636 :
3637 : static void
3638 432 : tm_memopt_compute_avin (basic_block bb)
3639 : {
3640 432 : edge e;
3641 432 : unsigned ix;
3642 :
3643 : /* Seed with the AVOUT of any predecessor. */
3644 461 : for (ix = 0; ix < EDGE_COUNT (bb->preds); ix++)
3645 : {
3646 461 : e = EDGE_PRED (bb, ix);
3647 : /* Make sure we have already visited this BB, and is thus
3648 : initialized.
3649 :
3650 : If e->src->aux is NULL, this predecessor is actually on an
3651 : enclosing transaction. We only care about the current
3652 : transaction, so ignore it. */
3653 461 : if (e->src->aux && BB_VISITED_P (e->src))
3654 : {
3655 432 : bitmap_copy (STORE_AVAIL_IN (bb), STORE_AVAIL_OUT (e->src));
3656 432 : bitmap_copy (READ_AVAIL_IN (bb), READ_AVAIL_OUT (e->src));
3657 432 : break;
3658 : }
3659 : }
3660 :
3661 968 : for (; ix < EDGE_COUNT (bb->preds); ix++)
3662 : {
3663 536 : e = EDGE_PRED (bb, ix);
3664 536 : if (e->src->aux && BB_VISITED_P (e->src))
3665 : {
3666 513 : bitmap_and_into (STORE_AVAIL_IN (bb), STORE_AVAIL_OUT (e->src));
3667 513 : bitmap_and_into (READ_AVAIL_IN (bb), READ_AVAIL_OUT (e->src));
3668 : }
3669 : }
3670 :
3671 432 : BB_VISITED_P (bb) = true;
3672 432 : }
3673 :
3674 : /* Compute the STORE_ANTIC_IN for the basic block BB. */
3675 :
3676 : static void
3677 230 : tm_memopt_compute_antin (basic_block bb)
3678 : {
3679 230 : edge e;
3680 230 : unsigned ix;
3681 :
3682 : /* Seed with the ANTIC_OUT of any successor. */
3683 251 : for (ix = 0; ix < EDGE_COUNT (bb->succs); ix++)
3684 : {
3685 205 : e = EDGE_SUCC (bb, ix);
3686 : /* Make sure we have already visited this BB, and is thus
3687 : initialized. */
3688 205 : if (BB_VISITED_P (e->dest))
3689 : {
3690 184 : bitmap_copy (STORE_ANTIC_IN (bb), STORE_ANTIC_OUT (e->dest));
3691 184 : break;
3692 : }
3693 : }
3694 :
3695 512 : for (; ix < EDGE_COUNT (bb->succs); ix++)
3696 : {
3697 282 : e = EDGE_SUCC (bb, ix);
3698 282 : if (BB_VISITED_P (e->dest))
3699 282 : bitmap_and_into (STORE_ANTIC_IN (bb), STORE_ANTIC_OUT (e->dest));
3700 : }
3701 :
3702 230 : BB_VISITED_P (bb) = true;
3703 230 : }
3704 :
3705 : /* Compute the AVAIL sets for every basic block in BLOCKS.
3706 :
3707 : We compute {STORE,READ}_AVAIL_{OUT,IN} as follows:
3708 :
3709 : AVAIL_OUT[bb] = union (AVAIL_IN[bb], LOCAL[bb])
3710 : AVAIL_IN[bb] = intersect (AVAIL_OUT[predecessors])
3711 :
3712 : This is basically what we do in lcm's compute_available(), but here
3713 : we calculate two sets of sets (one for STOREs and one for READs),
3714 : and we work on a region instead of the entire CFG.
3715 :
3716 : REGION is the TM region.
3717 : BLOCKS are the basic blocks in the region. */
3718 :
3719 : static void
3720 221 : tm_memopt_compute_available (struct tm_region *region,
3721 : vec<basic_block> blocks)
3722 : {
3723 221 : edge e;
3724 221 : basic_block *worklist, *qin, *qout, *qend, bb;
3725 221 : unsigned int qlen, i;
3726 221 : edge_iterator ei;
3727 221 : bool changed;
3728 :
3729 : /* Allocate a worklist array/queue. Entries are only added to the
3730 : list if they were not already on the list. So the size is
3731 : bounded by the number of basic blocks in the region. */
3732 221 : gcc_assert (!blocks.is_empty ());
3733 221 : qlen = blocks.length () - 1;
3734 221 : qin = qout = worklist = XNEWVEC (basic_block, qlen);
3735 :
3736 : /* Put every block in the region on the worklist. */
3737 1063 : for (i = 0; blocks.iterate (i, &bb); ++i)
3738 : {
3739 : /* Seed AVAIL_OUT with the LOCAL set. */
3740 621 : bitmap_ior_into (STORE_AVAIL_OUT (bb), STORE_LOCAL (bb));
3741 621 : bitmap_ior_into (READ_AVAIL_OUT (bb), READ_LOCAL (bb));
3742 :
3743 621 : AVAIL_IN_WORKLIST_P (bb) = true;
3744 : /* No need to insert the entry block, since it has an AVIN of
3745 : null, and an AVOUT that has already been seeded in. */
3746 621 : if (bb != region->entry_block)
3747 400 : *qin++ = bb;
3748 : }
3749 :
3750 : /* The entry block has been initialized with the local sets. */
3751 221 : BB_VISITED_P (region->entry_block) = true;
3752 :
3753 221 : qin = worklist;
3754 221 : qend = &worklist[qlen];
3755 :
3756 : /* Iterate until the worklist is empty. */
3757 653 : while (qlen)
3758 : {
3759 : /* Take the first entry off the worklist. */
3760 432 : bb = *qout++;
3761 432 : qlen--;
3762 :
3763 432 : if (qout >= qend)
3764 95 : qout = worklist;
3765 :
3766 : /* This block can be added to the worklist again if necessary. */
3767 432 : AVAIL_IN_WORKLIST_P (bb) = false;
3768 432 : tm_memopt_compute_avin (bb);
3769 :
3770 : /* Note: We do not add the LOCAL sets here because we already
3771 : seeded the AVAIL_OUT sets with them. */
3772 432 : changed = bitmap_ior_into (STORE_AVAIL_OUT (bb), STORE_AVAIL_IN (bb));
3773 432 : changed |= bitmap_ior_into (READ_AVAIL_OUT (bb), READ_AVAIL_IN (bb));
3774 432 : if (changed
3775 432 : && (region->exit_blocks == NULL
3776 150 : || !bitmap_bit_p (region->exit_blocks, bb->index)))
3777 : /* If the out state of this block changed, then we need to add
3778 : its successors to the worklist if they are not already in. */
3779 454 : FOR_EACH_EDGE (e, ei, bb->succs)
3780 236 : if (!AVAIL_IN_WORKLIST_P (e->dest)
3781 32 : && e->dest != EXIT_BLOCK_PTR_FOR_FN (cfun))
3782 : {
3783 32 : *qin++ = e->dest;
3784 32 : AVAIL_IN_WORKLIST_P (e->dest) = true;
3785 32 : qlen++;
3786 :
3787 32 : if (qin >= qend)
3788 0 : qin = worklist;
3789 : }
3790 : }
3791 :
3792 221 : free (worklist);
3793 :
3794 221 : if (dump_file)
3795 2 : dump_tm_memopt_sets (blocks);
3796 221 : }
3797 :
3798 : /* Compute ANTIC sets for every basic block in BLOCKS.
3799 :
3800 : We compute STORE_ANTIC_OUT as follows:
3801 :
3802 : STORE_ANTIC_OUT[bb] = union(STORE_ANTIC_IN[bb], STORE_LOCAL[bb])
3803 : STORE_ANTIC_IN[bb] = intersect(STORE_ANTIC_OUT[successors])
3804 :
3805 : REGION is the TM region.
3806 : BLOCKS are the basic blocks in the region. */
3807 :
3808 : static void
3809 221 : tm_memopt_compute_antic (struct tm_region *region,
3810 : vec<basic_block> blocks)
3811 : {
3812 221 : edge e;
3813 221 : basic_block *worklist, *qin, *qout, *qend, bb;
3814 221 : unsigned int qlen;
3815 221 : int i;
3816 221 : edge_iterator ei;
3817 :
3818 : /* Allocate a worklist array/queue. Entries are only added to the
3819 : list if they were not already on the list. So the size is
3820 : bounded by the number of basic blocks in the region. */
3821 442 : qin = qout = worklist = XNEWVEC (basic_block, blocks.length ());
3822 :
3823 1063 : for (qlen = 0, i = blocks.length () - 1; i >= 0; --i)
3824 : {
3825 621 : bb = blocks[i];
3826 :
3827 : /* Seed ANTIC_OUT with the LOCAL set. */
3828 621 : bitmap_ior_into (STORE_ANTIC_OUT (bb), STORE_LOCAL (bb));
3829 :
3830 : /* Put every block in the region on the worklist. */
3831 621 : AVAIL_IN_WORKLIST_P (bb) = true;
3832 : /* No need to insert exit blocks, since their ANTIC_IN is NULL,
3833 : and their ANTIC_OUT has already been seeded in. */
3834 621 : if (region->exit_blocks
3835 621 : && !bitmap_bit_p (region->exit_blocks, bb->index))
3836 : {
3837 230 : qlen++;
3838 230 : *qin++ = bb;
3839 : }
3840 : }
3841 :
3842 : /* The exit blocks have been initialized with the local sets. */
3843 221 : if (region->exit_blocks)
3844 : {
3845 181 : unsigned int i;
3846 181 : bitmap_iterator bi;
3847 381 : EXECUTE_IF_SET_IN_BITMAP (region->exit_blocks, 0, i, bi)
3848 200 : BB_VISITED_P (BASIC_BLOCK_FOR_FN (cfun, i)) = true;
3849 : }
3850 :
3851 221 : qin = worklist;
3852 221 : qend = &worklist[qlen];
3853 :
3854 : /* Iterate until the worklist is empty. */
3855 451 : while (qlen)
3856 : {
3857 : /* Take the first entry off the worklist. */
3858 230 : bb = *qout++;
3859 230 : qlen--;
3860 :
3861 230 : if (qout >= qend)
3862 62 : qout = worklist;
3863 :
3864 : /* This block can be added to the worklist again if necessary. */
3865 230 : AVAIL_IN_WORKLIST_P (bb) = false;
3866 230 : tm_memopt_compute_antin (bb);
3867 :
3868 : /* Note: We do not add the LOCAL sets here because we already
3869 : seeded the ANTIC_OUT sets with them. */
3870 230 : if (bitmap_ior_into (STORE_ANTIC_OUT (bb), STORE_ANTIC_IN (bb))
3871 230 : && bb != region->entry_block)
3872 : /* If the out state of this block changed, then we need to add
3873 : its predecessors to the worklist if they are not already in. */
3874 0 : FOR_EACH_EDGE (e, ei, bb->preds)
3875 0 : if (!AVAIL_IN_WORKLIST_P (e->src))
3876 : {
3877 0 : *qin++ = e->src;
3878 0 : AVAIL_IN_WORKLIST_P (e->src) = true;
3879 0 : qlen++;
3880 :
3881 0 : if (qin >= qend)
3882 0 : qin = worklist;
3883 : }
3884 : }
3885 :
3886 221 : free (worklist);
3887 :
3888 221 : if (dump_file)
3889 2 : dump_tm_memopt_sets (blocks);
3890 221 : }
3891 :
3892 : /* Offsets of load variants from TM_LOAD. For example,
3893 : BUILT_IN_TM_LOAD_RAR* is an offset of 1 from BUILT_IN_TM_LOAD*.
3894 : See gtm-builtins.def. */
3895 : #define TRANSFORM_RAR 1
3896 : #define TRANSFORM_RAW 2
3897 : #define TRANSFORM_RFW 3
3898 : /* Offsets of store variants from TM_STORE. */
3899 : #define TRANSFORM_WAR 1
3900 : #define TRANSFORM_WAW 2
3901 :
3902 : /* Inform about a load/store optimization. */
3903 :
3904 : static void
3905 87 : dump_tm_memopt_transform (gimple *stmt)
3906 : {
3907 87 : if (dump_file)
3908 : {
3909 7 : fprintf (dump_file, "TM memopt: transforming: ");
3910 7 : print_gimple_stmt (dump_file, stmt, 0);
3911 7 : fprintf (dump_file, "\n");
3912 : }
3913 87 : }
3914 :
3915 : /* Perform a read/write optimization. Replaces the TM builtin in STMT
3916 : by a builtin that is OFFSET entries down in the builtins table in
3917 : gtm-builtins.def. */
3918 :
3919 : static void
3920 87 : tm_memopt_transform_stmt (unsigned int offset,
3921 : gcall *stmt,
3922 : gimple_stmt_iterator *gsi)
3923 : {
3924 87 : tree fn = gimple_call_fn (stmt);
3925 87 : gcc_assert (TREE_CODE (fn) == ADDR_EXPR);
3926 87 : TREE_OPERAND (fn, 0)
3927 174 : = builtin_decl_explicit ((enum built_in_function)
3928 87 : (DECL_FUNCTION_CODE (TREE_OPERAND (fn, 0))
3929 87 : + offset));
3930 87 : gimple_call_set_fn (stmt, fn);
3931 87 : gsi_replace (gsi, stmt, true);
3932 87 : dump_tm_memopt_transform (stmt);
3933 87 : }
3934 :
3935 : /* Perform the actual TM memory optimization transformations in the
3936 : basic blocks in BLOCKS. */
3937 :
3938 : static void
3939 221 : tm_memopt_transform_blocks (vec<basic_block> blocks)
3940 : {
3941 221 : size_t i;
3942 221 : basic_block bb;
3943 221 : gimple_stmt_iterator gsi;
3944 :
3945 842 : for (i = 0; blocks.iterate (i, &bb); ++i)
3946 : {
3947 3209 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
3948 : {
3949 1967 : gimple *stmt = gsi_stmt (gsi);
3950 1967 : bitmap read_avail = READ_AVAIL_IN (bb);
3951 1967 : bitmap store_avail = STORE_AVAIL_IN (bb);
3952 1967 : bitmap store_antic = STORE_ANTIC_OUT (bb);
3953 1967 : unsigned int loc;
3954 :
3955 1967 : if (is_tm_simple_load (stmt))
3956 : {
3957 221 : gcall *call_stmt = as_a <gcall *> (stmt);
3958 221 : loc = tm_memopt_value_number (stmt, NO_INSERT);
3959 221 : if (store_avail && bitmap_bit_p (store_avail, loc))
3960 4 : tm_memopt_transform_stmt (TRANSFORM_RAW, call_stmt, &gsi);
3961 217 : else if (store_antic && bitmap_bit_p (store_antic, loc))
3962 : {
3963 39 : tm_memopt_transform_stmt (TRANSFORM_RFW, call_stmt, &gsi);
3964 39 : bitmap_set_bit (store_avail, loc);
3965 : }
3966 178 : else if (read_avail && bitmap_bit_p (read_avail, loc))
3967 0 : tm_memopt_transform_stmt (TRANSFORM_RAR, call_stmt, &gsi);
3968 : else
3969 178 : bitmap_set_bit (read_avail, loc);
3970 : }
3971 1746 : else if (is_tm_simple_store (stmt))
3972 : {
3973 111 : gcall *call_stmt = as_a <gcall *> (stmt);
3974 111 : loc = tm_memopt_value_number (stmt, NO_INSERT);
3975 111 : if (store_avail && bitmap_bit_p (store_avail, loc))
3976 40 : tm_memopt_transform_stmt (TRANSFORM_WAW, call_stmt, &gsi);
3977 : else
3978 : {
3979 71 : if (read_avail && bitmap_bit_p (read_avail, loc))
3980 4 : tm_memopt_transform_stmt (TRANSFORM_WAR, call_stmt, &gsi);
3981 71 : bitmap_set_bit (store_avail, loc);
3982 : }
3983 : }
3984 : }
3985 : }
3986 221 : }
3987 :
3988 : /* Return a new set of bitmaps for a BB. */
3989 :
3990 : static struct tm_memopt_bitmaps *
3991 621 : tm_memopt_init_sets (void)
3992 : {
3993 621 : struct tm_memopt_bitmaps *b
3994 621 : = XOBNEW (&tm_memopt_obstack.obstack, struct tm_memopt_bitmaps);
3995 621 : b->store_avail_in = BITMAP_ALLOC (&tm_memopt_obstack);
3996 621 : b->store_avail_out = BITMAP_ALLOC (&tm_memopt_obstack);
3997 621 : b->store_antic_in = BITMAP_ALLOC (&tm_memopt_obstack);
3998 621 : b->store_antic_out = BITMAP_ALLOC (&tm_memopt_obstack);
3999 621 : b->store_avail_out = BITMAP_ALLOC (&tm_memopt_obstack);
4000 621 : b->read_avail_in = BITMAP_ALLOC (&tm_memopt_obstack);
4001 621 : b->read_avail_out = BITMAP_ALLOC (&tm_memopt_obstack);
4002 621 : b->read_local = BITMAP_ALLOC (&tm_memopt_obstack);
4003 621 : b->store_local = BITMAP_ALLOC (&tm_memopt_obstack);
4004 621 : return b;
4005 : }
4006 :
4007 : /* Free sets computed for each BB. */
4008 :
4009 : static void
4010 221 : tm_memopt_free_sets (vec<basic_block> blocks)
4011 : {
4012 221 : size_t i;
4013 221 : basic_block bb;
4014 :
4015 842 : for (i = 0; blocks.iterate (i, &bb); ++i)
4016 621 : bb->aux = NULL;
4017 221 : }
4018 :
4019 : /* Clear the visited bit for every basic block in BLOCKS. */
4020 :
4021 : static void
4022 442 : tm_memopt_clear_visited (vec<basic_block> blocks)
4023 : {
4024 442 : size_t i;
4025 442 : basic_block bb;
4026 :
4027 1684 : for (i = 0; blocks.iterate (i, &bb); ++i)
4028 1242 : BB_VISITED_P (bb) = false;
4029 442 : }
4030 :
4031 : /* Replace TM load/stores with hints for the runtime. We handle
4032 : things like read-after-write, write-after-read, read-after-read,
4033 : read-for-write, etc. */
4034 :
4035 : static unsigned int
4036 163 : execute_tm_memopt (void)
4037 : {
4038 163 : struct tm_region *region;
4039 163 : vec<basic_block> bbs;
4040 :
4041 163 : tm_memopt_value_id = 0;
4042 163 : tm_memopt_value_numbers = new hash_table<tm_memop_hasher> (10);
4043 :
4044 384 : for (region = all_tm_regions; region; region = region->next)
4045 : {
4046 : /* All the TM stores/loads in the current region. */
4047 221 : size_t i;
4048 221 : basic_block bb;
4049 :
4050 221 : bitmap_obstack_initialize (&tm_memopt_obstack);
4051 :
4052 : /* Save all BBs for the current region. */
4053 221 : bbs = get_tm_region_blocks (region->entry_block,
4054 : region->exit_blocks,
4055 : region->irr_blocks,
4056 : NULL,
4057 : false);
4058 :
4059 : /* Collect all the memory operations. */
4060 1063 : for (i = 0; bbs.iterate (i, &bb); ++i)
4061 : {
4062 621 : bb->aux = tm_memopt_init_sets ();
4063 621 : tm_memopt_accumulate_memops (bb);
4064 : }
4065 :
4066 : /* Solve data flow equations and transform each block accordingly. */
4067 221 : tm_memopt_clear_visited (bbs);
4068 221 : tm_memopt_compute_available (region, bbs);
4069 221 : tm_memopt_clear_visited (bbs);
4070 221 : tm_memopt_compute_antic (region, bbs);
4071 221 : tm_memopt_transform_blocks (bbs);
4072 :
4073 221 : tm_memopt_free_sets (bbs);
4074 221 : bbs.release ();
4075 221 : bitmap_obstack_release (&tm_memopt_obstack);
4076 384 : tm_memopt_value_numbers->empty ();
4077 : }
4078 :
4079 163 : delete tm_memopt_value_numbers;
4080 163 : tm_memopt_value_numbers = NULL;
4081 163 : return 0;
4082 : }
4083 :
4084 : namespace {
4085 :
4086 : const pass_data pass_data_tm_memopt =
4087 : {
4088 : GIMPLE_PASS, /* type */
4089 : "tmmemopt", /* name */
4090 : OPTGROUP_NONE, /* optinfo_flags */
4091 : TV_TRANS_MEM, /* tv_id */
4092 : ( PROP_ssa | PROP_cfg ), /* properties_required */
4093 : 0, /* properties_provided */
4094 : 0, /* properties_destroyed */
4095 : 0, /* todo_flags_start */
4096 : 0, /* todo_flags_finish */
4097 : };
4098 :
4099 : class pass_tm_memopt : public gimple_opt_pass
4100 : {
4101 : public:
4102 285722 : pass_tm_memopt (gcc::context *ctxt)
4103 571444 : : gimple_opt_pass (pass_data_tm_memopt, ctxt)
4104 : {}
4105 :
4106 : /* opt_pass methods: */
4107 479 : bool gate (function *) final override { return flag_tm && optimize > 0; }
4108 163 : unsigned int execute (function *) final override
4109 : {
4110 163 : return execute_tm_memopt ();
4111 : }
4112 :
4113 : }; // class pass_tm_memopt
4114 :
4115 : } // anon namespace
4116 :
4117 : gimple_opt_pass *
4118 285722 : make_pass_tm_memopt (gcc::context *ctxt)
4119 : {
4120 285722 : return new pass_tm_memopt (ctxt);
4121 : }
4122 :
4123 :
4124 : /* Interprocedual analysis for the creation of transactional clones.
4125 : The aim of this pass is to find which functions are referenced in
4126 : a non-irrevocable transaction context, and for those over which
4127 : we have control (or user directive), create a version of the
4128 : function which uses only the transactional interface to reference
4129 : protected memories. This analysis proceeds in several steps:
4130 :
4131 : (1) Collect the set of all possible transactional clones:
4132 :
4133 : (a) For all local public functions marked tm_callable, push
4134 : it onto the tm_callee queue.
4135 :
4136 : (b) For all local functions, scan for calls in transaction blocks.
4137 : Push the caller and callee onto the tm_caller and tm_callee
4138 : queues. Count the number of callers for each callee.
4139 :
4140 : (c) For each local function on the callee list, assume we will
4141 : create a transactional clone. Push *all* calls onto the
4142 : callee queues; count the number of clone callers separately
4143 : to the number of original callers.
4144 :
4145 : (2) Propagate irrevocable status up the dominator tree:
4146 :
4147 : (a) Any external function on the callee list that is not marked
4148 : tm_callable is irrevocable. Push all callers of such onto
4149 : a worklist.
4150 :
4151 : (b) For each function on the worklist, mark each block that
4152 : contains an irrevocable call. Use the AND operator to
4153 : propagate that mark up the dominator tree.
4154 :
4155 : (c) If we reach the entry block for a possible transactional
4156 : clone, then the transactional clone is irrevocable, and
4157 : we should not create the clone after all. Push all
4158 : callers onto the worklist.
4159 :
4160 : (d) Place tm_irrevocable calls at the beginning of the relevant
4161 : blocks. Special case here is the entry block for the entire
4162 : transaction region; there we mark it GTMA_DOES_GO_IRREVOCABLE for
4163 : the library to begin the region in serial mode. Decrement
4164 : the call count for all callees in the irrevocable region.
4165 :
4166 : (3) Create the transactional clones:
4167 :
4168 : Any tm_callee that still has a non-zero call count is cloned.
4169 : */
4170 :
4171 : /* This structure is stored in the AUX field of each cgraph_node. */
4172 : struct tm_ipa_cg_data
4173 : {
4174 : /* The clone of the function that got created. */
4175 : struct cgraph_node *clone;
4176 :
4177 : /* The tm regions in the normal function. */
4178 : struct tm_region *all_tm_regions;
4179 :
4180 : /* The blocks of the normal/clone functions that contain irrevocable
4181 : calls, or blocks that are post-dominated by irrevocable calls. */
4182 : bitmap irrevocable_blocks_normal;
4183 : bitmap irrevocable_blocks_clone;
4184 :
4185 : /* The blocks of the normal function that are involved in transactions. */
4186 : bitmap transaction_blocks_normal;
4187 :
4188 : /* The number of callers to the transactional clone of this function
4189 : from normal and transactional clones respectively. */
4190 : unsigned tm_callers_normal;
4191 : unsigned tm_callers_clone;
4192 :
4193 : /* True if all calls to this function's transactional clone
4194 : are irrevocable. Also automatically true if the function
4195 : has no transactional clone. */
4196 : bool is_irrevocable;
4197 :
4198 : /* Flags indicating the presence of this function in various queues. */
4199 : bool in_callee_queue;
4200 : bool in_worklist;
4201 :
4202 : /* Flags indicating the kind of scan desired while in the worklist. */
4203 : bool want_irr_scan_normal;
4204 : };
4205 :
4206 : typedef vec<cgraph_node *> cgraph_node_queue;
4207 :
4208 : /* Return the ipa data associated with NODE, allocating zeroed memory
4209 : if necessary. TRAVERSE_ALIASES is true if we must traverse aliases
4210 : and set *NODE accordingly. */
4211 :
4212 : static struct tm_ipa_cg_data *
4213 9068 : get_cg_data (struct cgraph_node **node, bool traverse_aliases)
4214 : {
4215 9068 : struct tm_ipa_cg_data *d;
4216 :
4217 9068 : if (traverse_aliases && (*node)->alias)
4218 185 : *node = (*node)->get_alias_target ();
4219 :
4220 9068 : d = (struct tm_ipa_cg_data *) (*node)->aux;
4221 :
4222 9068 : if (d == NULL)
4223 : {
4224 2926 : d = (struct tm_ipa_cg_data *)
4225 1463 : obstack_alloc (&tm_obstack.obstack, sizeof (*d));
4226 1463 : (*node)->aux = (void *) d;
4227 1463 : memset (d, 0, sizeof (*d));
4228 : }
4229 :
4230 9068 : return d;
4231 : }
4232 :
4233 : /* Add NODE to the end of QUEUE, unless IN_QUEUE_P indicates that
4234 : it is already present. */
4235 :
4236 : static void
4237 1878 : maybe_push_queue (struct cgraph_node *node,
4238 : cgraph_node_queue *queue_p, bool *in_queue_p)
4239 : {
4240 0 : if (!*in_queue_p)
4241 : {
4242 1480 : *in_queue_p = true;
4243 1351 : queue_p->safe_push (node);
4244 : }
4245 0 : }
4246 :
4247 : /* A subroutine of ipa_tm_scan_calls_transaction and ipa_tm_scan_calls_clone.
4248 : Queue all callees within block BB. */
4249 :
4250 : static void
4251 1476 : ipa_tm_scan_calls_block (cgraph_node_queue *callees_p,
4252 : basic_block bb, bool for_clone)
4253 : {
4254 1476 : gimple_stmt_iterator gsi;
4255 :
4256 7276 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4257 : {
4258 4324 : gimple *stmt = gsi_stmt (gsi);
4259 5613 : if (is_gimple_call (stmt) && !is_tm_pure_call (stmt))
4260 : {
4261 1134 : tree fndecl = gimple_call_fndecl (stmt);
4262 1134 : if (fndecl)
4263 : {
4264 1117 : struct tm_ipa_cg_data *d;
4265 1117 : unsigned *pcallers;
4266 1117 : struct cgraph_node *node;
4267 :
4268 1117 : if (is_tm_ending_fndecl (fndecl))
4269 624 : continue;
4270 627 : if (find_tm_replacement_function (fndecl))
4271 134 : continue;
4272 :
4273 493 : node = cgraph_node::get (fndecl);
4274 493 : gcc_assert (node != NULL);
4275 493 : d = get_cg_data (&node, true);
4276 :
4277 493 : pcallers = (for_clone ? &d->tm_callers_clone
4278 : : &d->tm_callers_normal);
4279 493 : *pcallers += 1;
4280 :
4281 839 : maybe_push_queue (node, callees_p, &d->in_callee_queue);
4282 : }
4283 : }
4284 : }
4285 1476 : }
4286 :
4287 : /* Scan all calls in NODE that are within a transaction region,
4288 : and push the resulting nodes into the callee queue. */
4289 :
4290 : static void
4291 284 : ipa_tm_scan_calls_transaction (struct tm_ipa_cg_data *d,
4292 : cgraph_node_queue *callees_p)
4293 : {
4294 284 : d->transaction_blocks_normal = BITMAP_ALLOC (&tm_obstack);
4295 284 : d->all_tm_regions = all_tm_regions;
4296 :
4297 634 : for (tm_region *r = all_tm_regions; r; r = r->next)
4298 : {
4299 350 : vec<basic_block> bbs;
4300 350 : basic_block bb;
4301 350 : unsigned i;
4302 :
4303 350 : bbs = get_tm_region_blocks (r->entry_block, r->exit_blocks, NULL,
4304 : d->transaction_blocks_normal, false, false);
4305 :
4306 1464 : FOR_EACH_VEC_ELT (bbs, i, bb)
4307 764 : ipa_tm_scan_calls_block (callees_p, bb, false);
4308 :
4309 350 : bbs.release ();
4310 : }
4311 284 : }
4312 :
4313 : /* Scan all calls in NODE as if this is the transactional clone,
4314 : and push the destinations into the callee queue. */
4315 :
4316 : static void
4317 337 : ipa_tm_scan_calls_clone (struct cgraph_node *node,
4318 : cgraph_node_queue *callees_p)
4319 : {
4320 337 : struct function *fn = DECL_STRUCT_FUNCTION (node->decl);
4321 337 : basic_block bb;
4322 :
4323 1049 : FOR_EACH_BB_FN (bb, fn)
4324 712 : ipa_tm_scan_calls_block (callees_p, bb, true);
4325 337 : }
4326 :
4327 : /* The function NODE has been detected to be irrevocable. Push all
4328 : of its callers onto WORKLIST for the purpose of re-scanning them. */
4329 :
4330 : static void
4331 124 : ipa_tm_note_irrevocable (struct cgraph_node *node,
4332 : cgraph_node_queue *worklist_p)
4333 : {
4334 124 : struct tm_ipa_cg_data *d = get_cg_data (&node, true);
4335 124 : struct cgraph_edge *e;
4336 :
4337 124 : d->is_irrevocable = true;
4338 :
4339 317 : for (e = node->callers; e ; e = e->next_caller)
4340 : {
4341 193 : basic_block bb;
4342 193 : struct cgraph_node *caller;
4343 :
4344 : /* Don't examine recursive calls. */
4345 193 : if (e->caller == node)
4346 21 : continue;
4347 : /* Even if we think we can go irrevocable, believe the user
4348 : above all. */
4349 193 : if (is_tm_safe_or_pure (e->caller->decl))
4350 21 : continue;
4351 :
4352 172 : caller = e->caller;
4353 172 : d = get_cg_data (&caller, true);
4354 :
4355 : /* Check if the callee is in a transactional region. If so,
4356 : schedule the function for normal re-scan as well. */
4357 172 : bb = gimple_bb (e->call_stmt);
4358 172 : gcc_assert (bb != NULL);
4359 172 : if (d->transaction_blocks_normal
4360 172 : && bitmap_bit_p (d->transaction_blocks_normal, bb->index))
4361 63 : d->want_irr_scan_normal = true;
4362 :
4363 195 : maybe_push_queue (caller, worklist_p, &d->in_worklist);
4364 : }
4365 124 : }
4366 :
4367 : /* A subroutine of ipa_tm_scan_irr_blocks; return true iff any statement
4368 : within the block is irrevocable. */
4369 :
4370 : static bool
4371 1750 : ipa_tm_scan_irr_block (basic_block bb)
4372 : {
4373 1750 : gimple_stmt_iterator gsi;
4374 1750 : tree fn;
4375 :
4376 7491 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4377 : {
4378 4119 : gimple *stmt = gsi_stmt (gsi);
4379 4119 : switch (gimple_code (stmt))
4380 : {
4381 1610 : case GIMPLE_ASSIGN:
4382 1610 : if (gimple_assign_single_p (stmt))
4383 : {
4384 1262 : tree lhs = gimple_assign_lhs (stmt);
4385 1262 : tree rhs = gimple_assign_rhs1 (stmt);
4386 1262 : if (volatile_lvalue_p (lhs) || volatile_lvalue_p (rhs))
4387 : return true;
4388 : }
4389 : break;
4390 :
4391 1181 : case GIMPLE_CALL:
4392 1181 : {
4393 1181 : tree lhs = gimple_call_lhs (stmt);
4394 1181 : if (lhs && volatile_lvalue_p (lhs))
4395 : return true;
4396 :
4397 1181 : if (is_tm_pure_call (stmt))
4398 : break;
4399 :
4400 1048 : fn = gimple_call_fn (stmt);
4401 :
4402 : /* Functions with the attribute are by definition irrevocable. */
4403 1048 : if (is_tm_irrevocable (fn))
4404 : return true;
4405 :
4406 : /* For direct function calls, go ahead and check for replacement
4407 : functions, or transitive irrevocable functions. For indirect
4408 : functions, we'll ask the runtime. */
4409 1039 : if (TREE_CODE (fn) == ADDR_EXPR)
4410 : {
4411 1022 : struct tm_ipa_cg_data *d;
4412 1022 : struct cgraph_node *node;
4413 :
4414 1022 : fn = TREE_OPERAND (fn, 0);
4415 1022 : if (is_tm_ending_fndecl (fn))
4416 : break;
4417 600 : if (find_tm_replacement_function (fn))
4418 : break;
4419 :
4420 466 : node = cgraph_node::get (fn);
4421 466 : d = get_cg_data (&node, true);
4422 :
4423 : /* Return true if irrevocable, but above all, believe
4424 : the user. */
4425 466 : if (d->is_irrevocable
4426 466 : && !is_tm_safe_or_pure (fn))
4427 94 : return true;
4428 : }
4429 : break;
4430 : }
4431 :
4432 18 : case GIMPLE_ASM:
4433 : /* ??? The Approved Method of indicating that an inline
4434 : assembly statement is not relevant to the transaction
4435 : is to wrap it in a __tm_waiver block. This is not
4436 : yet implemented, so we can't check for it. */
4437 18 : if (is_tm_safe (current_function_decl))
4438 1 : error_at (gimple_location (stmt),
4439 : "%<asm%> not allowed in %<transaction_safe%> function");
4440 : return true;
4441 :
4442 : default:
4443 : break;
4444 : }
4445 : }
4446 :
4447 : return false;
4448 : }
4449 :
4450 : /* For each of the blocks seeded witin PQUEUE, walk the CFG looking
4451 : for new irrevocable blocks, marking them in NEW_IRR. Don't bother
4452 : scanning past OLD_IRR or EXIT_BLOCKS. */
4453 :
4454 : static bool
4455 708 : ipa_tm_scan_irr_blocks (vec<basic_block> *pqueue, bitmap new_irr,
4456 : bitmap old_irr, bitmap exit_blocks)
4457 : {
4458 708 : bool any_new_irr = false;
4459 708 : edge e;
4460 708 : edge_iterator ei;
4461 708 : bitmap visited_blocks = BITMAP_ALLOC (NULL);
4462 :
4463 1750 : do
4464 : {
4465 1750 : basic_block bb = pqueue->pop ();
4466 :
4467 : /* Don't re-scan blocks we know already are irrevocable. */
4468 1750 : if (old_irr && bitmap_bit_p (old_irr, bb->index))
4469 0 : continue;
4470 :
4471 1750 : if (ipa_tm_scan_irr_block (bb))
4472 : {
4473 128 : bitmap_set_bit (new_irr, bb->index);
4474 128 : any_new_irr = true;
4475 : }
4476 1622 : else if (exit_blocks == NULL || !bitmap_bit_p (exit_blocks, bb->index))
4477 : {
4478 2517 : FOR_EACH_EDGE (e, ei, bb->succs)
4479 1219 : if (!bitmap_bit_p (visited_blocks, e->dest->index))
4480 : {
4481 1042 : bitmap_set_bit (visited_blocks, e->dest->index);
4482 1042 : pqueue->safe_push (e->dest);
4483 : }
4484 : }
4485 : }
4486 3500 : while (!pqueue->is_empty ());
4487 :
4488 708 : BITMAP_FREE (visited_blocks);
4489 :
4490 708 : return any_new_irr;
4491 : }
4492 :
4493 : /* Propagate the irrevocable property both up and down the dominator tree.
4494 : BB is the current block being scanned; EXIT_BLOCKS are the edges of the
4495 : TM regions; OLD_IRR are the results of a previous scan of the dominator
4496 : tree which has been fully propagated; NEW_IRR is the set of new blocks
4497 : which are gaining the irrevocable property during the current scan. */
4498 :
4499 : static void
4500 124 : ipa_tm_propagate_irr (basic_block entry_block, bitmap new_irr,
4501 : bitmap old_irr, bitmap exit_blocks)
4502 : {
4503 124 : vec<basic_block> bbs;
4504 124 : bitmap all_region_blocks;
4505 :
4506 : /* If this block is in the old set, no need to rescan. */
4507 124 : if (old_irr && bitmap_bit_p (old_irr, entry_block->index))
4508 0 : return;
4509 :
4510 124 : all_region_blocks = BITMAP_ALLOC (&tm_obstack);
4511 124 : bbs = get_tm_region_blocks (entry_block, exit_blocks, NULL,
4512 : all_region_blocks, false);
4513 376 : do
4514 : {
4515 376 : basic_block bb = bbs.pop ();
4516 376 : bool this_irr = bitmap_bit_p (new_irr, bb->index);
4517 376 : bool all_son_irr = false;
4518 376 : edge_iterator ei;
4519 376 : edge e;
4520 :
4521 : /* Propagate up. If my children are, I am too, but we must have
4522 : at least one child that is. */
4523 376 : if (!this_irr)
4524 : {
4525 267 : FOR_EACH_EDGE (e, ei, bb->succs)
4526 : {
4527 162 : if (!bitmap_bit_p (new_irr, e->dest->index))
4528 : {
4529 : all_son_irr = false;
4530 : break;
4531 : }
4532 : else
4533 19 : all_son_irr = true;
4534 : }
4535 248 : if (all_son_irr)
4536 : {
4537 : /* Add block to new_irr if it hasn't already been processed. */
4538 7 : if (!old_irr || !bitmap_bit_p (old_irr, bb->index))
4539 : {
4540 7 : bitmap_set_bit (new_irr, bb->index);
4541 7 : this_irr = true;
4542 : }
4543 : }
4544 : }
4545 :
4546 : /* Propagate down to everyone we immediately dominate. */
4547 7 : if (this_irr)
4548 : {
4549 135 : basic_block son;
4550 135 : for (son = first_dom_son (CDI_DOMINATORS, bb);
4551 225 : son;
4552 90 : son = next_dom_son (CDI_DOMINATORS, son))
4553 : {
4554 : /* Make sure block is actually in a TM region, and it
4555 : isn't already in old_irr. */
4556 2 : if ((!old_irr || !bitmap_bit_p (old_irr, son->index))
4557 91 : && bitmap_bit_p (all_region_blocks, son->index))
4558 89 : bitmap_set_bit (new_irr, son->index);
4559 : }
4560 : }
4561 : }
4562 500 : while (!bbs.is_empty ());
4563 :
4564 124 : BITMAP_FREE (all_region_blocks);
4565 124 : bbs.release ();
4566 : }
4567 :
4568 : static void
4569 216 : ipa_tm_decrement_clone_counts (basic_block bb, bool for_clone)
4570 : {
4571 216 : gimple_stmt_iterator gsi;
4572 :
4573 1120 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4574 : {
4575 688 : gimple *stmt = gsi_stmt (gsi);
4576 987 : if (is_gimple_call (stmt) && !is_tm_pure_call (stmt))
4577 : {
4578 264 : tree fndecl = gimple_call_fndecl (stmt);
4579 264 : if (fndecl)
4580 : {
4581 264 : struct tm_ipa_cg_data *d;
4582 264 : unsigned *pcallers;
4583 264 : struct cgraph_node *tnode;
4584 :
4585 264 : if (is_tm_ending_fndecl (fndecl))
4586 65 : continue;
4587 199 : if (find_tm_replacement_function (fndecl))
4588 0 : continue;
4589 :
4590 199 : tnode = cgraph_node::get (fndecl);
4591 199 : d = get_cg_data (&tnode, true);
4592 :
4593 199 : pcallers = (for_clone ? &d->tm_callers_clone
4594 : : &d->tm_callers_normal);
4595 :
4596 199 : gcc_assert (*pcallers > 0);
4597 199 : *pcallers -= 1;
4598 : }
4599 : }
4600 : }
4601 216 : }
4602 :
4603 : /* (Re-)Scan the transaction blocks in NODE for calls to irrevocable functions,
4604 : as well as other irrevocable actions such as inline assembly. Mark all
4605 : such blocks as irrevocable and decrement the number of calls to
4606 : transactional clones. Return true if, for the transactional clone, the
4607 : entire function is irrevocable. */
4608 :
4609 : static bool
4610 812 : ipa_tm_scan_irr_function (struct cgraph_node *node, bool for_clone)
4611 : {
4612 812 : struct tm_ipa_cg_data *d;
4613 812 : bitmap new_irr, old_irr;
4614 812 : bool ret = false;
4615 :
4616 : /* Builtin operators (operator new, and such). */
4617 812 : if (DECL_STRUCT_FUNCTION (node->decl) == NULL
4618 812 : || DECL_STRUCT_FUNCTION (node->decl)->cfg == NULL)
4619 : return false;
4620 :
4621 642 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
4622 642 : calculate_dominance_info (CDI_DOMINATORS);
4623 :
4624 642 : d = get_cg_data (&node, true);
4625 642 : auto_vec<basic_block, 10> queue;
4626 642 : new_irr = BITMAP_ALLOC (&tm_obstack);
4627 :
4628 : /* Scan each tm region, propagating irrevocable status through the tree. */
4629 642 : if (for_clone)
4630 : {
4631 351 : old_irr = d->irrevocable_blocks_clone;
4632 351 : queue.quick_push (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)));
4633 351 : if (ipa_tm_scan_irr_blocks (&queue, new_irr, old_irr, NULL))
4634 : {
4635 59 : ipa_tm_propagate_irr (single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)),
4636 : new_irr,
4637 : old_irr, NULL);
4638 118 : ret = bitmap_bit_p (new_irr,
4639 59 : single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun))->index);
4640 : }
4641 : }
4642 : else
4643 : {
4644 291 : struct tm_region *region;
4645 :
4646 291 : old_irr = d->irrevocable_blocks_normal;
4647 648 : for (region = d->all_tm_regions; region; region = region->next)
4648 : {
4649 357 : queue.quick_push (region->entry_block);
4650 357 : if (ipa_tm_scan_irr_blocks (&queue, new_irr, old_irr,
4651 : region->exit_blocks))
4652 65 : ipa_tm_propagate_irr (region->entry_block, new_irr, old_irr,
4653 : region->exit_blocks);
4654 : }
4655 : }
4656 :
4657 : /* If we found any new irrevocable blocks, reduce the call count for
4658 : transactional clones within the irrevocable blocks. Save the new
4659 : set of irrevocable blocks for next time. */
4660 642 : if (!bitmap_empty_p (new_irr))
4661 : {
4662 112 : bitmap_iterator bmi;
4663 112 : unsigned i;
4664 :
4665 328 : EXECUTE_IF_SET_IN_BITMAP (new_irr, 0, i, bmi)
4666 216 : ipa_tm_decrement_clone_counts (BASIC_BLOCK_FOR_FN (cfun, i),
4667 : for_clone);
4668 :
4669 112 : if (old_irr)
4670 : {
4671 1 : bitmap_ior_into (old_irr, new_irr);
4672 1 : BITMAP_FREE (new_irr);
4673 : }
4674 111 : else if (for_clone)
4675 58 : d->irrevocable_blocks_clone = new_irr;
4676 : else
4677 53 : d->irrevocable_blocks_normal = new_irr;
4678 :
4679 112 : if (dump_file && new_irr)
4680 : {
4681 8 : const char *dname;
4682 8 : bitmap_iterator bmi;
4683 8 : unsigned i;
4684 :
4685 8 : dname = lang_hooks.decl_printable_name (current_function_decl, 2);
4686 25 : EXECUTE_IF_SET_IN_BITMAP (new_irr, 0, i, bmi)
4687 17 : fprintf (dump_file, "%s: bb %d goes irrevocable\n", dname, i);
4688 : }
4689 : }
4690 : else
4691 530 : BITMAP_FREE (new_irr);
4692 :
4693 642 : pop_cfun ();
4694 :
4695 642 : return ret;
4696 642 : }
4697 :
4698 : /* Return true if, for the transactional clone of NODE, any call
4699 : may enter irrevocable mode. */
4700 :
4701 : static bool
4702 508 : ipa_tm_mayenterirr_function (struct cgraph_node *node)
4703 : {
4704 508 : struct tm_ipa_cg_data *d;
4705 508 : tree decl;
4706 508 : unsigned flags;
4707 :
4708 508 : d = get_cg_data (&node, true);
4709 508 : decl = node->decl;
4710 508 : flags = flags_from_decl_or_type (decl);
4711 :
4712 : /* Handle some TM builtins. Ordinarily these aren't actually generated
4713 : at this point, but handling these functions when written in by the
4714 : user makes it easier to build unit tests. */
4715 508 : if (flags & ECF_TM_BUILTIN)
4716 : return false;
4717 :
4718 : /* Filter out all functions that are marked. */
4719 507 : if (flags & ECF_TM_PURE)
4720 : return false;
4721 500 : if (is_tm_safe (decl))
4722 : return false;
4723 276 : if (is_tm_irrevocable (decl))
4724 : return true;
4725 270 : if (is_tm_callable (decl))
4726 : return true;
4727 229 : if (find_tm_replacement_function (decl))
4728 : return true;
4729 :
4730 : /* If we aren't seeing the final version of the function we don't
4731 : know what it will contain at runtime. */
4732 229 : if (node->get_availability () < AVAIL_AVAILABLE)
4733 : return true;
4734 :
4735 : /* If the function must go irrevocable, then of course true. */
4736 175 : if (d->is_irrevocable)
4737 : return true;
4738 :
4739 : /* If there are any blocks marked irrevocable, then the function
4740 : as a whole may enter irrevocable. */
4741 150 : if (d->irrevocable_blocks_clone)
4742 : return true;
4743 :
4744 : /* We may have previously marked this function as tm_may_enter_irr;
4745 : see pass_diagnose_tm_blocks. */
4746 147 : if (node->tm_may_enter_irr)
4747 : return true;
4748 :
4749 : /* Recurse on the main body for aliases. In general, this will
4750 : result in one of the bits above being set so that we will not
4751 : have to recurse next time. */
4752 147 : if (node->alias)
4753 0 : return ipa_tm_mayenterirr_function
4754 0 : (cgraph_node::get (thunk_info::get (node)->alias));
4755 :
4756 : /* What remains is unmarked local functions without items that force
4757 : the function to go irrevocable. */
4758 : return false;
4759 : }
4760 :
4761 : /* Diagnose calls from transaction_safe functions to unmarked
4762 : functions that are determined to not be safe. */
4763 :
4764 : static void
4765 130 : ipa_tm_diagnose_tm_safe (struct cgraph_node *node)
4766 : {
4767 130 : struct cgraph_edge *e;
4768 :
4769 311 : for (e = node->callees; e ; e = e->next_callee)
4770 181 : if (!is_tm_callable (e->callee->decl)
4771 181 : && e->callee->tm_may_enter_irr)
4772 21 : error_at (gimple_location (e->call_stmt),
4773 : "unsafe function call %qD within "
4774 : "%<transaction_safe%> function", e->callee->decl);
4775 130 : }
4776 :
4777 : /* Diagnose call from atomic transactions to unmarked functions
4778 : that are determined to not be safe. */
4779 :
4780 : static void
4781 275 : ipa_tm_diagnose_transaction (struct cgraph_node *node,
4782 : struct tm_region *all_tm_regions)
4783 : {
4784 275 : struct tm_region *r;
4785 :
4786 610 : for (r = all_tm_regions; r ; r = r->next)
4787 335 : if (gimple_transaction_subcode (r->get_transaction_stmt ())
4788 : & GTMA_IS_RELAXED)
4789 : {
4790 : /* Atomic transactions can be nested inside relaxed. */
4791 66 : if (r->inner)
4792 0 : ipa_tm_diagnose_transaction (node, r->inner);
4793 : }
4794 : else
4795 : {
4796 269 : vec<basic_block> bbs;
4797 269 : gimple_stmt_iterator gsi;
4798 269 : basic_block bb;
4799 269 : size_t i;
4800 :
4801 269 : bbs = get_tm_region_blocks (r->entry_block, r->exit_blocks,
4802 : r->irr_blocks, NULL, false);
4803 :
4804 820 : for (i = 0; bbs.iterate (i, &bb); ++i)
4805 2555 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
4806 : {
4807 1745 : gimple *stmt = gsi_stmt (gsi);
4808 1745 : tree fndecl;
4809 :
4810 1745 : if (gimple_code (stmt) == GIMPLE_ASM)
4811 : {
4812 4 : error_at (gimple_location (stmt),
4813 : "%<asm%> not allowed in atomic transaction");
4814 4 : continue;
4815 : }
4816 :
4817 1741 : if (!is_gimple_call (stmt))
4818 1116 : continue;
4819 625 : fndecl = gimple_call_fndecl (stmt);
4820 :
4821 : /* Indirect function calls have been diagnosed already. */
4822 625 : if (!fndecl)
4823 7 : continue;
4824 :
4825 : /* Stop at the end of the transaction. */
4826 618 : if (is_tm_ending_fndecl (fndecl))
4827 : {
4828 332 : if (bitmap_bit_p (r->exit_blocks, bb->index))
4829 : break;
4830 40 : continue;
4831 : }
4832 :
4833 : /* Marked functions have been diagnosed already. */
4834 286 : if (is_tm_pure_call (stmt))
4835 76 : continue;
4836 210 : if (is_tm_callable (fndecl))
4837 69 : continue;
4838 :
4839 141 : if (cgraph_node::local_info_node (fndecl)->tm_may_enter_irr)
4840 13 : error_at (gimple_location (stmt),
4841 : "unsafe function call %qD within "
4842 : "atomic transaction", fndecl);
4843 : }
4844 :
4845 269 : bbs.release ();
4846 : }
4847 275 : }
4848 :
4849 : /* Return a transactional mangled name for the DECL_ASSEMBLER_NAME in
4850 : OLD_DECL. The returned value is a freshly malloced pointer that
4851 : should be freed by the caller. */
4852 :
4853 : static tree
4854 440 : tm_mangle (tree old_asm_id)
4855 : {
4856 440 : const char *old_asm_name;
4857 440 : char *tm_name;
4858 440 : void *alloc = NULL;
4859 440 : struct demangle_component *dc;
4860 440 : tree new_asm_id;
4861 :
4862 : /* Determine if the symbol is already a valid C++ mangled name. Do this
4863 : even for C, which might be interfacing with C++ code via appropriately
4864 : ugly identifiers. */
4865 : /* ??? We could probably do just as well checking for "_Z" and be done. */
4866 440 : old_asm_name = IDENTIFIER_POINTER (old_asm_id);
4867 440 : dc = cplus_demangle_v3_components (old_asm_name, DMGL_NO_OPTS, &alloc);
4868 :
4869 440 : if (dc == NULL)
4870 : {
4871 66 : char length[12];
4872 :
4873 66 : do_unencoded:
4874 66 : sprintf (length, "%u", IDENTIFIER_LENGTH (old_asm_id));
4875 66 : tm_name = concat ("_ZGTt", length, old_asm_name, NULL);
4876 : }
4877 : else
4878 : {
4879 374 : old_asm_name += 2; /* Skip _Z */
4880 :
4881 374 : switch (dc->type)
4882 : {
4883 0 : case DEMANGLE_COMPONENT_TRANSACTION_CLONE:
4884 0 : case DEMANGLE_COMPONENT_NONTRANSACTION_CLONE:
4885 : /* Don't play silly games, you! */
4886 0 : goto do_unencoded;
4887 :
4888 0 : case DEMANGLE_COMPONENT_HIDDEN_ALIAS:
4889 : /* I'd really like to know if we can ever be passed one of
4890 : these from the C++ front end. The Logical Thing would
4891 : seem that hidden-alias should be outer-most, so that we
4892 : get hidden-alias of a transaction-clone and not vice-versa. */
4893 0 : old_asm_name += 2;
4894 0 : break;
4895 :
4896 : default:
4897 : break;
4898 : }
4899 :
4900 374 : tm_name = concat ("_ZGTt", old_asm_name, NULL);
4901 : }
4902 440 : free (alloc);
4903 :
4904 440 : new_asm_id = get_identifier (tm_name);
4905 440 : free (tm_name);
4906 :
4907 440 : return new_asm_id;
4908 : }
4909 :
4910 : static inline void
4911 221 : ipa_tm_mark_force_output_node (struct cgraph_node *node)
4912 : {
4913 221 : node->mark_force_output ();
4914 221 : node->analyzed = true;
4915 221 : }
4916 :
4917 : static inline void
4918 3 : ipa_tm_mark_forced_by_abi_node (struct cgraph_node *node)
4919 : {
4920 3 : node->forced_by_abi = true;
4921 3 : node->analyzed = true;
4922 3 : }
4923 :
4924 : /* Callback data for ipa_tm_create_version_alias. */
4925 : struct create_version_alias_info
4926 : {
4927 : struct cgraph_node *old_node;
4928 : tree new_decl;
4929 : };
4930 :
4931 : /* A subroutine of ipa_tm_create_version, called via
4932 : cgraph_for_node_and_aliases. Create new tm clones for each of
4933 : the existing aliases. */
4934 : static bool
4935 440 : ipa_tm_create_version_alias (struct cgraph_node *node, void *data)
4936 : {
4937 440 : struct create_version_alias_info *info
4938 : = (struct create_version_alias_info *)data;
4939 440 : tree old_decl, new_decl, tm_name;
4940 440 : struct cgraph_node *new_node;
4941 :
4942 440 : if (!node->cpp_implicit_alias)
4943 : return false;
4944 :
4945 58 : old_decl = node->decl;
4946 58 : tm_name = tm_mangle (DECL_ASSEMBLER_NAME (old_decl));
4947 58 : new_decl = build_decl (DECL_SOURCE_LOCATION (old_decl),
4948 58 : TREE_CODE (old_decl), tm_name,
4949 58 : TREE_TYPE (old_decl));
4950 :
4951 58 : SET_DECL_ASSEMBLER_NAME (new_decl, tm_name);
4952 58 : SET_DECL_RTL (new_decl, NULL);
4953 :
4954 : /* Based loosely on C++'s make_alias_for(). */
4955 58 : TREE_PUBLIC (new_decl) = TREE_PUBLIC (old_decl);
4956 58 : DECL_CONTEXT (new_decl) = DECL_CONTEXT (old_decl);
4957 58 : DECL_LANG_SPECIFIC (new_decl) = DECL_LANG_SPECIFIC (old_decl);
4958 58 : TREE_READONLY (new_decl) = TREE_READONLY (old_decl);
4959 58 : DECL_EXTERNAL (new_decl) = 0;
4960 58 : DECL_ARTIFICIAL (new_decl) = 1;
4961 58 : TREE_ADDRESSABLE (new_decl) = 1;
4962 58 : TREE_USED (new_decl) = 1;
4963 58 : TREE_SYMBOL_REFERENCED (tm_name) = 1;
4964 :
4965 : /* Perform the same remapping to the comdat group. */
4966 58 : if (DECL_ONE_ONLY (new_decl))
4967 0 : varpool_node::get (new_decl)->set_comdat_group
4968 0 : (tm_mangle (decl_comdat_group_id (old_decl)));
4969 :
4970 58 : new_node = cgraph_node::create_same_body_alias (new_decl, info->new_decl);
4971 58 : new_node->tm_clone = true;
4972 58 : new_node->externally_visible = info->old_node->externally_visible;
4973 58 : new_node->no_reorder = info->old_node->no_reorder;
4974 : /* ?? Do not traverse aliases here. */
4975 58 : get_cg_data (&node, false)->clone = new_node;
4976 :
4977 58 : record_tm_clone_pair (old_decl, new_decl);
4978 :
4979 58 : if (info->old_node->force_output
4980 58 : || info->old_node->ref_list.first_referring ())
4981 58 : ipa_tm_mark_force_output_node (new_node);
4982 58 : if (info->old_node->forced_by_abi)
4983 0 : ipa_tm_mark_forced_by_abi_node (new_node);
4984 : return false;
4985 : }
4986 :
4987 : /* Create a copy of the function (possibly declaration only) of OLD_NODE,
4988 : appropriate for the transactional clone. */
4989 :
4990 : static void
4991 382 : ipa_tm_create_version (struct cgraph_node *old_node)
4992 : {
4993 382 : tree new_decl, old_decl, tm_name;
4994 382 : struct cgraph_node *new_node;
4995 :
4996 382 : old_decl = old_node->decl;
4997 382 : new_decl = copy_node (old_decl);
4998 :
4999 : /* DECL_ASSEMBLER_NAME needs to be set before we call
5000 : cgraph_copy_node_for_versioning below, because cgraph_node will
5001 : fill the assembler_name_hash. */
5002 382 : tm_name = tm_mangle (DECL_ASSEMBLER_NAME (old_decl));
5003 382 : SET_DECL_ASSEMBLER_NAME (new_decl, tm_name);
5004 382 : SET_DECL_RTL (new_decl, NULL);
5005 382 : TREE_SYMBOL_REFERENCED (tm_name) = 1;
5006 :
5007 : /* Perform the same remapping to the comdat group. */
5008 382 : if (DECL_ONE_ONLY (new_decl))
5009 0 : varpool_node::get (new_decl)->set_comdat_group
5010 0 : (tm_mangle (DECL_COMDAT_GROUP (old_decl)));
5011 :
5012 382 : gcc_assert (!old_node->ipa_transforms_to_apply.exists ());
5013 382 : new_node = old_node->create_version_clone (new_decl, vNULL, NULL);
5014 382 : new_node->local = false;
5015 382 : new_node->externally_visible = old_node->externally_visible;
5016 382 : new_node->lowered = true;
5017 382 : new_node->tm_clone = 1;
5018 382 : if (!old_node->implicit_section)
5019 382 : new_node->set_section (*old_node);
5020 382 : get_cg_data (&old_node, true)->clone = new_node;
5021 :
5022 382 : if (old_node->get_availability () >= AVAIL_INTERPOSABLE)
5023 : {
5024 : /* Remap extern inline to static inline. */
5025 : /* ??? Is it worth trying to use make_decl_one_only? */
5026 272 : if (DECL_DECLARED_INLINE_P (new_decl) && DECL_EXTERNAL (new_decl))
5027 : {
5028 3 : DECL_EXTERNAL (new_decl) = 0;
5029 3 : TREE_PUBLIC (new_decl) = 0;
5030 3 : DECL_WEAK (new_decl) = 0;
5031 : }
5032 :
5033 272 : tree_function_versioning (old_decl, new_decl,
5034 : NULL, NULL, false, NULL, NULL);
5035 : }
5036 :
5037 382 : record_tm_clone_pair (old_decl, new_decl);
5038 :
5039 382 : symtab->call_cgraph_insertion_hooks (new_node);
5040 382 : if (old_node->force_output
5041 382 : || old_node->ref_list.first_referring ())
5042 163 : ipa_tm_mark_force_output_node (new_node);
5043 382 : if (old_node->forced_by_abi)
5044 3 : ipa_tm_mark_forced_by_abi_node (new_node);
5045 :
5046 : /* Do the same thing, but for any aliases of the original node. */
5047 382 : {
5048 382 : struct create_version_alias_info data;
5049 382 : data.old_node = old_node;
5050 382 : data.new_decl = new_decl;
5051 382 : old_node->call_for_symbol_thunks_and_aliases (ipa_tm_create_version_alias,
5052 : &data, true);
5053 : }
5054 382 : }
5055 :
5056 : /* Construct a call to TM_IRREVOCABLE and insert it at the beginning of BB. */
5057 :
5058 : static void
5059 43 : ipa_tm_insert_irr_call (struct cgraph_node *node, struct tm_region *region,
5060 : basic_block bb)
5061 : {
5062 43 : gimple_stmt_iterator gsi;
5063 43 : gcall *g;
5064 :
5065 43 : transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE);
5066 :
5067 43 : g = gimple_build_call (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE),
5068 : 1, build_int_cst (NULL_TREE, MODE_SERIALIRREVOCABLE));
5069 :
5070 43 : split_block_after_labels (bb);
5071 43 : gsi = gsi_after_labels (bb);
5072 43 : gsi_insert_before (&gsi, g, GSI_SAME_STMT);
5073 :
5074 43 : node->create_edge (cgraph_node::get_create
5075 : (builtin_decl_explicit (BUILT_IN_TM_IRREVOCABLE)),
5076 43 : g, gimple_bb (g)->count);
5077 43 : }
5078 :
5079 : /* Construct a call to TM_GETTMCLONE and insert it before GSI. */
5080 :
5081 : static bool
5082 24 : ipa_tm_insert_gettmclone_call (struct cgraph_node *node,
5083 : struct tm_region *region,
5084 : gimple_stmt_iterator *gsi, gcall *stmt)
5085 : {
5086 24 : tree gettm_fn, ret, old_fn, callfn;
5087 24 : gcall *g;
5088 24 : gassign *g2;
5089 24 : bool safe;
5090 :
5091 24 : old_fn = gimple_call_fn (stmt);
5092 :
5093 24 : if (TREE_CODE (old_fn) == ADDR_EXPR)
5094 : {
5095 7 : tree fndecl = TREE_OPERAND (old_fn, 0);
5096 7 : tree clone = get_tm_clone_pair (fndecl);
5097 :
5098 : /* By transforming the call into a TM_GETTMCLONE, we are
5099 : technically taking the address of the original function and
5100 : its clone. Explain this so inlining will know this function
5101 : is needed. */
5102 7 : cgraph_node::get (fndecl)->mark_address_taken () ;
5103 7 : if (clone)
5104 0 : cgraph_node::get (clone)->mark_address_taken ();
5105 : }
5106 :
5107 24 : safe = is_tm_safe (TREE_TYPE (old_fn));
5108 36 : gettm_fn = builtin_decl_explicit (safe ? BUILT_IN_TM_GETTMCLONE_SAFE
5109 : : BUILT_IN_TM_GETTMCLONE_IRR);
5110 24 : ret = create_tmp_var (ptr_type_node);
5111 :
5112 24 : if (!safe)
5113 12 : transaction_subcode_ior (region, GTMA_MAY_ENTER_IRREVOCABLE);
5114 :
5115 : /* Discard OBJ_TYPE_REF, since we weren't able to fold it. */
5116 24 : if (TREE_CODE (old_fn) == OBJ_TYPE_REF)
5117 14 : old_fn = OBJ_TYPE_REF_EXPR (old_fn);
5118 :
5119 24 : g = gimple_build_call (gettm_fn, 1, old_fn);
5120 24 : ret = make_ssa_name (ret, g);
5121 24 : gimple_call_set_lhs (g, ret);
5122 :
5123 24 : gsi_insert_before (gsi, g, GSI_SAME_STMT);
5124 :
5125 24 : node->create_edge (cgraph_node::get_create (gettm_fn), g, gimple_bb (g)->count);
5126 :
5127 : /* Cast return value from tm_gettmclone* into appropriate function
5128 : pointer. */
5129 24 : callfn = create_tmp_var (TREE_TYPE (old_fn));
5130 24 : g2 = gimple_build_assign (callfn,
5131 24 : fold_build1 (NOP_EXPR, TREE_TYPE (callfn), ret));
5132 24 : callfn = make_ssa_name (callfn, g2);
5133 24 : gimple_assign_set_lhs (g2, callfn);
5134 24 : gsi_insert_before (gsi, g2, GSI_SAME_STMT);
5135 :
5136 : /* ??? This is a hack to preserve the NOTHROW bit on the call,
5137 : which we would have derived from the decl. Failure to save
5138 : this bit means we might have to split the basic block. */
5139 24 : if (gimple_call_nothrow_p (stmt))
5140 6 : gimple_call_set_nothrow (stmt, true);
5141 :
5142 24 : gimple_call_set_fn (stmt, callfn);
5143 :
5144 : /* Discarding OBJ_TYPE_REF above may produce incompatible LHS and RHS
5145 : for a call statement. Fix it. */
5146 24 : {
5147 24 : tree lhs = gimple_call_lhs (stmt);
5148 48 : tree rettype = TREE_TYPE (gimple_call_fntype (stmt));
5149 24 : if (lhs
5150 24 : && !useless_type_conversion_p (TREE_TYPE (lhs), rettype))
5151 : {
5152 0 : tree temp;
5153 :
5154 0 : temp = create_tmp_reg (rettype);
5155 0 : gimple_call_set_lhs (stmt, temp);
5156 :
5157 0 : g2 = gimple_build_assign (lhs,
5158 0 : fold_build1 (VIEW_CONVERT_EXPR,
5159 : TREE_TYPE (lhs), temp));
5160 0 : gsi_insert_after (gsi, g2, GSI_SAME_STMT);
5161 : }
5162 : }
5163 :
5164 24 : update_stmt (stmt);
5165 24 : cgraph_edge *e = cgraph_node::get (current_function_decl)->get_edge (stmt);
5166 24 : if (e)
5167 : {
5168 24 : cgraph_polymorphic_indirect_info *pii
5169 24 : = dyn_cast <cgraph_polymorphic_indirect_info *> (e->indirect_info);
5170 14 : if (pii)
5171 14 : pii->mark_unusable ();
5172 : }
5173 24 : return true;
5174 : }
5175 :
5176 : /* Helper function for ipa_tm_transform_calls*. Given a call
5177 : statement in GSI which resides inside transaction REGION, redirect
5178 : the call to either its wrapper function, or its clone. */
5179 :
5180 : static void
5181 813 : ipa_tm_transform_calls_redirect (struct cgraph_node *node,
5182 : struct tm_region *region,
5183 : gimple_stmt_iterator *gsi,
5184 : bool *need_ssa_rename_p)
5185 : {
5186 813 : gcall *stmt = as_a <gcall *> (gsi_stmt (*gsi));
5187 813 : struct cgraph_node *new_node;
5188 813 : struct cgraph_edge *e = node->get_edge (stmt);
5189 813 : tree fndecl = gimple_call_fndecl (stmt);
5190 :
5191 : /* For indirect calls, pass the address through the runtime. */
5192 813 : if (fndecl == NULL)
5193 : {
5194 34 : *need_ssa_rename_p |=
5195 17 : ipa_tm_insert_gettmclone_call (node, region, gsi, stmt);
5196 17 : return;
5197 : }
5198 :
5199 : /* Handle some TM builtins. Ordinarily these aren't actually generated
5200 : at this point, but handling these functions when written in by the
5201 : user makes it easier to build unit tests. */
5202 796 : if (flags_from_decl_or_type (fndecl) & ECF_TM_BUILTIN)
5203 : return;
5204 :
5205 : /* Fixup recursive calls inside clones. */
5206 : /* ??? Why did cgraph_copy_node_for_versioning update the call edges
5207 : for recursion but not update the call statements themselves? */
5208 384 : if (e->caller == e->callee && decl_is_tm_clone (current_function_decl))
5209 : {
5210 0 : gimple_call_set_fndecl (stmt, current_function_decl);
5211 0 : return;
5212 : }
5213 :
5214 : /* If there is a replacement, use it. */
5215 383 : fndecl = find_tm_replacement_function (fndecl);
5216 383 : if (fndecl)
5217 : {
5218 134 : new_node = cgraph_node::get_create (fndecl);
5219 :
5220 : /* ??? Mark all transaction_wrap functions tm_may_enter_irr.
5221 :
5222 : We can't do this earlier in record_tm_replacement because
5223 : cgraph_remove_unreachable_nodes is called before we inject
5224 : references to the node. Further, we can't do this in some
5225 : nice central place in ipa_tm_execute because we don't have
5226 : the exact list of wrapper functions that would be used.
5227 : Marking more wrappers than necessary results in the creation
5228 : of unnecessary cgraph_nodes, which can cause some of the
5229 : other IPA passes to crash.
5230 :
5231 : We do need to mark these nodes so that we get the proper
5232 : result in expand_call_tm. */
5233 : /* ??? This seems broken. How is it that we're marking the
5234 : CALLEE as may_enter_irr? Surely we should be marking the
5235 : CALLER. Also note that find_tm_replacement_function also
5236 : contains mappings into the TM runtime, e.g. memcpy. These
5237 : we know won't go irrevocable. */
5238 134 : new_node->tm_may_enter_irr = 1;
5239 : }
5240 : else
5241 : {
5242 249 : struct tm_ipa_cg_data *d;
5243 249 : struct cgraph_node *tnode = e->callee;
5244 :
5245 249 : d = get_cg_data (&tnode, true);
5246 249 : new_node = d->clone;
5247 :
5248 : /* As we've already skipped pure calls and appropriate builtins,
5249 : and we've already marked irrevocable blocks, if we can't come
5250 : up with a static replacement, then ask the runtime. */
5251 249 : if (new_node == NULL)
5252 : {
5253 14 : *need_ssa_rename_p |=
5254 7 : ipa_tm_insert_gettmclone_call (node, region, gsi, stmt);
5255 7 : return;
5256 : }
5257 :
5258 242 : fndecl = new_node->decl;
5259 : }
5260 :
5261 376 : e->redirect_callee (new_node);
5262 376 : gimple_call_set_fndecl (stmt, fndecl);
5263 : }
5264 :
5265 : /* Helper function for ipa_tm_transform_calls. For a given BB,
5266 : install calls to tm_irrevocable when IRR_BLOCKS are reached,
5267 : redirect other calls to the generated transactional clone. */
5268 :
5269 : static bool
5270 1388 : ipa_tm_transform_calls_1 (struct cgraph_node *node, struct tm_region *region,
5271 : basic_block bb, bitmap irr_blocks)
5272 : {
5273 1388 : gimple_stmt_iterator gsi;
5274 1388 : bool need_ssa_rename = false;
5275 :
5276 1388 : if (irr_blocks && bitmap_bit_p (irr_blocks, bb->index))
5277 : {
5278 43 : ipa_tm_insert_irr_call (node, region, bb);
5279 43 : return true;
5280 : }
5281 :
5282 5862 : for (gsi = gsi_start_bb (bb); !gsi_end_p (gsi); gsi_next (&gsi))
5283 : {
5284 3172 : gimple *stmt = gsi_stmt (gsi);
5285 :
5286 3172 : if (!is_gimple_call (stmt))
5287 2241 : continue;
5288 931 : if (is_tm_pure_call (stmt))
5289 118 : continue;
5290 :
5291 : /* Redirect edges to the appropriate replacement or clone. */
5292 813 : ipa_tm_transform_calls_redirect (node, region, &gsi, &need_ssa_rename);
5293 : }
5294 :
5295 1345 : return need_ssa_rename;
5296 : }
5297 :
5298 : /* Walk the CFG for REGION, beginning at BB. Install calls to
5299 : tm_irrevocable when IRR_BLOCKS are reached, redirect other calls to
5300 : the generated transactional clone. */
5301 :
5302 : static bool
5303 466 : ipa_tm_transform_calls (struct cgraph_node *node, struct tm_region *region,
5304 : basic_block bb, bitmap irr_blocks)
5305 : {
5306 466 : bool need_ssa_rename = false;
5307 466 : edge e;
5308 466 : edge_iterator ei;
5309 466 : auto_vec<basic_block> queue;
5310 466 : bitmap visited_blocks = BITMAP_ALLOC (NULL);
5311 :
5312 466 : queue.safe_push (bb);
5313 1388 : do
5314 : {
5315 1388 : bb = queue.pop ();
5316 :
5317 2776 : need_ssa_rename |=
5318 1388 : ipa_tm_transform_calls_1 (node, region, bb, irr_blocks);
5319 :
5320 1388 : if (irr_blocks && bitmap_bit_p (irr_blocks, bb->index))
5321 43 : continue;
5322 :
5323 1345 : if (region && bitmap_bit_p (region->exit_blocks, bb->index))
5324 316 : continue;
5325 :
5326 2093 : FOR_EACH_EDGE (e, ei, bb->succs)
5327 1064 : if (!bitmap_bit_p (visited_blocks, e->dest->index))
5328 : {
5329 922 : bitmap_set_bit (visited_blocks, e->dest->index);
5330 922 : queue.safe_push (e->dest);
5331 : }
5332 : }
5333 2776 : while (!queue.is_empty ());
5334 :
5335 466 : BITMAP_FREE (visited_blocks);
5336 :
5337 466 : return need_ssa_rename;
5338 466 : }
5339 :
5340 : /* Transform the calls within the TM regions within NODE. */
5341 :
5342 : static void
5343 284 : ipa_tm_transform_transaction (struct cgraph_node *node)
5344 : {
5345 284 : struct tm_ipa_cg_data *d;
5346 284 : struct tm_region *region;
5347 284 : bool need_ssa_rename = false;
5348 :
5349 284 : d = get_cg_data (&node, true);
5350 :
5351 284 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
5352 284 : calculate_dominance_info (CDI_DOMINATORS);
5353 :
5354 634 : for (region = d->all_tm_regions; region; region = region->next)
5355 : {
5356 : /* If we're sure to go irrevocable, don't transform anything. */
5357 408 : if (d->irrevocable_blocks_normal
5358 417 : && bitmap_bit_p (d->irrevocable_blocks_normal,
5359 67 : region->entry_block->index))
5360 : {
5361 58 : transaction_subcode_ior (region, GTMA_DOES_GO_IRREVOCABLE
5362 : | GTMA_MAY_ENTER_IRREVOCABLE
5363 : | GTMA_HAS_NO_INSTRUMENTATION);
5364 58 : continue;
5365 : }
5366 :
5367 292 : need_ssa_rename |=
5368 292 : ipa_tm_transform_calls (node, region, region->entry_block,
5369 : d->irrevocable_blocks_normal);
5370 : }
5371 :
5372 284 : if (need_ssa_rename)
5373 13 : update_ssa (TODO_update_ssa_only_virtuals);
5374 :
5375 284 : pop_cfun ();
5376 284 : }
5377 :
5378 : /* Transform the calls within the transactional clone of NODE. */
5379 :
5380 : static void
5381 272 : ipa_tm_transform_clone (struct cgraph_node *node)
5382 : {
5383 272 : struct tm_ipa_cg_data *d;
5384 272 : bool need_ssa_rename;
5385 :
5386 272 : d = get_cg_data (&node, true);
5387 :
5388 : /* If this function makes no calls and has no irrevocable blocks,
5389 : then there's nothing to do. */
5390 : /* ??? Remove non-aborting top-level transactions. */
5391 272 : if (!node->callees && !node->indirect_calls && !d->irrevocable_blocks_clone)
5392 : return;
5393 :
5394 174 : push_cfun (DECL_STRUCT_FUNCTION (d->clone->decl));
5395 174 : calculate_dominance_info (CDI_DOMINATORS);
5396 :
5397 174 : need_ssa_rename =
5398 174 : ipa_tm_transform_calls (d->clone, NULL,
5399 174 : single_succ (ENTRY_BLOCK_PTR_FOR_FN (cfun)),
5400 : d->irrevocable_blocks_clone);
5401 :
5402 174 : if (need_ssa_rename)
5403 48 : update_ssa (TODO_update_ssa_only_virtuals);
5404 :
5405 174 : pop_cfun ();
5406 : }
5407 :
5408 : /* Main entry point for the transactional memory IPA pass. */
5409 :
5410 : static unsigned int
5411 400 : ipa_tm_execute (void)
5412 : {
5413 400 : cgraph_node_queue tm_callees = cgraph_node_queue ();
5414 : /* List of functions that will go irrevocable. */
5415 400 : cgraph_node_queue irr_worklist = cgraph_node_queue ();
5416 :
5417 400 : struct cgraph_node *node;
5418 400 : struct tm_ipa_cg_data *d;
5419 400 : enum availability a;
5420 400 : unsigned int i;
5421 :
5422 400 : cgraph_node::checking_verify_cgraph_nodes ();
5423 :
5424 400 : bitmap_obstack_initialize (&tm_obstack);
5425 400 : initialize_original_copy_tables ();
5426 :
5427 : /* For all local functions marked tm_callable, queue them. */
5428 1503 : FOR_EACH_DEFINED_FUNCTION (node)
5429 1103 : if (is_tm_callable (node->decl)
5430 1103 : && node->get_availability () >= AVAIL_INTERPOSABLE)
5431 : {
5432 162 : d = get_cg_data (&node, true);
5433 324 : maybe_push_queue (node, &tm_callees, &d->in_callee_queue);
5434 : }
5435 :
5436 : /* For all local reachable functions... */
5437 1524 : FOR_EACH_DEFINED_FUNCTION (node)
5438 1124 : if (node->lowered
5439 1124 : && node->get_availability () >= AVAIL_INTERPOSABLE)
5440 : {
5441 : /* ... marked tm_pure, record that fact for the runtime by
5442 : indicating that the pure function is its own tm_callable.
5443 : No need to do this if the function's address can't be taken. */
5444 958 : if (is_tm_pure (node->decl))
5445 : {
5446 27 : if (!node->local)
5447 23 : record_tm_clone_pair (node->decl, node->decl);
5448 27 : continue;
5449 : }
5450 :
5451 931 : push_cfun (DECL_STRUCT_FUNCTION (node->decl));
5452 931 : calculate_dominance_info (CDI_DOMINATORS);
5453 :
5454 931 : tm_region_init (NULL);
5455 931 : if (all_tm_regions)
5456 : {
5457 284 : d = get_cg_data (&node, true);
5458 :
5459 : /* Scan for calls that are in each transaction, and
5460 : generate the uninstrumented code path. */
5461 284 : ipa_tm_scan_calls_transaction (d, &tm_callees);
5462 :
5463 : /* Put it in the worklist so we can scan the function
5464 : later (ipa_tm_scan_irr_function) and mark the
5465 : irrevocable blocks. */
5466 284 : maybe_push_queue (node, &irr_worklist, &d->in_worklist);
5467 284 : d->want_irr_scan_normal = true;
5468 : }
5469 :
5470 931 : pop_cfun ();
5471 : }
5472 :
5473 : /* For every local function on the callee list, scan as if we will be
5474 : creating a transactional clone, queueing all new functions we find
5475 : along the way. */
5476 1594 : for (i = 0; i < tm_callees.length (); ++i)
5477 : {
5478 508 : node = tm_callees[i];
5479 508 : a = node->get_availability ();
5480 508 : d = get_cg_data (&node, true);
5481 :
5482 : /* Put it in the worklist so we can scan the function later
5483 : (ipa_tm_scan_irr_function) and mark the irrevocable
5484 : blocks. */
5485 508 : maybe_push_queue (node, &irr_worklist, &d->in_worklist);
5486 :
5487 : /* Some callees cannot be arbitrarily cloned. These will always be
5488 : irrevocable. Mark these now, so that we need not scan them. */
5489 508 : if (is_tm_irrevocable (node->decl))
5490 6 : ipa_tm_note_irrevocable (node, &irr_worklist);
5491 502 : else if (a <= AVAIL_NOT_AVAILABLE
5492 502 : && !is_tm_safe_or_pure (node->decl))
5493 64 : ipa_tm_note_irrevocable (node, &irr_worklist);
5494 438 : else if (a >= AVAIL_INTERPOSABLE)
5495 : {
5496 337 : if (!tree_versionable_function_p (node->decl))
5497 0 : ipa_tm_note_irrevocable (node, &irr_worklist);
5498 337 : else if (!d->is_irrevocable)
5499 : {
5500 : /* If this is an alias, make sure its base is queued as well.
5501 : we need not scan the callees now, as the base will do. */
5502 337 : if (node->alias)
5503 : {
5504 0 : node = cgraph_node::get (thunk_info::get (node)->alias);
5505 0 : d = get_cg_data (&node, true);
5506 0 : maybe_push_queue (node, &tm_callees, &d->in_callee_queue);
5507 0 : continue;
5508 : }
5509 :
5510 : /* Add all nodes called by this function into
5511 : tm_callees as well. */
5512 337 : ipa_tm_scan_calls_clone (node, &tm_callees);
5513 : }
5514 : }
5515 : }
5516 :
5517 : /* Iterate scans until no more work to be done. Prefer not to use
5518 : vec::pop because the worklist tends to follow a breadth-first
5519 : search of the callgraph, which should allow convergance with a
5520 : minimum number of scans. But we also don't want the worklist
5521 : array to grow without bound, so we shift the array up periodically. */
5522 1201 : for (i = 0; i < irr_worklist.length (); ++i)
5523 : {
5524 801 : if (i > 256 && i == irr_worklist.length () / 8)
5525 : {
5526 0 : irr_worklist.block_remove (0, i);
5527 0 : i = 0;
5528 : }
5529 :
5530 801 : node = irr_worklist[i];
5531 801 : d = get_cg_data (&node, true);
5532 801 : d->in_worklist = false;
5533 :
5534 801 : if (d->want_irr_scan_normal)
5535 : {
5536 291 : d->want_irr_scan_normal = false;
5537 291 : ipa_tm_scan_irr_function (node, false);
5538 : }
5539 801 : if (d->in_callee_queue && ipa_tm_scan_irr_function (node, true))
5540 54 : ipa_tm_note_irrevocable (node, &irr_worklist);
5541 : }
5542 :
5543 : /* For every function on the callee list, collect the tm_may_enter_irr
5544 : bit on the node. */
5545 400 : irr_worklist.truncate (0);
5546 1994 : for (i = 0; i < tm_callees.length (); ++i)
5547 : {
5548 508 : node = tm_callees[i];
5549 508 : if (ipa_tm_mayenterirr_function (node))
5550 : {
5551 129 : d = get_cg_data (&node, true);
5552 129 : gcc_assert (d->in_worklist == false);
5553 129 : maybe_push_queue (node, &irr_worklist, &d->in_worklist);
5554 : }
5555 : }
5556 :
5557 : /* Propagate the tm_may_enter_irr bit to callers until stable. */
5558 571 : for (i = 0; i < irr_worklist.length (); ++i)
5559 : {
5560 171 : struct cgraph_node *caller;
5561 171 : struct cgraph_edge *e;
5562 171 : struct ipa_ref *ref;
5563 :
5564 171 : if (i > 256 && i == irr_worklist.length () / 8)
5565 : {
5566 0 : irr_worklist.block_remove (0, i);
5567 0 : i = 0;
5568 : }
5569 :
5570 171 : node = irr_worklist[i];
5571 171 : d = get_cg_data (&node, true);
5572 171 : d->in_worklist = false;
5573 171 : node->tm_may_enter_irr = true;
5574 :
5575 : /* Propagate back to normal callers. */
5576 357 : for (e = node->callers; e ; e = e->next_caller)
5577 : {
5578 186 : caller = e->caller;
5579 186 : if (!is_tm_safe_or_pure (caller->decl)
5580 186 : && !caller->tm_may_enter_irr)
5581 : {
5582 122 : d = get_cg_data (&caller, true);
5583 160 : maybe_push_queue (caller, &irr_worklist, &d->in_worklist);
5584 : }
5585 : }
5586 :
5587 : /* Propagate back to referring aliases as well. */
5588 179 : FOR_EACH_ALIAS (node, ref)
5589 : {
5590 8 : caller = dyn_cast<cgraph_node *> (ref->referring);
5591 8 : if (!caller->tm_may_enter_irr)
5592 : {
5593 : /* ?? Do not traverse aliases here. */
5594 8 : d = get_cg_data (&caller, false);
5595 12 : maybe_push_queue (caller, &irr_worklist, &d->in_worklist);
5596 : }
5597 : }
5598 : }
5599 :
5600 : /* Now validate all tm_safe functions, and all atomic regions in
5601 : other functions. */
5602 1524 : FOR_EACH_DEFINED_FUNCTION (node)
5603 1124 : if (node->lowered
5604 1124 : && node->get_availability () >= AVAIL_INTERPOSABLE)
5605 : {
5606 958 : d = get_cg_data (&node, true);
5607 958 : if (is_tm_safe (node->decl))
5608 130 : ipa_tm_diagnose_tm_safe (node);
5609 828 : else if (d->all_tm_regions)
5610 275 : ipa_tm_diagnose_transaction (node, d->all_tm_regions);
5611 : }
5612 :
5613 : /* Create clones. Do those that are not irrevocable and have a
5614 : positive call count. Do those publicly visible functions that
5615 : the user directed us to clone. */
5616 1594 : for (i = 0; i < tm_callees.length (); ++i)
5617 : {
5618 508 : bool doit = false;
5619 :
5620 508 : node = tm_callees[i];
5621 508 : if (node->cpp_implicit_alias)
5622 0 : continue;
5623 :
5624 508 : a = node->get_availability ();
5625 508 : d = get_cg_data (&node, true);
5626 :
5627 508 : if (a <= AVAIL_NOT_AVAILABLE)
5628 170 : doit = is_tm_callable (node->decl);
5629 338 : else if (a <= AVAIL_AVAILABLE && is_tm_callable (node->decl))
5630 : doit = true;
5631 179 : else if (!d->is_irrevocable
5632 153 : && d->tm_callers_normal + d->tm_callers_clone > 0)
5633 : doit = true;
5634 :
5635 170 : if (doit)
5636 382 : ipa_tm_create_version (node);
5637 : }
5638 :
5639 : /* Redirect calls to the new clones, and insert irrevocable marks. */
5640 908 : for (i = 0; i < tm_callees.length (); ++i)
5641 : {
5642 508 : node = tm_callees[i];
5643 508 : if (node->analyzed)
5644 : {
5645 338 : d = get_cg_data (&node, true);
5646 338 : if (d->clone)
5647 272 : ipa_tm_transform_clone (node);
5648 : }
5649 : }
5650 1854 : FOR_EACH_DEFINED_FUNCTION (node)
5651 1454 : if (node->lowered
5652 1454 : && node->get_availability () >= AVAIL_INTERPOSABLE)
5653 : {
5654 1230 : d = get_cg_data (&node, true);
5655 1230 : if (d->all_tm_regions)
5656 284 : ipa_tm_transform_transaction (node);
5657 : }
5658 :
5659 : /* Free and clear all data structures. */
5660 400 : tm_callees.release ();
5661 400 : irr_worklist.release ();
5662 400 : bitmap_obstack_release (&tm_obstack);
5663 400 : free_original_copy_tables ();
5664 :
5665 2937 : FOR_EACH_FUNCTION (node)
5666 2537 : node->aux = NULL;
5667 :
5668 400 : cgraph_node::checking_verify_cgraph_nodes ();
5669 :
5670 400 : return 0;
5671 : }
5672 :
5673 : namespace {
5674 :
5675 : const pass_data pass_data_ipa_tm =
5676 : {
5677 : SIMPLE_IPA_PASS, /* type */
5678 : "tmipa", /* name */
5679 : OPTGROUP_NONE, /* optinfo_flags */
5680 : TV_TRANS_MEM, /* tv_id */
5681 : ( PROP_ssa | PROP_cfg ), /* properties_required */
5682 : 0, /* properties_provided */
5683 : 0, /* properties_destroyed */
5684 : 0, /* todo_flags_start */
5685 : 0, /* todo_flags_finish */
5686 : };
5687 :
5688 : class pass_ipa_tm : public simple_ipa_opt_pass
5689 : {
5690 : public:
5691 285722 : pass_ipa_tm (gcc::context *ctxt)
5692 571444 : : simple_ipa_opt_pass (pass_data_ipa_tm, ctxt)
5693 : {}
5694 :
5695 : /* opt_pass methods: */
5696 229960 : bool gate (function *) final override { return flag_tm; }
5697 400 : unsigned int execute (function *) final override { return ipa_tm_execute (); }
5698 :
5699 : }; // class pass_ipa_tm
5700 :
5701 : } // anon namespace
5702 :
5703 : simple_ipa_opt_pass *
5704 285722 : make_pass_ipa_tm (gcc::context *ctxt)
5705 : {
5706 285722 : return new pass_ipa_tm (ctxt);
5707 : }
5708 :
5709 : #include "gt-trans-mem.h"
|
