Line data Source code
1 : // wb.cc -- Add write barriers as needed.
2 :
3 : // Copyright 2017 The Go Authors. All rights reserved.
4 : // Use of this source code is governed by a BSD-style
5 : // license that can be found in the LICENSE file.
6 :
7 : #include "go-system.h"
8 :
9 : #include "go-c.h"
10 : #include "go-diagnostics.h"
11 : #include "operator.h"
12 : #include "lex.h"
13 : #include "types.h"
14 : #include "expressions.h"
15 : #include "statements.h"
16 : #include "runtime.h"
17 : #include "gogo.h"
18 :
19 : // Mark variables whose addresses are taken and do some other
20 : // cleanups. This has to be done before the write barrier pass and
21 : // after the escape analysis pass. It would be nice to do this
22 : // elsewhere but there isn't an obvious place.
23 :
24 8504 : class Mark_address_taken : public Traverse
25 : {
26 : public:
27 4252 : Mark_address_taken(Gogo* gogo)
28 4252 : : Traverse(traverse_functions
29 : | traverse_statements
30 : | traverse_expressions),
31 4252 : gogo_(gogo), function_(NULL)
32 : { }
33 :
34 : int
35 : function(Named_object*);
36 :
37 : int
38 : statement(Block*, size_t*, Statement*);
39 :
40 : int
41 : expression(Expression**);
42 :
43 : private:
44 : // General IR.
45 : Gogo* gogo_;
46 : // The function we are traversing.
47 : Named_object* function_;
48 : };
49 :
50 : // Record a function.
51 :
52 : int
53 287649 : Mark_address_taken::function(Named_object* no)
54 : {
55 287649 : go_assert(this->function_ == NULL);
56 287649 : this->function_ = no;
57 287649 : int t = no->func_value()->traverse(this);
58 287649 : this->function_ = NULL;
59 :
60 287649 : if (t == TRAVERSE_EXIT)
61 0 : return t;
62 : return TRAVERSE_SKIP_COMPONENTS;
63 : }
64 :
65 : // Traverse a statement.
66 :
67 : int
68 5489440 : Mark_address_taken::statement(Block* block, size_t* pindex, Statement* s)
69 : {
70 : // If this is an assignment of the form s = append(s, ...), expand
71 : // it now, so that we can assign it to the left hand side in the
72 : // middle of the expansion and possibly skip a write barrier.
73 5489440 : Assignment_statement* as = s->assignment_statement();
74 967766 : if (as != NULL && !as->lhs()->is_sink_expression())
75 : {
76 961573 : Call_expression* rce = as->rhs()->call_expression();
77 15094 : if (rce != NULL
78 15091 : && rce->builtin_call_expression() != NULL
79 15091 : && (rce->builtin_call_expression()->code()
80 : == Builtin_call_expression::BUILTIN_APPEND)
81 14637 : && Expression::is_same_variable(as->lhs(), rce->args()->front()))
82 : {
83 13022 : Statement_inserter inserter = Statement_inserter(block, pindex);
84 13022 : Expression* a =
85 13022 : rce->builtin_call_expression()->flatten_append(this->gogo_,
86 : this->function_,
87 : &inserter,
88 : as->lhs(),
89 : block);
90 13022 : go_assert(a == NULL);
91 : // That does the assignment, so remove this statement.
92 13022 : Expression* e = Expression::make_boolean(true, s->location());
93 13022 : Statement* dummy = Statement::make_statement(e, true);
94 13022 : block->replace_statement(*pindex, dummy);
95 : }
96 : }
97 5489440 : return TRAVERSE_CONTINUE;
98 : }
99 :
100 : // Mark variable addresses taken.
101 :
102 : int
103 13742706 : Mark_address_taken::expression(Expression** pexpr)
104 : {
105 13742706 : Expression* expr = *pexpr;
106 13742706 : Unary_expression* ue = expr->unary_expression();
107 826646 : if (ue != NULL)
108 826646 : ue->check_operand_address_taken(this->gogo_);
109 :
110 13742706 : Array_index_expression* aie = expr->array_index_expression();
111 178736 : if (aie != NULL
112 178736 : && aie->end() != NULL
113 37197 : && !aie->array()->type()->is_slice_type())
114 : {
115 : // Slice of an array. The escape analysis models this with
116 : // a child Node representing the address of the array.
117 13328 : bool escapes = false;
118 13328 : Node* n = Node::make_node(expr);
119 13328 : if (n->child() == NULL
120 13328 : || (n->child()->encoding() & ESCAPE_MASK) != Node::ESCAPE_NONE)
121 : escapes = true;
122 13328 : aie->array()->address_taken(escapes);
123 : }
124 :
125 13742706 : if (expr->allocation_expression() != NULL)
126 : {
127 7761 : Node* n = Node::make_node(expr);
128 7761 : if ((n->encoding() & ESCAPE_MASK) == Node::ESCAPE_NONE)
129 1931 : expr->allocation_expression()->set_allocate_on_stack();
130 : }
131 13742706 : if (expr->heap_expression() != NULL)
132 : {
133 48107 : Node* n = Node::make_node(expr);
134 48107 : if ((n->encoding() & ESCAPE_MASK) == Node::ESCAPE_NONE)
135 7540 : expr->heap_expression()->set_allocate_on_stack();
136 : }
137 13742706 : if (expr->slice_literal() != NULL)
138 : {
139 100556 : Node* n = Node::make_node(expr);
140 100556 : if ((n->encoding() & ESCAPE_MASK) == Node::ESCAPE_NONE)
141 33704 : expr->slice_literal()->set_storage_does_not_escape();
142 : }
143 :
144 : // Rewrite non-escaping makeslice with constant size to stack allocation.
145 13742706 : Slice_value_expression* sve = expr->slice_value_expression();
146 8576 : if (sve != NULL)
147 : {
148 8576 : std::pair<Call_expression*, Temporary_statement*> p =
149 8576 : Expression::find_makeslice_call(sve);
150 8576 : Call_expression* call = p.first;
151 8576 : Temporary_statement* ts = p.second;
152 8576 : if (call != NULL
153 8576 : && Node::make_node(call)->encoding() == Node::ESCAPE_NONE)
154 : {
155 1713 : Expression* len_arg = call->args()->at(1);
156 1713 : Expression* cap_arg = call->args()->at(2);
157 1713 : Numeric_constant nclen;
158 1713 : Numeric_constant nccap;
159 1713 : unsigned long vlen;
160 1713 : unsigned long vcap;
161 1713 : if (len_arg->numeric_constant_value(&nclen)
162 841 : && cap_arg->numeric_constant_value(&nccap)
163 588 : && nclen.to_unsigned_long(&vlen) == Numeric_constant::NC_UL_VALID
164 2301 : && nccap.to_unsigned_long(&vcap) == Numeric_constant::NC_UL_VALID)
165 : {
166 : // Stack allocate an array and make a slice value from it.
167 588 : Location loc = expr->location();
168 1176 : Type* elmt_type = expr->type()->array_type()->element_type();
169 588 : Expression* len_expr =
170 588 : Expression::make_integer_ul(vcap, cap_arg->type(), loc);
171 588 : Type* array_type = Type::make_array_type(elmt_type, len_expr);
172 588 : Expression* alloc = Expression::make_allocation(array_type, loc);
173 588 : alloc->allocation_expression()->set_allocate_on_stack();
174 588 : Type* ptr_type = Type::make_pointer_type(elmt_type);
175 588 : Expression* ptr = Expression::make_unsafe_cast(ptr_type, alloc,
176 : loc);
177 588 : Expression* slice =
178 588 : Expression::make_slice_value(expr->type(), ptr, len_arg,
179 : cap_arg, loc);
180 588 : *pexpr = slice;
181 588 : if (ts != NULL && ts->uses() == 1)
182 71 : ts->set_init(Expression::make_nil(loc));
183 : }
184 1713 : }
185 : }
186 13742706 : return TRAVERSE_CONTINUE;
187 : }
188 :
189 : // Check variables and closures do not escape when compiling runtime.
190 :
191 14 : class Check_escape : public Traverse
192 : {
193 : public:
194 7 : Check_escape()
195 7 : : Traverse(traverse_expressions | traverse_variables)
196 : { }
197 :
198 : int
199 : expression(Expression**);
200 :
201 : int
202 : variable(Named_object*);
203 : };
204 :
205 : int
206 64228 : Check_escape::variable(Named_object* no)
207 : {
208 64228 : if ((no->is_variable() && no->var_value()->is_in_heap())
209 117343 : || (no->is_result_variable()
210 11113 : && no->result_var_value()->is_in_heap()))
211 0 : go_error_at(no->location(),
212 : "%s escapes to heap, not allowed in runtime",
213 0 : no->message_name().c_str());
214 64228 : return TRAVERSE_CONTINUE;
215 : }
216 :
217 : int
218 847982 : Check_escape::expression(Expression** pexpr)
219 : {
220 847982 : Expression* expr = *pexpr;
221 847982 : Func_expression* fe = expr->func_expression();
222 63590 : if (fe != NULL && fe->closure() != NULL)
223 : {
224 406 : Node* n = Node::make_node(expr);
225 406 : if (n->encoding() == Node::ESCAPE_HEAP)
226 0 : go_error_at(expr->location(),
227 : "heap-allocated closure, not allowed in runtime");
228 : }
229 847982 : return TRAVERSE_CONTINUE;
230 : }
231 :
232 : // Collect all writebarrierrec functions. This is used when compiling
233 : // the runtime package, to propagate //go:nowritebarrierrec.
234 :
235 14 : class Collect_writebarrierrec_functions : public Traverse
236 : {
237 : public:
238 7 : Collect_writebarrierrec_functions(std::vector<Named_object*>* worklist)
239 7 : : Traverse(traverse_functions),
240 7 : worklist_(worklist)
241 : { }
242 :
243 : private:
244 : int
245 : function(Named_object*);
246 :
247 : // The collected functions are put here.
248 : std::vector<Named_object*>* worklist_;
249 : };
250 :
251 : int
252 17220 : Collect_writebarrierrec_functions::function(Named_object* no)
253 : {
254 17220 : if (no->is_function()
255 17220 : && no->func_value()->enclosing() == NULL
256 33639 : && (no->func_value()->pragmas() & GOPRAGMA_NOWRITEBARRIERREC) != 0)
257 : {
258 451 : go_assert((no->func_value()->pragmas() & GOPRAGMA_MARK) == 0);
259 451 : this->worklist_->push_back(no);
260 : }
261 17220 : return TRAVERSE_CONTINUE;
262 : }
263 :
264 : // Collect all callees of this function. We only care about locally
265 : // defined, known, functions.
266 :
267 7 : class Collect_callees : public Traverse
268 : {
269 : public:
270 7 : Collect_callees(std::vector<Named_object*>* worklist)
271 7 : : Traverse(traverse_expressions),
272 7 : worklist_(worklist)
273 7 : { }
274 :
275 : private:
276 : int
277 : expression(Expression**);
278 :
279 : // The collected callees are put here.
280 : std::vector<Named_object*>* worklist_;
281 : };
282 :
283 : int
284 188954 : Collect_callees::expression(Expression** pexpr)
285 : {
286 188954 : Call_expression* ce = (*pexpr)->call_expression();
287 17080 : if (ce != NULL)
288 : {
289 17080 : Func_expression* fe = ce->fn()->func_expression();
290 16913 : if (fe != NULL)
291 : {
292 16913 : Named_object* no = fe->named_object();
293 16913 : if (no->package() == NULL && no->is_function())
294 : {
295 : // The function runtime.systemstack is special, in that
296 : // it is a common way to call a function in the runtime:
297 : // mark its argument if we can.
298 11721 : if (Gogo::unpack_hidden_name(no->name()) != "systemstack")
299 11637 : this->worklist_->push_back(no);
300 84 : else if (ce->args()->size() > 0)
301 : {
302 16997 : fe = ce->args()->front()->func_expression();
303 84 : if (fe != NULL)
304 : {
305 84 : no = fe->named_object();
306 84 : if (no->package() == NULL && no->is_function())
307 84 : this->worklist_->push_back(no);
308 : }
309 : }
310 : }
311 : }
312 : }
313 188954 : return TRAVERSE_CONTINUE;
314 : }
315 :
316 : // When compiling the runtime package, propagate //go:nowritebarrierrec
317 : // annotations. A function marked as //go:nowritebarrierrec does not
318 : // permit write barriers, and also all the functions that it calls,
319 : // recursively, do not permit write barriers. Except that a
320 : // //go:yeswritebarrierrec annotation permits write barriers even if
321 : // called by a //go:nowritebarrierrec function. Here we turn
322 : // //go:nowritebarrierrec into //go:nowritebarrier, as appropriate.
323 :
324 : void
325 7 : Gogo::propagate_writebarrierrec()
326 : {
327 7 : std::vector<Named_object*> worklist;
328 7 : Collect_writebarrierrec_functions cwf(&worklist);
329 7 : this->traverse(&cwf);
330 :
331 7 : Collect_callees cc(&worklist);
332 :
333 12179 : while (!worklist.empty())
334 : {
335 12172 : Named_object* no = worklist.back();
336 12172 : worklist.pop_back();
337 :
338 12172 : unsigned int pragmas = no->func_value()->pragmas();
339 12172 : if ((pragmas & GOPRAGMA_MARK) != 0)
340 : {
341 : // We've already seen this function.
342 8606 : continue;
343 : }
344 3566 : if ((pragmas & GOPRAGMA_YESWRITEBARRIERREC) != 0)
345 : {
346 : // We don't want to propagate //go:nowritebarrierrec into
347 : // this function or it's callees.
348 126 : continue;
349 : }
350 :
351 3440 : no->func_value()->set_pragmas(pragmas
352 : | GOPRAGMA_NOWRITEBARRIER
353 : | GOPRAGMA_MARK);
354 :
355 3440 : no->func_value()->traverse(&cc);
356 : }
357 7 : }
358 :
359 : // Add write barriers to the IR. This are required by the concurrent
360 : // garbage collector. A write barrier is needed for any write of a
361 : // pointer into memory controlled by the garbage collector. Write
362 : // barriers are not required for writes to local variables that live
363 : // on the stack. Write barriers are only required when the runtime
364 : // enables them, which can be checked using a run time check on
365 : // runtime.writeBarrier.enabled.
366 : //
367 : // Essentially, for each assignment A = B, where A is or contains a
368 : // pointer, and where A is not, or at any rate may not be, a stack
369 : // variable, we rewrite it into
370 : // if runtime.writeBarrier.enabled {
371 : // typedmemmove(typeof(A), &A, &B)
372 : // } else {
373 : // A = B
374 : // }
375 : //
376 : // The test of runtime.writeBarrier.Enabled is implemented by treating
377 : // the variable as a *uint32, and testing *runtime.writeBarrier != 0.
378 : // This is compatible with the definition in the runtime package.
379 : //
380 : // For types that are pointer shared (pointers, maps, chans, funcs),
381 : // we replaced the call to typedmemmove with gcWriteBarrier(&A, B).
382 : // As far as the GC is concerned, all pointers are the same, so it
383 : // doesn't need the type descriptor.
384 : //
385 : // There are possible optimizations that are not implemented.
386 : //
387 : // runtime.writeBarrier can only change when the goroutine is
388 : // preempted, which in practice means when a call is made into the
389 : // runtime package, so we could optimize by only testing it once
390 : // between function calls.
391 : //
392 : // A slice could be handled with a call to gcWriteBarrier plus two
393 : // integer moves.
394 :
395 : // Traverse the IR adding write barriers.
396 :
397 : class Write_barriers : public Traverse
398 : {
399 : public:
400 4252 : Write_barriers(Gogo* gogo)
401 4252 : : Traverse(traverse_functions
402 : | traverse_blocks
403 : | traverse_variables
404 : | traverse_statements),
405 4252 : gogo_(gogo), function_(NULL), statements_added_(),
406 4252 : nonwb_pointers_()
407 4252 : { }
408 :
409 : int
410 : function(Named_object*);
411 :
412 : int
413 : block(Block*);
414 :
415 : int
416 : variable(Named_object*);
417 :
418 : int
419 : statement(Block*, size_t* pindex, Statement*);
420 :
421 : private:
422 : // General IR.
423 : Gogo* gogo_;
424 : // Current function.
425 : Function* function_;
426 : // Statements introduced.
427 : Statement_inserter::Statements statements_added_;
428 : // Within a single block, pointer variables that point to values
429 : // that do not need write barriers.
430 : Unordered_set(const Named_object*) nonwb_pointers_;
431 : };
432 :
433 : // Traverse a function. Just record it for later.
434 :
435 : int
436 287649 : Write_barriers::function(Named_object* no)
437 : {
438 287649 : go_assert(this->function_ == NULL);
439 287649 : this->function_ = no->func_value();
440 287649 : int t = this->function_->traverse(this);
441 287649 : this->function_ = NULL;
442 :
443 287649 : if (t == TRAVERSE_EXIT)
444 0 : return t;
445 : return TRAVERSE_SKIP_COMPONENTS;
446 : }
447 :
448 : // Traverse a block. Clear anything we know about local pointer
449 : // variables.
450 :
451 : int
452 2193254 : Write_barriers::block(Block*)
453 : {
454 2193254 : this->nonwb_pointers_.clear();
455 2193254 : return TRAVERSE_CONTINUE;
456 : }
457 :
458 : // Insert write barriers for a global variable: ensure that variable
459 : // initialization is handled correctly. This is rarely needed, since
460 : // we currently don't enable background GC until after all global
461 : // variables are initialized. But we do need this if an init function
462 : // calls runtime.GC.
463 :
464 : int
465 1409155 : Write_barriers::variable(Named_object* no)
466 : {
467 : // We handle local variables in the variable declaration statement.
468 : // We only have to handle global variables here.
469 1409155 : if (!no->is_variable())
470 : return TRAVERSE_CONTINUE;
471 1171468 : Variable* var = no->var_value();
472 1171468 : if (!var->is_global())
473 : return TRAVERSE_CONTINUE;
474 :
475 : // Nothing to do if there is no initializer.
476 329302 : Expression* init = var->init();
477 329302 : if (init == NULL)
478 : return TRAVERSE_CONTINUE;
479 :
480 : // Nothing to do for variables that do not contain any pointers.
481 19671 : if (!var->type()->has_pointer())
482 : return TRAVERSE_CONTINUE;
483 :
484 : // Nothing to do if the initializer is static.
485 17263 : init = Expression::make_cast(var->type(), init, var->location());
486 33549 : if (!var->has_pre_init() && init->is_static_initializer())
487 : return TRAVERSE_CONTINUE;
488 :
489 : // Nothing to do for a type that can not be in the heap, or a
490 : // pointer to a type that can not be in the heap.
491 11992 : if (!var->type()->in_heap())
492 : return TRAVERSE_CONTINUE;
493 11992 : if (var->type()->points_to() != NULL && !var->type()->points_to()->in_heap())
494 : return TRAVERSE_CONTINUE;
495 :
496 : // Otherwise change the initializer into a pre_init assignment
497 : // statement with a write barrier.
498 :
499 : // We can't check for a dependency of the variable on itself after
500 : // we make this change, because the preinit statement will always
501 : // depend on the variable (since it assigns to it). So check for a
502 : // self-dependency now.
503 11992 : this->gogo_->check_self_dep(no);
504 :
505 : // Replace the initializer.
506 11992 : Location loc = init->location();
507 11992 : Expression* ref = Expression::make_var_reference(no, loc);
508 :
509 11992 : Statement_inserter inserter(this->gogo_, var, &this->statements_added_);
510 11992 : Statement* s = this->gogo_->assign_with_write_barrier(NULL, NULL, &inserter,
511 11992 : ref, init, loc);
512 11992 : this->statements_added_.insert(s);
513 :
514 11992 : var->add_preinit_statement(this->gogo_, s);
515 11992 : var->clear_init();
516 :
517 11992 : return TRAVERSE_CONTINUE;
518 : }
519 :
520 : // Insert write barriers for statements.
521 :
522 : int
523 5673760 : Write_barriers::statement(Block* block, size_t* pindex, Statement* s)
524 : {
525 5673760 : if (this->statements_added_.find(s) != this->statements_added_.end())
526 : return TRAVERSE_SKIP_COMPONENTS;
527 :
528 5630164 : switch (s->classification())
529 : {
530 : default:
531 : break;
532 :
533 382208 : case Statement::STATEMENT_VARIABLE_DECLARATION:
534 382208 : {
535 382208 : Variable_declaration_statement* vds =
536 382208 : s->variable_declaration_statement();
537 382208 : Named_object* no = vds->var();
538 382208 : Variable* var = no->var_value();
539 :
540 : // We may need to emit a write barrier for the initialization
541 : // of the variable.
542 :
543 : // Nothing to do for a variable with no initializer.
544 382208 : Expression* init = var->init();
545 382208 : if (init == NULL)
546 : break;
547 :
548 : // Nothing to do if the variable is not in the heap. Only
549 : // local variables get declaration statements, and local
550 : // variables on the stack do not require write barriers.
551 280530 : if (!var->is_in_heap())
552 : {
553 : // If this is a pointer variable, and assigning through
554 : // the initializer does not require a write barrier,
555 : // record that fact.
556 273129 : if (var->type()->points_to() != NULL
557 273129 : && this->gogo_->is_nonwb_pointer(init, &this->nonwb_pointers_))
558 1038 : this->nonwb_pointers_.insert(no);
559 :
560 : break;
561 : }
562 :
563 : // Nothing to do if the variable does not contain any pointers.
564 7401 : if (!var->type()->has_pointer())
565 : break;
566 :
567 : // Nothing to do for a type that can not be in the heap, or a
568 : // pointer to a type that can not be in the heap.
569 6212 : if (!var->type()->in_heap())
570 : break;
571 6212 : if (var->type()->points_to() != NULL
572 6212 : && !var->type()->points_to()->in_heap())
573 : break;
574 :
575 : // Otherwise initialize the variable with a write barrier.
576 :
577 6212 : Function* function = this->function_;
578 6212 : Location loc = init->location();
579 6212 : Statement_inserter inserter(block, pindex, &this->statements_added_);
580 :
581 : // Insert the variable declaration statement with no
582 : // initializer, so that the variable exists.
583 6212 : var->clear_init();
584 6212 : inserter.insert(s);
585 :
586 : // Create a statement that initializes the variable with a
587 : // write barrier.
588 6212 : Expression* ref = Expression::make_var_reference(no, loc);
589 6212 : Statement* assign = this->gogo_->assign_with_write_barrier(function,
590 : block,
591 : &inserter,
592 : ref, init,
593 6212 : loc);
594 6212 : this->statements_added_.insert(assign);
595 :
596 : // Replace the old variable declaration statement with the new
597 : // initialization.
598 6212 : block->replace_statement(*pindex, assign);
599 : }
600 6212 : break;
601 :
602 1005541 : case Statement::STATEMENT_ASSIGNMENT:
603 1005541 : {
604 1005541 : Assignment_statement* as = s->assignment_statement();
605 :
606 1005541 : Expression* lhs = as->lhs();
607 1005541 : Expression* rhs = as->rhs();
608 :
609 : // Keep track of variables whose values do not escape.
610 1005541 : Var_expression* lhsve = lhs->var_expression();
611 660588 : if (lhsve != NULL && lhsve->type()->points_to() != NULL)
612 : {
613 47868 : Named_object* no = lhsve->named_object();
614 47868 : if (this->gogo_->is_nonwb_pointer(rhs, &this->nonwb_pointers_))
615 995 : this->nonwb_pointers_.insert(no);
616 : else
617 46873 : this->nonwb_pointers_.erase(no);
618 : }
619 :
620 1005541 : if (as->omit_write_barrier())
621 : break;
622 :
623 : // We may need to emit a write barrier for the assignment.
624 :
625 989319 : if (!this->gogo_->assign_needs_write_barrier(lhs,
626 : &this->nonwb_pointers_))
627 : break;
628 :
629 : // Change the assignment to use a write barrier.
630 52932 : Function* function = this->function_;
631 52932 : Location loc = as->location();
632 52932 : Statement_inserter inserter =
633 52932 : Statement_inserter(block, pindex, &this->statements_added_);
634 52932 : Statement* assign = this->gogo_->assign_with_write_barrier(function,
635 : block,
636 : &inserter,
637 : lhs, rhs,
638 52932 : loc);
639 52932 : this->statements_added_.insert(assign);
640 52932 : block->replace_statement(*pindex, assign);
641 : }
642 52932 : break;
643 : }
644 :
645 : return TRAVERSE_CONTINUE;
646 : }
647 :
648 : // The write barrier pass.
649 :
650 : void
651 4646 : Gogo::add_write_barriers()
652 : {
653 4646 : if (saw_errors())
654 394 : return;
655 :
656 4252 : Mark_address_taken mat(this);
657 4252 : this->traverse(&mat);
658 :
659 4252 : if (this->compiling_runtime() && this->package_name() == "runtime")
660 : {
661 7 : this->propagate_writebarrierrec();
662 :
663 7 : Check_escape chk;
664 7 : this->traverse(&chk);
665 7 : }
666 :
667 4252 : Write_barriers wb(this);
668 4252 : this->traverse(&wb);
669 4252 : }
670 :
671 : // Return the runtime.writeBarrier variable.
672 :
673 : Named_object*
674 202412 : Gogo::write_barrier_variable()
675 : {
676 202412 : static Named_object* write_barrier_var;
677 202412 : if (write_barrier_var == NULL)
678 : {
679 2602 : Location bloc = Linemap::predeclared_location();
680 :
681 2602 : Type* bool_type = Type::lookup_bool_type();
682 2602 : Array_type* pad_type =
683 2602 : Type::make_array_type(Type::lookup_integer_type("byte"),
684 : Expression::make_integer_ul(3, NULL, bloc));
685 2602 : Type* uint64_type = Type::lookup_integer_type("uint64");
686 2602 : Type* wb_type = Type::make_builtin_struct_type(5,
687 : "enabled", bool_type,
688 : "pad", pad_type,
689 : "needed", bool_type,
690 : "cgo", bool_type,
691 : "alignme", uint64_type);
692 :
693 2602 : Variable* var = new Variable(wb_type, NULL,
694 2602 : true, false, false, bloc);
695 :
696 2602 : bool add_to_globals;
697 5204 : Package* package = this->add_imported_package("runtime", "_", false,
698 5204 : "runtime", "runtime",
699 : bloc, &add_to_globals);
700 2602 : write_barrier_var = Named_object::make_variable("writeBarrier",
701 : package, var);
702 : }
703 :
704 202412 : return write_barrier_var;
705 : }
706 :
707 : // Return whether an assignment that sets LHS needs a write barrier.
708 : // NONWB_POINTERS is a set of variables that point to values that do
709 : // not need write barriers.
710 :
711 : bool
712 995825 : Gogo::assign_needs_write_barrier(
713 : Expression* lhs,
714 : Unordered_set(const Named_object*)* nonwb_pointers)
715 : {
716 : // Nothing to do if the variable does not contain any pointers.
717 995825 : if (!lhs->type()->has_pointer())
718 : return false;
719 :
720 : // An assignment to a field or an array index is handled like an
721 : // assignment to the struct.
722 436622 : while (true)
723 : {
724 : // Nothing to do for a type that can not be in the heap, or a
725 : // pointer to a type that can not be in the heap. We check this
726 : // at each level of a struct.
727 436622 : if (!lhs->type()->in_heap())
728 : return false;
729 436468 : if (lhs->type()->points_to() != NULL
730 516677 : && !lhs->type()->points_to()->in_heap())
731 : return false;
732 :
733 : // For a struct assignment, we don't need a write barrier if all
734 : // the field types can not be in the heap.
735 436468 : Struct_type* st = lhs->type()->struct_type();
736 72223 : if (st != NULL)
737 : {
738 72223 : bool in_heap = false;
739 72223 : const Struct_field_list* fields = st->fields();
740 102923 : for (Struct_field_list::const_iterator p = fields->begin();
741 102923 : p != fields->end();
742 30700 : p++)
743 : {
744 102923 : Type* ft = p->type();
745 102923 : if (!ft->has_pointer())
746 30700 : continue;
747 72223 : if (!ft->in_heap())
748 0 : continue;
749 72223 : if (ft->points_to() != NULL && !ft->points_to()->in_heap())
750 0 : continue;
751 : in_heap = true;
752 : break;
753 : }
754 72223 : if (!in_heap)
755 : return false;
756 : }
757 :
758 436468 : Field_reference_expression* fre = lhs->field_reference_expression();
759 436468 : if (fre != NULL)
760 : {
761 46310 : lhs = fre->expr();
762 46310 : continue;
763 : }
764 :
765 390158 : Array_index_expression* aie = lhs->array_index_expression();
766 26720 : if (aie != NULL
767 26720 : && aie->end() == NULL
768 26720 : && !aie->array()->type()->is_slice_type())
769 : {
770 12369 : lhs = aie->array();
771 12369 : continue;
772 : }
773 :
774 377789 : break;
775 : }
776 :
777 : // Nothing to do for an assignment to a temporary.
778 1268721 : if (lhs->temporary_reference_expression() != NULL)
779 : return false;
780 :
781 : // Nothing to do for an assignment to a sink.
782 328985 : if (lhs->is_sink_expression())
783 : return false;
784 :
785 : // Nothing to do for an assignment to a local variable that is not
786 : // on the heap.
787 328985 : Var_expression* ve = lhs->var_expression();
788 279720 : if (ve != NULL)
789 : {
790 279720 : Named_object* no = ve->named_object();
791 279720 : if (no->is_variable())
792 : {
793 92266 : Variable* var = no->var_value();
794 1084391 : if (!var->is_global() && !var->is_in_heap())
795 : return false;
796 : }
797 187454 : else if (no->is_result_variable())
798 : {
799 187454 : Result_variable* rvar = no->result_var_value();
800 187454 : if (!rvar->is_in_heap())
801 : return false;
802 : }
803 : }
804 :
805 : // Nothing to do for an assignment to *(convert(&x)) where
806 : // x is local variable or a temporary variable.
807 56413 : Unary_expression* ue = lhs->unary_expression();
808 33111 : if (ue != NULL
809 33111 : && ue->op() == OPERATOR_MULT
810 33111 : && this->is_nonwb_pointer(ue->operand(), nonwb_pointers))
811 : return false;
812 :
813 : // Write barrier needed in other cases.
814 : return true;
815 : }
816 :
817 : // Return whether EXPR is the address of a variable that can be set
818 : // without a write barrier. That is, if this returns true, then an
819 : // assignment to *EXPR does not require a write barrier.
820 : // NONWB_POINTERS is a set of variables that point to values that do
821 : // not need write barriers.
822 :
823 : bool
824 129317 : Gogo::is_nonwb_pointer(Expression* expr,
825 : Unordered_set(const Named_object*)* nonwb_pointers)
826 : {
827 141549 : while (true)
828 : {
829 141549 : if (expr->conversion_expression() != NULL)
830 8791 : expr = expr->conversion_expression()->expr();
831 132758 : else if (expr->unsafe_conversion_expression() != NULL)
832 3441 : expr = expr->unsafe_conversion_expression()->expr();
833 : else
834 : break;
835 : }
836 :
837 129317 : Var_expression* ve = expr->var_expression();
838 129317 : if (ve != NULL
839 129317 : && nonwb_pointers != NULL
840 129317 : && nonwb_pointers->find(ve->named_object()) != nonwb_pointers->end())
841 265 : return true;
842 :
843 131817 : Unary_expression* ue = expr->unary_expression();
844 4857 : if (ue == NULL || ue->op() != OPERATOR_AND)
845 : return false;
846 4033 : if (this->assign_needs_write_barrier(ue->operand(), nonwb_pointers))
847 : return false;
848 : return true;
849 : }
850 :
851 : // Return a statement that sets LHS to RHS using a write barrier.
852 : // ENCLOSING is the enclosing block.
853 :
854 : Statement*
855 72352 : Gogo::assign_with_write_barrier(Function* function, Block* enclosing,
856 : Statement_inserter* inserter, Expression* lhs,
857 : Expression* rhs, Location loc)
858 : {
859 72352 : if (function != NULL && (function->pragmas() & GOPRAGMA_NOWRITEBARRIER) != 0)
860 0 : go_error_at(loc, "write barrier prohibited");
861 :
862 72352 : Type* type = lhs->type();
863 72352 : go_assert(type->has_pointer());
864 :
865 72352 : Expression* addr;
866 72352 : if (lhs->unary_expression() != NULL
867 5360 : && lhs->unary_expression()->op() == OPERATOR_MULT)
868 5360 : addr = lhs->unary_expression()->operand();
869 : else
870 : {
871 66992 : addr = Expression::make_unary(OPERATOR_AND, lhs, loc);
872 66992 : addr->unary_expression()->set_does_not_escape();
873 : }
874 72352 : Temporary_statement* lhs_temp = Statement::make_temporary(NULL, addr, loc);
875 72352 : lhs_temp->determine_types(this);
876 72352 : inserter->insert(lhs_temp);
877 72352 : lhs = Expression::make_temporary_reference(lhs_temp, loc);
878 :
879 72352 : if (!Type::are_identical(type, rhs->type(),
880 : Type::COMPARE_ERRORS | Type::COMPARE_TAGS,
881 : NULL)
882 3155 : && rhs->type()->interface_type() != NULL
883 72352 : && !rhs->is_multi_eval_safe())
884 : {
885 : // May need a temporary for interface conversion.
886 0 : Temporary_statement* temp = Statement::make_temporary(NULL, rhs, loc);
887 0 : temp->determine_types(this);
888 0 : inserter->insert(temp);
889 0 : rhs = Expression::make_temporary_reference(temp, loc);
890 : }
891 72352 : rhs = Expression::convert_for_assignment(this, type, rhs, loc);
892 72352 : Temporary_statement* rhs_temp = NULL;
893 72352 : if (!rhs->is_multi_eval_safe())
894 : {
895 46427 : rhs_temp = Statement::make_temporary(NULL, rhs, loc);
896 46427 : rhs_temp->determine_types(this);
897 46427 : inserter->insert(rhs_temp);
898 46427 : rhs = Expression::make_temporary_reference(rhs_temp, loc);
899 : }
900 :
901 72352 : Expression* indir =
902 72352 : Expression::make_dereference(lhs, Expression::NIL_CHECK_DEFAULT, loc);
903 72352 : Statement* assign = Statement::make_assignment(indir, rhs, loc);
904 :
905 72352 : lhs = Expression::make_temporary_reference(lhs_temp, loc);
906 72352 : if (rhs_temp != NULL)
907 46427 : rhs = Expression::make_temporary_reference(rhs_temp, loc);
908 :
909 72352 : Type* unsafe_ptr_type = Type::make_pointer_type(Type::make_void_type());
910 72352 : lhs = Expression::make_unsafe_cast(unsafe_ptr_type, lhs, loc);
911 :
912 72352 : Type* uintptr_type = Type::lookup_integer_type("uintptr");
913 72352 : Expression* call;
914 72352 : switch (type->base()->classification())
915 : {
916 0 : default:
917 0 : go_unreachable();
918 :
919 : case Type::TYPE_ERROR:
920 : return assign;
921 :
922 26011 : case Type::TYPE_POINTER:
923 26011 : case Type::TYPE_FUNCTION:
924 26011 : case Type::TYPE_MAP:
925 26011 : case Type::TYPE_CHANNEL:
926 26011 : {
927 : // These types are all represented by a single pointer.
928 26011 : rhs = Expression::make_unsafe_cast(uintptr_type, rhs, loc);
929 26011 : call = Runtime::make_call(this, Runtime::GCWRITEBARRIER, loc, 2,
930 : lhs, rhs);
931 : }
932 26011 : break;
933 :
934 10939 : case Type::TYPE_STRING:
935 10939 : {
936 : // Assign the length field directly.
937 10939 : Expression* llen =
938 10939 : Expression::make_string_info(indir->copy(),
939 : Expression::STRING_INFO_LENGTH,
940 : loc);
941 10939 : Expression* rlen =
942 10939 : Expression::make_string_info(rhs,
943 : Expression::STRING_INFO_LENGTH,
944 : loc);
945 10939 : Statement* as = Statement::make_assignment(llen, rlen, loc);
946 10939 : as->determine_types(this);
947 10939 : inserter->insert(as);
948 :
949 : // Assign the data field with a write barrier.
950 10939 : lhs =
951 10939 : Expression::make_string_info(indir->copy(),
952 : Expression::STRING_INFO_DATA,
953 : loc);
954 10939 : rhs =
955 10939 : Expression::make_string_info(rhs,
956 : Expression::STRING_INFO_DATA,
957 : loc);
958 10939 : assign = Statement::make_assignment(lhs, rhs, loc);
959 10939 : lhs = Expression::make_unary(OPERATOR_AND, lhs, loc);
960 10939 : rhs = Expression::make_unsafe_cast(uintptr_type, rhs, loc);
961 10939 : call = Runtime::make_call(this, Runtime::GCWRITEBARRIER, loc, 2,
962 : lhs, rhs);
963 : }
964 10939 : break;
965 :
966 13140 : case Type::TYPE_INTERFACE:
967 13140 : {
968 : // Assign the first field directly.
969 : // The first field is either a type descriptor or a method table.
970 : // Type descriptors are either statically created, or created by
971 : // the reflect package. For the latter the reflect package keeps
972 : // all references.
973 : // Method tables are either statically created or persistently
974 : // allocated.
975 : // In all cases they don't need a write barrier.
976 13140 : Expression* ltab =
977 13140 : Expression::make_interface_info(indir->copy(),
978 : Expression::INTERFACE_INFO_METHODS,
979 : loc);
980 13140 : Expression* rtab =
981 13140 : Expression::make_interface_info(rhs,
982 : Expression::INTERFACE_INFO_METHODS,
983 : loc);
984 13140 : Statement* as = Statement::make_assignment(ltab, rtab, loc);
985 13140 : as->determine_types(this);
986 13140 : inserter->insert(as);
987 :
988 : // Assign the data field with a write barrier.
989 13140 : lhs =
990 13140 : Expression::make_interface_info(indir->copy(),
991 : Expression::INTERFACE_INFO_OBJECT,
992 : loc);
993 13140 : rhs =
994 13140 : Expression::make_interface_info(rhs,
995 : Expression::INTERFACE_INFO_OBJECT,
996 : loc);
997 13140 : assign = Statement::make_assignment(lhs, rhs, loc);
998 13140 : lhs = Expression::make_unary(OPERATOR_AND, lhs, loc);
999 13140 : rhs = Expression::make_unsafe_cast(uintptr_type, rhs, loc);
1000 13140 : call = Runtime::make_call(this, Runtime::GCWRITEBARRIER, loc, 2,
1001 : lhs, rhs);
1002 : }
1003 13140 : break;
1004 :
1005 15621 : case Type::TYPE_ARRAY:
1006 15621 : if (type->is_slice_type())
1007 : {
1008 : // Assign the lenth fields directly.
1009 15546 : Expression* llen =
1010 15546 : Expression::make_slice_info(indir->copy(),
1011 : Expression::SLICE_INFO_LENGTH,
1012 : loc);
1013 15546 : Expression* rlen =
1014 15546 : Expression::make_slice_info(rhs,
1015 : Expression::SLICE_INFO_LENGTH,
1016 : loc);
1017 15546 : Statement* as = Statement::make_assignment(llen, rlen, loc);
1018 15546 : as->determine_types(this);
1019 15546 : inserter->insert(as);
1020 :
1021 : // Assign the capacity fields directly.
1022 15546 : Expression* lcap =
1023 15546 : Expression::make_slice_info(indir->copy(),
1024 : Expression::SLICE_INFO_CAPACITY,
1025 : loc);
1026 15546 : Expression* rcap =
1027 15546 : Expression::make_slice_info(rhs,
1028 : Expression::SLICE_INFO_CAPACITY,
1029 : loc);
1030 15546 : as = Statement::make_assignment(lcap, rcap, loc);
1031 15546 : as->determine_types(this);
1032 15546 : inserter->insert(as);
1033 :
1034 : // Assign the data field with a write barrier.
1035 15546 : lhs =
1036 15546 : Expression::make_slice_info(indir->copy(),
1037 : Expression::SLICE_INFO_VALUE_POINTER,
1038 : loc);
1039 15546 : rhs =
1040 15546 : Expression::make_slice_info(rhs,
1041 : Expression::SLICE_INFO_VALUE_POINTER,
1042 : loc);
1043 15546 : assign = Statement::make_assignment(lhs, rhs, loc);
1044 15546 : lhs = Expression::make_unary(OPERATOR_AND, lhs, loc);
1045 15546 : rhs = Expression::make_unsafe_cast(uintptr_type, rhs, loc);
1046 15546 : call = Runtime::make_call(this, Runtime::GCWRITEBARRIER, loc, 2,
1047 : lhs, rhs);
1048 15546 : break;
1049 : }
1050 : // fallthrough
1051 :
1052 6716 : case Type::TYPE_STRUCT:
1053 6716 : if (type->is_direct_iface_type())
1054 : {
1055 50 : rhs = Expression::unpack_direct_iface(rhs, loc);
1056 50 : rhs = Expression::make_unsafe_cast(uintptr_type, rhs, loc);
1057 50 : call = Runtime::make_call(this, Runtime::GCWRITEBARRIER, loc, 2,
1058 : lhs, rhs);
1059 : }
1060 : else
1061 : {
1062 : // TODO: split assignments for small struct/array?
1063 6666 : rhs = Expression::make_unary(OPERATOR_AND, rhs, loc);
1064 6666 : rhs->unary_expression()->set_does_not_escape();
1065 6666 : call = Runtime::make_call(this, Runtime::TYPEDMEMMOVE, loc, 3,
1066 : Expression::make_type_descriptor(type, loc),
1067 : lhs, rhs);
1068 : }
1069 : break;
1070 : }
1071 :
1072 72352 : return this->check_write_barrier(enclosing, assign,
1073 72352 : Statement::make_statement(call, false));
1074 : }
1075 :
1076 : // Return a statement that tests whether write barriers are enabled
1077 : // and executes either the efficient code or the write barrier
1078 : // function call, depending.
1079 :
1080 : Statement*
1081 202412 : Gogo::check_write_barrier(Block* enclosing, Statement* without,
1082 : Statement* with)
1083 : {
1084 202412 : Location loc = without->location();
1085 202412 : Named_object* wb = this->write_barrier_variable();
1086 : // We pretend that writeBarrier is a uint32, so that we do a
1087 : // 32-bit load. That is what the gc toolchain does.
1088 202412 : Type* void_type = Type::make_void_type();
1089 202412 : Type* unsafe_pointer_type = Type::make_pointer_type(void_type);
1090 202412 : Type* uint32_type = Type::lookup_integer_type("uint32");
1091 202412 : Type* puint32_type = Type::make_pointer_type(uint32_type);
1092 202412 : Expression* ref = Expression::make_var_reference(wb, loc);
1093 202412 : ref = Expression::make_unary(OPERATOR_AND, ref, loc);
1094 202412 : ref = Expression::make_cast(unsafe_pointer_type, ref, loc);
1095 202412 : ref = Expression::make_cast(puint32_type, ref, loc);
1096 202412 : ref = Expression::make_dereference(ref,
1097 : Expression::NIL_CHECK_NOT_NEEDED, loc);
1098 202412 : Expression* zero = Expression::make_integer_ul(0, ref->type(), loc);
1099 202412 : Expression* cond = Expression::make_binary(OPERATOR_EQEQ, ref, zero, loc);
1100 :
1101 202412 : Block* then_block = new Block(enclosing, loc);
1102 202412 : then_block->add_statement(without);
1103 :
1104 202412 : Block* else_block = new Block(enclosing, loc);
1105 202412 : else_block->add_statement(with);
1106 :
1107 202412 : Statement* s = Statement::make_if_statement(cond, then_block, else_block,
1108 : loc);
1109 202412 : s->determine_types(this);
1110 202412 : return s;
1111 : }
|
