Skip to content

Commit 1612916

Browse files
wkswks
committed
StickyImmix support
1 parent 4d7e5e5 commit 1612916

File tree

8 files changed

+163
-35
lines changed

8 files changed

+163
-35
lines changed

‎array.c

+51
Original file line numberDiff line numberDiff line change
@@ -37,6 +37,9 @@
3737
#endif
3838
#include "ruby_assert.h"
3939

40+
// Conditional compilation macros for MMTk.
41+
#include "internal/mmtk_macros.h"
42+
4043
VALUE rb_cArray;
4144

4245
/* Flags of RArray
@@ -230,6 +233,11 @@ rb_ary_size_as_embedded(VALUE ary)
230233

231234
#if USE_MMTK
232235
void
236+
rb_mmtk_remember_array_content_holder(VALUE ary)
237+
{
238+
rb_gc_writebarrier_remember(rb_mmtk_array_content_holder(ary));
239+
}
240+
void
233241
rb_mmtk_ary_set_objbuf(VALUE ary, VALUE objbuf)
234242
{
235243
RUBY_ASSERT(rb_mmtk_enabled_p());
@@ -381,6 +389,14 @@ memfill(register VALUE *mem, register long size, register VALUE val)
381389
static void
382390
ary_memfill(VALUE ary, long beg, long size, VALUE val)
383391
{
392+
WHEN_USING_MMTK({
393+
rb_mmtk_remember_array_content_holder(ary);
394+
RARRAY_PTR_USE(ary, ptr, {
395+
memfill(ptr + beg, size, val);
396+
});
397+
return;
398+
})
399+
384400
RARRAY_PTR_USE(ary, ptr, {
385401
memfill(ptr + beg, size, val);
386402
RB_OBJ_WRITTEN(ary, Qundef, val);
@@ -392,6 +408,14 @@ ary_memcpy0(VALUE ary, long beg, long argc, const VALUE *argv, VALUE buff_owner_
392408
{
393409
RUBY_ASSERT(!ARY_SHARED_P(buff_owner_ary));
394410

411+
WHEN_USING_MMTK({
412+
rb_mmtk_remember_array_content_holder(buff_owner_ary);
413+
RARRAY_PTR_USE(ary, ptr, {
414+
MEMCPY(ptr+beg, argv, VALUE, argc);
415+
});
416+
return;
417+
})
418+
395419
if (argc > (int)(128/sizeof(VALUE)) /* is magic number (cache line size) */) {
396420
rb_gc_writebarrier_remember(buff_owner_ary);
397421
RARRAY_PTR_USE(ary, ptr, {
@@ -689,7 +713,11 @@ rb_ary_cancel_sharing(VALUE ary)
689713
ARY_SET_CAPA(ary, len);
690714
}
691715

716+
WHEN_USING_MMTK2({
717+
rb_mmtk_remember_array_content_holder(ary);
718+
}, { // when not using MMTk
692719
rb_gc_writebarrier_remember(ary);
720+
})
693721
}
694722
ary_verify(ary);
695723
}
@@ -1475,6 +1503,7 @@ ary_make_partial_step(VALUE ary, VALUE klass, long offset, long len, long step)
14751503
VALUE *ptr = (VALUE *)ARY_EMBED_PTR(result);
14761504
const VALUE *values = RARRAY_CONST_PTR(ary);
14771505

1506+
// MMTk note: Since `result` is embedded, the content holder must be `result` itself.
14781507
RB_OBJ_WRITE(result, ptr, values[offset]);
14791508
ARY_SET_EMBED_LEN(result, 1);
14801509
return result;
@@ -1495,6 +1524,7 @@ ary_make_partial_step(VALUE ary, VALUE klass, long offset, long len, long step)
14951524
const VALUE *values = RARRAY_CONST_PTR(ary);
14961525

14971526
for (i = 0; i < len; ++i) {
1527+
// MMTk note: Since `result` is embedded, the content holder must be `result` itself.
14981528
RB_OBJ_WRITE(result, ptr+i, values[j]);
14991529
j += step;
15001530
}
@@ -1503,12 +1533,22 @@ ary_make_partial_step(VALUE ary, VALUE klass, long offset, long len, long step)
15031533
else {
15041534
const VALUE *values = RARRAY_CONST_PTR(ary);
15051535

1536+
WHEN_USING_MMTK2({
1537+
VALUE holder = rb_mmtk_array_content_holder(ary);
1538+
RARRAY_PTR_USE(result, ptr, {
1539+
for (i = 0; i < len; ++i) {
1540+
RB_OBJ_WRITE(holder, ptr+i, values[j]);
1541+
j += step;
1542+
}
1543+
});
1544+
}, { // When not using MMTk
15061545
RARRAY_PTR_USE(result, ptr, {
15071546
for (i = 0; i < len; ++i) {
15081547
RB_OBJ_WRITE(result, ptr+i, values[j]);
15091548
j += step;
15101549
}
15111550
});
1551+
})
15121552
ARY_SET_LEN(result, len);
15131553
}
15141554

@@ -1578,6 +1618,9 @@ rb_ary_push(VALUE ary, VALUE item)
15781618
{
15791619
long idx = RARRAY_LEN((ary_verify(ary), ary));
15801620
VALUE target_ary = ary_ensure_room_for_push(ary, 1);
1621+
WHEN_USING_MMTK(
1622+
target_ary = rb_mmtk_array_content_holder(target_ary);
1623+
)
15811624
RARRAY_PTR_USE(ary, ptr, {
15821625
RB_OBJ_WRITE(target_ary, &ptr[idx], item);
15831626
});
@@ -2446,7 +2489,15 @@ rb_ary_splice(VALUE ary, long beg, long len, const VALUE *rptr, long rlen)
24462489
}
24472490
if (rlen > 0) {
24482491
if (rofs == -1) {
2492+
WHEN_USING_MMTK2({
2493+
// MMTk note: If rofs == -1, it means it is copying contents from a different
2494+
// array, and this will introduce new edges from `ary`'s content holder to
2495+
// other objects; If rofs >= 0, it's copying within the array, and the object
2496+
// graph doesn't change (ignoring the differences between slots).
2497+
rb_mmtk_remember_array_content_holder(ary);
2498+
}, { // when not using MMTk
24492499
rb_gc_writebarrier_remember(ary);
2500+
})
24502501
}
24512502
else {
24522503
/* In this case, we're copying from a region in this array, so

‎gc.c

+35-10
Original file line numberDiff line numberDiff line change
@@ -153,6 +153,9 @@
153153
#include "builtin.h"
154154
#include "shape.h"
155155

156+
// Conditional compilation macros for MMTk.
157+
#include "internal/mmtk_macros.h"
158+
156159
#define rb_setjmp(env) RUBY_SETJMP(env)
157160
#define rb_jmp_buf rb_jmpbuf_t
158161
#undef rb_data_object_wrap
@@ -1775,6 +1778,10 @@ RVALUE_PINNED(VALUE obj)
17751778
static inline int
17761779
RVALUE_WB_UNPROTECTED(VALUE obj)
17771780
{
1781+
WHEN_USING_MMTK({
1782+
return mmtk_is_object_wb_unprotected((MMTk_ObjectReference)obj);
1783+
})
1784+
17781785
check_rvalue_consistency(obj);
17791786
return RVALUE_WB_UNPROTECTED_BITMAP(obj) != 0;
17801787
}
@@ -2717,16 +2724,18 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
27172724
RB_VM_LOCK_LEAVE_NO_BARRIER();
27182725
#endif
27192726

2727+
if (UNLIKELY(wb_protected == FALSE)) {
27202728
#if USE_MMTK
2721-
if (!rb_mmtk_enabled_p()) {
2729+
if (rb_mmtk_enabled_p()) {
2730+
mmtk_register_wb_unprotected_object((MMTk_ObjectReference)obj);
2731+
} else {
27222732
#endif
2723-
if (UNLIKELY(wb_protected == FALSE)) {
27242733
ASSERT_vm_locking();
27252734
MARK_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
2726-
}
27272735
#if USE_MMTK
2728-
}
2736+
}
27292737
#endif
2738+
}
27302739

27312740
#if RGENGC_PROFILE
27322741
if (wb_protected) {
@@ -3523,6 +3532,7 @@ obj_free(rb_objspace_t *objspace, VALUE obj)
35233532
#endif
35243533

35253534
#if USE_MMTK
3535+
// Note: MMTk removes dead objects from the set of WB-unprotected objects after each GC.
35263536
if (!rb_mmtk_enabled_p()) {
35273537
#endif
35283538
if (RVALUE_WB_UNPROTECTED(obj)) CLEAR_IN_BITMAP(GET_HEAP_WB_UNPROTECTED_BITS(obj), obj);
@@ -9184,6 +9194,13 @@ gc_writebarrier_incremental(VALUE a, VALUE b, rb_objspace_t *objspace)
91849194
void
91859195
rb_gc_writebarrier(VALUE a, VALUE b)
91869196
{
9197+
#if USE_MMTK
9198+
if (rb_mmtk_enabled_p()) {
9199+
mmtk_object_reference_write_post(GET_THREAD()->mutator, (MMTk_ObjectReference)a);
9200+
return;
9201+
}
9202+
#endif
9203+
91879204
rb_objspace_t *objspace = &rb_objspace;
91889205

91899206
if (RGENGC_CHECK_MODE) {
@@ -9224,6 +9241,7 @@ rb_gc_writebarrier_unprotect(VALUE obj)
92249241
{
92259242
#if USE_MMTK
92269243
if (rb_mmtk_enabled_p()) {
9244+
mmtk_register_wb_unprotected_object((MMTk_ObjectReference)obj);
92279245
return;
92289246
}
92299247
#endif
@@ -9268,6 +9286,13 @@ rb_gc_writebarrier_unprotect(VALUE obj)
92689286
void
92699287
rb_gc_writebarrier_remember(VALUE obj)
92709288
{
9289+
#if USE_MMTK
9290+
if (rb_mmtk_enabled_p()) {
9291+
mmtk_object_reference_write_post(GET_THREAD()->mutator, (MMTk_ObjectReference)obj);
9292+
return;
9293+
}
9294+
#endif
9295+
92719296
rb_objspace_t *objspace = &rb_objspace;
92729297

92739298
gc_report(1, objspace, "rb_gc_writebarrier_remember: %s\n", obj_info(obj));
@@ -9287,15 +9312,9 @@ rb_gc_writebarrier_remember(VALUE obj)
92879312
void
92889313
rb_gc_copy_attributes(VALUE dest, VALUE obj)
92899314
{
9290-
#if USE_MMTK
9291-
if (!rb_mmtk_enabled_p()) {
9292-
#endif
92939315
if (RVALUE_WB_UNPROTECTED(obj)) {
92949316
rb_gc_writebarrier_unprotect(dest);
92959317
}
9296-
#if USE_MMTK
9297-
}
9298-
#endif
92999318
rb_gc_copy_finalizer(dest, obj);
93009319
}
93019320

@@ -10146,6 +10165,12 @@ gc_is_moveable_obj(rb_objspace_t *objspace, VALUE obj)
1014610165
static VALUE
1014710166
gc_move(rb_objspace_t *objspace, VALUE scan, VALUE free, size_t src_slot_size, size_t slot_size)
1014810167
{
10168+
#if USE_MMTK
10169+
if (rb_mmtk_enabled_p()) {
10170+
rb_bug("gc_move should not be reached when using MMTk");
10171+
}
10172+
#endif
10173+
1014910174
int marked;
1015010175
int wb_unprotected;
1015110176
int uncollectible;

‎hash.c

+15
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,9 @@
4949
#include "ruby/ractor.h"
5050
#include "vm_sync.h"
5151

52+
// Conditional compilation macros for MMTk.
53+
#include "internal/mmtk_macros.h"
54+
5255
#if USE_MMTK
5356
#include "internal/mmtk.h" // For mmtk_register_ppp
5457
#endif
@@ -3603,7 +3606,11 @@ rb_hash_keys(VALUE hash)
36033606
size = st_keys(table, ptr, size);
36043607
}
36053608
});
3609+
WHEN_USING_MMTK2({
3610+
rb_mmtk_remember_array_content_holder(keys);
3611+
}, { // When not using MMTk
36063612
rb_gc_writebarrier_remember(keys);
3613+
})
36073614
rb_ary_set_len(keys, size);
36083615
}
36093616
else {
@@ -3640,14 +3647,22 @@ rb_hash_values(VALUE hash)
36403647

36413648
if (ST_DATA_COMPATIBLE_P(VALUE)) {
36423649
if (RHASH_AR_TABLE_P(hash)) {
3650+
WHEN_USING_MMTK2({
3651+
rb_mmtk_remember_array_content_holder(values);
3652+
}, { // When not using MMTk
36433653
rb_gc_writebarrier_remember(values);
3654+
})
36443655
RARRAY_PTR_USE(values, ptr, {
36453656
size = ar_values(hash, ptr, size);
36463657
});
36473658
}
36483659
else if (RHASH_ST_TABLE_P(hash)) {
36493660
st_table *table = RHASH_ST_TABLE(hash);
3661+
WHEN_USING_MMTK2({
3662+
rb_mmtk_remember_array_content_holder(values);
3663+
}, { // When not using MMTk
36503664
rb_gc_writebarrier_remember(values);
3665+
})
36513666
RARRAY_PTR_USE(values, ptr, {
36523667
size = st_values(table, ptr, size);
36533668
});

‎include/ruby/internal/core/rarray.h

+38-10
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,10 @@
3535
#include "ruby/internal/value_type.h"
3636
#include "ruby/assert.h"
3737

38+
#if USE_MMTK
39+
#include "ruby/internal/memory.h" // for RB_GC_GUARD
40+
#endif
41+
3842
/**
3943
* Convenient casting macro.
4044
*
@@ -338,29 +342,33 @@ rb_mmtk_array_content_holder(VALUE a)
338342
return RARRAY_EXT(a)->objbuf;
339343
}
340344
}
345+
346+
void rb_mmtk_remember_array_content_holder(VALUE ary);
341347
#endif
342348

343349
#if USE_MMTK
344350
// Defined in mmtk_support.c
345351
bool rb_mmtk_enabled_p(void);
346-
void rb_mmtk_pin_array_buffer(VALUE array, volatile VALUE *stack_slot);
347352

348-
// When using MMTk, we need to pin the underlying buffer if the array is not embedded becuase the
349-
// buffer is in the GC heap. Otherwise, if C calls back to Ruby, and Ruby triggers GCm and GC
350-
// moves the array buffer, the C function will be operating on the old address of the buffer.
351-
#define RBIMPL_RARRAY_STMT(ary, var, expr) do { \
353+
// When using MMTk, we need to
354+
// 1. Apply RB_GC_GUARD to the underlying buffer if the array is not embedded
355+
// becuase the buffer is in the GC heap. Otherwise, if C calls back to
356+
// Ruby, and Ruby triggers GC, and GC moves the array buffer, the C
357+
// function will be operating on the old address of the buffer.
358+
// 2. Apply write barrier to the actual array content holder if the caller
359+
// writes to the array. Currently this is done by the caller. It's better
360+
// to find a way to encapsulate the write barrier access in this macro.
361+
#define RBIMPL_RARRAY_STMT(ary, var, expr) do { \
352362
RBIMPL_ASSERT_TYPE((ary), RUBY_T_ARRAY); \
353363
const VALUE rbimpl_ary = (ary); \
354-
volatile VALUE rb_mmtk_impl_pinned; \
364+
volatile VALUE rb_mmtk_actual_array_content_holder = 0; \
355365
if (rb_mmtk_enabled_p()) { \
356-
rb_mmtk_pin_array_buffer(ary, &rb_mmtk_impl_pinned); \
357-
} else { \
358-
rb_mmtk_impl_pinned = 0; \
366+
rb_mmtk_actual_array_content_holder = rb_mmtk_array_content_holder(ary); \
359367
} \
360368
VALUE *var = rb_ary_ptr_use_start(rbimpl_ary); \
361369
expr; \
362370
rb_ary_ptr_use_end(rbimpl_ary); \
363-
rb_mmtk_impl_pinned = 0; \
371+
RB_GC_GUARD(rb_mmtk_actual_array_content_holder); \
364372
} while (0)
365373
#else
366374
/**
@@ -443,8 +451,28 @@ RARRAY_PTR(VALUE ary)
443451
static inline void
444452
RARRAY_ASET(VALUE ary, long i, VALUE v)
445453
{
454+
#if USE_MMTK
455+
if (rb_mmtk_enabled_p()) {
456+
// When using MMTk, a non-embedded array consists of two heap objects: the array itself and
457+
// and underlying buffer (imemo:mmtk_objbuf). The GC considers them as two separate
458+
// objects. So when writing to a non-embedded array, we should apply write barrier to the
459+
// underlying buffer (imemo:mmtk_objbuf) instead of the array itself.
460+
// We also don't need to use RARRAY_PTR_USE. It is for keeping a pointer to the ary on the
461+
// stack so that it will be treated as a pinning root if GC is triggered inside the
462+
// operation which RARRAY_PTR_USE wraps. But when using MMTk, write barriers never trigger
463+
// GC.
464+
465+
// Determine the source object. Select the objbuf if it is not embedded.
466+
VALUE src_obj = FL_TEST_RAW(ary, RARRAY_EMBED_FLAG) ? ary : RARRAY_EXT(ary)->objbuf;
467+
// Apply the (subsuming) write barrier.
468+
RB_OBJ_WRITE(src_obj, &RARRAY_CONST_PTR(ary)[i], v);
469+
} else {
470+
#endif
446471
RARRAY_PTR_USE(ary, ptr,
447472
RB_OBJ_WRITE(ary, &ptr[i], v));
473+
#if USE_MMTK
474+
}
475+
#endif
448476
}
449477

450478
/**

‎internal/mmtk.h

+6
Original file line numberDiff line numberDiff line change
@@ -207,4 +207,10 @@ bool mmtk_unpin_object(MMTk_ObjectReference object);
207207

208208
bool mmtk_is_pinned(MMTk_ObjectReference object);
209209

210+
void mmtk_register_wb_unprotected_object(MMTk_ObjectReference object);
211+
212+
bool mmtk_is_object_wb_unprotected(MMTk_ObjectReference object);
213+
214+
void mmtk_object_reference_write_post(MMTk_Mutator *mutator, MMTk_ObjectReference object);
215+
210216
#endif /* MMTK_H */

0 commit comments

Comments
 (0)