Support heap traversal

We use the object enumeration feature in mmtk-core to implement heap
traversal.  This enables `ObjectSpace.each_object` as well as the
`TracePoint` utility which needs heap traversal to enumerate objects and
set hooks.

Test cases involving each_object and TracePoint are no longer excluded.
The TestTracepointObj test case remains excluded because we have not
implemented querying GC statistics from TracePoint.

Author: wks

Diff for: gc/default.c

@@ -3207,9 +3207,105 @@ objspace_each_objects(rb_objspace_t *objspace, each_obj_callback *callback, void
     objspace_each_exec(protected, &each_obj_data);
 }
 
+#if USE_MMTK
+struct rb_mmtk_build_obj_array_data {
+    VALUE **array_ptr;
+    size_t len;
+    size_t capa;
+};
+
+void
+rb_mmtk_build_obj_array_i(MMTk_ObjectReference object, void *data)
+{
+    struct rb_mmtk_build_obj_array_data *build_array_data = (struct rb_mmtk_build_obj_array_data*)data;
+    VALUE *array = *build_array_data->array_ptr;
+    size_t len = build_array_data->len;
+    size_t capa = build_array_data->capa;
+    if (len == capa) {
+        size_t new_capa = capa * 2;
+        VALUE *new_array = (VALUE*)realloc(array, sizeof(VALUE) * new_capa);
+        *build_array_data->array_ptr = new_array;
+        build_array_data->capa = new_capa;
+        array = new_array;
+    }
+
+    RUBY_ASSERT(build_array_data->len < build_array_data->capa);
+
+    array[len] = (VALUE)object;
+    build_array_data->len = len + 1;
+}
+
+void
+rb_mmtk_each_objects_safe(each_obj_callback *callback, void *data)
+{
+    // Allocate a tmpbuf object.  It's OK if it triggers GC now.
+    volatile VALUE tmpbuf = rb_imemo_tmpbuf_auto_free_pointer();
+
+    // Build an array of object references.
+    const size_t initial_capacity = 512;
+    // We must not trigger GC while running `mmtk_enumerate_objects`,
+    // so we use `malloc` directly.
+    // It will be realloced as we add more objects.
+    VALUE *array = (VALUE*)malloc(sizeof(VALUE) * initial_capacity);
+    struct rb_mmtk_build_obj_array_data build_array_data = {
+        .array_ptr = &array,
+        .len = 0,
+        .capa = initial_capacity,
+    };
+
+    // No GC from now on.
+    mmtk_enumerate_objects(rb_mmtk_build_obj_array_i, &build_array_data);
+
+    // Root the array.
+    rb_imemo_tmpbuf_set_ptr(tmpbuf, array);
+    ((rb_imemo_tmpbuf_t*)tmpbuf)->cnt = build_array_data.len;
+    // GC is OK from now on.
+
+    // Inform the VM about malloc memory usage.
+    // Since elements of `array` are rooted by `tmpbuf`, it is safe to trigger GC.
+    // The GC won't free any object because we have just rooted every object.
+    // But the GC may adjust the threshold for triggering the next GC.
+    rb_gc_adjust_memory_usage(sizeof(VALUE) * build_array_data.capa);
+
+    RUBY_DEBUG_LOG("Begin enumerating %zu objects\n", build_array_data.len);
+
+    // Now enumerate objects.
+    // If GC is triggered in `callback`, `tmpbuf` will keep elements of `array` alive.
+    for (size_t i = 0; i < build_array_data.len; i++) {
+        volatile VALUE object = array[i];
+        size_t object_size = rb_mmtk_get_object_size(object);
+        uintptr_t object_end = object + object_size;
+
+        RUBY_DEBUG_LOG("Enumerating object: %p\n", (void*)object);
+        callback((void*)object, (void*)object_end, object_size, data);
+        RB_GC_GUARD(object);
+
+        // Clear the element so that it no longer pins the object if it dies.
+        array[i] = 0;
+    }
+
+    RUBY_DEBUG_LOG("End enumerating %zu objects\n", build_array_data.len);
+
+    // Explicitly free `array` because we know it is no longer used.
+    // Don't wait for GC to free it because `free()` is a bottleneck during GC.
+    // Adjust memory usage accordingly.
+    rb_imemo_tmpbuf_set_ptr(tmpbuf, NULL);
+    ((rb_imemo_tmpbuf_t*)tmpbuf)->cnt = 0;
+    free(array);
+    rb_gc_adjust_memory_usage(-(ssize_t)(sizeof(VALUE) * build_array_data.capa));
+
+    RB_GC_GUARD(tmpbuf);
+}
+#endif
+
 void
 rb_gc_impl_each_objects(void *objspace_ptr, each_obj_callback *callback, void *data)
 {
+    WHEN_USING_MMTK({
+        rb_mmtk_each_objects_safe(callback, data);
+        return;
+    })
+
     objspace_each_objects(objspace_ptr, callback, data, TRUE);
 }
 

Diff for: internal/mmtk.h

@@ -254,4 +254,10 @@ bool mmtk_is_object_wb_unprotected(MMTk_ObjectReference object);
 
 void mmtk_object_reference_write_post(MMTk_Mutator *mutator, MMTk_ObjectReference object);
 
+/**
+ * Enumerate objects.  This function will call `callback(object, data)` for each object. It has
+ * undefined behavior if allocation or GC happens while this function is running.
+ */
+void mmtk_enumerate_objects(void (*callback)(MMTk_ObjectReference, void*), void *data);
+
 #endif /* MMTK_H */

Diff for: internal/mmtk_support.h

@@ -49,6 +49,7 @@ void rb_mmtk_destroy_mutator(MMTk_VMMutatorThread cur_thread, bool at_fork);
 // Object layout
 size_t rb_mmtk_prefix_size(void);
 size_t rb_mmtk_suffix_size(void);
+size_t rb_mmtk_get_object_size(VALUE object);
 
 // Allocation
 VALUE rb_mmtk_alloc_obj(size_t mmtk_alloc_size, size_t size_pool_size, size_t prefix_size);

Diff for: mmtk_support.c

@@ -395,6 +395,13 @@ rb_mmtk_suffix_size(void)
     return ruby_binding_options.suffix_size;
 }
 
+size_t
+rb_mmtk_get_object_size(VALUE object)
+{
+    return *(size_t*)(object - sizeof(VALUE));
+}
+
+
 ////////////////////////////////////////////////////////////////////////////////
 // Allocation
 ////////////////////////////////////////////////////////////////////////////////

Diff for: test/.excludes-mmtk/TestISeq.rb

@@ -1 +1 @@
-exclude(/test_/, "tracepoints are not supported")
+exclude(/test_/, "tracepoints does not supported GC stats yet")