Split global roots

Split gc_mark_roots into multiple functions so that they can be called
from different work packets.

Author: wks

gc.c

@@ -7315,6 +7315,11 @@ rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr)
 static inline void
 gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
 {
+    WHEN_USING_MMTK({
+        // MMTk scans object concurrently.  This function only adds one more source of data race.
+        return;
+    })
+
     if (RVALUE_OLD_P(obj)) {
         objspace->rgengc.parent_object = obj;
     }
@@ -7670,7 +7675,17 @@ gc_mark_roots(rb_objspace_t *objspace, const char **categoryp)
 #if USE_MMTK
     const bool mmtk_enabled_local = rb_mmtk_enabled_p(); // Allows control-flow sensitive analysis of ec etc
     if (mmtk_enabled_local) {
-        rb_mmtk_assert_mmtk_worker();
+        if (GET_RACTOR()->mfd == NULL) {
+            // When using MMTk, we don't directly call `gc_mark_roots` for scanning roots.
+            // Instead, we split this function into multiple functions named `rb_mmtk_scan_*_roots`
+            // so that they can be paralleled by calling them from different work packets.
+            rb_bug("gc_mark_roots should not be called when using MMTk.");
+        } else {
+            // We still call `gc_mark_roots` when enumerating objects reachable from roots.
+            // In this case, this function will be called from mutator.
+            rb_mmtk_assert_mutator();
+        }
+
         vm = GET_VM();
     } else {
 #endif
@@ -7759,6 +7774,57 @@ gc_mark_roots(rb_objspace_t *objspace, const char **categoryp)
 #undef MARK_CHECKPOINT
 }
 
+#if USE_MMTK
+void
+rb_mmtk_scan_vm_roots(void)
+{
+    rb_vm_t *vm = GET_VM();
+    rb_objspace_t *objspace = vm->objspace;
+
+    rb_vm_mark(vm);
+    if (vm->self) gc_mark(objspace, vm->self);
+}
+
+void
+rb_mmtk_scan_finalizer_tbl_roots(void)
+{
+    rb_vm_t *vm = GET_VM();
+    rb_objspace_t *objspace = vm->objspace;
+
+    mark_finalizer_tbl(objspace, finalizer_table);
+}
+
+void
+rb_mmtk_scan_end_proc_roots(void)
+{
+    rb_mark_end_proc();
+}
+
+void
+rb_mmtk_scan_global_tbl_roots(void)
+{
+    rb_gc_mark_global_tbl();
+}
+
+void
+rb_mmtk_scan_obj_to_id_tbl_roots(void)
+{
+    rb_vm_t *vm = GET_VM();
+    rb_objspace_t *objspace = vm->objspace;
+
+    mark_tbl_no_pin(objspace, objspace->obj_to_id_tbl); /* Only mark ids */
+}
+
+void
+rb_mmtk_scan_misc_roots(void)
+{
+    rb_vm_t *vm = GET_VM();
+    rb_objspace_t *objspace = vm->objspace;
+
+    if (stress_to_class) rb_gc_mark(stress_to_class);
+}
+#endif
+
 #if RGENGC_CHECK_MODE >= 4
 
 #define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
@@ -14778,11 +14844,8 @@ rb_mmtk_update_ci_table(void)
 
 // Workaround private functions
 void
-rb_mmtk_mark_roots(void)
+rb_mmtk_scan_final_jobs_roots(void)
 {
-    rb_vm_t *vm = GET_VM();
-    const char *phase;
-    gc_mark_roots(vm->objspace, &phase);
     rb_mmtk_mark_final_jobs();
 }
 

internal/gc.h

@@ -235,7 +235,14 @@ void rb_gc_initial_stress_set(VALUE flag);
 } while (0)
 
 #if USE_MMTK
-void rb_mmtk_mark_roots(void);
+void rb_mmtk_scan_vm_roots(void);
+void rb_mmtk_scan_finalizer_tbl_roots(void);
+void rb_mmtk_scan_end_proc_roots(void);
+void rb_mmtk_scan_global_tbl_roots(void);
+void rb_mmtk_scan_obj_to_id_tbl_roots(void);
+void rb_mmtk_scan_misc_roots(void);
+void rb_mmtk_scan_final_jobs_roots(void);
+
 void rb_mmtk_mark_children(VALUE obj);
 void rb_mmtk_update_object_references(VALUE obj);
 void rb_mmtk_obj_free(VALUE obj);

internal/mmtk.h

@@ -62,7 +62,13 @@ typedef struct MMTk_RubyUpcalls {
     void (*block_for_gc)(MMTk_VMMutatorThread tls);
     size_t (*number_of_mutators)(void);
     void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data);
-    void (*scan_vm_specific_roots)(void);
+    void (*scan_vm_roots)(void);
+    void (*scan_finalizer_tbl_roots)(void);
+    void (*scan_end_proc_roots)(void);
+    void (*scan_global_tbl_roots)(void);
+    void (*scan_obj_to_id_tbl_roots)(void);
+    void (*scan_misc_roots)(void);
+    void (*scan_final_jobs_roots)(void);
     void (*scan_roots_in_mutator_thread)(MMTk_VMMutatorThread mutator_tls,
                                          MMTk_VMWorkerThread worker_tls);
     void (*scan_object_ruby_style)(MMTk_ObjectReference object);

mmtk_support.c

@@ -1556,16 +1556,6 @@ rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), v
     }
 }
 
-static void
-rb_mmtk_scan_vm_specific_roots(void)
-{
-    rb_mmtk_assert_mmtk_worker();
-
-    RUBY_DEBUG_LOG("Scanning VM-specific roots...");
-
-    rb_mmtk_mark_roots();
-}
-
 static void
 rb_mmtk_scan_roots_in_mutator_thread(MMTk_VMMutatorThread mutator, MMTk_VMWorkerThread worker)
 {
@@ -1616,7 +1606,13 @@ MMTk_RubyUpcalls ruby_upcalls = {
     rb_mmtk_block_for_gc,
     rb_mmtk_number_of_mutators,
     rb_mmtk_get_mutators,
-    rb_mmtk_scan_vm_specific_roots,
+    rb_mmtk_scan_vm_roots,
+    rb_mmtk_scan_finalizer_tbl_roots,
+    rb_mmtk_scan_end_proc_roots,
+    rb_mmtk_scan_global_tbl_roots,
+    rb_mmtk_scan_obj_to_id_tbl_roots,
+    rb_mmtk_scan_misc_roots,
+    rb_mmtk_scan_final_jobs_roots,
     rb_mmtk_scan_roots_in_mutator_thread,
     rb_mmtk_scan_object_ruby_style,
     rb_mmtk_call_gc_mark_children,