Split global roots

Split gc_mark_roots into multiple functions so that they can be called
from different work packets.

Author: wks

gc.c

@@ -7315,6 +7315,11 @@ rb_gc_remove_weak(VALUE parent_obj, VALUE *ptr)
 static inline void
 gc_mark_set_parent(rb_objspace_t *objspace, VALUE obj)
 {
+    WHEN_USING_MMTK({
+        // MMTk scans object concurrently.  This function only adds one more source of data race.
+        return;
+    })
+
     if (RVALUE_OLD_P(obj)) {
         objspace->rgengc.parent_object = obj;
     }
@@ -7664,21 +7669,14 @@ show_mark_ticks(void)
 static void
 gc_mark_roots(rb_objspace_t *objspace, const char **categoryp)
 {
-    rb_execution_context_t *ec;
-    rb_vm_t *vm;
+    WHEN_USING_MMTK({
+        // When using MMTk, we parallelize root scanning by splitting this function into multiple
+        // functions, and call each one in a separate work packet.
+        rb_bug("gc_mark_roots should not be called when using MMTk.");
+    })
 
-#if USE_MMTK
-    const bool mmtk_enabled_local = rb_mmtk_enabled_p(); // Allows control-flow sensitive analysis of ec etc
-    if (mmtk_enabled_local) {
-        rb_mmtk_assert_mmtk_worker();
-        vm = GET_VM();
-    } else {
-#endif
-        ec = GET_EC();
-        vm = rb_ec_vm_ptr(ec);
-#if USE_MMTK
-    }
-#endif
+    rb_execution_context_t *ec = GET_EC();
+    rb_vm_t *vm = rb_ec_vm_ptr(ec);
 
 #if PRINT_ROOT_TICKS
     tick_t start_tick = tick();
@@ -7715,32 +7713,15 @@ gc_mark_roots(rb_objspace_t *objspace, const char **categoryp)
 } while (0)
 
     MARK_CHECKPOINT("vm");
-
-#if USE_MMTK
-    // Note that when using MMTk, this function is executed by a GC worker thread, not a mutator.
-    // Therefore we don't set stack end or scan the current stack.
-    if (!mmtk_enabled_local) {
-#endif
     SET_STACK_END;
-#if USE_MMTK
-    }
-#endif
-
     rb_vm_mark(vm);
     if (vm->self) gc_mark(objspace, vm->self);
 
     MARK_CHECKPOINT("finalizers");
     mark_finalizer_tbl(objspace, finalizer_table);
 
-#if USE_MMTK
-    if (!mmtk_enabled_local) {
-#endif
-    // When using MMTk, the current thread is a GC worker.  Mutators are scanned separately.
     MARK_CHECKPOINT("machine_context");
     mark_current_machine_context(objspace, ec);
-#if USE_MMTK
-    }
-#endif
 
     /* mark protected global variables */
 
@@ -7759,6 +7740,57 @@ gc_mark_roots(rb_objspace_t *objspace, const char **categoryp)
 #undef MARK_CHECKPOINT
 }
 
+#if USE_MMTK
+void
+rb_mmtk_scan_vm_roots(void)
+{
+    rb_vm_t *vm = GET_VM();
+    rb_objspace_t *objspace = vm->objspace;
+
+    rb_vm_mark(vm);
+    if (vm->self) gc_mark(objspace, vm->self);
+}
+
+void
+rb_mmtk_scan_finalizer_tbl_roots(void)
+{
+    rb_vm_t *vm = GET_VM();
+    rb_objspace_t *objspace = vm->objspace;
+
+    mark_finalizer_tbl(objspace, finalizer_table);
+}
+
+void
+rb_mmtk_scan_end_proc_roots(void)
+{
+    rb_mark_end_proc();
+}
+
+void
+rb_mmtk_scan_global_tbl_roots(void)
+{
+    rb_gc_mark_global_tbl();
+}
+
+void
+rb_mmtk_scan_obj_to_id_tbl_roots(void)
+{
+    rb_vm_t *vm = GET_VM();
+    rb_objspace_t *objspace = vm->objspace;
+
+    mark_tbl_no_pin(objspace, objspace->obj_to_id_tbl); /* Only mark ids */
+}
+
+void
+rb_mmtk_scan_misc_roots(void)
+{
+    rb_vm_t *vm = GET_VM();
+    rb_objspace_t *objspace = vm->objspace;
+
+    if (stress_to_class) rb_gc_mark(stress_to_class);
+}
+#endif
+
 #if RGENGC_CHECK_MODE >= 4
 
 #define MAKE_ROOTSIG(obj) (((VALUE)(obj) << 1) | 0x01)
@@ -14778,11 +14810,8 @@ rb_mmtk_update_ci_table(void)
 
 // Workaround private functions
 void
-rb_mmtk_mark_roots(void)
+rb_mmtk_scan_final_jobs_roots(void)
 {
-    rb_vm_t *vm = GET_VM();
-    const char *phase;
-    gc_mark_roots(vm->objspace, &phase);
     rb_mmtk_mark_final_jobs();
 }
 

internal/gc.h

@@ -235,7 +235,14 @@ void rb_gc_initial_stress_set(VALUE flag);
 } while (0)
 
 #if USE_MMTK
-void rb_mmtk_mark_roots(void);
+void rb_mmtk_scan_vm_roots(void);
+void rb_mmtk_scan_finalizer_tbl_roots(void);
+void rb_mmtk_scan_end_proc_roots(void);
+void rb_mmtk_scan_global_tbl_roots(void);
+void rb_mmtk_scan_obj_to_id_tbl_roots(void);
+void rb_mmtk_scan_misc_roots(void);
+void rb_mmtk_scan_final_jobs_roots(void);
+
 void rb_mmtk_mark_children(VALUE obj);
 void rb_mmtk_update_object_references(VALUE obj);
 void rb_mmtk_obj_free(VALUE obj);

internal/mmtk.h

@@ -62,7 +62,13 @@ typedef struct MMTk_RubyUpcalls {
     void (*block_for_gc)(MMTk_VMMutatorThread tls);
     size_t (*number_of_mutators)(void);
     void (*get_mutators)(void (*visit_mutator)(MMTk_Mutator*, void*), void *data);
-    void (*scan_vm_specific_roots)(void);
+    void (*scan_vm_roots)(void);
+    void (*scan_finalizer_tbl_roots)(void);
+    void (*scan_end_proc_roots)(void);
+    void (*scan_global_tbl_roots)(void);
+    void (*scan_obj_to_id_tbl_roots)(void);
+    void (*scan_misc_roots)(void);
+    void (*scan_final_jobs_roots)(void);
     void (*scan_roots_in_mutator_thread)(MMTk_VMMutatorThread mutator_tls,
                                          MMTk_VMWorkerThread worker_tls);
     void (*scan_object_ruby_style)(MMTk_ObjectReference object);

mmtk_support.c

@@ -1556,16 +1556,6 @@ rb_mmtk_get_mutators(void (*visit_mutator)(MMTk_Mutator *mutator, void *data), v
     }
 }
 
-static void
-rb_mmtk_scan_vm_specific_roots(void)
-{
-    rb_mmtk_assert_mmtk_worker();
-
-    RUBY_DEBUG_LOG("Scanning VM-specific roots...");
-
-    rb_mmtk_mark_roots();
-}
-
 static void
 rb_mmtk_scan_roots_in_mutator_thread(MMTk_VMMutatorThread mutator, MMTk_VMWorkerThread worker)
 {
@@ -1616,7 +1606,13 @@ MMTk_RubyUpcalls ruby_upcalls = {
     rb_mmtk_block_for_gc,
     rb_mmtk_number_of_mutators,
     rb_mmtk_get_mutators,
-    rb_mmtk_scan_vm_specific_roots,
+    rb_mmtk_scan_vm_roots,
+    rb_mmtk_scan_finalizer_tbl_roots,
+    rb_mmtk_scan_end_proc_roots,
+    rb_mmtk_scan_global_tbl_roots,
+    rb_mmtk_scan_obj_to_id_tbl_roots,
+    rb_mmtk_scan_misc_roots,
+    rb_mmtk_scan_final_jobs_roots,
     rb_mmtk_scan_roots_in_mutator_thread,
     rb_mmtk_scan_object_ruby_style,
     rb_mmtk_call_gc_mark_children,