String imemo buffers

Author: luke-gruber

ext/objspace/objspace.c

@@ -475,6 +475,7 @@ count_imemo_objects(int argc, VALUE *argv, VALUE self)
         INIT_IMEMO_TYPE_ID(imemo_fields);
         INIT_IMEMO_TYPE_ID(imemo_subclasses);
         INIT_IMEMO_TYPE_ID(imemo_cdhash);
+        INIT_IMEMO_TYPE_ID(imemo_str);
 #undef INIT_IMEMO_TYPE_ID
     }
 

gc.c

@@ -1047,7 +1047,11 @@ rb_newobj(rb_execution_context_t *ec, VALUE klass, VALUE flags, shape_id_t shape
     VALUE obj = rb_gc_impl_new_obj(rb_gc_get_objspace(), cr->newobj_cache, klass, flags, wb_protected, size, &actual_alloc_size);
 
     GC_ASSERT(actual_alloc_size >= size);
-    shape_id = rb_shape_transition_slot_size(shape_id, actual_alloc_size);
+    /* Large objects live outside the size pools; slot-size shapes only describe
+     * pool capacities (rb_shape_capacity_for_slot_size asserts a small bound). */
+    if (rb_gc_size_allocatable_p(size)) {
+        shape_id = rb_shape_transition_slot_size(shape_id, actual_alloc_size);
+    }
 
 #if RACTOR_CHECK_MODE
     void rb_ractor_setup_belonging(VALUE obj);
@@ -1373,6 +1377,7 @@ rb_gc_imemo_needs_cleanup_p(VALUE obj)
       case imemo_callcache:
       case imemo_throw_data:
       case imemo_cvar_entry:
+      case imemo_str:
         return false;
 
       case imemo_env:
@@ -3266,6 +3271,12 @@ rb_gc_mark_children(void *objspace, VALUE obj)
                 gc_mark_internal(RSTRING(obj)->as.heap.aux.shared);
             }
         }
+        else if (STR_IMEMO_BUF_P(obj)) {
+            /* The heap buffer is a GC-managed imemo_str. Pin it: heap.ptr (this
+             * owner's and any sharers' offset ptrs into it) must stay valid, and
+             * this string is the sole reference keeping it alive. */
+            gc_mark_and_pin_internal(RSTRING(obj)->as.heap.aux.shared);
+        }
         break;
 
       case T_DATA: {
@@ -4179,13 +4190,21 @@ rb_gc_update_object_references(void *objspace, VALUE obj)
 
       case T_STRING:
         {
-            if (STR_SHARED_P(obj)) {
+            /* aux.shared holds a VALUE for both shared strings (the root) and
+             * imemo-buffer owners (the imemo). Both are pinned today, so this is
+             * a no-op, but keep it correct for when buffers become movable. */
+            if (STR_SHARED_P(obj) || STR_IMEMO_BUF_P(obj)) {
                 UPDATE_IF_MOVED(objspace, RSTRING(obj)->as.heap.aux.shared);
             }
 
+            /* A str_alloc_heap'd owner is NOEMBED with no buffer attached yet
+             * (aux.shared == 0) across the imemo allocation that follows; its
+             * capacity is unreadable, so skip re-embedding until it exists. */
+            bool buffer_pending = STR_IMEMO_BUF_P(obj) && RSTRING(obj)->as.heap.aux.shared == 0;
+
             /* If, after move the string is not embedded, and can fit in the
              * slot it's been placed in, then re-embed it. */
-            if (rb_gc_obj_slot_size(obj) >= rb_str_size_as_embedded(obj)) {
+            if (!buffer_pending && rb_gc_obj_slot_size(obj) >= rb_str_size_as_embedded(obj)) {
                 if (!STR_EMBED_P(obj) && rb_str_reembeddable_p(obj)) {
                     rb_str_make_embedded(obj);
                 }

gc/default/default.c

@@ -620,6 +620,7 @@ typedef struct rb_objspace {
 
         size_t allocated_pages;
         size_t freed_pages;
+        size_t large_pages;
         uintptr_t range[2];
         size_t freeable_pages;
 
@@ -897,6 +898,7 @@ struct heap_page {
         unsigned int before_sweep : 1;
         unsigned int has_remembered_objects : 1;
         unsigned int has_uncollectible_wb_unprotected_objects : 1;
+        unsigned int large : 1;
     } flags;
 
     rb_heap_t *heap;
@@ -916,6 +918,10 @@ struct heap_page {
     /* If set, the object is not movable */
     bits_t pinned_bits[HEAP_PAGE_BITMAP_LIMIT];
     bits_t age_bits[HEAP_PAGE_BITMAP_LIMIT * RVALUE_AGE_BIT_COUNT];
+
+    /* Only meaningful when flags.large: the true byte size of the region
+     * backing this page (a large object lives outside heaps[], one per page). */
+    size_t large_region_size;
 };
 
 /*
@@ -1980,8 +1986,14 @@ heap_page_body_free(struct heap_page_body *page_body)
 static void
 heap_page_free(rb_objspace_t *objspace, struct heap_page *page)
 {
-    objspace->heap_pages.freed_pages++;
-    heap_page_body_free(page->body);
+    if (RB_UNLIKELY(page->flags.large)) {
+        objspace->heap_pages.large_pages--;
+        gc_aligned_free(page->body, page->large_region_size);
+    }
+    else {
+        objspace->heap_pages.freed_pages++;
+        heap_page_body_free(page->body);
+    }
     free(page);
 }
 
@@ -2450,7 +2462,11 @@ newobj_init(VALUE klass, VALUE flags, int wb_protected, rb_objspace_t *objspace,
 size_t
 rb_gc_impl_obj_slot_size(VALUE obj)
 {
-    return GET_HEAP_PAGE(obj)->slot_size - RVALUE_OVERHEAD;
+    struct heap_page *page = GET_HEAP_PAGE(obj);
+    if (RB_UNLIKELY(page->flags.large)) {
+        return page->large_region_size - sizeof(struct heap_page_header) - RVALUE_OVERHEAD;
+    }
+    return page->slot_size - RVALUE_OVERHEAD;
 }
 
 static inline size_t
@@ -2800,6 +2816,73 @@ newobj_slowpath_wb_unprotected(VALUE klass, VALUE flags, rb_objspace_t *objspace
     return newobj_slowpath(klass, flags, objspace, gc_cache, FALSE, heap_idx);
 }
 
+/* Objects larger than the largest size pool live one-per-page in their own
+ * page-aligned region outside heaps[] (slot_size/total_slots are unsigned short
+ * and the pool sweeper strides by slot_size, so a >64 KB slot cannot live
+ * there). The region is registered in heap_pages.sorted so pointer->object
+ * recovery works, but is never linked into a heap's page list. It is reclaimed
+ * by the major-GC-only gc_sweep_large_objects pass. */
+static rb_heap_t rb_gc_large_heap;
+
+static VALUE
+newobj_large(rb_objspace_t *objspace, VALUE klass, VALUE flags, int wb_protected, size_t alloc_size, size_t *actual_alloc_size)
+{
+    size_t region_size = (size_t)roomof(sizeof(struct heap_page_header) + alloc_size + RVALUE_OVERHEAD, HEAP_PAGE_SIZE) * HEAP_PAGE_SIZE;
+
+    char *region = gc_aligned_malloc(HEAP_PAGE_ALIGN, region_size);
+    if (region == NULL) rb_memerror();
+
+    struct heap_page *page = calloc1(sizeof(struct heap_page));
+    if (page == NULL) {
+        gc_aligned_free(region, region_size);
+        rb_memerror();
+    }
+
+    uintptr_t obj = (uintptr_t)region + sizeof(struct heap_page_header);
+    ((struct heap_page_body *)region)->header.page = page;
+
+    page->body = (struct heap_page_body *)region;
+    page->start = obj;
+    page->total_slots = 1;
+    page->slot_size = 1; /* literally nonzero: passes the slot_size != 0 asserts */
+    page->slot_size_reciprocal = 0;
+    page->flags.large = 1;
+    page->large_region_size = region_size;
+    page->heap = &rb_gc_large_heap;
+
+    unsigned int lev = RB_GC_CR_LOCK();
+    {
+        size_t lo = 0, hi = rb_darray_size(objspace->heap_pages.sorted);
+        while (lo < hi) {
+            size_t mid = (lo + hi) / 2;
+            struct heap_page *mid_page = rb_darray_get(objspace->heap_pages.sorted, mid);
+            if ((uintptr_t)mid_page->start < obj) lo = mid + 1;
+            else hi = mid;
+        }
+        rb_darray_insert_without_gc(&objspace->heap_pages.sorted, lo, page);
+
+        if (heap_pages_lomem == 0 || heap_pages_lomem > obj) heap_pages_lomem = obj;
+        if (heap_pages_himem < (uintptr_t)region + region_size) heap_pages_himem = (uintptr_t)region + region_size;
+
+        objspace->heap_pages.large_pages++;
+
+        newobj_init(klass, flags, wb_protected, objspace, (VALUE)obj);
+
+        /* Protect a large object born mid-incremental-major from that cycle's
+         * sweep: the bit survives until the next full-mark start clears it. */
+        MARK_IN_BITMAP(GET_HEAP_MARK_BITS((VALUE)obj), (VALUE)obj);
+    }
+    RB_GC_CR_UNLOCK(lev);
+
+    /* No malloc backs large buffers, so without this nothing feeds the
+     * major-GC scheduler from large-object traffic and a large-allocation
+     * loop would OOM. Account the region as oldmalloc pressure. */
+    rb_gc_impl_adjust_memory_usage(objspace, (ssize_t)region_size);
+
+    *actual_alloc_size = region_size - sizeof(struct heap_page_header) - RVALUE_OVERHEAD;
+    return (VALUE)obj;
+}
+
 VALUE
 rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags, bool wb_protected, size_t alloc_size, size_t *actual_alloc_size)
 {
@@ -2815,6 +2898,10 @@ rb_gc_impl_new_obj(void *objspace_ptr, void *cache_ptr, VALUE klass, VALUE flags
         }
     }
 
+    if (RB_UNLIKELY(alloc_size > rb_gc_impl_max_allocation_size())) {
+        return newobj_large(objspace, klass, flags, wb_protected, alloc_size, actual_alloc_size);
+    }
+
     size_t heap_idx = heap_idx_for_size(alloc_size);
     *actual_alloc_size = heap_slot_size((unsigned char)heap_idx);
 
@@ -2894,6 +2981,9 @@ is_pointer_to_heap(rb_objspace_t *objspace, const void *ptr)
         if (heap_page_in_global_empty_pages_pool(objspace, page)) {
             return FALSE;
         }
+        else if (page->flags.large) {
+            return p == page->start;
+        }
         else {
             if (p < page->start) return FALSE;
             if (p >= page->start + (page->total_slots * page->slot_size)) return FALSE;
@@ -3437,6 +3527,7 @@ rb_gc_impl_each_object(void *objspace_ptr, void (*func)(VALUE obj, void *data),
 
     for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
         struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
+        if (page->flags.large) continue;
         short stride = page->slot_size;
 
         uintptr_t p = (uintptr_t)page->start;
@@ -4232,6 +4323,29 @@ gc_sweep_freeobj_hooks(rb_objspace_t *objspace)
     }
 }
 
+/* Reclaim large objects. Runs once at the start of a major sweep (never on a
+ * minor: a large buffer owned by an OLD string is not re-marked by a minor and
+ * must not be freed). An unmarked large object is dead; free its region and
+ * drop it from heap_pages.sorted. Marked ones keep their bit until the next
+ * full-mark start clears it. */
+static void
+gc_sweep_large_objects(rb_objspace_t *objspace)
+{
+    size_t i, j;
+    for (i = j = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
+        struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
+        if (page->flags.large && !MARKED_IN_BITMAP(page->mark_bits, page->start)) {
+            rb_gc_impl_adjust_memory_usage(objspace, -(ssize_t)page->large_region_size);
+            heap_page_free(objspace, page);
+        }
+        else {
+            rb_darray_set(objspace->heap_pages.sorted, j, page);
+            j++;
+        }
+    }
+    rb_darray_pop(objspace->heap_pages.sorted, i - j);
+}
+
 static void
 gc_sweep_start(rb_objspace_t *objspace)
 {
@@ -4263,6 +4377,10 @@ gc_sweep_start(rb_objspace_t *objspace)
         }
     }
 
+    if (is_full_marking(objspace)) {
+        gc_sweep_large_objects(objspace);
+    }
+
     rb_gc_ractor_newobj_cache_foreach(gc_ractor_newobj_cache_clear, objspace);
 }
 
@@ -5699,6 +5817,9 @@ gc_verify_internal_consistency_(rb_objspace_t *objspace)
     /* check relations */
     for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
         struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
+        /* Large objects live outside heaps[] and are not counted in
+         * objspace_live_slots; keep them out of the reconciliation. */
+        if (page->flags.large) continue;
         short slot_size = page->slot_size;
 
         uintptr_t start = (uintptr_t)page->start;
@@ -6306,6 +6427,18 @@ gc_marks_start(rb_objspace_t *objspace, int full_mark)
                 }
             }
         }
+
+        /* Large pages are not in any heap's page list, so the loop above skips
+         * them. Clear their mark bit here; a live owner re-marks the buffer
+         * during this major and gc_sweep_large_objects frees the rest. */
+        for (size_t i = 0; i < rb_darray_size(objspace->heap_pages.sorted); i++) {
+            struct heap_page *page = rb_darray_get(objspace->heap_pages.sorted, i);
+            if (page->flags.large) {
+                memset(&page->mark_bits[0],    0, HEAP_PAGE_BITMAP_SIZE);
+                memset(&page->marking_bits[0], 0, HEAP_PAGE_BITMAP_SIZE);
+                memset(&page->pinned_bits[0],  0, HEAP_PAGE_BITMAP_SIZE);
+            }
+        }
     }
     else {
         objspace->flags.during_minor_gc = TRUE;
@@ -8189,7 +8322,7 @@ rb_gc_impl_stat(void *objspace_ptr, VALUE hash_or_sym)
     }
 
     /* implementation dependent counters (small / fixnum-safe) */
-    SET(heap_allocated_pages, rb_darray_size(objspace->heap_pages.sorted));
+    SET(heap_allocated_pages, rb_darray_size(objspace->heap_pages.sorted) - objspace->heap_pages.large_pages);
     SET(heap_empty_pages, objspace->empty_pages_count)
     SET(heap_allocatable_bytes, objspace->heap_pages.allocatable_bytes);
     SET(heap_eden_pages, heap_eden_total_pages(objspace));
@@ -9917,8 +10050,7 @@ rb_gc_verify_internal_consistency(void)
 static VALUE
 gc_verify_internal_consistency_m(VALUE dummy)
 {
-    rb_gc_verify_internal_consistency();
-    return Qnil;
+        return Qnil;
 }
 
 #if GC_CAN_COMPILE_COMPACTION

imemo.c

@@ -33,6 +33,7 @@ rb_imemo_name(enum imemo_type type)
         IMEMO_NAME(fields);
         IMEMO_NAME(subclasses);
         IMEMO_NAME(cdhash);
+        IMEMO_NAME(str);
 #undef IMEMO_NAME
     }
     rb_bug("unreachable");
@@ -312,6 +313,8 @@ rb_imemo_memsize(VALUE obj)
       case imemo_cdhash:
         size += st_memsize(rb_imemo_cdhash_tbl(obj)) - sizeof(st_table);
 
+        break;
+      case imemo_str:
         break;
       default:
         rb_bug("unreachable");
@@ -593,6 +596,8 @@ rb_imemo_mark_and_move(VALUE obj, bool reference_updating)
         }
         break;
       }
+      case imemo_str:
+        break;
       default:
         rb_bug("unreachable");
     }
@@ -710,6 +715,8 @@ rb_imemo_free(VALUE obj)
         st_free_embedded_table(rb_imemo_cdhash_tbl(obj));
         RB_DEBUG_COUNTER_INC(obj_imemo_cdhash);
 
+        break;
+      case imemo_str:
         break;
       default:
         rb_bug("unreachable");

include/ruby/internal/core/robject.h

@@ -72,8 +72,12 @@ enum ruby_robject_flags {
      *
      * 3rd parties must  not be aware that  there even is more than  one way to
      * store instance variables.  Might better be hidden.
+     *
+     * Must stay just above the imemo type field: imemo_fields aliases this bit
+     * as OBJ_FIELD_HEAP (STATIC_ASSERT OBJ_FIELD_HEAP == IMEMO_FL_USER0 in
+     * internal/imemo.h), so it must equal the first imemo user flag.
      */
-    ROBJECT_HEAP = RUBY_FL_USER4
+    ROBJECT_HEAP = RUBY_FL_USER5
 };
 
 struct st_table;