C: Implement Tracking Allocator and add --leak-check to analyze (#1304)

This pull request implements a new `hb_alloctor_T` implementation that
tracks allocations and checks if all allocations are also being freed
again.

We also expose a new `--leak-check` CLI flag for the `analyze`
subcommand so we can easily check a whole project if any of the files
leak.

**`herb analyze ../rubyevents --leak-check`**

<img width="1900" height="882" alt="CleanShot 2026-03-05 at 08 40 16@2x"
src="https://github.com/user-attachments/assets/3ed26685-dbf9-4661-8e55-2379a967779a"
/>


**With leaks:**

<img width="2844" height="1762" alt="CleanShot 2026-03-05 at 08 39
40@2x"
src="https://github.com/user-attachments/assets/6b95f474-2479-4f8a-be1b-514847fbc86a"
/>


**Calling `delloc` on a pointer that wasn't allocated using our
allocator:**

<img width="2812" height="1722" alt="CleanShot 2026-03-05 at 08 40
36@2x"
src="https://github.com/user-attachments/assets/2190fefc-023b-4398-9d24-8193d6381646"
/>

Author: marcoroth

ext/herb/extension.c

@@ -315,6 +315,98 @@ static VALUE Herb_arena_stats(int argc, VALUE* argv, VALUE self) {
   return hash;
 }
 
+static VALUE make_tracking_hash(hb_allocator_tracking_stats_T* stats) {
+  VALUE hash = rb_hash_new();
+  rb_hash_aset(hash, ID2SYM(rb_intern("allocations")), SIZET2NUM(stats->allocation_count));
+  rb_hash_aset(hash, ID2SYM(rb_intern("deallocations")), SIZET2NUM(stats->deallocation_count));
+  rb_hash_aset(hash, ID2SYM(rb_intern("bytes_allocated")), SIZET2NUM(stats->bytes_allocated));
+  rb_hash_aset(hash, ID2SYM(rb_intern("bytes_deallocated")), SIZET2NUM(stats->bytes_deallocated));
+  rb_hash_aset(hash, ID2SYM(rb_intern("untracked_deallocations")), SIZET2NUM(stats->untracked_deallocation_count));
+
+  VALUE leaks = rb_ary_new();
+  for (size_t i = 0; i < stats->buckets_capacity; i++) {
+    if (stats->buckets[i].pointer != NULL && stats->buckets[i].pointer != (void*) 1) {
+      rb_ary_push(leaks, SIZET2NUM(stats->buckets[i].size));
+    }
+  }
+  rb_hash_aset(hash, ID2SYM(rb_intern("leaks")), leaks);
+
+  VALUE untracked = rb_ary_new_capa((long) stats->untracked_pointers_size);
+  for (size_t i = 0; i < stats->untracked_pointers_size; i++) {
+    rb_ary_push(untracked, rb_sprintf("%p", stats->untracked_pointers[i]));
+  }
+  rb_hash_aset(hash, ID2SYM(rb_intern("untracked_pointers")), untracked);
+
+  return hash;
+}
+
+static VALUE Herb_leak_check(VALUE self, VALUE source) {
+  char* string = (char*) check_string(source);
+  VALUE result = rb_hash_new();
+
+  {
+    hb_allocator_T allocator;
+    if (!hb_allocator_init(&allocator, HB_ALLOCATOR_TRACKING)) { return Qnil; }
+
+    hb_array_T* tokens = herb_lex(string, &allocator);
+    if (tokens != NULL) { herb_free_tokens(&tokens, &allocator); }
+
+    hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+    rb_hash_aset(result, ID2SYM(rb_intern("lex")), make_tracking_hash(stats));
+
+    hb_allocator_destroy(&allocator);
+  }
+
+  {
+    hb_allocator_T allocator;
+    if (!hb_allocator_init(&allocator, HB_ALLOCATOR_TRACKING)) { return Qnil; }
+
+    parser_options_T parser_options = HERB_DEFAULT_PARSER_OPTIONS;
+    AST_DOCUMENT_NODE_T* root = herb_parse(string, &parser_options, &allocator);
+    if (root != NULL) { ast_node_free((AST_NODE_T*) root, &allocator); }
+
+    hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+    rb_hash_aset(result, ID2SYM(rb_intern("parse")), make_tracking_hash(stats));
+
+    hb_allocator_destroy(&allocator);
+  }
+
+  {
+    hb_buffer_T output;
+    if (!hb_buffer_init(&output, strlen(string))) { return Qnil; }
+
+    hb_allocator_T allocator;
+    if (!hb_allocator_init(&allocator, HB_ALLOCATOR_TRACKING)) { return Qnil; }
+
+    herb_extract_ruby_options_T extract_options = HERB_EXTRACT_RUBY_DEFAULT_OPTIONS;
+    herb_extract_ruby_to_buffer_with_options(string, &output, &extract_options, &allocator);
+
+    hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+    rb_hash_aset(result, ID2SYM(rb_intern("extract_ruby")), make_tracking_hash(stats));
+
+    hb_allocator_destroy(&allocator);
+    free(output.value);
+  }
+
+  {
+    hb_buffer_T output;
+    if (!hb_buffer_init(&output, strlen(string))) { return Qnil; }
+
+    hb_allocator_T allocator;
+    if (!hb_allocator_init(&allocator, HB_ALLOCATOR_TRACKING)) { return Qnil; }
+
+    herb_extract_html_to_buffer(string, &output, &allocator);
+
+    hb_allocator_tracking_stats_T* stats = hb_allocator_tracking_stats(&allocator);
+    rb_hash_aset(result, ID2SYM(rb_intern("extract_html")), make_tracking_hash(stats));
+
+    hb_allocator_destroy(&allocator);
+    free(output.value);
+  }
+
+  return result;
+}
+
 static VALUE Herb_version(VALUE self) {
   VALUE gem_version = rb_const_get(self, rb_intern("VERSION"));
   VALUE libherb_version = rb_utf8_str_new_cstr(herb_version());
@@ -345,5 +437,6 @@ __attribute__((__visibility__("default"))) void Init_herb(void) {
   rb_define_singleton_method(mHerb, "extract_ruby", Herb_extract_ruby, -1);
   rb_define_singleton_method(mHerb, "extract_html", Herb_extract_html, 1);
   rb_define_singleton_method(mHerb, "arena_stats", Herb_arena_stats, -1);
+  rb_define_singleton_method(mHerb, "leak_check", Herb_leak_check, 1);
   rb_define_singleton_method(mHerb, "version", Herb_version, 0);
 }

lib/herb/cli.rb

@@ -8,7 +8,7 @@
 class Herb::CLI
   include Herb::Colors
 
-  attr_accessor :json, :silent, :log_file, :no_timing, :local, :escape, :no_escape, :freeze, :debug, :tool, :strict, :analyze, :track_whitespace, :verbose, :isolate, :arena_stats
+  attr_accessor :json, :silent, :log_file, :no_timing, :local, :escape, :no_escape, :freeze, :debug, :tool, :strict, :analyze, :track_whitespace, :verbose, :isolate, :arena_stats, :leak_check
 
   def initialize(args)
     @args = args
@@ -150,6 +150,7 @@ def result
                   project.isolate = isolate
                   project.validate_ruby = true
                   project.arena_stats = arena_stats
+                  project.leak_check = leak_check
                   has_issues = project.analyze!
                   exit(has_issues ? 1 : 0)
                 when "report"
@@ -304,6 +305,10 @@ def option_parser
       parser.on("--arena-stats", "Print arena memory statistics (for lex/parse/analyze commands)") do
         self.arena_stats = true
       end
+
+      parser.on("--leak-check", "Check for memory leaks in lex/parse/extract operations (for analyze command)") do
+        self.leak_check = true
+      end
     end
   end
 

lib/herb/project.rb

@@ -12,7 +12,7 @@ module Herb
   class Project
     include Colors
 
-    attr_accessor :project_path, :output_file, :no_log_file, :no_timing, :silent, :verbose, :isolate, :validate_ruby, :file_paths, :arena_stats
+    attr_accessor :project_path, :output_file, :no_log_file, :no_timing, :silent, :verbose, :isolate, :validate_ruby, :file_paths, :arena_stats, :leak_check
 
     # Known error types that indicate issues in the user's template, not bugs in the parser.
     TEMPLATE_ERRORS = [
@@ -260,6 +260,10 @@ def analyze!
           print_arena_summary(file_results)
         end
 
+        if leak_check
+          print_leak_check_summary(file_results)
+        end
+
         unless no_log_file
           puts "\n #{separator}"
           puts "\n #{dimmed("Results saved to #{output_file}")}"
@@ -347,6 +351,10 @@ def process_file_direct(file_path)
         result[:arena_stats] = capture_arena_stats(file_content)
       end
 
+      if leak_check
+        result[:leak_check] = capture_leak_check(file_content)
+      end
+
       Timeout.timeout(1) do
         parse_result = Herb.parse(file_content)
 
@@ -887,6 +895,90 @@ def format_duration(seconds)
       end
     end
 
+    def capture_leak_check(file_content)
+      Herb.leak_check(file_content)
+    rescue StandardError
+      { lex: { allocations: 0, deallocations: 0, bytes_allocated: 0, bytes_deallocated: 0 },
+        parse: { allocations: 0, deallocations: 0, bytes_allocated: 0, bytes_deallocated: 0 },
+        extract_ruby: { allocations: 0, deallocations: 0, bytes_allocated: 0, bytes_deallocated: 0 },
+        extract_html: { allocations: 0, deallocations: 0, bytes_allocated: 0, bytes_deallocated: 0 } }
+    end
+
+    def print_leak_check_summary(file_results)
+      leaky_files = file_results.filter_map { |result|
+        next unless result[:leak_check]
+
+        ops = result[:leak_check]
+        leaks = ops.select { |_op, stats| stats[:leaks]&.any? || stats[:allocations] != stats[:deallocations] || stats[:untracked_deallocations]&.positive? }
+        next if leaks.empty?
+
+        { file: result[:file_path], leaks: leaks, all: ops }
+      }
+
+      puts "\n #{separator}"
+      puts "\n"
+      puts " #{bold("Leak check:")}"
+
+      if leaky_files.empty?
+        puts ""
+        puts "  #{bold(green("✓"))} #{green("No leaks detected across all files.")}"
+        return
+      end
+
+      puts "  #{red("#{leaky_files.size} #{pluralize(leaky_files.size, "file")} with potential leaks:")}"
+      puts ""
+
+      leaky_files.each do |entry|
+        relative = relative_path(entry[:file])
+        puts "  #{cyan(relative)}:"
+
+        entry[:all].each do |op, stats|
+          leaks = stats[:leaks] || []
+          untracked_count = stats[:untracked_deallocations] || 0
+          untracked_ptrs = stats[:untracked_pointers] || []
+          leaked_bytes = stats[:bytes_allocated] - stats[:bytes_deallocated]
+
+          if leaks.any?
+            puts "    #{red("✗")} #{op}: #{stats[:allocations]} allocs, #{stats[:deallocations]} deallocs (#{bold(red("#{leaks.size} unfreed, #{format_bytes(leaked_bytes)}"))})"
+            leaks.each_with_index do |size, i|
+              puts "      #{dimmed("#{i + 1}.")} #{format_bytes(size)}"
+            end
+          elsif untracked_count.positive?
+            puts "    #{yellow("~")} #{op}: #{stats[:allocations]} allocs, #{stats[:deallocations]} deallocs"
+          else
+            puts "    #{green("✓")} #{op}: #{stats[:allocations]} allocs, #{stats[:deallocations]} deallocs"
+          end
+
+          next unless untracked_count.positive?
+
+          puts "      #{yellow("#{untracked_count} untracked #{pluralize(untracked_count, "deallocation")}")} #{dimmed("(freed through allocator but not allocated through it)")}"
+          untracked_ptrs.each_with_index do |ptr, i|
+            puts "      #{dimmed("#{i + 1}.")} #{ptr}"
+          end
+        end
+
+        puts ""
+      end
+
+      op_to_command = { lex: "lex", parse: "parse", extract_ruby: "ruby", extract_html: "html" }
+
+      commands = leaky_files.flat_map { |entry|
+        entry[:leaks].keys.map { |op| { command: op_to_command[op] || op.to_s, file: entry[:file] } }
+      }
+
+      puts "  #{dimmed("To debug, run the following from the herb repo root (build with `make` first):")}"
+      puts ""
+      puts "  #{dimmed("# macOS")}"
+      commands.each do |cmd|
+        puts "  leaks --atExit -- ./herb #{cmd[:command]} #{cmd[:file]}"
+      end
+      puts ""
+      puts "  #{dimmed("# Linux")}"
+      commands.each do |cmd|
+        puts "  valgrind --leak-check=full ./herb #{cmd[:command]} #{cmd[:file]}"
+      end
+    end
+
     def capture_arena_stats(file_content)
       stats = Herb.arena_stats(file_content)
 

src/include/util/hb_allocator.h

@@ -13,8 +13,28 @@
 typedef enum {
   HB_ALLOCATOR_MALLOC,
   HB_ALLOCATOR_ARENA,
+  HB_ALLOCATOR_TRACKING,
 } hb_allocator_type_T;
 
+typedef struct {
+  void* pointer;
+  size_t size;
+} hb_allocator_tracking_entry_T;
+
+typedef struct {
+  size_t allocation_count;
+  size_t deallocation_count;
+  size_t untracked_deallocation_count;
+  size_t bytes_allocated;
+  size_t bytes_deallocated;
+  hb_allocator_tracking_entry_T* buckets;
+  size_t buckets_capacity;
+  size_t buckets_used;
+  void** untracked_pointers;
+  size_t untracked_pointers_size;
+  size_t untracked_pointers_capacity;
+} hb_allocator_tracking_stats_T;
+
 typedef struct hb_allocator {
   void* (*alloc)(struct hb_allocator* self, size_t size);
   void (*dealloc)(struct hb_allocator* self, void* pointer);
@@ -30,6 +50,9 @@ void hb_allocator_destroy(hb_allocator_T* allocator);
 
 hb_allocator_T hb_allocator_with_malloc(void);
 hb_allocator_T hb_allocator_with_arena(hb_arena_T* arena);
+hb_allocator_T hb_allocator_with_tracking(void);
+
+hb_allocator_tracking_stats_T* hb_allocator_tracking_stats(hb_allocator_T* allocator);
 
 static inline void* hb_allocator_alloc(hb_allocator_T* allocator, size_t size) {
   return allocator->alloc(allocator, size);