feat: Store log data in a separate table

.github/workflows/rspec.yml

@@ -32,6 +32,12 @@ jobs:
           gemfile: rails8.gemfile
           fx: "false"
           after_trigger: "false"
+        - ruby: "3.4"
+          postgres: "16"
+          gemfile: rails8.gemfile
+          fx: "true"
+          after_trigger: "false"
+          detached: "true"
         - ruby: "3.3"
           postgres: "16"
           gemfile: rails8.gemfile
@@ -109,6 +115,7 @@ jobs:
         USE_FX: ${{ matrix.fx }}
         TABLE_NAME_PREFIX: ${{ matrix.table_name_prefix }}
         TABLE_NAME_SUFFIX: ${{ matrix.table_name_suffix }}
+        LOGIDZE_DETACHED: ${{ matrix.detached }}
       run: |
         bundle exec rspec --exclude-pattern=spec/acceptance/**/* -f d --force-color
     - name: Run RSpec (acceptance)

.gitignore

@@ -36,6 +36,7 @@ spec/dummy/db/functions/
 spec/dummy/db/triggers/
 spec/dummy/db/migrate/*_enable_hstore.rb
 spec/dummy/db/migrate/*_logidze_install.rb
+spec/dummy/db/migrate/*_logidze_data.rb
 
 Gemfile.lock
 Gemfile.local

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## master (unreleased)
 
 - Support Rails 7.2 ([@atomaka][])
+- Add `--detached` option to store `log_data` in a separate `logidze_data` table to avoid table bloat.
 
 ## 1.3.1 (2024-10-23)
 

README.md

@@ -37,6 +37,7 @@ Other requirements:
   - [Logs timestamps](#logs-timestamps)
   - [Undoing a Generated Invocation](#undoing-a-generated-invocation)
   - [Using with partitioned tables](#using-with-partitioned-tables)
+  - [Storing history data in a separate table](#storing-history-data-in-a-separate-table)
 - [Usage](#usage)
   - [Basic API](#basic-api)
   - [Track meta information](#track-meta-information)
@@ -203,6 +204,17 @@ bundle exec rails generate logidze:model Post --after-trigger
 
 **IMPORTANT:** Using Logidze for partitioned tables in PostgreSQL 10 is not supported.
 
+### Storing history data in a separate table
+
+By default, Logidze stores history data in the `log_data` column in the origin record table, which might lead to table bloat.
+If it concerns you, you may configure Logidze to store history data in a separate table by providing `--detached` option to the migration:
+
+```sh
+bundle exec rails logidze:model Post --detached
+```
+
+**IMPORTANT:** Using `--detached` mode for storing historic data slightly decreases performance. Check [bench results] for the details.
+
 ## Usage
 
 ### Basic API
@@ -645,3 +657,4 @@ Bug reports and pull requests are welcome on GitHub at [https://github.com/palka
 The gem is available as open source under the terms of the [MIT License](http://opensource.org/licenses/MIT).
 
 [fx]: https://github.com/teoljungberg/fx
+[bench results]: https://github.com/palkan/logidze/blob/feat-log-data-separate-storage/bench/performance/README.md

app/models/logidze/logidze_data.rb

@@ -0,0 +1,9 @@
+# frozen_string_literal: true
+
+module Logidze
+  class LogidzeData < ::ApplicationRecord
+    attribute :log_data, Logidze::History::Type.new
+
+    belongs_to :loggable, polymorphic: true
+  end
+end

bench/performance/Gemfile

@@ -2,10 +2,10 @@
 
 source "https://rubygems.org"
 
-gem "rails", "~> 6.0"
+gem "rails", "~> 7.0"
 gem "pg", "~> 1.0"
 
-gem "paper_trail", "~> 10.0"
+gem "paper_trail", "~> 15.0"
 gem "logidze", path: "../../.."
 gem "fx"
 

bench/performance/README.md

@@ -19,9 +19,10 @@ bundle exec ruby benchmarks/insert_bench.rb
 
 ```sh
 Comparison:
-        Plain INSERT:      535.5 i/s
-      Logidze INSERT:      498.2 i/s - same-ish: difference falls within error
-   PaperTrail INSERT:      276.5 i/s - 1.94x  (± 0.00) slower
+          Plain INSERT:      450.9 i/s
+        Logidze INSERT:      408.2 i/s - same-ish: difference falls within error
+LogidzeDetached INSERT:      359.8 i/s - same-ish: difference falls within error
+     PaperTrail INSERT:      183.9 i/s - same-ish: difference falls within error
 ```
 
 ## Update ([source](benchmarks/update_bench.rb))
@@ -30,18 +31,20 @@ When changeset has 2 fields:
 
 ```sh
 Comparison:
-     Plain UPDATE #1:      775.2 i/s
-   Logidze UPDATE #1:      518.6 i/s - 1.49x  (± 0.00) slower
-        PT UPDATE #1:      471.4 i/s - 1.64x  (± 0.00) slower
+          Plain UPDATE #1:      204.8 i/s
+LogidzeDetached UPDATE #1:      188.4 i/s - same-ish: difference falls within error
+        Logidze UPDATE #1:      175.7 i/s - same-ish: difference falls within error
+             PT UPDATE #1:      114.2 i/s - same-ish: difference falls within error
 ```
 
 When changeset has 5 fields:
 
 ```sh
 Comparison:
-     Plain UPDATE #2:      726.8 i/s
-   Logidze UPDATE #2:      468.0 i/s - 1.55x  (± 0.00) slower
-        PT UPDATE #2:      431.2 i/s - 1.69x  (± 0.00) slower
+           Plain UPDATE #2:      117.8 i/s
+Logidze Detached UPDATE #2:      116.1 i/s - same-ish: difference falls within error
+         Logidze UPDATE #2:      115.4 i/s - same-ish: difference falls within error
+              PT UPDATE #2:       57.6 i/s - 2.04x  slower
 ```
 
 ## Getting diff ([source](benchmarks/diff_bench.rb))
@@ -53,18 +56,21 @@ When each record has 10 versions:
 
 ```sh
 Comparison:
-        Logidze DIFF:      129.2 i/s
-      PT (join) DIFF:        4.5 i/s - 28.57x  (± 0.00) slower
-             PT DIFF:        3.9 i/s - 32.83x  (± 0.00) slower
+        Logidze DIFF:      189.2 i/s
+LogidzeDetached DIFF:      124.0 i/s - 1.52x  slower
+      PT (join) DIFF:        4.4 i/s - 42.82x  slower
+             PT DIFF:        3.1 i/s - 60.69x  slower
 ```
 
 When each record has 100 versions:
 
 ```sh
+
 Comparison:
-        Logidze DIFF:       32.8 i/s
-      PT (join) DIFF:        0.5 i/s - 69.48x  (± 0.00) slower
-             PT DIFF:        0.4 i/s - 88.81x  (± 0.00) slower
+        Logidze DIFF:       78.5 i/s
+LogidzeDetached DIFF:       69.8 i/s - 1.13x  slower
+             PT DIFF:        0.4 i/s - 200.20x  slower
+      PT (join) DIFF:        0.3 i/s - 227.26x  slower
 ```
 
 ## Getting version at the specified time ([source](benchmarks/version_at_bench.rb))
@@ -75,24 +81,28 @@ When each record has 10 versions:
 
 ```sh
 Comparison:
-   Logidze AT single:      882.1 i/s
-        PT AT single:      336.7 i/s - 2.62x  (± 0.00) slower
+        Logidze AT single:      468.3 i/s
+LogidzeDetached AT single:      262.3 i/s - same-ish: difference falls within error
+             PT AT single:      169.6 i/s - 2.76x  slower
 
 Comparison:
-     Logidze AT many:      265.0 i/s
-          PT AT many:       43.9 i/s - 6.04x  (± 0.00) slower
+        Logidze AT many:      283.7 i/s
+LogidzeDetached AT many:      199.3 i/s - 1.42x  slower
+             PT AT many:       27.0 i/s - 10.52x  slower
 ```
 
 When each record has 100 versions:
 
 ```sh
 Comparison:
-   Logidze AT single:      543.0 i/s
-        PT AT single:      337.8 i/s - 1.61x  (± 0.00) slower
+        Logidze AT single:      339.2 i/s
+LogidzeDetached AT single:      219.6 i/s - same-ish: difference falls within error
+             PT AT single:      150.3 i/s - 2.26x  slower
 
 Comparison:
-     Logidze AT many:       92.0 i/s
-          PT AT many:       45.3 i/s - 2.03x  (± 0.00) slower
+        Logidze AT many:      167.9 i/s
+LogidzeDetached AT many:      130.2 i/s - 1.29x  slower
+             PT AT many:       24.2 i/s - 6.94x  slower
 ```
 
 **NOTE:** PaperTrail has N+1 problem when loading multiple records at the specified time (due to the usage of the `versions.subsequent` method).
@@ -105,36 +115,46 @@ When each record has 10 versions:
 
 ```sh
 Plain records
-Total Allocated:                                15.24 KB
-Total Retained:                                 9.68 KB
-Retained_memsize memory (per record):           1.12 KB
+Total Allocated:				44.38 KB
+Total Retained:					22.52 KB
+Retained_memsize memory (per record):		2.35 KB
 
 PT with versions
-Total Allocated:                                124.38 KB
-Total Retained:                                 95.52 KB
-Retained_memsize memory (per record):           81.82 KB
+Total Allocated:				387.58 KB
+Total Retained:					223.77 KB
+Retained_memsize memory (per record):		203.13 KB
 
 Logidze records
-Total Allocated:                                38.68 KB
-Total Retained:                                 32.12 KB
-Retained_memsize memory (per record):           3.54 KB
+Total Allocated:				76.37 KB
+Total Retained:					38.46 KB
+Retained_memsize memory (per record):		3.72 KB
+
+LogidzeDetached records
+Total Allocated:				53.84 KB
+Total Retained:					24.1 KB
+Retained_memsize memory (per record):		-1768 B
 ```
 
 When each record has 100 versions:
 
 ```sh
 Plain records
-Total Allocated:                                15.22 KB
-Total Retained:                                 9.66 KB
-Retained_memsize memory (per record):           1.11 KB
+Total Allocated:				44.3 KB
+Total Retained:					22.48 KB
+Retained_memsize memory (per record):		2.23 KB
 
 PT with versions
-Total Allocated:                                998.71 KB
-Total Retained:                                 864.26 KB
-Retained_memsize memory (per record):           850.6 KB
+Total Allocated:				3.32 MB
+Total Retained:					1.99 MB
+Retained_memsize memory (per record):		1.97 MB
 
 Logidze records
-Total Allocated:                                140.16 KB
-Total Retained:                                 133.6 KB
-Retained_memsize memory (per record):           13.91 KB
+Total Allocated:				281.22 KB
+Total Retained:					140.89 KB
+Retained_memsize memory (per record):		14.16 KB
+
+LogidzeDetached records
+Total Allocated:				53.29 KB
+Total Retained:					23.83 KB
+Retained_memsize memory (per record):		-1608 B
 ```

bench/performance/app/lib/benchmarker.rb

@@ -13,6 +13,7 @@ def populate(n = 1_000, skip_user: false)
       User.create!(params) unless skip_user
       PaperTrailUser.create!(params)
       LogidzeUser.create!(params)
+      LogidzeDetachedUser.create!(params)
     end
 
     $stdout.puts "Done in #{Time.current - ts}s"
@@ -23,6 +24,7 @@ def cleanup
     PaperTrailUser.delete_all
     User.delete_all
     PaperTrail::Version.delete_all
+    LogidzeDetachedUser.delete_all
   end
 
   def generate_versions(num = 1)
@@ -39,6 +41,10 @@ def generate_versions(num = 1)
         u.update!(fake_params(sample: true))
       end
 
+      LogidzeDetachedUser.find_each do |u|
+        u.update!(fake_params(sample: true))
+      end
+
       # make at least 1 second between versions
       sleep 1
     end

bench/performance/app/models/logidze_detached_user.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+class LogidzeDetachedUser < ApplicationRecord
+  has_logidze detached: true
+end

bench/performance/benchmarks/diff_bench.rb

@@ -36,5 +36,9 @@
     LogidzeUser.random(N / 2).diff_from(time: ts1)
   end
 
+  x.report("LogidzeDetached DIFF") do
+    LogidzeDetachedUser.random(N / 2).diff_from(time: ts1)
+  end
+
   x.compare!
 end

bench/performance/benchmarks/insert_bench.rb

@@ -24,5 +24,9 @@
     LogidzeUser.create!(params)
   end
 
+  x.report("LogidzeDetached INSERT") do
+    LogidzeDetachedUser.create!(params)
+  end
+
   x.compare!
 end

bench/performance/benchmarks/memory_profile.rb

@@ -20,7 +20,9 @@ module MemoryReport
 
   module_function
 
-  def call(msg, relation)
+  def call(msg, relation, gc_disable: true)
+    GC.disable if gc_disable
+
     buffer = nil
     delta = N / 10
     r0 = MemoryProfiler.report do
@@ -54,3 +56,4 @@ def to_human_size(size)
 MemoryReport.call("Plain records", User.all)
 MemoryReport.call("PT with versions", PaperTrailUser.joins(:versions).all)
 MemoryReport.call("Logidze records", LogidzeUser.all)
+MemoryReport.call("LogidzeDetached records", LogidzeDetachedUser.joins(:logidze_data).all)

bench/performance/benchmarks/update_bench.rb

@@ -36,6 +36,10 @@ class LogidzeUser
     LogidzeUser.random.update!(params)
   end
 
+  x.report("LogidzeDetached UPDATE #1") do
+    LogidzeDetachedUser.random.update!(params)
+  end
+
   x.compare!
 end
 
@@ -60,5 +64,11 @@ class LogidzeUser
     user.update!(params2.slice(*JSON_COLUMNS))
   end
 
+  x.report("Logidze Detached UPDATE #2") do
+    user = LogidzeDetachedUser.random
+    user.update!(params2.except(*JSON_COLUMNS))
+    user.update!(params2.slice(*JSON_COLUMNS))
+  end
+
   x.compare!
 end

bench/performance/benchmarks/version_at_bench.rb

@@ -34,6 +34,10 @@
     LogidzeUser.random.at(time: ts1)
   end
 
+  x.report("LogidzeDetached AT single") do
+    LogidzeDetachedUser.random.at(time: ts1)
+  end
+
   x.compare!
 end
 
@@ -48,5 +52,9 @@
     LogidzeUser.random(M).at(time: ts1)
   end
 
+  x.report("LogidzeDetached AT many") do
+    LogidzeDetachedUser.random(M).at(time: ts1)
+  end
+
   x.compare!
 end

bench/performance/config/application.rb

@@ -14,5 +14,10 @@ class Application < Rails::Application
 
     config.logger = ENV["LOG"] ? Logger.new($stdout) : Logger.new(IO::NULL)
     config.active_record.dump_schema_after_migration = false
+    config.active_record.yaml_column_permitted_classes = [
+      ActiveSupport::TimeWithZone,
+      ActiveSupport::TimeZone,
+      Time
+    ]
   end
 end