Enable bulk execution for SeedDo.seed

Add a `bulk: true` option to the `SeedDo.seed` method. Normally, `SeedDo.seed` runs queries for each model, but `SeedDo.seed(bulk: true)` combines multiple queries into one. This should give a large performance improvement when a project defines a lot of seed data.

Allow the number of records in each query to be set with the `batch_size` option. The default is 1000.

```ruby
SeedDo.seed(bulk: { batch_size: 100 })
```

## Requirements

Use Rails [upsert_all](https://api.rubyonrails.org/classes/ActiveRecord/Relation.html#method-i-upsert_all) internally to implement this feature. To keep the behavior close to the existing seed-do behavior, use the `:unique_by` option, so the related constraints must have unique indexes. Also, `:unique_by` is supported only by PostgreSQL and SQLite, so this feature is not available on MySQL.

## Notes

`SeedDo.seed` and `SeedDo.seed(bulk: true)` do not always produce the same results.

For example, with the following seed definition:

```ruby
User.seed(:name) do |u|
  u.name = 'first'
end

User.create!(name: 'second')

User.seed(:name) do |u|
  u.name = 'third'
end
```

With `SeedDo.seed`, records are created in the order `first`, `second`, `third`. However, with `SeedDo.seed(bulk: true)`, the order becomes `second`, `first`, `third`.

Because of this, verify the behavior carefully before using `SeedDo.seed(bulk: true)` in an existing production environment.

Author: willnet

lib/seed-do.rb

@@ -17,12 +17,20 @@ module SeedDo
   # plugin, SeedDo will set it to contain `Rails.root/db/fixtures` and
   # `Rails.root/db/fixtures/Rails.env`
   mattr_accessor :fixture_paths, default: ['db/fixtures']
-
   # Load seed data from files
   # @param [Array] fixture_paths The paths to look for seed files in
   # @param [Regexp] filter If given, only filenames matching this expression will be loaded
-  def self.seed(fixture_paths = SeedDo.fixture_paths, filter = nil)
-    Runner.new(fixture_paths, filter).run
+  # @param [Boolean] bulk If true, bulk insert/upsert will be used
+  def self.seed(fixture_paths = SeedDo.fixture_paths, filter = nil, bulk: false)
+    Runner.new(fixture_paths, filter, bulk: bulk).run
+  end
+
+  def self.current_runner
+    @current_runner
+  end
+
+  def self.current_runner=(runner)
+    @current_runner = runner
   end
 end
 

lib/seed-do/active_record_extension.rb

@@ -29,7 +29,13 @@ module ActiveRecordExtension
     #       { :x => 5, :y => 9,  :name => "Office" }
     #     )
     def seed(*args, &block)
-      SeedDo::Seeder.new(self, *parse_seed_do_args(args, block)).seed
+      runner = SeedDo.current_runner
+      if runner&.bulk && runner.buffer
+        constraints, data = parse_seed_do_args(args, block)
+        runner.buffer[:seed] << { model: self, constraints: constraints, data: data }
+      else
+        SeedDo::Seeder.new(self, *parse_seed_do_args(args, block)).seed
+      end
     end
 
     # Has the same syntax as {#seed}, but if a record already exists with the same values for
@@ -41,7 +47,12 @@ def seed(*args, &block)
     #   Person.seed_once(:id, :id => 1, :name => "Harry") # => Name *not* changed
     def seed_once(*args, &block)
       constraints, data = parse_seed_do_args(args, block)
-      SeedDo::Seeder.new(self, constraints, data, insert_only: true).seed
+      runner = SeedDo.current_runner
+      if runner&.bulk && runner.buffer
+        runner.buffer[:seed_once] << { model: self, constraints: constraints, data: data }
+      else
+        SeedDo::Seeder.new(self, constraints, data, insert_only: true).seed
+      end
     end
 
     private

lib/seed-do/runner.rb

@@ -3,28 +3,51 @@
 
 module SeedDo
   # Runs seed files.
-  #
-  # It is not recommended to use this class directly. Instead, use {SeedDo.seed SeedDo.seed}, which creates
-  # an instead of {Runner} and calls {#run #run}.
+  # It is not recommended to use this class directly. Instead, use {SeedDo.seed SeedDo.seed}, which creates an instead of {Runner} and calls {#run #run}.
   #
   # @see SeedDo.seed SeedDo.seed
   class Runner
+    attr_accessor :buffer
+    attr_reader :bulk
+
     # @param [Array<String>] fixture_paths The paths where fixtures are located. Will use
     #   `SeedDo.fixture_paths` if {nil}. If the argument is not an array, it will be wrapped by one.
     # @param [Regexp] filter If given, only seed files with a file name matching this pattern will
     #   be used
-    def initialize(fixture_paths = nil, filter = nil)
+    # @param [Boolean, Hash] bulk If true, use upsert_all to insert/update records in bulk.
+    #   If a hash, can include :batch_size (default: 1000) to control the number of records
+    #   per upsert_all call.
+    def initialize(fixture_paths = nil, filter = nil, bulk: false)
       @fixture_paths = Array.wrap(fixture_paths || SeedDo.fixture_paths)
       @filter        = filter
+      if bulk.is_a?(Hash)
+        @bulk = true
+        @batch_size = bulk.fetch(:batch_size, 1000)
+      else
+        @bulk = !bulk.nil? && bulk != false
+        @batch_size = 1000
+      end
+
+      validate_bulk_support! if @bulk
+    end
+
+    def validate_bulk_support!
+      return if ActiveRecord::Base.connection.supports_insert_conflict_target?
+
+      raise ArgumentError,
+            "Bulk mode is not supported for #{ActiveRecord::Base.connection.adapter_name}. " \
+            'The database does not support upsert operations with conflict targets. ' \
+            'Please use SeedDo.seed without the bulk option.'
     end
 
     # Run the seed files.
     def run
+      SeedDo.current_runner = self
       puts "\n== Filtering seed files against regexp: #{@filter.inspect}" if @filter && !SeedDo.quiet
 
-      filenames.each do |filename|
-        run_file(filename)
-      end
+      filenames.each { |filename| run_file(filename) }
+    ensure
+      SeedDo.current_runner = nil
     end
 
     private
@@ -33,17 +56,61 @@ def run_file(filename)
       puts "\n== Seed from #{filename}" unless SeedDo.quiet
 
       ActiveRecord::Base.transaction do
-        open(filename) do |file|
-          chunked_ruby = +''
-          file.each_line do |line|
-            if line == "# BREAK EVAL\n"
-              eval(chunked_ruby)
-              chunked_ruby = +''
-            else
-              chunked_ruby << line
-            end
+        if bulk
+          with_buffer { _run_file(filename) }
+        else
+          _run_file(filename)
+        end
+      end
+    end
+
+    def with_buffer
+      self.buffer = { seed: [], seed_once: [] }
+      yield
+      process_buffer
+    ensure
+      self.buffer = nil
+    end
+
+    def _run_file(filename)
+      open(filename) do |file|
+        chunked_ruby = +''
+        file.each_line do |line|
+          if line == "# BREAK EVAL\n"
+            eval(chunked_ruby)
+            chunked_ruby = +''
+          else
+            chunked_ruby << line
           end
-          eval(chunked_ruby) unless chunked_ruby == ''
+        end
+        eval(chunked_ruby) unless chunked_ruby == ''
+      end
+    end
+
+    def process_buffer
+      return unless buffer
+
+      buffer.each do |type, operations|
+        next if operations.empty?
+
+        operations.chunk { |op| [op[:model], op[:constraints]] }
+                  .each do |(model, constraints), chunk|
+                    flush_chunk(model, constraints, type, chunk)
+        end
+      end
+    end
+
+    def flush_chunk(model, constraints, type, chunk)
+      all_data = chunk.flat_map { |op| op[:data] }
+
+      options = {}
+      options[:unique_by] = constraints if model.connection.supports_insert_conflict_target?
+      all_data.each_slice(@batch_size) do |batch|
+        if type == :seed
+          model.upsert_all(batch, **options)
+        elsif type == :seed_once
+          options[:on_duplicate] = :skip
+          model.upsert_all(batch, **options)
         end
       end
     end

spec/bulk_spec.rb

@@ -0,0 +1,97 @@
+require 'spec_helper'
+
+describe 'Bulk Insertion' do
+  before(:all) do
+    skip 'Bulk mode is not supported on databases without insert conflict target support' unless ActiveRecord::Base.connection.supports_insert_conflict_target?
+  end
+  it 'uses upsert_all when bulk option is true' do
+    expect(BulkSeededModel).to receive(:upsert_all).with(
+      contain_exactly(
+        hash_including('title' => 'Bulk 1', 'login' => 'bulk1'),
+        hash_including('title' => 'Bulk 2', 'login' => 'bulk2')
+      ),
+      hash_including(unique_by: [:title])
+    ).and_call_original
+
+    SeedDo.seed("#{File.dirname(__FILE__)}/fixtures", /bulk_insert/, bulk: true)
+
+    expect(BulkSeededModel.count).to eq(2)
+    item1 = BulkSeededModel.find_by(title: 'Bulk 1')
+    expect(item1.login).to eq 'bulk1'
+    item2 = BulkSeededModel.find_by(title: 'Bulk 2')
+    expect(item2.login).to eq 'bulk2'
+  end
+
+  it 'uses upsert_all with skip option for seed_once in bulk mode' do
+    # Pre-create record
+    BulkSeededModel.create!(title: 'Existing', login: 'original')
+
+    expect(BulkSeededModel).to receive(:upsert_all).with(
+      contain_exactly(
+        hash_including('title' => 'Existing', 'login' => 'new'),
+        hash_including('title' => 'New', 'login' => 'created')
+      ),
+      hash_including(unique_by: [:title], on_duplicate: :skip)
+    ).and_call_original
+
+    SeedDo.seed("#{File.dirname(__FILE__)}/fixtures", /bulk_seed_once/, bulk: true)
+
+    expect(BulkSeededModel.count).to eq(2)
+    existing = BulkSeededModel.find_by(title: 'Existing')
+    expect(existing.login).to eq 'original' # Should NOT change
+    new_rec = BulkSeededModel.find_by(title: 'New')
+    expect(new_rec.login).to eq 'created'
+  end
+
+  it 'respects batch_size option when specified' do
+    # With batch_size: 10, 25 records should result in 3 upsert_all calls (10, 10, 5)
+    call_count = 0
+    allow(BulkSeededModel).to receive(:upsert_all).and_wrap_original do |method, data, **kwargs|
+      call_count += 1
+      method.call(data, **kwargs)
+    end
+
+    SeedDo.seed("#{File.dirname(__FILE__)}/fixtures", /bulk_large/, bulk: { batch_size: 10 })
+
+    expect(call_count).to eq(3)
+    expect(BulkSeededModel.count).to eq(25)
+
+    # Verify all records were created
+    (1..25).each do |i|
+      record = BulkSeededModel.find_by(title: "Bulk #{i}")
+      expect(record).to be_present
+      expect(record.login).to eq "bulk#{i}"
+    end
+  end
+
+  it 'uses default batch_size of 1000 when bulk: true' do
+    call_count = 0
+    allow(BulkSeededModel).to receive(:upsert_all).and_wrap_original do |method, data, **kwargs|
+      call_count += 1
+      method.call(data, **kwargs)
+    end
+
+    SeedDo.seed("#{File.dirname(__FILE__)}/fixtures", /bulk_insert/, bulk: true)
+
+    # With default batch_size of 1000, 2 records should result in 1 upsert_all call
+    expect(call_count).to eq(1)
+    expect(BulkSeededModel.count).to eq(2)
+  end
+
+  it 'splits large dataset across multiple batches with custom batch_size' do
+    call_count = 0
+    batch_sizes = []
+    allow(BulkSeededModel).to receive(:upsert_all).and_wrap_original do |method, data, **options|
+      call_count += 1
+      batch_sizes << data.size
+      method.call(data, **options)
+    end
+
+    SeedDo.seed("#{File.dirname(__FILE__)}/fixtures", /bulk_large/, bulk: { batch_size: 7 })
+
+    # With batch_size: 7, 25 records should result in 4 calls: [7, 7, 7, 4]
+    expect(call_count).to eq(4)
+    expect(batch_sizes).to eq([7, 7, 7, 4])
+    expect(BulkSeededModel.count).to eq(25)
+  end
+end

spec/fixtures/bulk_insert.rb

@@ -0,0 +1,9 @@
+BulkSeededModel.seed(:title) do |s|
+  s.title = 'Bulk 1'
+  s.login = 'bulk1'
+end
+
+BulkSeededModel.seed(:title) do |s|
+  s.title = 'Bulk 2'
+  s.login = 'bulk2'
+end

spec/fixtures/bulk_large.rb

@@ -0,0 +1,7 @@
+# Create 25 records for batch_size testing
+(1..25).each do |i|
+  BulkSeededModel.seed(:title) do |s|
+    s.title = "Bulk #{i}"
+    s.login = "bulk#{i}"
+  end
+end

spec/fixtures/bulk_seed_once.rb

@@ -0,0 +1,9 @@
+BulkSeededModel.seed_once(:title) do |s|
+  s.title = 'Existing'
+  s.login = 'new'
+end
+
+BulkSeededModel.seed_once(:title) do |s|
+  s.title = 'New'
+  s.login = 'created'
+end

spec/runner_spec.rb

@@ -21,4 +21,34 @@
     SeedDo.seed
     expect(SeededModel.count).to eq 3
   end
+
+  describe 'bulk mode validation' do
+    context 'when database does not support insert conflict target' do
+      it 'raises ArgumentError when bulk option is true' do
+        skip 'Test only runs on databases without insert conflict target support' if ActiveRecord::Base.connection.supports_insert_conflict_target?
+
+        expect do
+          SeedDo::Runner.new(File.dirname(__FILE__) + '/fixtures', nil, bulk: true)
+        end.to raise_error(ArgumentError, /Bulk mode is not supported/)
+      end
+
+      it 'raises ArgumentError when bulk option is a hash' do
+        skip 'Test only runs on databases without insert conflict target support' if ActiveRecord::Base.connection.supports_insert_conflict_target?
+
+        expect do
+          SeedDo::Runner.new(File.dirname(__FILE__) + '/fixtures', nil, bulk: { batch_size: 500 })
+        end.to raise_error(ArgumentError, /Bulk mode is not supported/)
+      end
+    end
+
+    context 'when database supports insert conflict target' do
+      it 'does not raise error when bulk option is true' do
+        skip 'Test only runs on databases with insert conflict target support' unless ActiveRecord::Base.connection.supports_insert_conflict_target?
+
+        expect do
+          SeedDo::Runner.new(File.dirname(__FILE__) + '/fixtures', nil, bulk: true)
+        end.not_to raise_error
+      end
+    end
+  end
 end

spec/spec_helper.rb

@@ -20,6 +20,14 @@
     t.column :title, :string
   end
 
+  create_table :bulk_seeded_models, force: true do |t|
+    t.column :login, :string
+    t.column :first_name, :string
+    t.column :last_name, :string
+    t.column :title, :string
+    t.index :title, unique: true
+  end
+
   create_table :seeded_model_no_primary_keys, id: false, force: true do |t|
     t.column :id, :string
   end
@@ -38,14 +46,21 @@ class SeededModel < ActiveRecord::Base
   before_save { throw(:abort) if fail_to_save }
 end
 
-class SeededModelNoPrimaryKey < ActiveRecord::Base # rubocop:disable Style/OneClassPerFile
+class SeededModelNoPrimaryKey < ActiveRecord::Base
+end
+
+class BulkSeededModel < ActiveRecord::Base
+end
+
+class SeededModelNoPrimaryKey < ActiveRecord::Base
 end
 
-class SeededModelNoSequence < ActiveRecord::Base # rubocop:disable Style/OneClassPerFile
+class SeededModelNoSequence < ActiveRecord::Base
 end
 
 RSpec.configure do |config|
   config.before do
     SeededModel.delete_all
+    BulkSeededModel.delete_all
   end
 end