Fix BC-10008678752: tame self-hosted import disk demands

A self-hosted import paid ~4x the export size in peak disk: multipart
buffer + attach copy + a blob.open tempfile copy per read pass + the
imported attachment blobs. ENOSPC mid-import crashes the shared
puma/Solid Queue container, and SQLite reports it as 'database or disk
is full' — misdirecting operators at the database.

- ZipFile.read_from_disk now opens the stored file in place via
  service.path_for (zip reading only needs random access), eliminating
  a full-size tempfile copy in each of check and process
- Account::Import#check preflights free disk (df -Pk on the service
  root) and fails fast with a clear insufficient_disk reason instead of
  crashing the container mid-import

Card: https://app.basecamp.com/2914079/buckets/27/card_tables/cards/10008678752

Claude-Session: https://claude.ai/code/session_01CSpm1M4ZQwmurqqNb5uQx8

Author: jeremy

app/jobs/account/data_import_job.rb

@@ -2,7 +2,8 @@ class Account::DataImportJob < ApplicationJob
   include ActiveJob::Continuable
 
   queue_as :backend
-  discard_on Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError
+  discard_on Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError,
+    Account::Import::InsufficientDiskSpaceError
 
   def perform(import)
     step :check do |step|

app/models/account/import.rb

@@ -1,4 +1,12 @@
+require "shellwords"
+
 class Account::Import < ApplicationRecord
+  class InsufficientDiskSpaceError < StandardError; end
+
+  # Peak disk demand beyond the stored zip itself: the imported attachment
+  # blobs roughly mirror the zip's contents, plus database growth and margin.
+  REQUIRED_DISK_FACTOR = 2
+
   broadcasts_refreshes
 
   belongs_to :account
@@ -7,7 +15,7 @@ class Account::Import < ApplicationRecord
   has_one_attached :file, dependent: :purge_later
 
   enum :status, %w[ pending processing completed failed ].index_by(&:itself), default: :pending
-  enum :failure_reason, %w[ conflict invalid_export ].index_by(&:itself), prefix: :failed_due_to, scopes: false
+  enum :failure_reason, %w[ conflict invalid_export insufficient_disk ].index_by(&:itself), prefix: :failed_due_to, scopes: false
 
   scope :expired, -> { where(completed_at: ...24.hours.ago).or(where(status: :failed, created_at: ...7.days.ago)) }
 
@@ -21,6 +29,7 @@ def process_later
 
   def check(start: nil, callback: nil)
     processing!
+    ensure_sufficient_disk_space
 
     ZipFile.read_from(file.blob) do |zip|
       Account::DataTransfer::Manifest.new(account).each_record_set(start: start) do |record_set, last_id|
@@ -33,6 +42,9 @@ def check(start: nil, callback: nil)
   rescue Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError => e
     mark_as_failed(:invalid_export)
     raise e
+  rescue InsufficientDiskSpaceError => e
+    mark_as_failed(:insufficient_disk)
+    raise e
   rescue => e
     mark_as_failed
     raise e
@@ -69,6 +81,26 @@ def cleanup
   end
 
   private
+    # Running out of disk mid-import is the worst self-hosted failure mode:
+    # Solid Queue shares the web process, so once its own SQLite writes start
+    # failing ("database or disk is full") the whole app goes down with the
+    # import. Fail fast with a clear reason before consuming the space.
+    def ensure_sufficient_disk_space
+      return unless file.blob.service.respond_to?(:root)
+
+      required = file.blob.byte_size * REQUIRED_DISK_FACTOR
+      available = available_disk_space
+
+      if available && available < required
+        raise InsufficientDiskSpaceError, "import needs ~#{required / 1.gigabyte} GB free, found #{available / 1.gigabyte} GB"
+      end
+    end
+
+    def available_disk_space
+      fields = `df -Pk #{Shellwords.escape(file.blob.service.root.to_s)} 2>/dev/null`.lines.last&.split
+      fields && fields[3].to_i * 1024
+    end
+
     def mark_completed
       update!(status: :completed, completed_at: Time.current)
       ImportMailer.completed(identity, account).deliver_later

app/models/zip_file.rb

@@ -91,9 +91,21 @@ def read_from_s3(blob)
       end
 
       def read_from_disk(blob)
-        blob.open do |file|
-          reader = Reader.new(file)
-          yield reader
+        # Read the stored file in place when the service exposes its path.
+        # blob.open copies the whole blob to a tempfile first — for large
+        # self-hosted imports that's a multi-gigabyte extra disk demand paid
+        # on every read pass, and zip reading only needs random access, which
+        # a plain File handle provides.
+        if blob.service.respond_to?(:path_for)
+          File.open(blob.service.path_for(blob.key), "rb") do |file|
+            reader = Reader.new(file)
+            yield reader
+          end
+        else
+          blob.open do |file|
+            reader = Reader.new(file)
+            yield reader
+          end
         end
       end
   end

app/views/account/imports/show.html.erb

@@ -22,6 +22,8 @@
         <div>The account you’re trying to import already exists. Make sure you’re importing a <%= Fizzy.saas? ? "self-hosted" : "fizzy.do" %> account.</div>
       <% elsif @import.failed_due_to_invalid_export? %>
         <div>The .zip file you uploaded doesn’t look like a Fizzy account export.</div>
+      <% elsif @import.failed_due_to_insufficient_disk? %>
+        <div>Your server doesn’t have enough free disk space for this import. Free up at least twice the export file’s size and try again.</div>
       <% else %>
         <div>This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export.</div>
       <% end %>

app/views/mailers/import_mailer/failed.html.erb

@@ -4,6 +4,8 @@
   <p>It looks like the account you are trying to import already exists.</p>
 <% elsif @import.failed_due_to_invalid_export? %>
   <p>The ZIP file isn't a Fizzy account export.</p>
+<% elsif @import.failed_due_to_insufficient_disk? %>
+  <p>Your server doesn't have enough free disk space for this import. Free up at least twice the export file's size and try again.</p>
 <% else %>
   <p>This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export, or reach out for help if the problem persists.</p>
 <% end %>

app/views/mailers/import_mailer/failed.text.erb

@@ -4,6 +4,8 @@ Unfortunately, we couldn't import your Fizzy account.
 It looks like the account you are trying to import already exists.
 <% elsif @import.failed_due_to_invalid_export? -%>
 The ZIP file isn't a Fizzy account export.
+<% elsif @import.failed_due_to_insufficient_disk? -%>
+Your server doesn't have enough free disk space for this import. Free up at least twice the export file's size and try again.
 <% else -%>
 This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export, or reach out for help if the problem persists.
 <% end -%>

test/models/account/import_test.rb

@@ -220,7 +220,33 @@ class Account::ImportTest < ActiveSupport::TestCase
     export_tempfile&.unlink
   end
 
+  test "check fails fast with a clear reason when free disk space is insufficient" do
+    import = import_with_attached_zip
+    import.stubs(:available_disk_space).returns(import.file.blob.byte_size)
+
+    error = assert_raises(Account::Import::InsufficientDiskSpaceError) { import.check }
+    assert_match(/GB free/, error.message)
+    assert import.reload.failed_due_to_insufficient_disk?
+  end
+
+  test "check proceeds when free disk space cannot be determined" do
+    import = import_with_attached_zip
+    import.stubs(:available_disk_space).returns(nil)
+
+    assert_raises(ZipFile::InvalidFileError) { import.check }
+    assert import.reload.failed_due_to_invalid_export?
+  end
+
   private
+    def import_with_attached_zip
+      account = Account.create!(name: "Disk Check")
+      import = Account::Import.create!(account: account, identity: identities(:david))
+      Current.set(account: account) do
+        import.file.attach(io: StringIO.new("not actually a zip"), filename: "export.zip", content_type: "application/zip")
+      end
+      import
+    end
+
     def account_digest(account)
       {
         name: account.name,