Fix BC-10008678752: tame self-hosted import disk demands

app/jobs/account/data_import_job.rb

@@ -2,7 +2,8 @@ class Account::DataImportJob < ApplicationJob
   include ActiveJob::Continuable
 
   queue_as :backend
-  discard_on Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError
+  discard_on Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError,
+    Account::Import::InsufficientDiskSpaceError
 
   def perform(import)
     step :check do |step|

app/models/account/import.rb

@@ -1,4 +1,11 @@
+require "shellwords"
+
 class Account::Import < ApplicationRecord
+  class InsufficientDiskSpaceError < StandardError; end
+
+  # Imported blobs roughly mirror the zip's contents, plus database growth and margin.
+  REQUIRED_DISK_SPACE_FACTOR = 2
+
   broadcasts_refreshes
 
   belongs_to :account
@@ -7,7 +14,7 @@ class Account::Import < ApplicationRecord
   has_one_attached :file, dependent: :purge_later
 
   enum :status, %w[ pending processing completed failed ].index_by(&:itself), default: :pending
-  enum :failure_reason, %w[ conflict invalid_export ].index_by(&:itself), prefix: :failed_due_to, scopes: false
+  enum :failure_reason, %w[ conflict invalid_export insufficient_disk_space ].index_by(&:itself), prefix: :failed_due_to, scopes: false
 
   scope :expired, -> { where(completed_at: ...24.hours.ago).or(where(status: :failed, created_at: ...7.days.ago)) }
 
@@ -21,6 +28,7 @@ def process_later
 
   def check(start: nil, callback: nil)
     processing!
+    ensure_sufficient_disk_space
 
     ZipFile.read_from(file.blob) do |zip|
       Account::DataTransfer::Manifest.new(account).each_record_set(start: start) do |record_set, last_id|
@@ -33,6 +41,9 @@ def check(start: nil, callback: nil)
   rescue Account::DataTransfer::RecordSet::IntegrityError, ZipFile::InvalidFileError => e
     mark_as_failed(:invalid_export)
     raise e
+  rescue InsufficientDiskSpaceError => e
+    mark_as_failed(:insufficient_disk_space)
+    raise e
   rescue => e
     mark_as_failed
     raise e
@@ -69,6 +80,25 @@ def cleanup
   end
 
   private
+    # Fail fast before consuming the space: self-hosted Solid Queue shares the web
+    # process, so mid-import ENOSPC ("database or disk is full") takes the app down.
+    def ensure_sufficient_disk_space
+      return unless path = ZipFile.path_on_disk(file.blob)
+
+      required = file.blob.byte_size * REQUIRED_DISK_SPACE_FACTOR
+      available = available_disk_space(path)
+
+      if available && available < required
+        raise InsufficientDiskSpaceError, "import needs ~#{required / 1.gigabyte} GB free, found #{available / 1.gigabyte} GB"
+      end
+    end
+
+    def available_disk_space(path)
+      fields = `df -Pk #{Shellwords.escape(path)} 2>/dev/null`.lines.last&.split
+      kilobytes = fields && Integer(fields[3] || "", exception: false)
+      kilobytes && kilobytes * 1024
+    end
+
     def mark_completed
       update!(status: :completed, completed_at: Time.current)
       ImportMailer.completed(identity, account).deliver_later

app/models/zip_file.rb

@@ -26,6 +26,10 @@ def read_from(blob)
       end
     end
 
+    def path_on_disk(blob)
+      blob.service.path_for(blob.key) if blob.service.respond_to?(:path_for)
+    end
+
     private
       def s3_service?(service)
         # The S3 service doesn't get loaded in development unless it's used
@@ -91,9 +95,18 @@ def read_from_s3(blob)
       end
 
       def read_from_disk(blob)
-        blob.open do |file|
-          reader = Reader.new(file)
-          yield reader
+        # blob.open copies the whole blob to a tempfile; zip reading only needs
+        # random access, so read the stored file in place when we can.
+        if path = path_on_disk(blob)
+          File.open(path, "rb") do |file|
+            reader = Reader.new(file)
+            yield reader
+          end
+        else
+          blob.open do |file|
+            reader = Reader.new(file)
+            yield reader
+          end
         end
       end
   end

app/views/account/imports/show.html.erb

@@ -22,6 +22,8 @@
         <div>The account you’re trying to import already exists. Make sure you’re importing a <%= Fizzy.saas? ? "self-hosted" : "fizzy.do" %> account.</div>
       <% elsif @import.failed_due_to_invalid_export? %>
         <div>The .zip file you uploaded doesn’t look like a Fizzy account export.</div>
+      <% elsif @import.failed_due_to_insufficient_disk_space? %>
+        <div>Your server doesn’t have enough free disk space for this import. Free up at least twice the export file’s size and try again.</div>
       <% else %>
         <div>This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export.</div>
       <% end %>

app/views/mailers/import_mailer/failed.html.erb

@@ -4,6 +4,8 @@
   <p>It looks like the account you are trying to import already exists.</p>
 <% elsif @import.failed_due_to_invalid_export? %>
   <p>The ZIP file isn't a Fizzy account export.</p>
+<% elsif @import.failed_due_to_insufficient_disk_space? %>
+  <p>Your server doesn't have enough free disk space for this import. Free up at least twice the export file's size and try again.</p>
 <% else %>
   <p>This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export, or reach out for help if the problem persists.</p>
 <% end %>

app/views/mailers/import_mailer/failed.text.erb

@@ -4,6 +4,8 @@ Unfortunately, we couldn't import your Fizzy account.
 It looks like the account you are trying to import already exists.
 <% elsif @import.failed_due_to_invalid_export? -%>
 The ZIP file isn't a Fizzy account export.
+<% elsif @import.failed_due_to_insufficient_disk_space? -%>
+Your server doesn't have enough free disk space for this import. Free up at least twice the export file's size and try again.
 <% else -%>
 This may be due to corrupted export data or a conflict with existing data. Please try again with a fresh export, or reach out for help if the problem persists.
 <% end -%>

docs/docker-deployment.md

@@ -153,6 +153,17 @@ This is for convenience: typically when you self-host you'll be running a single
 
 If you do want to allow multiple accounts to be created in your instance, set `MULTI_TENANT=true`
 
+## Importing an existing Fizzy account
+
+You can move an account between Fizzy instances by exporting it on the old instance and uploading the export zip to the new one during signup.
+
+Imports need free disk space: at least twice the export file's size, beyond the export itself, since the imported attachments roughly mirror the zip's contents. If there isn't enough, the import fails fast with "Your server doesn't have enough free disk space for this import" — free up space (or grow the volume) and try again.
+
+For very large exports:
+
+- Browser uploads pass through Thruster, which drops slow uploads after its read timeout (a 502 before the import ever starts). Raise `THRUSTER_HTTP_READ_TIMEOUT` (seconds) and recreate the container so the setting takes effect.
+- `script/import-account` runs the import directly on the server from a zip already on disk, bypassing the browser upload entirely — handy for multi-gigabyte exports.
+
 ## Example
 
 Here's an example of a `docker-compose.yml` that you could use to run Fizzy via `docker compose up`

test/mailers/import_mailer_test.rb

@@ -47,4 +47,15 @@ class ImportMailerTest < ActionMailer::TestCase
 
     assert_match "isn't a Fizzy account export", email.body.encoded
   end
+
+  test "failed with insufficient_disk_space reason" do
+    import = Account::Import.create!(account: Current.account, identity: identities(:david), status: :failed, failure_reason: :insufficient_disk_space)
+    email = ImportMailer.failed(import)
+
+    assert_emails 1 do
+      email.deliver_now
+    end
+
+    assert_match "twice the export file", email.body.encoded
+  end
 end

test/models/account/import_test.rb

@@ -220,7 +220,40 @@ class Account::ImportTest < ActiveSupport::TestCase
     export_tempfile&.unlink
   end
 
+  test "check fails fast with a clear reason when free disk space is insufficient" do
+    import = import_with_attached_zip
+    import.stubs(:available_disk_space).returns(import.file.blob.byte_size)
+
+    error = assert_raises(Account::Import::InsufficientDiskSpaceError) { import.check }
+    assert_match(/GB free/, error.message)
+    assert import.reload.failed_due_to_insufficient_disk_space?
+  end
+
+  test "check proceeds when free disk space cannot be determined" do
+    import = import_with_attached_zip
+    import.stubs(:available_disk_space).returns(nil)
+
+    assert_raises(ZipFile::InvalidFileError) { import.check }
+    assert import.reload.failed_due_to_invalid_export?
+  end
+
+  test "available_disk_space is indeterminate when df output is unparseable" do
+    import = import_with_attached_zip
+    import.stubs(:`).returns("Filesystem 1024-blocks Used Available Capacity Mounted on\n")
+
+    assert_nil import.send(:available_disk_space, "/tmp")
+  end
+
   private
+    def import_with_attached_zip
+      account = Account.create!(name: "Disk Check")
+      import = Account::Import.create!(account: account, identity: identities(:david))
+      Current.set(account: account) do
+        import.file.attach(io: StringIO.new("not actually a zip"), filename: "export.zip", content_type: "application/zip")
+      end
+      import
+    end
+
     def account_digest(account)
       {
         name: account.name,