Data exports (#2517)

* Import / export UI

* Data exports

* Lint fixes, brakeman update

Author: zachgoll

Gemfile

@@ -72,6 +72,7 @@ gem "plaid"
 gem "rotp", "~> 6.3"
 gem "rqrcode", "~> 3.0"
 gem "activerecord-import"
+gem "rubyzip", "~> 2.3"
 
 # State machines
 gem "aasm"

Gemfile.lock

@@ -672,6 +672,7 @@ DEPENDENCIES
   rubocop-rails-omakase
   ruby-lsp-rails
   ruby-openai
+  rubyzip (~> 2.3)
   selenium-webdriver
   sentry-rails
   sentry-ruby

app/controllers/family_exports_controller.rb

@@ -0,0 +1,47 @@
+class FamilyExportsController < ApplicationController
+  include StreamExtensions
+
+  before_action :require_admin
+  before_action :set_export, only: [ :download ]
+
+  def new
+    # Modal view for initiating export
+  end
+
+  def create
+    @export = Current.family.family_exports.create!
+    FamilyDataExportJob.perform_later(@export)
+
+    respond_to do |format|
+      format.html { redirect_to settings_profile_path, notice: "Export started. You'll be able to download it shortly." }
+      format.turbo_stream {
+        stream_redirect_to settings_profile_path, notice: "Export started. You'll be able to download it shortly."
+      }
+    end
+  end
+
+  def index
+    @exports = Current.family.family_exports.ordered.limit(10)
+    render layout: false # For turbo frame
+  end
+
+  def download
+    if @export.downloadable?
+      redirect_to @export.export_file, allow_other_host: true
+    else
+      redirect_to settings_profile_path, alert: "Export not ready for download"
+    end
+  end
+
+  private
+
+    def set_export
+      @export = Current.family.family_exports.find(params[:id])
+    end
+
+    def require_admin
+      unless Current.user.admin?
+        redirect_to root_path, alert: "Access denied"
+      end
+    end
+end

app/jobs/family_data_export_job.rb

@@ -0,0 +1,22 @@
+class FamilyDataExportJob < ApplicationJob
+  queue_as :default
+
+  def perform(family_export)
+    family_export.update!(status: :processing)
+
+    exporter = Family::DataExporter.new(family_export.family)
+    zip_file = exporter.generate_export
+
+    family_export.export_file.attach(
+      io: zip_file,
+      filename: family_export.filename,
+      content_type: "application/zip"
+    )
+
+    family_export.update!(status: :completed)
+  rescue => e
+    Rails.logger.error "Family export failed: #{e.message}"
+    Rails.logger.error e.backtrace.join("\n")
+    family_export.update!(status: :failed)
+  end
+end

app/models/family.rb

@@ -18,6 +18,7 @@ class Family < ApplicationRecord
   has_many :invitations, dependent: :destroy
 
   has_many :imports, dependent: :destroy
+  has_many :family_exports, dependent: :destroy
 
   has_many :entries, through: :accounts
   has_many :transactions, through: :accounts

app/models/family/data_exporter.rb

@@ -0,0 +1,238 @@
+require "zip"
+require "csv"
+
+class Family::DataExporter
+  def initialize(family)
+    @family = family
+  end
+
+  def generate_export
+    # Create a StringIO to hold the zip data in memory
+    zip_data = Zip::OutputStream.write_buffer do |zipfile|
+      # Add accounts.csv
+      zipfile.put_next_entry("accounts.csv")
+      zipfile.write generate_accounts_csv
+
+      # Add transactions.csv
+      zipfile.put_next_entry("transactions.csv")
+      zipfile.write generate_transactions_csv
+
+      # Add trades.csv
+      zipfile.put_next_entry("trades.csv")
+      zipfile.write generate_trades_csv
+
+      # Add categories.csv
+      zipfile.put_next_entry("categories.csv")
+      zipfile.write generate_categories_csv
+
+      # Add all.ndjson
+      zipfile.put_next_entry("all.ndjson")
+      zipfile.write generate_ndjson
+    end
+
+    # Rewind and return the StringIO
+    zip_data.rewind
+    zip_data
+  end
+
+  private
+
+    def generate_accounts_csv
+      CSV.generate do |csv|
+        csv << [ "id", "name", "type", "subtype", "balance", "currency", "created_at" ]
+
+        # Only export accounts belonging to this family
+        @family.accounts.includes(:accountable).find_each do |account|
+          csv << [
+            account.id,
+            account.name,
+            account.accountable_type,
+            account.subtype,
+            account.balance.to_s,
+            account.currency,
+            account.created_at.iso8601
+          ]
+        end
+      end
+    end
+
+    def generate_transactions_csv
+      CSV.generate do |csv|
+        csv << [ "date", "account_name", "amount", "name", "category", "tags", "notes", "currency" ]
+
+        # Only export transactions from accounts belonging to this family
+        @family.transactions
+          .includes(:category, :tags, entry: :account)
+          .find_each do |transaction|
+            csv << [
+              transaction.entry.date.iso8601,
+              transaction.entry.account.name,
+              transaction.entry.amount.to_s,
+              transaction.entry.name,
+              transaction.category&.name,
+              transaction.tags.pluck(:name).join(","),
+              transaction.entry.notes,
+              transaction.entry.currency
+            ]
+          end
+      end
+    end
+
+    def generate_trades_csv
+      CSV.generate do |csv|
+        csv << [ "date", "account_name", "ticker", "quantity", "price", "amount", "currency" ]
+
+        # Only export trades from accounts belonging to this family
+        @family.trades
+          .includes(:security, entry: :account)
+          .find_each do |trade|
+            csv << [
+              trade.entry.date.iso8601,
+              trade.entry.account.name,
+              trade.security.ticker,
+              trade.qty.to_s,
+              trade.price.to_s,
+              trade.entry.amount.to_s,
+              trade.currency
+            ]
+          end
+      end
+    end
+
+    def generate_categories_csv
+      CSV.generate do |csv|
+        csv << [ "name", "color", "parent_category", "classification" ]
+
+        # Only export categories belonging to this family
+        @family.categories.includes(:parent).find_each do |category|
+          csv << [
+            category.name,
+            category.color,
+            category.parent&.name,
+            category.classification
+          ]
+        end
+      end
+    end
+
+    def generate_ndjson
+      lines = []
+
+      # Export accounts with full accountable data
+      @family.accounts.includes(:accountable).find_each do |account|
+        lines << {
+          type: "Account",
+          data: account.as_json(
+            include: {
+              accountable: {}
+            }
+          )
+        }.to_json
+      end
+
+      # Export categories
+      @family.categories.find_each do |category|
+        lines << {
+          type: "Category",
+          data: category.as_json
+        }.to_json
+      end
+
+      # Export tags
+      @family.tags.find_each do |tag|
+        lines << {
+          type: "Tag",
+          data: tag.as_json
+        }.to_json
+      end
+
+      # Export merchants (only family merchants)
+      @family.merchants.find_each do |merchant|
+        lines << {
+          type: "Merchant",
+          data: merchant.as_json
+        }.to_json
+      end
+
+      # Export transactions with full data
+      @family.transactions.includes(:category, :merchant, :tags, entry: :account).find_each do |transaction|
+        lines << {
+          type: "Transaction",
+          data: {
+            id: transaction.id,
+            entry_id: transaction.entry.id,
+            account_id: transaction.entry.account_id,
+            date: transaction.entry.date,
+            amount: transaction.entry.amount,
+            currency: transaction.entry.currency,
+            name: transaction.entry.name,
+            notes: transaction.entry.notes,
+            excluded: transaction.entry.excluded,
+            category_id: transaction.category_id,
+            merchant_id: transaction.merchant_id,
+            tag_ids: transaction.tag_ids,
+            kind: transaction.kind,
+            created_at: transaction.created_at,
+            updated_at: transaction.updated_at
+          }
+        }.to_json
+      end
+
+      # Export trades with full data
+      @family.trades.includes(:security, entry: :account).find_each do |trade|
+        lines << {
+          type: "Trade",
+          data: {
+            id: trade.id,
+            entry_id: trade.entry.id,
+            account_id: trade.entry.account_id,
+            security_id: trade.security_id,
+            ticker: trade.security.ticker,
+            date: trade.entry.date,
+            qty: trade.qty,
+            price: trade.price,
+            amount: trade.entry.amount,
+            currency: trade.currency,
+            created_at: trade.created_at,
+            updated_at: trade.updated_at
+          }
+        }.to_json
+      end
+
+      # Export valuations
+      @family.entries.valuations.includes(:account, :entryable).find_each do |entry|
+        lines << {
+          type: "Valuation",
+          data: {
+            id: entry.entryable.id,
+            entry_id: entry.id,
+            account_id: entry.account_id,
+            date: entry.date,
+            amount: entry.amount,
+            currency: entry.currency,
+            name: entry.name,
+            created_at: entry.created_at,
+            updated_at: entry.updated_at
+          }
+        }.to_json
+      end
+
+      # Export budgets
+      @family.budgets.find_each do |budget|
+        lines << {
+          type: "Budget",
+          data: budget.as_json
+        }.to_json
+      end
+
+      # Export budget categories
+      @family.budget_categories.includes(:budget, :category).find_each do |budget_category|
+        lines << {
+          type: "BudgetCategory",
+          data: budget_category.as_json
+        }.to_json
+      end
+
+      lines.join("\n")
+    end
+end

app/models/family_export.rb

@@ -0,0 +1,22 @@
+class FamilyExport < ApplicationRecord
+  belongs_to :family
+
+  has_one_attached :export_file
+
+  enum :status, {
+    pending: "pending",
+    processing: "processing",
+    completed: "completed",
+    failed: "failed"
+  }, default: :pending, validate: true
+
+  scope :ordered, -> { order(created_at: :desc) }
+
+  def filename
+    "maybe_export_#{created_at.strftime('%Y%m%d_%H%M%S')}.zip"
+  end
+
+  def downloadable?
+    completed? && export_file.attached?
+  end
+end

app/views/family_exports/_list.html.erb

@@ -0,0 +1,39 @@
+<%= turbo_frame_tag "family_exports",
+    data: exports.any? { |e| e.pending? || e.processing? } ? {
+      turbo_refresh_url: family_exports_path,
+      turbo_refresh_interval: 3000
+    } : {} do %>
+  <div class="mt-4 space-y-3 max-h-96 overflow-y-auto">
+    <% if exports.any? %>
+      <% exports.each do |export| %>
+        <div class="flex items-center justify-between bg-container p-4 rounded-lg border border-primary">
+          <div>
+            <p class="text-sm font-medium text-primary">Export from <%= export.created_at.strftime("%B %d, %Y at %I:%M %p") %></p>
+            <p class="text-xs text-secondary"><%= export.filename %></p>
+          </div>
+
+          <% if export.processing? || export.pending? %>
+            <div class="flex items-center gap-2 text-secondary">
+              <div class="animate-spin h-4 w-4 border-2 border-secondary border-t-transparent rounded-full"></div>
+              <span class="text-sm">Exporting...</span>
+            </div>
+          <% elsif export.completed? %>
+            <%= link_to download_family_export_path(export),
+                class: "flex items-center gap-2 text-primary hover:text-primary-hover",
+                data: { turbo_frame: "_top" } do %>
+              <%= icon "download", class: "w-5 h-5" %>
+              <span class="text-sm font-medium">Download</span>
+            <% end %>
+          <% elsif export.failed? %>
+            <div class="flex items-center gap-2 text-destructive">
+              <%= icon "alert-circle", class: "w-4 h-4" %>
+              <span class="text-sm">Failed</span>
+            </div>
+          <% end %>
+        </div>
+      <% end %>
+    <% else %>
+      <p class="text-sm text-secondary text-center py-4">No exports yet</p>
+    <% end %>
+  </div>
+<% end %>

app/views/family_exports/index.html.erb

@@ -0,0 +1 @@
+<%= render "list", exports: @exports %>