Add account data enrichment (#1532)

* Add data enrichment

* Make data enrichment optional for self-hosters

* Add categories to data enrichment

* Only update category and merchant if nil

* Fix name overrides

* Lint fixes

Author: zachgoll

app/controllers/settings/hostings_controller.rb

@@ -26,6 +26,10 @@ def update
       Setting.synth_api_key = hosting_params[:synth_api_key]
     end
 
+    if hosting_params.key?(:data_enrichment_enabled)
+      Setting.data_enrichment_enabled = hosting_params[:data_enrichment_enabled]
+    end
+
     redirect_to settings_hosting_path, notice: t(".success")
   rescue ActiveRecord::RecordInvalid => error
     flash.now[:alert] = t(".failure")
@@ -34,7 +38,7 @@ def update
 
   private
     def hosting_params
-      params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key)
+      params.require(:setting).permit(:render_deploy_hook, :upgrades_setting, :require_invite_for_signup, :synth_api_key, :data_enrichment_enabled)
     end
 
     def raise_if_not_self_hosted

app/jobs/enrich_data_job.rb

@@ -0,0 +1,7 @@
+class EnrichDataJob < ApplicationJob
+  queue_as :default
+
+  def perform(account)
+    account.enrich_data
+  end
+end

app/models/account.rb

@@ -126,6 +126,14 @@ def favorable_direction
     classification == "asset" ? "up" : "down"
   end
 
+  def enrich_data
+    DataEnricher.new(self).run
+  end
+
+  def enrich_data_later
+    EnrichDataJob.perform_later(self)
+  end
+
   def update_with_sync!(attributes)
     transaction do
       update!(attributes)

app/models/account/data_enricher.rb

@@ -0,0 +1,61 @@
+class Account::DataEnricher
+  include Providable
+
+  attr_reader :account
+
+  def initialize(account)
+    @account = account
+  end
+
+  def run
+    enrich_transactions
+  end
+
+  private
+    def enrich_transactions
+      candidates = account.entries.account_transactions.includes(entryable: [ :merchant, :category ])
+
+      Rails.logger.info("Enriching #{candidates.count} transactions for account #{account.id}")
+
+      merchants = {}
+      categories = {}
+
+      candidates.each do |entry|
+        if entry.enriched_at.nil? || entry.entryable.merchant_id.nil? || entry.entryable.category_id.nil?
+          begin
+            info = self.class.synth_provider.enrich_transaction(entry.name).info
+
+            next unless info.present?
+
+            if info.name.present?
+              merchant = merchants[info.name] ||= account.family.merchants.find_or_create_by(name: info.name)
+
+              if info.icon_url.present?
+                merchant.icon_url = info.icon_url
+              end
+            end
+
+            if info.category.present?
+              category = categories[info.category] ||= account.family.categories.find_or_create_by(name: info.category)
+            end
+
+            entryable_attributes = { id: entry.entryable_id }
+            entryable_attributes[:merchant_id] = merchant.id if merchant.present? && entry.entryable.merchant_id.nil?
+            entryable_attributes[:category_id] = category.id if category.present? && entry.entryable.category_id.nil?
+
+            Account.transaction do
+              merchant.save! if merchant.present?
+              category.save! if category.present?
+              entry.update!(
+                enriched_at: Time.current,
+                name: entry.enriched_at.nil? ? info.name : entry.name,
+                entryable_attributes: entryable_attributes
+              )
+            end
+          rescue => e
+            Rails.logger.warn("Error enriching transaction #{entry.id}: #{e.message}")
+          end
+        end
+      end
+    end
+end

app/models/account/syncer.rb

@@ -10,6 +10,12 @@ def run
     account.reload
     update_account_info(balances, holdings) unless account.plaid_account_id.present?
     convert_records_to_family_currency(balances, holdings) unless account.currency == account.family.currency
+
+    if Setting.data_enrichment_enabled || Rails.configuration.app_mode.managed?
+      account.enrich_data_later
+    else
+      Rails.logger.info("Data enrichment is disabled, skipping enrichment for account #{account.id}")
+    end
   end
 
   private

app/models/concerns/providable.rb

@@ -23,8 +23,10 @@ def git_repository_provider
     end
 
     def synth_provider
-      api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
-      api_key.present? ? Provider::Synth.new(api_key) : nil
+      @synth_provider ||= begin
+        api_key = self_hosted? ? Setting.synth_api_key : ENV["SYNTH_API_KEY"]
+        api_key.present? ? Provider::Synth.new(api_key) : nil
+      end
     end
 
     private

app/models/provider/synth.rb

@@ -167,6 +167,35 @@ def fetch_security_info(ticker:, mic_code:)
       raw_response: response
   end
 
+  def enrich_transaction(description, amount: nil, date: nil, city: nil, state: nil, country: nil)
+    params = {
+      description: description,
+      amount: amount,
+      date: date,
+      city: city,
+      state: state,
+      country: country
+    }.compact
+
+    response = client.get("#{base_url}/enrich", params)
+
+    parsed = JSON.parse(response.body)
+
+    EnrichTransactionResponse.new \
+      info: EnrichTransactionInfo.new(
+        name: parsed.dig("merchant"),
+        icon_url: parsed.dig("icon"),
+        category: parsed.dig("category")
+      ),
+      success?: true,
+      raw_response: response
+  rescue StandardError => error
+    EnrichTransactionResponse.new \
+      success?: false,
+      error: error,
+      raw_response: error
+  end
+
   private
 
     attr_reader :api_key
@@ -177,6 +206,8 @@ def fetch_security_info(ticker:, mic_code:)
     UsageResponse = Struct.new :used, :limit, :utilization, :plan, :success?, :error, :raw_response, keyword_init: true
     SearchSecuritiesResponse = Struct.new :securities, :success?, :error, :raw_response, keyword_init: true
     SecurityInfoResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
+    EnrichTransactionResponse = Struct.new :info, :success?, :error, :raw_response, keyword_init: true
+    EnrichTransactionInfo = Struct.new :name, :icon_url, :category, keyword_init: true
 
     def base_url
       ENV["SYNTH_URL"] || "https://api.synthfinance.com"

app/models/setting.rb

@@ -17,6 +17,10 @@ class Setting < RailsSettings::Base
         default: ENV.fetch("UPGRADES_TARGET", "release"),
         validates: { inclusion: { in: %w[release commit] } }
 
+  field :data_enrichment_enabled,
+        type: :boolean,
+        default: true
+
   field :synth_api_key, type: :string, default: ENV["SYNTH_API_KEY"]
 
   field :require_invite_for_signup, type: :boolean, default: false

app/views/account/transactions/_transaction.html.erb

@@ -11,15 +11,17 @@
 
     <div class="max-w-full">
       <%= content_tag :div, class: ["flex items-center gap-2"] do %>
-        <div class="flex h-8 w-8 shrink-0 items-center justify-center rounded-full bg-gray-600/5 text-gray-600">
-          <%= transaction.name.first.upcase %>
-        </div>
+        <% if entry.account_transaction.merchant&.icon_url %>
+          <%= image_tag entry.account_transaction.merchant.icon_url, class: "w-6 h-6 rounded-full" %>
+        <% else %>
+          <%= render "shared/circle_logo", name: entry.name, size: "sm" %>
+        <% end %>
 
         <div class="truncate">
           <% if entry.new_record? %>
-            <%= content_tag :p, transaction.name %>
+            <%= content_tag :p, entry.name %>
           <% else %>
-            <%= link_to transaction.name,
+            <%= link_to entry.name,
                         entry.transfer.present? ? account_transfer_path(entry.transfer) : account_entry_path(entry),
                         data: { turbo_frame: "drawer", turbo_prefetch: false },
                         class: "hover:underline hover:text-gray-800" %>

app/views/merchants/_merchant.html.erb

@@ -2,7 +2,14 @@
 
 <div class="flex justify-between items-center p-4 bg-white">
   <div class="flex w-full items-center gap-2.5">
-    <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %>
+    <% if merchant.icon_url %>
+      <div class="w-8 h-8 rounded-full flex justify-center items-center">
+        <%= image_tag merchant.icon_url, class: "w-8 h-8 rounded-full" %>
+      </div>
+    <% else %>
+      <%= render partial: "shared/color_avatar", locals: { name: merchant.name, color: merchant.color } %>
+    <% end %>
+
     <p class="text-gray-900 text-sm truncate">
       <%= merchant.name %>
     </p>