perf(sync): Preload and cache exchange rate data when syncing.

Author: Joelute

app/models/balance/sync_cache.rb

@@ -15,32 +15,69 @@ def get_entries(date)
     converted_entries.select { |e| e.date == date && (e.transaction? || e.trade?) }
   end
 
+  def find_rate_by_cache(amount_money, to_currency, date: Date.current, fallback_rate: 1)
+    raise TypeError unless amount_money.respond_to?(:amount) && amount_money.respond_to?(:currency)
+
+    iso_code = Money::Currency.new(amount_money.currency).iso_code
+    other_iso_code = Money::Currency.new(to_currency).iso_code
+
+    return amount_money if iso_code == other_iso_code
+
+    exchange_rate = exchange_rates(to_currency)[[ iso_code, date ]]&.last&.rate ||
+      ExchangeRate.fetch_rate(from: iso_code, to: other_iso_code, date: date)&.rate ||
+      fallback_rate
+
+    raise Money::ConversionError.new(from_currency: iso_code, to_currency: other_iso_code, date: date) unless exchange_rate
+
+    Money.new(amount_money.amount * exchange_rate, other_iso_code)
+  end
+
   private
     attr_reader :account
 
     def converted_entries
-      @converted_entries ||= account.entries.order(:date).to_a.map do |e|
+      @converted_entries ||= entries.map do |e|
         converted_entry = e.dup
-        converted_entry.amount = converted_entry.amount_money.exchange_to(
+        converted_entry.amount = find_rate_by_cache(
+          converted_entry.amount_money,
           account.currency,
           date: e.date,
-          fallback_rate: 1
         ).amount
         converted_entry.currency = account.currency
         converted_entry
       end
     end
 
     def converted_holdings
-      @converted_holdings ||= account.holdings.map do |h|
+      @converted_holdings ||= holdings.map do |h|
         converted_holding = h.dup
-        converted_holding.amount = converted_holding.amount_money.exchange_to(
+        converted_holding.amount = find_rate_by_cache(
+          converted_holding.amount_money,
           account.currency,
           date: h.date,
-          fallback_rate: 1
         ).amount
         converted_holding.currency = account.currency
         converted_holding
       end
     end
+
+    def entries
+      @entries ||= account.entries.order(:date).to_a
+    end
+
+    def holdings
+      @holdings ||= account.holdings
+    end
+
+    def exchange_rates(to_currency)
+      combined = entries + holdings
+      all_dates = combined.map(&:date).uniq
+      all_currencies = combined.map(&:currency).uniq
+
+      @exchange_rates ||= ExchangeRate
+        .where(from_currency: all_currencies, to_currency: to_currency)
+        .where(date: all_dates)
+        .order(:date)
+        .group_by { |r| [ r.from_currency, r.date.to_date ] }
+    end
 end

app/models/exchange_rate/provided.rb

@@ -27,6 +27,23 @@ def find_or_fetch_rate(from:, to:, date: Date.current, cache: true)
       rate
     end
 
+    def fetch_rate(from:, to:, date: Date.current, cache: true)
+      return nil unless provider.present? # No provider configured (some self-hosted apps)
+
+      response = provider.fetch_exchange_rate(from: from, to: to, date: date)
+
+      return nil unless response.success? # Provider error
+
+      rate = response.data
+      ExchangeRate.find_or_create_by!(
+        from_currency: rate.from,
+        to_currency: rate.to,
+        date: rate.date,
+        rate: rate.rate
+      ) if cache
+      rate
+    end
+
     # @return [Integer] The number of exchange rates synced
     def import_provider_rates(from:, to:, start_date:, end_date:, clear_cache: false)
       unless provider.present?

test/models/balance/forward_calculator_test.rb

@@ -2,6 +2,7 @@
 
 class Balance::ForwardCalculatorTest < ActiveSupport::TestCase
   include EntriesTestHelper
+  include ExchangeRateTestHelper
 
   setup do
     @account = families(:empty).accounts.create!(
@@ -55,8 +56,10 @@ class Balance::ForwardCalculatorTest < ActiveSupport::TestCase
     create_transaction(account: @account, date: 2.days.ago.to_date, amount: 100) # expense
 
     expected = [ 0, 500, 500, 400, 400, 400 ]
-    calculated = Balance::ForwardCalculator.new(@account).calculate.sort_by(&:date).map(&:balance)
-
+    calculated = nil
+    assert_queries_count(3) do
+      calculated = Balance::ForwardCalculator.new(@account).calculate.sort_by(&:date).map(&:balance)
+    end
     assert_equal expected, calculated
   end
 
@@ -75,17 +78,30 @@ class Balance::ForwardCalculatorTest < ActiveSupport::TestCase
   end
 
   test "multi-currency sync" do
-    ExchangeRate.create! date: 1.day.ago.to_date, from_currency: "EUR", to_currency: "USD", rate: 1.2
+    load_exchange_prices
 
-    create_transaction(account: @account, date: 3.days.ago.to_date, amount: -100, currency: "USD")
-    create_transaction(account: @account, date: 2.days.ago.to_date, amount: -300, currency: "USD")
+    expected = [ 0.0 ]
 
-    # Transaction in different currency than the account's main currency
-    create_transaction(account: @account, date: 1.day.ago.to_date, amount: -500, currency: "EUR") # €500 * 1.2 = $600
+    for i in (3).downto(0) do
+      amount = 0
+      create_transaction(account: @account, date: i.days.ago.to_date, amount: -100, currency: "USD")
+      create_transaction(account: @account, date: i.days.ago.to_date, amount: -100, currency: "CAD")
+      create_transaction(account: @account, date: i.days.ago.to_date, amount: -100, currency: "EUR")
 
-    expected = [ 0, 100, 400, 1000, 1000 ]
-    calculated = Balance::ForwardCalculator.new(@account).calculate.sort_by(&:date).map(&:balance)
+      amount = @account.entries.where(date: i.days.ago.to_date).map do |e|
+        e.amount_money.exchange_to(
+          @account.currency,
+          date: e.date,
+        ).amount
+      end
 
+      expected << expected.last + (amount.sum * -1)
+    end
+
+    calculated = nil
+    assert_queries_count(4) do
+      calculated = Balance::ForwardCalculator.new(@account).calculate.sort_by(&:date).map(&:balance)
+    end
     assert_equal expected, calculated
   end
 

test/models/balance/sync_cache_test.rb

@@ -0,0 +1,85 @@
+require "test_helper"
+
+class Balance::SyncCacheTest < ActiveSupport::TestCase
+  include EntriesTestHelper
+  include ExchangeRateTestHelper
+
+  setup do
+    @account = families(:empty).accounts.create!(
+      name: "Test",
+      balance: 20000,
+      cash_balance: 20000,
+      currency: "USD",
+      accountable: Investment.new
+    )
+    create_transaction(account: @account, date: 1.day.ago.to_date, amount: 100, currency: "CAD")
+    @sync_cache = Balance::SyncCache.new(@account)
+  end
+
+  test "convert currency when rate available by cache" do
+    load_exchange_prices
+    money = Money.new(100, "CAD")
+    converted_money = nil
+    assert_queries_count(3) do
+      converted_money = @sync_cache.find_rate_by_cache(money, @account.currency, date: 1.day.ago.to_date)
+    end
+    expected_money = money.exchange_to(@account.currency, date: 1.day.ago.to_date)
+
+    assert_equal expected_money.amount, converted_money.amount
+    assert_equal expected_money.currency, converted_money.currency
+  end
+
+  test "convert currency after fetching rate" do
+    ExchangeRate.expects(:fetch_rate).returns(Provider::ExchangeRateConcept::Rate.new(date: 1.day.ago.to_date, from: "JPY", to: "USD", rate: 0.007))
+    money = Money.new(1000, "JPY")
+
+    assert_equal Money.new(7, "USD"), @sync_cache.find_rate_by_cache(money, @account.currency, date: 1.day.ago.to_date)
+  end
+
+  test "converts currency with a fallback rate" do
+    ExchangeRate.expects(:fetch_rate).returns(nil).twice
+    money = Money.new(1000, "CAD")
+
+    assert_queries_count(3) do
+      assert_equal Money.new(0, "USD"), @sync_cache.find_rate_by_cache(money, @account.currency, date: 1.day.ago.to_date, fallback_rate: 0)
+    end
+
+    assert_equal Money.new(1000, "USD"), @sync_cache.find_rate_by_cache(money, @account.currency, date: 1.day.ago.to_date, fallback_rate: 1)
+  end
+
+  test "raises when no conversion rate available and no fallback rate provided" do
+    money = Money.new(1000, "JPY")
+    ExchangeRate.expects(:fetch_rate).returns(nil)
+
+    assert_raises Money::ConversionError do
+      @sync_cache.find_rate_by_cache(money, @account.currency, date: 1.day.ago.to_date, fallback_rate: nil)
+    end
+  end
+
+  test "raises if input is not Money-like" do
+    assert_raises(TypeError) do
+      @sync_cache.find_rate_by_cache(12345, "USD")
+    end
+  end
+
+  test "returns original money when source and target currency are the same" do
+    money = Money.new(500, "USD")
+    result = nil
+    assert_queries_count(0) do
+      result = @sync_cache.find_rate_by_cache(money, "USD", date: Date.today)
+    end
+    assert_same money, result
+  end
+
+  test "query and cache exchange rates" do
+    load_exchange_prices
+    assert_queries_count(3) do
+      @sync_cache.find_rate_by_cache(Money.new(100, "CAD"), @account.currency, date: 1.day.ago.to_date)
+    end
+
+    assert_queries_count(0) do
+      @sync_cache.find_rate_by_cache(Money.new(100, "CAD"), @account.currency, date: 1.day.ago.to_date)
+      @sync_cache.find_rate_by_cache(Money.new(200, "CAD"), @account.currency, date: 1.day.ago.to_date)
+    end
+  end
+end

test/models/family/auto_transfer_matchable_test.rb

@@ -2,6 +2,7 @@
 
 class Family::AutoTransferMatchableTest < ActiveSupport::TestCase
   include EntriesTestHelper
+  include ExchangeRateTestHelper
 
   setup do
     @family = families(:dylan_family)
@@ -116,33 +117,4 @@ class Family::AutoTransferMatchableTest < ActiveSupport::TestCase
       @family.auto_match_transfers!
     end
   end
-
-  private
-    def load_exchange_prices
-      rates = {
-        4.days.ago.to_date => 1.36,
-        3.days.ago.to_date => 1.37,
-        2.days.ago.to_date => 1.38,
-        1.day.ago.to_date  => 1.39,
-        Date.current => 1.40
-      }
-
-      rates.each do |date, rate|
-        # USD to CAD
-        ExchangeRate.create!(
-          from_currency: "USD",
-          to_currency: "CAD",
-          date: date,
-          rate: rate
-        )
-
-        # CAD to USD (inverse)
-        ExchangeRate.create!(
-          from_currency: "CAD",
-          to_currency: "USD",
-          date: date,
-          rate: (1.0 / rate).round(6)
-        )
-      end
-    end
 end

test/support/exchange_rate_test_helper.rb

@@ -0,0 +1,55 @@
+module ExchangeRateTestHelper
+  def load_exchange_prices
+    cad_rates = {
+      4.days.ago.to_date => 1.36,
+      3.days.ago.to_date => 1.37,
+      2.days.ago.to_date => 1.38,
+      1.day.ago.to_date  => 1.39,
+      Date.current => 1.40
+    }
+
+    eur_rates = {
+      4.days.ago.to_date => 1.17,
+      3.days.ago.to_date => 1.18,
+      2.days.ago.to_date => 1.19,
+      1.day.ago.to_date  => 1.2,
+      Date.current => 1.21
+    }
+
+    cad_rates.each do |date, rate|
+      # USD to CAD
+      ExchangeRate.create!(
+        from_currency: "USD",
+        to_currency: "CAD",
+        date: date,
+        rate: rate
+      )
+
+      # CAD to USD (inverse)
+      ExchangeRate.create!(
+        from_currency: "CAD",
+        to_currency: "USD",
+        date: date,
+        rate: (1.0 / rate).round(6)
+      )
+    end
+
+    eur_rates.each do |date, rate|
+      # EUR to USD
+      ExchangeRate.create!(
+        from_currency: "EUR",
+        to_currency: "USD",
+        date: date,
+        rate: rate
+      )
+
+      # USD to EUR (inverse)
+      ExchangeRate.create!(
+        from_currency: "USD",
+        to_currency: "EUR",
+        date: date,
+        rate: (1.0 / rate).round(6)
+      )
+    end
+  end
+end