Skip to content

Commit a7dc760

Browse files
harsh-98harsh-98
committed
fix: tvl
1 parent c67e76f commit a7dc760

File tree

16 files changed

+131
-79
lines changed

16 files changed

+131
-79
lines changed

‎db_scripts/local_testing/anvil_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ psql -U $SUPERUSER -d $TMP_DB < $PARENT_DIR/../../migrations/000016_rankings.up.
2727
migrate -path $PARENT_DIR/../../migrations/ -database "$TMP_DB_URL" up
2828

2929

30-
psql -U $SUPERUSER -d $TMP_DB < <(cat $PARENT_DIR/reset_to_blocknum.sql | sed "s/18246321/$FORK_BLOCK/" )
30+
psql -U $SUPERUSER -d $TMP_DB < <(cat $PARENT_DIR/reset_to_blocknum.sql | sed "s/18246321/$FORK_BLOCK/g" )
3131
set +e
3232
psql -U $SUPERUSER -d postgres -c "drop database $FINAL_DB"
3333
set -e

‎db_scripts/local_testing/local_test.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,4 +4,4 @@ psql -U $SUPERUSER -d postgres -c "create database sample with owner $SUPERUSER"
44
pg_dump "$DCDB" | psql -U $SUPERUSER -d sample
55
psql -U $SUPERUSER -d sample < migrations/000016_rankings.up.sql
66
FORK_BLOCK=`jq .forkBlock.number < <(curl https://anvil.gearbox.foundation/api/forks/$2 )`
7-
psql -U $SUPERUSER -d sample < <(cat db_scripts/local_testing/reset_to_blocknum.sql | sed "s/18246321/$FORK_BLOCK/" )
7+
psql -U $SUPERUSER -d sample < <(cat db_scripts/local_testing/reset_to_blocknum.sql | sed "s/18246321/$FORK_BLOCK/g" )

‎db_scripts/local_testing/reset_to_blocknum.sql‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -8,7 +8,6 @@ delete from blocks where id > 18246321;
88

99

1010
delete from debts where block_num> 18246321;
11-
update debt_sync set last_calculated_at=18246321;
1211
delete from token_oracle where block_num> 18246321;
1312
delete from transfer_account_allowed where block_num> 18246321;
1413
delete from no_session_transfers where block_num> 18246321;
@@ -66,8 +65,8 @@ delete from rebase_details where block_num > 18246321;
6665
update sync_adapters set last_sync=18246321 where type in ('ContractRegister', 'ACL');
6766
delete from debts where block_num > 18246321;
6867
delete from current_debts where block_num > 18246321;
69-
update debt_sync set last_calculated_at=18246321;
70-
68+
update debt_sync set tvl_block=18246321, debt_block=18246321;
69+
delete from tvl_snapshots where block_num> 18246321;
7170

7271
delete from closed_trading_sessions;
7372
----

‎debts/debt_db.go‎

Lines changed: 31 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ import (
88
"github.com/Gearbox-protocol/sdk-go/log"
99
"github.com/Gearbox-protocol/sdk-go/utils"
1010
"github.com/Gearbox-protocol/third-eye/ds"
11+
"gorm.io/gorm"
1112
"gorm.io/gorm/clause"
1213
)
1314

@@ -141,16 +142,38 @@ func (eng *DebtEngine) addLastDebt(debt *schemas.Debt) {
141142
eng.lastDebts[debt.SessionId] = debt
142143
}
143144

144-
func (eng *DebtEngine) flushDebt(newDebtSyncTill int64) {
145-
debtLen := len(eng.debts)
146-
if debtLen == 0 && len(eng.tvlSnapshots) == 0 {
145+
func (eng *DebtEngine) flushTvl(tvlDebtSync int64, tx *gorm.DB, lastSync schemas.LastSync) {
146+
tvls := []*schemas.TvlSnapshots{}
147+
for _, tvl := range eng.tvlSnapshots {
148+
if tvl.BlockNum > lastSync.Tvl {
149+
tvls = append(tvls, tvl)
150+
}
151+
}
152+
if len(tvls) == 0 {
153+
return
154+
}
155+
log.Infof("Flushing tvl %d till block:%d", len(tvls), tvlDebtSync)
156+
err := tx.Exec(`UPDATE debt=?, field_set='t'`, tvlDebtSync).Error
157+
log.CheckFatal(err)
158+
err = tx.Clauses(clause.OnConflict{UpdateAll: true}).CreateInBatches(tvls, 50).Error
159+
log.CheckFatal(err)
160+
}
161+
func (eng *DebtEngine) flushDebt(newDebtSyncTill int64, tx *gorm.DB, lastSync schemas.LastSync) {
162+
debts := []*schemas.Debt{}
163+
for _, d := range eng.debts {
164+
if d.BlockNumber > lastSync.Debt {
165+
debts = append(debts, d)
166+
}
167+
}
168+
debtLen := len(debts)
169+
if debtLen == 0 {
147170
return
148171
}
149172
log.Infof("Flushing %d till block:%d", debtLen, newDebtSyncTill)
150-
tx := eng.db.Begin()
151-
err := tx.Clauses(clause.OnConflict{
152-
UpdateAll: true,
153-
}).Create(&schemas.DebtSync{LastCalculatedAt: newDebtSyncTill, FieldSet: true}).Error
173+
err := tx.Exec(`UPDATE debt_sync set debt=?, field_set='t'`, newDebtSyncTill).Error
174+
// err := tx.Clauses(clause.OnConflict{
175+
// UpdateAll: true,
176+
// }).Create(&schemas.DebtSync{Debt: newDebtSyncTill, FieldSet: true}).Error
154177
log.CheckFatal(err)
155178
liquidableAccounts := []*schemas.LiquidableAccount{}
156179
for _, la := range eng.liquidableBlockTracker {
@@ -166,14 +189,6 @@ func (eng *DebtEngine) flushDebt(newDebtSyncTill int64) {
166189
}).CreateInBatches(liquidableAccounts, 50).Error
167190
log.CheckFatal(err)
168191
}
169-
err = tx.CreateInBatches(eng.debts, 50).Error
192+
err = tx.CreateInBatches(debts, 50).Error
170193
log.CheckFatal(err)
171-
err = tx.CreateInBatches(eng.tvlSnapshots, 50).Error
172-
log.CheckFatal(err)
173-
info := tx.Commit()
174-
if info.Error != nil {
175-
log.Fatal(info.Error)
176-
}
177-
eng.debts = []*schemas.Debt{}
178-
eng.tvlSnapshots = []*schemas.TvlSnapshots{}
179194
}

‎debts/engine.go‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,7 @@ func (eng *DebtEngine) updateLocalState(blockNum int64, block *schemas.Block) (p
6565
BlockNum: ps.BlockNum,
6666
CumulativeIndexRAY: ps.CumulativeIndexRAY,
6767
AvailableLiquidityBI: ps.AvailableLiquidityBI,
68+
ExpectedLiqBI: ps.ExpectedLiquidityBI,
6869
BaseBorrowAPYBI: ps.BaseBorrowAPYBI,
6970
Timestamp: block.Timestamp,
7071
})
@@ -180,7 +181,7 @@ func (eng *DebtEngine) createTvlSnapshots(blockNum int64, caTotalValueInUSD floa
180181
if eng.lastTvlSnapshot != nil && blockNum-eng.lastTvlSnapshot.BlockNum < core.NoOfBlocksPerHr { // tvl snapshot every hour
181182
return
182183
}
183-
var totalAvailableLiquidityInUSD float64 = 0
184+
var totalAvailableLiquidityInUSD, expectedLiqInUSD float64 = 0, 0
184185
log.Info("tvl for block", blockNum)
185186
for _, entry := range eng.poolLastInterestData {
186187
adapter := eng.repo.GetAdapter(entry.Address)
@@ -203,11 +204,17 @@ func (eng *DebtEngine) createTvlSnapshots(blockNum int64, caTotalValueInUSD floa
203204
eng.GetAmountInUSD(
204205
underlyingToken,
205206
entry.AvailableLiquidityBI.Convert(), version), 8)
207+
expectedLiqInUSD += utils.GetFloat64Decimal(
208+
eng.GetAmountInUSD(
209+
underlyingToken,
210+
entry.ExpectedLiqBI.Convert(), version), 8)
211+
206212
}
207213
// save as last tvl snapshot and add to db
208214
tvls := &schemas.TvlSnapshots{
209215
BlockNum: blockNum,
210216
AvailableLiquidity: totalAvailableLiquidityInUSD,
217+
ExpectedLiq: expectedLiqInUSD,
211218
CATotalValue: caTotalValueInUSD,
212219
}
213220
eng.tvlSnapshots = append(eng.tvlSnapshots, tvls)

‎debts/index.go‎

Lines changed: 38 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -73,31 +73,32 @@ func (eng *DebtEngine) ProcessBackLogs() {
7373
return
7474
}
7575
// synced till
76-
lastDebtSynced := eng.repo.LoadLastDebtSync()
76+
lastSync := eng.repo.LoadLastDebtSync()
77+
minSynced := lastSync.Min()
7778
// lastDebtSynced = 227143579
78-
log.Info("Debt engine started, from", lastDebtSynced)
79+
log.Info("Debt engine started, from", minSynced)
7980
eng.loadLastTvlSnapshot()
80-
eng.loadLastCSS(lastDebtSynced)
81-
eng.loadLastRebaseDetails(lastDebtSynced)
82-
eng.loadTokenLastPrice(lastDebtSynced)
83-
eng.loadAllowedTokenThreshold(lastDebtSynced)
84-
eng.loadLastLTRamp(lastDebtSynced)
85-
eng.loadPoolLastInterestData(lastDebtSynced)
86-
eng.loadLastDebts(lastDebtSynced)
87-
eng.loadParameters(lastDebtSynced)
88-
eng.loadLiquidableAccounts(lastDebtSynced)
81+
eng.loadLastCSS(minSynced)
82+
eng.loadLastRebaseDetails(minSynced)
83+
eng.loadTokenLastPrice(minSynced)
84+
eng.loadAllowedTokenThreshold(minSynced)
85+
eng.loadLastLTRamp(minSynced)
86+
eng.loadPoolLastInterestData(minSynced)
87+
eng.loadLastDebts(minSynced)
88+
eng.loadParameters(minSynced)
89+
eng.loadLiquidableAccounts(minSynced)
8990
// v3
9091
// eng.loadAccounQuotaInfo(lastDebtSynced, eng.db)
91-
eng.loadPoolQuotaDetails(lastDebtSynced, eng.db)
92+
eng.loadPoolQuotaDetails(minSynced, eng.db)
9293
//
9394
// process blocks for calculating debts
9495
adaptersSyncedTill := eng.repo.LoadLastAdapterSync()
9596
// adaptersSyncedTill = 227143580
9697
batchSize := eng.config.BatchSizeForHistory
97-
for ; lastDebtSynced+batchSize < adaptersSyncedTill; lastDebtSynced += batchSize {
98-
eng.processBlocksInBatch(lastDebtSynced, lastDebtSynced+batchSize)
98+
for ; minSynced+batchSize < adaptersSyncedTill; minSynced += batchSize {
99+
eng.processBlocksInBatch(minSynced, minSynced+batchSize, lastSync)
99100
}
100-
eng.processBlocksInBatch(lastDebtSynced, adaptersSyncedTill)
101+
eng.processBlocksInBatch(minSynced, adaptersSyncedTill, lastSync)
101102
}
102103
func (eng *DebtEngine) loadLastTvlSnapshot() {
103104
lastTvlSnapshot := &schemas.TvlSnapshots{}
@@ -108,23 +109,34 @@ func (eng *DebtEngine) loadLastTvlSnapshot() {
108109
}
109110

110111
// load blocks from > and to <=
111-
func (eng *DebtEngine) processBlocksInBatch(from, to int64) {
112+
func (eng *DebtEngine) processBlocksInBatch(from, to int64, lastSync schemas.LastSync) {
112113
if from == to {
113114
return
114115
}
115116
eng.repo.LoadBlocks(from, to)
116117
if len(eng.repo.GetBlocks()) > 0 {
117-
eng.CalculateDebtAndClear(to)
118+
eng.CalculateDebtAndClear(to, lastSync)
118119
}
119120
}
120121

121122
// called for the engine/index.go and the debt engine
122-
func (eng *DebtEngine) CalculateDebtAndClear(to int64) {
123+
func (eng *DebtEngine) CalculateDebtAndClear(to int64, lastSync schemas.LastSync) {
123124
if !eng.config.DisableDebtEngine {
124125
eng.CalculateDebt()
125-
eng.flushDebt(to)
126-
eng.CalCurrentDebts(to)
127-
eng.flushCurrentDebts(to)
126+
//
127+
tx := eng.db.Begin()
128+
eng.flushDebt(to, tx, lastSync)
129+
eng.flushTvl(to, tx, lastSync)
130+
if info := tx.Commit(); info.Error != nil {
131+
log.Fatal(info.Error)
132+
}
133+
eng.tvlSnapshots = []*schemas.TvlSnapshots{}
134+
eng.debts = []*schemas.Debt{}
135+
//
136+
if to > lastSync.Debt {
137+
eng.CalCurrentDebts(to)
138+
eng.flushCurrentDebts(to)
139+
}
128140
}
129141
eng.Clear()
130142
}
@@ -168,14 +180,16 @@ func (eng *DebtEngine) notifiedIfLiquidable(sessionId string, notified bool) {
168180

169181
// QueryPriceFeed is updated only till the lastFetchedBlock, not the syncTill that is provided to the aqfwrapper's aftersynchook from engine/index.go in the syncmodel. So, ignore that for updating the debts.
170182
func (eng *DebtEngine) AreActiveAdapterSynchronized() bool {
171-
data := schemas.DebtSync{}
172-
query := `SELECT count(distinct last_sync) as last_calculated_at FROM sync_adapters
183+
data := struct {
184+
LastSync int64 `json:"last_sync"`
185+
}{}
186+
query := `SELECT count(distinct last_sync) as last_sync FROM sync_adapters
173187
WHERE disabled=false AND type NOT IN ('QueryPriceFeed','RebaseToken','Treasury','LMRewardsv2','LMRewardsv3','GearToken')`
174188
err := eng.db.Raw(query).Find(&data).Error
175189
if err != nil {
176190
log.Fatal(err)
177191
}
178-
val := data.LastCalculatedAt <= 1
192+
val := data.LastSync <= 1
179193
if !val {
180194
log.Warn("DebtEngine disabled active adapters are not synchronised")
181195
}

‎ds/debt.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ import (
1313
type DebtEngineI interface {
1414
Clear()
1515
ProcessBackLogs()
16-
CalculateDebtAndClear(to int64)
16+
CalculateDebtAndClear(to int64, lastSync schemas.LastSync)
1717
CalCurrentDebts(to int64)
1818
CalculateDebt()
1919
GetDebts() core.Json
@@ -34,7 +34,7 @@ type DebtProfile struct {
3434
Tokens map[string]TokenDetails `json:"tokens"`
3535
UnderlyingDecimals int8 `json:"underlyingDecimals"`
3636
*CumIndexAndUToken `json:"poolDetails"`
37-
CalcString string `json:"calcString"`
37+
CalcString string `json:"calcString"`
3838
}
3939

4040
type CumIndexAndUToken struct {

‎ds/repo.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -125,7 +125,7 @@ type RepositoryI interface {
125125
// oracle and uni
126126
GetRetryFeedForDebts() []QueryPriceFeedI
127127
//
128-
LoadLastDebtSync() int64
128+
LoadLastDebtSync() schemas.LastSync
129129
LoadLastAdapterSync() int64
130130
Clear()
131131
// multicall

‎ds/repo_dummy.go‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -225,8 +225,8 @@ func (DummyRepo) GetRetryFeedForDebts() []QueryPriceFeedI {
225225
func (DummyRepo) AddNewPriceOracleEvent(tokenOracle *schemas.TokenOracle, bounded bool, forChainlinkNewFeed ...bool) {
226226
}
227227

228-
func (DummyRepo) LoadLastDebtSync() int64 {
229-
return 0
228+
func (DummyRepo) LoadLastDebtSync() schemas.LastSync {
229+
return schemas.LastSync{}
230230
}
231231
func (DummyRepo) LoadLastAdapterSync() int64 {
232232
return 0

‎engine/index.go‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -141,7 +141,10 @@ func (e *Engine) syncLoop(syncedTill, latestBlockNum int64) int64 {
141141
func (e *Engine) SyncAndFlush(syncTill int64) {
142142
e.Sync(syncTill)
143143
e.repo.Flush(syncTill)
144-
e.debtEng.CalculateDebtAndClear(syncTill)
144+
e.debtEng.CalculateDebtAndClear(syncTill, schemas.LastSync{
145+
Debt: 0,
146+
Tvl: 0,
147+
})
145148
if syncTill > e.syncedBlock.Load().(int64) {
146149
e.syncedBlock.Store(syncTill)
147150
}

0 commit comments

Comments
 (0)