V310

db_scripts/local_testing/anvil_test.sh

@@ -13,7 +13,7 @@ TMP_DB="tmp_$FINAL_DB"
 export TMP_DB_URL="postgres://$SUPERUSER@localhost:5432/$TMP_DB?sslmode=disable"
 
 set +e
-psql -U $SUPERUSER -d postgres -c " SELECT  pg_terminate_backend(pid) FROM  pg_stat_activity WHERE  pid <> pg_backend_pid() AND datname = '$TMP_DB';"
+psql -U $SUPERUSER -d postgres -c "SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE pid <> pg_backend_pid() AND datname = '""$TMP_DB""';"
 psql -U $SUPERUSER -d postgres -c "drop database $TMP_DB"
 psql -U $SUPERUSER -d postgres -c "create database $TMP_DB"
 pg_dump --no-owner "$REMOTE_DB" | psql  -U $SUPERUSER -d $TMP_DB
@@ -23,7 +23,7 @@ pg_dump "$REMOTE_DB" --table public.schema_migrations  | psql  -U $SUPERUSER -d
 set -e
 
 # psql -U $SUPERUSER -d sample < db_scripts/local_testing/missing_table_from_download_db.sql
-psql -U $SUPERUSER -d $TMP_DB < $PARENT_DIR/../../migrations/000016_rankings.up.sql
+# psql -U $SUPERUSER -d $TMP_DB < $PARENT_DIR/../../migrations/000016_rankings.up.sql
 migrate -path $PARENT_DIR/../../migrations/ -database "$TMP_DB_URL" up
 
 
@@ -36,7 +36,7 @@ set -e
 PWD=`pwd`
 LOCAL_DB="host=localhost user=$SUPERUSER  dbname=$TMP_DB"
 cd /home/debian/$FINAL_DB-third-eye
-go run "scripts/08_merged_pf_version_reset/main.go" "$LOCAL_DB" $FORK_BLOCK
+# go run "scripts/08_merged_pf_version_reset/main.go" "$LOCAL_DB" $FORK_BLOCK
 cd $PWD
 
 createdb -O $SUPERUSER -T $TMP_DB $FINAL_DB

debts/calc_details.go

@@ -51,11 +51,14 @@ type storeForCalc struct {
 	inner *DebtEngine
 }
 
-func (s storeForCalc) GetToken(token string) *schemas.Token {
-	return s.inner.repo.GetToken(token)
+func (s storeForCalc) GetToken(token string) (*schemas.Token, error) {
+	return s.inner.repo.GetToken(token), nil
 }
-func (s storeForCalc) GetPrices(token string, version schemas.PFVersion, blockNums ...int64) *big.Int {
-	return s.inner.GetTokenLastPrice(token, version)
+func (s storeForCalc) GetDecimals(token string) int8 {
+	return s.inner.repo.GetToken(token).Decimals
+}
+func (s storeForCalc) GetPriceOnBlock(cm, token string, version core.VersionType, blockNums ...int64) *big.Int {
+	return s.inner.priceHandler.GetLastPrice(cm, token, version)
 }
 
 func (s storeForCalc) GetLiqThreshold(ts uint64, cm, token string) *big.Int {

debts/debt_db.go

@@ -109,6 +109,9 @@ func (eng *DebtEngine) AddDebt(debt *schemas.Debt, forceAdd bool) {
 			core.DiffMoreThanFraction(lastDebt.CalDebtBI, debt.CalDebtBI, big.NewFloat(0.05)) ||
 			// add debt when the health factor is on different side of 10000 from the lastdebt
 			core.ValueDifferSideOf10000(debt.CalHealthFactor, lastDebt.CalHealthFactor) {
+			if lastDebt.CalHealthFactor.Cmp(debt.CalDebtBI) == 0 && lastDebt.CalTotalValueBI.Cmp(debt.CalTotalValueBI) == 0 {
+				return
+			}
 			eng.addDebt(debt)
 		}
 	} else {

debts/engine.go

@@ -1,7 +1,6 @@
 package debts
 
 import (
-	"fmt"
 	"math/big"
 	"sort"
 	"time"
@@ -93,11 +92,30 @@ func (eng *DebtEngine) updateLocalState(blockNum int64, block *schemas.Block) (p
 		eng.AddTokenLTRamp(ltRamp)
 	}
 
+	for _, to := range block.TokenOracles {
+		if !to.Reserve { // only main
+			eng.priceHandler.AddTokenOracle(to)
+		}
+	}
+	//
+	for _, relation := range block.Relations {
+		if relation.Type == "PoolOracle" {
+			eng.priceHandler.poolToPriceOracle[relation.Owner] = schemas.PriceOracleT(relation.Dependent)
+		}
+	}
 	// C3.b: updated price
 	for _, pf := range block.GetPriceFeeds() {
 		// L5
-		eng.AddTokenLastPrice(pf)
-		tokensUpdated[pf.Token] = true
+		eng.priceHandler.AddTokenLastPrice(pf)
+		// TIMECOMPLEX
+		// use mdl.repo.getsyncadapter(pf.Feed).gettokens(block_num)
+		for _, oracles := range eng.priceHandler.poTotokenOracle {
+			for _, oracle := range oracles {
+				if pf.Feed == oracle.Feed {
+					tokensUpdated[oracle.Token] = true
+				}
+			}
+		}
 	}
 	// updates complete
 	return poolsUpdated, tokensUpdated, sessionsUpdated
@@ -123,7 +141,7 @@ func (eng *DebtEngine) CalculateDebt() {
 		cmToPoolDetails := eng.GetCumulativeIndexAndDecimalForCMs(blockNum, block.Timestamp)
 
 		//
-		var caTotalValueInUSD float64 = 0
+		marketToTvl := make(MarketToTvl)
 		// check if session's debt needs to be recalculated
 		for _, session := range sessions {
 			if (session.ClosedAt != 0 && session.ClosedAt <= blockNum) || session.Since > blockNum {
@@ -135,11 +153,23 @@ func (eng *DebtEngine) CalculateDebt() {
 			}
 			// #C3
 			sessionSnapshot := eng.lastCSS[session.ID]
-			caTotalValueInUSD += utils.GetFloat64Decimal(
+			caValue := utils.GetFloat64Decimal(
 				eng.GetAmountInUSD(
+					session.CreditManager,
 					cmToPoolDetails[session.CreditManager].Token,
-					sessionSnapshot.TotalValueBI.Convert(), schemas.VersionToPFVersion(session.Version, false),
+					sessionSnapshot.TotalValueBI.Convert(), session.Version,
 				), 8)
+			{
+				pool := eng.priceHandler.GetPoolFromCM(session.CreditManager)
+				adapter := eng.repo.GetAdapter(pool)
+				state := adapter.GetUnderlyingState()
+				if state == nil {
+					log.Fatal("State for pool not found for address: ", pool)
+				}
+				//
+				market := state.(*schemas.PoolState).Market
+				marketToTvl.add(market, caValue, 0, 0)
+			}
 
 			for token, tokenBalance := range *sessionSnapshot.Balances {
 				if tokenBalance.IsEnabled && tokenBalance.HasBalanceMoreThanOne() {
@@ -171,19 +201,17 @@ func (eng *DebtEngine) CalculateDebt() {
 			}
 		}
 		//
-		eng.createTvlSnapshots(blockNum, caTotalValueInUSD)
+		eng.createTvlSnapshots(blockNum, marketToTvl)
 		if len(sessionsUpdated) > 0 {
 			log.Debugf("Calculated %d debts for block %d", len(sessionsUpdated), blockNum)
 		}
 	}
 }
 
-func (eng *DebtEngine) createTvlSnapshots(blockNum int64, caTotalValueInUSD float64) {
-	if eng.lastTvlSnapshot != nil && blockNum-eng.lastTvlSnapshot.BlockNum < core.BlockPer(core.GetChainId(eng.client), time.Hour) { // tvl snapshot every hour
-		return
-	}
-	var totalAvailableLiquidityInUSD, expectedLiqInUSD float64 = 0, 0
-	log.Info("tvl for block", blockNum)
+func (eng *DebtEngine) createTvlSnapshots(blockNum int64, marketToTvl MarketToTvl) {
+	// if eng.lastTvlSnapshot != nil && blockNum-eng.lastTvlSnapshot.BlockNum < core.NoOfBlocksPerHr { // tvl snapshot every hour
+	// 	return
+	// }
 	for _, entry := range eng.poolLastInterestData {
 		adapter := eng.repo.GetAdapter(entry.Address)
 		state := adapter.GetUnderlyingState()
@@ -193,33 +221,41 @@ func (eng *DebtEngine) createTvlSnapshots(blockNum int64, caTotalValueInUSD floa
 		//
 		underlyingToken := state.(*schemas.PoolState).UnderlyingToken
 		//
-		version := schemas.V1PF
-		if eng.tokenLastPrice[schemas.V2PF] != nil && eng.tokenLastPrice[schemas.V2PF][underlyingToken] != nil {
-			version = schemas.V2PF
+		latestOracle, version, err := eng.repo.GetActivePriceOracleByBlockNum(blockNum)
+		log.CheckFatal(err)
+		//
+		fn := func(amount *core.BigInt) float64 {
+			return utils.GetFloat64Decimal(
+				eng.GetAmountInUSDByOracle(
+					latestOracle,
+					underlyingToken,
+					amount.Convert(), version), 8)
 		}
-		if eng.tokenLastPrice[schemas.V3PF_MAIN] != nil && eng.tokenLastPrice[schemas.V3PF_MAIN][underlyingToken] != nil {
-			version = schemas.V3PF_MAIN
+		availLiq := fn(entry.AvailableLiquidityBI)
+		expectedLiqInUSD := fn(entry.ExpectedLiqBI)
+		marketToTvl.add(state.(*schemas.PoolState).Market, 0, availLiq, expectedLiqInUSD)
+	}
+	// save as last tvl snapshot and add to db
+	addedMarket := []string{}
+	for market, details := range marketToTvl {
+		if lastTvlBlock, ok := eng.marketTolastTvlBlock[market]; ok && blockNum-lastTvlBlock < core.BlockPer(core.GetBaseChainId(eng.client), time.Hour) { // only snap her hr.
+			continue
 		}
 		//
-		totalAvailableLiquidityInUSD += utils.GetFloat64Decimal(
-			eng.GetAmountInUSD(
-				underlyingToken,
-				entry.AvailableLiquidityBI.Convert(), version), 8)
-		expectedLiqInUSD += utils.GetFloat64Decimal(
-			eng.GetAmountInUSD(
-				underlyingToken,
-				entry.ExpectedLiqBI.Convert(), version), 8)
-
+		tvl := &schemas.TvlSnapshots{
+			BlockNum:           blockNum,
+			AvailableLiquidity: details.totalAvailableLiquidity,
+			CATotalValue:       details.caTotalValue,
+			ExpectedLiq:        details.expectedLiq,
+			Market:             market,
+		}
+		addedMarket = append(addedMarket, market)
+		eng.tvlSnapshots = append(eng.tvlSnapshots, tvl)
+		eng.marketTolastTvlBlock[tvl.Market] = tvl.BlockNum
 	}
-	// save as last tvl snapshot and add to db
-	tvls := &schemas.TvlSnapshots{
-		BlockNum:           blockNum,
-		AvailableLiquidity: totalAvailableLiquidityInUSD,
-		ExpectedLiq:        expectedLiqInUSD,
-		CATotalValue:       caTotalValueInUSD,
+	if len(addedMarket) > 0 {
+		log.Infof("%d:Tvl snapshot added for market %s", blockNum, addedMarket)
 	}
-	eng.tvlSnapshots = append(eng.tvlSnapshots, tvls)
-	eng.lastTvlSnapshot = tvls
 }
 
 func (eng *DebtEngine) ifAccountLiquidated(sessionId, cmAddr string, closedAt int64, status int) {
@@ -291,10 +327,6 @@ func (eng *DebtEngine) GetCumulativeIndexAndDecimalForCMs(blockNum int64, ts uin
 	return cmToCumIndex
 }
 
-func (eng *DebtEngine) getTokenPriceFeed(token string, pfVersion schemas.PFVersion) *schemas.PriceFeed {
-	return eng.tokenLastPrice[pfVersion][token]
-}
-
 func (eng *DebtEngine) SessionDebtHandler(blockNum int64, session *schemas.CreditSession, cumIndexAndUToken *ds.CumIndexAndUToken) {
 	sessionId := session.ID
 	sessionSnapshot := eng.lastCSS[sessionId]
@@ -307,14 +339,16 @@ func (eng *DebtEngine) SessionDebtHandler(blockNum int64, session *schemas.Credi
 		retryFeeds := eng.repo.GetRetryFeedForDebts()
 		for tokenAddr, details := range *sessionSnapshot.Balances {
 			if details.IsEnabled && details.HasBalanceMoreThanOne() {
-				lastPriceEvent := eng.getTokenPriceFeed(tokenAddr, schemas.VersionToPFVersion(session.Version, false)) // don't use reserve
 				//
-				if tokenAddr != eng.repo.GetWETHAddr() && lastPriceEvent.BlockNumber != blockNum {
-					feedAddr := lastPriceEvent.Feed
-					for _, retryFeed := range retryFeeds {
-						if retryFeed.GetAddress() == feedAddr {
-							eng.requestPriceFeed(blockNum, retryFeed, tokenAddr, lastPriceEvent.MergedPFVersion)
-
+				if tokenAddr != eng.repo.GetWETHAddr() { // REDUNDANT
+					lastPriceEvent := eng.priceHandler.GetLastPriceFeed(session.CreditManager, tokenAddr, session.Version) // don't use reserve
+					if lastPriceEvent.BlockNumber != blockNum {
+						feedAddr := lastPriceEvent.Feed
+						for _, retryFeed := range retryFeeds {
+							if retryFeed.GetAddress() == feedAddr {
+								// log.Info("hf ", debt.CalHealthFactor.Convert(), "of", sessionId, "at", blockNum)
+								eng.priceHandler.requestPriceFeed(blockNum, eng.client, retryFeed, tokenAddr, profile != nil)
+							}
 						}
 					}
 				}
@@ -400,7 +434,7 @@ func (eng *DebtEngine) CalculateSessionDebt(blockNum int64, session *schemas.Cre
 	for tokenAddr, details := range *sessionSnapshot.Balances {
 		if details.IsEnabled {
 			profile.Tokens[tokenAddr] = ds.TokenDetails{
-				Price:             eng.GetTokenLastPrice(tokenAddr, schemas.VersionToPFVersion(session.Version, false)), // don't use reserve
+				Price:             eng.priceHandler.GetLastPrice(session.CreditManager, tokenAddr, session.Version), // don't use reserve
 				Decimals:          eng.repo.GetToken(tokenAddr).Decimals,
 				TokenLiqThreshold: eng.tokenLTRamp[session.CreditManager][tokenAddr].GetLTForTs(eng.currentTs),
 			}
@@ -424,12 +458,15 @@ func (eng *DebtEngine) CalculateSessionDebt(blockNum int64, session *schemas.Cre
 			notMatched = true
 		}
 		// even if data compressor matching is disabled check the calc values  with session data at block where last credit snapshot was taken
-	} else if sessionSnapshot.BlockNum == blockNum && sessionSnapshot.HealthFactor.Convert().Cmp(new(big.Int)) != 0 { // it is 0 when the issuccessful is false for redstone credit accounts
+		//  // 20563217 and 0xe8f5F52842D7AF4BbcF5Fe731A336147B51F09D5_19980779_297 on mainnet has creditsessionsnapshot but isSuccessful for dv3 is false.
+	} else if sessionSnapshot.BlockNum == blockNum && sessionSnapshot.HealthFactor.Convert().Cmp(new(big.Int)) != 0 &&
+		!(utils.Contains([]int64{20563217, 20671148}, blockNum) && sessionSnapshot.TotalValueBI.Convert().Cmp(new(big.Int)) == 0) {
+		// it is 0 when the issuccessful is false for redstone credit accounts
 		if IsChangeMoreThanFraction(debt.CalTotalValueBI, sessionSnapshot.TotalValueBI, big.NewFloat(0.001)) ||
 			// hf value calculated are on different side of 1
 			core.ValueDifferSideOf10000(debt.CalHealthFactor, sessionSnapshot.HealthFactor) ||
 			// if healhFactor diff by 4 %
-			core.DiffMoreThanFraction(debt.CalHealthFactor, sessionSnapshot.HealthFactor, big.NewFloat(0.04)) {
+			core.DiffMoreThanFraction(debt.CalHealthFactor, sessionSnapshot.HealthFactor, big.NewFloat(0.01)) {
 			// log.Info(debt.CalHealthFactor, sessionSnapshot.HealthFactor, blockNum)
 			// log.Info(debt.CalTotalValueBI, sessionSnapshot.TotalValueBI, blockNum)
 			if log.GetBaseNet(core.GetChainId(eng.client)) == "ARBITRUM" {
@@ -446,11 +483,12 @@ func (eng *DebtEngine) CalculateSessionDebt(blockNum int64, session *schemas.Cre
 				notMatched = true
 			}
 			log.Info(
+				debt.CalTotalValueBI, sessionSnapshot.TotalValueBI, sessionId,
 				IsChangeMoreThanFraction(debt.CalTotalValueBI, sessionSnapshot.TotalValueBI, big.NewFloat(0.001)),
 				// hf value calculated are on different side of 1
 				core.ValueDifferSideOf10000(debt.CalHealthFactor, sessionSnapshot.HealthFactor),
 				// if healhFactor diff by 4 %
-				core.DiffMoreThanFraction(debt.CalHealthFactor, sessionSnapshot.HealthFactor, big.NewFloat(0.04)),
+				core.DiffMoreThanFraction(debt.CalHealthFactor, sessionSnapshot.HealthFactor, big.NewFloat(0.01)),
 			)
 		}
 	}
@@ -467,15 +505,31 @@ func (eng *DebtEngine) CalculateSessionDebt(blockNum int64, session *schemas.Cre
 }
 
 // helper methods
-func (eng *DebtEngine) GetAmountInUSD(tokenAddr string, amount *big.Int, pfVersion schemas.PFVersion) *big.Int {
-	tokenPrice := eng.GetTokenLastPrice(tokenAddr, pfVersion)
+func (eng *DebtEngine) GetAmountInUSDByOracle(priceOracle schemas.PriceOracleT, tokenAddr string, amount *big.Int, version core.VersionType) *big.Int {
+	tokenPrice := eng.priceHandler.GetLastPriceFeedByOracle(priceOracle, tokenAddr, version)
+	tokenDecimals := eng.repo.GetToken(tokenAddr).Decimals
+	if version.MoreThan(core.NewVersion(1)) { // than decimals 8
+		return utils.GetInt64(new(big.Int).Mul(tokenPrice.PriceBI.Convert(), amount), tokenDecimals)
+	}
+	// for v1
+	usdcAddr := eng.repo.GetUSDCAddr()
+	usdcPrice := eng.priceHandler.GetLastPriceFeedByOracle(priceOracle, usdcAddr, version)
+	usdcDecimals := eng.repo.GetToken(usdcAddr).Decimals
+
+	value := new(big.Int).Mul(amount, tokenPrice.PriceBI.Convert())
+	value = utils.GetInt64(value, tokenDecimals-usdcDecimals)
+	value = new(big.Int).Quo(value, usdcPrice.PriceBI.Convert())
+	return new(big.Int).Mul(value, big.NewInt(100))
+}
+func (eng *DebtEngine) GetAmountInUSD(cm string, tokenAddr string, amount *big.Int, version core.VersionType) *big.Int {
+	tokenPrice := eng.priceHandler.GetLastPrice(cm, tokenAddr, version)
 	tokenDecimals := eng.repo.GetToken(tokenAddr).Decimals
-	if pfVersion.Decimals() == 8 {
+	if version.MoreThan(core.NewVersion(1)) { // than decimals 8
 		return utils.GetInt64(new(big.Int).Mul(tokenPrice, amount), tokenDecimals)
 	}
 	// for v1
 	usdcAddr := eng.repo.GetUSDCAddr()
-	usdcPrice := eng.GetTokenLastPrice(usdcAddr, pfVersion)
+	usdcPrice := eng.priceHandler.GetLastPrice(cm, usdcAddr, version)
 	usdcDecimals := eng.repo.GetToken(usdcAddr).Decimals
 
 	value := new(big.Int).Mul(amount, tokenPrice)
@@ -484,21 +538,6 @@ func (eng *DebtEngine) GetAmountInUSD(tokenAddr string, amount *big.Int, pfVersi
 	return new(big.Int).Mul(value, big.NewInt(100))
 }
 
-func (eng *DebtEngine) GetTokenLastPrice(addr string, pfVersion schemas.PFVersion, dontFail ...bool) *big.Int {
-	if pfVersion.ToVersion().Eq(1) && eng.repo.GetWETHAddr() == addr {
-		return core.WETHPrice
-	}
-	if eng.tokenLastPrice[pfVersion][addr] != nil {
-		return eng.tokenLastPrice[pfVersion][addr].PriceBI.Convert()
-	}
-	//
-	if len(dontFail) > 0 && dontFail[0] {
-		return nil
-	}
-	log.Fatal(fmt.Sprintf("Price not found for %s pfversion: %d", addr, pfVersion))
-	return nil
-}
-
 func (eng *DebtEngine) SessionDataFromDC(version core.VersionType, blockNum int64, cmAddr, borrower, account string) dc.CreditAccountCallData {
 	call, resultFn, err := eng.repo.GetDCWrapper().GetCreditAccountData(version, blockNum,
 		common.HexToAddress(cmAddr),
@@ -518,28 +557,3 @@ func (eng *DebtEngine) SessionDataFromDC(version core.VersionType, blockNum int6
 	}
 	return data
 }
-
-func (eng *DebtEngine) requestPriceFeed(blockNum int64, retryFeed ds.QueryPriceFeedI, token string, pfVersion schemas.MergedPFVersion) {
-	// defer func() {
-	// 	if err:= recover(); err != nil {
-	// 		log.Warn("err", err, "in getting yearn price feed in debt", feed, token, blockNum, pfVersion)
-	// 	}
-	// }()
-	// PFFIX
-	calls, isQueryable := retryFeed.GetCalls(blockNum)
-	if !isQueryable {
-		return
-	}
-	log.Info("getting price for ", token, "at", blockNum)
-	results := core.MakeMultiCall(eng.client, blockNum, false, calls)
-	price := retryFeed.ProcessResult(blockNum, results, true)
-	eng.AddTokenLastPrice(&schemas.PriceFeed{
-		BlockNumber:     blockNum,
-		Token:           token,
-		Feed:            retryFeed.GetAddress(),
-		RoundId:         price.RoundId,
-		PriceBI:         price.PriceBI,
-		Price:           price.Price,
-		MergedPFVersion: pfVersion,
-	})
-}