feat: poolCompressor, tvl calulation, AfterSyncHook on addrprovider

Author: harsh-98

debts/engine.go

@@ -167,7 +167,7 @@ func (eng *DebtEngine) CalculateDebt() {
 					log.Fatal("State for pool not found for address: ", pool)
 				}
 				//
-				market := state.(*schemas.PoolState).UnderlyingToken
+				market := state.(*schemas.PoolState).Market
 				marketToTvl.add(market, caValue, 0, 0)
 			}
 
@@ -212,7 +212,6 @@ func (eng *DebtEngine) createTvlSnapshots(blockNum int64, marketToTvl MarketToTv
 	// if eng.lastTvlSnapshot != nil && blockNum-eng.lastTvlSnapshot.BlockNum < core.NoOfBlocksPerHr { // tvl snapshot every hour
 	// 	return
 	// }
-	log.Info("tvl for block", blockNum)
 	for _, entry := range eng.poolLastInterestData {
 		adapter := eng.repo.GetAdapter(entry.Address)
 		state := adapter.GetUnderlyingState()
@@ -237,8 +236,10 @@ func (eng *DebtEngine) createTvlSnapshots(blockNum int64, marketToTvl MarketToTv
 		marketToTvl.add(state.(*schemas.PoolState).Market, 0, availLiq, expectedLiqInUSD)
 	}
 	// save as last tvl snapshot and add to db
+	addedMarket := []string{}
 	for market, details := range marketToTvl {
 		if lastTvlBlock, ok := eng.marketTolastTvlBlock[market]; ok && blockNum-lastTvlBlock < core.BlockPer(core.GetBaseChainId(eng.client), time.Hour) { // only snap her hr.
+			addedMarket = append(addedMarket, market)
 			continue
 		}
 		//
@@ -252,6 +253,9 @@ func (eng *DebtEngine) createTvlSnapshots(blockNum int64, marketToTvl MarketToTv
 		eng.tvlSnapshots = append(eng.tvlSnapshots, tvl)
 		eng.marketTolastTvlBlock[tvl.Market] = tvl.BlockNum
 	}
+	if len(addedMarket) > 0 {
+		log.Infof("%d:Tvl snapshot added for market %s", blockNum, addedMarket)
+	}
 }
 
 func (eng *DebtEngine) ifAccountLiquidated(sessionId, cmAddr string, closedAt int64, status int) {

ds/dc_wrapper/wrapper.go

@@ -14,8 +14,8 @@ import (
 	dcv2 "github.com/Gearbox-protocol/sdk-go/artifacts/dataCompressor/dataCompressorv2"
 	"github.com/Gearbox-protocol/sdk-go/artifacts/dataCompressor/mainnet"
 	dcv3 "github.com/Gearbox-protocol/sdk-go/artifacts/dataCompressorv3"
-	"github.com/Gearbox-protocol/sdk-go/artifacts/marketCompressor"
 	"github.com/Gearbox-protocol/sdk-go/artifacts/multicall"
+	"github.com/Gearbox-protocol/sdk-go/artifacts/poolCompressor"
 	"github.com/Gearbox-protocol/sdk-go/core"
 	"github.com/Gearbox-protocol/sdk-go/log"
 	"github.com/Gearbox-protocol/sdk-go/pkg/dc"
@@ -158,19 +158,19 @@ func (dcw *DataCompressorWrapper) addDataCompressorv300(version core.VersionType
 }
 
 func (dcw *DataCompressorWrapper) AddCompressorType(addr common.Address, cType CompressorType, discoveredAt int64) {
-	if !utils.Contains([]CompressorType{MARKET_COMPRESSOR, CREDIT_ACCOUNT_COMPRESSOR}, cType) {
+	if !utils.Contains([]CompressorType{MARKET_COMPRESSOR, CREDIT_ACCOUNT_COMPRESSOR, POOL_COMPRESSOR}, cType) {
 		log.Fatal("ctype is wrong, ", cType)
 	}
 	if len(dcw.compressorByBlock[cType]) > 0 {
 		last := dcw.compressorByBlock[cType][len(dcw.compressorByBlock[cType])-1]
 		if last.block > discoveredAt {
 			log.Fatalf("last %s has blocknum more than (%s) and new addr is %s", last, discoveredAt, addr)
 		}
-		dcw.compressorByBlock[cType] = append(dcw.compressorByBlock[cType], addressAndBlock{
-			address: addr,
-			block:   discoveredAt,
-		})
 	}
+	dcw.compressorByBlock[cType] = append(dcw.compressorByBlock[cType], addressAndBlock{
+		address: addr,
+		block:   discoveredAt,
+	})
 }
 
 func (dcw *DataCompressorWrapper) LoadMultipleDC(multiDCs interface{}) {
@@ -212,8 +212,8 @@ func (dcw *DataCompressorWrapper) LoadMultipleDC(multiDCs interface{}) {
 type CompressorType string
 
 const (
-	MARKET_COMPRESSOR CompressorType = "MARKET"
-	// CM_COMPRESSOR             CompressorType = "CM"
+	MARKET_COMPRESSOR         CompressorType = "MARKET"
+	POOL_COMPRESSOR           CompressorType = "POOL"
 	CREDIT_ACCOUNT_COMPRESSOR CompressorType = "ACCOUNT"
 )
 
@@ -399,15 +399,17 @@ func (dcw *DataCompressorWrapper) GetCreditManagerData(version core.VersionType,
 			if err != nil {
 				return dc.CMCallData{}, err
 			}
-			type debts struct {
+			debt := struct {
 				MinDebt *big.Int
 				MaxDebt *big.Int
-			}
-			cmData := *abi.ConvertType(out[0], new(debts)).(*debts)
+			}{}
+			debt.MinDebt = *abi.ConvertType(out[0], new(*big.Int)).(**big.Int)
+			debt.MaxDebt = *abi.ConvertType(out[1], new(*big.Int)).(**big.Int)
+			// cmData := *abi.ConvertType(out[0], new(debts)).(*debts)
 			return dc.CMCallData{
 				Addr:    _creditManager,
-				MinDebt: (*core.BigInt)(cmData.MinDebt),
-				MaxDebt: (*core.BigInt)(cmData.MaxDebt),
+				MinDebt: (*core.BigInt)(debt.MinDebt),
+				MaxDebt: (*core.BigInt)(debt.MaxDebt),
 			}, nil
 		case DCV3:
 			out, err := core.GetAbi("DataCompressorv3").Unpack("getCreditManagerData", bytes)
@@ -446,61 +448,85 @@ type PoolZapperInfo struct {
 }
 
 // zapper info from v300 onwards
-func (dcw *DataCompressorWrapper) GetZapperInfo(blockNum int64, poolAddr ...common.Address) (ans []PoolZapperInfo, found bool) {
-	key, compressor := dcw.GetKeyAndAddress(core.NewVersion(300), blockNum, MARKET_COMPRESSOR)
+func (dcw *DataCompressorWrapper) getZapperInfov3(blockNum int64, poolAddrs ...common.Address) ([]PoolZapperInfo, error) {
+	compressor, found := dcw.GetLatestv3DC()
+	if !found {
+		return nil, nil
+	}
 	opts := &bind.CallOpts{BlockNumber: big.NewInt(blockNum)}
-	switch key {
-	case DCV310:
-		marketConfigs := GetMarketConfigurators()
-		con, err := marketCompressor.NewMarketCompressor(compressor, dcw.client)
-		log.CheckFatal(err)
-		markets, err := con.GetMarkets(opts, marketCompressor.MarketFilter{Pools: poolAddr})
-		for _, market := range markets {
-			if !utils.Contains(marketConfigs, market.Configurator) {
-				continue
-			}
-			obj := &PoolZapperInfo{
-				Addr:        market.Pool.BaseParams.Addr,
-				Underlying:  market.Pool.Underlying,
-				DieselToken: market.Pool.BaseParams.Addr,
-			}
-			for _, zapper := range market.Zappers {
-				obj.Zappers = append(obj.Zappers, dcv3.ZapperInfo{
-					Zapper:   zapper.BaseParams.Addr,
-					TokenIn:  zapper.TokenIn.Addr,
-					TokenOut: zapper.TokenOut.Addr,
-				})
+	con, err := dcv3.NewDataCompressorv3(compressor, dcw.client)
+	log.CheckFatal(err)
+	poolList := []dcv3.PoolData{}
+	zapperInfo := []PoolZapperInfo{}
+	if len(poolAddrs) != 0 {
+		for _, addr := range poolAddrs {
+			data, err := con.GetPoolData(opts, addr)
+			if err != nil {
+				return nil, fmt.Errorf("pool ZapperInfo not found in dc %d: %v", blockNum, poolAddrs)
 			}
-			ans = append(ans, *obj)
+			poolList = append(poolList, data)
 		}
-		return ans, true
-	case DCV3:
-		con, err := dcv3.NewDataCompressorv3(compressor, dcw.client)
+	} else {
+		var err error
+		poolList, err = con.GetPoolsV3List(nil)
 		log.CheckFatal(err)
-		poolList := []dcv3.PoolData{}
-		if len(poolAddr) != 0 {
-			for _, addr := range poolAddr {
-				data, err := con.GetPoolData(opts, addr)
-				log.CheckFatal(err)
-				poolList = append(poolList, data)
-			}
-		} else {
-			var err error
-			poolList, err = con.GetPoolsV3List(nil)
-			log.CheckFatal(err)
+	}
+	for _, pool := range poolList {
+		obj := &PoolZapperInfo{
+			Addr:        pool.Addr,
+			Underlying:  pool.Underlying,
+			DieselToken: pool.DieselToken,
+			Zappers:     pool.Zappers,
 		}
-		for _, pool := range poolList {
-			obj := &PoolZapperInfo{
-				Addr:        pool.Addr,
-				Underlying:  pool.Underlying,
-				DieselToken: pool.DieselToken,
-				Zappers:     pool.Zappers,
-			}
-			ans = append(ans, *obj)
+		zapperInfo = append(zapperInfo, *obj)
+	}
+	return zapperInfo, nil
+}
+
+func (dcw *DataCompressorWrapper) GetZapperInfo(blockNum int64, poolAddrs ...common.Address) (ans []PoolZapperInfo) {
+	key, _ := dcw.GetKeyAndAddress(core.NewVersion(300), blockNum, MARKET_COMPRESSOR)
+	switch key {
+	case DCV310:
+		data, err := dcw.getZapperInfov3(blockNum, poolAddrs...)
+		if err != nil {
+			log.Warn(err)
 		}
+		return data
+		// marketConfigs := GetMarketConfigurators()
+		// con, err := marketCompressor.NewMarketCompressor(compressor, dcw.client)
+		// log.CheckFatal(err)
+		// poolAddrs = []common.Address{}
+		// markets, err := con.GetMarkets(opts, marketCompressor.MarketFilter{Pools: poolAddrs})
+		// log.CheckFatal(err)
+		// for _, market := range markets {
+		// 	log.Info(market.Configurator, market.Pool.BaseParams.Addr)
+		// 	if !utils.Contains(marketConfigs, market.Configurator) {
+		// 		continue
+		// 	}
+		// 	obj := &PoolZapperInfo{
+		// 		Addr:        market.Pool.BaseParams.Addr,
+		// 		Underlying:  market.Pool.Underlying,
+		// 		DieselToken: market.Pool.BaseParams.Addr,
+		// 	}
+		// 	for _, zapper := range market.Zappers {
+		// 		obj.Zappers = append(obj.Zappers, dcv3.ZapperInfo{
+		// 			Zapper:   zapper.BaseParams.Addr,
+		// 			TokenIn:  zapper.TokenIn.Addr,
+		// 			TokenOut: zapper.TokenOut.Addr,
+		// 		})
+		// 	}
+		// 	ans = append(ans, *obj)
+		// }
+		// log.Infof("%s: zapper info, v310: %d pooladdr:%v", log.DetectFuncAtStackN(2), len(data), poolAddrs)
+		// return ans
+	case DCV3:
+		data, notv3 := dcw.getZapperInfov3(blockNum, poolAddrs...)
+		log.CheckFatal(notv3)
+		log.Infof("%s: zapper info, v3:%d pooladdr:%v", log.DetectFuncAtStackN(2), len(data), poolAddrs)
+		return data
 	default:
 		// log.Fatal("No data compressor found for zapper info")
-		return nil, false
+		return nil
 	}
 }
 
@@ -510,12 +536,12 @@ func (dcw *DataCompressorWrapper) GetPoolData(version core.VersionType, blockNum
 	resultFn func([]byte) (dc.PoolCallData, error),
 	errReturn error) {
 	//
-	key, dcAddr := dcw.GetKeyAndAddress(version, blockNum, MARKET_COMPRESSOR)
+	key, dcAddr := dcw.GetKeyAndAddress(version, blockNum, POOL_COMPRESSOR)
 	switch key {
 	case NODC:
 		errReturn = NO_DC_FOUND_ERR
 	case DCV310:
-		data, err := core.GetAbi("MarketCompressor").Pack("getMarketData0", _pool)
+		data, err := core.GetAbi("PoolCompressor").Pack("getPoolState", _pool)
 		call, errReturn = multicall.Multicall2Call{
 			Target:   dcAddr,
 			CallData: data,
@@ -553,12 +579,12 @@ func (dcw *DataCompressorWrapper) GetPoolData(version core.VersionType, blockNum
 		case NODC:
 			log.Fatal("No data compressor found for pool data")
 		case DCV310:
-			out, err := core.GetAbi("MarketCompressor").Unpack("getMarketData0", bytes)
+			out, err := core.GetAbi("PoolCompressor").Unpack("getPoolState", bytes)
 			if err != nil {
 				return dc.PoolCallData{}, err
 			}
-			poolData := *abi.ConvertType(out[0], new(marketCompressor.MarketData)).(*marketCompressor.MarketData)
-			return dc.GetPoolDataFromDCCall(poolData.Pool)
+			poolData := *abi.ConvertType(out[0], new(poolCompressor.PoolState)).(*poolCompressor.PoolState)
+			return dc.GetPoolDataFromDCCall(poolData)
 		case DCV3:
 			out, err := core.GetAbi("DataCompressorv3").Unpack("getPoolData", bytes)
 			if err != nil {

ds/repo.go

@@ -2,6 +2,8 @@ package ds
 
 import (
 	"math/big"
+	"reflect"
+	"strconv"
 	"time"
 
 	"github.com/Gearbox-protocol/sdk-go/core"
@@ -154,3 +156,19 @@ func IsTestnet(client core.ClientI) bool {
 	chainid := core.GetChainId(client)
 	return log.GetNetworkName(chainid) != log.GetBaseNet(chainid)
 }
+
+func ToInt(i interface{}) int64 {
+	switch i := i.(type) {
+	case string:
+		x, err := strconv.ParseInt(i, 10, 64)
+		log.WrapErrWithLineN(err, 3)
+		return x
+	case float64:
+		return int64(i)
+	case int64:
+		return int64(i)
+	default:
+		log.Fatal("", reflect.TypeOf(i))
+		return 0
+	}
+}

go.mod

@@ -5,7 +5,7 @@ go 1.23
 // toolchain go1.23.2
 
 require (
-	github.com/Gearbox-protocol/sdk-go v0.0.0-20250402073851-44589b7f10df
+	github.com/Gearbox-protocol/sdk-go v0.0.0-20250403170805-67506ff30e20
 	github.com/ethereum/go-ethereum v1.13.14
 	github.com/go-playground/validator/v10 v10.4.1
 	github.com/google/go-cmp v0.6.0
@@ -84,6 +84,6 @@ require (
 
 replace github.com/btcsuite/btcd => github.com/btcsuite/btcd v0.22.1
 
-replace github.com/Gearbox-protocol/sdk-go v0.0.0-20250402070712-3f6d98a522ee => ../sdk-go
+replace github.com/Gearbox-protocol/sdk-go v0.0.0-20250402073851-44589b7f10df => ../sdk-go
 
 replace github.com/ethereum/go-ethereum v1.13.14 => github.com/OffchainLabs/go-ethereum v1.13.4-0.20240313010929-e5d8587e7227

go.sum

@@ -6,8 +6,8 @@ github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/DataDog/zstd v1.4.5 h1:EndNeuB0l9syBZhut0wns3gV1hL8zX8LIu6ZiVHWLIQ=
 github.com/DataDog/zstd v1.4.5/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo=
-github.com/Gearbox-protocol/sdk-go v0.0.0-20250402073851-44589b7f10df h1:5El7QxuRDMNLy0uAbjICMA1Mzax/hTL8ffLM7HuF7K0=
-github.com/Gearbox-protocol/sdk-go v0.0.0-20250402073851-44589b7f10df/go.mod h1:MZsiiSAwDdnHY6GEILYtHEqhb9wmbUUE34UBQgNlUyM=
+github.com/Gearbox-protocol/sdk-go v0.0.0-20250403170805-67506ff30e20 h1:PUfrZ/We3p4JByvN3cKJXZ1gzJOLpv7tf4rc71q1LBI=
+github.com/Gearbox-protocol/sdk-go v0.0.0-20250403170805-67506ff30e20/go.mod h1:MZsiiSAwDdnHY6GEILYtHEqhb9wmbUUE34UBQgNlUyM=
 github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
 github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/OffchainLabs/go-ethereum v1.13.4-0.20240313010929-e5d8587e7227 h1:+/3TrD+q+BP36jGj2Bycdmrc/joKLNbc5ImePQzKRLM=

jsonnet/sync_adapters/adapters.jsonnet

@@ -11,6 +11,9 @@
           '3': '#PriceOracle_1',
           '5': '#PriceOracle_2',
         },
+        all: {
+          '#AddressProvider_1': 10,
+        },
       },
       disabled: false,
       disabled_at: 0,