finalise decoder

Author: go-dockly

pkg/datasource/csvsource/csv_tick_decoder.go

@@ -2,6 +2,10 @@ package csvsource
 
 import (
 	"encoding/csv"
+	"fmt"
+	"math/big"
+	"strconv"
+	"strings"
 
 	"github.com/c9s/bbgo/pkg/fixedpoint"
 	"github.com/c9s/bbgo/pkg/types"
@@ -23,14 +27,34 @@ func BinanceCSVTickDecoder(row []string, _ int) (*CsvTick, error) {
 	if len(row) < 7 {
 		return nil, ErrNotEnoughColumns
 	}
-	size := fixedpoint.MustNewFromString(row[2])
+	// example csv row
+	// id, price, qty, base_qty, time, is_buyer_maker,
+	// 11782578,6.00000000,1.00000000,14974844,14974844,1698623884463,True
+	qty := fixedpoint.MustNewFromString(row[2])
+	baseQty := fixedpoint.MustNewFromString(row[2])
 	price := fixedpoint.MustNewFromString(row[1])
-	hn := price.Mul(size)
+	// isBuyerMaker=false trade will qualify as BUY.
+	id, err := strconv.ParseUint(row[0], 10, 64)
+	if err != nil {
+		return nil, err
+	}
+	isBuyerMaker, err := strconv.ParseBool(row[5])
+	if err != nil {
+		return nil, err
+	}
+	side := types.SideTypeBuy
+	if isBuyerMaker {
+		side = types.SideTypeSell
+	}
 	return &CsvTick{
-		Timestamp:    types.MustParseMillisecondTimestamp(row[5]),
-		Size:         size,
-		Price:        price,
-		HomeNotional: hn,
+		TradeID:         id,
+		Exchange:        types.ExchangeBinance,
+		Side:            side,
+		Size:            qty,
+		Price:           price,
+		HomeNotional:    price.Mul(qty),
+		ForeignNotional: price.Mul(baseQty),
+		Timestamp:       types.MustParseMillisecondTimestamp(row[5]),
 	}, nil
 }
 
@@ -44,6 +68,9 @@ func NewBybitCSVTickReader(csv *csv.Reader) *CSVTickReader {
 
 // BybitCSVTickDecoder decodes a CSV record from Bybit into a CsvTick.
 func BybitCSVTickDecoder(row []string, index int) (*CsvTick, error) {
+	// example csv row
+	// timestamp,symbol,side,size,price,tickDirection,trdMatchID,grossValue,homeNotional,foreignNotional
+	// 1649054912,FXSUSDT,Buy,0.01,38.32,PlusTick,9c30abaf-80ae-5ebf-9850-58fe7ed4bac8,3.832e+07,0.01,0.3832
 	if len(row) < 9 {
 		return nil, ErrNotEnoughColumns
 	}
@@ -54,14 +81,33 @@ func BybitCSVTickDecoder(row []string, index int) (*CsvTick, error) {
 	if err != nil {
 		return nil, ErrInvalidOrderSideFormat
 	}
+	id, err := uuidStringToUInt(row[6])
+	if err != nil {
+		return nil, ErrInvalidOrderSideFormat
+	}
 	return &CsvTick{
-		Timestamp:       types.MustParseMillisecondTimestamp(row[0]),
+		TradeID:         id,
 		Symbol:          row[1],
+		Exchange:        types.ExchangeBybit,
 		Side:            side,
-		TickDirection:   row[5],
 		Size:            fixedpoint.MustNewFromString(row[3]),
 		Price:           fixedpoint.MustNewFromString(row[4]),
 		HomeNotional:    fixedpoint.MustNewFromString(row[8]),
 		ForeignNotional: fixedpoint.MustNewFromString(row[9]),
+		TickDirection:   row[5], // todo does this seem promising to define for other exchanges too?
+		Timestamp:       types.MustParseMillisecondTimestamp(row[0]),
 	}, nil
 }
+
+func uuidStringToUInt(uuidStr string) (uint64, error) {
+	// Remove hyphens from the UUID string
+	uuidStr = strings.Replace(uuidStr, "-", "", -1)
+
+	// Parse the hexadecimal string into a big integer
+	uuidBigInt, success := new(big.Int).SetString(uuidStr, 16)
+	if !success {
+		return 0, fmt.Errorf("Failed to parse UUID as a big integer")
+	}
+
+	return uuidBigInt.Uint64(), nil
+}

pkg/datasource/csvsource/stream.go

@@ -69,6 +69,7 @@ func (s *Stream) simulateEvents() error {
 		if err != nil {
 			return err
 		}
+		tick.Symbol = s.config.Symbol // not every csv provides symbol information
 		trade, err := tick.toGlobalTrade()
 		if err != nil {
 			return err

pkg/datasource/csvsource/types.go

@@ -6,6 +6,8 @@ import (
 )
 
 type CsvTick struct {
+	Exchange        types.ExchangeName         `json:"exchange"`
+	TradeID         uint64                     `json:"tradeID"`
 	Symbol          string                     `json:"symbol"`
 	TickDirection   string                     `json:"tickDirection"`
 	Side            types.SideType             `json:"side"`
@@ -19,21 +21,21 @@ type CsvTick struct {
 // todo
 func (c *CsvTick) toGlobalTrade() (*types.Trade, error) {
 	return &types.Trade{
-		// ID:            tradeIdNum,
-		// OrderID:       orderIdNum,
-		// Exchange:      types.ExchangeBybit,
-		// Price:         trade.OrderPrice,
-		// Quantity:      trade.OrderQty,
-		// QuoteQuantity: trade.OrderPrice.Mul(trade.OrderQty),
-		// Symbol:        trade.Symbol,
-		// Side:          side,
-		// IsBuyer:       side == types.SideTypeBuy,
-		// IsMaker:       isMaker,
-		// Time:          types.Time(trade.ExecutionTime),
-		// Fee:           trade.ExecFee,
+		ID: c.TradeID,
+		// OrderID:    // not implemented
+		Exchange:      c.Exchange,
+		Price:         c.Price,
+		Quantity:      c.Size,
+		QuoteQuantity: c.Price.Mul(c.Size), // todo this does not seem right use of propert.. looses info on foreign notional
+		Symbol:        c.Symbol,
+		Side:          c.Side,
+		IsBuyer:       c.Side == types.SideTypeBuy,
+		// IsMaker:       isMaker, // todo property isBuyer and isMaker seem to get confused and duplicated
+		Time: types.Time(c.Timestamp),
+		// Fee:           trade.ExecFee, // todo how to get this info?
 		// FeeCurrency:   trade.FeeTokenId,
-		// IsMargin:      false,
-		// IsFutures:     false,
-		// IsIsolated:    false,
+		IsMargin:   false,
+		IsFutures:  false, // todo make future dataset source type as config
+		IsIsolated: false,
 	}, nil
 }