diff --git a/go.sum b/go.sum index 6195615bc5..8b380f0f67 100644 --- a/go.sum +++ b/go.sum @@ -82,12 +82,6 @@ github.com/c-bata/goptuna v0.8.1 h1:25+n1MLv0yvCsD56xv4nqIus3oLHL9GuPAZDLIqmX1U= github.com/c-bata/goptuna v0.8.1/go.mod h1:knmS8+Iyq5PPy1YUeIEq0pMFR4Y6x7z/CySc9HlZTCY= github.com/c9s/requestgen v1.3.6 h1:ul7dZ2uwGYjNBjreooRfSY10WTXvQmQSjZsHebz6QfE= github.com/c9s/requestgen v1.3.6/go.mod h1:QwkZudcv84kJ8g9+E0RDTj+13btFXbTvv2aI+zbuLbc= -github.com/c9s/rockhopper/v2 v2.0.0 h1:HZx8epjZpp+bt7wkl6eCTU47uy5q6BWki1wzBWUw17c= -github.com/c9s/rockhopper/v2 v2.0.0/go.mod h1:xJ+sHtboZP6Y7wBswLD+Qs4RXyi562skFiI24KcXDXM= -github.com/c9s/rockhopper/v2 v2.0.1 h1:GlqO5BJX1gA/jsP3OMpZwS4qLGm0CbpIjme+EkkvzxU= -github.com/c9s/rockhopper/v2 v2.0.1/go.mod h1:QltQJDqEGppqrAhgXCG98Efm38UVQ0RdnJNEhbnTqYI= -github.com/c9s/rockhopper/v2 v2.0.2 h1:ccOY4hHry2KbI8XPbFxy841tlCVYcYlhlfYwYGvTx94= -github.com/c9s/rockhopper/v2 v2.0.2/go.mod h1:QltQJDqEGppqrAhgXCG98Efm38UVQ0RdnJNEhbnTqYI= github.com/c9s/rockhopper/v2 v2.0.3-0.20240124055428-2473c6221858 h1:VLvl82XQCtH12axHdroyrfprPYKtA3TueEWicx8yT4A= github.com/c9s/rockhopper/v2 v2.0.3-0.20240124055428-2473c6221858/go.mod h1:QltQJDqEGppqrAhgXCG98Efm38UVQ0RdnJNEhbnTqYI= github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= @@ -504,12 +498,12 @@ github.com/nxadm/tail v1.4.8/go.mod h1:+ncqLTQzXmGhMZNUePPaPqPvBxHAIsmXswZKocGu+ github.com/oklog/ulid v1.3.1/go.mod h1:CirwcVhetQ6Lv90oh/F+FBtV6XMibvdAFo93nm5qn4U= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= +github.com/onsi/ginkgo v1.15.0 h1:1V1NfVQR87RtWAgp1lv9JZJ5Jap+XFGKPi00andXGi4= github.com/onsi/ginkgo v1.15.0/go.mod h1:hF8qUzuuC8DJGygJH3726JnCZX4MYbRB8yFfISqnKUg= -github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/onsi/gomega v1.10.5 h1:7n6FEkpFmfCoo2t+YYqXH0evK+a9ICQz0xcAy9dYcaQ= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= -github.com/onsi/gomega v1.19.0 h1:4ieX6qQjPP/BfC3mpsAtIGGlxTWPeA3Inl/7DtXw1tw= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI= diff --git a/pkg/service/backtest.go b/pkg/service/backtest.go index 5193e83b5e..c64d713930 100644 --- a/pkg/service/backtest.go +++ b/pkg/service/backtest.go @@ -144,7 +144,7 @@ func (s *BacktestService) QueryKLine( ) (*types.KLine, error) { log.Infof("querying last kline exchange = %s AND symbol = %s AND interval = %s", ex, symbol, interval) - tableName := targetKlineTable(ex) + tableName := s.targetKlineTable(ex, symbol) // make the SQL syntax IDE friendly, so that it can analyze it. sql := fmt.Sprintf("SELECT * FROM `%s` WHERE `symbol` = :symbol AND `interval` = :interval ORDER BY end_time "+orderBy+" LIMIT "+strconv.Itoa(limit), tableName) @@ -175,7 +175,7 @@ func (s *BacktestService) QueryKLine( func (s *BacktestService) QueryKLinesForward( exchange types.Exchange, symbol string, interval types.Interval, startTime time.Time, limit int, ) ([]types.KLine, error) { - tableName := targetKlineTable(exchange) + tableName := s.targetKlineTable(exchange, symbol) sql := "SELECT * FROM `binance_klines` WHERE `end_time` >= :start_time AND `symbol` = :symbol AND `interval` = :interval and exchange = :exchange ORDER BY end_time ASC LIMIT :limit" sql = strings.ReplaceAll(sql, "binance_klines", tableName) @@ -196,7 +196,7 @@ func (s *BacktestService) QueryKLinesForward( func (s *BacktestService) QueryKLinesBackward( exchange types.Exchange, symbol string, interval types.Interval, endTime time.Time, limit int, ) ([]types.KLine, error) { - tableName := targetKlineTable(exchange) + tableName := s.targetKlineTable(exchange, symbol) sql := "SELECT * FROM `binance_klines` WHERE `end_time` <= :end_time and exchange = :exchange AND `symbol` = :symbol AND `interval` = :interval ORDER BY end_time DESC LIMIT :limit" sql = strings.ReplaceAll(sql, "binance_klines", tableName) @@ -223,24 +223,21 @@ func (s *BacktestService) QueryKLinesCh( return returnError(errors.Errorf("symbols is empty when querying kline, plesae check your strategy setting. ")) } - tableName := targetKlineTable(exchange) - var query string + var queries []string + for _, symbol := range symbols { + queries = append(queries, fmt.Sprintf("SELECT * FROM `%s` WHERE `end_time` BETWEEN :since AND :until AND `symbol` = %s AND `interval` IN (:intervals)", s.targetKlineTable(exchange, symbol), symbol)) + } - // need to sort by start_time desc in order to let matching engine process 1m first - // otherwise any other close event could peek on the final close price - if len(symbols) == 1 { - query = "SELECT * FROM `binance_klines` WHERE `end_time` BETWEEN :since AND :until AND `symbol` = :symbols AND `interval` IN (:intervals) ORDER BY end_time ASC, start_time DESC" + var query string + if len(queries) == 1 { + query = queries[0] + " ORDER BY end_time ASC, start_time DESC" } else { - query = "SELECT * FROM `binance_klines` WHERE `end_time` BETWEEN :since AND :until AND `symbol` IN (:symbols) AND `interval` IN (:intervals) ORDER BY end_time ASC, start_time DESC" + query = "SELECT * FROM (" + strings.Join(queries, " UNION ALL ") + ") a ORDER BY end_time ASC, start_time DESC" } - query = strings.ReplaceAll(query, "binance_klines", tableName) - sql, args, err := sqlx.Named(query, map[string]interface{}{ "since": since, "until": until, - "symbol": symbols[0], - "symbols": symbols, "intervals": types.IntervalSlice(intervals), }) @@ -316,15 +313,20 @@ func (s *BacktestService) scanRows(rows *sqlx.Rows) (klines []types.KLine, err e return klines, rows.Err() } -func targetKlineTable(exchange types.Exchange) string { +func (s *BacktestService) targetKlineTable(exchange types.Exchange, symbol string) string { _, isFutures, _, _ := exchange2.GetSessionAttributes(exchange) - tableName := strings.ToLower(exchange.Name().String()) + tableName := strings.ToLower(exchange.Name().String()) + strings.ToLower(symbol) if isFutures { - return tableName + "_futures_klines" + tableName = tableName + "_futures_klines" } else { - return tableName + "_klines" + tableName = tableName + "_spot_klines" } + + // Create table on the fly + _, _ = s.DB.Exec(fmt.Sprintf("CREATE TABLE IF NOT EXISTS `%s` LIKE `klines`;", tableName)) + + return tableName } var errExchangeFieldIsUnset = errors.New("kline.Exchange field should not be empty") @@ -334,7 +336,7 @@ func (s *BacktestService) Insert(kline types.KLine, ex types.Exchange) error { return errExchangeFieldIsUnset } - tableName := targetKlineTable(ex) + tableName := s.targetKlineTable(ex, kline.Symbol) sql := fmt.Sprintf("INSERT INTO `%s` (`exchange`, `start_time`, `end_time`, `symbol`, `interval`, `open`, `high`, `low`, `close`, `closed`, `volume`, `quote_volume`, `taker_buy_base_volume`, `taker_buy_quote_volume`)"+ "VALUES (:exchange, :start_time, :end_time, :symbol, :interval, :open, :high, :low, :close, :closed, :volume, :quote_volume, :taker_buy_base_volume, :taker_buy_quote_volume)", tableName) @@ -343,13 +345,13 @@ func (s *BacktestService) Insert(kline types.KLine, ex types.Exchange) error { return err } -// BatchInsert Note: all kline should be same exchange, or it will cause issue. +// BatchInsert Note: all kline should be same exchange and same symbol, or it will cause issue. func (s *BacktestService) BatchInsert(kline []types.KLine, ex types.Exchange) error { if len(kline) == 0 { return nil } - tableName := targetKlineTable(ex) + tableName := s.targetKlineTable(ex, kline[0].Symbol) sql := fmt.Sprintf("INSERT INTO `%s` (`exchange`, `start_time`, `end_time`, `symbol`, `interval`, `open`, `high`, `low`, `close`, `closed`, `volume`, `quote_volume`, `taker_buy_base_volume`, `taker_buy_quote_volume`)"+ " VALUES (:exchange, :start_time, :end_time, :symbol, :interval, :open, :high, :low, :close, :closed, :volume, :quote_volume, :taker_buy_base_volume, :taker_buy_quote_volume); ", tableName) @@ -534,7 +536,7 @@ func (s *BacktestService) SelectKLineTimePoints( conditions = append(conditions, sq.Expr("`start_time` BETWEEN ? AND ?", since, until)) } - tableName := targetKlineTable(ex) + tableName := s.targetKlineTable(ex, symbol) return sq.Select("start_time"). From(tableName). @@ -560,7 +562,7 @@ func (s *BacktestService) SelectKLineTimeRange( conditions = append(conditions, sq.Expr("`start_time` BETWEEN ? AND ?", since, until)) } - tableName := targetKlineTable(ex) + tableName := s.targetKlineTable(ex, symbol) return sq.Select("MIN(start_time) AS t1, MAX(start_time) AS t2"). From(tableName). @@ -571,7 +573,7 @@ func (s *BacktestService) SelectKLineTimeRange( func (s *BacktestService) SelectLastKLines( ex types.Exchange, symbol string, interval types.Interval, startTime, endTime time.Time, limit uint64, ) sq.SelectBuilder { - tableName := targetKlineTable(ex) + tableName := s.targetKlineTable(ex, symbol) return sq.Select("*"). From(tableName). Where(sq.And{