Merge pull request #1442 from bnb-chain/develop

release: v1.10.1

Author: annielz

CHANGELOG.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## v1.10.1
+FEATURES
+* [#1440](https://github.com/bnb-chain/greenfield-storage-provider/pull/1440) feat: add index to permission table and add piece store no such key error as 404 code
+
 ## v1.10.0
 BUGFIXES
 * [#1435](https://github.com/bnb-chain/greenfield-storage-provider/pull/1435) fix: gc objects

go.mod

@@ -286,7 +286,7 @@ require (
 	google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect
 	google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect
-	google.golang.org/protobuf v1.33.0 // indirect
+	google.golang.org/protobuf v1.33.0
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -308,7 +308,7 @@ replace (
 	github.com/confio/ics23/go => github.com/cosmos/cosmos-sdk/ics23/go v0.8.0
 	github.com/cosmos/cosmos-sdk => github.com/bnb-chain/greenfield-cosmos-sdk v1.9.0
 	github.com/cosmos/iavl => github.com/bnb-chain/greenfield-iavl v0.20.1
-	github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20240604033531-028f2cc8f76d
+	github.com/forbole/juno/v4 => github.com/bnb-chain/juno/v4 v4.0.0-20241213040233-b35dbdd3eccc
 	github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1
 	github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7
 )

go.sum

@@ -192,8 +192,8 @@ github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965b
 github.com/bnb-chain/greenfield-cosmos-sdk/math v0.0.0-20231206043955-0855e0965bc8/go.mod h1:An0MllWJY6PxibUpnwGk8jOm+a/qIxlKmL5Zyp9NnaM=
 github.com/bnb-chain/greenfield-iavl v0.20.1 h1:y3L64GU99otNp27/xLVBTDbv4eroR6CzoYz0rbaVotM=
 github.com/bnb-chain/greenfield-iavl v0.20.1/go.mod h1:oLksTs8dfh7DYIKBro7hbRQ+ewls7ghJ27pIXlbEXyI=
-github.com/bnb-chain/juno/v4 v4.0.0-20240604033531-028f2cc8f76d h1:a5Q944ZvZK4K3lDkbIqweZd1CqEE8gmxph+wpTQzE3k=
-github.com/bnb-chain/juno/v4 v4.0.0-20240604033531-028f2cc8f76d/go.mod h1:p+KkPIHURpqUJOdXanhhYgZpTLQxGZWkrAwtUaOuPlM=
+github.com/bnb-chain/juno/v4 v4.0.0-20241213040233-b35dbdd3eccc h1:+dDQk5ayB51Qgw5hZOzAkqJTyX6PDQFEj3aDEWmCLss=
+github.com/bnb-chain/juno/v4 v4.0.0-20241213040233-b35dbdd3eccc/go.mod h1:p+KkPIHURpqUJOdXanhhYgZpTLQxGZWkrAwtUaOuPlM=
 github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps=
 github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g=
 github.com/bradfitz/gomemcache v0.0.0-20170208213004-1952afaa557d/go.mod h1:PmM6Mmwb0LSuEubjR8N7PtNe1KxZLtOUHtbeikc5h60=

modular/blocksyncer/modules/permission/module.go

@@ -63,5 +63,5 @@ func (m *Module) PrepareTables() error {
 
 // AutoMigrate implements
 func (m *Module) AutoMigrate() error {
-	return m.db.AutoMigrate(context.TODO(), []schema.Tabler{&models.Statements{}})
+	return m.db.AutoMigrate(context.TODO(), []schema.Tabler{&models.Permission{}, &models.Statements{}})
 }

modular/downloader/download_task.go

@@ -4,9 +4,11 @@ import (
 	"context"
 	"errors"
 	"net/http"
+	"strings"
 	"sync/atomic"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/s3"
 	"gorm.io/gorm"
 
 	"github.com/bnb-chain/greenfield-storage-provider/base/types/gfsperrors"
@@ -39,6 +41,10 @@ func ErrPieceStoreWithDetail(detail string) *gfsperrors.GfSpError {
 	return gfsperrors.Register(module.ReceiveModularName, http.StatusInternalServerError, 85101, detail)
 }
 
+func ErrPieceStoreNoSuchKeyWithDetail(detail string) *gfsperrors.GfSpError {
+	return gfsperrors.Register(module.ReceiveModularName, http.StatusNotFound, 85102, detail)
+}
+
 func ErrGfSpDBWithDetail(detail string) *gfsperrors.GfSpError {
 	return gfsperrors.Register(module.ReceiveModularName, http.StatusInternalServerError, 85201, detail)
 }
@@ -175,8 +181,12 @@ func (d *DownloadModular) HandleDownloadObjectTask(ctx context.Context, download
 			int64(pInfo.Offset), int64(pInfo.Length))
 		if getPieceErr != nil {
 			log.CtxErrorw(ctx, "failed to get piece data from piece store", "task_info", downloadObjectTask.Info(), "piece_info", pInfo, "error", getPieceErr)
-			err = ErrPieceStoreWithDetail("failed to get piece data from piece store, error: " + getPieceErr.Error())
-			return nil, err
+			pieceStoreErrDetail := "failed to get piece data from piece store, task_info: " + downloadObjectTask.Info() + ", error: " + err.Error()
+			if isErrNoSuchKey(err) {
+				return nil, ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail)
+			} else {
+				return nil, ErrPieceStoreWithDetail(pieceStoreErrDetail)
+			}
 		}
 		d.pieceCache.Add(key, piece)
 		data = append(data, piece...)
@@ -389,7 +399,11 @@ func (d *DownloadModular) HandleDownloadPieceTask(ctx context.Context, downloadP
 		int64(downloadPieceTask.GetPieceOffset()), int64(downloadPieceTask.GetPieceLength())); err != nil {
 		metrics.PerfGetObjectTimeHistogram.WithLabelValues("get_object_put_piece_time").Observe(time.Since(putPieceTime).Seconds())
 		log.CtxErrorw(ctx, "failed to get piece data from piece store", "task_info", downloadPieceTask.Info(), "error", err)
-		return nil, ErrPieceStoreWithDetail("failed to get piece data from piece store, task_info: " + downloadPieceTask.Info() + ", error: " + err.Error())
+		pieceStoreErrDetail := "failed to get piece data from piece store, task_info: " + downloadPieceTask.Info() + ", error: " + err.Error()
+		if isErrNoSuchKey(err) {
+			return nil, ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail)
+		}
+		return nil, ErrPieceStoreWithDetail(pieceStoreErrDetail)
 	}
 	metrics.PerfGetObjectTimeHistogram.WithLabelValues("get_object_put_piece_time").Observe(time.Since(putPieceTime).Seconds())
 	return pieceData, nil
@@ -462,7 +476,12 @@ func (d *DownloadModular) HandleChallengePiece(ctx context.Context, challengePie
 	metrics.PerfChallengeTimeHistogram.WithLabelValues("challenge_get_piece_time").Observe(time.Since(getPieceTime).Seconds())
 	if err != nil {
 		log.CtxErrorw(ctx, "failed to get piece data", "task", challengePieceTask, "error", err)
-		return nil, nil, nil, ErrPieceStoreWithDetail("failed to get piece data, error: " + err.Error())
+		pieceStoreErrDetail := "failed to get piece data, task: " + challengePieceTask.Info() + ", error: " + err.Error()
+		if isErrNoSuchKey(err) {
+			return nil, nil, nil, ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail)
+		} else {
+			return nil, nil, nil, ErrPieceStoreWithDetail(pieceStoreErrDetail)
+		}
 	}
 
 	return integrity.IntegrityChecksum, integrity.PieceChecksumList, data, nil
@@ -474,3 +493,8 @@ func (d *DownloadModular) PostChallengePiece(context.Context, task.ChallengePiec
 func (d *DownloadModular) QueryTasks(context.Context, task.TKey) ([]task.Task, error) {
 	return nil, nil
 }
+
+func isErrNoSuchKey(err error) bool {
+	msg := err.Error()
+	return strings.Contains(msg, s3.ErrCodeNoSuchKey)
+}

modular/gater/admin_handler.go

@@ -11,6 +11,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/aws/aws-sdk-go/service/s3"
 	sdktypes "github.com/cosmos/cosmos-sdk/types"
 
 	commonhash "github.com/bnb-chain/greenfield-common/go/hash"
@@ -1001,7 +1002,12 @@ func (g *GateModular) getRecoverPiece(ctx context.Context, objectInfo *storagety
 	pieceData, err := g.baseApp.GfSpClient().GetPiece(ctx, pieceTask)
 	if err != nil {
 		log.CtxErrorw(ctx, "failed to download piece", "error", err)
-		return nil, downloader.ErrPieceStoreWithDetail("failed to download piece, error: " + err.Error())
+		pieceStoreErrDetail := "failed to download piece, error: " + err.Error()
+		if isErrNoSuchKey(err) {
+			return nil, downloader.ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail)
+		} else {
+			return nil, downloader.ErrPieceStoreWithDetail(pieceStoreErrDetail)
+		}
 	}
 
 	return pieceData, nil
@@ -1091,7 +1097,12 @@ func (g *GateModular) getRecoverSegment(ctx context.Context, objectInfo *storage
 		pieceData, err := g.baseApp.GfSpClient().GetPiece(ctx, pieceTask)
 		if err != nil {
 			log.CtxErrorw(ctx, "failed to download piece", "error", err)
-			return nil, downloader.ErrPieceStoreWithDetail("failed to download piece, error: " + err.Error())
+			pieceStoreErrDetail := "failed to download piece, error: " + err.Error()
+			if isErrNoSuchKey(err) {
+				return nil, downloader.ErrPieceStoreNoSuchKeyWithDetail(pieceStoreErrDetail)
+			} else {
+				return nil, downloader.ErrPieceStoreWithDetail(pieceStoreErrDetail)
+			}
 		}
 		return pieceData, nil
 	}
@@ -1117,3 +1128,8 @@ func (g *GateModular) getRecoverSegment(ctx context.Context, objectInfo *storage
 
 	return ecData[redundancyIdx], nil
 }
+
+func isErrNoSuchKey(err error) bool {
+	msg := err.Error()
+	return strings.Contains(msg, s3.ErrCodeNoSuchKey)
+}

modular/gater/object_handler.go

@@ -663,7 +663,7 @@ func (g *GateModular) downloadObject(w http.ResponseWriter, reqCtx *RequestConte
 			log.CtxErrorw(reqCtx.Context(), "failed to download piece", "error", err)
 			downloaderErr := gfsperrors.MakeGfSpError(err)
 			// if it is the first piece and the quota db is not updated, no extra data need to updated
-			if idx >= 1 || (idx == 0 && downloaderErr.GetInnerCode() == 85101) {
+			if idx >= 1 || (idx == 0 && (downloaderErr.GetInnerCode() == 85101 || downloaderErr.GetInnerCode() == 85102)) {
 				extraQuota = downloadSize - consumedQuota
 			}
 			return err