feat(dl): add credential support for dl service (#163)

Signed-off-by: wuhuizuo <[email protected]>

Signed-off-by: wuhuizuo <[email protected]>

Author: wuhuizuo

dl/cmd/server/main.go

@@ -28,6 +28,7 @@ func main() {
 		secureF     = flag.Bool("secure", false, "Use secure scheme (https or grpcs)")
 		dbgF        = flag.Bool("debug", false, "Log request and response bodies")
 		ks3CfgPathF = flag.String("ks3-config", "ks3.yaml", "ks3 config yaml file path")
+		ociCfgPathF = flag.String("oci-config", "oci.yaml", "oci config yaml file path")
 	)
 	flag.Parse()
 
@@ -47,7 +48,7 @@ func main() {
 	)
 	{
 		healthSvc = dl.NewHealth(logger)
-		ociSvc = dl.NewOci(logger)
+		ociSvc = dl.NewOci(logger, ociCfgPathF)
 		ks3Svc = dl.NewKs3(logger, *ks3CfgPathF)
 	}
 

dl/ks3.go

@@ -25,7 +25,10 @@ type ks3srvc struct {
 }
 
 func newKS3Client(cfg *pkgks3.Config) *s3.S3 {
-	var cre = credentials.NewStaticCredentials(cfg.AccessKey, cfg.SecretKey, "")
+	var cre *credentials.Credentials
+	if cfg != nil && cfg.AccessKey != "" && cfg.SecretKey != "" {
+		cre = credentials.NewStaticCredentials(cfg.AccessKey, cfg.SecretKey, "")
+	}
 	awsConfig := aws.Config{
 		Region:           cfg.Region, // Ref: https://docs.ksyun.com/documents/6761
 		Credentials:      cre,

dl/oci.go

@@ -5,40 +5,87 @@ import (
 	"io"
 	"log"
 	"net/url"
+	"os"
 	"strings"
 
 	oci "github.com/PingCAP-QE/ee-apps/dl/gen/oci"
 	pkgoci "github.com/PingCAP-QE/ee-apps/dl/pkg/oci"
+	"gopkg.in/yaml.v3"
+	"oras.land/oras-go/v2/registry/remote"
+	"oras.land/oras-go/v2/registry/remote/auth"
+	"oras.land/oras-go/v2/registry/remote/retry"
 )
 
 // oci service example implementation.
 // The example methods log the requests and return zero values.
 type ocisrvc struct {
-	logger *log.Logger
+	logger     *log.Logger
+	credential *auth.Credential
 }
 
 // NewOci returns the oci service implementation.
-func NewOci(logger *log.Logger) oci.Service {
-	return &ocisrvc{logger}
+func NewOci(logger *log.Logger, cfgFile *string) oci.Service {
+	var cfg pkgoci.Config
+	if cfgFile == nil {
+		return &ocisrvc{logger: logger, credential: &auth.EmptyCredential}
+	}
+
+	cfgBytes, err := os.ReadFile(*cfgFile)
+	if err != nil {
+		logger.Fatalf("Failed to load configuration: %v", err)
+	}
+	if err := yaml.Unmarshal(cfgBytes, &cfg); err != nil {
+		logger.Fatalf("Failed to load configuration: %v", err)
+	}
+
+	return &ocisrvc{logger: logger, credential: &auth.Credential{
+		Username: cfg.Username,
+		Password: cfg.Password,
+	}}
 }
 
 // ListFiles implements list-files.
 func (s *ocisrvc) ListFiles(ctx context.Context, p *oci.ListFilesPayload) (res []string, err error) {
 	s.logger.Print("oci.list-files")
 
-	files, err := pkgoci.ListFiles(ctx, p.Repository, p.Tag)
+	repository, err := s.getTargetRepo(p.Repository)
+	if err != nil {
+		return nil, err
+	}
+
+	files, err := pkgoci.ListFiles(ctx, repository, p.Tag)
 	if err != nil {
 		return nil, oci.MakeInvalidFilePath(err)
 	}
 
 	return files, nil
 }
 
+func (s *ocisrvc) getTargetRepo(repo string) (*remote.Repository, error) {
+	repository, err := remote.NewRepository(repo)
+	if err != nil {
+		return nil, err
+	}
+
+	reg := strings.SplitN(repo, "/", 2)[0]
+	repository.Client = &auth.Client{
+		Client:     retry.DefaultClient,
+		Cache:      auth.DefaultCache,
+		Credential: auth.StaticCredential(reg, *s.credential),
+	}
+
+	return repository, nil
+}
+
 // DownloadFile implements download-file.
 func (s *ocisrvc) DownloadFile(ctx context.Context, p *oci.DownloadFilePayload) (res *oci.DownloadFileResult, resp io.ReadCloser, err error) {
 	s.logger.Print("oci.download-files")
 
-	rc, length, err := pkgoci.NewFileReadCloser(ctx, p.Repository, p.Tag, p.File)
+	repository, err := s.getTargetRepo(p.Repository)
+	if err != nil {
+		return nil, nil, err
+	}
+	rc, length, err := pkgoci.NewFileReadCloser(ctx, repository, p.Tag, p.File)
 	if err != nil {
 		return nil, nil, err
 	}
@@ -54,7 +101,11 @@ func (s *ocisrvc) DownloadFile(ctx context.Context, p *oci.DownloadFilePayload)
 func (s *ocisrvc) DownloadFileSha256(ctx context.Context, p *oci.DownloadFileSha256Payload) (res *oci.DownloadFileSha256Result, resp io.ReadCloser, err error) {
 	s.logger.Print("oci.download-file-sha256")
 
-	value, err := pkgoci.GetFileSHA256(ctx, p.Repository, p.Tag, p.File)
+	repository, err := s.getTargetRepo(p.Repository)
+	if err != nil {
+		return nil, nil, err
+	}
+	value, err := pkgoci.GetFileSHA256(ctx, repository, p.Tag, p.File)
 	if err != nil {
 		return nil, nil, err
 	}

dl/pkg/oci/cfg.go

@@ -0,0 +1,6 @@
+package oci
+
+type Config struct {
+	Username string `yaml:"username,omitempty" json:"username,omitempty"`
+	Password string `yaml:"password,omitempty" json:"password,omitempty"`
+}

dl/pkg/oci/download.go

@@ -6,31 +6,14 @@ import (
 	"fmt"
 	"io"
 
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 	oras "oras.land/oras-go/v2"
 	"oras.land/oras-go/v2/registry/remote"
-
-	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
 )
 
 const AnnotationKeyFileName = "org.opencontainers.image.title"
 
-func ListFiles(ctx context.Context, repo, tag string) ([]string, error) {
-	repository, err := remote.NewRepository(repo)
-	if err != nil {
-		return nil, err
-	}
-
-	// Note: The below code can be omitted if authentication is not required
-	// reg := strings.SplitN(repo, "/", 2)[0]
-	// repository.Client = &auth.Client{
-	// 	Client: retry.DefaultClient,
-	// 	Cache:  auth.DefaultCache,
-	// 	Credential: auth.StaticCredential(reg, auth.Credential{
-	// 		Username: "username",
-	// 		Password: "password",
-	// 	}),
-	// }
-
+func ListFiles(ctx context.Context, repository *remote.Repository, tag string) ([]string, error) {
 	layers, err := listArtifactLayers(ctx, repository, tag)
 	if err != nil {
 		return nil, err
@@ -44,12 +27,7 @@ func ListFiles(ctx context.Context, repo, tag string) ([]string, error) {
 	return ret, nil
 }
 
-func NewFileReadCloser(ctx context.Context, repo, tag, filename string) (io.ReadCloser, int64, error) {
-	repository, err := remote.NewRepository(repo)
-	if err != nil {
-		return nil, 0, err
-	}
-
+func NewFileReadCloser(ctx context.Context, repository *remote.Repository, tag, filename string) (io.ReadCloser, int64, error) {
 	// 1. get desired file descriptor in the artifact.
 	// destination := strings.Join([]string{repo, tag}, ":")
 	desiredFileDescriptor, err := fetchFileDescriptor(ctx, repository, tag, filename)
@@ -67,12 +45,7 @@ func NewFileReadCloser(ctx context.Context, repo, tag, filename string) (io.Read
 	return rc, desiredFileDescriptor.Size, nil
 }
 
-func GetFileSHA256(ctx context.Context, repo, tag, filename string) (string, error) {
-	repository, err := remote.NewRepository(repo)
-	if err != nil {
-		return "", err
-	}
-
+func GetFileSHA256(ctx context.Context, repository oras.ReadOnlyTarget, tag, filename string) (string, error) {
 	// 1. get desired file descriptor in the artifact.
 	// destination := strings.Join([]string{repo, tag}, ":")
 	desiredFileDescriptor, err := fetchFileDescriptor(ctx, repository, tag, filename)