Merge pull request #6 from wu0407/optimization

优化处理效率

Author: malc0lm

.gitignore

@@ -1,4 +1,5 @@
 _output/*
 *.yaml
 ccr_to_tcr_rules
-
+.vscode/*
+image-transfer

configs/config.go

@@ -41,7 +41,7 @@ var (
 
 const (
 	maxRatelimit int = 30000
-	maxRoutineNums int = 10
+	maxRoutineNums int = 50
 )
 
 // Configs struct save of all config

go.mod

@@ -8,6 +8,7 @@ require (
 	github.com/docker/docker v1.13.1 // indirect
 	github.com/emicklei/go-restful v2.15.0+incompatible
 	github.com/gorilla/mux v1.8.0 // indirect
+	github.com/opencontainers/go-digest v1.0.0
 	github.com/pkg/errors v0.9.1
 	github.com/skipor/goenv v0.0.0-20170219222015-cf3a15e6b664
 	github.com/spf13/cobra v1.1.1

go.sum

@@ -19,11 +19,9 @@
 package ccrapis
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
 	"net/http"
 	"strings"
+	"sync"
 
 	"github.com/pkg/errors"
 	"github.com/tencentcloud/tencentcloud-sdk-go/tencentcloud/common"
@@ -40,7 +38,8 @@ type CCRAPIClient struct {
 	url        string
 }
 
-var regionPrefix = map[string]string{
+// RegionPrefix is map tencent cloud region to ccr domain prefix
+var RegionPrefix = map[string]string{
 	"ap-guangzhou":     "ccr",
 	"ap-shanghai":      "ccr",
 	"ap-nanjing":       "ccr",
@@ -101,6 +100,7 @@ func (ai *CCRAPIClient) GetAllNamespaceByName(secret map[string]configs.Secret,
 		for _, ns := range resp.Response.Data.NamespaceInfo {
 			nsList = append(nsList, *ns.Namespace)
 		}
+		log.Debugf("ccr namespace offset %d limit %d resp is %s", offset, limit, resp.ToJsonString())
 
 		if int64(count) >= namespaceCount {
 			break
@@ -114,73 +114,59 @@ func (ai *CCRAPIClient) GetAllNamespaceByName(secret map[string]configs.Secret,
 
 }
 
-//GenerateAllCcrRules generate all ccr rules
-func (ai *CCRAPIClient) GenerateAllCcrRules(secret map[string]configs.Secret, ccrRegion string,
-	failedNsList []string, tcrRegion string, tcrName string) (map[string]string, error) {
-
-	rulesMap := make(map[string]string)
-
+//GetAllCcrRepo get all ccr repo send to repoChan
+func (ai *CCRAPIClient) GetAllCcrRepo(secret map[string]configs.Secret, ccrRegion string,
+	failedNsList []string, tcrRegion string, tcrName string, repoChan chan string, toltalCount int64) error {
 	secretID, secretKey, err := GetCcrSecret(secret)
-
 	if err != nil {
-		log.Errorf("GetCcrSecret error: ", err)
-		return rulesMap, err
+		log.Errorf("GetCcrSecret error: %s", err)
+		return err
 	}
 
 	offset := int64(0)
-	count := 0
 	limit := int64(100)
-
-	for {
-		resp, err := ai.DescribeRepositoryOwnerPersonal(secretID, secretKey, ccrRegion, offset, limit)
-		if err != nil {
-			log.Errorf("get ccr repo error, ", err)
-			return rulesMap, err
-		}
-		repoCount := *resp.Response.Data.TotalCount
-		count += len(resp.Response.Data.RepoInfo)
-
-		for _, repo := range resp.Response.Data.RepoInfo {
-			ns := strings.Split(*repo.RepoName, "/")[0]
-			if len(failedNsList) == 0 || !utils.IsContain(failedNsList, ns) {
-				tags, err := ai.getRepoTags(secretID, secretKey, ccrRegion, *repo.RepoName)
-				if err != nil {
-					return nil, err
-				}
-				if len(tags) == 0 {
-					continue
-				}
-				tagStr := strings.Join(tags, ",")
-				source := fmt.Sprintf("%s%s%s:%s", regionPrefix[ccrRegion], ".ccs.tencentyun.com/", *repo.RepoName, tagStr)
-				target := tcrName + ".tencentcloudcr.com/" + *repo.RepoName
-				rulesMap[target] = source
-			}
-		}
-
-		if int64(count) >= repoCount {
-			break
-		} else {
-			offset += limit
-		}
-
-	}
-
-	jsonStr, err := json.Marshal(rulesMap)
-	if err != nil {
-		log.Errorf("Marshal ccr rules map error %v, ", err)
+	maxGorutineNum := 5
+	page := toltalCount/limit
+	if toltalCount%limit >0 {
+		page++
 	}
+	wg := sync.WaitGroup{}
+	defer close(repoChan)
+	leakCh := make(chan struct{}, maxGorutineNum)
+	wg.Add(1)
 	go func() {
-		err = ioutil.WriteFile("./ccr_to_tcr_rules", []byte(jsonStr), 0666)
-		if err != nil {
-			log.Errorf("WriteFile ccr rules error %v, ", err)
+		defer wg.Done()
+		for i := 0; i < maxGorutineNum; i++ {
+			leakCh <- struct{}{}
 		}
 	}()
 
-	return rulesMap, nil
-
+	for j :=int64(0);j< page;j++ {
+		wg.Add(1)
+		<-leakCh
+		go func(n int64) {
+			defer wg.Done()
+			offset = n*limit
+			resp, err := ai.DescribeRepositoryOwnerPersonal(secretID, secretKey, ccrRegion, offset, limit)
+			if err != nil {
+				log.Errorf("get ccr repo offset %d limit %d error, %s", offset, limit, err)
+				return
+			}
+			for _, repo := range resp.Response.Data.RepoInfo {
+				ns := strings.Split(*repo.RepoName, "/")[0]
+				if len(failedNsList) == 0 || !utils.IsContain(failedNsList, ns) {
+					repoChan <- *repo.RepoName
+				}
+			}
+			leakCh <- struct{}{}
+		}(j)	
+	}
+	wg.Wait()
+	return nil
 }
 
-func (ai *CCRAPIClient) getRepoTags(secretID, secretKey, ccrRegion, repoName string) ([]string, error) {
+// GetRepoTags get ccr repo tags
+func (ai *CCRAPIClient) GetRepoTags(secretID, secretKey, ccrRegion, repoName string) ([]string, error) {
 
 	offset := int64(0)
 	count := int64(0)

pkg/apis/ccrapis/ccrapi.go

@@ -77,6 +77,7 @@ func (ai *TCRAPIClient) GetAllNamespaceByName(secret map[string]configs.Secret,
 			log.Errorf("DescribeNamespaces error, ", err)
 			return nsList, tcrID, err
 		}
+		log.Debugf("tcr namespace offset %d limit %d resp is %s", offset, limit, resp.ToJsonString())
 		namespaceCount := *resp.Response.TotalCount
 		count += len(resp.Response.NamespaceList)
 		for _, ns := range resp.Response.NamespaceList {
@@ -204,3 +205,73 @@ func GetTcrSecret(secret map[string]configs.Secret) (string, string, error) {
 
 	return secretID, secretKey, nil
 }
+
+// DescribeImages get the images list of tcr repo
+func (ai *TCRAPIClient) DescribeImages(secretID, secretKey, region, registryID, nsName, repositoryName string, offset, limit int64) (*tcr.DescribeImagesResponse, error) {
+	credential := common.NewCredential(
+		secretID,
+		secretKey,
+	)
+	cpf := profile.NewClientProfile()
+	cpf.HttpProfile.Endpoint = "tcr.tencentcloudapi.com"
+	client, _ := tcr.NewClient(credential, region, cpf)
+
+	request := tcr.NewDescribeImagesRequest()
+	request.RegistryId = common.StringPtr(registryID)
+	request.NamespaceName = common.StringPtr(nsName)
+	request.RepositoryName = common.StringPtr(repositoryName)
+	request.Limit = common.Int64Ptr(limit)
+	request.Offset = common.Int64Ptr(offset)
+	
+	response, err := client.DescribeImages(request)
+
+	if err != nil {
+		log.Errorf("An error has returned: %s", err)
+		return nil, err
+	}
+	return response, nil
+}
+
+// GetRepoTags get tcr repo tag list
+func (ai *TCRAPIClient) GetRepoTags(secretID, secretKey, region, tcrName, nsName, repositoryName, instanceName string) ([]string, error) {
+	var tags []string
+
+	tcrID := instanceName
+	if tcrID == "" { 
+		//get tcrId by tcr name
+		filterValues := []string{tcrName}
+		resp, err := ai.DescribeInstances(secretID, secretKey, region, 0, 100, "RegistryName", filterValues)
+		if err != nil {
+			log.Errorf("DescribeInstances error, %s", err)
+			return tags, err
+		}
+
+		tcrID = *resp.Response.Registries[0].RegistryId
+	}
+
+	// tcr offset means page number, currently :(
+	offset := int64(1)
+	count := 0
+	limit := int64(100)
+
+	for {
+		resp, err := ai.DescribeImages(secretID, secretKey, region, tcrID, nsName, repositoryName, offset, limit)
+		if err != nil {
+			log.Errorf("DescribeImages error, %s", err)
+			return tags, err
+		}
+		tagsCount := *resp.Response.TotalCount
+		count += len(resp.Response.ImageInfoList)
+		for _, tagInfo := range resp.Response.ImageInfoList {
+			tags = append(tags, *tagInfo.ImageVersion)
+		}
+
+		log.Debugf("tcr get tag repo %s/%s offset %d, limit %d, total count %d, current count %d, resp %v", nsName, repositoryName, offset, limit, tagsCount, count, resp.ToJsonString())
+		if int64(count) >= tagsCount {
+			break
+		} else {
+			offset++
+		}
+	}
+	return tags, nil
+}

pkg/apis/tcrapis/tcrapi.go

@@ -36,7 +36,8 @@ type ConfigOptions struct {
 	TCRRegion string
 	TCRName string
 	SecretFile string
-
+	// if target tag is exist override it
+	TagExistOverridden bool
 }
 
 // NewConfigOptions creates a NewConfigOptions object with default
@@ -67,7 +68,7 @@ func (o *ConfigOptions) AddFlags(fs *pflag.FlagSet) {
 		"default destinate namespace when destinate namespace is not" +
 		" given in the config file, can also be set with DEFAULT_NAMESPACE environment value")
 	fs.IntVar(&o.RoutineNums, "routines", 5,
-		"number of goroutines, default value is 5, max routines is 10")
+		"number of goroutines, default value is 5, max routines is 50")
 	fs.IntVar(&o.RetryNums, "retry", 2,
 		"number of retries, default value is 2")
 	fs.IntVar(&o.QPS, "qps", 100,
@@ -82,4 +83,5 @@ func (o *ConfigOptions) AddFlags(fs *pflag.FlagSet) {
 		"tcr name. this flag is used when flag ccrToTcr=true")
 	fs.StringVar(&o.SecretFile, "secretFile", o.SecretFile,
 		"Tencent Cloud secretId 、secretKey for access ccr and tcr. this flag is used when flag ccrToTcr=true")
+	fs.BoolVar(&o.TagExistOverridden, "tag-exist-overridden", true, "if target tag is exist, override it")
 }