Skip to content

Commit 6d2be30

Browse files
eusebiu-constantin-petu-dbkeusebiu-constantin-petu-dbk
committed
feat(sync): specific images
added the option to sync specific images(docker/oci) specific repo:tags can be set in sync config Signed-off-by: Petu Eusebiu <[email protected]>
1 parent 4cb7a6c commit 6d2be30

11 files changed

+864
-80
lines changed

‎Makefile

+1
Original file line numberDiff line numberDiff line change
@@ -430,6 +430,7 @@ run-blackbox-ci: check-blackbox-prerequisites binary binary-minimal cli
430430
$(BATS) $(BATS_FLAGS) test/blackbox/cve.bats && \
431431
$(BATS) $(BATS_FLAGS) test/blackbox/sync.bats && \
432432
$(BATS) $(BATS_FLAGS) test/blackbox/sync_docker.bats && \
433+
$(BATS) $(BATS_FLAGS) test/blackbox/sync_specific_images.bats && \
433434
$(BATS) $(BATS_FLAGS) test/blackbox/sync_replica_cluster.bats && \
434435
$(BATS) $(BATS_FLAGS) test/blackbox/scrub.bats && \
435436
$(BATS) $(BATS_FLAGS) test/blackbox/garbage_collect.bats && \

‎pkg/extensions/config/sync/config.go

+1
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ type RegistryConfig struct {
3131
}
3232

3333
type Content struct {
34+
Images map[string][]string
3435
Prefix string
3536
Tags *Tags
3637
Destination string `mapstructure:",omitempty"`

‎pkg/extensions/extension_sync.go

+33-2
Original file line numberDiff line numberDiff line change
@@ -38,10 +38,11 @@ func EnableSyncExtension(config *config.Config, metaDB mTypes.MetaDB,
3838
return nil, zerr.ErrSyncNoURLsLeft
3939
}
4040

41-
isPeriodical := len(registryConfig.Content) != 0 && registryConfig.PollInterval != 0
41+
isPeriodical := isPeriodical(registryConfig)
4242
isOnDemand := registryConfig.OnDemand
43+
hasSpecificImagesToSync := hasSpecificImages(registryConfig)
4344

44-
if isPeriodical || isOnDemand {
45+
if isPeriodical || isOnDemand || hasSpecificImagesToSync {
4546
service, err := sync.New(registryConfig, config.Extensions.Sync.CredentialsFile,
4647
storeController, metaDB, log)
4748
if err != nil {
@@ -54,6 +55,12 @@ func EnableSyncExtension(config *config.Config, metaDB mTypes.MetaDB,
5455
sch.SubmitGenerator(gen, registryConfig.PollInterval, scheduler.MediumPriority)
5556
}
5657

58+
if hasSpecificImagesToSync {
59+
// add to task scheduler periodically sync specific images
60+
gen := sync.NewSpecificImagesGenerator(service, log)
61+
sch.SubmitGenerator(gen, registryConfig.PollInterval, scheduler.MediumPriority)
62+
}
63+
5764
if isOnDemand {
5865
// onDemand services used in routes.go
5966
onDemand.Add(service)
@@ -171,3 +178,27 @@ func removeSelfURLs(config *config.Config, registryConfig *syncconf.RegistryConf
171178

172179
return nil
173180
}
181+
182+
func hasSpecificImages(registryConfig syncconf.RegistryConfig) bool {
183+
if registryConfig.PollInterval != 0 {
184+
for _, content := range registryConfig.Content {
185+
if len(content.Images) > 0 {
186+
return true
187+
}
188+
}
189+
}
190+
191+
return false
192+
}
193+
194+
func isPeriodical(registryConfig syncconf.RegistryConfig) bool {
195+
if registryConfig.PollInterval != 0 {
196+
for _, content := range registryConfig.Content {
197+
if content.Prefix != "" {
198+
return true
199+
}
200+
}
201+
}
202+
203+
return false
204+
}

‎pkg/extensions/sync/content.go

+19-6
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ import (
1010
"github.com/Masterminds/semver"
1111
glob "github.com/bmatcuk/doublestar/v4"
1212

13+
zerr "zotregistry.io/zot/errors"
1314
"zotregistry.io/zot/pkg/common"
1415
syncconf "zotregistry.io/zot/pkg/extensions/config/sync"
1516
"zotregistry.io/zot/pkg/log"
@@ -75,27 +76,35 @@ func (cm ContentManager) FilterTags(repo string, tags []string) ([]string, error
7576
GetRepoDestination applies content destination config rule and returns the final repo namespace.
7677
- used by periodically sync.
7778
*/
78-
func (cm ContentManager) GetRepoDestination(repo string) string {
79+
func (cm ContentManager) GetRepoDestination(repo string) (string, error) {
80+
if len(cm.contents) == 0 {
81+
return repo, nil
82+
}
83+
7984
content := cm.getContentByUpstreamRepo(repo)
8085
if content == nil {
81-
return ""
86+
return "", zerr.ErrSyncImageFilteredOut
8287
}
8388

84-
return getRepoDestination(repo, *content)
89+
return getRepoDestination(repo, *content), nil
8590
}
8691

8792
/*
8893
GetRepoSource is the inverse function of GetRepoDestination, needed in on demand to find out
8994
the remote name of a repo given a local repo.
9095
- used by on demand sync.
9196
*/
92-
func (cm ContentManager) GetRepoSource(repo string) string {
97+
func (cm ContentManager) GetRepoSource(repo string) (string, error) {
98+
if len(cm.contents) == 0 {
99+
return repo, nil
100+
}
101+
93102
content := cm.getContentByLocalRepo(repo)
94103
if content == nil {
95-
return ""
104+
return "", zerr.ErrSyncImageFilteredOut
96105
}
97106

98-
return getRepoSource(repo, *content)
107+
return getRepoSource(repo, *content), nil
99108
}
100109

101110
// utilies functions.
@@ -109,6 +118,10 @@ func (cm ContentManager) getContentByUpstreamRepo(repo string) *syncconf.Content
109118
prefix = content.Prefix
110119
}
111120

121+
if _, ok := content.Images[repo]; ok {
122+
return &content
123+
}
124+
112125
matched, err := glob.Match(prefix, repo)
113126
if err != nil {
114127
cm.log.Error().Str("errorType", common.TypeOf(err)).

‎pkg/extensions/sync/content_internal_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -68,7 +68,7 @@ func TestContentManager(t *testing.T) {
6868
Convey("Test GetRepoDestination()", t, func() {
6969
for _, test := range testCases {
7070
cm := NewContentManager([]syncconf.Content{test.content}, log.Logger{})
71-
actualResult := cm.GetRepoDestination(test.expected)
71+
actualResult, _ := cm.GetRepoDestination(test.expected)
7272
So(actualResult, ShouldEqual, test.repo)
7373
}
7474
})
@@ -77,7 +77,7 @@ func TestContentManager(t *testing.T) {
7777
Convey("Test GetRepoSource()", t, func() {
7878
for _, test := range testCases {
7979
cm := NewContentManager([]syncconf.Content{test.content}, log.Logger{})
80-
actualResult := cm.GetRepoSource(test.repo)
80+
actualResult, _ := cm.GetRepoSource(test.repo)
8181
So(actualResult, ShouldEqual, test.expected)
8282
}
8383
})

‎pkg/extensions/sync/on_demand.go

+32-10
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ func (onDemand *BaseOnDemand) SyncImage(ctx context.Context, repo, reference str
8383
return err
8484
}
8585

86-
func (onDemand *BaseOnDemand) SyncReference(ctx context.Context, repo string,
86+
func (onDemand *BaseOnDemand) SyncReference(ctx context.Context, localRepo string,
8787
subjectDigestStr string, referenceType string,
8888
) error {
8989
var err error
@@ -94,7 +94,17 @@ func (onDemand *BaseOnDemand) SyncReference(ctx context.Context, repo string,
9494
return err
9595
}
9696

97-
err = service.SyncReference(ctx, repo, subjectDigestStr, referenceType)
97+
remoteRepo, err := service.GetRemoteRepoName(localRepo)
98+
if err != nil {
99+
// filtered out
100+
onDemand.log.Info().Str("local repository", localRepo).
101+
Str("repository", localRepo).Str("subject", subjectDigestStr).
102+
Msg("will not sync image, filtered out by content")
103+
104+
continue
105+
}
106+
107+
err = service.SyncReference(ctx, localRepo, remoteRepo, subjectDigestStr, referenceType)
98108
if err != nil {
99109
continue
100110
} else {
@@ -105,7 +115,7 @@ func (onDemand *BaseOnDemand) SyncReference(ctx context.Context, repo string,
105115
return err
106116
}
107117

108-
func (onDemand *BaseOnDemand) syncImage(ctx context.Context, repo, reference string, syncResult chan error) {
118+
func (onDemand *BaseOnDemand) syncImage(ctx context.Context, localRepo, reference string, syncResult chan error) {
109119
var err error
110120
for serviceID, service := range onDemand.services {
111121
err = service.SetNextAvailableURL()
@@ -115,16 +125,25 @@ func (onDemand *BaseOnDemand) syncImage(ctx context.Context, repo, reference str
115125
return
116126
}
117127

118-
err = service.SyncImage(ctx, repo, reference)
128+
remoteRepo, err := service.GetRemoteRepoName(localRepo)
129+
if err != nil {
130+
// filtered out
131+
onDemand.log.Info().Str("local repository", localRepo).
132+
Str("remote repository", remoteRepo).Str("repository", localRepo).Str("reference", reference).
133+
Msg("will not sync image, filtered out by content")
134+
135+
continue
136+
}
137+
138+
err = service.SyncImage(ctx, localRepo, remoteRepo, reference)
119139
if err != nil {
120140
if errors.Is(err, zerr.ErrManifestNotFound) ||
121-
errors.Is(err, zerr.ErrSyncImageFilteredOut) ||
122141
errors.Is(err, zerr.ErrSyncImageNotSigned) {
123142
continue
124143
}
125144

126145
req := request{
127-
repo: repo,
146+
repo: localRepo,
128147
reference: reference,
129148
serviceID: serviceID,
130149
isBackground: true,
@@ -143,22 +162,25 @@ func (onDemand *BaseOnDemand) syncImage(ctx context.Context, repo, reference str
143162
// remove image after syncing
144163
defer func() {
145164
onDemand.requestStore.Delete(req)
146-
onDemand.log.Info().Str("repo", repo).Str("reference", reference).
165+
onDemand.log.Info().Str("local repository", localRepo).
166+
Str("remote repository", remoteRepo).Str("reference", reference).
147167
Msg("sync routine for image exited")
148168
}()
149169

150-
onDemand.log.Info().Str("repo", repo).Str(reference, "reference").Str("err", err.Error()).
170+
onDemand.log.Info().Str("local repository", localRepo).
171+
Str("remote repository", remoteRepo).Str(reference, "reference").Str("err", err.Error()).
151172
Msg("sync routine: starting routine to copy image, because of error")
152173

153174
time.Sleep(retryOptions.Delay)
154175

155176
// retrying in background, can't use the same context which should be cancelled by now.
156177
if err = retry.RetryIfNecessary(context.Background(), func() error {
157-
err := service.SyncImage(context.Background(), repo, reference)
178+
err := service.SyncImage(context.Background(), localRepo, remoteRepo, reference)
158179

159180
return err
160181
}, retryOptions); err != nil {
161-
onDemand.log.Error().Str("errorType", common.TypeOf(err)).Str("repo", repo).Str("reference", reference).
182+
onDemand.log.Error().Str("errorType", common.TypeOf(err)).Str("local repository", localRepo).
183+
Str("remote repository", remoteRepo).Str("reference", reference).
162184
Err(err).Msg("sync routine: error while copying image")
163185
}
164186
}(service)

0 commit comments

Comments
 (0)