Skip to content

Commit 96bab7b

Browse files
CamrynCarteramartin120zackbradys
authored
path rewrites (#475)
* image reference rewrite * remove need for rewrite-provider * handle charts * handle leading slash and missing tags * tests * tool cleanup * removed intermediate store cleanup * fix? * test cleanup * clean up tools folder again * debug test * clear tempdir after each filename provided by load Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * update tar command in load integration test Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * clean up debug prints * clean up debug prints * fix typo --------- Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> Signed-off-by: Zack Brady <zackbrady123@gmail.com> Co-authored-by: Adam Martin <adam.martin@ranchergovernment.com> Co-authored-by: Zack Brady <zackbrady123@gmail.com>
1 parent 5ea9b29 commit 96bab7b

11 files changed

Lines changed: 202 additions & 14 deletions

File tree

‎.github/workflows/tests.yaml‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,21 @@ jobs:
154154
# verify via the hauler store contents
155155
hauler store info
156156
157+
- name: Verify - hauler store add chart --rewrite
158+
run: |
159+
# add chart with rewrite flag
160+
hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable --version 2.8.4 --rewrite custom-path/rancher:2.8.4
161+
# verify new ref in store
162+
hauler store info | grep 'custom-path/rancher:2.8.4'
163+
# confrim leading slash trimmed from rewrite
164+
hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable --version 2.8.4 --rewrite /custom-path/rancher:2.8.4
165+
# verify no leading slash
166+
! hauler store info | grep '/custom-path/rancher:2.8.4'
167+
# confirm old tag used if not specified
168+
hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable --version 2.8.4 --rewrite /custom-path/rancher
169+
# confirm tag
170+
hauler store info | grep '2.8.4'
171+
157172
- name: Verify - hauler store add file
158173
run: |
159174
hauler store add file --help
@@ -178,6 +193,21 @@ jobs:
178193
# verify via the hauler store contents
179194
hauler store info
180195
196+
- name: Verify - hauler store add image --rewrite
197+
run: |
198+
# add image with rewrite flag
199+
hauler store add image ghcr.io/hauler-dev/library/busybox --rewrite custom-registry.io/custom-path/busybox:latest
200+
# verify new ref in store
201+
hauler store info | grep 'custom-registry.io/custom-path/busybox:latest'
202+
# confrim leading slash trimmed from rewrite
203+
hauler store add image ghcr.io/hauler-dev/library/busybox --rewrite /custom-path/busybox:latest
204+
# verify no leading slash
205+
! hauler store info | grep '/custom-path/busybox:latest'
206+
# confirm old tag used if not specified
207+
hauler store add image ghcr.io/hauler-dev/library/busybox:stable --rewrite /custom-path/busybox
208+
# confirm tag
209+
hauler store info | grep ':stable'
210+
181211
- name: Verify - hauler store copy
182212
run: |
183213
hauler store copy --help
@@ -229,6 +259,8 @@ jobs:
229259
hauler store load
230260
# verify via load with multiple files
231261
hauler store load --filename haul.tar.zst --filename store.tar.zst
262+
# confirm store contents
263+
tar -xOf store.tar.zst index.json
232264
# verify via load with filename and temp directory
233265
hauler store load --filename store.tar.zst --tempdir /opt
234266
# verify via load with filename and platform (amd64)

‎cmd/hauler/cli/store/add.go‎

Lines changed: 121 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -2,11 +2,12 @@ package store
22

33
import (
44
"context"
5+
"fmt"
56
"os"
7+
"strings"
68

79
"github.com/google/go-containerregistry/pkg/name"
8-
"helm.sh/helm/v3/pkg/action"
9-
10+
ocispec "github.com/opencontainers/image-spec/specs-go/v1"
1011
"hauler.dev/go/hauler/internal/flags"
1112
v1 "hauler.dev/go/hauler/pkg/apis/hauler.cattle.io/v1"
1213
"hauler.dev/go/hauler/pkg/artifacts/file"
@@ -17,6 +18,7 @@ import (
1718
"hauler.dev/go/hauler/pkg/log"
1819
"hauler.dev/go/hauler/pkg/reference"
1920
"hauler.dev/go/hauler/pkg/store"
21+
"helm.sh/helm/v3/pkg/action"
2022
)
2123

2224
func AddFileCmd(ctx context.Context, o *flags.AddFileOpts, s *store.Layout, reference string) error {
@@ -57,7 +59,8 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re
5759
l := log.FromContext(ctx)
5860

5961
cfg := v1.Image{
60-
Name: reference,
62+
Name: reference,
63+
Rewrite: o.Rewrite,
6164
}
6265

6366
// Check if the user provided a key.
@@ -78,10 +81,10 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re
7881
l.Infof("keyless signature verified for image [%s]", cfg.Name)
7982
}
8083

81-
return storeImage(ctx, s, cfg, o.Platform, rso, ro)
84+
return storeImage(ctx, s, cfg, o.Platform, rso, ro, o.Rewrite)
8285
}
8386

84-
func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts) error {
87+
func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform string, rso *flags.StoreRootOpts, ro *flags.CliRootOpts, rewrite string) error {
8588
l := log.FromContext(ctx)
8689

8790
if !ro.IgnoreErrors {
@@ -104,6 +107,7 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
104107
}
105108
}
106109

110+
// copy and sig verification
107111
err = cosign.SaveImage(ctx, s, r.Name(), platform, rso, ro)
108112
if err != nil {
109113
if ro.IgnoreErrors {
@@ -115,21 +119,82 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
115119
}
116120
}
117121

122+
if rewrite != "" {
123+
rewrite = strings.TrimPrefix(rewrite, "/")
124+
if !strings.Contains(rewrite, ":") {
125+
rewrite = strings.Join([]string{rewrite, r.(name.Tag).TagStr()}, ":")
126+
}
127+
// rename image name in store
128+
newRef, err := name.ParseReference(rewrite)
129+
if err != nil {
130+
l.Errorf("unable to parse rewrite name: %w", err)
131+
}
132+
rewriteReference(ctx, s, r, newRef)
133+
}
134+
118135
l.Infof("successfully added image [%s]", r.Name())
119136
return nil
120137
}
121138

139+
func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference) error {
140+
l := log.FromContext(ctx)
141+
142+
l.Infof("rewriting [%s] to [%s]", oldRef.Name(), newRef.Name())
143+
144+
s.OCI.LoadIndex()
145+
146+
//TODO: improve string manipulation
147+
oldRefContext := oldRef.Context()
148+
newRefContext := newRef.Context()
149+
150+
oldRepo := oldRefContext.RepositoryStr()
151+
newRepo := newRefContext.RepositoryStr()
152+
oldTag := oldRef.(name.Tag).TagStr()
153+
newTag := newRef.(name.Tag).TagStr()
154+
oldRegistry := strings.TrimPrefix(oldRefContext.RegistryStr(), "index.")
155+
newRegistry := strings.TrimPrefix(newRefContext.RegistryStr(), "index.")
156+
157+
oldTotal := oldRepo + ":" + oldTag
158+
newTotal := newRepo + ":" + newTag
159+
oldTotalReg := oldRegistry + "/" + oldTotal
160+
newTotalReg := newRegistry + "/" + newTotal
161+
162+
//find and update reference
163+
found := false
164+
if err := s.OCI.Walk(func(k string, d ocispec.Descriptor) error {
165+
if d.Annotations[ocispec.AnnotationRefName] == oldTotal && d.Annotations[consts.ContainerdImageNameKey] == oldTotalReg {
166+
d.Annotations[ocispec.AnnotationRefName] = newTotal
167+
d.Annotations[consts.ContainerdImageNameKey] = newTotalReg
168+
found = true
169+
}
170+
return nil
171+
}); err != nil {
172+
return err
173+
}
174+
175+
if !found {
176+
return fmt.Errorf("could not find image [%s] in store", oldRef.Name())
177+
}
178+
179+
return s.OCI.SaveIndex()
180+
181+
}
182+
122183
func AddChartCmd(ctx context.Context, o *flags.AddChartOpts, s *store.Layout, chartName string) error {
123184
cfg := v1.Chart{
124185
Name: chartName,
125186
RepoURL: o.ChartOpts.RepoURL,
126187
Version: o.ChartOpts.Version,
127188
}
128189

129-
return storeChart(ctx, s, cfg, o.ChartOpts)
190+
rewrite := ""
191+
if o.Rewrite != "" {
192+
rewrite = o.Rewrite
193+
}
194+
return storeChart(ctx, s, cfg, o.ChartOpts, rewrite)
130195
}
131196

132-
func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action.ChartPathOptions) error {
197+
func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action.ChartPathOptions, rewrite string) error {
133198
l := log.FromContext(ctx)
134199

135200
l.Infof("adding chart [%s] to the store", cfg.Name)
@@ -152,11 +217,60 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *action
152217
if err != nil {
153218
return err
154219
}
220+
155221
_, err = s.AddOCI(ctx, chrt, ref.Name())
156222
if err != nil {
157223
return err
224+
} else {
225+
s.OCI.SaveIndex()
158226
}
159227

228+
if rewrite != "" {
229+
rewrite = strings.TrimPrefix(rewrite, "/")
230+
newRef, err := name.ParseReference(rewrite)
231+
if err != nil {
232+
l.Errorf("unable to parse rewrite name: %w", err)
233+
}
234+
235+
s.OCI.LoadIndex()
236+
237+
oldRefContext := ref.Context()
238+
newRefContext := newRef.Context()
239+
240+
oldRepo := oldRefContext.RepositoryStr()
241+
newRepo := newRefContext.RepositoryStr()
242+
oldTag := ref.(name.Tag).TagStr()
243+
244+
var newTag string
245+
if strings.Contains(rewrite, ":") {
246+
newTag = newRef.(name.Tag).TagStr()
247+
} else {
248+
newTag = oldTag
249+
}
250+
251+
oldTotal := oldRepo + ":" + oldTag
252+
newTotal := newRepo + ":" + newTag
253+
254+
found := false
255+
if err := s.OCI.Walk(func(k string, d ocispec.Descriptor) error {
256+
if d.Annotations[ocispec.AnnotationRefName] == oldTotal {
257+
d.Annotations[ocispec.AnnotationRefName] = newTotal
258+
found = true
259+
}
260+
return nil
261+
}); err != nil {
262+
return err
263+
}
264+
265+
if !found {
266+
return fmt.Errorf("could not find chart [%s] in store", ref.Name())
267+
}
268+
269+
cfg.Name = newRef.Name()
270+
fmt.Println("chart name (new): ", cfg.Name)
271+
272+
s.OCI.SaveIndex()
273+
}
160274
l.Infof("successfully added chart [%s]", ref.Name())
161275
return nil
162276
}

‎cmd/hauler/cli/store/load.go‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@ func LoadCmd(ctx context.Context, o *flags.LoadOpts, rso *flags.StoreRootOpts, r
4444
if err != nil {
4545
return err
4646
}
47+
clearDir(tempDir)
4748
}
4849

4950
return nil
@@ -137,3 +138,19 @@ func unarchiveLayoutTo(ctx context.Context, haulPath string, dest string, tempDi
137138
_, err = s.CopyAll(ctx, ts, nil)
138139
return err
139140
}
141+
142+
func clearDir(path string) error {
143+
entries, err := os.ReadDir(path)
144+
if err != nil {
145+
return err
146+
}
147+
148+
for _, entry := range entries {
149+
err = os.RemoveAll(filepath.Join(path, entry.Name()))
150+
if err != nil {
151+
return err
152+
}
153+
}
154+
155+
return nil
156+
}

‎cmd/hauler/cli/store/sync.go‎

Lines changed: 17 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ func SyncCmd(ctx context.Context, o *flags.SyncOpts, s *store.Layout, rso *flags
6363
img := v1.Image{
6464
Name: manifestLoc,
6565
}
66-
err := storeImage(ctx, s, img, o.Platform, rso, ro)
66+
err := storeImage(ctx, s, img, o.Platform, rso, ro, "")
6767
if err != nil {
6868
return err
6969
}
@@ -338,7 +338,12 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
338338
platform = i.Platform
339339
}
340340

341-
if err := storeImage(ctx, s, i, platform, rso, ro); err != nil {
341+
rewrite := ""
342+
if i.Rewrite != "" {
343+
rewrite = i.Rewrite
344+
}
345+
346+
if err := storeImage(ctx, s, i, platform, rso, ro, rewrite); err != nil {
342347
return err
343348
}
344349
}
@@ -473,7 +478,12 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
473478
platform = i.Platform
474479
}
475480

476-
if err := storeImage(ctx, s, i, platform, rso, ro); err != nil {
481+
rewrite := ""
482+
if i.Rewrite != "" {
483+
rewrite = i.Rewrite
484+
}
485+
486+
if err := storeImage(ctx, s, i, platform, rso, ro, rewrite); err != nil {
477487
return err
478488
}
479489
}
@@ -496,8 +506,8 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
496506
if err := convert.ConvertCharts(&alphaCfg, &v1Cfg); err != nil {
497507
return err
498508
}
499-
for _, ch := range v1Cfg.Spec.Charts {
500-
if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}); err != nil {
509+
for i, ch := range v1Cfg.Spec.Charts {
510+
if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}, v1Cfg.Spec.Charts[i].Rewrite); err != nil {
501511
return err
502512
}
503513
}
@@ -507,8 +517,8 @@ func processContent(ctx context.Context, fi *os.File, o *flags.SyncOpts, s *stor
507517
if err := yaml.Unmarshal(doc, &cfg); err != nil {
508518
return err
509519
}
510-
for _, ch := range cfg.Spec.Charts {
511-
if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}); err != nil {
520+
for i, ch := range cfg.Spec.Charts {
521+
if err := storeChart(ctx, s, ch, &action.ChartPathOptions{}, cfg.Spec.Charts[i].Rewrite); err != nil {
512522
return err
513523
}
514524
}

‎internal/flags/add.go‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ type AddImageOpts struct {
1616
CertGithubWorkflowRepository string
1717
Tlog bool
1818
Platform string
19+
Rewrite string
1920
}
2021

2122
func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
@@ -28,8 +29,11 @@ func (o *AddImageOpts) AddFlags(cmd *cobra.Command) {
2829
f.StringVar(&o.CertGithubWorkflowRepository, "certificate-github-workflow-repository", "", "(Optional) Cosign certificate-github-workflow-repository option")
2930
f.BoolVarP(&o.Tlog, "use-tlog-verify", "v", false, "(Optional) Allow transparency log verification. (defaults to false)")
3031
f.StringVarP(&o.Platform, "platform", "p", "", "(Optional) Specifiy the platform of the image... i.e. linux/amd64 (defaults to all)")
32+
f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string")
3133
}
3234

35+
//func (o *AddImageOpts) RewriteValue() string { return o.Rewrite }
36+
3337
type AddFileOpts struct {
3438
*StoreRootOpts
3539
Name string
@@ -44,6 +48,7 @@ type AddChartOpts struct {
4448
*StoreRootOpts
4549

4650
ChartOpts *action.ChartPathOptions
51+
Rewrite string
4752
}
4853

4954
func (o *AddChartOpts) AddFlags(cmd *cobra.Command) {
@@ -58,4 +63,5 @@ func (o *AddChartOpts) AddFlags(cmd *cobra.Command) {
5863
f.StringVar(&o.ChartOpts.KeyFile, "key-file", "", "(Optional) Location of the TLS Key to use for client authenication")
5964
f.BoolVar(&o.ChartOpts.InsecureSkipTLSverify, "insecure-skip-tls-verify", false, "(Optional) Skip TLS certificate verification")
6065
f.StringVar(&o.ChartOpts.CaFile, "ca-file", "", "(Optional) Location of CA Bundle to enable certification verification")
66+
f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string")
6167
}

‎internal/flags/sync.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@ type SyncOpts struct {
2020
ProductRegistry string
2121
TempOverride string
2222
Tlog bool
23+
Rewrite string
2324
}
2425

2526
func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
@@ -38,4 +39,7 @@ func (o *SyncOpts) AddFlags(cmd *cobra.Command) {
3839
f.StringVarP(&o.ProductRegistry, "product-registry", "c", "", "(Optional) Specify the product registry. Defaults to RGS Carbide Registry (rgcrprod.azurecr.us)")
3940
f.StringVarP(&o.TempOverride, "tempdir", "t", "", "(Optional) Override the default temporary directiory determined by the OS")
4041
f.BoolVarP(&o.Tlog, "use-tlog-verify", "v", false, "(Optional) Allow transparency log verification. (defaults to false)")
42+
f.StringVar(&o.Rewrite, "rewrite", "", "(Optional) Rewrite artifact path to specified string")
4143
}
44+
45+
//func (o *SyncOpts) RewriteValue() string { return o.Rewrite }

‎pkg/apis/hauler.cattle.io/v1/chart.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ type Chart struct {
1919
Name string `json:"name,omitempty"`
2020
RepoURL string `json:"repoURL,omitempty"`
2121
Version string `json:"version,omitempty"`
22+
Rewrite string `json:"rewrite,omitempty"`
2223
}
2324

2425
type ThickCharts struct {

0 commit comments

Comments
 (0)