Skip to content

Commit e2a5950

Browse files
amartin120amartin120
authored
over-"haul": replace oras v1 and cosign fork with native containerd-based implementation (#515)
* remove oras from hauler Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * remove cosign fork and use upstream cosign for verification Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * added support for oci referrers Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * updated README.md projects list Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * updates for copilot PR review Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * bug fix for unsafe type assertions Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * bug fix for http getter and dead code Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * fixes for more clarity and better error handling Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * bug fix for resource leaks and unchecked errors Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * bug fix for rewrite logic for docker.io images due to cosign removal Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * bug fix for sigs and referrers Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * bug fix for index.json missing mediatype Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> * bug fix to make sure manifest.json doesnt include anything other than actual container images Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com> --------- Signed-off-by: Adam Martin <adam.martin@ranchergovernment.com>
1 parent 26b11d5 commit e2a5950

28 files changed

Lines changed: 2310 additions & 562 deletions

File tree

‎README.md‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -70,7 +70,6 @@ brew install hauler
7070
## Acknowledgements
7171

7272
`Hauler` wouldn't be possible without the open-source community, but there are a few projects that stand out:
73-
74-
- [oras cli](https://github.com/oras-project/oras)
75-
- [cosign](https://github.com/sigstore/cosign)
73+
- [containerd](https://github.com/containerd/containerd)
7674
- [go-containerregistry](https://github.com/google/go-containerregistry)
75+
- [cosign](https://github.com/sigstore/cosign)

‎cmd/hauler/cli/cli.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,7 @@ import (
44
"context"
55

66
cranecmd "github.com/google/go-containerregistry/cmd/crane/cmd"
7+
"github.com/sirupsen/logrus"
78
"github.com/spf13/cobra"
89
"hauler.dev/go/hauler/internal/flags"
910
"hauler.dev/go/hauler/pkg/consts"
@@ -20,6 +21,14 @@ func New(ctx context.Context, ro *flags.CliRootOpts) *cobra.Command {
2021
l.SetLevel(ro.LogLevel)
2122
l.Debugf("running cli command [%s]", cmd.CommandPath())
2223

24+
// Suppress WARN-level messages from containerd and other
25+
// libraries that use the global logrus logger.
26+
if ro.LogLevel == "debug" {
27+
logrus.SetLevel(logrus.DebugLevel)
28+
} else {
29+
logrus.SetLevel(logrus.ErrorLevel)
30+
}
31+
2332
return nil
2433
},
2534
RunE: func(cmd *cobra.Command, args []string) error {

‎cmd/hauler/cli/store.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -216,7 +216,7 @@ func addStoreSave(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
216216
func addStoreInfo(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Command {
217217
o := &flags.InfoOpts{StoreRootOpts: rso}
218218

219-
var allowedValues = []string{"image", "chart", "file", "sigs", "atts", "sbom", "all"}
219+
var allowedValues = []string{"image", "chart", "file", "sigs", "atts", "sbom", "referrer", "all"}
220220

221221
cmd := &cobra.Command{
222222
Use: "info",

‎cmd/hauler/cli/store/add.go‎

Lines changed: 57 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,7 @@ import (
2525
"hauler.dev/go/hauler/pkg/getter"
2626
"hauler.dev/go/hauler/pkg/log"
2727
"hauler.dev/go/hauler/pkg/reference"
28+
"hauler.dev/go/hauler/pkg/retry"
2829
"hauler.dev/go/hauler/pkg/store"
2930
)
3031

@@ -52,7 +53,7 @@ func storeFile(ctx context.Context, s *store.Layout, fi v1.File) error {
5253
}
5354

5455
l.Infof("adding file [%s] to the store as [%s]", fi.Path, ref.Name())
55-
_, err = s.AddOCI(ctx, f, ref.Name())
56+
_, err = s.AddArtifact(ctx, f, ref.Name())
5657
if err != nil {
5758
return err
5859
}
@@ -73,15 +74,18 @@ func AddImageCmd(ctx context.Context, o *flags.AddImageOpts, s *store.Layout, re
7374
// Check if the user provided a key.
7475
if o.Key != "" {
7576
// verify signature using the provided key.
76-
err := cosign.VerifySignature(ctx, s, o.Key, o.Tlog, cfg.Name, rso, ro)
77+
err := cosign.VerifySignature(ctx, o.Key, o.Tlog, cfg.Name, rso, ro)
7778
if err != nil {
7879
return err
7980
}
8081
l.Infof("signature verified for image [%s]", cfg.Name)
8182
} else if o.CertIdentityRegexp != "" || o.CertIdentity != "" {
82-
// verify signature using keyless details
83+
// verify signature using keyless details.
84+
// Keyless (Fulcio) certificates expire after ~10 minutes, so the transparency
85+
// log is always required to prove the cert was valid at signing time — ignore
86+
// --use-tlog-verify for this path and always check tlog.
8387
l.Infof("verifying keyless signature for [%s]", cfg.Name)
84-
err := cosign.VerifyKeylessSignature(ctx, s, o.CertIdentity, o.CertIdentityRegexp, o.CertOidcIssuer, o.CertOidcIssuerRegexp, o.CertGithubWorkflowRepository, o.Tlog, cfg.Name, rso, ro)
88+
err := cosign.VerifyKeylessSignature(ctx, o.CertIdentity, o.CertIdentityRegexp, o.CertOidcIssuer, o.CertOidcIssuerRegexp, o.CertGithubWorkflowRepository, cfg.Name, rso, ro)
8589
if err != nil {
8690
return err
8791
}
@@ -114,8 +118,10 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
114118
}
115119
}
116120

117-
// copy and sig verification
118-
err = cosign.SaveImage(ctx, s, r.Name(), platform, rso, ro)
121+
// fetch image along with any associated signatures and attestations
122+
err = retry.Operation(ctx, rso, ro, func() error {
123+
return s.AddImage(ctx, r.Name(), platform)
124+
})
119125
if err != nil {
120126
if ro.IgnoreErrors {
121127
l.Warnf("unable to add image [%s] to store: %v... skipping...", r.Name(), err)
@@ -129,14 +135,20 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
129135
if rewrite != "" {
130136
rewrite = strings.TrimPrefix(rewrite, "/")
131137
if !strings.Contains(rewrite, ":") {
132-
rewrite = strings.Join([]string{rewrite, r.(name.Tag).TagStr()}, ":")
138+
if tag, ok := r.(name.Tag); ok {
139+
rewrite = rewrite + ":" + tag.TagStr()
140+
} else {
141+
return fmt.Errorf("cannot rewrite digest reference [%s] without an explicit tag in the rewrite", r.Name())
142+
}
133143
}
134144
// rename image name in store
135145
newRef, err := name.ParseReference(rewrite)
136146
if err != nil {
137-
l.Errorf("unable to parse rewrite name: %w", err)
147+
return fmt.Errorf("unable to parse rewrite name [%s]: %w", rewrite, err)
148+
}
149+
if err := rewriteReference(ctx, s, r, newRef); err != nil {
150+
return err
138151
}
139-
rewriteReference(ctx, s, r, newRef)
140152
}
141153

142154
l.Infof("successfully added image [%s]", r.Name())
@@ -146,19 +158,33 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
146158
func rewriteReference(ctx context.Context, s *store.Layout, oldRef name.Reference, newRef name.Reference) error {
147159
l := log.FromContext(ctx)
148160

149-
s.OCI.LoadIndex()
161+
if err := s.OCI.LoadIndex(); err != nil {
162+
return fmt.Errorf("failed to load index: %w", err)
163+
}
150164

151165
//TODO: improve string manipulation
152166
oldRefContext := oldRef.Context()
153167
newRefContext := newRef.Context()
154168
oldRepo := oldRefContext.RepositoryStr()
155169
newRepo := newRefContext.RepositoryStr()
156-
oldTag := oldRef.(name.Tag).TagStr()
157-
newTag := newRef.(name.Tag).TagStr()
158-
oldRegistry := strings.TrimPrefix(oldRefContext.RegistryStr(), "index.")
159-
newRegistry := strings.TrimPrefix(newRefContext.RegistryStr(), "index.")
160-
// If new registry not set in rewrite, keep old registry instead of defaulting to docker.io
161-
if newRegistry == "docker.io" && oldRegistry != "docker.io" {
170+
171+
oldTag := oldRef.Identifier()
172+
if tag, ok := oldRef.(name.Tag); ok {
173+
oldTag = tag.TagStr()
174+
}
175+
newTag := newRef.Identifier()
176+
if tag, ok := newRef.(name.Tag); ok {
177+
newTag = tag.TagStr()
178+
}
179+
180+
// ContainerdImageNameKey stores annotationRef.Name() verbatim, which includes the
181+
// "index.docker.io" prefix for docker.io images. Do not strip "index." here or the
182+
// comparison will never match images stored by writeImage/writeIndex.
183+
oldRegistry := oldRefContext.RegistryStr()
184+
newRegistry := newRefContext.RegistryStr()
185+
// If user omitted a registry in the rewrite string, go-containerregistry defaults to
186+
// index.docker.io. Preserve the original registry when the source is non-docker.
187+
if newRegistry == "index.docker.io" && oldRegistry != "index.docker.io" {
162188
newRegistry = oldRegistry
163189
}
164190
oldTotal := oldRepo + ":" + oldTag
@@ -349,7 +375,7 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags.
349375
return err
350376
}
351377

352-
if _, err := s.AddOCI(ctx, chrt, ref.Name()); err != nil {
378+
if _, err := s.AddArtifact(ctx, chrt, ref.Name()); err != nil {
353379
return err
354380
}
355381
if err := s.OCI.SaveIndex(); err != nil {
@@ -501,7 +527,9 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags.
501527
}
502528
return fmt.Errorf("failed to store image [%s]: %w", image, err)
503529
}
504-
s.OCI.LoadIndex()
530+
if err := s.OCI.LoadIndex(); err != nil {
531+
return err
532+
}
505533
if err := s.OCI.SaveIndex(); err != nil {
506534
return err
507535
}
@@ -558,7 +586,10 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags.
558586
}
559587

560588
// if rewrite omits a tag... keep the existing tag
561-
oldTag := ref.(name.Tag).TagStr()
589+
oldTag := ref.Identifier()
590+
if tag, ok := ref.(name.Tag); ok {
591+
oldTag = tag.TagStr()
592+
}
562593
if !strings.Contains(rewrite, ":") {
563594
rewrite = strings.Join([]string{rewrite, oldTag}, ":")
564595
newRef, err = name.ParseReference(rewrite)
@@ -568,14 +599,19 @@ func storeChart(ctx context.Context, s *store.Layout, cfg v1.Chart, opts *flags.
568599
}
569600

570601
// rename chart name in store
571-
s.OCI.LoadIndex()
602+
if err := s.OCI.LoadIndex(); err != nil {
603+
return err
604+
}
572605

573606
oldRefContext := ref.Context()
574607
newRefContext := newRef.Context()
575608

576609
oldRepo := oldRefContext.RepositoryStr()
577610
newRepo := newRefContext.RepositoryStr()
578-
newTag := newRef.(name.Tag).TagStr()
611+
newTag := newRef.Identifier()
612+
if tag, ok := newRef.(name.Tag); ok {
613+
newTag = tag.TagStr()
614+
}
579615

580616
oldTotal := oldRepo + ":" + oldTag
581617
newTotal := newRepo + ":" + newTag

0 commit comments

Comments
 (0)