Skip to content

Commit 00f1ccf

Browse files
zackbradyszackbradys
committed
added hauler copy for direct oci copies
1 parent a5d9343 commit 00f1ccf

10 files changed

Lines changed: 447 additions & 229 deletions

File tree

‎cmd/hauler/cli/cli.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -69,6 +69,7 @@ func New(ctx context.Context, ro *flags.CliRootOpts) *cobra.Command {
6969
cmd.AddCommand(cranecmd.NewCmdAuthLogin("hauler"))
7070
cmd.AddCommand(cranecmd.NewCmdAuthLogout("hauler"))
7171
addStore(cmd, ro)
72+
addCopy(cmd, ro)
7273
addVersion(cmd, ro)
7374
addCompletion(cmd, ro)
7475

‎cmd/hauler/cli/copy.go‎

Lines changed: 176 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,176 @@
1+
package cli
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"time"
7+
8+
"github.com/google/go-containerregistry/pkg/authn"
9+
gname "github.com/google/go-containerregistry/pkg/name"
10+
gv1 "github.com/google/go-containerregistry/pkg/v1"
11+
"github.com/google/go-containerregistry/pkg/v1/remote"
12+
"github.com/spf13/cobra"
13+
14+
"hauler.dev/go/hauler/v2/internal/flags"
15+
"hauler.dev/go/hauler/v2/pkg/audit"
16+
"hauler.dev/go/hauler/v2/pkg/content"
17+
"hauler.dev/go/hauler/v2/pkg/log"
18+
"hauler.dev/go/hauler/v2/pkg/retry"
19+
"hauler.dev/go/hauler/v2/pkg/store"
20+
)
21+
22+
func addCopy(parent *cobra.Command, ro *flags.CliRootOpts) {
23+
o := &flags.ImageCopyOpts{}
24+
25+
cmd := &cobra.Command{
26+
Use: "copy SRC DST",
27+
Aliases: []string{"cp"},
28+
Short: "Copy an artifact between registries",
29+
Example: ` # copy an image to another registry
30+
hauler copy busybox:latest registry.example.com/busybox:latest
31+
32+
# copy a specific platform out of a multi-arch image
33+
hauler copy ghcr.io/hauler-dev/hauler-debug:v2.0.3 registry.example.com/hauler-debug:v2.0.3 --platform linux/amd64
34+
35+
# copy to a registry with a self-signed certificate
36+
hauler copy busybox:latest registry.example.com/busybox:latest --insecure-skip-tls-verify
37+
38+
# copy to a registry with no TLS at all
39+
hauler copy busybox:latest registry.example.com/busybox:latest --plain-http`,
40+
Args: cobra.ExactArgs(2),
41+
RunE: func(cmd *cobra.Command, args []string) error {
42+
return CopyImageCmd(cmd.Context(), o, args[0], args[1], ro)
43+
},
44+
}
45+
o.AddFlags(cmd)
46+
parent.AddCommand(cmd)
47+
}
48+
49+
// CopyImageCmd copies src to dst directly, registry to registry, no store involved.
50+
func CopyImageCmd(ctx context.Context, o *flags.ImageCopyOpts, src, dst string, ro *flags.CliRootOpts) error {
51+
l := log.FromContext(ctx)
52+
53+
retries, err := flags.ResolveRetries(o.Retries)
54+
if err != nil {
55+
return err
56+
}
57+
58+
tr, err := content.BuildTransport(o.InsecureSkipTLSVerify, o.CaFile)
59+
if err != nil {
60+
return err
61+
}
62+
63+
opts := []remote.Option{
64+
remote.WithAuthFromKeychain(authn.DefaultKeychain),
65+
remote.WithContext(ctx),
66+
remote.WithTransport(tr),
67+
}
68+
69+
var nameOpts []gname.Option
70+
if o.PlainHTTP {
71+
nameOpts = append(nameOpts, gname.Insecure)
72+
}
73+
74+
srcRef, err := gname.ParseReference(src, nameOpts...)
75+
if err != nil {
76+
return fmt.Errorf("parsing source reference %q: %w", src, err)
77+
}
78+
dstRef, err := gname.ParseReference(dst, nameOpts...)
79+
if err != nil {
80+
return fmt.Errorf("parsing destination reference %q: %w", dst, err)
81+
}
82+
83+
l.Infof("copying [%s] to [%s]", src, dst)
84+
85+
start := time.Now()
86+
var digest string
87+
err = retry.Operation(ctx, &flags.StoreRootOpts{Retries: retries}, ro, func() error {
88+
d, copyErr := copyOnce(srcRef, dstRef, o.Platform, opts)
89+
if copyErr == nil {
90+
digest = d
91+
}
92+
return copyErr
93+
})
94+
if err != nil {
95+
l.Errorf("unable to copy [%s] to [%s]: %v", src, dst, err)
96+
return err
97+
}
98+
99+
if flags.AuditLevel(ro) != "none" {
100+
e := audit.Entry{
101+
Command: "copy",
102+
Args: []string{src, dst},
103+
Type: "image",
104+
Reference: dst,
105+
Digest: digest,
106+
}
107+
if flags.AuditLevel(ro) == "verbose" {
108+
sys := audit.BuildSystem()
109+
g := audit.BuildGlobal(ro, nil)
110+
e.System = &sys
111+
e.Global = &g
112+
e.Flags = map[string]any{
113+
"insecure-skip-tls-verify": o.InsecureSkipTLSVerify,
114+
"plain-http": o.PlainHTTP,
115+
"ca-file": o.CaFile,
116+
"platform": o.Platform,
117+
}
118+
}
119+
if err := audit.Append(ro.HaulerDir, e); err != nil {
120+
l.Warnf("failed to write audit entry: %v", err)
121+
}
122+
l.Debugf("generated audit id of [%s]", audit.ID())
123+
} else {
124+
l.Debugf("generated audit id of [none]")
125+
}
126+
127+
l.Infof("✓ copied [%s] to [%s] (%.1fs)", src, dst, time.Since(start).Seconds())
128+
129+
return nil
130+
}
131+
132+
// copyOnce copies srcRef to dstRef and returns the digest copied. No
133+
// platform filter keeps a multi-arch index intact; platform picks one child.
134+
func copyOnce(srcRef, dstRef gname.Reference, platform string, opts []remote.Option) (string, error) {
135+
desc, err := remote.Get(srcRef, opts...)
136+
if err != nil {
137+
return "", fmt.Errorf("fetching descriptor for %q: %w", srcRef.Name(), err)
138+
}
139+
140+
if idx, idxErr := desc.ImageIndex(); idxErr == nil && platform == "" {
141+
if err := remote.WriteIndex(dstRef, idx, opts...); err != nil {
142+
return "", fmt.Errorf("writing index for %q: %w", dstRef.Name(), err)
143+
}
144+
d, err := idx.Digest()
145+
if err != nil {
146+
return "", fmt.Errorf("getting index digest for %q: %w", srcRef.Name(), err)
147+
}
148+
return d.String(), nil
149+
}
150+
151+
var img gv1.Image
152+
if platform != "" {
153+
p, err := store.ParsePlatform(platform)
154+
if err != nil {
155+
return "", err
156+
}
157+
img, err = remote.Image(srcRef, append(append([]remote.Option{}, opts...), remote.WithPlatform(p))...)
158+
if err != nil {
159+
return "", fmt.Errorf("fetching image %q: %w", srcRef.Name(), err)
160+
}
161+
} else {
162+
img, err = desc.Image()
163+
if err != nil {
164+
return "", fmt.Errorf("fetching image %q: %w", srcRef.Name(), err)
165+
}
166+
}
167+
168+
if err := remote.Write(dstRef, img, opts...); err != nil {
169+
return "", fmt.Errorf("writing image for %q: %w", dstRef.Name(), err)
170+
}
171+
d, err := img.Digest()
172+
if err != nil {
173+
return "", fmt.Errorf("getting image digest for %q: %w", srcRef.Name(), err)
174+
}
175+
return d.String(), nil
176+
}

‎cmd/hauler/cli/store.go‎

Lines changed: 56 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -310,8 +310,8 @@ func addStoreCopy(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
310310
Use: "copy",
311311
Short: "Copy all store content to another location",
312312
Example: ` # supported copy target prefixes
313-
registry:// | reg:// | oci:// - Pushes the store to an OCI registry
314-
directory:// | dir:// - Extracts the store to a directory`,
313+
registry:// | reg:// | oci:// - Pushes the store to an OCI registry
314+
directory:// | dir:// - Extracts the store to a directory`,
315315
Args: cobra.ExactArgs(1),
316316
RunE: func(cmd *cobra.Command, args []string) error {
317317
ctx := cmd.Context()
@@ -354,13 +354,13 @@ func addStoreAddFile(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Com
354354
Use: "file",
355355
Short: "Add a file to the store",
356356
Example: ` # fetch local file
357-
hauler store add file file.txt
357+
hauler store add file file.txt
358358
359-
# fetch remote file
360-
hauler store add file https://get.rke2.io/install.sh
359+
# fetch remote file
360+
hauler store add file https://get.rke2.io/install.sh
361361
362-
# fetch remote file and assign new name
363-
hauler store add file https://get.hauler.dev --name hauler-install.sh`,
362+
# fetch remote file and assign new name
363+
hauler store add file https://get.hauler.dev --name hauler-install.sh`,
364364
Args: cobra.ExactArgs(1),
365365
RunE: func(cmd *cobra.Command, args []string) error {
366366
ctx := cmd.Context()
@@ -384,27 +384,27 @@ func addStoreAddImage(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co
384384
cmd := &cobra.Command{
385385
Use: "image",
386386
Short: "Add a image to the store",
387-
Example: ` # fetch image
388-
hauler store add image busybox
387+
Example: ` # fetch image
388+
hauler store add image busybox
389389
390-
# fetch image with repository and tag
391-
hauler store add image library/busybox:stable
390+
# fetch image with repository and tag
391+
hauler store add image library/busybox:stable
392392
393-
# fetch image with full image reference and specific platform
394-
hauler store add image ghcr.io/hauler-dev/hauler-debug:v1.2.0 --platform linux/amd64
393+
# fetch image with full image reference and specific platform
394+
hauler store add image ghcr.io/hauler-dev/hauler-debug:v1.2.0 --platform linux/amd64
395395
396-
# fetch image with full image reference via digest
397-
hauler store add image gcr.io/distroless/base@sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5
396+
# fetch image with full image reference via digest
397+
hauler store add image gcr.io/distroless/base@sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5
398398
399-
# fetch image with full image reference, specific platform, and signature verification
400-
curl -sfOL https://raw.githubusercontent.com/rancherfederal/carbide-releases/main/carbide-key.pub
401-
hauler store add image rgcrprod.azurecr.us/rancher/rke2-runtime:v1.31.5-rke2r1 --platform linux/amd64 --key carbide-key.pub
399+
# fetch image with full image reference, specific platform, and signature verification
400+
curl -sfOL https://raw.githubusercontent.com/rancherfederal/carbide-releases/main/carbide-key.pub
401+
hauler store add image rgcrprod.azurecr.us/rancher/rke2-runtime:v1.31.5-rke2r1 --platform linux/amd64 --key carbide-key.pub
402402
403-
# fetch image and rewrite path
404-
hauler store add image busybox --rewrite custom-path/busybox:latest
403+
# fetch image and rewrite path
404+
hauler store add image busybox --rewrite custom-path/busybox:latest
405405
406-
# add image from local Docker daemon
407-
hauler store add image my-local-app:latest --local`,
406+
# add image from local Docker daemon
407+
hauler store add image my-local-app:latest --local`,
408408
Args: cobra.ExactArgs(1),
409409
PreRunE: func(cmd *cobra.Command, args []string) error {
410410
// Check for ca-file & insecure-skip-tls-verify env variables
@@ -417,6 +417,12 @@ func addStoreAddImage(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co
417417
o.InsecureSkipTLSVerify = &b
418418
}
419419
}
420+
421+
// resolve *bool: nil unless the user explicitly passed the flag
422+
if cmd.Flags().Changed("insecure-skip-tls-verify") {
423+
v, _ := cmd.Flags().GetBool("insecure-skip-tls-verify")
424+
o.InsecureSkipTLSVerify = &v
425+
}
420426
return nil
421427
},
422428
RunE: func(cmd *cobra.Command, args []string) error {
@@ -441,26 +447,26 @@ func addStoreAddChart(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co
441447
cmd := &cobra.Command{
442448
Use: "chart",
443449
Short: "Add a helm chart to the store",
444-
Example: ` # fetch local helm chart
445-
hauler store add chart path/to/chart/directory --repo .
450+
Example: ` # fetch local helm chart
451+
hauler store add chart path/to/chart/directory --repo .
446452
447-
# fetch local compressed helm chart
448-
hauler store add chart path/to/chart.tar.gz --repo .
453+
# fetch local compressed helm chart
454+
hauler store add chart path/to/chart.tar.gz --repo .
449455
450-
# fetch remote oci helm chart
451-
hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev
456+
# fetch remote oci helm chart
457+
hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev
452458
453-
# fetch remote oci helm chart with version
454-
hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --version 1.2.0
459+
# fetch remote oci helm chart with version
460+
hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --version 1.2.0
455461
456-
# fetch remote helm chart
457-
hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable
462+
# fetch remote helm chart
463+
hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable
458464
459-
# fetch remote helm chart with specific version
460-
hauler store add chart rancher --repo https://releases.rancher.com/server-charts/latest --version 2.10.1
465+
# fetch remote helm chart with specific version
466+
hauler store add chart rancher --repo https://releases.rancher.com/server-charts/latest --version 2.10.1
461467
462-
# fetch remote helm chart and rewrite path
463-
hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --rewrite custom-path/hauler-chart:latest`,
468+
# fetch remote helm chart and rewrite path
469+
hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --rewrite custom-path/hauler-chart:latest`,
464470
Args: cobra.ExactArgs(1),
465471
PreRunE: func(cmd *cobra.Command, args []string) error {
466472
n, err := flags.ResolveConcurrency(cmd.Flags().Changed("concurrency"), o.Concurrency)
@@ -502,25 +508,25 @@ func addStoreRemove(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comm
502508
Use: "remove <artifact-ref>",
503509
Short: "Remove an artifact from the content store",
504510
Example: ` # remove an image using full store reference
505-
hauler store info
506-
hauler store remove index.docker.io/library/busybox:stable
511+
hauler store info
512+
hauler store remove index.docker.io/library/busybox:stable
507513
508-
# remove a chart using full store reference
509-
hauler store info
510-
hauler store remove hauler/rancher:2.8.4
514+
# remove a chart using full store reference
515+
hauler store info
516+
hauler store remove hauler/rancher:2.8.4
511517
512-
# remove a file using full store reference
513-
hauler store info
514-
hauler store remove hauler/rke2-install.sh
518+
# remove a file using full store reference
519+
hauler store info
520+
hauler store remove hauler/rke2-install.sh
515521
516-
# remove any artifact with the latest tag
517-
hauler store remove :latest
522+
# remove any artifact with the latest tag
523+
hauler store remove :latest
518524
519-
# remove any artifact with 'busybox' in the reference
520-
hauler store remove busybox
525+
# remove any artifact with 'busybox' in the reference
526+
hauler store remove busybox
521527
522-
# force remove without verification
523-
hauler store remove busybox:latest --force`,
528+
# force remove without verification
529+
hauler store remove busybox:latest --force`,
524530
Args: cobra.ExactArgs(1),
525531
RunE: func(cmd *cobra.Command, args []string) error {
526532
ctx := cmd.Context()

‎cmd/hauler/cli/store/add.go‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -389,7 +389,7 @@ func storeLocalImage(ctx context.Context, s *store.Layout, i v1.Image, _ *flags.
389389
start := time.Now()
390390
ignoreErrors := flags.ShouldIgnoreErrors(ro)
391391

392-
l.Debugf("adding image [%s] from local Docker daemon to the store", i.Name)
392+
l.Debugf("resolving image [%s] from local Docker daemon (rewrite=%q)", i.Name, rewrite)
393393

394394
r, err := name.ParseReference(i.Name)
395395
if err != nil {
@@ -480,7 +480,11 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
480480
return err
481481
}
482482

483-
log.BaseFromContext(ctx).Debugf("adding image [%s] to the store", i.Name)
483+
insecureSkipTLSVerify := derefInsecure(i.InsecureSkipTLSVerify)
484+
caFile := i.CaFile
485+
486+
log.BaseFromContext(ctx).Debugf("resolving image [%s] (platform=%q, excludeExtras=%t, verified=%t, insecureSkipTLSVerify=%t, caFile=%q, rewrite=%q, digest=%q)",
487+
i.Name, platform, excludeExtras, verified, insecureSkipTLSVerify, caFile, rewrite, pinnedDigest)
484488

485489
r, err := name.ParseReference(i.Name)
486490
if err != nil {
@@ -493,9 +497,6 @@ func storeImage(ctx context.Context, s *store.Layout, i v1.Image, platform strin
493497
}
494498
}
495499

496-
insecureSkipTLSVerify := derefInsecure(i.InsecureSkipTLSVerify)
497-
caFile := i.CaFile
498-
499500
// fetch image along with any associated signatures and attestations.
500501
// A fresh store.ImageStats is built inside the closure on every attempt,
501502
// not once outside it, so a failed attempt's partial layer/byte counts

‎cmd/hauler/cli/store/audit.go‎

Lines changed: 1 addition & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,5 @@ import "hauler.dev/go/hauler/v2/internal/flags"
44

55
// auditLevel returns the resolved audit level (none, standard, verbose)
66
func auditLevel(ro *flags.CliRootOpts) string {
7-
if ro == nil {
8-
return "none"
9-
}
10-
if ro.AuditLevel == "" {
11-
return "standard"
12-
}
13-
return ro.AuditLevel
7+
return flags.AuditLevel(ro)
148
}

0 commit comments

Comments
 (0)