.gitattributes

@@ -5,3 +5,4 @@
 **/zz_deepcopy_generated.go linguist-generated=true
 cmd/crane/doc/crane*.md linguist-generated=true
 go.sum linguist-generated=true
+**/testdata/** ignore-lint=true

.github/workflows/release.yml

@@ -20,7 +20,7 @@ jobs:
       - uses: goreleaser/goreleaser-action@v4.2.0
         id: run-goreleaser
         with:
-          version: v1.18.2
+          version: "~> v1.19"
           args: release --rm-dist
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.gitignore

@@ -3,6 +3,7 @@
 bazel*
 .idea
 *.iml
+dist/
 
 cmd/crane/crane
 cmd/gcrane/gcrane

.goreleaser.yml

@@ -91,13 +91,13 @@ builds:
 source:
   enabled: true
 archives:
-- replacements:
-    darwin: Darwin
-    linux: Linux
-    windows: Windows
-    386: i386
-    amd64: x86_64
-  name_template: "{{ .ProjectName }}_{{ .Os }}_{{ .Arch }}{{ if .Arm }}v{{ .Arm }}{{ end }}"
+- name_template: >-
+      {{ .ProjectName }}_
+      {{- title .Os }}_
+      {{- if eq .Arch "amd64" }}x86_64
+      {{- else if eq .Arch "386" }}i386
+      {{- else }}{{ .Arch }}{{ end }}
+      {{- if .Arm }}v{{ .Arm }}{{ end -}}
 checksum:
   name_template: 'checksums.txt'
 snapshot:

cmd/crane/cmd/gc.go

@@ -0,0 +1,66 @@
+// Copyright 2018 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	"github.com/google/go-containerregistry/pkg/v1/layout"
+	"github.com/spf13/cobra"
+)
+
+func NewCmdLayout() *cobra.Command {
+	cmd := &cobra.Command{
+		Use: "layout",
+	}
+	cmd.AddCommand(newCmdGc())
+	return cmd
+}
+
+// NewCmdGc creates a new cobra.Command for the pull subcommand.
+func newCmdGc() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:    "gc OCI-LAYOUT",
+		Short:  "Garbage collect unreferenced blobs in a local oci-layout",
+		Args:   cobra.ExactArgs(1),
+		Hidden: true, // TODO: promote to public once theres some milage
+		RunE: func(_ *cobra.Command, args []string) error {
+			path := args[0]
+
+			p, err := layout.FromPath(path)
+
+			if err != nil {
+				return err
+			}
+
+			blobs, err := p.GarbageCollect()
+			if err != nil {
+				return err
+			}
+
+			for _, blob := range blobs {
+				if err := p.RemoveBlob(blob); err != nil {
+					return err
+				}
+				fmt.Fprintf(os.Stderr, "garbage collecting: %s\n", blob.String())
+			}
+
+			return nil
+		},
+	}
+
+	return cmd
+}

cmd/crane/cmd/push.go

@@ -71,7 +71,9 @@ func NewCmdPush(options *[]crane.Option) *cobra.Command {
 
 			digest := ref.Context().Digest(h.String())
 			if imageRefs != "" {
-				return os.WriteFile(imageRefs, []byte(digest.String()), 0600)
+				if err := os.WriteFile(imageRefs, []byte(digest.String()), 0600); err != nil {
+					return fmt.Errorf("failed to write image refs to %s: %w", imageRefs, err)
+				}
 			}
 
 			// Print the digest of the pushed image to stdout to facilitate command composition.

cmd/crane/cmd/root.go

@@ -129,7 +129,8 @@ func New(use, short string, options []crane.Option) *cobra.Command {
 		NewCmdTag(&options),
 		NewCmdValidate(&options),
 		NewCmdVersion(),
-		newCmdRegistry(),
+		NewCmdRegistry(),
+		NewCmdLayout(),
 	)
 
 	root.PersistentFlags().BoolVarP(&verbose, "verbose", "v", false, "Enable debug logs")

cmd/crane/cmd/serve.go

@@ -28,7 +28,7 @@ import (
 	"github.com/google/go-containerregistry/pkg/registry"
 )
 
-func newCmdRegistry() *cobra.Command {
+func NewCmdRegistry() *cobra.Command {
 	cmd := &cobra.Command{
 		Use: "registry",
 	}
@@ -37,15 +37,16 @@ func newCmdRegistry() *cobra.Command {
 }
 
 func newCmdServe() *cobra.Command {
-	var disk bool
+	var address, disk string
+	var blobsToDisk bool
 	cmd := &cobra.Command{
 		Use:   "serve",
-		Short: "Serve an in-memory registry implementation",
-		Long: `This sub-command serves an in-memory registry implementation on an automatically chosen port (or $PORT)
+		Short: "Serve a registry implementation",
+		Long: `This sub-command serves a registry implementation on an automatically chosen port (:0), $PORT or --address
 
 The command blocks while the server accepts pushes and pulls.
 
-Contents are only stored in memory, and when the process exits, pushed data is lost.`,
+Contents are can be stored in memory (when the process exits, pushed data is lost.), and disk (--disk).`,
 		Args: cobra.NoArgs,
 		RunE: func(cmd *cobra.Command, _ []string) error {
 			ctx := cmd.Context()
@@ -54,18 +55,34 @@ Contents are only stored in memory, and when the process exits, pushed data is l
 			if port == "" {
 				port = "0"
 			}
-			listener, err := net.Listen("tcp", ":"+port)
+			listenOn := ":" + port
+			if address != "" {
+				listenOn = address
+			}
+
+			listener, err := net.Listen("tcp", listenOn)
 			if err != nil {
 				log.Fatalln(err)
 			}
 			porti := listener.Addr().(*net.TCPAddr).Port
 			port = fmt.Sprintf("%d", porti)
 
 			bh := registry.NewInMemoryBlobHandler()
-			if disk {
-				tmp := os.TempDir()
-				log.Printf("storing blobs in %s", tmp)
-				bh = registry.NewDiskBlobHandler(tmp)
+
+			diskp := disk
+			if cmd.Flags().Changed("blobs-to-disk") {
+				if disk != "" {
+					return fmt.Errorf("--disk and --blobs-to-disk can't be used together")
+				}
+				diskp, err = os.MkdirTemp(os.TempDir(), "craneregistry*")
+				if err != nil {
+					return err
+				}
+			}
+
+			if diskp != "" {
+				log.Printf("storing blobs in %s", diskp)
+				bh = registry.NewDiskBlobHandler(diskp)
 			}
 
 			s := &http.Server{
@@ -89,7 +106,12 @@ Contents are only stored in memory, and when the process exits, pushed data is l
 			return nil
 		},
 	}
-	cmd.Flags().BoolVar(&disk, "blobs-to-disk", false, "Store blobs on disk")
+	// TODO: remove --blobs-to-disk in a future release.
+	cmd.Flags().BoolVarP(&blobsToDisk, "blobs-to-disk", "", false, "Store blobs on disk on tmpdir")
 	cmd.Flags().MarkHidden("blobs-to-disk")
+	cmd.Flags().MarkDeprecated("blobs-to-disk", "and will stop working in a future release. use --disk=$(mktemp -d) instead.")
+	cmd.Flags().StringVarP(&disk, "disk", "", "", "Path to a directory where blobs will be stored")
+	cmd.Flags().StringVar(&address, "address", "", "Address to listen on")
+
 	return cmd
 }

pkg/v1/layout/gc.go

@@ -0,0 +1,137 @@
+// Copyright 2018 Google LLC All Rights Reserved.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//    http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// This is an EXPERIMENTAL package, and may change in arbitrary ways without notice.
+package layout
+
+import (
+	"fmt"
+	"io/fs"
+	"path/filepath"
+	"strings"
+
+	v1 "github.com/google/go-containerregistry/pkg/v1"
+)
+
+// GarbageCollect removes unreferenced blobs from the oci-layout
+//
+//	This is an experimental api, and not subject to any stability guarantees
+//	We may abandon it at any time, without prior notice.
+//	Deprecated: Use it at your own risk!
+func (l Path) GarbageCollect() ([]v1.Hash, error) {
+	idx, err := l.ImageIndex()
+	if err != nil {
+		return nil, err
+	}
+	blobsToKeep := map[string]bool{}
+	if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil {
+		return nil, err
+	}
+	blobsDir := l.path("blobs")
+	removedBlobs := []v1.Hash{}
+
+	err = filepath.WalkDir(blobsDir, func(path string, d fs.DirEntry, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if d.IsDir() {
+			return nil
+		}
+
+		rel, err := filepath.Rel(blobsDir, path)
+		if err != nil {
+			return err
+		}
+		hashString := strings.Replace(rel, "/", ":", 1)
+		if present := blobsToKeep[hashString]; !present {
+			h, err := v1.NewHash(hashString)
+			if err != nil {
+				return err
+			}
+			removedBlobs = append(removedBlobs, h)
+		}
+		return nil
+	})
+
+	if err != nil {
+		return nil, err
+	}
+
+	return removedBlobs, nil
+}
+
+func (l Path) garbageCollectImageIndex(index v1.ImageIndex, blobsToKeep map[string]bool) error {
+	idxm, err := index.IndexManifest()
+	if err != nil {
+		return err
+	}
+
+	h, err := index.Digest()
+	if err != nil {
+		return err
+	}
+
+	blobsToKeep[h.String()] = true
+
+	for _, descriptor := range idxm.Manifests {
+		if descriptor.MediaType.IsImage() {
+			img, err := index.Image(descriptor.Digest)
+			if err != nil {
+				return err
+			}
+			if err := l.garbageCollectImage(img, blobsToKeep); err != nil {
+				return err
+			}
+		} else if descriptor.MediaType.IsIndex() {
+			idx, err := index.ImageIndex(descriptor.Digest)
+			if err != nil {
+				return err
+			}
+			if err := l.garbageCollectImageIndex(idx, blobsToKeep); err != nil {
+				return err
+			}
+		} else {
+			return fmt.Errorf("gc: unknown media type: %s", descriptor.MediaType)
+		}
+	}
+	return nil
+}
+
+func (l Path) garbageCollectImage(image v1.Image, blobsToKeep map[string]bool) error {
+	h, err := image.Digest()
+	if err != nil {
+		return err
+	}
+	blobsToKeep[h.String()] = true
+
+	h, err = image.ConfigName()
+	if err != nil {
+		return err
+	}
+	blobsToKeep[h.String()] = true
+
+	ls, err := image.Layers()
+	if err != nil {
+		return err
+	}
+	for _, l := range ls {
+		h, err := l.Digest()
+		if err != nil {
+			return err
+		}
+		blobsToKeep[h.String()] = true
+	}
+	return nil
+}