delete artifacts from store

.github/workflows/tests.yaml

@@ -322,6 +322,105 @@ jobs:
           # verify fileserver directory structure
           tree -hC fileserver
 
+      - name: Verify - hauler store delete-artifact (image)
+        run: |
+          hauler store delete-artifact --help
+          # add test images
+          hauler store add image docker.io/library/nginx:1.25-alpine
+          hauler store add image docker.io/library/nginx:1.26-alpine
+          # confirm artifacts
+          hauler store info | grep 'nginx:1.25'
+          hauler store info | grep 'nginx:1.26'
+          # count blobs before delete
+          BLOBS_BEFORE=$(find store/blobs/sha256 -type f | wc -l | xargs)
+          echo "blobs before deletion: $BLOBS_BEFORE"
+          # delete one artifact
+          hauler store delete-artifact nginx:1.25 --force
+          # verify artifact removed
+          ! hauler store info | grep -q "nginx:1.25"
+          # non-deleted artifact exists
+          hauler store info | grep -q "nginx:1.26"
+          # count blobs after delete
+          BLOBS_AFTER=$(find store/blobs/sha256 -type f | wc -l | xargs)
+          echo "blobs after deletion: $BLOBS_AFTER"
+          # verify only unreferenced blobs removed
+          if [ "$BLOBS_AFTER" -ge "$BLOBS_BEFORE" ]; then
+            echo "ERROR: No blobs were cleaned up"
+            exit 1
+          fi
+          if [ "$BLOBS_AFTER" -eq 0 ]; then
+            echo "ERROR: All blobs deleted (shared layers removed)"
+            exit 1
+          fi
+          # verify remaining image not missing layers 
+          hauler store extract docker.io/library/nginx:1.26-alpine
+
+      - name: Verify - hauler store delete-artifact (chart)
+        run: |
+          hauler store delete-artifact --help
+          # add test images
+          hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable --version 2.8.4
+          hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable --version 2.8.5
+          # confirm artifacts
+          hauler store info | grep '2.8.4'
+          hauler store info | grep '2.8.5'
+          # count blobs before delete
+          BLOBS_BEFORE=$(find store/blobs/sha256 -type f | wc -l | xargs)
+          echo "blobs before deletion: $BLOBS_BEFORE"
+          # delete one artifact
+          hauler store delete-artifact 2.8.4 --force
+          # verify artifact removed
+          ! hauler store info | grep -q "2.8.4"
+          # non-deleted artifact exists
+          hauler store info | grep -q "2.8.5"
+          # count blobs after delete
+          BLOBS_AFTER=$(find store/blobs/sha256 -type f | wc -l | xargs)
+          echo "blobs after deletion: $BLOBS_AFTER"
+          # verify only unreferenced blobs removed
+          if [ "$BLOBS_AFTER" -ge "$BLOBS_BEFORE" ]; then
+            echo "ERROR: No blobs were cleaned up"
+            exit 1
+          fi
+          if [ "$BLOBS_AFTER" -eq 0 ]; then
+            echo "ERROR: All blobs deleted (shared layers removed)"
+            exit 1
+          fi
+          # verify remaining image not missing layers 
+          hauler store extract hauler/rancher:2.8.5
+
+      - name: Verify - hauler store delete-artifact (file)
+        run: |
+          hauler store delete-artifact --help
+          # add test images
+          hauler store add file https://get.hauler.dev
+          hauler store add file https://get.rke2.io/install.sh
+          # confirm artifacts
+          hauler store info | grep 'get.hauler.dev'
+          hauler store info | grep 'install.sh'
+          # count blobs before delete
+          BLOBS_BEFORE=$(find store/blobs/sha256 -type f | wc -l | xargs)
+          echo "blobs before deletion: $BLOBS_BEFORE"
+          # delete one artifact
+          hauler store delete-artifact get.hauler.dev --force
+          # verify artifact removed
+          ! hauler store info | grep -q "get.hauler.dev"
+          # non-deleted artifact exists
+          hauler store info | grep -q "install.sh"
+          # count blobs after delete
+          BLOBS_AFTER=$(find store/blobs/sha256 -type f | wc -l | xargs)
+          echo "blobs after deletion: $BLOBS_AFTER"
+          # verify only unreferenced blobs removed
+          if [ "$BLOBS_AFTER" -ge "$BLOBS_BEFORE" ]; then
+            echo "ERROR: No blobs were cleaned up"
+            exit 1
+          fi
+          if [ "$BLOBS_AFTER" -eq 0 ]; then
+            echo "ERROR: All blobs deleted (shared layers removed)"
+            exit 1
+          fi
+          # verify remaining image not missing layers 
+          hauler store extract hauler/install.sh:latest
+
       - name: Create Hauler Report
         run: |
           hauler version >> hauler-report.txt

cmd/hauler/cli/store.go

@@ -32,6 +32,7 @@ func addStore(parent *cobra.Command, ro *flags.CliRootOpts) {
 		addStoreInfo(rso, ro),
 		addStoreCopy(rso, ro),
 		addStoreAdd(rso, ro),
+		addStoreDeleteArtifact(rso, ro),
 	)
 
 	parent.AddCommand(cmd)
@@ -384,3 +385,26 @@ hauler store add chart rancher --repo https://releases.rancher.com/server-charts
 
 	return cmd
 }
+
+func addStoreDeleteArtifact(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Command {
+	o := &flags.DeleteArtifactOpts{}
+	cmd := &cobra.Command{
+		Use:     "delete-artifact <artifact-ref>",
+		Short:   "Delete an artifact from the content store",
+		Aliases: []string{"del"},
+		Args:    cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			ctx := cmd.Context()
+
+			s, err := rso.Store(ctx)
+			if err != nil {
+				return err
+			}
+
+			return store.DeleteArtifactCmd(ctx, o, s, args[0])
+		},
+	}
+	o.AddFlags(cmd)
+
+	return cmd
+}

cmd/hauler/cli/store/delete-artifact.go

@@ -0,0 +1,89 @@
+package store
+
+import (
+	"context"
+	"fmt"
+	"strings"
+
+	ocispec "github.com/opencontainers/image-spec/specs-go/v1"
+
+	"hauler.dev/go/hauler/internal/flags"
+	"hauler.dev/go/hauler/pkg/log"
+	"hauler.dev/go/hauler/pkg/store"
+)
+
+func DeleteArtifactCmd(ctx context.Context, o *flags.DeleteArtifactOpts, s *store.Layout, ref string) error {
+	l := log.FromContext(ctx)
+
+	// collect matching artifacts
+	type match struct {
+		reference string
+		desc      ocispec.Descriptor
+	}
+	var matches []match
+
+	if err := s.Walk(func(reference string, desc ocispec.Descriptor) error {
+		if !strings.Contains(reference, ref) {
+			return nil
+		}
+
+		matches = append(matches, match{
+			reference: reference,
+			desc:      desc,
+		})
+
+		return nil // continue walking
+	}); err != nil {
+		return err
+	}
+
+	if len(matches) == 0 {
+		return fmt.Errorf("reference [%s] not found in store (hint: use `hauler store info` to list store contents)", ref)
+	}
+
+	if len(matches) >= 1 {
+		l.Infof("found %d matching references:", len(matches))
+		for _, m := range matches {
+			l.Infof(" - %s", m.reference)
+		}
+	}
+
+	if !o.Force {
+		fmt.Printf("are you sure you want to delete %d artifact(s) from the store? (yes/no) ", len(matches))
+
+		var response string
+		_, err := fmt.Scanln(&response)
+		if err != nil {
+			return fmt.Errorf("failed to read response: %w", err)
+		}
+		switch response {
+		case "yes", "y":
+			l.Infof("deleting artifacts from store...")
+		case "no", "n":
+			l.Infof("deletion cancelled")
+			return nil
+		default:
+			return fmt.Errorf("invalid response '%s' - please answer 'yes' or 'no'", response)
+		}
+	}
+
+	//delete artifact(s)
+	for _, m := range matches {
+		if err := s.DeleteArtifact(ctx, m.reference, m.desc); err != nil {
+			return fmt.Errorf("failed to delete artifact %s: %w", m.reference, err)
+		}
+
+		l.Infof("deleted [%s] of type %s with digest [%s]", m.reference, m.desc.MediaType, m.desc.Digest.String())
+	}
+
+	// clean up unreferenced blobs
+	l.Infof("cleaning up unreferenced blobs...")
+	deletedCount, deletedSize, err := s.CleanUp(ctx)
+	if err != nil {
+		l.Warnf("garbrage collection failed: %v", err)
+	} else if deletedCount > 0 {
+		l.Infof("removed %d unreferenced blobs (freed %d bytes)", deletedCount, deletedSize)
+	}
+
+	return nil
+}

internal/flags/delete-artifact.go

@@ -0,0 +1,11 @@
+package flags
+
+import "github.com/spf13/cobra"
+
+type DeleteArtifactOpts struct {
+	Force bool // skip delete confirmation
+}
+
+func (o *DeleteArtifactOpts) AddFlags(cmd *cobra.Command) {
+	cmd.Flags().BoolVarP(&o.Force, "force", "f", false, "(Optional) Delete artifacts without confirmation")
+}

pkg/content/oci.go

@@ -312,3 +312,7 @@ func (p *ociPusher) Push(ctx context.Context, d ocispec.Descriptor) (ccontent.Wr
 	w := content.NewIoContentWriter(f, content.WithInputHash(d.Digest), content.WithOutputHash(d.Digest))
 	return w, nil
 }
+
+func (o *OCI) RemoveFromIndex(ref string) {
+	o.nameMap.Delete(ref)
+}

pkg/store/store.go

@@ -3,6 +3,7 @@ package store
 import (
 	"context"
 	"encoding/json"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
@@ -260,3 +261,125 @@ func (l *Layout) writeLayer(layer v1.Layer) error {
 	_, err = io.Copy(w, r)
 	return err
 }
+
+// Delete artifact reference from the store
+func (l *Layout) DeleteArtifact(ctx context.Context, reference string, desc ocispec.Descriptor) error {
+	if err := l.OCI.LoadIndex(); err != nil {
+		return err
+	}
+
+	l.OCI.RemoveFromIndex(reference)
+	return l.OCI.SaveIndex()
+}
+
+func (l *Layout) CleanUp(ctx context.Context) (int, int64, error) {
+	referencedDigests := make(map[string]bool)
+
+	if err := l.OCI.LoadIndex(); err != nil {
+		return 0, 0, fmt.Errorf("failed to load index: %w", err)
+	}
+
+	var processManifest func(desc ocispec.Descriptor) error
+	processManifest = func(desc ocispec.Descriptor) error {
+		if desc.Digest.Validate() != nil {
+			return nil
+		}
+
+		// mark digest as referenced by existing artifact
+		referencedDigests[desc.Digest.Hex()] = true
+
+		// fetch and parse manifests for layer digests
+		rc, err := l.OCI.Fetch(ctx, desc)
+		if err != nil {
+			return nil // skip if can't be read
+		}
+		defer rc.Close()
+
+		var manifest struct {
+			Config struct {
+				Digest digest.Digest `json:"digest"`
+			} `json:"config"`
+			Layers []struct {
+				digest.Digest `json:"digest"`
+			} `json:"layers"`
+			Manifests []struct {
+				Digest    digest.Digest `json:"digest"`
+				MediaType string        `json:"mediaType"`
+				Size      int64         `json:"size"`
+			} `json:"manifests"`
+		}
+
+		if err := json.NewDecoder(rc).Decode(&manifest); err != nil {
+			return nil
+		}
+
+		// handle image manifest
+		if manifest.Config.Digest.Validate() == nil {
+			referencedDigests[manifest.Config.Digest.Hex()] = true
+		}
+
+		for _, layer := range manifest.Layers {
+			if layer.Digest.Validate() == nil {
+				referencedDigests[layer.Digest.Hex()] = true
+			}
+		}
+
+		// handle manifest list
+		for _, m := range manifest.Manifests {
+			if m.Digest.Validate() == nil {
+				// mark manifest
+				referencedDigests[m.Digest.Hex()] = true
+				// process manifest for layers
+				manifestDesc := ocispec.Descriptor{
+					MediaType: m.MediaType,
+					Digest:    m.Digest,
+					Size:      m.Size,
+				}
+				processManifest(manifestDesc) // calls helper func on manifests in list
+			}
+		}
+
+		return nil
+	}
+
+	// walk through artifacts
+	if err := l.OCI.Walk(func(reference string, desc ocispec.Descriptor) error {
+		return processManifest(desc)
+	}); err != nil {
+		return 0, 0, fmt.Errorf("failed to walk artifacts: %w", err)
+	}
+
+	// read all entries
+	blobsPath := filepath.Join(l.Root, "blobs", "sha256")
+	entries, err := os.ReadDir(blobsPath)
+	if err != nil {
+		return 0, 0, fmt.Errorf("failed to read blobs directory: %w", err)
+	}
+
+	// track count and size of deletions
+	deletedCount := 0
+	var deletedSize int64
+
+	// scan blobs
+	for _, entry := range entries {
+		if entry.IsDir() {
+			continue
+		}
+
+		digest := entry.Name()
+
+		if !referencedDigests[digest] {
+			blobPath := filepath.Join(blobsPath, digest)
+			if info, err := entry.Info(); err == nil {
+				deletedSize += info.Size()
+			}
+
+			if err := os.Remove(blobPath); err != nil {
+				return deletedCount, deletedSize, fmt.Errorf("failed to remove blob %s: %w", digest, err)
+			}
+			deletedCount++
+		}
+	}
+
+	return deletedCount, deletedSize, nil
+}