Merge branch 'main' into docker-library-behavior

Author: CamrynCarter

.github/workflows/tests.yaml

@@ -216,7 +216,20 @@ jobs:
       - name: Verify - hauler store copy
         run: |
           hauler store copy --help
-          # need more tests here
+          # verify via registry target
+          hauler store serve registry --port 5002 --readonly=false &
+          until curl -sf http://localhost:5002/v2/_catalog; do : ; done
+          hauler store copy registry://localhost:5002 --plain-http
+          # verify blobs actually landed in the registry backend
+          [ $(find registry -type f | wc -l) -gt 0 ]
+          # verify via --only
+          hauler store copy registry://localhost:5002 --plain-http --only busybox
+          pkill -f "hauler store serve registry --port 5002"
+          rm -rf registry
+          # verify via directory target (files and charts only; images and cosign artifacts are skipped)
+          hauler store copy dir://./store-copy-output
+          test -f ./store-copy-output/hauler-manifest.yaml
+          rm -rf ./store-copy-output
 
       - name: Verify - hauler store extract
         run: |
@@ -354,9 +367,9 @@ jobs:
           until curl -sf http://localhost:5001/v2/_catalog; do : ; done
           pkill -f "hauler store serve registry --port 5001 --readonly"
           # verify via registry with different port with readonly with tls
-          # hauler store serve registry --port 5001 --readonly --tls-cert testdata/certs/server-cert.crt --tls-key testdata/certs/server-cert.key &
-          # until curl -sf --cacert testdata/certs/cacerts.pem https://localhost:5001/v2/_catalog; do : ; done
-          # pkill -f "hauler store serve registry --port 5001 --readonly --tls-cert testdata/certs/server-cert.crt --tls-key testdata/certs/server-cert.key"
+          hauler store serve registry --port 5001 --readonly --tls-cert testdata/certs/server-cert.crt --tls-key testdata/certs/server-cert.key &
+          until curl -sf --cacert testdata/certs/cacerts.pem https://localhost:5001/v2/_catalog; do : ; done
+          pkill -f "hauler store serve registry --port 5001 --readonly --tls-cert testdata/certs/server-cert.crt --tls-key testdata/certs/server-cert.key"
 
       - name: Verify - hauler store serve fileserver
         run: |
@@ -374,9 +387,9 @@ jobs:
           until curl -sf http://localhost:8000; do : ; done
           pkill -f "hauler store serve fileserver --port 8000 --timeout 120"
           # verify via fileserver with different port with timeout and tls
-          # hauler store serve fileserver --port 8000 --timeout 120 --tls-cert testdata/certs/server-cert.crt --tls-key testdata/certs/server-cert.key &
-          # until curl -sf --cacert testdata/certs/cacerts.pem https://localhost:8000; do : ; done
-          # pkill -f "hauler store serve fileserver --port 8000 --timeout 120 --tls-cert testdata/certs/server-cert.crt --tls-key testdata/certs/server-cert.key"
+          hauler store serve fileserver --port 8000 --timeout 120 --tls-cert testdata/certs/server-cert.crt --tls-key testdata/certs/server-cert.key &
+          until curl -sf --cacert testdata/certs/cacerts.pem https://localhost:8000; do : ; done
+          pkill -f "hauler store serve fileserver --port 8000 --timeout 120 --tls-cert testdata/certs/server-cert.crt --tls-key testdata/certs/server-cert.key"
 
       - name: Verify Hauler Store Contents
         run: |

.gitignore

@@ -20,4 +20,5 @@ vulncheck.out
 trivy.out
 CLAUDE.md
 **/CLAUDE.*
-.claude**
+.claude**
+__debug_bin*

.goreleaser.yaml

@@ -69,7 +69,7 @@ dockers_v2:
       "classification": "UNCLASSIFIED"
       "org.opencontainers.image.created": "{{.Date}}"
       "org.opencontainers.image.description": "Hauler: Airgap Swiss Army Knife"
-      "org.opencontainers.image.name": "{{.ProjectName}}-debug"
+      "org.opencontainers.image.name": "{{.ProjectName}}"
       "org.opencontainers.image.revision": "{{.FullCommit}}"
       "org.opencontainers.image.source": "{{.GitURL}}"
       "org.opencontainers.image.version": "{{.Version}}"

Dockerfile

@@ -1,5 +1,5 @@
 # builder stage
-FROM registry.suse.com/bci/bci-base:15.7 AS builder
+FROM registry.suse.com/bci/bci-base:16.1 AS builder
 ARG TARGETPLATFORM
 
 # fetched from goreleaser build process
@@ -29,15 +29,13 @@ USER hauler
 ENTRYPOINT [ "/hauler" ]
 
 # debug stage
-FROM alpine AS debug
+FROM registry.suse.com/bci/bci-base:16.1 AS debug
 
 COPY --from=builder /var/lib/ca-certificates/ca-bundle.pem /etc/ssl/certs/ca-certificates.crt
 COPY --from=builder /etc/passwd /etc/passwd
 COPY --from=builder /etc/group /etc/group
 COPY --from=builder --chown=hauler:hauler /home/hauler/. /home/hauler
 COPY --from=builder --chown=hauler:hauler /hauler /usr/local/bin/hauler
 
-RUN apk --no-cache add curl
-
 USER hauler
 WORKDIR /home/hauler

cmd/hauler/cli/cli.go

@@ -18,7 +18,7 @@ func New(ctx context.Context, ro *flags.CliRootOpts) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:     "hauler",
 		Short:   "Airgap Swiss Army Knife",
-		Example: "  View the Docs: https://docs.hauler.dev\n  Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerStoreDir + " | " + consts.HaulerIgnoreErrors + " | " + consts.HaulerLogLevel + " | " + consts.HaulerAuditLevel + " | " + consts.HaulerConcurrency + " | " + consts.HaulerBlobConcurrency + "\n  Warnings: Hauler commands and flags marked with (EXPERIMENTAL) are not yet stable and may change in the future.",
+		Example: "  View the Docs: https://docs.hauler.dev\n  Environment Variables: " + consts.HaulerDir + " | " + consts.HaulerTempDir + " | " + consts.HaulerStoreDir + " | " + consts.HaulerIgnoreErrors + " | " + consts.HaulerRetries + " | " + consts.HaulerLogLevel + " | " + consts.HaulerAuditLevel + " | " + consts.HaulerConcurrency + " | " + consts.HaulerBlobConcurrency + "\n  Warnings: Hauler commands and flags marked with (EXPERIMENTAL) are not yet stable and may change in the future.",
 		PersistentPreRunE: func(cmd *cobra.Command, args []string) error {
 			// check for log level env variable or flag
 			if ro.LogLevel == "" {

cmd/hauler/cli/store.go

@@ -2,12 +2,14 @@ package cli
 
 import (
 	"fmt"
+	"os"
 
 	"github.com/spf13/cobra"
 	"helm.sh/helm/v4/pkg/action"
 
 	"hauler.dev/go/hauler/v2/cmd/hauler/cli/store"
 	"hauler.dev/go/hauler/v2/internal/flags"
+	"hauler.dev/go/hauler/v2/pkg/consts"
 	"hauler.dev/go/hauler/v2/pkg/log"
 )
 
@@ -71,6 +73,24 @@ func addStoreSync(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
 		Short: "Sync content to the content store",
 		Args:  cobra.ExactArgs(0),
 		PreRunE: func(cmd *cobra.Command, args []string) error {
+			// Check for ca-file & insecure-skip-tls-verify env variables
+			if o.CaFile == "" {
+				o.CaFile = os.Getenv(consts.CaFile)
+			}
+
+			// record which precedence-carrying flags the user explicitly set, so
+			// the resolvers can let an explicit CLI value win over per-item/annotation
+			o.TlogChanged = cmd.Flags().Changed("use-tlog-verify")
+			o.ExcludeExtrasChanged = cmd.Flags().Changed("exclude-extras")
+			o.InsecureChanged = cmd.Flags().Changed("insecure-skip-tls-verify")
+			o.StoreChanged = cmd.Flags().Changed("store")
+			o.RetriesChanged = cmd.Flags().Changed("retries")
+
+			// env var only applies when the flag wasn't set, so an explicit --insecure-skip-tls-verify=false still wins
+			if !o.InsecureChanged && os.Getenv(consts.InsecureSkipTLSVerify) == "true" {
+				o.InsecureSkipTLSVerify = true
+			}
+
 			// --dry-run requires --products
 			if o.DryRun && len(o.Products) == 0 {
 				return fmt.Errorf("--dry-run requires --products")
@@ -146,9 +166,8 @@ func addStoreLoad(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
 			if err != nil {
 				return err
 			}
-			_ = s
 
-			return store.LoadCmd(ctx, o, rso, ro)
+			return store.LoadCmd(ctx, o, s, rso, ro)
 		},
 	}
 	o.AddFlags(cmd)
@@ -232,9 +251,8 @@ func addStoreSave(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
 			if err != nil {
 				return err
 			}
-			_ = s
 
-			return store.SaveCmd(ctx, o, rso, ro)
+			return store.SaveCmd(ctx, o, s, rso, ro)
 		},
 	}
 	o.AddFlags(cmd)
@@ -292,8 +310,8 @@ func addStoreCopy(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comman
 		Use:   "copy",
 		Short: "Copy all store content to another location",
 		Example: `  # supported copy target prefixes
-	registry://  | reg:// | oci:// - Pushes the store to an OCI registry
-	directory:// | dir://          - Extracts the store to a directory`,
+  registry://  | reg:// | oci:// - Pushes the store to an OCI registry
+  directory:// | dir://          - Extracts the store to a directory`,
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
@@ -336,13 +354,13 @@ func addStoreAddFile(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Com
 		Use:   "file",
 		Short: "Add a file to the store",
 		Example: `  # fetch local file
-	hauler store add file file.txt
+  hauler store add file file.txt
 
-	# fetch remote file
-	hauler store add file https://get.rke2.io/install.sh
+  # fetch remote file
+  hauler store add file https://get.rke2.io/install.sh
 
-	# fetch remote file and assign new name
-	hauler store add file https://get.hauler.dev --name hauler-install.sh`,
+  # fetch remote file and assign new name
+  hauler store add file https://get.hauler.dev --name hauler-install.sh`,
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
@@ -366,28 +384,39 @@ func addStoreAddImage(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co
 	cmd := &cobra.Command{
 		Use:   "image",
 		Short: "Add a image to the store",
-		Example: `	# fetch image
-	hauler store add image busybox
+		Example: `  # fetch image
+  hauler store add image busybox
 
-	# fetch image with repository and tag
-	hauler store add image library/busybox:stable
+  # fetch image with repository and tag
+  hauler store add image library/busybox:stable
 
-	# fetch image with full image reference and specific platform
-	hauler store add image ghcr.io/hauler-dev/hauler-debug:v1.2.0 --platform linux/amd64
+  # fetch image with full image reference and specific platform
+  hauler store add image ghcr.io/hauler-dev/hauler-debug:v1.2.0 --platform linux/amd64
 
-	# fetch image with full image reference via digest
-	hauler store add image gcr.io/distroless/base@sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5
+  # fetch image with full image reference via digest
+  hauler store add image gcr.io/distroless/base@sha256:7fa7445dfbebae4f4b7ab0e6ef99276e96075ae42584af6286ba080750d6dfe5
 
-	# fetch image with full image reference, specific platform, and signature verification
-	curl -sfOL https://raw.githubusercontent.com/rancherfederal/carbide-releases/main/carbide-key.pub
-	hauler store add image rgcrprod.azurecr.us/rancher/rke2-runtime:v1.31.5-rke2r1 --platform linux/amd64 --key carbide-key.pub
+  # fetch image with full image reference, specific platform, and signature verification
+  curl -sfOL https://raw.githubusercontent.com/rancherfederal/carbide-releases/main/carbide-key.pub
+  hauler store add image rgcrprod.azurecr.us/rancher/rke2-runtime:v1.31.5-rke2r1 --platform linux/amd64 --key carbide-key.pub
 
-	# fetch image and rewrite path
-	hauler store add image busybox --rewrite custom-path/busybox:latest
+  # fetch image and rewrite path
+  hauler store add image busybox --rewrite custom-path/busybox:latest
 
-	# add image from local Docker daemon
-	hauler store add image my-local-app:latest --local`,
+  # add image from local Docker daemon
+  hauler store add image my-local-app:latest --local`,
 		Args: cobra.ExactArgs(1),
+		PreRunE: func(cmd *cobra.Command, args []string) error {
+			// Check for ca-file & insecure-skip-tls-verify env variables
+			if o.CaFile == "" {
+				o.CaFile = os.Getenv(consts.CaFile)
+			}
+			// env var only applies when the flag wasn't set, so an explicit --insecure-skip-tls-verify=false still wins
+			if !cmd.Flags().Changed("insecure-skip-tls-verify") && os.Getenv(consts.InsecureSkipTLSVerify) == "true" {
+				o.InsecureSkipTLSVerify = true
+			}
+			return nil
+		},
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()
 
@@ -410,26 +439,26 @@ func addStoreAddChart(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Co
 	cmd := &cobra.Command{
 		Use:   "chart",
 		Short: "Add a helm chart to the store",
-		Example: `	# fetch local helm chart
-	hauler store add chart path/to/chart/directory --repo .
+		Example: `  # fetch local helm chart
+  hauler store add chart path/to/chart/directory --repo .
 
-	# fetch local compressed helm chart
-	hauler store add chart path/to/chart.tar.gz --repo .
+  # fetch local compressed helm chart
+  hauler store add chart path/to/chart.tar.gz --repo .
 
-	# fetch remote oci helm chart
-	hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev
+  # fetch remote oci helm chart
+  hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev
 
-	# fetch remote oci helm chart with version
-	hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --version 1.2.0
+  # fetch remote oci helm chart with version
+  hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --version 1.2.0
 
-	# fetch remote helm chart
-	hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable
+  # fetch remote helm chart
+  hauler store add chart rancher --repo https://releases.rancher.com/server-charts/stable
 
-	# fetch remote helm chart with specific version
-	hauler store add chart rancher --repo https://releases.rancher.com/server-charts/latest --version 2.10.1
+  # fetch remote helm chart with specific version
+  hauler store add chart rancher --repo https://releases.rancher.com/server-charts/latest --version 2.10.1
 
-	# fetch remote helm chart and rewrite path
-	hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --rewrite custom-path/hauler-chart:latest`,
+  # fetch remote helm chart and rewrite path
+  hauler store add chart hauler-helm --repo oci://ghcr.io/hauler-dev --rewrite custom-path/hauler-chart:latest`,
 		Args: cobra.ExactArgs(1),
 		PreRunE: func(cmd *cobra.Command, args []string) error {
 			n, err := flags.ResolveConcurrency(cmd.Flags().Changed("concurrency"), o.Concurrency)
@@ -471,25 +500,25 @@ func addStoreRemove(rso *flags.StoreRootOpts, ro *flags.CliRootOpts) *cobra.Comm
 		Use:   "remove <artifact-ref>",
 		Short: "Remove an artifact from the content store",
 		Example: `  # remove an image using full store reference
-	hauler store info
-	hauler store remove index.docker.io/library/busybox:stable
+  hauler store info
+  hauler store remove index.docker.io/library/busybox:stable
 
-	# remove a chart using full store reference
-	hauler store info
-	hauler store remove hauler/rancher:2.8.4
+  # remove a chart using full store reference
+  hauler store info
+  hauler store remove hauler/rancher:2.8.4
 
-	# remove a file using full store reference
-	hauler store info
-	hauler store remove hauler/rke2-install.sh
+  # remove a file using full store reference
+  hauler store info
+  hauler store remove hauler/rke2-install.sh
 
-	# remove any artifact with the latest tag
-	hauler store remove :latest
+  # remove any artifact with the latest tag
+  hauler store remove :latest
 
-	# remove any artifact with 'busybox' in the reference
-	hauler store remove busybox
+  # remove any artifact with 'busybox' in the reference
+  hauler store remove busybox
 
-	# force remove without verification
-	hauler store remove busybox:latest --force`,
+  # force remove without verification
+  hauler store remove busybox:latest --force`,
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			ctx := cmd.Context()