Honor registry mirrors when fetching SOCI artifacts

Dockerfile

@@ -56,6 +56,7 @@ FROM public.ecr.aws/amazonlinux/amazonlinux:2023 AS rapidgzip-builder
 
 ARG RAPIDGZIP_VERSION
 ARG TARGETARCH
+ENV SOCI_SNAPSHOTTER_SRC=/src/github.com/awslabs/soci-snapshotter
 
 RUN dnf update -y && dnf install -y \
     binutils \
@@ -90,7 +91,7 @@ RUN git clone https://github.com/mxmlnkn/rapidgzip.git && \
           -DCMAKE_C_FLAGS="-O2 -fPIC" \
           ${ISAL_FLAGS} \
           -DCMAKE_BUILD_TYPE=Release .. && \
-    make -j$(nproc) && \
+    cmake --build . --target rapidgzip --parallel "$(nproc)" && \
     cp src/tools/rapidgzip /opt/rapidgzip/usr/local/bin/ && \
     chmod +x /opt/rapidgzip/usr/local/bin/rapidgzip && \
     cd ../.. && \
@@ -104,9 +105,10 @@ ARG NERDCTL_VERSION
 ARG TARGETARCH
 ENV GOPROXY=direct
 ENV GOCOVERDIR=/test_coverage
+ENV SOCI_SNAPSHOTTER_SRC=/src/github.com/awslabs/soci-snapshotter
 
 COPY ./integ_entrypoint.sh /integ_entrypoint.sh
-COPY . $GOPATH/src/github.com/awslabs/soci-snapshotter
+COPY . ${SOCI_SNAPSHOTTER_SRC}
 RUN dnf update && dnf upgrade && dnf install -y \
     diffutils \
     findutils \
@@ -123,14 +125,14 @@ RUN dnf update && dnf upgrade && dnf install -y \
 COPY --from=igzip-builder /opt/igzip/usr /usr/local
 COPY --from=rapidgzip-builder /opt/rapidgzip/usr/local /usr/local
 
-RUN cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci /usr/local/bin/ \
-    && cp $GOPATH/src/github.com/awslabs/soci-snapshotter/out/soci-snapshotter-grpc /usr/local/bin/ \
+RUN cp ${SOCI_SNAPSHOTTER_SRC}/out/soci /usr/local/bin/ \
+    && cp ${SOCI_SNAPSHOTTER_SRC}/out/soci-snapshotter-grpc /usr/local/bin/ \
     && mkdir /etc/soci-snapshotter-grpc \
     && mkdir /etc/containerd/ \
-    && cp $GOPATH/src/github.com/awslabs/soci-snapshotter/integration/config/etc/soci-snapshotter-grpc/config.toml /etc/soci-snapshotter-grpc/ \
-    && cp $GOPATH/src/github.com/awslabs/soci-snapshotter/integration/config/etc/containerd/config.toml /etc/containerd/ \
-    && cp $GOPATH/src/github.com/awslabs/soci-snapshotter/soci-snapshotter.service /etc/systemd/system \
-    && cp $GOPATH/src/github.com/awslabs/soci-snapshotter/soci-snapshotter.socket /etc/systemd/system
+    && cp ${SOCI_SNAPSHOTTER_SRC}/integration/config/etc/soci-snapshotter-grpc/config.toml /etc/soci-snapshotter-grpc/ \
+    && cp ${SOCI_SNAPSHOTTER_SRC}/integration/config/etc/containerd/config.toml /etc/containerd/ \
+    && cp ${SOCI_SNAPSHOTTER_SRC}/soci-snapshotter.service /etc/systemd/system \
+    && cp ${SOCI_SNAPSHOTTER_SRC}/soci-snapshotter.socket /etc/systemd/system
 RUN curl -sSL --output /tmp/containerd.tgz https://github.com/containerd/containerd/releases/download/v${CONTAINERD_VERSION}/containerd-${CONTAINERD_VERSION}-linux-${TARGETARCH:-amd64}.tar.gz \
     && tar zxvf /tmp/containerd.tgz -C /usr/local/ \
     && rm -f /tmp/containerd.tgz

Makefile

@@ -178,7 +178,12 @@ $(COVDIR)/unit: $(COVDIR)
 integration: build
 	@echo "$@"
 	@echo "SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT)"
-	@GO111MODULE=$(GO111MODULE_VALUE) SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT) ENABLE_INTEGRATION_TEST=true gotestsum --rerun-fails --format standard-verbose --packages ./integration -- -timeout 0 $(GO_TEST_FLAGS)
+	@if command -v gotestsum >/dev/null 2>&1; then \
+		GO111MODULE=$(GO111MODULE_VALUE) SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT) ENABLE_INTEGRATION_TEST=true gotestsum --rerun-fails --format standard-verbose --packages ./integration -- -timeout 0 $(GO_TEST_FLAGS); \
+	else \
+		echo "gotestsum not found; falling back to go test ./integration"; \
+		GO111MODULE=$(GO111MODULE_VALUE) SOCI_SNAPSHOTTER_PROJECT_ROOT=$(SOCI_SNAPSHOTTER_PROJECT_ROOT) ENABLE_INTEGRATION_TEST=true go test -v ./integration -timeout 0 $(GO_TEST_FLAGS); \
+	fi
 
 show-integration-coverage: $(COVDIR)/unit
 	go tool covdata percent -i $(COVDIR)/integration

docs/build.md

@@ -29,6 +29,7 @@ to install them on your machine:
 
 - **[go](https://go.dev/doc/install) >= 1.25** - required to build the project;
 to confirm please check with `go version`.
+- **[gotestsum](https://github.com/gotestyourself/gotestsum)** - used by `make integration` when available for richer test output; otherwise the Makefile falls back to `go test ./integration`.
 - **[containerd](https://github.com/containerd/containerd/blob/main/docs/getting-started.md) >= 1.4** -
 required to run the SOCI snapshotter; to confirm please check with `sudo containerd --version`.
 - **fuse** - used for mounting without root access (`sudo yum install fuse`).

fs/artifact_fetcher.go

@@ -25,7 +25,9 @@ import (
 	"mime"
 	"net/http"
 	"net/url"
+	"path"
 	"strconv"
+	"strings"
 
 	sociremote "github.com/awslabs/soci-snapshotter/fs/remote"
 	socihttp "github.com/awslabs/soci-snapshotter/internal/http"
@@ -75,8 +77,8 @@ type orasBlobStore struct {
 	*remote.Repository
 }
 
-func newRemoteBlobStore(refspec reference.Spec, client *http.Client) (*orasBlobStore, error) {
-	repo, err := newRemoteStore(refspec, client)
+func newRemoteBlobStoreFromHost(refspec reference.Spec, host docker.RegistryHost) (*orasBlobStore, error) {
+	repo, err := newRemoteStoreFromHost(refspec, host)
 	if err != nil {
 		return nil, fmt.Errorf("cannot create remote store: %w", err)
 	}
@@ -96,7 +98,7 @@ func (r *orasBlobStore) Resolve(ctx context.Context, reference string) (ocispec.
 	}
 
 	tr := &clientWrapper{r.Client}
-	url := sociremote.CraftBlobURL(reference, ref)
+	url := sociremote.CraftBlobURL(r.PlainHTTP, ref)
 	resp, err := sociremote.GetHeader(ctx, url, tr)
 	if err != nil {
 		return ocispec.Descriptor{}, err
@@ -164,7 +166,7 @@ func (r *orasBlobStore) FetchRange(ctx context.Context, reference string, lower,
 	}
 
 	tr := &clientWrapper{r.Client}
-	realURL := sociremote.CraftBlobURL(reference, ref)
+	realURL := sociremote.CraftBlobURL(r.PlainHTTP, ref)
 	resp, err := GetContentWithRange(ctx, realURL, tr, lower, upper)
 	if err != nil {
 		return nil, cleanFetchErrors(err)
@@ -203,7 +205,7 @@ func (r *orasBlobStore) doInitialFetch(ctx context.Context, reference string) (b
 	}
 
 	tr := &clientWrapper{r.Client}
-	url := sociremote.CraftBlobURL(reference, ref)
+	url := sociremote.CraftBlobURL(r.PlainHTTP, ref)
 	resp, err := sociremote.GetHeaderWithGet(ctx, url, tr)
 	if err != nil {
 		return false, fmt.Errorf("error getting header info: %v", err)
@@ -228,6 +230,38 @@ func (c *clientWrapper) RoundTrip(req *http.Request) (*http.Response, error) {
 	return c.Client.Do(req)
 }
 
+func withLocatorHost(refspec reference.Spec, host docker.RegistryHost) (reference.Spec, error) {
+	if host.Host == "" || strings.Contains(host.Host, "/") {
+		return reference.Spec{}, fmt.Errorf("invalid registry host %q", host.Host)
+	}
+
+	repoPath := strings.TrimPrefix(refspec.Locator, refspec.Hostname())
+	repoPath = strings.TrimPrefix(repoPath, "/")
+	if repoPath == "" {
+		return reference.Spec{}, fmt.Errorf("invalid image locator %q", refspec.Locator)
+	}
+
+	repoPrefix, err := repositoryPrefixFromHostPath(host.Path)
+	if err != nil {
+		return reference.Spec{}, err
+	}
+
+	refspec.Locator = path.Join(host.Host, repoPrefix, repoPath)
+	return refspec, nil
+}
+
+func repositoryPrefixFromHostPath(hostPath string) (string, error) {
+	clean := path.Clean(hostPath)
+	switch {
+	case clean == ".", clean == "/", clean == "/v2":
+		return "", nil
+	case strings.HasPrefix(clean, "/v2/"):
+		return strings.TrimPrefix(clean, "/v2/"), nil
+	default:
+		return "", fmt.Errorf("unsupported registry host path %q; expected /v2 or /v2/<prefix>", hostPath)
+	}
+}
+
 func newRemoteStore(refspec reference.Spec, client *http.Client) (*remote.Repository, error) {
 	repo, err := remote.NewRepository(refspec.Locator)
 	if err != nil {
@@ -242,6 +276,26 @@ func newRemoteStore(refspec reference.Spec, client *http.Client) (*remote.Reposi
 	return repo, nil
 }
 
+func newRemoteStoreFromHost(refspec reference.Spec, host docker.RegistryHost) (*remote.Repository, error) {
+	repo, err := newRemoteStore(refspec, host.Client)
+	if err != nil {
+		return nil, err
+	}
+
+	switch strings.ToLower(host.Scheme) {
+	case "http":
+		repo.PlainHTTP = true
+	case "", "https":
+		// Keep the default behavior from newRemoteStore:
+		// - localhost uses plain HTTP
+		// - all other hosts use HTTPS
+	default:
+		return nil, fmt.Errorf("unsupported registry scheme %q for host %q", host.Scheme, host.Host)
+	}
+
+	return repo, nil
+}
+
 // Constructs a new artifact fetcher
 // Takes in the image reference, the local store and the resolver
 func newArtifactFetcher(refspec reference.Spec, localStore store.BasicStore, remoteStore resolverStorage) (*artifactFetcher, error) {
@@ -266,7 +320,6 @@ func constructRef(refspec reference.Spec, desc ocispec.Descriptor) string {
 // It first checks the local store for the artifact.
 // If not found, if constructs the ref and fetches it from remote.
 func (f *artifactFetcher) Fetch(ctx context.Context, desc ocispec.Descriptor) (io.ReadCloser, bool, error) {
-
 	// Check local store first
 	rc, err := f.localStore.Fetch(ctx, desc)
 	if err == nil {