Add support for AWS backend

.github/workflows/test.yml

@@ -80,8 +80,10 @@ jobs:
         with:
           go-version-file: './go.mod'
           check-latest: true
-      - name: Run Go tests
-        run: go test -covermode atomic -coverprofile coverage.txt $(go list ./... | grep -v third_party/)
+      - name: Run Go tests (AWS backend)
+        run: go test -tags aws -covermode atomic -coverprofile coverage.txt $(go list -tags aws ./... | grep -v third_party/)
+      - name: Run Go tests (GCP backend)
+        run: go test -tags gcp $(go list -tags gcp ./... | grep -v third_party/)
       - name: Workaround buggy Codecov OIDC auth
         run: |
           # only set CODECOV_TOKEN if OIDC token is available
@@ -98,9 +100,12 @@ jobs:
           # When github.com/codecov/codecov-action/issues/1791 is fixed,
           # remove workaround step above and uncomment:
           # use_oidc: true
-      - name: Run Go tests w/ `-race`
+      - name: Run Go tests w/ `-race` (AWS backend)
         if: ${{ runner.os == 'Linux' }}
-        run: go test -race $(go list ./... | grep -v third_party/)
+        run: go test -tags aws -race $(go list -tags aws ./... | grep -v third_party/)
+      - name: Run Go tests w/ `-race` (GCP backend)
+        if: ${{ runner.os == 'Linux' }}
+        run: go test -tags gcp -race $(go list -tags gcp ./... | grep -v third_party/)
 
   e2e-tests:
     name: Run E2E tests
@@ -111,10 +116,24 @@ jobs:
       - uses: actions/checkout@1af3b93b6815bc44a9784bd300feb67ff0d1eeb3 # v6.0.0
         with:
           persist-credentials: false
-      - name: Run docker compose
+
+      # Test GCP backend
+      - name: Run docker compose for GCP
         run: docker compose -f compose.yml up -d --build --wait --wait-timeout 60
-      - name: Run e2e tests
-        run: go test -v -tags=e2e ./tests/
+      - name: Run GCP e2e tests
+        run: TEST_BACKENDS=gcp go test -v -tags=e2e ./tests/
+      - name: Stop GCP docker compose
+        if: always()
+        run: docker compose -f compose.yml down
+
+      # Test AWS backend
+      - name: Run docker compose for AWS
+        run: docker compose -f docker-compose-aws.yml up -d --build --wait --wait-timeout 60
+      - name: Run AWS e2e tests
+        run: TEST_BACKENDS=aws go test -v -tags=e2e ./tests/
+      - name: Stop AWS docker compose
+        if: always()
+        run: docker compose -f docker-compose-aws.yml down
 
   sharding-freeze:
     name: Run freeze log tests

.golangci.yml

@@ -16,6 +16,8 @@
 version: "2"
 run:
   issues-exit-code: 1
+  build-tags:
+    - gcp
 linters:
   enable:
     - errcheck

Dockerfile

@@ -16,6 +16,7 @@
 FROM --platform=$BUILDPLATFORM golang:1.25.3@sha256:6d4e5e74f47db00f7f24da5f53c1b4198ae46862a47395e30477365458347bf2 AS builder
 ARG TARGETOS
 ARG TARGETARCH
+ARG CLOUD_PROVIDER=gcp
 ENV APP_ROOT=/opt/app-root
 ENV GOPATH=$APP_ROOT
 
@@ -30,9 +31,9 @@ ADD ./internal/ $APP_ROOT/src/internal/
 
 ARG SERVER_LDFLAGS
 # Build server for deployment
-RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -ldflags "${SERVER_LDFLAGS}" ./cmd/rekor-server
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -tags ${CLOUD_PROVIDER} -ldflags "${SERVER_LDFLAGS}" -o rekor-server ./cmd/rekor-server-${CLOUD_PROVIDER}
 # Build server for debugger
-RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -gcflags "all=-N -l" -ldflags "${SERVER_LDFLAGS}" -o rekor-server_debug ./cmd/rekor-server
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -tags ${CLOUD_PROVIDER} -gcflags "all=-N -l" -ldflags "${SERVER_LDFLAGS}" -o rekor-server_debug ./cmd/rekor-server-${CLOUD_PROVIDER}
 
 # Multi-stage deployment build
 FROM golang:1.25.3@sha256:6d4e5e74f47db00f7f24da5f53c1b4198ae46862a47395e30477365458347bf2 AS deploy

Dockerfile.release

@@ -17,6 +17,7 @@
 FROM --platform=$BUILDPLATFORM golang:1.25.3-bookworm@sha256:4f43b271f9673eb7bd0cb3a49cc17b08d8d6ee110277e26dbacc93c43a5a7793 AS builder
 ARG TARGETOS
 ARG TARGETARCH
+ARG CLOUD_PROVIDER=gcp
 ENV APP_ROOT=/opt/app-root
 ENV GOPATH=$APP_ROOT
 
@@ -31,7 +32,7 @@ ADD ./internal/ $APP_ROOT/src/internal/
 
 ARG SERVER_LDFLAGS
 # Build server for deployment. Build without cgo since distroless/static-debian12 doesn't include lib/cgo
-RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -ldflags "${SERVER_LDFLAGS}" ./cmd/rekor-server
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 go build -ldflags "${SERVER_LDFLAGS}" -o rekor-server ./cmd/rekor-server-${CLOUD_PROVIDER}
 
 # Multi-stage deployment build
 FROM gcr.io/distroless/static-debian12:nonroot@sha256:2b7c93f6d6648c11f0e80a48558c8f77885eb0445213b8e69a6a0d7c89fc6ae4 AS deploy

Makefile

@@ -15,7 +15,7 @@
 
 .PHONY: all test clean lint gosec ko-local tools ldflags
 
-all: protos rekor-server
+all: protos rekor-server-gcp rekor-server-aws
 
 GIT_VERSION ?= $(shell git describe --tags --always --dirty)
 GIT_HASH ?= $(shell git rev-parse HEAD)
@@ -73,20 +73,35 @@ lint:
 gosec: ## Run gosec security scanner
 	$(GOBIN)/gosec ./...
 
-rekor-server: $(SRC) $(PROTO_SRC)
-	CGO_ENABLED=0 go build -trimpath -ldflags "$(SERVER_LDFLAGS)" -o rekor-server ./cmd/rekor-server
+rekor-server-gcp: $(SRC) $(PROTO_SRC)
+	CGO_ENABLED=0 go build -trimpath -tags gcp -ldflags "$(SERVER_LDFLAGS)" -o rekor-server-gcp ./cmd/rekor-server-gcp
+
+rekor-server-aws: $(SRC) $(PROTO_SRC)
+	CGO_ENABLED=0 go build -trimpath -tags aws -ldflags "$(SERVER_LDFLAGS)" -o rekor-server-aws ./cmd/rekor-server-aws
+
+# Legacy target for backwards compatibility - builds GCP version
+rekor-server: rekor-server-gcp
+	cp rekor-server-gcp rekor-server
 
 ldflags: ## Print ldflags
 	@echo $(SERVER_LDFLAGS)
 
 test: ## Run all tests
-	go test ./...
+	@echo "Running tests with AWS backend..."
+	go test -tags aws ./...
+	@echo "Running tests with GCP backend..."
+	go test -tags gcp ./...
 
 ko-local: ## Build container images locally using ko
-	KO_DOCKER_REPO=ko.local LDFLAGS="$(SERVER_LDFLAGS)" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
+	KO_DOCKER_REPO=ko.local LDFLAGS="$(SERVER_LDFLAGS)" GOFLAGS="-tags=gcp" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
+	ko publish --base-import-paths \
+		--tags $(GIT_VERSION)-gcp --tags $(GIT_HASH) --image-refs rekorImagerefs-gcp \
+		github.com/sigstore/rekor-tiles/v2/cmd/rekor-server-gcp
+	KO_DOCKER_REPO=ko.local LDFLAGS="$(SERVER_LDFLAGS)" GOFLAGS="-tags=aws" GIT_HASH=$(GIT_HASH) GIT_VERSION=$(GIT_VERSION) \
 	ko publish --base-import-paths \
-		--tags $(GIT_VERSION) --tags $(GIT_HASH) --image-refs rekorImagerefs \
-		github.com/sigstore/rekor-tiles/v2/cmd/rekor-server
+		--tags $(GIT_VERSION)-aws --tags $(GIT_HASH) --image-refs rekorImagerefs-aws \
+		github.com/sigstore/rekor-tiles/v2/cmd/rekor-server-aws
+	cp rekorImagerefs-gcp rekorImagerefs
 
 # generate Go protobuf code
 protos:
@@ -106,7 +121,7 @@ clean: ## Remove built binaries and artifacts
 	rm -rf pkg/generated/protobuf/*
 	rm -rf dist
 	rm -rf hack/tools/bin
-	rm -rf rekor-server
+	rm -rf rekor-server rekor-server-gcp rekor-server-aws
 
 ##################
 # help

README.md

@@ -58,15 +58,84 @@ We provide prebuilt binaries and containers for private deployments.
 If you find any issues, follow Sigstore's [security policy](https://github.com/sigstore/rekor-tiles/security/policy)
 to report them.
 
+## Storage Backends
+
+Rekor v2 supports multiple storage backends. To avoid binary bloat from unused dependencies, separate binaries for each backend are provided:
+
+- `rekor-server-gcp`: GCP-specific binary (includes only Google Cloud dependencies)
+- `rekor-server-aws`: AWS-specific binary (includes only AWS dependencies)
+
+### Google Cloud Platform (GCP)
+- **Binary**: `rekor-server-gcp`
+- **Object Storage**: Google Cloud Storage (GCS)
+- **Database**: Cloud Spanner
+- **Use case**: Preferred for global deployments requiring strong consistency and automatic scaling
+
+### Amazon Web Services (AWS)
+- **Binary**: `rekor-server-aws`
+- **Object Storage**: Amazon S3
+- **Database**: Aurora MySQL (or RDS MySQL)
+- **Use case**: Cost-effective option for regional deployments with MySQL compatibility
+
 ## Local Development
 
-### Deployment
+### Deployment with GCP Emulators (Default)
 
 Run `docker compose up --build --wait` to start the service along with emulated Google Cloud Storage and Spanner instances.
 
 Run `docker compose down` to turn down the service, or `docker compose down --volumes` to turn down the service and delete
 persisted tiles.
 
+### Deployment with AWS Emulators
+
+Run `docker compose -f docker-compose-aws.yml up --build --wait` to start the service with MinIO (S3-compatible) and MySQL.
+
+Run `docker compose -f docker-compose-aws.yml down` to turn down the service, or add `--volumes` to delete persisted data.
+
+### Server Configuration
+
+When deploying your own instance, configure the storage backend using command-line flags:
+
+**GCP Backend:**
+```bash
+rekor-server-gcp serve \
+  --hostname=your-hostname \
+  --gcp-bucket=your-gcs-bucket \
+  --gcp-spanner=projects/PROJECT/instances/INSTANCE/databases/DATABASE \
+  --signer-filepath=/path/to/key.pem
+```
+
+**AWS Backend:**
+```bash
+rekor-server-aws serve \
+  --hostname=your-hostname \
+  --aws-bucket=your-s3-bucket \
+  --aws-mysql-dsn="user:password@tcp(host:3306)/database?parseTime=true" \
+  --signer-filepath=/path/to/key.pem
+```
+
+**AWS Environment Variables:**
+
+The AWS backend requires standard AWS SDK environment variables for authentication and configuration:
+
+Required:
+- `AWS_ACCESS_KEY_ID`: AWS access key ID for authentication
+- `AWS_SECRET_ACCESS_KEY`: AWS secret access key for authentication
+- `AWS_REGION`: AWS region for S3 bucket (e.g., `us-east-1`)
+
+Optional (for S3-compatible storage like MinIO):
+- `AWS_ENDPOINT_URL`: Custom S3 endpoint URL (e.g., `http://localhost:9000`)
+- `AWS_S3_FORCE_PATH_STYLE`: Set to `true` to use path-style addressing instead of virtual-hosted-style
+
+The `--aws-mysql-dsn` format is `user:password@tcp(host:port)/database?parseTime=true`. The `parseTime=true` parameter is required for proper timestamp handling.
+
+Optional flags for both backends:
+- `--persistent-antispam`: Enable persistent deduplication (requires Spanner or MySQL)
+- `--checkpoint-interval`: Frequency of checkpoint publishing (default: 30s)
+- `--batch-max-size`: Maximum entries per batch (default: 1024)
+
+See `rekor-server-gcp serve --help` or `rekor-server-aws serve --help` for all available options.
+
 ### Making a request
 
 Follow the [client documentation](https://github.com/sigstore/rekor-tiles/blob/main/CLIENTS.md#rekor-v2-the-bash-way)