feat(verify): offline / air-gapped verification (--insecure-ignore-tlog) (#1798)

Signed-off-by: Brian Lockwood <lockwobr@gmail.com>

Author: lockwobr

docs/user/cli-reference.md

@@ -2649,6 +2649,7 @@ aicr verify <bundle-dir> [flags]
 | `--certificate-identity-regexp` | string | | Override the certificate identity pattern for binary attestation verification. Must contain `"NVIDIA/aicr"`. For testing only. |
 | `--key` | string | | Verify a key-signed bundle attestation against a KMS key URI (`awskms://` \| `gcpkms://` \| `azurekms://` \| `hashivault://`) or a local PEM public-key file. This is the counterpart to `bundle --signing-key`. It coexists with `--certificate-identity-regexp`, which pins the binary attestation; the two verify different attestations. |
 | `--trust-root` | string | | Verify the bundle attestation against a private Sigstore trusted root (a `trusted_root.json` from a self-hosted Fulcio/Rekor). Additive to AICR's built-in public-good root, so NVIDIA-signed and privately-signed bundles both verify. Composes with `--key` and `--certificate-identity-regexp`. The verify counterpart to `bundle --fulcio-url`/`--rekor-url`. |
+| `--insecure-ignore-tlog` | bool | `false` | Offline/air-gapped verification: skip the transparency-log (and observer-timestamp) requirement so a bundle signed with `bundle --signing-key ... --tlog-upload=false` verifies against `--key` with no transparency-log network calls. A local PEM `--key` is then fully offline; a KMS `--key` URI still makes a live `GetPublicKey` call to resolve the key (export a PEM with `cosign public-key` for a truly offline verify). Requires `--key`; the air-gapped path is key-based, not keyless. Named "insecure" because, with no transparency log, there is no trusted timestamp proving when the signature was made. Does not affect the binary attestation, which always requires a transparency log. |
 | `--format` | string | `text` | Output format: `text` or `json`. |
 
 #### Trust Levels
@@ -2699,11 +2700,18 @@ aicr verify ./bundles/<bundle-dir> --key ./bundle-signer.pub
 
 # Verify a privately-signed bundle against an org trusted root
 aicr verify ./my-bundle --trust-root ./trusted_root.json
+
+# Verify an offline/air-gapped bundle: no transparency log, no network calls.
+# Sign on a connected host (signing needs KMS access), then export the public key once:
+aicr bundle -r recipe.yaml --attest --signing-key awskms://alias/my-key --tlog-upload=false -o ./bundles
+cosign public-key --key awskms://alias/my-key > bundle-signer.pub
+# Verify anywhere offline against the exported PEM (a KMS --key URI would make a live GetPublicKey call):
+aicr verify ./bundles/<bundle-dir> --key ./bundle-signer.pub --insecure-ignore-tlog
 ```
 
 > **`--key` network behavior:** Resolving a **KMS URI** (`awskms://`, `gcpkms://`, `azurekms://`, `hashivault://`) makes network calls to the KMS provider to fetch the public key, so credentials for that provider must be available in the environment. A **local PEM** public-key file is read from disk with no provider calls; export it once with `cosign public-key --key <kms-uri>` (or your provider's console) and verify anywhere.
 >
-> Resolving the key is only part of verification: by default the bundle's Rekor transparency-log entry is also checked. Its inclusion proof is embedded in the bundle, so no live Rekor call is made, but the check needs the Sigstore trusted root. That root is loaded from the local cache and falls back to the embedded trusted root on a cache miss, so no network fetch happens on the verify path and `aicr trust update` is not required for offline use. Verification that drops the transparency-log requirement entirely, for true air-gapped use, is tracked in [#1154](https://github.com/NVIDIA/aicr/issues/1154).
+> Resolving the key is only part of verification: by default the bundle's Rekor transparency-log entry is also checked. Its inclusion proof is embedded in the bundle, so no live Rekor call is made, but the check needs the Sigstore trusted root. That root is loaded from the local cache and falls back to the embedded trusted root on a cache miss, so no network fetch happens on the verify path and `aicr trust update` is not required for offline use. For a bundle signed without a transparency-log entry at all (`bundle --signing-key ... --tlog-upload=false`), pass `--insecure-ignore-tlog` alongside `--key` to drop the transparency-log requirement entirely; verification then runs with no network calls and no trusted timestamp, so use it only for true air-gapped bundles you signed yourself.
 >
 > **Stale root:** If verification fails with certificate chain errors, run `aicr trust update` to refresh the Sigstore trusted root.
 

pkg/bundler/attestation/verifying_test.go

@@ -16,11 +16,15 @@ package attestation
 
 import (
 	"context"
+	"crypto/ecdsa"
+	"crypto/elliptic"
+	"crypto/rand"
 	stderrors "errors"
 	"strings"
 	"testing"
 
 	"github.com/sigstore/sigstore-go/pkg/fulcio/certificate"
+	"github.com/sigstore/sigstore-go/pkg/root"
 	"github.com/sigstore/sigstore-go/pkg/verify"
 
 	"github.com/NVIDIA/aicr/pkg/errors"
@@ -194,3 +198,63 @@ func TestRequireTLogPolicy_VerifierOptions(t *testing.T) {
 		t.Fatalf("VerifierOptions() returned %d options, want 2", len(opts))
 	}
 }
+
+func TestNoTLogVerifyPolicy_VerifierOptions(t *testing.T) {
+	opts := NewNoTLogVerifyPolicy().VerifierOptions()
+	// The offline policy carries exactly one option (WithNoObserverTimestamps),
+	// which sigstore-go requires be specified exclusively.
+	if len(opts) != 1 {
+		t.Fatalf("VerifierOptions() returned %d options, want 1", len(opts))
+	}
+	if opts[0] == nil {
+		t.Fatal("VerifierOptions()[0] is nil")
+	}
+}
+
+// TestNoTLogVerifyPolicy_RoundTrip proves the offline relaxation is load-bearing:
+// a key-signed bundle produced with NewNoTLogPolicy() (no Rekor entry, no
+// timestamp) verifies fully offline with NewNoTLogVerifyPolicy(), while the same
+// bundle FAILS under the default NewRequireTLogPolicy() (which demands a tlog
+// inclusion proof / observer timestamp the air-gapped bundle does not carry).
+func TestNoTLogVerifyPolicy_RoundTrip(t *testing.T) {
+	ctx := context.Background()
+
+	// One key drives both the offline signer and the matching key verifier.
+	priv, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader)
+	if err != nil {
+		t.Fatalf("generate key: %v", err)
+	}
+	signID := &fakeKeyIdentity{signer: &localECDSASigner{priv: priv}, uri: "awskms://test/key"}
+
+	res, err := SignStatementWith(ctx, validStatementJSON(t), signID, NewNoTLogPolicy())
+	if err != nil {
+		t.Fatalf("SignStatementWith (offline, no tlog): %v", err)
+	}
+
+	// Build the key verification identity from the public half. Inject an empty
+	// trust-root source (not the default public-good root, which would require a
+	// TUF fetch) to keep the round trip fully offline: a key-based, no-timestamp
+	// verify needs only the key material, exactly the air-gapped guarantee.
+	offlineSrc := func(context.Context) (root.TrustedMaterial, error) {
+		return root.TrustedMaterialCollection{}, nil
+	}
+	pemPath := writeTempPubPEM(t, &priv.PublicKey)
+	verID, err := NewKeyVerificationIdentity(ctx, pemPath, offlineSrc)
+	if err != nil {
+		t.Fatalf("NewKeyVerificationIdentity: %v", err)
+	}
+
+	// validStatementJSON binds subject checksums.txt to an all-zero sha256, so
+	// the artifact digest is 32 zero bytes.
+	digest := make([]byte, 32)
+
+	if _, err := VerifyStatementWith(ctx, res.BundleJSON, verID, NewNoTLogVerifyPolicy(), digest); err != nil {
+		t.Fatalf("offline verify with NewNoTLogVerifyPolicy() must succeed, got %v", err)
+	}
+
+	// The relaxation is load-bearing: the same tlog-less bundle must fail the
+	// default policy, which requires a transparency-log / observer timestamp.
+	if _, err := VerifyStatementWith(ctx, res.BundleJSON, verID, NewRequireTLogPolicy(), digest); err == nil {
+		t.Fatal("verify with NewRequireTLogPolicy() must fail for a tlog-less bundle, got nil error")
+	}
+}

pkg/bundler/attestation/verifytransparency.go

@@ -24,8 +24,8 @@ import (
 // records a Rekor entry.
 //
 // The offline relaxation (no transparency log, key-based verification of an
-// air-gapped signature) is tracked as #1154 and will add a sibling policy
-// here, mirroring how noTLogPolicy complements rekorPolicy on the signing side.
+// air-gapped signature) is #1154, now implemented as noTLogVerifyPolicy below,
+// mirroring how noTLogPolicy complements rekorPolicy on the signing side.
 type requireTLogPolicy struct{}
 
 // NewRequireTLogPolicy returns a VerifyTransparencyPolicy that requires a
@@ -38,3 +38,33 @@ func (requireTLogPolicy) VerifierOptions() []verify.VerifierOption {
 		verify.WithObserverTimestamps(1),
 	}
 }
+
+// noTLogVerifyPolicy is the offline/air-gapped verification dual of noTLogPolicy
+// (the signing side, #409): it requires no transparency-log inclusion proof and
+// no observer timestamp, so a key-signed bundle with neither can verify with no
+// transparency-log network calls. This policy governs only the transparency/
+// timestamp axis; whether verification is fully offline also depends on the
+// verification identity and trusted-material source (a local PEM key + offline
+// trusted root is fully offline, whereas a KMS key URI still resolves remotely).
+// INSECURE relative to the default: without a tlog/timestamp there is no trusted
+// proof of WHEN the signature was made. Only valid for key-based (--key)
+// verification of an air-gapped signature.
+//
+// verify.WithNoObserverTimestamps() sets the sigstore-go verifier's
+// allowNoTimestamp flag, which the verifier's config validation requires to be
+// specified exclusively (no WithTransparencyLog / WithObserverTimestamps
+// alongside it); it is only usable for key-based, not certificate-based,
+// verification, which is exactly the air-gapped KMS/PEM path here.
+type noTLogVerifyPolicy struct{}
+
+// NewNoTLogVerifyPolicy returns a VerifyTransparencyPolicy that requires no
+// transparency-log inclusion proof and no observer timestamp, so an air-gapped
+// key-signed bundle verifies with no transparency-log network calls (fully
+// offline use additionally requires an offline key and trusted-material source).
+func NewNoTLogVerifyPolicy() VerifyTransparencyPolicy { return noTLogVerifyPolicy{} }
+
+func (noTLogVerifyPolicy) VerifierOptions() []verify.VerifierOption {
+	return []verify.VerifierOption{
+		verify.WithNoObserverTimestamps(),
+	}
+}

pkg/bundler/verifier/verifier.go

@@ -154,6 +154,19 @@ type VerifyOptions struct {
 	// Composable with Key. Does NOT affect the binary attestation, which is
 	// always NVIDIA-public-CI-signed and stays pinned to the public-good root.
 	TrustRoot string
+
+	// IgnoreTLog enables offline/air-gapped verification of the key-signed
+	// bundle attestation: it skips the transparency-log (and observer-timestamp)
+	// requirement so a bundle produced by `bundle --signing-key ... --tlog-upload=false`
+	// (#409) verifies with no transparency-log network calls. Full offline
+	// operation additionally requires a local PEM Key: a KMS Key URI still makes a
+	// live GetPublicKey call via NewKeyVerificationIdentity to resolve the key.
+	// ONLY valid with Key set (the air-gapped path is key-based, not keyless);
+	// Verify rejects it otherwise. INSECURE relative to the default: without a
+	// tlog/timestamp there is no trusted proof of when the signature was made.
+	// Does NOT affect the keyless or binary-attestation paths, which always
+	// require a transparency log.
+	IgnoreTLog bool
 }
 
 // resolveBundleTrustRoot returns the TrustedRootSource for the bundle
@@ -262,17 +275,30 @@ func verifyWithDependencies(
 	if err := ctx.Err(); err != nil {
 		return nil, errors.Wrap(errors.ErrCodeTimeout, "context cancelled before verification", err)
 	}
+
+	// Resolve and validate options before touching the filesystem so a malformed
+	// request (e.g. IgnoreTLog without Key) fails with InvalidRequest regardless
+	// of whether the bundle directory exists, rather than being masked by a
+	// NotFound from the os.Stat below.
+	if opts == nil {
+		opts = &VerifyOptions{}
+	}
+	// Offline/air-gapped verification is key-based only: rejecting the transparency
+	// log without a public key would leave the keyless path with no trusted
+	// timestamp and nothing to bind the signature to. Fail closed here (not just
+	// in the CLI) so library callers get the same guarantee.
+	if opts.IgnoreTLog && opts.Key == "" {
+		return nil, errors.New(errors.ErrCodeInvalidRequest,
+			"IgnoreTLog (offline verification) requires Key: the air-gapped path is key-based")
+	}
+
 	if _, err := os.Stat(bundleDir); err != nil {
 		if os.IsNotExist(err) {
 			return nil, errors.New(errors.ErrCodeNotFound, "bundle directory not found: "+bundleDir)
 		}
 		return nil, errors.Wrap(errors.ErrCodeInternal, "cannot access bundle directory", err)
 	}
 
-	// Resolve options
-	if opts == nil {
-		opts = &VerifyOptions{}
-	}
 	identityPattern := opts.CertificateIdentityRegexp
 	if identityPattern == "" {
 		identityPattern = TrustedRepositoryPattern
@@ -360,7 +386,7 @@ func verifyStagedSnapshot(
 		err           error
 	)
 	if opts.Key != "" {
-		bundleCreator, err = verifyKeySignedBundle(ctx, bundleAttestPath, checksumDigest, opts.Key, bundleSource)
+		bundleCreator, err = verifyKeySignedBundle(ctx, bundleAttestPath, checksumDigest, opts.Key, bundleSource, opts.IgnoreTLog)
 	} else {
 		bundleCreator, err = verifySigstoreBundle(ctx, bundleAttestPath, checksumDigest, bundleSource)
 	}
@@ -700,8 +726,12 @@ func verifySigstoreBundle(ctx context.Context, bundlePath string, artifactDigest
 // verifyKeySignedBundle verifies a public-key-signed bundle attestation (#407
 // KMS signing or a local PEM) against the key named by keyRef, binding it to
 // artifactDigest. Returns the key identity (KMS URI or pem:<fp>) as the bundle
-// creator, since a key-signed bundle has no certificate SAN.
-func verifyKeySignedBundle(ctx context.Context, bundlePath string, artifactDigest []byte, keyRef string, src attestation.TrustedRootSource) (string, error) {
+// creator, since a key-signed bundle has no certificate SAN. When ignoreTLog is
+// true (offline/air-gapped, #1154), the transparency-log requirement is relaxed
+// so a bundle signed with `bundle --signing-key ... --tlog-upload=false` (#409)
+// verifies with no transparency-log network calls (a local PEM keyRef is then
+// fully offline; a KMS URI keyRef still resolves the key remotely).
+func verifyKeySignedBundle(ctx context.Context, bundlePath string, artifactDigest []byte, keyRef string, src attestation.TrustedRootSource, ignoreTLog bool) (string, error) {
 	id, err := attestation.NewKeyVerificationIdentity(ctx, keyRef, src)
 	if err != nil {
 		return "", err // already classified (ErrCodeInvalidRequest / ErrCodeUnavailable)
@@ -710,7 +740,11 @@ func verifyKeySignedBundle(ctx context.Context, bundlePath string, artifactDiges
 	if err != nil {
 		return "", err // already coded by readBoundedFileContext
 	}
-	signer, err := attestation.VerifyStatementWith(ctx, data, id, attestation.NewRequireTLogPolicy(), artifactDigest)
+	tlogPolicy := attestation.NewRequireTLogPolicy()
+	if ignoreTLog {
+		tlogPolicy = attestation.NewNoTLogVerifyPolicy()
+	}
+	signer, err := attestation.VerifyStatementWith(ctx, data, id, tlogPolicy, artifactDigest)
 	if err != nil {
 		return "", err
 	}

pkg/bundler/verifier/verifier_test.go

@@ -185,6 +185,51 @@ func TestVerify_NilOptions(t *testing.T) {
 	}
 }
 
+// TestVerify_IgnoreTLogRequiresKey pins the defense-in-depth guard: offline
+// verification (IgnoreTLog) is key-based only, so Verify must reject the option
+// with ErrCodeInvalidRequest when Key is empty, before any bundle inspection.
+func TestVerify_IgnoreTLogRequiresKey(t *testing.T) {
+	dir := createTestBundle(t)
+
+	tests := []struct {
+		name    string
+		opts    *VerifyOptions
+		wantErr bool
+	}{
+		{"ignore tlog without key", &VerifyOptions{IgnoreTLog: true}, true},
+		{"ignore tlog with key", &VerifyOptions{IgnoreTLog: true, Key: "awskms://test/key"}, false},
+		{"no ignore tlog, no key", &VerifyOptions{}, false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			_, err := Verify(context.Background(), dir, tt.opts)
+			if tt.wantErr {
+				if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) {
+					t.Fatalf("Verify() error = %v, want ErrCodeInvalidRequest", err)
+				}
+				return
+			}
+			// The non-error cases here assert only that the guard did NOT trip;
+			// downstream verification (e.g. a bogus KMS key) may still fail, so we
+			// only require the error, if any, is not the guard's InvalidRequest.
+			if err != nil && strings.Contains(err.Error(), "offline verification") {
+				t.Fatalf("Verify() unexpectedly tripped IgnoreTLog guard: %v", err)
+			}
+		})
+	}
+
+	// The guard runs before the bundle-directory stat, so an invalid
+	// IgnoreTLog/Key combination is reported as InvalidRequest even when the
+	// directory does not exist (it must not be masked by NotFound).
+	t.Run("guard precedes NotFound for a missing directory", func(t *testing.T) {
+		_, err := Verify(context.Background(), filepath.Join(t.TempDir(), "does-not-exist"),
+			&VerifyOptions{IgnoreTLog: true})
+		if !stderrors.Is(err, errors.New(errors.ErrCodeInvalidRequest, "")) {
+			t.Fatalf("Verify() error = %v, want ErrCodeInvalidRequest", err)
+		}
+	})
+}
+
 func TestExtractToolVersion(t *testing.T) {
 	t.Run("valid bundle with tool version", func(t *testing.T) {
 		// Build a minimal sigstore bundle JSON with a DSSE envelope

pkg/cli/bundle_verify.go

@@ -65,6 +65,10 @@ Require a minimum CLI version (bare version defaults to >= semantics):
 Verify a privately-signed bundle against an org trusted root:
   aicr verify ./my-bundle --trust-root ./trusted_root.json
 
+Verify an offline/air-gapped bundle (no transparency-log network calls; use a
+local PEM key for fully offline operation, since a KMS URI still resolves remotely):
+  aicr verify ./bundle --key ./bundle-signer.pub --insecure-ignore-tlog
+
 Output as JSON:
   aicr verify ./my-bundle --format json
 `,
@@ -99,6 +103,10 @@ Output as JSON:
 				Name:  "trust-root",
 				Usage: "Verify the bundle attestation against a private Sigstore trusted root (a trusted_root.json from a self-hosted Fulcio/Rekor). Additive to AICR's built-in public-good root, so NVIDIA-signed and privately-signed bundles both verify. Composes with --key and --certificate-identity-regexp. The verify counterpart to `bundle --fulcio-url`/`--rekor-url`.",
 			},
+			&cli.BoolFlag{
+				Name:  "insecure-ignore-tlog",
+				Usage: "Skip transparency-log verification for an offline/air-gapped bundle signed with `bundle --signing-key ... --tlog-upload=false`. Verifies the signature against --key with no transparency-log network calls. Requires --key (the air-gapped path is key-based); a local PEM key is fully offline, while a KMS URI still makes a live GetPublicKey call to resolve the key. \"insecure\" because, with no transparency log, there is no trusted timestamp proving when the signature was made.",
+			},
 			withCompletions(&cli.StringFlag{
 				Name:  flagFormat,
 				Value: verifyFormatText,
@@ -139,6 +147,14 @@ func runBundleVerifyCmd(ctx context.Context, cmd *cli.Command) error {
 	}
 	verifyOpts.Key = cmd.String("key")
 	verifyOpts.TrustRoot = cmd.String("trust-root")
+	verifyOpts.IgnoreTLog = cmd.Bool("insecure-ignore-tlog")
+
+	// Offline/air-gapped verification is key-based only: reject the flag combo up
+	// front so the user gets a clear message instead of a downstream failure.
+	if verifyOpts.IgnoreTLog && verifyOpts.Key == "" {
+		return errors.New(errors.ErrCodeInvalidRequest,
+			"--insecure-ignore-tlog requires --key: offline verification is key-based (verify a bundle signed with `bundle --signing-key ... --tlog-upload=false`)")
+	}
 
 	// Run verification
 	result, err := verifier.Verify(ctx, absDir, verifyOpts)