feat(session): manage KAP mTLS agent credentials (#1264)

## Summary

- keep only the KAP mTLS status and credential-update session commands;
kubeconfig is now fixed during Lepton bootstrap
- validate each per-machine certificate, identity, endpoint, and CA
fingerprint before writing an immutable credential generation
- atomically switch the `current` symlink, restart the agent, and
acknowledge the update only after `/readyz` succeeds
- remove enable/disable state, kubeconfig mutation, pending markers,
nested TLS preflight, and automatic rollback
- keep certificate and private-key material redacted from session audit
logs

## Rollout

- the companion Lepton MR
[dgxc-lepton/lepton!7510](https://gitlab-master.nvidia.com/dgxc-lepton/lepton/-/merge_requests/7510)
installs the agent by default on new nodes and points kubelet at
loopback from bootstrap
- existing nodes are not backfilled; Lepton reconciles only Machines
that report the `kap-mtls-agent` package
- release this gpud change before or with the Lepton node-init rollout;
failures leave new nodes blocked before kubelet registration

## Verification

- `GOFLAGS=-mod=mod go test -count=1 ./pkg/kapmtls`
- `GOFLAGS=-mod=mod go test -count=1 -gcflags='all=-N -l' ./pkg/session`
- `GOFLAGS=-mod=mod go test -race ./pkg/kapmtls`
- `GOFLAGS=-mod=mod go vet ./pkg/kapmtls ./pkg/session`

The repository-wide default `pkg/session` test command requires Mockey's
`-gcflags='all=-N -l'` on macOS/arm64; the changed package scope passes
with that required flag.

Jira: LEP-5336

---------

Co-authored-by: Codex <noreply@openai.com>
Co-authored-by: Gyuho Lee <gyuhol@nvidia.com>

Author: 3 people

api/v1/login.go

@@ -70,6 +70,9 @@ type LoginResponse struct {
 	// Token is the token used to report data from the machine.
 	Token string `json:"token,omitempty"`
 
+	// MachineProof binds privileged session commands to this machine identity.
+	MachineProof string `json:"machineProof,omitempty"`
+
 	// ValidationResults is the validation results for the login request.
 	// The validation results are done by the control plane.
 	ValidationResults []ValidationResult `json:"validationResults,omitempty"`

cmd/gpud/metadata/command.go

@@ -38,6 +38,18 @@ type metadataJSONOutput struct {
 	Updated       *metadataUpdatedEntry `json:"updated,omitempty"`
 }
 
+func maskMetadataValue(key, value string) string {
+	switch key {
+	case pkgmetadata.MetadataKeyToken:
+		return pkgmetadata.MaskToken(value)
+	case pkgmetadata.MetadataKeyMachineProof:
+		// Machine proof authorizes privileged session commands, so reveal none of it.
+		return "<redacted>"
+	default:
+		return value
+	}
+}
+
 func Command(cliContext *cli.Context) error {
 	outputFormat, err := gpudcommon.ParseOutputFormat(cliContext.String("output-format"))
 	if err != nil {
@@ -92,9 +104,7 @@ func Command(cliContext *cli.Context) error {
 
 	maskedMetadata := make(map[string]string, len(metadata))
 	for k, v := range metadata {
-		if k == pkgmetadata.MetadataKeyToken {
-			v = pkgmetadata.MaskToken(v)
-		}
+		v = maskMetadataValue(k, v)
 		maskedMetadata[k] = v
 		if outputFormat == gpudcommon.OutputFormatPlain {
 			fmt.Printf("%s: %s\n", k, v)
@@ -129,13 +139,14 @@ func Command(cliContext *cli.Context) error {
 		}()
 		log.Logger.Debugw("successfully opened state file for writing")
 
-		log.Logger.Debugw("setting metadata", "key", setKey, "value", setValue)
+		maskedSetValue := maskMetadataValue(setKey, setValue)
+		log.Logger.Debugw("setting metadata", "key", setKey, "value", maskedSetValue)
 		if err := pkgmetadata.SetMetadata(rootCtx, dbRW, setKey, setValue); err != nil {
 			return wrapErr("failed_to_update_metadata", fmt.Errorf("failed to update metadata: %w", err))
 		}
 		log.Logger.Debugw("successfully updated metadata")
 
-		updated = &metadataUpdatedEntry{Key: setKey, Value: setValue}
+		updated = &metadataUpdatedEntry{Key: setKey, Value: maskedSetValue}
 		if outputFormat == gpudcommon.OutputFormatPlain {
 			fmt.Printf("%s successfully updated metadata\n", cmdcommon.CheckMark)
 		}

cmd/gpud/metadata/mock_command_test.go

@@ -441,12 +441,14 @@ func TestCommand_JSONOutput(t *testing.T) {
 		stateFile := filepath.Join(dataDir, "gpud.state")
 
 		rawToken := "nvapi-stg-1234567890abcdef"
+		machineProof := "machine-proof-secret"
 		evTime := time.Date(2025, 1, 2, 3, 4, 5, 0, time.UTC)
 		evMessage := "test reboot event"
 
 		createMetadataDB(t, stateFile, map[string]string{
-			pkgmetadata.MetadataKeyMachineID: "machine-1",
-			pkgmetadata.MetadataKeyToken:     rawToken,
+			pkgmetadata.MetadataKeyMachineID:    "machine-1",
+			pkgmetadata.MetadataKeyToken:        rawToken,
+			pkgmetadata.MetadataKeyMachineProof: machineProof,
 		})
 		insertRebootEvent(t, stateFile, evTime, evMessage)
 
@@ -475,13 +477,47 @@ func TestCommand_JSONOutput(t *testing.T) {
 		require.NotNil(t, out.Metadata)
 		assert.Equal(t, "machine-1", out.Metadata[pkgmetadata.MetadataKeyMachineID])
 		assert.Equal(t, pkgmetadata.MaskToken(rawToken), out.Metadata[pkgmetadata.MetadataKeyToken])
+		assert.Equal(t, "<redacted>", out.Metadata[pkgmetadata.MetadataKeyMachineProof])
+		assert.NotContains(t, stdout, machineProof)
 		require.Len(t, out.RebootHistory, 1)
 		assert.Equal(t, evTime.Format(time.RFC3339), out.RebootHistory[0].TimeUTC)
 		assert.Equal(t, evMessage, out.RebootHistory[0].Message)
 		assert.NotContains(t, stdout, "Reboot History:")
 	})
 }
 
+func TestCommand_JSONOutputRedactsUpdatedMachineProof(t *testing.T) {
+	mockey.PatchConvey("json output redacts updated machine proof", t, func() {
+		mockey.Mock(osutil.RequireRoot).To(func() error { return nil }).Build()
+
+		dataDir := t.TempDir()
+		stateFile := filepath.Join(dataDir, "gpud.state")
+		createMetadataDB(t, stateFile, map[string]string{})
+
+		cliContext := newCLIContext(t, []string{
+			"--data-dir", dataDir,
+			"--set-key", pkgmetadata.MetadataKeyMachineProof,
+			"--set-value", "machine-proof-secret",
+			"--output-format", "json",
+		})
+		stdout, stderr := captureOutput(t, func() { require.NoError(t, Command(cliContext)) })
+		assert.Empty(t, stderr)
+		assert.NotContains(t, stdout, "machine-proof-secret")
+
+		var out metadataJSONOutput
+		require.NoError(t, json.Unmarshal([]byte(stdout), &out))
+		require.NotNil(t, out.Updated)
+		assert.Equal(t, "<redacted>", out.Updated.Value)
+
+		dbRO, err := sqlite.Open(stateFile, sqlite.WithReadOnly(true))
+		require.NoError(t, err)
+		defer func() { _ = dbRO.Close() }()
+		stored, err := pkgmetadata.ReadMetadata(context.Background(), dbRO, pkgmetadata.MetadataKeyMachineProof)
+		require.NoError(t, err)
+		assert.Equal(t, "machine-proof-secret", stored)
+	})
+}
+
 func TestCommand_JSONOutputSetKeyValue(t *testing.T) {
 	mockey.PatchConvey("json output set key value", t, func() {
 		mockey.Mock(osutil.RequireRoot).To(func() error {

cmd/gpud/run/mock_command_test.go

@@ -260,15 +260,53 @@ func TestGetSessionCredentialsOptions_AllCredentialsPresent(t *testing.T) {
 		}).Build()
 
 		mockey.Mock(pkgmetadata.ReadMetadata).To(func(ctx context.Context, db *sql.DB, key string) (string, error) {
-			if key == pkgmetadata.MetadataKeyEndpoint {
+			switch key {
+			case pkgmetadata.MetadataKeyEndpoint:
 				return "https://stored.endpoint.com", nil
+			case pkgmetadata.MetadataKeyMachineProof:
+				return "machine-proof", nil
 			}
 			return "", nil
 		}).Build()
 
 		opts := getSessionCredentialsOptions(true, tmpDir, "")
 		assert.NotEmpty(t, opts)
-		assert.Len(t, opts, 3) // token, machine ID, endpoint
+		assert.Len(t, opts, 4) // token, machine ID, endpoint, machine proof
+
+		op := &config.Op{}
+		require.NoError(t, op.ApplyOpts(opts))
+		assert.Equal(t, "machine-proof", op.SessionMachineProof)
+	})
+}
+
+func TestGetSessionCredentialsOptions_MachineProofReadError(t *testing.T) {
+	tmpDir := t.TempDir()
+
+	mockey.PatchConvey("getSessionCredentialsOptions machine proof read error", t, func() {
+		createTestStateFile(t, tmpDir)
+
+		mockey.Mock(config.ResolveDataDir).To(func(string) (string, error) {
+			return tmpDir, nil
+		}).Build()
+		mockey.Mock(pkgsqlite.Open).To(func(string, ...pkgsqlite.OpOption) (*sql.DB, error) {
+			return &sql.DB{}, nil
+		}).Build()
+		mockey.Mock((*sql.DB).Close).To(func(*sql.DB) error { return nil }).Build()
+		mockey.Mock(pkgmetadata.ReadToken).To(func(context.Context, *sql.DB) (string, error) {
+			return "session-token", nil
+		}).Build()
+		mockey.Mock(pkgmetadata.ReadMachineID).To(func(context.Context, *sql.DB) (string, error) {
+			return "machine-id-123", nil
+		}).Build()
+		mockey.Mock(pkgmetadata.ReadMetadata).To(func(_ context.Context, _ *sql.DB, key string) (string, error) {
+			if key == pkgmetadata.MetadataKeyEndpoint {
+				return "https://stored.endpoint.com", nil
+			}
+			return "", errors.New("machine proof unavailable")
+		}).Build()
+
+		opts := getSessionCredentialsOptions(true, tmpDir, "")
+		require.Len(t, opts, 3)
 	})
 }
 

cmd/gpud/run/mock_session_test.go

@@ -5,6 +5,7 @@ import (
 	"database/sql"
 	"errors"
 	"os"
+	"path/filepath"
 	"testing"
 
 	"github.com/bytedance/mockey"
@@ -38,13 +39,39 @@ func TestReadSessionCredentialsFromPersistentFile(t *testing.T) {
 	require.NoError(t, pkgmetadata.SetMetadata(ctx, dbRW, pkgmetadata.MetadataKeyToken, "session-token"))
 	require.NoError(t, pkgmetadata.SetMetadata(ctx, dbRW, pkgmetadata.MetadataKeyMachineID, "assigned-machine-id"))
 	require.NoError(t, pkgmetadata.SetMetadata(ctx, dbRW, pkgmetadata.MetadataKeyEndpoint, "gpud-manager.example.com"))
+	require.NoError(t, pkgmetadata.SetMetadata(ctx, dbRW, pkgmetadata.MetadataKeyMachineProof, "machine-proof"))
 
 	gotToken, gotMachineID, gotEndpoint, err := readSessionCredentialsFromPersistentFile(ctx, tmpDir)
 	require.NoError(t, err)
 
 	assert.Equal(t, "session-token", gotToken)
 	assert.Equal(t, "assigned-machine-id", gotMachineID)
 	assert.Equal(t, "gpud-manager.example.com", gotEndpoint)
+
+	gotMachineProof, err := readMachineProofFromPersistentFile(ctx, tmpDir)
+	require.NoError(t, err)
+	assert.Equal(t, "machine-proof", gotMachineProof)
+}
+
+func TestReadMachineProofFromPersistentFileErrors(t *testing.T) {
+	t.Run("resolve data directory", func(t *testing.T) {
+		parentFile := filepath.Join(t.TempDir(), "file")
+		require.NoError(t, os.WriteFile(parentFile, []byte("data"), 0600))
+
+		_, err := readMachineProofFromPersistentFile(context.Background(), filepath.Join(parentFile, "child"))
+		require.ErrorContains(t, err, "failed to resolve data dir")
+	})
+
+	t.Run("open state file", func(t *testing.T) {
+		mockey.PatchConvey("open state file error", t, func() {
+			mockey.Mock(pkgsqlite.Open).To(func(string, ...pkgsqlite.OpOption) (*sql.DB, error) {
+				return nil, errors.New("open failed")
+			}).Build()
+
+			_, err := readMachineProofFromPersistentFile(context.Background(), t.TempDir())
+			require.ErrorContains(t, err, "failed to open state file")
+		})
+	})
 }
 
 func TestReadSessionCredentialsFromPersistentFile_StateFileNotFound(t *testing.T) {

cmd/gpud/run/session.go

@@ -95,6 +95,20 @@ func readSessionCredentialsFromPersistentFile(ctx context.Context, dataDir strin
 	return sessionToken, assignedMachineID, endpoint, nil
 }
 
+func readMachineProofFromPersistentFile(ctx context.Context, dataDir string) (string, error) {
+	resolvedDataDir, err := config.ResolveDataDir(dataDir)
+	if err != nil {
+		return "", fmt.Errorf("failed to resolve data dir: %w", err)
+	}
+	stateFile := config.StateFilePath(resolvedDataDir)
+	dbRO, err := pkgsqlite.Open(stateFile, pkgsqlite.WithReadOnly(true))
+	if err != nil {
+		return "", fmt.Errorf("failed to open state file: %w", err)
+	}
+	defer func() { _ = dbRO.Close() }()
+	return pkgmetadata.ReadMetadata(ctx, dbRO, pkgmetadata.MetadataKeyMachineProof)
+}
+
 func getSessionCredentialsOptions(dbInMemory bool, dataDir string, controlPlaneEndpoint string) []config.OpOption {
 	// When --db-in-memory is enabled, read session credentials from the persistent state file
 	// and pass them to the config so the server can seed them into the in-memory database.
@@ -144,11 +158,18 @@ func getSessionCredentialsOptions(dbInMemory bool, dataDir string, controlPlaneE
 			"machineID", assignedMachineID,
 			"endpoint", endpoint,
 		)
-		return []config.OpOption{
+		options := []config.OpOption{
 			config.WithSessionToken(sessionToken),
 			config.WithSessionMachineID(assignedMachineID),
 			config.WithSessionEndpoint(endpoint),
 		}
+		machineProof, proofErr := readMachineProofFromPersistentFile(readCtx, dataDir)
+		if proofErr != nil {
+			log.Logger.Warnw("failed to read machine proof from persistent state", "error", proofErr)
+		} else if machineProof != "" {
+			options = append(options, config.WithSessionMachineProof(machineProof))
+		}
+		return options
 	}
 
 	// Credentials were read but are incomplete - this may indicate a partial state

pkg/config/config.go

@@ -119,6 +119,9 @@ type Config struct {
 	// This allows gpud up to pass the assigned machine ID from login to gpud run.
 	SessionMachineID string `json:"-"`
 
+	// SessionMachineProof is the per-machine proof returned by login.
+	SessionMachineProof string `json:"-"`
+
 	// SessionEndpoint is the control plane endpoint.
 	// Used when DBInMemory is true and session credentials are passed via CLI flags.
 	// This allows gpud up to pass the endpoint from login to gpud run.

pkg/config/default.go

@@ -66,9 +66,10 @@ func DefaultConfig(ctx context.Context, opts ...OpOption) (*Config, error) {
 
 		ContainerdServiceActiveCommands: options.ContainerdServiceActiveCommands,
 
-		SessionToken:     options.SessionToken,
-		SessionMachineID: options.SessionMachineID,
-		SessionEndpoint:  options.SessionEndpoint,
+		SessionToken:        options.SessionToken,
+		SessionMachineID:    options.SessionMachineID,
+		SessionMachineProof: options.SessionMachineProof,
+		SessionEndpoint:     options.SessionEndpoint,
 	}
 
 	if cfg.State == "" {

pkg/config/options.go

@@ -45,6 +45,9 @@ type Op struct {
 	// this machine ID into the in-memory database.
 	SessionMachineID string
 
+	// SessionMachineProof is the per-machine proof for db-in-memory mode.
+	SessionMachineProof string
+
 	// SessionEndpoint is the control plane endpoint for db-in-memory mode.
 	// When DBInMemory is true and this is set, the server will seed
 	// this endpoint into the in-memory database.
@@ -170,6 +173,13 @@ func WithSessionMachineID(machineID string) OpOption {
 	}
 }
 
+// WithSessionMachineProof sets the per-machine proof for db-in-memory mode.
+func WithSessionMachineProof(machineProof string) OpOption {
+	return func(op *Op) {
+		op.SessionMachineProof = machineProof
+	}
+}
+
 // WithSessionEndpoint sets the control plane endpoint for db-in-memory mode.
 // When DBInMemory is true and this is set, the server will seed
 // this endpoint into the in-memory database.

pkg/config/options_test.go

@@ -147,6 +147,7 @@ func TestDBInMemoryWithSessionCredentials(t *testing.T) {
 		WithDBInMemory(true),
 		WithSessionToken("session-token-from-login-response"),
 		WithSessionMachineID("assigned-machine-id-from-login-response"),
+		WithSessionMachineProof("machine-proof-from-login-response"),
 		WithSessionEndpoint("https://api.example.com"),
 	}
 
@@ -156,6 +157,7 @@ func TestDBInMemoryWithSessionCredentials(t *testing.T) {
 	assert.True(t, op.DBInMemory)
 	assert.Equal(t, "session-token-from-login-response", op.SessionToken)
 	assert.Equal(t, "assigned-machine-id-from-login-response", op.SessionMachineID)
+	assert.Equal(t, "machine-proof-from-login-response", op.SessionMachineProof)
 	assert.Equal(t, "https://api.example.com", op.SessionEndpoint)
 }