fix(auth-callout): restore nv-config permissions watcher (#26)

Signed-off-by: Frank Spitulski <fspitulski@nvidia.com>

Author: FrankSpitulski

auth-callout/go.mod

@@ -21,14 +21,14 @@ require (
 )
 
 require (
-	github.com/fsnotify/fsnotify v1.9.0
 	github.com/knadh/koanf/maps v0.1.2
 	github.com/knadh/koanf/parsers/yaml v1.1.0
 	github.com/knadh/koanf/providers/env v1.1.0
 	github.com/knadh/koanf/providers/file v1.2.0
 	github.com/knadh/koanf/providers/rawbytes v1.0.0
 	github.com/knadh/koanf/v2 v2.3.0
 	github.com/prometheus/client_golang v1.23.2
+	gitlab-master.nvidia.com/ncp/vmaas/libs/golang/nv-config v0.1.7
 	go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc v1.38.0
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.38.0
 	go.opentelemetry.io/otel/exporters/prometheus v0.60.0
@@ -44,6 +44,7 @@ require (
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fsnotify/fsnotify v1.9.0 // indirect
 	github.com/go-logr/logr v1.4.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-viper/mapstructure/v2 v2.4.0 // indirect
@@ -53,6 +54,7 @@ require (
 	github.com/grpc-ecosystem/grpc-gateway/v2 v2.27.2 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
+	github.com/knadh/koanf/providers/confmap v1.0.0 // indirect
 	github.com/minio/highwayhash v1.0.3 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect

auth-callout/go.sum

@@ -54,6 +54,8 @@ github.com/knadh/koanf/maps v0.1.2 h1:RBfmAW5CnZT+PJ1CVc1QSJKf4Xu9kxfQgYVQSu8hpb
 github.com/knadh/koanf/maps v0.1.2/go.mod h1:npD/QZY3V6ghQDdcQzl1W4ICNVTkohC8E73eI2xW4yI=
 github.com/knadh/koanf/parsers/yaml v1.1.0 h1:3ltfm9ljprAHt4jxgeYLlFPmUaunuCgu1yILuTXRdM4=
 github.com/knadh/koanf/parsers/yaml v1.1.0/go.mod h1:HHmcHXUrp9cOPcuC+2wrr44GTUB0EC+PyfN3HZD9tFg=
+github.com/knadh/koanf/providers/confmap v1.0.0 h1:mHKLJTE7iXEys6deO5p6olAiZdG5zwp8Aebir+/EaRE=
+github.com/knadh/koanf/providers/confmap v1.0.0/go.mod h1:txHYHiI2hAtF0/0sCmcuol4IDcuQbKTybiB1nOcUo1A=
 github.com/knadh/koanf/providers/env v1.1.0 h1:U2VXPY0f+CsNDkvdsG8GcsnK4ah85WwWyJgef9oQMSc=
 github.com/knadh/koanf/providers/env v1.1.0/go.mod h1:QhHHHZ87h9JxJAn2czdEl6pdkNnDh/JS1Vtsyt65hTY=
 github.com/knadh/koanf/providers/file v1.2.0 h1:hrUJ6Y9YOA49aNu/RSYzOTFlqzXSCpmYIDXI7OJU6+U=
@@ -120,6 +122,8 @@ github.com/uptrace/opentelemetry-go-extra/otelutil v0.3.2 h1:3/aHKUq7qaFMWxyQV0W
 github.com/uptrace/opentelemetry-go-extra/otelutil v0.3.2/go.mod h1:Zit4b8AQXaXvA68+nzmbyDzqiyFRISyw1JiD5JqUBjw=
 github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.4 h1:/4mU8NB88+6u9JVKlkdD6HjrhRM1V1KRTsJaU8FSr8I=
 github.com/uptrace/opentelemetry-go-extra/otelzap v0.2.4/go.mod h1:JoL6Kg6zYo9WtK5Y715GWItSUNpWprRYj5wgO01h00g=
+gitlab-master.nvidia.com/ncp/vmaas/libs/golang/nv-config v0.1.7 h1:GU2CMTh5HCdPX0bwZFig9oOGtuCn0m+5zWQuICqCmrU=
+gitlab-master.nvidia.com/ncp/vmaas/libs/golang/nv-config v0.1.7/go.mod h1:Q0VFXFw0iM1/YM2DHyG7wROkjhCgTHU1wE8kWv/MupI=
 go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64=
 go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y=
 go.opentelemetry.io/otel v1.38.0 h1:RkfdswUDRimDg0m2Az18RKOsnI8UDzppJAtj01/Ymk8=

auth-callout/src/internal/config/permissions.go

@@ -7,15 +7,18 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
-	"path/filepath"
 	"sync/atomic"
+	"time"
 
-	"github.com/fsnotify/fsnotify"
 	"github.com/nats-io/jwt/v2"
 	"github.com/uptrace/opentelemetry-go-extra/otelzap"
 	"go.uber.org/zap"
+
+	nvconfig "gitlab-master.nvidia.com/ncp/vmaas/libs/golang/nv-config"
 )
 
+const permissionsReloadPollingInterval = 5 * time.Second
+
 // UserProfile contains the account and permissions for a user
 type UserProfile struct {
 	Name        string          `json:"-"` // Friendly name from config (not serialized)
@@ -66,7 +69,7 @@ type PermissionsManager struct {
 	mtlsLookup    atomic.Pointer[map[string]UserProfile]          // identity -> profile
 	nkeyLookup    atomic.Pointer[map[string]UserProfile]          // public_key -> profile
 	noauthProfile atomic.Pointer[UserProfile]                     // optional no-auth profile
-	fileWatcher   *fsnotify.Watcher
+	fileWatcher   nvconfig.FileWatcher
 	logger        *otelzap.Logger
 }
 
@@ -92,7 +95,22 @@ func NewPermissionsManager(filePath string, logger *otelzap.Logger) (*Permission
 		return nil, fmt.Errorf("failed to load initial config: %w", err)
 	}
 
-	if err := pm.startWatcher(filePath); err != nil {
+	fw, err := nvconfig.NewFileWatcher(nvconfig.WithPollingInterval(permissionsReloadPollingInterval))
+	if err != nil {
+		return nil, fmt.Errorf("failed to create file watcher: %w", err)
+	}
+	pm.fileWatcher = fw
+
+	if err := fw.WatchFile(filePath, func(_ string) error {
+		pm.logger.Info("Config file changed, reloading...")
+		if err := pm.load(filePath); err != nil {
+			pm.logger.Error("Error reloading config", zap.Error(err))
+			return err
+		}
+		pm.logger.Info("Config reloaded successfully")
+		return nil
+	}); err != nil {
+		_ = fw.Stop()
 		return nil, fmt.Errorf("failed to watch config file: %w", err)
 	}
 
@@ -198,53 +216,6 @@ func (pm *PermissionsManager) load(filePath string) error {
 	return nil
 }
 
-func (pm *PermissionsManager) startWatcher(filePath string) error {
-	watcher, err := fsnotify.NewWatcher()
-	if err != nil {
-		return fmt.Errorf("failed to create file watcher: %w", err)
-	}
-	pm.fileWatcher = watcher
-
-	if err := watcher.Add(filepath.Dir(filePath)); err != nil {
-		_ = watcher.Close()
-		return fmt.Errorf("failed to watch config directory: %w", err)
-	}
-
-	go func() {
-		for {
-			select {
-			case event, ok := <-watcher.Events:
-				if !ok {
-					return
-				}
-				if !shouldReload(event, filePath) {
-					continue
-				}
-				pm.logger.Info("Config file changed, reloading...")
-				if err := pm.load(filePath); err != nil {
-					pm.logger.Error("Error reloading config", zap.Error(err))
-					continue
-				}
-				pm.logger.Info("Config reloaded successfully")
-			case err, ok := <-watcher.Errors:
-				if !ok {
-					return
-				}
-				pm.logger.Error("Config watcher error", zap.Error(err))
-			}
-		}
-	}()
-
-	return nil
-}
-
-func shouldReload(event fsnotify.Event, filePath string) bool {
-	if filepath.Clean(event.Name) != filepath.Clean(filePath) {
-		return false
-	}
-	return event.Has(fsnotify.Write) || event.Has(fsnotify.Create) || event.Has(fsnotify.Rename)
-}
-
 // GetOAuth2Profile returns the user profile and required scope for OAuth2 claims (subject and azp)
 func (pm *PermissionsManager) GetOAuth2Profile(subject, azp string) (UserProfile, string, bool) {
 	lookup := pm.oauth2Lookup.Load()
@@ -293,8 +264,5 @@ func (pm *PermissionsManager) GetNoAuthProfile() (UserProfile, bool) {
 
 // Close stops the file watcher
 func (pm *PermissionsManager) Close() error {
-	if pm.fileWatcher == nil {
-		return nil
-	}
-	return pm.fileWatcher.Close()
+	return pm.fileWatcher.Stop()
 }

auth-callout/src/internal/config/permissions_test.go

@@ -0,0 +1,83 @@
+// Copyright 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+// SPDX-License-Identifier: Apache-2.0
+
+package config
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/require"
+	"github.com/uptrace/opentelemetry-go-extra/otelzap"
+	"go.uber.org/zap"
+)
+
+func TestPermissionsManagerReloadsKubernetesConfigMapVolume(t *testing.T) {
+	volumeDir := t.TempDir()
+	permissionsFile := filepath.Join(volumeDir, "permissions.json")
+
+	writeKubernetesConfigMapVersion(t, volumeDir, "2026_05_27_00_00_00.000000001", "APP1")
+
+	pm, err := NewPermissionsManager(permissionsFile, testLogger())
+	require.NoError(t, err)
+	t.Cleanup(func() {
+		require.NoError(t, pm.Close())
+	})
+
+	requireNoAuthAccount(t, pm, "APP1")
+
+	writeKubernetesConfigMapVersion(t, volumeDir, "2026_05_27_00_00_01.000000002", "APP2")
+
+	require.Eventually(t, func() bool {
+		profile, ok := pm.GetNoAuthProfile()
+		return ok && profile.Account == "APP2"
+	}, 10*time.Second, 100*time.Millisecond)
+}
+
+func testLogger() *otelzap.Logger {
+	return otelzap.New(zap.NewNop())
+}
+
+func writeKubernetesConfigMapVersion(t *testing.T, volumeDir, version, account string) {
+	t.Helper()
+
+	versionName := ".." + version
+	versionDir := filepath.Join(volumeDir, versionName)
+	require.NoError(t, os.Mkdir(versionDir, 0o755))
+	writeNoAuthPermissions(t, filepath.Join(versionDir, "permissions.json"), account)
+
+	tmpLink := filepath.Join(volumeDir, "..data_tmp")
+	require.NoError(t, os.RemoveAll(tmpLink))
+	require.NoError(t, os.Symlink(versionName, tmpLink))
+	require.NoError(t, os.Rename(tmpLink, filepath.Join(volumeDir, "..data")))
+
+	permissionsLink := filepath.Join(volumeDir, "permissions.json")
+	if _, err := os.Lstat(permissionsLink); os.IsNotExist(err) {
+		require.NoError(t, os.Symlink(filepath.Join("..data", "permissions.json"), permissionsLink))
+	} else {
+		require.NoError(t, err)
+	}
+}
+
+func writeNoAuthPermissions(t *testing.T, path, account string) {
+	t.Helper()
+
+	data, err := json.Marshal(PermissionsConfig{
+		NoAuth: &NoAuthEntry{
+			Account: account,
+		},
+	})
+	require.NoError(t, err)
+	require.NoError(t, os.WriteFile(path, data, 0o644))
+}
+
+func requireNoAuthAccount(t *testing.T, pm *PermissionsManager, account string) {
+	t.Helper()
+
+	profile, ok := pm.GetNoAuthProfile()
+	require.True(t, ok)
+	require.Equal(t, account, profile.Account)
+}