Skip to content

Commit 20c7ae2

Browse files
authored
feat(infra): add gpud session method to update token and get token (#1188)
feat(infra): add gpud session method to update token and get token Co-authored-by: calmp <calmp@nvidia.com>
1 parent 47581b4 commit 20c7ae2

9 files changed

Lines changed: 560 additions & 9 deletions

File tree

pkg/process/pids_test.go

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,23 +27,35 @@ func TestCountProcessesByStatus(t *testing.T) {
2727
}
2828

2929
// Verify that each status has at least one process
30+
validProcessCount := 0
3031
for status, procs := range processes {
3132
if len(procs) == 0 {
3233
t.Fatalf("Expected at least one process for status %s, got none", status)
3334
}
3435

3536
// Verify that each process has a valid PID
37+
// Note: Some processes may disappear during testing, so we don't fail on individual errors
3638
for _, proc := range procs {
3739
status, err := proc.Status()
3840
if err != nil {
39-
t.Fatal(err)
41+
// Process may have terminated, log and continue
42+
t.Logf("Warning: Could not get status for process (may have terminated): %v", err)
43+
continue
4044
}
4145
if len(status) == 0 {
42-
t.Fatalf("Expected at least one status, got none")
46+
t.Logf("Warning: Process returned empty status")
47+
continue
4348
}
49+
validProcessCount++
4450
}
4551
}
4652

53+
// Verify we successfully checked at least some processes
54+
if validProcessCount == 0 {
55+
t.Fatal("Could not verify any processes - all processes terminated during test")
56+
}
57+
t.Logf("Successfully verified %d processes", validProcessCount)
58+
4759
// Test with canceled context
4860
canceledCtx, cancel := context.WithCancel(context.Background())
4961
cancel() // Cancel immediately

pkg/server/server.go

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -593,6 +593,7 @@ func (s *Server) updateToken(ctx context.Context, metricsStore pkgmetrics.Store,
593593
return pkgcustomplugins.SaveSpecs(s.pluginSpecsFile, specs)
594594
}),
595595
session.WithFaultInjector(s.faultInjector),
596+
session.WithDB(s.dbRW, s.dbRO),
596597
)
597598
if err != nil {
598599
log.Logger.Errorw("error creating session", "error", err)
@@ -656,6 +657,7 @@ func (s *Server) updateToken(ctx context.Context, metricsStore pkgmetrics.Store,
656657
return pkgcustomplugins.SaveSpecs(s.pluginSpecsFile, specs)
657658
}),
658659
session.WithFaultInjector(s.faultInjector),
660+
session.WithDB(s.dbRW, s.dbRO),
659661
)
660662
if err != nil {
661663
log.Logger.Errorw("error creating session", "error", err)

pkg/session/session.go

Lines changed: 36 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,7 @@ package session
22

33
import (
44
"context"
5+
"database/sql"
56
"encoding/json"
67
"errors"
78
"fmt"
@@ -49,6 +50,8 @@ type Op struct {
4950
metricsStore pkgmetrics.Store
5051
savePluginSpecsFunc func(context.Context, pkgcustomplugins.Specs) (bool, error)
5152
faultInjector pkgfaultinjector.Injector
53+
dbRW *sql.DB
54+
dbRO *sql.DB
5255
}
5356

5457
type OpOption func(*Op)
@@ -146,6 +149,13 @@ func WithFaultInjector(faultInjector pkgfaultinjector.Injector) OpOption {
146149
}
147150
}
148151

152+
func WithDB(dbRW *sql.DB, dbRO *sql.DB) OpOption {
153+
return func(op *Op) {
154+
op.dbRW = dbRW
155+
op.dbRO = dbRO
156+
}
157+
}
158+
149159
// Triggers an auto update of GPUd itself by exiting the process with the given exit code.
150160
// Useful when the machine is managed by the Kubernetes daemonset and we want to
151161
// trigger an auto update when the daemonset restarts the machine.
@@ -170,10 +180,14 @@ type Session struct {
170180
// epControlPlane is the endpoint of the control plane
171181
epControlPlane string
172182

183+
tokenMu sync.RWMutex
173184
token string
174185
dataDir string
175186
dbInMemory bool
176187

188+
dbRW *sql.DB
189+
dbRO *sql.DB
190+
177191
createGossipRequestFunc func(machineID string, nvmlInstance nvidianvml.Instance) (*apiv1.GossipRequest, error)
178192

179193
setDefaultIbExpectedPortStatesFunc func(states componentsnvidiainfinibanditypes.ExpectedPortStates)
@@ -225,7 +239,7 @@ type Session struct {
225239

226240
// checkServerHealthFunc checks server health
227241
// In production: s.checkServerHealth, in tests: can be mocked
228-
checkServerHealthFunc func(ctx context.Context, jar *cookiejar.Jar) error
242+
checkServerHealthFunc func(ctx context.Context, jar *cookiejar.Jar, token string) error
229243
}
230244

231245
type closeOnce struct {
@@ -277,6 +291,10 @@ func NewSession(ctx context.Context, epLocalGPUdServer string, epControlPlane st
277291
dataDir: dataDir,
278292
dbInMemory: op.dbInMemory,
279293

294+
tokenMu: sync.RWMutex{},
295+
dbRW: op.dbRW,
296+
dbRO: op.dbRO,
297+
280298
createGossipRequestFunc: pkgmachineinfo.CreateGossipRequest,
281299

282300
setDefaultIbExpectedPortStatesFunc: componentsnvidiainfiniband.SetDefaultExpectedPortStates,
@@ -503,7 +521,7 @@ func (s *Session) startReader(ctx context.Context, readerExit chan any, jar *coo
503521
s.processReaderResponse(resp, goroutineCloseCh, pipeFinishCh)
504522
}
505523

506-
func (s *Session) checkServerHealth(ctx context.Context, jar *cookiejar.Jar) error {
524+
func (s *Session) checkServerHealth(ctx context.Context, jar *cookiejar.Jar, token string) error {
507525
u, err := url.Parse(s.epControlPlane)
508526
if err != nil {
509527
return err
@@ -518,7 +536,10 @@ func (s *Session) checkServerHealth(ctx context.Context, jar *cookiejar.Jar) err
518536
if err != nil {
519537
return err
520538
}
521-
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", s.token))
539+
if token == "" {
540+
token = s.getToken()
541+
}
542+
req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token))
522543

523544
client := createHTTPClient(jar)
524545
resp, err := client.Do(req)
@@ -547,6 +568,18 @@ func (s *Session) getLastPackageTimestamp() time.Time {
547568
return s.lastPackageTimestamp
548569
}
549570

571+
func (s *Session) setToken(token string) {
572+
s.tokenMu.Lock()
573+
defer s.tokenMu.Unlock()
574+
s.token = token
575+
}
576+
577+
func (s *Session) getToken() string {
578+
s.tokenMu.RLock()
579+
defer s.tokenMu.RUnlock()
580+
return s.token
581+
}
582+
550583
func (s *Session) processReaderResponse(resp *http.Response, goroutineCloseCh, pipeFinishCh chan any) {
551584
s.setLastPackageTimestamp(time.Now())
552585

pkg/session/session_keepalive.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ func (s *Session) keepAlive() {
6767

6868
// DO NOT CHANGE OR REMOVE THIS SERVER HEALTH CHECK, DEPEND ON IT FOR STICKY SESSION
6969
// TODO: we can remove it once we migrate to gpud-gateway
70-
if err := s.checkServerHealthFunc(ctx, jar); err != nil {
70+
if err := s.checkServerHealthFunc(ctx, jar, ""); err != nil {
7171
log.Logger.Errorf("session keep alive: error checking server health: %v", err)
7272
cancel()
7373
continue

pkg/session/session_keepalive_test.go

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@ func TestKeepAliveReconnectionDelay(t *testing.T) {
2424
}
2525

2626
// Mock checkServerHealth to always succeed
27-
s.checkServerHealthFunc = func(ctx context.Context, jar *cookiejar.Jar) error {
27+
s.checkServerHealthFunc = func(ctx context.Context, jar *cookiejar.Jar, token string) error {
2828
return nil
2929
}
3030

@@ -135,7 +135,7 @@ func TestKeepAliveDeadlockPrevention(t *testing.T) {
135135
}
136136

137137
// Mock checkServerHealth to always succeed
138-
s.checkServerHealthFunc = func(ctx context.Context, jar *cookiejar.Jar) error {
138+
s.checkServerHealthFunc = func(ctx context.Context, jar *cookiejar.Jar, token string) error {
139139
return nil
140140
}
141141

@@ -268,7 +268,7 @@ func TestKeepAliveNoRapidReconnection(t *testing.T) {
268268

269269
// Mock checkServerHealth to always fail (simulate rapid failures)
270270
healthCheckCount := int32(0)
271-
s.checkServerHealthFunc = func(ctx context.Context, jar *cookiejar.Jar) error {
271+
s.checkServerHealthFunc = func(ctx context.Context, jar *cookiejar.Jar, token string) error {
272272
atomic.AddInt32(&healthCheckCount, 1)
273273
// Fail health checks to test reconnection behavior
274274
return fmt.Errorf("simulated health check failure")

pkg/session/session_process_request.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,12 @@ func (s *Session) processRequest(ctx context.Context, reqID string, payload Requ
125125

126126
case "getPluginSpecs":
127127
s.processGetPluginSpecs(response)
128+
129+
case "updateToken":
130+
s.processUpdateToken(payload, response)
131+
132+
case "getToken":
133+
s.processGetToken(response)
128134
}
129135

130136
return false // Request is handled synchronously

pkg/session/session_serve.go

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,9 @@ type Request struct {
4444

4545
// CustomPluginSpecs is the specs for the custom plugins to register or overwrite.
4646
CustomPluginSpecs pkgcustomplugins.Specs `json:"custom_plugin_specs,omitempty"`
47+
48+
// Token is the new token to update on the agent side.
49+
Token string `json:"token,omitempty"`
4750
}
4851

4952
// Response is the response from GPUd to the control plane.
@@ -68,6 +71,9 @@ type Response struct {
6871

6972
// CustomPluginSpecs lists the specs for the custom plugins.
7073
CustomPluginSpecs pkgcustomplugins.Specs `json:"custom_plugin_specs,omitempty"`
74+
75+
// Token is the current token value from the agent.
76+
Token string `json:"token,omitempty"`
7177
}
7278

7379
type BootstrapRequest struct {

pkg/session/token.go

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
package session
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"net/http/cookiejar"
7+
"time"
8+
9+
"github.com/leptonai/gpud/pkg/log"
10+
pkgmetadata "github.com/leptonai/gpud/pkg/metadata"
11+
)
12+
13+
// processUpdateToken updates the session token in the database with the new token from the control plane.
14+
// It validates the new token before updating, and immediately reconnects the session with the new token.
15+
func (s *Session) processUpdateToken(payload Request, response *Response) {
16+
if payload.Token == "" {
17+
response.Error = "token cannot be empty"
18+
log.Logger.Errorw("updateToken request with empty token")
19+
return
20+
}
21+
22+
// if in-memory cached token equal update token, just return is ok
23+
cacheToken := s.getToken()
24+
if cacheToken == payload.Token {
25+
log.Logger.Info("cached token already matches the requested token")
26+
return
27+
}
28+
29+
if s.dbRW == nil {
30+
response.Error = "database connection not available"
31+
log.Logger.Errorw("updateToken failed: database connection is nil")
32+
return
33+
}
34+
35+
// Validate the new token before updating
36+
log.Logger.Infow("validating new token with control plane")
37+
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
38+
defer cancel()
39+
40+
if err := s.validateTokenWithHealthCheck(ctx, payload.Token); err != nil {
41+
response.Error = fmt.Sprintf("token validation failed: %v", err)
42+
log.Logger.Errorw("new token validation failed", "error", err)
43+
return
44+
}
45+
log.Logger.Infow("new token validated successfully")
46+
47+
// Save the new token to database
48+
if err := pkgmetadata.SetMetadata(ctx, s.dbRW, pkgmetadata.MetadataKeyToken, payload.Token); err != nil {
49+
response.Error = err.Error()
50+
log.Logger.Errorw("failed to update token in database", "error", err)
51+
return
52+
}
53+
54+
// Update in-memory token
55+
s.setToken(payload.Token)
56+
57+
log.Logger.Infow("token updated successfully in database and memory")
58+
59+
// Immediately close the current session to force reconnection with the new token
60+
log.Logger.Infow("closing current session to immediately reconnect with new token")
61+
go func() {
62+
// Give some time for the response to be sent back to control plane
63+
time.Sleep(2 * time.Second)
64+
s.closer.Close()
65+
}()
66+
}
67+
68+
// validateTokenWithHealthCheck validates the new token using the checkServerHealth mechanism
69+
func (s *Session) validateTokenWithHealthCheck(ctx context.Context, newToken string) error {
70+
// Create a new cookie jar for validation
71+
jar, err := cookiejar.New(nil)
72+
if err != nil {
73+
return fmt.Errorf("failed to create cookie jar: %w", err)
74+
}
75+
76+
// Use the existing checkServerHealth function to validate the token
77+
// Pass newToken directly to avoid modifying the session's current token
78+
if err := s.checkServerHealthFunc(ctx, jar, newToken); err != nil {
79+
return fmt.Errorf("health check with new token failed: %w", err)
80+
}
81+
82+
return nil
83+
}
84+
85+
// processGetToken retrieves the current token value from the database.
86+
func (s *Session) processGetToken(response *Response) {
87+
if s.dbRO == nil {
88+
response.Error = "database connection not available"
89+
log.Logger.Errorw("getToken failed: database connection is nil")
90+
return
91+
}
92+
93+
// if session token cache is exist, just return it
94+
tokenCache := s.getToken()
95+
if tokenCache != "" {
96+
response.Token = tokenCache
97+
return
98+
}
99+
100+
// else, get token from metadata
101+
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
102+
defer cancel()
103+
104+
token, err := pkgmetadata.ReadToken(ctx, s.dbRO)
105+
if err != nil {
106+
response.Error = err.Error()
107+
log.Logger.Errorw("failed to read token from database", "error", err)
108+
return
109+
}
110+
s.setToken(token)
111+
response.Token = token
112+
log.Logger.Infow("token retrieved successfully from database")
113+
}

0 commit comments

Comments
 (0)