Skip to content

Commit be0333e

Browse files
feniofenio
committed
feat: proactive storage session recovery after NAS reconnection
After a NAS reboot, iSCSI sessions go transport-offline and NVMe-oF controllers die, but the CSI driver was reusing stale connections because the device files still existed. This caused pods to crash-loop with I/O errors until manual intervention. Changes: - iSCSI: check session state before reusing existing device, force logout/re-login if not LOGGED_IN - NVMe-oF: check controller state (not just device size) before reuse, disconnect if not live - Add proactive recovery via OnReconnect callback: scan all iSCSI sessions and NVMe controllers immediately after WebSocket reconnects, recover any that are stale - Bump Go to 1.26.2, grpc to v1.80.0
1 parent ddbe3bb commit be0333e

7 files changed

Lines changed: 440 additions & 9 deletions

File tree

‎go.mod‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
module github.com/nasty-project/nasty-csi
22

3-
go 1.26.1
3+
go 1.26.2
44

55
require (
66
github.com/container-storage-interface/spec v1.12.0
@@ -11,7 +11,7 @@ require (
1111
github.com/prometheus/client_golang v1.23.2
1212
github.com/prometheus/client_model v0.6.2
1313
golang.org/x/sync v0.20.0
14-
google.golang.org/grpc v1.79.3
14+
google.golang.org/grpc v1.80.0
1515
google.golang.org/protobuf v1.36.11
1616
k8s.io/api v0.35.3
1717
k8s.io/apimachinery v0.35.3

‎go.sum‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -197,12 +197,12 @@ golang.org/x/tools v0.43.0/go.mod h1:uHkMso649BX2cZK6+RpuIPXS3ho2hZo4FVwfoy1vIk0
197197
golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
198198
golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
199199
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
200-
gonum.org/v1/gonum v0.16.0 h1:5+ul4Swaf3ESvrOnidPp4GZbzf0mxVQpDCYUQE7OJfk=
201-
gonum.org/v1/gonum v0.16.0/go.mod h1:fef3am4MQ93R2HHpKnLk4/Tbh/s0+wqD5nfa6Pnwy4E=
200+
gonum.org/v1/gonum v0.17.0 h1:VbpOemQlsSMrYmn7T2OUvQ4dqxQXU+ouZFQsZOx50z4=
201+
gonum.org/v1/gonum v0.17.0/go.mod h1:El3tOrEuMpv2UdMrbNlKEh9vd86bmQ6vqIcDwxEOc1E=
202202
google.golang.org/genproto/googleapis/rpc v0.0.0-20260330182312-d5a96adf58d8 h1:OHkuo1i98/05rzpm9NBbfEtpJH/k3abEgZUKaAuCI7Y=
203203
google.golang.org/genproto/googleapis/rpc v0.0.0-20260330182312-d5a96adf58d8/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
204-
google.golang.org/grpc v1.79.3 h1:sybAEdRIEtvcD68Gx7dmnwjZKlyfuc61Dyo9pGXXkKE=
205-
google.golang.org/grpc v1.79.3/go.mod h1:KmT0Kjez+0dde/v2j9vzwoAScgEPx/Bw1CYChhHLrHQ=
204+
google.golang.org/grpc v1.80.0 h1:Xr6m2WmWZLETvUNvIUmeD5OAagMw3FiKmMlTdViWsHM=
205+
google.golang.org/grpc v1.80.0/go.mod h1:ho/dLnxwi3EDJA4Zghp7k2Ec1+c2jqup0bFkw07bwF4=
206206
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=
207207
google.golang.org/protobuf v1.36.11/go.mod h1:HTf+CrKn2C3g5S8VImy6tdcUvCska2kB7j23XfzDpco=
208208
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

‎pkg/driver/driver.go‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,16 @@ func NewDriver(cfg Config) (*Driver, error) {
6161
return nil, err
6262
}
6363

64-
return NewDriverWithClient(cfg, apiClient)
64+
d, err := NewDriverWithClient(cfg, apiClient)
65+
if err != nil {
66+
return nil, err
67+
}
68+
69+
// Register reconnection callback to proactively recover storage sessions
70+
// after the NAS comes back from a reboot or network interruption.
71+
apiClient.SetOnReconnect(d.node.recoverVolumes)
72+
73+
return d, nil
6574
}
6675

6776
// NewDriverWithClient creates a new driver instance with a custom client.

‎pkg/driver/node_iscsi.go‎

Lines changed: 19 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -78,8 +78,25 @@ func (s *NodeService) stageISCSIVolume(ctx context.Context, req *csi.NodeStageVo
7878

7979
// Try to reuse existing connection (idempotency)
8080
if devicePath, findErr := s.findISCSIDevice(ctx, params); findErr == nil && devicePath != "" {
81-
klog.V(4).Infof("iSCSI device already connected at %s - reusing existing connection", devicePath)
82-
return s.stageISCSIDevice(ctx, volumeID, devicePath, stagingTargetPath, volumeCapability, isBlockVolume, volumeContext)
81+
// Verify the iSCSI session is healthy before reusing.
82+
// After a NAS reboot, the device may still exist in sysfs but the session
83+
// is in "transport-offline" state, causing I/O errors.
84+
sessionState, stateErr := getISCSISessionState(ctx, devicePath)
85+
if stateErr != nil {
86+
klog.Warningf("Failed to check iSCSI session state for %s: %v - forcing re-login", devicePath, stateErr)
87+
} else if sessionState == "LOGGED_IN" {
88+
klog.V(4).Infof("iSCSI device already connected at %s (session healthy) - reusing existing connection", devicePath)
89+
return s.stageISCSIDevice(ctx, volumeID, devicePath, stagingTargetPath, volumeCapability, isBlockVolume, volumeContext)
90+
} else {
91+
klog.Warningf("iSCSI session for %s is %q (not LOGGED_IN) - forcing logout and re-login", devicePath, sessionState)
92+
}
93+
94+
// Session is stale — logout before attempting a fresh login
95+
if logoutErr := s.logoutISCSITarget(ctx, params); logoutErr != nil {
96+
klog.Warningf("Failed to logout stale iSCSI session for %s: %v (continuing with fresh login)", params.iqn, logoutErr)
97+
}
98+
// Small delay for kernel to clean up the stale SCSI device
99+
time.Sleep(2 * time.Second)
83100
}
84101

85102
// Check if iscsiadm is installed

‎pkg/driver/node_nvmeof.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -136,6 +136,21 @@ func (s *NodeService) tryReuseExistingConnection(ctx context.Context, params *nv
136136
klog.V(4).Infof("NVMe-oF device already connected at %s for NQN=%s - checking if connection is healthy",
137137
devicePath, params.nqn)
138138

139+
// Check NVMe controller state first — after a NAS reboot the controller
140+
// may be in "connecting" or "dead" state even if the device still exists
141+
// and reports its cached size.
142+
ctrlState, ctrlErr := getNVMeControllerState(devicePath)
143+
if ctrlErr != nil {
144+
klog.V(4).Infof("Failed to get NVMe controller state for %s: %v (will check device health)", devicePath, ctrlErr)
145+
} else if ctrlState != nvmeSubsystemStateLive {
146+
klog.Warningf("NVMe controller for %s is %q (not live) - disconnecting to force reconnect", devicePath, ctrlState)
147+
if disconnectErr := s.disconnectNVMeOF(ctx, params.nqn); disconnectErr != nil {
148+
klog.Warningf("Failed to disconnect stale NVMe-oF connection: %v", disconnectErr)
149+
}
150+
time.Sleep(2 * time.Second)
151+
return nil, "", nil
152+
}
153+
139154
// Rescan the namespace to ensure we have fresh data from the target
140155
if rescanErr := s.rescanNVMeNamespace(ctx, devicePath); rescanErr != nil {
141156
klog.Warningf("Failed to rescan NVMe namespace %s: %v (continuing anyway)", devicePath, rescanErr)

0 commit comments

Comments
 (0)