Skip to content

Commit 3dab8cd

Browse files
committed
Update Azure Spot Termination checking
* Query the Azure metadata service for scheduled termination events and decode the response JSON. In the past it may have been sufficient to look for OK/200 responses as the indicator, but now it seems that the default is a JSON response with zero events. * When any scheduled termination event is observed, start the stop procedures (e.g. drain and/or delete, depending) Docs: https://learn.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-terminate-notification#get-terminate-notifications
1 parent 5243bcc commit 3dab8cd

1 file changed

Lines changed: 24 additions & 0 deletions

File tree

internal/scuttle.go

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@ package scuttle
88

99
import (
1010
"context"
11+
"encoding/json"
1112
"fmt"
1213
"net/http"
1314
"os"
@@ -217,6 +218,21 @@ func (w *Scuttle) pendingShutdown(ctx context.Context) bool {
217218
return false
218219
}
219220

221+
if w.config.Platform == "azure" {
222+
scheduledEvents := new(AzureScheduledEvents)
223+
err = json.NewDecoder(resp.Body).Decode(scheduledEvents)
224+
if err != nil {
225+
w.log.WithFields(fields).Errorf("scuttle: error decoding Azure scheduled events: %v, %v", err, scheduledEvents)
226+
}
227+
if len(scheduledEvents.Events) > 0 {
228+
w.log.WithFields(fields).Info("scuttle: Spot Instance interruption notice!")
229+
return true
230+
}
231+
w.log.WithFields(fields).Debugf("scuttle: no scheduled events: %v", scheduledEvents)
232+
return false
233+
}
234+
235+
// by default, a simple 200 response code indicates an interruption notice
220236
switch resp.StatusCode {
221237
case 200:
222238
w.log.WithFields(fields).Info("scuttle: Spot Instance interruption notice!")
@@ -226,3 +242,11 @@ func (w *Scuttle) pendingShutdown(ctx context.Context) bool {
226242
return false
227243
}
228244
}
245+
246+
// Github Issue (abbreviated)
247+
type AzureScheduledEvents struct {
248+
DocumentIncarnation int `json:"DocumentIncarnation"`
249+
Events []struct {
250+
ID string `json:"EventId"`
251+
} `json:"Events"`
252+
}

0 commit comments

Comments
 (0)