Skip to content

Commit 9c3a978

Browse files
authored
Merge pull request #47 from platform9/pushkar/host-404-window
fix(resmgr): a host being deauthorised has not gone away
2 parents 8cba4fe + 0451d89 commit 9c3a978

8 files changed

Lines changed: 192 additions & 31 deletions

File tree

CHANGELOG.md

Lines changed: 18 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,24 @@ All notable changes to this project are documented here. The format is based on
44
[Keep a Changelog](https://keepachangelog.com/en/1.1.0/), and this project adheres to
55
[Semantic Versioning](https://semver.org/spec/v2.0.0.html).
66

7-
## [Unreleased]
7+
## [0.1.9] - 2026-08-18
8+
9+
### Fixed
10+
11+
- `pcd_host_config_assignment`, `pcd_host_cluster_role` and `pcd_host_role` no longer disappear from
12+
state while a host is being deauthorised. resmgr answers the per-host endpoints with `404` for
13+
minutes after a host's last role is removed, while `GET /resmgr/v2/hosts` keeps reporting the host,
14+
its roles and its `hostconfig_id` throughout; the reads believed the `404` and removed resources
15+
that still existed, and the next apply then failed against the reality they never left — `409
16+
HostToHostconfigConflict` re-creating an assignment, `403 HostInAuthState` re-adding a role. A
17+
`404` is now checked against the host list before anything leaves state, and an unreadable list is
18+
an error rather than an absence. An ordinary read still costs one request.
19+
20+
### Security
21+
22+
- Bumped the indirect `google.golang.org/grpc` dependency to 1.82.1 (GHSA: gRPC-Go xDS RBAC and
23+
HTTP/2 vulnerabilities). The provider's gRPC server only ever serves the local Terraform CLI over
24+
a private channel and does not use xDS, so exposure was minimal.
825

926
## [0.1.8] - 2026-08-18
1027

go.mod

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,7 @@ require (
8080
golang.org/x/tools v0.47.0 // indirect
8181
google.golang.org/appengine v1.6.8 // indirect
8282
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
83-
google.golang.org/grpc v1.82.0 // indirect
83+
google.golang.org/grpc v1.82.1 // indirect
8484
google.golang.org/protobuf v1.36.11 // indirect
8585
gopkg.in/yaml.v2 v2.4.0 // indirect
8686
)

go.sum

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -286,8 +286,8 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs
286286
google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
287287
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw=
288288
google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
289-
google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU=
290-
google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
289+
google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
290+
google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
291291
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
292292
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
293293
google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

internal/services/resmgr/absence_internal_test.go

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -176,3 +176,120 @@ func TestWaitUnassigned(t *testing.T) {
176176
}
177177
})
178178
}
179+
180+
// windowed serves resmgr's post-deauth behaviour: the per-host endpoint 404s while the
181+
// list keeps reporting whatever `list` says.
182+
func windowed(t *testing.T, perHostStatus int, perHost, list string) *gophercloud.ServiceClient {
183+
t.Helper()
184+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
185+
w.Header().Set("Content-Type", "application/json")
186+
if strings.TrimSuffix(r.URL.Path, "/") == "/hosts" {
187+
_, _ = w.Write([]byte(list))
188+
return
189+
}
190+
w.WriteHeader(perHostStatus)
191+
_, _ = w.Write([]byte(perHost))
192+
}))
193+
t.Cleanup(srv.Close)
194+
return &gophercloud.ServiceClient{ProviderClient: &gophercloud.ProviderClient{}, Endpoint: srv.URL + "/"}
195+
}
196+
197+
// A host being deauthorised 404s on its per-host endpoint for minutes while the list still
198+
// reports it. Believing that 404 drops a live assignment or role out of state, and the next
199+
// apply then fails against a reality it never left.
200+
func TestHostRecordDoesNotBelieveThePostDeauthWindow(t *testing.T) {
201+
const listed = `[{"id":"host-a","roles":["hypervisor"],"hostconfig_id":"hc-1"}]`
202+
203+
t.Run("404 while the list still has it is not gone", func(t *testing.T) {
204+
host, known, err := hostRecord(context.Background(),
205+
windowed(t, 404, `{"message":"HostNotFound"}`, listed), "host-a")
206+
if err != nil {
207+
t.Fatalf("unexpected error: %v", err)
208+
}
209+
if !known {
210+
t.Fatal("reported the host as gone; a Read would drop a live resource from state")
211+
}
212+
if host.HostConfigID != "hc-1" || len(host.Roles) != 1 {
213+
t.Fatalf("the list record did not come back: %+v", host)
214+
}
215+
})
216+
217+
t.Run("404 and absent from the list is gone", func(t *testing.T) {
218+
_, known, err := hostRecord(context.Background(),
219+
windowed(t, 404, `{"message":"HostNotFound"}`, `[{"id":"host-b"}]`), "host-a")
220+
if err != nil {
221+
t.Fatalf("unexpected error: %v", err)
222+
}
223+
if known {
224+
t.Fatal("a host resmgr does not list anywhere is gone and must leave state")
225+
}
226+
})
227+
228+
t.Run("an unreadable list fails closed", func(t *testing.T) {
229+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
230+
if strings.TrimSuffix(r.URL.Path, "/") == "/hosts" {
231+
w.WriteHeader(http.StatusInternalServerError)
232+
return
233+
}
234+
w.WriteHeader(http.StatusNotFound)
235+
}))
236+
defer srv.Close()
237+
client := &gophercloud.ServiceClient{ProviderClient: &gophercloud.ProviderClient{}, Endpoint: srv.URL + "/"}
238+
if _, known, err := hostRecord(context.Background(), client, "host-a"); err == nil || known {
239+
t.Fatal("an unverified 404 must surface as an error, not as an absence")
240+
}
241+
})
242+
243+
// A per-host failure that is not a 404 says nothing about whether the host exists, so it
244+
// has to reach the caller as an error. All three Read paths check err before known, which
245+
// makes this guard the thing standing between a transient 500 or an expired token and a
246+
// live assignment or role being dropped from state — the outcome this whole fix exists to
247+
// prevent. Without this case the guard can be deleted and the suite stays green.
248+
t.Run("a per-host failure that is not a 404 is not an absence", func(t *testing.T) {
249+
for _, status := range []int{
250+
http.StatusInternalServerError,
251+
http.StatusUnauthorized,
252+
http.StatusForbidden,
253+
http.StatusServiceUnavailable,
254+
} {
255+
_, known, err := hostRecord(context.Background(),
256+
windowed(t, status, `{"message":"boom"}`, listed), "host-a")
257+
if err == nil {
258+
t.Errorf("status %d: got no error; a failed read would be reported as a deleted host", status)
259+
}
260+
if known {
261+
t.Errorf("status %d: reported the host as known off a read that never answered", status)
262+
}
263+
}
264+
})
265+
266+
t.Run("the ordinary path is one request", func(t *testing.T) {
267+
var n int
268+
srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
269+
n++
270+
w.Header().Set("Content-Type", "application/json")
271+
_, _ = w.Write([]byte(`{"id":"host-a","roles":["hypervisor"],"hostconfig_id":"hc-1"}`))
272+
}))
273+
defer srv.Close()
274+
client := &gophercloud.ServiceClient{ProviderClient: &gophercloud.ProviderClient{}, Endpoint: srv.URL + "/"}
275+
if _, known, err := hostRecord(context.Background(), client, "host-a"); err != nil || !known {
276+
t.Fatalf("known=%v err=%v", known, err)
277+
}
278+
if n != 1 {
279+
t.Fatalf("a host resmgr describes cost %d requests, want 1", n)
280+
}
281+
})
282+
}
283+
284+
func TestHostHasRoleThroughTheWindow(t *testing.T) {
285+
const listed = `[{"id":"host-a","roles":["hypervisor","image-library"]}]`
286+
client := windowed(t, 404, `{"message":"HostNotFound"}`, listed)
287+
288+
has, known, err := hostHasRole(context.Background(), client, "host-a", "hypervisor")
289+
if err != nil || !known || !has {
290+
t.Fatalf("has=%v known=%v err=%v; the role is still on the host", has, known, err)
291+
}
292+
if has, known, _ = hostHasRole(context.Background(), client, "host-a", "persistent-storage"); has || !known {
293+
t.Fatalf("has=%v known=%v; the host is known, the role is not on it", has, known)
294+
}
295+
}

internal/services/resmgr/host_cluster_role_resource.go

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -281,17 +281,15 @@ func (r *hostClusterRoleResource) Read(ctx context.Context, req resource.ReadReq
281281

282282
// The v2 host view reports cluster roles under their uber-role names, so
283283
// membership is checked directly against the configured role.
284-
var host struct {
285-
Roles []string `json:"roles"`
286-
}
287-
if err := getJSON(ctx, client, client.ServiceURL("hosts", state.HostID.ValueString()), &host); err != nil {
288-
if isNotFound(err) {
289-
resp.State.RemoveResource(ctx)
290-
return
291-
}
284+
host, known, err := hostRecord(ctx, client, state.HostID.ValueString())
285+
if err != nil {
292286
resp.Diagnostics.AddError("resmgr: reading host", err.Error())
293287
return
294288
}
289+
if !known {
290+
resp.State.RemoveResource(ctx)
291+
return
292+
}
295293
found := false
296294
for _, r := range host.Roles {
297295
if r == state.Role.ValueString() {

internal/services/resmgr/host_config_assignment_resource.go

Lines changed: 3 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -98,16 +98,12 @@ func (r *hostConfigAssignmentResource) Read(ctx context.Context, req resource.Re
9898
return
9999
}
100100

101-
var host hostAPI
102-
if err := getJSON(ctx, client, client.ServiceURL("hosts", state.HostID.ValueString()), &host); err != nil {
103-
if isNotFound(err) {
104-
resp.State.RemoveResource(ctx)
105-
return
106-
}
101+
host, known, err := hostRecord(ctx, client, state.HostID.ValueString())
102+
if err != nil {
107103
resp.Diagnostics.AddError("resmgr: reading host", err.Error())
108104
return
109105
}
110-
if host.HostConfigID != state.HostConfigID.ValueString() {
106+
if !known || host.HostConfigID != state.HostConfigID.ValueString() {
111107
resp.State.RemoveResource(ctx)
112108
return
113109
}

internal/services/resmgr/host_role_resource.go

Lines changed: 10 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -105,16 +105,12 @@ func (r *hostRoleResource) Read(ctx context.Context, req resource.ReadRequest, r
105105
return
106106
}
107107

108-
has, err := hostHasRole(ctx, client, state.HostID.ValueString(), state.RoleName.ValueString())
108+
has, known, err := hostHasRole(ctx, client, state.HostID.ValueString(), state.RoleName.ValueString())
109109
if err != nil {
110-
if isNotFound(err) {
111-
resp.State.RemoveResource(ctx)
112-
return
113-
}
114110
resp.Diagnostics.AddError("resmgr: reading host roles", err.Error())
115111
return
116112
}
117-
if !has {
113+
if !known || !has {
118114
resp.State.RemoveResource(ctx)
119115
return
120116
}
@@ -168,15 +164,17 @@ type hostAPI struct {
168164
HostConfigID string `json:"hostconfig_id"`
169165
}
170166

171-
func hostHasRole(ctx context.Context, client *gophercloud.ServiceClient, hostID, roleName string) (bool, error) {
172-
var host hostAPI
173-
if err := getJSON(ctx, client, client.ServiceURL("hosts", hostID), &host); err != nil {
174-
return false, err
167+
// hostHasRole reports whether the host carries the role, and whether resmgr knows the host
168+
// at all — a host in the post-deauth window is not gone, however its per-host endpoint answers.
169+
func hostHasRole(ctx context.Context, client *gophercloud.ServiceClient, hostID, roleName string) (has, known bool, err error) {
170+
host, known, err := hostRecord(ctx, client, hostID)
171+
if err != nil || !known {
172+
return false, known, err
175173
}
176174
for _, r := range host.Roles {
177175
if r == roleName {
178-
return true, nil
176+
return true, true, nil
179177
}
180178
}
181-
return false, nil
179+
return false, true, nil
182180
}

internal/services/resmgr/resmgr.go

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -120,6 +120,41 @@ func hostsAssignedTo(ctx context.Context, client *gophercloud.ServiceClient, hos
120120
return assigned, nil
121121
}
122122

123+
// hostRecord returns the host as resmgr reports it, and whether resmgr knows it at all.
124+
//
125+
// The per-host endpoints answer 404 for minutes after a host's last role is removed, while
126+
// GET /resmgr/v2/hosts keeps reporting the host, its roles and its hostconfig_id throughout
127+
// (observed on 2026.4). A Read that believes that 404 removes from state a resource that
128+
// still exists, and the next apply then fails against the reality it never left: 409
129+
// HostToHostconfigConflict re-creating an assignment, 403 HostInAuthState re-adding a role.
130+
//
131+
// So a 404 is not taken at face value: only the list can say the host is really gone. The
132+
// extra call happens on the 404 path alone, leaving an ordinary read at one request. If
133+
// resmgr ever stops 404ing a host it still lists, this degrades to that fast path and can
134+
// be retired.
135+
func hostRecord(ctx context.Context, client *gophercloud.ServiceClient, hostID string) (hostAPI, bool, error) {
136+
var host hostAPI
137+
err := getJSON(ctx, client, client.ServiceURL("hosts", hostID), &host)
138+
if err == nil {
139+
return host, true, nil
140+
}
141+
if !isNotFound(err) {
142+
return host, false, err
143+
}
144+
var hosts []hostAPI
145+
if err := getJSONList(ctx, client, client.ServiceURL("hosts"), &hosts); err != nil {
146+
// Fail closed: dropping a live resource from state on an unverified 404 is what
147+
// this exists to prevent.
148+
return host, false, err
149+
}
150+
for _, h := range hosts {
151+
if h.ID == hostID {
152+
return h, true, nil
153+
}
154+
}
155+
return host, false, nil
156+
}
157+
123158
// unassignPollInterval / unassignPollTimeout bound the wait for an unassign to show up
124159
// in the host list. Short: this confirms a write resmgr has already accepted.
125160
// var, not const, so a test can drive the clock instead of sleeping through it.

0 commit comments

Comments
 (0)