fix(resmgr): a host being deauthorised has not gone away

resmgr answers the per-host endpoints with 404 for minutes after a host's last
role is removed, while GET /resmgr/v2/hosts keeps reporting the host, its roles
and its hostconfig_id throughout. Three reads believed that 404 and removed
resources that still existed:

    pcd_host_config_assignment   -> next apply: 409 HostToHostconfigConflict
    pcd_host_cluster_role        -> next apply: 403 HostInAuthState
    pcd_host_role                -> same

Worse than the failed apply, the assignment leaving state is what let a host
config be deleted as unused, which strands the host for good — the case the
delete guard now refuses outright.

So a 404 is no longer taken at face value: hostRecord falls back to the host
list, which is the only view that stays truthful through the window, and a list
that cannot be read is an error rather than an absence. Nothing leaves state on
an unverified 404. The extra request happens on the 404 path alone, so an
ordinary read still costs one call.

Waiting the window out belongs in the writes that must not race it, not in a
Read: a refresh must not block for minutes.

Also bumps the indirect google.golang.org/grpc to 1.82.1 (xDS RBAC and HTTP/2
advisories). The provider's gRPC server only serves the local Terraform CLI over
a private channel and does not use xDS, so exposure was minimal.

Author: Pushkar Mulay

CHANGELOG.md

@@ -6,6 +6,23 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+### Fixed
+
+- `pcd_host_config_assignment`, `pcd_host_cluster_role` and `pcd_host_role` no longer disappear from
+  state while a host is being deauthorised. resmgr answers the per-host endpoints with `404` for
+  minutes after a host's last role is removed, while `GET /resmgr/v2/hosts` keeps reporting the host,
+  its roles and its `hostconfig_id` throughout; the reads believed the `404` and removed resources
+  that still existed, and the next apply then failed against the reality they never left — `409
+  HostToHostconfigConflict` re-creating an assignment, `403 HostInAuthState` re-adding a role. A
+  `404` is now checked against the host list before anything leaves state, and an unreadable list is
+  an error rather than an absence. An ordinary read still costs one request.
+
+### Security
+
+- Bumped the indirect `google.golang.org/grpc` dependency to 1.82.1 (GHSA: gRPC-Go xDS RBAC and
+  HTTP/2 vulnerabilities). The provider's gRPC server only ever serves the local Terraform CLI over
+  a private channel and does not use xDS, so exposure was minimal.
+
 ## [0.1.8] - 2026-08-18
 
 ### Changed

go.mod

@@ -80,7 +80,7 @@ require (
 	golang.org/x/tools v0.47.0 // indirect
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 // indirect
-	google.golang.org/grpc v1.82.0 // indirect
+	google.golang.org/grpc v1.82.1 // indirect
 	google.golang.org/protobuf v1.36.11 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 )

go.sum

@@ -286,8 +286,8 @@ google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAs
 google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478 h1:RmoJA1ujG+/lRGNfUnOMfhCy5EipVMyvUE+KNbPbTlw=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20260414002931-afd174a4e478/go.mod h1:4Hqkh8ycfw05ld/3BWL7rJOSfebL2Q+DVDeRgYgxUU8=
-google.golang.org/grpc v1.82.0 h1:vguDnZUPjE26w09A63VoxZPnvPjB5Riyc0mkXPFmAIU=
-google.golang.org/grpc v1.82.0/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
+google.golang.org/grpc v1.82.1 h1:NnAxzGRA0677vCa4BUkOAnO5+FfQqVl9iUXeD0IqcGE=
+google.golang.org/grpc v1.82.1/go.mod h1:yzTZ1TB1Z3SG+LIYaI+WiE8D5+PZ3ArnrSp8zF3+/ZA=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 google.golang.org/protobuf v1.36.11 h1:fV6ZwhNocDyBLK0dj+fg8ektcVegBBuEolpbTQyBNVE=

internal/services/resmgr/absence_internal_test.go

@@ -176,3 +176,120 @@ func TestWaitUnassigned(t *testing.T) {
 		}
 	})
 }
+
+// windowed serves resmgr's post-deauth behaviour: the per-host endpoint 404s while the
+// list keeps reporting whatever `list` says.
+func windowed(t *testing.T, perHostStatus int, perHost, list string) *gophercloud.ServiceClient {
+	t.Helper()
+	srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		w.Header().Set("Content-Type", "application/json")
+		if strings.TrimSuffix(r.URL.Path, "/") == "/hosts" {
+			_, _ = w.Write([]byte(list))
+			return
+		}
+		w.WriteHeader(perHostStatus)
+		_, _ = w.Write([]byte(perHost))
+	}))
+	t.Cleanup(srv.Close)
+	return &gophercloud.ServiceClient{ProviderClient: &gophercloud.ProviderClient{}, Endpoint: srv.URL + "/"}
+}
+
+// A host being deauthorised 404s on its per-host endpoint for minutes while the list still
+// reports it. Believing that 404 drops a live assignment or role out of state, and the next
+// apply then fails against a reality it never left.
+func TestHostRecordDoesNotBelieveThePostDeauthWindow(t *testing.T) {
+	const listed = `[{"id":"host-a","roles":["hypervisor"],"hostconfig_id":"hc-1"}]`
+
+	t.Run("404 while the list still has it is not gone", func(t *testing.T) {
+		host, known, err := hostRecord(context.Background(),
+			windowed(t, 404, `{"message":"HostNotFound"}`, listed), "host-a")
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if !known {
+			t.Fatal("reported the host as gone; a Read would drop a live resource from state")
+		}
+		if host.HostConfigID != "hc-1" || len(host.Roles) != 1 {
+			t.Fatalf("the list record did not come back: %+v", host)
+		}
+	})
+
+	t.Run("404 and absent from the list is gone", func(t *testing.T) {
+		_, known, err := hostRecord(context.Background(),
+			windowed(t, 404, `{"message":"HostNotFound"}`, `[{"id":"host-b"}]`), "host-a")
+		if err != nil {
+			t.Fatalf("unexpected error: %v", err)
+		}
+		if known {
+			t.Fatal("a host resmgr does not list anywhere is gone and must leave state")
+		}
+	})
+
+	t.Run("an unreadable list fails closed", func(t *testing.T) {
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			if strings.TrimSuffix(r.URL.Path, "/") == "/hosts" {
+				w.WriteHeader(http.StatusInternalServerError)
+				return
+			}
+			w.WriteHeader(http.StatusNotFound)
+		}))
+		defer srv.Close()
+		client := &gophercloud.ServiceClient{ProviderClient: &gophercloud.ProviderClient{}, Endpoint: srv.URL + "/"}
+		if _, known, err := hostRecord(context.Background(), client, "host-a"); err == nil || known {
+			t.Fatal("an unverified 404 must surface as an error, not as an absence")
+		}
+	})
+
+	// A per-host failure that is not a 404 says nothing about whether the host exists, so it
+	// has to reach the caller as an error. All three Read paths check err before known, which
+	// makes this guard the thing standing between a transient 500 or an expired token and a
+	// live assignment or role being dropped from state — the outcome this whole fix exists to
+	// prevent. Without this case the guard can be deleted and the suite stays green.
+	t.Run("a per-host failure that is not a 404 is not an absence", func(t *testing.T) {
+		for _, status := range []int{
+			http.StatusInternalServerError,
+			http.StatusUnauthorized,
+			http.StatusForbidden,
+			http.StatusServiceUnavailable,
+		} {
+			_, known, err := hostRecord(context.Background(),
+				windowed(t, status, `{"message":"boom"}`, listed), "host-a")
+			if err == nil {
+				t.Errorf("status %d: got no error; a failed read would be reported as a deleted host", status)
+			}
+			if known {
+				t.Errorf("status %d: reported the host as known off a read that never answered", status)
+			}
+		}
+	})
+
+	t.Run("the ordinary path is one request", func(t *testing.T) {
+		var n int
+		srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
+			n++
+			w.Header().Set("Content-Type", "application/json")
+			_, _ = w.Write([]byte(`{"id":"host-a","roles":["hypervisor"],"hostconfig_id":"hc-1"}`))
+		}))
+		defer srv.Close()
+		client := &gophercloud.ServiceClient{ProviderClient: &gophercloud.ProviderClient{}, Endpoint: srv.URL + "/"}
+		if _, known, err := hostRecord(context.Background(), client, "host-a"); err != nil || !known {
+			t.Fatalf("known=%v err=%v", known, err)
+		}
+		if n != 1 {
+			t.Fatalf("a host resmgr describes cost %d requests, want 1", n)
+		}
+	})
+}
+
+func TestHostHasRoleThroughTheWindow(t *testing.T) {
+	const listed = `[{"id":"host-a","roles":["hypervisor","image-library"]}]`
+	client := windowed(t, 404, `{"message":"HostNotFound"}`, listed)
+
+	has, known, err := hostHasRole(context.Background(), client, "host-a", "hypervisor")
+	if err != nil || !known || !has {
+		t.Fatalf("has=%v known=%v err=%v; the role is still on the host", has, known, err)
+	}
+	if has, known, _ = hostHasRole(context.Background(), client, "host-a", "persistent-storage"); has || !known {
+		t.Fatalf("has=%v known=%v; the host is known, the role is not on it", has, known)
+	}
+}

internal/services/resmgr/host_cluster_role_resource.go

@@ -281,17 +281,15 @@ func (r *hostClusterRoleResource) Read(ctx context.Context, req resource.ReadReq
 
 	// The v2 host view reports cluster roles under their uber-role names, so
 	// membership is checked directly against the configured role.
-	var host struct {
-		Roles []string `json:"roles"`
-	}
-	if err := getJSON(ctx, client, client.ServiceURL("hosts", state.HostID.ValueString()), &host); err != nil {
-		if isNotFound(err) {
-			resp.State.RemoveResource(ctx)
-			return
-		}
+	host, known, err := hostRecord(ctx, client, state.HostID.ValueString())
+	if err != nil {
 		resp.Diagnostics.AddError("resmgr: reading host", err.Error())
 		return
 	}
+	if !known {
+		resp.State.RemoveResource(ctx)
+		return
+	}
 	found := false
 	for _, r := range host.Roles {
 		if r == state.Role.ValueString() {

internal/services/resmgr/host_config_assignment_resource.go

@@ -98,16 +98,12 @@ func (r *hostConfigAssignmentResource) Read(ctx context.Context, req resource.Re
 		return
 	}
 
-	var host hostAPI
-	if err := getJSON(ctx, client, client.ServiceURL("hosts", state.HostID.ValueString()), &host); err != nil {
-		if isNotFound(err) {
-			resp.State.RemoveResource(ctx)
-			return
-		}
+	host, known, err := hostRecord(ctx, client, state.HostID.ValueString())
+	if err != nil {
 		resp.Diagnostics.AddError("resmgr: reading host", err.Error())
 		return
 	}
-	if host.HostConfigID != state.HostConfigID.ValueString() {
+	if !known || host.HostConfigID != state.HostConfigID.ValueString() {
 		resp.State.RemoveResource(ctx)
 		return
 	}

internal/services/resmgr/host_role_resource.go

@@ -105,16 +105,12 @@ func (r *hostRoleResource) Read(ctx context.Context, req resource.ReadRequest, r
 		return
 	}
 
-	has, err := hostHasRole(ctx, client, state.HostID.ValueString(), state.RoleName.ValueString())
+	has, known, err := hostHasRole(ctx, client, state.HostID.ValueString(), state.RoleName.ValueString())
 	if err != nil {
-		if isNotFound(err) {
-			resp.State.RemoveResource(ctx)
-			return
-		}
 		resp.Diagnostics.AddError("resmgr: reading host roles", err.Error())
 		return
 	}
-	if !has {
+	if !known || !has {
 		resp.State.RemoveResource(ctx)
 		return
 	}
@@ -168,15 +164,17 @@ type hostAPI struct {
 	HostConfigID string   `json:"hostconfig_id"`
 }
 
-func hostHasRole(ctx context.Context, client *gophercloud.ServiceClient, hostID, roleName string) (bool, error) {
-	var host hostAPI
-	if err := getJSON(ctx, client, client.ServiceURL("hosts", hostID), &host); err != nil {
-		return false, err
+// hostHasRole reports whether the host carries the role, and whether resmgr knows the host
+// at all — a host in the post-deauth window is not gone, however its per-host endpoint answers.
+func hostHasRole(ctx context.Context, client *gophercloud.ServiceClient, hostID, roleName string) (has, known bool, err error) {
+	host, known, err := hostRecord(ctx, client, hostID)
+	if err != nil || !known {
+		return false, known, err
 	}
 	for _, r := range host.Roles {
 		if r == roleName {
-			return true, nil
+			return true, true, nil
 		}
 	}
-	return false, nil
+	return false, true, nil
 }

internal/services/resmgr/resmgr.go

@@ -120,6 +120,41 @@ func hostsAssignedTo(ctx context.Context, client *gophercloud.ServiceClient, hos
 	return assigned, nil
 }
 
+// hostRecord returns the host as resmgr reports it, and whether resmgr knows it at all.
+//
+// The per-host endpoints answer 404 for minutes after a host's last role is removed, while
+// GET /resmgr/v2/hosts keeps reporting the host, its roles and its hostconfig_id throughout
+// (observed on 2026.4). A Read that believes that 404 removes from state a resource that
+// still exists, and the next apply then fails against the reality it never left: 409
+// HostToHostconfigConflict re-creating an assignment, 403 HostInAuthState re-adding a role.
+//
+// So a 404 is not taken at face value: only the list can say the host is really gone. The
+// extra call happens on the 404 path alone, leaving an ordinary read at one request. If
+// resmgr ever stops 404ing a host it still lists, this degrades to that fast path and can
+// be retired.
+func hostRecord(ctx context.Context, client *gophercloud.ServiceClient, hostID string) (hostAPI, bool, error) {
+	var host hostAPI
+	err := getJSON(ctx, client, client.ServiceURL("hosts", hostID), &host)
+	if err == nil {
+		return host, true, nil
+	}
+	if !isNotFound(err) {
+		return host, false, err
+	}
+	var hosts []hostAPI
+	if err := getJSONList(ctx, client, client.ServiceURL("hosts"), &hosts); err != nil {
+		// Fail closed: dropping a live resource from state on an unverified 404 is what
+		// this exists to prevent.
+		return host, false, err
+	}
+	for _, h := range hosts {
+		if h.ID == hostID {
+			return h, true, nil
+		}
+	}
+	return host, false, nil
+}
+
 // unassignPollInterval / unassignPollTimeout bound the wait for an unassign to show up
 // in the host list. Short: this confirms a write resmgr has already accepted.
 // var, not const, so a test can drive the clock instead of sleeping through it.