host volumes: -force flag for delete

.changelog/25902.txt

@@ -0,0 +1,3 @@
+```release-note:improvement
+host volumes: Add -force flag to volume delete command for removing volumes from GC'd nodes
+```

api/host_volumes.go

@@ -174,7 +174,8 @@ type HostVolumeListRequest struct {
 }
 
 type HostVolumeDeleteRequest struct {
-	ID string
+	ID    string
+	Force bool
 }
 
 type HostVolumeDeleteResponse struct{}
@@ -244,6 +245,9 @@ func (hv *HostVolumes) Delete(req *HostVolumeDeleteRequest, opts *WriteOptions)
 	if err != nil {
 		return nil, nil, err
 	}
+	if req.Force {
+		path = path + "?force=true"
+	}
 	wm, err := hv.client.delete(path, nil, resp, opts)
 	return resp, wm, err
 }

command/volume_delete.go

@@ -29,16 +29,21 @@ Usage: nomad volume delete [options] <vol id>
   unpublished. If the volume no longer exists, this command will silently
   return without an error.
 
-  When ACLs are enabled, this command requires a token with the
-  'csi-write-volume' and 'csi-read-volume' capabilities for the volume's
-  namespace.
+  When ACLs are enabled, this command requires a token with the appropriate
+  capability in the volume's namespace: the 'csi-write-volume' capability for
+  CSI volumes or 'host-volume-create' for dynamic host volumes.
 
 General Options:
 
   ` + generalOptionsUsage(usageOptsDefault) + `
 
 Delete Options:
 
+  -force
+    Delete the volume from the Nomad state store if the node has been garbage
+    collected. You should only use -force if the node will never rejoin the
+    cluster. Only available for dynamic host volumes.
+
   -secret
     Secrets to pass to the plugin to delete the snapshot. Accepts multiple
     flags in the form -secret key=value. Only available for CSI volumes.
@@ -88,10 +93,12 @@ func (c *VolumeDeleteCommand) Name() string { return "volume delete" }
 func (c *VolumeDeleteCommand) Run(args []string) int {
 	var secretsArgs flaghelper.StringFlag
 	var typeArg string
+	var force bool
 	flags := c.Meta.FlagSet(c.Name(), FlagSetClient)
 	flags.Usage = func() { c.Ui.Output(c.Help()) }
 	flags.Var(&secretsArgs, "secret", "secrets for snapshot, ex. -secret key=value")
 	flags.StringVar(&typeArg, "type", "csi", "type of volume (csi or host)")
+	flags.BoolVar(&force, "force", false, "force delete from garbage collected node")
 
 	if err := flags.Parse(args); err != nil {
 		c.Ui.Error(fmt.Sprintf("Error parsing arguments %s", err))
@@ -118,7 +125,7 @@ func (c *VolumeDeleteCommand) Run(args []string) int {
 	case "csi":
 		return c.deleteCSIVolume(client, volID, secretsArgs)
 	case "host":
-		return c.deleteHostVolume(client, volID)
+		return c.deleteHostVolume(client, volID, force)
 	default:
 		c.Ui.Error(fmt.Sprintf("No such volume type %q", typeArg))
 		return 1
@@ -174,7 +181,7 @@ func (c *VolumeDeleteCommand) deleteCSIVolume(client *api.Client, volID string,
 	return 0
 }
 
-func (c *VolumeDeleteCommand) deleteHostVolume(client *api.Client, volID string) int {
+func (c *VolumeDeleteCommand) deleteHostVolume(client *api.Client, volID string, force bool) int {
 
 	if !helper.IsUUID(volID) {
 		stub, possible, err := getHostVolumeByPrefix(client, volID, c.namespace)
@@ -195,7 +202,8 @@ func (c *VolumeDeleteCommand) deleteHostVolume(client *api.Client, volID string)
 		c.namespace = stub.Namespace
 	}
 
-	_, _, err := client.HostVolumes().Delete(&api.HostVolumeDeleteRequest{ID: volID}, nil)
+	_, _, err := client.HostVolumes().Delete(&api.HostVolumeDeleteRequest{
+		ID: volID, Force: force}, nil)
 	if err != nil {
 		c.Ui.Error(fmt.Sprintf("Error deleting volume: %s", err))
 		return 1

nomad/host_volume_endpoint.go

@@ -671,7 +671,14 @@ func (v *HostVolume) Delete(args *structs.HostVolumeDeleteRequest, reply *struct
 	// serialize client RPC and raft write per volume ID
 	index, err := v.serializeCall(vol.ID, "delete", func() (uint64, error) {
 		if err := v.deleteVolume(vol); err != nil {
-			return 0, err
+			if structs.IsErrUnknownNode(err) {
+				if !args.Force {
+					return 0, fmt.Errorf(
+						"volume cannot be removed from unknown node without force=true")
+				}
+			} else {
+				return 0, err
+			}
 		}
 		_, idx, err := v.srv.raftApply(structs.HostVolumeDeleteRequestType, args)
 		if err != nil {

nomad/host_volume_endpoint_test.go

@@ -392,16 +392,25 @@ func TestHostVolumeEndpoint_CreateRegisterGetDelete(t *testing.T) {
 		must.Nil(t, getResp.Volume)
 	})
 
+	index++
+	must.NoError(t, srv.State().DeleteNode(structs.MsgTypeTestSetup, index, []string{vol1.NodeID}))
+
 	// delete vol1 to finish cleaning up
-	var delResp structs.HostVolumeDeleteResponse
-	err := msgpackrpc.CallWithCodec(codec, "HostVolume.Delete", &structs.HostVolumeDeleteRequest{
+	delReq := &structs.HostVolumeDeleteRequest{
 		VolumeID: vol1.ID,
 		WriteRequest: structs.WriteRequest{
 			Region:    srv.Region(),
 			Namespace: vol1.Namespace,
 			AuthToken: powerToken,
 		},
-	}, &delResp)
+	}
+
+	var delResp structs.HostVolumeDeleteResponse
+	err := msgpackrpc.CallWithCodec(codec, "HostVolume.Delete", delReq, &delResp)
+	must.EqError(t, err, "volume cannot be removed from unknown node without force=true")
+
+	delReq.Force = true
+	err = msgpackrpc.CallWithCodec(codec, "HostVolume.Delete", delReq, &delResp)
 	must.NoError(t, err)
 
 	// should be no volumes left

nomad/structs/host_volumes.go

@@ -401,6 +401,7 @@ type HostVolumeRegisterResponse struct {
 
 type HostVolumeDeleteRequest struct {
 	VolumeID string
+	Force    bool
 	WriteRequest
 }
 

website/content/docs/commands/volume/delete.mdx

@@ -39,6 +39,10 @@ volumes or `host-volume-delete` for dynamic host volumes.
 
 ## Delete options
 
+- `-force`: Delete the volume from the Nomad state store if the node has been
+  garbage collected. You should only use `-force` if the node will never rejoin
+  the cluster. Only available for dynamic host volumes.
+
 - `-secret`: Secrets to pass to the plugin to delete the snapshot. Accepts
   multiple flags in the form `-secret key=value`. Only available for CSI
   volumes.