fix: improve k8s-dqlite node removal handling

`dqlite.Client.RemoveNodeByAddress` now returns `ErrNotFound` error if the member
is not found. This error is now being handled properly in `removeNodeFromK8sDqlite`
(`cluster_remove.go`).

By doing so, we're making the k8s-dqlite node removal operation idempotent on
subsequent calls.

Signed-off-by: Claudiu Belu <cbelu@cloudbasesolutions.com>

Author: claudiubelu

src/k8s/pkg/client/dqlite/client.go

@@ -4,13 +4,17 @@ import (
 	"context"
 	"crypto/tls"
 	"crypto/x509"
+	"errors"
 	"fmt"
 	"os"
 
 	"github.com/canonical/go-dqlite/v2/app"
 	"github.com/canonical/go-dqlite/v2/client"
 )
 
+// ErrNotFound is returned when a dqlite member is not found.
+var ErrNotFound = errors.New("dqlite member not found")
+
 type ClientOpts struct {
 	// ClusterYAML is the path cluster.yaml, containing the list of known dqlite nodes.
 	ClusterYAML string

src/k8s/pkg/client/dqlite/remove.go

@@ -46,7 +46,7 @@ func (c *Client) RemoveNodeByAddress(ctx context.Context, address string) error
 	}
 
 	if !memberExists {
-		return fmt.Errorf("cluster does not have a node with address %v", address)
+		return fmt.Errorf("%w: %s", ErrNotFound, address)
 	}
 
 	if !clusterHasOtherVoters {

src/k8s/pkg/k8sd/api/cluster_remove.go

@@ -3,6 +3,7 @@ package api
 import (
 	"context"
 	"database/sql"
+	"errors"
 	"fmt"
 	"net"
 	"net/http"
@@ -11,6 +12,7 @@ import (
 
 	apiv1 "github.com/canonical/k8s-snap-api/api/v1"
 	apiv1_annotations "github.com/canonical/k8s-snap-api/api/v1/annotations"
+	dqliteclient "github.com/canonical/k8s/pkg/client/dqlite"
 	databaseutil "github.com/canonical/k8s/pkg/k8sd/database/util"
 	"github.com/canonical/k8s/pkg/k8sd/types"
 	"github.com/canonical/k8s/pkg/log"
@@ -207,6 +209,10 @@ func removeNodeFromK8sDqlite(ctx context.Context, s state.State, snap snap.Snap,
 
 	log.Info("Removing node from k8s-dqlite using address", "address", nodeAddress)
 	if err := client.RemoveNodeByAddress(ctx, nodeAddress); err != nil {
+		if errors.Is(err, dqliteclient.ErrNotFound) {
+			log.Info("Node not found in k8s-dqlite cluster, nothing to remove", "address", nodeAddress)
+			return nil
+		}
 		return fmt.Errorf("failed to remove node from k8s-dqlite cluster: %w", err)
 	}
 	return nil