add feature flags for spicedb client options

Tool: gitpod/catfood.gitpod.cloud

components/server/src/authorization/spicedb.ts

@@ -7,6 +7,8 @@
 import { v1 } from "@authzed/authzed-node";
 import { log } from "@gitpod/gitpod-protocol/lib/util/logging";
 import * as grpc from "@grpc/grpc-js";
+import { getExperimentsClientForBackend } from "@gitpod/gitpod-protocol/lib/experiments/configcat-server";
+import { TrustedValue } from "@gitpod/gitpod-protocol/lib/util/scrubbing";
 
 export interface SpiceDBClientConfig {
     address: string;
@@ -15,6 +17,34 @@ export interface SpiceDBClientConfig {
 
 export type SpiceDBClient = v1.ZedPromiseClientInterface;
 type Client = v1.ZedClientInterface & grpc.Client;
+const DEFAULT_FEATURE_FLAG_VAULE = "undefined";
+const DefaultClientOptions: grpc.ClientOptions = {
+    // we ping frequently to check if the connection is still alive
+    "grpc.keepalive_time_ms": 1000,
+    "grpc.keepalive_timeout_ms": 1000,
+
+    "grpc.max_reconnect_backoff_ms": 5000,
+    "grpc.initial_reconnect_backoff_ms": 500,
+    "grpc.service_config": JSON.stringify({
+        methodConfig: [
+            {
+                name: [{}],
+                retryPolicy: {
+                    maxAttempts: 10,
+                    initialBackoff: "0.1s",
+                    maxBackoff: "5s",
+                    backoffMultiplier: 2.0,
+                    retryableStatusCodes: ["UNAVAILABLE", "DEADLINE_EXCEEDED"],
+                },
+            },
+        ],
+    }),
+    "grpc.enable_retries": 1, //TODO enabled by default
+
+    // Governs how log DNS resolution results are cached (at minimum!)
+    // default is 30s, which is too long for us during rollouts (where service DNS entries are updated)
+    "grpc.dns_min_time_between_resolutions_ms": 2000,
+};
 
 export function spiceDBConfigFromEnv(): SpiceDBClientConfig | undefined {
     const token = process.env["SPICEDB_PRESHARED_KEY"];
@@ -36,48 +66,80 @@ export function spiceDBConfigFromEnv(): SpiceDBClientConfig | undefined {
 
 export class SpiceDBClientProvider {
     private client: Client | undefined;
+    private previousClientOptionsString: string = DEFAULT_FEATURE_FLAG_VAULE;
+    private clientOptions: grpc.ClientOptions;
 
     constructor(
         private readonly clientConfig: SpiceDBClientConfig,
         private readonly interceptors: grpc.Interceptor[] = [],
-    ) {}
+    ) {
+        this.clientOptions = DefaultClientOptions;
+        this.reconcileClientOptions()
+            .then(() => {})
+            .catch((e) => {
+                log.error("[spicedb] Failed to reconcile client options", e);
+            });
+    }
 
-    getClient(): SpiceDBClient {
-        if (!this.client) {
-            this.client = v1.NewClient(
-                this.clientConfig.token,
-                this.clientConfig.address,
-                v1.ClientSecurity.INSECURE_PLAINTEXT_CREDENTIALS,
-                undefined, //
-                {
-                    // we ping frequently to check if the connection is still alive
-                    "grpc.keepalive_time_ms": 1000,
-                    "grpc.keepalive_timeout_ms": 1000,
+    private async reconcileClientOptions() {
+        const doReconcileClientOptions = async () => {
+            const customClientOptions = await getExperimentsClientForBackend().getValueAsync(
+                "spicedb_client_options",
+                DEFAULT_FEATURE_FLAG_VAULE,
+                {},
+            );
+            if (customClientOptions === this.previousClientOptionsString) {
+                return;
+            }
+            try {
+                let clientOptions = DefaultClientOptions;
+                if (customClientOptions && customClientOptions != DEFAULT_FEATURE_FLAG_VAULE) {
+                    clientOptions = JSON.parse(customClientOptions);
+                }
+                if (this.client != null) {
+                    const newClient = this.createClient(clientOptions);
+                    const oldClient = this.client;
+                    this.client = newClient;
 
-                    "grpc.max_reconnect_backoff_ms": 5000,
-                    "grpc.initial_reconnect_backoff_ms": 500,
-                    "grpc.service_config": JSON.stringify({
-                        methodConfig: [
-                            {
-                                name: [{}],
-                                retryPolicy: {
-                                    maxAttempts: 10,
-                                    initialBackoff: "0.1s",
-                                    maxBackoff: "5s",
-                                    backoffMultiplier: 2.0,
-                                    retryableStatusCodes: ["UNAVAILABLE", "DEADLINE_EXCEEDED"],
-                                },
-                            },
-                        ],
-                    }),
-                    "grpc.enable_retries": 1, //TODO enabled by default
+                    log.info("[spicedb] client options changes", {
+                        clientOptions: new TrustedValue(clientOptions),
+                    });
 
-                    // Governs how log DNS resolution results are cached (at minimum!)
-                    // default is 30s, which is too long for us during rollouts (where service DNS entries are updated)
-                    "grpc.dns_min_time_between_resolutions_ms": 2000,
-                    interceptors: this.interceptors,
-                },
-            ) as Client;
+                    setTimeout(() => {
+                        oldClient.close();
+                    }, 10 * 1000);
+                }
+                this.clientOptions = clientOptions;
+                this.previousClientOptionsString = customClientOptions;
+            } catch (e) {
+                log.error("[spicedb] Failed to parse custom client options", e);
+            }
+        };
+        while (true) {
+            await doReconcileClientOptions();
+            await new Promise((resolve) => setTimeout(resolve, 60 * 1000));
+        }
+    }
+
+    private createClient(clientOptions: grpc.ClientOptions): Client {
+        log.debug("[spicedb] creating client", {
+            clientOptions: new TrustedValue(clientOptions),
+        });
+        return v1.NewClient(
+            this.clientConfig.token,
+            this.clientConfig.address,
+            v1.ClientSecurity.INSECURE_PLAINTEXT_CREDENTIALS,
+            undefined, //
+            {
+                ...clientOptions,
+                interceptors: this.interceptors,
+            },
+        ) as Client;
+    }
+
+    getClient(): SpiceDBClient {
+        if (!this.client) {
+            this.client = this.createClient(this.clientOptions);
         }
         return this.client.promises;
     }