chore: unknown flags (#9837)

https://linear.app/unleash/issue/2-3406/hold-unknown-flags-in-memory-and-show-them-in-the-ui-somehow

This PR introduces a suggestion for a “unknown flags” feature.

When clients report metrics for flags that don’t exist in Unleash (e.g.
due to typos), we now track a limited set of these unknown flag names
along with the appnames that reported them. The goal is to help users
identify and clean up incorrect flag usage across their apps.

We store up to 10 unknown flag + appName combinations, keeping only the
most recent reports. Data is collected in-memory and flushed
periodically to the DB, with deduplication and merging to ensure we
don’t exceed the cap even across pods.

We were especially careful to make this implementation defensive, as
unknown flags could be reported in very high volumes. Writes are
batched, deduplicated, and hard-capped to avoid DB pressure.

No UI has been added yet — this is backend-only for now and intended as
a step toward better visibility into client misconfigurations.

I would suggest starting with a simple banner that opens a dialog
showing the list of unknown flags and which apps reported them.

<img width="497" alt="image"
src="https://github.com/user-attachments/assets/b7348e0d-0163-4be4-a7f8-c072e8464331"
/>

Author: nunogois

src/lib/db/index.ts

@@ -66,6 +66,7 @@ import { UserSubscriptionsReadModel } from '../features/user-subscriptions/user-
 import { UniqueConnectionStore } from '../features/unique-connection/unique-connection-store';
 import { UniqueConnectionReadModel } from '../features/unique-connection/unique-connection-read-model';
 import { FeatureLinkStore } from '../features/feature-links/feature-link-store';
+import { UnknownFlagsStore } from '../features/metrics/unknown-flags/unknown-flags-store';
 
 export const createStores = (
     config: IUnleashConfig,
@@ -203,6 +204,7 @@ export const createStores = (
         releasePlanMilestoneStrategyStore:
             new ReleasePlanMilestoneStrategyStore(db, config),
         featureLinkStore: new FeatureLinkStore(db, config),
+        unknownFlagsStore: new UnknownFlagsStore(db),
     };
 };
 

src/lib/features/metrics/client-metrics/metrics-service-v2.test.ts

@@ -11,6 +11,7 @@ import type {
 } from '../../../../lib/types';
 import { endOfDay, startOfHour, subDays, subHours } from 'date-fns';
 import type { IClientMetricsEnv } from './client-metrics-store-v2-type';
+import { UnknownFlagsService } from '../unknown-flags/unknown-flags-service';
 
 function initClientMetrics(flagEnabled = true) {
     const stores = createStores();
@@ -35,8 +36,19 @@ function initClientMetrics(flagEnabled = true) {
         config,
     );
     lastSeenService.updateLastSeen = jest.fn();
+    const unknownFlagsService = new UnknownFlagsService(
+        {
+            unknownFlagsStore: stores.unknownFlagsStore,
+        },
+        config,
+    );
 
-    const service = new ClientMetricsServiceV2(stores, config, lastSeenService);
+    const service = new ClientMetricsServiceV2(
+        stores,
+        config,
+        lastSeenService,
+        unknownFlagsService,
+    );
     return { clientMetricsService: service, eventBus, lastSeenService };
 }
 
@@ -161,10 +173,12 @@ test('get daily client metrics for a toggle', async () => {
         getLogger() {},
     } as unknown as IUnleashConfig;
     const lastSeenService = {} as LastSeenService;
+    const unknownFlagsService = {} as UnknownFlagsService;
     const service = new ClientMetricsServiceV2(
         { clientMetricsStoreV2 },
         config,
         lastSeenService,
+        unknownFlagsService,
     );
 
     const metrics = await service.getClientMetricsForToggle('feature', 3 * 24);
@@ -217,10 +231,12 @@ test('get hourly client metrics for a toggle', async () => {
         getLogger() {},
     } as unknown as IUnleashConfig;
     const lastSeenService = {} as LastSeenService;
+    const unknownFlagsService = {} as UnknownFlagsService;
     const service = new ClientMetricsServiceV2(
         { clientMetricsStoreV2 },
         config,
         lastSeenService,
+        unknownFlagsService,
     );
 
     const metrics = await service.getClientMetricsForToggle('feature', 2);
@@ -287,10 +303,12 @@ const setupMetricsService = ({
         },
     } as unknown as IUnleashConfig;
     const lastSeenService = {} as LastSeenService;
+    const unknownFlagsService = {} as UnknownFlagsService;
     const service = new ClientMetricsServiceV2(
         { clientMetricsStoreV2 },
         config,
         lastSeenService,
+        unknownFlagsService,
     );
     return {
         service,

src/lib/features/metrics/client-metrics/metrics-service-v2.ts

@@ -26,6 +26,10 @@ import {
 import type { ClientMetricsSchema } from '../../../../lib/openapi';
 import { nameSchema } from '../../../schema/feature-schema';
 import memoizee from 'memoizee';
+import {
+    MAX_UNKNOWN_FLAGS,
+    type UnknownFlagsService,
+} from '../unknown-flags/unknown-flags-service';
 
 export default class ClientMetricsServiceV2 {
     private config: IUnleashConfig;
@@ -36,6 +40,8 @@ export default class ClientMetricsServiceV2 {
 
     private lastSeenService: LastSeenService;
 
+    private unknownFlagsService: UnknownFlagsService;
+
     private flagResolver: Pick<IFlagResolver, 'isEnabled' | 'getVariant'>;
 
     private logger: Logger;
@@ -46,9 +52,11 @@ export default class ClientMetricsServiceV2 {
         { clientMetricsStoreV2 }: Pick<IUnleashStores, 'clientMetricsStoreV2'>,
         config: IUnleashConfig,
         lastSeenService: LastSeenService,
+        unknownFlagsService: UnknownFlagsService,
     ) {
         this.clientMetricsStoreV2 = clientMetricsStoreV2;
         this.lastSeenService = lastSeenService;
+        this.unknownFlagsService = unknownFlagsService;
         this.config = config;
         this.logger = config.getLogger(
             '/services/client-metrics/client-metrics-service-v2.ts',
@@ -113,25 +121,55 @@ export default class ClientMetricsServiceV2 {
         }
     }
 
-    async filterExistingToggleNames(toggleNames: string[]): Promise<string[]> {
-        if (this.flagResolver.isEnabled('filterExistingFlagNames')) {
+    async filterExistingToggleNames(toggleNames: string[]): Promise<{
+        validatedToggleNames: string[];
+        unknownToggleNames: string[];
+    }> {
+        let toggleNamesToValidate: string[] = toggleNames;
+        let unknownToggleNames: string[] = [];
+
+        const shouldFilter = this.flagResolver.isEnabled(
+            'filterExistingFlagNames',
+        );
+        const shouldReport = this.flagResolver.isEnabled('reportUnknownFlags');
+
+        if (shouldFilter || shouldReport) {
             try {
-                const validNames = await this.cachedFeatureNames();
+                const existingFlags = await this.cachedFeatureNames();
 
                 const existingNames = toggleNames.filter((name) =>
-                    validNames.includes(name),
+                    existingFlags.includes(name),
+                );
+                const nonExistingNames = toggleNames.filter(
+                    (name) => !existingFlags.includes(name),
                 );
-                if (existingNames.length !== toggleNames.length) {
-                    this.logger.info(
-                        `Filtered out ${toggleNames.length - existingNames.length} toggles with non-existing names`,
+
+                if (shouldFilter) {
+                    toggleNamesToValidate = existingNames;
+
+                    if (existingNames.length !== toggleNames.length) {
+                        this.logger.info(
+                            `Filtered out ${toggleNames.length - existingNames.length} toggles with non-existing names`,
+                        );
+                    }
+                }
+
+                if (shouldReport) {
+                    unknownToggleNames = nonExistingNames.slice(
+                        0,
+                        MAX_UNKNOWN_FLAGS,
                     );
                 }
-                return this.filterValidToggleNames(existingNames);
             } catch (e) {
                 this.logger.error(e);
             }
         }
-        return this.filterValidToggleNames(toggleNames);
+
+        const validatedToggleNames = await this.filterValidToggleNames(
+            toggleNamesToValidate,
+        );
+
+        return { validatedToggleNames, unknownToggleNames };
     }
 
     async filterValidToggleNames(toggleNames: string[]): Promise<string[]> {
@@ -181,7 +219,7 @@ export default class ClientMetricsServiceV2 {
                 ),
         );
 
-        const validatedToggleNames =
+        const { validatedToggleNames, unknownToggleNames } =
             await this.filterExistingToggleNames(toggleNames);
 
         this.logger.debug(
@@ -204,6 +242,15 @@ export default class ClientMetricsServiceV2 {
             this.config.eventBus.emit(CLIENT_REGISTER, heartbeatEvent);
         }
 
+        if (unknownToggleNames.length > 0) {
+            const unknownFlags = unknownToggleNames.map((name) => ({
+                name,
+                appName: value.appName,
+                seenAt: value.bucket.stop,
+            }));
+            this.unknownFlagsService.register(unknownFlags);
+        }
+
         if (validatedToggleNames.length > 0) {
             const clientMetrics: IClientMetricsEnv[] = validatedToggleNames.map(
                 (name) => ({

src/lib/features/metrics/unknown-flags/fake-unknown-flags-store.ts

@@ -0,0 +1,37 @@
+import type { IUnknownFlagsStore, UnknownFlag } from './unknown-flags-store';
+
+export class FakeUnknownFlagsStore implements IUnknownFlagsStore {
+    private unknownFlagMap = new Map<string, UnknownFlag>();
+
+    private getKey(flag: UnknownFlag): string {
+        return `${flag.name}:${flag.appName}`;
+    }
+
+    async replaceAll(flags: UnknownFlag[]): Promise<void> {
+        this.unknownFlagMap.clear();
+        for (const flag of flags) {
+            this.unknownFlagMap.set(this.getKey(flag), flag);
+        }
+    }
+
+    async getAll(): Promise<UnknownFlag[]> {
+        return Array.from(this.unknownFlagMap.values());
+    }
+
+    async clear(hoursAgo: number): Promise<void> {
+        const cutoff = Date.now() - hoursAgo * 60 * 60 * 1000;
+        for (const [key, flag] of this.unknownFlagMap.entries()) {
+            if (flag.seenAt.getTime() < cutoff) {
+                this.unknownFlagMap.delete(key);
+            }
+        }
+    }
+
+    async deleteAll(): Promise<void> {
+        this.unknownFlagMap.clear();
+    }
+
+    async count(): Promise<number> {
+        return this.unknownFlagMap.size;
+    }
+}

src/lib/features/metrics/unknown-flags/unknown-flags-controller.ts

@@ -0,0 +1,74 @@
+import type { Response } from 'express';
+import {
+    unknownFlagsResponseSchema,
+    type UnknownFlagsResponseSchema,
+} from '../../../openapi';
+import { createResponseSchema } from '../../../openapi/util/create-response-schema';
+import Controller from '../../../routes/controller';
+import type { IAuthRequest } from '../../../routes/unleash-types';
+import type { OpenApiService } from '../../../services/openapi-service';
+import type { IFlagResolver } from '../../../types/experimental';
+import type { IUnleashConfig } from '../../../types/option';
+import { NONE } from '../../../types/permissions';
+import { serializeDates } from '../../../types/serialize-dates';
+import type { IUnleashServices } from '../../../types/services';
+import type { UnknownFlagsService } from './unknown-flags-service';
+import { NotFoundError } from '../../../error';
+
+export default class UnknownFlagsController extends Controller {
+    private unknownFlagsService: UnknownFlagsService;
+
+    private flagResolver: IFlagResolver;
+
+    private openApiService: OpenApiService;
+
+    constructor(
+        config: IUnleashConfig,
+        {
+            unknownFlagsService,
+            openApiService,
+        }: Pick<IUnleashServices, 'unknownFlagsService' | 'openApiService'>,
+    ) {
+        super(config);
+        this.unknownFlagsService = unknownFlagsService;
+        this.flagResolver = config.flagResolver;
+        this.openApiService = openApiService;
+
+        this.route({
+            method: 'get',
+            path: '',
+            handler: this.getUnknownFlags,
+            permission: NONE,
+            middleware: [
+                openApiService.validPath({
+                    operationId: 'getUnknownFlags',
+                    tags: ['Unstable'],
+                    summary: 'Get latest reported unknown flag names',
+                    description:
+                        'Returns a list of unknown flag names reported in the last 24 hours, if any. Maximum of 10.',
+                    responses: {
+                        200: createResponseSchema('unknownFlagsResponseSchema'),
+                    },
+                }),
+            ],
+        });
+    }
+
+    async getUnknownFlags(
+        _: IAuthRequest,
+        res: Response<UnknownFlagsResponseSchema>,
+    ): Promise<void> {
+        if (!this.flagResolver.isEnabled('reportUnknownFlags')) {
+            throw new NotFoundError();
+        }
+        const unknownFlags =
+            await this.unknownFlagsService.getGroupedUnknownFlags();
+
+        this.openApiService.respondWithValidation(
+            200,
+            res,
+            unknownFlagsResponseSchema.$id,
+            serializeDates({ unknownFlags }),
+        );
+    }
+}