diff --git a/intelligence/ts/src/engines/caching/nodeStorage.ts b/intelligence/ts/src/engines/caching/nodeStorage.ts new file mode 100644 index 000000000000..1070024257cd --- /dev/null +++ b/intelligence/ts/src/engines/caching/nodeStorage.ts @@ -0,0 +1,48 @@ +// Copyright 2025 Flower Labs GmbH. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +import { join } from 'path'; +import os from 'os'; +import { mkdir, readFile, writeFile } from 'fs/promises'; +import { CachedMapping, CacheStorage } from './storage'; + +export class NodeCacheStorage extends CacheStorage { + private filePath: string | undefined; + + private async getCacheFilePath(): Promise { + if (!this.filePath) { + const homeDir = os.homedir(); + const cacheFolder = join(homeDir, '.flwr', 'cache'); + await mkdir(cacheFolder, { recursive: true }); + this.filePath = join(cacheFolder, 'intelligence-model-names.json'); + } + return this.filePath; + } + + protected async load(): Promise { + try { + const filePath = await this.getCacheFilePath(); + const data = await readFile(filePath, 'utf-8'); + return JSON.parse(data) as CachedMapping; + } catch (_) { + return null; + } + } + + protected async save(cache: CachedMapping): Promise { + const filePath = await this.getCacheFilePath(); + await writeFile(filePath, JSON.stringify(cache), 'utf-8'); + } +} diff --git a/intelligence/ts/src/engines/caching/storage.ts b/intelligence/ts/src/engines/caching/storage.ts new file mode 100644 index 000000000000..9ce898b06441 --- /dev/null +++ b/intelligence/ts/src/engines/caching/storage.ts @@ -0,0 +1,75 @@ +// Copyright 2025 Flower Labs GmbH. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +export interface CachedEntry { + engineModel: string; + timestamp: number; +} + +export interface CachedMapping { + mapping: Record; +} + +export class CacheStorage { + protected async load(): Promise { + await Promise.resolve(); + throw new Error('Method not implemented'); + } + protected async save(_cache: CachedMapping): Promise { + await Promise.resolve(); + throw new Error('Method not implemented'); + } + + /** + * Gets a model mapping if it was saved in the cache, otherwise null. + */ + async getItem(key: string): Promise { + let cache = await this.load(); + if (!cache) { + cache = { mapping: {} }; + } + + if (key in cache.mapping) { + return cache.mapping[key]; + } + return null; + } + + /** + * Adds or updates a model mapping in the cache with the current timestamp. + */ + async setItem(key: string, value: string): Promise { + const now = Date.now(); + let cache = await this.load(); + if (!cache) { + cache = { mapping: {} }; + } + cache.mapping[key] = { engineModel: value, timestamp: now }; + await this.save(cache); + } + + /** + * Removes a model from the cache. + */ + async remove(model: string, engine: string): Promise { + const cache = await this.load(); + const key = `${model}_${engine}`; + if (cache && key in cache.mapping) { + const { [key]: removed, ...rest } = cache.mapping; + cache.mapping = rest; + await this.save(cache); + } + } +} diff --git a/intelligence/ts/src/engines/caching/webStorage.ts b/intelligence/ts/src/engines/caching/webStorage.ts new file mode 100644 index 000000000000..ebaafccd6381 --- /dev/null +++ b/intelligence/ts/src/engines/caching/webStorage.ts @@ -0,0 +1,38 @@ +// Copyright 2025 Flower Labs GmbH. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +import { CachedMapping, CacheStorage } from './storage'; + +export class WebCacheStorage extends CacheStorage { + private readonly CACHE_KEY = 'flwr-mdl-cache'; + + protected async load(): Promise { + await Promise.resolve(); + const data = localStorage.getItem(this.CACHE_KEY); + if (data) { + try { + return JSON.parse(data) as CachedMapping; + } catch { + return null; + } + } + return null; + } + + protected async save(cache: CachedMapping): Promise { + await Promise.resolve(); + localStorage.setItem(this.CACHE_KEY, JSON.stringify(cache)); + } +} diff --git a/intelligence/ts/src/engines/common.ts b/intelligence/ts/src/engines/common.ts index 862375238ffc..75d3c1aad025 100644 --- a/intelligence/ts/src/engines/common.ts +++ b/intelligence/ts/src/engines/common.ts @@ -14,7 +14,13 @@ // ============================================================================= import { REMOTE_URL, SDK, VERSION } from '../constants'; +import { isNode } from '../env'; import { FailureCode, Result } from '../typing'; +import { NodeCacheStorage } from './caching/nodeStorage'; +import { WebCacheStorage } from './caching/webStorage'; +import { CacheStorage } from './caching/storage'; + +const STALE_TIMEOUT_MS = 24 * 60 * 60 * 1000; // 24 hours. interface ModelResponse { is_supported: boolean; @@ -22,7 +28,9 @@ interface ModelResponse { model: string | undefined; } -export async function getEngineModelName(model: string, engine: string): Promise> { +export const cacheStorage: CacheStorage = isNode ? new NodeCacheStorage() : new WebCacheStorage(); + +async function updateModel(model: string, engine: string): Promise> { try { const response = await fetch(`${REMOTE_URL}/v1/fetch-model-config`, { method: 'POST', @@ -48,8 +56,10 @@ export async function getEngineModelName(model: string, engine: string): Promise const data = (await response.json()) as ModelResponse; if (data.is_supported && data.engine_model) { + await cacheStorage.setItem(`${model}_${engine}`, data.engine_model); return { ok: true, value: data.engine_model }; } else { + await cacheStorage.remove(model, engine); return { ok: false, failure: { @@ -68,3 +78,28 @@ export async function getEngineModelName(model: string, engine: string): Promise }; } } + +/** + * Checks if a model is supported. + * - If the model exists in the cache and its timestamp is fresh, return it. + * - If it exists but is stale (older than 24 hours), trigger a background update (and return the stale mapping). + * - If it does not exist, update synchronously. + */ +export async function getEngineModelName(model: string, engine: string): Promise> { + const now = Date.now(); + + const cachedEntry = await cacheStorage.getItem(`${model}_${engine}`); + if (cachedEntry) { + // If the cached entry is stale, trigger a background update. + if (now - cachedEntry.timestamp > STALE_TIMEOUT_MS) { + updateModel(model, engine).catch((err: unknown) => { + console.warn(`Background update failed for model ${model}:`, String(err)); + }); + } + // Return the (possibly stale) cached result. + return { ok: true, value: cachedEntry.engineModel }; + } else { + // Not in cache, call updateModel synchronously. + return await updateModel(model, engine); + } +} diff --git a/intelligence/ts/src/env.ts b/intelligence/ts/src/env.ts new file mode 100644 index 000000000000..d0991b77ac1b --- /dev/null +++ b/intelligence/ts/src/env.ts @@ -0,0 +1,18 @@ +// Copyright 2025 Flower Labs GmbH. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// ============================================================================= + +/* eslint-disable @typescript-eslint/no-unnecessary-condition */ +export const isNode: boolean = typeof process !== 'undefined' && process.versions?.node != null; +/* eslint-enable @typescript-eslint/no-unnecessary-condition */ diff --git a/intelligence/ts/src/flowerintelligence.ts b/intelligence/ts/src/flowerintelligence.ts index 9355eeea0576..6661b7cbe5fb 100644 --- a/intelligence/ts/src/flowerintelligence.ts +++ b/intelligence/ts/src/flowerintelligence.ts @@ -19,10 +19,7 @@ import { TransformersEngine } from './engines/transformersEngine'; import { ChatOptions, ChatResponseResult, FailureCode, Message, Progress, Result } from './typing'; import { WebllmEngine } from './engines/webllmEngine'; import { DEFAULT_MODEL } from './constants'; - -/* eslint-disable @typescript-eslint/no-unnecessary-condition */ -const isNode = typeof process !== 'undefined' && process.versions.node != null; -/* eslint-enable @typescript-eslint/no-unnecessary-condition */ +import { isNode } from './env'; /** * Class representing the core intelligence service for Flower Labs. diff --git a/intelligence/ts/vite.config.ts b/intelligence/ts/vite.config.ts index dc51e1e9957f..4fe298db0da7 100644 --- a/intelligence/ts/vite.config.ts +++ b/intelligence/ts/vite.config.ts @@ -18,7 +18,15 @@ export default defineConfig({ fileName: (format) => `flowerintelligence.${format}.js`, }, rollupOptions: { - external: ['@huggingface/transformers', '@mlc-ai/web-llm', 'crypto'], + external: [ + '@huggingface/transformers', + '@mlc-ai/web-llm', + 'crypto', + 'fs', + 'fs/promises', + 'path', + 'os', + ], }, }, });