Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
39 changes: 39 additions & 0 deletions src/main/libs/openclawConfigSync.runtime.test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -221,6 +221,45 @@ describe('OpenClawConfigSync runtime config output', () => {
expect(config.models.pricing).toEqual({ enabled: false });
});

test('defaults memory search to local FTS-only when embeddings are disabled', async () => {
const sync = await createSync({
getCoworkConfig: () => ({
workingDirectory: tmpDir,
systemPrompt: '',
executionMode: 'local',
agentEngine: 'openclaw',
memoryEnabled: true,
memoryImplicitUpdateEnabled: false,
memoryLlmJudgeEnabled: false,
memoryGuardLevel: 'balanced',
memoryUserMemoriesMaxItems: 100,
skipMissedJobs: false,
embeddingEnabled: false,
embeddingProvider: 'openai',
embeddingModel: '',
embeddingLocalModelPath: '',
embeddingVectorWeight: 0.7,
embeddingRemoteBaseUrl: '',
embeddingRemoteApiKey: '',
}),
});

const result = sync.sync('memory-search-default-fts');
expect(result.ok).toBe(true);

const config = JSON.parse(fs.readFileSync(configPath, 'utf8'));
expect(config.agents.defaults.memorySearch).toMatchObject({
enabled: true,
provider: 'none',
fallback: 'none',
store: {
fts: { tokenizer: 'trigram' },
vector: { enabled: false },
},
});
expect(config.agents.defaults.memorySearch.remote).toBeUndefined();
});

test('configures OpenClaw chat image attachment limit to 30MB', async () => {
const sync = await createSync();

Expand Down
31 changes: 18 additions & 13 deletions src/main/libs/openclawConfigSync.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1854,30 +1854,35 @@ loopDetection: MANAGED_TOOL_LOOP_DETECTION,
workspace: path.resolve(mainWorkspacePath),
mediaMaxMb: 30,
...(taskWorkingDirectory ? { cwd: path.resolve(taskWorkingDirectory) } : {}),
...(coworkConfig.embeddingEnabled ? {
memorySearch: {
enabled: true,
provider: (['openai', 'gemini', 'voyage', 'mistral', 'ollama'].includes(coworkConfig.embeddingProvider)
memorySearch: {
enabled: true,
provider: coworkConfig.embeddingEnabled
? (['openai', 'gemini', 'voyage', 'mistral', 'ollama'].includes(coworkConfig.embeddingProvider)
? coworkConfig.embeddingProvider
: 'openai'),
...(coworkConfig.embeddingModel ? { model: coworkConfig.embeddingModel } : {}),
: 'openai')
: 'none',
...(coworkConfig.embeddingEnabled && coworkConfig.embeddingModel ? { model: coworkConfig.embeddingModel } : {}),
...(coworkConfig.embeddingEnabled ? {
remote: {
...(coworkConfig.embeddingRemoteBaseUrl ? { baseUrl: coworkConfig.embeddingRemoteBaseUrl } : {}),
...(coworkConfig.embeddingRemoteApiKey ? { apiKey: coworkConfig.embeddingRemoteApiKey } : {}),
},
store: {
// Use trigram tokenizer for FTS5 — unicode61 (the openclaw default)
// cannot tokenize CJK characters, so Chinese/Japanese/Korean memory
// content is invisible to keyword search.
fts: { tokenizer: 'trigram' },
},
query: {
hybrid: {
vectorWeight: coworkConfig.embeddingVectorWeight ?? 0.7,
},
},
} : {
fallback: 'none',
}),
store: {
// Use trigram tokenizer for FTS5 — unicode61 (the openclaw default)
// cannot tokenize CJK characters, so Chinese/Japanese/Korean memory
// content is invisible to keyword search.
fts: { tokenizer: 'trigram' },
...(!coworkConfig.embeddingEnabled ? { vector: { enabled: false } } : {}),
},
} : {}),
},
heartbeat: {
every: coworkConfig.openClawHeartbeatEnabled === false
? OPENCLAW_HEARTBEAT_EVERY_DISABLED
Expand Down
9 changes: 9 additions & 0 deletions src/main/libs/openclawEngineManager.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@ import {
import { getCodexHomeDir } from './openaiCodexAuth';
import { migrateLegacyCronStorageWithDoctor } from './openclawCronLegacyMigration';
import { cleanupStaleThirdPartyPluginsFromBundledDir, listLocalOpenClawExtensionIds,syncLocalOpenClawExtensionsIntoRuntime } from './openclawLocalExtensions';
import { migrateMainFtsOnlyMemoryIndex } from './openclawMemoryIndexMigration';
import { ensureOpenClawWorkerShims } from './openclawWorkerShims';
import { appendPythonRuntimeToEnv } from './pythonRuntime';

Expand Down Expand Up @@ -599,6 +600,14 @@ export class OpenClawEngineManager extends EventEmitter {
env,
});

await migrateMainFtsOnlyMemoryIndex({
stateDir: this.stateDir,
configPath: this.configPath,
runtimeRoot: runtime.root,
electronNodeRuntimePath,
env,
});

const forkArgs = ['gateway', '--bind', 'loopback', '--port', String(port), '--token', token, '--verbose'];
const gatewayExecArgv = buildOpenClawGatewayExecArgv(process.env.NODE_OPTIONS);
if (gatewayExecArgv.length > 0) {
Expand Down
281 changes: 281 additions & 0 deletions src/main/libs/openclawMemoryIndexMigration.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,281 @@
import Database from 'better-sqlite3';
import fs from 'fs';
import os from 'os';
import path from 'path';
import { afterEach, beforeEach, describe, expect, test, vi } from 'vitest';

import {
type MemoryIndexMigrationRunner,
migrateMainFtsOnlyMemoryIndex,
resolveMainMemoryIndexMigrationNeed,
} from './openclawMemoryIndexMigration';

let tmpDir = '';
let stateDir = '';
let runtimeRoot = '';
let configPath = '';
let dbPath = '';
let electronNodeRuntimePath = '';

function mkdirp(dir: string): void {
fs.mkdirSync(dir, { recursive: true });
}

function writeFile(filePath: string, content: string): void {
mkdirp(path.dirname(filePath));
fs.writeFileSync(filePath, content, 'utf8');
}

function writeConfig(memorySearch: Record<string, unknown>, agentMemorySearch?: Record<string, unknown>): void {
const mainAgent: Record<string, unknown> = { id: 'main', default: true };
if (agentMemorySearch) {
mainAgent.memorySearch = agentMemorySearch;
}
writeFile(
configPath,
`${JSON.stringify({
gateway: { mode: 'local' },
agents: {
defaults: {
workspace: path.join(stateDir, 'workspace-main'),
memorySearch,
},
list: [mainAgent],
},
}, null, 2)}\n`,
);
}

function writeIndexMeta(meta: Record<string, unknown> | null): void {
mkdirp(path.dirname(dbPath));
const db = new Database(dbPath);
try {
db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT NOT NULL)');
if (meta) {
db
.prepare('INSERT OR REPLACE INTO meta (key, value) VALUES (?, ?)')
.run('memory_index_meta_v1', JSON.stringify(meta));
}
} finally {
db.close();
}
}

function writeEmptySqlite(): void {
mkdirp(path.dirname(dbPath));
const db = new Database(dbPath);
db.close();
}

function ftsOnlyMemorySearch(): Record<string, unknown> {
return {
enabled: true,
provider: 'none',
fallback: 'none',
store: {
fts: { tokenizer: 'trigram' },
vector: { enabled: false },
},
};
}

describe('openclawMemoryIndexMigration', () => {
beforeEach(() => {
tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), 'lobsterai-openclaw-memory-migration-'));
stateDir = path.join(tmpDir, 'openclaw', 'state');
runtimeRoot = path.join(tmpDir, 'runtime');
configPath = path.join(stateDir, 'openclaw.json');
dbPath = path.join(stateDir, 'memory', 'main.sqlite');
electronNodeRuntimePath = process.execPath;
mkdirp(runtimeRoot);
writeFile(path.join(runtimeRoot, 'openclaw.mjs'), 'console.log("openclaw");\n');
});

afterEach(() => {
vi.restoreAllMocks();
if (tmpDir) {
fs.rmSync(tmpDir, { recursive: true, force: true });
}
});

test('skips when config still uses an embedding provider', async () => {
writeConfig({
enabled: true,
provider: 'gemini',
model: 'gemini-embedding-001',
store: { fts: { tokenizer: 'trigram' } },
});
writeIndexMeta({
model: 'gemini-embedding-001',
provider: 'gemini',
ftsTokenizer: 'trigram',
});
const runner = vi.fn<MemoryIndexMigrationRunner>();

const result = await migrateMainFtsOnlyMemoryIndex({
stateDir,
configPath,
runtimeRoot,
electronNodeRuntimePath,
env: {},
runner,
});

expect(result).toEqual({ status: 'skipped', reason: 'not-fts-only-config' });
expect(runner).not.toHaveBeenCalled();
});

test('skips when main agent overrides defaults with an embedding provider', async () => {
writeConfig(ftsOnlyMemorySearch(), {
enabled: true,
provider: 'gemini',
model: 'gemini-embedding-001',
store: { fts: { tokenizer: 'trigram' } },
});
writeIndexMeta({
model: 'gemini-embedding-001',
provider: 'gemini',
ftsTokenizer: 'trigram',
});

expect(resolveMainMemoryIndexMigrationNeed({ configPath, stateDir })).toEqual({
shouldMigrate: false,
reason: 'not-fts-only-config',
});
});

test('skips when FTS-only index metadata is current', async () => {
writeConfig(ftsOnlyMemorySearch());
writeIndexMeta({
model: 'fts-only',
provider: 'none',
ftsTokenizer: 'trigram',
});
const runner = vi.fn<MemoryIndexMigrationRunner>();

const result = await migrateMainFtsOnlyMemoryIndex({
stateDir,
configPath,
runtimeRoot,
electronNodeRuntimePath,
env: {},
runner,
});

expect(result).toEqual({ status: 'skipped', reason: 'index-meta-current' });
expect(runner).not.toHaveBeenCalled();
});

test('runs official forced reindex when old embedding metadata exists under FTS-only config', async () => {
writeConfig(ftsOnlyMemorySearch());
writeIndexMeta({
model: 'gemini-embedding-001',
provider: 'gemini',
ftsTokenizer: 'unicode61',
});
const runner = vi.fn<MemoryIndexMigrationRunner>().mockResolvedValue({
code: 0,
stdout: 'updated',
stderr: '',
});

const result = await migrateMainFtsOnlyMemoryIndex({
stateDir,
configPath,
runtimeRoot,
electronNodeRuntimePath,
env: { EXISTING: '1' },
runner,
});

expect(result.status).toBe('migrated');
expect(runner).toHaveBeenCalledTimes(1);
const [command, args, options] = runner.mock.calls[0];
expect(command).toBe(electronNodeRuntimePath);
expect(args).toEqual([
path.join(runtimeRoot, 'openclaw.mjs'),
'memory',
'index',
'--force',
'--agent',
'main',
]);
expect(options.cwd).toBe(runtimeRoot);
expect(options.timeoutMs).toBeGreaterThan(0);
expect(options.env.EXISTING).toBe('1');
expect(options.env.OPENCLAW_HOME).toBe(path.dirname(stateDir));
expect(options.env.OPENCLAW_STATE_DIR).toBe(stateDir);
expect(options.env.OPENCLAW_CONFIG_PATH).toBe(configPath);
expect(options.env.ELECTRON_RUN_AS_NODE).toBe('1');
});

test('runs forced reindex when metadata is missing', async () => {
writeConfig(ftsOnlyMemorySearch());
writeIndexMeta(null);

expect(resolveMainMemoryIndexMigrationNeed({ configPath, stateDir })).toMatchObject({
shouldMigrate: true,
dbPath,
reason: 'index metadata is missing',
});
});

test('runs forced reindex when sqlite meta table is missing', async () => {
writeConfig(ftsOnlyMemorySearch());
writeEmptySqlite();

expect(resolveMainMemoryIndexMigrationNeed({ configPath, stateDir })).toMatchObject({
shouldMigrate: true,
dbPath,
reason: 'index metadata is missing',
});
});

test('skips when bundled OpenClaw CLI is missing', async () => {
fs.rmSync(path.join(runtimeRoot, 'openclaw.mjs'), { force: true });
writeConfig(ftsOnlyMemorySearch());
writeIndexMeta({
model: 'gemini-embedding-001',
provider: 'gemini',
ftsTokenizer: 'unicode61',
});
const runner = vi.fn<MemoryIndexMigrationRunner>();

const result = await migrateMainFtsOnlyMemoryIndex({
stateDir,
configPath,
runtimeRoot,
electronNodeRuntimePath,
env: {},
runner,
});

expect(result).toEqual({ status: 'skipped', reason: 'missing-openclaw-cli' });
expect(runner).not.toHaveBeenCalled();
});

test('returns failed when forced reindex exits non-zero', async () => {
writeConfig(ftsOnlyMemorySearch());
writeIndexMeta({
model: 'gemini-embedding-001',
provider: 'gemini',
ftsTokenizer: 'unicode61',
});
const runner = vi.fn<MemoryIndexMigrationRunner>().mockResolvedValue({
code: 2,
stdout: '',
stderr: 'failed',
});

const result = await migrateMainFtsOnlyMemoryIndex({
stateDir,
configPath,
runtimeRoot,
electronNodeRuntimePath,
env: {},
runner,
});

expect(result).toEqual({ status: 'failed', code: 2 });
});
});
Loading
Loading