From 4e7eacb8eb579d663f6a1b6d7e781a6f65288653 Mon Sep 17 00:00:00 2001 From: Gergana Mihaylova Date: Wed, 17 Jun 2026 17:58:04 +0300 Subject: [PATCH 1/5] Add: POC testing the accountOps migration to indexedDb --- src/controllers/activity/activity.ts | 213 +++++++++- src/interfaces/activityIdb.ts | 68 +++ src/services/storage/activityIdb.ts | 600 +++++++++++++++++++++++++++ src/services/storage/baseIdbStore.ts | 163 ++++++++ 4 files changed, 1035 insertions(+), 9 deletions(-) create mode 100644 src/interfaces/activityIdb.ts create mode 100644 src/services/storage/activityIdb.ts create mode 100644 src/services/storage/baseIdbStore.ts diff --git a/src/controllers/activity/activity.ts b/src/controllers/activity/activity.ts index d2cc4b87a1..7d4436fba6 100644 --- a/src/controllers/activity/activity.ts +++ b/src/controllers/activity/activity.ts @@ -6,9 +6,11 @@ import { pickBetterPoisoningMatch, ScoredAddressPoisoningMatch } from '@/libs/transfer/address-poisoning' +import { ActivityIdbStorage } from '@/services/storage/activityIdb' import { Account, AccountId, IAccountsController } from '../../interfaces/account' import { IActivityController } from '../../interfaces/activity' +import { IActivityIdbStorage } from '../../interfaces/activityIdb' import { Banner } from '../../interfaces/banner' import { IEventEmitterRegistryController } from '../../interfaces/eventEmitter' import { Fetch } from '../../interfaces/fetch' @@ -96,6 +98,9 @@ export interface ExternalAccountOps { // We are limiting items array to include no more than 1000 records, // as we trim out the oldest ones (in the beginning of the items array). // We do this to maintain optimal storage and performance. +// Set to true to disable IndexedDB and benchmark against chrome.storage.local +const DISABLE_ACTIVITY_IDB = false + const trim = (items: T[], maxSize = 1000): void => { if (items.length > maxSize) { // If the array size is greater than maxSize, remove the last (oldest) item @@ -244,6 +249,8 @@ export class ActivityController extends EventEmitter implements IActivityControl #fetch: Fetch + #activityIdb?: IActivityIdbStorage + #initialLoadPromise?: Promise #accounts: IAccountsController @@ -322,6 +329,18 @@ export class ActivityController extends EventEmitter implements IActivityControl super(eventEmitterRegistry) this.#storage = storage this.#fetch = fetch + + // Initialize ActivityIdbStorage if available (browser environment) + if (!DISABLE_ACTIVITY_IDB && typeof indexedDB !== 'undefined') { + try { + this.#activityIdb = new ActivityIdbStorage() + console.log('[ActivityController] ActivityIdbStorage initialized') + } catch (error) { + console.error('[ActivityController] Failed to initialize ActivityIdbStorage', error) + } + } + + console.log('[ActivityController] Constructor: activityIdb =', !!this.#activityIdb) this.#callRelayer = callRelayer this.#accounts = accounts this.#selectedAccount = selectedAccount @@ -338,8 +357,52 @@ export class ActivityController extends EventEmitter implements IActivityControl async #load(): Promise { await this.#accounts.initialLoadPromise await this.#selectedAccount.initialLoadPromise - const [accountsOps, externalAccountOps, signedMessages] = await Promise.all([ - this.#storage.get('accountsOps', {}), + + let accountsOps: InternalAccountsOps = {} + let externalAccountOps: ExternalAccountOps = {} + let signedMessages: InternalSignedMessages = {} + + // Migrate from chrome.storage.local to IDB if needed (first load) + if (this.#activityIdb) { + try { + const idbIsEmpty = await this.#activityIdb.isEmpty() + if (idbIsEmpty) { + const storedOps = await this.#storage.get('accountsOps', {}) + if (Object.keys(storedOps).length > 0) { + console.log('[ActivityController] Migrating accountsOps to IDB') + await this.#activityIdb.migrateFromStorage(storedOps) + console.log('[ActivityController] accountsOps migration complete') + } + } + } catch (error) { + console.error('[ActivityController] Failed to migrate to IDB', error) + } + } + + // Load from IDB or fallback to storage + if (this.#activityIdb) { + try { + console.time('[ActivityController] startup-load:idb') + console.log('[ActivityController] Loading startup ops from IDB...') + accountsOps = await this.#activityIdb.loadStartupOps() + console.timeEnd('[ActivityController] startup-load:idb') + console.log('[ActivityController] Loaded startup ops from IDB') + } catch (error) { + console.error('[ActivityController] Failed to load from IDB, falling back to storage', error) + console.time('[ActivityController] startup-load:storage-fallback') + accountsOps = await this.#storage.get('accountsOps', {}) + console.timeEnd('[ActivityController] startup-load:storage-fallback') + console.log('[ActivityController] Loaded from chrome.storage.local fallback') + } + } else { + console.log('[ActivityController] IDB not available, using chrome.storage.local') + console.time('[ActivityController] startup-load:storage') + accountsOps = await this.#storage.get('accountsOps', {}) + console.timeEnd('[ActivityController] startup-load:storage') + } + + // Always load externalAccountOps from storage (not migrated to IDB yet) + [externalAccountOps, signedMessages] = await Promise.all([ this.#storage.get('externalAccountOps', {}), this.#storage.get('signedMessages', {}) ]) @@ -348,6 +411,10 @@ export class ActivityController extends EventEmitter implements IActivityControl this.#externalAccountOps = externalAccountOps this.#signedMessages = signedMessages + console.log('[ActivityController] Storage snapshot:') + console.log('accountsOps:', JSON.stringify(accountsOps, (_, v) => typeof v === 'bigint' ? v.toString() : v, 2)) + + console.log('[ActivityController] Load complete, emitting update') this.emitUpdate() } @@ -446,14 +513,79 @@ export class ActivityController extends EventEmitter implements IActivityControl ) const enabledNetworkChainIds = this.#networks.networks.map(({ chainId }) => String(chainId)) - const internalAccountOpsByChain = this.#accountsOps[filters.account] || {} + let internalAccountOpsByChain = this.#accountsOps[filters.account] || {} const externalAccountOpsByChain = this.#externalAccountOps[filters.account] || {} + + console.log('[ActivityController] filterAccountsOps called', { + account: filters.account, + chainId: filters.chainId?.toString(), + page: pagination.fromPage, + knownAccounts: Object.keys(this.#accountsOps), + internalChains: Object.entries(internalAccountOpsByChain).map( + ([c, ops]) => `${c}:${ops.length}ops` + ), + externalChains: Object.entries(externalAccountOpsByChain).map( + ([c, ops]) => `${c}:${ops.length}ops` + ), + enabledNetworkChainIds + }) + + // Lazy-load from IDB if requesting pages beyond the in-memory window and IDB is available. + // Skip if in-memory already holds the full history (>= STARTUP_RECENT_OPS_LIMIT means it was + // previously expanded, so a second IDB round-trip would return the same data). + const chainIdString = filters.chainId?.toString() + const inMemoryCount = chainIdString + ? (internalAccountOpsByChain[chainIdString]?.length ?? 0) + : 0 + // IDB startup window loads 20 ops per chain; more than that means the cache was already expanded. + // Lazy-load whenever a chain filter is active and the cache is still in the startup window — + // including page 0, so the correct total page count is shown immediately on chain switch. + const idbStartupWindowSize = 20 + const alreadyFullyLoaded = inMemoryCount > idbStartupWindowSize + if (this.#activityIdb && filters.chainId && chainIdString && !alreadyFullyLoaded) { + try { + console.time(`[ActivityController] lazy-load:idb page=${pagination.fromPage}`) + const fullOpsFromIdb = await this.#activityIdb.getOpsForAccountAndChain( + filters.account, + filters.chainId + ) + console.timeEnd(`[ActivityController] lazy-load:idb page=${pagination.fromPage}`) + if (fullOpsFromIdb) { + console.log( + `[ActivityController] lazy-load: in-memory had ${inMemoryCount} ops → IDB returned ${fullOpsFromIdb.length} ops for ${filters.account}:${chainIdString}` + ) + // Update in-memory cache with full array from IDB + if (!internalAccountOpsByChain[chainIdString]) { + internalAccountOpsByChain[chainIdString] = [] + } + internalAccountOpsByChain[chainIdString] = fullOpsFromIdb + this.#accountsOps[filters.account] = internalAccountOpsByChain + } + } catch (error) { + console.error('ActivityController: Failed to lazy-load from IDB', error) + // Continue with in-memory data + } + } else { + console.log( + `[ActivityController] pagination page=${pagination.fromPage}: served from in-memory (${inMemoryCount} ops cached)` + ) + } + const internalAccountOpsEntriesOnEnabledNetworks = Object.entries( internalAccountOpsByChain ).filter(([chainId]) => enabledNetworkChainIds.includes(chainId)) const internalAccountOps = new Set( internalAccountOpsEntriesOnEnabledNetworks.flatMap(([, accountOps]) => accountOps) ) + + // Build a set of all txnIds from internal ops for dedup at the merge point. + // External ops whose txnId matches an internal op are filtered out here — they are + // duplicates that #removeExternalAccountOpsMatchingInternalOps missed because the + // internal op was outside the startup window when the scanner ran. + const internalTxnIds = new Set( + [...internalAccountOps].flatMap((op) => getInternalAccountOpTxnIds(op).map(normalizeTxnId)) + ) + const accountOpsEntriesOnEnabledNetworks = enabledNetworkChainIds .map( (chainId) => @@ -461,7 +593,9 @@ export class ActivityController extends EventEmitter implements IActivityControl chainId, [ ...(internalAccountOpsByChain[chainId] || []), - ...(externalAccountOpsByChain[chainId] || []) + ...(externalAccountOpsByChain[chainId] || []).filter( + (extOp) => !extOp.txnId || !internalTxnIds.has(normalizeTxnId(extOp.txnId)) + ) ] ] as const ) @@ -490,7 +624,12 @@ export class ActivityController extends EventEmitter implements IActivityControl ) } + console.time(`[ActivityController] paginate page=${pagination.fromPage}`) const result = paginate(filteredItems, pagination.fromPage, pagination.itemsPerPage) + console.timeEnd(`[ActivityController] paginate page=${pagination.fromPage}`) + console.log( + `[ActivityController] paginate: ${filteredItems.length} total ops → page ${pagination.fromPage} has ${result.items.length} items (${result.maxPages} pages total)` + ) this.setDashboardBannersSeen(sessionId, filters.account) this.accountsOps[sessionId] = { result, filters, pagination } @@ -563,7 +702,36 @@ export class ActivityController extends EventEmitter implements IActivityControl } private async persistAccountsOps() { - await this.#storage.set('accountsOps', this.#accountsOps) + if (this.#activityIdb) { + try { + // Collect all (accountAddr, chainId) records to persist + const recordsToPersist: Array<{ + accountAddr: string + chainId: bigint | string + ops: SubmittedAccountOp[] + }> = [] + + for (const [accountAddr, chainMap] of Object.entries(this.#accountsOps)) { + for (const [chainIdString, ops] of Object.entries(chainMap)) { + recordsToPersist.push({ + accountAddr, + chainId: chainIdString, + ops + }) + } + } + + if (recordsToPersist.length > 0) { + await this.#activityIdb.putMultiple(recordsToPersist) + } + } catch (error) { + console.error('ActivityController: Failed to persist to IDB, falling back to storage', error) + await this.#storage.set('accountsOps', this.#accountsOps) + } + } else { + await this.#storage.set('accountsOps', this.#accountsOps) + } + await this.syncFilteredAccountsOps() this.emitUpdate() } @@ -682,7 +850,22 @@ export class ActivityController extends EventEmitter implements IActivityControl await this.syncFilteredAccountsOps() - await this.#storage.set('accountsOps', this.#accountsOps) + // Write to IDB if available, otherwise fallback to chrome.storage.local + if (this.#activityIdb) { + try { + await this.#activityIdb.putOpsForAccountAndChain( + accountAddr, + chainId, + this.#accountsOps[accountAddr][chainId.toString()]! + ) + } catch (error) { + console.error('ActivityController: Failed to write to IDB, falling back to storage', error) + await this.#storage.set('accountsOps', this.#accountsOps) + } + } else { + await this.#storage.set('accountsOps', this.#accountsOps) + } + this.emitUpdate() } @@ -832,7 +1015,8 @@ export class ActivityController extends EventEmitter implements IActivityControl externalAccountOps.unshift(submittedAccountOpLike) trim(externalAccountOps) - await this.#storage.set('externalAccountOps', this.#externalAccountOps) + // externalAccountOps: using chrome.storage.local only (not migrated to IDB yet) + await this.#storage.set('externalAccountOps', this.#externalAccountOps); await this.syncFilteredAccountsOps() this.emitUpdate() } @@ -1444,8 +1628,19 @@ export class ActivityController extends EventEmitter implements IActivityControl await this.syncFilteredAccountsOps() await this.syncSignedMessages() - await this.#storage.set('accountsOps', this.#accountsOps) - await this.#storage.set('signedMessages', this.#signedMessages) + // Delete from IDB if available + if (this.#activityIdb) { + try { + await this.#activityIdb.deleteAccount(address) + } catch (error) { + console.error('ActivityController: Failed to delete from IDB, falling back to storage', error) + await this.#storage.set('accountsOps', this.#accountsOps) + await this.#storage.set('signedMessages', this.#signedMessages) + } + } else { + await this.#storage.set('accountsOps', this.#accountsOps) + await this.#storage.set('signedMessages', this.#signedMessages) + } this.emitUpdate() } diff --git a/src/interfaces/activityIdb.ts b/src/interfaces/activityIdb.ts new file mode 100644 index 0000000000..a59f3bdecf --- /dev/null +++ b/src/interfaces/activityIdb.ts @@ -0,0 +1,68 @@ +import { SubmittedAccountOp, SubmittedAccountOpLike } from '../libs/accountOp/submittedAccountOp' + +/** + * Interface for IndexedDB-backed storage of account operations. + * Provides minimal, targeted reads/writes compared to full blob serialization. + */ +export interface IActivityIdbStorage { + /** + * Load minimal startup dataset: all pending ops + 20 most recent per (account, chainId) + * Used during ActivityController initialization. + */ + loadStartupOps(): Promise + + /** + * Fetch full history for a specific (account, chainId) pair. + * Used for lazy-loading older history during pagination. + */ + getOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string + ): Promise + + /** + * Write ops for a single (account, chainId) pair. + * Accepts both internal and external account ops. + */ + putOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string, + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + ): Promise + + /** + * Batch write multiple (account, chainId) records in a single transaction. + * More efficient than multiple individual puts. + * Accepts both internal and external account ops. + */ + putMultiple( + records: Array<{ + accountAddr: string + chainId: bigint | string + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + }> + ): Promise + + /** + * Delete all ops for an account across all chains. + */ + deleteAccount(accountAddr: string): Promise + + /** + * One-time migration: import all ops from chrome.storage.local into IDB. + * After successful import, the caller should remove the key from chrome.storage.local. + */ + migrateFromStorage(data: InternalAccountsOps): Promise + + /** + * Check if IDB has any data (used to detect if migration is needed). + */ + isEmpty(): Promise +} + +/** + * Re-export for convenience (mirrors ActivityController's internal structure) + */ +export interface InternalAccountsOps { + [accountAddr: string]: { [chainIdString: string]: SubmittedAccountOp[] } +} diff --git a/src/services/storage/activityIdb.ts b/src/services/storage/activityIdb.ts new file mode 100644 index 0000000000..64e137a660 --- /dev/null +++ b/src/services/storage/activityIdb.ts @@ -0,0 +1,600 @@ +import { SubmittedAccountOp, SubmittedAccountOpLike } from '../../libs/accountOp/submittedAccountOp' +import { IActivityIdbStorage, InternalAccountsOps } from '../../interfaces/activityIdb' +import { BaseIdbStore } from './baseIdbStore' +import { AccountOpStatus } from '../../libs/accountOp/types' + +const STARTUP_RECENT_OPS_LIMIT = 20 +const MAX_OPS_PER_ACCOUNT_CHAIN = 1000 + +// Known bigint field names — used by the JSON reviver to restore bigint values after deserialization +const BIGINT_FIELDS = new Set([ + 'chainId', + 'nonce', + 'eoaNonce', + 'amount', + 'amountBefore', + 'amountAfter', + 'balanceChange', + 'feeTokenChainId', + 'simulatedGasLimit', + 'gasPrice', + 'maxPriorityFeePerGas' +]) + +function serializeOp(op: SubmittedAccountOp | SubmittedAccountOpLike): string { + return JSON.stringify(op, (_key, value) => { + if (typeof value === 'bigint') return value.toString() + return value + }) +} + +function deserializeOp(serialized: string): SubmittedAccountOp { + return JSON.parse(serialized, (key, value) => { + if (BIGINT_FIELDS.has(key) && typeof value === 'string' && /^\d+$/.test(value)) { + return BigInt(value) + } + return value + }) as SubmittedAccountOp +} + +/** + * One IDB row per submitted account op (v2 schema). + * keyPath: ['accountAddr', 'chainId', 'id'] + * index 'by-account-chain-timestamp': ['accountAddr', 'chainId', 'timestamp'] + */ +interface IdbAccountOpRow { + accountAddr: string + chainId: string // bigint converted to string for IDB compatibility + id: string // op.id + timestamp: number + status: AccountOpStatus + serializedOp: string // JSON.stringify with bigint→string replacement +} + +// ──────────────────────────────────────────────────────────────────────────────── +// V1 shape — only used during migration in onupgradeneeded +// ──────────────────────────────────────────────────────────────────────────────── +interface V1IdbAccountOpsRecord { + accountAddr: string + chainId: string + ops: any[] +} + +// The highest unicode character — used as a range upper bound to select all keys +// that start with a given prefix, without matching the prefix itself as a key. +const RANGE_HIGH = '￿' + +/** + * IndexedDB-backed storage for account operations — v2 row-per-op schema. + * + * One IDB record per SubmittedAccountOp, indexed by timestamp so that + * loadStartupOps() can use a cursor to fetch only the needed subset instead of + * reading the full history for every (account, chain). + * + * Extends BaseIdbStore for common IDB operations and error handling. + */ +export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStorage { + constructor() { + super({ + dbName: 'ambire-activity', + storeName: 'accountsOps', + keyPath: ['accountAddr', 'chainId', 'id'], + dbVersion: 3 + }) + } + + /** + * Override doInit to: + * 1. Create the object store with the v2 compound keyPath + * 2. Add the 'by-account-chain-timestamp' index + * 3. Migrate existing v1 blob records to individual rows when upgrading from v1 + */ + protected doInit(): Promise { + return new Promise((resolve, reject) => { + const request = indexedDB.open(this.config.dbName, this.config.dbVersion) + + request.onerror = () => { + console.error(`BaseIdbStore: Failed to open ${this.config.dbName}`, request.error) + reject(request.error) + } + + request.onsuccess = () => { + this.db = request.result + console.log( + `[BaseIdbStore] Opened ${this.config.dbName} v${this.config.dbVersion} with store "${this.config.storeName}"` + ) + resolve() + } + + request.onupgradeneeded = (event) => { + const db = (event.target as IDBOpenDBRequest).result + const tx = (event.target as IDBOpenDBRequest).transaction! + const oldVersion = event.oldVersion + + if (oldVersion < 1) { + // Fresh install — create store directly at v3 shape + const store = db.createObjectStore(this.config.storeName, { + keyPath: this.config.keyPath + }) + store.createIndex('by-account-chain-timestamp', ['accountAddr', 'chainId', 'timestamp']) + console.log(`[BaseIdbStore] Created store "${this.config.storeName}"`) + return + } + + if (oldVersion === 1 || oldVersion === 2) { + // v1→v3: expand blob-per-(account,chain) records into individual rows. + // v2→v3: re-run migration to fix bad chainId data — v2 used op.chainId which + // could be a non-string (bigint, object) depending on serialization history. + // v3 always uses record.chainId (the safe IDB key string) instead. + console.log(`[ActivityIdbStorage] Migrating from v${oldVersion} to v3 schema...`) + + const oldStore = tx.objectStore(this.config.storeName) + const getAllRequest = oldStore.getAll() + + getAllRequest.onsuccess = () => { + const oldRecords = (getAllRequest.result || []) as any[] + + db.deleteObjectStore(this.config.storeName) + const newStore = db.createObjectStore(this.config.storeName, { + keyPath: this.config.keyPath + }) + newStore.createIndex( + 'by-account-chain-timestamp', + ['accountAddr', 'chainId', 'timestamp'] + ) + console.log(`[ActivityIdbStorage] Recreated store for v3`) + + let migratedOps = 0 + for (const record of oldRecords) { + const { accountAddr, chainId } = record + + if (oldVersion === 1) { + // v1 shape: { accountAddr, chainId, ops: any[] } + // Use record.chainId — already the correct string from the v1 IDB key. + // Do NOT use op.chainId — it may be a bigint, object, or custom-serialized value. + const ops: any[] = record.ops || [] + for (const op of ops) { + if (!op?.id) continue + const row: IdbAccountOpRow = { + accountAddr, + chainId, + id: op.id, + timestamp: op.timestamp ?? 0, + status: op.status, + serializedOp: serializeOp(op) + } + newStore.put(row) + migratedOps++ + } + } else { + // v2 shape: already a row — { accountAddr, chainId, id, timestamp, status, serializedOp } + // chainId may be "[object Object]" — fix it using the record key components directly. + // The v2 IDB key was ['accountAddr', 'chainId', 'id'] so the values are on the record. + // If chainId is bad, we have no reliable source for the correct value and must drop the row. + if (!chainId || chainId === '[object Object]') { + console.warn( + `[ActivityIdbStorage] v2→v3: dropping row with bad chainId for ${accountAddr}:${record.id}` + ) + continue + } + const row: IdbAccountOpRow = { + accountAddr, + chainId, + id: record.id, + timestamp: record.timestamp ?? 0, + status: record.status, + serializedOp: record.serializedOp + } + newStore.put(row) + migratedOps++ + } + } + console.log( + `[ActivityIdbStorage] v${oldVersion}→v3 migration complete: ${oldRecords.length} records → ${migratedOps} rows` + ) + } + + getAllRequest.onerror = () => { + console.error( + `[ActivityIdbStorage] v${oldVersion}→v3 migration failed to read old records`, + getAllRequest.error + ) + // Don't reject — let the upgrade complete even if data is lost; + // a broken upgrade prevents the DB from opening at all. + } + } + } + }) + } + + // ────────────────────────────────────────────────────────────────────────────── + // Public API + // ────────────────────────────────────────────────────────────────────────────── + + /** + * Load minimal startup dataset using a cursor on 'by-account-chain-timestamp' + * (newest-first). For each (account, chain) group we collect: + * - All pending ops (BroadcastedButNotConfirmed | Pending) + * - Up to STARTUP_RECENT_OPS_LIMIT finalized ops + * Once a group has enough finalized ops the cursor jumps to the next chain. + */ + async loadStartupOps(): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readonly') + const store = tx.objectStore(this.config.storeName) + const index = store.index('by-account-chain-timestamp') + + // 'prev' gives us newest first within each (accountAddr, chainId) group + const cursorRequest = index.openCursor(null, 'prev') + + const result: InternalAccountsOps = {} + // Tracks how many finalized ops we've collected per group key + const finalizedCount = new Map() + // Accumulates rows before we sort and deserialize at the end + const rowsByGroup = new Map() + + let totalRecords = 0 + + cursorRequest.onerror = () => { + console.error('[ActivityIdbStorage] loadStartupOps cursor error', cursorRequest.error) + reject(cursorRequest.error) + } + + cursorRequest.onsuccess = () => { + const cursor = cursorRequest.result as IDBCursorWithValue | null + + if (!cursor) { + // Cursor exhausted — build the result object + let totalOps = 0 + for (const [groupKey, rows] of rowsByGroup) { + // Sort descending by timestamp (cursor already delivered newest-first + // within a chain, but groups may interleave so re-sort to be safe) + rows.sort((a, b) => b.timestamp - a.timestamp) + const [accountAddr, chainIdStr] = groupKey.split(':') + if (!result[accountAddr]) result[accountAddr] = {} + result[accountAddr][chainIdStr] = rows.map((r) => deserializeOp(r.serializedOp)) + totalOps += rows.length + } + console.log( + `[ActivityIdbStorage] loadStartupOps complete: ${totalRecords} records, ${totalOps} ops loaded` + ) + resolve(result) + return + } + + const row = cursor.value as IdbAccountOpRow + totalRecords++ + + const groupKey = `${row.accountAddr}:${row.chainId}` + const isPending = + row.status === AccountOpStatus.BroadcastedButNotConfirmed || + row.status === AccountOpStatus.Pending + + if (isPending) { + // Always include pending ops — just collect and advance + if (!rowsByGroup.has(groupKey)) rowsByGroup.set(groupKey, []) + rowsByGroup.get(groupKey)!.push(row) + cursor.continue() + return + } + + // Finalized op + const count = finalizedCount.get(groupKey) ?? 0 + if (count < STARTUP_RECENT_OPS_LIMIT) { + if (!rowsByGroup.has(groupKey)) rowsByGroup.set(groupKey, []) + rowsByGroup.get(groupKey)!.push(row) + finalizedCount.set(groupKey, count + 1) + } + // Always advance without a key argument — cursor.continue(key) in 'prev' + // direction requires the key to be strictly less than the current position, + // so the chainId + RANGE_HIGH skip trick only works in 'next' direction. + // Saturated groups are skipped in JS above; we still iterate their rows. + cursor.continue() + } + }) + } + + /** + * Fetch all ops for a specific (account, chainId) pair (full history, no limit). + * Used for lazy-loading older history during pagination. + * Returns undefined if no ops found (matches existing caller checks). + */ + async getOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string + ): Promise { + await this.init() + + const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readonly') + const store = tx.objectStore(this.config.storeName) + + // Bound on [accountAddr, chainIdStr, ''] → [accountAddr, chainIdStr, RANGE_HIGH] + const range = IDBKeyRange.bound( + [accountAddr, chainIdStr, ''], + [accountAddr, chainIdStr, RANGE_HIGH] + ) + const request = store.getAll(range) + + request.onerror = () => { + console.error( + `ActivityIdbStorage: Failed to get ops for ${accountAddr}:${chainIdStr}`, + request.error + ) + reject(request.error) + } + + request.onsuccess = () => { + const rows = (request.result || []) as IdbAccountOpRow[] + if (rows.length === 0) { + console.log( + `[ActivityIdbStorage] getOpsForAccountAndChain ${accountAddr}:${chainIdStr} - found 0 ops` + ) + resolve(undefined) + return + } + // Sort descending by timestamp + rows.sort((a, b) => b.timestamp - a.timestamp) + const ops = rows.map((r) => deserializeOp(r.serializedOp)) + console.log( + `[ActivityIdbStorage] getOpsForAccountAndChain ${accountAddr}:${chainIdStr} - found ${ops.length} ops` + ) + resolve(ops) + } + }) + } + + /** + * Write ops for a single (account, chainId) pair. + * Deletes existing rows for this pair first, then inserts new ones. + */ + async putOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string, + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + ): Promise { + await this.init() + + const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readwrite') + const store = tx.objectStore(this.config.storeName) + + // Delete all existing rows for this (account, chain) + const deleteRange = IDBKeyRange.bound( + [accountAddr, chainIdStr, ''], + [accountAddr, chainIdStr, RANGE_HIGH] + ) + store.delete(deleteRange) + + // Insert each op as an individual row + for (const op of ops) { + const row = this.#opToRow(accountAddr, chainIdStr, op) + store.put(row) + } + + tx.onerror = () => { + console.error( + `ActivityIdbStorage: Failed to put ops for ${accountAddr}:${chainIdStr}`, + tx.error + ) + reject(tx.error) + } + + tx.oncomplete = () => { + console.log( + `[ActivityIdbStorage] putOpsForAccountAndChain ${accountAddr}:${chainIdStr} - wrote ${ops.length} ops` + ) + this.checkQuota() + resolve() + } + }) + } + + /** + * Batch write multiple (account, chainId) pairs in a single transaction. + * More efficient than multiple individual puts. + */ + async putMultiple( + records: Array<{ + accountAddr: string + chainId: bigint | string + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + }> + ): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readwrite') + const store = tx.objectStore(this.config.storeName) + + for (const record of records) { + const chainIdStr = + typeof record.chainId === 'bigint' ? record.chainId.toString() : record.chainId + + // Delete existing rows for this (account, chain) + const deleteRange = IDBKeyRange.bound( + [record.accountAddr, chainIdStr, ''], + [record.accountAddr, chainIdStr, RANGE_HIGH] + ) + store.delete(deleteRange) + + // Insert individual rows + for (const op of record.ops) { + const row = this.#opToRow(record.accountAddr, chainIdStr, op) + store.put(row) + } + } + + tx.onerror = () => { + console.error('ActivityIdbStorage: Batch put failed', tx.error) + reject(tx.error) + } + + tx.oncomplete = () => { + const totalOps = records.reduce((sum, r) => sum + r.ops.length, 0) + console.log( + `[ActivityIdbStorage] putMultiple complete - wrote ${records.length} records (${totalOps} ops)` + ) + this.checkQuota() + resolve() + } + }) + } + + /** + * Delete all ops for an account across all chains. + */ + async deleteAccount(accountAddr: string): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readwrite') + const store = tx.objectStore(this.config.storeName) + + // IDB compound key range: all keys where first component === accountAddr + const range = IDBKeyRange.bound([accountAddr, '', ''], [accountAddr, RANGE_HIGH, RANGE_HIGH]) + const request = store.delete(range) + + request.onerror = () => { + console.error( + `ActivityIdbStorage: Failed to delete account ${accountAddr}`, + request.error + ) + reject(request.error) + } + + tx.oncomplete = () => { + console.log(`[ActivityIdbStorage] deleteAccount ${accountAddr} - deleted all rows`) + resolve() + } + + tx.onerror = () => { + console.error( + `ActivityIdbStorage: Failed to delete account ${accountAddr} during transaction`, + tx.error + ) + reject(tx.error) + } + }) + } + + /** + * One-time migration: import all ops from chrome.storage.local into IDB. + * After successful import, the caller should remove the key from chrome.storage.local. + */ + async migrateFromStorage(data: InternalAccountsOps): Promise { + await this.init() + + const records: Array<{ + accountAddr: string + chainId: string + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + }> = [] + let totalOps = 0 + + for (const [accountAddr, chainMap] of Object.entries(data)) { + for (const [chainIdString, ops] of Object.entries(chainMap)) { + records.push({ accountAddr, chainId: chainIdString, ops }) + totalOps += ops.length + } + } + + console.log( + `[ActivityIdbStorage] migrateFromStorage - importing ${records.length} records with ${totalOps} total ops` + ) + + return this.putMultiple(records) + } + + /** + * Check if IDB has any data (used to detect if migration is needed). + */ + async isEmpty(): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readonly') + const store = tx.objectStore(this.config.storeName) + const request = store.count() + + request.onerror = () => { + console.error('ActivityIdbStorage: Failed to check if empty', request.error) + reject(request.error) + } + + request.onsuccess = () => { + const empty = request.result === 0 + console.log( + `[ActivityIdbStorage] isEmpty check - ${empty ? 'empty' : `${request.result} records found`}` + ) + resolve(empty) + } + }) + } + + /** + * Debug: dump all data in the database without trimming (for testing only). + */ + async debugDumpAll(): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readonly') + const store = tx.objectStore(this.config.storeName) + const request = store.getAll() + + request.onerror = () => reject(request.error) + request.onsuccess = () => { + const rows = (request.result || []) as IdbAccountOpRow[] + const result: InternalAccountsOps = {} + + for (const row of rows) { + if (!result[row.accountAddr]) result[row.accountAddr] = {} + if (!result[row.accountAddr][row.chainId]) result[row.accountAddr][row.chainId] = [] + result[row.accountAddr][row.chainId].push(deserializeOp(row.serializedOp)) + } + + // Sort each group descending by timestamp + for (const chainMap of Object.values(result)) { + for (const ops of Object.values(chainMap)) { + ops.sort((a, b) => b.timestamp - a.timestamp) + } + } + + for (const [accountAddr, chainMap] of Object.entries(result)) { + for (const [chainId, ops] of Object.entries(chainMap)) { + console.log(`[ActivityIdbStorage] DEBUG: ${accountAddr}:${chainId} = ${ops.length} ops`) + } + } + + console.log(`[ActivityIdbStorage] DEBUG dump complete: ${rows.length} rows`) + resolve(result) + } + }) + } + + // ────────────────────────────────────────────────────────────────────────────── + // Private helpers + // ────────────────────────────────────────────────────────────────────────────── + + #opToRow( + accountAddr: string, + chainIdStr: string, + op: SubmittedAccountOp | SubmittedAccountOpLike + ): IdbAccountOpRow { + return { + accountAddr, + chainId: chainIdStr, + id: (op as any).id as string, + timestamp: (op as any).timestamp as number, + status: (op as any).status as AccountOpStatus, + serializedOp: serializeOp(op) + } + } +} diff --git a/src/services/storage/baseIdbStore.ts b/src/services/storage/baseIdbStore.ts new file mode 100644 index 0000000000..b0f811fe71 --- /dev/null +++ b/src/services/storage/baseIdbStore.ts @@ -0,0 +1,163 @@ +/** + * Configuration for an IndexedDB store. + */ +export interface IdbStoreConfig { + dbName: string + storeName: string + keyPath: string | string[] + dbVersion?: number +} + +/** + * Base IndexedDB storage service. + * Handles common operations: open/upgrade, transactions, error handling. + * Extend this for specific data types. + */ +export class BaseIdbStore { + protected db: IDBDatabase | null = null + protected initPromise: Promise | null = null + protected config: IdbStoreConfig + + constructor(config: IdbStoreConfig) { + this.config = { + dbVersion: 1, + ...config + } + } + + protected async init(): Promise { + if (this.db) return + if (this.initPromise) return this.initPromise + + this.initPromise = this.doInit() + await this.initPromise + } + + protected async doInit(): Promise { + return new Promise((resolve, reject) => { + const request = indexedDB.open(this.config.dbName, this.config.dbVersion) + + request.onerror = () => { + console.error(`BaseIdbStore: Failed to open ${this.config.dbName}`, request.error) + reject(request.error) + } + + request.onsuccess = () => { + this.db = request.result + console.log( + `[BaseIdbStore] Opened ${this.config.dbName} v${this.config.dbVersion} with store "${this.config.storeName}"` + ) + resolve() + } + + request.onupgradeneeded = (event) => { + const db = (event.target as IDBOpenDBRequest).result + if (!db.objectStoreNames.contains(this.config.storeName)) { + db.createObjectStore(this.config.storeName, { keyPath: this.config.keyPath }) + console.log(`[BaseIdbStore] Created store "${this.config.storeName}"`) + } + } + }) + } + + protected async getAll(): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readonly') + const store = tx.objectStore(this.config.storeName) + const request = store.getAll() + + request.onerror = () => reject(request.error) + request.onsuccess = () => resolve(request.result as T[]) + }) + } + + protected async put(record: any): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readwrite') + const store = tx.objectStore(this.config.storeName) + store.put(record) + + tx.onerror = () => reject(tx.error) + tx.oncomplete = () => resolve() + }) + } + + protected async putMultiple(records: any[]): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readwrite') + const store = tx.objectStore(this.config.storeName) + + records.forEach((record) => store.put(record)) + + tx.onerror = () => reject(tx.error) + tx.oncomplete = () => resolve() + }) + } + + protected async delete(key: any): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readwrite') + const store = tx.objectStore(this.config.storeName) + const request = store.delete(key) + + request.onerror = () => reject(request.error) + tx.oncomplete = () => resolve() + }) + } + + protected async clear(): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readwrite') + const store = tx.objectStore(this.config.storeName) + const request = store.clear() + + request.onerror = () => reject(request.error) + tx.oncomplete = () => resolve() + }) + } + + protected async isEmpty(): Promise { + await this.init() + + return new Promise((resolve, reject) => { + const tx = this.db!.transaction([this.config.storeName], 'readonly') + const store = tx.objectStore(this.config.storeName) + const request = store.count() + + request.onerror = () => reject(request.error) + request.onsuccess = () => resolve(request.result === 0) + }) + } + + protected checkQuota(): void { + if (!navigator.storage?.estimate) return + + navigator.storage.estimate().then((estimate) => { + if (!estimate.quota || !estimate.usage) return + + const percentUsed = (estimate.usage / estimate.quota) * 100 + const usedMB = (estimate.usage / 1024 / 1024).toFixed(1) + const quotaMB = (estimate.quota / 1024 / 1024).toFixed(1) + + console.log( + `[BaseIdbStore] ${this.config.storeName} quota: ${usedMB}MB / ${quotaMB}MB (${percentUsed.toFixed(1)}%)` + ) + + if (percentUsed > 80) { + console.warn( + `[BaseIdbStore] ${this.config.storeName} quota usage high (${percentUsed.toFixed(1)}%)` + ) + } + }) + } +} From 8c58f67bd3e94953a7e61ce5e5e812945832e4c0 Mon Sep 17 00:00:00 2001 From: Gergana Mihaylova Date: Fri, 19 Jun 2026 18:19:38 +0300 Subject: [PATCH 2/5] Update: simplification of activity idb --- package-lock.json | 15 +- package.json | 1 + src/controllers/activity/activity.ts | 7 +- src/interfaces/activity.ts | 66 ++++ src/interfaces/activityIdb.ts | 68 ---- src/services/storage/activityIdb.test.ts | 421 +++++++++++++++++++++++ src/services/storage/activityIdb.ts | 222 ++---------- 7 files changed, 540 insertions(+), 260 deletions(-) delete mode 100644 src/interfaces/activityIdb.ts create mode 100644 src/services/storage/activityIdb.test.ts diff --git a/package-lock.json b/package-lock.json index 1ce9782884..b75a246577 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ambire-common", - "version": "2.99.0", + "version": "2.101.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ambire-common", - "version": "2.99.0", + "version": "2.101.0", "dependencies": { "@ambire/signature-validator": "^1.5.0", "@lifi/types": "^17.7.1", @@ -56,6 +56,7 @@ "eslint-plugin-prettier": "5.2.1", "eslint-plugin-react": "7.37.5", "eslint-plugin-react-hooks": "7.1.1", + "fake-indexeddb": "^6.2.5", "globals": "15.9.0", "hardhat": "2.24.1", "hardhat-gas-reporter": "1.0.10", @@ -8832,6 +8833,16 @@ "node": "^14.15.0 || ^16.10.0 || >=18.0.0" } }, + "node_modules/fake-indexeddb": { + "version": "6.2.5", + "resolved": "https://registry.npmjs.org/fake-indexeddb/-/fake-indexeddb-6.2.5.tgz", + "integrity": "sha512-CGnyrvbhPlWYMngksqrSSUT1BAVP49dZocrHuK0SvtR0D5TMs5wP0o3j7jexDJW01KSadjBp1M/71o/KR3nD1w==", + "dev": true, + "license": "Apache-2.0", + "engines": { + "node": ">=18" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", diff --git a/package.json b/package.json index 2a4a26bc92..038d345837 100644 --- a/package.json +++ b/package.json @@ -75,6 +75,7 @@ "eslint-plugin-prettier": "5.2.1", "eslint-plugin-react": "7.37.5", "eslint-plugin-react-hooks": "7.1.1", + "fake-indexeddb": "^6.2.5", "globals": "15.9.0", "hardhat": "2.24.1", "hardhat-gas-reporter": "1.0.10", diff --git a/src/controllers/activity/activity.ts b/src/controllers/activity/activity.ts index 7d4436fba6..79a67d0087 100644 --- a/src/controllers/activity/activity.ts +++ b/src/controllers/activity/activity.ts @@ -9,8 +9,7 @@ import { import { ActivityIdbStorage } from '@/services/storage/activityIdb' import { Account, AccountId, IAccountsController } from '../../interfaces/account' -import { IActivityController } from '../../interfaces/activity' -import { IActivityIdbStorage } from '../../interfaces/activityIdb' +import { IActivityController, IActivityIdbStorage, InternalAccountsOps } from '../../interfaces/activity' import { Banner } from '../../interfaces/banner' import { IEventEmitterRegistryController } from '../../interfaces/eventEmitter' import { Fetch } from '../../interfaces/fetch' @@ -86,10 +85,6 @@ export interface Filters { identifiedBy?: AccountOpIdentifiedBy } -export interface InternalAccountsOps { - // account => network => SubmittedAccountOp[] - [key: string]: { [key: string]: SubmittedAccountOp[] } -} export interface ExternalAccountOps { [account: string]: { [network: string]: SubmittedAccountOpLike[] } diff --git a/src/interfaces/activity.ts b/src/interfaces/activity.ts index d090771981..36ad04274d 100644 --- a/src/interfaces/activity.ts +++ b/src/interfaces/activity.ts @@ -1,5 +1,71 @@ +import { SubmittedAccountOp, SubmittedAccountOpLike } from '../libs/accountOp/submittedAccountOp' import { ControllerInterface } from './controller' export type IActivityController = ControllerInterface< InstanceType > + +export interface InternalAccountsOps { + // account => network => SubmittedAccountOp[] + [key: string]: { [key: string]: SubmittedAccountOp[] } +} + +/** + * Interface for IndexedDB-backed storage of account operations. + * Provides minimal, targeted reads/writes compared to full blob serialization. + */ +export interface IActivityIdbStorage { + /** + * Load minimal startup dataset: all pending ops + 20 most recent per (account, chainId) + * Used during ActivityController initialization. + */ + loadStartupOps(): Promise + + /** + * Fetch full history for a specific (account, chainId) pair. + * Used for lazy-loading older history during pagination. + */ + getOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string + ): Promise + + /** + * Write ops for a single (account, chainId) pair. + * Accepts both internal and external account ops. + */ + putOpsForAccountAndChain( + accountAddr: string, + chainId: bigint | string, + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + ): Promise + + /** + * Batch write multiple (account, chainId) records in a single transaction. + * More efficient than multiple individual puts. + * Accepts both internal and external account ops. + */ + putMultiple( + records: Array<{ + accountAddr: string + chainId: bigint | string + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + }> + ): Promise + + /** + * Delete all ops for an account across all chains. + */ + deleteAccount(accountAddr: string): Promise + + /** + * One-time migration: import all ops from chrome.storage.local into IDB. + * After successful import, the caller should remove the key from chrome.storage.local. + */ + migrateFromStorage(data: InternalAccountsOps): Promise + + /** + * Check if IDB has any data (used to detect if migration is needed). + */ + isEmpty(): Promise +} diff --git a/src/interfaces/activityIdb.ts b/src/interfaces/activityIdb.ts deleted file mode 100644 index a59f3bdecf..0000000000 --- a/src/interfaces/activityIdb.ts +++ /dev/null @@ -1,68 +0,0 @@ -import { SubmittedAccountOp, SubmittedAccountOpLike } from '../libs/accountOp/submittedAccountOp' - -/** - * Interface for IndexedDB-backed storage of account operations. - * Provides minimal, targeted reads/writes compared to full blob serialization. - */ -export interface IActivityIdbStorage { - /** - * Load minimal startup dataset: all pending ops + 20 most recent per (account, chainId) - * Used during ActivityController initialization. - */ - loadStartupOps(): Promise - - /** - * Fetch full history for a specific (account, chainId) pair. - * Used for lazy-loading older history during pagination. - */ - getOpsForAccountAndChain( - accountAddr: string, - chainId: bigint | string - ): Promise - - /** - * Write ops for a single (account, chainId) pair. - * Accepts both internal and external account ops. - */ - putOpsForAccountAndChain( - accountAddr: string, - chainId: bigint | string, - ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] - ): Promise - - /** - * Batch write multiple (account, chainId) records in a single transaction. - * More efficient than multiple individual puts. - * Accepts both internal and external account ops. - */ - putMultiple( - records: Array<{ - accountAddr: string - chainId: bigint | string - ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] - }> - ): Promise - - /** - * Delete all ops for an account across all chains. - */ - deleteAccount(accountAddr: string): Promise - - /** - * One-time migration: import all ops from chrome.storage.local into IDB. - * After successful import, the caller should remove the key from chrome.storage.local. - */ - migrateFromStorage(data: InternalAccountsOps): Promise - - /** - * Check if IDB has any data (used to detect if migration is needed). - */ - isEmpty(): Promise -} - -/** - * Re-export for convenience (mirrors ActivityController's internal structure) - */ -export interface InternalAccountsOps { - [accountAddr: string]: { [chainIdString: string]: SubmittedAccountOp[] } -} diff --git a/src/services/storage/activityIdb.test.ts b/src/services/storage/activityIdb.test.ts new file mode 100644 index 0000000000..7f45e024ea --- /dev/null +++ b/src/services/storage/activityIdb.test.ts @@ -0,0 +1,421 @@ +import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' + +import { beforeEach, describe, expect, test } from '@jest/globals' + +import { SubmittedAccountOpLike } from '../../libs/accountOp/submittedAccountOp' +import { AccountOpStatus } from '../../libs/accountOp/types' +import { ActivityIdbStorage } from './activityIdb' + +// ───────────────────────────────────────────────────────────────────────────── +// Test constants +// ───────────────────────────────────────────────────────────────────────────── + +const ACC_A = '0xAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA' +const ACC_B = '0xBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBBB' +const CHAIN_1 = 1n +const CHAIN_137 = 137n + +// ───────────────────────────────────────────────────────────────────────────── +// Helpers +// ───────────────────────────────────────────────────────────────────────────── + +function makeOp( + id: string, + accountAddr: string, + chainId: bigint, + status: AccountOpStatus, + timestamp: number +): SubmittedAccountOpLike { + return { + id, + accountAddr, + chainId, + calls: [], + gasFeePayment: null as any, + status, + timestamp, + identifiedBy: { type: 'Transaction', identifier: `0x${id}` } + } as SubmittedAccountOpLike +} + +beforeEach(() => { + // Give each test a completely isolated IndexedDB environment. + global.indexedDB = new IDBFactory() + global.IDBKeyRange = IDBKeyRange + // checkQuota() reads navigator.storage — stub it to avoid ReferenceError in Node. + ;(global as any).navigator = {} +}) + +// ───────────────────────────────────────────────────────────────────────────── +// Tests +// ───────────────────────────────────────────────────────────────────────────── + +describe('ActivityIdbStorage', () => { + describe('isEmpty', () => { + test('returns true on a fresh store', async () => { + const store = new ActivityIdbStorage() + expect(await store.isEmpty()).toBe(true) + }) + + test('returns false after data is written', async () => { + const store = new ActivityIdbStorage() + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + expect(await store.isEmpty()).toBe(false) + }) + }) + + describe('putOpsForAccountAndChain + getOpsForAccountAndChain', () => { + test('stores ops and returns them sorted by timestamp descending', async () => { + const store = new ActivityIdbStorage() + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000), + makeOp('op-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 3000), + makeOp('op-3', ACC_A, CHAIN_1, AccountOpStatus.Success, 2000) + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result?.map((op) => op.id)).toEqual(['op-2', 'op-3', 'op-1']) + }) + + test('returns undefined when no ops exist for the pair', async () => { + const store = new ActivityIdbStorage() + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_1)).toBeUndefined() + }) + + test('accepts bigint chainId — retrieve with bigint or equivalent string', async () => { + const store = new ActivityIdbStorage() + await store.putOpsForAccountAndChain(ACC_A, CHAIN_137, [ + makeOp('op-1', ACC_A, CHAIN_137, AccountOpStatus.Success, 1000) + ]) + + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_137)).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_A, '137')).toHaveLength(1) + }) + + test('replaces existing ops on second write to the same pair', async () => { + const store = new ActivityIdbStorage() + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('old', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('new-1', ACC_A, CHAIN_1, AccountOpStatus.Failure, 2000), + makeOp('new-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 3000) + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(2) + expect(result?.map((op) => op.id)).not.toContain('old') + }) + + test('different chains for the same account are stored independently', async () => { + const store = new ActivityIdbStorage() + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('chain1-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) + ]) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_137, [ + makeOp('chain137-op', ACC_A, CHAIN_137, AccountOpStatus.Success, 2000) + ]) + + expect((await store.getOpsForAccountAndChain(ACC_A, CHAIN_1))![0].id).toBe('chain1-op') + expect((await store.getOpsForAccountAndChain(ACC_A, CHAIN_137))![0].id).toBe('chain137-op') + }) + }) + + describe('putMultiple', () => { + test('writes all (account, chainId) pairs atomically', async () => { + const store = new ActivityIdbStorage() + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('a1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + }, + { + accountAddr: ACC_A, + chainId: CHAIN_137, + ops: [makeOp('a137', ACC_A, CHAIN_137, AccountOpStatus.Success, 2)] + }, + { + accountAddr: ACC_B, + chainId: CHAIN_1, + ops: [makeOp('b1', ACC_B, CHAIN_1, AccountOpStatus.Success, 3)] + } + ]) + + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_1)).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_137)).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_B, CHAIN_1)).toHaveLength(1) + }) + + test('replaces existing ops per pair', async () => { + const store = new ActivityIdbStorage() + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('old', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + } + ]) + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('new', ACC_A, CHAIN_1, AccountOpStatus.Success, 2)] + } + ]) + + const result = await store.getOpsForAccountAndChain(ACC_A, CHAIN_1) + expect(result).toHaveLength(1) + expect(result![0].id).toBe('new') + }) + }) + + describe('deleteAccount', () => { + test('removes all chains for the given account', async () => { + const store = new ActivityIdbStorage() + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('a1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + }, + { + accountAddr: ACC_A, + chainId: CHAIN_137, + ops: [makeOp('a137', ACC_A, CHAIN_137, AccountOpStatus.Success, 2)] + } + ]) + + await store.deleteAccount(ACC_A) + + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_1)).toBeUndefined() + expect(await store.getOpsForAccountAndChain(ACC_A, CHAIN_137)).toBeUndefined() + }) + + test('does not affect other accounts', async () => { + const store = new ActivityIdbStorage() + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('a', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + }, + { + accountAddr: ACC_B, + chainId: CHAIN_1, + ops: [makeOp('b', ACC_B, CHAIN_1, AccountOpStatus.Success, 2)] + } + ]) + + await store.deleteAccount(ACC_A) + + expect(await store.getOpsForAccountAndChain(ACC_B, CHAIN_1)).toHaveLength(1) + }) + + test('is a no-op when the account has no ops', async () => { + const store = new ActivityIdbStorage() + await expect(store.deleteAccount(ACC_A)).resolves.not.toThrow() + }) + }) + + describe('migrateFromStorage', () => { + test('imports all ops from InternalAccountsOps format', async () => { + const store = new ActivityIdbStorage() + await store.migrateFromStorage({ + [ACC_A]: { + '1': [makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1000) as any], + '137': [makeOp('op-2', ACC_A, CHAIN_137, AccountOpStatus.Failure, 2000) as any] + }, + [ACC_B]: { + '1': [makeOp('op-3', ACC_B, CHAIN_1, AccountOpStatus.Success, 3000) as any] + } + }) + + expect(await store.getOpsForAccountAndChain(ACC_A, '1')).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_A, '137')).toHaveLength(1) + expect(await store.getOpsForAccountAndChain(ACC_B, '1')).toHaveLength(1) + }) + + test('preserves op ids and timestamps after migration', async () => { + const store = new ActivityIdbStorage() + await store.migrateFromStorage({ + [ACC_A]: { + '1': [makeOp('migrate-op', ACC_A, CHAIN_1, AccountOpStatus.Success, 42000) as any] + } + }) + + const result = await store.getOpsForAccountAndChain(ACC_A, '1') + expect(result![0].id).toBe('migrate-op') + expect(result![0].timestamp).toBe(42000) + }) + }) + + describe('loadStartupOps', () => { + test('returns an empty object for an empty store', async () => { + const store = new ActivityIdbStorage() + expect(await store.loadStartupOps()).toEqual({}) + }) + + test('returns all pending ops regardless of how many there are', async () => { + const store = new ActivityIdbStorage() + // 25 pending — more than the 20-op finalized limit + const ops = Array.from({ length: 25 }, (_, i) => + makeOp(`op-${i}`, ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, i * 100) + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, ops) + + const result = await store.loadStartupOps() + expect(result[ACC_A]?.['1']).toHaveLength(25) + }) + + test('limits finalized ops to 20 per (account, chainId) group', async () => { + const store = new ActivityIdbStorage() + const ops = Array.from({ length: 25 }, (_, i) => + makeOp(`op-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Success, i * 100) + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, ops) + + const result = await store.loadStartupOps() + expect(result[ACC_A]?.['1']).toHaveLength(20) + }) + + test('always includes pending ops even when the finalized limit is already reached', async () => { + const store = new ActivityIdbStorage() + const finalized = Array.from({ length: 20 }, (_, i) => + makeOp(`fin-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Success, i * 10) + ) + const pending = Array.from({ length: 3 }, (_, i) => + makeOp( + `pend-${i}`, + ACC_A, + CHAIN_1, + AccountOpStatus.BroadcastedButNotConfirmed, + 1000 + i + ) + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [...finalized, ...pending]) + + const result = await store.loadStartupOps() + const ids = result[ACC_A]!['1'].map((op) => op.id) + expect(ids.filter((id) => id.startsWith('pend-'))).toHaveLength(3) + expect(ids.filter((id) => id.startsWith('fin-'))).toHaveLength(20) + }) + + test('returns ops sorted by timestamp descending within each group', async () => { + const store = new ActivityIdbStorage() + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 100), + makeOp('op-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 300), + makeOp('op-3', ACC_A, CHAIN_1, AccountOpStatus.Success, 200) + ]) + + const result = await store.loadStartupOps() + expect(result[ACC_A]!['1'].map((op) => op.timestamp)).toEqual([300, 200, 100]) + }) + + test('handles multiple accounts and chains independently', async () => { + const store = new ActivityIdbStorage() + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('a1', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + }, + { + accountAddr: ACC_A, + chainId: CHAIN_137, + ops: [makeOp('a137', ACC_A, CHAIN_137, AccountOpStatus.Success, 2)] + }, + { + accountAddr: ACC_B, + chainId: CHAIN_1, + ops: [makeOp('b1', ACC_B, CHAIN_1, AccountOpStatus.Success, 3)] + } + ]) + + const result = await store.loadStartupOps() + expect(result[ACC_A]?.['1']).toHaveLength(1) + expect(result[ACC_A]?.['137']).toHaveLength(1) + expect(result[ACC_B]?.['1']).toHaveLength(1) + }) + + test('selects the 20 newest finalized ops (highest timestamps) per group', async () => { + const store = new ActivityIdbStorage() + // ops 0-24 — op with index 24 has the highest timestamp + const ops = Array.from({ length: 25 }, (_, i) => + makeOp(`op-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Success, i * 100) + ) + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, ops) + + const result = await store.loadStartupOps() + const timestamps = result[ACC_A]!['1'].map((op) => op.timestamp) + // Expect 2400, 2300, ..., 500 (the 20 newest) + expect(Math.min(...timestamps)).toBe(500) + expect(Math.max(...timestamps)).toBe(2400) + }) + }) + + describe('debugDumpAll', () => { + test('returns all stored ops grouped by account and chain', async () => { + const store = new ActivityIdbStorage() + await store.putMultiple([ + { + accountAddr: ACC_A, + chainId: CHAIN_1, + ops: [makeOp('a', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] + }, + { + accountAddr: ACC_B, + chainId: CHAIN_137, + ops: [makeOp('b', ACC_B, CHAIN_137, AccountOpStatus.Success, 2)] + } + ]) + + const dump = await store.debugDumpAll() + expect(dump[ACC_A]?.['1']).toHaveLength(1) + expect(dump[ACC_B]?.['137']).toHaveLength(1) + }) + + test('returns an empty object for an empty store', async () => { + const store = new ActivityIdbStorage() + expect(await store.debugDumpAll()).toEqual({}) + }) + + test('returns ops sorted by timestamp descending', async () => { + const store = new ActivityIdbStorage() + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ + makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 100), + makeOp('op-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 300) + ]) + + const dump = await store.debugDumpAll() + expect(dump[ACC_A]!['1'].map((op) => op.timestamp)).toEqual([300, 100]) + }) + }) + + describe('bigint serialization roundtrip', () => { + test('chainId and nonce survive serialize → store → deserialize', async () => { + const store = new ActivityIdbStorage() + const op = { + id: 'bigint-op', + accountAddr: ACC_A, + chainId: CHAIN_1, + nonce: 42n, + calls: [], + gasFeePayment: null, + status: AccountOpStatus.Success, + timestamp: 1000, + identifiedBy: { type: 'Transaction', identifier: '0xhash' } + } as unknown as SubmittedAccountOpLike + + await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [op]) + const [retrieved] = (await store.getOpsForAccountAndChain(ACC_A, CHAIN_1))! + + expect(retrieved.chainId).toBe(CHAIN_1) + expect((retrieved as any).nonce).toBe(42n) + }) + }) + +}) diff --git a/src/services/storage/activityIdb.ts b/src/services/storage/activityIdb.ts index 64e137a660..61659b25a9 100644 --- a/src/services/storage/activityIdb.ts +++ b/src/services/storage/activityIdb.ts @@ -1,10 +1,9 @@ +import { IActivityIdbStorage, InternalAccountsOps } from '../../interfaces/activity' import { SubmittedAccountOp, SubmittedAccountOpLike } from '../../libs/accountOp/submittedAccountOp' -import { IActivityIdbStorage, InternalAccountsOps } from '../../interfaces/activityIdb' -import { BaseIdbStore } from './baseIdbStore' import { AccountOpStatus } from '../../libs/accountOp/types' +import { BaseIdbStore } from './baseIdbStore' const STARTUP_RECENT_OPS_LIMIT = 20 -const MAX_OPS_PER_ACCOUNT_CHAIN = 1000 // Known bigint field names — used by the JSON reviver to restore bigint values after deserialization const BIGINT_FIELDS = new Set([ @@ -30,10 +29,12 @@ function serializeOp(op: SubmittedAccountOp | SubmittedAccountOpLike): string { function deserializeOp(serialized: string): SubmittedAccountOp { return JSON.parse(serialized, (key, value) => { - if (BIGINT_FIELDS.has(key) && typeof value === 'string' && /^\d+$/.test(value)) { + if (!BIGINT_FIELDS.has(key) || typeof value !== 'string') return value + try { return BigInt(value) + } catch { + return value } - return value }) as SubmittedAccountOp } @@ -51,15 +52,6 @@ interface IdbAccountOpRow { serializedOp: string // JSON.stringify with bigint→string replacement } -// ──────────────────────────────────────────────────────────────────────────────── -// V1 shape — only used during migration in onupgradeneeded -// ──────────────────────────────────────────────────────────────────────────────── -interface V1IdbAccountOpsRecord { - accountAddr: string - chainId: string - ops: any[] -} - // The highest unicode character — used as a range upper bound to select all keys // that start with a given prefix, without matching the prefix itself as a key. const RANGE_HIGH = '￿' @@ -84,10 +76,9 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor } /** - * Override doInit to: - * 1. Create the object store with the v2 compound keyPath - * 2. Add the 'by-account-chain-timestamp' index - * 3. Migrate existing v1 blob records to individual rows when upgrading from v1 + * Override doInit to create the object store with the compound keyPath and + * the 'by-account-chain-timestamp' index. Any existing store from an older + * schema version is dropped — no data migration. */ protected doInit(): Promise { return new Promise((resolve, reject) => { @@ -108,101 +99,16 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor request.onupgradeneeded = (event) => { const db = (event.target as IDBOpenDBRequest).result - const tx = (event.target as IDBOpenDBRequest).transaction! - const oldVersion = event.oldVersion - - if (oldVersion < 1) { - // Fresh install — create store directly at v3 shape - const store = db.createObjectStore(this.config.storeName, { - keyPath: this.config.keyPath - }) - store.createIndex('by-account-chain-timestamp', ['accountAddr', 'chainId', 'timestamp']) - console.log(`[BaseIdbStore] Created store "${this.config.storeName}"`) - return - } - if (oldVersion === 1 || oldVersion === 2) { - // v1→v3: expand blob-per-(account,chain) records into individual rows. - // v2→v3: re-run migration to fix bad chainId data — v2 used op.chainId which - // could be a non-string (bigint, object) depending on serialization history. - // v3 always uses record.chainId (the safe IDB key string) instead. - console.log(`[ActivityIdbStorage] Migrating from v${oldVersion} to v3 schema...`) - - const oldStore = tx.objectStore(this.config.storeName) - const getAllRequest = oldStore.getAll() - - getAllRequest.onsuccess = () => { - const oldRecords = (getAllRequest.result || []) as any[] - - db.deleteObjectStore(this.config.storeName) - const newStore = db.createObjectStore(this.config.storeName, { - keyPath: this.config.keyPath - }) - newStore.createIndex( - 'by-account-chain-timestamp', - ['accountAddr', 'chainId', 'timestamp'] - ) - console.log(`[ActivityIdbStorage] Recreated store for v3`) - - let migratedOps = 0 - for (const record of oldRecords) { - const { accountAddr, chainId } = record - - if (oldVersion === 1) { - // v1 shape: { accountAddr, chainId, ops: any[] } - // Use record.chainId — already the correct string from the v1 IDB key. - // Do NOT use op.chainId — it may be a bigint, object, or custom-serialized value. - const ops: any[] = record.ops || [] - for (const op of ops) { - if (!op?.id) continue - const row: IdbAccountOpRow = { - accountAddr, - chainId, - id: op.id, - timestamp: op.timestamp ?? 0, - status: op.status, - serializedOp: serializeOp(op) - } - newStore.put(row) - migratedOps++ - } - } else { - // v2 shape: already a row — { accountAddr, chainId, id, timestamp, status, serializedOp } - // chainId may be "[object Object]" — fix it using the record key components directly. - // The v2 IDB key was ['accountAddr', 'chainId', 'id'] so the values are on the record. - // If chainId is bad, we have no reliable source for the correct value and must drop the row. - if (!chainId || chainId === '[object Object]') { - console.warn( - `[ActivityIdbStorage] v2→v3: dropping row with bad chainId for ${accountAddr}:${record.id}` - ) - continue - } - const row: IdbAccountOpRow = { - accountAddr, - chainId, - id: record.id, - timestamp: record.timestamp ?? 0, - status: record.status, - serializedOp: record.serializedOp - } - newStore.put(row) - migratedOps++ - } - } - console.log( - `[ActivityIdbStorage] v${oldVersion}→v3 migration complete: ${oldRecords.length} records → ${migratedOps} rows` - ) - } - - getAllRequest.onerror = () => { - console.error( - `[ActivityIdbStorage] v${oldVersion}→v3 migration failed to read old records`, - getAllRequest.error - ) - // Don't reject — let the upgrade complete even if data is lost; - // a broken upgrade prevents the DB from opening at all. - } + if (db.objectStoreNames.contains(this.config.storeName)) { + db.deleteObjectStore(this.config.storeName) } + + const store = db.createObjectStore(this.config.storeName, { + keyPath: this.config.keyPath + }) + store.createIndex('by-account-chain-timestamp', ['accountAddr', 'chainId', 'timestamp']) + console.log(`[BaseIdbStore] Created store "${this.config.storeName}"`) } }) } @@ -331,9 +237,6 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor request.onsuccess = () => { const rows = (request.result || []) as IdbAccountOpRow[] if (rows.length === 0) { - console.log( - `[ActivityIdbStorage] getOpsForAccountAndChain ${accountAddr}:${chainIdStr} - found 0 ops` - ) resolve(undefined) return } @@ -357,43 +260,7 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor chainId: bigint | string, ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] ): Promise { - await this.init() - - const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId - - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readwrite') - const store = tx.objectStore(this.config.storeName) - - // Delete all existing rows for this (account, chain) - const deleteRange = IDBKeyRange.bound( - [accountAddr, chainIdStr, ''], - [accountAddr, chainIdStr, RANGE_HIGH] - ) - store.delete(deleteRange) - - // Insert each op as an individual row - for (const op of ops) { - const row = this.#opToRow(accountAddr, chainIdStr, op) - store.put(row) - } - - tx.onerror = () => { - console.error( - `ActivityIdbStorage: Failed to put ops for ${accountAddr}:${chainIdStr}`, - tx.error - ) - reject(tx.error) - } - - tx.oncomplete = () => { - console.log( - `[ActivityIdbStorage] putOpsForAccountAndChain ${accountAddr}:${chainIdStr} - wrote ${ops.length} ops` - ) - this.checkQuota() - resolve() - } - }) + return this.putMultiple([{ accountAddr, chainId, ops }]) } /** @@ -413,22 +280,9 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor const tx = this.db!.transaction([this.config.storeName], 'readwrite') const store = tx.objectStore(this.config.storeName) - for (const record of records) { - const chainIdStr = - typeof record.chainId === 'bigint' ? record.chainId.toString() : record.chainId - - // Delete existing rows for this (account, chain) - const deleteRange = IDBKeyRange.bound( - [record.accountAddr, chainIdStr, ''], - [record.accountAddr, chainIdStr, RANGE_HIGH] - ) - store.delete(deleteRange) - - // Insert individual rows - for (const op of record.ops) { - const row = this.#opToRow(record.accountAddr, chainIdStr, op) - store.put(row) - } + for (const { accountAddr, chainId, ops } of records) { + const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + this.#writeRecordToStore(store, accountAddr, chainIdStr, ops) } tx.onerror = () => { @@ -489,26 +343,13 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor * After successful import, the caller should remove the key from chrome.storage.local. */ async migrateFromStorage(data: InternalAccountsOps): Promise { - await this.init() - - const records: Array<{ - accountAddr: string - chainId: string - ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] - }> = [] - let totalOps = 0 - - for (const [accountAddr, chainMap] of Object.entries(data)) { - for (const [chainIdString, ops] of Object.entries(chainMap)) { - records.push({ accountAddr, chainId: chainIdString, ops }) - totalOps += ops.length - } - } - + const records = Object.entries(data).flatMap(([accountAddr, chainMap]) => + Object.entries(chainMap).map(([chainId, ops]) => ({ accountAddr, chainId, ops })) + ) + const totalOps = records.reduce((sum, r) => sum + r.ops.length, 0) console.log( `[ActivityIdbStorage] migrateFromStorage - importing ${records.length} records with ${totalOps} total ops` ) - return this.putMultiple(records) } @@ -583,6 +424,19 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor // Private helpers // ────────────────────────────────────────────────────────────────────────────── + #writeRecordToStore( + store: IDBObjectStore, + accountAddr: string, + chainIdStr: string, + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + ): void { + // Delete existing rows for this (account, chain), then insert fresh ones + store.delete(IDBKeyRange.bound([accountAddr, chainIdStr, ''], [accountAddr, chainIdStr, RANGE_HIGH])) + for (const op of ops) { + store.put(this.#opToRow(accountAddr, chainIdStr, op)) + } + } + #opToRow( accountAddr: string, chainIdStr: string, From d257096ee2112255043dee0a237fd0d85d1c7264 Mon Sep 17 00:00:00 2001 From: Gergana Mihaylova Date: Mon, 29 Jun 2026 18:21:38 +0300 Subject: [PATCH 3/5] Update: remove debug functions and simplify --- package-lock.json | 24 ++- src/controllers/activity/activity.ts | 10 +- src/services/storage/activityIdb.test.ts | 38 ---- src/services/storage/activityIdb.ts | 211 ++++++++--------------- 4 files changed, 99 insertions(+), 184 deletions(-) diff --git a/package-lock.json b/package-lock.json index 9f342e36c5..e651ee0a1b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ambire-common", - "version": "2.102.1", + "version": "2.102.2", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ambire-common", - "version": "2.102.1", + "version": "2.102.2", "dependencies": { "@ambire/signature-validator": "^1.5.0", "@lifi/types": "^17.7.1", @@ -196,6 +196,7 @@ "integrity": "sha512-UlLAnTPrFdNGoFtbSXwcGFQBtQZJCNjaN6hQNP3UPvuNXT1i82N26KL3dZeIpNalWywr9IuQuncaAfUaS1g6sQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@ampproject/remapping": "^2.2.0", "@babel/code-frame": "^7.27.1", @@ -4032,6 +4033,7 @@ "integrity": "sha512-zhOZ4hdRORls31DTOqg+GmEZM0ujly8GGIuRY7t7szEk2zW/arY1qDug/py8AEktT00v5K+b6RvbVog+va51IA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "debug": "^4.1.1", "lodash.isequal": "^4.5.0" @@ -5127,6 +5129,7 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-24.10.11.tgz", "integrity": "sha512-/Af7O8r1frCVgOz0I62jWUtMohJ0/ZQU/ZoketltOJPZpnb17yoNc9BSoVuV9qlaIXJiPNOpsfq4ByFajSArNQ==", "license": "MIT", + "peer": true, "dependencies": { "undici-types": "~7.16.0" } @@ -5249,6 +5252,7 @@ "integrity": "sha512-plR3pp6D+SSUn1HM7xvSkx12/DhoHInI2YF35KAcVFNZvlC0gtrWqx7Qq1oH2Ssgi0vlFRCTbP+DZc7B9+TtsQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@typescript-eslint/scope-manager": "8.59.2", "@typescript-eslint/types": "8.59.2", @@ -5789,6 +5793,7 @@ "integrity": "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg==", "dev": true, "license": "MIT", + "peer": true, "bin": { "acorn": "bin/acorn" }, @@ -6572,6 +6577,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "caniuse-lite": "^1.0.30001726", "electron-to-chromium": "^1.5.173", @@ -6751,6 +6757,7 @@ "resolved": "https://registry.npmjs.org/chai/-/chai-4.3.7.tgz", "integrity": "sha512-HLnAzZ2iupm25PlN0xFreAlBA5zaBSv3og0DdeGA4Ar6h6rJ3A0rolRUKJhSF2V10GZKDgWF/VmAEsNWjCRB+A==", "dev": true, + "peer": true, "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.2", @@ -7631,7 +7638,6 @@ "resolved": "https://registry.npmjs.org/encoding/-/encoding-0.1.13.tgz", "integrity": "sha512-ETBauow1T35Y/WZMkio9jiM0Z5xjHHmJ4XmjZOq1l/dXz3lr2sRn87nJy20RupqSh1F2m3HHPSp8ShIPQJrJ3A==", "optional": true, - "peer": true, "dependencies": { "iconv-lite": "^0.6.2" } @@ -7641,7 +7647,6 @@ "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.6.3.tgz", "integrity": "sha512-4fCk79wshMdzMp2rH06qWrJE4iolqLhCUH+OiuIgU++RB0+94NlDL81atO7GX55uUKueo0txHNtvEyI6D7WdMw==", "optional": true, - "peer": true, "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" }, @@ -7910,6 +7915,7 @@ "integrity": "sha512-XoMjdBOwe/esVgEvLmNsD3IRHkm7fbKIUGvrleloJXUZgDHig2IPWNniv+GwjyJXzuNqVjlr5+4yVUZjycJwfQ==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@eslint-community/eslint-utils": "^4.8.0", "@eslint-community/regexpp": "^4.12.1", @@ -7970,6 +7976,7 @@ "integrity": "sha512-NSWl5BFQWEPi1j4TjVNItzYV7dZXZ+wP6I6ZhrBGpChQhZRUaElihE9uRRkcbRnNb76UMKDF3r+WTmNcGPKsqw==", "dev": true, "license": "MIT", + "peer": true, "bin": { "eslint-config-prettier": "bin/cli.js" }, @@ -8684,6 +8691,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@adraffy/ens-normalize": "1.10.1", "@noble/curves": "1.2.0", @@ -9349,6 +9357,7 @@ "integrity": "sha512-3iwrO2liEGCw1rz/l/mlB1rSNexCc4CFcMj0DlvjXGChzmD3sGUgLwWDOZPf+ya8MEm5ZhO1oprRVmb/wVi0YA==", "dev": true, "license": "MIT", + "peer": true, "dependencies": { "@ethereumjs/util": "^9.1.0", "@ethersproject/abi": "^5.1.2", @@ -15637,6 +15646,7 @@ "integrity": "sha512-i2tDNA0O5IrMO757lfrdQZCc2jPNDVntV0m/+4whiDfWaTKfMNgR7Qz0NAeGz/nRqF4m5/6CLzbP4/liHt12Ew==", "dev": true, "license": "MIT", + "peer": true, "bin": { "prettier": "bin/prettier.cjs" }, @@ -17316,6 +17326,7 @@ "integrity": "sha512-5gTmgEY/sqK6gFXLIsQNH19lWb4ebPDLA4SdLP7dsWkIXHWlG66oPuVvXSGFPppYZz8ZDZq0dYYrbHfBCVUb1Q==", "dev": true, "license": "MIT", + "peer": true, "engines": { "node": ">=12" }, @@ -17503,6 +17514,7 @@ "resolved": "https://registry.npmjs.org/ts-node/-/ts-node-10.9.1.tgz", "integrity": "sha512-NtVysVPkxxrwFGUUxGYhfux8k78pQB3JqYBXlLRZgdGUqTO5wU/UyHop5p70iEbGhB7q5KmiZiU0Y3KlJrScEw==", "dev": true, + "peer": true, "dependencies": { "@cspotcode/source-map-support": "^0.8.0", "@tsconfig/node10": "^1.0.7", @@ -17724,6 +17736,7 @@ "integrity": "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw==", "devOptional": true, "license": "Apache-2.0", + "peer": true, "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" @@ -17931,6 +17944,7 @@ } ], "license": "MIT", + "peer": true, "dependencies": { "@noble/curves": "1.9.1", "@noble/hashes": "1.8.0", @@ -18406,6 +18420,7 @@ "version": "7.4.6", "resolved": "https://registry.npmjs.org/ws/-/ws-7.4.6.tgz", "integrity": "sha512-YmhHDO4MzaDLB+M9ym/mDA5z0naX8j7SIlT8f8z+I0VtzsRbekxEutHSme7NPS2qE8StCYQNUnfWdXta/Yu85A==", + "peer": true, "engines": { "node": ">=8.3.0" }, @@ -18583,6 +18598,7 @@ "integrity": "sha512-ytENFjIJFl2UwYglde2jchW2Hwm4GJFLDiSXWdTrJQBIN9Fcyp7n4DhxJEiWNAJMV1/BqWfW/kkg71UDcHJyTQ==", "devOptional": true, "license": "MIT", + "peer": true, "funding": { "url": "https://github.com/sponsors/colinhacks" } diff --git a/src/controllers/activity/activity.ts b/src/controllers/activity/activity.ts index 424968ed2f..9a2f29fd4a 100644 --- a/src/controllers/activity/activity.ts +++ b/src/controllers/activity/activity.ts @@ -9,7 +9,11 @@ import { import { ActivityIdbStorage } from '@/services/storage/activityIdb' import { Account, AccountId, IAccountsController } from '../../interfaces/account' -import { IActivityController, IActivityIdbStorage, InternalAccountsOps } from '../../interfaces/activity' +import { + IActivityController, + IActivityIdbStorage, + InternalAccountsOps +} from '../../interfaces/activity' import { Banner } from '../../interfaces/banner' import { IEventEmitterRegistryController } from '../../interfaces/eventEmitter' import { Fetch } from '../../interfaces/fetch' @@ -859,6 +863,7 @@ export class ActivityController extends EventEmitter implements IActivityControl await this.#initialLoadPromise const { accountAddr, chainId } = accountOp + console.log(`[ActivityController] addAccountOp: ${accountAddr} chain=${chainId} id=${accountOp.id} status=${accountOp.status} txnId=${accountOp.txnId}`) if (!this.#accountsOps[accountAddr]) this.#accountsOps[accountAddr] = {} if (!this.#accountsOps[accountAddr][chainId.toString()]) @@ -869,6 +874,7 @@ export class ActivityController extends EventEmitter implements IActivityControl trim(this.#accountsOps[accountAddr][chainId.toString()]!) await this.syncFilteredAccountsOps() + this.emitUpdate() // Write to IDB if available, otherwise fallback to chrome.storage.local if (this.#activityIdb) { @@ -885,8 +891,6 @@ export class ActivityController extends EventEmitter implements IActivityControl } else { await this.#storage.set('accountsOps', this.#accountsOps) } - - this.emitUpdate() } async addExternalAccountOp({ diff --git a/src/services/storage/activityIdb.test.ts b/src/services/storage/activityIdb.test.ts index 7f45e024ea..15a7be2a0c 100644 --- a/src/services/storage/activityIdb.test.ts +++ b/src/services/storage/activityIdb.test.ts @@ -357,44 +357,6 @@ describe('ActivityIdbStorage', () => { }) }) - describe('debugDumpAll', () => { - test('returns all stored ops grouped by account and chain', async () => { - const store = new ActivityIdbStorage() - await store.putMultiple([ - { - accountAddr: ACC_A, - chainId: CHAIN_1, - ops: [makeOp('a', ACC_A, CHAIN_1, AccountOpStatus.Success, 1)] - }, - { - accountAddr: ACC_B, - chainId: CHAIN_137, - ops: [makeOp('b', ACC_B, CHAIN_137, AccountOpStatus.Success, 2)] - } - ]) - - const dump = await store.debugDumpAll() - expect(dump[ACC_A]?.['1']).toHaveLength(1) - expect(dump[ACC_B]?.['137']).toHaveLength(1) - }) - - test('returns an empty object for an empty store', async () => { - const store = new ActivityIdbStorage() - expect(await store.debugDumpAll()).toEqual({}) - }) - - test('returns ops sorted by timestamp descending', async () => { - const store = new ActivityIdbStorage() - await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [ - makeOp('op-1', ACC_A, CHAIN_1, AccountOpStatus.Success, 100), - makeOp('op-2', ACC_A, CHAIN_1, AccountOpStatus.Success, 300) - ]) - - const dump = await store.debugDumpAll() - expect(dump[ACC_A]!['1'].map((op) => op.timestamp)).toEqual([300, 100]) - }) - }) - describe('bigint serialization roundtrip', () => { test('chainId and nonce survive serialize → store → deserialize', async () => { const store = new ActivityIdbStorage() diff --git a/src/services/storage/activityIdb.ts b/src/services/storage/activityIdb.ts index 61659b25a9..bba005ed0c 100644 --- a/src/services/storage/activityIdb.ts +++ b/src/services/storage/activityIdb.ts @@ -38,11 +38,6 @@ function deserializeOp(serialized: string): SubmittedAccountOp { }) as SubmittedAccountOp } -/** - * One IDB row per submitted account op (v2 schema). - * keyPath: ['accountAddr', 'chainId', 'id'] - * index 'by-account-chain-timestamp': ['accountAddr', 'chainId', 'timestamp'] - */ interface IdbAccountOpRow { accountAddr: string chainId: string // bigint converted to string for IDB compatibility @@ -52,33 +47,23 @@ interface IdbAccountOpRow { serializedOp: string // JSON.stringify with bigint→string replacement } -// The highest unicode character — used as a range upper bound to select all keys +// The highest BMP Unicode character — used as a range upper bound to select all keys // that start with a given prefix, without matching the prefix itself as a key. -const RANGE_HIGH = '￿' - -/** - * IndexedDB-backed storage for account operations — v2 row-per-op schema. - * - * One IDB record per SubmittedAccountOp, indexed by timestamp so that - * loadStartupOps() can use a cursor to fetch only the needed subset instead of - * reading the full history for every (account, chain). - * - * Extends BaseIdbStore for common IDB operations and error handling. - */ +const RANGE_HIGH = '\uffff' + export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStorage { constructor() { super({ - dbName: 'ambire-activity', + dbName: 'ambire', storeName: 'accountsOps', keyPath: ['accountAddr', 'chainId', 'id'], - dbVersion: 3 + dbVersion: 1 }) } /** - * Override doInit to create the object store with the compound keyPath and - * the 'by-account-chain-timestamp' index. Any existing store from an older - * schema version is dropped — no data migration. + * Override doInit to create the object store with the compound keyPath. + * Any existing store from an older schema version is dropped — no data migration. */ protected doInit(): Promise { return new Promise((resolve, reject) => { @@ -104,10 +89,7 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor db.deleteObjectStore(this.config.storeName) } - const store = db.createObjectStore(this.config.storeName, { - keyPath: this.config.keyPath - }) - store.createIndex('by-account-chain-timestamp', ['accountAddr', 'chainId', 'timestamp']) + db.createObjectStore(this.config.storeName, { keyPath: this.config.keyPath }) console.log(`[BaseIdbStore] Created store "${this.config.storeName}"`) } }) @@ -118,86 +100,48 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor // ────────────────────────────────────────────────────────────────────────────── /** - * Load minimal startup dataset using a cursor on 'by-account-chain-timestamp' - * (newest-first). For each (account, chain) group we collect: - * - All pending ops (BroadcastedButNotConfirmed | Pending) - * - Up to STARTUP_RECENT_OPS_LIMIT finalized ops - * Once a group has enough finalized ops the cursor jumps to the next chain. + * Load minimal startup dataset: all pending ops + up to STARTUP_RECENT_OPS_LIMIT + * finalized ops per (account, chain). Reads all rows then filters in JS. */ async loadStartupOps(): Promise { await this.init() return new Promise((resolve, reject) => { const tx = this.db!.transaction([this.config.storeName], 'readonly') - const store = tx.objectStore(this.config.storeName) - const index = store.index('by-account-chain-timestamp') + const request = tx.objectStore(this.config.storeName).getAll() - // 'prev' gives us newest first within each (accountAddr, chainId) group - const cursorRequest = index.openCursor(null, 'prev') - - const result: InternalAccountsOps = {} - // Tracks how many finalized ops we've collected per group key - const finalizedCount = new Map() - // Accumulates rows before we sort and deserialize at the end - const rowsByGroup = new Map() + request.onerror = () => { + console.error('[ActivityIdbStorage] loadStartupOps error', request.error) + reject(request.error) + } - let totalRecords = 0 + request.onsuccess = () => { + const rows = (request.result as IdbAccountOpRow[]).sort( + (a, b) => b.timestamp - a.timestamp + ) - cursorRequest.onerror = () => { - console.error('[ActivityIdbStorage] loadStartupOps cursor error', cursorRequest.error) - reject(cursorRequest.error) - } + const result: InternalAccountsOps = {} + const finalizedCount = new Map() - cursorRequest.onsuccess = () => { - const cursor = cursorRequest.result as IDBCursorWithValue | null - - if (!cursor) { - // Cursor exhausted — build the result object - let totalOps = 0 - for (const [groupKey, rows] of rowsByGroup) { - // Sort descending by timestamp (cursor already delivered newest-first - // within a chain, but groups may interleave so re-sort to be safe) - rows.sort((a, b) => b.timestamp - a.timestamp) - const [accountAddr, chainIdStr] = groupKey.split(':') - if (!result[accountAddr]) result[accountAddr] = {} - result[accountAddr][chainIdStr] = rows.map((r) => deserializeOp(r.serializedOp)) - totalOps += rows.length - } - console.log( - `[ActivityIdbStorage] loadStartupOps complete: ${totalRecords} records, ${totalOps} ops loaded` - ) - resolve(result) - return - } + for (const row of rows) { + const isPending = + row.status === AccountOpStatus.BroadcastedButNotConfirmed || + row.status === AccountOpStatus.Pending + const groupKey = `${row.accountAddr}:${row.chainId}` + const count = finalizedCount.get(groupKey) ?? 0 - const row = cursor.value as IdbAccountOpRow - totalRecords++ + if (!isPending && count >= STARTUP_RECENT_OPS_LIMIT) continue - const groupKey = `${row.accountAddr}:${row.chainId}` - const isPending = - row.status === AccountOpStatus.BroadcastedButNotConfirmed || - row.status === AccountOpStatus.Pending + if (!result[row.accountAddr]) result[row.accountAddr] = {} + const chainMap = result[row.accountAddr]! + if (!chainMap[row.chainId]) chainMap[row.chainId] = [] + chainMap[row.chainId]!.push(deserializeOp(row.serializedOp)) - if (isPending) { - // Always include pending ops — just collect and advance - if (!rowsByGroup.has(groupKey)) rowsByGroup.set(groupKey, []) - rowsByGroup.get(groupKey)!.push(row) - cursor.continue() - return + if (!isPending) finalizedCount.set(groupKey, count + 1) } - // Finalized op - const count = finalizedCount.get(groupKey) ?? 0 - if (count < STARTUP_RECENT_OPS_LIMIT) { - if (!rowsByGroup.has(groupKey)) rowsByGroup.set(groupKey, []) - rowsByGroup.get(groupKey)!.push(row) - finalizedCount.set(groupKey, count + 1) - } - // Always advance without a key argument — cursor.continue(key) in 'prev' - // direction requires the key to be strictly less than the current position, - // so the chainId + RANGE_HIGH skip trick only works in 'next' direction. - // Saturated groups are skipped in JS above; we still iterate their rows. - cursor.continue() + console.log(`[ActivityIdbStorage] loadStartupOps: loaded from ${rows.length} rows`) + resolve(result) } }) } @@ -282,7 +226,7 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor for (const { accountAddr, chainId, ops } of records) { const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId - this.#writeRecordToStore(store, accountAddr, chainIdStr, ops) + this.#writeRecordToStore(store, accountAddr, chainIdStr, this.#dedupeOpsById(ops)) } tx.onerror = () => { @@ -339,8 +283,8 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor } /** - * One-time migration: import all ops from chrome.storage.local into IDB. - * After successful import, the caller should remove the key from chrome.storage.local. + * One-time migration: import all ops from legacy blob storage into IDB. + * After successful import, the caller should remove the key from legacy storage. */ async migrateFromStorage(data: InternalAccountsOps): Promise { const records = Object.entries(data).flatMap(([accountAddr, chainMap]) => @@ -379,47 +323,6 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor }) } - /** - * Debug: dump all data in the database without trimming (for testing only). - */ - async debugDumpAll(): Promise { - await this.init() - - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readonly') - const store = tx.objectStore(this.config.storeName) - const request = store.getAll() - - request.onerror = () => reject(request.error) - request.onsuccess = () => { - const rows = (request.result || []) as IdbAccountOpRow[] - const result: InternalAccountsOps = {} - - for (const row of rows) { - if (!result[row.accountAddr]) result[row.accountAddr] = {} - if (!result[row.accountAddr][row.chainId]) result[row.accountAddr][row.chainId] = [] - result[row.accountAddr][row.chainId].push(deserializeOp(row.serializedOp)) - } - - // Sort each group descending by timestamp - for (const chainMap of Object.values(result)) { - for (const ops of Object.values(chainMap)) { - ops.sort((a, b) => b.timestamp - a.timestamp) - } - } - - for (const [accountAddr, chainMap] of Object.entries(result)) { - for (const [chainId, ops] of Object.entries(chainMap)) { - console.log(`[ActivityIdbStorage] DEBUG: ${accountAddr}:${chainId} = ${ops.length} ops`) - } - } - - console.log(`[ActivityIdbStorage] DEBUG dump complete: ${rows.length} rows`) - resolve(result) - } - }) - } - // ────────────────────────────────────────────────────────────────────────────── // Private helpers // ────────────────────────────────────────────────────────────────────────────── @@ -431,23 +334,53 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] ): void { // Delete existing rows for this (account, chain), then insert fresh ones - store.delete(IDBKeyRange.bound([accountAddr, chainIdStr, ''], [accountAddr, chainIdStr, RANGE_HIGH])) + store.delete( + IDBKeyRange.bound([accountAddr, chainIdStr, ''], [accountAddr, chainIdStr, RANGE_HIGH]) + ) for (const op of ops) { store.put(this.#opToRow(accountAddr, chainIdStr, op)) } } + #dedupeOpsById( + ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] + ): (SubmittedAccountOp | SubmittedAccountOpLike)[] { + const deduped = new Map() + + for (const op of ops) { + if (typeof op.id !== 'string' || !op.id) { + console.warn('[ActivityIdbStorage] Skipping op without a valid id', op) + continue + } + deduped.set(op.id, op) + } + + return Array.from(deduped.values()) + } + #opToRow( accountAddr: string, chainIdStr: string, op: SubmittedAccountOp | SubmittedAccountOpLike ): IdbAccountOpRow { + if (typeof op.id !== 'string' || !op.id) { + throw new Error('[ActivityIdbStorage] Cannot store op without a valid id') + } + + if (typeof op.timestamp !== 'number') { + throw new Error(`[ActivityIdbStorage] Cannot store op ${op.id} without a valid timestamp`) + } + + if (op.status === undefined) { + throw new Error(`[ActivityIdbStorage] Cannot store op ${op.id} without a valid status`) + } + return { accountAddr, chainId: chainIdStr, - id: (op as any).id as string, - timestamp: (op as any).timestamp as number, - status: (op as any).status as AccountOpStatus, + id: op.id, + timestamp: op.timestamp, + status: op.status, serializedOp: serializeOp(op) } } From fca057fdc1b339ab7a865ac76fec5ce749a0be23 Mon Sep 17 00:00:00 2001 From: Gergana Mihaylova Date: Wed, 8 Jul 2026 16:42:15 +0300 Subject: [PATCH 4/5] Update: integrate idb for account operations and enhance storage management --- package-lock.json | 7 + package.json | 1 + src/controllers/activity/activity.ts | 91 ++--- src/interfaces/activity.ts | 17 + src/services/storage/activityIdb.test.ts | 7 +- src/services/storage/activityIdb.ts | 419 +++++++++++------------ src/services/storage/baseIdbStore.ts | 221 +++++++----- 7 files changed, 402 insertions(+), 361 deletions(-) diff --git a/package-lock.json b/package-lock.json index e651ee0a1b..986be81e3f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -21,6 +21,7 @@ "ethers": "^6.8.0", "events": "^3.3.0", "hash-wasm": "^4.12.0", + "idb": "^8.0.3", "js-yaml": "^4.1.0", "scrypt-js": "^3.0.1", "tldts": "7.0.17", @@ -9898,6 +9899,12 @@ "node": ">=0.10.0" } }, + "node_modules/idb": { + "version": "8.0.3", + "resolved": "https://registry.npmjs.org/idb/-/idb-8.0.3.tgz", + "integrity": "sha512-LtwtVyVYO5BqRvcsKuB2iUMnHwPVByPCXFXOpuU96IZPPoPN6xjOGxZQ74pgSVVLQWtUOYgyeL4GE98BY5D3wg==", + "license": "ISC" + }, "node_modules/ignore": { "version": "5.3.2", "resolved": "https://registry.npmjs.org/ignore/-/ignore-5.3.2.tgz", diff --git a/package.json b/package.json index 38b939c570..99b8752c6a 100644 --- a/package.json +++ b/package.json @@ -28,6 +28,7 @@ "ethers": "^6.8.0", "events": "^3.3.0", "hash-wasm": "^4.12.0", + "idb": "^8.0.3", "js-yaml": "^4.1.0", "scrypt-js": "^3.0.1", "tldts": "7.0.17", diff --git a/src/controllers/activity/activity.ts b/src/controllers/activity/activity.ts index 9a2f29fd4a..6e8648d0e4 100644 --- a/src/controllers/activity/activity.ts +++ b/src/controllers/activity/activity.ts @@ -339,7 +339,6 @@ export class ActivityController extends EventEmitter implements IActivityControl } } - console.log('[ActivityController] Constructor: activityIdb =', !!this.#activityIdb) this.#callRelayer = callRelayer this.#accounts = accounts this.#selectedAccount = selectedAccount @@ -382,19 +381,15 @@ export class ActivityController extends EventEmitter implements IActivityControl if (this.#activityIdb) { try { console.time('[ActivityController] startup-load:idb') - console.log('[ActivityController] Loading startup ops from IDB...') accountsOps = await this.#activityIdb.loadStartupOps() console.timeEnd('[ActivityController] startup-load:idb') - console.log('[ActivityController] Loaded startup ops from IDB') } catch (error) { console.error('[ActivityController] Failed to load from IDB, falling back to storage', error) console.time('[ActivityController] startup-load:storage-fallback') accountsOps = await this.#storage.get('accountsOps', {}) console.timeEnd('[ActivityController] startup-load:storage-fallback') - console.log('[ActivityController] Loaded from chrome.storage.local fallback') } } else { - console.log('[ActivityController] IDB not available, using chrome.storage.local') console.time('[ActivityController] startup-load:storage') accountsOps = await this.#storage.get('accountsOps', {}) console.timeEnd('[ActivityController] startup-load:storage') @@ -409,11 +404,6 @@ export class ActivityController extends EventEmitter implements IActivityControl this.#accountsOps = accountsOps this.#externalAccountOps = externalAccountOps this.#signedMessages = signedMessages - - console.log('[ActivityController] Storage snapshot:') - console.log('accountsOps:', JSON.stringify(accountsOps, (_, v) => typeof v === 'bigint' ? v.toString() : v, 2)) - - console.log('[ActivityController] Load complete, emitting update') this.emitUpdate() } @@ -725,37 +715,28 @@ export class ActivityController extends EventEmitter implements IActivityControl await Promise.all(promises) } - private async persistAccountsOps() { - if (this.#activityIdb) { - try { - // Collect all (accountAddr, chainId) records to persist - const recordsToPersist: Array<{ - accountAddr: string - chainId: bigint | string - ops: SubmittedAccountOp[] - }> = [] - - for (const [accountAddr, chainMap] of Object.entries(this.#accountsOps)) { - for (const [chainIdString, ops] of Object.entries(chainMap)) { - recordsToPersist.push({ - accountAddr, - chainId: chainIdString, - ops - }) - } - } - - if (recordsToPersist.length > 0) { - await this.#activityIdb.putMultiple(recordsToPersist) - } - } catch (error) { - console.error('ActivityController: Failed to persist to IDB, falling back to storage', error) - await this.#storage.set('accountsOps', this.#accountsOps) - } - } else { + /** + * Writes accountsOps via IDB if available, otherwise falls back to chrome.storage.local. + * When IDB is available, errors are logged but do NOT fall back to storage — writing + * this.#accountsOps (startup subset) to storage would permanently truncate full history. + */ + async #persistToIdb(fn: () => Promise): Promise { + if (!this.#activityIdb) { await this.#storage.set('accountsOps', this.#accountsOps) + return + } + try { + await fn() + } catch (error) { + console.error('ActivityController: Failed to persist to IDB', error) } + } + /** + * Persist changed ops to IDB (or storage fallback), sync filtered views, and emit an update. + */ + private async persistAccountsOps(changedOps: SubmittedAccountOp[]) { + await this.#persistToIdb(() => this.#activityIdb!.updateOps(changedOps)) await this.syncFilteredAccountsOps() this.emitUpdate() } @@ -863,34 +844,26 @@ export class ActivityController extends EventEmitter implements IActivityControl await this.#initialLoadPromise const { accountAddr, chainId } = accountOp - console.log(`[ActivityController] addAccountOp: ${accountAddr} chain=${chainId} id=${accountOp.id} status=${accountOp.status} txnId=${accountOp.txnId}`) if (!this.#accountsOps[accountAddr]) this.#accountsOps[accountAddr] = {} if (!this.#accountsOps[accountAddr][chainId.toString()]) this.#accountsOps[accountAddr][chainId.toString()] = [] + // Capture the oldest op's id before mutating — it becomes the trimmed id if the + // group is already at capacity (trim() removes exactly one op via .pop()). + const group = this.#accountsOps[accountAddr][chainId.toString()]! + const trimmedId = group.length >= 1000 ? group[group.length - 1]?.id : undefined + // newest SubmittedAccountOp goes first in the list - this.#accountsOps[accountAddr]![chainId.toString()]!.unshift({ ...accountOp }) - trim(this.#accountsOps[accountAddr][chainId.toString()]!) + group.unshift({ ...accountOp }) + trim(group) await this.syncFilteredAccountsOps() this.emitUpdate() - // Write to IDB if available, otherwise fallback to chrome.storage.local - if (this.#activityIdb) { - try { - await this.#activityIdb.putOpsForAccountAndChain( - accountAddr, - chainId, - this.#accountsOps[accountAddr][chainId.toString()]! - ) - } catch (error) { - console.error('ActivityController: Failed to write to IDB, falling back to storage', error) - await this.#storage.set('accountsOps', this.#accountsOps) - } - } else { - await this.#storage.set('accountsOps', this.#accountsOps) - } + await this.#persistToIdb(() => + this.#activityIdb!.putSingleOp(accountAddr, chainId, accountOp, trimmedId) + ) } async addExternalAccountOp({ @@ -1078,7 +1051,7 @@ export class ActivityController extends EventEmitter implements IActivityControl */ async backfillAccountOpBalanceChangesAndPersist(accountOps: SubmittedAccountOp[]) { await Promise.all(accountOps.map((accOp) => this.backfillAccountOpBalanceChanges(accOp))) - await this.persistAccountsOps() + await this.persistAccountsOps(accountOps) } /** @@ -1247,7 +1220,7 @@ export class ActivityController extends EventEmitter implements IActivityControl ) ) ) - await this.persistAccountsOps() + await this.persistAccountsOps(balanceChangesTasks.map((t) => t.accountOp)) } /** @@ -1560,7 +1533,7 @@ export class ActivityController extends EventEmitter implements IActivityControl if (shouldEmitUpdate) { // remove duplicates if encountered during a race condition await this.#removeExternalAccountOpsMatchingInternalOps(updatedAccountsOps) - await this.persistAccountsOps() + await this.persistAccountsOps(updatedAccountsOps) } // record the balance changes but do not await them diff --git a/src/interfaces/activity.ts b/src/interfaces/activity.ts index 36ad04274d..3a49248eb6 100644 --- a/src/interfaces/activity.ts +++ b/src/interfaces/activity.ts @@ -68,4 +68,21 @@ export interface IActivityIdbStorage { * Check if IDB has any data (used to detect if migration is needed). */ isEmpty(): Promise + + /** + * Write a single new op and optionally delete the op evicted by the in-memory trim. + * Use instead of putOpsForAccountAndChain when adding one op at a time. + */ + putSingleOp( + accountAddr: string, + chainId: bigint | string, + op: SubmittedAccountOp, + trimmedId?: string + ): Promise + + /** + * Update existing rows in place (e.g. status or balance-change updates). + * Only the provided ops are written — no range-delete. + */ + updateOps(ops: SubmittedAccountOp[]): Promise } diff --git a/src/services/storage/activityIdb.test.ts b/src/services/storage/activityIdb.test.ts index 15a7be2a0c..ec51ec4543 100644 --- a/src/services/storage/activityIdb.test.ts +++ b/src/services/storage/activityIdb.test.ts @@ -1,10 +1,12 @@ -import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' +import 'fake-indexeddb/auto' +import { IDBFactory, IDBKeyRange } from 'fake-indexeddb' import { beforeEach, describe, expect, test } from '@jest/globals' import { SubmittedAccountOpLike } from '../../libs/accountOp/submittedAccountOp' import { AccountOpStatus } from '../../libs/accountOp/types' import { ActivityIdbStorage } from './activityIdb' +import { BaseIdbStore } from './baseIdbStore' // ───────────────────────────────────────────────────────────────────────────── // Test constants @@ -39,7 +41,8 @@ function makeOp( } beforeEach(() => { - // Give each test a completely isolated IndexedDB environment. + // Reset the shared DB connection so each test gets a completely isolated environment. + BaseIdbStore.resetDb('ambire') global.indexedDB = new IDBFactory() global.IDBKeyRange = IDBKeyRange // checkQuota() reads navigator.storage — stub it to avoid ReferenceError in Node. diff --git a/src/services/storage/activityIdb.ts b/src/services/storage/activityIdb.ts index bba005ed0c..994c5e1a10 100644 --- a/src/services/storage/activityIdb.ts +++ b/src/services/storage/activityIdb.ts @@ -4,52 +4,28 @@ import { AccountOpStatus } from '../../libs/accountOp/types' import { BaseIdbStore } from './baseIdbStore' const STARTUP_RECENT_OPS_LIMIT = 20 - -// Known bigint field names — used by the JSON reviver to restore bigint values after deserialization -const BIGINT_FIELDS = new Set([ - 'chainId', - 'nonce', - 'eoaNonce', - 'amount', - 'amountBefore', - 'amountAfter', - 'balanceChange', - 'feeTokenChainId', - 'simulatedGasLimit', - 'gasPrice', - 'maxPriorityFeePerGas' -]) - -function serializeOp(op: SubmittedAccountOp | SubmittedAccountOpLike): string { - return JSON.stringify(op, (_key, value) => { - if (typeof value === 'bigint') return value.toString() - return value - }) -} - -function deserializeOp(serialized: string): SubmittedAccountOp { - return JSON.parse(serialized, (key, value) => { - if (!BIGINT_FIELDS.has(key) || typeof value !== 'string') return value - try { - return BigInt(value) - } catch { - return value - } - }) as SubmittedAccountOp -} +// Hard cap on the number of IDB rows per (account, chainId) group. +// The in-memory cap is enforced by ActivityController; this guards against IDB +// accumulating more rows than the in-memory limit (e.g. after a startup that +// loaded only the 20-op subset before the limit was enforced). +const MAX_IDB_GROUP_SIZE = 1000 interface IdbAccountOpRow { accountAddr: string - chainId: string // bigint converted to string for IDB compatibility - id: string // op.id + // String copy of op.chainId — BigInt is not a valid IDB key type so it cannot + // be used directly in the compound keyPath or index keys. + chainId: string + id: string timestamp: number status: AccountOpStatus - serializedOp: string // JSON.stringify with bigint→string replacement + // The full op is stored via the Structured Clone Algorithm, which preserves + // BigInt natively — no JSON serialization needed. + op: SubmittedAccountOp | SubmittedAccountOpLike } // The highest BMP Unicode character — used as a range upper bound to select all keys // that start with a given prefix, without matching the prefix itself as a key. -const RANGE_HIGH = '\uffff' +const RANGE_HIGH = '￿' export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStorage { constructor() { @@ -57,41 +33,11 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor dbName: 'ambire', storeName: 'accountsOps', keyPath: ['accountAddr', 'chainId', 'id'], - dbVersion: 1 - }) - } - - /** - * Override doInit to create the object store with the compound keyPath. - * Any existing store from an older schema version is dropped — no data migration. - */ - protected doInit(): Promise { - return new Promise((resolve, reject) => { - const request = indexedDB.open(this.config.dbName, this.config.dbVersion) - - request.onerror = () => { - console.error(`BaseIdbStore: Failed to open ${this.config.dbName}`, request.error) - reject(request.error) - } - - request.onsuccess = () => { - this.db = request.result - console.log( - `[BaseIdbStore] Opened ${this.config.dbName} v${this.config.dbVersion} with store "${this.config.storeName}"` - ) - resolve() - } - - request.onupgradeneeded = (event) => { - const db = (event.target as IDBOpenDBRequest).result - - if (db.objectStoreNames.contains(this.config.storeName)) { - db.deleteObjectStore(this.config.storeName) - } - - db.createObjectStore(this.config.storeName, { keyPath: this.config.keyPath }) - console.log(`[BaseIdbStore] Created store "${this.config.storeName}"`) - } + dbVersion: 1, + indexes: [ + { name: 'by-account-chain-timestamp', keyPath: ['accountAddr', 'chainId', 'timestamp'] }, + { name: 'by-account-chain-status', keyPath: ['accountAddr', 'chainId', 'status'] } + ] }) } @@ -101,49 +47,164 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor /** * Load minimal startup dataset: all pending ops + up to STARTUP_RECENT_OPS_LIMIT - * finalized ops per (account, chain). Reads all rows then filters in JS. + * finalized ops per (account, chain). + * + * Two transactions: + * 1. Key-only cursor enumerates (accountAddr, chainId) groups — O(N_groups) reads. + * 2. Per-group queries run in parallel within one transaction: + * - 'by-account-chain-timestamp' cursor (prev) → top N finalized, stops early + * - 'by-account-chain-status' getAll × 2 → all pending ops + * + * All per-group requests are fired before any await resolves, keeping the + * transaction open for the duration. */ async loadStartupOps(): Promise { await this.init() - - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readonly') - const request = tx.objectStore(this.config.storeName).getAll() - - request.onerror = () => { - console.error('[ActivityIdbStorage] loadStartupOps error', request.error) - reject(request.error) + const db = this.db! + + // Step 1: enumerate (accountAddr, chainId) groups — key-only cursor, O(N_groups) reads + const groups: [string, string][] = [] + { + const tx = db.transaction(this.config.storeName, 'readonly') + let cursor = await tx.objectStore(this.config.storeName).openKeyCursor() + while (cursor) { + const [accountAddr, chainId] = cursor.primaryKey as [string, string, string] + groups.push([accountAddr, chainId]) + cursor = await cursor.continue([accountAddr, chainId, RANGE_HIGH]) } + } - request.onsuccess = () => { - const rows = (request.result as IdbAccountOpRow[]).sort( - (a, b) => b.timestamp - a.timestamp - ) - - const result: InternalAccountsOps = {} - const finalizedCount = new Map() + if (groups.length === 0) return {} - for (const row of rows) { - const isPending = - row.status === AccountOpStatus.BroadcastedButNotConfirmed || - row.status === AccountOpStatus.Pending - const groupKey = `${row.accountAddr}:${row.chainId}` - const count = finalizedCount.get(groupKey) ?? 0 + // Step 2: fetch per-group data — all groups run in parallel within one transaction. + // Each group's async function fires its IDB requests (1 cursor + 2 getAlls) before + // the first await resolves, so the transaction always has pending requests. + const result: InternalAccountsOps = {} + { + const tx = db.transaction(this.config.storeName, 'readonly') + const store = tx.objectStore(this.config.storeName) + const tsIndex = store.index('by-account-chain-timestamp') + const statusIndex = store.index('by-account-chain-status') + + await Promise.all( + groups.map(async ([accountAddr, chainId]) => { + if (!result[accountAddr]) result[accountAddr] = {} + if (!result[accountAddr]![chainId]) result[accountAddr]![chainId] = [] + const groupOps = result[accountAddr]![chainId]! + + const tsRange = IDBKeyRange.bound( + [accountAddr, chainId, 0], + [accountAddr, chainId, Number.MAX_SAFE_INTEGER] + ) + + // Run timestamp cursor + 2 pending getAlls in parallel for this group. + // The getAlls are fired synchronously (before any await), the cursor IIFE + // fires its first request synchronously too — all 3 are pending at once. + const [, pendingBroadcasted, pendingQueued] = await Promise.all([ + (async () => { + let finalizedCount = 0 + let cur = await tsIndex.openCursor(tsRange, 'prev') + while (cur && finalizedCount < STARTUP_RECENT_OPS_LIMIT) { + const row = cur.value as IdbAccountOpRow + const isPending = + row.status === AccountOpStatus.BroadcastedButNotConfirmed || + row.status === AccountOpStatus.Pending + if (!isPending) { + groupOps.push(row.op as SubmittedAccountOp) + finalizedCount++ + } + cur = await cur.continue() + } + })(), + statusIndex.getAll( + IDBKeyRange.only([accountAddr, chainId, AccountOpStatus.BroadcastedButNotConfirmed]) + ), + statusIndex.getAll(IDBKeyRange.only([accountAddr, chainId, AccountOpStatus.Pending])) + ]) + + for (const row of [...pendingBroadcasted, ...pendingQueued] as IdbAccountOpRow[]) { + groupOps.push(row.op as SubmittedAccountOp) + } + }) + ) + } - if (!isPending && count >= STARTUP_RECENT_OPS_LIMIT) continue + // Sort each group descending by timestamp + for (const chainMap of Object.values(result)) { + for (const ops of Object.values(chainMap)) { + ops.sort((a, b) => b.timestamp - a.timestamp) + } + } - if (!result[row.accountAddr]) result[row.accountAddr] = {} - const chainMap = result[row.accountAddr]! - if (!chainMap[row.chainId]) chainMap[row.chainId] = [] - chainMap[row.chainId]!.push(deserializeOp(row.serializedOp)) + console.log(`[ActivityIdbStorage] loadStartupOps: ${groups.length} groups loaded`) + return result + } - if (!isPending) finalizedCount.set(groupKey, count + 1) + /** + * Write a single new op and optionally delete the op evicted by the in-memory trim. + * O(1) IDB operations vs. the full-group rewrite of putOpsForAccountAndChain. + */ + async putSingleOp( + accountAddr: string, + chainId: bigint | string, + op: SubmittedAccountOp, + trimmedId?: string + ): Promise { + await this.init() + const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + const tx = this.db!.transaction(this.config.storeName, 'readwrite') + const store = tx.objectStore(this.config.storeName) + + // Fire put before any await so the transaction has a pending request. + // .catch(() => {}) suppresses unhandled-rejection warnings; tx.done still + // rejects on failure and is awaited below. + store.put(this.#opToRow(accountAddr, chainIdStr, op)).catch(() => {}) + + if (trimmedId) { + // In-memory trim already identified the op to evict. + store.delete([accountAddr, chainIdStr, trimmedId]).catch(() => {}) + } else { + // The in-memory group is within its cap, but IDB may have accumulated more + // rows than the in-memory limit (e.g. after a startup that only loaded the + // 20-op subset). Count after the put (IDB serializes requests within a tx) + // and evict the oldest row when the group exceeds the hard cap. + const groupRange = IDBKeyRange.bound( + [accountAddr, chainIdStr, ''], + [accountAddr, chainIdStr, RANGE_HIGH] + ) + const count = await store.count(groupRange) + if (count > MAX_IDB_GROUP_SIZE) { + const tsIndex = store.index('by-account-chain-timestamp') + const cursor = await tsIndex.openCursor( + IDBKeyRange.bound( + [accountAddr, chainIdStr, 0], + [accountAddr, chainIdStr, Number.MAX_SAFE_INTEGER] + ) + ) + if (cursor) { + store.delete(cursor.primaryKey as IDBValidKey).catch(() => {}) } - - console.log(`[ActivityIdbStorage] loadStartupOps: loaded from ${rows.length} rows`) - resolve(result) } - }) + } + + await tx.done + this.checkQuota() + } + + /** + * Update existing rows in place (status or balance-change updates). + * Uses store.put() per op — no range-delete, only touched rows are written. + */ + async updateOps(ops: SubmittedAccountOp[]): Promise { + if (ops.length === 0) return + await this.init() + + const tx = this.db!.transaction(this.config.storeName, 'readwrite') + const store = tx.objectStore(this.config.storeName) + for (const op of ops) { + store.put(this.#opToRow(op.accountAddr, op.chainId.toString(), op)).catch(() => {}) + } + await tx.done } /** @@ -158,41 +219,20 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor await this.init() const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + const range = IDBKeyRange.bound( + [accountAddr, chainIdStr, ''], + [accountAddr, chainIdStr, RANGE_HIGH] + ) + const rows = (await this.db!.getAll(this.config.storeName, range)) as IdbAccountOpRow[] - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readonly') - const store = tx.objectStore(this.config.storeName) - - // Bound on [accountAddr, chainIdStr, ''] → [accountAddr, chainIdStr, RANGE_HIGH] - const range = IDBKeyRange.bound( - [accountAddr, chainIdStr, ''], - [accountAddr, chainIdStr, RANGE_HIGH] - ) - const request = store.getAll(range) - - request.onerror = () => { - console.error( - `ActivityIdbStorage: Failed to get ops for ${accountAddr}:${chainIdStr}`, - request.error - ) - reject(request.error) - } + if (rows.length === 0) return undefined - request.onsuccess = () => { - const rows = (request.result || []) as IdbAccountOpRow[] - if (rows.length === 0) { - resolve(undefined) - return - } - // Sort descending by timestamp - rows.sort((a, b) => b.timestamp - a.timestamp) - const ops = rows.map((r) => deserializeOp(r.serializedOp)) - console.log( - `[ActivityIdbStorage] getOpsForAccountAndChain ${accountAddr}:${chainIdStr} - found ${ops.length} ops` - ) - resolve(ops) - } - }) + rows.sort((a, b) => b.timestamp - a.timestamp) + const result = rows.map((r) => r.op as SubmittedAccountOp) + console.log( + `[ActivityIdbStorage] getOpsForAccountAndChain ${accountAddr}:${chainIdStr} - found ${result.length} ops` + ) + return result } /** @@ -220,29 +260,20 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor ): Promise { await this.init() - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readwrite') - const store = tx.objectStore(this.config.storeName) + const tx = this.db!.transaction(this.config.storeName, 'readwrite') + const store = tx.objectStore(this.config.storeName) - for (const { accountAddr, chainId, ops } of records) { - const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId - this.#writeRecordToStore(store, accountAddr, chainIdStr, this.#dedupeOpsById(ops)) - } - - tx.onerror = () => { - console.error('ActivityIdbStorage: Batch put failed', tx.error) - reject(tx.error) - } + for (const { accountAddr, chainId, ops } of records) { + const chainIdStr = typeof chainId === 'bigint' ? chainId.toString() : chainId + this.#writeRecordToStore(store, accountAddr, chainIdStr, this.#dedupeOpsById(ops)) + } - tx.oncomplete = () => { - const totalOps = records.reduce((sum, r) => sum + r.ops.length, 0) - console.log( - `[ActivityIdbStorage] putMultiple complete - wrote ${records.length} records (${totalOps} ops)` - ) - this.checkQuota() - resolve() - } - }) + await tx.done + const totalOps = records.reduce((sum, r) => sum + r.ops.length, 0) + console.log( + `[ActivityIdbStorage] putMultiple complete - wrote ${records.length} records (${totalOps} ops)` + ) + this.checkQuota() } /** @@ -251,35 +282,11 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor async deleteAccount(accountAddr: string): Promise { await this.init() - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readwrite') - const store = tx.objectStore(this.config.storeName) - - // IDB compound key range: all keys where first component === accountAddr - const range = IDBKeyRange.bound([accountAddr, '', ''], [accountAddr, RANGE_HIGH, RANGE_HIGH]) - const request = store.delete(range) - - request.onerror = () => { - console.error( - `ActivityIdbStorage: Failed to delete account ${accountAddr}`, - request.error - ) - reject(request.error) - } - - tx.oncomplete = () => { - console.log(`[ActivityIdbStorage] deleteAccount ${accountAddr} - deleted all rows`) - resolve() - } - - tx.onerror = () => { - console.error( - `ActivityIdbStorage: Failed to delete account ${accountAddr} during transaction`, - tx.error - ) - reject(tx.error) - } - }) + const range = IDBKeyRange.bound([accountAddr, '', ''], [accountAddr, RANGE_HIGH, RANGE_HIGH]) + const tx = this.db!.transaction(this.config.storeName, 'readwrite') + await tx.objectStore(this.config.storeName).delete(range) + await tx.done + console.log(`[ActivityIdbStorage] deleteAccount ${accountAddr} - deleted all rows`) } /** @@ -303,24 +310,12 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor async isEmpty(): Promise { await this.init() - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readonly') - const store = tx.objectStore(this.config.storeName) - const request = store.count() - - request.onerror = () => { - console.error('ActivityIdbStorage: Failed to check if empty', request.error) - reject(request.error) - } - - request.onsuccess = () => { - const empty = request.result === 0 - console.log( - `[ActivityIdbStorage] isEmpty check - ${empty ? 'empty' : `${request.result} records found`}` - ) - resolve(empty) - } - }) + const count = await this.db!.count(this.config.storeName) + const empty = count === 0 + console.log( + `[ActivityIdbStorage] isEmpty check - ${empty ? 'empty' : `${count} records found`}` + ) + return empty } // ────────────────────────────────────────────────────────────────────────────── @@ -328,17 +323,19 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor // ────────────────────────────────────────────────────────────────────────────── #writeRecordToStore( - store: IDBObjectStore, + store: any, accountAddr: string, chainIdStr: string, ops: (SubmittedAccountOp | SubmittedAccountOpLike)[] ): void { // Delete existing rows for this (account, chain), then insert fresh ones - store.delete( - IDBKeyRange.bound([accountAddr, chainIdStr, ''], [accountAddr, chainIdStr, RANGE_HIGH]) - ) + store + .delete( + IDBKeyRange.bound([accountAddr, chainIdStr, ''], [accountAddr, chainIdStr, RANGE_HIGH]) + ) + .catch(() => {}) for (const op of ops) { - store.put(this.#opToRow(accountAddr, chainIdStr, op)) + store.put(this.#opToRow(accountAddr, chainIdStr, op)).catch(() => {}) } } @@ -381,7 +378,7 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor id: op.id, timestamp: op.timestamp, status: op.status, - serializedOp: serializeOp(op) + op } } } diff --git a/src/services/storage/baseIdbStore.ts b/src/services/storage/baseIdbStore.ts index b0f811fe71..ecce08c05f 100644 --- a/src/services/storage/baseIdbStore.ts +++ b/src/services/storage/baseIdbStore.ts @@ -1,3 +1,10 @@ +import { IDBPDatabase, openDB } from 'idb' + +export interface IdbIndexDef { + name: string + keyPath: string | string[] +} + /** * Configuration for an IndexedDB store. */ @@ -6,137 +13,173 @@ export interface IdbStoreConfig { storeName: string keyPath: string | string[] dbVersion?: number + indexes?: IdbIndexDef[] +} + +interface DbRegistry { + stores: Map + db: IDBPDatabase | null + openPromise: Promise> | null + version: number } /** * Base IndexedDB storage service. - * Handles common operations: open/upgrade, transactions, error handling. - * Extend this for specific data types. + * + * Maintains a static registry of all stores per database name so that a single + * database connection is shared across all stores. The one onupgradeneeded + * handler creates every registered store, preventing version conflicts when + * multiple stores share the same database. + * + * Subclasses pass their full config (including indexes) to super() — that is + * enough to participate in the shared connection. No doInit override needed. */ export class BaseIdbStore { - protected db: IDBDatabase | null = null + private static readonly registries = new Map() + + protected db: IDBPDatabase | null = null protected initPromise: Promise | null = null protected config: IdbStoreConfig constructor(config: IdbStoreConfig) { - this.config = { - dbVersion: 1, - ...config - } + this.config = { dbVersion: 1, ...config } + BaseIdbStore.#registerStore(this.config) } - protected async init(): Promise { - if (this.db) return - if (this.initPromise) return this.initPromise + // ────────────────────────────────────────────────────────────────────────────── + // Static registry helpers + // ────────────────────────────────────────────────────────────────────────────── - this.initPromise = this.doInit() - await this.initPromise - } + static #registerStore(config: IdbStoreConfig): void { + const { dbName, dbVersion = 1 } = config - protected async doInit(): Promise { - return new Promise((resolve, reject) => { - const request = indexedDB.open(this.config.dbName, this.config.dbVersion) + if (!BaseIdbStore.registries.has(dbName)) { + BaseIdbStore.registries.set(dbName, { + stores: new Map(), + db: null, + openPromise: null, + version: dbVersion + }) + } - request.onerror = () => { - console.error(`BaseIdbStore: Failed to open ${this.config.dbName}`, request.error) - reject(request.error) - } + const registry = BaseIdbStore.registries.get(dbName)! - request.onsuccess = () => { - this.db = request.result - console.log( - `[BaseIdbStore] Opened ${this.config.dbName} v${this.config.dbVersion} with store "${this.config.storeName}"` - ) - resolve() - } + if (registry.db || registry.openPromise) { + // The DB is already open — a store was constructed after the first init() call. + // Its schema will not be applied until the next version bump. + console.error( + `[BaseIdbStore] Store "${config.storeName}" registered after "${dbName}" is already open — schema will not be applied until the next version bump.` + ) + } - request.onupgradeneeded = (event) => { - const db = (event.target as IDBOpenDBRequest).result - if (!db.objectStoreNames.contains(this.config.storeName)) { - db.createObjectStore(this.config.storeName, { keyPath: this.config.keyPath }) - console.log(`[BaseIdbStore] Created store "${this.config.storeName}"`) + registry.stores.set(config.storeName, config) + registry.version = Math.max(registry.version, dbVersion) + } + + static #getDb( + dbName: string, + version: number, + stores: Map + ): Promise> { + const registry = BaseIdbStore.registries.get(dbName)! + + if (registry.db) return Promise.resolve(registry.db) + if (registry.openPromise) return registry.openPromise + + registry.openPromise = openDB(dbName, version, { + upgrade(db, oldVersion) { + console.log(`[BaseIdbStore] Upgrading "${dbName}" from v${oldVersion} → v${version}`) + for (const storeDef of stores.values()) { + if (!db.objectStoreNames.contains(storeDef.storeName)) { + const store = db.createObjectStore(storeDef.storeName, { + keyPath: storeDef.keyPath + }) + for (const idx of storeDef.indexes ?? []) { + store.createIndex(idx.name, idx.keyPath) + } + console.log(`[BaseIdbStore] Created store "${storeDef.storeName}"`) + } + // When oldVersion > 0 and the store already exists, add migration branches here. } } + }).then((db) => { + registry.db = db + console.log(`[BaseIdbStore] Opened "${dbName}" v${version}`) + return db + }).catch((error) => { + // Allow a subsequent openDB attempt after a transient failure. + registry.openPromise = null + throw error }) - } - - protected async getAll(): Promise { - await this.init() - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readonly') - const store = tx.objectStore(this.config.storeName) - const request = store.getAll() + return registry.openPromise + } - request.onerror = () => reject(request.error) - request.onsuccess = () => resolve(request.result as T[]) - }) + /** + * Reset the shared connection for a database — for use in tests only. + * Call in beforeEach after replacing global.indexedDB with a fresh IDBFactory. + */ + static resetDb(dbName: string): void { + const registry = BaseIdbStore.registries.get(dbName) + if (!registry) return + registry.db?.close() + registry.db = null + registry.openPromise = null + registry.stores.clear() + BaseIdbStore.registries.delete(dbName) } - protected async put(record: any): Promise { - await this.init() + // ────────────────────────────────────────────────────────────────────────────── + // Instance methods + // ────────────────────────────────────────────────────────────────────────────── - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readwrite') - const store = tx.objectStore(this.config.storeName) - store.put(record) + protected async init(): Promise { + if (this.db) return + if (this.initPromise) return this.initPromise - tx.onerror = () => reject(tx.error) - tx.oncomplete = () => resolve() + this.initPromise = this.doInit().catch((error) => { + // Allow a subsequent init() call to retry after a transient failure. + this.initPromise = null + throw error }) + await this.initPromise } - protected async putMultiple(records: any[]): Promise { - await this.init() - - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readwrite') - const store = tx.objectStore(this.config.storeName) - - records.forEach((record) => store.put(record)) + protected async doInit(): Promise { + const registry = BaseIdbStore.registries.get(this.config.dbName)! + this.db = await BaseIdbStore.#getDb(this.config.dbName, registry.version, registry.stores) + } - tx.onerror = () => reject(tx.error) - tx.oncomplete = () => resolve() - }) + protected async getAll(): Promise { + await this.init() + return this.db!.getAll(this.config.storeName) as Promise } - protected async delete(key: any): Promise { + protected async put(record: unknown): Promise { await this.init() + await this.db!.put(this.config.storeName, record) + } - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readwrite') - const store = tx.objectStore(this.config.storeName) - const request = store.delete(key) + protected async putMultiple(records: unknown[]): Promise { + await this.init() + const tx = this.db!.transaction(this.config.storeName, 'readwrite') + await Promise.all(records.map((r) => tx.objectStore(this.config.storeName).put(r))) + await tx.done + } - request.onerror = () => reject(request.error) - tx.oncomplete = () => resolve() - }) + protected async delete(key: IDBValidKey | IDBKeyRange): Promise { + await this.init() + await this.db!.delete(this.config.storeName, key as any) } protected async clear(): Promise { await this.init() - - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readwrite') - const store = tx.objectStore(this.config.storeName) - const request = store.clear() - - request.onerror = () => reject(request.error) - tx.oncomplete = () => resolve() - }) + await this.db!.clear(this.config.storeName) } protected async isEmpty(): Promise { await this.init() - - return new Promise((resolve, reject) => { - const tx = this.db!.transaction([this.config.storeName], 'readonly') - const store = tx.objectStore(this.config.storeName) - const request = store.count() - - request.onerror = () => reject(request.error) - request.onsuccess = () => resolve(request.result === 0) - }) + return (await this.db!.count(this.config.storeName)) === 0 } protected checkQuota(): void { @@ -158,6 +201,6 @@ export class BaseIdbStore { `[BaseIdbStore] ${this.config.storeName} quota usage high (${percentUsed.toFixed(1)}%)` ) } - }) + }).catch(() => {}) } } From 1bdfff93eedf977022383a1d48f88b53ae55fcaf Mon Sep 17 00:00:00 2001 From: Gergana Mihaylova Date: Thu, 9 Jul 2026 14:09:15 +0300 Subject: [PATCH 5/5] add logs for testing --- package-lock.json | 4 +- src/controllers/activity/activity.ts | 16 ++++--- src/services/storage/activityIdb.test.ts | 9 +--- src/services/storage/activityIdb.ts | 15 +++++++ src/services/storage/baseIdbStore.ts | 57 +++++++++++++++--------- 5 files changed, 64 insertions(+), 37 deletions(-) diff --git a/package-lock.json b/package-lock.json index ed54a65eb3..bf4e51fd2f 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "ambire-common", - "version": "2.102.3", + "version": "2.103.0", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "ambire-common", - "version": "2.102.3", + "version": "2.103.0", "dependencies": { "@ambire/signature-validator": "^1.5.0", "@lifi/types": "^17.7.1", diff --git a/src/controllers/activity/activity.ts b/src/controllers/activity/activity.ts index 6e8648d0e4..426574b912 100644 --- a/src/controllers/activity/activity.ts +++ b/src/controllers/activity/activity.ts @@ -89,7 +89,6 @@ export interface Filters { identifiedBy?: AccountOpIdentifiedBy } - export interface ExternalAccountOps { [account: string]: { [network: string]: SubmittedAccountOpLike[] } } @@ -384,7 +383,10 @@ export class ActivityController extends EventEmitter implements IActivityControl accountsOps = await this.#activityIdb.loadStartupOps() console.timeEnd('[ActivityController] startup-load:idb') } catch (error) { - console.error('[ActivityController] Failed to load from IDB, falling back to storage', error) + console.error( + '[ActivityController] Failed to load from IDB, falling back to storage', + error + ) console.time('[ActivityController] startup-load:storage-fallback') accountsOps = await this.#storage.get('accountsOps', {}) console.timeEnd('[ActivityController] startup-load:storage-fallback') @@ -396,7 +398,7 @@ export class ActivityController extends EventEmitter implements IActivityControl } // Always load externalAccountOps from storage (not migrated to IDB yet) - [externalAccountOps, signedMessages] = await Promise.all([ + ;[externalAccountOps, signedMessages] = await Promise.all([ this.#storage.get('externalAccountOps', {}), this.#storage.get('signedMessages', {}) ]) @@ -727,6 +729,7 @@ export class ActivityController extends EventEmitter implements IActivityControl } try { await fn() + console.log('ActivityController: IDB persist succeeded') } catch (error) { console.error('ActivityController: Failed to persist to IDB', error) } @@ -1013,7 +1016,7 @@ export class ActivityController extends EventEmitter implements IActivityControl trim(externalAccountOps) // externalAccountOps: using chrome.storage.local only (not migrated to IDB yet) - await this.#storage.set('externalAccountOps', this.#externalAccountOps); + await this.#storage.set('externalAccountOps', this.#externalAccountOps) await this.syncFilteredAccountsOps() this.emitUpdate() } @@ -1630,7 +1633,10 @@ export class ActivityController extends EventEmitter implements IActivityControl try { await this.#activityIdb.deleteAccount(address) } catch (error) { - console.error('ActivityController: Failed to delete from IDB, falling back to storage', error) + console.error( + 'ActivityController: Failed to delete from IDB, falling back to storage', + error + ) await this.#storage.set('accountsOps', this.#accountsOps) await this.#storage.set('signedMessages', this.#signedMessages) } diff --git a/src/services/storage/activityIdb.test.ts b/src/services/storage/activityIdb.test.ts index ec51ec4543..0b52d1d11d 100644 --- a/src/services/storage/activityIdb.test.ts +++ b/src/services/storage/activityIdb.test.ts @@ -290,13 +290,7 @@ describe('ActivityIdbStorage', () => { makeOp(`fin-${i}`, ACC_A, CHAIN_1, AccountOpStatus.Success, i * 10) ) const pending = Array.from({ length: 3 }, (_, i) => - makeOp( - `pend-${i}`, - ACC_A, - CHAIN_1, - AccountOpStatus.BroadcastedButNotConfirmed, - 1000 + i - ) + makeOp(`pend-${i}`, ACC_A, CHAIN_1, AccountOpStatus.BroadcastedButNotConfirmed, 1000 + i) ) await store.putOpsForAccountAndChain(ACC_A, CHAIN_1, [...finalized, ...pending]) @@ -382,5 +376,4 @@ describe('ActivityIdbStorage', () => { expect((retrieved as any).nonce).toBe(42n) }) }) - }) diff --git a/src/services/storage/activityIdb.ts b/src/services/storage/activityIdb.ts index 994c5e1a10..ebd101345f 100644 --- a/src/services/storage/activityIdb.ts +++ b/src/services/storage/activityIdb.ts @@ -158,10 +158,14 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor // Fire put before any await so the transaction has a pending request. // .catch(() => {}) suppresses unhandled-rejection warnings; tx.done still // rejects on failure and is awaited below. + console.log(`[ActivityIdbStorage] putSingleOp ${accountAddr}:${chainIdStr} op=${op.id}`) store.put(this.#opToRow(accountAddr, chainIdStr, op)).catch(() => {}) if (trimmedId) { // In-memory trim already identified the op to evict. + console.log( + `[ActivityIdbStorage] putSingleOp evicting in-memory trimmed op=${trimmedId} for ${accountAddr}:${chainIdStr}` + ) store.delete([accountAddr, chainIdStr, trimmedId]).catch(() => {}) } else { // The in-memory group is within its cap, but IDB may have accumulated more @@ -173,6 +177,9 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor [accountAddr, chainIdStr, RANGE_HIGH] ) const count = await store.count(groupRange) + console.log( + `[ActivityIdbStorage] putSingleOp IDB group size after put: ${count} for ${accountAddr}:${chainIdStr}` + ) if (count > MAX_IDB_GROUP_SIZE) { const tsIndex = store.index('by-account-chain-timestamp') const cursor = await tsIndex.openCursor( @@ -182,12 +189,16 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor ) ) if (cursor) { + console.log( + `[ActivityIdbStorage] putSingleOp IDB cap hit (${count}/${MAX_IDB_GROUP_SIZE}) — evicting oldest op id=${cursor.value.id} ts=${cursor.value.timestamp} for ${accountAddr}:${chainIdStr}` + ) store.delete(cursor.primaryKey as IDBValidKey).catch(() => {}) } } } await tx.done + console.log(`[ActivityIdbStorage] putSingleOp committed for ${accountAddr}:${chainIdStr}`) this.checkQuota() } @@ -199,12 +210,16 @@ export class ActivityIdbStorage extends BaseIdbStore implements IActivityIdbStor if (ops.length === 0) return await this.init() + console.log( + `[ActivityIdbStorage] updateOps ${ops.length} op(s): ${ops.map((o) => o.id).join(', ')}` + ) const tx = this.db!.transaction(this.config.storeName, 'readwrite') const store = tx.objectStore(this.config.storeName) for (const op of ops) { store.put(this.#opToRow(op.accountAddr, op.chainId.toString(), op)).catch(() => {}) } await tx.done + console.log(`[ActivityIdbStorage] updateOps committed`) } /** diff --git a/src/services/storage/baseIdbStore.ts b/src/services/storage/baseIdbStore.ts index ecce08c05f..cd6b102d30 100644 --- a/src/services/storage/baseIdbStore.ts +++ b/src/services/storage/baseIdbStore.ts @@ -102,15 +102,17 @@ export class BaseIdbStore { // When oldVersion > 0 and the store already exists, add migration branches here. } } - }).then((db) => { - registry.db = db - console.log(`[BaseIdbStore] Opened "${dbName}" v${version}`) - return db - }).catch((error) => { - // Allow a subsequent openDB attempt after a transient failure. - registry.openPromise = null - throw error }) + .then((db) => { + registry.db = db + console.log(`[BaseIdbStore] Opened "${dbName}" v${version}`) + return db + }) + .catch((error) => { + // Allow a subsequent openDB attempt after a transient failure. + registry.openPromise = null + throw error + }) return registry.openPromise } @@ -137,12 +139,20 @@ export class BaseIdbStore { if (this.db) return if (this.initPromise) return this.initPromise + console.log( + `[BaseIdbStore] init() starting for "${this.config.dbName}/${this.config.storeName}"` + ) this.initPromise = this.doInit().catch((error) => { // Allow a subsequent init() call to retry after a transient failure. + console.error( + `[BaseIdbStore] init() failed for "${this.config.dbName}/${this.config.storeName}", will retry on next call`, + error + ) this.initPromise = null throw error }) await this.initPromise + console.log(`[BaseIdbStore] init() ready for "${this.config.dbName}/${this.config.storeName}"`) } protected async doInit(): Promise { @@ -185,22 +195,25 @@ export class BaseIdbStore { protected checkQuota(): void { if (!navigator.storage?.estimate) return - navigator.storage.estimate().then((estimate) => { - if (!estimate.quota || !estimate.usage) return + navigator.storage + .estimate() + .then((estimate) => { + if (!estimate.quota || !estimate.usage) return - const percentUsed = (estimate.usage / estimate.quota) * 100 - const usedMB = (estimate.usage / 1024 / 1024).toFixed(1) - const quotaMB = (estimate.quota / 1024 / 1024).toFixed(1) + const percentUsed = (estimate.usage / estimate.quota) * 100 + const usedMB = (estimate.usage / 1024 / 1024).toFixed(1) + const quotaMB = (estimate.quota / 1024 / 1024).toFixed(1) - console.log( - `[BaseIdbStore] ${this.config.storeName} quota: ${usedMB}MB / ${quotaMB}MB (${percentUsed.toFixed(1)}%)` - ) - - if (percentUsed > 80) { - console.warn( - `[BaseIdbStore] ${this.config.storeName} quota usage high (${percentUsed.toFixed(1)}%)` + console.log( + `[BaseIdbStore] ${this.config.storeName} quota: ${usedMB}MB / ${quotaMB}MB (${percentUsed.toFixed(1)}%)` ) - } - }).catch(() => {}) + + if (percentUsed > 80) { + console.warn( + `[BaseIdbStore] ${this.config.storeName} quota usage high (${percentUsed.toFixed(1)}%)` + ) + } + }) + .catch(() => {}) } }