diff --git a/packages/keyring-sdk/CHANGELOG.md b/packages/keyring-sdk/CHANGELOG.md index 094f7b36c..db2ca5e13 100644 --- a/packages/keyring-sdk/CHANGELOG.md +++ b/packages/keyring-sdk/CHANGELOG.md @@ -7,6 +7,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Added + +- Add keyring state migration framework ([#505](https://github.com/MetaMask/accounts/pull/505)) + - Use `createMigrations().add(...)` to build a chain of versioned migration steps, and `.apply(state)` to migrate the internal state of keyrings + ### Changed - Bump `@metamask/keyring-api` from `^23.2.0` to `^23.5.0` ([#569](https://github.com/MetaMask/accounts/pull/569), [#583](https://github.com/MetaMask/accounts/pull/583), [#587](https://github.com/MetaMask/accounts/pull/587)) diff --git a/packages/keyring-sdk/docs/migrations.md b/packages/keyring-sdk/docs/migrations.md new file mode 100644 index 000000000..ae1f8b087 --- /dev/null +++ b/packages/keyring-sdk/docs/migrations.md @@ -0,0 +1,119 @@ +# Keyring State Migrations + +A framework for evolving keyring serialized state across versions. Migrations run during `deserialize()` and transform old state into the current format. + +Versioned state is stored as an envelope: + +```json5 +{ + version: 1, + data: { + // Keyring state + }, +} +``` + +Unversioned state (vaults created before migration support was added) has no envelope. The framework treats it as version 0 and applies all steps. + +## Key Concepts + +- **`createMigrations()`**: Starts an empty migration chain. +- **`.add(step)`**: Appends a step to the chain and returns a new chain typed to that step's output. +- **`outputSchema`**: (Optional) Validates the **output** of a step at runtime. +- **`inputSchema`**: (Optional) Validates the **input** before the `migrate` function is called. +- **Positional versions**: The first `.add()` call produces version 1, the second version 2, and so on. + +## Example + +### 1. Define State Schemas + +Define a schema for each version of your state. + +```typescript +import { object, array, number, string } from '@metamask/superstruct'; +import type { Infer } from '@metamask/superstruct'; + +const HdStateV0Schema = object({ + numberOfAccounts: number(), // legacy field name + mnemonic: array(number()), + hdPath: string(), +}); + +const HdStateV1Schema = object({ + accountCount: number(), // renamed from numberOfAccounts + mnemonic: array(number()), + hdPath: string(), +}); + +const HdStateV2Schema = object({ + accountCount: number(), + mnemonic: array(number()), + hdPath: string(), + createdAt: number(), // new field +}); + +type HdStateV0 = Infer; +type HdStateV1 = Infer; +type HdStateV2 = Infer; +``` + +### 2. Define the Migration Chain + +```typescript +import { createMigrations } from '@metamask/keyring-sdk'; + +const migrations = createMigrations() + .add({ + inputSchema: HdStateV0Schema, + outputSchema: HdStateV1Schema, + migrate: (data) => ({ + accountCount: data.numberOfAccounts, + mnemonic: data.mnemonic, + hdPath: data.hdPath, + }), + }) + .add({ + outputSchema: HdStateV2Schema, + migrate: (data) => ({ ...data, createdAt: Date.now() }), // data is typed as HdStateV1, no cast needed + }); +``` + +### 3. Implement in your Keyring + +```typescript +import type { VersionedState } from '@metamask/keyring-sdk'; +import type { Json } from '@metamask/utils'; + +class MyKeyring { + async deserialize(state: Json): Promise { + const { data } = await migrations.apply(state); + + // data is typed as HdStateV2 + this.#mnemonic = data.mnemonic; + this.#accountCount = data.accountCount; + this.#hdPath = data.hdPath; + } + + async serialize(): Promise> { + return { + version: migrations.version, + data: { + mnemonic: this.#mnemonic, + accountCount: this.#accountCount, + hdPath: this.#hdPath, + }, + }; + } +} +``` + +## Best Practices + +- **Idempotent migrations**: Design steps so re-running them on already-migrated data is harmless. +- **Immutability**: Treat the input `data` as immutable within the `migrate` function. +- **Schema coverage**: Ensure `outputSchema` covers all fields expected in the new version to prevent runtime errors. +- **Non-mutating chains**: `.add()` returns a new chain rather than mutating the one it's called on, so it's safe to branch multiple chains off a shared base. + +## Constraints + +- **Forward-only**: there is no downgrade path. Code that does not understand the versioned envelope will fail on migrated state. diff --git a/packages/keyring-sdk/src/index.ts b/packages/keyring-sdk/src/index.ts index 170478da5..874f77ce5 100644 --- a/packages/keyring-sdk/src/index.ts +++ b/packages/keyring-sdk/src/index.ts @@ -1,3 +1,4 @@ export * from './keyring-account-registry'; +export * from './migration'; export * from './mnemonic'; export * from './eth'; diff --git a/packages/keyring-sdk/src/migration.test-d.ts b/packages/keyring-sdk/src/migration.test-d.ts new file mode 100644 index 000000000..2a339e57b --- /dev/null +++ b/packages/keyring-sdk/src/migration.test-d.ts @@ -0,0 +1,63 @@ +import { object, number, string } from '@metamask/superstruct'; +import type { Json } from '@metamask/utils'; +import { expectType } from 'tsd'; + +import { createMigrations } from './migration'; +import type { MigrationChain, MigrationResult } from './migration'; + +// `createMigrations()` starts an empty chain typed to accept `Json`. +expectType>(createMigrations()); + +// A step's `migrate` receives the previous step's output shape, with no cast needed. +createMigrations() + .add({ migrate: (): { count: number } => ({ count: 1 }) }) + .add({ + migrate: (data) => { + expectType(data.count); + return data; + }, + }); + +// A step's `migrate` must accept the previous step's output shape. +createMigrations() + .add({ migrate: (): { count: number } => ({ count: 1 }) }) + // @ts-expect-error [test] `data` is `{ count: number }`, not `{ label: string }`. + .add({ migrate: (data: { label: string }) => data.label }); + +// A step's inferred `migrate` parameter is the previous step's output shape, not `Json`. +createMigrations() + .add({ migrate: (): { count: number } => ({ count: 1 }) }) + .add({ + migrate: (data) => { + expectType<{ count: number }>(data); + return data; + }, + }); + +// `inputSchema` narrows `migrate`'s input to the schema's inferred type, with no cast +// needed. +createMigrations().add({ + inputSchema: object({ oldCount: number() }), + migrate: (data) => { + expectType(data.oldCount); + return { count: data.oldCount }; + }, +}); + +// `inputSchema` must be compatible with the chain's current data type, just like +// `migrate`. +createMigrations() + .add({ migrate: (): { count: number } => ({ count: 1 }) }) + .add({ + // @ts-expect-error [test] `inputSchema`'s inferred type doesn't extend `{ count: number }`. + inputSchema: object({ label: string() }), + migrate: () => 'x', + }); + +// `apply()` resolves to the final step's output type. +const migrationsForApply = createMigrations().add({ + migrate: (): { count: number } => ({ count: 1 }), +}); +expectType>>( + migrationsForApply.apply({}), +); diff --git a/packages/keyring-sdk/src/migration.test.ts b/packages/keyring-sdk/src/migration.test.ts new file mode 100644 index 000000000..efff8610d --- /dev/null +++ b/packages/keyring-sdk/src/migration.test.ts @@ -0,0 +1,420 @@ +import { object, number, string, array } from '@metamask/superstruct'; +import type { Infer } from '@metamask/superstruct'; +import type { Json } from '@metamask/utils'; + +import { createMigrations, isVersionedState } from './migration'; + +describe('isVersionedState', () => { + it('returns true for a valid versioned state', () => { + expect(isVersionedState({ version: 1, data: { foo: 'bar' } })).toBe(true); + }); + + it('returns true when data is an array', () => { + expect(isVersionedState({ version: 0, data: ['a', 'b'] })).toBe(true); + }); + + it('returns true when data is null', () => { + expect(isVersionedState({ version: 0, data: null })).toBe(true); + }); + + it('returns false for a plain object without version', () => { + expect(isVersionedState({ foo: 'bar' })).toBe(false); + }); + + it('returns false for an array', () => { + expect(isVersionedState(['a', 'b'] as unknown as Json)).toBe(false); + }); + + it('returns false for null', () => { + expect(isVersionedState(null)).toBe(false); + }); + + it('returns false when version is not a number', () => { + expect(isVersionedState({ version: '1', data: {} })).toBe(false); + }); + + it('returns false when data is missing', () => { + expect(isVersionedState({ version: 1 })).toBe(false); + }); +}); + +describe('createMigrations', () => { + it('starts at version 0 with no steps', () => { + expect(createMigrations().version).toBe(0); + }); + + it('increments version by 1 for each added step', () => { + const chain = createMigrations() + .add({ migrate: (data) => data }) + .add({ migrate: (data) => data }) + .add({ migrate: (data) => data }); + + expect(chain.version).toBe(3); + }); + + it('does not mutate the base chain when a step is added', () => { + const base = createMigrations().add({ migrate: () => ({ count: 1 }) }); + const extended = base.add({ + migrate: (data: { count: number }) => ({ ...data, label: 'x' }), + }); + + expect(base.version).toBe(1); + expect(extended.version).toBe(2); + }); +}); + +describe('apply', () => { + describe('when given unversioned state', () => { + it('applies all steps to an unversioned object', async () => { + type UnversionedHdState = { oldField: string; existing: boolean }; + type HdStateV2 = { + existing: boolean; + newField: string; + renamedField: string; + }; + + const migrations = createMigrations() + .add({ + migrate: (data: UnversionedHdState) => ({ + ...data, + newField: 'added', + }), + }) + .add({ + migrate: (data): HdStateV2 => ({ + existing: data.existing, + newField: data.newField, + renamedField: data.oldField, + }), + }); + + const result = await migrations.apply({ + oldField: 'value', + existing: true, + } satisfies UnversionedHdState); + + expect(result).toStrictEqual({ + version: 2, + data: { + existing: true, + newField: 'added', + renamedField: 'value', + } satisfies HdStateV2, + migrated: true, + }); + }); + + it('applies all steps to an unversioned array', async () => { + type PrivateKeysV1 = { keys: string[]; format: string }; + + const migrations = createMigrations().add({ + migrate: (data: string[]): PrivateKeysV1 => ({ + keys: data, + format: 'v1', + }), + }); + + const result = await migrations.apply([ + 'key1', + 'key2', + ] as unknown as Json); + + expect(result).toStrictEqual({ + version: 1, + data: { keys: ['key1', 'key2'], format: 'v1' } satisfies PrivateKeysV1, + migrated: true, + }); + }); + + it('wraps in envelope at version 0 when the chain has no steps', async () => { + const result = await createMigrations().apply({ foo: 'bar' }); + + expect(result).toStrictEqual({ + version: 0, + data: { foo: 'bar' }, + migrated: false, + }); + }); + + it('wraps array state in envelope at version 0 when the chain has no steps', async () => { + const result = await createMigrations().apply([ + 'a', + 'b', + ] as unknown as Json); + + expect(result).toStrictEqual({ + version: 0, + data: ['a', 'b'], + migrated: false, + }); + }); + }); + + describe('when given versioned state', () => { + it('skips already-applied steps', async () => { + type StateV2 = { existing: boolean; v2: boolean }; + + const migrateFn = jest.fn((data: { existing: boolean }) => ({ + ...data, + v2: true, + })); + + const migrations = createMigrations() + .add({ migrate: (data: { existing: boolean }) => data }) + .add({ migrate: migrateFn }); + + const result = await migrations.apply({ + version: 1, + data: { existing: true }, + }); + + expect(migrateFn).toHaveBeenCalledWith({ existing: true }); + expect(result).toStrictEqual({ + version: 2, + data: { existing: true, v2: true } satisfies StateV2, + migrated: true, + }); + }); + + it('returns state unchanged when already at the latest version', async () => { + const migrateFn = jest.fn((data) => data); + + const migrations = createMigrations().add({ migrate: migrateFn }); + + const result = await migrations.apply({ + version: 1, + data: { foo: 'bar' }, + }); + + expect(migrateFn).not.toHaveBeenCalled(); + expect(result).toStrictEqual({ + version: 1, + data: { foo: 'bar' }, + migrated: false, + }); + }); + + it('applies multiple pending steps in order', async () => { + type StateV3 = { original: boolean; migrated: boolean }; + + const order: number[] = []; + + const migrations = createMigrations() + .add({ + migrate: (data) => { + order.push(1); + return data; + }, + }) + .add({ + migrate: (data) => { + order.push(2); + return data; + }, + }) + .add({ + migrate: (data: { original: boolean }): StateV3 => { + order.push(3); + return { ...data, migrated: true }; + }, + }); + + const result = await migrations.apply({ + version: 1, + data: { original: true }, + }); + + expect(order).toStrictEqual([2, 3]); + expect(result).toStrictEqual({ + version: 3, + data: { original: true, migrated: true } satisfies StateV3, + migrated: true, + }); + }); + + it('treats explicitly versioned state at version 0 as unversioned', async () => { + type UnversionedHdState = { oldField: string; existing: boolean }; + type HdStateV1 = { + oldField: string; + existing: boolean; + newField: string; + }; + + const migrations = createMigrations().add({ + migrate: (data: UnversionedHdState): HdStateV1 => ({ + ...data, + newField: 'added', + }), + }); + + const result = await migrations.apply({ + version: 0, + data: { oldField: 'value', existing: true }, + }); + + expect(result).toStrictEqual({ + version: 1, + data: { + oldField: 'value', + existing: true, + newField: 'added', + } satisfies HdStateV1, + migrated: true, + }); + }); + + it('throws when state version is newer than the chain', async () => { + const migrations = createMigrations().add({ migrate: (data) => data }); + + await expect(migrations.apply({ version: 5, data: {} })).rejects.toThrow( + 'State version 5 is newer than the latest migration version 1', + ); + }); + + it('throws when state version is negative', async () => { + const migrations = createMigrations().add({ migrate: (data) => data }); + + await expect(migrations.apply({ version: -1, data: {} })).rejects.toThrow( + 'State version -1 is invalid; it cannot be negative', + ); + }); + }); + + describe('when migrate is async', () => { + it('applies the step successfully', async () => { + type StateV1 = { foo: string; async: boolean }; + + const migrations = createMigrations().add({ + migrate: async (data: { foo: string }): Promise => { + await new Promise((resolve) => setTimeout(resolve, 1)); + return { ...data, async: true }; + }, + }); + + const result = await migrations.apply({ foo: 'bar' }); + + expect(result).toStrictEqual({ + version: 1, + data: { foo: 'bar', async: true } satisfies StateV1, + migrated: true, + }); + }); + }); + + describe('when migrate throws', () => { + it('propagates the error', async () => { + const migrations = createMigrations().add({ + migrate: (): never => { + throw new Error('Migration failed'); + }, + }); + + await expect(migrations.apply({})).rejects.toThrow('Migration failed'); + }); + }); + + describe('when a step declares an outputSchema', () => { + it('applies the step when the output matches the outputSchema', async () => { + const OutputSchema = object({ name: string(), count: number() }); + + const migrations = createMigrations().add({ + outputSchema: OutputSchema, + migrate: () => ({ name: 'test', count: 42 }), + }); + + const result = await migrations.apply({}); + + expect(result).toStrictEqual({ + version: 1, + data: { name: 'test', count: 42 }, + migrated: true, + }); + }); + + it('throws when the output does not match the outputSchema', async () => { + const OutputSchema = object({ name: string(), count: number() }); + type OutputState = Infer; + + const migrations = createMigrations().add({ + outputSchema: OutputSchema, + // @ts-expect-error - intentionally invalid return for test + migrate: (): OutputState => ({ name: 'test', count: 'not a number' }), + }); + + await expect(migrations.apply({})).rejects.toThrow('Expected a number'); + }); + + it('validates each step independently', async () => { + const V1Schema = object({ items: array(string()) }); + type StateV1 = Infer; + + const V2Schema = object({ items: array(string()), total: number() }); + type StateV2 = Infer; + + const migrations = createMigrations() + .add({ + outputSchema: V1Schema, + migrate: (): StateV1 => ({ items: ['a', 'b'] }), + }) + .add({ + outputSchema: V2Schema, + migrate: (data): StateV2 => ({ ...data, total: data.items.length }), + }); + + const result = await migrations.apply({}); + + expect(result).toStrictEqual({ + version: 2, + data: { items: ['a', 'b'], total: 2 } satisfies StateV2, + migrated: true, + }); + }); + }); + + describe('when a step declares an inputSchema', () => { + it('applies the step when input matches the inputSchema', async () => { + const V0Schema = object({ oldCount: number() }); + + const migrations = createMigrations().add({ + inputSchema: V0Schema, + migrate: (data) => ({ count: data.oldCount }), + }); + + const result = await migrations.apply({ oldCount: 7 }); + + expect(result).toStrictEqual({ + version: 1, + data: { count: 7 }, + migrated: true, + }); + }); + + it('throws before calling migrate when input does not match the inputSchema', async () => { + const V0Schema = object({ oldCount: number() }); + const migrateFn = jest.fn(); + + const migrations = createMigrations().add({ + inputSchema: V0Schema, + migrate: migrateFn, + }); + + await expect(migrations.apply({ wrongField: 'oops' })).rejects.toThrow( + 'Expected a number', + ); + expect(migrateFn).not.toHaveBeenCalled(); + }); + + it('validates input as JSON even when inputSchema is omitted', async () => { + const migrateFn = jest.fn(); + const migrations = createMigrations().add({ migrate: migrateFn }); + + // undefined is not valid JSON + await expect( + migrations.apply(undefined as unknown as Json), + ).rejects.toThrow( + 'Expected a value of type `JSON`, but received: `undefined`', + ); + expect(migrateFn).not.toHaveBeenCalled(); + }); + }); +}); diff --git a/packages/keyring-sdk/src/migration.ts b/packages/keyring-sdk/src/migration.ts new file mode 100644 index 000000000..9bab37321 --- /dev/null +++ b/packages/keyring-sdk/src/migration.ts @@ -0,0 +1,219 @@ +import type { Infer, Struct } from '@metamask/superstruct'; +import { assert, integer, is, object } from '@metamask/superstruct'; +import { JsonStruct } from '@metamask/utils'; +import type { Json } from '@metamask/utils'; + +/** + * Superstruct schema for the versioned state envelope. + */ +export const VersionedStateStruct = object({ + version: integer(), + data: JsonStruct, +}); + +/** + * Versioned state envelope wrapping the actual keyring data. + * + * After migrations are applied, state is always wrapped in this format. `serialize()` + * should produce this envelope so that subsequent `deserialize()` calls can detect the + * version. + */ +export type VersionedState = Omit< + Infer, + 'data' +> & { + data: Data; +}; + +/** + * Return value of {@link MigrationChain.apply}. + * + * Extends {@link VersionedState} with a `migrated` flag that is `true` when at least + * one step was applied during the call. Callers can use this to detect that the + * in-memory state has been upgraded and schedule a persist so the new version is + * written to storage, even when no other state change happens in the session. + */ +export type MigrationResult = VersionedState & { + migrated: boolean; +}; + +/** + * Type guard to check if a value is a {@link VersionedState} envelope. + * + * @param state - The value to check. + * @returns `true` if the value is a versioned state envelope. + */ +export function isVersionedState( + state: State | VersionedState, +): state is VersionedState { + return is(state, VersionedStateStruct); +} + +/** + * Get the version and data from state, treating unversioned state as version 0. + * + * @param state - The state to check. + * @returns The version and data. + */ +function getVersionAndData( + state: State | VersionedState, +): VersionedState { + return isVersionedState(state) + ? { version: state.version, data: state.data } + : { version: 0, data: state }; +} + +/** + * A single migration step, added to a {@link MigrationChain} via `.add()`. + * + * `Input` is bound automatically to the chain's current data type when the step is + * passed to `.add()`, so `migrate` receives a correctly typed argument with no manual + * cast. + */ +export type MigrationStep< + Input extends Json = Json, + Output extends Json = Json, +> = { + /** + * Transform state from the previous step's output to this step's output. + * + * Receives the raw inner data (not the versioned envelope). May be sync or async to + * support complex operations like re-deriving data from secrets. + * + * @param data - The state data from the previous step. + * @returns The migrated data. + */ + migrate(data: Input): Output | Promise; + /** + * Optional schema validating this step's input before `migrate` is called. Defaults + * to a generic JSON-shape check when omitted. + */ + inputSchema?: Struct; + /** + * Optional schema validating this step's output at runtime. + */ + outputSchema?: Struct; +}; + +/** + * A chain of migration steps for evolving keyring serialized state across versions. + * + * Steps are versioned by position: the first `.add()` call produces version 1, the + * second version 2, and so on. Create one with {@link createMigrations}. + */ +export type MigrationChain = { + /** + * The number of steps added so far. Since steps are versioned by position starting at + * 1, this also doubles as "the latest version" (use it in `serialize()`). + */ + readonly version: number; + /** + * Append a step to the chain. + * + * Returns a new chain typed to `Output`. It does not mutate the chain it's called on, + * so branching from a shared base chain is safe. + * + * `Input` defaults to the chain's current `Data` type. Providing `inputSchema` lets + * TypeScript infer a narrower `Input` (bounded to extend `Data`), so `migrate` + * receives a schema-typed argument when narrowing raw state into a specific shape, + * typically on a chain's first step, where `Data` is `Json`. + * + * @param step - The migration step to append. + * @returns A new chain whose data type is the step's `Output`. + */ + add( + step: MigrationStep, + ): MigrationChain; + /** + * Apply all pending steps to `state`. + * + * Handles both versioned state (wrapped in `{ version, data }` envelope) and + * unversioned legacy state (treated as version 0). + * + * @param state - The serialized keyring state (from vault or previous serialize). + * @returns The migrated state wrapped in a versioned envelope, plus a `migrated` + * flag. + * @throws If `state`'s version is newer than this chain's latest version, or if a + * step's `inputSchema`/`outputSchema` validation fails. + */ + apply(state: Json): Promise>; +}; + +/** + * Apply the pending steps of a chain to `state`. + * + * Implements {@link MigrationChain.apply} for the chain built from `steps`. + * + * @param steps - All steps of the chain. + * @param state - The serialized keyring state. + * @returns The migrated state wrapped in a versioned envelope, plus a `migrated` flag. + */ +async function applySteps( + steps: readonly MigrationStep[], + state: Json, +): Promise> { + const latestVersion = steps.length; + const { version, data: initialData } = getVersionAndData(state); + + if (version < 0) { + throw new Error( + `State version ${version} is invalid; it cannot be negative`, + ); + } + + if (version > latestVersion) { + throw new Error( + `State version ${version} is newer than the latest migration version ${latestVersion}`, + ); + } + + const pendingSteps = steps.slice(version); + let data = initialData; + + for (const step of pendingSteps) { + assert(data, step.inputSchema ?? JsonStruct); + data = await step.migrate(data); + assert(data, step.outputSchema ?? JsonStruct); + } + + return { + version: latestVersion, + data, + migrated: pendingSteps.length > 0, + } as MigrationResult; +} + +/** + * Build a {@link MigrationChain} wrapping the given internal steps. + * + * @param steps - The steps accumulated so far. + * @returns A chain exposing `add`, `version`, and `apply`. + */ +function buildChain( + steps: readonly MigrationStep[], +): MigrationChain { + return { + version: steps.length, + add: ( + step: MigrationStep, + ) => buildChain([...steps, step as unknown as MigrationStep]), + apply: async (state) => applySteps(steps, state), + }; +} + +/** + * Start a new, empty migration chain. + * + * @example + * ```typescript + * const migrations = createMigrations() + * .add({ migrate: (data) => ({ count: data.numberOfItems }) }) + * .add({ migrate: (data) => ({ ...data, createdAt: Date.now() }) }); // data typed as the first step's output, no cast + * + * const { data } = await migrations.apply(state); + * ``` + * @returns An empty chain, typed to accept `Json` as its first step's input. + */ +export function createMigrations(): MigrationChain { + return buildChain([]); +}