fix(ocap-kernel): enforce one delivery per crank, fix rollback cache staleness

- Restructure run queue generator to yield exactly one item per
  startCrank/endCrank pair, preventing rollback from undoing
  unrelated earlier deliveries in the same crank
- Refresh StoredQueue after rollback so cached head/tail pointers
  are re-read from DB, fixing dequeue returning undefined
- Invalidate runQueueLengthCache after rollback
- Bypass VatManager.terminateVat() in KernelQueue callback to avoid
  waitForCrank() deadlock when terminating from within a crank
- Handle vanished endpoints in KernelRouter.deliverSend with
  try/catch, treating as splat instead of crashing
- Change KernelQueue subscriptions to {resolve, reject} so aborted
  sends can reject the caller's JS promise immediately
- Distinguish rejected vs fulfilled in invokeKernelSubscription
- Improve splat error messages to describe cause without leaking
  internal identifiers (krefs, endpoint IDs)
- Add integration test for orphaned ephemeral exo rejection
- Standardize KernelQueue test loop-exit pattern using sentinel

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

Author: rekmarks

docs/glossary.md

@@ -66,8 +66,9 @@ runtime environment for vat code and handles object persistence, promise managem
 ### crank
 
 A single execution cycle in the kernel's [run queue](#run-queue). Each crank processes one
-item from the run queue, delivering a single message or notification to [a vat](#vat).
-Cranks can be aborted and rolled back if errors occur. See the
+item from the run queue, delivering a single message or notification to [a vat](#vat). The
+"message or notification" is whatever item is taken of the run queue. Cranks can be
+aborted and rolled back if errors occur. See the
 [KernelQueue](../packages/ocap-kernel/src/KernelQueue.ts) for the run loop implementation.
 
 ### syscall

packages/kernel-node-runtime/test/e2e/orphaned-ephemeral-exo.test.ts

@@ -55,10 +55,11 @@ describe('Orphaned ephemeral exo', { timeout: 30_000 }, () => {
         // The consumer's E(ephemeral).increment() targets an orphaned vref.
         // Liveslots in the provider throws "I don't remember allocating",
         // which terminates the provider and rejects the caller's promise.
+        // This is surfaced to the caller as "target object has no owner".
         await expect(
           kernel.queueMessage(rootKref, 'useEphemeral', []),
         ).rejects.toMatchObject({
-          body: expect.stringContaining("I don't remember allocating"),
+          body: expect.stringContaining('target object has no owner'),
         });
       } finally {
         await kernel.stop();

packages/kernel-test/src/orphaned-ephemeral-exo.test.ts

@@ -0,0 +1,50 @@
+import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs';
+import { waitUntilQuiescent } from '@metamask/kernel-utils';
+import { kunser } from '@metamask/ocap-kernel';
+import { describe, expect, it } from 'vitest';
+
+import { getBundleSpec, makeKernel, makeTestLogger } from './utils.ts';
+
+describe('orphaned ephemeral exo', () => {
+  it('rejects when provider vat restarts', async () => {
+    const { logger } = makeTestLogger();
+    const database = await makeSQLKernelDatabase({});
+    const kernel = await makeKernel(database, true, logger);
+
+    const { rootKref, subclusterId } = await kernel.launchSubcluster({
+      bootstrap: 'consumer',
+      vats: {
+        provider: {
+          bundleSpec: getBundleSpec('orphaned-ephemeral-provider'),
+          parameters: {},
+        },
+        consumer: {
+          bundleSpec: getBundleSpec('orphaned-ephemeral-consumer'),
+          parameters: {},
+        },
+      },
+    });
+    await waitUntilQuiescent();
+
+    // Works before restart
+    const r1 = await kernel.queueMessage(rootKref, 'useEphemeral', []);
+    expect(kunser(r1)).toBe(999);
+
+    // Restart only the provider — the consumer still holds the
+    // ephemeral ref, but the exo behind it no longer exists.
+    const subcluster = kernel.getSubcluster(subclusterId);
+    expect(subcluster).toBeDefined();
+    await kernel.restartVat(subcluster!.vats.provider);
+    await waitUntilQuiescent();
+
+    // The consumer's E(ephemeral).increment() targets an orphaned vref.
+    // Liveslots in the provider throws "I don't remember allocating",
+    // which terminates the provider vat. The message is retried in a new
+    // crank, but the endpoint is gone — so it splats and rejects.
+    await expect(
+      kernel.queueMessage(rootKref, 'useEphemeral', []),
+    ).rejects.toMatchObject({
+      body: expect.stringContaining('has no owner'),
+    });
+  });
+});

packages/kernel-test/src/syscall-validation.test.ts

@@ -94,10 +94,12 @@ describe('Syscall Validation & Revoked Objects', { timeout: 30_000 }, () => {
     // Revoke the object
     kernelStore.setRevoked(objectKRef, true);
     await waitUntilQuiescent();
-    // Try to send message to revoked object
-    const revokedResult = await kernel.queueMessage(objectKRef, 'getValue', []);
-    // Should get proper error response for revoked object
-    expect(revokedResult.body).toContain('revoked object');
+    // Try to send message to revoked object — kernel rejects the promise
+    await expect(
+      kernel.queueMessage(objectKRef, 'getValue', []),
+    ).rejects.toMatchObject({
+      body: expect.stringContaining('target object has been revoked'),
+    });
     // Verify kernel doesn't crash and exporter vat remains operational
     const exporterStatus = await kernel.queueMessage(exporterKRef, 'noop', []);
     expect(exporterStatus.body).toContain('noop');
@@ -140,21 +142,22 @@ describe('Syscall Validation & Revoked Objects', { timeout: 30_000 }, () => {
     // Revoke the object
     kernelStore.setRevoked(objectKRef, true);
     await waitUntilQuiescent();
-    // Send message to revoked object that would return a promise
-    const promiseResult = await kernel.queueMessage(objectKRef, 'getValue', []);
-    // Verify the promise is rejected with revocation error
-    expect(promiseResult.body).toContain('revoked object');
+    // Send message to revoked object — kernel rejects the promise
+    await expect(
+      kernel.queueMessage(objectKRef, 'getValue', []),
+    ).rejects.toMatchObject({
+      body: expect.stringContaining('target object has been revoked'),
+    });
     // Verify exporter vat is still operational
     const exporterStatus = await kernel.queueMessage(exporterKRef, 'noop', []);
     expect(exporterStatus.body).toContain('noop');
     // Verify kernel can handle multiple revoked object accesses
     for (let i = 0; i < 5; i++) {
-      const multipleResult = await kernel.queueMessage(
-        objectKRef,
-        'getValue',
-        [],
-      );
-      expect(multipleResult.body).toContain('revoked object');
+      await expect(
+        kernel.queueMessage(objectKRef, 'getValue', []),
+      ).rejects.toMatchObject({
+        body: expect.stringContaining('target object has been revoked'),
+      });
     }
     // Verify kernel remains stable
     const finalStatus = await kernel.queueMessage(exporterKRef, 'noop', []);
@@ -210,17 +213,14 @@ describe('Syscall Validation & Revoked Objects', { timeout: 30_000 }, () => {
     }
     await waitUntilQuiescent();
 
-    // Try to access all revoked objects
-    const revokedResults = await Promise.all(
-      objectKRefs.map(async (objectKRef) =>
+    // Try to access all revoked objects — all should reject
+    for (const objectKRef of objectKRefs) {
+      await expect(
         kernel.queueMessage(objectKRef, 'getValue', []),
-      ),
-    );
-
-    // All should return revocation errors
-    revokedResults.forEach((result) => {
-      expect(result.body).toContain('revoked object');
-    });
+      ).rejects.toMatchObject({
+        body: expect.stringContaining('target object has been revoked'),
+      });
+    }
 
     // Verify exporter vat is still operational
     const exporterStatus = await kernel.queueMessage(exporterKRef, 'noop', []);

packages/kernel-test/src/vat-lifecycle.test.ts

@@ -1,6 +1,6 @@
 import { makeSQLKernelDatabase } from '@metamask/kernel-store/sqlite/nodejs';
 import { waitUntilQuiescent } from '@metamask/kernel-utils';
-import { kunser, makeKernelStore } from '@metamask/ocap-kernel';
+import { makeKernelStore } from '@metamask/ocap-kernel';
 import { describe, expect, it, beforeEach } from 'vitest';
 
 import {
@@ -131,13 +131,12 @@ describe('Vat Lifecycle', { timeout: 30_000 }, () => {
     expect(remainingVats).toHaveLength(1);
     expect(remainingVats[0]?.id).toBe(liveVatId);
 
-    // Try to send a message to the terminated vat's root object
-    const messageResult = await kernel.queueMessage(
-      deadRootObject,
-      'resume',
-      [],
-    );
-    expect(kunser(messageResult)).toBe('no endpoint');
+    // Try to send a message to the terminated vat's root object — rejects
+    await expect(
+      kernel.queueMessage(deadRootObject, 'resume', []),
+    ).rejects.toMatchObject({
+      body: expect.stringContaining('has no owner'),
+    });
 
     // Verify that messaging works as expected
     expect(await runResume(kernel, liveRootObject)).toBe(

packages/kernel-test/src/vats/orphaned-ephemeral-consumer.ts

@@ -0,0 +1,23 @@
+import { E } from '@endo/eventual-send';
+import { makeDefaultExo } from '@metamask/kernel-utils/exo';
+
+/**
+ * A consumer vat that obtains an ephemeral exo reference from the provider
+ * during bootstrap and calls it on demand.
+ *
+ * @returns The root object.
+ */
+// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+export function buildRootObject() {
+  let ephemeralRef: unknown;
+
+  return makeDefaultExo('root', {
+    async bootstrap(vats: { provider: unknown }) {
+      ephemeralRef = await E(vats.provider).getEphemeral();
+    },
+
+    async useEphemeral() {
+      return E(ephemeralRef).increment();
+    },
+  });
+}

packages/kernel-test/src/vats/orphaned-ephemeral-provider.ts

@@ -0,0 +1,22 @@
+import { makeDefaultExo } from '@metamask/kernel-utils/exo';
+
+/**
+ * A provider vat that exposes a single ephemeral (non-durable) exo.
+ * The exo will not survive a vat restart.
+ *
+ * @returns The root object.
+ */
+// eslint-disable-next-line @typescript-eslint/explicit-function-return-type
+export function buildRootObject() {
+  const ephemeral = makeDefaultExo('EphemeralCounter', {
+    increment() {
+      return 999;
+    },
+  });
+
+  return makeDefaultExo('root', {
+    getEphemeral() {
+      return ephemeral;
+    },
+  });
+}

packages/ocap-kernel/src/Kernel.ts

@@ -128,9 +128,14 @@ export class Kernel {
       this.#resetKernelState({ resetIdentity: Boolean(options.mnemonic) });
     }
 
+    // Bypass VatManager.terminateVat() here because it calls waitForCrank(),
+    // which would deadlock — this callback is invoked from within a crank.
     this.#kernelQueue = new KernelQueue(
       this.#kernelStore,
-      async (vatId, reason) => this.#vatManager.terminateVat(vatId, reason),
+      async (vatId, reason) => {
+        await this.#vatManager.stopVat(vatId, true, reason);
+        this.#kernelStore.markVatAsTerminated(vatId);
+      },
     );
 
     this.#vatManager = new VatManager({