fix(local-inference): harden desktop local ASR path

Author: NubsCarson

packages/app-core/src/api/server-cors.test.ts

@@ -0,0 +1,12 @@
+import { describe, expect, it } from "vitest";
+import { isAllowedOrigin } from "./server-cors";
+
+describe("server CORS origin allowlist", () => {
+  it("allows the packaged Electrobun views scheme used by the desktop renderer", () => {
+    expect(isAllowedOrigin("views://")).toBe(true);
+  });
+
+  it("continues to reject untrusted custom browser schemes", () => {
+    expect(isAllowedOrigin("evil://localhost")).toBe(false);
+  });
+});

packages/app-core/src/api/server-cors.ts

@@ -91,6 +91,7 @@ const CAPACITOR_WEBVIEW_ORIGINS: ReadonlySet<string> = new Set([
  * these origins; they are used by packaged/native app shells.
  */
 const NATIVE_WEBVIEW_PROTOCOLS: ReadonlySet<string> = new Set([
+  "views:",
   "capacitor:",
   "capacitor-electron:",
   "ionic:",

packages/ui/src/voice/local-asr-capture.test.ts

@@ -0,0 +1,34 @@
+import { describe, expect, it } from "vitest";
+import {
+  encodeMonoPcm16Wav,
+  isSilentPcmAudio,
+  measurePcmAudio,
+} from "./local-asr-capture";
+
+describe("local ASR capture", () => {
+  it("detects truly silent PCM before sending it to ASR", () => {
+    const pcm = new Float32Array(16000);
+
+    expect(measurePcmAudio(pcm)).toEqual({ rms: 0, peak: 0 });
+    expect(isSilentPcmAudio(pcm)).toBe(true);
+  });
+
+  it("keeps low but real microphone signal eligible for ASR", () => {
+    const pcm = new Float32Array(16000);
+    pcm[1200] = 0.001;
+    pcm[1201] = -0.001;
+
+    expect(measurePcmAudio(pcm).peak).toBeCloseTo(0.001);
+    expect(isSilentPcmAudio(pcm)).toBe(false);
+  });
+
+  it("encodes mono PCM16 WAV with the requested sample rate", () => {
+    const wav = encodeMonoPcm16Wav(new Float32Array([0, 1, -1]), 16000);
+    const view = new DataView(wav.buffer);
+
+    expect(String.fromCharCode(...wav.slice(0, 4))).toBe("RIFF");
+    expect(String.fromCharCode(...wav.slice(8, 12))).toBe("WAVE");
+    expect(view.getUint32(24, true)).toBe(16000);
+    expect(view.getUint32(40, true)).toBe(6);
+  });
+});

packages/ui/src/voice/local-asr-capture.ts

@@ -3,6 +3,11 @@ export interface LocalAsrRecorder {
   cancel(): void;
 }
 
+export interface PcmAudioStats {
+  rms: number;
+  peak: number;
+}
+
 type AudioContextConstructor = typeof AudioContext;
 
 type WindowWithAudioContext = Window & {
@@ -45,6 +50,27 @@ function clampPcm16(value: number): number {
   return Math.max(-1, Math.min(1, value));
 }
 
+export function measurePcmAudio(pcm: Float32Array): PcmAudioStats {
+  let sumSquares = 0;
+  let peak = 0;
+  for (const sample of pcm) {
+    if (!Number.isFinite(sample)) continue;
+    const abs = Math.abs(sample);
+    peak = Math.max(peak, abs);
+    sumSquares += sample * sample;
+  }
+
+  return {
+    rms: Math.sqrt(sumSquares / Math.max(1, pcm.length)),
+    peak,
+  };
+}
+
+export function isSilentPcmAudio(pcm: Float32Array): boolean {
+  const { rms, peak } = measurePcmAudio(pcm);
+  return peak < 0.0001 && rms < 0.00001;
+}
+
 export function encodeMonoPcm16Wav(
   pcm: Float32Array,
   sampleRateHz: number,
@@ -154,6 +180,11 @@ export async function startLocalAsrRecorder(): Promise<LocalAsrRecorder> {
       if (pcm.length === 0) {
         throw new Error("No microphone audio was captured for local ASR");
       }
+      if (isSilentPcmAudio(pcm)) {
+        throw new Error(
+          "Microphone audio was silent; check the selected input device and try again",
+        );
+      }
       return encodeMonoPcm16Wav(pcm, sampleRate);
     },
     cancel() {

plugins/plugin-local-inference/src/services/voice/engine-bridge.ts

@@ -93,9 +93,11 @@ import {
 	SpeakerPresetCache,
 } from "./speaker-preset-cache";
 import {
+	type AsrBackendPreference,
 	ASR_SAMPLE_RATE,
 	AsrUnavailableError,
 	createStreamingTranscriber,
+	readAsrBackendPreferenceFromEnv,
 	resampleLinear,
 } from "./transcriber";
 import type {
@@ -1488,6 +1490,8 @@ export class EngineVoiceBridge {
 	 */
 	createStreamingTranscriber(opts?: {
 		vad?: VadEventSource;
+		prefer?: AsrBackendPreference;
+		allowOpenVinoWhisper?: boolean;
 	}): StreamingTranscriber {
 		this.assertVoiceOn("create streaming transcriber");
 		const contextRef = this.ffiContextRef;
@@ -1496,6 +1500,8 @@ export class EngineVoiceBridge {
 			getContext: contextRef ? () => contextRef.ensure() : undefined,
 			asrBundlePresent: this.asrAvailable,
 			vad: opts?.vad,
+			prefer: opts?.prefer,
+			allowOpenVinoWhisper: opts?.allowOpenVinoWhisper,
 		});
 	}
 
@@ -1518,6 +1524,36 @@ export class EngineVoiceBridge {
 				? signal.reason
 				: new DOMException("Aborted", "AbortError");
 		}
+		const transcribeWithStreaming = async (
+			prefer?: AsrBackendPreference,
+		): Promise<string> => {
+			const transcriber = this.createStreamingTranscriber(
+				prefer ? { prefer } : undefined,
+			);
+			const abort = () => transcriber.dispose();
+			try {
+				signal?.addEventListener("abort", abort, { once: true });
+				transcriber.feed({
+					pcm: args.pcm,
+					sampleRate: args.sampleRate,
+					timestampMs: 0,
+				});
+				const final = await transcriber.flush();
+				if (signal?.aborted) {
+					throw signal.reason instanceof Error
+						? signal.reason
+						: new DOMException("Aborted", "AbortError");
+				}
+				return final.partial;
+			} finally {
+				signal?.removeEventListener("abort", abort);
+				transcriber.dispose();
+			}
+		};
+		const asrPreference = readAsrBackendPreferenceFromEnv();
+		if (asrPreference === "openvino-whisper") {
+			return await transcribeWithStreaming("openvino-whisper");
+		}
 		const backendBatch = this.backend as OmniVoiceBackend & {
 			transcribe?: (args: TranscriptionAudio) => Promise<string>;
 		};
@@ -1554,26 +1590,7 @@ export class EngineVoiceBridge {
 			}
 			return transcript;
 		}
-		const transcriber = this.createStreamingTranscriber();
-		const abort = () => transcriber.dispose();
-		try {
-			signal?.addEventListener("abort", abort, { once: true });
-			transcriber.feed({
-				pcm: args.pcm,
-				sampleRate: args.sampleRate,
-				timestampMs: 0,
-			});
-			const final = await transcriber.flush();
-			if (signal?.aborted) {
-				throw signal.reason instanceof Error
-					? signal.reason
-					: new DOMException("Aborted", "AbortError");
-			}
-			return final.partial;
-		} finally {
-			signal?.removeEventListener("abort", abort);
-			transcriber.dispose();
-		}
+		return await transcribeWithStreaming(asrPreference ?? undefined);
 	}
 
 	/**

plugins/plugin-local-inference/src/services/voice/openvino-whisper-asr.test.ts

@@ -69,6 +69,9 @@ beforeEach(() => {
 
 afterEach(() => {
 	mockState.runtimeFixture = null;
+	vi.unstubAllEnvs();
+	delete process.env.ELIZA_LOCAL_ASR_BACKEND;
+	delete process.env.ELIZA_LOCAL_ASR_ALLOW_OPENVINO;
 });
 
 describe("createStreamingTranscriber — OpenVINO Whisper tier", () => {

plugins/plugin-local-inference/src/services/voice/transcriber.ts

@@ -68,6 +68,59 @@ export class AsrUnavailableError extends Error {
 	}
 }
 
+export type AsrBackendPreference =
+	| "auto"
+	| "fused"
+	| "ffi-batch"
+	| "openvino-whisper";
+
+function normalizeBooleanEnv(value: string | undefined): boolean {
+	const normalized = value?.trim().toLowerCase();
+	return (
+		normalized === "1" ||
+		normalized === "true" ||
+		normalized === "yes" ||
+		normalized === "on"
+	);
+}
+
+export function normalizeAsrBackendPreference(
+	value: string | null | undefined,
+): AsrBackendPreference | null {
+	const normalized = value?.trim().toLowerCase().replace(/_/g, "-");
+	if (!normalized) return null;
+	switch (normalized) {
+		case "auto":
+			return "auto";
+		case "fused":
+		case "streaming":
+		case "fused-streaming":
+			return "fused";
+		case "batch":
+		case "ffi-batch":
+		case "fused-batch":
+			return "ffi-batch";
+		case "openvino":
+		case "openvino-whisper":
+		case "whisper-openvino":
+			return "openvino-whisper";
+		default:
+			return null;
+	}
+}
+
+export function readAsrBackendPreferenceFromEnv(
+	env: NodeJS.ProcessEnv = process.env,
+): AsrBackendPreference | null {
+	return normalizeAsrBackendPreference(env.ELIZA_LOCAL_ASR_BACKEND);
+}
+
+function allowOpenVinoWhisperFromEnv(
+	env: NodeJS.ProcessEnv = process.env,
+): boolean {
+	return normalizeBooleanEnv(env.ELIZA_LOCAL_ASR_ALLOW_OPENVINO);
+}
+
 /* ==================================================================== *
  * Shared base — event fan-out, VAD gating, word detection.
  * ==================================================================== */
@@ -744,7 +797,7 @@ export interface CreateStreamingTranscriberOptions {
 	 *   `"auto"`             (default) → fused streaming → fused batch →
 	 *                                    OpenVINO whisper (when enabled) → throw.
 	 */
-	prefer?: "auto" | "fused" | "ffi-batch" | "openvino-whisper";
+	prefer?: AsrBackendPreference;
 	/**
 	 * Permit the OpenVINO Whisper adapter (NPU→CPU autoprobe). Off by default
 	 * — Eliza-1 voice bridges run only the fused path. Set explicitly to `true`
@@ -771,8 +824,11 @@ export interface CreateStreamingTranscriberOptions {
 export function createStreamingTranscriber(
 	opts: CreateStreamingTranscriberOptions = {},
 ): StreamingTranscriber {
-	const prefer = opts.prefer ?? "auto";
-	const allowOpenVinoWhisper = opts.allowOpenVinoWhisper === true;
+	const prefer = opts.prefer ?? readAsrBackendPreferenceFromEnv() ?? "auto";
+	const allowOpenVinoWhisper =
+		opts.allowOpenVinoWhisper === true ||
+		prefer === "openvino-whisper" ||
+		allowOpenVinoWhisperFromEnv();
 
 	const tryFusedStreaming = (): StreamingTranscriber | null => {
 		if (!opts.ffi || !opts.getContext) return null;