Skip to content

Commit 3bfc888

Browse files
Lolle2000laKuuuube
Lolle2000la
and
Kuuuube
authored
Fix LNA issues when using local audio (#2448)
* Proxy local audio fetch via background and play using Web Audio API * Use runtime type guards, shared AudioContext, and toError() utility * Support IPv6 loopback and HTTPS for local audio URL detection * Adjust copyright year * Make isLocalhostUrl an export from utilities * Remove redundant _isLocalUrl * Use existing api messaging channels to communicate with backend * Remove handlers from background-main * Allow AudioSystem to be initialized without API * Fallback to regular fetching of audio if api is not present instead of erroring * Use promises instead of callbacks * Remove unused --------- Co-authored-by: kuuuube <hexagonisalie@gmail.com>
1 parent c0c3702 commit 3bfc888

10 files changed

Lines changed: 206 additions & 31 deletions

File tree

‎ext/js/background/backend.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -189,6 +189,7 @@ export class Backend {
189189
['getLanguageSummaries', this._onApiGetLanguageSummaries.bind(this)],
190190
['heartbeat', this._onApiHeartbeat.bind(this)],
191191
['forceSync', this._onApiForceSync.bind(this)],
192+
['fetchLocalAudioData', this._onApiFetchLocalAudioData.bind(this)],
192193
]);
193194

194195
/** @type {import('api').PmApiMap} */
@@ -1167,6 +1168,29 @@ export class Backend {
11671168
return void 0;
11681169
}
11691170

1171+
/** @type {import('api').ApiHandler<'fetchLocalAudioData'>} */
1172+
async _onApiFetchLocalAudioData({url}) {
1173+
const response = await fetch(url);
1174+
if (!response.ok) {
1175+
log.error(`Local server responded with HTTP status code ${response.status}`);
1176+
return null;
1177+
}
1178+
1179+
const contentType = response.headers.get('content-type') || 'audio/mpeg';
1180+
const arrayBuffer = await response.arrayBuffer();
1181+
1182+
let binary = '';
1183+
const bytes = new Uint8Array(arrayBuffer);
1184+
for (let i = 0; i < bytes.byteLength; i++) {
1185+
binary += String.fromCharCode(bytes[i]);
1186+
}
1187+
1188+
return {
1189+
data: btoa(binary),
1190+
contentType: contentType,
1191+
};
1192+
}
1193+
11701194
// Command handlers
11711195

11721196
/**

‎ext/js/comm/api.js‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -435,6 +435,14 @@ export class API {
435435
return this._invoke('forceSync', void 0);
436436
}
437437

438+
/**
439+
* @param {string} url
440+
* @returns {Promise<{data: string, contentType: string}|null>}
441+
*/
442+
fetchLocalAudioData(url) {
443+
return this._invoke('fetchLocalAudioData', {url});
444+
}
445+
438446
// Utilities
439447

440448
/**

‎ext/js/core/utilities.js‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -344,3 +344,27 @@ export async function unsafeArrayBufferDigest(algorithm, arrayBuffer) {
344344
// @ts-expect-error - Allow SHA-1 here
345345
return arrayBufferDigest(algorithm, arrayBuffer);
346346
}
347+
348+
/**
349+
* @param {string} urlString
350+
* @returns {boolean}
351+
*/
352+
export function isLocalhostUrl(urlString) {
353+
try {
354+
const url = new URL(urlString);
355+
switch (url.hostname.toLowerCase()) {
356+
case 'localhost':
357+
case '127.0.0.1':
358+
case '[::1]':
359+
switch (url.protocol.toLowerCase()) {
360+
case 'http:':
361+
case 'https:':
362+
return true;
363+
}
364+
break;
365+
}
366+
} catch (e) {
367+
// NOP
368+
}
369+
return false;
370+
}

‎ext/js/display/display-audio.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@ export class DisplayAudio {
3232
/** @type {?import('display-audio').GenericAudio} */
3333
this._audioPlaying = null;
3434
/** @type {AudioSystem} */
35-
this._audioSystem = new AudioSystem();
35+
this._audioSystem = new AudioSystem(this._display.application.api);
3636
/** @type {number} */
3737
this._playbackVolume = 1;
3838
/** @type {boolean} */

‎ext/js/media/audio-system.js‎

Lines changed: 23 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -16,19 +16,27 @@
1616
* along with this program. If not, see <https://www.gnu.org/licenses/>.
1717
*/
1818

19+
import {API} from '../comm/api.js';
1920
import {EventDispatcher} from '../core/event-dispatcher.js';
21+
import {isLocalhostUrl} from '../core/utilities.js';
2022
import {TextToSpeechAudio} from './text-to-speech-audio.js';
23+
import {WebAudioLocalAudio} from './web-audio-local-audio.js';
2124

2225
/**
2326
* @augments EventDispatcher<import('audio-system').Events>
2427
*/
2528
export class AudioSystem extends EventDispatcher {
26-
constructor() {
29+
/**
30+
* @param {API?} api
31+
*/
32+
constructor(api) {
2733
super();
2834
/** @type {?HTMLAudioElement} */
2935
this._fallbackAudio = null;
3036
/** @type {?import('settings').FallbackSoundType} */
3137
this._fallbackSoundType = null;
38+
/** @type {API?} */
39+
this._api = api;
3240
}
3341

3442
/**
@@ -70,9 +78,22 @@ export class AudioSystem extends EventDispatcher {
7078
/**
7179
* @param {string} url
7280
* @param {import('settings').AudioSourceType} sourceType
73-
* @returns {Promise<HTMLAudioElement>}
81+
* @returns {Promise<HTMLAudioElement|WebAudioLocalAudio>}
7482
*/
7583
async createAudio(url, sourceType) {
84+
if (isLocalhostUrl(url) && this._api) {
85+
/** @type {{data: string, contentType: string}|null} */
86+
const response = await this._api.fetchLocalAudioData(url);
87+
88+
if (!response) {
89+
throw new Error('Failed to fetch local audio from background context');
90+
}
91+
92+
const localAudio = new WebAudioLocalAudio(response.data, response.contentType || 'audio/mpeg');
93+
await localAudio.prepare();
94+
return localAudio;
95+
}
96+
7697
const audio = new Audio(url);
7798
await this._waitForData(audio);
7899
if (!this._isAudioValid(audio, sourceType)) {

‎ext/js/media/web-audio-local-audio.js‎

Lines changed: 114 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,114 @@
1+
/*
2+
* Copyright (C) 2026 Yomitan Authors
3+
*
4+
* This program is free software: you can redistribute it and/or modify
5+
* it under the terms of the GNU General Public License as published by
6+
* the Free Software Foundation, either version 3 of the License, or
7+
* (at your option) any later version.
8+
*
9+
* This program is distributed in the hope that it will be useful,
10+
* but WITHOUT ANY WARRANTY; without even the implied warranty of
11+
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
12+
* GNU General Public License for more details.
13+
*
14+
* You should have received a copy of the GNU General Public License
15+
* along with this program. If not, see <https://www.gnu.org/licenses/>.
16+
*/
17+
18+
19+
/** @type {?AudioContext} */
20+
let sharedAudioContext = null;
21+
22+
/**
23+
* @returns {AudioContext}
24+
*/
25+
function getSharedAudioContext() {
26+
if (!sharedAudioContext || sharedAudioContext.state === 'closed') {
27+
sharedAudioContext = new AudioContext();
28+
}
29+
return sharedAudioContext;
30+
}
31+
32+
export class WebAudioLocalAudio {
33+
/**
34+
* @param {string} base64Data
35+
* @param {string} contentType
36+
*/
37+
constructor(base64Data, contentType) {
38+
/** @type {string} */
39+
this._base64Data = base64Data;
40+
/** @type {string} */
41+
this._contentType = contentType;
42+
/** @type {number} */
43+
this._volume = 1;
44+
/** @type {number} */
45+
this._currentTime = 0;
46+
/** @type {AudioContext} */
47+
this._audioContext = getSharedAudioContext();
48+
/** @type {?AudioBufferSourceNode} */
49+
this._bufferSource = null;
50+
/** @type {?GainNode} */
51+
this._gainNode = null;
52+
/** @type {?AudioBuffer} */
53+
this._decodedBuffer = null;
54+
}
55+
56+
/** @type {number} */
57+
get currentTime() { return this._currentTime; }
58+
59+
set currentTime(value) { this._currentTime = value; }
60+
61+
/** @type {number} */
62+
get volume() { return this._volume; }
63+
64+
set volume(value) {
65+
this._volume = value;
66+
if (this._gainNode) { this._gainNode.gain.value = value; }
67+
}
68+
69+
/** @type {number} */
70+
get duration() { return this._decodedBuffer ? this._decodedBuffer.duration : 0; }
71+
72+
/** */
73+
async prepare() {
74+
const byteCharacters = atob(this._base64Data);
75+
const byteNumbers = new Array(byteCharacters.length);
76+
for (let i = 0; i < byteCharacters.length; i++) {
77+
byteNumbers[i] = byteCharacters.charCodeAt(i);
78+
}
79+
const byteArray = new Uint8Array(byteNumbers);
80+
81+
this._decodedBuffer = await this._audioContext.decodeAudioData(byteArray.buffer);
82+
}
83+
84+
/**
85+
* @returns {Promise<void>}
86+
*/
87+
async play() {
88+
if (!this._decodedBuffer || !this._audioContext) { return; }
89+
if (this._audioContext.state === 'suspended') {
90+
await this._audioContext.resume();
91+
}
92+
this.pause();
93+
94+
this._bufferSource = this._audioContext.createBufferSource();
95+
this._bufferSource.buffer = this._decodedBuffer;
96+
97+
this._gainNode = this._audioContext.createGain();
98+
this._gainNode.gain.value = this._volume;
99+
100+
this._bufferSource.connect(this._gainNode);
101+
this._gainNode.connect(this._audioContext.destination);
102+
this._bufferSource.start(0, this._currentTime);
103+
}
104+
105+
/**
106+
* @returns {void}
107+
*/
108+
pause() {
109+
if (this._bufferSource) {
110+
try { this._bufferSource.stop(); } catch (e) { /* NOP */ }
111+
this._bufferSource = null;
112+
}
113+
}
114+
}

‎ext/js/pages/settings/audio-controller.js‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ export class AudioController extends EventDispatcher {
3636
/** @type {import('./modal-controller.js').ModalController} */
3737
this._modalController = modalController;
3838
/** @type {AudioSystem} */
39-
this._audioSystem = new AudioSystem();
39+
this._audioSystem = new AudioSystem(null);
4040
/** @type {HTMLElement} */
4141
this._audioSourceContainer = querySelectorNotNull(document, '#audio-source-list');
4242
/** @type {HTMLButtonElement} */

‎ext/js/pages/settings/backup-controller.js‎

Lines changed: 3 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@ import {parseJson} from '../../core/json.js';
2222
import {log} from '../../core/log.js';
2323
import {isObjectNotArray} from '../../core/object-utilities.js';
2424
import {toError} from '../../core/to-error.js';
25+
import {isLocalhostUrl} from '../../core/utilities.js';
2526
import {arrayBufferUtf8Decode} from '../../data/array-buffer-util.js';
2627
import {OptionsUtil} from '../../data/options-util.js';
2728
import {getAllPermissions} from '../../data/permissions-util.js';
@@ -325,30 +326,6 @@ export class BackupController {
325326
});
326327
}
327328

328-
/**
329-
* @param {string} urlString
330-
* @returns {boolean}
331-
*/
332-
_isLocalhostUrl(urlString) {
333-
try {
334-
const url = new URL(urlString);
335-
switch (url.hostname.toLowerCase()) {
336-
case 'localhost':
337-
case '127.0.0.1':
338-
case '[::1]':
339-
switch (url.protocol.toLowerCase()) {
340-
case 'http:':
341-
case 'https:':
342-
return true;
343-
}
344-
break;
345-
}
346-
} catch (e) {
347-
// NOP
348-
}
349-
return false;
350-
}
351-
352329
/**
353330
* @param {import('settings').ProfileOptions} options
354331
* @param {boolean} dryRun
@@ -367,7 +344,7 @@ export class BackupController {
367344
}
368345
}
369346
const server = anki.server;
370-
if (typeof server === 'string' && server.length > 0 && !this._isLocalhostUrl(server)) {
347+
if (typeof server === 'string' && server.length > 0 && !isLocalhostUrl(server)) {
371348
warnings.push('anki.server uses a non-localhost URL');
372349
if (!dryRun) {
373350
anki.server = 'http://127.0.0.1:8765';
@@ -383,7 +360,7 @@ export class BackupController {
383360
const source = sources[i];
384361
if (!isObjectNotArray(source)) { continue; }
385362
const {url} = source;
386-
if (typeof url === 'string' && url.length > 0 && !this._isLocalhostUrl(url)) {
363+
if (typeof url === 'string' && url.length > 0 && !isLocalhostUrl(url)) {
387364
warnings.push(`audio.sources[${i}].url uses a non-localhost URL`);
388365
if (!dryRun) {
389366
sources[i].url = '';

‎types/ext/api.d.ts‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -415,6 +415,12 @@ type ApiSurface = {
415415
params: void;
416416
return: void;
417417
};
418+
fetchLocalAudioData: {
419+
params: {
420+
url: string;
421+
};
422+
return: {data: string, contentType: string} | null;
423+
};
418424
};
419425

420426
type ApiExtraArgs = [sender: chrome.runtime.MessageSender];

‎types/ext/display-audio.d.ts‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
*/
1717

1818
import type {TextToSpeechAudio} from '../../ext/js/media/text-to-speech-audio';
19+
import type {WebAudioLocalAudio} from '../../ext/js/media/web-audio-local-audio';
1920
import type * as Audio from './audio';
2021
import type * as AudioDownloader from './audio-downloader';
2122
import type * as Settings from './settings';
@@ -92,7 +93,7 @@ export type CreateAudioResult = {
9293
cacheUpdated: boolean;
9394
};
9495

95-
export type GenericAudio = HTMLAudioElement | TextToSpeechAudio;
96+
export type GenericAudio = HTMLAudioElement | TextToSpeechAudio | WebAudioLocalAudio;
9697

9798
export type MenuItemEntry = {
9899
valid: boolean | null;

0 commit comments

Comments
 (0)