From 459b7c91242c64714c8cf479a32c0cae5f888852 Mon Sep 17 00:00:00 2001 From: zhifu gao <18321252+LauraGPT@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:18:48 +0000 Subject: [PATCH 1/3] docs(frontend): document local FunASR transcription --- autogpt_platform/frontend/.env.default | 5 ++ docs/platform/copilot-local-llm.md | 67 ++++++++++++++++++++++++++ 2 files changed, 72 insertions(+) diff --git a/autogpt_platform/frontend/.env.default b/autogpt_platform/frontend/.env.default index 593755240bbd..3f327dfade2e 100644 --- a/autogpt_platform/frontend/.env.default +++ b/autogpt_platform/frontend/.env.default @@ -43,3 +43,8 @@ NEXT_PUBLIC_VAPID_PUBLIC_KEY=BBg49iVTWthVbRYphwmZNvZyiSJDqtSO4nmLxDzLKe3Oo9jbtu0 # OpenAI (for voice transcription) OPENAI_API_KEY= + +# Optional OpenAI-compatible voice transcription endpoint +TRANSCRIPTION_API_BASE_URL= +TRANSCRIPTION_MODEL= +TRANSCRIPTION_API_KEY= diff --git a/docs/platform/copilot-local-llm.md b/docs/platform/copilot-local-llm.md index 03e6d01cd309..a848e3d47ea5 100644 --- a/docs/platform/copilot-local-llm.md +++ b/docs/platform/copilot-local-llm.md @@ -5,6 +5,8 @@ > Generator block (used inside agent graphs you build yourself), see > [Running Ollama with AutoGPT](ollama.md). The two paths read different > env vars, so configuring one does not configure the other. +> The optional voice transcription section below is also separate and +> reads server-side variables from the frontend environment. > > Self-hosting only — the cloud `agpt.co` deployment routes AutoPilot > through Anthropic / OpenRouter and ignores the variables below. @@ -83,6 +85,71 @@ CHAT_FAST_STANDARD_MODEL=hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M CHAT_FAST_ADVANCED_MODEL=qwen3:14b-q4_K_M ``` +## Optional: local voice transcription with FunASR + +AutoPilot's microphone input can use a self-hosted +[FunASR](https://github.com/modelscope/FunASR) server instead of the +default OpenAI transcription endpoint. FunASR exposes the compatible +`POST /v1/audio/transcriptions` route and uses SenseVoice by default. + +Install FunASR and start the server on the Docker host: + +```bash +python -m pip install -U funasr + +# CPU +funasr-server --model sensevoice --device cpu + +# Or, on a CUDA host +funasr-server --model sensevoice --device cuda +``` + +The first start downloads the model. The server listens on +`0.0.0.0:8000` and does not require an API key by default. Before +connecting AutoGPT, verify it with a local audio file: + +```bash +curl http://localhost:8000/v1/audio/transcriptions \ + -F file=@sample.wav \ + -F model=sensevoice +``` + +Keep port 8000 on a trusted host or private network. If the endpoint is +reachable from an untrusted network, put it behind an authenticated +HTTPS reverse proxy. + +Then add the transcription settings to +`autogpt_platform/frontend/.env`: + +```bash +# Docker Desktop on macOS or Windows +TRANSCRIPTION_API_BASE_URL=http://host.docker.internal:8000/v1 +TRANSCRIPTION_MODEL=sensevoice +TRANSCRIPTION_API_KEY= +``` + +On native Linux Docker, replace `host.docker.internal` with the host's +LAN or bridge-reachable IP. Alternatively, add +`host.docker.internal:host-gateway` to the frontend service's +`extra_hosts`. For frontend development outside Docker, use +`http://localhost:8000/v1`. + +Recreate the frontend container so the server-side transcription route +receives the new environment: + +```bash +cd autogpt_platform +docker compose up -d --force-recreate frontend +``` + +Leave `TRANSCRIPTION_API_KEY` empty for a default FunASR server. Set it +only when a gateway or reverse proxy in front of FunASR requires bearer +authentication. If the `TRANSCRIPTION_*` variables are absent, AutoGPT +continues to use OpenAI's `whisper-1` model and `OPENAI_API_KEY`. + +For model choices and deployment options, see the +[FunASR model selection guide](https://github.com/modelscope/FunASR/blob/main/docs/model_selection.md). + ## Picking a model The platform's chat loop calls **OpenAI-style tool-calling** on every From 40ca1b1a903b7f1b0cc0fb2a5fd5bf12bad4b8be Mon Sep 17 00:00:00 2001 From: zhifu gao <18321252+LauraGPT@users.noreply.github.com> Date: Thu, 23 Jul 2026 21:44:31 +0000 Subject: [PATCH 2/3] docs(frontend): address transcription review feedback --- autogpt_platform/frontend/.env.default | 2 +- .../src/app/api/transcribe/__tests__/route.test.ts | 4 ++-- docs/platform/copilot-local-llm.md | 10 +++++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/autogpt_platform/frontend/.env.default b/autogpt_platform/frontend/.env.default index 3f327dfade2e..4067587060ac 100644 --- a/autogpt_platform/frontend/.env.default +++ b/autogpt_platform/frontend/.env.default @@ -46,5 +46,5 @@ OPENAI_API_KEY= # Optional OpenAI-compatible voice transcription endpoint TRANSCRIPTION_API_BASE_URL= -TRANSCRIPTION_MODEL= TRANSCRIPTION_API_KEY= +TRANSCRIPTION_MODEL= diff --git a/autogpt_platform/frontend/src/app/api/transcribe/__tests__/route.test.ts b/autogpt_platform/frontend/src/app/api/transcribe/__tests__/route.test.ts index e553c735c748..c481934ee61c 100644 --- a/autogpt_platform/frontend/src/app/api/transcribe/__tests__/route.test.ts +++ b/autogpt_platform/frontend/src/app/api/transcribe/__tests__/route.test.ts @@ -31,7 +31,7 @@ describe("transcribe route", () => { it("sends audio to a configured OpenAI-compatible transcription endpoint", async () => { process.env.TRANSCRIPTION_API_BASE_URL = "http://funasr.local:8000/v1/"; - process.env.TRANSCRIPTION_MODEL = "iic/SenseVoiceSmall"; + process.env.TRANSCRIPTION_MODEL = "sensevoice"; const fetchMock = vi.fn().mockResolvedValue( new Response(JSON.stringify({ text: "hello from funasr" }), { status: 200, @@ -56,7 +56,7 @@ describe("transcribe route", () => { expect(fetchInit.headers).toBeInstanceOf(Headers); expect((fetchInit.headers as Headers).has("Authorization")).toBe(false); const upstreamBody = fetchInit.body as FormData; - expect(upstreamBody.get("model")).toBe("iic/SenseVoiceSmall"); + expect(upstreamBody.get("model")).toBe("sensevoice"); const file = upstreamBody.get("file") as File; expect(file.name).toBe("recording.webm"); }); diff --git a/docs/platform/copilot-local-llm.md b/docs/platform/copilot-local-llm.md index a848e3d47ea5..432ed766ce2c 100644 --- a/docs/platform/copilot-local-llm.md +++ b/docs/platform/copilot-local-llm.md @@ -95,7 +95,7 @@ default OpenAI transcription endpoint. FunASR exposes the compatible Install FunASR and start the server on the Docker host: ```bash -python -m pip install -U funasr +python -m pip install -U funasr fastapi uvicorn python-multipart # CPU funasr-server --model sensevoice --device cpu @@ -144,8 +144,12 @@ docker compose up -d --force-recreate frontend Leave `TRANSCRIPTION_API_KEY` empty for a default FunASR server. Set it only when a gateway or reverse proxy in front of FunASR requires bearer -authentication. If the `TRANSCRIPTION_*` variables are absent, AutoGPT -continues to use OpenAI's `whisper-1` model and `OPENAI_API_KEY`. +authentication. If no `TRANSCRIPTION_*` overrides are set, AutoGPT uses +`whisper-1` and sends requests to `OPENAI_API_BASE_URL` when configured, +or to the default OpenAI endpoint otherwise. `OPENAI_API_KEY` is attached +only when the resulting base URL is the default OpenAI URL. Set +`TRANSCRIPTION_API_KEY` explicitly when a custom `OPENAI_API_BASE_URL` +requires bearer authentication. For model choices and deployment options, see the [FunASR model selection guide](https://github.com/modelscope/FunASR/blob/main/docs/model_selection.md). From e5ab161b206554cdaf188aa3b00ebee3a9f6ee69 Mon Sep 17 00:00:00 2001 From: zhifu gao <18321252+LauraGPT@users.noreply.github.com> Date: Fri, 24 Jul 2026 05:03:06 +0000 Subject: [PATCH 3/3] docs(frontend): generalize transcription setup --- autogpt_platform/frontend/.env.default | 6 ++-- docs/platform/copilot-local-llm.md | 41 ++++++++++++++------------ 2 files changed, 25 insertions(+), 22 deletions(-) diff --git a/autogpt_platform/frontend/.env.default b/autogpt_platform/frontend/.env.default index 4067587060ac..2568ef3eb423 100644 --- a/autogpt_platform/frontend/.env.default +++ b/autogpt_platform/frontend/.env.default @@ -45,6 +45,6 @@ NEXT_PUBLIC_VAPID_PUBLIC_KEY=BBg49iVTWthVbRYphwmZNvZyiSJDqtSO4nmLxDzLKe3Oo9jbtu0 OPENAI_API_KEY= # Optional OpenAI-compatible voice transcription endpoint -TRANSCRIPTION_API_BASE_URL= -TRANSCRIPTION_API_KEY= -TRANSCRIPTION_MODEL= +#TRANSCRIPTION_API_BASE_URL= +#TRANSCRIPTION_API_KEY= +#TRANSCRIPTION_MODEL= diff --git a/docs/platform/copilot-local-llm.md b/docs/platform/copilot-local-llm.md index 432ed766ce2c..e193b66ef3b9 100644 --- a/docs/platform/copilot-local-llm.md +++ b/docs/platform/copilot-local-llm.md @@ -85,14 +85,15 @@ CHAT_FAST_STANDARD_MODEL=hf.co/unsloth/Qwen3.5-4B-GGUF:Q4_K_M CHAT_FAST_ADVANCED_MODEL=qwen3:14b-q4_K_M ``` -## Optional: local voice transcription with FunASR +## Optional: local OpenAI-compatible voice transcription -AutoPilot's microphone input can use a self-hosted -[FunASR](https://github.com/modelscope/FunASR) server instead of the -default OpenAI transcription endpoint. FunASR exposes the compatible -`POST /v1/audio/transcriptions` route and uses SenseVoice by default. +AutoPilot's microphone input can use any service that exposes the +OpenAI-compatible `POST /v1/audio/transcriptions` route. Configure the +base URL and model name for your service. The example below uses +[FunASR](https://github.com/modelscope/FunASR), a self-hosted server that +uses SenseVoice by default. -Install FunASR and start the server on the Docker host: +Install the example server and start it on the Docker host: ```bash python -m pip install -U funasr fastapi uvicorn python-multipart @@ -104,7 +105,7 @@ funasr-server --model sensevoice --device cpu funasr-server --model sensevoice --device cuda ``` -The first start downloads the model. The server listens on +The first start downloads the configured model. The server listens on `0.0.0.0:8000` and does not require an API key by default. Before connecting AutoGPT, verify it with a local audio file: @@ -116,7 +117,9 @@ curl http://localhost:8000/v1/audio/transcriptions \ Keep port 8000 on a trusted host or private network. If the endpoint is reachable from an untrusted network, put it behind an authenticated -HTTPS reverse proxy. +HTTPS reverse proxy. For another compatible service, substitute its +startup command, port, and model name; the AutoGPT configuration below +is otherwise the same. Then add the transcription settings to `autogpt_platform/frontend/.env`: @@ -142,17 +145,17 @@ cd autogpt_platform docker compose up -d --force-recreate frontend ``` -Leave `TRANSCRIPTION_API_KEY` empty for a default FunASR server. Set it -only when a gateway or reverse proxy in front of FunASR requires bearer -authentication. If no `TRANSCRIPTION_*` overrides are set, AutoGPT uses -`whisper-1` and sends requests to `OPENAI_API_BASE_URL` when configured, -or to the default OpenAI endpoint otherwise. `OPENAI_API_KEY` is attached -only when the resulting base URL is the default OpenAI URL. Set -`TRANSCRIPTION_API_KEY` explicitly when a custom `OPENAI_API_BASE_URL` -requires bearer authentication. - -For model choices and deployment options, see the -[FunASR model selection guide](https://github.com/modelscope/FunASR/blob/main/docs/model_selection.md). +Leave `TRANSCRIPTION_API_KEY` empty when the transcription service does +not require authentication. Set it when the endpoint or a reverse proxy +requires bearer authentication. If no `TRANSCRIPTION_*` overrides are +set, AutoGPT uses `whisper-1` and sends requests to +`OPENAI_API_BASE_URL` when configured, or to the default OpenAI endpoint +otherwise. `OPENAI_API_KEY` is attached only when the resulting base URL +is the default OpenAI URL. Set `TRANSCRIPTION_API_KEY` explicitly when a +custom `OPENAI_API_BASE_URL` requires bearer authentication. + +For the example server's model choices and deployment options, see the +[model selection guide](https://github.com/modelscope/FunASR/blob/main/docs/model_selection.md). ## Picking a model