-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathtest_api.py
More file actions
358 lines (299 loc) · 13.2 KB
/
Copy pathtest_api.py
File metadata and controls
358 lines (299 loc) · 13.2 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
from __future__ import annotations
from pathlib import Path
from uuid import uuid4
import httpx
import pytest
from conftest import TOKEN, FakeEngine, FakeNormalizer
from starlette.status import (
HTTP_200_OK,
HTTP_401_UNAUTHORIZED,
HTTP_404_NOT_FOUND,
HTTP_413_CONTENT_TOO_LARGE,
HTTP_415_UNSUPPORTED_MEDIA_TYPE,
HTTP_422_UNPROCESSABLE_CONTENT,
HTTP_503_SERVICE_UNAVAILABLE,
)
from app.config import Settings
from app.errors import LanguageUnsupportedError
from app.main import create_app
from app.models.base import EngineHealth, TranscriptionOptions
from app.schemas import CreateSessionRequest
TEST_AUDIO_SIZE = 200
TEST_AUDIO_BYTES = b"x" * TEST_AUDIO_SIZE
OVERSIZED_AUDIO_SIZE = 20_001
OVERSIZED_AUDIO_BYTES = b"x" * OVERSIZED_AUDIO_SIZE
SESSIONS_API_PATH = "/v1/sessions"
CLIENT_SESSION_ID_KEY = "client_session_id"
STYLE_KEY = "style"
CONTENT_TYPE_HEADER = "Content-Type"
WAV_CONTENT_TYPE = "audio/wav"
STATUS_KEY = "status"
LANGUAGES_KEY = "languages"
JOB_ID_KEY = "job_id"
TRANSCRIPT_KEY = "transcript"
class UnreadyEngine:
async def health(self) -> EngineHealth:
return EngineHealth(ready=False, name="missing-model")
class WrongLanguageEngine:
"""Stands in for a loaded model whose language list excludes the request."""
async def health(self) -> EngineHealth:
return EngineHealth(ready=True, name="english-only-model")
async def transcribe(self, audio_path: Path, options: TranscriptionOptions) -> str:
raise LanguageUnsupportedError(
"The selected model does not support hi. Choose Auto, en, or another model."
)
def test_session_schema_accepts_roman_hinglish_output_contract() -> None:
request = CreateSessionRequest(client_session_id=uuid4(), language="hinglish_roman")
assert request.language == "hinglish_roman"
with pytest.raises(ValueError):
CreateSessionRequest(client_session_id=uuid4(), language="not_a_language")
async def test_unsupported_language_is_reported_a_d68a4(
settings: Settings, authorization: dict[str, str], audio_bytes: bytes
) -> None:
"""A language the loaded model cannot serve must not look like a transient fault.
The clients decide whether to keep audio for Retry from this code, and no
number of retries will make an English-only model transcribe Hindi.
"""
async with httpx.AsyncClient(
transport=httpx.ASGITransport(
app=create_app(settings, engine=WrongLanguageEngine(), normalizer=FakeNormalizer())
),
base_url="http://gateway",
) as client:
session_id = uuid4()
await client.post(
SESSIONS_API_PATH,
headers=authorization,
json={CLIENT_SESSION_ID_KEY: str(session_id), "language": "hi", STYLE_KEY: "raw"},
)
await client.put(
f"/v1/sessions/{session_id}/audio",
headers={**authorization, CONTENT_TYPE_HEADER: WAV_CONTENT_TYPE},
content=audio_bytes,
)
finished = await client.post(f"/v1/sessions/{session_id}/finish", headers=authorization)
assert finished.status_code == HTTP_422_UNPROCESSABLE_CONTENT
error = finished.json()["error"]
assert error["code"] == "language_unsupported"
assert error["recoverable"] is False
assert "does not support hi" in error["message"]
session = await client.get(f"/v1/sessions/{session_id}", headers=authorization)
assert session.json()["error_code"] == "language_unsupported"
async def test_health_is_public_and_separates_eng_aa(
client: httpx.AsyncClient, fake_engine: FakeEngine
) -> None:
response = await client.get("/health")
assert response.status_code == HTTP_200_OK
assert response.json() == {
STATUS_KEY: "ok",
"engine_ready": True,
"engine": "fake-local-model",
"streaming_supported": False,
# A stub engine has no catalog entry, so the gateway makes no claim and
# clients keep every language selectable.
LANGUAGES_KEY: [],
"detects_language_automatically": False,
}
liveness = await client.get("/health/live")
readiness = await client.get("/health/ready")
repeated = await client.get("/health")
assert liveness.status_code == HTTP_200_OK
assert liveness.json()[STATUS_KEY] == "ok"
assert liveness.json()["uptime_seconds"] >= 0
assert readiness.status_code == HTTP_200_OK
assert readiness.json()[STATUS_KEY] == "ready"
assert readiness.json()["engine"] == "fake-local-model"
assert repeated.status_code == HTTP_200_OK
assert fake_engine.health_calls == 1
async def test_private_endpoints_require_bearer_token(client: httpx.AsyncClient) -> None:
response = await client.get("/v1/models")
assert response.status_code == HTTP_401_UNAUTHORIZED
assert response.json()["error"]["code"] == "unauthorized"
assert TOKEN not in response.text
async def test_readiness_can_fail_without_failing_ca0a7(tmp_path) -> None:
settings = Settings(
token=TOKEN,
data_dir=tmp_path,
whisper_binary=tmp_path / "missing-whisper",
whisper_model=tmp_path / "missing-model",
)
app = create_app(settings, engine=UnreadyEngine(), normalizer=FakeNormalizer())
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://test"
) as test_client:
liveness = await test_client.get("/health/live")
readiness = await test_client.get("/health/ready")
assert liveness.status_code == HTTP_200_OK
assert readiness.status_code == HTTP_503_SERVICE_UNAVAILABLE
assert readiness.json()[STATUS_KEY] == "not_ready"
assert readiness.json()["engine"] == "missing-model"
async def test_complete_flow_is_idempotent_and_de_aaa(
client: httpx.AsyncClient,
authorization: dict[str, str],
audio_bytes: bytes,
) -> None:
session_id = uuid4()
created = await client.post(
SESSIONS_API_PATH,
headers=authorization,
json={CLIENT_SESSION_ID_KEY: str(session_id), "language": "auto", STYLE_KEY: "raw"},
)
repeated = await client.post(
SESSIONS_API_PATH,
headers=authorization,
json={CLIENT_SESSION_ID_KEY: str(session_id), "language": "auto", STYLE_KEY: "raw"},
)
assert created.status_code == HTTP_200_OK
assert repeated.json()[JOB_ID_KEY] == created.json()[JOB_ID_KEY]
assert (
await client.put(
f"/v1/sessions/{session_id}/audio",
headers={**authorization, CONTENT_TYPE_HEADER: WAV_CONTENT_TYPE},
content=audio_bytes,
)
).status_code == HTTP_200_OK
finished = await client.post(f"/v1/sessions/{session_id}/finish", headers=authorization)
finished_again = await client.post(f"/v1/sessions/{session_id}/finish", headers=authorization)
assert finished.status_code == HTTP_200_OK
assert finished.json()[TRANSCRIPT_KEY] == "hello from the local model"
assert finished_again.json()[TRANSCRIPT_KEY] == finished.json()[TRANSCRIPT_KEY]
assert finished_again.json()[JOB_ID_KEY] == finished.json()[JOB_ID_KEY]
async def test_session_accepts_writing_styles_and_aaaa(
client: httpx.AsyncClient,
authorization: dict[str, str],
) -> None:
for style in ("formal", "casual", "very_casual", "excited"):
response = await client.post(
SESSIONS_API_PATH,
headers=authorization,
json={CLIENT_SESSION_ID_KEY: str(uuid4()), STYLE_KEY: style},
)
assert response.status_code == HTTP_200_OK
assert response.json()[STYLE_KEY] == style
invalid = await client.post(
SESSIONS_API_PATH,
headers=authorization,
json={CLIENT_SESSION_ID_KEY: str(uuid4()), STYLE_KEY: "pirate"},
)
assert invalid.status_code == HTTP_422_UNPROCESSABLE_CONTENT
async def test_writing_style_is_applied_to_the_lo_aaaaa(
client: httpx.AsyncClient,
authorization: dict[str, str],
audio_bytes: bytes,
) -> None:
session_id = uuid4()
await client.post(
SESSIONS_API_PATH,
headers=authorization,
json={CLIENT_SESSION_ID_KEY: str(session_id), STYLE_KEY: "formal"},
)
await client.put(
f"/v1/sessions/{session_id}/audio",
headers={**authorization, CONTENT_TYPE_HEADER: WAV_CONTENT_TYPE},
content=audio_bytes,
)
finished = await client.post(f"/v1/sessions/{session_id}/finish", headers=authorization)
assert finished.status_code == HTTP_200_OK
assert finished.json()[TRANSCRIPT_KEY] == "Hello from the local model."
async def test_upload_rejects_unsupported_empty_a_f2c1d(
client: httpx.AsyncClient,
authorization: dict[str, str],
) -> None:
async def create() -> str:
session_id = str(uuid4())
await client.post(
SESSIONS_API_PATH,
headers=authorization,
json={CLIENT_SESSION_ID_KEY: session_id},
)
return session_id
unsupported = await client.put(
f"/v1/sessions/{await create()}/audio",
headers={**authorization, CONTENT_TYPE_HEADER: "text/plain"},
content=TEST_AUDIO_BYTES,
)
empty = await client.put(
f"/v1/sessions/{await create()}/audio",
headers={**authorization, CONTENT_TYPE_HEADER: WAV_CONTENT_TYPE},
content=b"x",
)
oversized = await client.put(
f"/v1/sessions/{await create()}/audio",
headers={**authorization, CONTENT_TYPE_HEADER: WAV_CONTENT_TYPE},
content=OVERSIZED_AUDIO_BYTES,
)
assert unsupported.status_code == HTTP_415_UNSUPPORTED_MEDIA_TYPE
assert empty.status_code == HTTP_422_UNPROCESSABLE_CONTENT
assert oversized.status_code == HTTP_413_CONTENT_TOO_LARGE
async def test_delete_is_idempotent(
client: httpx.AsyncClient,
authorization: dict[str, str],
) -> None:
session_id = uuid4()
await client.post(
SESSIONS_API_PATH,
headers=authorization,
json={CLIENT_SESSION_ID_KEY: str(session_id)},
)
first = await client.delete(f"/v1/sessions/{session_id}", headers=authorization)
second = await client.delete(f"/v1/sessions/{session_id}", headers=authorization)
assert first.status_code == HTTP_200_OK
assert first.json() == {"deleted": True}
assert second.status_code == HTTP_404_NOT_FOUND
assert second.json() == {"deleted": False}
async def test_health_reports_what_the_loaded_mod_a(settings: Settings, audio_bytes: bytes) -> None:
"""Clients cannot offer a sensible language picker without knowing whether the
loaded model can be pinned at all. An engine holding a catalog entry reports
that entry's languages; one that picks its own language says so."""
from app.catalog import DEFAULT_CATALOG
from app.models.base import EngineHealth
dolphin = next(
model for model in DEFAULT_CATALOG if model.id == "sherpa-onnx:dolphin-small-ctc-int8"
)
class DolphinLikeEngine:
catalog_model = dolphin
async def health(self) -> EngineHealth:
return EngineHealth(ready=True, name="sherpa-onnx:dolphin")
async def transcribe(self, audio_path: Path, options: TranscriptionOptions) -> str:
return "unused"
app = create_app(settings, engine=DolphinLikeEngine(), normalizer=FakeNormalizer())
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://gateway") as client:
payload = (await client.get("/health")).json()
assert payload["detects_language_automatically"] is True
assert "hi" in payload[LANGUAGES_KEY] and "bn" in payload[LANGUAGES_KEY]
assert "en" not in payload[LANGUAGES_KEY] # Dolphin is not trained on English
class BoomEngine:
"""Unexpected failure during transcription (not a typed engine error)."""
async def health(self) -> EngineHealth:
return EngineHealth(ready=True, name="boom-model")
async def transcribe(self, audio_path: Path, options: TranscriptionOptions) -> str:
raise RuntimeError("engine exploded")
async def test_unexpected_finish_error_leaves_ses_f5519(
settings: Settings, authorization: dict[str, str], audio_bytes: bytes
) -> None:
"""Bare exceptions must not leave the session stuck in 'transcribing'.
Retry only accepts failed/uploaded/completed, and finish rejects
transcription_in_progress, so a stuck transcribing state blocks recovery.
"""
app = create_app(settings, engine=BoomEngine(), normalizer=FakeNormalizer())
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=app), base_url="http://gateway"
) as client:
session_id = uuid4()
await client.post(
SESSIONS_API_PATH,
headers=authorization,
json={CLIENT_SESSION_ID_KEY: str(session_id), "language": "en", STYLE_KEY: "raw"},
)
await client.put(
f"/v1/sessions/{session_id}/audio",
headers={**authorization, CONTENT_TYPE_HEADER: WAV_CONTENT_TYPE},
content=audio_bytes,
)
with pytest.raises(RuntimeError, match="engine exploded"):
# ASGI client surfaces the unhandled exception from the route.
await client.post(f"/v1/sessions/{session_id}/finish", headers=authorization)
session = await client.get(f"/v1/sessions/{session_id}", headers=authorization)
assert session.json()["state"] == "failed"
assert session.json()["error_code"] == "internal_error"