Skip to content

Commit c431591

Browse files
MomiJiSanclaude
andauthored
feat(study): 伴学插件 Phase 9 — 语音过滤、KaTeX 数学渲染、无障碍与国际化 (#1606)
* feat(study): add voice filter, KaTeX rendering, and i18n for Phase 9 UX Study Companion Plugin-only changes: - voice_filter.py: self-contained rule engine for realtime voice transcript filtering (name-call detection, question intent, OCR overlap suppression) - study_panel.tsx: KaTeX math rendering pipeline (inline/block), Escape key to cancel in-flight explain requests, ARIA accessibility enhancements - 72 KaTeX font files + katex.min.js/css + katex-render.js - 8 i18n locale files: new UI labels for study features - onboarding.md: first-time user guide - handle_voice_transcript custom event handler using standard @custom_event decorator (safe to register before bridge infrastructure) Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com> * fix: address study companion review feedback --------- Co-authored-by: Claude Opus 4.8 <noreply@anthropic.com>
1 parent fa634e2 commit c431591

75 files changed

Lines changed: 2150 additions & 9 deletions

File tree

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

plugin/plugins/study_companion/__init__.py

Lines changed: 110 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
NekoPluginBase,
1818
Ok,
1919
SdkError,
20+
custom_event,
2021
lifecycle,
2122
neko_plugin,
2223
plugin_entry,
@@ -78,6 +79,17 @@
7879
from .ui_api import build_open_ui_payload
7980
from .ui_api import build_contribution_settings_payload, build_knowledge_map_payload
8081
from .ui_api import build_habit_dashboard_payload, build_pomodoro_status_payload
82+
from .voice_filter import VoiceFilter, _derive_subject, build_context_for_catgirl
83+
84+
85+
def _voice_session_key(lanlan_name: str, metadata: Mapping[str, Any] | None) -> str:
86+
for key in ("voice_session_id", "session_id", "conversation_id", "request_session_id"):
87+
value = metadata.get(key) if isinstance(metadata, Mapping) else None
88+
text = str(value or "").strip()
89+
if text:
90+
return f"session:{text}"
91+
name = str(lanlan_name or "").strip()
92+
return f"lanlan:{name}" if name else "__default__"
8193

8294

8395
def _register_install_routes() -> None:
@@ -216,6 +228,7 @@ def __init__(self, ctx):
216228
self._awareness_task: asyncio.Task[None] | None = None
217229
self._last_awareness_push_at = 0.0
218230
self._awareness_idle_ticks = 0
231+
self._voice_filter = VoiceFilter()
219232
self._review_due_task: asyncio.Task[None] | None = None
220233
self._command_queue: asyncio.Queue[tuple[str, dict[str, Any]]] = asyncio.Queue()
221234
self._command_worker_task: asyncio.Task[None] | None = None
@@ -231,6 +244,9 @@ async def startup(self, **_):
231244
try:
232245
raw = await self.config.dump(timeout=5.0)
233246
self._cfg = build_config(raw if isinstance(raw, dict) else {})
247+
self._voice_filter = VoiceFilter(
248+
plugin_config=raw if isinstance(raw, dict) else {}
249+
)
234250
await asyncio.to_thread(self._store.open)
235251
self._cfg = await asyncio.to_thread(self._store.load_config, self._cfg)
236252
self._knowledge_tracker = KnowledgeTracker(
@@ -841,6 +857,100 @@ def _state_snapshot(self) -> dict[str, Any]:
841857
def _screen_classification_context(self) -> dict[str, Any]:
842858
return dict(self._state.last_screen_classification)
843859

860+
@custom_event(
861+
event_type="voice_transcript",
862+
id="handle_transcript",
863+
name="Handle study voice transcript",
864+
description="Filter realtime study voice transcripts and return a voice-session action.",
865+
input_schema={
866+
"type": "object",
867+
"properties": {
868+
"transcript": {"type": "string"},
869+
"lanlan_name": {"type": "string"},
870+
"metadata": {"type": "object"},
871+
},
872+
"required": ["transcript"],
873+
},
874+
trigger_method="manual",
875+
)
876+
async def handle_voice_transcript(
877+
self,
878+
transcript: str = "",
879+
lanlan_name: str = "",
880+
metadata: dict[str, Any] | None = None,
881+
**_,
882+
):
883+
def voice_noop(reason: str, filter_result: Mapping[str, Any] | None = None):
884+
filter_payload = dict(filter_result or {})
885+
original_method = str(filter_payload.get("method") or "")
886+
if original_method and original_method != reason:
887+
filter_payload["source_method"] = original_method
888+
filter_payload["method"] = reason
889+
return Ok({"action": "noop", "reason": reason, "filter": filter_payload})
890+
891+
text = str(transcript or "").strip()
892+
if not text:
893+
return voice_noop("empty_transcript")
894+
metadata_payload = metadata if isinstance(metadata, dict) else {}
895+
session_key = _voice_session_key(lanlan_name, metadata_payload)
896+
897+
async with self._lock:
898+
if self._state.status != STATUS_READY:
899+
return voice_noop("not_ready")
900+
state_snapshot_payload = self._state.to_dict()
901+
902+
# Voice filtering only needs a point-in-time view; avoid holding the
903+
# plugin lock while building OCR context or applying filter rules.
904+
screen_text = str(state_snapshot_payload.get("last_ocr_text") or "")
905+
screen_classification = (
906+
state_snapshot_payload.get("last_screen_classification")
907+
if isinstance(
908+
state_snapshot_payload.get("last_screen_classification"), dict
909+
)
910+
else {}
911+
)
912+
screen_type = str(screen_classification.get("screen_type") or "")
913+
session_seed = (
914+
state_snapshot_payload.get("session_summary_seed")
915+
if isinstance(state_snapshot_payload.get("session_summary_seed"), dict)
916+
else {}
917+
)
918+
screen_context = {
919+
"topic": str(session_seed.get("last_topic") or "").strip(),
920+
"subject": _derive_subject(screen_text),
921+
}
922+
filter_result = self._voice_filter.filter(
923+
text,
924+
screen_text=screen_text,
925+
screen_type=screen_type,
926+
subject=screen_context["subject"],
927+
session_key=session_key,
928+
extra_names=[lanlan_name],
929+
)
930+
if filter_result is None:
931+
return voice_noop("not_matched")
932+
if not bool(filter_result.get("should_relay")):
933+
return Ok({"action": "cancel_response", "filter": dict(filter_result)})
934+
935+
state_snapshot = SimpleNamespace(**state_snapshot_payload)
936+
context_text = build_context_for_catgirl(
937+
text,
938+
state_snapshot,
939+
screen_context,
940+
filter_result,
941+
).strip()
942+
if not context_text:
943+
return voice_noop("empty_context", filter_result)
944+
return Ok(
945+
{
946+
"action": "prime_context",
947+
"context": context_text,
948+
"skipped": True,
949+
"filter": dict(filter_result),
950+
"lanlan_name": str(lanlan_name or ""),
951+
}
952+
)
953+
844954
async def _update_screen_classification(
845955
self, text: str, *, window_title: str = "", update_empty: bool = True
846956
) -> dict[str, Any]:
Lines changed: 60 additions & 0 deletions

plugin/plugins/study_companion/plugin.toml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -192,6 +192,12 @@ title = "Study Companion Quickstart"
192192
entry = "surfaces/quickstart.tsx"
193193
permissions = ["state:read"]
194194

195+
[[plugin.ui.docs]]
196+
id = "onboarding"
197+
title = "Study Companion Onboarding"
198+
entry = "onboarding.md"
199+
permissions = ["state:read"]
200+
195201
[plugin.i18n]
196202
default_locale = "zh-CN"
197203
locales_dir = "i18n"
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.

0 commit comments

Comments
 (0)