diff --git a/data/placement/en.json b/data/placement/en.json new file mode 100644 index 0000000..18cf8d1 --- /dev/null +++ b/data/placement/en.json @@ -0,0 +1,32 @@ +{ + "id": "placement_en_v1", + "language": "en", + "title": "English app placement pack", + "license": "MIT — synthetic prompts; no third-party copyrighted content", + "prompts": [ + { + "id": "en_p1", + "skill": "writing", + "prompt": "Write 2–3 sentences about your daily routine.", + "demo_answer": "I usually wake up early and drink coffee. Then I work on my laptop and take a short walk after lunch." + }, + { + "id": "en_p2", + "skill": "speaking", + "prompt": "Imagine you are introducing yourself to a new classmate. What do you say?", + "demo_answer": "Hi, my name is Alex. I am learning English because I want to travel and meet people from other countries." + }, + { + "id": "en_p3", + "skill": "reading", + "prompt": "A notice says the library closes at 6pm on Fridays. In one sentence, explain what that means for a student who arrives at 6:10pm.", + "demo_answer": "The student cannot enter because the library is already closed at that time." + }, + { + "id": "en_p4", + "skill": "grammar", + "prompt": "Complete: Yesterday I ___ to the market and ___ some fruit.", + "demo_answer": "Yesterday I went to the market and bought some fruit." + } + ] +} diff --git a/data/placement/ja.json b/data/placement/ja.json new file mode 100644 index 0000000..198eed8 --- /dev/null +++ b/data/placement/ja.json @@ -0,0 +1,32 @@ +{ + "id": "placement_ja_v1", + "language": "ja", + "title": "Japanese app placement pack", + "license": "MIT — synthetic prompts; no third-party copyrighted content", + "prompts": [ + { + "id": "ja_p1", + "skill": "writing", + "prompt": "毎日の生活について、2〜3文で書いてください。", + "demo_answer": "朝早く起きてコーヒーを飲みます。そのあと勉強をして、夕方は少し散歩します。" + }, + { + "id": "ja_p2", + "skill": "speaking", + "prompt": "新しいクラスメイトに自己紹介するとしたら、何と言いますか。", + "demo_answer": "こんにちは。私はユキです。日本語を勉強していて、旅行が好きです。" + }, + { + "id": "ja_p3", + "skill": "reading", + "prompt": "掲示に図書館は金曜日の午後6時に閉まるとあります。6時10分に来た学生にとって何を意味するか、一文で説明してください。", + "demo_answer": "すでに閉まっているので、その学生は入れません。" + }, + { + "id": "ja_p4", + "skill": "grammar", + "prompt": "適切な語で完成させてください：昨日市場へ___、果物を___。", + "demo_answer": "昨日市場へ行って、果物を買いました。" + } + ] +} diff --git a/data/placement/ko.json b/data/placement/ko.json new file mode 100644 index 0000000..8ec7894 --- /dev/null +++ b/data/placement/ko.json @@ -0,0 +1,32 @@ +{ + "id": "placement_ko_v1", + "language": "ko", + "title": "Korean app placement pack", + "license": "MIT — synthetic prompts; no third-party copyrighted content", + "prompts": [ + { + "id": "ko_p1", + "skill": "writing", + "prompt": "하루에 하는 일을 두세 문장으로 적어 보세요.", + "demo_answer": "저는 아침에 일어나서 커피를 마십니다. 그다음에 공부하고 저녁에는 산책을 합니다." + }, + { + "id": "ko_p2", + "skill": "speaking", + "prompt": "새 친구에게 자기소개를 한다면 뭐라고 말하겠어요?", + "demo_answer": "안녕하세요. 저는 민수입니다. 한국어를 배우고 있고 여행하는 것을 좋아합니다." + }, + { + "id": "ko_p3", + "skill": "reading", + "prompt": "안내문에 도서관이 금요일 오후 6시에 닫힌다고 합니다. 6시 10분에 온 학생에게 무슨 뜻인지 한 문장으로 설명하세요.", + "demo_answer": "이미 문이 닫혀서 그 학생은 들어갈 수 없습니다." + }, + { + "id": "ko_p4", + "skill": "grammar", + "prompt": "알맞은 말로 완성하세요: 어제 시장에 ___ 과일을 ___.", + "demo_answer": "어제 시장에 가서 과일을 샀습니다." + } + ] +} diff --git a/docs/PLACEMENT_PACK.md b/docs/PLACEMENT_PACK.md new file mode 100644 index 0000000..e91948e --- /dev/null +++ b/docs/PLACEMENT_PACK.md @@ -0,0 +1,49 @@ +# Placement packs for language apps + +NokaMan ships **license-safe** placement packs under `data/placement/` for embedding +in a learning app. Minimum languages: **EN, KO, JA**. + +## Single product path + +CLI: + +```bash +# Sample evidence reports (bundled demo answers) +nokaman eval placement-pack --lang en --demo +nokaman eval placement-pack --lang ko --demo +nokaman eval placement-pack --lang ja --demo + +# Real learner answers (same order as pack prompts) +nokaman eval placement-pack --lang en \ + -a "I wake up early..." \ + -a "Hi, my name is..." \ + -a "The library is closed..." \ + -a "Yesterday I went..." +``` + +API (`nokaman[api]`): + +```http +POST /assess/placement-pack +{"language":"en","use_demo_answers":true} +``` + +or with learner answers: + +```http +POST /assess/placement-pack +{"language":"ko","answers":["...","...","...","..."]} +``` + +Reports include `pack_id`, `overall`, `cefr`, per-item `prompt`/`answer`/`score`, +and `ready_for_ui: true`. Sample demo runs set `sample_report: true`. + +## Pack files + +| File | Language | +| --- | --- | +| `data/placement/en.json` | English | +| `data/placement/ko.json` | Korean | +| `data/placement/ja.json` | Japanese | + +Each pack lists prompts + synthetic `demo_answer` text (no scraped PII, MIT-safe). diff --git a/docs/examples/placement_pack_en_sample.json b/docs/examples/placement_pack_en_sample.json new file mode 100644 index 0000000..7b2efd5 --- /dev/null +++ b/docs/examples/placement_pack_en_sample.json @@ -0,0 +1,78 @@ +{ + "pack_id": "placement_en_v1", + "title": "English app placement pack", + "license": "MIT — synthetic prompts; no third-party copyrighted content", + "language": "en", + "n_items": 4, + "overall": 55.0, + "cefr": "B1", + "items": [ + { + "prompt_id": "en_p1", + "skill": "writing", + "prompt": "Write 2–3 sentences about your daily routine.", + "answer": "I usually wake up early and drink coffee. Then I work on my laptop and take a short walk after lunch.", + "score": 55.39, + "cefr": "B1", + "skills": { + "vocabulary": 71.32, + "grammar": 29.26, + "reading": 52.87, + "writing": 55.65, + "listening": 50.09, + "speaking": 73.07 + } + }, + { + "prompt_id": "en_p2", + "skill": "speaking", + "prompt": "Imagine you are introducing yourself to a new classmate. What do you say?", + "answer": "Hi, my name is Alex. I am learning English because I want to travel and meet people from other countries.", + "score": 57.03, + "cefr": "B1", + "skills": { + "vocabulary": 73.92, + "grammar": 30.96, + "reading": 53.84, + "writing": 56.68, + "listening": 51.01, + "speaking": 75.84 + } + }, + { + "prompt_id": "en_p3", + "skill": "reading", + "prompt": "A notice says the library closes at 6pm on Fridays. In one sentence, explain what that means for a student who arrives at 6:10pm.", + "answer": "The student cannot enter because the library is already closed at that time.", + "score": 53.88, + "cefr": "B1", + "skills": { + "vocabulary": 70.43, + "grammar": 30.99, + "reading": 49.9, + "writing": 52.53, + "listening": 47.28, + "speaking": 72.42 + } + }, + { + "prompt_id": "en_p4", + "skill": "grammar", + "prompt": "Complete: Yesterday I ___ to the market and ___ some fruit.", + "answer": "Yesterday I went to the market and bought some fruit.", + "score": 53.69, + "cefr": "B1", + "skills": { + "vocabulary": 73.07, + "grammar": 28.59, + "reading": 48.49, + "writing": 51.04, + "listening": 45.94, + "speaking": 75.52 + } + } + ], + "ready_for_ui": true, + "sample_report": true, + "source": "/Users/sawko/Documents/Projekty/SideJob/bounties/NokaMan/data/placement/en.json" +} diff --git a/docs/examples/placement_pack_ja_sample.json b/docs/examples/placement_pack_ja_sample.json new file mode 100644 index 0000000..30977ff --- /dev/null +++ b/docs/examples/placement_pack_ja_sample.json @@ -0,0 +1,78 @@ +{ + "pack_id": "placement_ja_v1", + "title": "Japanese app placement pack", + "license": "MIT — synthetic prompts; no third-party copyrighted content", + "language": "ja", + "n_items": 4, + "overall": 59.86, + "cefr": "B1", + "items": [ + { + "prompt_id": "ja_p1", + "skill": "writing", + "prompt": "毎日の生活について、2〜3文で書いてください。", + "answer": "朝早く起きてコーヒーを飲みます。そのあと勉強をして、夕方は少し散歩します。", + "score": 61.64, + "cefr": "B1", + "skills": { + "vocabulary": 76.97, + "grammar": 48.87, + "reading": 55.22, + "writing": 58.12, + "listening": 52.31, + "speaking": 79.06 + } + }, + { + "prompt_id": "ja_p2", + "skill": "speaking", + "prompt": "新しいクラスメイトに自己紹介するとしたら、何と言いますか。", + "answer": "こんにちは。私はユキです。日本語を勉強していて、旅行が好きです。", + "score": 58.36, + "cefr": "B1", + "skills": { + "vocabulary": 75.25, + "grammar": 41.33, + "reading": 52.26, + "writing": 55.01, + "listening": 49.51, + "speaking": 77.5 + } + }, + { + "prompt_id": "ja_p3", + "skill": "reading", + "prompt": "掲示に図書館は金曜日の午後6時に閉まるとあります。6時10分に来た学生にとって何を意味するか、一文で説明してください。", + "answer": "すでに閉まっているので、その学生は入れません。", + "score": 60.81, + "cefr": "B1", + "skills": { + "vocabulary": 76.45, + "grammar": 47.56, + "reading": 54.32, + "writing": 57.18, + "listening": 51.46, + "speaking": 78.59 + } + }, + { + "prompt_id": "ja_p4", + "skill": "grammar", + "prompt": "適切な語で完成させてください：昨日市場へ___、果物を___。", + "answer": "昨日市場へ行って、果物を買いました。", + "score": 58.63, + "cefr": "B1", + "skills": { + "vocabulary": 75.26, + "grammar": 42.93, + "reading": 52.27, + "writing": 55.02, + "listening": 49.52, + "speaking": 77.51 + } + } + ], + "ready_for_ui": true, + "sample_report": true, + "source": "/Users/sawko/Documents/Projekty/SideJob/bounties/NokaMan/data/placement/ja.json" +} diff --git a/docs/examples/placement_pack_ko_sample.json b/docs/examples/placement_pack_ko_sample.json new file mode 100644 index 0000000..d4c0b84 --- /dev/null +++ b/docs/examples/placement_pack_ko_sample.json @@ -0,0 +1,78 @@ +{ + "pack_id": "placement_ko_v1", + "title": "Korean app placement pack", + "license": "MIT — synthetic prompts; no third-party copyrighted content", + "language": "ko", + "n_items": 4, + "overall": 52.22, + "cefr": "B1", + "items": [ + { + "prompt_id": "ko_p1", + "skill": "writing", + "prompt": "하루에 하는 일을 두세 문장으로 적어 보세요.", + "answer": "저는 아침에 일어나서 커피를 마십니다. 그다음에 공부하고 저녁에는 산책을 합니다.", + "score": 53.89, + "cefr": "B1", + "skills": { + "vocabulary": 73.62, + "grammar": 25.75, + "reading": 49.44, + "writing": 52.04, + "listening": 46.84, + "speaking": 76.02 + } + }, + { + "prompt_id": "ko_p2", + "skill": "speaking", + "prompt": "새 친구에게 자기소개를 한다면 뭐라고 말하겠어요?", + "answer": "안녕하세요. 저는 민수입니다. 한국어를 배우고 있고 여행하는 것을 좋아합니다.", + "score": 53.92, + "cefr": "B1", + "skills": { + "vocabulary": 73.43, + "grammar": 27.4, + "reading": 49.1, + "writing": 51.68, + "listening": 46.52, + "speaking": 75.84 + } + }, + { + "prompt_id": "ko_p3", + "skill": "reading", + "prompt": "안내문에 도서관이 금요일 오후 6시에 닫힌다고 합니다. 6시 10분에 온 학생에게 무슨 뜻인지 한 문장으로 설명하세요.", + "answer": "이미 문이 닫혀서 그 학생은 들어갈 수 없습니다.", + "score": 51.02, + "cefr": "B1", + "skills": { + "vocabulary": 71.99, + "grammar": 20.08, + "reading": 46.63, + "writing": 49.08, + "listening": 44.17, + "speaking": 74.54 + } + }, + { + "prompt_id": "ko_p4", + "skill": "grammar", + "prompt": "알맞은 말로 완성하세요: 어제 시장에 ___ 과일을 ___.", + "answer": "어제 시장에 가서 과일을 샀습니다.", + "score": 50.06, + "cefr": "B1", + "skills": { + "vocabulary": 71.03, + "grammar": 21.37, + "reading": 44.96, + "writing": 47.32, + "listening": 42.59, + "speaking": 73.66 + } + } + ], + "ready_for_ui": true, + "sample_report": true, + "source": "/Users/sawko/Documents/Projekty/SideJob/bounties/NokaMan/data/placement/ko.json" +} diff --git a/src/nokaman/api/app.py b/src/nokaman/api/app.py index 3a988b5..e08503d 100644 --- a/src/nokaman/api/app.py +++ b/src/nokaman/api/app.py @@ -2,7 +2,7 @@ from nokaman import __version__ from nokaman.eval.adaptive import adaptive_session -from nokaman.eval.metrics import placement_test +from nokaman.eval.metrics import placement_test, run_placement_pack from nokaman.eval.pipeline import evaluate_demo, evaluate_text from nokaman.rubrics.registry import SUPPORTED_LANGUAGES, get_language_meta @@ -39,6 +39,12 @@ class PlacementReq(BaseModel): answers: list[str] = Field(..., min_length=1) +class PlacementPackReq(BaseModel): + language: str = "en" + answers: list[str] | None = None + use_demo_answers: bool = False + + class AdaptiveReq(BaseModel): language: str = "en" answers: list[str] = Field(default_factory=list) @@ -86,6 +92,20 @@ def assess_placement(req: PlacementReq) -> dict: return placement_test(req.language, req.answers) +@app.post("/assess/placement-pack") +def assess_placement_pack(req: PlacementPackReq) -> dict: + if req.language not in SUPPORTED_LANGUAGES: + raise HTTPException(400, f"unsupported language {req.language}") + try: + return run_placement_pack( + req.language, + answers=req.answers, + use_demo_answers=req.use_demo_answers, + ) + except (FileNotFoundError, ValueError) as exc: + raise HTTPException(400, str(exc)) from exc + + @app.post("/assess/adaptive") def assess_adaptive(req: AdaptiveReq) -> dict: if req.language not in SUPPORTED_LANGUAGES: diff --git a/src/nokaman/cli.py b/src/nokaman/cli.py index b104b41..645cc56 100644 --- a/src/nokaman/cli.py +++ b/src/nokaman/cli.py @@ -12,7 +12,7 @@ from nokaman.config import OUT_DIR, RUNS_DIR from nokaman.data.coverage import language_skill_coverage from nokaman.data.loader import list_sample_files, list_rubric_files, load_rubric -from nokaman.eval.metrics import batch_evaluate, placement_test +from nokaman.eval.metrics import batch_evaluate, list_placement_packs, placement_test, run_placement_pack from nokaman.eval.pipeline import evaluate_demo, evaluate_sample_file, evaluate_text from nokaman.eval.session import SessionManager from nokaman.rubrics.registry import ( @@ -296,6 +296,41 @@ def eval_placement( _print_json(data=result) +@eval_app.command("placement-pack") +def eval_placement_pack( + lang: str = typer.Option("en", "--lang", "-l", help="Pack language: en, ko, or ja"), + answer: Optional[list[str]] = typer.Option( + None, + "--answer", + "-a", + help="Learner answers in pack order (omit with --demo for sample report)", + ), + demo: bool = typer.Option( + False, + "--demo", + help="Score bundled demo answers (sample evidence report)", + ), +) -> None: + """Run the multi-lang placement pack (prompts + scoring + app-ready JSON).""" + try: + if demo: + result = run_placement_pack(lang, use_demo_answers=True) + elif answer: + result = run_placement_pack(lang, answers=list(answer)) + else: + console.print("[red]Provide --answer … or --demo[/red]") + raise typer.Exit(code=1) + except (FileNotFoundError, ValueError) as exc: + console.print(f"[red]{exc}[/red]") + console.print(f"[dim]Available packs:[/dim] {', '.join(list_placement_packs()) or '(none)'}") + raise typer.Exit(code=1) from exc + _print_json(data=result) + OUT_DIR.mkdir(parents=True, exist_ok=True) + out_path = OUT_DIR / f"placement_pack_{result['language']}.json" + out_path.write_text(json.dumps(result, indent=2, ensure_ascii=False) + "\n", encoding="utf-8") + console.print(f"[dim]saved[/dim] {out_path}") + + @train_app.command("toy") def train_toy_cmd( config: Path | None = typer.Option( diff --git a/src/nokaman/config.py b/src/nokaman/config.py index eb21f13..e09d3e0 100644 --- a/src/nokaman/config.py +++ b/src/nokaman/config.py @@ -20,5 +20,6 @@ def data_dir() -> Path: SAMPLES_DIR = data_dir() / "samples" LISTENING_DIR = data_dir() / "listening" RUBRICS_DIR = data_dir() / "rubrics" +PLACEMENT_DIR = data_dir() / "placement" OUT_DIR = data_dir() / "out" RUNS_DIR = data_dir() / "runs" diff --git a/src/nokaman/eval/metrics.py b/src/nokaman/eval/metrics.py index 49c12db..8659f8e 100644 --- a/src/nokaman/eval/metrics.py +++ b/src/nokaman/eval/metrics.py @@ -1,7 +1,9 @@ from __future__ import annotations +import json from pathlib import Path +from nokaman.config import PLACEMENT_DIR from nokaman.data.loader import list_sample_files, load_sample from nokaman.eval.pipeline import evaluate_sample_file from nokaman.models.cefr import cefr_rank @@ -63,6 +65,33 @@ def batch_evaluate(sample_dir: Path | None = None) -> dict: } +def load_placement_pack(language: str, directory: Path | None = None) -> dict: + """Load a license-safe placement pack JSON for a language (en/ko/ja minimum).""" + code = language.strip().lower() + root = directory or PLACEMENT_DIR + path = root / f"{code}.json" + if not path.exists(): + raise FileNotFoundError( + f"No placement pack for language '{code}'. Expected file: {path}" + ) + pack = json.loads(path.read_text(encoding="utf-8")) + if not isinstance(pack, dict): + raise ValueError(f"Placement pack must be a JSON object: {path}") + prompts = pack.get("prompts") or [] + if not isinstance(prompts, list) or len(prompts) < 3: + raise ValueError(f"Placement pack needs at least 3 prompts: {path}") + pack["language"] = str(pack.get("language") or code).strip().lower() + pack["source"] = str(path) + return pack + + +def list_placement_packs(directory: Path | None = None) -> list[str]: + root = directory or PLACEMENT_DIR + if not root.exists(): + return [] + return sorted(p.stem for p in root.glob("*.json")) + + def placement_test(language: str, answers: list[str]) -> dict: """ Simple placement: score each short answer text, average overall CEFR. @@ -88,3 +117,58 @@ def placement_test(language: str, answers: list[str]) -> dict: "items": details, "ready_for_ui": True, } + + +def run_placement_pack( + language: str, + answers: list[str] | None = None, + *, + use_demo_answers: bool = False, + directory: Path | None = None, +) -> dict: + """ + Single product path: load prompts for a language, score answers, return app-ready report. + + If answers is None and use_demo_answers is True, score bundled demo answers + (sample evidence reports for EN/KO/JA). + """ + pack = load_placement_pack(language, directory=directory) + prompts = pack["prompts"] + if answers is None: + if not use_demo_answers: + raise ValueError("Provide answers or set use_demo_answers=True for sample reports") + answers = [str(p.get("demo_answer") or "") for p in prompts] + if len(answers) != len(prompts): + raise ValueError( + f"Expected {len(prompts)} answers for pack {pack.get('id')}, got {len(answers)}" + ) + if any(not str(a).strip() for a in answers): + raise ValueError("All placement answers must be non-empty") + + scored = placement_test(pack["language"], answers) + items = [] + for prompt, detail, answer in zip(prompts, scored["items"], answers, strict=True): + items.append( + { + "prompt_id": prompt.get("id"), + "skill": prompt.get("skill"), + "prompt": prompt.get("prompt"), + "answer": answer, + "score": detail.get("overall"), + "cefr": detail.get("cefr"), + "skills": detail.get("skills"), + } + ) + return { + "pack_id": pack.get("id"), + "title": pack.get("title"), + "license": pack.get("license"), + "language": pack["language"], + "n_items": len(items), + "overall": scored["overall"], + "cefr": scored["cefr"], + "items": items, + "ready_for_ui": True, + "sample_report": bool(use_demo_answers), + "source": pack.get("source"), + } diff --git a/tests/test_placement_pack.py b/tests/test_placement_pack.py new file mode 100644 index 0000000..8a27ec3 --- /dev/null +++ b/tests/test_placement_pack.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +import json +from pathlib import Path + +from typer.testing import CliRunner + +from nokaman.cli import app +from nokaman.eval.metrics import list_placement_packs, run_placement_pack + + +def test_placement_packs_cover_en_ko_ja() -> None: + packs = list_placement_packs() + assert set(packs) >= {"en", "ko", "ja"} + + +def test_sample_reports_for_three_languages(tmp_path: Path) -> None: + for lang in ("en", "ko", "ja"): + report = run_placement_pack(lang, use_demo_answers=True) + assert report["language"] == lang + assert report["n_items"] >= 3 + assert report["ready_for_ui"] is True + assert report["sample_report"] is True + assert report["cefr"] + out = tmp_path / f"placement_pack_{lang}.json" + out.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") + assert out.exists() + + +def test_cli_placement_pack_demo() -> None: + result = CliRunner().invoke(app, ["eval", "placement-pack", "--lang", "en", "--demo"]) + assert result.exit_code == 0, result.output + assert "placement_en_v1" in result.output + assert "ready_for_ui" in result.output