build_wordlist.py

"""Build filtered word list from a language corpus."""

import importlib
import json

import lang as _lang


def build_wordlist(language: str) -> list[tuple[str, int]]:
    """Build word list for the given language. Requires set_lang() called first."""
    mod = importlib.import_module(f"languages.{language}.parse_corpus")
    words = mod.parse_corpus(_lang.corpus_dir())

    out_path = _lang.wordlist_path()
    out_path.parent.mkdir(parents=True, exist_ok=True)
    with open(out_path, "w", encoding="utf-8") as f:
        json.dump(words, f, ensure_ascii=False, indent=2)

    return words


if __name__ == "__main__":
    import argparse

    parser = argparse.ArgumentParser(description="Build word list from corpus")
    parser.add_argument(
        "--lang", required=True, choices=_lang.available_languages(), help="language code"
    )
    args = parser.parse_args()

    _lang.set_lang(args.lang)
    words = build_wordlist(args.lang)

    print(f"Words: {len(words)}")
    print(f"Top 10: {words[:10]}")
    print(f"Bottom 10: {words[-10:]}")
    print(f"Written to {_lang.wordlist_path()}")