evals.json

{
  "skill_name": "multilingual-caption-video",
  "evals": [
    {
      "id": "english-url-to-arabic-file",
      "prompt": "Add Arabic captions to this English video: https://example.com/video.mp4",
      "expected_output": "A verified MP4 with burned-in Arabic captions delivered as a local file.",
      "assertions": [
        "Uses the explicitly requested Arabic language without asking for it again",
        "Asks whether the user wants the current choices persisted before writing preferences",
        "Announces end-to-end processing once and avoids routine progress updates unless the user opts in",
        "Downloads only the requested video without a playlist through the bundled uv-managed yt-dlp Python API rather than a global yt-dlp executable",
        "Uses timestamped multilingual transcription and Arabic translation",
        "Selects an installed Arabic-capable font using operating-system-appropriate discovery",
        "Samples the subtitle region at multiple points per cue and selects one stable high-contrast or translucent-background style for that cue",
        "Checks preview byte size against the vision tool limit and resizes or recompresses it before inspection when necessary",
        "Visually inspects the rendered captions before delivery",
        "Stores the original video title in the delivered MP4 title metadata instead of its filename",
        "Copies the generated MP4 to the operating system's Downloads directory",
        "Reports the actual font used and offers font, size, or subtitle-style changes",
        "Announces and schedules cleanup only after successful delivery"
      ]
    },
    {
      "id": "non-english-local",
      "prompt": "Take /tmp/spanish-interview.mov and add Arabic subtitles.",
      "expected_output": "The agent produces a locally delivered, verified MP4 preserving the Spanish video's timing and audio with readable Arabic captions.",
      "assertions": [
        "Does not overwrite or delete the local source file",
        "Transcribes the non-English source with the multilingual model",
        "Uses the requested or saved font size when present; otherwise scales the default from the video dimensions",
        "Uses a compatible font already installed on the current operating system without requiring fc-match",
        "Uses a single low-resolution video pass to assign stable per-cue styles based on the subtitle region's luminance and variation",
        "Keeps visual-inspection previews below the image tool's upload-size limit",
        "Probes and visually checks the final MP4 before delivery",
        "Copies file deliveries to the operating system's Downloads directory with a lowercase language-code_YYYY-MM-DD_HH.mm.ss.mp4 name without overwriting an existing file",
        "Reports the selected font and offers font, size, or subtitle-style changes",
        "Deletes only the marked generated work directory after the cleanup grace period while retaining the delivered Downloads file"
      ]
    },
    {
      "id": "missing-runtime-requirements",
      "prompt": "Add English subtitles to /tmp/interview.mp4 and send me the file. The environment has neither uv nor a usable FFmpeg installation, but Nix and Homebrew are available.",
      "expected_output": "The agent stops before processing, identifies uv and FFmpeg as missing, offers temporary and persistent setup choices, and waits for explicit approval.",
      "assertions": [
        "Runs the shell-based requirement preflight before preferences, downloads, transcription, or work-directory creation",
        "Describes the missing requirements as uv and FFmpeg with libass and libx264 support without unnecessarily mentioning FFmpeg companion executables",
        "Offers a temporary Nix environment and explains that it is scoped to this job even though packages may remain cached",
        "Offers an available persistent setup and explains that it is a one-time installation reusable by future skill runs",
        "States that provisioning may download packages and asks the user to select and approve an option before proceeding",
        "Provides the official uv and FFmpeg installation pages for self-installation",
        "Does not install packages, mutate package-manager configuration, or begin caption processing before explicit approval",
        "Reruns the capability preflight after any approved provisioning and refuses to continue if libass or libx264 remains unavailable"
      ]
    }
  ]
}