build-desktop-linux.yml

name: Build Desktop App (Linux only)

# Manual-trigger Linux-only mirror of build-desktop.yml. The cross-platform
# workflow already produces Linux artifacts on its nightly schedule; this one
# exists so contributors without a local Linux build host can request a fresh
# Linux bundle on demand without spinning up the full Win+macOS+Linux matrix.
# Artifacts are uploaded only — no nightly Release is published (the main
# workflow owns the `nightly` tag).

on:
  workflow_dispatch:
    inputs:
      version:
        description: 'Release version (e.g. 1.0.0-beta.1)'
        required: false
        default: ''
      electron_repo:
        description: 'Electron frontend repo (owner/repo)'
        required: false
        default: 'Project-N-E-K-O/N.E.K.O.-PC'
      electron_ref:
        description: 'Electron repo branch/tag/commit'
        required: false
        default: 'main'
      skip_signing:
        description: 'Skip code signing'
        required: false
        default: 'true'
        type: choice
        options:
          - 'true'
          - 'false'

env:
  PYTHON_VERSION: '3.11'
  NODE_VERSION: '20'
  # Anonymous embedding profile id (compatibility contract — bumped only when
  # the upstream weights/tokenizer become incompatible with previously-cached
  # vectors). The repo + revision together pin the actual artifacts.
  EMBEDDING_MODEL_REPO: jinaai/jina-embeddings-v5-text-nano-retrieval
  EMBEDDING_MODEL_PROFILE_ID: local-text-retrieval-v1
  EMBEDDING_MODEL_REVISION: ac5d898c8d382b17167c33e5c8af644a3519b47d

jobs:
  # ──────────────────────────────────────────────
  # Step 0: Calculate version
  # ──────────────────────────────────────────────
  version:
    if: github.repository == 'Project-N-E-K-O/N.E.K.O'
    runs-on: ubuntu-latest
    outputs:
      version: ${{ steps.calc.outputs.VERSION }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Calculate version
        id: calc
        run: |
          if [[ "$GITHUB_REF" == refs/tags/v* ]]; then
            VERSION="${GITHUB_REF#refs/tags/v}"
          elif [[ -n "${{ github.event.inputs.version }}" ]]; then
            VERSION="${{ github.event.inputs.version }}"
          else
            COMMIT_SHORT=$(git rev-parse --short HEAD)
            DATE=$(date +%Y%m%d)
            VERSION="${DATE}-${COMMIT_SHORT}"
          fi
          echo "VERSION=$VERSION" >> $GITHUB_OUTPUT
          echo "Resolved version: $VERSION"

  # ──────────────────────────────────────────────
  # Step 1: Build Python backend with Nuitka (Linux only)
  # ──────────────────────────────────────────────
  build-python:
    if: github.repository == 'Project-N-E-K-O/N.E.K.O'
    needs: version
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            platform: linux
            artifact_name: python-backend-linux
            output_binary: projectneko_server

    runs-on: ${{ matrix.os }}
    timeout-minutes: 300
    steps:
      - name: Checkout backend repo
        uses: actions/checkout@v4

      - name: Set up Python ${{ env.PYTHON_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ env.PYTHON_VERSION }}

      - name: Install uv
        uses: astral-sh/setup-uv@v4

      - name: Create venv and install dependencies
        shell: bash
        run: |
          uv venv .venv
          # Embedding runtime deps (onnxruntime, tokenizers; CPU SIMD detect
          # reads numpy's __cpu_features__, no py-cpuinfo) are in
          # [project.dependencies] now — plain `uv sync` covers both the
          # build-time prepare step and the bundled runtime.
          # `--group galgame` adds rapidocr-onnxruntime + opencv-python-headless
          # for the bundled OCR backend; pyproject's [tool.uv].override-dependencies
          # blocks opencv-python (full) so only the headless cv2 ships.
          uv sync --group galgame
          uv pip install --python .venv nuitka ordered-set zstandard

      # --- Generate runtime config files (not tracked in git) ---
      - name: Generate default config files
        shell: bash
        run: |
          python -c "
          import json, os
          configs = {
            'config/characters.json': {'主人': {'档案名': '', '性别': '', '昵称': ''}, '猫娘': {}, '当前猫娘': ''},
            'config/characters.en.json': {'主人': {'档案名': '', '性别': '', '昵称': ''}, '猫娘': {}, '当前猫娘': ''},
            'config/characters.ja.json': {'主人': {'档案名': '', '性别': '', '昵称': ''}, '猫娘': {}, '当前猫娘': ''},
            'config/characters.ko.json': {'主人': {'档案名': '', '性别': '', '昵称': ''}, '猫娘': {}, '当前猫娘': ''},
            'config/characters.ru.json': {'主人': {'档案名': '', '性别': '', '昵称': ''}, '猫娘': {}, '当前猫娘': ''},
            'config/characters.zh-CN.json': {'主人': {'档案名': '', '性别': '', '昵称': ''}, '猫娘': {}, '当前猫娘': ''},
            'config/characters.zh-TW.json': {'主人': {'档案名': '', '性别': '', '昵称': ''}, '猫娘': {}, '当前猫娘': ''},
            'config/core_config.json': {},
            'config/user_preferences.json': [],
          }
          for path, data in configs.items():
            if not os.path.exists(path):
              with open(path, 'w', encoding='utf-8') as f:
                json.dump(data, f, ensure_ascii=False, indent=2)
              print(f'Created {path}')
          "
          # Copy browser_use prompt .md templates from installed package
          mkdir -p data/browser_use_prompts
          .venv/bin/python -c "
          import shutil
          from pathlib import Path
          import browser_use.agent.system_prompts as sp
          src = Path(sp.__file__).parent
          dst = Path('data/browser_use_prompts')
          dst.mkdir(parents=True, exist_ok=True)
          for md in src.glob('*.md'):
              shutil.copy2(md, dst / md.name)
              print(f'Copied {md.name}')
          "
          echo "=== browser_use_prompts ==="
          ls -la data/browser_use_prompts/
          echo "Default config files and directories ready."

      # --- Platform-specific system dependencies ---
      - name: Install Linux system dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            build-essential \
            pkg-config \
            portaudio19-dev \
            patchelf \
            ccache

      # --- Pre-install Playwright Chromium for bundle ---
      - name: Install Playwright Chromium
        shell: bash
        run: |
          mkdir -p playwright_browsers
          PLAYWRIGHT_BROWSERS_PATH="$(pwd)/playwright_browsers" \
            .venv/bin/python -m playwright install chromium || \
            echo "[WARNING] Playwright Chromium install failed; will download on first run."

      # --- Prepare local embedding model assets ---
      # Mirrors the upstream Hugging Face repo at a pinned revision into the
      # anonymous profile folder so the bundled app can run vector memory
      # offline. The runtime (memory/embeddings.py) auto-falls back to bundled
      # assets when the user's app-data profile is incomplete.
      # Cache the weights by revision so huggingface.co is hit at most once per
      # pin (per OS) instead of every build — the same per-IP HTTP 429 throttle
      # that broke the Docker build can bite here too. Cache hit -> the next step
      # is a no-op ("keep existing"); only a cold cache actually downloads.
      - name: Cache embedding model weights
        uses: actions/cache@v4
        with:
          path: data/embedding_models
          key: embedding-model-${{ runner.os }}-${{ env.EMBEDDING_MODEL_REVISION }}-both

      - name: Prepare embedding model assets
        shell: bash
        run: |
          .venv/bin/python scripts/prepare_embedding_model.py \
            --repo "$EMBEDDING_MODEL_REPO" \
            --revision "$EMBEDDING_MODEL_REVISION" \
            --profile-id "$EMBEDDING_MODEL_PROFILE_ID" \
            --output-root data/embedding_models \
            --variant both

      # --- Warm tiktoken o200k_base cache for offline use (PR #929) ---
      # tiktoken downloads encoding blobs (~1.5 MB each) on first use into
      # TIKTOKEN_CACHE_DIR; without warming, the bundled app would either
      # need network at startup or fall back to the heuristic counter.
      # launcher.py points TIKTOKEN_CACHE_DIR at data/tiktoken_cache when it
      # exists in the bundle.
      - name: Warm tiktoken cache (o200k_base)
        shell: bash
        run: |
          mkdir -p data/tiktoken_cache
          TIKTOKEN_CACHE_DIR="$(pwd)/data/tiktoken_cache" \
            .venv/bin/python -c "import tiktoken; tiktoken.get_encoding('o200k_base')"
          echo "=== tiktoken_cache ==="
          ls -la data/tiktoken_cache/

      - name: Set up Node.js for frontend builds
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}

      - name: Build all frontend projects
        shell: bash
        run: |
          ./build_frontend.sh
          bash scripts/verify_frontend_build.sh

      # --- Build with Nuitka ---
      - name: Build with Nuitka
        shell: bash
        run: |
          NUITKA_OPTS="--standalone"
          NUITKA_OPTS="$NUITKA_OPTS --output-dir=dist"
          NUITKA_OPTS="$NUITKA_OPTS --output-filename=projectneko_server"

          # Config data files
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/__init__.py=config/__init__.py"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/api_providers.json=config/api_providers.json"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/characters.json=config/characters.json"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/characters.en.json=config/characters.en.json"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/characters.ja.json=config/characters.ja.json"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/characters.ko.json=config/characters.ko.json"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/characters.ru.json=config/characters.ru.json"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/characters.zh-CN.json=config/characters.zh-CN.json"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/characters.zh-TW.json=config/characters.zh-TW.json"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/core_config.json=config/core_config.json"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=config/user_preferences.json=config/user_preferences.json"

          # Data directories
          NUITKA_OPTS="$NUITKA_OPTS --include-data-dir=assets=assets"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-dir=templates=templates"
          NUITKA_OPTS="$NUITKA_OPTS --include-data-dir=data/browser_use_prompts=data/browser_use_prompts"
          # Plugin manager UI (vite dist; gitignored)
          if [ -d "frontend/plugin-manager/dist" ]; then
            NUITKA_OPTS="$NUITKA_OPTS --include-data-dir=frontend/plugin-manager/dist=frontend/plugin-manager/dist"
          else
            echo "[WARNING] frontend/plugin-manager/dist missing; /ui will be empty in bundle."
          fi
          if [ -d "plugin/plugins" ]; then
            NUITKA_OPTS="$NUITKA_OPTS --include-data-dir=plugin/plugins=plugin/plugins"
          fi
          NUITKA_OPTS="$NUITKA_OPTS --include-data-dir=playwright_browsers=playwright_browsers"
          if [ -d "data/tiktoken_cache" ]; then
            NUITKA_OPTS="$NUITKA_OPTS --include-data-dir=data/tiktoken_cache=data/tiktoken_cache"
          fi
          if [ -d "data/embedding_models" ]; then
            NUITKA_OPTS="$NUITKA_OPTS --include-data-dir=data/embedding_models=data/embedding_models"
          fi
          NUITKA_OPTS="$NUITKA_OPTS --include-data-dir=static=static"

          # Packages
          NUITKA_OPTS="$NUITKA_OPTS --include-package=uvicorn"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=fastapi"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=starlette"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=jinja2"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=websockets"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=app"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=config"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=plugin"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=brain"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=main_logic"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=main_routers"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=memory"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=utils"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=steamworks"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=browser_use"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=browser_use_sdk"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=playwright"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=tiktoken"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=tiktoken_ext"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=onnxruntime"
          # onnxruntime.transformers/ is a benchmark + model-conversion toolbox
          # nothing at runtime imports (rapidocr uses only the core InferenceSession).
          # --include-package=onnxruntime would recurse and compile the whole subtree,
          # including a 130k+ line gpt2 benchmark C unit that crashes the C backend.
          # Mirrors build-desktop.yml.
          NUITKA_OPTS="$NUITKA_OPTS --nofollow-import-to=onnxruntime.transformers"
          NUITKA_OPTS="$NUITKA_OPTS --include-package=tokenizers"

          # Package data
          NUITKA_OPTS="$NUITKA_OPTS --include-package-data=audiolab"
          NUITKA_OPTS="$NUITKA_OPTS --include-package-data=pyrnnoise"
          NUITKA_OPTS="$NUITKA_OPTS --include-package-data=browser_use"
          # NOTE: do NOT use --include-package-data=playwright here,
          # Nuitka's built-in playwright plugin handles driver/node automatically
          # and adding package-data would conflict with the exe at the same path.
          NUITKA_OPTS="$NUITKA_OPTS --include-package-data=jinja2"
          NUITKA_OPTS="$NUITKA_OPTS --include-package-data=certifi"
          NUITKA_OPTS="$NUITKA_OPTS --include-package-data=bilibili_api"

          # Exclusions
          NUITKA_OPTS="$NUITKA_OPTS --nofollow-import-to=plugin.plugins"
          NUITKA_OPTS="$NUITKA_OPTS --nofollow-import-to=audiolab"
          NUITKA_OPTS="$NUITKA_OPTS --nofollow-import-to=pyrnnoise"
          NUITKA_OPTS="$NUITKA_OPTS --nofollow-import-to=matplotlib"
          NUITKA_OPTS="$NUITKA_OPTS --nofollow-import-to=pytest"

          # Plugin & CI flags
          NUITKA_OPTS="$NUITKA_OPTS --enable-plugin=dill-compat"
          NUITKA_OPTS="$NUITKA_OPTS --assume-yes-for-downloads"

          # Steam files (include if present in repo).
          # PR #1264 moved native libs into steamworks/; runtime still loads them
          # from the exe sibling dir (see steamworks/__init__.py), so dest stays at root.
          # --include-package=steamworks above also picks up every native lib in
          # steamworks/ as package data, so strip the non-Linux copies from the
          # source tree before invoking nuitka (an x-platform .so/.dll/.dylib
          # mix trips Nuitka's arch check).
          NUITKA_OPTS="$NUITKA_OPTS --include-data-files=steam_appid.txt=steam_appid.txt"
          rm -f steamworks/SteamworksPy.dylib steamworks/libsteam_api.dylib \
                steamworks/SteamworksPy64.dll steamworks/steam_api64.dll steamworks/steam_api64.lib
          if [ -f "steamworks/libsteam_api.so" ]; then
            NUITKA_OPTS="$NUITKA_OPTS --include-data-files=steamworks/libsteam_api.so=libsteam_api.so"
          fi
          if [ -f "steamworks/SteamworksPy.so" ]; then
            NUITKA_OPTS="$NUITKA_OPTS --include-data-files=steamworks/SteamworksPy.so=SteamworksPy.so"
          fi

          echo "=== Nuitka options ==="
          echo "$NUITKA_OPTS" | tr ' ' '\n'
          echo "======================"

          .venv/bin/python -m nuitka $NUITKA_OPTS launcher.py

      # --- Rename Nuitka output directory ---
      - name: Rename build output
        shell: bash
        run: |
          if [ -d "dist/launcher.dist" ]; then
            mv dist/launcher.dist dist/Xiao8
          fi
          echo "=== Build output ==="
          ls -la dist/Xiao8/ || echo "dist/Xiao8 not found!"

      # --- Verify bundled embedding + tiktoken assets ---
      # memory/embeddings.py treats every onnx_data sidecar plus both fp32
      # and int8 variants as required at session-load time; tiktoken
      # o200k_base must also be present so utils.tokenize doesn't fall
      # back to the heuristic counter. Use -s (non-empty) so a zero-byte
      # file from an interrupted download fails the build, matching
      # prepare_embedding_model.py's own non-empty contract.
      - name: Verify bundled offline assets
        shell: bash
        run: |
          set -euo pipefail
          if [ ! -d "dist/Xiao8" ]; then
            echo "::error::dist/Xiao8 not found, cannot verify"
            exit 1
          fi
          embedding_required=(
            "tokenizer.json"
            "onnx/model.onnx"
            "onnx/model.onnx_data"
            "onnx/model_quantized.onnx"
            "onnx/model_quantized.onnx_data"
          )
          missing=0
          for rel in "${embedding_required[@]}"; do
            if [ ! -s "dist/Xiao8/data/embedding_models/$EMBEDDING_MODEL_PROFILE_ID/$rel" ]; then
              echo "::error::missing or empty bundled embedding asset: $rel"
              missing=1
            fi
          done
          if ! ls dist/Xiao8/data/tiktoken_cache/ 2>/dev/null | grep -q .; then
            echo "::error::data/tiktoken_cache empty in bundle (o200k_base would re-download at runtime)"
            missing=1
          fi
          [ "$missing" -eq 0 ]

      # --- Generic dist invariant check ---
      # Catches the broader class of "Nuitka silently dropped a required
      # asset" bugs: missing config/, static/, plugin/plugins, frontend
      # build output, or built-in plugin.toml files. See
      # scripts/check_nuitka_dist.py for the full asset list and rationale.
      - name: Verify Nuitka dist invariants
        shell: bash
        run: |
          set -euo pipefail
          .venv/bin/python scripts/check_nuitka_dist.py dist/Xiao8

      - name: Upload Python backend artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.artifact_name }}
          path: dist/Xiao8/
          retention-days: 7
          compression-level: 6

  # ──────────────────────────────────────────────
  # Step 2: Build Electron desktop app (Linux only)
  # ──────────────────────────────────────────────
  build-electron:
    if: github.repository == 'Project-N-E-K-O/N.E.K.O'
    needs: [version, build-python]
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            platform: linux
            python_artifact: python-backend-linux
            electron_args: '--linux'
            artifact_name: desktop-linux-x64

    runs-on: ${{ matrix.os }}
    timeout-minutes: 180
    steps:
      - name: Checkout Electron frontend
        uses: actions/checkout@v4
        with:
          repository: ${{ github.event.inputs.electron_repo || 'Project-N-E-K-O/N.E.K.O.-PC' }}
          ref: ${{ github.event.inputs.electron_ref || 'main' }}
          token: ${{ secrets.ELECTRON_REPO_TOKEN || secrets.GITHUB_TOKEN }}
          path: electron-app

      - name: Set up Node.js ${{ env.NODE_VERSION }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ env.NODE_VERSION }}
          cache: 'npm'
          cache-dependency-path: electron-app/package-lock.json

      - name: Download Python backend artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ matrix.python_artifact }}
          path: electron-app/bin

      - name: Make server binary executable
        run: chmod +x electron-app/bin/projectneko_server

      # --- Cache Electron binary to avoid download failures ---
      - name: Cache Electron binary
        uses: actions/cache@v4
        with:
          path: |
            ~/.cache/electron
          key: electron-${{ matrix.os }}-${{ matrix.platform }}-${{ hashFiles('electron-app/package-lock.json') }}
          restore-keys: |
            electron-${{ matrix.os }}-${{ matrix.platform }}-

      - name: Install npm dependencies
        working-directory: electron-app
        run: npm ci || npm install

      - name: Update version in package.json
        working-directory: electron-app
        run: |
          node -e "
            const fs = require('fs');
            const pkg = JSON.parse(fs.readFileSync('package.json', 'utf8'));
            pkg.version = '${{ needs.version.outputs.version }}'.replace(/^[^0-9]*/, '');
            if (!/^\d+\.\d+\.\d+/.test(pkg.version)) {
              pkg.version = '0.0.0-' + pkg.version;
            }
            fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');
            console.log('Updated version to:', pkg.version);
          "

      - name: Install Linux build dependencies
        run: |
          sudo apt-get update
          sudo apt-get install -y --no-install-recommends \
            dpkg \
            fakeroot

      # Linux .deb requires a maintainer email; N.E.K.O.-PC package.json
      # only has author name without email, so patch it here
      - name: Set Linux deb maintainer
        working-directory: electron-app
        run: |
          node -e "
            const fs = require('fs');
            const pkg = JSON.parse(fs.readFileSync('package.json', 'utf8'));
            pkg.build = pkg.build || {};
            pkg.build.linux = pkg.build.linux || {};
            pkg.build.linux.maintainer = 'Project N.E.K.O. <projectneko@yahoo.com>';
            fs.writeFileSync('package.json', JSON.stringify(pkg, null, 2) + '\n');
            console.log('Set Linux deb maintainer');
          "

      # --- Build Electron app ---
      - name: Build Electron app
        working-directory: electron-app
        run: npx electron-builder ${{ matrix.electron_args }} --publish never
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: List build output
        working-directory: electron-app/dist
        shell: bash
        run: |
          echo "=== Build artifacts ==="
          ls -lahR . || true

      - name: Upload desktop artifact
        uses: actions/upload-artifact@v4
        with:
          name: ${{ matrix.artifact_name }}
          path: |
            electron-app/dist/*.AppImage
            electron-app/dist/*.deb
            electron-app/dist/*.tar.gz
            electron-app/dist/*.blockmap
            electron-app/dist/latest*.yml
          if-no-files-found: warn
          retention-days: 14