ci.yml

name: CI

permissions:
  contents: read

on:
  workflow_call:
  push:
    branches: [main]
  workflow_dispatch:
  schedule:
    - cron: '23 8 * * 1'
jobs:
  release-linux-smoke:
    strategy:
      fail-fast: false
      matrix:
        include:
          - goarch: amd64
            runner: ubuntu-latest
            go_arch_name: amd64
          - goarch: arm64
            runner: ubuntu-24.04-arm
            go_arch_name: arm64
    runs-on: ${{ contains(fromJSON('["ubuntu-latest","ubuntu-24.04","ubuntu-22.04"]'), matrix.runner) && github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || matrix.runner }}
    container:
      image: ubuntu:22.04
    steps:
      - name: Install build tools
        env:
          DEBIAN_FRONTEND: noninteractive
        run: |
          apt-get update
          apt-get install -y gcc g++ make git curl tar gzip file binutils libsqlite3-dev

      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Install Go
        run: |
          GO_VERSION=$(grep '^go ' go.mod | awk '{print $2}')
          curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${{ matrix.go_arch_name }}.tar.gz" -o go.tar.gz
          tar -C /usr/local -xzf go.tar.gz
          rm go.tar.gz
          echo "/usr/local/go/bin" >> "$GITHUB_PATH"
          echo "$HOME/go/bin" >> "$GITHUB_PATH"

      - name: Build Linux release binary
        env:
          GOOS: linux
          GOARCH: ${{ matrix.goarch }}
          CGO_ENABLED: '1'
        run: |
          export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH"
          COMMIT=$(printf '%s' "$GITHUB_SHA" | cut -c1-8)
          VERSION="ci-$COMMIT"

          mkdir -p dist
          LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=${COMMIT} -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -extldflags '-lstdc++ -lm'"
          go build -tags "fts5 sqlite_vec" -trimpath -buildvcs=false -ldflags="$LDFLAGS" -o dist/msgvault ./cmd/msgvault

          echo "--- Binary info ---"
          file dist/msgvault
          ldd dist/msgvault || true

          echo "--- Runtime requirements ---"
          objdump -T dist/msgvault 2>/dev/null | grep -oP 'GLIBC_\d+\.\d+' | sort -uV | tail -1 || true
          objdump -T dist/msgvault 2>/dev/null | grep -oP 'GLIBCXX_\d+\.\d+(\.\d+)?' | sort -uV | tail -1 || true

          echo "--- Smoke test ---"
          SMOKE_OUT=$(dist/msgvault version 2>&1)
          echo "$SMOKE_OUT"
          echo "$SMOKE_OUT" | grep -q "$VERSION" || { echo "FATAL: version output doesn't match CI build version"; exit 1; }

  test:
    runs-on: macos-15
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      - name: Install golangci-lint
        run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2

      - name: Test
        run: make test

      - name: Check OpenAPI artifacts
        run: make openapi-check

      - name: Lint
        run: make lint-ci

      - name: Install govulncheck
        run: go install golang.org/x/vuln/cmd/govulncheck@v1.1.4

      - name: Vulnerability check
        run: govulncheck -tags "fts5 sqlite_vec" ./...

  frontend:
    runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-24.04' }}
    container:
      image: mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - name: Install container build tools
        # Mirror hiccups have hung plain apt-get for 15+ minutes; bounded
        # timeouts plus retries turn an outage into a short delay.
        run: |
          for attempt in 1 2 3; do
            if apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30 \
              && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip; then
              exit 0
            fi
            echo "apt attempt $attempt failed; retrying" >&2
            sleep 30
          done
          exit 1

      - name: Trust the checkout in the container
        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
        with:
          bun-version-file: web/package.json

      - name: Install frontend dependencies
        working-directory: web
        run: bun install --frozen-lockfile

      - name: Check, test, and build frontend
        run: make web-check web-test web-build

      # The e2e specs generate their archive fixture by running a Go test;
      # compile it here so the first Playwright test doesn't pay the cold
      # cgo build inside its own timeout.
      - name: Precompile e2e archive fixture
        run: go test -tags "fts5 sqlite_vec" -run '^$' -count=1 ./internal/query

      - name: Run deterministic browser and visual tests
        run: make web-test-browser

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: playwright-browser-failures
          path: |
            web/playwright-report
            web/test-results
          if-no-files-found: ignore

  web-e2e:
    runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-24.04' }}
    container:
      image: mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      - name: Install container build tools
        # Mirror hiccups have hung plain apt-get for 15+ minutes; bounded
        # timeouts plus retries turn an outage into a short delay.
        run: |
          for attempt in 1 2 3; do
            if apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30 \
              && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip; then
              exit 0
            fi
            echo "apt attempt $attempt failed; retrying" >&2
            sleep 30
          done
          exit 1

      - name: Trust the checkout in the container
        run: git config --global --add safe.directory "$GITHUB_WORKSPACE"

      - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
        with:
          bun-version-file: web/package.json

      - name: Install frontend dependencies
        working-directory: web
        run: bun install --frozen-lockfile

      # See the frontend job: warm the cgo fixture compile outside test timeouts.
      - name: Precompile e2e archive fixture
        run: go test -tags "fts5 sqlite_vec" -run '^$' -count=1 ./internal/query

      - name: Run accessibility, keyboard, security, and mixed-archive gates
        run: make web-e2e

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: failure()
        with:
          name: playwright-e2e-failures
          path: |
            web/playwright-report
            web/test-results
          if-no-files-found: ignore

  large-archive-benchmark:
    if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
    runs-on: macos-15
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      - name: Run mixed-archive analytical reference benchmark
        run: |
          go test -tags "fts5 sqlite_vec" ./internal/query -run '^$' \
            -bench '^BenchmarkExploreLargeArchive$' -benchtime=3x -count=1 \
            | tee large-archive-benchmark.txt

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: large-archive-benchmark
          path: large-archive-benchmark.txt
          if-no-files-found: error

  test-windows:
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      # sqlite-vec's CGo bindings #include <sqlite3.h>, which isn't on
      # the default windows-latest include path. Install it from MSYS2
      # (already present on the runner) and point CGO_CFLAGS at it.
      #
      # arrow-go/v18's cdata helpers.h declares its helpers as plain
      # `inline` (not `static inline`). Under C99 that requires exactly
      # one external definition somewhere, and no TU in arrow-go
      # provides one. -fgnu89-inline forces every TU to emit the
      # external copy, and --allow-multiple-definition lets MinGW's ld
      # pick the first and continue instead of erroring on the
      # duplicates that flag produces.
      - name: Install sqlite3 dev headers
        shell: pwsh
        run: |
          C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3

      - name: Build
        shell: pwsh
        env:
          CGO_ENABLED: "1"
          CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline"
          CGO_LDFLAGS: "-Wl,--allow-multiple-definition"
        run: go build -tags "fts5 sqlite_vec" -o msgvault.exe ./cmd/msgvault

      - name: Test
        shell: pwsh
        env:
          CGO_ENABLED: "1"
          CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline"
          CGO_LDFLAGS: "-Wl,--allow-multiple-definition"
        run: |
          $separatePackages = @(
            "go.kenn.io/msgvault/cmd/msgvault/cmd"
            "go.kenn.io/msgvault/internal/api"
            "go.kenn.io/msgvault/internal/importer"
            "go.kenn.io/msgvault/internal/store"
            "go.kenn.io/msgvault/internal/sync"
            "go.kenn.io/msgvault/internal/vector/embed"
            "go.kenn.io/msgvault/internal/vector/sqlitevec"
            "go.kenn.io/msgvault/scripts"
          )
          $packages = go list -tags "fts5 sqlite_vec" ./... |
            Where-Object { $_ -notin $separatePackages }
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          go test -timeout 20m -tags "fts5 sqlite_vec" $packages

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        with:
          name: msgvault-windows-amd64
          path: msgvault.exe

  test-windows-slow-packages:
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      - name: Install sqlite3 dev headers
        shell: pwsh
        run: |
          C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3

      - name: Test slow packages
        shell: pwsh
        env:
          CGO_ENABLED: "1"
          CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline"
          CGO_LDFLAGS: "-Wl,--allow-multiple-definition"
        run: |
          go test -timeout 20m -tags "fts5 sqlite_vec" `
            ./internal/api `
            ./internal/importer `
            ./internal/store `
            ./scripts

  test-windows-sharded:
    name: test-windows-sharded (${{ matrix.name }})
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: cli
            package: ./cmd/msgvault/cmd
          - name: sync-vector-embed
            package: ./internal/sync
            package2: ./internal/vector/embed
          - name: vector-sqlite
            package: ./internal/vector/sqlitevec
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      - name: Install sqlite3 dev headers
        shell: pwsh
        run: |
          C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3

      - name: Test package in isolated shards
        shell: pwsh
        env:
          CGO_ENABLED: "1"
          CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline"
          CGO_LDFLAGS: "-Wl,--allow-multiple-definition"
        run: |
          $packages = @('${{ matrix.package }}', '${{ matrix.package2 }}') |
            Where-Object { $_ }
          foreach ($package in $packages) {
            ./scripts/test-package-shards.ps1 `
              -Package $package `
              -ShardCount 4 `
              -Tags "fts5 sqlite_vec" `
              -Timeout 20m
            if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }
          }

  test-windows-pack-restore:
    runs-on: windows-latest
    timeout-minutes: 20
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      - name: Install sqlite3 dev headers
        shell: pwsh
        run: |
          C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3

      - name: Measure packed and loose restore layouts
        shell: pwsh
        env:
          CGO_ENABLED: "1"
          CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline"
          CGO_LDFLAGS: "-Wl,--allow-multiple-definition"
        run: |
          go test -tags "fts5 sqlite_vec" ./internal/backupapp `
            -run '^$' -bench '^BenchmarkBackupRestoreLayouts$' -benchtime=1x -count=3 `
            2>&1 | Tee-Object -FilePath pack-restore-benchmark.txt
          if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE }

      - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
        if: always()
        with:
          name: msgvault-windows-pack-restore-benchmark
          path: pack-restore-benchmark.txt
          if-no-files-found: ignore

  # pgvector lane: runs every pgvector-tagged test against a live
  # pgvector/pgvector image — backend, embed queue/worker/enqueue, hybrid
  # filter, scheduler embed-job, and serve/search CLI. The `pgvector` build
  # tag is required; without it these files compile out and ship no tests,
  # so this code would otherwise have zero CI coverage.
  test-pgvector:
    runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-latest' }}
    services:
      postgres:
        image: pgvector/pgvector:pg16
        env:
          POSTGRES_USER: msgvault_test
          POSTGRES_PASSWORD: testpw
          POSTGRES_DB: msgvault_test
        ports:
          - 5432:5432
        options: >-
          --health-cmd "pg_isready -U msgvault_test -d msgvault_test"
          --health-interval 5s
          --health-timeout 5s
          --health-retries 10
    env:
      MSGVAULT_TEST_DB: postgres://msgvault_test:testpw@localhost:5432/msgvault_test
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      - name: Install psql client
        run: |
          sudo apt-get update
          sudo apt-get install -y postgresql-client

      - name: Enable pgvector extension
        env:
          PGPASSWORD: testpw
        run: |
          psql -h localhost -U msgvault_test -d msgvault_test -c "CREATE EXTENSION IF NOT EXISTS vector"

      - name: Test (pgvector-tagged)
        run: go test -tags "fts5 sqlite_vec pgvector" -count=1 ./internal/vector/... ./internal/scheduler/... ./cmd/msgvault/cmd/...

  nix-build:
    runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-latest' }}
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22

      - name: Build
        run: nix build

  # PostgreSQL backend lane: runs the full Go test suite against a live
  # PostgreSQL service. Tests that use testutil.NewTestStore read
  # MSGVAULT_TEST_DB and switch backends without a recompile. Concurrency
  # tests use -count=5 so non-deterministic races (lost updates, missing
  # unique constraints) surface here rather than slipping past a single
  # pass.
  test-postgres:
    runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-latest' }}
    services:
      postgres:
        image: postgres:16
        env:
          POSTGRES_USER: msgvault_test
          POSTGRES_PASSWORD: msgvault_test
          POSTGRES_DB: msgvault_test
        ports:
          - 5432:5432
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
    env:
      MSGVAULT_TEST_DB: postgres://msgvault_test:msgvault_test@localhost:5432/msgvault_test?sslmode=disable
    steps:
      - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0

      - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0
        with:
          go-version-file: go.mod

      # This lane uses a stock postgres:16 image (no pgvector extension), so it
      # must NOT build the pgvector tag — those tests CREATE EXTENSION "vector"
      # and fail here. pgvector is covered by the dedicated test-pgvector job
      # against a pgvector/pgvector image. (Run go test directly rather than
      # `make test-pg`, whose PG_TEST_TAGS include pgvector.)
      - name: Test (PostgreSQL)
        run: go test -tags "fts5 sqlite_vec" ./...

      - name: Concurrency tests (-count=5)
        run: |
          go test -tags fts5 -count=5 -run 'Concurrent' ./internal/store