Route trusted Linux builds to managed runners (#508) #1761
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: CI | |
| permissions: | |
| contents: read | |
| on: | |
| workflow_call: | |
| push: | |
| branches: [main] | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '23 8 * * 1' | |
| jobs: | |
| release-linux-smoke: | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - goarch: amd64 | |
| runner: ubuntu-latest | |
| go_arch_name: amd64 | |
| - goarch: arm64 | |
| runner: ubuntu-24.04-arm | |
| go_arch_name: arm64 | |
| runs-on: ${{ contains(fromJSON('["ubuntu-latest","ubuntu-24.04","ubuntu-22.04"]'), matrix.runner) && github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || matrix.runner }} | |
| container: | |
| image: ubuntu:22.04 | |
| steps: | |
| - name: Install build tools | |
| env: | |
| DEBIAN_FRONTEND: noninteractive | |
| run: | | |
| apt-get update | |
| apt-get install -y gcc g++ make git curl tar gzip file binutils libsqlite3-dev | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Install Go | |
| run: | | |
| GO_VERSION=$(grep '^go ' go.mod | awk '{print $2}') | |
| curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${{ matrix.go_arch_name }}.tar.gz" -o go.tar.gz | |
| tar -C /usr/local -xzf go.tar.gz | |
| rm go.tar.gz | |
| echo "/usr/local/go/bin" >> "$GITHUB_PATH" | |
| echo "$HOME/go/bin" >> "$GITHUB_PATH" | |
| - name: Build Linux release binary | |
| env: | |
| GOOS: linux | |
| GOARCH: ${{ matrix.goarch }} | |
| CGO_ENABLED: '1' | |
| run: | | |
| export PATH="/usr/local/go/bin:$HOME/go/bin:$PATH" | |
| COMMIT=$(printf '%s' "$GITHUB_SHA" | cut -c1-8) | |
| VERSION="ci-$COMMIT" | |
| mkdir -p dist | |
| LDFLAGS="-s -w -X go.kenn.io/msgvault/cmd/msgvault/cmd.Version=${VERSION} -X go.kenn.io/msgvault/cmd/msgvault/cmd.Commit=${COMMIT} -X go.kenn.io/msgvault/cmd/msgvault/cmd.BuildDate=$(date -u +%Y-%m-%dT%H:%M:%SZ) -extldflags '-lstdc++ -lm'" | |
| go build -tags "fts5 sqlite_vec" -trimpath -buildvcs=false -ldflags="$LDFLAGS" -o dist/msgvault ./cmd/msgvault | |
| echo "--- Binary info ---" | |
| file dist/msgvault | |
| ldd dist/msgvault || true | |
| echo "--- Runtime requirements ---" | |
| objdump -T dist/msgvault 2>/dev/null | grep -oP 'GLIBC_\d+\.\d+' | sort -uV | tail -1 || true | |
| objdump -T dist/msgvault 2>/dev/null | grep -oP 'GLIBCXX_\d+\.\d+(\.\d+)?' | sort -uV | tail -1 || true | |
| echo "--- Smoke test ---" | |
| SMOKE_OUT=$(dist/msgvault version 2>&1) | |
| echo "$SMOKE_OUT" | |
| echo "$SMOKE_OUT" | grep -q "$VERSION" || { echo "FATAL: version output doesn't match CI build version"; exit 1; } | |
| test: | |
| runs-on: macos-15 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Install golangci-lint | |
| run: go install github.com/golangci/golangci-lint/v2/cmd/golangci-lint@v2.12.2 | |
| - name: Test | |
| run: make test | |
| - name: Check OpenAPI artifacts | |
| run: make openapi-check | |
| - name: Lint | |
| run: make lint-ci | |
| - name: Install govulncheck | |
| run: go install golang.org/x/vuln/cmd/govulncheck@v1.1.4 | |
| - name: Vulnerability check | |
| run: govulncheck -tags "fts5 sqlite_vec" ./... | |
| frontend: | |
| runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-24.04' }} | |
| container: | |
| image: mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - name: Install container build tools | |
| # Mirror hiccups have hung plain apt-get for 15+ minutes; bounded | |
| # timeouts plus retries turn an outage into a short delay. | |
| run: | | |
| for attempt in 1 2 3; do | |
| if apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30 \ | |
| && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip; then | |
| exit 0 | |
| fi | |
| echo "apt attempt $attempt failed; retrying" >&2 | |
| sleep 30 | |
| done | |
| exit 1 | |
| - name: Trust the checkout in the container | |
| run: git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 | |
| with: | |
| bun-version-file: web/package.json | |
| - name: Install frontend dependencies | |
| working-directory: web | |
| run: bun install --frozen-lockfile | |
| - name: Check, test, and build frontend | |
| run: make web-check web-test web-build | |
| # The e2e specs generate their archive fixture by running a Go test; | |
| # compile it here so the first Playwright test doesn't pay the cold | |
| # cgo build inside its own timeout. | |
| - name: Precompile e2e archive fixture | |
| run: go test -tags "fts5 sqlite_vec" -run '^$' -count=1 ./internal/query | |
| - name: Run deterministic browser and visual tests | |
| run: make web-test-browser | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: failure() | |
| with: | |
| name: playwright-browser-failures | |
| path: | | |
| web/playwright-report | |
| web/test-results | |
| if-no-files-found: ignore | |
| web-e2e: | |
| runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-24.04' }} | |
| container: | |
| image: mcr.microsoft.com/playwright:v1.61.1-noble@sha256:5b8f294aff9041b7191c34a4bab3ac270157a28774d4b0660e9743297b697e48 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Install container build tools | |
| # Mirror hiccups have hung plain apt-get for 15+ minutes; bounded | |
| # timeouts plus retries turn an outage into a short delay. | |
| run: | | |
| for attempt in 1 2 3; do | |
| if apt-get update -o Acquire::Retries=3 -o Acquire::http::Timeout=30 \ | |
| && apt-get install -y --no-install-recommends -o Acquire::Retries=3 -o Acquire::http::Timeout=30 gcc g++ make unzip; then | |
| exit 0 | |
| fi | |
| echo "apt attempt $attempt failed; retrying" >&2 | |
| sleep 30 | |
| done | |
| exit 1 | |
| - name: Trust the checkout in the container | |
| run: git config --global --add safe.directory "$GITHUB_WORKSPACE" | |
| - uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0 | |
| with: | |
| bun-version-file: web/package.json | |
| - name: Install frontend dependencies | |
| working-directory: web | |
| run: bun install --frozen-lockfile | |
| # See the frontend job: warm the cgo fixture compile outside test timeouts. | |
| - name: Precompile e2e archive fixture | |
| run: go test -tags "fts5 sqlite_vec" -run '^$' -count=1 ./internal/query | |
| - name: Run accessibility, keyboard, security, and mixed-archive gates | |
| run: make web-e2e | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: failure() | |
| with: | |
| name: playwright-e2e-failures | |
| path: | | |
| web/playwright-report | |
| web/test-results | |
| if-no-files-found: ignore | |
| large-archive-benchmark: | |
| if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' | |
| runs-on: macos-15 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Run mixed-archive analytical reference benchmark | |
| run: | | |
| go test -tags "fts5 sqlite_vec" ./internal/query -run '^$' \ | |
| -bench '^BenchmarkExploreLargeArchive$' -benchtime=3x -count=1 \ | |
| | tee large-archive-benchmark.txt | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: always() | |
| with: | |
| name: large-archive-benchmark | |
| path: large-archive-benchmark.txt | |
| if-no-files-found: error | |
| test-windows: | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| # sqlite-vec's CGo bindings #include <sqlite3.h>, which isn't on | |
| # the default windows-latest include path. Install it from MSYS2 | |
| # (already present on the runner) and point CGO_CFLAGS at it. | |
| # | |
| # arrow-go/v18's cdata helpers.h declares its helpers as plain | |
| # `inline` (not `static inline`). Under C99 that requires exactly | |
| # one external definition somewhere, and no TU in arrow-go | |
| # provides one. -fgnu89-inline forces every TU to emit the | |
| # external copy, and --allow-multiple-definition lets MinGW's ld | |
| # pick the first and continue instead of erroring on the | |
| # duplicates that flag produces. | |
| - name: Install sqlite3 dev headers | |
| shell: pwsh | |
| run: | | |
| C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3 | |
| - name: Build | |
| shell: pwsh | |
| env: | |
| CGO_ENABLED: "1" | |
| CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline" | |
| CGO_LDFLAGS: "-Wl,--allow-multiple-definition" | |
| run: go build -tags "fts5 sqlite_vec" -o msgvault.exe ./cmd/msgvault | |
| - name: Test | |
| shell: pwsh | |
| env: | |
| CGO_ENABLED: "1" | |
| CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline" | |
| CGO_LDFLAGS: "-Wl,--allow-multiple-definition" | |
| run: | | |
| $separatePackages = @( | |
| "go.kenn.io/msgvault/cmd/msgvault/cmd" | |
| "go.kenn.io/msgvault/internal/api" | |
| "go.kenn.io/msgvault/internal/importer" | |
| "go.kenn.io/msgvault/internal/store" | |
| "go.kenn.io/msgvault/internal/sync" | |
| "go.kenn.io/msgvault/internal/vector/embed" | |
| "go.kenn.io/msgvault/internal/vector/sqlitevec" | |
| "go.kenn.io/msgvault/scripts" | |
| ) | |
| $packages = go list -tags "fts5 sqlite_vec" ./... | | |
| Where-Object { $_ -notin $separatePackages } | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| go test -timeout 20m -tags "fts5 sqlite_vec" $packages | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| with: | |
| name: msgvault-windows-amd64 | |
| path: msgvault.exe | |
| test-windows-slow-packages: | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Install sqlite3 dev headers | |
| shell: pwsh | |
| run: | | |
| C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3 | |
| - name: Test slow packages | |
| shell: pwsh | |
| env: | |
| CGO_ENABLED: "1" | |
| CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline" | |
| CGO_LDFLAGS: "-Wl,--allow-multiple-definition" | |
| run: | | |
| go test -timeout 20m -tags "fts5 sqlite_vec" ` | |
| ./internal/api ` | |
| ./internal/importer ` | |
| ./internal/store ` | |
| ./scripts | |
| test-windows-sharded: | |
| name: test-windows-sharded (${{ matrix.name }}) | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: cli | |
| package: ./cmd/msgvault/cmd | |
| - name: sync-vector-embed | |
| package: ./internal/sync | |
| package2: ./internal/vector/embed | |
| - name: vector-sqlite | |
| package: ./internal/vector/sqlitevec | |
| runs-on: windows-latest | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Install sqlite3 dev headers | |
| shell: pwsh | |
| run: | | |
| C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3 | |
| - name: Test package in isolated shards | |
| shell: pwsh | |
| env: | |
| CGO_ENABLED: "1" | |
| CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline" | |
| CGO_LDFLAGS: "-Wl,--allow-multiple-definition" | |
| run: | | |
| $packages = @('${{ matrix.package }}', '${{ matrix.package2 }}') | | |
| Where-Object { $_ } | |
| foreach ($package in $packages) { | |
| ./scripts/test-package-shards.ps1 ` | |
| -Package $package ` | |
| -ShardCount 4 ` | |
| -Tags "fts5 sqlite_vec" ` | |
| -Timeout 20m | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| } | |
| test-windows-pack-restore: | |
| runs-on: windows-latest | |
| timeout-minutes: 20 | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Install sqlite3 dev headers | |
| shell: pwsh | |
| run: | | |
| C:\msys64\usr\bin\pacman.exe -S --noconfirm --needed mingw-w64-x86_64-sqlite3 | |
| - name: Measure packed and loose restore layouts | |
| shell: pwsh | |
| env: | |
| CGO_ENABLED: "1" | |
| CGO_CFLAGS: "-IC:/msys64/mingw64/include -fgnu89-inline" | |
| CGO_LDFLAGS: "-Wl,--allow-multiple-definition" | |
| run: | | |
| go test -tags "fts5 sqlite_vec" ./internal/backupapp ` | |
| -run '^$' -bench '^BenchmarkBackupRestoreLayouts$' -benchtime=1x -count=3 ` | |
| 2>&1 | Tee-Object -FilePath pack-restore-benchmark.txt | |
| if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } | |
| - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 | |
| if: always() | |
| with: | |
| name: msgvault-windows-pack-restore-benchmark | |
| path: pack-restore-benchmark.txt | |
| if-no-files-found: ignore | |
| # pgvector lane: runs every pgvector-tagged test against a live | |
| # pgvector/pgvector image — backend, embed queue/worker/enqueue, hybrid | |
| # filter, scheduler embed-job, and serve/search CLI. The `pgvector` build | |
| # tag is required; without it these files compile out and ship no tests, | |
| # so this code would otherwise have zero CI coverage. | |
| test-pgvector: | |
| runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-latest' }} | |
| services: | |
| postgres: | |
| image: pgvector/pgvector:pg16 | |
| env: | |
| POSTGRES_USER: msgvault_test | |
| POSTGRES_PASSWORD: testpw | |
| POSTGRES_DB: msgvault_test | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd "pg_isready -U msgvault_test -d msgvault_test" | |
| --health-interval 5s | |
| --health-timeout 5s | |
| --health-retries 10 | |
| env: | |
| MSGVAULT_TEST_DB: postgres://msgvault_test:testpw@localhost:5432/msgvault_test | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| - name: Install psql client | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y postgresql-client | |
| - name: Enable pgvector extension | |
| env: | |
| PGPASSWORD: testpw | |
| run: | | |
| psql -h localhost -U msgvault_test -d msgvault_test -c "CREATE EXTENSION IF NOT EXISTS vector" | |
| - name: Test (pgvector-tagged) | |
| run: go test -tags "fts5 sqlite_vec pgvector" -count=1 ./internal/vector/... ./internal/scheduler/... ./cmd/msgvault/cmd/... | |
| nix-build: | |
| runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-latest' }} | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: DeterminateSystems/nix-installer-action@ef8a148080ab6020fd15196c2084a2eea5ff2d25 # v22 | |
| - name: Build | |
| run: nix build | |
| # PostgreSQL backend lane: runs the full Go test suite against a live | |
| # PostgreSQL service. Tests that use testutil.NewTestStore read | |
| # MSGVAULT_TEST_DB and switch backends without a recompile. Concurrency | |
| # tests use -count=5 so non-deterministic races (lost updates, missing | |
| # unique constraints) surface here rather than slipping past a single | |
| # pass. | |
| test-postgres: | |
| runs-on: ${{ github.repository == 'kenn-io/msgvault' && (github.ref == 'refs/heads/main' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.event.pull_request.base.repo.full_name == github.repository)) && 'kenn-linux-x64-public' || 'ubuntu-latest' }} | |
| services: | |
| postgres: | |
| image: postgres:16 | |
| env: | |
| POSTGRES_USER: msgvault_test | |
| POSTGRES_PASSWORD: msgvault_test | |
| POSTGRES_DB: msgvault_test | |
| ports: | |
| - 5432:5432 | |
| options: >- | |
| --health-cmd pg_isready | |
| --health-interval 10s | |
| --health-timeout 5s | |
| --health-retries 5 | |
| env: | |
| MSGVAULT_TEST_DB: postgres://msgvault_test:msgvault_test@localhost:5432/msgvault_test?sslmode=disable | |
| steps: | |
| - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 | |
| - uses: actions/setup-go@924ae3a1cded613372ab5595356fb5720e22ba16 # v6.5.0 | |
| with: | |
| go-version-file: go.mod | |
| # This lane uses a stock postgres:16 image (no pgvector extension), so it | |
| # must NOT build the pgvector tag — those tests CREATE EXTENSION "vector" | |
| # and fail here. pgvector is covered by the dedicated test-pgvector job | |
| # against a pgvector/pgvector image. (Run go test directly rather than | |
| # `make test-pg`, whose PG_TEST_TAGS include pgvector.) | |
| - name: Test (PostgreSQL) | |
| run: go test -tags "fts5 sqlite_vec" ./... | |
| - name: Concurrency tests (-count=5) | |
| run: | | |
| go test -tags fts5 -count=5 -run 'Concurrent' ./internal/store |