Add CI liveness pipeline for SUQL stack

.github/workflows/test.yml

@@ -0,0 +1,136 @@
+name: SUQL liveness check
+
+# Runs the full pipeline on every push and PR: bring up Postgres, fetch the
+# sample ACLED CSV from a private fixtures repo, ingest it, start SUQL's two
+# servers, then issue one canned `answer()` query and assert it returns rows.
+#
+# Required GitHub Secrets on this fork:
+#   FIXTURES_TOKEN     fine-grained PAT with `Contents: Read` on the private
+#                      fixtures repo (see tests/SETUP.md for one-time setup)
+#   OPENAI_API_KEY     LLM proxy credential
+#   OPENAI_API_BASE    proxy base URL (e.g. https://azureopenai.genie.stanford.edu/)
+#
+# Optional GitHub Variable:
+#   FIXTURES_REPO      owner/name of the private repo holding the sample CSV
+#                      (defaults to skyxiath/suql-test-fixtures)
+
+on:
+  push:
+    branches: [main, add-testing-framework]
+  workflow_dispatch:
+  # No pull_request trigger: the workflow needs secrets that can't be
+  # available on cross-repo PRs from forks anyway. Maintainers who fork
+  # and want CI on their PRs should add `pull_request:` here once their
+  # FIXTURES_TOKEN secret is configured.
+
+jobs:
+  liveness:
+    runs-on: ubuntu-latest
+    timeout-minutes: 15
+
+    services:
+      postgres:
+        image: postgres:16
+        env:
+          POSTGRES_USER:     oval
+          POSTGRES_PASSWORD: oval
+          POSTGRES_DB:       acled
+        ports: ['5432:5432']
+        options: >-
+          --health-cmd     "pg_isready -U oval"
+          --health-interval 5s
+          --health-timeout  5s
+          --health-retries  10
+
+    env:
+      PGHOST:     127.0.0.1
+      PGPORT:     '5432'
+      PGDATABASE: acled
+      PGUSER:     oval
+      PGPASSWORD: oval
+
+    steps:
+      - name: Checkout SUQL repo
+        uses: actions/checkout@v4
+
+      - name: Fetch sample CSV from private fixtures repo
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ vars.FIXTURES_REPO || 'skyxiath/suql-test-fixtures' }}
+          token: ${{ secrets.FIXTURES_TOKEN }}
+          path: _fixtures
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.11'
+          cache: pip
+
+      - name: Cache HuggingFace model (BAAI/bge-large-en-v1.5, ~1.3GB)
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/huggingface
+          key: hf-bge-large-en-v1.5
+
+      - name: Install SUQL with embedding deps
+        run: |
+          pip install --upgrade pip
+          pip install -e .[embedding]
+          pip install python-dotenv
+
+      - name: Install postgresql-client
+        run: sudo apt-get update -qq && sudo apt-get install -y -qq postgresql-client
+
+      - name: Ingest sample data
+        env:
+          SAMPLE_CSV: ${{ github.workspace }}/_fixtures/acled_sample.csv
+        run: ./tests/ingest.sh
+
+      - name: Start free-text server (port 8500)
+        run: |
+          nohup python -m suql.free_text_fcns_server > /tmp/freetext.log 2>&1 &
+          echo "$!" > /tmp/freetext.pid
+
+      - name: Start embedding server (port 8505)
+        # Embedding server connects to PG, reads notes, computes embeddings.
+        # First run on this runner downloads BAAI/bge-large-en-v1.5 (~60s);
+        # subsequent runs hit the HF cache. CPU mode is auto-detected.
+        run: |
+          export SUQL_EMBED_PORT=8505
+          nohup python tests/start_embedding_server.py > /tmp/embed.log 2>&1 &
+          echo "$!" > /tmp/embed.pid
+
+      - name: Wait for SUQL servers
+        run: |
+          for port in 8500 8505; do
+            for i in $(seq 1 60); do
+              if (echo > /dev/tcp/127.0.0.1/$port) 2>/dev/null; then
+                echo "✓ port $port open after ${i}s"; break
+              fi
+              sleep 2
+            done
+            if ! (echo > /dev/tcp/127.0.0.1/$port) 2>/dev/null; then
+              echo "✗ port $port never came up. Logs:"
+              echo "--- freetext.log ---"; tail -50 /tmp/freetext.log || true
+              echo "--- embed.log ---";    tail -50 /tmp/embed.log    || true
+              exit 1
+            fi
+          done
+
+      - name: Run liveness probe
+        env:
+          OPENAI_API_KEY:    ${{ secrets.OPENAI_API_KEY }}
+          OPENAI_API_BASE:   ${{ secrets.OPENAI_API_BASE }}
+          SUQL_EMBED_URL:    http://127.0.0.1:8505
+          SUQL_FREETEXT_URL: http://127.0.0.1:8500
+        run: python tests/check_alive.py
+
+      - name: Upload server logs (always, for debugging)
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: server-logs
+          path: |
+            /tmp/freetext.log
+            /tmp/embed.log
+          retention-days: 7

.gitignore

@@ -154,4 +154,10 @@ tests/private/
 # Local Gradio demo — OVAL/Stanford-specific defaults, not for the public repo
 demo/
 
-src/suql/loaders/documents.py
+src/suql/loaders/documents.py
+
+# Sample data lives in a separate private fixtures repo (see tests/SETUP.md).
+# These belt-and-suspenders entries make it impossible to accidentally commit
+# the plaintext CSV into this repo even if someone drops one in for local dev.
+tests/fixtures/*.csv
+_fixtures/

setup.py

@@ -32,6 +32,13 @@
     "FlagEmbedding~=1.2.5",
 ]
 
+install_embedding_requires = [
+    # ~=1.2.5 included a version with FlagEmbedding/BGE_M3/trainer.py
+    # referencing `Optional` without importing it — upstream-fixed in 1.3+.
+    "FlagEmbedding>=1.3",
+    "faiss-cpu>=1.7.4",
+]
+
 # Additional package information
 classifiers = [
     "License :: OSI Approved :: Apache Software License",
@@ -54,7 +61,10 @@
     packages=packages,
     package_dir={"": "src"},
     install_requires=install_requires,
-    extra_requires={"dev": install_dev_requires},
+    extras_require={
+        "dev": install_dev_requires,
+        "embedding": install_embedding_requires,
+    },
     url=url,
     classifiers=classifiers,
     package_data={"": ["*.prompt"]},

tests/README.md

@@ -0,0 +1,114 @@
+# SUQL liveness pipeline
+
+End-to-end CI check: bring up Postgres, ingest a private data sample,
+start SUQL's two servers, run one canned `answer()` query, confirm it
+returns rows. Runs on every push via `.github/workflows/test.yml`.
+
+The answer to "did this change break anything obvious?" is a green or
+red dot on the commit — no local environment needed to find out.
+
+## Layout
+
+```
+tests/
+├── README.md                       this file
+├── SETUP.md                        one-time fixtures repo + PAT setup
+├── check_alive.py                  liveness probe — runs the canned query
+├── ingest.sh                       applies schema + \COPY (auth-agnostic)
+├── start_embedding_server.py       embedding-server bootstrap
+└── fixtures/
+    └── schema.sql                  example events table + select_user / creator_role
+```
+
+No CSV ever lives in this repo. The data sits in a separate private
+fixtures repo that CI fetches with a scoped credential. See `SETUP.md`.
+
+## Required GitHub secrets
+
+| Secret             | What it is                                                 |
+| ------------------ | ---------------------------------------------------------- |
+| `FIXTURES_TOKEN`   | fine-grained PAT, `Contents: Read` on the fixtures repo    |
+| `OPENAI_API_KEY`   | LLM proxy credential                                       |
+| `OPENAI_API_BASE`  | proxy base URL                                             |
+
+## Optional GitHub variable
+
+| Variable          | Default                       | What it is                       |
+| ----------------- | ----------------------------- | -------------------------------- |
+| `FIXTURES_REPO`   | `skyxiath/suql-test-fixtures` | `owner/name` of the fixtures repo |
+
+Set these at: **repo → Settings → Secrets and variables → Actions**.
+
+See `SETUP.md` for the walkthrough.
+
+## Adapting the probe to your data
+
+The example schema + canned query assume an ACLED-shaped events table.
+For a different dataset:
+
+1. Replace `fixtures/schema.sql` with your table DDL. Keep the
+   `select_user` and `creator_role` blocks — SUQL's compiler needs them.
+2. Push a CSV matching that schema to the fixtures repo.
+3. Update `start_embedding_server.py`'s call to `store.add(...)` to
+   point at your table / primary key / free-text column. Or override
+   via `SUQL_TABLE`, `SUQL_ID_COL`, `SUQL_TEXT_COL` env vars.
+4. Override the probe via `SUQL_QUERY` in the workflow's "Run liveness
+   probe" step. Any query that exercises one `answer()` clause and
+   returns rows from your sample works.
+
+## Refreshing the sample
+
+Sample lives in the fixtures repo, not here. Update it there:
+
+```bash
+cd /path/to/your-fixtures-repo
+# (generate / copy in a new sample.csv however you do)
+git add sample.csv && git commit -m "Refresh sample" && git push
+```
+
+CI on the next workflow run picks up the new sample automatically — no
+change to this repo needed.
+
+## Running locally
+
+```bash
+# 1. Start Postgres locally:
+docker run -d --name suql-test-pg -p 5432:5432 \
+    -e POSTGRES_USER=oval -e POSTGRES_PASSWORD=oval -e POSTGRES_DB=acled \
+    postgres:16
+
+# 2. Install SUQL with embedding deps:
+pip install -e .[embedding]
+pip install python-dotenv
+
+# 3. Drop a sample CSV at /tmp/sample.csv (from wherever you keep it).
+
+# 4. Ingest:
+PGPASSWORD=oval SAMPLE_CSV=/tmp/sample.csv ./tests/ingest.sh
+
+# 5. Start servers:
+python -m suql.free_text_fcns_server &
+SUQL_EMBED_PORT=8505 python tests/start_embedding_server.py &
+
+# 6. Run probe:
+OPENAI_API_KEY=... OPENAI_API_BASE=... python tests/check_alive.py
+```
+
+## What "liveness" means here
+
+The probe runs one canned `answer(...) = 'yes'` query. Exit codes:
+
+| Code | Meaning                                                     |
+| ---- | ----------------------------------------------------------- |
+| 0    | `suql_execute` returned >= 1 row                            |
+| 1    | `suql_execute` raised (PG, embedding, free-text, or LLM)    |
+| 2    | Ran cleanly but returned 0 rows                             |
+| 3    | Preflight failed — TCP target unreachable or creds missing  |
+
+The workflow fails on any non-zero exit.
+
+## Scope
+
+This catches *plumbing* breaks (server boot failure, SQL compilation,
+prompt loading, network connectivity). It does not measure result
+quality — for that you want benchmarks / evals, not a CI smoke test.

tests/SETUP.md

@@ -0,0 +1,105 @@
+# One-time setup: fixtures repo + PAT + secrets
+
+This walkthrough wires up the auth-gated fetch the CI workflow depends on.
+Do it once; CI handles itself from there.
+
+Notation: `<owner>` is your GitHub username/org. `<fixtures-repo>` is the
+name you'll pick for the data repo (e.g. `suql-test-fixtures`).
+
+## Step 1 — Create the private fixtures repo
+
+The sample CSV needs a home separate from the SUQL code. Create a new
+**private** repo:
+
+```bash
+gh repo create <owner>/<fixtures-repo> --private --description \
+    "Sample data + fixtures consumed by SUQL CI"
+```
+
+That's it for now — it can stay empty until you push the first sample.
+
+## Step 2 — Drop in the first sample
+
+Push a CSV named (by default) `acled_sample.csv` matching the column
+order in `tests/fixtures/schema.sql`:
+
+```bash
+gh repo clone <owner>/<fixtures-repo> /tmp/fixtures
+cp /path/to/your/sample.csv /tmp/fixtures/acled_sample.csv
+cd /tmp/fixtures
+git add acled_sample.csv
+git commit -m "Initial sample"
+git push
+```
+
+For a different filename, set the `SAMPLE_CSV` env in the workflow's
+"Ingest sample data" step to match.
+
+## Step 3 — Create a fine-grained PAT
+
+GitHub's fine-grained PATs let you scope a token to a single repo with
+read-only permission. That's exactly what CI needs.
+
+1. Open: **GitHub → Settings → Developer settings → Personal access tokens
+   → Fine-grained tokens → Generate new token**
+2. **Token name:** something memorable, e.g. `suql-ci-fixtures-readonly`
+3. **Expiration:** pick the longest you're comfortable with (max 1 year).
+   Set a calendar reminder to rotate before expiry.
+4. **Repository access:** *Only select repositories* → `<owner>/<fixtures-repo>`
+5. **Permissions → Repository permissions:**
+   - `Contents`: **Read-only**
+   - `Metadata`: **Read-only** (required by GitHub for any access)
+6. Click **Generate token**, copy the value — GitHub only shows it once.
+
+## Step 4 — Add secrets + variable to the SUQL repo
+
+1. Open: **SUQL fork → Settings → Secrets and variables → Actions →
+   Secrets tab → New repository secret**
+2. Add three secrets:
+
+| Name              | Value                                          |
+| ----------------- | ---------------------------------------------- |
+| `FIXTURES_TOKEN`  | the PAT from Step 3                            |
+| `OPENAI_API_KEY`  | your LLM proxy key                             |
+| `OPENAI_API_BASE` | your LLM proxy base URL                        |
+
+3. Switch to the **Variables** tab and add one:
+
+| Name             | Value                              |
+| ---------------- | ---------------------------------- |
+| `FIXTURES_REPO`  | `<owner>/<fixtures-repo>`          |
+
+## Step 5 — Verify by triggering a CI run
+
+```bash
+git push
+# or
+gh workflow run "SUQL liveness check"
+```
+
+Watch: **repo → Actions → SUQL liveness check → most-recent run**.
+
+The "Fetch sample CSV" step is where the PAT is exercised. If it fails:
+- 404 → PAT can see fewer repos than expected; recheck Step 3's
+  *Repository access* selection.
+- 403 → PAT lacks `Contents: Read`; recheck Step 3's *Permissions*.
+- Anything else → check the run logs.
+
+The "Run liveness probe" step is the actual end-to-end check. On
+success it prints `✓ suql_execute returned in Ns` with row count and
+cost.
+
+## Rotation
+
+When the PAT approaches expiration (GitHub emails you ahead of time):
+
+1. Generate a new PAT with the same scope (Step 3)
+2. Update the `FIXTURES_TOKEN` secret with the new value (Step 4)
+3. Delete the old PAT
+
+## Revocation (if the PAT leaks)
+
+1. **Immediately** delete the leaked PAT at GitHub → Settings → Developer
+   settings → Personal access tokens → Fine-grained tokens
+2. Generate a new PAT, update the secret, push a no-op commit to verify
+3. Review whatever process exposed the PAT