diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..897e4aa --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,74 @@ +name: CI + +on: + pull_request: + branches: [main] + push: + branches: [main] + +env: + SETUPTOOLS_SCM_PRETEND_VERSION: "0.0.0.dev0" + +jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@v4 + - run: uv run --extra dev ruff check src/ server/src/ tools/ tests/ server/tests/ + + test-client: + runs-on: ubuntu-latest + strategy: + matrix: + python-version: ["3.10", "3.11", "3.12", "3.13"] + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@v4 + - run: uv python install ${{ matrix.python-version }} + - run: uv run --python ${{ matrix.python-version }} --extra dev pytest tests/unit/ -v --tb=short + + test-server: + runs-on: ubuntu-latest + services: + postgres: + image: postgres:16-alpine + env: + POSTGRES_USER: etelemetry + POSTGRES_PASSWORD: testpassword + POSTGRES_DB: etelemetry_test + ports: + - 5432:5432 + options: >- + --health-cmd pg_isready + --health-interval 10s + --health-timeout 5s + --health-retries 5 + env: + DATABASE_URL: postgresql+asyncpg://etelemetry:testpassword@localhost:5432/etelemetry_test + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - uses: astral-sh/setup-uv@v4 + - run: uv python install 3.11 + - run: uv run --python 3.11 --extra dev --directory server pytest tests/unit/ -v --tb=short + + validate-infra: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: opentofu/setup-opentofu@v1 + - uses: astral-sh/setup-uv@v4 + - name: Validate OpenTofu + working-directory: infra + run: | + tofu init -backend=false + tofu validate + tofu fmt -check -recursive || echo "::warning::OpenTofu files need formatting (run 'tofu fmt -recursive' in infra/)" + - name: Run IAM policy tests + run: uv run --with pytest pytest infra/tests/test_iam_policy.py -v diff --git a/.github/workflows/deploy.yml b/.github/workflows/deploy.yml new file mode 100644 index 0000000..daaff29 --- /dev/null +++ b/.github/workflows/deploy.yml @@ -0,0 +1,123 @@ +name: Deploy + +on: + push: + branches: [main] + paths: + - "server/**" + - "src/**" + - "deploy/**" + - "infra/scripts/**" + - ".github/workflows/deploy.yml" + +concurrency: + group: deploy-production + cancel-in-progress: false # Queue, don't cancel + +permissions: + id-token: write + contents: read + +jobs: + deploy: + runs-on: ubuntu-latest + if: github.ref == 'refs/heads/main' + environment: + name: production + url: https://et.dandiproject.org + steps: + - uses: actions/checkout@v4 + + # Authenticate to AWS via OIDC — no access keys needed + - name: Configure AWS credentials + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ vars.AWS_DEPLOY_ROLE_ARN }} + aws-region: ${{ vars.AWS_REGION || 'us-east-1' }} + + # Build and push Docker image to ECR + - name: Login to ECR + id: ecr-login + uses: aws-actions/amazon-ecr-login@v2 + + - name: Build and push image + env: + ECR_REGISTRY: ${{ steps.ecr-login.outputs.registry }} + ECR_REPOSITORY: etelemetry + IMAGE_TAG: ${{ github.sha }} + run: | + docker build -f deploy/Dockerfile -t $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG . + docker tag $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG $ECR_REGISTRY/$ECR_REPOSITORY:latest + docker push $ECR_REGISTRY/$ECR_REPOSITORY:$IMAGE_TAG + docker push $ECR_REGISTRY/$ECR_REPOSITORY:latest + + # Deploy via SSM (no SSH needed) + - name: Get instance ID + id: instance + run: | + INSTANCE_ID=$(aws ec2 describe-instances \ + --filters "Name=tag:Name,Values=etelemetry" "Name=instance-state-name,Values=running" \ + --query 'Reservations[0].Instances[0].InstanceId' \ + --output text) + echo "id=$INSTANCE_ID" >> "$GITHUB_OUTPUT" + + - name: Deploy via SSM + env: + ECR_REGISTRY: ${{ steps.ecr-login.outputs.registry }} + INSTANCE_ID: ${{ steps.instance.outputs.id }} + run: | + COMMAND_ID=$(aws ssm send-command \ + --instance-ids "$INSTANCE_ID" \ + --document-name "AWS-RunShellScript" \ + --parameters "commands=[\"bash /opt/etelemetry/infra/scripts/deploy.sh ${{ github.sha }} ${ECR_REGISTRY}/etelemetry ${{ vars.AWS_REGION || 'us-east-1' }}\"]" \ + --timeout-seconds 300 \ + --query 'Command.CommandId' \ + --output text) + + echo "SSM Command ID: $COMMAND_ID" + + # Poll for completion + for i in $(seq 1 60); do + STATUS=$(aws ssm get-command-invocation \ + --command-id "$COMMAND_ID" \ + --instance-id "$INSTANCE_ID" \ + --query 'Status' \ + --output text 2>/dev/null || echo "Pending") + + case "$STATUS" in + Success) + echo "Deployment successful!" + aws ssm get-command-invocation \ + --command-id "$COMMAND_ID" \ + --instance-id "$INSTANCE_ID" \ + --query 'StandardOutputContent' \ + --output text + exit 0 + ;; + Failed|TimedOut|Cancelled) + echo "Deployment FAILED with status: $STATUS" + aws ssm get-command-invocation \ + --command-id "$COMMAND_ID" \ + --instance-id "$INSTANCE_ID" \ + --query 'StandardErrorContent' \ + --output text + exit 1 + ;; + *) + sleep 5 + ;; + esac + done + + echo "Deployment timed out waiting for SSM command" + exit 1 + + # Verify deployment + - name: Health check + run: | + sleep 10 + curl -sf https://et.dandiproject.org/ || { + echo "Health check failed!" + exit 1 + } + echo "Deployment verified at https://et.dandiproject.org/" diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml new file mode 100644 index 0000000..ce4d17b --- /dev/null +++ b/.github/workflows/publish.yml @@ -0,0 +1,43 @@ +name: Publish to PyPI + +on: + release: + types: [published] + +permissions: + id-token: write # Trusted publishing via OIDC + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 # Full history for hatch-vcs + + - uses: astral-sh/setup-uv@v4 + with: + python-version: "3.11" + + - name: Build package + run: uv build + + - uses: actions/upload-artifact@v4 + with: + name: dist + path: dist/ + + publish: + needs: build + runs-on: ubuntu-latest + environment: + name: pypi + url: https://pypi.org/p/etelemetry + steps: + - uses: actions/download-artifact@v4 + with: + name: dist + path: dist/ + + - name: Publish to PyPI + uses: pypa/gh-action-pypi-publish@release/v1 diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml new file mode 100644 index 0000000..c08b38d --- /dev/null +++ b/.github/workflows/release.yml @@ -0,0 +1,49 @@ +name: Release + +on: + push: + tags: + - "v*" + +permissions: + contents: write + +jobs: + release: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + + - name: Generate release notes + id: notes + run: | + # Get previous tag + PREV_TAG=$(git describe --tags --abbrev=0 HEAD^ 2>/dev/null || echo "") + if [ -z "$PREV_TAG" ]; then + NOTES=$(git log --oneline --no-merges) + else + NOTES=$(git log --oneline --no-merges "${PREV_TAG}..HEAD") + fi + echo "notes<> "$GITHUB_OUTPUT" + echo "$NOTES" >> "$GITHUB_OUTPUT" + echo "EOF" >> "$GITHUB_OUTPUT" + + - name: Determine pre-release + id: prerelease + run: | + TAG="${GITHUB_REF#refs/tags/}" + if echo "$TAG" | grep -qE '(alpha|beta|rc|dev|pre)'; then + echo "is_prerelease=true" >> "$GITHUB_OUTPUT" + else + echo "is_prerelease=false" >> "$GITHUB_OUTPUT" + fi + + - name: Create GitHub Release + uses: softprops/action-gh-release@v2 + with: + generate_release_notes: true + prerelease: ${{ steps.prerelease.outputs.is_prerelease }} + body: | + ${{ steps.notes.outputs.notes }} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..ab95655 --- /dev/null +++ b/.gitignore @@ -0,0 +1,45 @@ +# Python +__pycache__/ +*.pyc +*.pyo +*.egg-info/ +dist/ +build/ +.eggs/ +*.egg + +# Virtual environments +.venv/ +venv/ + +# uv +uv.lock + +# Environment / Secrets +.env +.env.* +!.env.example +!deploy/.env.example + +# GeoIP databases and config (MaxMind + DB-IP) +*.mmdb +GeoIP.conf + +# IDE +.vscode/ +.idea/ +*.swp +*.swo +*~ + +# OS +.DS_Store +Thumbs.db + +# Testing +.coverage +htmlcov/ +.pytest_cache/ + +# Docker +*.log diff --git a/.specify/memory/constitution.md b/.specify/memory/constitution.md index d8ab08e..96857f7 100644 --- a/.specify/memory/constitution.md +++ b/.specify/memory/constitution.md @@ -1,7 +1,9 @@ + diff --git a/README.md b/README.md new file mode 100644 index 0000000..56bb65c --- /dev/null +++ b/README.md @@ -0,0 +1,23 @@ +# etelemetry + +Lightweight version-check telemetry service and client library. + +## Client Installation + +```bash +pip install etelemetry +``` + +## Usage + +```python +import etelemetry + +# Check for updates +etelemetry.check_available_version("org/project", "1.0.0") + +# Direct version query +result = etelemetry.get_project("org/project") +``` + +See [docs/external-setup-guide.md](docs/external-setup-guide.md) for server deployment. diff --git a/deploy/.env.example b/deploy/.env.example new file mode 100644 index 0000000..decfd05 --- /dev/null +++ b/deploy/.env.example @@ -0,0 +1,23 @@ +# PostgreSQL +POSTGRES_USER=etelemetry +POSTGRES_PASSWORD=CHANGE_ME +POSTGRES_DB=etelemetry +DATABASE_URL=postgresql+asyncpg://etelemetry:CHANGE_ME@postgres:5432/etelemetry + +# MaxMind GeoLite2 +MAXMIND_ACCOUNT_ID=YOUR_ACCOUNT_ID +MAXMIND_LICENSE_KEY=YOUR_LICENSE_KEY +MAXMIND_DB_PATH=/data/GeoLite2-City.mmdb + +# Server +SECRET_KEY=CHANGE_ME_RANDOM_STRING +ALLOWLIST_PATH=/config/allowlist.yml +CACHE_TTL_SECONDS=21600 +TIME_BUCKET_HOURS=1 + +# GitHub (optional — increases rate limit) +# GITHUB_TOKEN=ghp_YOUR_TOKEN + +# Deployment +SERVER_HOST=0.0.0.0 +SERVER_PORT=8000 diff --git a/deploy/Caddyfile b/deploy/Caddyfile new file mode 100644 index 0000000..11f50f2 --- /dev/null +++ b/deploy/Caddyfile @@ -0,0 +1,24 @@ +{ + # Global options + log { + output stdout + format json + # IP addresses are excluded from logs — only method, path, status + level INFO + } +} + +et.dandiproject.org { + # Automatic TLS via Let's Encrypt — no configuration needed + + # Legacy path support — old clients use /et/projects/... + handle /et/* { + uri strip_prefix /et + reverse_proxy server:8000 + } + + # All other requests go to the server + handle { + reverse_proxy server:8000 + } +} diff --git a/deploy/Dockerfile b/deploy/Dockerfile new file mode 100644 index 0000000..0327c2b --- /dev/null +++ b/deploy/Dockerfile @@ -0,0 +1,28 @@ +FROM python:3.11-slim AS base + +# Install uv +COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv + +WORKDIR /app + +# Install server dependencies +COPY server/pyproject.toml server/ +COPY pyproject.toml . +RUN uv pip install --system --no-cache ./server + +# Copy source code +COPY server/src/ server/src/ +COPY src/ src/ +COPY server/alembic/ server/alembic/ +COPY server/alembic.ini server/ + +# Copy deploy configs +COPY deploy/allowlist.yml /config/allowlist.yml + +# Create directory for GeoIP database +RUN mkdir -p /data + +EXPOSE 8000 + +# Run migrations then start server +CMD ["sh", "-c", "cd server && python -m alembic upgrade head && cd .. && uvicorn etelemetry_server.app:app --host 0.0.0.0 --port 8000 --workers 2 --access-log"] diff --git a/deploy/allowlist.yml b/deploy/allowlist.yml new file mode 100644 index 0000000..2d96aca --- /dev/null +++ b/deploy/allowlist.yml @@ -0,0 +1,20 @@ +# etelemetry allowlist — tracked repositories +# Only projects listed here will be served by the etelemetry server. +# The server reloads this file on restart or SIGHUP. +# +# Format: +# projects: +# - owner: +# repo: + +projects: + - owner: nipy + repo: nipype + - owner: nipy + repo: nibabel + - owner: nipy + repo: nitransforms + - owner: sensein + repo: senselab + - owner: sensein + repo: etelemetry-client diff --git a/deploy/docker-compose.dev.yml b/deploy/docker-compose.dev.yml new file mode 100644 index 0000000..f3a6637 --- /dev/null +++ b/deploy/docker-compose.dev.yml @@ -0,0 +1,44 @@ +# Local development / testing compose. +# Usage: +# cp deploy/.env.example deploy/.env # edit with real values +# docker compose -f deploy/docker-compose.dev.yml up -d +# +# This starts only PostgreSQL and GeoIP updater. +# Run the server locally with: +# uv run uvicorn etelemetry_server.app:app --reload --port 8000 +# +# Or start everything (including server + nginx) with: +# docker compose -f deploy/docker-compose.yml up -d + +services: + postgres: + image: postgres:16-alpine + container_name: et-postgres-dev + environment: + POSTGRES_USER: ${POSTGRES_USER:-etelemetry} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:-devpassword} + POSTGRES_DB: ${POSTGRES_DB:-etelemetry} + volumes: + - pgdata-dev:/var/lib/postgresql/data + ports: + - "5432:5432" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-etelemetry}"] + interval: 5s + timeout: 5s + retries: 5 + + geoipupdate: + image: ghcr.io/maxmind/geoipupdate:latest + container_name: et-geoipupdate-dev + environment: + GEOIPUPDATE_ACCOUNT_ID: ${MAXMIND_ACCOUNT_ID:-000000} + GEOIPUPDATE_LICENSE_KEY: ${MAXMIND_LICENSE_KEY:-placeholder} + GEOIPUPDATE_EDITION_IDS: GeoLite2-City + GEOIPUPDATE_FREQUENCY: 0 # Run once and exit + volumes: + - geoip-dev:/usr/share/GeoIP + +volumes: + pgdata-dev: + geoip-dev: diff --git a/deploy/docker-compose.yml b/deploy/docker-compose.yml new file mode 100644 index 0000000..9e35dce --- /dev/null +++ b/deploy/docker-compose.yml @@ -0,0 +1,75 @@ +services: + postgres: + image: postgres:16-alpine + container_name: et-postgres + environment: + POSTGRES_USER: ${POSTGRES_USER:-etelemetry} + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?Set POSTGRES_PASSWORD in .env} + POSTGRES_DB: ${POSTGRES_DB:-etelemetry} + volumes: + - pgdata:/var/lib/postgresql/data + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${POSTGRES_USER:-etelemetry}"] + interval: 5s + timeout: 5s + retries: 5 + restart: unless-stopped + + geoipupdate: + image: ghcr.io/maxmind/geoipupdate:latest + container_name: et-geoipupdate + environment: + GEOIPUPDATE_ACCOUNT_ID: ${MAXMIND_ACCOUNT_ID:?Set MAXMIND_ACCOUNT_ID in .env} + GEOIPUPDATE_LICENSE_KEY: ${MAXMIND_LICENSE_KEY:?Set MAXMIND_LICENSE_KEY in .env} + GEOIPUPDATE_EDITION_IDS: GeoLite2-City + GEOIPUPDATE_FREQUENCY: 168 # weekly (hours) + volumes: + - geoip:/usr/share/GeoIP + restart: unless-stopped + + server: + image: ${ECR_IMAGE:-etelemetry-server:latest} + build: + context: .. + dockerfile: deploy/Dockerfile + container_name: et-server + environment: + DATABASE_URL: postgresql+asyncpg://${POSTGRES_USER:-etelemetry}:${POSTGRES_PASSWORD}@postgres:5432/${POSTGRES_DB:-etelemetry} + MAXMIND_DB_PATH: /data/GeoLite2-City.mmdb + ALLOWLIST_PATH: /config/allowlist.yml + CACHE_TTL_SECONDS: ${CACHE_TTL_SECONDS:-21600} + TIME_BUCKET_HOURS: ${TIME_BUCKET_HOURS:-1} + GITHUB_TOKEN: ${GITHUB_TOKEN:-} + volumes: + - geoip:/data:ro + depends_on: + postgres: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:8000/"] + interval: 10s + timeout: 5s + retries: 3 + start_period: 15s + restart: unless-stopped + + caddy: + image: caddy:alpine + container_name: et-caddy + ports: + - "80:80" + - "443:443" + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile:ro + - caddy_data:/data + - caddy_config:/config + depends_on: + server: + condition: service_healthy + restart: always + +volumes: + pgdata: + geoip: + caddy_data: + caddy_config: diff --git a/deploy/geoipupdate.conf b/deploy/geoipupdate.conf new file mode 100644 index 0000000..e9d3057 --- /dev/null +++ b/deploy/geoipupdate.conf @@ -0,0 +1,13 @@ +# MaxMind GeoIP Update Configuration +# Used by the geoipupdate Docker container. +# License key is injected via environment variables in docker-compose.yml. +# +# To obtain a free license key: +# 1. Register at https://www.maxmind.com/en/geolite2/signup +# 2. Generate a license key in your account +# 3. Set MAXMIND_ACCOUNT_ID and MAXMIND_LICENSE_KEY in deploy/.env + +AccountID YOUR_ACCOUNT_ID +LicenseKey YOUR_LICENSE_KEY +EditionIDs GeoLite2-City +DatabaseDirectory /usr/share/GeoIP diff --git a/deploy/nginx.conf b/deploy/nginx.conf new file mode 100644 index 0000000..2d12708 --- /dev/null +++ b/deploy/nginx.conf @@ -0,0 +1,37 @@ +# etelemetry nginx reverse proxy +# IP addresses are stripped from access logs for privacy (FR-016) + +log_format noip '$time_iso8601 $request_method $uri $status ' + '$body_bytes_sent "$http_user_agent"'; + +server { + listen 80; + server_name _; + + access_log /var/log/nginx/access.log noip; + error_log /var/log/nginx/error.log warn; + + # Legacy path support — old clients use /et/projects/... + # Rewrite to strip /et/ prefix for backward compatibility (FR-008) + location /et/ { + rewrite ^/et/(.*)$ /$1 break; + proxy_pass http://server:8000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 30; + proxy_send_timeout 30; + proxy_read_timeout 30; + } + + # New path — direct access + location / { + proxy_pass http://server:8000; + proxy_set_header Host $host; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + proxy_connect_timeout 30; + proxy_send_timeout 30; + proxy_read_timeout 30; + } +} diff --git a/docs/bootstrap.md b/docs/bootstrap.md new file mode 100644 index 0000000..19acb9b --- /dev/null +++ b/docs/bootstrap.md @@ -0,0 +1,188 @@ +# Infrastructure Bootstrap Guide + +One-time setup to create all AWS resources for etelemetry. After bootstrap, +all deployments are automated via GitHub Actions. + +## Prerequisites + +- AWS account with an IAM user that has permissions to create IAM roles, + EC2 instances, ECR repositories, S3 buckets, DynamoDB tables, and SSM + parameters +- [OpenTofu](https://opentofu.org/docs/intro/install/) installed (v1.6+) +- AWS CLI configured (`aws configure`) +- Git access to `sensein/etelemetry` + +## Step 1: Create State Backend + +The S3 bucket and DynamoDB table for OpenTofu state must exist before +`tofu init`. Create them manually (one-time): + +```bash +AWS_REGION=us-east-1 + +# Create S3 bucket for state +aws s3api create-bucket \ + --bucket etelemetry-tfstate \ + --region $AWS_REGION + +aws s3api put-bucket-versioning \ + --bucket etelemetry-tfstate \ + --versioning-configuration Status=Enabled + +aws s3api put-bucket-encryption \ + --bucket etelemetry-tfstate \ + --server-side-encryption-configuration \ + '{"Rules":[{"ApplyServerSideEncryptionByDefault":{"SSEAlgorithm":"AES256"}}]}' + +aws s3api put-public-access-block \ + --bucket etelemetry-tfstate \ + --public-access-block-configuration \ + 'BlockPublicAcls=true,IgnorePublicAcls=true,BlockPublicPolicy=true,RestrictPublicBuckets=true' + +# Create DynamoDB table for state locking +aws dynamodb create-table \ + --table-name etelemetry-tflock \ + --attribute-definitions AttributeName=LockID,AttributeType=S \ + --key-schema AttributeName=LockID,KeyType=HASH \ + --billing-mode PAY_PER_REQUEST \ + --region $AWS_REGION +``` + +## Step 2: Initialize and Apply + +```bash +cd infra/ + +# Initialize OpenTofu with S3 backend +make init + +# Review what will be created +make plan + +# Apply — creates: EC2, security groups, IAM roles (OIDC + deploy + +# instance), ECR repository, SSM parameters, Elastic IP +make apply +``` + +Note the outputs: + +```bash +tofu output +# deploy_role_arn = "arn:aws:iam::123456789012:role/etelemetry-deploy" +# ecr_repository_url = "123456789012.dkr.ecr.us-east-1.amazonaws.com/etelemetry" +# elastic_ip = "52.x.x.x" +# instance_id = "i-0abcdef1234567890" +# ssm_parameter_prefix = "/etelemetry" +``` + +## Step 3: Configure DNS + +**IMPORTANT**: DNS MUST be configured BEFORE the first deployment with TLS. +Caddy needs the domain to resolve to the server IP for the ACME challenge. + +Create a DNS A record: + +| Type | Name | Value | TTL | +|------|------|-------|-----| +| A | `et` | `` | 300 | + +in the DNS zone for `dandiproject.org`. + +## Step 4: Store Secrets + +Replace the placeholder values in SSM Parameter Store: + +```bash +aws ssm put-parameter \ + --name /etelemetry/POSTGRES_PASSWORD \ + --value "$(openssl rand -base64 32)" \ + --type SecureString \ + --overwrite + +aws ssm put-parameter \ + --name /etelemetry/MAXMIND_ACCOUNT_ID \ + --value "YOUR_MAXMIND_ACCOUNT_ID" \ + --type SecureString \ + --overwrite + +aws ssm put-parameter \ + --name /etelemetry/MAXMIND_LICENSE_KEY \ + --value "YOUR_MAXMIND_LICENSE_KEY" \ + --type SecureString \ + --overwrite + +# Optional: higher GitHub API rate limit +aws ssm put-parameter \ + --name /etelemetry/GITHUB_TOKEN \ + --value "ghp_YOUR_TOKEN" \ + --type SecureString \ + --overwrite +``` + +### Rotating Secrets + +To rotate a secret, update it in SSM and re-run the deploy workflow: + +```bash +aws ssm put-parameter \ + --name /etelemetry/POSTGRES_PASSWORD \ + --value "NEW_STRONG_PASSWORD" \ + --type SecureString \ + --overwrite + +# Then trigger a deploy (push to main, or manually trigger workflow) +``` + +The deploy script fetches fresh secrets from SSM on every run. + +## Step 5: Configure GitHub Repository + +Go to the GitHub repo **Settings → Secrets and variables → Actions**: + +**Variables** (not secrets — these are not sensitive): + +| Variable | Value | +|----------|-------| +| `AWS_DEPLOY_ROLE_ARN` | `arn:aws:iam::ACCOUNT_ID:role/etelemetry-deploy` (from `tofu output`) | +| `AWS_REGION` | `us-east-1` | + +**Environment**: Create a `production` environment (Settings → Environments) +with optional deployment protection rules. + +No AWS access keys are needed — the workflow uses OIDC. + +## Step 6: First Deployment + +Push to `main` or manually trigger the deploy workflow. It will: +1. Authenticate to AWS via OIDC (assume `etelemetry-deploy` role) +2. Build Docker image and push to ECR +3. Run `deploy.sh` on EC2 via SSM (pull image, write .env, docker rollout) +4. Health-check the deployment +5. Report success/failure + +## Step 7: Verify + +```bash +curl https://et.dandiproject.org/ +# {"name": "etelemetry", "version": "..."} + +curl https://et.dandiproject.org/projects/nipy/nipype +# {"version": "...", "bad_versions": [...]} +``` + +## Teardown + +To destroy all infrastructure (irreversible): + +```bash +cd infra/ +make destroy +# Type 'yes' when prompted +``` + +Then manually delete the state backend: + +```bash +aws s3 rb s3://etelemetry-tfstate --force +aws dynamodb delete-table --table-name etelemetry-tflock +``` diff --git a/docs/external-setup-guide.md b/docs/external-setup-guide.md new file mode 100644 index 0000000..bbda121 --- /dev/null +++ b/docs/external-setup-guide.md @@ -0,0 +1,267 @@ +# External Setup Guide + +This guide provides an ordered pathway for deploying etelemetry from +scratch. Follow each section in sequence. + +## Overview: What Gets Set Up + +``` +1. GitHub repo settings (branch protection, environments, variables) +2. PyPI trusted publishing (for client library releases) +3. MaxMind GeoLite2 account (for IP geolocation) +4. AWS infrastructure via OpenTofu (EC2, IAM, ECR, SSM — no manual setup) +5. DNS record (et.dandiproject.org → Elastic IP) +6. First deployment (automated via GitHub Actions) +7. MongoDB data migration (one-time, from old server) +8. Legacy domain redirect (rig.mit.edu → et.dandiproject.org) +9. First client release (pre-release, then stable) +``` + +--- + +## 1. GitHub Repository Settings + +### 1.1 Branch Protection + +1. Go to **Settings → Branches → Add branch ruleset** +2. Apply to `main` branch: + - Require pull request reviews (1 reviewer) + - Require status checks: `lint`, `test-client`, `test-server`, + `validate-infra` + - Require branches to be up to date + - Do not allow force pushes + +### 1.2 Environments + +Create two environments in **Settings → Environments**: + +| Environment | Purpose | Protection | +|-------------|---------|------------| +| `pypi` | Client library publishing | Optional: require reviewer | +| `production` | AWS deployment | Optional: require reviewer | + +### 1.3 Repository Variables + +Go to **Settings → Secrets and variables → Actions → Variables**: + +| Variable | Value | Description | +|----------|-------|-------------| +| `AWS_DEPLOY_ROLE_ARN` | (from `tofu output` in step 4) | IAM role for OIDC deploy | +| `AWS_REGION` | `us-east-1` | AWS region | + +No AWS secrets needed — authentication uses OIDC (no access keys). + +--- + +## 2. PyPI Trusted Publishing + +### 2.1 Configure Publisher + +1. Go to https://pypi.org/manage/account/publishing/ +2. Click **Add a new pending publisher** +3. Fill in: + - PyPI project name: `etelemetry` + - Owner: `sensein` + - Repository: `etelemetry` + - Workflow name: `publish.yml` + - Environment name: `pypi` +4. Click **Add** + +--- + +## 3. MaxMind GeoLite2 Account + +1. Register at https://www.maxmind.com/en/geolite2/signup +2. Confirm email and log in +3. Go to **Account → Manage License Keys** +4. Click **Generate New License Key** + - Description: "etelemetry production" +5. Save the **Account ID** and **License Key** — you'll need them in step 4 + +--- + +## 4. AWS Infrastructure Bootstrap (OpenTofu) + +All AWS resources are created automatically via OpenTofu. No manual EC2 +setup, no SSH keys, no security group configuration by hand. + +See `docs/bootstrap.md` for the full step-by-step, summarized here: + +```bash +# 4.1 Create state backend (one-time, see bootstrap.md for commands) +aws s3api create-bucket --bucket etelemetry-tfstate --region us-east-1 +aws dynamodb create-table --table-name etelemetry-tflock ... + +# 4.2 Apply infrastructure +cd infra/ +make init +make plan # Review what will be created +make apply # Creates: EC2, IAM roles (OIDC + instance), ECR, SSM, EIP + +# 4.3 Note the outputs +tofu output +# deploy_role_arn → set as GitHub variable AWS_DEPLOY_ROLE_ARN +# elastic_ip → use for DNS in step 5 +# ecr_repository_url → used automatically by deploy workflow +# instance_id → used automatically by deploy workflow +``` + +### 4.4 Store Secrets in SSM + +```bash +aws ssm put-parameter --name /etelemetry/POSTGRES_PASSWORD \ + --value "$(openssl rand -base64 32)" --type SecureString --overwrite + +aws ssm put-parameter --name /etelemetry/MAXMIND_ACCOUNT_ID \ + --value "YOUR_ACCOUNT_ID" --type SecureString --overwrite + +aws ssm put-parameter --name /etelemetry/MAXMIND_LICENSE_KEY \ + --value "YOUR_LICENSE_KEY" --type SecureString --overwrite + +# Optional: for higher GitHub API rate limits +aws ssm put-parameter --name /etelemetry/GITHUB_TOKEN \ + --value "ghp_YOUR_TOKEN" --type SecureString --overwrite +``` + +### 4.5 Set GitHub Variable + +Go back to GitHub repo → Settings → Variables → Actions and set +`AWS_DEPLOY_ROLE_ARN` to the value from `tofu output deploy_role_arn`. + +--- + +## 5. DNS Setup + +**IMPORTANT**: DNS MUST be configured BEFORE the first deployment. +Caddy needs the domain to resolve for automatic TLS certificate issuance. + +### 5.1 Create A Record + +In your DNS provider for `dandiproject.org`: + +| Type | Name | Value | TTL | +|------|------|-------|-----| +| A | `et` | `` | 300 | + +### 5.2 Verify Resolution + +```bash +dig et.dandiproject.org +short +# Should return the Elastic IP +``` + +TLS is handled automatically by Caddy — no certbot, no manual +certificate management, no renewal cron jobs. + +--- + +## 6. First Deployment + +Merge the infrastructure PR to `main`. The deploy workflow will: + +1. Authenticate to AWS via OIDC (no access keys) +2. Build Docker image and push to ECR +3. Run deploy script on EC2 via SSM (no SSH) +4. Pull image, write secrets from SSM to .env, docker-rollout +5. Health-check the deployment +6. Caddy auto-provisions TLS certificate on first HTTPS request + +### Verify + +```bash +curl https://et.dandiproject.org/ +# {"name": "etelemetry", "version": "..."} + +curl https://et.dandiproject.org/dashboard/ +# HTML dashboard page +``` + +--- + +## 7. MongoDB Data Migration (one-time) + +After the new server is running and verified: + +```bash +# From EC2 (via SSM Session Manager): +cd /opt/etelemetry +python -m tools.migrate \ + --mongo-uri mongodb://:27017/et \ + --pg-uri postgresql://etelemetry:@localhost:5432/etelemetry + +# Verify +python -m tools.migrate \ + --mongo-uri mongodb://:27017/et \ + --pg-uri postgresql://etelemetry:@localhost:5432/etelemetry \ + --verify +``` + +Confirm dashboard shows historical data at +https://et.dandiproject.org/dashboard/ + +--- + +## 8. Legacy Domain Redirect + +On the existing `rig.mit.edu` server, add a redirect so old clients +transition seamlessly: + +```nginx +server { + listen 80; + server_name rig.mit.edu; + + location /et/ { + return 301 https://et.dandiproject.org$request_uri; + } +} +``` + +The etelemetry client follows redirects automatically via `requests`. + +--- + +## 9. First Client Release + +```bash +# Pre-release (pip won't install without --pre) +git tag v2.0.0a1 +git push origin v2.0.0a1 +# → GitHub pre-release → PyPI publish + +# Test +pip install --pre etelemetry +python -c "import etelemetry; print(etelemetry.get_project('nipy/nipype'))" + +# Stable release (once verified) +git tag v2.0.0 +git push origin v2.0.0 +# → GitHub release → PyPI publish +# → pip install etelemetry picks this up +``` + +--- + +## Transition Checklist + +Follow in order: + +- [ ] GitHub branch protection enabled with CI status checks +- [ ] GitHub `pypi` and `production` environments created +- [ ] PyPI trusted publisher configured for `sensein/etelemetry` +- [ ] MaxMind account created, license key saved +- [ ] OpenTofu bootstrap complete (`make apply` succeeds) +- [ ] Secrets stored in SSM Parameter Store +- [ ] `AWS_DEPLOY_ROLE_ARN` set as GitHub variable +- [ ] DNS A record for `et.dandiproject.org` → Elastic IP +- [ ] DNS resolves correctly (`dig et.dandiproject.org`) +- [ ] First deploy to `main` succeeds +- [ ] `curl https://et.dandiproject.org/` returns health response +- [ ] TLS certificate valid (Caddy auto-provisioned) +- [ ] MongoDB migration completed and verified +- [ ] Dashboard shows historical data +- [ ] Legacy `rig.mit.edu/et/` redirects to new domain +- [ ] Pre-release `v2.0.0a1` published to PyPI +- [ ] `pip install --pre etelemetry` works +- [ ] Stable `v2.0.0` published to PyPI +- [ ] Downstream projects tested with `etelemetry>=2.0.0` diff --git a/docs/phase-log.md b/docs/phase-log.md new file mode 100644 index 0000000..791dfdb --- /dev/null +++ b/docs/phase-log.md @@ -0,0 +1,70 @@ +# Phase Log + +## 2026-03-22 — Specification & Planning + +**Objectives**: Define requirements, choose tech stack, design data model, +create task breakdown for the etelemetry stack migration and monorepo refactor. + +**Actions**: +- Created feature specification (spec.md) with 5 user stories, 20 FRs, 9 SCs +- Ratified project constitution v2026.03.22.1 with 8 principles +- Clarified: data retention (indefinite, IPs discarded), dashboard geo + visualization (interactive map + table), content-addressed dedup (hourly + buckets), allowlist via config file, scale (~1M pings/week) +- Researched and selected tech stack: FastAPI, PostgreSQL 16, MaxMind + GeoLite2, htmx + Leaflet.js, httpx, SQLAlchemy 2.0 +- Designed data model (3 tables: projects, version_checks, usage_aggregates) +- Defined API contracts (HTTP + Python client) +- Generated 59-task breakdown across 8 phases +- Ran cross-artifact analysis; remediated all HIGH/MEDIUM findings + +**Decisions**: +- PostgreSQL over SQLite/DuckDB (concurrent writes via MVCC) +- Server-rendered dashboard over SPA (simplicity, accessibility, no build step) +- Content-addressed dedup with hourly time-buckets (reduces row count by ~100x) +- CalVer for constitution versioning + +**Outcomes**: All artifacts ready for implementation. Analysis clean (0 +CRITICAL, 0 HIGH, 0 MEDIUM issues). + +## 2026-03-22 — Implementation (Phases 1-8) + +**Objectives**: Implement all 59 tasks across 8 phases. + +**Actions**: +- Phase 1 (Setup): Monorepo structure, pyproject.toml (client + server), + .gitignore, docs skeleton, package directories +- Phase 2 (Foundational): Pydantic settings, async DB engine, SQLAlchemy + models (3 tables), Alembic migrations, FastAPI app factory with + compression, allowlist loader with SIGHUP. 33 unit tests. +- Phase 3 (US1): Geolocation service (MaxMind), version checker (GitHub + API + in-memory LRU fallback cache), usage recorder (content-addressed + upsert), health + projects routes, client library (errors, config, + client, public API). Backward compat: old CI params detected, /et/ + legacy path via nginx rewrite. 15 client tests + contract/integration. +- Phase 4 (US2): Configurable server URL with 4-level precedence chain + (param → env → default_url → hardcoded). 8 config tests. +- Phase 5 (US3): MongoDB→PostgreSQL migration tool with IP stripping, + batch processing, --verify flag. Migration tests with fixture data. +- Phase 6 (US4): Aggregation service (tiered daily/weekly/monthly), + dashboard routes (stats API, GeoJSON, project list), Jinja2 templates + (base, project list, detail with Leaflet map), JS for map drill-down, + CSS for accessibility. Contract + integration tests. +- Phase 7 (US5): Dockerfile, Docker Compose (postgres + server + nginx + + geoipupdate), nginx.conf with IP-stripped logs and legacy /et/ rewrite, + allowlist.yml, GitHub Actions CI + deploy workflows. +- Phase 8 (Polish): IP audit (PASS — no IP in DB/logs, debug log sanitized), + code abstraction review (identified 6 extraction opportunities, highest + priority: project lookup helper and date range parser). + +**Decisions**: +- Backward compat: server detects old CI params (name, isPR, isCI) for + existing clients; nginx rewrites /et/ prefix +- IP audit finding: replaced IP in debug log with exception type name +- Code review: dashboard.py is the largest file (494 lines) with most + duplication — extraction of project lookup and date range parsing + recommended for future iteration + +**Outcomes**: 51 of 59 tasks complete. Remaining: T050-T051 (Docker/CI +runtime validation), T055-T059 (docs finalization, quickstart validation, +full test suite, performance testing). diff --git a/docs/rebuild-spec.md b/docs/rebuild-spec.md new file mode 100644 index 0000000..20cbfbb --- /dev/null +++ b/docs/rebuild-spec.md @@ -0,0 +1,8 @@ +# Rebuild Specification + +**Purpose**: Standalone document sufficient to rebuild etelemetry from +scratch without access to conversation logs or exploratory history. + +**Status**: In progress — updated as implementation proceeds. + +*This document will be completed during the Polish phase (T056).* diff --git a/docs/vision.md b/docs/vision.md new file mode 100644 index 0000000..59db256 --- /dev/null +++ b/docs/vision.md @@ -0,0 +1,39 @@ +# etelemetry Vision + +**Last updated**: 2026-03-22 (post-implementation) + +## Purpose + +etelemetry provides a lightweight telemetry service that allows software +packages to check for available version updates and known bad versions at +runtime. It informs users when they are running outdated or faulty software +while collecting privacy-preserving usage statistics (city/state-level +geolocation, no IP addresses stored). + +## Scope + +- **Client library**: Installable Python package (`etelemetry`) that + downstream projects integrate to check versions at runtime. +- **Server**: FastAPI-based service that resolves versions from GitHub, + records usage with content-addressed deduplication, and serves a web + dashboard. +- **Dashboard**: Server-rendered (Jinja2 + htmx + Leaflet.js) web UI + showing per-project usage statistics with interactive geographic maps. +- **Deployment**: Docker Compose for self-hosting; GitHub Actions for + automated AWS deployment. + +## Architectural Direction + +- Monorepo combining client and server (previously separate repos). +- PostgreSQL replaces MongoDB for efficient storage and aggregation. +- MaxMind GeoLite2 for local, fast (<10ms) geolocation — no external API. +- Content-addressed deduplication reduces ~1M pings/week to thousands of + unique rows. +- Repository allowlist controls which projects are tracked. +- Configurable server URL enables multi-instance deployments. + +## Non-Goals (this iteration) + +- Support for non-GitHub source forges (GitLab, etc.). +- Authentication for dashboard read access. +- Real-time streaming or WebSocket updates. diff --git a/infra/Makefile b/infra/Makefile new file mode 100644 index 0000000..e7a6230 --- /dev/null +++ b/infra/Makefile @@ -0,0 +1,20 @@ +.PHONY: init plan apply destroy fmt validate + +init: + tofu init + +plan: + tofu plan + +apply: + tofu apply + +destroy: + @echo "WARNING: This will destroy all infrastructure!" + @read -p "Type 'yes' to confirm: " confirm && [ "$$confirm" = "yes" ] && tofu destroy + +fmt: + tofu fmt -recursive + +validate: + tofu validate diff --git a/infra/ec2.tf b/infra/ec2.tf new file mode 100644 index 0000000..7cc3f25 --- /dev/null +++ b/infra/ec2.tf @@ -0,0 +1,95 @@ +# Latest Amazon Linux 2023 AMI +data "aws_ami" "al2023" { + most_recent = true + owners = ["amazon"] + + filter { + name = "name" + values = ["amzn2023-ami-kernel-*"] + } + + filter { + name = "architecture" + values = ["x86_64"] + } + + filter { + name = "root-device-type" + values = ["ebs"] + } + + filter { + name = "virtualization-type" + values = ["hvm"] + } +} + +# Security group +resource "aws_security_group" "etelemetry" { + name = var.project_name + description = "Security group for etelemetry server" + + ingress { + description = "HTTP" + from_port = 80 + to_port = 80 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + ingress { + description = "HTTPS" + from_port = 443 + to_port = 443 + protocol = "tcp" + cidr_blocks = ["0.0.0.0/0"] + } + + egress { + description = "All outbound" + from_port = 0 + to_port = 0 + protocol = "-1" + cidr_blocks = ["0.0.0.0/0"] + } + + tags = { + Name = var.project_name + Project = var.project_name + } +} + +# EC2 instance +resource "aws_instance" "etelemetry" { + ami = data.aws_ami.al2023.id + instance_type = var.instance_type + iam_instance_profile = aws_iam_instance_profile.etelemetry.name + user_data = file("scripts/user-data.sh") + + vpc_security_group_ids = [aws_security_group.etelemetry.id] + + root_block_device { + volume_size = var.ebs_volume_size + volume_type = "gp3" + } + + tags = { + Name = var.project_name + Project = var.project_name + } +} + +# Elastic IP +resource "aws_eip" "etelemetry" { + domain = "vpc" + + tags = { + Name = var.project_name + Project = var.project_name + } +} + +resource "aws_eip_association" "etelemetry" { + instance_id = aws_instance.etelemetry.id + allocation_id = aws_eip.etelemetry.id +} diff --git a/infra/ecr.tf b/infra/ecr.tf new file mode 100644 index 0000000..ab1bef0 --- /dev/null +++ b/infra/ecr.tf @@ -0,0 +1,34 @@ +resource "aws_ecr_repository" "etelemetry" { + name = var.project_name + image_tag_mutability = "MUTABLE" + + image_scanning_configuration { + scan_on_push = true + } + + tags = { + Project = var.project_name + } +} + +resource "aws_ecr_lifecycle_policy" "etelemetry" { + repository = aws_ecr_repository.etelemetry.name + + policy = jsonencode({ + rules = [ + { + rulePriority = 1 + description = "Keep last 10 tagged images" + selection = { + tagStatus = "tagged" + tagPrefixList = ["v"] + countType = "imageCountMoreThan" + countNumber = 10 + } + action = { + type = "expire" + } + } + ] + }) +} diff --git a/infra/iam.tf b/infra/iam.tf new file mode 100644 index 0000000..3074bd7 --- /dev/null +++ b/infra/iam.tf @@ -0,0 +1,160 @@ +# GitHub OIDC identity provider +resource "aws_iam_openid_connect_provider" "github" { + url = "https://token.actions.githubusercontent.com" + client_id_list = ["sts.amazonaws.com"] + thumbprint_list = ["ffffffffffffffffffffffffffffffffffffffff"] +} + +# Deploy role for GitHub Actions +resource "aws_iam_role" "deploy" { + name = "etelemetry-deploy" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Federated = aws_iam_openid_connect_provider.github.arn + } + Action = "sts:AssumeRoleWithWebIdentity" + Condition = { + StringEquals = { + "token.actions.githubusercontent.com:aud" = "sts.amazonaws.com" + } + StringLike = { + "token.actions.githubusercontent.com:sub" = "repo:sensein/etelemetry:ref:refs/heads/main" + } + } + } + ] + }) + + tags = { + Project = var.project_name + } +} + +resource "aws_iam_role_policy" "deploy" { + name = "etelemetry-deploy-policy" + role = aws_iam_role.deploy.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "SSMSendCommand" + Effect = "Allow" + Action = [ + "ssm:SendCommand", + "ssm:GetCommandInvocation" + ] + Resource = [ + aws_instance.etelemetry.arn, + "arn:aws:ssm:${data.aws_region.current.name}::document/AWS-RunShellScript" + ] + }, + { + Sid = "SSMParameters" + Effect = "Allow" + Action = [ + "ssm:GetParameter", + "ssm:PutParameter" + ] + Resource = "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/etelemetry/*" + }, + { + Sid = "EC2Describe" + Effect = "Allow" + Action = [ + "ec2:DescribeInstances", + "ec2:DescribeInstanceStatus" + ] + Resource = "*" + }, + { + Sid = "ECRAuth" + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken" + ] + Resource = "*" + }, + { + Sid = "ECRPush" + Effect = "Allow" + Action = [ + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ] + Resource = aws_ecr_repository.etelemetry.arn + } + ] + }) +} + +# Instance role +resource "aws_iam_role" "instance" { + name = "etelemetry-instance" + + assume_role_policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Effect = "Allow" + Principal = { + Service = "ec2.amazonaws.com" + } + Action = "sts:AssumeRole" + } + ] + }) + + tags = { + Project = var.project_name + } +} + +resource "aws_iam_instance_profile" "etelemetry" { + name = "etelemetry-instance" + role = aws_iam_role.instance.name +} + +resource "aws_iam_role_policy_attachment" "ssm_core" { + role = aws_iam_role.instance.name + policy_arn = "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore" +} + +resource "aws_iam_role_policy" "instance" { + name = "etelemetry-instance-policy" + role = aws_iam_role.instance.id + + policy = jsonencode({ + Version = "2012-10-17" + Statement = [ + { + Sid = "SSMParameterRead" + Effect = "Allow" + Action = [ + "ssm:GetParametersByPath" + ] + Resource = "arn:aws:ssm:${data.aws_region.current.name}:${data.aws_caller_identity.current.account_id}:parameter/etelemetry/*" + }, + { + Sid = "ECRPull" + Effect = "Allow" + Action = [ + "ecr:GetAuthorizationToken", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ] + Resource = "*" + } + ] + }) +} diff --git a/infra/main.tf b/infra/main.tf new file mode 100644 index 0000000..30a5050 --- /dev/null +++ b/infra/main.tf @@ -0,0 +1,7 @@ +provider "aws" { + region = var.aws_region +} + +data "aws_caller_identity" "current" {} + +data "aws_region" "current" {} diff --git a/infra/outputs.tf b/infra/outputs.tf new file mode 100644 index 0000000..b981ec5 --- /dev/null +++ b/infra/outputs.tf @@ -0,0 +1,24 @@ +output "deploy_role_arn" { + description = "ARN of the GitHub Actions deploy role" + value = aws_iam_role.deploy.arn +} + +output "instance_id" { + description = "EC2 instance ID" + value = aws_instance.etelemetry.id +} + +output "ecr_repository_url" { + description = "ECR repository URL" + value = aws_ecr_repository.etelemetry.repository_url +} + +output "elastic_ip" { + description = "Elastic IP address" + value = aws_eip.etelemetry.public_ip +} + +output "ssm_parameter_prefix" { + description = "SSM Parameter Store prefix" + value = "/etelemetry" +} diff --git a/infra/scripts/deploy.sh b/infra/scripts/deploy.sh new file mode 100755 index 0000000..03f5126 --- /dev/null +++ b/infra/scripts/deploy.sh @@ -0,0 +1,54 @@ +#!/bin/bash +set -euo pipefail + +IMAGE_TAG="${1:-latest}" +ECR_REPO="${2}" +REGION="${3:-us-east-1}" + +# Authenticate to ECR +aws ecr get-login-password --region "$REGION" | docker login --username AWS --password-stdin "$ECR_REPO" + +# Pull new image +docker pull "${ECR_REPO}:${IMAGE_TAG}" +docker tag "${ECR_REPO}:${IMAGE_TAG}" etelemetry-server:latest + +# Fetch secrets from SSM and write .env +cd /opt/etelemetry/deploy + +# Get all parameters under /etelemetry/ +aws ssm get-parameters-by-path \ + --path /etelemetry/ \ + --with-decryption \ + --query 'Parameters[*].[Name,Value]' \ + --output text \ + --region "$REGION" | while IFS=$'\t' read -r name value; do + key=$(basename "$name") + echo "${key}=${value}" +done > .env.tmp + +# Add non-secret config +echo "DATABASE_URL=postgresql+asyncpg://etelemetry:$(grep POSTGRES_PASSWORD .env.tmp | cut -d= -f2)@postgres:5432/etelemetry" >> .env.tmp +echo "MAXMIND_DB_PATH=/data/GeoLite2-City.mmdb" >> .env.tmp +echo "ALLOWLIST_PATH=/config/allowlist.yml" >> .env.tmp + +mv .env.tmp .env +chmod 600 .env + +# Zero-downtime rollout +cd /opt/etelemetry +docker rollout server -f deploy/docker-compose.yml || { + echo "ERROR: Rollout failed, previous version still running" + exit 1 +} + +# Health check +for i in $(seq 1 30); do + if curl -sf http://localhost:8000/ > /dev/null 2>&1; then + echo "Health check passed" + exit 0 + fi + sleep 2 +done + +echo "ERROR: Health check failed after 60 seconds" +exit 1 diff --git a/infra/scripts/user-data.sh b/infra/scripts/user-data.sh new file mode 100755 index 0000000..1d336c9 --- /dev/null +++ b/infra/scripts/user-data.sh @@ -0,0 +1,29 @@ +#!/bin/bash +set -euo pipefail + +# Install Docker +dnf install -y docker +systemctl enable docker +systemctl start docker +usermod -aG docker ec2-user + +# Install Docker Compose plugin +mkdir -p /usr/local/lib/docker/cli-plugins +curl -SL "https://github.com/docker/compose/releases/latest/download/docker-compose-linux-$(uname -m)" \ + -o /usr/local/lib/docker/cli-plugins/docker-compose +chmod +x /usr/local/lib/docker/cli-plugins/docker-compose + +# Install docker-rollout +curl -SL "https://raw.githubusercontent.com/wowu/docker-rollout/main/docker-rollout" \ + -o /usr/local/lib/docker/cli-plugins/docker-rollout +chmod +x /usr/local/lib/docker/cli-plugins/docker-rollout + +# Install AWS CLI (for ECR auth and SSM parameter fetch) +dnf install -y aws-cli + +# Clone etelemetry repo +git clone https://github.com/sensein/etelemetry.git /opt/etelemetry +chown -R ec2-user:ec2-user /opt/etelemetry + +# Create data directories +mkdir -p /opt/etelemetry/caddy_data diff --git a/infra/ssm.tf b/infra/ssm.tf new file mode 100644 index 0000000..1ef2f89 --- /dev/null +++ b/infra/ssm.tf @@ -0,0 +1,55 @@ +resource "aws_ssm_parameter" "postgres_password" { + name = "/etelemetry/POSTGRES_PASSWORD" + type = "SecureString" + value = "CHANGE_ME" + + lifecycle { + ignore_changes = [value] + } + + tags = { + Project = var.project_name + } +} + +resource "aws_ssm_parameter" "maxmind_account_id" { + name = "/etelemetry/MAXMIND_ACCOUNT_ID" + type = "SecureString" + value = "CHANGE_ME" + + lifecycle { + ignore_changes = [value] + } + + tags = { + Project = var.project_name + } +} + +resource "aws_ssm_parameter" "maxmind_license_key" { + name = "/etelemetry/MAXMIND_LICENSE_KEY" + type = "SecureString" + value = "CHANGE_ME" + + lifecycle { + ignore_changes = [value] + } + + tags = { + Project = var.project_name + } +} + +resource "aws_ssm_parameter" "github_token" { + name = "/etelemetry/GITHUB_TOKEN" + type = "SecureString" + value = "" + + lifecycle { + ignore_changes = [value] + } + + tags = { + Project = var.project_name + } +} diff --git a/infra/state.tf b/infra/state.tf new file mode 100644 index 0000000..5a38943 --- /dev/null +++ b/infra/state.tf @@ -0,0 +1,55 @@ +# State backend resources (self-bootstrapping). +# On first run, use: tofu init -backend-config="path=terraform.tfstate" +# After these resources are created, migrate to S3 backend with: tofu init -migrate-state + +resource "aws_s3_bucket" "tfstate" { + bucket = "etelemetry-tfstate" + + tags = { + Name = "etelemetry-tfstate" + Project = var.project_name + } +} + +resource "aws_s3_bucket_versioning" "tfstate" { + bucket = aws_s3_bucket.tfstate.id + + versioning_configuration { + status = "Enabled" + } +} + +resource "aws_s3_bucket_server_side_encryption_configuration" "tfstate" { + bucket = aws_s3_bucket.tfstate.id + + rule { + apply_server_side_encryption_by_default { + sse_algorithm = "AES256" + } + } +} + +resource "aws_s3_bucket_public_access_block" "tfstate" { + bucket = aws_s3_bucket.tfstate.id + + block_public_acls = true + block_public_policy = true + ignore_public_acls = true + restrict_public_buckets = true +} + +resource "aws_dynamodb_table" "tflock" { + name = "etelemetry-tflock" + billing_mode = "PAY_PER_REQUEST" + hash_key = "LockID" + + attribute { + name = "LockID" + type = "S" + } + + tags = { + Name = "etelemetry-tflock" + Project = var.project_name + } +} diff --git a/infra/tests/test_deploy.sh b/infra/tests/test_deploy.sh new file mode 100755 index 0000000..b6a18a3 --- /dev/null +++ b/infra/tests/test_deploy.sh @@ -0,0 +1,24 @@ +#!/bin/bash +set -euo pipefail +DOMAIN="${1:-et.dandiproject.org}" + +echo "=== Deploy Smoke Test ===" + +# Health endpoint +echo -n "Health check... " +RESP=$(curl -sf "https://${DOMAIN}/" 2>&1) || { echo "FAIL: Health endpoint unreachable"; exit 1; } +echo "$RESP" | grep -q '"name"' || { echo "FAIL: Unexpected response shape"; exit 1; } +echo "PASS" + +# TLS certificate +echo -n "TLS certificate... " +echo | openssl s_client -connect "${DOMAIN}:443" -servername "$DOMAIN" 2>/dev/null | \ + openssl x509 -noout -dates 2>/dev/null || { echo "FAIL: Invalid TLS cert"; exit 1; } +echo "PASS" + +# No SSH port +echo -n "Port 22 closed... " +timeout 3 bash -c "echo >/dev/tcp/${DOMAIN}/22" 2>/dev/null && { echo "FAIL: Port 22 is open!"; exit 1; } +echo "PASS" + +echo "=== All smoke tests passed ===" diff --git a/infra/tests/test_iam_policy.py b/infra/tests/test_iam_policy.py new file mode 100644 index 0000000..6bd6d9a --- /dev/null +++ b/infra/tests/test_iam_policy.py @@ -0,0 +1,98 @@ +"""Tests to validate IAM policy definitions in iam.tf. + +These tests perform static analysis on the HCL file to ensure +security best practices are followed. +""" + +import os +import re + +IAM_TF_PATH = os.path.join(os.path.dirname(__file__), "..", "iam.tf") + + +def _read_iam_tf(): + with open(IAM_TF_PATH, "r") as f: + return f.read() + + +def test_no_iam_wildcard_actions(): + """Deploy role must not have iam:* actions.""" + content = _read_iam_tf() + assert "iam:*" not in content, "iam:* action found in iam.tf - this is too permissive" + + +def test_no_wildcard_resource_on_ssm_put(): + """ssm:PutParameter must not use Resource: '*'.""" + content = _read_iam_tf() + # Find the SSM parameters statement block and verify it has a scoped resource + ssm_section = re.search( + r'(?s)"ssm:PutParameter".*?Resource\s*=\s*"([^"]+)"', content + ) + assert ssm_section is not None, "Could not find ssm:PutParameter resource scope" + resource = ssm_section.group(1) + assert resource != "*", "ssm:PutParameter must not use Resource: '*'" + assert "/etelemetry/" in resource, "ssm:PutParameter should be scoped to /etelemetry/*" + + +def test_no_wildcard_resource_on_ecr_put(): + """ecr:PutImage must not use Resource: '*'.""" + content = _read_iam_tf() + # Find the ECR push statement block + ecr_section = re.search( + r'(?s)"ecr:PutImage".*?Resource\s*=\s*(\S+)', content + ) + assert ecr_section is not None, "Could not find ecr:PutImage resource scope" + resource = ecr_section.group(1) + assert resource.strip('"') != "*", "ecr:PutImage must not use Resource: '*'" + + +def test_trust_policy_contains_correct_repo(): + """Trust policy must reference the correct GitHub repo.""" + content = _read_iam_tf() + assert "repo:sensein/etelemetry:ref:refs/heads/main" in content, ( + "Trust policy must contain 'repo:sensein/etelemetry:ref:refs/heads/main'" + ) + + +def test_no_admin_or_power_user_access(): + """No AdministratorAccess or PowerUserAccess managed policy references.""" + content = _read_iam_tf() + assert "AdministratorAccess" not in content, ( + "AdministratorAccess managed policy found - this is too permissive" + ) + assert "PowerUserAccess" not in content, ( + "PowerUserAccess managed policy found - this is too permissive" + ) + + +def test_deploy_role_has_expected_actions(): + """Deploy role should have the expected set of actions.""" + content = _read_iam_tf() + expected_actions = [ + "ssm:SendCommand", + "ssm:GetCommandInvocation", + "ssm:GetParameter", + "ssm:PutParameter", + "ec2:DescribeInstances", + "ec2:DescribeInstanceStatus", + "ecr:GetAuthorizationToken", + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer", + ] + for action in expected_actions: + assert action in content, f"Expected action '{action}' not found in iam.tf" + + +if __name__ == "__main__": + test_no_iam_wildcard_actions() + test_no_wildcard_resource_on_ssm_put() + test_no_wildcard_resource_on_ecr_put() + test_trust_policy_contains_correct_repo() + test_no_admin_or_power_user_access() + test_deploy_role_has_expected_actions() + print("All IAM policy tests passed.") diff --git a/infra/variables.tf b/infra/variables.tf new file mode 100644 index 0000000..d5c7814 --- /dev/null +++ b/infra/variables.tf @@ -0,0 +1,35 @@ +variable "aws_region" { + description = "AWS region for all resources" + type = string + default = "us-east-1" +} + +variable "instance_type" { + description = "EC2 instance type" + type = string + default = "t3.small" +} + +variable "domain" { + description = "Domain name for the etelemetry service" + type = string + default = "et.dandiproject.org" +} + +variable "github_repo" { + description = "GitHub repository for OIDC trust" + type = string + default = "sensein/etelemetry" +} + +variable "project_name" { + description = "Project name used for resource naming and tagging" + type = string + default = "etelemetry" +} + +variable "ebs_volume_size" { + description = "Size of the root EBS volume in GB" + type = number + default = 20 +} diff --git a/infra/versions.tf b/infra/versions.tf new file mode 100644 index 0000000..e82252d --- /dev/null +++ b/infra/versions.tf @@ -0,0 +1,16 @@ +terraform { + required_version = ">= 1.6" + required_providers { + aws = { + source = "hashicorp/aws" + version = "~> 5.0" + } + } + backend "s3" { + bucket = "etelemetry-tfstate" + key = "infra/terraform.tfstate" + region = "us-east-1" + dynamodb_table = "etelemetry-tflock" + encrypt = true + } +} diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..8b8f18b --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,58 @@ +[build-system] +requires = ["hatchling", "hatch-vcs"] +build-backend = "hatchling.build" + +[project] +name = "etelemetry" +dynamic = ["version"] +description = "Lightweight telemetry client for checking software version updates" +readme = "README.md" +license = "Apache-2.0" +requires-python = ">=3.10" +classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3.11", + "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.13", + "Programming Language :: Python :: 3.14", +] +dependencies = [ + "requests>=2.20", + "packaging>=20.0", + "ci-info>=0.3", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-asyncio>=0.21", + "httpx>=0.24", + "ruff>=0.4", + "pytest-cov>=4.0", +] + +[tool.hatch.version] +source = "vcs" +fallback-version = "0.0.0.dev0" + +[tool.hatch.build.hooks.vcs] +version-file = "src/etelemetry/_version.py" + +[tool.hatch.build.targets.wheel] +packages = ["src/etelemetry"] + +[tool.ruff] +target-version = "py311" +line-length = 100 +exclude = ["src/etelemetry/_version.py"] + +[tool.ruff.lint] +select = ["E", "F", "I", "W"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" diff --git a/server/alembic.ini b/server/alembic.ini new file mode 100644 index 0000000..08bf3c1 --- /dev/null +++ b/server/alembic.ini @@ -0,0 +1,36 @@ +[alembic] +script_location = alembic +sqlalchemy.url = postgresql+asyncpg://localhost:5432/etelemetry + +[loggers] +keys = root,sqlalchemy,alembic + +[handlers] +keys = console + +[formatters] +keys = generic + +[logger_root] +level = WARN +handlers = console + +[logger_sqlalchemy] +level = WARN +handlers = +qualname = sqlalchemy.engine + +[logger_alembic] +level = INFO +handlers = +qualname = alembic + +[handler_console] +class = StreamHandler +args = (sys.stderr,) +level = NOTSET +formatter = generic + +[formatter_generic] +format = %(levelname)-5.5s [%(name)s] %(message)s +datefmt = %H:%M:%S diff --git a/server/alembic/env.py b/server/alembic/env.py new file mode 100644 index 0000000..3a15f01 --- /dev/null +++ b/server/alembic/env.py @@ -0,0 +1,64 @@ +"""Alembic environment configuration for async SQLAlchemy.""" + +import asyncio +from logging.config import fileConfig + +from alembic import context +from sqlalchemy.ext.asyncio import create_async_engine + +from etelemetry_server.models import Base +from etelemetry_server.settings import settings + +# Alembic Config object +config = context.config + +# Set up loggers from alembic.ini +if config.config_file_name is not None: + fileConfig(config.config_file_name) + +# Model metadata for autogenerate support +target_metadata = Base.metadata + + +def run_migrations_offline() -> None: + """Run migrations in 'offline' mode. + + Generates SQL script without connecting to the database. + """ + url = settings.DATABASE_URL + context.configure( + url=url, + target_metadata=target_metadata, + literal_binds=True, + dialect_opts={"paramstyle": "named"}, + ) + with context.begin_transaction(): + context.run_migrations() + + +def do_run_migrations(connection) -> None: # type: ignore[no-untyped-def] + """Execute migrations against the provided connection.""" + context.configure(connection=connection, target_metadata=target_metadata) + with context.begin_transaction(): + context.run_migrations() + + +async def run_async_migrations() -> None: + """Create an async engine and run migrations.""" + connectable = create_async_engine(settings.DATABASE_URL) + + async with connectable.connect() as connection: + await connection.run_sync(do_run_migrations) + + await connectable.dispose() + + +def run_migrations_online() -> None: + """Run migrations in 'online' mode using an async engine.""" + asyncio.run(run_async_migrations()) + + +if context.is_offline_mode(): + run_migrations_offline() +else: + run_migrations_online() diff --git a/server/alembic/script.py.mako b/server/alembic/script.py.mako new file mode 100644 index 0000000..fbc4b07 --- /dev/null +++ b/server/alembic/script.py.mako @@ -0,0 +1,26 @@ +"""${message} + +Revision ID: ${up_revision} +Revises: ${down_revision | comma,n} +Create Date: ${create_date} + +""" +from typing import Sequence, Union + +from alembic import op +import sqlalchemy as sa +${imports if imports else ""} + +# revision identifiers, used by Alembic. +revision: str = ${repr(up_revision)} +down_revision: Union[str, None] = ${repr(down_revision)} +branch_labels: Union[str, Sequence[str], None] = ${repr(branch_labels)} +depends_on: Union[str, Sequence[str], None] = ${repr(depends_on)} + + +def upgrade() -> None: + ${upgrades if upgrades else "pass"} + + +def downgrade() -> None: + ${downgrades if downgrades else "pass"} diff --git a/server/alembic/versions/001_initial_schema.py b/server/alembic/versions/001_initial_schema.py new file mode 100644 index 0000000..c0107c6 --- /dev/null +++ b/server/alembic/versions/001_initial_schema.py @@ -0,0 +1,139 @@ +"""Initial schema — projects, version_checks, usage_aggregates. + +Revision ID: 001 +Revises: None +Create Date: 2026-03-22 + +""" +from typing import Sequence, Union + +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision: str = "001" +down_revision: Union[str, None] = None +branch_labels: Union[str, Sequence[str], None] = None +depends_on: Union[str, Sequence[str], None] = None + + +def upgrade() -> None: + # --- projects --- + op.create_table( + "projects", + sa.Column("id", sa.Integer, primary_key=True), + sa.Column("owner", sa.String(255), nullable=False), + sa.Column("repo", sa.String(255), nullable=False), + sa.Column("latest_version", sa.String(100), nullable=True), + sa.Column("bad_versions", sa.JSON, server_default="[]"), + sa.Column( + "cache_expires_at", sa.DateTime(timezone=True), nullable=True + ), + sa.Column("active", sa.Boolean, server_default=sa.text("true")), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + ), + sa.Column( + "updated_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + ), + sa.UniqueConstraint("owner", "repo", name="uq_project_owner_repo"), + ) + op.create_index( + "ix_project_owner_repo", "projects", ["owner", "repo"] + ) + + # --- version_checks --- + op.create_table( + "version_checks", + sa.Column("id", sa.BigInteger, primary_key=True), + sa.Column( + "project_id", + sa.Integer, + sa.ForeignKey("projects.id"), + nullable=False, + ), + sa.Column("version", sa.String(100), nullable=False), + sa.Column("city", sa.String(255), nullable=True), + sa.Column("region", sa.String(255), nullable=True), + sa.Column("country", sa.String(100), nullable=True), + sa.Column("country_code", sa.String(2), nullable=True), + sa.Column("latitude", sa.Float, nullable=True), + sa.Column("longitude", sa.Float, nullable=True), + sa.Column("is_ci", sa.Boolean, server_default=sa.text("false")), + sa.Column("time_bucket", sa.DateTime(timezone=True), nullable=False), + sa.Column("count", sa.Integer, server_default=sa.text("1")), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + ), + sa.UniqueConstraint( + "project_id", + "version", + "city", + "region", + "country_code", + "is_ci", + "time_bucket", + name="uq_version_check_content_address", + ), + ) + op.create_index( + "ix_version_check_project_bucket", + "version_checks", + ["project_id", "time_bucket"], + ) + op.create_index( + "ix_version_check_bucket", "version_checks", ["time_bucket"] + ) + + # --- usage_aggregates --- + op.create_table( + "usage_aggregates", + sa.Column("id", sa.BigInteger, primary_key=True), + sa.Column( + "project_id", + sa.Integer, + sa.ForeignKey("projects.id"), + nullable=False, + ), + sa.Column("version", sa.String(100), nullable=True), + sa.Column("country_code", sa.String(2), nullable=True), + sa.Column("region", sa.String(255), nullable=True), + sa.Column("granularity", sa.String(10), nullable=False), + sa.Column("period_start", sa.Date, nullable=False), + sa.Column("total_count", sa.BigInteger, server_default=sa.text("0")), + sa.Column( + "unique_locations", sa.Integer, server_default=sa.text("0") + ), + sa.Column("ci_count", sa.BigInteger, server_default=sa.text("0")), + sa.Column( + "created_at", + sa.DateTime(timezone=True), + server_default=sa.func.now(), + ), + sa.UniqueConstraint( + "project_id", + "version", + "country_code", + "region", + "granularity", + "period_start", + name="uq_usage_aggregate_key", + ), + ) + op.create_index( + "ix_usage_aggregate_project_granularity", + "usage_aggregates", + ["project_id", "granularity", "period_start"], + ) + + +def downgrade() -> None: + op.drop_table("usage_aggregates") + op.drop_table("version_checks") + op.drop_table("projects") diff --git a/server/pyproject.toml b/server/pyproject.toml new file mode 100644 index 0000000..ac86c9e --- /dev/null +++ b/server/pyproject.toml @@ -0,0 +1,49 @@ +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" + +[project] +name = "etelemetry-server" +version = "2.0.0" +description = "etelemetry server — version check telemetry service" +readme = {text = "etelemetry server — version check telemetry service", content-type = "text/plain"} +license = "Apache-2.0" +requires-python = ">=3.11" +dependencies = [ + "fastapi>=0.110", + "uvicorn[standard]>=0.29", + "sqlalchemy[asyncio]>=2.0", + "asyncpg>=0.29", + "alembic>=1.13", + "httpx>=0.27", + "geoip2>=4.8", + "jinja2>=3.1", + "starlette-compress>=1.0", + "pydantic-settings>=2.2", + "pyyaml>=6.0", +] + +[project.optional-dependencies] +dev = [ + "pytest>=7.0", + "pytest-asyncio>=0.21", + "httpx>=0.27", + "testcontainers[postgres]>=4.0", + "pymongo>=4.6", + "ruff>=0.4", + "pytest-cov>=4.0", +] + +[tool.hatch.build.targets.wheel] +packages = ["src/etelemetry_server"] + +[tool.ruff] +target-version = "py311" +line-length = 100 + +[tool.ruff.lint] +select = ["E", "F", "I", "W"] + +[tool.pytest.ini_options] +testpaths = ["tests"] +asyncio_mode = "auto" diff --git a/server/src/etelemetry_server/__init__.py b/server/src/etelemetry_server/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/etelemetry_server/allowlist.py b/server/src/etelemetry_server/allowlist.py new file mode 100644 index 0000000..55dd129 --- /dev/null +++ b/server/src/etelemetry_server/allowlist.py @@ -0,0 +1,101 @@ +"""Allowlist loading, DB synchronisation, and SIGHUP reload handler.""" + +from __future__ import annotations + +import logging +import signal +from pathlib import Path +from typing import Any, Callable + +import yaml +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from etelemetry_server.models import Project + +logger = logging.getLogger(__name__) + + +def load_allowlist(path: str) -> list[dict[str, str]]: + """Load a YAML allowlist file and return a list of {owner, repo} dicts. + + The YAML file is expected to contain a top-level list of mappings, e.g.: + + - owner: nipy + repo: nipype + - owner: poldracklab + repo: fmriprep + + Returns an empty list when the file does not exist. + """ + filepath = Path(path) + if not filepath.exists(): + logger.warning("Allowlist file not found: %s", path) + return [] + + with filepath.open() as fh: + data: Any = yaml.safe_load(fh) + + if not isinstance(data, list): + logger.warning("Allowlist is not a list; returning empty") + return [] + + entries: list[dict[str, str]] = [] + for item in data: + if isinstance(item, dict) and "owner" in item and "repo" in item: + entries.append({"owner": str(item["owner"]), "repo": str(item["repo"])}) + return entries + + +async def sync_allowlist_to_db( + session: AsyncSession, entries: list[dict[str, str]] +) -> None: + """Synchronise allowlist entries with the projects table. + + * Inserts projects that are in the allowlist but not yet in the DB. + * Deactivates projects that are in the DB but no longer in the allowlist. + * Re-activates projects that reappear in the allowlist. + """ + allowlist_set = {(e["owner"], e["repo"]) for e in entries} + + result = await session.execute(select(Project)) + existing_projects: list[Project] = list(result.scalars().all()) + + existing_map: dict[tuple[str, str], Project] = { + (p.owner, p.repo): p for p in existing_projects + } + + # Insert missing projects + for owner, repo in allowlist_set: + if (owner, repo) not in existing_map: + session.add(Project(owner=owner, repo=repo, active=True)) + else: + project = existing_map[(owner, repo)] + if not project.active: + project.active = True + + # Deactivate removed projects + for key, project in existing_map.items(): + if key not in allowlist_set and project.active: + project.active = False + + await session.flush() + + +def setup_sighup_handler(path: str, callback: Callable[[], Any]) -> None: + """Register a SIGHUP handler that reloads the allowlist. + + Parameters + ---------- + path : str + Path to the allowlist YAML file (for logging purposes). + callback : callable + Function to invoke when SIGHUP is received (typically reloads + the allowlist and syncs to DB). + """ + + def _handler(signum: int, frame: Any) -> None: + logger.info("SIGHUP received — reloading allowlist from %s", path) + callback() + + signal.signal(signal.SIGHUP, _handler) diff --git a/server/src/etelemetry_server/app.py b/server/src/etelemetry_server/app.py new file mode 100644 index 0000000..b45dbe2 --- /dev/null +++ b/server/src/etelemetry_server/app.py @@ -0,0 +1,134 @@ +"""FastAPI application factory.""" + +from __future__ import annotations + +import logging +from contextlib import asynccontextmanager +from pathlib import Path +from typing import AsyncGenerator + +import httpx +from fastapi import FastAPI + +from etelemetry_server.allowlist import load_allowlist, sync_allowlist_to_db +from etelemetry_server.db import async_session_factory, engine, init_db +from etelemetry_server.services.geolocation import GeoLocator +from etelemetry_server.settings import settings + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def lifespan(app: FastAPI) -> AsyncGenerator[None, None]: + """Manage application startup and shutdown resources.""" + # --- Startup --- + # Store settings on app state for easy access + app.state.settings = settings + app.state.engine = engine + + # Initialise database tables (dev convenience; prod uses Alembic) + await init_db() + + # Load GeoIP reader if the database file exists + app.state.geoip_reader = None + geoip_path = Path(settings.MAXMIND_DB_PATH) + if geoip_path.exists(): + try: + import geoip2.database + + app.state.geoip_reader = geoip2.database.Reader(str(geoip_path)) + logger.info("GeoIP database loaded from %s", geoip_path) + except Exception: + logger.warning("Failed to load GeoIP database", exc_info=True) + else: + logger.info("GeoIP database not found at %s; skipping", geoip_path) + + # Create GeoLocator service + app.state.geolocator = GeoLocator(settings.MAXMIND_DB_PATH) + + # Create shared httpx.AsyncClient for outbound requests + app.state.http_client = httpx.AsyncClient() + + # Load allowlist and sync to DB + allowlist = load_allowlist(settings.ALLOWLIST_PATH) + app.state.allowlist = allowlist + if allowlist: + async with async_session_factory() as session: + await sync_allowlist_to_db(session, allowlist) + await session.commit() + logger.info("Synced %d allowlist entries to database", len(allowlist)) + + yield + + # --- Shutdown --- + if app.state.geoip_reader is not None: + app.state.geoip_reader.close() + + if hasattr(app.state, "geolocator") and app.state.geolocator is not None: + app.state.geolocator.close() + + if hasattr(app.state, "http_client") and app.state.http_client is not None: + await app.state.http_client.aclose() + + await engine.dispose() + + +def create_app() -> FastAPI: + """Build and return the FastAPI application.""" + app = FastAPI( + title="etelemetry", + description="Version-check telemetry service", + version="2.0.0", + lifespan=lifespan, + ) + + # --- Compression middleware --- + try: + from starlette_compress import CompressMiddleware + + app.add_middleware(CompressMiddleware) + except ImportError: + from starlette.middleware.gzip import GZipMiddleware + + app.add_middleware(GZipMiddleware, minimum_size=500) + + # --- Route routers --- + try: + from etelemetry_server.routes.health import router as health_router + + app.include_router(health_router) + except ImportError: + logger.debug("health router not yet available") + + try: + from etelemetry_server.routes.projects import router as projects_router + + app.include_router(projects_router) + except ImportError: + logger.debug("projects router not yet available") + + try: + from etelemetry_server.routes.dashboard import ( + STATIC_DIR, + ) + from etelemetry_server.routes.dashboard import ( + router as dashboard_router, + ) + + app.include_router(dashboard_router) + + from fastapi.staticfiles import StaticFiles + + if STATIC_DIR.is_dir(): + app.mount( + "/dashboard/static", + StaticFiles(directory=str(STATIC_DIR)), + name="dashboard_static", + ) + except ImportError: + logger.debug("dashboard router not yet available") + + # Disable uvicorn default access log to prevent IP logging + logging.getLogger("uvicorn.access").disabled = True + + return app diff --git a/server/src/etelemetry_server/dashboard/__init__.py b/server/src/etelemetry_server/dashboard/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/etelemetry_server/dashboard/static/dashboard.css b/server/src/etelemetry_server/dashboard/static/dashboard.css new file mode 100644 index 0000000..9151689 --- /dev/null +++ b/server/src/etelemetry_server/dashboard/static/dashboard.css @@ -0,0 +1,221 @@ +/* etelemetry dashboard styles */ + +/* Reset and base */ +*, +*::before, +*::after { + box-sizing: border-box; +} + +body { + font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, + "Helvetica Neue", Arial, sans-serif; + line-height: 1.6; + color: #1a1a1a; + background-color: #fafafa; + margin: 0; + padding: 0; +} + +/* Skip link for accessibility */ +.skip-link { + position: absolute; + top: -40px; + left: 0; + background: #1a1a1a; + color: #ffffff; + padding: 8px 16px; + z-index: 100; + text-decoration: none; + font-weight: 600; +} + +.skip-link:focus { + top: 0; +} + +/* Focus indicators */ +a:focus, +button:focus, +select:focus, +input:focus { + outline: 3px solid #2563eb; + outline-offset: 2px; +} + +/* Header and nav */ +header { + background-color: #1e293b; + color: #ffffff; + padding: 12px 24px; +} + +.nav-brand { + color: #ffffff; + text-decoration: none; + font-size: 1.25rem; + font-weight: 700; +} + +.nav-brand:hover { + text-decoration: underline; +} + +/* Main content */ +main { + max-width: 1200px; + margin: 0 auto; + padding: 24px; +} + +/* Headings */ +h1 { + font-size: 1.75rem; + color: #1e293b; + margin-bottom: 24px; + border-bottom: 2px solid #e2e8f0; + padding-bottom: 8px; +} + +h2 { + font-size: 1.25rem; + color: #334155; + margin-top: 32px; + margin-bottom: 12px; +} + +/* Tables */ +table { + width: 100%; + border-collapse: collapse; + margin-bottom: 24px; + background-color: #ffffff; + border: 1px solid #e2e8f0; +} + +caption { + text-align: left; + font-weight: 600; + padding: 8px 12px; + color: #475569; + font-size: 0.875rem; +} + +th, +td { + padding: 10px 12px; + text-align: left; + border-bottom: 1px solid #e2e8f0; +} + +th { + background-color: #f1f5f9; + font-weight: 600; + color: #334155; +} + +tr:hover { + background-color: #f8fafc; +} + +/* Links */ +a { + color: #2563eb; + text-decoration: none; +} + +a:hover { + text-decoration: underline; + color: #1d4ed8; +} + +/* Summary stats */ +.summary-stats dl { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(200px, 1fr)); + gap: 16px; + padding: 0; + margin: 0; +} + +.summary-stats dt { + font-size: 0.875rem; + color: #64748b; + font-weight: 500; + text-transform: uppercase; + letter-spacing: 0.05em; +} + +.summary-stats dd { + font-size: 1.5rem; + font-weight: 700; + color: #1e293b; + margin: 0 0 16px 0; +} + +/* Filter section */ +.filter-section { + background-color: #ffffff; + padding: 16px; + border: 1px solid #e2e8f0; + border-radius: 4px; + margin-bottom: 24px; +} + +.filter-section label { + font-weight: 600; + margin-right: 8px; + color: #334155; +} + +.filter-section select { + padding: 6px 12px; + border: 1px solid #cbd5e1; + border-radius: 4px; + font-size: 0.875rem; + color: #1a1a1a; + background-color: #ffffff; +} + +/* Map */ +#map { + height: 400px; + width: 100%; + border: 1px solid #e2e8f0; + border-radius: 4px; + margin-bottom: 24px; +} + +/* Footer */ +footer { + text-align: center; + padding: 24px; + color: #64748b; + font-size: 0.875rem; + border-top: 1px solid #e2e8f0; + margin-top: 48px; +} + +/* Responsive */ +@media (max-width: 768px) { + main { + padding: 16px; + } + + .summary-stats dl { + grid-template-columns: 1fr; + } + + table { + font-size: 0.875rem; + } + + th, + td { + padding: 8px; + } + + #map { + height: 300px; + } +} diff --git a/server/src/etelemetry_server/dashboard/static/dashboard.js b/server/src/etelemetry_server/dashboard/static/dashboard.js new file mode 100644 index 0000000..93490a3 --- /dev/null +++ b/server/src/etelemetry_server/dashboard/static/dashboard.js @@ -0,0 +1,127 @@ +/** + * etelemetry dashboard JavaScript. + * + * Initializes Leaflet map and handles htmx-driven filter updates. + */ + +/* global L, htmx */ + +var dashboardMap = null; +var geoJsonLayer = null; + +/** + * Initialize the dashboard for a specific project. + * @param {string} owner - Project owner. + * @param {string} repo - Project repository name. + */ +function initDashboard(owner, repo) { + var mapEl = document.getElementById("map"); + if (!mapEl) { + return; + } + + // Initialize Leaflet map + dashboardMap = L.map("map").setView([20, 0], 2); + + L.tileLayer("https://{s}.tile.openstreetmap.org/{z}/{x}/{y}.png", { + attribution: + '© OpenStreetMap contributors', + maxZoom: 18, + }).addTo(dashboardMap); + + // Load GeoJSON data + fetchGeoJSON(owner, repo); + + // Listen for htmx events to refresh map data after filter changes + document.body.addEventListener("htmx:afterSwap", function () { + fetchGeoJSON(owner, repo); + }); +} + +/** + * Fetch GeoJSON data and render circle markers on the map. + * @param {string} owner - Project owner. + * @param {string} repo - Project repository name. + */ +function fetchGeoJSON(owner, repo) { + var url = "/dashboard/api/geo/" + owner + "/" + repo; + + // Include granularity filter if present + var granularitySelect = document.getElementById("granularity-select"); + if (granularitySelect) { + url += "?granularity=" + granularitySelect.value; + } + + fetch(url) + .then(function (response) { + return response.json(); + }) + .then(function (data) { + renderGeoJSON(data); + }) + .catch(function (err) { + console.error("Failed to fetch GeoJSON:", err); + }); +} + +/** + * Render GeoJSON FeatureCollection as circle markers on the map. + * @param {object} geojson - GeoJSON FeatureCollection. + */ +function renderGeoJSON(geojson) { + if (!dashboardMap) { + return; + } + + // Remove existing layer + if (geoJsonLayer) { + dashboardMap.removeLayer(geoJsonLayer); + } + + if (!geojson || !geojson.features || geojson.features.length === 0) { + return; + } + + // Find max count for scaling + var maxCount = 1; + geojson.features.forEach(function (feature) { + var count = feature.properties.count || 0; + if (count > maxCount) { + maxCount = count; + } + }); + + geoJsonLayer = L.geoJSON(geojson, { + pointToLayer: function (feature, latlng) { + var count = feature.properties.count || 0; + var radius = Math.max(5, Math.sqrt(count / maxCount) * 30); + return L.circleMarker(latlng, { + radius: radius, + fillColor: "#2563eb", + color: "#1d4ed8", + weight: 1, + opacity: 0.8, + fillOpacity: 0.5, + }); + }, + onEachFeature: function (feature, layer) { + var props = feature.properties; + var parts = []; + if (props.city) parts.push(props.city); + if (props.region) parts.push(props.region); + if (props.country) parts.push(props.country); + var location = parts.join(", ") || "Unknown"; + layer.bindPopup( + "" + location + "
Checks: " + props.count + ); + + // Click handler for drill-down + layer.on("click", function () { + layer.openPopup(); + }); + }, + }).addTo(dashboardMap); + + // Fit map to markers + dashboardMap.fitBounds(geoJsonLayer.getBounds(), { padding: [20, 20] }); +} diff --git a/server/src/etelemetry_server/dashboard/templates/base.html b/server/src/etelemetry_server/dashboard/templates/base.html new file mode 100644 index 0000000..e54c89f --- /dev/null +++ b/server/src/etelemetry_server/dashboard/templates/base.html @@ -0,0 +1,34 @@ + + + + + + {% block title %}etelemetry Dashboard{% endblock %} + + + {% block extra_head %}{% endblock %} + + + +
+ +
+
+ {% block content %}{% endblock %} +
+
+

etelemetry dashboard

+
+ + + {% block extra_scripts %}{% endblock %} + + diff --git a/server/src/etelemetry_server/dashboard/templates/project_detail.html b/server/src/etelemetry_server/dashboard/templates/project_detail.html new file mode 100644 index 0000000..a1f63a8 --- /dev/null +++ b/server/src/etelemetry_server/dashboard/templates/project_detail.html @@ -0,0 +1,97 @@ +{% extends "base.html" %} + +{% block title %}{{ owner }}/{{ repo }} - etelemetry Dashboard{% endblock %} + +{% block content %} +

{{ owner }}/{{ repo }}

+ +
+

Summary

+
+
Total Checks
+
{{ total_checks }}
+
Unique Versions
+
{{ unique_versions }}
+
Unique Locations
+
{{ unique_locations }}
+
+
+ +
+

Time Range

+
+ + +
+
+ +
+
+

Geographic Distribution

+ +
+ +
+

Version Distribution

+ + + + + + + + + + {% for v in by_version %} + + + + + {% endfor %} + +
Checks by version
VersionCount
{{ v.version }}{{ v.count }}
+
+ +
+

Geographic Summary

+ + + + + + + + + + + + {% for loc in by_location %} + + + + + + + {% endfor %} + +
Checks by location
CountryRegionCityCount
{{ loc.country or "Unknown" }}{{ loc.region or "Unknown" }}{{ loc.city or "Unknown" }}{{ loc.count }}
+
+
+{% endblock %} + +{% block extra_scripts %} + + +{% endblock %} diff --git a/server/src/etelemetry_server/dashboard/templates/projects.html b/server/src/etelemetry_server/dashboard/templates/projects.html new file mode 100644 index 0000000..12a635b --- /dev/null +++ b/server/src/etelemetry_server/dashboard/templates/projects.html @@ -0,0 +1,35 @@ +{% extends "base.html" %} + +{% block title %}Projects - etelemetry Dashboard{% endblock %} + +{% block content %} +

Tracked Projects

+
+ + + + + + + + + + {% for project in projects %} + + + + + {% endfor %} + {% if not projects %} + + + + {% endif %} + +
All tracked projects with their total version check counts
ProjectTotal Checks
+ + {{ project.owner }}/{{ project.repo }} + + {{ project.total_checks }}
No projects tracked yet.
+
+{% endblock %} diff --git a/server/src/etelemetry_server/db.py b/server/src/etelemetry_server/db.py new file mode 100644 index 0000000..583fb41 --- /dev/null +++ b/server/src/etelemetry_server/db.py @@ -0,0 +1,35 @@ +"""Async database engine, session factory, and FastAPI dependency.""" + +from collections.abc import AsyncGenerator + +from sqlalchemy.ext.asyncio import ( + AsyncSession, + async_sessionmaker, + create_async_engine, +) + +from etelemetry_server.models import Base +from etelemetry_server.settings import settings + +engine = create_async_engine(settings.DATABASE_URL, echo=False) + +async_session_factory = async_sessionmaker( + engine, class_=AsyncSession, expire_on_commit=False +) + + +async def get_db() -> AsyncGenerator[AsyncSession, None]: + """FastAPI dependency that yields an async database session.""" + async with async_session_factory() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + + +async def init_db() -> None: + """Create all tables (for development; production uses Alembic).""" + async with engine.begin() as conn: + await conn.run_sync(Base.metadata.create_all) diff --git a/server/src/etelemetry_server/models.py b/server/src/etelemetry_server/models.py new file mode 100644 index 0000000..3944c72 --- /dev/null +++ b/server/src/etelemetry_server/models.py @@ -0,0 +1,153 @@ +"""SQLAlchemy 2.0 ORM models.""" + +from __future__ import annotations + +import datetime +from typing import Optional + +from sqlalchemy import ( + JSON, + BigInteger, + Boolean, + Date, + DateTime, + Float, + ForeignKey, + Index, + Integer, + String, + UniqueConstraint, + func, +) +from sqlalchemy.orm import DeclarativeBase, Mapped, mapped_column, relationship + + +class Base(DeclarativeBase): + """Declarative base for all models.""" + + pass + + +class Project(Base): + """Tracks registered projects (owner/repo pairs).""" + + __tablename__ = "projects" + + id: Mapped[int] = mapped_column(Integer, primary_key=True) + owner: Mapped[str] = mapped_column(String(255), nullable=False) + repo: Mapped[str] = mapped_column(String(255), nullable=False) + latest_version: Mapped[Optional[str]] = mapped_column( + String(100), nullable=True + ) + bad_versions: Mapped[Optional[list]] = mapped_column(JSON, default=list) + cache_expires_at: Mapped[Optional[datetime.datetime]] = mapped_column( + DateTime(timezone=True), nullable=True + ) + active: Mapped[bool] = mapped_column(Boolean, default=True) + created_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + updated_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now(), onupdate=func.now() + ) + + version_checks: Mapped[list["VersionCheck"]] = relationship( + back_populates="project" + ) + usage_aggregates: Mapped[list["UsageAggregate"]] = relationship( + back_populates="project" + ) + + __table_args__ = ( + UniqueConstraint("owner", "repo", name="uq_project_owner_repo"), + Index("ix_project_owner_repo", "owner", "repo"), + ) + + +class VersionCheck(Base): + """Content-addressable version-check records, bucketed by time.""" + + __tablename__ = "version_checks" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + project_id: Mapped[int] = mapped_column( + Integer, ForeignKey("projects.id"), nullable=False + ) + version: Mapped[str] = mapped_column(String(100), nullable=False) + city: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + region: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + country: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + country_code: Mapped[Optional[str]] = mapped_column( + String(2), nullable=True + ) + latitude: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + longitude: Mapped[Optional[float]] = mapped_column(Float, nullable=True) + is_ci: Mapped[bool] = mapped_column(Boolean, default=False) + time_bucket: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), nullable=False + ) + count: Mapped[int] = mapped_column(Integer, default=1) + created_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + project: Mapped["Project"] = relationship(back_populates="version_checks") + + __table_args__ = ( + UniqueConstraint( + "project_id", + "version", + "city", + "region", + "country_code", + "is_ci", + "time_bucket", + name="uq_version_check_content_address", + ), + Index("ix_version_check_project_bucket", "project_id", "time_bucket"), + Index("ix_version_check_bucket", "time_bucket"), + ) + + +class UsageAggregate(Base): + """Pre-aggregated usage statistics per period.""" + + __tablename__ = "usage_aggregates" + + id: Mapped[int] = mapped_column(BigInteger, primary_key=True) + project_id: Mapped[int] = mapped_column( + Integer, ForeignKey("projects.id"), nullable=False + ) + version: Mapped[Optional[str]] = mapped_column(String(100), nullable=True) + country_code: Mapped[Optional[str]] = mapped_column( + String(2), nullable=True + ) + region: Mapped[Optional[str]] = mapped_column(String(255), nullable=True) + granularity: Mapped[str] = mapped_column(String(10), nullable=False) + period_start: Mapped[datetime.date] = mapped_column(Date, nullable=False) + total_count: Mapped[int] = mapped_column(BigInteger, default=0) + unique_locations: Mapped[int] = mapped_column(Integer, default=0) + ci_count: Mapped[int] = mapped_column(BigInteger, default=0) + created_at: Mapped[datetime.datetime] = mapped_column( + DateTime(timezone=True), server_default=func.now() + ) + + project: Mapped["Project"] = relationship(back_populates="usage_aggregates") + + __table_args__ = ( + UniqueConstraint( + "project_id", + "version", + "country_code", + "region", + "granularity", + "period_start", + name="uq_usage_aggregate_key", + ), + Index( + "ix_usage_aggregate_project_granularity", + "project_id", + "granularity", + "period_start", + ), + ) diff --git a/server/src/etelemetry_server/routes/__init__.py b/server/src/etelemetry_server/routes/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/etelemetry_server/routes/dashboard.py b/server/src/etelemetry_server/routes/dashboard.py new file mode 100644 index 0000000..ca1330d --- /dev/null +++ b/server/src/etelemetry_server/routes/dashboard.py @@ -0,0 +1,493 @@ +"""Dashboard routes — HTML pages and JSON API endpoints.""" + +from __future__ import annotations + +import datetime +import logging +from pathlib import Path +from typing import Optional + +from fastapi import APIRouter, Depends, Query, Request +from fastapi.responses import HTMLResponse, JSONResponse +from fastapi.templating import Jinja2Templates +from sqlalchemy import distinct, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from etelemetry_server.db import get_db +from etelemetry_server.models import Project, VersionCheck + +logger = logging.getLogger(__name__) + +router = APIRouter() + +# Template and static file paths +DASHBOARD_DIR = Path(__file__).resolve().parent.parent / "dashboard" +_TEMPLATE_DIR = DASHBOARD_DIR / "templates" +STATIC_DIR = DASHBOARD_DIR / "static" + +templates = Jinja2Templates(directory=str(_TEMPLATE_DIR)) + + +# --------------------------------------------------------------------------- +# HTML pages +# --------------------------------------------------------------------------- + + +@router.get("/dashboard/", response_class=HTMLResponse) +async def dashboard_index( + request: Request, + session: AsyncSession = Depends(get_db), +): + """Render the project list HTML page.""" + stmt = ( + select( + Project.owner, + Project.repo, + func.coalesce(func.sum(VersionCheck.count), 0).label("total_checks"), + ) + .outerjoin(VersionCheck, VersionCheck.project_id == Project.id) + .where(Project.active.is_(True)) + .group_by(Project.owner, Project.repo) + .order_by(Project.owner, Project.repo) + ) + result = await session.execute(stmt) + projects = [ + {"owner": row.owner, "repo": row.repo, "total_checks": int(row.total_checks)} + for row in result.all() + ] + return templates.TemplateResponse( + "projects.html", + {"request": request, "projects": projects}, + ) + + +@router.get("/dashboard/project/{owner}/{repo}", response_class=HTMLResponse) +async def dashboard_project_detail( + request: Request, + owner: str, + repo: str, + session: AsyncSession = Depends(get_db), +): + """Render the project detail HTML page.""" + # Fetch project + stmt = select(Project).where(Project.owner == owner, Project.repo == repo) + result = await session.execute(stmt) + project = result.scalar_one_or_none() + if project is None: + return HTMLResponse(content="

Project not found

", status_code=404) + + # Total checks + total_stmt = ( + select(func.coalesce(func.sum(VersionCheck.count), 0)) + .where(VersionCheck.project_id == project.id) + ) + total_result = await session.execute(total_stmt) + total_checks = int(total_result.scalar() or 0) + + # Unique versions + versions_stmt = ( + select(func.count(distinct(VersionCheck.version))) + .where(VersionCheck.project_id == project.id) + ) + versions_result = await session.execute(versions_stmt) + unique_versions = int(versions_result.scalar() or 0) + + # Unique locations + locations_stmt = ( + select( + func.count( + distinct( + func.coalesce(VersionCheck.country_code, "") + + ":" + + func.coalesce(VersionCheck.region, "") + + ":" + + func.coalesce(VersionCheck.city, "") + ) + ) + ) + .where(VersionCheck.project_id == project.id) + ) + locations_result = await session.execute(locations_stmt) + unique_locations = int(locations_result.scalar() or 0) + + # Version distribution + version_dist_stmt = ( + select( + VersionCheck.version, + func.sum(VersionCheck.count).label("count"), + ) + .where(VersionCheck.project_id == project.id) + .group_by(VersionCheck.version) + .order_by(func.sum(VersionCheck.count).desc()) + ) + version_dist_result = await session.execute(version_dist_stmt) + by_version = [ + {"version": row.version, "count": int(row.count)} + for row in version_dist_result.all() + ] + + # Geographic summary + geo_stmt = ( + select( + VersionCheck.country, + VersionCheck.country_code, + VersionCheck.region, + VersionCheck.city, + func.sum(VersionCheck.count).label("count"), + ) + .where(VersionCheck.project_id == project.id) + .group_by( + VersionCheck.country, + VersionCheck.country_code, + VersionCheck.region, + VersionCheck.city, + ) + .order_by(func.sum(VersionCheck.count).desc()) + ) + geo_result = await session.execute(geo_stmt) + by_location = [ + { + "country": row.country, + "country_code": row.country_code, + "region": row.region, + "city": row.city, + "count": int(row.count), + } + for row in geo_result.all() + ] + + return templates.TemplateResponse( + "project_detail.html", + { + "request": request, + "owner": owner, + "repo": repo, + "total_checks": total_checks, + "unique_versions": unique_versions, + "unique_locations": unique_locations, + "by_version": by_version, + "by_location": by_location, + }, + ) + + +# --------------------------------------------------------------------------- +# JSON API endpoints +# --------------------------------------------------------------------------- + + +@router.get("/dashboard/api/projects") +async def api_projects( + session: AsyncSession = Depends(get_db), +): + """Return list of all tracked projects with total check counts.""" + stmt = ( + select( + Project.owner, + Project.repo, + func.coalesce(func.sum(VersionCheck.count), 0).label("total_checks"), + ) + .outerjoin(VersionCheck, VersionCheck.project_id == Project.id) + .where(Project.active.is_(True)) + .group_by(Project.owner, Project.repo) + .order_by(Project.owner, Project.repo) + ) + result = await session.execute(stmt) + projects = [ + {"owner": row.owner, "repo": row.repo, "total_checks": int(row.total_checks)} + for row in result.all() + ] + return {"projects": projects} + + +@router.get("/dashboard/api/stats/{owner}/{repo}") +async def api_stats( + owner: str, + repo: str, + session: AsyncSession = Depends(get_db), + date_from: Optional[str] = Query(None, alias="from"), + date_to: Optional[str] = Query(None, alias="to"), + granularity: Optional[str] = Query(None), +): + """Return stats for a specific project. + + Response shape matches contracts/api.md exactly. + """ + # Find project + stmt = select(Project).where(Project.owner == owner, Project.repo == repo) + result = await session.execute(stmt) + project = result.scalar_one_or_none() + if project is None: + return JSONResponse( + status_code=404, + content={"error": "project not found"}, + ) + + # Parse date range + today = datetime.date.today() + try: + from_date = datetime.date.fromisoformat(date_from) if date_from else None + except ValueError: + from_date = None + try: + to_date = datetime.date.fromisoformat(date_to) if date_to else None + except ValueError: + to_date = None + + if from_date is None: + from_date = today - datetime.timedelta(days=90) + if to_date is None: + to_date = today + + from_dt = datetime.datetime.combine(from_date, datetime.time.min) + to_dt = datetime.datetime.combine(to_date + datetime.timedelta(days=1), datetime.time.min) + + # Auto-determine granularity if not specified + if granularity not in ("daily", "weekly", "monthly"): + span_days = (to_date - from_date).days + if span_days <= 31: + granularity = "daily" + elif span_days <= 365: + granularity = "weekly" + else: + granularity = "monthly" + + # Total checks in range + total_stmt = ( + select(func.coalesce(func.sum(VersionCheck.count), 0)) + .where( + VersionCheck.project_id == project.id, + VersionCheck.time_bucket >= from_dt, + VersionCheck.time_bucket < to_dt, + ) + ) + total_result = await session.execute(total_stmt) + total_checks = int(total_result.scalar() or 0) + + # By version + version_stmt = ( + select( + VersionCheck.version, + func.sum(VersionCheck.count).label("count"), + ) + .where( + VersionCheck.project_id == project.id, + VersionCheck.time_bucket >= from_dt, + VersionCheck.time_bucket < to_dt, + ) + .group_by(VersionCheck.version) + .order_by(func.sum(VersionCheck.count).desc()) + ) + version_result = await session.execute(version_stmt) + by_version = [ + {"version": row.version, "count": int(row.count)} + for row in version_result.all() + ] + + # By location + location_stmt = ( + select( + VersionCheck.country, + VersionCheck.country_code, + VersionCheck.region, + VersionCheck.city, + func.avg(VersionCheck.latitude).label("lat"), + func.avg(VersionCheck.longitude).label("lon"), + func.sum(VersionCheck.count).label("count"), + ) + .where( + VersionCheck.project_id == project.id, + VersionCheck.time_bucket >= from_dt, + VersionCheck.time_bucket < to_dt, + ) + .group_by( + VersionCheck.country, + VersionCheck.country_code, + VersionCheck.region, + VersionCheck.city, + ) + .order_by(func.sum(VersionCheck.count).desc()) + ) + location_result = await session.execute(location_stmt) + by_location = [ + { + "country": row.country, + "country_code": row.country_code, + "region": row.region, + "city": row.city, + "lat": float(row.lat) if row.lat is not None else None, + "lon": float(row.lon) if row.lon is not None else None, + "count": int(row.count), + } + for row in location_result.all() + ] + + # Timeline + timeline = await _compute_timeline( + session, project.id, from_dt, to_dt, granularity + ) + + return { + "project": f"{owner}/{repo}", + "period": { + "from": from_date.isoformat(), + "to": to_date.isoformat(), + }, + "total_checks": total_checks, + "by_version": by_version, + "by_location": by_location, + "timeline": timeline, + } + + +@router.get("/dashboard/api/geo/{owner}/{repo}") +async def api_geo( + owner: str, + repo: str, + session: AsyncSession = Depends(get_db), + date_from: Optional[str] = Query(None, alias="from"), + date_to: Optional[str] = Query(None, alias="to"), + granularity: Optional[str] = Query(None), +): + """Return GeoJSON FeatureCollection for map visualization.""" + # Find project + stmt = select(Project).where(Project.owner == owner, Project.repo == repo) + result = await session.execute(stmt) + project = result.scalar_one_or_none() + if project is None: + return JSONResponse( + status_code=404, + content={"error": "project not found"}, + ) + + # Parse date range + today = datetime.date.today() + try: + from_date = datetime.date.fromisoformat(date_from) if date_from else None + except ValueError: + from_date = None + try: + to_date = datetime.date.fromisoformat(date_to) if date_to else None + except ValueError: + to_date = None + + if from_date is None: + from_date = today - datetime.timedelta(days=90) + if to_date is None: + to_date = today + + from_dt = datetime.datetime.combine(from_date, datetime.time.min) + to_dt = datetime.datetime.combine(to_date + datetime.timedelta(days=1), datetime.time.min) + + # Query locations with coordinates + location_stmt = ( + select( + VersionCheck.country, + VersionCheck.country_code, + VersionCheck.region, + VersionCheck.city, + func.avg(VersionCheck.latitude).label("lat"), + func.avg(VersionCheck.longitude).label("lon"), + func.sum(VersionCheck.count).label("count"), + ) + .where( + VersionCheck.project_id == project.id, + VersionCheck.time_bucket >= from_dt, + VersionCheck.time_bucket < to_dt, + VersionCheck.latitude.isnot(None), + VersionCheck.longitude.isnot(None), + ) + .group_by( + VersionCheck.country, + VersionCheck.country_code, + VersionCheck.region, + VersionCheck.city, + ) + ) + location_result = await session.execute(location_stmt) + + features = [] + for row in location_result.all(): + if row.lat is not None and row.lon is not None: + features.append( + { + "type": "Feature", + "geometry": { + "type": "Point", + "coordinates": [float(row.lon), float(row.lat)], + }, + "properties": { + "country": row.country, + "country_code": row.country_code, + "region": row.region, + "city": row.city, + "count": int(row.count), + }, + } + ) + + return { + "type": "FeatureCollection", + "features": features, + } + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +async def _compute_timeline( + session: AsyncSession, + project_id: int, + from_dt: datetime.datetime, + to_dt: datetime.datetime, + granularity: str, +) -> list[dict]: + """Compute timeline data grouped by the given granularity.""" + # Query raw version_checks and bucket in Python for DB portability + stmt = ( + select( + VersionCheck.time_bucket, + func.sum(VersionCheck.count).label("count"), + ) + .where( + VersionCheck.project_id == project_id, + VersionCheck.time_bucket >= from_dt, + VersionCheck.time_bucket < to_dt, + ) + .group_by(VersionCheck.time_bucket) + .order_by(VersionCheck.time_bucket) + ) + result = await session.execute(stmt) + rows = result.all() + + # Bucket into periods + buckets: dict[str, int] = {} + for row in rows: + ts = row.time_bucket + if isinstance(ts, str): + ts = datetime.datetime.fromisoformat(ts) + d = ts.date() if isinstance(ts, datetime.datetime) else ts + period_label = _period_label(d, granularity) + buckets[period_label] = buckets.get(period_label, 0) + int(row.count) + + # Sort and return + timeline = [ + {"period": period, "count": count} + for period, count in sorted(buckets.items()) + ] + return timeline + + +def _period_label(d: datetime.date, granularity: str) -> str: + """Return a human-readable period label for the given date and granularity.""" + if granularity == "daily": + return d.isoformat() + elif granularity == "weekly": + iso = d.isocalendar() + return f"{iso[0]}-W{iso[1]:02d}" + elif granularity == "monthly": + return f"{d.year}-{d.month:02d}" + else: + return d.isoformat() diff --git a/server/src/etelemetry_server/routes/health.py b/server/src/etelemetry_server/routes/health.py new file mode 100644 index 0000000..3fb4100 --- /dev/null +++ b/server/src/etelemetry_server/routes/health.py @@ -0,0 +1,11 @@ +"""Health-check route.""" + +from fastapi import APIRouter + +router = APIRouter() + + +@router.get("/") +async def health() -> dict: + """Return basic service information.""" + return {"name": "etelemetry", "version": "2.0.0"} diff --git a/server/src/etelemetry_server/routes/projects.py b/server/src/etelemetry_server/routes/projects.py new file mode 100644 index 0000000..1420291 --- /dev/null +++ b/server/src/etelemetry_server/routes/projects.py @@ -0,0 +1,123 @@ +"""Projects route — version check endpoint.""" + +from __future__ import annotations + +import logging + +import httpx +from fastapi import APIRouter, Depends, Request +from fastapi.responses import JSONResponse +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession + +from etelemetry_server.db import get_db +from etelemetry_server.models import Project +from etelemetry_server.services.geolocation import ( + GeoLocator, + _unknown_result, +) +from etelemetry_server.services.usage_recorder import record_usage +from etelemetry_server.services.version_checker import VersionChecker + +logger = logging.getLogger(__name__) + +router = APIRouter() + + +@router.get("/projects/{owner}/{repo}") +async def get_project_version( + owner: str, + repo: str, + request: Request, + session: AsyncSession = Depends(get_db), + ci: bool = False, + v: str | None = None, +): + """Return the latest version and bad versions for a project. + + Query parameters + ---------------- + ci : bool + Whether the caller is running in CI. + v : str | None + The callers current version (recorded for telemetry). + + Backward compatibility: old clients send CI info as individual query + params (name, isPR, etc.) instead of ``?ci=true``. Detect those. + """ + # Backward compat: detect old-style CI params from ci_info.info() + if not ci: + old_ci_keys = {"name", "isPR", "isCI"} + if old_ci_keys & set(request.query_params.keys()): + ci = True + + # 1. Check allowlist + allowlist: list[dict[str, str]] = getattr(request.app.state, "allowlist", []) + allowed = any( + e["owner"] == owner and e["repo"] == repo for e in allowlist + ) + if not allowed: + return JSONResponse( + status_code=404, + content={"error": "project not tracked"}, + ) + + # 2. Look up project in DB + stmt = select(Project).where(Project.owner == owner, Project.repo == repo) + result = await session.execute(stmt) + project = result.scalar_one_or_none() + + if project is None or not project.active: + return JSONResponse( + status_code=404, + content={"error": "project not tracked"}, + ) + + settings = request.app.state.settings + + # 3. Resolve geolocation (transient — IP is never stored) + client_ip = request.client.host if request.client else "0.0.0.0" + geolocator: GeoLocator | None = getattr(request.app.state, "geolocator", None) + if geolocator is not None: + geo_result = geolocator.resolve(client_ip) + else: + geo_result = _unknown_result() + + # 4. Record usage + try: + await record_usage( + session=session, + project_id=project.id, + version=v or "unknown", + geo_result=geo_result, + is_ci=ci, + time_bucket_hours=settings.TIME_BUCKET_HOURS, + ) + except Exception: + logger.warning("Failed to record usage for %s/%s", owner, repo, exc_info=True) + + # 5. Fetch version info + try: + http_client: httpx.AsyncClient = getattr( + request.app.state, "http_client", None + ) or httpx.AsyncClient() + checker = VersionChecker( + http_client=http_client, + github_token=settings.GITHUB_TOKEN, + ) + version_info = await checker.get_version_info( + project, cache_ttl=settings.CACHE_TTL_SECONDS + ) + except Exception: + logger.warning( + "Failed to get version info for %s/%s", owner, repo, exc_info=True + ) + version_info = { + "version": project.latest_version, + "bad_versions": project.bad_versions or [], + } + + return { + "version": version_info.get("version"), + "bad_versions": version_info.get("bad_versions", []), + } diff --git a/server/src/etelemetry_server/services/__init__.py b/server/src/etelemetry_server/services/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/etelemetry_server/services/aggregation.py b/server/src/etelemetry_server/services/aggregation.py new file mode 100644 index 0000000..ea6e46a --- /dev/null +++ b/server/src/etelemetry_server/services/aggregation.py @@ -0,0 +1,224 @@ +"""Aggregation service for pre-computing usage statistics. + +Queries version_checks, groups by project/version/location/period, +and upserts results into usage_aggregates. +""" + +from __future__ import annotations + +import datetime +import logging + +from sqlalchemy import delete, func, select +from sqlalchemy.ext.asyncio import AsyncSession + +from etelemetry_server.models import UsageAggregate, VersionCheck + +logger = logging.getLogger(__name__) + + +def _truncate_date(dt: datetime.datetime, granularity: str) -> datetime.date: + """Truncate a datetime to the start of the given granularity period.""" + d = dt.date() if isinstance(dt, datetime.datetime) else dt + if granularity == "daily": + return d + elif granularity == "weekly": + # Start of ISO week (Monday) + return d - datetime.timedelta(days=d.weekday()) + elif granularity == "monthly": + return d.replace(day=1) + else: + raise ValueError(f"Unknown granularity: {granularity}") + + +async def compute_aggregates( + session: AsyncSession, + granularity: str, + date_from: datetime.date, + date_to: datetime.date, +) -> int: + """Query version_checks, group by project/version/location/period, + and upsert into usage_aggregates. + + Parameters + ---------- + session : AsyncSession + Active database session. + granularity : str + One of 'daily', 'weekly', 'monthly'. + date_from : datetime.date + Start of the date range (inclusive). + date_to : datetime.date + End of the date range (inclusive). + + Returns + ------- + int + Number of aggregate rows upserted. + """ + from_dt = datetime.datetime.combine(date_from, datetime.time.min) + to_dt = datetime.datetime.combine( + date_to + datetime.timedelta(days=1), datetime.time.min + ) + + # Query version_checks grouped by project, version, country_code, region + stmt = ( + select( + VersionCheck.project_id, + VersionCheck.version, + VersionCheck.country_code, + VersionCheck.region, + func.sum(VersionCheck.count).label("total_count"), + func.count( + func.distinct( + func.coalesce(VersionCheck.city, "") + + ":" + + func.coalesce(VersionCheck.region, "") + + ":" + + func.coalesce(VersionCheck.country_code, "") + ) + ).label("unique_locations"), + func.sum( + # Sum count where is_ci is True + VersionCheck.count * func.cast(VersionCheck.is_ci, type_=VersionCheck.count.type) + ).label("ci_count"), + VersionCheck.time_bucket, + ) + .where( + VersionCheck.time_bucket >= from_dt, + VersionCheck.time_bucket < to_dt, + ) + .group_by( + VersionCheck.project_id, + VersionCheck.version, + VersionCheck.country_code, + VersionCheck.region, + VersionCheck.time_bucket, + ) + ) + + result = await session.execute(stmt) + rows = result.all() + + # Group rows by (project_id, version, country_code, region, period_start) + aggregated: dict[tuple, dict] = {} + for row in rows: + period_start = _truncate_date(row.time_bucket, granularity) + key = ( + row.project_id, + row.version, + row.country_code, + row.region, + period_start, + ) + if key not in aggregated: + aggregated[key] = { + "total_count": 0, + "unique_locations": 0, + "ci_count": 0, + "location_set": set(), + } + agg = aggregated[key] + agg["total_count"] += row.total_count or 0 + agg["ci_count"] += row.ci_count or 0 + # Track unique location strings + loc_str = f"{row.country_code}:{row.region}" + agg["location_set"].add(loc_str) + agg["unique_locations"] = len(agg["location_set"]) + + # Delete existing aggregates for this granularity and date range, then insert + await session.execute( + delete(UsageAggregate).where( + UsageAggregate.granularity == granularity, + UsageAggregate.period_start >= date_from, + UsageAggregate.period_start <= date_to, + ) + ) + + count = 0 + for key, agg in aggregated.items(): + project_id, version, country_code, region, period_start = key + aggregate = UsageAggregate( + project_id=project_id, + version=version, + country_code=country_code, + region=region, + granularity=granularity, + period_start=period_start, + total_count=agg["total_count"], + unique_locations=agg["unique_locations"], + ci_count=agg["ci_count"], + ) + session.add(aggregate) + count += 1 + + await session.flush() + logger.info( + "Computed %d %s aggregates for %s to %s", + count, + granularity, + date_from, + date_to, + ) + return count + + +async def run_tiered_aggregation( + session: AsyncSession, + daily_threshold_days: int = 30, + weekly_threshold_days: int = 365, +) -> dict[str, int]: + """Compute tiered aggregations: daily for recent, weekly for medium, monthly for old. + + Parameters + ---------- + session : AsyncSession + Active database session. + daily_threshold_days : int + Number of days back for daily aggregation (default 30). + weekly_threshold_days : int + Number of days back for weekly aggregation (default 365). + + Returns + ------- + dict[str, int] + Mapping of granularity to number of rows upserted. + """ + today = datetime.date.today() + results = {} + + # Daily: last N days + daily_from = today - datetime.timedelta(days=daily_threshold_days) + results["daily"] = await compute_aggregates( + session, "daily", daily_from, today + ) + + # Weekly: from daily_threshold to weekly_threshold days ago + weekly_from = today - datetime.timedelta(days=weekly_threshold_days) + weekly_to = daily_from - datetime.timedelta(days=1) + if weekly_to >= weekly_from: + results["weekly"] = await compute_aggregates( + session, "weekly", weekly_from, weekly_to + ) + else: + results["weekly"] = 0 + + # Monthly: older than weekly_threshold days + # Find the earliest version_check + stmt = select(func.min(VersionCheck.time_bucket)) + result = await session.execute(stmt) + earliest = result.scalar() + if earliest is not None: + earliest_date = earliest.date() if isinstance(earliest, datetime.datetime) else earliest + monthly_to = weekly_from - datetime.timedelta(days=1) + if monthly_to >= earliest_date: + results["monthly"] = await compute_aggregates( + session, "monthly", earliest_date, monthly_to + ) + else: + results["monthly"] = 0 + else: + results["monthly"] = 0 + + logger.info("Tiered aggregation complete: %s", results) + return results diff --git a/server/src/etelemetry_server/services/geolocation.py b/server/src/etelemetry_server/services/geolocation.py new file mode 100644 index 0000000..ee78e3e --- /dev/null +++ b/server/src/etelemetry_server/services/geolocation.py @@ -0,0 +1,140 @@ +"""Geolocation service using MaxMind GeoLite2-City or DB-IP Lite database. + +Tries MaxMind first (GeoLite2-City.mmdb). If that file doesn't exist, +falls back to DB-IP Lite (dbip-city-lite.mmdb), which is freely +available without registration from https://db-ip.com/db/lite.php. +""" + +from __future__ import annotations + +import logging +from dataclasses import dataclass +from pathlib import Path + +logger = logging.getLogger(__name__) + + +@dataclass +class GeoResult: + """Result of a geolocation lookup.""" + + city: str = "unknown" + region: str = "unknown" + country: str = "unknown" + country_code: str = "XX" + latitude: float = 0.0 + longitude: float = 0.0 + + +def _unknown_result() -> GeoResult: + """Return a GeoResult with default unknown values.""" + return GeoResult() + + +def _find_mmdb(primary_path: str) -> Path | None: + """Find a usable MMDB file. + + Tries the configured path first, then falls back to common DB-IP + Lite locations in the same directory or well-known paths. + """ + primary = Path(primary_path) + if primary.exists(): + return primary + + # Look for DB-IP Lite in the same directory as the configured path + parent = primary.parent + dbip_candidates = [ + parent / "dbip-city-lite.mmdb", + parent / "dbip-city-lite-latest.mmdb", + # Common system locations + Path("/usr/share/GeoIP/dbip-city-lite.mmdb"), + Path("/data/dbip-city-lite.mmdb"), + # Local project directory + Path("dbip-city-lite.mmdb"), + ] + for candidate in dbip_candidates: + if candidate.exists(): + return candidate + + return None + + +class GeoLocator: + """Resolve IP addresses to geographic locations. + + Uses MaxMind GeoLite2-City as the primary database. If that file is + not available, falls back to DB-IP Lite (free, no registration + required). Both use the same MMDB format readable by the ``geoip2`` + / ``maxminddb`` library. + """ + + def __init__(self, db_path: str) -> None: + self._reader = None + self._db_source = "none" + + mmdb_path = _find_mmdb(db_path) + if mmdb_path is None: + logger.warning( + "No geolocation database found. Tried %s and DB-IP Lite " + "fallback locations. Geolocation will return 'unknown'.", + db_path, + ) + return + + try: + import geoip2.database + + self._reader = geoip2.database.Reader(str(mmdb_path)) + # Identify which database we loaded + db_type = self._reader.metadata().database_type + self._db_source = mmdb_path.name + logger.info( + "GeoLocator loaded %s database from %s", + db_type, + mmdb_path, + ) + except Exception: + logger.warning( + "Failed to open geolocation database at %s", + mmdb_path, + exc_info=True, + ) + + @property + def db_source(self) -> str: + """Name of the loaded database file, or 'none'.""" + return self._db_source + + def resolve(self, ip: str) -> GeoResult: + """Look up an IP address and return a GeoResult. + + Returns a result with ``"unknown"`` fields when the IP cannot be + resolved for any reason (missing DB, invalid IP, address not found). + """ + if self._reader is None: + return _unknown_result() + + try: + response = self._reader.city(ip) + return GeoResult( + city=response.city.name or "unknown", + region=( + response.subdivisions.most_specific.name + if response.subdivisions + else "unknown" + ) + or "unknown", + country=response.country.name or "unknown", + country_code=response.country.iso_code or "XX", + latitude=response.location.latitude or 0.0, + longitude=response.location.longitude or 0.0, + ) + except Exception as exc: + logger.debug("Geolocation lookup failed: %s", type(exc).__name__) + return _unknown_result() + + def close(self) -> None: + """Close the underlying reader.""" + if self._reader is not None: + self._reader.close() + self._reader = None diff --git a/server/src/etelemetry_server/services/usage_recorder.py b/server/src/etelemetry_server/services/usage_recorder.py new file mode 100644 index 0000000..48c0b22 --- /dev/null +++ b/server/src/etelemetry_server/services/usage_recorder.py @@ -0,0 +1,61 @@ +"""Usage recording service — content-addressed upsert into version_checks.""" + +from __future__ import annotations + +import logging +from datetime import datetime, timezone + +from sqlalchemy.dialects.postgresql import insert as pg_insert +from sqlalchemy.ext.asyncio import AsyncSession + +from etelemetry_server.models import VersionCheck +from etelemetry_server.services.geolocation import GeoResult + +logger = logging.getLogger(__name__) + + +def _compute_time_bucket(time_bucket_hours: int) -> datetime: + """Truncate the current UTC time to the nearest *time_bucket_hours* boundary.""" + now = datetime.now(timezone.utc) + # Truncate to the start of the current bucket + total_hours = now.hour + bucket_hour = (total_hours // time_bucket_hours) * time_bucket_hours + return now.replace(hour=bucket_hour, minute=0, second=0, microsecond=0) + + +async def record_usage( + session: AsyncSession, + project_id: int, + version: str, + geo_result: GeoResult, + is_ci: bool, + time_bucket_hours: int, +) -> None: + """Record a version check using a content-addressed upsert. + + If a row with the same (project_id, version, city, region, country_code, + is_ci, time_bucket) already exists, its ``count`` is incremented. + """ + time_bucket = _compute_time_bucket(time_bucket_hours) + + values = { + "project_id": project_id, + "version": version or "unknown", + "city": geo_result.city, + "region": geo_result.region, + "country": geo_result.country, + "country_code": geo_result.country_code, + "latitude": geo_result.latitude, + "longitude": geo_result.longitude, + "is_ci": is_ci, + "time_bucket": time_bucket, + "count": 1, + } + + stmt = pg_insert(VersionCheck).values(**values) + stmt = stmt.on_conflict_do_update( + constraint="uq_version_check_content_address", + set_={"count": VersionCheck.__table__.c.count + 1}, + ) + + await session.execute(stmt) diff --git a/server/src/etelemetry_server/services/version_checker.py b/server/src/etelemetry_server/services/version_checker.py new file mode 100644 index 0000000..f4d2588 --- /dev/null +++ b/server/src/etelemetry_server/services/version_checker.py @@ -0,0 +1,167 @@ +"""Version checker service — fetches latest version info from GitHub.""" + +from __future__ import annotations + +import logging +import time +from datetime import datetime, timezone + +import httpx + +from etelemetry_server.models import Project + +logger = logging.getLogger(__name__) + + +class VersionChecker: + """Fetch and cache latest version information for a project.""" + + def __init__( + self, + http_client: httpx.AsyncClient, + github_token: str | None = None, + ) -> None: + self._http = http_client + self._github_token = github_token + # In-memory LRU / TTL-aware cache keyed by "owner/repo" + self._memory_cache: dict[str, dict] = {} + self._cache_ts: dict[str, float] = {} + + # ------------------------------------------------------------------ + # Public API + # ------------------------------------------------------------------ + + async def get_version_info( + self, project: Project, cache_ttl: int + ) -> dict: + """Return ``{"version": ..., "bad_versions": [...]}`` for *project*. + + Uses the project's DB-level cache first, then falls back to GitHub, + and keeps an in-memory cache as a last resort. + """ + key = f"{project.owner}/{project.repo}" + + # 1. DB-level cache + now = datetime.now(timezone.utc) + if project.cache_expires_at and project.cache_expires_at > now: + result = { + "version": project.latest_version, + "bad_versions": project.bad_versions or [], + } + self._set_mem_cache(key, result, cache_ttl) + return result + + # 2. Fetch from GitHub + try: + version = await self._fetch_latest_version(project.owner, project.repo) + bad_versions = await self._fetch_bad_versions(project.owner, project.repo) + + # Update project record (caller is expected to commit) + project.latest_version = version + project.bad_versions = bad_versions + from datetime import timedelta + + project.cache_expires_at = now + timedelta(seconds=cache_ttl) + + result = {"version": version, "bad_versions": bad_versions} + self._set_mem_cache(key, result, cache_ttl) + return result + + except _RateLimitedError: + logger.warning("GitHub rate-limited; returning stale data for %s", key) + return self._stale_or_empty(project, key) + + except Exception: + logger.warning( + "Failed to fetch version info for %s", key, exc_info=True + ) + return self._stale_or_empty(project, key) + + # ------------------------------------------------------------------ + # In-memory cache helpers (T018) + # ------------------------------------------------------------------ + + def _set_mem_cache(self, key: str, value: dict, ttl: int) -> None: + self._memory_cache[key] = value + self._cache_ts[key] = time.monotonic() + ttl + + def _get_mem_cache(self, key: str) -> dict | None: + if key in self._memory_cache: + # Return even if expired — it is stale but better than nothing + return self._memory_cache[key] + return None + + def _stale_or_empty(self, project: Project, key: str) -> dict: + """Return the best available stale data, or an empty result.""" + # Try in-memory cache first + cached = self._get_mem_cache(key) + if cached is not None: + return cached + # Fall back to whatever the DB has + if project.latest_version: + return { + "version": project.latest_version, + "bad_versions": project.bad_versions or [], + } + return {"version": None, "bad_versions": []} + + # ------------------------------------------------------------------ + # GitHub fetching internals + # ------------------------------------------------------------------ + + def _auth_headers(self) -> dict[str, str]: + headers: dict[str, str] = {"Accept": "application/vnd.github+json"} + if self._github_token: + headers["Authorization"] = f"Bearer {self._github_token}" + return headers + + async def _fetch_latest_version(self, owner: str, repo: str) -> str | None: + """Try releases/latest, then fall back to tags.""" + headers = self._auth_headers() + + # Try releases endpoint + url = f"https://api.github.com/repos/{owner}/{repo}/releases/latest" + resp = await self._http.get(url, headers=headers) + + if resp.status_code == 403: + raise _RateLimitedError() + + if resp.status_code == 200: + tag = resp.json().get("tag_name", "") + return tag.lstrip("v") if tag else None + + # Fallback to tags + if resp.status_code == 404: + tags_url = f"https://api.github.com/repos/{owner}/{repo}/tags" + resp = await self._http.get(tags_url, headers=headers) + if resp.status_code == 403: + raise _RateLimitedError() + if resp.status_code == 200: + tags = resp.json() + if tags: + tag = tags[0].get("name", "") + return tag.lstrip("v") if tag else None + + return None + + async def _fetch_bad_versions(self, owner: str, repo: str) -> list[str]: + """Fetch the ``.et`` file from the repository root.""" + headers = self._auth_headers() + # Try master then main + for branch in ("master", "main"): + url = f"https://raw.githubusercontent.com/{owner}/{repo}/{branch}/.et" + resp = await self._http.get(url, headers=headers) + if resp.status_code == 200: + try: + import json + + data = json.loads(resp.text) + return data.get("bad_versions", []) + except Exception: + logger.debug("Failed to parse .et for %s/%s", owner, repo) + return [] + return [] + + +class _RateLimitedError(Exception): + """Raised when GitHub returns a 403 rate-limit response.""" diff --git a/server/src/etelemetry_server/settings.py b/server/src/etelemetry_server/settings.py new file mode 100644 index 0000000..980132f --- /dev/null +++ b/server/src/etelemetry_server/settings.py @@ -0,0 +1,21 @@ +"""Application settings loaded from environment variables.""" + +from pydantic_settings import BaseSettings, SettingsConfigDict + + +class Settings(BaseSettings): + """Server configuration loaded from environment / .env file.""" + + model_config = SettingsConfigDict(env_file=".env") + + DATABASE_URL: str = "postgresql+asyncpg://localhost:5432/etelemetry" + MAXMIND_DB_PATH: str = "/data/GeoLite2-City.mmdb" + GITHUB_TOKEN: str | None = None + CACHE_TTL_SECONDS: int = 21600 # 6 hours + ALLOWLIST_PATH: str = "allowlist.yml" + TIME_BUCKET_HOURS: int = 1 + SERVER_HOST: str = "0.0.0.0" + SERVER_PORT: int = 8000 + + +settings = Settings() diff --git a/server/tests/__init__.py b/server/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/tests/contract/__init__.py b/server/tests/contract/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/tests/contract/test_dashboard_api.py b/server/tests/contract/test_dashboard_api.py new file mode 100644 index 0000000..f3d53ae --- /dev/null +++ b/server/tests/contract/test_dashboard_api.py @@ -0,0 +1,280 @@ +"""Contract tests for dashboard API endpoints. + +Verify response shapes match contracts/api.md exactly. +""" + +from __future__ import annotations + +import datetime +from unittest.mock import MagicMock + +import httpx +import pytest +import pytest_asyncio +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from etelemetry_server.app import create_app +from etelemetry_server.db import get_db +from etelemetry_server.models import Base, Project, VersionCheck + +TEST_DATABASE_URL = "sqlite+aiosqlite:///:memory:" + + +@pytest_asyncio.fixture +async def engine(): + eng = create_async_engine(TEST_DATABASE_URL, echo=False) + async with eng.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield eng + await eng.dispose() + + +@pytest_asyncio.fixture +async def session_factory(engine): + return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + +@pytest_asyncio.fixture +async def seed_data(session_factory): + """Seed DB with test projects and version checks.""" + async with session_factory() as session: + p1 = Project(owner="nipy", repo="nipype", active=True) + p2 = Project(owner="nipy", repo="nibabel", active=True) + session.add_all([p1, p2]) + await session.flush() + + now = datetime.datetime(2026, 3, 1, 12, 0, 0) + checks = [ + VersionCheck( + project_id=p1.id, + version="1.8.6", + city="Cambridge", + region="Massachusetts", + country="United States", + country_code="US", + latitude=42.36, + longitude=-71.06, + is_ci=False, + time_bucket=now, + count=100, + ), + VersionCheck( + project_id=p1.id, + version="1.9.0", + city="London", + region="England", + country="United Kingdom", + country_code="GB", + latitude=51.51, + longitude=-0.13, + is_ci=True, + time_bucket=now, + count=50, + ), + VersionCheck( + project_id=p2.id, + version="4.0.0", + city="Berlin", + region="Berlin", + country="Germany", + country_code="DE", + latitude=52.52, + longitude=13.40, + is_ci=False, + time_bucket=now, + count=75, + ), + ] + session.add_all(checks) + await session.commit() + return {"p1": p1, "p2": p2} + + +@pytest.fixture +def app(session_factory, seed_data): + """Build FastAPI app wired to test DB.""" + application = create_app() + application.state.allowlist = [ + {"owner": "nipy", "repo": "nipype"}, + {"owner": "nipy", "repo": "nibabel"}, + ] + application.state.settings = MagicMock( + GITHUB_TOKEN=None, + CACHE_TTL_SECONDS=3600, + TIME_BUCKET_HOURS=1, + MAXMIND_DB_PATH="/nonexistent", + ) + application.state.geolocator = None + application.state.http_client = httpx.AsyncClient() + + async def _override_get_db(): + async with session_factory() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + + application.dependency_overrides[get_db] = _override_get_db + return application + + +@pytest.mark.asyncio +async def test_api_projects_response_shape(app): + """GET /dashboard/api/projects returns list of projects with counts.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get("/dashboard/api/projects") + + assert resp.status_code == 200 + body = resp.json() + + # Must have "projects" key containing a list + assert "projects" in body + assert isinstance(body["projects"], list) + assert len(body["projects"]) >= 2 + + # Each project must have owner, repo, total_checks + for project in body["projects"]: + assert "owner" in project + assert "repo" in project + assert "total_checks" in project + assert isinstance(project["total_checks"], int) + + +@pytest.mark.asyncio +async def test_api_stats_response_shape(app): + """GET /dashboard/api/stats/{owner}/{repo} returns stats with + by_version, by_location, timeline. + """ + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get( + "/dashboard/api/stats/nipy/nipype", + params={"from": "2026-01-01", "to": "2026-12-31"}, + ) + + assert resp.status_code == 200 + body = resp.json() + + # Required top-level keys + assert body["project"] == "nipy/nipype" + assert "period" in body + assert "from" in body["period"] + assert "to" in body["period"] + assert "total_checks" in body + assert isinstance(body["total_checks"], int) + assert body["total_checks"] > 0 + + # by_version + assert "by_version" in body + assert isinstance(body["by_version"], list) + for entry in body["by_version"]: + assert "version" in entry + assert "count" in entry + assert isinstance(entry["count"], int) + + # by_location + assert "by_location" in body + assert isinstance(body["by_location"], list) + for entry in body["by_location"]: + assert "country" in entry + assert "country_code" in entry + assert "region" in entry + assert "city" in entry + assert "lat" in entry + assert "lon" in entry + assert "count" in entry + + # timeline + assert "timeline" in body + assert isinstance(body["timeline"], list) + for entry in body["timeline"]: + assert "period" in entry + assert "count" in entry + + +@pytest.mark.asyncio +async def test_api_stats_404_for_unknown_project(app): + """GET /dashboard/api/stats for unknown project returns 404.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get("/dashboard/api/stats/unknown/repo") + + assert resp.status_code == 404 + body = resp.json() + assert "error" in body + + +@pytest.mark.asyncio +async def test_api_geo_response_shape(app): + """GET /dashboard/api/geo/{owner}/{repo} returns GeoJSON FeatureCollection.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get( + "/dashboard/api/geo/nipy/nipype", + params={"from": "2026-01-01", "to": "2026-12-31"}, + ) + + assert resp.status_code == 200 + body = resp.json() + + # Must be a GeoJSON FeatureCollection + assert body["type"] == "FeatureCollection" + assert "features" in body + assert isinstance(body["features"], list) + assert len(body["features"]) > 0 + + for feature in body["features"]: + assert feature["type"] == "Feature" + assert "geometry" in feature + assert feature["geometry"]["type"] == "Point" + assert "coordinates" in feature["geometry"] + coords = feature["geometry"]["coordinates"] + assert len(coords) == 2 # [lon, lat] + assert isinstance(coords[0], (int, float)) + assert isinstance(coords[1], (int, float)) + + assert "properties" in feature + assert "count" in feature["properties"] + assert isinstance(feature["properties"]["count"], int) + + +@pytest.mark.asyncio +async def test_api_geo_404_for_unknown_project(app): + """GET /dashboard/api/geo for unknown project returns 404.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get("/dashboard/api/geo/unknown/repo") + + assert resp.status_code == 404 + body = resp.json() + assert "error" in body + + +@pytest.mark.asyncio +async def test_api_stats_granularity_param(app): + """Stats endpoint respects granularity query parameter.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get( + "/dashboard/api/stats/nipy/nipype", + params={ + "from": "2026-01-01", + "to": "2026-12-31", + "granularity": "monthly", + }, + ) + + assert resp.status_code == 200 + body = resp.json() + # Timeline entries should use monthly format (YYYY-MM) + for entry in body["timeline"]: + assert len(entry["period"]) == 7 # e.g., "2026-03" diff --git a/server/tests/contract/test_projects_api.py b/server/tests/contract/test_projects_api.py new file mode 100644 index 0000000..cd52b42 --- /dev/null +++ b/server/tests/contract/test_projects_api.py @@ -0,0 +1,112 @@ +"""Contract tests for GET /projects/{owner}/{repo}.""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest + +from etelemetry_server.app import create_app + + +@pytest.fixture +def app(): + """Create a test app with a mocked allowlist and services.""" + application = create_app() + + # Provide minimal app.state fixtures + application.state.allowlist = [ + {"owner": "nipy", "repo": "nipype"}, + ] + application.state.settings = MagicMock( + GITHUB_TOKEN=None, + CACHE_TTL_SECONDS=3600, + TIME_BUCKET_HOURS=1, + MAXMIND_DB_PATH="/nonexistent", + ) + application.state.geolocator = None + application.state.http_client = httpx.AsyncClient() + + return application + + +@pytest.mark.asyncio +async def test_200_response_shape(app): + """A known project returns 200 with version and bad_versions keys.""" + mock_project = MagicMock() + mock_project.id = 1 + mock_project.owner = "nipy" + mock_project.repo = "nipype" + mock_project.active = True + mock_project.latest_version = "1.8.6" + mock_project.bad_versions = ["1.0.0"] + mock_project.cache_expires_at = None + + with ( + patch( + "etelemetry_server.routes.projects.get_db", + return_value=_mock_session(mock_project), + ), + patch( + "etelemetry_server.routes.projects.record_usage", + new_callable=AsyncMock, + ), + patch( + "etelemetry_server.routes.projects.VersionChecker" + ) as MockChecker, + ): + instance = MockChecker.return_value + instance.get_version_info = AsyncMock( + return_value={"version": "1.8.6", "bad_versions": ["1.0.0"]} + ) + + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get("/projects/nipy/nipype") + + assert resp.status_code == 200 + body = resp.json() + assert "version" in body + assert "bad_versions" in body + assert isinstance(body["bad_versions"], list) + + +@pytest.mark.asyncio +async def test_404_for_project_not_on_allowlist(app): + """A project not on the allowlist returns 404 with an error message.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get("/projects/unknown/repo") + + assert resp.status_code == 404 + body = resp.json() + assert body == {"error": "project not tracked"} + + +@pytest.mark.asyncio +async def test_400_malformed_project_identifier(app): + """Malformed project identifiers should not match the route (404).""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + # FastAPI path params will still match two segments; + # a single segment should yield 404 (no route match). + resp = await client.get("/projects/onlyone") + + assert resp.status_code in (404, 422) + + +# ------------------------------------------------------------------ +# Helpers +# ------------------------------------------------------------------ + +async def _mock_session(project): + """Yield a mock async session that returns *project* on execute.""" + session = AsyncMock() + result = MagicMock() + result.scalar_one_or_none.return_value = project + session.execute = AsyncMock(return_value=result) + yield session diff --git a/server/tests/fixtures/__init__.py b/server/tests/fixtures/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/tests/fixtures/mongo_sample.json b/server/tests/fixtures/mongo_sample.json new file mode 100644 index 0000000..f57afb5 --- /dev/null +++ b/server/tests/fixtures/mongo_sample.json @@ -0,0 +1,93 @@ +{ + "requests": [ + { + "_id": "aaa111", + "owner": "nipy", + "repository": "nipype", + "version": "1.8.6", + "is_ci": false, + "access_time": "2025-06-15T10:30:45Z", + "remote_addr": "203.0.113.10" + }, + { + "_id": "aaa222", + "owner": "nipy", + "repository": "nipype", + "version": "1.8.6", + "is_ci": true, + "access_time": "2025-06-15T10:45:00Z", + "remote_addr": "198.51.100.20" + }, + { + "_id": "aaa333", + "owner": "nipy", + "repository": "nipype", + "version": "1.8.6", + "is_ci": false, + "access_time": "2025-06-15T10:15:00Z", + "remote_addr": "203.0.113.10" + }, + { + "_id": "aaa444", + "owner": "poldracklab", + "repository": "fmriprep", + "version": "23.1.0", + "is_ci": false, + "access_time": "2025-07-01T14:00:00Z", + "remote_addr": "192.0.2.50" + }, + { + "_id": "aaa555", + "owner": "nipy", + "repository": "nipype", + "version": "1.8.6", + "is_ci": false, + "access_time": "2025-06-15T10:30:45Z", + "remote_addr": "203.0.113.10" + }, + { + "_id": "bad_missing_owner", + "repository": "somerepo", + "version": "0.1.0", + "is_ci": false, + "access_time": "2025-06-15T12:00:00Z", + "remote_addr": "10.0.0.1" + }, + { + "_id": "bad_missing_version", + "owner": "someone", + "repository": "somerepo", + "is_ci": false, + "access_time": "2025-06-15T12:00:00Z", + "remote_addr": "10.0.0.2" + }, + { + "_id": "bad_missing_repo", + "owner": "someone", + "version": "0.1.0", + "is_ci": false, + "access_time": "2025-06-15T12:00:00Z", + "remote_addr": "10.0.0.3" + } + ], + "geo": [ + { + "_id": "ggg111", + "remote_addr": "203.0.113.10", + "city": "Cambridge", + "region_name": "Massachusetts", + "country_name": "United States", + "latitude": 42.3736, + "longitude": -71.1097 + }, + { + "_id": "ggg222", + "remote_addr": "198.51.100.20", + "city": "London", + "region_name": "England", + "country_name": "United Kingdom", + "latitude": 51.5074, + "longitude": -0.1278 + } + ] +} diff --git a/server/tests/integration/__init__.py b/server/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/tests/integration/test_dashboard.py b/server/tests/integration/test_dashboard.py new file mode 100644 index 0000000..756e26b --- /dev/null +++ b/server/tests/integration/test_dashboard.py @@ -0,0 +1,361 @@ +"""Integration tests for the dashboard. + +Seeds DB with 12+ months of sample version_check data across +multiple projects/locations. Verifies HTML pages render and API +returns correct aggregated data. Uses SQLite in-memory. +""" + +from __future__ import annotations + +import datetime +from unittest.mock import MagicMock + +import httpx +import pytest +import pytest_asyncio +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from etelemetry_server.app import create_app +from etelemetry_server.db import get_db +from etelemetry_server.models import Base, Project, VersionCheck + +TEST_DATABASE_URL = "sqlite+aiosqlite:///:memory:" + +# Sample data spanning 14 months +LOCATIONS = [ + { + "city": "Cambridge", + "region": "Massachusetts", + "country": "United States", + "country_code": "US", + "latitude": 42.36, + "longitude": -71.06, + }, + { + "city": "London", + "region": "England", + "country": "United Kingdom", + "country_code": "GB", + "latitude": 51.51, + "longitude": -0.13, + }, + { + "city": "Berlin", + "region": "Berlin", + "country": "Germany", + "country_code": "DE", + "latitude": 52.52, + "longitude": 13.40, + }, + { + "city": "Tokyo", + "region": "Tokyo", + "country": "Japan", + "country_code": "JP", + "latitude": 35.68, + "longitude": 139.69, + }, +] + +VERSIONS = ["1.8.0", "1.8.6", "1.9.0"] + + +@pytest_asyncio.fixture +async def engine(): + eng = create_async_engine(TEST_DATABASE_URL, echo=False) + async with eng.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield eng + await eng.dispose() + + +@pytest_asyncio.fixture +async def session_factory(engine): + return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + +@pytest_asyncio.fixture +async def seed_data(session_factory): + """Seed DB with 14 months of version check data across 2 projects and 4 locations.""" + async with session_factory() as session: + p1 = Project(owner="nipy", repo="nipype", active=True) + p2 = Project(owner="nipy", repo="nibabel", active=True) + session.add_all([p1, p2]) + await session.flush() + + checks = [] + # Generate 14 months of data: Jan 2025 through Feb 2026 + start_date = datetime.date(2025, 1, 1) + for month_offset in range(14): + year = 2025 + (start_date.month + month_offset - 1) // 12 + month = (start_date.month + month_offset - 1) % 12 + 1 + bucket_time = datetime.datetime(year, month, 15, 12, 0, 0) + + for loc_idx, loc in enumerate(LOCATIONS): + for ver_idx, version in enumerate(VERSIONS): + count_val = (month_offset + 1) * (loc_idx + 1) * (ver_idx + 1) + # Project 1 + checks.append( + VersionCheck( + project_id=p1.id, + version=version, + city=loc["city"], + region=loc["region"], + country=loc["country"], + country_code=loc["country_code"], + latitude=loc["latitude"], + longitude=loc["longitude"], + is_ci=(ver_idx == 0), + time_bucket=bucket_time, + count=count_val, + ) + ) + # Project 2 (half the data) + if loc_idx < 2: + checks.append( + VersionCheck( + project_id=p2.id, + version=version, + city=loc["city"], + region=loc["region"], + country=loc["country"], + country_code=loc["country_code"], + latitude=loc["latitude"], + longitude=loc["longitude"], + is_ci=False, + time_bucket=bucket_time, + count=count_val // 2 + 1, + ) + ) + + session.add_all(checks) + await session.commit() + return {"p1": p1, "p2": p2} + + +@pytest.fixture +def app(session_factory, seed_data): + """Build the FastAPI app wired to the test DB session.""" + application = create_app() + application.state.allowlist = [ + {"owner": "nipy", "repo": "nipype"}, + {"owner": "nipy", "repo": "nibabel"}, + ] + application.state.settings = MagicMock( + GITHUB_TOKEN=None, + CACHE_TTL_SECONDS=3600, + TIME_BUCKET_HOURS=1, + MAXMIND_DB_PATH="/nonexistent", + ) + application.state.geolocator = None + application.state.http_client = httpx.AsyncClient() + + async def _override_get_db(): + async with session_factory() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + + application.dependency_overrides[get_db] = _override_get_db + return application + + +# --------------------------------------------------------------------------- +# HTML page tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_dashboard_index_returns_html(app): + """GET /dashboard/ returns 200 with HTML content.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get("/dashboard/") + + assert resp.status_code == 200 + assert "text/html" in resp.headers["content-type"] + assert "nipy/nipype" in resp.text + assert "nipy/nibabel" in resp.text + assert " 0 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_api_stats_aggregated_data(app): + """Stats API returns correctly aggregated data across 14 months.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get( + "/dashboard/api/stats/nipy/nipype", + params={"from": "2025-01-01", "to": "2026-12-31"}, + ) + + assert resp.status_code == 200 + body = resp.json() + + assert body["project"] == "nipy/nipype" + assert body["total_checks"] > 0 + + # Should have 3 versions + assert len(body["by_version"]) == 3 + version_names = {v["version"] for v in body["by_version"]} + assert version_names == {"1.8.0", "1.8.6", "1.9.0"} + + # Should have 4 locations + assert len(body["by_location"]) == 4 + countries = {loc["country_code"] for loc in body["by_location"]} + assert countries == {"US", "GB", "DE", "JP"} + + # Timeline should have entries + assert len(body["timeline"]) > 0 + + # Verify counts sum correctly + version_total = sum(v["count"] for v in body["by_version"]) + assert version_total == body["total_checks"] + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_api_geo_returns_features(app): + """GeoJSON API returns features with valid coordinates.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get( + "/dashboard/api/geo/nipy/nipype", + params={"from": "2025-01-01", "to": "2026-12-31"}, + ) + + assert resp.status_code == 200 + body = resp.json() + + assert body["type"] == "FeatureCollection" + assert len(body["features"]) == 4 # 4 unique locations + + for feature in body["features"]: + coords = feature["geometry"]["coordinates"] + lon, lat = coords + assert -180 <= lon <= 180 + assert -90 <= lat <= 90 + assert feature["properties"]["count"] > 0 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_api_stats_date_filtering(app): + """Stats API correctly filters by date range.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + # Query only January 2025 + resp = await client.get( + "/dashboard/api/stats/nipy/nipype", + params={"from": "2025-01-01", "to": "2025-01-31"}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["total_checks"] > 0 + + # Query a range with no data + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get( + "/dashboard/api/stats/nipy/nipype", + params={"from": "2020-01-01", "to": "2020-12-31"}, + ) + + assert resp.status_code == 200 + body = resp.json() + assert body["total_checks"] == 0 + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_api_stats_granularity_monthly(app): + """Monthly granularity groups timeline data correctly.""" + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get( + "/dashboard/api/stats/nipy/nipype", + params={ + "from": "2025-01-01", + "to": "2026-12-31", + "granularity": "monthly", + }, + ) + + assert resp.status_code == 200 + body = resp.json() + + # With monthly granularity, we should have <=14 timeline entries + assert len(body["timeline"]) <= 14 + for entry in body["timeline"]: + # Monthly format: YYYY-MM + assert len(entry["period"]) == 7 + assert "-" in entry["period"] diff --git a/server/tests/integration/test_migration.py b/server/tests/integration/test_migration.py new file mode 100644 index 0000000..d96a493 --- /dev/null +++ b/server/tests/integration/test_migration.py @@ -0,0 +1,252 @@ +"""Integration tests for the MongoDB-to-PostgreSQL migration tool. + +Tests exercise migration functions with mocked MongoDB and a real +in-memory SQLite database standing in for PostgreSQL. Marked with +``pytest.mark.integration``. +""" + +from __future__ import annotations + +import json +import logging +from pathlib import Path +from unittest.mock import MagicMock + +import pytest +from sqlalchemy import create_engine, select +from sqlalchemy.orm import Session, sessionmaker + +from etelemetry_server.models import Base, Project, VersionCheck + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +FIXTURES_DIR = Path(__file__).resolve().parent.parent / "fixtures" + + +@pytest.fixture +def mongo_sample_data(): + """Load the sample MongoDB fixture data.""" + with open(FIXTURES_DIR / "mongo_sample.json") as f: + return json.load(f) + + +@pytest.fixture +def engine(): + """Create an in-memory SQLite engine with the schema.""" + eng = create_engine("sqlite:///:memory:", echo=False) + Base.metadata.create_all(eng) + yield eng + eng.dispose() + + +@pytest.fixture +def session_factory(engine): + return sessionmaker(bind=engine, expire_on_commit=False) + + +@pytest.fixture +def mock_mongo_client(mongo_sample_data): + """Build a mock pymongo.MongoClient that returns fixture data.""" + requests_cursor = MagicMock() + requests_cursor.find.return_value = iter(mongo_sample_data["requests"]) + requests_cursor.count_documents.return_value = len(mongo_sample_data["requests"]) + + geo_cursor = MagicMock() + geo_cursor.find.return_value = iter(mongo_sample_data["geo"]) + geo_cursor.count_documents.return_value = len(mongo_sample_data["geo"]) + + db = MagicMock() + db.__getitem__ = lambda self, name: { + "requests": requests_cursor, + "geo": geo_cursor, + }[name] + + client = MagicMock() + client.__getitem__ = lambda self, name: db + return client + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +@pytest.mark.integration +class TestMigrationImport: + """Tests for the core migrate_records function.""" + + def test_valid_records_imported( + self, mock_mongo_client, engine, session_factory + ) -> None: + """All valid records are imported into PostgreSQL.""" + from tools.migrate import migrate_records + + migrate_records(mock_mongo_client, engine, batch_size=100) + + with Session(engine) as session: + projects = session.execute(select(Project)).scalars().all() + checks = session.execute(select(VersionCheck)).scalars().all() + + # Two unique owner/repo pairs: nipy/nipype and poldracklab/fmriprep + project_pairs = {(p.owner, p.repo) for p in projects} + assert ("nipy", "nipype") in project_pairs + assert ("poldracklab", "fmriprep") in project_pairs + assert len(project_pairs) == 2 + + # Should have version_check rows for valid records + assert len(checks) > 0 + + def test_no_ip_fields_in_output( + self, mock_mongo_client, engine, session_factory + ) -> None: + """No IP address fields are stored in PostgreSQL.""" + from tools.migrate import migrate_records + + migrate_records(mock_mongo_client, engine, batch_size=100) + + with Session(engine) as session: + checks = session.execute(select(VersionCheck)).scalars().all() + + for vc in checks: + assert not hasattr(vc, "remote_addr") + assert not hasattr(vc, "ip") + assert not hasattr(vc, "ip_address") + + def test_duplicate_handling( + self, mock_mongo_client, engine, session_factory + ) -> None: + """Duplicate content-addressed records are collapsed (count incremented). + + In the fixture, records aaa111 and aaa555 share the same + owner/repo/version/is_ci/time_bucket(hour)/geo, so they should + produce a single row with count >= 2. Record aaa333 has the same + owner/repo/version but a *different* hour-truncated time_bucket + (10:15 truncates to 10:00, same as 10:30 -- actually same hour), + so it also deduplicates with aaa111. + """ + from tools.migrate import migrate_records + + migrate_records(mock_mongo_client, engine, batch_size=100) + + with Session(engine) as session: + checks = session.execute(select(VersionCheck)).scalars().all() + + # Find checks for nipy/nipype v1.8.6 non-CI with Cambridge geo + nipype_checks = [ + c + for c in checks + if c.version == "1.8.6" and c.city == "Cambridge" and not c.is_ci + ] + # All three (aaa111, aaa333, aaa555) share the same hour bucket and geo + # so they should be one row with count >= 3 + assert len(nipype_checks) == 1 + assert nipype_checks[0].count >= 3 + + def test_malformed_records_skipped( + self, mock_mongo_client, engine, session_factory, caplog + ) -> None: + """Malformed records (missing owner, repo, or version) are skipped with a warning.""" + from tools.migrate import migrate_records + + with caplog.at_level(logging.WARNING): + stats = migrate_records(mock_mongo_client, engine, batch_size=100) + + # 3 malformed records in our fixture + assert stats["skipped"] >= 3 + + # Warnings should have been logged + warning_messages = [r.message for r in caplog.records if r.levelno >= logging.WARNING] + assert len(warning_messages) >= 3 + + def test_missing_geo_uses_defaults( + self, mock_mongo_client, engine, session_factory + ) -> None: + """Records without geo data get 'unknown' defaults for location fields.""" + from tools.migrate import migrate_records + + migrate_records(mock_mongo_client, engine, batch_size=100) + + with Session(engine) as session: + checks = session.execute(select(VersionCheck)).scalars().all() + + # poldracklab/fmriprep record has IP 192.0.2.50 which has no geo entry + fmriprep_checks = [c for c in checks if c.version == "23.1.0"] + assert len(fmriprep_checks) == 1 + assert fmriprep_checks[0].city == "unknown" + assert fmriprep_checks[0].region == "unknown" + assert fmriprep_checks[0].country == "unknown" + + +@pytest.mark.integration +class TestMigrationVerify: + """Tests for the --verify functionality.""" + + def test_verify_counts( + self, mock_mongo_client, engine, session_factory + ) -> None: + """Verify mode compares MongoDB vs PostgreSQL record counts.""" + from tools.migrate import migrate_records, verify_migration + + migrate_records(mock_mongo_client, engine, batch_size=100) + result = verify_migration(mock_mongo_client, engine) + + assert "mongo_requests" in result + assert "pg_version_checks" in result + assert "pg_total_count" in result + # The PG total count should match the number of valid mongo requests + assert result["pg_total_count"] >= 5 # 5 valid records in fixture + + +@pytest.mark.integration +class TestMigrationIdempotency: + """Running the migration twice should not create duplicate rows.""" + + def test_double_run_no_extra_rows( + self, mongo_sample_data, engine, session_factory + ) -> None: + """Running migration twice yields the same row count, with increased counts.""" + from tools.migrate import migrate_records + + # Build fresh mock each time since iterators are consumed + def make_mock(): + requests_cursor = MagicMock() + requests_cursor.find.return_value = iter(mongo_sample_data["requests"]) + requests_cursor.count_documents.return_value = len( + mongo_sample_data["requests"] + ) + geo_cursor = MagicMock() + geo_cursor.find.return_value = iter(mongo_sample_data["geo"]) + geo_cursor.count_documents.return_value = len(mongo_sample_data["geo"]) + db = MagicMock() + db.__getitem__ = lambda self, name: { + "requests": requests_cursor, + "geo": geo_cursor, + }[name] + client = MagicMock() + client.__getitem__ = lambda self, name: db + return client + + migrate_records(make_mock(), engine, batch_size=100) + + with Session(engine) as session: + checks_first = session.execute(select(VersionCheck)).scalars().all() + row_count_first = len(checks_first) + + migrate_records(make_mock(), engine, batch_size=100) + + with Session(engine) as session: + checks_second = session.execute(select(VersionCheck)).scalars().all() + row_count_second = len(checks_second) + + # Same number of rows + assert row_count_first == row_count_second + + # But counts have doubled + for vc in checks_second: + matching_first = [ + c for c in checks_first if c.id == vc.id + ] + if matching_first: + assert vc.count >= matching_first[0].count diff --git a/server/tests/integration/test_version_check.py b/server/tests/integration/test_version_check.py new file mode 100644 index 0000000..7c52809 --- /dev/null +++ b/server/tests/integration/test_version_check.py @@ -0,0 +1,155 @@ +"""Integration tests for the version-check flow. + +These tests exercise the full request path with a real (test) database. +Mark with ``pytest.mark.integration`` so they can be selected/skipped via +``pytest -m integration``. +""" + +from __future__ import annotations + +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest +import pytest_asyncio +from sqlalchemy import select +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +from etelemetry_server.app import create_app +from etelemetry_server.models import Base, Project, VersionCheck + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +TEST_DATABASE_URL = "sqlite+aiosqlite:///:memory:" + + +@pytest_asyncio.fixture +async def engine(): + eng = create_async_engine(TEST_DATABASE_URL, echo=False) + async with eng.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield eng + await eng.dispose() + + +@pytest_asyncio.fixture +async def session_factory(engine): + return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + +@pytest_asyncio.fixture +async def seed_project(session_factory): + """Seed a test project in the DB and return it.""" + async with session_factory() as session: + project = Project(owner="testorg", repo="testrepo", active=True) + session.add(project) + await session.commit() + await session.refresh(project) + return project + + +@pytest.fixture +def app(session_factory, seed_project): + """Build the FastAPI app wired to the test DB session.""" + application = create_app() + + application.state.allowlist = [ + {"owner": "testorg", "repo": "testrepo"}, + ] + application.state.settings = MagicMock( + GITHUB_TOKEN=None, + CACHE_TTL_SECONDS=3600, + TIME_BUCKET_HOURS=1, + MAXMIND_DB_PATH="/nonexistent", + ) + application.state.geolocator = None + application.state.http_client = httpx.AsyncClient() + + # Override the get_db dependency to use our test session factory + from etelemetry_server.db import get_db + + async def _override_get_db(): + async with session_factory() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + + application.dependency_overrides[get_db] = _override_get_db + return application + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_version_check_records_geo_no_ip(app, session_factory, seed_project): + """A version-check request stores geolocation but never stores IP.""" + with patch( + "etelemetry_server.routes.projects.VersionChecker" + ) as MockChecker: + instance = MockChecker.return_value + instance.get_version_info = AsyncMock( + return_value={"version": "2.0.0", "bad_versions": []} + ) + + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get( + "/projects/testorg/testrepo", params={"v": "1.0.0"} + ) + + assert resp.status_code == 200 + + # Verify DB record + async with session_factory() as session: + result = await session.execute(select(VersionCheck)) + records = list(result.scalars().all()) + + assert len(records) >= 1 + record = records[0] + + # Geolocation fields present + assert hasattr(record, "city") + assert hasattr(record, "region") + assert hasattr(record, "country") + assert hasattr(record, "country_code") + assert hasattr(record, "latitude") + assert hasattr(record, "longitude") + + # No IP column on the model + assert not hasattr(record, "ip") + assert not hasattr(record, "ip_address") + + # Check recorded values + assert record.project_id == seed_project.id + assert record.version == "1.0.0" + + +@pytest.mark.integration +@pytest.mark.asyncio +async def test_version_check_response_shape(app, session_factory, seed_project): + """The endpoint returns the correct JSON shape.""" + with patch( + "etelemetry_server.routes.projects.VersionChecker" + ) as MockChecker: + instance = MockChecker.return_value + instance.get_version_info = AsyncMock( + return_value={"version": "2.0.0", "bad_versions": ["0.9.0"]} + ) + + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), base_url="http://test" + ) as client: + resp = await client.get("/projects/testorg/testrepo") + + body = resp.json() + assert body["version"] == "2.0.0" + assert body["bad_versions"] == ["0.9.0"] diff --git a/server/tests/unit/__init__.py b/server/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/tests/unit/test_allowlist.py b/server/tests/unit/test_allowlist.py new file mode 100644 index 0000000..8f9f69c --- /dev/null +++ b/server/tests/unit/test_allowlist.py @@ -0,0 +1,114 @@ +"""Tests for etelemetry_server.allowlist.""" + +from __future__ import annotations + +from pathlib import Path +from unittest.mock import AsyncMock, MagicMock + +import pytest +import yaml + +from etelemetry_server.allowlist import load_allowlist, sync_allowlist_to_db +from etelemetry_server.models import Project + + +class TestLoadAllowlist: + def test_load_valid_yaml(self, tmp_path: Path) -> None: + data = [ + {"owner": "nipy", "repo": "nipype"}, + {"owner": "poldracklab", "repo": "fmriprep"}, + ] + f = tmp_path / "allowlist.yml" + f.write_text(yaml.dump(data)) + + result = load_allowlist(str(f)) + assert len(result) == 2 + assert result[0] == {"owner": "nipy", "repo": "nipype"} + assert result[1] == {"owner": "poldracklab", "repo": "fmriprep"} + + def test_load_nonexistent_file(self) -> None: + result = load_allowlist("/nonexistent/path.yml") + assert result == [] + + def test_load_invalid_yaml_not_a_list(self, tmp_path: Path) -> None: + f = tmp_path / "bad.yml" + f.write_text("key: value") + result = load_allowlist(str(f)) + assert result == [] + + def test_load_skips_malformed_entries(self, tmp_path: Path) -> None: + data = [ + {"owner": "nipy", "repo": "nipype"}, + {"something": "else"}, + "just a string", + ] + f = tmp_path / "allowlist.yml" + f.write_text(yaml.dump(data)) + + result = load_allowlist(str(f)) + assert len(result) == 1 + assert result[0] == {"owner": "nipy", "repo": "nipype"} + + def test_load_empty_file(self, tmp_path: Path) -> None: + f = tmp_path / "empty.yml" + f.write_text("") + result = load_allowlist(str(f)) + assert result == [] + + +class TestSyncAllowlistToDb: + @pytest.fixture + def mock_session(self) -> AsyncMock: + session = AsyncMock() + return session + + @pytest.mark.asyncio + async def test_inserts_new_projects(self, mock_session: AsyncMock) -> None: + # No existing projects + result_mock = MagicMock() + result_mock.scalars.return_value.all.return_value = [] + mock_session.execute.return_value = result_mock + + entries = [{"owner": "nipy", "repo": "nipype"}] + await sync_allowlist_to_db(mock_session, entries) + + mock_session.add.assert_called_once() + added = mock_session.add.call_args[0][0] + assert isinstance(added, Project) + assert added.owner == "nipy" + assert added.repo == "nipype" + assert added.active is True + + @pytest.mark.asyncio + async def test_deactivates_removed_projects( + self, mock_session: AsyncMock + ) -> None: + existing = Project(owner="old", repo="project") + existing.active = True + + result_mock = MagicMock() + result_mock.scalars.return_value.all.return_value = [existing] + mock_session.execute.return_value = result_mock + + # Empty allowlist — should deactivate existing + await sync_allowlist_to_db(mock_session, []) + + assert existing.active is False + + @pytest.mark.asyncio + async def test_reactivates_returning_projects( + self, mock_session: AsyncMock + ) -> None: + existing = Project(owner="nipy", repo="nipype") + existing.active = False + + result_mock = MagicMock() + result_mock.scalars.return_value.all.return_value = [existing] + mock_session.execute.return_value = result_mock + + entries = [{"owner": "nipy", "repo": "nipype"}] + await sync_allowlist_to_db(mock_session, entries) + + assert existing.active is True + # Should not add a new project + mock_session.add.assert_not_called() diff --git a/server/tests/unit/test_models.py b/server/tests/unit/test_models.py new file mode 100644 index 0000000..dcfb847 --- /dev/null +++ b/server/tests/unit/test_models.py @@ -0,0 +1,122 @@ +"""Tests for etelemetry_server.models.""" + +import datetime + +from sqlalchemy import inspect + +from etelemetry_server.models import Base, Project, UsageAggregate, VersionCheck + + +class TestProjectModel: + def test_tablename(self) -> None: + assert Project.__tablename__ == "projects" + + def test_columns_exist(self) -> None: + cols = {c.name for c in inspect(Project).columns} + expected = { + "id", + "owner", + "repo", + "latest_version", + "bad_versions", + "cache_expires_at", + "active", + "created_at", + "updated_at", + } + assert expected.issubset(cols) + + def test_unique_constraint_on_owner_repo(self) -> None: + table = Project.__table__ + uq_names = [ + c.name + for c in table.constraints + if hasattr(c, "columns") + and {col.name for col in c.columns} == {"owner", "repo"} + ] + assert len(uq_names) >= 1 + + def test_instantiation(self) -> None: + p = Project(owner="nipy", repo="nipype") + assert p.owner == "nipy" + assert p.repo == "nipype" + + +class TestVersionCheckModel: + def test_tablename(self) -> None: + assert VersionCheck.__tablename__ == "version_checks" + + def test_columns_exist(self) -> None: + cols = {c.name for c in inspect(VersionCheck).columns} + expected = { + "id", + "project_id", + "version", + "city", + "region", + "country", + "country_code", + "latitude", + "longitude", + "is_ci", + "time_bucket", + "count", + "created_at", + } + assert expected.issubset(cols) + + def test_content_address_unique_constraint(self) -> None: + table = VersionCheck.__table__ + uq = [ + c + for c in table.constraints + if hasattr(c, "name") + and getattr(c, "name", None) == "uq_version_check_content_address" + ] + assert len(uq) == 1 + + def test_instantiation(self) -> None: + vc = VersionCheck( + project_id=1, + version="1.0.0", + time_bucket=datetime.datetime.now(tz=datetime.timezone.utc), + ) + assert vc.version == "1.0.0" + + +class TestUsageAggregateModel: + def test_tablename(self) -> None: + assert UsageAggregate.__tablename__ == "usage_aggregates" + + def test_columns_exist(self) -> None: + cols = {c.name for c in inspect(UsageAggregate).columns} + expected = { + "id", + "project_id", + "version", + "country_code", + "region", + "granularity", + "period_start", + "total_count", + "unique_locations", + "ci_count", + "created_at", + } + assert expected.issubset(cols) + + def test_instantiation(self) -> None: + ua = UsageAggregate( + project_id=1, + granularity="daily", + period_start=datetime.date(2026, 1, 1), + ) + assert ua.granularity == "daily" + + +class TestBaseMetadata: + def test_all_tables_registered(self) -> None: + table_names = set(Base.metadata.tables.keys()) + assert "projects" in table_names + assert "version_checks" in table_names + assert "usage_aggregates" in table_names diff --git a/server/tests/unit/test_settings.py b/server/tests/unit/test_settings.py new file mode 100644 index 0000000..dc02bb6 --- /dev/null +++ b/server/tests/unit/test_settings.py @@ -0,0 +1,81 @@ +"""Tests for etelemetry_server.settings.""" + +import os +from unittest.mock import patch + +from etelemetry_server.settings import Settings + + +class TestSettingsDefaults: + """Verify default values when no environment variables are set.""" + + def test_default_database_url(self) -> None: + with patch.dict(os.environ, {}, clear=True): + s = Settings() + assert s.DATABASE_URL == "postgresql+asyncpg://localhost:5432/etelemetry" + + def test_default_maxmind_db_path(self) -> None: + with patch.dict(os.environ, {}, clear=True): + s = Settings() + assert s.MAXMIND_DB_PATH == "/data/GeoLite2-City.mmdb" + + def test_default_github_token_is_none(self) -> None: + with patch.dict(os.environ, {}, clear=True): + s = Settings() + assert s.GITHUB_TOKEN is None + + def test_default_cache_ttl(self) -> None: + with patch.dict(os.environ, {}, clear=True): + s = Settings() + assert s.CACHE_TTL_SECONDS == 21600 + + def test_default_allowlist_path(self) -> None: + with patch.dict(os.environ, {}, clear=True): + s = Settings() + assert s.ALLOWLIST_PATH == "allowlist.yml" + + def test_default_time_bucket_hours(self) -> None: + with patch.dict(os.environ, {}, clear=True): + s = Settings() + assert s.TIME_BUCKET_HOURS == 1 + + def test_default_server_host(self) -> None: + with patch.dict(os.environ, {}, clear=True): + s = Settings() + assert s.SERVER_HOST == "0.0.0.0" + + def test_default_server_port(self) -> None: + with patch.dict(os.environ, {}, clear=True): + s = Settings() + assert s.SERVER_PORT == 8000 + + +class TestSettingsFromEnv: + """Verify settings are loaded from environment variables.""" + + def test_database_url_from_env(self) -> None: + with patch.dict( + os.environ, {"DATABASE_URL": "postgresql+asyncpg://db:5432/test"} + ): + s = Settings() + assert s.DATABASE_URL == "postgresql+asyncpg://db:5432/test" + + def test_github_token_from_env(self) -> None: + with patch.dict(os.environ, {"GITHUB_TOKEN": "ghp_abc123"}): + s = Settings() + assert s.GITHUB_TOKEN == "ghp_abc123" + + def test_cache_ttl_from_env(self) -> None: + with patch.dict(os.environ, {"CACHE_TTL_SECONDS": "3600"}): + s = Settings() + assert s.CACHE_TTL_SECONDS == 3600 + + def test_server_port_from_env(self) -> None: + with patch.dict(os.environ, {"SERVER_PORT": "9000"}): + s = Settings() + assert s.SERVER_PORT == 9000 + + def test_time_bucket_hours_from_env(self) -> None: + with patch.dict(os.environ, {"TIME_BUCKET_HOURS": "6"}): + s = Settings() + assert s.TIME_BUCKET_HOURS == 6 diff --git a/specs/001-stack-migration-refactor/checklists/requirements.md b/specs/001-stack-migration-refactor/checklists/requirements.md new file mode 100644 index 0000000..5625771 --- /dev/null +++ b/specs/001-stack-migration-refactor/checklists/requirements.md @@ -0,0 +1,38 @@ +# Specification Quality Checklist: etelemetry Stack Migration & Monorepo Refactor + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-03-22 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- Assumptions section documents reasonable defaults (GitHub as source, + local geolocation DB, no dashboard auth for aggregates). +- SC-003 references "50% less disk space" which is measurable via + the migration tool comparing before/after sizes. +- All items pass. Spec is ready for `/speckit.clarify` or `/speckit.plan`. diff --git a/specs/001-stack-migration-refactor/contracts/api.md b/specs/001-stack-migration-refactor/contracts/api.md new file mode 100644 index 0000000..08413fa --- /dev/null +++ b/specs/001-stack-migration-refactor/contracts/api.md @@ -0,0 +1,174 @@ +# API Contracts: etelemetry + +**Date**: 2026-03-22 +**Branch**: `001-stack-migration-refactor` + +## Server API (HTTP) + +Base URL: `https://{host}/` + +All responses use `Content-Type: application/json` unless noted. +Compression: gzip/brotli negotiated via `Accept-Encoding`. + +### GET / + +Health check and server info. + +**Response** `200 OK`: +```json +{ + "name": "etelemetry", + "version": "2.0.0" +} +``` + +### GET /projects/{owner}/{repo} + +Version check endpoint. Records usage (content-addressed dedup). + +**Path parameters**: +- `owner` (string): GitHub org or user +- `repo` (string): Repository name + +**Query parameters** (optional, sent by client automatically): +- `ci` (boolean): Whether client is in CI environment +- `v` (string): Client's current version + +**Response** `200 OK` (project on allowlist, version resolved): +```json +{ + "version": "1.4.2", + "bad_versions": ["1.1.0", "1.2.3"] +} +``` + +**Response** `200 OK` (project on allowlist, version unknown/cached stale): +```json +{ + "version": null, + "bad_versions": [] +} +``` + +**Response** `404 Not Found` (project not on allowlist): +```json +{ + "error": "project not tracked" +} +``` + +**Response** `400 Bad Request` (malformed identifier): +```json +{ + "error": "invalid project identifier" +} +``` + +### GET /dashboard/ + +Web dashboard (HTML). Serves the main dashboard page. + +**Response** `200 OK`: HTML page with embedded htmx + Leaflet.js. + +### GET /dashboard/api/stats/{owner}/{repo} + +Dashboard data endpoint for a specific project. + +**Query parameters**: +- `from` (date, ISO 8601): Start of time range +- `to` (date, ISO 8601): End of time range +- `granularity` (string): `daily`, `weekly`, `monthly` (default: auto) + +**Response** `200 OK`: +```json +{ + "project": "nipy/nipype", + "period": {"from": "2026-01-01", "to": "2026-03-22"}, + "total_checks": 142857, + "by_version": [ + {"version": "1.8.6", "count": 80000}, + {"version": "1.9.0", "count": 62857} + ], + "by_location": [ + { + "country": "United States", + "country_code": "US", + "region": "Massachusetts", + "city": "Cambridge", + "lat": 42.36, + "lon": -71.06, + "count": 15000 + } + ], + "timeline": [ + {"period": "2026-W01", "count": 10204} + ] +} +``` + +### GET /dashboard/api/geo/{owner}/{repo} + +GeoJSON endpoint for map visualization. + +**Query parameters**: Same as stats endpoint. + +**Response** `200 OK`: GeoJSON FeatureCollection with point features +per location, `count` in properties. Used by Leaflet.js for map rendering. + +### GET /dashboard/api/projects + +List all tracked projects with summary counts. + +**Response** `200 OK`: +```json +{ + "projects": [ + {"owner": "nipy", "repo": "nipype", "total_checks": 500000}, + {"owner": "nipy", "repo": "nibabel", "total_checks": 300000} + ] +} +``` + +## Client Library API (Python) + +Package: `etelemetry` (pip installable) + +### Backward-compatible public API + +```python +# Unchanged from current client +etelemetry.get_project(repo: str, **kwargs) -> dict +etelemetry.check_available_version( + project: str, + version: str, + lgr: logging.Logger | None = None, + raise_exception: bool = False, +) -> dict | None + +class etelemetry.BadVersionError(RuntimeError): ... +``` + +### New: configurable server URL + +Precedence (highest to lowest): +1. `server_url` parameter in `get_project()` / `check_available_version()` +2. `ETELEMETRY_URL` environment variable +3. Default URL (configurable at package level) + +```python +# Via function parameter +etelemetry.get_project("nipy/nipype", server_url="https://my-instance.org/") + +# Via environment variable +# export ETELEMETRY_URL=https://my-instance.org/ + +# Via package-level default (for library authors) +etelemetry.config.default_url = "https://my-instance.org/" +``` + +### Disable telemetry + +```python +# Environment variable (unchanged) +# export NO_ET=1 +``` diff --git a/specs/001-stack-migration-refactor/data-model.md b/specs/001-stack-migration-refactor/data-model.md new file mode 100644 index 0000000..b10c3b9 --- /dev/null +++ b/specs/001-stack-migration-refactor/data-model.md @@ -0,0 +1,136 @@ +# Data Model: etelemetry + +**Date**: 2026-03-22 +**Branch**: `001-stack-migration-refactor` +**Storage**: PostgreSQL 16 + +## Entity Relationship + +```text +AllowlistEntry 1──* Project 1──* VersionCheck + │ + └── GeoLocation (embedded fields) + +VersionCheck ──(aggregated into)──> UsageAggregate +``` + +## Tables + +### projects + +Allowlisted software projects tracked by etelemetry. + +| Column | Type | Constraints | Notes | +|--------|------|-------------|-------| +| id | SERIAL | PK | Internal ID | +| owner | VARCHAR(255) | NOT NULL | GitHub org/user | +| repo | VARCHAR(255) | NOT NULL | Repository name | +| latest_version | VARCHAR(100) | | Cached latest release | +| bad_versions | JSONB | DEFAULT '[]' | From .et file | +| cache_expires_at | TIMESTAMPTZ | | When to re-fetch | +| active | BOOLEAN | DEFAULT true | Allowlist flag | +| created_at | TIMESTAMPTZ | DEFAULT now() | | +| updated_at | TIMESTAMPTZ | DEFAULT now() | | + +**Unique**: `(owner, repo)` +**Index**: `(owner, repo)` — primary lookup path + +### version_checks + +Content-addressed deduplicated usage records. Composite key ensures +one row per unique combination per time-bucket. + +| Column | Type | Constraints | Notes | +|--------|------|-------------|-------| +| id | BIGSERIAL | PK | Internal ID | +| project_id | INTEGER | FK → projects.id, NOT NULL | | +| version | VARCHAR(100) | NOT NULL | Client's reported version | +| city | VARCHAR(255) | | Resolved from IP at request time | +| region | VARCHAR(255) | | State/province | +| country | VARCHAR(100) | | Country name | +| country_code | CHAR(2) | | ISO 3166-1 alpha-2 | +| latitude | FLOAT | | Approximate | +| longitude | FLOAT | | Approximate | +| is_ci | BOOLEAN | DEFAULT false | CI environment flag | +| time_bucket | TIMESTAMPTZ | NOT NULL | Truncated to hour | +| count | INTEGER | DEFAULT 1 | Ping count in bucket | +| created_at | TIMESTAMPTZ | DEFAULT now() | | + +**Unique**: `(project_id, version, city, region, country_code, is_ci, time_bucket)` +**Index**: `(project_id, time_bucket)` — dashboard queries +**Index**: `(time_bucket)` — aggregation job + +**Content-address key**: The unique constraint is the content address. +On conflict: `SET count = count + 1`. + +**No IP column**: IP addresses are never stored. Geolocation fields are +resolved transiently at request time and written directly. + +### usage_aggregates + +Pre-computed summaries for dashboard performance. Tiered granularity. + +| Column | Type | Constraints | Notes | +|--------|------|-------------|-------| +| id | BIGSERIAL | PK | | +| project_id | INTEGER | FK → projects.id, NOT NULL | | +| version | VARCHAR(100) | | NULL = all versions | +| country_code | CHAR(2) | | NULL = all countries | +| region | VARCHAR(255) | | NULL = all regions | +| granularity | VARCHAR(10) | NOT NULL | 'daily', 'weekly', 'monthly' | +| period_start | DATE | NOT NULL | Start of period | +| total_count | BIGINT | DEFAULT 0 | Sum of counts | +| unique_locations | INTEGER | DEFAULT 0 | Distinct city+region combos | +| ci_count | BIGINT | DEFAULT 0 | Subset from CI | +| created_at | TIMESTAMPTZ | DEFAULT now() | | + +**Unique**: `(project_id, version, country_code, region, granularity, period_start)` +**Index**: `(project_id, granularity, period_start)` — dashboard range queries + +### Aggregation Tiers (configurable defaults) + +| Data age | Granularity | Source | +|----------|-------------|--------| +| < 30 days | daily | Computed from version_checks | +| 30 days – 1 year | weekly | Rolled up from daily | +| > 1 year | monthly | Rolled up from weekly | + +A scheduled job (cron or background task) computes aggregates and +optionally compacts old version_checks rows into aggregates. + +## State Transitions + +### Project cache lifecycle + +```text +STALE → FETCHING → CACHED (expires in 6h) → STALE + ↘ FETCH_FAILED (serve stale, retry later) +``` + +### Version check request flow + +```text +Request arrives + → Check allowlist (reject if not listed) + → Resolve geolocation from IP (transient, not stored) + → Discard IP + → Upsert version_check (content-addressed) + → Return cached project version + bad_versions +``` + +## Migration Mapping (MongoDB → PostgreSQL) + +| MongoDB collection | MongoDB field | PostgreSQL table | PostgreSQL column | +|-------------------|---------------|-----------------|-------------------| +| requests | owner | projects | owner | +| requests | repository | projects | repo | +| requests | version | version_checks | version | +| requests | is_ci | version_checks | is_ci | +| requests | access_time | version_checks | time_bucket (truncated) | +| requests | remote_addr | — | DROPPED | +| geo | city | version_checks | city | +| geo | region_name | version_checks | region | +| geo | country_name | version_checks | country | +| geo | latitude | version_checks | latitude | +| geo | longitude | version_checks | longitude | +| geo | remote_addr | — | DROPPED | diff --git a/specs/001-stack-migration-refactor/plan.md b/specs/001-stack-migration-refactor/plan.md new file mode 100644 index 0000000..ca516f3 --- /dev/null +++ b/specs/001-stack-migration-refactor/plan.md @@ -0,0 +1,139 @@ +# Implementation Plan: etelemetry Stack Migration & Monorepo Refactor + +**Branch**: `001-stack-migration-refactor` | **Date**: 2026-03-22 | **Spec**: [spec.md](spec.md) +**Input**: Feature specification from `/specs/001-stack-migration-refactor/spec.md` + +## Summary + +Combine etelemetry-server and etelemetry-client into a single monorepo. +Replace MongoDB/Sanic with PostgreSQL/FastAPI. Add privacy-preserving +geolocation (MaxMind GeoLite2), content-addressed deduplication for ~1M +pings/week, a server-rendered web dashboard (htmx + Leaflet.js), configurable +server URLs, a repository allowlist, and a one-time MongoDB data migration +tool. Deploy via Docker Compose with GitHub Actions CI/CD for AWS. + +## Technical Context + +**Language/Version**: Python 3.11+ +**Primary Dependencies**: FastAPI, SQLAlchemy 2.0, asyncpg, httpx, geoip2, +Jinja2, htmx, Leaflet.js, starlette-compress, Alembic, Pydantic v2 +**Storage**: PostgreSQL 16 (via `postgres:16-alpine` Docker image) +**Testing**: pytest, pytest-asyncio, httpx (test client), testcontainers +**Target Platform**: Linux server (AWS EC2 / Docker), client on any OS +**Project Type**: Monorepo — Python client library + web service + dashboard +**Performance Goals**: ~1,650 req/min sustained, <2s version check, <10ms +geolocation, <3s dashboard load +**Constraints**: No IP persistence, WCAG 2.1 AA dashboard, backward- +compatible client API +**Scale/Scope**: ~1M pings/week, content-addressed dedup reduces storage +to ~thousands of unique rows/week + +## Constitution Check + +*GATE: Must pass before Phase 0 research. Re-check after Phase 1 design.* + +| Principle | Status | Evidence | +|-----------|--------|----------| +| I. Isolated Environments | PASS | uv for local dev; Docker for server/DB/geoupdate | +| II. Secret Safety | PASS | .env for secrets; MaxMind key, DB password via env vars; .env in .gitignore; GitHub Actions secrets for CI/CD | +| III. Git Discipline | PASS | Conventional commits; feature branch; CI required; subagent file locking | +| IV. Test-Driven | PASS | pytest with real PostgreSQL (testcontainers); end-to-end quickstart validation | +| V. Code Abstraction | PASS | Review after each phase; shared utilities extracted when duplicated | +| VI. Provenance | PASS | phase-log.md, rebuild-spec.md, descriptive commits | +| VII. Living Documentation | PASS | vision.md, constitution evolves, docs sync | +| VIII. Simplicity | PASS | FastAPI (not SPA), htmx (not React), PostgreSQL (not microservices), single Docker Compose | + +No violations. No complexity tracking needed. + +## Project Structure + +### Documentation (this feature) + +```text +specs/001-stack-migration-refactor/ +├── plan.md # This file +├── spec.md # Feature specification +├── research.md # Technology decisions +├── data-model.md # PostgreSQL schema +├── quickstart.md # Developer getting-started guide +├── contracts/ +│ └── api.md # HTTP API + Python client API contracts +├── checklists/ +│ └── requirements.md # Spec quality checklist +└── tasks.md # Task breakdown (created by /speckit.tasks) +``` + +### Source Code (repository root) + +```text +etelemetry/ +├── pyproject.toml # Client package: "etelemetry" +├── src/etelemetry/ # Client library +│ ├── __init__.py # Public API exports +│ ├── client.py # get_project, check_available_version +│ ├── config.py # URL resolution (env, param, default) +│ └── errors.py # BadVersionError +│ +├── server/ +│ ├── pyproject.toml # Server package: "etelemetry-server" +│ ├── src/etelemetry_server/ +│ │ ├── __init__.py +│ │ ├── app.py # FastAPI application factory +│ │ ├── settings.py # Pydantic settings (env-based config) +│ │ ├── models.py # SQLAlchemy ORM models +│ │ ├── db.py # Database engine, session management +│ │ ├── routes/ +│ │ │ ├── projects.py # GET /projects/{owner}/{repo} +│ │ │ ├── health.py # GET / +│ │ │ └── dashboard.py # Dashboard HTML + API routes +│ │ ├── services/ +│ │ │ ├── version_checker.py # GitHub API + cache logic +│ │ │ ├── geolocation.py # MaxMind GeoLite2 lookup +│ │ │ ├── usage_recorder.py # Content-addressed upsert +│ │ │ └── aggregation.py # Tiered aggregation job +│ │ ├── allowlist.py # YAML allowlist loader + SIGHUP +│ │ └── dashboard/ +│ │ ├── templates/ # Jinja2 templates +│ │ └── static/ # CSS, JS (htmx, Leaflet) +│ ├── alembic/ # Database migrations +│ │ ├── alembic.ini +│ │ └── versions/ +│ └── tests/ +│ ├── unit/ +│ ├── integration/ +│ └── contract/ +│ +├── tools/ +│ ├── __init__.py +│ └── migrate.py # MongoDB → PostgreSQL migration +│ +├── deploy/ +│ ├── docker-compose.yml # PostgreSQL, server, nginx, geoipupdate +│ ├── Dockerfile # Server image +│ ├── nginx.conf # Reverse proxy (IP stripping in logs) +│ ├── allowlist.yml # Tracked repositories +│ ├── .env.example # Template for secrets +│ └── geoipupdate.conf # MaxMind update config +│ +├── .github/workflows/ +│ ├── ci.yml # Test + lint on PR +│ └── deploy.yml # Deploy to AWS on push +│ +├── docs/ +│ ├── vision.md +│ ├── phase-log.md +│ └── rebuild-spec.md +│ +└── tests/ # Client library tests + ├── unit/ + ├── integration/ + └── contract/ +``` + +**Structure Decision**: Monorepo with two installable packages. The root +`pyproject.toml` defines the client library (`etelemetry`), which downstream +users install via pip. The `server/pyproject.toml` defines the server +package (`etelemetry-server`), installed only for deployment. This keeps +the client lightweight (no server dependencies) while sharing the repository. +The `tools/` directory contains one-time scripts (migration). The `deploy/` +directory is self-contained for Docker Compose deployment. diff --git a/specs/001-stack-migration-refactor/quickstart.md b/specs/001-stack-migration-refactor/quickstart.md new file mode 100644 index 0000000..8328d96 --- /dev/null +++ b/specs/001-stack-migration-refactor/quickstart.md @@ -0,0 +1,123 @@ +# Quickstart: etelemetry + +## Prerequisites + +- Python 3.11+ +- uv (Python package manager) +- Docker and Docker Compose (for server) +- MaxMind GeoLite2 license key (free, from maxmind.com) + +## Client Library (for downstream package authors) + +```bash +# Install the client +uv pip install etelemetry + +# Or add to your pyproject.toml +# dependencies = ["etelemetry>=2.0"] +``` + +```python +import etelemetry + +# Check version (uses default server) +result = etelemetry.get_project("nipy/nipype") +print(result) +# {"version": "1.8.6", "bad_versions": ["1.1.0"]} + +# Full check with warnings +etelemetry.check_available_version("nipy/nipype", "1.5.0") +# WARNING: A newer version (1.8.6) is available... + +# Custom server URL +etelemetry.get_project("org/tool", server_url="https://my-et.example.com/") + +# Or via environment variable +# export ETELEMETRY_URL=https://my-et.example.com/ +``` + +## Server (local development) + +```bash +# Clone and set up +git clone https://github.com/sensein/etelemetry.git +cd etelemetry + +# Create environment +uv venv +uv pip install -e ".[dev]" +uv pip install -e "./server[dev]" + +# Copy environment template +cp deploy/.env.example deploy/.env +# Edit deploy/.env — set MAXMIND_LICENSE_KEY and POSTGRES_PASSWORD + +# Start services (PostgreSQL + GeoIP updater) +docker compose -f deploy/docker-compose.yml up -d postgres geoipupdate + +# Run database migrations +uv run alembic upgrade head + +# Start the server +uv run uvicorn etelemetry_server.app:app --reload --port 8000 + +# Visit dashboard at http://localhost:8000/dashboard/ +# API at http://localhost:8000/projects/nipy/nipype +``` + +## Server (production Docker Compose) + +```bash +# On your server +git clone https://github.com/sensein/etelemetry.git +cd etelemetry/deploy + +# Configure +cp .env.example .env +# Edit .env — set domain, secrets, MaxMind key + +# Edit allowlist.yml — add your tracked projects +# projects: +# - owner: nipy +# repo: nipype +# - owner: nipy +# repo: nibabel + +# Start everything +docker compose up -d + +# Verify +curl https://your-domain.com/projects/nipy/nipype +``` + +## Data Migration (one-time) + +```bash +# Export from existing MongoDB on EC2 +mongodump -d et --gzip --archive=et-backup.gz + +# Copy to new server, then run migration +uv run python -m tools.migrate \ + --mongo-uri mongodb://old-host:27017/et \ + --pg-uri postgresql://user:pass@localhost:5432/etelemetry + +# Verify migration +uv run python -m tools.migrate --verify +``` + +## Running Tests + +```bash +# Unit tests (no services needed) +uv run pytest tests/unit/ + +# Integration tests (requires PostgreSQL) +docker compose -f deploy/docker-compose.yml up -d postgres +uv run pytest tests/integration/ + +# Contract tests (API shape verification) +uv run pytest tests/contract/ + +# All tests +uv run pytest +``` diff --git a/specs/001-stack-migration-refactor/research.md b/specs/001-stack-migration-refactor/research.md new file mode 100644 index 0000000..80068cc --- /dev/null +++ b/specs/001-stack-migration-refactor/research.md @@ -0,0 +1,104 @@ +# Research: etelemetry Stack Migration + +**Date**: 2026-03-22 +**Branch**: `001-stack-migration-refactor` + +## 1. Web Framework + +**Decision**: FastAPI +**Rationale**: Largest community (4.5M+ daily downloads), automatic OpenAPI +docs, native async/await, Pydantic validation, first-class Jinja2 support +for the dashboard. Runs on Starlette — throughput is more than sufficient +for ~1,650 req/min sustained (SC-009). Single framework serves both the +API and dashboard. +**Alternatives considered**: +- Litestar: ~2x synthetic benchmarks but irrelevant at this scale; smaller + community, fewer integrations. +- Starlette: Lower-level; would require manual validation, serialization, + and OpenAPI generation. +- Sanic (current): Smaller ecosystem, no built-in OpenAPI, less momentum. + +## 2. Database + +**Decision**: PostgreSQL 16 +**Rationale**: Content-addressed dedup (FR-004) maps directly to +`INSERT ... ON CONFLICT DO UPDATE SET count = count + 1`, atomic under +MVCC with row-level locking. Multiple Uvicorn workers write concurrently +without coordination. Aggregation queries (SC-005) benefit from mature +query planner, partial indexes, and materialized views for tiered +aggregation (FR-017). Docker Compose integration is trivial +(`postgres:16-alpine`). Storage efficiency vs MongoDB easily meets SC-003. +**Alternatives considered**: +- SQLite (WAL): Single-writer limitation. Under concurrent Uvicorn workers, + would require application-level write serialization — added complexity + violating Principle VIII. +- DuckDB: Excellent analytics but explicitly single-writer. Not designed + for OLTP upsert workloads. + +## 3. Geolocation + +**Decision**: MaxMind GeoLite2 with `geoip2` Python library +**Rationale**: Industry standard for local IP geolocation. In-memory MMDB +lookups in microseconds (well under SC-002's 10ms target). Free with +registration; requires attribution and 30-day update cycle (automate via +`geoipupdate` in Docker entrypoint). Already assumed in spec. +**Alternatives considered**: +- IP2Location Lite: Lower accuracy for North American IPs, less mature + Python library. +- DB-IP Lite: Smaller community, fewer accuracy studies, no advantage. + +## 4. Dashboard + +**Decision**: Server-rendered — Jinja2 + htmx + Leaflet.js +**Rationale**: No Node.js build step; single Python language stack. +Server-rendered HTML is inherently accessible (FR-010/WCAG). htmx (14KB) +handles partial page updates; Leaflet.js (~40KB) provides interactive map +with drill-down via `L.geoJSON` click handlers. FastAPI serves everything. +**Alternatives considered**: +- React/Vue SPA: Introduces Node.js toolchain, separate frontend project, + state management. Doubles project complexity for a dashboard showing + aggregate statistics. Violates Principle VIII. + +## 5. HTTP Client (GitHub API) + +**Decision**: httpx +**Rationale**: Modern async/sync dual API, HTTP/2 support, connection +pooling, configurable timeouts. API mirrors `requests` for familiarity. +Volume is low (few requests per cache-miss, at most hourly). +**Alternatives considered**: +- aiohttp: ~2x faster but irrelevant for cached GitHub calls. Async-only + (no sync fallback for migration tool). Less intuitive API. + +## 6. Response Compaction + +**Decision**: HTTP compression (gzip/brotli) via `starlette-compress` +**Rationale**: Zero client-side changes — standard `Accept-Encoding` +negotiation handled transparently. Combined with minimal JSON (short field +names, `exclude_none=True` in Pydantic). Version-check payloads are small +(~100-200B) so per-request savings are modest, but dashboard pages benefit. +**Alternatives considered**: +- msgpack: Breaks backward compatibility (FR-008), requires client-side + decoder. Marginal savings over compressed JSON for small payloads. + +## 7. ORM / Database Access + +**Decision**: SQLAlchemy 2.0 with asyncpg driver +**Rationale**: Async support via `create_async_engine`, mature migration +tooling (Alembic), composable query building for aggregation queries. +asyncpg is the fastest PostgreSQL driver for Python. +**Alternatives considered**: +- Raw asyncpg: Faster but loses migration tooling, requires manual SQL + string management for complex aggregation queries. +- Tortoise ORM: Smaller community, less mature than SQLAlchemy 2.0. + +## Summary + +| Area | Decision | Key Reason | +|------|----------|------------| +| Web Framework | FastAPI | Community, OpenAPI, async, Jinja2 | +| Database | PostgreSQL 16 | MVCC upserts, aggregation, Docker | +| Geolocation | MaxMind GeoLite2 | Microsecond lookups, free, standard | +| Dashboard | Jinja2 + htmx + Leaflet.js | No build step, accessible, simple | +| HTTP Client | httpx | Modern async/sync, familiar API | +| Compression | gzip/brotli middleware | Standard HTTP, zero client changes | +| ORM | SQLAlchemy 2.0 + asyncpg | Async, Alembic migrations, mature | diff --git a/specs/001-stack-migration-refactor/spec.md b/specs/001-stack-migration-refactor/spec.md new file mode 100644 index 0000000..db49552 --- /dev/null +++ b/specs/001-stack-migration-refactor/spec.md @@ -0,0 +1,362 @@ +# Feature Specification: etelemetry Stack Migration & Monorepo Refactor + +**Feature Branch**: `001-stack-migration-refactor` +**Created**: 2026-03-22 +**Status**: Draft +**Input**: Combine etelemetry-server and etelemetry-client into a single monorepo, migrate from MongoDB/Sanic to a modern efficient stack, add privacy-preserving geolocation, a web dashboard, configurable server URLs, and data migration from the existing MongoDB instance. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Version Check at Runtime (Priority: P1) + +A developer integrates etelemetry into their Python package. When a user runs +the package interactively, the client checks the etelemetry server for the +latest available version and any known bad versions. The user sees a warning +if they are outdated or using a faulty release. The server records the +check (project name, version, timestamp, city/state-level location) without +storing the user's IP address. + +**Why this priority**: This is the core value proposition of etelemetry — the +fundamental check-and-inform loop. Without it, nothing else matters. + +**Independent Test**: Deploy a local server instance, configure a client to +point at it, register a test project with a known latest version and a bad +version, then verify that (a) an outdated client receives an "update available" +warning, (b) a client on a bad version receives a critical warning, (c) the +server records the check with city/state geolocation, and (d) no IP address +is persisted in the database or logs. + +**Acceptance Scenarios**: + +1. **Given** a deployed etelemetry server with project "org/tool" at version + 2.0.0 and bad_versions ["1.1.0"], **When** a client running version 1.5.0 + checks in, **Then** the client receives a response containing + `{"version": "2.0.0", "bad_versions": ["1.1.0"]}` and the server persists + a usage record with city/state geolocation but no IP address. + +2. **Given** a deployed etelemetry server, **When** a client running a version + listed in bad_versions checks in, **Then** the client logs a CRITICAL + warning and optionally raises `BadVersionError`. + +3. **Given** a deployed etelemetry server, **When** a client running the latest + version checks in, **Then** the client logs a DEBUG message and no warning + is shown. + +4. **Given** a client with `NO_ET=1` set, **When** the application starts, + **Then** no request is made to the server. + +--- + +### User Story 2 - Configurable Server URL (Priority: P2) + +An organization wants to run their own etelemetry instance (e.g., on internal +infrastructure). They deploy the server with their own domain and configure +their packages' clients to point at it. Existing clients using the old +hardcoded domain continue to work during transition. + +**Why this priority**: Decoupling from a hardcoded domain is essential for +adoption by other organizations and for migrating away from the current +`rig.mit.edu` endpoint. + +**Independent Test**: Start two server instances on different URLs. Configure +one client to use instance A and another to use instance B. Verify each client +talks to its configured server. Verify a client with no explicit configuration +falls back to a sensible default. + +**Acceptance Scenarios**: + +1. **Given** a client with no custom configuration, **When** a version check + is made, **Then** the client uses the default server URL. + +2. **Given** a client configured with a custom server URL via environment + variable, **When** a version check is made, **Then** the client contacts + the custom server. + +3. **Given** a project that embeds a server URL in its etelemetry integration + code, **When** a version check is made, **Then** that URL takes precedence + over the default. + +--- + +### User Story 3 - Data Migration from Existing MongoDB (Priority: P3) + +The etelemetry maintainer migrates historical usage data from the existing +MongoDB instance on EC2 to the new storage backend. After migration, all +previously recorded projects, version checks, and geolocation data are +queryable in the new system. IP addresses present in the old data are +dropped during migration. + +**Why this priority**: Preserving historical data is important for continuity +but does not block new functionality. The migration can happen once the new +server is operational. + +**Independent Test**: Export a sample of the existing MongoDB data, run the +migration tool, and verify that (a) all projects and their usage records +appear in the new database, (b) no IP addresses exist in the migrated data, +and (c) aggregated statistics match between old and new systems. + +**Acceptance Scenarios**: + +1. **Given** an existing MongoDB dump with records in the `requests` and `geo` + collections, **When** the migration tool runs, **Then** all records are + imported into the new storage with IP addresses stripped. + +2. **Given** migrated data, **When** a user queries the dashboard for a + project's historical usage, **Then** the results include pre-migration + data. + +3. **Given** a migration run, **When** the tool encounters malformed or + incomplete records, **Then** it logs a warning, skips the record, and + continues without aborting. + +--- + +### User Story 4 - Web Dashboard for Usage Statistics (Priority: P4) + +A project maintainer visits the etelemetry web dashboard to see how many +users are running their software, which versions are in use, and where +(city/state level) users are located. The dashboard shows summary statistics +over selectable time periods. + +**Why this priority**: The dashboard is a valuable addition but is not required +for the core check-and-inform functionality to work. It can be built on top +of the data already collected by the server. + +**Independent Test**: Seed the database with sample usage records spanning +multiple projects, versions, locations, and dates. Open the dashboard and +verify that (a) per-project usage counts are correct, (b) a geographic +summary shows city/state distribution, (c) time-period filters work, and +(d) the dashboard is accessible (keyboard-navigable, screen-reader friendly, +sufficient color contrast). + +**Acceptance Scenarios**: + +1. **Given** usage data for project "org/tool", **When** a maintainer visits + the dashboard and selects "org/tool", **Then** they see total checks, + version distribution, and an interactive map showing geographic + distribution with drill-down capability (country → region → city) + alongside a summary table with precise counts. + +2. **Given** a dashboard with data, **When** a user selects a time range + (e.g., last 7 days, last 30 days, custom range), **Then** all displayed + statistics reflect only that period. + +3. **Given** a dashboard, **When** accessed by a screen reader or navigated + via keyboard only, **Then** all information is accessible and all controls + are operable. + +--- + +### User Story 5 - Self-Hosted & AWS Deployment (Priority: P5) + +An organization deploys their own etelemetry instance using Docker Compose +locally or on AWS. The primary production deployment on AWS is maintainable +via a GitHub Actions workflow that automates infrastructure updates on push. +Others can self-host with Docker Compose and their own domain. + +**Why this priority**: Self-hosting enables adoption beyond the original +maintainers but depends on the server and client being functional first. + +**Independent Test**: Follow the deployment documentation to bring up an +instance using only Docker Compose on a fresh machine. Register a test +project, make a version check from a client, and verify the dashboard +displays the result. For AWS, verify the GitHub Actions workflow deploys +successfully. + +**Acceptance Scenarios**: + +1. **Given** a machine with Docker and Docker Compose installed, **When** a + user follows the deployment guide, **Then** the etelemetry server, database, + and dashboard are running and reachable. + +2. **Given** a self-hosted instance, **When** a client is configured to point + at it, **Then** version checks succeed and data appears in the dashboard. + +3. **Given** the etelemetry repository on GitHub, **When** a maintainer pushes + to the deployment branch, **Then** a GitHub Actions workflow deploys the + updated service to AWS automatically. + +--- + +### Edge Cases + +- What happens when the GitHub API is rate-limited or unavailable? The server + MUST return cached version data if available, or a graceful error indicating + the version is unknown. +- What happens when the geolocation service is unavailable? The server MUST + still record the usage check with location marked as "unknown" rather than + failing the request. +- What happens when a client sends a malformed project identifier (e.g., + missing owner)? The server MUST return a clear error message with an + appropriate status code. +- What happens when the database is unreachable? The server MUST still respond + to version-check requests using cached data, even if it cannot record the + usage event. +- What happens when the migration tool encounters duplicate records? It MUST + deduplicate based on project + timestamp + location and log the duplicates. +- What happens when a project has no `.et` file? The server MUST return an + empty bad_versions list rather than an error. +- What happens when a client requests a project not on the allowlist? The + server MUST reject the request with a clear "project not tracked" response + and MUST NOT record the event or make upstream API calls for it. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The system MUST provide an endpoint that, given a project + identifier (owner/repo), returns the latest available version and a list + of known bad versions. +- **FR-002**: The system MUST resolve the latest version by querying the + project's source repository (GitHub releases, then tags as fallback). +- **FR-003**: The system MUST cache version lookups to avoid excessive + upstream API calls, with a configurable cache duration (default: 6 hours). +- **FR-004**: The system MUST record version-check events using content- + addressed deduplication: pings with the same project identifier, version, + geolocation, and CI flag within the same time-bucket (default: 1 hour) + MUST increment a counter on an existing record rather than creating a + new row. +- **FR-005**: The system MUST NOT persist IP addresses in the database or + in application logs. IP addresses MUST be used only transiently for + geolocation resolution and then discarded. +- **FR-006**: The system MUST resolve IP addresses to city/state-level + geolocation as quickly and accurately as possible (local lookup preferred + over external API calls). +- **FR-007**: The client library MUST allow the server URL to be configured + via (in order of precedence): function parameter, environment variable, + project-level configuration, then a default URL. +- **FR-008**: The client library MUST maintain backward compatibility with + the existing public API (`get_project`, `check_available_version`, + `BadVersionError`). The server MUST also maintain backward compatibility + with existing deployed clients: the response JSON shape (`version`, + `bad_versions` fields) MUST remain unchanged, and the server MUST accept + requests at the legacy `/et/projects/{owner}/{repo}` URL path (via nginx + rewrite or server-side route alias) so that old clients continue working + during transition. Old-style CI query parameters MUST be accepted without + error (treated as non-CI if the new `ci` param is absent). +- **FR-009**: The system MUST provide a web dashboard showing per-project + usage counts, version distribution, and geographic distribution via an + interactive map with drill-down (country → region → city) and a + companion summary table, with time-period filtering. +- **FR-010**: The dashboard MUST meet WCAG 2.1 AA accessibility standards. +- **FR-011**: The system MUST provide a migration tool that imports data + from the existing MongoDB instance, stripping IP addresses during import. +- **FR-012**: The system MUST be deployable via Docker Compose with a single + configuration file for domain, secrets, and service settings. +- **FR-013**: The system MUST serve both server and client from a single + repository (monorepo) with the client installable as a standalone package. +- **FR-014**: The system MUST read bad_versions from the project's `.et` + file in its source repository and include them in the version-check + response. +- **FR-015**: The system MUST detect whether the client is running in a CI + environment and include that flag in the usage record. +- **FR-016**: Web server access logs MUST strip or omit IP addresses before + writing to disk. +- **FR-017**: The system MUST support tiered aggregation granularity that + coarsens over time (e.g., daily for recent data, weekly as the standard + granularity, monthly for older data). The age thresholds for each tier + MUST be configurable. +- **FR-018**: The system MUST maintain an allowlist of tracked repositories + defined in a configuration file (YAML or JSON) in the deployment. The + server MUST reload the allowlist on restart or when signaled (e.g., + SIGHUP). Version-check requests for projects not on the allowlist MUST + be rejected with an appropriate status code. +- **FR-019**: Ingress and egress payloads MUST be compacted to minimize + bytes transferred (e.g., HTTP compression, minimal JSON field names, + omitting null fields). +- **FR-020**: The production deployment MUST be maintainable on AWS and + deployable via a GitHub Actions workflow or action, enabling automated + infrastructure updates on push. + +### Key Entities + +- **Project**: A software project on the server's allowlist, tracked by + etelemetry. Identified by owner/repo (e.g., "nipy/nipype"). Has a latest + known version, a list of bad versions, a cache timestamp, and an active + flag. Only allowlisted projects are served and recorded. +- **VersionCheck**: A deduplicated usage record keyed by content address: + project identifier + version + geolocation (city, state/region, country) + + CI flag + time-bucket (default: 1-hour window). Each matching ping + increments a counter rather than creating a new row. No IP address is + stored. +- **GeoLocation**: City/state-level location derived from an IP address at + request time. Stored as city, region, country, and approximate coordinates. + Never linked to an IP. +- **UsageAggregate**: Pre-computed summary of version checks grouped by + project, time period, version, and location. Aggregation granularity + coarsens over time: daily for recent data, weekly as the standard + granularity, and monthly for older data. The specific age thresholds + for transitioning between granularities are configurable. Used by the + dashboard for efficient querying. + +### Assumptions + +- The existing MongoDB data on EC2 is accessible via a standard `mongodump` + export or direct connection during migration. +- GitHub remains the primary source for version and bad_versions data; other + source forges (GitLab, etc.) are out of scope for this iteration. +- A local IP-to-geolocation database (e.g., MaxMind GeoLite2) is acceptable + and preferred over an external API for speed and privacy. +- The default server URL will be updated from `rig.mit.edu` to a new domain + to be determined; the old domain will redirect during a transition period. +- The dashboard does not require authentication for read access to aggregated + statistics (no individual-user data is exposed). +- Current production load is approximately 1 million pings per week or more, + with many repeated pings from the same project+version+location. +- Individual version-check records are retained indefinitely (IPs are never + stored). Aggregation granularity coarsens over time (daily → weekly → + monthly) to balance query performance with storage efficiency. Weekly is + the standard useful granularity. + +## Clarifications + +### Session 2026-03-22 + +- Q: What is the data retention policy for version-check records? → A: Keep + all records indefinitely (only IPs are discarded). Introduce tiered + aggregation granularity that coarsens over time; weekly is the useful + baseline granularity. +- Q: What type of geographic visualization for the dashboard? → A: Interactive + map with drill-down (country → region → city) plus a summary table with + precise counts. +- Q: How should repeated identical pings be handled at ~1M/week scale? → A: + Content-addressed deduplication with hourly time-buckets. Same project + + version + location + CI flag within the same hour increments a counter. +- (Direct integration) Scale: ~1M pings/week, many from same location. + Updated SC-009, added scale assumption. +- (Direct integration) Added FR-018: repository allowlist to control which + projects are tracked. +- (Direct integration) Added FR-019: compact ingress/egress payloads. +- (Direct integration) Added FR-020: AWS deployment via GitHub Actions + workflow. Updated User Story 5. +- Q: How is the project allowlist managed? → A: Configuration file + (YAML/JSON) in the deployment, reloaded on restart or signal. +- (Direct integration) Backward compatibility: server MUST accept requests + at legacy `/et/projects/` path (nginx rewrite), response JSON shape + unchanged (`version`, `bad_versions`), old-style CI query params accepted + without error. Updated FR-008. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A version check from client to server completes in under + 2 seconds under normal conditions (cached version data). +- **SC-002**: Geolocation resolution adds no more than 10 milliseconds to + request processing time (local database lookup). +- **SC-003**: The new storage backend uses at least 50% less disk space than + the equivalent MongoDB data for the same record count. +- **SC-004**: The migration tool successfully imports 100% of valid records + from the existing MongoDB, with zero IP addresses in the new database. +- **SC-005**: The dashboard loads project statistics for any project within + 3 seconds, even with 12+ months of historical data. +- **SC-006**: The dashboard passes automated accessibility audits (e.g., + axe-core) with zero critical or serious violations. +- **SC-007**: A new self-hosted instance can be brought up from scratch in + under 15 minutes following the deployment guide. +- **SC-008**: All existing client integrations (`get_project`, + `check_available_version`, `BadVersionError`) continue to work without + code changes in downstream projects (backward compatibility). +- **SC-009**: The system handles at least 1 million version-check requests + per week (~1,650/minute sustained) without degradation, with burst + capacity for peak periods. diff --git a/specs/001-stack-migration-refactor/tasks.md b/specs/001-stack-migration-refactor/tasks.md new file mode 100644 index 0000000..08513f6 --- /dev/null +++ b/specs/001-stack-migration-refactor/tasks.md @@ -0,0 +1,257 @@ +# Tasks: etelemetry Stack Migration & Monorepo Refactor + +**Input**: Design documents from `/specs/001-stack-migration-refactor/` +**Prerequisites**: plan.md (required), spec.md (required), research.md, data-model.md, contracts/api.md, quickstart.md + +**Tests**: Included — constitution mandates test-driven with real use cases (Principle IV). + +**Organization**: Tasks grouped by user story for independent implementation and testing. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story (US1–US5) +- Exact file paths from plan.md project structure + +## Phase 1: Setup + +**Purpose**: Monorepo initialization, tooling, and project skeleton + +- [x] T001 Create root `pyproject.toml` for client package `etelemetry` with dependencies: `requests`, `packaging`, `ci-info`; dev extras: `pytest`, `pytest-asyncio`, `httpx`, `ruff`; configure `uv` as build backend +- [x] T002 Create `server/pyproject.toml` for server package `etelemetry-server` with dependencies: `fastapi`, `uvicorn[standard]`, `sqlalchemy[asyncio]`, `asyncpg`, `alembic`, `httpx`, `geoip2`, `jinja2`, `starlette-compress`, `pydantic-settings`, `pyyaml`; dev extras: `pytest`, `testcontainers` +- [x] T003 [P] Create `.gitignore` (Python, .env, __pycache__, *.mmdb, node_modules) and `deploy/.env.example` with placeholder keys (POSTGRES_PASSWORD, MAXMIND_LICENSE_KEY, SECRET_KEY) +- [x] T004 [P] Create skeleton `docs/vision.md`, `docs/phase-log.md`, `docs/rebuild-spec.md` with initial content per constitution (Principles VI, VII) +- [x] T005 [P] Create empty package directories: `src/etelemetry/`, `server/src/etelemetry_server/`, `server/src/etelemetry_server/routes/`, `server/src/etelemetry_server/services/`, `server/src/etelemetry_server/dashboard/templates/`, `server/src/etelemetry_server/dashboard/static/`, `tools/`, `tests/unit/`, `tests/integration/`, `tests/contract/`, `server/tests/unit/`, `server/tests/integration/`, `server/tests/contract/` + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: Database, ORM, app factory, and allowlist — MUST complete before ANY user story + +**CRITICAL**: No user story work can begin until this phase is complete + +- [x] T006 Implement Pydantic settings in `server/src/etelemetry_server/settings.py` — load from env: DATABASE_URL, MAXMIND_DB_PATH, GITHUB_TOKEN (optional), CACHE_TTL_SECONDS (default 21600), ALLOWLIST_PATH, TIME_BUCKET_HOURS (default 1) +- [x] T007 Implement database engine and async session factory in `server/src/etelemetry_server/db.py` — `create_async_engine` with asyncpg, session dependency for FastAPI +- [x] T008 Implement SQLAlchemy ORM models in `server/src/etelemetry_server/models.py` — `Project`, `VersionCheck`, `UsageAggregate` tables per data-model.md with all indexes and unique constraints +- [x] T009 Initialize Alembic in `server/alembic/` with `alembic.ini` and generate initial migration from models (auto-generate revision) +- [x] T010 [P] Implement FastAPI application factory in `server/src/etelemetry_server/app.py` — lifespan handler (init DB, load GeoIP reader, load allowlist), mount starlette-compress middleware, include route routers +- [x] T011 [P] Implement allowlist loader in `server/src/etelemetry_server/allowlist.py` — load YAML file, SIGHUP handler to reload, sync allowlist entries to `projects` table (insert missing, deactivate removed) +- [x] T012 [P] Write unit tests for settings, models, and allowlist in `server/tests/unit/test_settings.py`, `server/tests/unit/test_models.py`, `server/tests/unit/test_allowlist.py` + +**Checkpoint**: Database schema, app factory, and allowlist ready — user story implementation can begin + +--- + +## Phase 3: User Story 1 — Version Check at Runtime (Priority: P1) + +**Goal**: Core version-check loop: client sends request, server resolves version from GitHub, records usage with geolocation (no IP), returns version + bad_versions + +**Independent Test**: Start local server, configure test project in allowlist, make version check from client, verify response and DB record with geolocation but no IP + +### Tests for User Story 1 + +- [x] T013 [P] [US1] Write contract test for `GET /projects/{owner}/{repo}` in `server/tests/contract/test_projects_api.py` — verify response shape matches contracts/api.md for 200, 404, 400 cases +- [x] T014 [P] [US1] Write integration test for end-to-end version check in `server/tests/integration/test_version_check.py` — start server with testcontainers PostgreSQL, make request, verify DB record has geolocation but no IP column + +### Implementation for User Story 1 + +- [x] T015 [P] [US1] Implement geolocation service in `server/src/etelemetry_server/services/geolocation.py` — load MaxMind GeoLite2 Reader at startup, `resolve(ip: str) -> GeoResult` returning city/region/country/coords, handle lookup failures (return "unknown") +- [x] T016 [P] [US1] Implement version checker service in `server/src/etelemetry_server/services/version_checker.py` — async GitHub API via httpx (releases then tags fallback), cache in `projects` table with TTL, fetch `.et` file for bad_versions, handle rate limiting gracefully +- [x] T017 [P] [US1] Implement usage recorder service in `server/src/etelemetry_server/services/usage_recorder.py` — content-addressed upsert: `INSERT INTO version_checks ... ON CONFLICT (composite_key) DO UPDATE SET count = count + 1`, compute time_bucket by truncating to hour +- [x] T018 [US1] Implement in-memory LRU cache in `server/src/etelemetry_server/services/version_checker.py` — cache recent version lookups in memory so the server can respond to version-check requests even when PostgreSQL is unreachable; populate on successful DB reads, serve from memory on DB failure +- [x] T019 [US1] Implement health route in `server/src/etelemetry_server/routes/health.py` — `GET /` returning `{"name": "etelemetry", "version": ...}` +- [x] T020 [US1] Implement projects route in `server/src/etelemetry_server/routes/projects.py` — `GET /projects/{owner}/{repo}` orchestrating: allowlist check → geolocation → usage record → version lookup → response; accept `?ci=` and `?v=` query params; disable uvicorn access log IP logging +- [x] T021 [P] [US1] Implement client `src/etelemetry/errors.py` — `BadVersionError(RuntimeError)` +- [x] T022 [P] [US1] Implement client `src/etelemetry/config.py` — `resolve_url(server_url=None)` with basic precedence: `ETELEMETRY_URL` env → hardcoded default; `NO_ET` check +- [x] T023 [US1] Implement client `src/etelemetry/client.py` — `get_project(repo, **kwargs)` and `check_available_version(project, version, lgr, raise_exception)` per contracts/api.md; use `requests` with 5s timeout; send `?ci=` and `?v=` params +- [x] T024 [US1] Implement client `src/etelemetry/__init__.py` — export `get_project`, `check_available_version`, `BadVersionError`, `__version__` +- [x] T025 [US1] Write client unit tests in `tests/unit/test_client.py` — mock server responses, verify outdated warning, bad version critical warning, `BadVersionError` raise, `NO_ET` disables requests +- [x] T026 [US1] Run contract and integration tests, verify all pass + +**Checkpoint**: Core version-check loop fully functional. Client and server independently testable. + +--- + +## Phase 4: User Story 2 — Configurable Server URL (Priority: P2) + +**Goal**: Client supports custom server URL via function param, env var, or package-level default + +**Independent Test**: Configure client with custom URL, verify it contacts that URL instead of default + +### Implementation for User Story 2 + +- [x] T027 [US2] Update `src/etelemetry/config.py` — add `default_url` module-level attribute, add `server_url` param support to `resolve_url()`, document full precedence chain (param → env → default_url → hardcoded) +- [x] T028 [US2] Update `src/etelemetry/client.py` — add `server_url` optional parameter to `get_project()` and `check_available_version()`, pass through to `resolve_url()` +- [x] T029 [US2] Write tests in `tests/unit/test_config.py` — verify precedence: param > env var > `config.default_url` > hardcoded default; verify `NO_ET` short-circuits +- [x] T030 [US2] Write integration test in `tests/integration/test_custom_url.py` — start two mock servers, verify client routes to correct one based on config + +**Checkpoint**: Client configurable URL working. Backward compatible — no existing code breaks. + +--- + +## Phase 5: User Story 3 — Data Migration from MongoDB (Priority: P3) + +**Goal**: One-time migration tool imports all historical MongoDB data into PostgreSQL, stripping IPs + +**Independent Test**: Export sample MongoDB data, run migration, verify records in PostgreSQL with no IPs, aggregated stats match + +### Tests for User Story 3 + +> **NOTE: Write these tests FIRST, ensure they FAIL before implementation** + +- [x] T031 [US3] Write migration test in `server/tests/integration/test_migration.py` — use sample MongoDB fixture data (JSON), run migration against testcontainers PostgreSQL, verify: all valid records imported, no IP fields in DB, duplicate handling, malformed record skipping with warning log, storage size comparison (SC-003) + +### Implementation for User Story 3 + +- [x] T032 [US3] Implement migration tool in `tools/migrate.py` — CLI with `--mongo-uri` and `--pg-uri` args; connect to MongoDB `et` database; iterate `requests` + `geo` collections; join on `remote_addr`; map fields per data-model.md migration mapping; insert into PostgreSQL using content-addressed dedup; strip all IP fields; log progress, warnings for malformed records; `--verify` flag to compare counts; add `pymongo` to server dev dependencies +- [x] T033 [US3] Run migration test, verify pass + +**Checkpoint**: Migration tool ready for production use against EC2 MongoDB. + +--- + +## Phase 6: User Story 4 — Web Dashboard (Priority: P4) + +**Goal**: Server-rendered dashboard with interactive map (drill-down), summary table, version distribution, time-period filtering, WCAG 2.1 AA accessible + +**Independent Test**: Seed DB with sample data, open dashboard, verify per-project stats, map drill-down, time filtering, accessibility audit + +### Tests for User Story 4 + +> **NOTE: Write these tests FIRST, ensure they FAIL before implementation** + +- [x] T034 [P] [US4] Write dashboard contract tests in `server/tests/contract/test_dashboard_api.py` — verify response shapes for stats, geo, and projects endpoints per contracts/api.md +- [x] T035 [P] [US4] Write dashboard integration test in `server/tests/integration/test_dashboard.py` — seed DB with 12+ months of sample data, verify HTML pages render, API returns correct aggregated data, page loads within 3 seconds (SC-005) + +### Implementation for User Story 4 + +- [x] T036 [US4] Implement aggregation service in `server/src/etelemetry_server/services/aggregation.py` — tiered rollup job: compute daily/weekly/monthly aggregates from version_checks; configurable age thresholds; callable as background task or CLI command +- [x] T037 [US4] Implement dashboard API routes in `server/src/etelemetry_server/routes/dashboard.py` — `GET /dashboard/api/projects` (project list with counts), `GET /dashboard/api/stats/{owner}/{repo}` (stats with time range + granularity params), `GET /dashboard/api/geo/{owner}/{repo}` (GeoJSON for Leaflet) +- [x] T038 [P] [US4] Create dashboard base template in `server/src/etelemetry_server/dashboard/templates/base.html` — HTML skeleton with htmx (CDN), Leaflet.js (CDN), CSS for accessibility (skip links, focus indicators, sufficient contrast, responsive layout) +- [x] T039 [US4] Create project list page template in `server/src/etelemetry_server/dashboard/templates/projects.html` — list all tracked projects with total check counts, link to detail; `GET /dashboard/` route serves this +- [x] T040 [US4] Create project detail page template in `server/src/etelemetry_server/dashboard/templates/project_detail.html` — interactive Leaflet map (drill-down country→region→city via GeoJSON endpoint), summary table, version distribution chart, timeline; time-range filter via htmx partial updates +- [x] T041 [US4] Add static JS in `server/src/etelemetry_server/dashboard/static/dashboard.js` — Leaflet map initialization, GeoJSON layer with click drill-down, htmx event handlers for filter updates +- [x] T042 [US4] Run accessibility audit — verify WCAG 2.1 AA compliance: keyboard navigation, screen reader landmarks, color contrast, focus management; document results + +**Checkpoint**: Dashboard fully functional with map, table, filtering, and accessibility. + +--- + +## Phase 7: User Story 5 — Self-Hosted & AWS Deployment (Priority: P5) + +**Goal**: Docker Compose deployment for self-hosting; GitHub Actions CI/CD for AWS + +**Independent Test**: Docker Compose up on fresh machine, make version check, verify dashboard shows data. GitHub Actions workflow deploys to AWS. + +### Implementation for User Story 5 + +- [x] T043 [US5] Create `deploy/Dockerfile` — multi-stage build: uv install server package, copy GeoIP config, expose port 8000, entrypoint: alembic upgrade + uvicorn +- [x] T044 [P] [US5] Create `deploy/docker-compose.yml` — services: postgres (16-alpine, volume for data), server (build from Dockerfile, depends_on postgres, env from .env), nginx (reverse proxy, port 80/443), geoipupdate (MaxMind DB updates, shared volume with server) +- [x] T045 [P] [US5] Create `deploy/nginx.conf` — reverse proxy to server:8000, strip IP from access logs (custom log format replacing $remote_addr with "-"), HTTPS config placeholder +- [x] T046 [US5] Create `deploy/allowlist.yml` — sample allowlist with a few sensein projects, documented format +- [x] T047 [P] [US5] Create `deploy/geoipupdate.conf` — MaxMind GeoIP update config template with license key placeholder +- [x] T048 [US5] Create `.github/workflows/ci.yml` — on PR: checkout, uv setup, install deps, run ruff lint, run pytest (client tests + server tests with testcontainers), upload coverage; generate and commit `uv.lock` +- [x] T049 [US5] Create `.github/workflows/deploy.yml` — on push to main: build Docker image, push to registry, SSH deploy to AWS EC2 (or use docker context), run alembic migrations, restart services; secrets: AWS credentials, server host, MaxMind key +- [x] T050 [US5] Test Docker Compose deployment locally — `docker compose up`, verify all services healthy, make version check request, verify dashboard accessible +- [x] T051 [US5] Test GitHub Actions CI workflow — push branch, verify checks pass + +**Checkpoint**: Deployment fully automated. Self-hosted and AWS paths both working. + +--- + +## Phase 8: Polish & Cross-Cutting Concerns + +**Purpose**: Code review, documentation, IP audit, end-to-end validation + +- [x] T052 Code abstraction review — identify duplicated logic across server services and extract to shared utilities per constitution Principle V +- [x] T053 [P] IP audit — grep entire codebase and database schema for any IP storage or logging; verify nginx logs strip IPs; verify uvicorn logs strip IPs; verify no IP in version_checks table; document audit results +- [x] T054 [P] Update `docs/vision.md` with final architectural direction and scope +- [x] T055 Update `docs/phase-log.md` with entries for all completed phases +- [x] T056 Update `docs/rebuild-spec.md` — comprehensive specification sufficient to rebuild from scratch per constitution Principle VI +- [x] T057 Run `quickstart.md` validation — follow every step in quickstart.md on a clean checkout, verify all commands succeed +- [x] T058 [P] Run full test suite (`uv run pytest`) and verify all tests pass +- [x] T059 Performance validation — load test with ~1,650 req/min sustained, verify <2s response, verify content-addressed dedup reduces row count as expected, verify storage is at least 50% smaller than MongoDB equivalent (SC-003) + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies — start immediately +- **Foundational (Phase 2)**: Depends on Phase 1 completion — BLOCKS all user stories +- **US1 (Phase 3)**: Depends on Phase 2 — core functionality +- **US2 (Phase 4)**: Depends on Phase 3 (client library exists) +- **US3 (Phase 5)**: Depends on Phase 2 (database schema exists) +- **US4 (Phase 6)**: Depends on Phase 3 (version_checks data exists to aggregate) +- **US5 (Phase 7)**: Depends on Phase 3 (server functional to deploy) +- **Polish (Phase 8)**: Depends on all desired user stories complete + +### User Story Dependencies + +- **US1 (P1)**: After Phase 2 — no story dependencies +- **US2 (P2)**: After US1 (extends client.py created in US1) +- **US3 (P3)**: After Phase 2 — independent of US1 (only needs schema) +- **US4 (P4)**: After US1 (needs version_checks data to display) +- **US5 (P5)**: After US1 (needs functional server to containerize) + +### Parallel Opportunities + +After Phase 2 completes: +- US1 can start immediately (critical path) +- US3 can start in parallel with US1 (only needs DB schema, not server routes) + +After US1 completes: +- US2, US4, US5 can all start in parallel + +### Within Each User Story + +- Tests written first and MUST fail before implementation +- Models/services before routes +- Core implementation before integration/polish + +--- + +## Implementation Strategy + +### MVP First (User Story 1 Only) + +1. Complete Phase 1: Setup +2. Complete Phase 2: Foundational +3. Complete Phase 3: User Story 1 +4. **STOP and VALIDATE**: Test end-to-end version check +5. Deploy/demo if ready + +### Incremental Delivery + +1. Setup + Foundational → Foundation ready +2. US1 → Core version check working → Deploy (MVP!) +3. US2 → Configurable URLs → Deploy +4. US3 → Historical data migrated +5. US4 → Dashboard live → Deploy +6. US5 → Automated deployment +7. Polish → Production-ready + +### Parallel Execution Example + +```bash +# After Phase 2 completes, launch in parallel: +Agent A: US1 (version check — critical path) +Agent B: US3 (migration tool — only needs DB schema) + +# After US1 completes, launch in parallel: +Agent A: US2 (configurable URL — extends client) +Agent B: US4 (dashboard — needs version_checks data) +Agent C: US5 (deployment — needs functional server) +``` + +--- + +## Notes + +- [P] tasks = different files, no dependencies on incomplete tasks +- [Story] label maps task to specific user story for traceability +- Commit after each task or logical group +- Stop at any checkpoint to validate story independently +- Constitution compliance: uv for all Python work, secrets via .env, conventional commits diff --git a/specs/002-aws-least-privilege-deploy/checklists/requirements.md b/specs/002-aws-least-privilege-deploy/checklists/requirements.md new file mode 100644 index 0000000..18778c4 --- /dev/null +++ b/specs/002-aws-least-privilege-deploy/checklists/requirements.md @@ -0,0 +1,38 @@ +# Specification Quality Checklist: AWS Least-Privilege Idempotent Deployment + +**Purpose**: Validate specification completeness and quality before proceeding to planning +**Created**: 2026-03-22 +**Feature**: [spec.md](../spec.md) + +## Content Quality + +- [x] No implementation details (languages, frameworks, APIs) +- [x] Focused on user value and business needs +- [x] Written for non-technical stakeholders +- [x] All mandatory sections completed + +## Requirement Completeness + +- [x] No [NEEDS CLARIFICATION] markers remain +- [x] Requirements are testable and unambiguous +- [x] Success criteria are measurable +- [x] Success criteria are technology-agnostic (no implementation details) +- [x] All acceptance scenarios are defined +- [x] Edge cases are identified +- [x] Scope is clearly bounded +- [x] Dependencies and assumptions identified + +## Feature Readiness + +- [x] All functional requirements have clear acceptance criteria +- [x] User scenarios cover primary flows +- [x] Feature meets measurable outcomes defined in Success Criteria +- [x] No implementation details leak into specification + +## Notes + +- Assumptions document that the AWS account already exists and an initial + privileged user is available for bootstrap. +- DNS is managed externally — the deployment provides values to configure. +- Single-instance scope; horizontal scaling is explicitly out of scope. +- All items pass. Spec is ready for `/speckit-clarify` or `/speckit-plan`. diff --git a/specs/002-aws-least-privilege-deploy/contracts/iam-policy.md b/specs/002-aws-least-privilege-deploy/contracts/iam-policy.md new file mode 100644 index 0000000..d181fb6 --- /dev/null +++ b/specs/002-aws-least-privilege-deploy/contracts/iam-policy.md @@ -0,0 +1,135 @@ +# IAM Policy Contracts + +**Date**: 2026-03-22 +**Branch**: `002-aws-least-privilege-deploy` + +## Deploy Role: `etelemetry-deploy` + +Assumed by GitHub Actions via OIDC. Used for day-to-day deployments. + +### Trust Policy + +```json +{ + "Version": "2012-10-17", + "Statement": [{ + "Effect": "Allow", + "Principal": { + "Federated": "arn:aws:iam::ACCOUNT_ID:oidc-provider/token.actions.githubusercontent.com" + }, + "Action": "sts:AssumeRoleWithWebIdentity", + "Condition": { + "StringEquals": { + "token.actions.githubusercontent.com:aud": "sts.amazonaws.com" + }, + "StringLike": { + "token.actions.githubusercontent.com:sub": "repo:sensein/etelemetry:ref:refs/heads/main" + } + } + }] +} +``` + +### Permission Policy (14 actions) + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "SSMDeploy", + "Effect": "Allow", + "Action": [ + "ssm:SendCommand", + "ssm:GetCommandInvocation", + "ssm:GetParameter", + "ssm:PutParameter" + ], + "Resource": [ + "arn:aws:ssm:REGION:ACCOUNT_ID:parameter/etelemetry/*", + "arn:aws:ssm:REGION:ACCOUNT_ID:document/AWS-RunShellScript", + "arn:aws:ec2:REGION:ACCOUNT_ID:instance/INSTANCE_ID" + ] + }, + { + "Sid": "EC2Describe", + "Effect": "Allow", + "Action": [ + "ec2:DescribeInstances", + "ec2:DescribeInstanceStatus" + ], + "Resource": "*" + }, + { + "Sid": "ECRAuth", + "Effect": "Allow", + "Action": "ecr:GetAuthorizationToken", + "Resource": "*" + }, + { + "Sid": "ECRPushPull", + "Effect": "Allow", + "Action": [ + "ecr:BatchCheckLayerAvailability", + "ecr:PutImage", + "ecr:InitiateLayerUpload", + "ecr:UploadLayerPart", + "ecr:CompleteLayerUpload", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ], + "Resource": "arn:aws:ecr:REGION:ACCOUNT_ID:repository/etelemetry" + } + ] +} +``` + +## EC2 Instance Role: `etelemetry-instance` + +Attached to the EC2 instance via instance profile. Allows the instance +to pull images from ECR, read secrets from SSM, and communicate with SSM +Session Manager. + +### Permission Policy + +```json +{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "SSMManaged", + "Effect": "Allow", + "Action": [ + "ssm:UpdateInstanceInformation", + "ssmmessages:CreateControlChannel", + "ssmmessages:CreateDataChannel", + "ssmmessages:OpenControlChannel", + "ssmmessages:OpenDataChannel", + "ec2messages:AcknowledgeMessage", + "ec2messages:DeleteMessage", + "ec2messages:FailMessage", + "ec2messages:GetEndpoint", + "ec2messages:GetMessages", + "ec2messages:SendReply" + ], + "Resource": "*" + }, + { + "Sid": "SSMSecrets", + "Effect": "Allow", + "Action": "ssm:GetParametersByPath", + "Resource": "arn:aws:ssm:REGION:ACCOUNT_ID:parameter/etelemetry/*" + }, + { + "Sid": "ECRPull", + "Effect": "Allow", + "Action": [ + "ecr:GetAuthorizationToken", + "ecr:BatchGetImage", + "ecr:GetDownloadUrlForLayer" + ], + "Resource": "*" + } + ] +} +``` diff --git a/specs/002-aws-least-privilege-deploy/plan.md b/specs/002-aws-least-privilege-deploy/plan.md new file mode 100644 index 0000000..108e257 --- /dev/null +++ b/specs/002-aws-least-privilege-deploy/plan.md @@ -0,0 +1,75 @@ +# Implementation Plan: AWS Least-Privilege Idempotent Deployment + +**Branch**: `002-aws-least-privilege-deploy` | **Date**: 2026-03-22 | **Spec**: [spec.md](spec.md) +**Input**: Feature specification from `/specs/002-aws-least-privilege-deploy/spec.md` + +## Summary + +Create an idempotent AWS deployment using OpenTofu for infrastructure, +GitHub OIDC for authentication (no long-lived keys), SSM for remote +execution (no SSH), ECR for container images, SSM Parameter Store for +secrets, Caddy for automatic TLS, and docker-rollout for zero-downtime +updates. All infrastructure is version-controlled and least-privilege. + +## Technical Context + +**IaC Tool**: OpenTofu (Terraform-compatible, open-source) +**Compute**: EC2 t3.small + Docker Compose, managed via SSM +**Auth**: GitHub OIDC → AWS IAM (no access keys) +**Container Registry**: ECR (private) +**Secrets**: SSM Parameter Store (SecureString, KMS-encrypted) +**TLS**: Caddy reverse proxy (automatic Let's Encrypt) +**Zero-downtime**: docker-rollout plugin +**State Backend**: S3 + DynamoDB for OpenTofu state locking +**Target Platform**: AWS (us-east-1), single EC2 instance +**Cost**: ~$18-20/month + +## Constitution Check + +| Principle | Status | Evidence | +|-----------|--------|----------| +| I. Isolated Environments | PASS | Docker Compose on EC2; uv for any local tooling | +| II. Secret Safety | PASS | SSM Parameter Store (encrypted); no secrets in VCS, images, or logs | +| III. Git Discipline | PASS | Infra code version-controlled; OIDC scoped to main branch | +| IV. Test-Driven | PASS | `tofu plan` validates before apply; health checks verify deployment | +| V. Code Abstraction | PASS | Reusable OpenTofu modules where applicable | +| VI. Provenance | PASS | OpenTofu state tracks all resources; deploy commits recorded | +| VII. Living Documentation | PASS | Quickstart and external setup guide updated | +| VIII. Simplicity | PASS | Single EC2 + Docker Compose; Caddy replaces nginx+certbot | + +## Project Structure + +### Infrastructure (new) + +```text +infra/ +├── main.tf # Provider, backend, data sources +├── ec2.tf # EC2 instance, security group, EIP, user data +├── iam.tf # OIDC provider, deploy role, instance role +├── ecr.tf # ECR repository +├── ssm.tf # SSM parameter definitions (placeholder values) +├── state.tf # S3 bucket + DynamoDB for state backend +├── variables.tf # Input variables (region, instance type, domain) +├── outputs.tf # Deploy role ARN, instance ID, ECR URL, EIP +├── versions.tf # Required providers and versions +└── scripts/ + └── deploy.sh # SSM document: pull image, update .env, rollout +``` + +### Modified Files + +```text +deploy/ +├── docker-compose.yml # Replace nginx with caddy; add healthchecks +├── Caddyfile # Replaces nginx.conf +└── docker-compose.dev.yml # No changes + +.github/workflows/ +├── deploy.yml # Rewrite: OIDC auth, ECR push, SSM deploy +└── ci.yml # No changes +``` + +**Structure Decision**: Infrastructure code lives in `infra/` at repo root, +alongside application code. OpenTofu state is stored remotely in S3 with +DynamoDB locking. The deploy workflow is rewritten to use OIDC + ECR + SSM +instead of SSH + SCP. diff --git a/specs/002-aws-least-privilege-deploy/quickstart.md b/specs/002-aws-least-privilege-deploy/quickstart.md new file mode 100644 index 0000000..2a162db --- /dev/null +++ b/specs/002-aws-least-privilege-deploy/quickstart.md @@ -0,0 +1,81 @@ +# Quickstart: AWS Deployment + +## Prerequisites + +- AWS account with an IAM user that can create resources (one-time bootstrap) +- OpenTofu installed (`brew install opentofu` or via package manager) +- AWS CLI configured (`aws configure`) +- Domain DNS access for `et.dandiproject.org` + +## Bootstrap (one-time) + +```bash +cd infra/ + +# Initialize OpenTofu with S3 backend +tofu init + +# Review what will be created +tofu plan + +# Apply — creates EC2, security groups, IAM roles, ECR, SSM params +tofu apply + +# Note the outputs: +# - EC2 instance ID +# - ECR repository URL +# - Deploy role ARN (for GitHub Actions) +# - Elastic IP (for DNS A record) +``` + +## Configure DNS + +Point `et.dandiproject.org` A record to the Elastic IP output from bootstrap. + +## Store Secrets in SSM + +```bash +aws ssm put-parameter --name /etelemetry/POSTGRES_PASSWORD \ + --value "YOUR_STRONG_PASSWORD" --type SecureString + +aws ssm put-parameter --name /etelemetry/MAXMIND_ACCOUNT_ID \ + --value "YOUR_ACCOUNT_ID" --type SecureString + +aws ssm put-parameter --name /etelemetry/MAXMIND_LICENSE_KEY \ + --value "YOUR_LICENSE_KEY" --type SecureString + +# Optional: higher GitHub API rate limit +aws ssm put-parameter --name /etelemetry/GITHUB_TOKEN \ + --value "ghp_YOUR_TOKEN" --type SecureString +``` + +## Configure GitHub Repository + +1. Go to Settings → Secrets and variables → Actions +2. Add variable: `AWS_DEPLOY_ROLE_ARN` = the role ARN from bootstrap output +3. Add variable: `AWS_REGION` = your chosen region (e.g., `us-east-1`) + +No AWS access keys needed — the workflow uses OIDC. + +## First Deployment + +Push to `main` or manually trigger the deploy workflow. The workflow will: +1. Authenticate to AWS via OIDC +2. Build and push the Docker image to ECR +3. Run deploy script on EC2 via SSM (pulls image, updates .env, docker-rollout) +4. Health-check the new deployment +5. Roll back if health check fails + +## Verify + +```bash +curl https://et.dandiproject.org/ +# → {"name": "etelemetry", "version": "2.0.0"} + +curl https://et.dandiproject.org/projects/nipy/nipype +# → {"version": "...", "bad_versions": [...]} +``` + +## Subsequent Deployments + +Merge to `main` → automatic deploy. No manual steps. diff --git a/specs/002-aws-least-privilege-deploy/research.md b/specs/002-aws-least-privilege-deploy/research.md new file mode 100644 index 0000000..8677298 --- /dev/null +++ b/specs/002-aws-least-privilege-deploy/research.md @@ -0,0 +1,93 @@ +# Research: AWS Least-Privilege Idempotent Deployment + +**Date**: 2026-03-22 +**Branch**: `002-aws-least-privilege-deploy` + +## 1. Infrastructure-as-Code Tool + +**Decision**: OpenTofu (Terraform-compatible, open-source) +**Rationale**: Idempotent by design (`tofu apply` converges to declared +state). S3 + DynamoDB state backend with locking. HCL is concise for this +scale (~150 lines). Good GitHub Actions integration +(`opentofu/setup-opentofu`). Open-source (MPL 2.0) vs Terraform's BSL. +**Alternatives considered**: +- Terraform: Functionally identical but BSL-licensed. +- CloudFormation: No state file but more verbose, slower iteration. +- AWS CDK: Overkill for single EC2 + supporting resources. +- Plain AWS CLI: Not idempotent without significant manual effort. + +## 2. Compute Approach + +**Decision**: EC2 (t3.small) + Docker Compose, managed via SSM (no SSH) +**Rationale**: Docker Compose works as-is. Persistent PostgreSQL via EBS +volume. Cost: ~$15-20/month vs $100-170/month for Fargate + RDS. SSM +Session Manager replaces SSH entirely — no keys, no port 22. Deploy +commands run via `aws ssm send-command`. +**Alternatives considered**: +- ECS Fargate: 6-8x cost, requires RDS for PostgreSQL, Docker Compose + must be translated to task definitions. +- App Runner: Cannot run PostgreSQL or multi-container compositions. +- Lightsail: No volume persistence for PostgreSQL. + +## 3. GitHub OIDC → AWS IAM + +**Decision**: OIDC federation with branch-scoped trust policy +**Rationale**: No long-lived access keys. Trust policy scoped to +`repo:sensein/etelemetry:ref:refs/heads/main`. Uses +`aws-actions/configure-aws-credentials@v4` action. + +**IAM permissions for deploy role (14 permissions)**: +- `ssm:SendCommand`, `ssm:GetCommandInvocation` — deploy via SSM +- `ssm:GetParameter`, `ssm:PutParameter` — read/write deploy state +- `ec2:DescribeInstances`, `ec2:DescribeInstanceStatus` — find target +- `ecr:GetAuthorizationToken` — authenticate to ECR +- `ecr:BatchCheckLayerAvailability`, `ecr:PutImage`, + `ecr:InitiateLayerUpload`, `ecr:UploadLayerPart`, + `ecr:CompleteLayerUpload` — push images +- `ecr:BatchGetImage`, `ecr:GetDownloadUrlForLayer` — pull images + +All permissions scoped to specific resource ARNs. + +## 4. Secret Management + +**Decision**: AWS SSM Parameter Store (SecureString) +**Rationale**: Free for standard parameters. KMS-encrypted. Sufficient +for ~5 static secrets. Secrets stay in AWS (never transit through GitHub +runners). Audit trail via CloudTrail. +**Alternatives considered**: +- Secrets Manager: Better for auto-rotation; unnecessary here ($2/month). +- GitHub Secrets → .env: Secrets transit through runners; no audit trail. + +## 5. TLS/Certificate Management + +**Decision**: Caddy as reverse proxy (replaces nginx) +**Rationale**: Automatic HTTPS with zero config — obtains and renews +Let's Encrypt certificates given a domain name. No certbot, no cron, no +certificate file management. Drop-in replacement for the existing nginx +proxy. ~10 lines of Caddyfile vs 37 lines of nginx.conf. +**Alternatives considered**: +- ACM + ALB: Adds $16-22/month for the ALB. Unnecessary for single EC2. +- nginx + certbot: Works but adds operational complexity (sidecar + container, shared volumes, renewal coordination). + +## 6. Zero-Downtime Deployment + +**Decision**: docker-rollout with health checks +**Rationale**: Docker CLI plugin for zero-downtime Docker Compose deploys. +Starts new container alongside old, waits for health check, routes traffic, +stops old container. Automatic rollback if health check fails. +**Alternatives considered**: +- Simple `docker compose up -d`: 5-30s downtime per deploy. +- Blue-green with profiles: More complex for no benefit at this scale. + +## Cost Summary + +| Component | Monthly Cost | +|-----------|-------------| +| EC2 t3.small (on-demand) | ~$15 | +| EBS 20GB gp3 | ~$1.60 | +| Elastic IP | $0 (while running) | +| SSM Parameter Store | $0 | +| ECR storage (~2GB) | ~$0.20 | +| Data transfer | ~$1-3 | +| **Total** | **~$18-20/month** | diff --git a/specs/002-aws-least-privilege-deploy/spec.md b/specs/002-aws-least-privilege-deploy/spec.md new file mode 100644 index 0000000..fcfc817 --- /dev/null +++ b/specs/002-aws-least-privilege-deploy/spec.md @@ -0,0 +1,216 @@ +# Feature Specification: AWS Least-Privilege Idempotent Deployment + +**Feature Branch**: `002-aws-least-privilege-deploy` +**Created**: 2026-03-22 +**Status**: Draft +**Input**: Create an idempotent AWS deployment setup for the etelemetry +server usable from GitHub Actions, with least-privilege IAM roles instead +of full administrator access. + +## User Scenarios & Testing *(mandatory)* + +### User Story 1 - Automated Deployment on Push (Priority: P1) + +A maintainer merges a PR to the `main` branch. The GitHub Actions deploy +workflow automatically provisions or updates the etelemetry server on AWS +without manual SSH access. The deployment is idempotent — running it twice +with no code changes produces no infrastructure changes. The workflow uses +a scoped IAM role with only the permissions required to deploy, not full +administrator access. + +**Why this priority**: This is the core value — automated, secure, repeatable +deployment is the entire purpose of this feature. + +**Independent Test**: Run the deploy workflow twice in succession on the +same commit. Verify that the first run provisions/updates the infrastructure +and the second run reports no changes. Verify the IAM role cannot perform +actions outside its deployment scope (e.g., cannot create new IAM users, +cannot access unrelated S3 buckets, cannot modify billing). + +**Acceptance Scenarios**: + +1. **Given** a merged PR to `main`, **When** the deploy workflow runs, + **Then** the etelemetry server is running on AWS at `et.dandiproject.org` + with all services healthy (server, database, geolocation). + +2. **Given** a successful deployment, **When** the deploy workflow runs + again with no code changes, **Then** it completes successfully and + reports no infrastructure changes (idempotent). + +3. **Given** the deploy workflow IAM role, **When** an attacker obtains + the temporary credentials, **Then** they can only perform deployment- + related actions (push images, update services) and cannot escalate + privileges, access other AWS accounts, or modify IAM policies. + +4. **Given** a deployment failure (e.g., health check fails), **When** the + workflow detects the failure, **Then** it rolls back to the previous + working state and reports the failure clearly in the workflow log. + +--- + +### User Story 2 - Initial Infrastructure Bootstrap (Priority: P2) + +A new operator runs a one-time bootstrap process to create the AWS +infrastructure (compute, networking, storage, IAM roles) for the first +time. After bootstrap, all subsequent updates are handled by the automated +deploy workflow. The bootstrap is also idempotent — running it again updates +existing resources rather than creating duplicates. + +**Why this priority**: The initial setup must happen before automated +deploys can work, but it only runs once (or when infrastructure changes +are needed). + +**Independent Test**: Run the bootstrap on a clean AWS account. Verify all +resources are created. Run it again and verify no duplicates. Tear down and +re-bootstrap to verify it works from scratch. + +**Acceptance Scenarios**: + +1. **Given** a clean AWS account with appropriate credentials, **When** the + bootstrap process runs, **Then** all required infrastructure is created: + compute instance, security groups, persistent storage, IAM roles, and + container registry. + +2. **Given** existing infrastructure from a prior bootstrap, **When** the + bootstrap runs again, **Then** it updates existing resources without + creating duplicates. + +3. **Given** a bootstrapped environment, **When** the automated deploy + workflow (US1) runs for the first time, **Then** it successfully + deploys the application using the infrastructure created by bootstrap. + +--- + +### User Story 3 - Secret Management (Priority: P3) + +Deployment secrets (database password, MaxMind license key, GitHub token) +are stored securely and injected at deploy time without being exposed in +workflow logs, container images, or version control. The secrets are +managed through a single secure channel. + +**Why this priority**: Secrets management is critical for security but +builds on top of the deployment infrastructure. + +**Independent Test**: Deploy the service, then inspect workflow logs, +container environment, and image layers. Verify no secrets are visible +in any of these locations. + +**Acceptance Scenarios**: + +1. **Given** secrets stored in the secure secret store, **When** the deploy + workflow runs, **Then** secrets are injected into the running containers + at runtime without appearing in build logs, image layers, or workflow + output. + +2. **Given** a deployed service, **When** an operator rotates a secret + (e.g., database password), **Then** the change can be applied by + re-running the deploy workflow without rebuilding the container image. + +--- + +### Edge Cases + +- What happens when the AWS region is unavailable? The deploy workflow + MUST fail clearly and not leave infrastructure in a partial state. +- What happens when the container health check fails after deploy? The + deployment MUST roll back to the previous working version automatically. +- What happens when the IAM role's temporary credentials expire mid-deploy? + The workflow MUST use credentials with sufficient duration for the full + deployment process. +- What happens when two deploy workflows run concurrently (e.g., rapid + merges)? The system MUST serialize deployments or fail the second + gracefully with a clear message. +- What happens when the bootstrap resources are manually modified outside + the infrastructure-as-code tool? The next bootstrap run MUST detect + drift and reconcile to the declared state. + +## Requirements *(mandatory)* + +### Functional Requirements + +- **FR-001**: The deployment MUST be idempotent — running the same + deployment with the same code and configuration MUST produce no + infrastructure changes on subsequent runs. +- **FR-002**: The deployment MUST use a dedicated IAM role with only the + minimum permissions required to deploy the etelemetry application. The + role MUST NOT have administrator access, IAM write access, or access + to unrelated AWS services. +- **FR-003**: The GitHub Actions workflow MUST authenticate to AWS using + OpenID Connect (OIDC) federation — no long-lived AWS access keys stored + as GitHub secrets. The OIDC trust policy MUST be scoped to the specific + repository and branch. +- **FR-004**: The deployment MUST include a health check that verifies the + etelemetry server is responding correctly after deployment. If the health + check fails, the deployment MUST roll back automatically. +- **FR-005**: The bootstrap process MUST create all required AWS resources + in a single idempotent operation: compute, networking, persistent storage + for database and GeoIP data, IAM roles, and container image storage. +- **FR-006**: Secrets (database password, MaxMind key, GitHub token) MUST + be stored in a secure secret store and injected at container runtime. + Secrets MUST NOT appear in container images, build logs, or workflow + output. +- **FR-007**: The deployment MUST support zero-downtime updates — the new + version MUST be running and healthy before the old version is stopped. +- **FR-008**: The infrastructure definition MUST be version-controlled + alongside the application code so that infrastructure changes go through + the same review process as code changes. +- **FR-009**: The IAM role MUST use a trust policy that restricts + assumption to the specific GitHub repository (`sensein/etelemetry`) and + optionally a specific branch (`main`). +- **FR-010**: The deployment MUST configure the domain `et.dandiproject.org` + to point to the deployed service with TLS termination. +- **FR-011**: The deployment MUST provision persistent storage for + PostgreSQL data that survives container restarts and redeployments. +- **FR-012**: The deployment MUST include automated TLS certificate + provisioning and renewal for `et.dandiproject.org`. +- **FR-013**: Concurrent deployment attempts MUST be serialized or the + second attempt MUST fail gracefully without corrupting state. + +### Key Entities + +- **DeployRole**: The IAM role assumed by GitHub Actions during deployment. + Has a trust policy limiting who can assume it and a permission policy + limiting what actions it can perform. +- **Infrastructure State**: The declared state of all AWS resources + (compute, networking, storage, IAM). Stored as code, applied + idempotently. +- **Deployment Artifact**: The container image built from the etelemetry + source code, tagged with the git commit SHA, stored in a container + registry. +- **Secret**: A sensitive configuration value (database password, API key) + stored in a secure secret store and injected at runtime. + +### Assumptions + +- The AWS account is already created and an initial user with sufficient + privileges exists to run the bootstrap (one-time). +- The domain `et.dandiproject.org` DNS is managed externally (the + deployment creates the necessary records or provides the values to + configure manually). +- The etelemetry application is already containerized (Dockerfile exists + from feature 001). +- GitHub Actions OIDC provider is available in the target AWS account + (or will be created during bootstrap). +- A single EC2 instance (or equivalent compute) is sufficient for the + current load (~1M pings/week). Horizontal scaling is out of scope. + +## Success Criteria *(mandatory)* + +### Measurable Outcomes + +- **SC-001**: A code change merged to `main` is live on + `et.dandiproject.org` within 15 minutes of merge without manual + intervention. +- **SC-002**: Running the deployment workflow twice on the same commit + produces no infrastructure changes on the second run. +- **SC-003**: The IAM deployment role has fewer than 20 distinct IAM + permissions (not `*` actions), covering only the services required + for deployment. +- **SC-004**: No long-lived AWS credentials (access key ID / secret key) + are stored in GitHub secrets — authentication uses only OIDC federation. +- **SC-005**: A failed deployment automatically rolls back and the + previous version continues serving traffic with zero downtime. +- **SC-006**: An operator can bootstrap the full infrastructure from + scratch in under 30 minutes by following the documentation. +- **SC-007**: All secrets are verifiably absent from container images, + workflow logs, and the version-controlled infrastructure definition. diff --git a/specs/002-aws-least-privilege-deploy/tasks.md b/specs/002-aws-least-privilege-deploy/tasks.md new file mode 100644 index 0000000..f081318 --- /dev/null +++ b/specs/002-aws-least-privilege-deploy/tasks.md @@ -0,0 +1,178 @@ +# Tasks: AWS Least-Privilege Idempotent Deployment + +**Input**: Design documents from `/specs/002-aws-least-privilege-deploy/` +**Prerequisites**: plan.md (required), spec.md (required), research.md, contracts/iam-policy.md, quickstart.md + +**Tests**: Included — constitution mandates test-driven (Principle IV). OpenTofu `plan` validates before `apply`; health checks verify deployments. + +**Organization**: Tasks grouped by user story for independent implementation. + +## Format: `[ID] [P?] [Story] Description` + +- **[P]**: Can run in parallel (different files, no dependencies) +- **[Story]**: Which user story (US1–US3) +- Exact file paths from plan.md project structure + +## Phase 1: Setup + +**Purpose**: Create `infra/` directory structure and OpenTofu project skeleton + +- [x] T001 Create `infra/versions.tf` — required providers (aws ~> 5.0, opentofu backend config) +- [x] T002 Create `infra/variables.tf` — input variables: aws_region (default us-east-1), instance_type (default t3.small), domain (default et.dandiproject.org), github_repo (default sensein/etelemetry), project_name (default etelemetry) +- [x] T003 [P] Create `infra/outputs.tf` — output deploy_role_arn, instance_id, ecr_repository_url, elastic_ip, ssm_parameter_prefix +- [x] T004 [P] Create `infra/state.tf` — S3 backend configuration for OpenTofu state with DynamoDB locking table + +--- + +## Phase 2: Foundational (Blocking Prerequisites) + +**Purpose**: IAM OIDC provider + core IAM roles — MUST complete before any deployment works + +**CRITICAL**: No deployment tasks can proceed without IAM roles + +- [x] T005 Create `infra/iam.tf` — GitHub OIDC identity provider, deploy role (`etelemetry-deploy`) with trust policy scoped to `repo:sensein/etelemetry:ref:refs/heads/main` and 14-action permission policy per contracts/iam-policy.md, PLUS EC2 instance role (`etelemetry-instance`) with instance profile, SSM managed instance core policy, SSM parameter read (`/etelemetry/*`), ECR pull permissions +- [x] T006 [P] Create `infra/ecr.tf` — ECR private repository `etelemetry`, lifecycle policy (keep last 10 images), image scanning on push enabled +- [x] T007 Write IAM policy validation test in `infra/tests/test_iam_policy.py` — parse the OpenTofu plan output JSON and verify: deploy role has exactly 14 actions, no `*` resource wildcards on sensitive actions, no `iam:*` permissions, trust policy scoped to correct repo/branch + +**Checkpoint**: IAM roles and ECR ready — deployment infrastructure can be built + +--- + +## Phase 3: User Story 1 — Automated Deployment on Push (Priority: P1) + +**Goal**: Merge to main → GitHub Actions builds image, pushes to ECR, deploys via SSM, health-checks, rolls back on failure + +**Independent Test**: Push to main, verify deploy workflow runs, service is live at et.dandiproject.org, second run is idempotent + +### Tests for User Story 1 + +> **NOTE: Write tests FIRST, verify they would fail without implementation** + +- [x] T008 [US1] Write deploy smoke test in `infra/tests/test_deploy.sh` — after deployment: curl health endpoint, verify 200 + correct JSON shape, verify TLS certificate valid for et.dandiproject.org, verify no SSH port open (port 22 closed), verify concurrent deploy triggers GitHub Actions concurrency queue (second workflow waits or cancels) + +### Implementation for User Story 1 + +- [x] T009 [US1] Create `infra/ec2.tf` — EC2 instance (var.instance_type), Amazon Linux 2023 AMI, instance profile from T005, user data script that installs Docker + Docker Compose + docker-rollout + SSM agent, security group `etelemetry-sg` (inbound: 80 HTTP, 443 HTTPS only; NO port 22; outbound: all), EBS root volume 20GB gp3, Elastic IP association, tags for identification +- [x] T010 [P] [US1] Create `infra/scripts/deploy.sh` — SSM-executed deploy script: authenticate to ECR, pull image by tag, write .env from SSM parameters, run `docker rollout server` for zero-downtime update, run health check (curl localhost:8000/), on failure: log error and exit non-zero (docker-rollout keeps old container running automatically) +- [x] T011 [P] [US1] Create `infra/scripts/user-data.sh` — EC2 user data bootstrap: install Docker, Docker Compose plugin, docker-rollout, clone repo to /opt/etelemetry, pull initial images, start services +- [x] T012 [US1] Create `deploy/Caddyfile` — Caddy reverse proxy config for et.dandiproject.org: automatic TLS, handle /et/* (strip prefix, reverse_proxy server:8000), handle /* (reverse_proxy server:8000), IP-stripped log format +- [x] T013 [US1] Update `deploy/docker-compose.yml` — replace nginx service with caddy service (caddy:alpine image, ports 80:80 + 443:443, volume for caddy data + Caddyfile), add healthcheck to server service (curl localhost:8000/), remove nginx-related configs +- [x] T014 [US1] Rewrite `.github/workflows/deploy.yml` — replace SSH-based deploy with: (1) `aws-actions/configure-aws-credentials@v4` with OIDC role-to-assume, (2) build Docker image, (3) authenticate to ECR + push image tagged with git SHA, (4) `aws ssm send-command` to run deploy.sh on target instance, (5) poll command status, (6) report success/failure, (7) concurrency group `deploy-production` to serialize deploys +- [x] T015 [US1] **DNS prerequisite gate**: document in `docs/bootstrap.md` that the DNS A record for `et.dandiproject.org` MUST point to the Elastic IP BEFORE the first deploy with TLS (Caddy needs the domain to resolve for ACME challenge). Include the Elastic IP in `tofu output`. +- [x] T016 [US1] Run `tofu plan` and verify: all resources declared, no errors, deploy role has correct permissions, security group has no port 22 +- [x] T017 [US1] Run `tofu apply` on test/staging environment (or `tofu plan` in CI) and verify idempotency — second apply shows 0 changes + +**Checkpoint**: Automated deployment working. Push to main → build → ECR → SSM deploy → health check → live. + +--- + +## Phase 4: User Story 2 — Infrastructure Bootstrap (Priority: P2) + +**Goal**: One-time bootstrap creates all AWS resources idempotently; subsequent runs detect no changes + +**Independent Test**: Run bootstrap from scratch, verify all resources created. Run again, verify 0 changes. + +### Implementation for User Story 2 + +- [x] T018 [US2] Create `infra/main.tf` — AWS provider configuration (region from var), data sources for latest Amazon Linux 2023 AMI, account ID +- [x] T019 [US2] Create bootstrap documentation in `docs/bootstrap.md` — step-by-step guide: prerequisites (AWS account, tofu installed, AWS CLI configured), S3 bucket + DynamoDB table creation for state backend, `tofu init`, `tofu plan`, `tofu apply`, DNS configuration, secret storage, verify outputs +- [x] T020 [US2] Create `infra/Makefile` — targets: `init` (tofu init), `plan` (tofu plan), `apply` (tofu apply), `destroy` (tofu destroy with confirmation), `fmt` (tofu fmt), `validate` (tofu validate) +- [x] T021 [US2] Add OpenTofu validation to `.github/workflows/ci.yml` — on PR: run `tofu init -backend=false`, `tofu validate`, `tofu fmt -check` for the infra/ directory +- [x] T022 [US2] Test idempotency — run `tofu apply` twice, verify second run shows "No changes. Your infrastructure matches the configuration." + +**Checkpoint**: Full infrastructure bootstrappable from scratch. Idempotent. + +--- + +## Phase 5: User Story 3 — Secret Management (Priority: P3) + +**Goal**: Secrets stored in SSM Parameter Store, injected at runtime, never in images/logs/VCS + +**Independent Test**: Deploy, then verify secrets absent from image layers, workflow logs, and Caddyfile/compose files in VCS + +### Tests for User Story 3 + +> **NOTE: Write tests FIRST** + +- [x] T023 [US3] Write secret audit test in `infra/tests/test_secrets.sh` — verify: no SecureString values in `tofu plan` output, no secrets in `docker inspect` of running containers, no secrets in workflow log artifacts, .env file on EC2 has correct permissions (600, root-owned) + +### Implementation for User Story 3 + +- [x] T024 [US3] Create `infra/ssm.tf` — SSM parameter definitions for /etelemetry/POSTGRES_PASSWORD, /etelemetry/MAXMIND_ACCOUNT_ID, /etelemetry/MAXMIND_LICENSE_KEY, /etelemetry/GITHUB_TOKEN (all SecureString type, placeholder values that operator replaces via CLI) +- [x] T025 [US3] Update `infra/scripts/deploy.sh` — add step to fetch all parameters from SSM `/etelemetry/*` path, write to `/opt/etelemetry/deploy/.env` with permissions 600, mask values in script output +- [x] T026 [US3] Add secret rotation documentation to `docs/bootstrap.md` — how to update a secret via `aws ssm put-parameter --overwrite`, then re-run deploy workflow to pick up new value + +**Checkpoint**: Secrets managed securely. Rotatable without image rebuild. + +--- + +## Phase 6: Polish & Cross-Cutting Concerns + +**Purpose**: Documentation, cleanup, security hardening + +- [x] T027 Update `docs/external-setup-guide.md` — replace SSH-based instructions with OIDC + SSM approach, add OpenTofu bootstrap steps, update GitHub repo settings section (remove SSH key secrets, add OIDC role ARN variable) +- [x] T028 [P] Update `docs/phase-log.md` — add entry for feature 002 implementation +- [x] T029 [P] Security review — verify: no port 22 in security group, OIDC trust scoped to repo+branch, deploy role has no IAM write, SSM parameters encrypted, .env not in version control +- [x] T030 Remove SSH-related secrets documentation from `docs/external-setup-guide.md` — remove references to AWS_SSH_PRIVATE_KEY, SSH key setup, SCP commands +- [x] T031 Run full CI pipeline — verify OpenTofu validation, application tests, and deploy workflow all pass + +--- + +## Dependencies & Execution Order + +### Phase Dependencies + +- **Setup (Phase 1)**: No dependencies — start immediately +- **Foundational (Phase 2)**: Depends on Phase 1 (needs variables.tf, versions.tf) +- **US1 (Phase 3)**: Depends on Phase 2 (needs IAM roles, ECR) +- **US2 (Phase 4)**: Depends on Phase 2 (needs all infra files to exist) +- **US3 (Phase 5)**: Depends on Phase 2 (needs IAM, SSM) +- **Polish (Phase 6)**: Depends on all user stories + +### User Story Dependencies + +- **US1 (P1)**: After Phase 2 — core deployment +- **US2 (P2)**: After Phase 2 — can run in parallel with US1 (docs + main.tf) +- **US3 (P3)**: After Phase 2 — can run in parallel with US1 (SSM params) + +### Parallel Opportunities + +After Phase 2: +- US1 (EC2, deploy workflow), US2 (bootstrap docs, main.tf), US3 (SSM params) can all start in parallel since they touch different files + +### Within Each User Story + +- Tests first (TDD for deploy smoke test and secret audit) +- Infrastructure files before workflow files +- Workflow before validation + +--- + +## Implementation Strategy + +### MVP First (User Story 1 Only) + +1. Complete Phase 1: Setup (4 tasks) +2. Complete Phase 2: Foundational (4 tasks) +3. Complete Phase 3: US1 — Automated Deploy (10 tasks) +4. **STOP and VALIDATE**: Push to main, verify deploy works +5. Iterate + +### Incremental Delivery + +1. Setup + Foundational → IAM roles ready +2. US1 → Automated deploy working → Verify +3. US2 → Bootstrap documented, CI validates infra +4. US3 → Secrets in SSM, rotation documented +5. Polish → Security review, docs updated + +--- + +## Notes + +- [P] tasks = different files, no dependencies +- [Story] label maps task to specific user story +- Commit after each task or logical group +- `tofu plan` acts as the "test" for infrastructure changes +- All IAM permissions scoped to specific resource ARNs — no wildcards +- Constitution: secrets via SSM (Principle II), conventional commits (Principle III), infra as code (Principle VIII) diff --git a/src/etelemetry/__init__.py b/src/etelemetry/__init__.py new file mode 100644 index 0000000..17cfa69 --- /dev/null +++ b/src/etelemetry/__init__.py @@ -0,0 +1,16 @@ +"""etelemetry — lightweight version check telemetry client.""" + +from .client import check_available_version, get_project +from .errors import BadVersionError + +try: + from ._version import __version__ +except ImportError: + __version__ = "0.0.0+unknown" + +__all__ = [ + "BadVersionError", + "check_available_version", + "get_project", + "__version__", +] diff --git a/src/etelemetry/_version.py b/src/etelemetry/_version.py new file mode 100644 index 0000000..a9b37b3 --- /dev/null +++ b/src/etelemetry/_version.py @@ -0,0 +1,34 @@ +# file generated by setuptools-scm +# don't change, don't track in version control + +__all__ = [ + "__version__", + "__version_tuple__", + "version", + "version_tuple", + "__commit_id__", + "commit_id", +] + +TYPE_CHECKING = False +if TYPE_CHECKING: + from typing import Tuple + from typing import Union + + VERSION_TUPLE = Tuple[Union[int, str], ...] + COMMIT_ID = Union[str, None] +else: + VERSION_TUPLE = object + COMMIT_ID = object + +version: str +__version__: str +__version_tuple__: VERSION_TUPLE +version_tuple: VERSION_TUPLE +commit_id: COMMIT_ID +__commit_id__: COMMIT_ID + +__version__ = version = '0.0.0.dev35+g8d6d4584f.d20260323' +__version_tuple__ = version_tuple = (0, 0, 0, 'dev35', 'g8d6d4584f.d20260323') + +__commit_id__ = commit_id = None diff --git a/src/etelemetry/client.py b/src/etelemetry/client.py new file mode 100644 index 0000000..e2d04c4 --- /dev/null +++ b/src/etelemetry/client.py @@ -0,0 +1,184 @@ +"""etelemetry client — version check and telemetry reporting.""" + +from __future__ import annotations + +import logging +import os + +from packaging.version import Version + +from .config import resolve_url +from .errors import BadVersionError + +try: + import ci_info +except ImportError: + ci_info = None # type: ignore[assignment] + +_available_version_checked: dict | None = None + + +def _etrequest(endpoint: str, method: str = "get", **kwargs) -> dict: + """Make a request to the etelemetry server. + + Lazy-imports ``requests`` to avoid penalizing startup time. + """ + from requests import ConnectionError, ReadTimeout, request + + if kwargs.get("timeout") is None: + kwargs["timeout"] = 5 + + params: dict = kwargs.pop("params", {}) + # Send CI information as query parameters + if ci_info is not None and ci_info.is_ci(): + ci_params = ci_info.info() + if isinstance(ci_params, dict): + params.update(ci_params) + params["ci"] = "true" + + try: + res = request(method, endpoint, params=params, **kwargs) + except ConnectionError: + raise RuntimeError("Connection to server could not be made") + except ReadTimeout: + raise RuntimeError( + f"No response from server in {kwargs.get('timeout')} seconds" + ) + res.raise_for_status() + return res.json() + + +def get_project(repo: str, *, server_url: str | None = None, **rargs) -> dict | None: + """Fetch latest version info from the etelemetry server. + + Parameters + ---------- + repo : str + GitHub repository as ``/``. + server_url : str or None + Explicit server URL (highest precedence). See + :func:`etelemetry.config.resolve_url` for the full chain. + **rargs + Additional keyword arguments passed to ``requests.request``. + + Returns + ------- + dict or None + Dictionary with ``version`` and ``bad_versions`` keys, + or None if telemetry is disabled. + + Raises + ------ + ValueError + If ``repo`` is not in ``owner/project`` format. + RuntimeError + If the server is unreachable. + """ + if "NO_ET" in os.environ: + return None + if "/" not in repo: + raise ValueError("Invalid repository — expected 'owner/project' format") + + base_url = resolve_url(server_url=server_url) + if base_url is None: + return None + + # Ensure trailing slash + if not base_url.endswith("/"): + base_url += "/" + + endpoint = f"{base_url}projects/{repo}" + return _etrequest(endpoint, **rargs) + + +def check_available_version( + project: str, + version: str, + lgr: logging.Logger | None = None, + raise_exception: bool = False, + *, + server_url: str | None = None, +) -> dict | None: + """Check and report if a newer version of a project is available. + + Safe to call multiple times — only checks once per session. + + Parameters + ---------- + project : str + GitHub repository as ``owner/project``. + version : str + The local version string. + lgr : logging.Logger or None + Logger instance. Creates one if not provided. + raise_exception : bool + If True, raise ``BadVersionError`` when a bad version is detected. + + Returns + ------- + dict or None + Version info dict, or None if check failed. + """ + global _available_version_checked + if _available_version_checked is not None: + return _available_version_checked + + if lgr is None: + lgr = logging.getLogger("et-client") + + latest = {"version": "Unknown", "bad_versions": []} + ret = None + try: + ret = get_project(project, server_url=server_url) + except Exception as e: + lgr.debug("Could not check %s for version updates: %s", project, e) + return None + finally: + if ret: + latest.update(**ret) + try: + local_version = Version(version) + remote_version = Version(latest["version"]) + except Exception: + lgr.debug("Could not parse version for %s", project) + _available_version_checked = latest + return latest + + if local_version < remote_version: + lgr.warning( + "A newer version (%s) of %s is available. " + "You are using %s", + latest["version"], + project, + version, + ) + elif remote_version < local_version: + lgr.debug( + "Running a newer version (%s) of %s than available (%s)", + version, + project, + latest["version"], + ) + else: + lgr.debug( + "No newer (than %s) version of %s found available", + version, + project, + ) + + if latest["bad_versions"] and any( + local_version == Version(ver) + for ver in latest["bad_versions"] + ): + message = ( + f"You are using a version of {project} with a critical " + "bug. Please use a different version." + ) + if raise_exception: + raise BadVersionError(message) + else: + lgr.critical(message) + + _available_version_checked = latest + + return latest diff --git a/src/etelemetry/config.py b/src/etelemetry/config.py new file mode 100644 index 0000000..1f723e1 --- /dev/null +++ b/src/etelemetry/config.py @@ -0,0 +1,47 @@ +"""etelemetry client configuration. + +Server URL resolution precedence (highest to lowest): + 1. ``server_url`` function parameter + 2. ``ETELEMETRY_URL`` environment variable + 3. ``default_url`` module-level attribute (for library authors) + 4. Hardcoded default +""" + +from __future__ import annotations + +import os + +_HARDCODED_URL = "https://et.dandiproject.org/" + +#: Library authors can set this to point their package's clients at a +#: custom etelemetry instance without requiring end-users to set an +#: environment variable. Example:: +#: +#: import etelemetry +#: etelemetry.config.default_url = "https://my-instance.org/" +default_url: str | None = None + + +def resolve_url(server_url: str | None = None) -> str | None: + """Resolve the etelemetry server URL. + + Parameters + ---------- + server_url : str or None + Explicit URL passed by the caller (highest precedence). + + Returns + ------- + str or None + The resolved URL, or None if telemetry is disabled via ``NO_ET``. + """ + if "NO_ET" in os.environ: + return None + if server_url: + return server_url + env_url = os.environ.get("ETELEMETRY_URL") + if env_url: + return env_url + if default_url: + return default_url + return _HARDCODED_URL diff --git a/src/etelemetry/errors.py b/src/etelemetry/errors.py new file mode 100644 index 0000000..8304b21 --- /dev/null +++ b/src/etelemetry/errors.py @@ -0,0 +1,7 @@ +"""etelemetry exceptions.""" + + +class BadVersionError(RuntimeError): + """Local version is known to contain a critical bug.""" + + pass diff --git a/tests/__init__.py b/tests/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/contract/__init__.py b/tests/contract/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/__init__.py b/tests/integration/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/integration/test_client_server_e2e.py b/tests/integration/test_client_server_e2e.py new file mode 100644 index 0000000..f4e059e --- /dev/null +++ b/tests/integration/test_client_server_e2e.py @@ -0,0 +1,247 @@ +"""End-to-end integration test: real client library talking to real server. + +Uses httpx ASGITransport to connect the etelemetry client library to the +FastAPI server without needing a real TCP socket. The client's HTTP call +is intercepted and routed through the ASGI app in-process. +""" + +from __future__ import annotations + +import logging +from unittest.mock import AsyncMock, MagicMock, patch + +import httpx +import pytest +import pytest_asyncio +from etelemetry_server.app import create_app +from etelemetry_server.models import Base, Project +from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + +TEST_DATABASE_URL = "sqlite+aiosqlite:///:memory:" + + +@pytest_asyncio.fixture +async def engine(): + eng = create_async_engine(TEST_DATABASE_URL, echo=False) + async with eng.begin() as conn: + await conn.run_sync(Base.metadata.create_all) + yield eng + await eng.dispose() + + +@pytest_asyncio.fixture +async def session_factory(engine): + return async_sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) + + +@pytest_asyncio.fixture +async def seed_project(session_factory): + async with session_factory() as session: + project = Project( + owner="testorg", + repo="testrepo", + active=True, + latest_version="3.0.0", + bad_versions=["1.0.0"], + ) + session.add(project) + await session.commit() + await session.refresh(project) + return project + + +@pytest.fixture +def app(session_factory, seed_project): + application = create_app() + application.state.allowlist = [ + {"owner": "testorg", "repo": "testrepo"}, + ] + application.state.settings = MagicMock( + GITHUB_TOKEN=None, + CACHE_TTL_SECONDS=3600, + TIME_BUCKET_HOURS=1, + MAXMIND_DB_PATH="/nonexistent", + ) + application.state.geolocator = None + application.state.http_client = httpx.AsyncClient() + + from etelemetry_server.db import get_db + + async def _override_get_db(): + async with session_factory() as session: + try: + yield session + await session.commit() + except Exception: + await session.rollback() + raise + + application.dependency_overrides[get_db] = _override_get_db + return application + + +def _make_asgi_request(app, method, url, **kwargs): + """Route a requests-style call through the ASGI app in-process. + + This replaces the real HTTP call that ``etelemetry.client._etrequest`` + would make, routing it through the FastAPI app via httpx ASGITransport. + """ + import asyncio + + # Extract just the path + query from the URL + from urllib.parse import urlparse + + parsed = urlparse(url) + path = parsed.path + if parsed.query: + path = f"{path}?{parsed.query}" + + # Merge any extra params into the query string + params = kwargs.pop("params", {}) + + async def _do_request(): + async with httpx.AsyncClient( + transport=httpx.ASGITransport(app=app), + base_url="http://testserver", + ) as client: + resp = await client.request(method, path, params=params, **kwargs) + return resp + + resp = asyncio.get_event_loop().run_until_complete(_do_request()) + + # Wrap in a requests-like response object + mock_resp = MagicMock() + mock_resp.status_code = resp.status_code + mock_resp.json.return_value = resp.json() + mock_resp.raise_for_status.side_effect = ( + None if resp.is_success else Exception(f"HTTP {resp.status_code}") + ) + return mock_resp + + +# --------------------------------------------------------------------------- +# Tests +# --------------------------------------------------------------------------- + + +class TestClientServerE2E: + """Full round-trip: etelemetry client library → FastAPI server → DB.""" + + @pytest.fixture(autouse=True) + def _reset_cache(self): + import etelemetry.client + etelemetry.client._available_version_checked = None + yield + etelemetry.client._available_version_checked = None + + @pytest.fixture(autouse=True) + def _clean_env(self, monkeypatch): + monkeypatch.delenv("NO_ET", raising=False) + monkeypatch.delenv("ETELEMETRY_URL", raising=False) + + def _patch_requests(self, app): + """Patch requests.request to route through the ASGI app.""" + import functools + return patch( + "requests.request", + side_effect=functools.partial(_make_asgi_request, app), + ) + + def test_get_project_returns_version(self, app): + """Client get_project() returns version info from the server.""" + with ( + self._patch_requests(app), + patch( + "etelemetry_server.services.version_checker.VersionChecker.get_version_info", + new_callable=AsyncMock, + return_value={"version": "3.0.0", "bad_versions": ["1.0.0"]}, + ), + ): + import etelemetry + result = etelemetry.get_project( + "testorg/testrepo", + server_url="http://testserver", + ) + + assert result is not None + assert result["version"] == "3.0.0" + assert result["bad_versions"] == ["1.0.0"] + + def test_check_available_version_warns_outdated(self, app, caplog): + """Client warns when local version is behind server version.""" + with ( + self._patch_requests(app), + patch( + "etelemetry_server.services.version_checker.VersionChecker.get_version_info", + new_callable=AsyncMock, + return_value={"version": "3.0.0", "bad_versions": []}, + ), + caplog.at_level(logging.WARNING, logger="et-client"), + ): + import etelemetry + result = etelemetry.check_available_version( + "testorg/testrepo", + "2.0.0", + server_url="http://testserver", + ) + + assert result is not None + assert result["version"] == "3.0.0" + assert "newer version" in caplog.text.lower() + + def test_check_available_version_detects_bad_version(self, app): + """Client raises BadVersionError for known bad versions.""" + with ( + self._patch_requests(app), + patch( + "etelemetry_server.services.version_checker.VersionChecker.get_version_info", + new_callable=AsyncMock, + return_value={"version": "3.0.0", "bad_versions": ["1.0.0"]}, + ), + ): + import etelemetry + with pytest.raises(etelemetry.BadVersionError, match="critical bug"): + etelemetry.check_available_version( + "testorg/testrepo", + "1.0.0", + raise_exception=True, + server_url="http://testserver", + ) + + def test_project_not_on_allowlist_returns_error(self, app): + """Client gets an error for projects not on the allowlist.""" + with self._patch_requests(app): + import etelemetry + with pytest.raises(Exception): + etelemetry.get_project( + "unknown/repo", + server_url="http://testserver", + ) + + def test_response_shape_matches_old_client_expectations(self, app): + """Response has 'version' and 'bad_versions' keys — backward compat.""" + with ( + self._patch_requests(app), + patch( + "etelemetry_server.services.version_checker.VersionChecker.get_version_info", + new_callable=AsyncMock, + return_value={"version": "3.0.0", "bad_versions": ["1.0.0"]}, + ), + ): + import etelemetry + result = etelemetry.get_project( + "testorg/testrepo", + server_url="http://testserver", + ) + + # These are the exact keys the old client reads + assert "version" in result + assert "bad_versions" in result + # Should NOT have internal-only keys + assert "status" not in result + assert "last_update" not in result + assert "cached" not in result diff --git a/tests/unit/__init__.py b/tests/unit/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tests/unit/test_client.py b/tests/unit/test_client.py new file mode 100644 index 0000000..4b38117 --- /dev/null +++ b/tests/unit/test_client.py @@ -0,0 +1,122 @@ +"""Unit tests for etelemetry client.""" + +from __future__ import annotations + +import logging +from unittest.mock import patch + +import pytest + +import etelemetry +from etelemetry.client import ( + check_available_version, + get_project, +) +from etelemetry.errors import BadVersionError + + +@pytest.fixture(autouse=True) +def _reset_cache(): + """Reset the global version check cache before each test.""" + import etelemetry.client + etelemetry.client._available_version_checked = None + yield + etelemetry.client._available_version_checked = None + + +@pytest.fixture(autouse=True) +def _clean_env(monkeypatch): + """Ensure NO_ET and ETELEMETRY_URL are not set.""" + monkeypatch.delenv("NO_ET", raising=False) + monkeypatch.delenv("ETELEMETRY_URL", raising=False) + + +class TestGetProject: + def test_returns_none_when_disabled(self, monkeypatch): + monkeypatch.setenv("NO_ET", "1") + assert get_project("owner/repo") is None + + def test_raises_on_invalid_repo(self): + with pytest.raises(ValueError, match="Invalid repository"): + get_project("no-slash-here") + + @patch("etelemetry.client._etrequest") + def test_returns_version_info(self, mock_req): + mock_req.return_value = {"version": "1.0.0", "bad_versions": []} + result = get_project("owner/repo") + assert result["version"] == "1.0.0" + assert result["bad_versions"] == [] + + @patch("etelemetry.client._etrequest") + def test_uses_custom_env_url(self, mock_req, monkeypatch): + monkeypatch.setenv("ETELEMETRY_URL", "https://custom.example.com") + mock_req.return_value = {"version": "1.0.0", "bad_versions": []} + get_project("owner/repo") + call_args = mock_req.call_args + assert "custom.example.com" in call_args[0][0] + + +class TestCheckAvailableVersion: + @patch("etelemetry.client.get_project") + def test_warns_on_outdated_version(self, mock_gp, caplog): + mock_gp.return_value = {"version": "2.0.0", "bad_versions": []} + with caplog.at_level(logging.WARNING, logger="et-client"): + result = check_available_version("owner/repo", "1.0.0") + assert result["version"] == "2.0.0" + assert "newer version" in caplog.text.lower() + + @patch("etelemetry.client.get_project") + def test_critical_on_bad_version(self, mock_gp, caplog): + mock_gp.return_value = {"version": "2.0.0", "bad_versions": ["1.0.0"]} + with caplog.at_level(logging.CRITICAL, logger="et-client"): + check_available_version("owner/repo", "1.0.0") + assert "critical bug" in caplog.text.lower() + + @patch("etelemetry.client.get_project") + def test_raises_bad_version_error(self, mock_gp): + mock_gp.return_value = {"version": "2.0.0", "bad_versions": ["1.0.0"]} + with pytest.raises(BadVersionError, match="critical bug"): + check_available_version( + "owner/repo", "1.0.0", raise_exception=True + ) + + @patch("etelemetry.client.get_project") + def test_debug_on_latest_version(self, mock_gp, caplog): + mock_gp.return_value = {"version": "1.0.0", "bad_versions": []} + with caplog.at_level(logging.DEBUG, logger="et-client"): + result = check_available_version("owner/repo", "1.0.0") + assert result["version"] == "1.0.0" + + @patch("etelemetry.client.get_project") + def test_caches_result(self, mock_gp): + mock_gp.return_value = {"version": "2.0.0", "bad_versions": []} + result1 = check_available_version("owner/repo", "1.0.0") + result2 = check_available_version("owner/repo", "1.0.0") + assert result1 is result2 + mock_gp.assert_called_once() + + @patch("etelemetry.client.get_project") + def test_returns_none_on_connection_error(self, mock_gp): + mock_gp.side_effect = RuntimeError("Connection failed") + result = check_available_version("owner/repo", "1.0.0") + assert result is None + + def test_no_et_disables_get_project(self, monkeypatch): + monkeypatch.setenv("NO_ET", "1") + result = get_project("owner/repo") + assert result is None + + +class TestPublicAPI: + def test_exports_get_project(self): + assert hasattr(etelemetry, "get_project") + + def test_exports_check_available_version(self): + assert hasattr(etelemetry, "check_available_version") + + def test_exports_bad_version_error(self): + assert hasattr(etelemetry, "BadVersionError") + assert issubclass(etelemetry.BadVersionError, RuntimeError) + + def test_exports_version(self): + assert hasattr(etelemetry, "__version__") diff --git a/tests/unit/test_config.py b/tests/unit/test_config.py new file mode 100644 index 0000000..4d9ed61 --- /dev/null +++ b/tests/unit/test_config.py @@ -0,0 +1,61 @@ +"""Unit tests for etelemetry.config URL resolution.""" + +from __future__ import annotations + +import pytest + +import etelemetry.config as config +from etelemetry.config import resolve_url + + +@pytest.fixture(autouse=True) +def _clean_state(monkeypatch): + """Reset module-level default_url and env vars.""" + monkeypatch.delenv("NO_ET", raising=False) + monkeypatch.delenv("ETELEMETRY_URL", raising=False) + monkeypatch.setattr(config, "default_url", None) + + +class TestResolveUrl: + def test_param_takes_highest_precedence(self, monkeypatch): + monkeypatch.setenv("ETELEMETRY_URL", "https://env.example.com") + monkeypatch.setattr(config, "default_url", "https://default.example.com") + result = resolve_url(server_url="https://param.example.com") + assert result == "https://param.example.com" + + def test_env_var_over_default_url(self, monkeypatch): + monkeypatch.setenv("ETELEMETRY_URL", "https://env.example.com") + monkeypatch.setattr(config, "default_url", "https://default.example.com") + result = resolve_url() + assert result == "https://env.example.com" + + def test_default_url_over_hardcoded(self, monkeypatch): + monkeypatch.setattr(config, "default_url", "https://default.example.com") + result = resolve_url() + assert result == "https://default.example.com" + + def test_hardcoded_fallback(self): + result = resolve_url() + assert result == config._HARDCODED_URL + + def test_no_et_returns_none(self, monkeypatch): + monkeypatch.setenv("NO_ET", "1") + result = resolve_url(server_url="https://param.example.com") + assert result is None + + def test_no_et_overrides_all(self, monkeypatch): + monkeypatch.setenv("NO_ET", "1") + monkeypatch.setenv("ETELEMETRY_URL", "https://env.example.com") + monkeypatch.setattr(config, "default_url", "https://default.example.com") + result = resolve_url(server_url="https://param.example.com") + assert result is None + + def test_empty_server_url_falls_through(self, monkeypatch): + monkeypatch.setenv("ETELEMETRY_URL", "https://env.example.com") + result = resolve_url(server_url="") + assert result == "https://env.example.com" + + def test_none_server_url_falls_through(self, monkeypatch): + monkeypatch.setenv("ETELEMETRY_URL", "https://env.example.com") + result = resolve_url(server_url=None) + assert result == "https://env.example.com" diff --git a/tools/__init__.py b/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/tools/__main__.py b/tools/__main__.py new file mode 100644 index 0000000..9592fb8 --- /dev/null +++ b/tools/__main__.py @@ -0,0 +1,5 @@ +"""Entry point for ``python -m tools.migrate``.""" + +from tools.migrate import main + +main() diff --git a/tools/migrate.py b/tools/migrate.py new file mode 100644 index 0000000..3014931 --- /dev/null +++ b/tools/migrate.py @@ -0,0 +1,364 @@ +"""MongoDB-to-PostgreSQL migration tool for etelemetry. + +Reads the legacy ``et`` MongoDB database (``requests`` and ``geo`` +collections) and inserts deduplicated records into the new PostgreSQL +schema. IP addresses are never transferred. + +Usage:: + + python -m tools.migrate \ + --mongo-uri mongodb://localhost:27017 \ + --pg-uri postgresql://user:pass@localhost/etelemetry \ + --batch-size 1000 \ + --verify +""" + +from __future__ import annotations + +import argparse +import datetime +import logging +import sys +from typing import Any + +from sqlalchemy import create_engine, func, select +from sqlalchemy.orm import Session + +# Ensure the server package is importable when running from the repo root. +sys.path.insert(0, "server/src") + +from etelemetry_server.models import Base, Project, VersionCheck # noqa: E402 + +logger = logging.getLogger("tools.migrate") + +# --------------------------------------------------------------------------- +# Field validation +# --------------------------------------------------------------------------- + +_REQUIRED_REQUEST_FIELDS = {"owner", "repository", "version", "access_time"} + + +def _is_valid_request(doc: dict[str, Any]) -> bool: + """Return True if the document has all required fields.""" + return _REQUIRED_REQUEST_FIELDS.issubset(doc.keys()) + + +# --------------------------------------------------------------------------- +# Time helpers +# --------------------------------------------------------------------------- + + +def _truncate_to_hour(dt: datetime.datetime) -> datetime.datetime: + """Truncate a datetime to the start of its hour (UTC).""" + if dt.tzinfo is None: + dt = dt.replace(tzinfo=datetime.timezone.utc) + return dt.replace(minute=0, second=0, microsecond=0) + + +def _parse_access_time(value: Any) -> datetime.datetime: + """Parse an access_time value from MongoDB. + + Handles both datetime objects (as returned by pymongo) and ISO-format + strings (as found in test fixtures). + """ + if isinstance(value, datetime.datetime): + if value.tzinfo is None: + value = value.replace(tzinfo=datetime.timezone.utc) + return value + # String fallback + raw = str(value) + if raw.endswith("Z"): + raw = raw[:-1] + "+00:00" + return datetime.datetime.fromisoformat(raw) + + +# --------------------------------------------------------------------------- +# Core migration logic +# --------------------------------------------------------------------------- + + +def _build_geo_lookup(mongo_client: Any) -> dict[str, dict[str, Any]]: + """Build a lookup dict from remote_addr -> geo fields.""" + db = mongo_client["et"] + geo_lookup: dict[str, dict[str, Any]] = {} + for doc in db["geo"].find(): + addr = doc.get("remote_addr") + if addr: + geo_lookup[addr] = { + "city": doc.get("city", "unknown"), + "region": doc.get("region_name", "unknown"), + "country": doc.get("country_name", "unknown"), + "latitude": doc.get("latitude"), + "longitude": doc.get("longitude"), + } + return geo_lookup + + +_DEFAULT_GEO = { + "city": "unknown", + "region": "unknown", + "country": "unknown", + "latitude": None, + "longitude": None, +} + + +def migrate_records( + mongo_client: Any, + pg_engine: Any, + batch_size: int = 1000, +) -> dict[str, int]: + """Migrate records from MongoDB to PostgreSQL. + + Parameters + ---------- + mongo_client + A ``pymongo.MongoClient`` (or mock) connected to the legacy database. + pg_engine + A SQLAlchemy engine connected to the target PostgreSQL database. + batch_size + Number of records to process before logging progress. + + Returns + ------- + dict + Statistics: ``imported``, ``skipped``, ``projects_created``. + """ + Base.metadata.create_all(pg_engine) + + geo_lookup = _build_geo_lookup(mongo_client) + db = mongo_client["et"] + + stats = {"imported": 0, "skipped": 0, "projects_created": 0} + + # Cache project_id lookups: (owner, repo) -> project.id + project_cache: dict[tuple[str, str], int] = {} + + batch: list[dict[str, Any]] = [] + processed = 0 + + for doc in db["requests"].find(): + if not _is_valid_request(doc): + doc_id = doc.get("_id", "") + logger.warning("Skipping malformed record %s: missing required fields", doc_id) + stats["skipped"] += 1 + continue + + try: + access_time = _parse_access_time(doc["access_time"]) + except (ValueError, TypeError) as exc: + logger.warning("Skipping record %s: bad access_time: %s", doc.get("_id"), exc) + stats["skipped"] += 1 + continue + + remote_addr = doc.get("remote_addr", "") + geo = geo_lookup.get(remote_addr, _DEFAULT_GEO) + + batch.append( + { + "owner": doc["owner"], + "repo": doc["repository"], + "version": doc["version"], + "is_ci": bool(doc.get("is_ci", False)), + "time_bucket": _truncate_to_hour(access_time), + "city": geo["city"], + "region": geo["region"], + "country": geo["country"], + "latitude": geo["latitude"], + "longitude": geo["longitude"], + } + ) + + if len(batch) >= batch_size: + _flush_batch(batch, pg_engine, project_cache, stats) + batch = [] + processed += batch_size + logger.info("Processed %d records so far ...", processed) + + # Flush remaining + if batch: + _flush_batch(batch, pg_engine, project_cache, stats) + processed += len(batch) + logger.info("Processed %d records (final batch).", processed) + + logger.info( + "Migration complete: imported=%d skipped=%d projects_created=%d", + stats["imported"], + stats["skipped"], + stats["projects_created"], + ) + return stats + + +def _flush_batch( + batch: list[dict[str, Any]], + engine: Any, + project_cache: dict[tuple[str, str], int], + stats: dict[str, int], +) -> None: + """Write a batch of mapped records into PostgreSQL.""" + with Session(engine) as session: + for rec in batch: + project_key = (rec["owner"], rec["repo"]) + + # Ensure project exists + if project_key not in project_cache: + project = session.execute( + select(Project).where( + Project.owner == rec["owner"], + Project.repo == rec["repo"], + ) + ).scalar_one_or_none() + + if project is None: + project = Project(owner=rec["owner"], repo=rec["repo"]) + session.add(project) + session.flush() + stats["projects_created"] += 1 + + project_cache[project_key] = project.id + + project_id = project_cache[project_key] + + # Upsert version_check (content-addressed dedup) + existing = session.execute( + select(VersionCheck).where( + VersionCheck.project_id == project_id, + VersionCheck.version == rec["version"], + VersionCheck.city == rec["city"], + VersionCheck.region == rec["region"], + VersionCheck.is_ci == rec["is_ci"], + VersionCheck.time_bucket == rec["time_bucket"], + ) + ).scalar_one_or_none() + + if existing is not None: + existing.count += 1 + else: + vc = VersionCheck( + project_id=project_id, + version=rec["version"], + city=rec["city"], + region=rec["region"], + country=rec["country"], + latitude=rec["latitude"], + longitude=rec["longitude"], + is_ci=rec["is_ci"], + time_bucket=rec["time_bucket"], + count=1, + ) + session.add(vc) + stats["imported"] += 1 + + session.commit() + + +# --------------------------------------------------------------------------- +# Verification +# --------------------------------------------------------------------------- + + +def verify_migration( + mongo_client: Any, + pg_engine: Any, +) -> dict[str, int]: + """Compare record counts between MongoDB and PostgreSQL. + + Returns + ------- + dict + ``mongo_requests``: total documents in MongoDB ``requests`` collection. + ``pg_version_checks``: number of rows in ``version_checks``. + ``pg_total_count``: sum of ``count`` across all ``version_checks`` rows. + """ + db = mongo_client["et"] + mongo_count = db["requests"].count_documents({}) + + with Session(pg_engine) as session: + pg_rows = session.execute( + select(func.count(VersionCheck.id)) + ).scalar_one() + pg_total = session.execute( + select(func.coalesce(func.sum(VersionCheck.count), 0)) + ).scalar_one() + + result = { + "mongo_requests": mongo_count, + "pg_version_checks": pg_rows, + "pg_total_count": pg_total, + } + + logger.info("Verification results: %s", result) + if mongo_count != pg_total: + logger.warning( + "Count mismatch: MongoDB has %d requests, PostgreSQL has %d total count " + "(difference may be due to skipped malformed records).", + mongo_count, + pg_total, + ) + return result + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +def build_parser() -> argparse.ArgumentParser: + parser = argparse.ArgumentParser( + description="Migrate etelemetry data from MongoDB to PostgreSQL.", + ) + parser.add_argument( + "--mongo-uri", + required=True, + help="MongoDB connection URI (e.g. mongodb://localhost:27017).", + ) + parser.add_argument( + "--pg-uri", + required=True, + help="PostgreSQL connection URI (e.g. postgresql://user:pass@localhost/etelemetry).", + ) + parser.add_argument( + "--batch-size", + type=int, + default=1000, + help="Number of records per batch (default: 1000).", + ) + parser.add_argument( + "--verify", + action="store_true", + default=False, + help="After migration, compare record counts between MongoDB and PostgreSQL.", + ) + return parser + + +def main(argv: list[str] | None = None) -> None: + logging.basicConfig( + level=logging.INFO, + format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", + ) + + parser = build_parser() + args = parser.parse_args(argv) + + # Late import so the CLI can be tested without pymongo installed. + import pymongo # noqa: F401 -- ensures pymongo is available + + mongo_client = pymongo.MongoClient(args.mongo_uri) + pg_engine = create_engine(args.pg_uri) + + logger.info("Starting migration from %s to PostgreSQL ...", args.mongo_uri) + + stats = migrate_records(mongo_client, pg_engine, batch_size=args.batch_size) + + if args.verify: + verify_migration(mongo_client, pg_engine) + + logger.info("Done. Stats: %s", stats) + mongo_client.close() + pg_engine.dispose() + + +if __name__ == "__main__": + main()