diff --git a/.env.example b/.env.example new file mode 100644 index 000000000..8257c7ea2 --- /dev/null +++ b/.env.example @@ -0,0 +1,5 @@ +# Local dev configuration read by docker compose and the Makefile. +# Copy to .env (or run `make .env` to generate one with a random password). +# Shared PostgreSQL password for the local stack (postgres container + every service). +# Use only URL-safe characters (letters, digits, -_.): the value is interpolated into database URLs in docker-compose.yml. +POSTGRES_PASSWORD= diff --git a/.github/workflows/dast-scan.yml b/.github/workflows/dast-scan.yml index 6facaf64c..d47dddc89 100644 --- a/.github/workflows/dast-scan.yml +++ b/.github/workflows/dast-scan.yml @@ -64,6 +64,7 @@ jobs: - name: Start the application stack run: | + make .env docker compose -f docker-compose.infra.yml -f docker-compose.yml up -d --build \ postgres redis localstack meilisearch \ auth-service api-gateway document-service search-service file-service diff --git a/Makefile b/Makefile index 1c6184107..3f90152d7 100644 --- a/Makefile +++ b/Makefile @@ -2,9 +2,25 @@ SHELL := /bin/bash +-include .env +export POSTGRES_PASSWORD +DB_PASSWORD ?= $(POSTGRES_PASSWORD) +export DB_PASSWORD + help: ## Show this help @grep -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}' +.env: ## Generate a local .env with a random POSTGRES_PASSWORD if none exists + @if [ ! -f .env ]; then \ + cp .env.example .env; \ + sed -i "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$$(openssl rand -hex 16)/" .env; \ + echo "Generated .env with a random POSTGRES_PASSWORD"; \ + if docker volume ls -q 2>/dev/null | grep -q postgres_data; then \ + echo "NOTE: an existing Postgres volume keeps the password it was initialised with;"; \ + echo " set POSTGRES_PASSWORD in .env to that value, or reset local data with 'make down && docker compose -f docker-compose.infra.yml -f docker-compose.yml down -v'"; \ + fi; \ + fi + PROCS_COMPOSE = docker compose -f docker-compose.procs.yml -p otterworks-procs-$(NS) PROCS_UV = uv run --with psycopg[binary]==3.2.9 --with pyyaml==6.0.2 PROCS_PORT_OFFSET = $(shell if command -v python3 >/dev/null 2>&1 && test -n "$(NS)"; then python3 -c "import zlib; print(zlib.crc32('$(NS)'.encode()) % 1000)"; fi) @@ -57,13 +73,13 @@ insurance-test: procs-validate ## Run the Commission Pay OLTP + OLAP test suites # --- Local Development --- -infra-up: ## Start local infrastructure (Postgres, Redis, LocalStack, MeiliSearch) +infra-up: .env ## Start local infrastructure (Postgres, Redis, LocalStack, MeiliSearch) docker compose -f docker-compose.infra.yml up -d infra-down: ## Stop local infrastructure docker compose -f docker-compose.infra.yml down -up: ## Start all services (add seed=1 to seed after start) +up: .env ## Start all services (add seed=1 to seed after start) docker compose -f docker-compose.infra.yml -f docker-compose.yml up -d --build ifdef seed @$(MAKE) --no-print-directory wait-for-db seed @@ -72,7 +88,7 @@ up: ## Start all services (add seed=1 to seed after start) down: ## Stop all application services docker compose -f docker-compose.infra.yml -f docker-compose.yml down -build: ## Build all service images +build: .env ## Build all service images docker compose -f docker-compose.infra.yml -f docker-compose.yml build seed: ## Seed development data (services must be running) @@ -98,7 +114,7 @@ COMPOSE := docker compose -f docker-compose.infra.yml -f docker-compose.yml # otherwise default to :8085, which only matches the k8s dev environment. COLLAB_WS_URL := ws://localhost:8084 -dev-backend: ## Start the Dockerized backend (all services except the frontend containers) +dev-backend: .env ## Start the Dockerized backend (all services except the frontend containers) $(COMPOSE) up -d $$($(COMPOSE) config --services | grep -vE '^(web-app|admin-dashboard)$$') @echo "Backend up - API gateway on http://localhost:8080 (fresh DB? run: make seed)" @@ -220,7 +236,7 @@ ifndef NS endif $(call validate_ns) @echo "Dropping schema otterworks_$(NS)..." - PGPASSWORD=$${DB_PASSWORD:-otterworks_dev} psql \ + PGPASSWORD=$${DB_PASSWORD} psql \ -h $${DB_HOST:-localhost} -p $${DB_PORT:-5432} \ -U $${DB_USER:-otterworks} -d $${DB_NAME:-otterworks} \ -c "DROP SCHEMA IF EXISTS otterworks_$(NS) CASCADE;" @@ -232,7 +248,7 @@ ifndef NS endif $(call validate_ns) @echo "Creating schema otterworks_$(NS)..." - PGPASSWORD=$${DB_PASSWORD:-otterworks_dev} psql \ + PGPASSWORD=$${DB_PASSWORD} psql \ -h $${DB_HOST:-localhost} -p $${DB_PORT:-5432} \ -U $${DB_USER:-otterworks} -d $${DB_NAME:-otterworks} \ -f testdata/harness/create_schema.sql \ diff --git a/docker-compose.infra.yml b/docker-compose.infra.yml index 97b7fbf43..a689c7641 100644 --- a/docker-compose.infra.yml +++ b/docker-compose.infra.yml @@ -5,7 +5,7 @@ services: container_name: otterworks-postgres environment: POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks ports: - 5432:5432 diff --git a/docker-compose.yml b/docker-compose.yml index bb09e0c0e..d25c20764 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 @@ -61,7 +61,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 @@ -69,7 +69,7 @@ services: OTEL_SERVICE_NAME: auth-service SPRING_DATASOURCE_URL: jdbc:postgresql://postgres:5432/otterworks SPRING_DATASOURCE_USERNAME: otterworks - SPRING_DATASOURCE_PASSWORD: otterworks_dev + SPRING_DATASOURCE_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} JWT_SECRET: ${JWT_SECRET:-otterworks-local-dev-jwt-secret-change-me-in-production} depends_on: postgres: @@ -103,7 +103,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 @@ -146,13 +146,13 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 OTEL_EXPORTER_OTLP_ENDPOINT: http://otel-collector:4318 OTEL_SERVICE_NAME: document-service - DOC_SVC_DATABASE_URL: postgresql+asyncpg://otterworks:otterworks_dev@postgres:5432/otterworks + DOC_SVC_DATABASE_URL: postgresql+asyncpg://otterworks:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env}@postgres:5432/otterworks DOC_SVC_SNS_TOPIC_ARN: arn:aws:sns:us-east-1:000000000000:otterworks-events DOC_SVC_AWS_ENDPOINT_URL: http://localstack:4566 DOC_SVC_SNS_ENABLED: 'true' @@ -189,7 +189,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 @@ -229,7 +229,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 @@ -272,7 +272,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 @@ -314,7 +314,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 @@ -356,7 +356,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 @@ -364,11 +364,11 @@ services: OTEL_SERVICE_NAME: admin-service RAILS_ENV: development JWT_SECRET: ${JWT_SECRET:-otterworks-local-dev-jwt-secret-change-me-in-production} - DATABASE_URL: postgres://otterworks:otterworks_dev@postgres:5432/otterworks + DATABASE_URL: postgres://otterworks:${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env}@postgres:5432/otterworks DATABASE_HOST: postgres DATABASE_PORT: 5432 DATABASE_USER: otterworks - DATABASE_PASSWORD: otterworks_dev + DATABASE_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} SECRET_KEY_BASE: dev-secret-key-base-otterworks-change-in-production DEVIN_API_KEY: ${DEVIN_API_KEY:-} DEVIN_ORG_ID: ${DEVIN_ORG_ID:-} @@ -407,7 +407,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 @@ -418,7 +418,7 @@ services: DB_PORT: 5432 DB_NAME: otterworks_reports DB_USER: otterworks - DB_PASSWORD: otterworks_dev + DB_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} ANALYTICS_SERVICE_URL: http://analytics-service:8088 AUDIT_SERVICE_URL: http://audit-service:8090 AUTH_SERVICE_URL: http://auth-service:8081 @@ -455,7 +455,7 @@ services: POSTGRES_HOST: postgres POSTGRES_PORT: 5432 POSTGRES_USER: otterworks - POSTGRES_PASSWORD: otterworks_dev + POSTGRES_PASSWORD: ${POSTGRES_PASSWORD:?POSTGRES_PASSWORD is not set; run 'make .env' or copy .env.example to .env} POSTGRES_DB: otterworks REDIS_HOST: redis REDIS_PORT: 6379 diff --git a/scripts/seed.py b/scripts/seed.py index 95b0f84ab..fd4611fd5 100644 --- a/scripts/seed.py +++ b/scripts/seed.py @@ -20,6 +20,7 @@ import sys import uuid from datetime import datetime, timedelta, timezone +from pathlib import Path import bcrypt import psycopg2 @@ -27,12 +28,27 @@ # ── Connection ──────────────────────────────────────────────────────────────── + +def _local_db_password() -> str: + """DB_PASSWORD, else POSTGRES_PASSWORD from the environment or the repo-root .env.""" + for name in ("DB_PASSWORD", "POSTGRES_PASSWORD"): + if name in os.environ: + return os.environ[name] + env_file = Path(__file__).resolve().parents[1] / ".env" + if env_file.is_file(): + for line in env_file.read_text().splitlines(): + key, sep, value = line.strip().partition("=") + if sep and key == "POSTGRES_PASSWORD": + return value.strip().strip("\"'") + return "" + + DB_CONFIG = { "host": os.getenv("DB_HOST", "localhost"), "port": int(os.getenv("DB_PORT", "5432")), "dbname": os.getenv("DB_NAME", "otterworks"), "user": os.getenv("DB_USER", "otterworks"), - "password": os.getenv("DB_PASSWORD", "otterworks_dev"), + "password": _local_db_password(), } # ── Helpers ─────────────────────────────────────────────────────────────────── diff --git a/services/document-service/app/config.py b/services/document-service/app/config.py index d98c1efa6..793fb1040 100644 --- a/services/document-service/app/config.py +++ b/services/document-service/app/config.py @@ -8,9 +8,7 @@ class Settings(BaseSettings): app_version: str = "0.1.0" debug: bool = False - database_url: str = ( - "postgresql+asyncpg://otterworks:otterworks_dev@localhost:5432/otterworks" - ) + database_url: str = "postgresql+asyncpg://otterworks@localhost:5432/otterworks" db_pool_size: int = 10 db_max_overflow: int = 20 diff --git a/services/document-service/tests/test_config.py b/services/document-service/tests/test_config.py new file mode 100644 index 000000000..9a10b626b --- /dev/null +++ b/services/document-service/tests/test_config.py @@ -0,0 +1,19 @@ +"""Settings must never carry an embedded database password by default.""" + +from urllib.parse import urlparse + +from app.config import Settings + + +def test_default_database_url_has_no_password(monkeypatch): + monkeypatch.delenv("DOC_SVC_DATABASE_URL", raising=False) + url = urlparse(Settings(_env_file=None).database_url) + assert url.password is None + assert "otterworks_dev" not in url.geturl() + + +def test_database_url_comes_from_environment(monkeypatch): + monkeypatch.setenv( + "DOC_SVC_DATABASE_URL", "postgresql+asyncpg://u:secret@db:5432/otterworks" + ) + assert Settings(_env_file=None).database_url.endswith("@db:5432/otterworks") diff --git a/testdata/generated/seed/helpers.py b/testdata/generated/seed/helpers.py index 6903791c8..599f4793d 100644 --- a/testdata/generated/seed/helpers.py +++ b/testdata/generated/seed/helpers.py @@ -9,6 +9,7 @@ import os import uuid from datetime import datetime, timedelta, timezone +from pathlib import Path import bcrypt import psycopg2 @@ -76,12 +77,27 @@ def hash_password(plain: str) -> str: # ── DB connection ───────────────────────────────────────────────────────────── + +def _local_db_password() -> str: + """DB_PASSWORD, else POSTGRES_PASSWORD from the environment or the repo-root .env.""" + for name in ("DB_PASSWORD", "POSTGRES_PASSWORD"): + if name in os.environ: + return os.environ[name] + env_file = Path(__file__).resolve().parents[3] / ".env" + if env_file.is_file(): + for line in env_file.read_text().splitlines(): + key, sep, value = line.strip().partition("=") + if sep and key == "POSTGRES_PASSWORD": + return value.strip().strip("\"'") + return "" + + DB_CONFIG = { "host": os.getenv("DB_HOST", "localhost"), "port": int(os.getenv("DB_PORT", "5432")), "dbname": os.getenv("DB_NAME", "otterworks"), "user": os.getenv("DB_USER", "otterworks"), - "password": os.getenv("DB_PASSWORD", "otterworks_dev"), + "password": _local_db_password(), } diff --git a/testdata/harness/validate.py b/testdata/harness/validate.py index eda9d0dda..2c46971b5 100644 --- a/testdata/harness/validate.py +++ b/testdata/harness/validate.py @@ -36,12 +36,27 @@ # ── Configuration ───────────────────────────────────────────────────────────── + +def _local_db_password() -> str: + """DB_PASSWORD, else POSTGRES_PASSWORD from the environment or the repo-root .env.""" + for name in ("DB_PASSWORD", "POSTGRES_PASSWORD"): + if name in os.environ: + return os.environ[name] + env_file = Path(__file__).resolve().parents[2] / ".env" + if env_file.is_file(): + for line in env_file.read_text().splitlines(): + key, sep, value = line.strip().partition("=") + if sep and key == "POSTGRES_PASSWORD": + return value.strip().strip("\"'") + return "" + + DB_CONFIG = { "host": os.getenv("DB_HOST", "localhost"), "port": int(os.getenv("DB_PORT", "5432")), "dbname": os.getenv("DB_NAME", "otterworks"), "user": os.getenv("DB_USER", "otterworks"), - "password": os.getenv("DB_PASSWORD", "otterworks_dev"), + "password": _local_db_password(), } # ── Data Classes ──────────────────────────────────────────────────────────────