CrateDB SSL: Add client programs and enable testing on CI

Author: amotl

.github/dependabot.yml

@@ -172,6 +172,13 @@ updates:
     schedule:
       interval: "daily"
 
+  # Operation.
+
+  - directory: "/operation/compose/ssl"
+    package-ecosystem: "docker-compose"
+    schedule:
+      interval: "daily"
+
   # Topics.
 
   - directory: "/topic/machine-learning/automl"

.github/workflows/operation-compose-ssl.yml

@@ -0,0 +1,68 @@
+name: "Compose: CrateDB with SSL"
+
+on:
+  pull_request:
+    paths:
+    - '.github/workflows/operation-compose-ssl.yml'
+    - 'operation/compose/ssl/**'
+    - '/requirements.txt'
+  push:
+    branches: [ main ]
+    paths:
+    - '.github/workflows/operation-compose-ssl.yml'
+    - 'operation/compose/ssl/**'
+    - '/requirements.txt'
+
+  # Allow job to be triggered manually.
+  workflow_dispatch:
+
+  # Run job each night after CrateDB nightly has been published.
+  schedule:
+    - cron: '0 3 * * *'
+
+# Cancel in-progress jobs when pushing to the same branch.
+concurrency:
+  cancel-in-progress: true
+  group: ${{ github.workflow }}-${{ github.ref }}
+
+jobs:
+  test:
+    name: "
+     Python ${{ matrix.python-version }}
+     on ${{ matrix.os }}"
+    runs-on: ${{ matrix.os }}
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ 'ubuntu-latest' ]
+        python-version: [ '3.13' ]
+
+    env:
+      UV_SYSTEM_PYTHON: true
+
+    steps:
+
+      - name: Acquire sources
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set up uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          cache-dependency-glob: |
+            requirements.txt
+          enable-cache: true
+          version: "latest"
+
+      - name: Install utilities
+        run: |
+          uv pip install -r requirements.txt
+
+      - name: Validate operation/compose/ssl
+        run: |
+          cd operation/compose/ssl
+          make test

operation/compose/ssl/Makefile

@@ -0,0 +1,19 @@
+.DEFAULT_GOAL:=help
+
+.PHONY: test
+test: ## Run the integration tests.
+	@\
+	docker compose up --detach
+	sh client_crash.sh
+	sh client_pgsql.sh
+	uv run client_dbapi.py
+	uv run client_sqlalchemy.py
+	uv run client_psycopg.py
+	docker compose down
+
+.PHONY: help
+help: ## Show this help message.
+	@echo 'usage: make [target]'
+	@echo
+	@echo 'targets:'
+	@grep -E '^[8+a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}'

operation/compose/ssl/README.md

@@ -0,0 +1,67 @@
+# CrateDB with SSL
+
+## About
+
+A service composition file (Docker or Podman) for running CrateDB
+with SSL enabled, using self-signed certificates.
+
+Accompanied are example client programs that connect to CrateDB/SSL
+using the canonical `?sslmode=require` connection option parameter,
+to use SSL, but turn off certificate validation. To turn it on,
+use `?sslmode=verify-ca` or `?sslmode=verify-full`.
+
+## Caveat
+
+Note this example is not generating fresh self-signed certificates,
+but uses the ones uploaded to the repository. In this spirit, anyone
+spinning an environment from this example will be using the same
+encryption keys. A future iteration might improve this, possibly by
+using [minica].
+
+## Usage
+
+```shell
+docker compose up
+```
+
+## Clients
+
+Client examples adjusted to disable certificate validation, allowing
+connections to the server with self-signed certificates.
+
+- [crash] » [client_crash.sh]
+- [DB API] » [client_dbapi.py]
+- [SQLAlchemy] » [client_sqlalchemy.py]
+
+## Tests
+
+```shell
+make test
+```
+
+## Rationale
+
+When not disabling host name verification, clients will bail out with
+`SSLError`/`SSLCertVerificationError`:
+```text
+[SSL: CERTIFICATE_VERIFY_FAILED] certificate verify failed: self-signed
+certificate in certificate chain (_ssl.c:1010).
+```
+
+## Backlog
+
+- Documentation is currently void of relevant ready-to-run examples
+  - [Install with containers](https://cratedb.com/docs/guide/install/container/)
+  - [Docker install guide](https://cratedb.com/docs/guide/install/container/docker.html)
+  - [SSL Reference](https://cratedb.com/docs/crate/reference/en/latest/admin/ssl.html)
+- Possibly synchronize with `crate-pdo`, where this is derived from
+  <https://github.com/crate/crate-pdo/tree/2.2.2/test/provisioning>
+
+
+[client_crash.sh]: ./client_crash.sh
+[client_dbapi.py]: ./client_dbapi.py
+[client_sqlalchemy.py]: ./client_sqlalchemy.py
+[crash]: https://cratedb.com/docs/crate/crash/
+[DB API]: https://cratedb.com/docs/python/
+[minica]: https://github.com/jsha/minica
+[SQLAlchemy]: https://cratedb.com/docs/sqlalchemy-cratedb/

operation/compose/ssl/client_crash.sh

@@ -0,0 +1,7 @@
+#!/usr/bin/env sh
+
+# Connect to CrateDB using SSL, with host name verification turned off.
+uvx crash -v --hosts https://localhost:4200 \
+  --user=crate --format=dynamic \
+  --verify-ssl=false \
+  --command "SELECT 42"

operation/compose/ssl/client_dbapi.py

@@ -0,0 +1,25 @@
+# Connect to CrateDB using SSL, with host name verification turned off.
+
+# /// script
+# requires-python = ">=3.9"
+# dependencies = [
+#     "crate",
+# ]
+# ///
+from crate import client
+
+
+def main():
+    connection = client.connect("https://localhost:4200", verify_ssl_cert=False)
+    cursor = connection.cursor()
+    cursor.execute("SELECT 42")
+    results = cursor.fetchall()
+    cursor.close()
+    connection.close()
+
+    print(results)
+
+
+if __name__ == "__main__":
+    main()
+

operation/compose/ssl/client_pgsql.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env sh
+
+psql "postgresql://crate@localhost:5432/?sslmode=require" -c "SELECT 42"

operation/compose/ssl/client_psycopg.py

@@ -0,0 +1,22 @@
+# Connect to CrateDB using SSL, with host name verification turned off.
+
+# /// script
+# requires-python = ">=3.9"
+# dependencies = [
+#     "psycopg[binary]",
+# ]
+# ///
+import psycopg
+
+
+def main():
+    connection = psycopg.connect("postgresql://crate@localhost:5432/?sslmode=require")
+    with connection.cursor(row_factory=psycopg.rows.dict_row) as cursor:
+        cursor.execute("SELECT 42")
+        result = cursor.fetchall()
+        print(result)
+    connection.close()
+
+
+if __name__ == "__main__":
+    main()

operation/compose/ssl/client_sqlalchemy.py

@@ -0,0 +1,20 @@
+# Connect to CrateDB using SSL, with host name verification turned off.
+
+# /// script
+# requires-python = ">=3.9"
+# dependencies = [
+#     "sqlalchemy-cratedb>=0.42.0.dev2",
+# ]
+# ///
+import sqlalchemy as sa
+
+
+def main():
+    engine = sa.create_engine("crate://localhost/?sslmode=require")
+    with engine.connect() as connection:
+        results = connection.execute(sa.text("SELECT 42;"))
+        print(results.fetchall())
+
+
+if __name__ == "__main__":
+    main()

operation/docker/ssl/compose.yml renamed to operation/compose/ssl/compose.yml

@@ -1,6 +1,6 @@
-# Purpose:
-# Start CrateDB with custom parameters and wait for the service being available,
-# even when invoked through `docker compose up --detach`.
+# CrateDB single-node configuration with SSL (self-signed certificates).
+
+name: cratedb-with-ssl
 
 services:
 
@@ -9,19 +9,22 @@ services:
     command: ["crate", "-Cstats.enabled=true"]
     ports:
       - 4200:4200
+      - 5432:5432
     volumes:
       - ./crate.yml:/crate/config/crate.yml
       - ./keystore:/crate/config/keystore
       - ./truststore:/crate/config/truststore
     healthcheck:
       test: ["CMD", "curl", "--fail", "--insecure", "https://localhost:4200"]
       start_period: 3s
-      interval: 0.5s
+      interval: 1.5s
       retries: 30
       timeout: 30s
 
+  # Wait for the service being available, even when
+  # invoked through `docker compose up --detach`.
   # https://marcopeg.com/2019/docker-compose-healthcheck/
-  start_dependencies:
+  wait-for-cratedb:
     image: dadarek/wait-for-dependencies
     depends_on:
       cratedb: