Skip to content

Commit cb2c70d

Browse files
nosahamanosahama
authored
oidc: add pingidentity support (#1307)
* oidc: refactor local OIDC compose setup - Move auth services into a dedicated compose overlay. - Share OIDC configuration across the local Keycloak and PingFederate setup. * oidc: refactor local OIDC auth workflow - Support local Keycloak and PingFederate OIDC flows in compose, tests, token helpers, middleware, and smoke tests. * oidc: scope related logic under oidc/ * oidc: add oidc request/response data - we can use this to debug and verify what each provider expects/returns * admin: wait for Kafka topic metadata propagation and add unit test coverage
1 parent 6dcc96f commit cb2c70d

20 files changed

Lines changed: 1258 additions & 364 deletions

File tree

‎.env.example‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,37 @@ DOCKER_COMPOSE=docker-compose
1010

1111
COVERAGE_FILE=.coverage.${PYTHON_VERSION}
1212
PYTEST_ARGS=--cov=karapace --cov-append --numprocesses 4 --log-file=/opt/test-tmp/test-logs/unit-tests-pytest.${PYTHON_VERSION}.log --showlocals
13+
14+
# Default OIDC settings for the local Keycloak auth stack.
15+
OIDC_PROVIDER=keycloak
16+
OIDC_REALM=karapace
17+
OIDC_JWKS_ENDPOINT_URL=http://keycloak:8080/realms/karapace/protocol/openid-connect/certs
18+
OIDC_ALLOW_INSECURE_JWKS=true
19+
OIDC_EXPECTED_ISSUER=http://keycloak:8080/realms/karapace
20+
OIDC_EXPECTED_AUDIENCE=karapace-audience
21+
OIDC_SUB_CLAIM_NAME=sub
22+
OIDC_CLIENT_ID=karapace-client
23+
OIDC_TOKEN_URL=http://keycloak:8080/realms/karapace/protocol/openid-connect/token
24+
OIDC_SCOPE=openid
25+
OIDC_VERIFY_TLS=true
26+
OIDC_ROLES_CLAIM_PATH=resource_access.karapace-client.roles
27+
OIDC_METHOD_ROLES={"GET": ["schema:read", "subject:read"], "POST": ["schema:write", "subject:write"], "PUT": ["config_subject:update","config_global:update"], "DELETE": ["schema:delete", "subject:delete"]}
28+
29+
# Optional PingFederate overrides.
30+
# Uncomment these to run Karapace against the local PingFederate profile instead of Keycloak.
31+
# PING_IDENTITY_DEVOPS_USER=<your-ping-devops-user>
32+
# PING_IDENTITY_DEVOPS_KEY=<your-ping-devops-key>
33+
# OIDC_PROVIDER=pingfederate
34+
# OIDC_JWKS_ENDPOINT_URL=https://pingfederate:9031/pf/JWKS
35+
# OIDC_ALLOW_INSECURE_JWKS=true
36+
# OIDC_EXPECTED_ISSUER=https://pingfederate:9031
37+
# OIDC_EXPECTED_AUDIENCE=karapace-audience
38+
# OIDC_SUB_CLAIM_NAME=client_id
39+
# OIDC_CLIENT_ID=karapace-client
40+
# OIDC_REALM=karapace
41+
# OIDC_CLIENT_SECRET=karapace-secret
42+
# OIDC_TOKEN_URL=https://pingfederate:9031/as/token.oauth2
43+
# OIDC_SCOPE=openid
44+
# OIDC_VERIFY_TLS=false
45+
# OIDC_ROLES_CLAIM_PATH=roles
46+
# OIDC_METHOD_ROLES={"GET": ["schema:read", "subject:read"], "POST": ["schema:write", "subject:write"], "PUT": ["config_subject:update","config_global:update"], "DELETE": ["schema:delete", "subject:delete"]}

‎GNUmakefile‎

Lines changed: 108 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,48 @@
1-
SHELL := /usr/bin/env bash
2-
31
ifneq ($(wildcard .env),)
42
include .env
53
endif
64

75
VENV_DIR ?= $(CURDIR)/venv
86
PYTEST_ARGS ?=
7+
OIDC_PROVIDER ?= keycloak
98
DOCKER_COMPOSE ?= docker compose
9+
DOCKER_COMPOSE_STANDARD ?= $(DOCKER_COMPOSE) -f container/compose.yml
10+
DOCKER_COMPOSE_AUTH ?= $(DOCKER_COMPOSE) -f container/compose.yml -f container/compose-auth.yml --profile auth
11+
DOCKER_COMPOSE_AUTH_PINGFEDERATE ?= $(DOCKER_COMPOSE) -f container/compose.yml -f container/compose-auth.yml --profile auth --profile pingfederate
12+
DOCKER_CONTAINERS ?= \
13+
karapace-cli \
14+
karapace-schema-registry \
15+
karapace-schema-registry-follower \
16+
karapace-rest-proxy \
17+
kafka \
18+
prometheus \
19+
grafana \
20+
statsd-exporter \
21+
opentelemetry-collector \
22+
jaeger \
23+
karapace-schema-registry-authn-only \
24+
karapace-rest-proxy-oidc \
25+
karapace-rest-proxy-no-forward \
26+
karapace-schema-registry-basic \
27+
karapace-rest-proxy-basic \
28+
keycloak \
29+
pingfederate
1030
PIP ?= pip3 --disable-pip-version-check --no-input --require-virtualenv
1131
PYTHON ?= python3
1232
PYTHON_VERSION ?= 3.12
1333
KARAPACE_VERSION ?= 5.0.3
1434
RUNNER_UID ?=
1535
RUNNER_GID ?=
1636
COVERAGE_FILE ?= .coverage.${PYTHON_VERSION}
17-
KARAPACE_CLI ?= $(DOCKER_COMPOSE) -f container/compose.yml run --rm karapace-cli
37+
KARAPACE_CLI ?= $(DOCKER_COMPOSE_STANDARD) run --rm karapace-cli
38+
KARAPACE_CLI_EXEC ?= $(DOCKER_COMPOSE_STANDARD) exec -T karapace-cli
1839
CERTS_FOLDER ?= /opt/karapace/certs
1940

2041
# Export variables needed by docker compose
21-
export PYTHON_VERSION KARAPACE_VERSION RUNNER_UID RUNNER_GID COVERAGE_FILE PYTEST_ARGS
42+
export PYTHON_VERSION KARAPACE_VERSION RUNNER_UID RUNNER_GID COVERAGE_FILE
43+
export OIDC_JWKS_ENDPOINT_URL OIDC_ALLOW_INSECURE_JWKS OIDC_EXPECTED_ISSUER OIDC_EXPECTED_AUDIENCE OIDC_SUB_CLAIM_NAME OIDC_CLIENT_ID OIDC_ROLES_CLAIM_PATH OIDC_METHOD_ROLES
44+
export OIDC_PROVIDER OIDC_TOKEN_URL OIDC_CLIENT_SECRET OIDC_SCOPE OIDC_VERIFY_TLS OIDC_REALM KEYCLOAK_URL
45+
export PING_IDENTITY_DEVOPS_USER PING_IDENTITY_DEVOPS_KEY
2246

2347
define PIN_VERSIONS_COMMAND
2448
pip install pip-tools && \
@@ -27,6 +51,13 @@ pip install pip-tools && \
2751
python -m piptools compile --upgrade --extra typing -o /karapace/requirements/requirements-typing.txt /karapace/pyproject.toml
2852
endef
2953

54+
define RUN_E2E_TESTS_IN_DOCKER_COMMAND
55+
rm -fr runtime/*; \
56+
sleep 10; \
57+
$1 exec -T karapace-cli $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/; \
58+
rm -fr runtime/*
59+
endef
60+
3061
export PATH := $(VENV_DIR)/bin:$(PATH)
3162
export PS4 := \e[0m\e[32m==> \e[0m
3263
export LC_ALL := C
@@ -73,7 +104,7 @@ venv/.deps-dev: venv/.make
73104
touch '$(@)'
74105

75106

76-
.PHONY: test
107+
.PHONY: tests
77108
tests: unit-tests integration-tests
78109

79110
.PHONY: unit-tests
@@ -103,7 +134,6 @@ cleanest: cleaner
103134
rm -fr '$(VENV_DIR)'
104135

105136
.PHONY: requirements
106-
requirements:
107137
requirements:
108138
$(PIP) install --upgrade pip setuptools pip-tools
109139
$(PIP) install .[dev,typing]
@@ -114,56 +144,108 @@ pin-requirements:
114144

115145
.PHONY: stop-karapace-docker-resources
116146
stop-karapace-docker-resources:
117-
$(DOCKER_COMPOSE) -f container/compose.yml down -v --remove-orphans
147+
$(DOCKER_COMPOSE_STANDARD) down -v --remove-orphans || true
148+
$(DOCKER_COMPOSE_AUTH_PINGFEDERATE) down -v --remove-orphans || true
149+
docker rm -f $(DOCKER_CONTAINERS) >/dev/null 2>&1 || true
118150

119-
.PHONY: start-karapace-docker-resources
120-
start-karapace-docker-resources:
121-
start-karapace-docker-resources:
151+
.PHONY: prepare-docker-resources
152+
prepare-docker-resources:
122153
touch .coverage.${PYTHON_VERSION} || sudo touch .coverage.${PYTHON_VERSION}
123154
chown ${RUNNER_UID}:${RUNNER_GID} .coverage.${PYTHON_VERSION} 2>/dev/null || sudo chown ${RUNNER_UID}:${RUNNER_GID} .coverage.${PYTHON_VERSION}
124155
mkdir -p test-tmp.${PYTHON_VERSION} || sudo mkdir -p test-tmp.${PYTHON_VERSION}
125156
chown -R ${RUNNER_UID}:${RUNNER_GID} test-tmp.${PYTHON_VERSION} 2>/dev/null || sudo chown -R ${RUNNER_UID}:${RUNNER_GID} test-tmp.${PYTHON_VERSION}
126-
$(DOCKER_COMPOSE) -f container/compose.yml up -d --build --wait --detach
157+
158+
.PHONY: start-karapace-docker-resources
159+
start-karapace-docker-resources: prepare-docker-resources
160+
$(DOCKER_COMPOSE_STANDARD) up -d --build --wait --detach
161+
162+
.PHONY: start-karapace-docker-auth-resources
163+
start-karapace-docker-auth-resources: prepare-docker-resources
164+
$(DOCKER_COMPOSE_AUTH) up -d --build --wait
165+
166+
.PHONY: start-karapace-docker-auth-pingfederate-resources
167+
start-karapace-docker-auth-pingfederate-resources: prepare-docker-resources
168+
$(DOCKER_COMPOSE_AUTH_PINGFEDERATE) up -d --build --wait
169+
170+
.PHONY: provision-pingfederate-oidc
171+
provision-pingfederate-oidc: start-karapace-docker-auth-pingfederate-resources
172+
$(DOCKER_COMPOSE_AUTH_PINGFEDERATE) exec -T karapace-cli $(PYTHON) /opt/karapace/bin/oidc/provision_pingfederate_oidc.py
173+
174+
.PHONY: print-keycloak-oidc-token
175+
print-keycloak-oidc-token: start-karapace-docker-auth-resources
176+
$(DOCKER_COMPOSE_AUTH) exec -T karapace-cli env -u OIDC_CLIENT_SECRET \
177+
OIDC_PROVIDER="keycloak" \
178+
KEYCLOAK_URL="http://keycloak:8080" \
179+
OIDC_REALM="karapace" \
180+
OIDC_CLIENT_ID="karapace-client" \
181+
OIDC_TOKEN_URL="http://keycloak:8080/realms/karapace/protocol/openid-connect/token" \
182+
OIDC_SCOPE="openid" \
183+
OIDC_VERIFY_TLS="true" \
184+
OIDC_ALLOW_INSECURE_JWKS="true" \
185+
$(PYTHON) /opt/karapace/bin/oidc/get_oidc_token.py
186+
187+
.PHONY: print-pingfederate-oidc-token
188+
print-pingfederate-oidc-token: start-karapace-docker-auth-pingfederate-resources
189+
$(DOCKER_COMPOSE_AUTH_PINGFEDERATE) exec -T karapace-cli \
190+
$(PYTHON) /opt/karapace/bin/oidc/provision_pingfederate_oidc.py >/dev/null
191+
$(DOCKER_COMPOSE_AUTH_PINGFEDERATE) exec -T karapace-cli env -u OIDC_CLIENT_SECRET \
192+
OIDC_PROVIDER="pingfederate" \
193+
OIDC_SUB_CLAIM_NAME="client_id" \
194+
OIDC_CLIENT_ID="karapace-client" \
195+
OIDC_CLIENT_SECRET="karapace-secret" \
196+
OIDC_TOKEN_URL="https://pingfederate:9031/as/token.oauth2" \
197+
OIDC_SCOPE="openid" \
198+
OIDC_VERIFY_TLS="false" \
199+
OIDC_ALLOW_INSECURE_JWKS="true" \
200+
$(PYTHON) /opt/karapace/bin/oidc/get_oidc_token.py
127201

128202
.PHONY: smoke-test-schema-registry
129-
smoke-test-schema-registry: start-karapace-docker-resources
130-
$(KARAPACE_CLI) /opt/karapace/bin/smoke-test-schema-registry.sh
203+
smoke-test-schema-registry: stop-karapace-docker-resources start-karapace-docker-auth-resources
204+
$(DOCKER_COMPOSE_AUTH) exec -T karapace-cli /opt/karapace/bin/smoke-test-schema-registry.sh
131205

132206
.PHONY: smoke-test-rest-proxy
133-
smoke-test-rest-proxy: start-karapace-docker-resources
134-
$(KARAPACE_CLI) /opt/karapace/bin/smoke-test-rest-proxy.sh
207+
smoke-test-rest-proxy: stop-karapace-docker-resources start-karapace-docker-resources
208+
$(KARAPACE_CLI_EXEC) /opt/karapace/bin/smoke-test-rest-proxy.sh
135209

136210
.PHONY: unit-tests-in-docker
137211
unit-tests-in-docker: start-karapace-docker-resources
138212
rm -fr runtime/*
139-
$(KARAPACE_CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
213+
$(KARAPACE_CLI_EXEC) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/unit/
140214
rm -fr runtime/*
141215

142216
.PHONY: e2e-tests-in-docker
143-
e2e-tests-in-docker: export COMPOSE_PROFILES = e2e
144-
e2e-tests-in-docker: stop-karapace-docker-resources start-karapace-docker-resources
145-
rm -fr runtime/*
146-
sleep 10
147-
$(KARAPACE_CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/e2e/
148-
rm -fr runtime/*
217+
e2e-tests-in-docker: stop-karapace-docker-resources start-karapace-docker-auth-resources
218+
$(call RUN_E2E_TESTS_IN_DOCKER_COMMAND,$(DOCKER_COMPOSE_AUTH))
219+
220+
.PHONY: e2e-tests-in-docker-keycloak
221+
e2e-tests-in-docker-keycloak: export OIDC_PROVIDER=keycloak
222+
e2e-tests-in-docker-keycloak: stop-karapace-docker-resources start-karapace-docker-auth-pingfederate-resources
223+
$(DOCKER_COMPOSE_AUTH_PINGFEDERATE) stop pingfederate >/dev/null 2>&1 || true
224+
$(call RUN_E2E_TESTS_IN_DOCKER_COMMAND,$(DOCKER_COMPOSE_AUTH_PINGFEDERATE))
225+
226+
.PHONY: e2e-tests-in-docker-pingfederate
227+
e2e-tests-in-docker-pingfederate: export OIDC_PROVIDER=pingfederate
228+
e2e-tests-in-docker-pingfederate: stop-karapace-docker-resources provision-pingfederate-oidc
229+
$(DOCKER_COMPOSE_AUTH_PINGFEDERATE) stop keycloak >/dev/null 2>&1 || true; \
230+
$(call RUN_E2E_TESTS_IN_DOCKER_COMMAND,$(DOCKER_COMPOSE_AUTH_PINGFEDERATE))
149231

150232
.PHONY: integration-tests-in-docker
151233
integration-tests-in-docker: start-karapace-docker-resources
152234
rm -fr runtime/*
153235
sleep 10
154-
$(KARAPACE_CLI) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/
236+
$(KARAPACE_CLI_EXEC) $(PYTHON) -m pytest -s -vvv $(PYTEST_ARGS) tests/integration/
155237
rm -fr runtime/*
156238

157239
.PHONY: type-check-mypy-in-docker
158240
type-check-mypy-in-docker: start-karapace-docker-resources
159-
$(KARAPACE_CLI) $(PYTHON) -m mypy src/karapace
241+
$(KARAPACE_CLI_EXEC) $(PYTHON) -m mypy src/karapace
160242

161243
.PHONY: cli
162244
cli: start-karapace-docker-resources
163245
$(KARAPACE_CLI) bash
164246

165247
.PHONY: generate-sr-https-certs
166-
generate-sr-https-certs:
248+
generate-sr-https-certs:
167249
$(info ====> Generating self-signed certificates <====)
168250
$(KARAPACE_CLI) mkcert -key-file $(CERTS_FOLDER)/key.pem -cert-file $(CERTS_FOLDER)/cert.pem \
169251
localhost \
@@ -178,6 +260,6 @@ cli: start-karapace-docker-resources
178260
curl-sr-https: header ?= 'Content-Type: application/vnd.schemaregistry.v1+json'
179261
curl-sr-https:
180262
$(info ====> Sending HTTPS $(method) request with data to $(url) <====)
181-
$(KARAPACE_CLI) curl -i -X $(method) --location $(url) --cacert /opt/karapace/certs/ca/rootCA.pem \
263+
$(KARAPACE_CLI) curl -i -X $(method) --location $(url) --cacert $(CERTS_FOLDER)/ca/rootCA.pem \
182264
--header $(header) \
183265
--data '$(data)'

‎README.rst‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -722,7 +722,8 @@ Below here is an example of karapace OpenId connect config ::
722722
sasl_oauthbearer_method_roles: dict[str, list[str]] = {"GET": [], "POST": [], "PUT": [], "DELETE": []}
723723

724724

725-
Below here is an example of karapace OpenId connect docker config ::
725+
Below here is an example of karapace OpenId connect docker config using the local
726+
Keycloak defaults in ``container/compose.yml`` ::
726727

727728
KARAPACE_SASL_OAUTHBEARER_AUTHENTICATION_ENABLED: True
728729
KARAPACE_SASL_OAUTHBEARER_JWKS_ENDPOINT_URL: http://keycloak:8080/realms/karapace/protocol/openid-connect/certs
@@ -769,6 +770,12 @@ and get a token like below. ::
769770

770771
Note : client id and client secret can be retrieved from Oidc provider
771772

773+
The helper script ``bin/oidc/get_oidc_token.py`` supports both providers. For local Keycloak::
774+
775+
OIDC_PROVIDER=keycloak python3 bin/oidc/get_oidc_token.py
776+
777+
For PingFederate examples, see ``container/oidc/pingidentity/README.md``.
778+
772779
Response of the above curl should be a access token, and other scope and expiry details.
773780
Export the token into ACCESS_TOKEN variable::
774781

‎bin/get_oidc_token.py‎

Lines changed: 0 additions & 64 deletions
This file was deleted.

0 commit comments

Comments
 (0)