diff --git a/README.md b/README.md index 30698f6..e9188b1 100644 --- a/README.md +++ b/README.md @@ -80,6 +80,7 @@ Each secret should expose a key with the same name so it becomes a file in - `pr-sandbox-pr-installation-id` - `pr-sandbox-cluster-installation-id` - `pr-sandbox-mysql-connection-string` +- `pr-sandbox-redis-connection-string` - `pr-sandbox-argocd-webhook-auth` - `pr-sandbox-argocd-readonly-user-password` - `pr-sandbox-extra-configs` diff --git a/app/handlers.py b/app/handlers.py index 97e6160..66cfece 100644 --- a/app/handlers.py +++ b/app/handlers.py @@ -5,6 +5,7 @@ import logging from fastapi import Depends, APIRouter, Header, BackgroundTasks from fastapi.responses import PlainTextResponse +from redis import Redis from typing import Annotated import uuid @@ -20,13 +21,20 @@ from app.helpers.constants import GithubActionTypes, GithubEventTypes, CheckRunStatus from app.helpers.db_utils import SessionDep from app.helpers.exceptions import ActiveCheckrunNotFoundException, DBOperationException -from app.helpers.validations import validate_signature, validate_request, validate_auth +from app.helpers.utils import get_secret +from app.helpers.validations import ( + validate_signature, + validate_request, + validate_auth, + validate_request_not_duplicate, +) from app.models.request_models import ( GithubWebhookRequest, GithubWebhookHeader, ArgoWebhookRequest, ) +redis_client = Redis.from_url(get_secret("pr-sandbox-redis-connection-string")) logger = logging.getLogger(__name__) github_webhook_router = APIRouter( @@ -181,6 +189,7 @@ def github_handler( """ logger.info(headers) github_event = headers.x_github_event + validate_request_not_duplicate(headers.x_hub_signature_256, redis_client) validate_request(github_event, request) handle_github_webhook(github_event, request, db_session, background_tasks) return { diff --git a/app/helpers/constants.py b/app/helpers/constants.py index eecf461..624af43 100644 --- a/app/helpers/constants.py +++ b/app/helpers/constants.py @@ -318,4 +318,4 @@ class ArgoCDSyncStatus(StrEnum): PR_SANDBOX_NAME_PATTERN = r"pr-\d*-[0-9a-f]{6}" -DEDUPLICATION_TTL = 300 +DEDUPLICATION_TTL = 30 diff --git a/app/helpers/validations.py b/app/helpers/validations.py index 6a2087a..c58638a 100644 --- a/app/helpers/validations.py +++ b/app/helpers/validations.py @@ -6,6 +6,7 @@ from fastapi import Request, HTTPException import hmac import hashlib +from redis import Redis from app.helpers.conf import config from app.helpers.constants import ( @@ -17,6 +18,7 @@ AUTHORIZATION_HEADER, AUTHORIZATION_PREFIX, WorkflowType, + DEDUPLICATION_TTL, ) from app.helpers.exceptions import UnactionableRequestException from app.helpers.utils import get_secret @@ -180,3 +182,11 @@ def validate_request(github_event: GithubEventTypes, request: GithubWebhookReque raise UnactionableRequestException("The provided installation id is invalid") _validate_request_actionable(github_event, request) + + +def validate_request_not_duplicate(unique_id: str, redis_client: Redis): + is_unique = redis_client.set(unique_id, "1", nx=True, ex=DEDUPLICATION_TTL) + if not bool(is_unique): + raise UnactionableRequestException( + f"The request with id {unique_id} is not unique" + ) diff --git a/charts/pr-sandbox-automation/Chart.yaml b/charts/pr-sandbox-automation/Chart.yaml index 398db10..29d8ef0 100644 --- a/charts/pr-sandbox-automation/Chart.yaml +++ b/charts/pr-sandbox-automation/Chart.yaml @@ -2,10 +2,14 @@ apiVersion: v2 name: pr-sandbox-automation description: Automatically manage Open edX sandbox instances for PRs type: application -version: 0.1.0 -appVersion: "0.1.0" +version: 0.1.1 +appVersion: "0.2.0" dependencies: - name: mysql version: 9.14.1 repository: https://charts.bitnami.com/bitnami condition: mysql.enabled + - name: redis + version: 24.1.3 + repository: https://charts.bitnami.com/bitnami + condition: redis.enabled diff --git a/charts/pr-sandbox-automation/templates/_helpers.tpl b/charts/pr-sandbox-automation/templates/_helpers.tpl index 5b171e8..93e7184 100644 --- a/charts/pr-sandbox-automation/templates/_helpers.tpl +++ b/charts/pr-sandbox-automation/templates/_helpers.tpl @@ -39,3 +39,11 @@ app.kubernetes.io/instance: {{ .Release.Name }} {{- printf "%s-mysql" .Release.Name -}} {{- end -}} {{- end -}} + +{{- define "pr-sandbox-automation.redisHost" -}} +{{- if .Values.redis.connectionSecret.host -}} +{{- .Values.redis.connectionSecret.host -}} +{{- else -}} +{{- printf "%s-redis-master" .Release.Name -}} +{{- end -}} +{{- end -}} diff --git a/charts/pr-sandbox-automation/templates/redis-connection-secret.yaml b/charts/pr-sandbox-automation/templates/redis-connection-secret.yaml new file mode 100644 index 0000000..98be73b --- /dev/null +++ b/charts/pr-sandbox-automation/templates/redis-connection-secret.yaml @@ -0,0 +1,18 @@ +{{- if and .Values.redis.enabled .Values.redis.connectionSecret.create -}} +apiVersion: v1 +kind: Secret +metadata: + name: {{ .Values.redis.connectionSecret.name }} + labels: + {{- include "pr-sandbox-automation.labels" . | nindent 4 }} +type: Opaque +stringData: + {{ .Values.redis.connectionSecret.key }}: {{- /* + Construct Redis connection string for the app. + Example: redis://:password@host:6379/0 + */}} + {{- $password := required "redis.auth.password is required when redis.connectionSecret.create is true" .Values.redis.auth.password -}} + {{- $host := include "pr-sandbox-automation.redisHost" . -}} + {{- $port := .Values.redis.connectionSecret.port | default 6379 -}} + {{ printf "redis://:%s@%s:%d/0" $password $host $port | quote }} +{{- end }} diff --git a/charts/pr-sandbox-automation/values.yaml b/charts/pr-sandbox-automation/values.yaml index 1d37eec..1feddad 100644 --- a/charts/pr-sandbox-automation/values.yaml +++ b/charts/pr-sandbox-automation/values.yaml @@ -86,6 +86,10 @@ secretMounts: items: - key: pr-sandbox-mysql-connection-string path: pr-sandbox-mysql-connection-string + - secretName: pr-sandbox-redis-connection-string + items: + - key: pr-sandbox-redis-connection-string + path: pr-sandbox-redis-connection-string - secretName: pr-sandbox-argocd-webhook-auth items: - key: pr-sandbox-argocd-webhook-auth @@ -127,3 +131,27 @@ mysql: key: pr-sandbox-mysql-connection-string host: "" port: 3306 + +redis: + enabled: false + architecture: standalone + auth: + enabled: true + password: "" + master: + persistence: + enabled: false + size: 1Gi + resources: + requests: + memory: "128Mi" + cpu: "100m" + limits: + memory: "256Mi" + cpu: "200m" + connectionSecret: + create: true + name: pr-sandbox-redis-connection-string + key: pr-sandbox-redis-connection-string + host: "" + port: 6379 diff --git a/dev/secrets/pr-sandbox-redis-connection-string b/dev/secrets/pr-sandbox-redis-connection-string new file mode 100644 index 0000000..c09fc4a --- /dev/null +++ b/dev/secrets/pr-sandbox-redis-connection-string @@ -0,0 +1 @@ +redis://dev-redis:6379 diff --git a/docker-compose.yml b/docker-compose.yml index a8dfb0b..6c61434 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,6 +9,8 @@ secrets: file: dev/secrets/pr-sandbox-cluster-installation-id pr-sandbox-mysql-connection-string: file: dev/secrets/pr-sandbox-mysql-connection-string + pr-sandbox-redis-connection-string: + file: dev/secrets/pr-sandbox-redis-connection-string pr-sandbox-argocd-webhook-auth: file: dev/secrets/pr-sandbox-argocd-webhook-auth pr-sandbox-argocd-readonly-user-password: @@ -30,6 +32,12 @@ services: volumes: - db_data:/var/lib/mysql + dev-redis: + container_name: redis-dev + image: redis:8.4 + ports: + - "6379:6379" + dev-app: container_name: pr-sandbox-automation-dev-app build: @@ -45,6 +53,7 @@ services: - pr-sandbox-pr-installation-id - pr-sandbox-cluster-installation-id - pr-sandbox-mysql-connection-string + - pr-sandbox-redis-connection-string - pr-sandbox-argocd-webhook-auth - pr-sandbox-argocd-readonly-user-password - pr-sandbox-extra-configs diff --git a/pyproject.toml b/pyproject.toml index fc8bbc2..961e65f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -11,6 +11,7 @@ dependencies = [ "pydantic-settings>=2.11.0", "pyjwt>=2.10.1", "pymysql>=1.1.2", + "redis>=7.1.0", "requests>=2.32.5", "sqlmodel>=0.0.27", ] diff --git a/uv.lock b/uv.lock index e62bbf5..afd0201 100644 --- a/uv.lock +++ b/uv.lock @@ -508,6 +508,7 @@ dependencies = [ { name = "pydantic-settings" }, { name = "pyjwt" }, { name = "pymysql" }, + { name = "redis" }, { name = "requests" }, { name = "sqlmodel" }, ] @@ -525,6 +526,7 @@ requires-dist = [ { name = "pydantic-settings", specifier = ">=2.11.0" }, { name = "pyjwt", specifier = ">=2.10.1" }, { name = "pymysql", specifier = ">=1.1.2" }, + { name = "redis", specifier = ">=7.1.0" }, { name = "requests", specifier = ">=2.32.5" }, { name = "sqlmodel", specifier = ">=0.0.27" }, ] @@ -705,6 +707,15 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, ] +[[package]] +name = "redis" +version = "7.1.0" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/43/c8/983d5c6579a411d8a99bc5823cc5712768859b5ce2c8afe1a65b37832c81/redis-7.1.0.tar.gz", hash = "sha256:b1cc3cfa5a2cb9c2ab3ba700864fb0ad75617b41f01352ce5779dabf6d5f9c3c", size = 4796669, upload-time = "2025-11-19T15:54:39.961Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/89/f0/8956f8a86b20d7bb9d6ac0187cf4cd54d8065bc9a1a09eb8011d4d326596/redis-7.1.0-py3-none-any.whl", hash = "sha256:23c52b208f92b56103e17c5d06bdc1a6c2c0b3106583985a76a18f83b265de2b", size = 354159, upload-time = "2025-11-19T15:54:38.064Z" }, +] + [[package]] name = "requests" version = "2.32.5"