[TT-16249]: Add vault to tyk-pro for integration tests (#78)

* Add Vault and upstream auth server for E2E tests

* Update tyk.conf to hardcode external_services.oauth.mtls to Vault

* added fixes

* fixed task implementation

---------

Co-authored-by: Leonid Bugaev <leonsbox@gmail.com>

Author: kofoworola

Taskfile.yml

@@ -21,7 +21,7 @@ tasks:
       DB: '{{ or .DB "mongo44" }}'
       CACHE: '{{ or .CACHE "redis6" }}'
       VARIATION: '{{or .VARIATION "murmur64" }}'
-      confs_dir: './{{ .FLAVOUR }}'
+      confs_dir: "./{{ .FLAVOUR }}"
       env_file: local-{{.DB}}.env
     cmds:
       - |
@@ -41,7 +41,7 @@ tasks:
       DB: '{{ or .DB "mongo7" }}'
       CACHE: '{{ or .CACHE "redis6" }}'
       VARIATION: '{{or .VARIATION "murmur64" }}'
-      confs_dir: './{{ .FLAVOUR }}'
+      confs_dir: "./{{ .FLAVOUR }}"
       env_file: upgrade.env
     cmds:
       - |
@@ -53,15 +53,14 @@ tasks:
           docker compose -p auto -f pro-ha.yml -f deps_pro-ha.yml -f ${DB}.yml -f ${CACHE}.yml --env-file master.env --profile slave-datacenter up --quiet-pull -d --build
         fi
 
-
   clean:
     desc: Tears down all local infrastructure including networking
     env:
       FLAVOUR: '{{ or .FLAVOUR "pro" }}'
       DB: '{{ or .DB "mongo44" }}'
       CACHE: '{{ or .CACHE "redis6" }}'
       VARIATION: '{{or .VARIATION "murmur64" }}'
-      confs_dir: './{{ .FLAVOUR }}'
+      confs_dir: "./{{ .FLAVOUR }}"
       env_file: local-{{.DB}}.env
     cmds:
       - docker compose -p auto -f ${FLAVOUR}.yml -f deps_${FLAVOUR}.yml -f ${DB}.yml -f ${CACHE}.yml --env-file master.env --env-file master.env --profile all down --remove-orphans
@@ -72,12 +71,31 @@ tasks:
       Deploys pytest container that runs against local infrastructure using docker compose manifest
       - For mdcb you may want to use the ARGS "--ci -m "mdcb""
       - For non mdcb deployments something like "-m "not local and not mdcb""
-    cmds: 
+    cmds:
       - |
         docker run --rm --network auto_default --env-file ./pytest.env \
         754489498669.dkr.ecr.eu-central-1.amazonaws.com/tyk-automated-tests:master \
         pytest -c pytest_ci.ini {{.CLI_ARGS}}
 
+  seed-vault:
+    desc: Seeds Vault with placeholder OAuth mTLS secrets required for local gateway startup
+    cmds:
+      - |
+        echo "Waiting for Vault to be ready..."
+        until curl -s http://localhost:8200/v1/sys/health | grep -q '"initialized":true'; do
+          sleep 2
+        done
+        echo "Seeding Vault with placeholder OAuth mTLS secrets..."
+        curl -s -X POST http://localhost:8200/v1/secret/data/oauth/cert_file \
+          -H "X-Vault-Token: root" \
+          -H "Content-Type: application/json" \
+          -d '{"data": {"cert_file": "placeholder", "value": "placeholder"}}'
+        curl -s -X POST http://localhost:8200/v1/secret/data/oauth/key_file \
+          -H "X-Vault-Token: root" \
+          -H "Content-Type: application/json" \
+          -d '{"data": {"key_file": "placeholder", "value": "placeholder"}}'
+        echo "Vault seeded successfully. You can now start your local gateway."
+
   healthcheck:
     desc: Usefull healthchecks to ensure services are up and running
     cmds:

deps_pro-ha.yml

@@ -102,7 +102,7 @@ services:
     container_name: rabbitmq
     image: rabbitmq:4.1.0-management-alpine
     ports:
-      - "5672:5672"   # AMQP protocol
+      - "5672:5672" # AMQP protocol
       - "15672:15672" # Management UI
       - "1883:1883"
     environment:
@@ -135,7 +135,11 @@ services:
     volumes:
       - ./keycloak/mcp-test-realm.json:/opt/keycloak/data/import/mcp-test-realm.json:ro
     healthcheck:
-      test: ["CMD-SHELL", "exec 3<>/dev/tcp/localhost/8080 && echo -e 'GET /health/ready HTTP/1.1\\r\\nHost: localhost\\r\\n\\r\\n' >&3 && cat <&3 | grep -q UP"]
+      test:
+        [
+          "CMD-SHELL",
+          "exec 3<>/dev/tcp/localhost/8080 && echo -e 'GET /health/ready HTTP/1.1\\r\\nHost: localhost\\r\\n\\r\\n' >&3 && cat <&3 | grep -q UP",
+        ]
       interval: 10s
       timeout: 10s
       retries: 15
@@ -153,8 +157,35 @@ services:
       - PORT=7878
       - DEBUG=false
     healthcheck:
-      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:7878/health"]
+      test:
+        [
+          "CMD",
+          "wget",
+          "--no-verbose",
+          "--tries=1",
+          "--spider",
+          "http://localhost:7878/health",
+        ]
       interval: 10s
       timeout: 3s
       retries: 3
       start_period: 5s
+
+  vault:
+    profiles: ["all", "master-datacenter"]
+    container_name: vault
+    image: hashicorp/vault:latest
+    ports:
+      - "8200:8200"
+    environment:
+      - VAULT_DEV_ROOT_TOKEN_ID=root
+      - VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200
+    cap_add:
+      - IPC_LOCK
+
+  upstream-auth-server:
+    profiles: ["all", "master-datacenter"]
+    container_name: upstream-auth-server
+    image: kennethreitz/httpbin
+    ports:
+      - "8090:80"

deps_pro.yml

@@ -97,8 +97,59 @@ services:
       - PORT=7878
       - DEBUG=false
     healthcheck:
-      test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:7878/health"]
+      test:
+        [
+          "CMD",
+          "wget",
+          "--no-verbose",
+          "--tries=1",
+          "--spider",
+          "http://localhost:7878/health",
+        ]
       interval: 10s
       timeout: 3s
       retries: 3
       start_period: 5s
+
+  vault:
+    profiles: ["all", "master-datacenter"]
+    container_name: vault
+    image: hashicorp/vault:latest
+    ports:
+      - "8200:8200"
+    environment:
+      - VAULT_DEV_ROOT_TOKEN_ID=root
+      - VAULT_DEV_LISTEN_ADDRESS=0.0.0.0:8200
+    cap_add:
+      - IPC_LOCK
+    healthcheck:
+      test: ["CMD", "vault", "status", "-address=http://127.0.0.1:8200"]
+      interval: 5s
+      retries: 10
+      start_period: 3s
+      timeout: 5s
+
+  vault-seed:
+    profiles: ["all", "master-datacenter"]
+    container_name: vault-seed
+    image: hashicorp/vault:latest
+    environment:
+      - VAULT_ADDR=http://vault:8200
+      - VAULT_TOKEN=root
+    depends_on:
+      vault:
+        condition: service_healthy
+    entrypoint: ["/bin/sh", "-c"]
+    command:
+      - |
+        vault kv put secret/oauth/cert_file cert_file="placeholder" value="placeholder"
+        vault kv put secret/oauth/key_file key_file="placeholder" value="placeholder"
+        echo "Vault seeded with placeholder OAuth mTLS cert_file and key_file secrets"
+    restart: "no"
+
+  upstream-auth-server:
+    profiles: ["all", "master-datacenter"]
+    container_name: upstream-auth-server
+    image: kennethreitz/httpbin
+    ports:
+      - "8090:80"

local-mongo44.env

@@ -1,4 +1,6 @@
 TYK_DB_MONGOURL=mongodb://mongo:27017/tyk_analytics
+TYK_GW_HASHKEYFUNCTION=sha256
+TYK_GW_HASHKEYS=true
 TYK_GW_SLAVEOPTIONS_APIKEY=<PLACEHOLDER>
 TYK_GW_SLAVEOPTIONS_RPCKEY=<PLACEHOLDER>
 TYK_LOGLEVEL=debug
@@ -28,4 +30,4 @@ TYK_PMP_PUMPS_MONGO_META_MONGOURL=mongodb://mongo:27017/tyk_analytics
 TYK_PMP_PUMPS_MONGO_TYPE=mongo
 TYK_PMP_UPTIMEPUMPCONFIG_COLLECTIONNAME=tyk_analytics
 TYK_PMP_UPTIMEPUMPCONFIG_MONGOURL=mongodb://mongo:27017/tyk_analytics
-TYK_PMP_UPTIMEPUMPCONFIG_UPTIMETYPE=mongo
+TYK_PMP_UPTIMEPUMPCONFIG_UPTIMETYPE=mongo