feat(iam): add in-memory introspection cache

Route IAM introspection and JWKS requests through a loopback Go proxy with bounded TTL caches and request coalescing. Install the service in Core images, preserve operator configuration on upgrades, and cover cache behavior with race-enabled tests.

Author: cassi-volkova

.github/workflows/tests.yml

@@ -16,6 +16,19 @@ on:
       - '*.md'
 
 jobs:
+  GoTests:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-go@v6
+        with:
+          go-version-file: services/iam-cache/go.mod
+          cache-dependency-path: services/iam-cache/go.mod
+      - name: Test IAM cache
+        working-directory: services/iam-cache
+        run: |
+          go test -race ./...
+          go vet ./...
   Lint:
     runs-on: ubuntu-24.04
     strategy:

etc/exordos_core/iam_cache.json.example

@@ -0,0 +1,10 @@
+{
+  "public_listen_address": "127.0.0.1:11110",
+  "internal_listen_address": "127.0.0.1:11111",
+  "core_url": "http://127.0.0.1:11010",
+  "request_timeout": "5s",
+  "introspection_cache_ttl": "15s",
+  "introspection_cache_max_entries": 100000,
+  "jwks_cache_ttl": "1m",
+  "jwks_cache_max_entries": 1000
+}

etc/systemd/exordos-iam-cache.service

@@ -0,0 +1,13 @@
+[Unit]
+Description=Exordos IAM Cache Service
+After=network-online.target ec-user-api.service
+
+[Service]
+TimeoutStopSec=10
+Restart=always
+RestartSec=5s
+KillSignal=SIGINT
+ExecStart=/usr/bin/exordos-iam-cache -config /etc/exordos_core/iam_cache.json
+
+[Install]
+WantedBy=multi-user.target

exordos/images/bootstrap.sh

@@ -85,6 +85,18 @@ if [[ -n "$PERSISTENT_DISK" ]]; then
     persist_migrate_complete
 fi
 
+# Existing persistent installations predate the IAM cache configuration. Add
+# its default config only when it is absent so operator changes survive future
+# image updates.
+if [[ ! -f "$GC_CFG_DIR/iam_cache.json" ]]; then
+    sudo install \
+      -o root \
+      -g root \
+      -m 0644 \
+      "$GC_PATH/etc/exordos_core/iam_cache.json.example" \
+      "$GC_CFG_DIR/iam_cache.json"
+fi
+
 # Create deprecated path
 mkdir -p /var/lib/exordos/data
 
@@ -159,6 +171,7 @@ fi
 log "systemctl enable --now ec-services"
 sudo systemctl enable --now \
     ec-user-api \
+    exordos-iam-cache \
     ec-orch-api \
     ec-status-api \
     ec-boot-api \

exordos/images/install.sh

@@ -32,6 +32,7 @@ GC_PG_PASS="exordos_core"
 GC_PG_DB="exordos_core"
 
 SYSTEMD_SERVICE_DIR=/etc/systemd/system/
+IAM_CACHE_GO_VERSION="1.23.12"
 
 DEV_SDK_PATH="/opt/gcl_sdk"
 SDK_DEV_MODE=$([ -d "$DEV_SDK_PATH" ] && echo "true" || echo "false")
@@ -141,12 +142,73 @@ sudo systemctl enable nginx
 # Install exordos core
 sudo mkdir -p $GC_CFG_DIR
 sudo cp "$GC_PATH/etc/exordos_core/logging.yaml" $GC_CFG_DIR/
+sudo install \
+  -o root \
+  -g root \
+  -m 0644 \
+  "$GC_PATH/etc/exordos_core/iam_cache.json.example" \
+  "$GC_CFG_DIR/iam_cache.json"
 # Drop-in config dir loaded by ec-user-api via --config-dir. The notification
 # element lands its [events] override and event_type_mapping.yaml here; must
 # exist (oslo --config-dir errors on a missing directory).
 sudo mkdir -p $GC_CFG_DIR/exordos_core.d
 sudo cp "$GC_PATH/exordos/images/bootstrap.sh" $BOOTSTRAP_PATH/0100-ec-bootstrap.sh
 
+# Build the IAM cache with a temporary Go toolchain. Only the stripped static
+# binary is installed into the image; the toolchain and every build cache are
+# removed both after a successful build and if the build fails.
+case "$(dpkg --print-architecture)" in
+  amd64)
+    IAM_CACHE_GO_ARCH="amd64"
+    IAM_CACHE_GO_SHA256="d3847fef834e9db11bf64e3fb34db9c04db14e068eeb064f49af747010454f90"
+    ;;
+  arm64)
+    IAM_CACHE_GO_ARCH="arm64"
+    IAM_CACHE_GO_SHA256="52ce172f96e21da53b1ae9079808560d49b02ac86cecfa457217597f9bc28ab3"
+    ;;
+  *)
+    echo "Unsupported architecture for the IAM cache: $(dpkg --print-architecture)" >&2
+    exit 1
+    ;;
+esac
+
+IAM_CACHE_BUILD_DIR=$(mktemp -d)
+cleanup_iam_cache_build() {
+    if [[ -n "${IAM_CACHE_BUILD_DIR:-}" && -d "$IAM_CACHE_BUILD_DIR" ]]; then
+        rm -rf -- "$IAM_CACHE_BUILD_DIR"
+    fi
+}
+trap cleanup_iam_cache_build EXIT
+
+curl -fsSLo "$IAM_CACHE_BUILD_DIR/go.tar.gz" \
+  "https://go.dev/dl/go${IAM_CACHE_GO_VERSION}.linux-${IAM_CACHE_GO_ARCH}.tar.gz"
+echo "$IAM_CACHE_GO_SHA256  $IAM_CACHE_BUILD_DIR/go.tar.gz" \
+  | sha256sum --check -
+tar -xzf "$IAM_CACHE_BUILD_DIR/go.tar.gz" -C "$IAM_CACHE_BUILD_DIR"
+
+(
+    cd "$GC_PATH/services/iam-cache"
+    CGO_ENABLED=0 \
+    GOCACHE="$IAM_CACHE_BUILD_DIR/go-cache" \
+    GOPATH="$IAM_CACHE_BUILD_DIR/gopath" \
+      "$IAM_CACHE_BUILD_DIR/go/bin/go" build \
+        -buildvcs=false \
+        -trimpath \
+        -ldflags="-s -w" \
+        -o "$IAM_CACHE_BUILD_DIR/exordos-iam-cache" \
+        ./cmd/exordos-iam-cache
+)
+sudo install \
+  -o root \
+  -g root \
+  -m 0755 \
+  "$IAM_CACHE_BUILD_DIR/exordos-iam-cache" \
+  /usr/bin/exordos-iam-cache
+
+cleanup_iam_cache_build
+trap - EXIT
+unset IAM_CACHE_BUILD_DIR
+
 cd "$GC_PATH"
 uv sync
 source "$GC_PATH"/.venv/bin/activate
@@ -206,6 +268,7 @@ sudo cp "$GC_PATH/etc/systemd/ec-core-agent.service" $SYSTEMD_SERVICE_DIR
 sudo cp "$GC_PATH/etc/systemd/exordos-universal-agent.service" $SYSTEMD_SERVICE_DIR
 sudo cp "$GC_PATH/etc/systemd/exordos-universal-scheduler.service" $SYSTEMD_SERVICE_DIR
 sudo cp "$GC_PATH/etc/systemd/exordos-repo-proxy-gservice.service" $SYSTEMD_SERVICE_DIR
+sudo cp "$GC_PATH/etc/systemd/exordos-iam-cache.service" $SYSTEMD_SERVICE_DIR
 
 # Prepare DNSaaS
 sudo systemctl disable --now pdns dnsdist@public dnsdist@private

exordos/manifests/core.yaml.j2

@@ -215,6 +215,14 @@ resources:
           host: 127.0.0.1
           port: 11010
           weight: 1
+    core_lb_iam_cache_backend_http:
+      project_id: "12345678-c625-4fee-81d5-f691897b8142"
+      parent: $core.network.lb.$core_lb:uuid
+      endpoints:
+        - kind: host
+          host: 127.0.0.1
+          port: 11110
+          weight: 1
   $core.network.lb.$core_lb.vhosts:
     core_lb_core_http:
       project_id: "12345678-c625-4fee-81d5-f691897b8142"
@@ -248,6 +256,30 @@ resources:
           - kind: rewrite_url
             regex: "^/api/core/(.*)"
             replacement: "/$1"
+    core_lb_iam_clients:
+      project_id: "12345678-c625-4fee-81d5-f691897b8142"
+      parent: $core.network.lb.$core_lb.vhosts.$core_lb_core_http:uuid
+      condition:
+        kind: prefix
+        value: /api/core/v1/iam/clients/
+        allowed_ips:
+          - 0.0.0.0/0
+        actions:
+          - kind: backend
+            pool: $core.network.lb.$core_lb.backend_pools.$core_lb_iam_cache_backend_http:uuid
+            protocol:
+              kind: http
+        modifiers:
+          - kind: auto_header
+            headers:
+              - 'Host'
+              - 'X-Forwarded-For'
+              - 'X-Forwarded-Port'
+              - 'X-Forwarded-Proto'
+              - 'X-Forwarded-Prefix'
+          - kind: rewrite_url
+            regex: "^/api/core/(.*)"
+            replacement: "/$1"
     core_lb_iam_default_client:
       project_id: "12345678-c625-4fee-81d5-f691897b8142"
       parent: $core.network.lb.$core_lb.vhosts.$core_lb_core_http:uuid
@@ -258,7 +290,7 @@ resources:
           - 0.0.0.0/0
         actions:
           - kind: backend
-            pool: $core.network.lb.$core_lb.backend_pools.$core_lb_core_backend_http:uuid
+            pool: $core.network.lb.$core_lb.backend_pools.$core_lb_iam_cache_backend_http:uuid
             protocol:
               kind: http
         modifiers:

services/iam-cache/README.md

@@ -0,0 +1,53 @@
+# Exordos IAM Cache
+
+`exordos-iam-cache` is an in-memory caching proxy for the Exordos Core IAM
+introspection and JWKS endpoints.
+
+The public listener preserves the existing Core routes:
+
+- `GET /v1/iam/clients/{client_uuid}/actions/introspect`
+- `GET /v1/iam/clients/{client_uuid}/actions/jwks`
+
+All other IAM client requests are forwarded unchanged to Core and are never
+cached. Any request carrying `X-OTP`, including a token request, also bypasses
+the cache. Successful introspection responses without `X-OTP` are cached by
+access token. The token UUID is read from the validated access token's `jti`
+claim and is used by the reverse index.
+
+The internal listener exposes an idempotent invalidation endpoint:
+
+```text
+DELETE /internal/v1/cache/introspection/{token_uuid}
+```
+
+Core does not call this endpoint in the first implementation. Until that
+integration is added, introspection entries expire only by their configured
+TTL, the access token expiration, or capacity eviction.
+
+JWKS responses use a separate cache keyed by IAM client UUID and a separate
+TTL.
+
+## Configuration
+
+The deployment example is
+[`../../etc/exordos_core/iam_cache.json.example`](../../etc/exordos_core/iam_cache.json.example).
+Cache lifetimes and the upstream request timeout use Go duration syntax such
+as `15s`, `5m`, or `1h`. The deployed defaults are 15 seconds for
+introspection and one minute for JWKS.
+
+The internal listener defaults to loopback. If it is exposed outside the host,
+protect it with the deployment's service-to-service authentication layer.
+
+## Run
+
+```bash
+go run ./cmd/exordos-iam-cache \
+  -config ../../etc/exordos_core/iam_cache.json.example
+```
+
+## Test
+
+```bash
+go test -race ./...
+go vet ./...
+```

services/iam-cache/cmd/exordos-iam-cache/main.go

@@ -0,0 +1,115 @@
+// Copyright 2026 Genesis Corporation
+//
+// All Rights Reserved.
+//
+//    Licensed under the Apache License, Version 2.0 (the "License"); you may
+//    not use this file except in compliance with the License. You may obtain
+//    a copy of the License at
+//
+//         http://www.apache.org/licenses/LICENSE-2.0
+//
+//    Unless required by applicable law or agreed to in writing, software
+//    distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+//    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+//    License for the specific language governing permissions and limitations
+//    under the License.
+
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"net/http"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"github.com/exordos/exordos_core/services/iam-cache/internal/app"
+)
+
+func main() {
+	if err := run(); err != nil {
+		log.Fatal(err)
+	}
+}
+
+func run() error {
+	configPath := flag.String(
+		"config",
+		"/etc/exordos_core/iam_cache.json",
+		"path to the JSON configuration file",
+	)
+	flag.Parse()
+
+	config, err := app.LoadConfig(*configPath)
+	if err != nil {
+		return err
+	}
+	proxy := app.NewProxy(config)
+
+	publicServer := newHTTPServer(
+		config.PublicListenAddress,
+		proxy.PublicHandler(),
+	)
+	internalServer := newHTTPServer(
+		config.InternalListenAddress,
+		proxy.InternalHandler(),
+	)
+
+	runContext, stop := signal.NotifyContext(
+		context.Background(),
+		syscall.SIGINT,
+		syscall.SIGTERM,
+	)
+	defer stop()
+
+	serverErrors := make(chan error, 2)
+	startServer("public", publicServer, serverErrors)
+	startServer("internal", internalServer, serverErrors)
+
+	var runErr error
+	select {
+	case <-runContext.Done():
+	case runErr = <-serverErrors:
+		stop()
+	}
+
+	shutdownContext, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+
+	publicErr := publicServer.Shutdown(shutdownContext)
+	internalErr := internalServer.Shutdown(shutdownContext)
+	return errors.Join(runErr, publicErr, internalErr)
+}
+
+func newHTTPServer(address string, handler http.Handler) *http.Server {
+	return &http.Server{
+		Addr:              address,
+		Handler:           handler,
+		ReadHeaderTimeout: 5 * time.Second,
+		IdleTimeout:       60 * time.Second,
+	}
+}
+
+func startServer(
+	name string,
+	server *http.Server,
+	errorsChannel chan<- error,
+) {
+	go func() {
+		log.Printf("%s listener started on %s", name, server.Addr)
+		err := server.ListenAndServe()
+		if err != nil && !errors.Is(err, http.ErrServerClosed) {
+			errorsChannel <- fmt.Errorf("%s listener: %w", name, err)
+		}
+	}()
+}
+
+func init() {
+	log.SetOutput(os.Stderr)
+	log.SetFlags(log.Ldate | log.Ltime | log.LUTC)
+}

services/iam-cache/go.mod

@@ -0,0 +1,3 @@
+module github.com/exordos/exordos_core/services/iam-cache
+
+go 1.23.0