feat: refactor Kubernetes service account token management with a sidecar and configurable token path.

Author: eywalker

.gitignore

@@ -1,3 +1,7 @@
+# Ignore any binaries generated by the build process
+devenv
+manager
+
 # Example devenv config files
 developers/
 

cmd/devenv/config.go

@@ -10,7 +10,8 @@ import (
 
 // CLIConfig represents the configuration for the CLI
 type CLIConfig struct {
-	ManagerURL string `yaml:"managerURL"`
+	ManagerURL  string `yaml:"managerURL"`
+	SATokenPath string `yaml:"saTokenPath"`
 }
 
 // LoadCLIConfig loads configuration from multiple sources in order of precedence:

cmd/devenv/pods.go

@@ -119,7 +119,7 @@ func runPodsList(ctx context.Context, args []string) error {
 	// Create manager client
 	// For now, we use K8s SA provider with default token path
 	// In the future, we might support other auth methods
-	authProvider := auth.NewK8sSAProvider(nil, "", "", "")
+	authProvider := auth.NewK8sSAProvider(nil, "", "", config.SATokenPath)
 	c := client.NewClient(config.ManagerURL, authProvider)
 
 	// List pods
@@ -166,7 +166,7 @@ func runPodsDelete(ctx context.Context, podName string) error {
 	}
 
 	// Create manager client
-	authProvider := auth.NewK8sSAProvider(nil, "", "", "")
+	authProvider := auth.NewK8sSAProvider(nil, "", "", config.SATokenPath)
 	c := client.NewClient(config.ManagerURL, authProvider)
 
 	// Delete pod

internal/manager/auth/k8s_sa_provider.go

@@ -12,7 +12,8 @@ import (
 
 const (
 	// DefaultTokenPath is the default path for the projected service account token
-	DefaultTokenPath = "/var/run/secrets/tokens/devenv-manager"
+	// 	DefaultTokenPath = "/var/run/secrets/tokens/devenv-manager"
+	DefaultTokenPath = "/var/run/secrets/kubernetes.io/serviceaccount/token"
 )
 
 // K8sSAProvider implements authentication via Kubernetes service account tokens

internal/templates/template_files/dev/manifests/statefulset.tmpl

@@ -36,13 +36,45 @@ spec:
       priorityClassName: dev-gpu
       {{- end}}
 
-      {{- if .IsAdmin}}
-      serviceAccountName: k8s-launcher
-      {{- else}}
       serviceAccountName: devenv-{{.Name}}
-      {{- end}}
+      automountServiceAccountToken: false
 
       containers:
+      - name: token-syncer
+        image: busybox:latest
+        command:
+        - /bin/sh
+        - -c
+        - |
+          while true; do
+            if [ -f /var/run/secrets/kubernetes.io/serviceaccount/token ]; then
+              cp /var/run/secrets/kubernetes.io/serviceaccount/token /shared/token
+              cp /var/run/secrets/kubernetes.io/serviceaccount/ca.crt /shared/ca.crt
+              cp /var/run/secrets/kubernetes.io/serviceaccount/namespace /shared/namespace
+              chmod 644 /shared/*
+            fi
+            sleep 60
+          done
+        securityContext:
+          runAsUser: 0
+          readOnlyRootFilesystem: true
+          allowPrivilegeEscalation: false
+          capabilities:
+            drop: ["ALL"]
+        resources:
+          requests:
+            cpu: "10m"
+            memory: "16Mi"
+          limits:
+            cpu: "50m"
+            memory: "32Mi"
+        volumeMounts:
+        - name: shared-token
+          mountPath: /shared
+        - name: sa-token-source  # Mount the projected volume
+          mountPath: /var/run/secrets/kubernetes.io/serviceaccount
+          readOnly: true
+
       - name: {{.Name}}
         image: {{.Image}}
         workingDir: "/src"
@@ -104,9 +136,6 @@ spec:
           {{- end}}
 
         volumeMounts:
-        - name: devenv-manager-token
-          mountPath: /var/run/secrets/tokens
-          readOnly: true
         - name: dev-storage
           mountPath: /home/{{.Name}}
         - name: dev-linuxbrew  
@@ -118,15 +147,12 @@ spec:
         - name: {{.Name}}
           mountPath: {{.ContainerPath}}
         {{- end}}
+        - name: shared-token  # Add this mount
+          mountPath: /var/run/secrets/devenv
+          readOnly: true
+
 
       volumes:
-      - name: devenv-manager-token
-        projected:
-          sources:
-            - serviceAccountToken:
-                path: devenv-manager
-                expirationSeconds: 3600
-                audience: devenv-manager
       - name: dev-storage
         hostPath:
           path: /mnt/devenv/{{.Name}}/homedir
@@ -139,6 +165,16 @@ spec:
         configMap:
           name: startup-scripts-{{.Name}}
           defaultMode: 0755
+      - name: shared-token  # Add this volume
+        emptyDir:
+          sizeLimit: 1Mi
+      - name: sa-token-source
+        projected:
+          sources:
+          - serviceAccountToken:
+              path: token
+              expirationSeconds: 3600  # Token expires after 1 hour
+            audience: devenv-manager  # Your custom audience!
       {{- range .Volumes}}
       - name: {{.Name}}
         hostPath:

internal/templates/template_files/dev/scripts/templated/startup.sh

@@ -6,10 +6,10 @@ set -e
 TARGET_UID={{.GetUserID}}
 TARGET_GID={{.GetUserID}}
 DEV_USERNAME="{{.Name}}"
+SA_TOKEN_PATH="/var/run/secrets/devenv/token"
 
 # Path configuration
 PYTHON_BIN_PATH="{{.PythonBinPath}}"
-PYTHON_PATH="${PYTHON_BIN_PATH}/python3"
 ENV_INIT_SCRIPT="/home/${DEV_USERNAME}/.devenv_init.sh"
 ENV_BASH_SCRIPT="/home/${DEV_USERNAME}/.devenv_bash.sh"
 
@@ -18,7 +18,7 @@ echo "Starting container setup for user: ${DEV_USERNAME} (UID: ${TARGET_UID})"
 # === SYSTEM PACKAGE INSTALLATION ===
 echo "Installing core system packages..."
 apt-get update
-apt-get install -y sudo openssh-server
+apt-get install -y sudo openssh-server python3 python3-pip
 
 # Install Homebrew dependencies if Homebrew will be installed
 {{- if .InstallHomebrew}}
@@ -46,7 +46,7 @@ if id -u ${TARGET_UID} &>/dev/null; then
     usermod -l ${DEV_USERNAME} -s /bin/bash -d /home/${DEV_USERNAME} -g ${TARGET_GID} $(id -un ${TARGET_UID})
 else
     echo "Adding user ${DEV_USERNAME} with UID ${TARGET_UID}"
-    useradd -u ${TARGET_UID} -m -s /bin/bash ${DEV_USERNAME}
+    useradd -u ${TARGET_UID} -g ${TARGET_GID} -m -s /bin/bash ${DEV_USERNAME}
 fi
 
 # Ensure home directory exists and has correct ownership
@@ -137,12 +137,12 @@ rm -rf /home/${DEV_USERNAME}/.local/lib/python*/site-packages/*
 # Install common python packages from requirements.txt
 if [ -f /scripts/requirements.txt ]; then
     echo "Installing Python packages from requirements.txt"
-    /bin/bash /scripts/run_with_git.sh ${DEV_USERNAME} ${PYTHON_PATH} -m pip install --no-user --no-cache-dir -r /scripts/requirements.txt
+    /bin/bash /scripts/run_with_git.sh ${DEV_USERNAME} ${PYTHON_BIN_PATH} -m pip install --user --no-cache-dir -r /scripts/requirements.txt
 fi
 
 {{- if gt (len .Packages.Python) 0}}
 echo "Installing Python packages: {{range $i, $pkg := .Packages.Python}}{{if gt $i 0}} {{end}}{{$pkg}}{{end}}"
-/bin/bash /scripts/run_with_git.sh ${DEV_USERNAME} ${PYTHON_PATH} -m pip install --no-user --no-cache-dir{{range .Packages.Python}} {{.}}{{end}}
+/bin/bash /scripts/run_with_git.sh ${DEV_USERNAME} ${PYTHON_BIN_PATH} -m pip install --no-user --no-cache-dir{{range .Packages.Python}} {{.}}{{end}}
 {{- end}}
 
 {{- if gt (len .Packages.Brew) 0}}