Alonragoler/multiagent (#6)

Author: alonr619

Makefile

@@ -61,6 +61,7 @@ KUBECTL := CLOUDSDK_CONFIG=$(GCLOUD_CONFIG_ABS) pixi run kubectl
 	gcp-admin-auth gcp-admin-project gcp-admin-kms-create-keyring gcp-admin-kms-create-key gcp-admin-kms-grant-user gcp-admin-kms-setup \
 	gke-auth gke-namespace gke-dummy-build gke-dummy-push gke-dummy-run-once gke-dummy-schedule gke-dummy-delete gke-dummy-logs \
 	sentiment-build sentiment-push sentiment-run-once sentiment-schedule sentiment-delete sentiment-logs \
+	redis-deploy redis-delete sentiment-rbac \
 	logout
 
 # ------------------------------------------------------------------------------------ #
@@ -277,7 +278,20 @@ sentiment-push: gcp-docker-auth gcp-artifact-registry-repo
 	$(GCLOUD) auth print-access-token | docker login -u oauth2accesstoken --password-stdin "https://$$REGISTRY_HOST"
 	@CLOUDSDK_CONFIG=$(GCLOUD_CONFIG_ABS) docker push "$(SENTIMENT_IMAGE)"
 
-sentiment-run-once: gke-namespace
+redis-deploy: gke-namespace
+	@sed -e 's|__NAMESPACE__|$(GKE_NAMESPACE)|g' cloud/k8s/redis/deployment.yaml | $(KUBECTL) apply -f -
+	@sed -e 's|__NAMESPACE__|$(GKE_NAMESPACE)|g' cloud/k8s/redis/service.yaml | $(KUBECTL) apply -f -
+	@echo "Redis deployed. Waiting for pod to be ready..."
+	@$(KUBECTL) -n "$(GKE_NAMESPACE)" rollout status deployment/redis --timeout=60s
+
+redis-delete: gke-auth
+	@$(KUBECTL) -n "$(GKE_NAMESPACE)" delete deployment redis --ignore-not-found
+	@$(KUBECTL) -n "$(GKE_NAMESPACE)" delete service redis-service --ignore-not-found
+
+sentiment-rbac: gke-namespace
+	@sed -e 's|__NAMESPACE__|$(GKE_NAMESPACE)|g' "$(SENTIMENT_MANIFEST_DIR)/rbac.yaml" | $(KUBECTL) apply -f -
+
+sentiment-run-once: gke-namespace sentiment-rbac
 	@test -n "$(OPENAI_API_KEY)" || (echo "Set OPENAI_API_KEY=... (e.g. make OPENAI_API_KEY=... sentiment-run-once)"; exit 1)
 	@test -n "$(TAVILY_API_KEY)" || (echo "Set TAVILY_API_KEY=... (e.g. make TAVILY_API_KEY=... sentiment-run-once)"; exit 1)
 	@sed -e 's|__IMAGE__|$(SENTIMENT_IMAGE)|g' \
@@ -287,7 +301,7 @@ sentiment-run-once: gke-namespace
 	     -e 's|__SENTIMENT_TOPIC__|$(SENTIMENT_TOPIC)|g' \
 	     "$(SENTIMENT_MANIFEST_DIR)/job.yaml" | $(KUBECTL) apply -f -
 
-sentiment-schedule: gke-namespace
+sentiment-schedule: gke-namespace sentiment-rbac
 	@test -n "$(OPENAI_API_KEY)" || (echo "Set OPENAI_API_KEY=... (e.g. make OPENAI_API_KEY=... sentiment-schedule)"; exit 1)
 	@test -n "$(TAVILY_API_KEY)" || (echo "Set TAVILY_API_KEY=... (e.g. make TAVILY_API_KEY=... sentiment-schedule)"; exit 1)
 	@sed -e 's|__IMAGE__|$(SENTIMENT_IMAGE)|g' \
@@ -300,6 +314,8 @@ sentiment-schedule: gke-namespace
 sentiment-delete: gke-auth
 	@$(KUBECTL) -n "$(GKE_NAMESPACE)" delete cronjob sentiment-agent --ignore-not-found
 	@$(KUBECTL) -n "$(GKE_NAMESPACE)" delete job sentiment-agent-once --ignore-not-found
+	@$(KUBECTL) -n "$(GKE_NAMESPACE)" delete jobs -l role=researcher --ignore-not-found
+	@$(KUBECTL) -n "$(GKE_NAMESPACE)" delete jobs -l role=synthesizer --ignore-not-found
 
 sentiment-logs: gke-auth
 	@$(KUBECTL) -n "$(GKE_NAMESPACE)" logs -l app=sentiment-agent --all-containers=true --tail=200 --prefix=true || ( \

cloud/k8s/redis/deployment.yaml

@@ -0,0 +1,30 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: redis
+  namespace: __NAMESPACE__
+  labels:
+    app: redis
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: redis
+  template:
+    metadata:
+      labels:
+        app: redis
+    spec:
+      containers:
+        - name: redis
+          image: redis:7-alpine
+          ports:
+            - containerPort: 6379
+          resources:
+            requests:
+              cpu: "50m"
+              memory: "64Mi"
+            limits:
+              cpu: "200m"
+              memory: "128Mi"
+          args: ["--maxmemory", "100mb", "--maxmemory-policy", "allkeys-lru"]

cloud/k8s/redis/service.yaml

@@ -0,0 +1,15 @@
+apiVersion: v1
+kind: Service
+metadata:
+  name: redis-service
+  namespace: __NAMESPACE__
+  labels:
+    app: redis
+spec:
+  type: ClusterIP
+  selector:
+    app: redis
+  ports:
+    - port: 6379
+      targetPort: 6379
+      protocol: TCP

cloud/k8s/sentiment-agent/cronjob.yaml

@@ -17,16 +17,26 @@ spec:
         metadata:
           labels:
             app: sentiment-agent
+            role: orchestrator
         spec:
+          serviceAccountName: sentiment-orchestrator
           restartPolicy: Never
           containers:
             - name: sentiment-agent
               image: __IMAGE__
               imagePullPolicy: IfNotPresent
               env:
+                - name: AGENT_ROLE
+                  value: "orchestrator"
                 - name: OPENAI_API_KEY
                   value: "__OPENAI_API_KEY__"
                 - name: TAVILY_API_KEY
                   value: "__TAVILY_API_KEY__"
                 - name: SENTIMENT_TOPIC
                   value: "__SENTIMENT_TOPIC__"
+                - name: REDIS_HOST
+                  value: "redis-service"
+                - name: SENTIMENT_IMAGE
+                  value: "__IMAGE__"
+                - name: K8S_NAMESPACE
+                  value: "__NAMESPACE__"

cloud/k8s/sentiment-agent/job.yaml

@@ -12,16 +12,26 @@ spec:
     metadata:
       labels:
         app: sentiment-agent
+        role: orchestrator
     spec:
+      serviceAccountName: sentiment-orchestrator
       restartPolicy: Never
       containers:
         - name: sentiment-agent
           image: __IMAGE__
           imagePullPolicy: Always
           env:
+            - name: AGENT_ROLE
+              value: "orchestrator"
             - name: OPENAI_API_KEY
               value: "__OPENAI_API_KEY__"
             - name: TAVILY_API_KEY
               value: "__TAVILY_API_KEY__"
             - name: SENTIMENT_TOPIC
               value: "__SENTIMENT_TOPIC__"
+            - name: REDIS_HOST
+              value: "redis-service"
+            - name: SENTIMENT_IMAGE
+              value: "__IMAGE__"
+            - name: K8S_NAMESPACE
+              value: "__NAMESPACE__"

cloud/k8s/sentiment-agent/rbac.yaml

@@ -0,0 +1,35 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: sentiment-orchestrator
+  namespace: __NAMESPACE__
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: sentiment-job-manager
+  namespace: __NAMESPACE__
+rules:
+  - apiGroups: ["batch"]
+    resources: ["jobs"]
+    verbs: ["create", "get", "list", "watch", "delete"]
+  - apiGroups: [""]
+    resources: ["pods"]
+    verbs: ["get", "list", "watch"]
+  - apiGroups: [""]
+    resources: ["pods/log"]
+    verbs: ["get"]
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: sentiment-orchestrator-binding
+  namespace: __NAMESPACE__
+subjects:
+  - kind: ServiceAccount
+    name: sentiment-orchestrator
+    namespace: __NAMESPACE__
+roleRef:
+  kind: Role
+  name: sentiment-job-manager
+  apiGroup: rbac.authorization.k8s.io