Merge pull request #58 from sallyom/update-llama-stack

update llama-stack manifests

Author: cooktheryan

kubernetes/llama-stack/configmap.yaml

@@ -1,4 +1,4 @@
-kind: ConfigMap
+kind: ConfigMap
 apiVersion: v1
 metadata:
   name: run-config
@@ -18,10 +18,17 @@ data:
     - vector_io
     providers:
       inference:
-      - provider_id: vllm-inference
+      - provider_id: llama-3b
         provider_type: remote::vllm
         config:
-          url: ${env.VLLM_URL}
+          url: ${env.LLAMA3B_URL}
+          max_tokens: 128000
+          api_token: fake
+          tls_verify: false
+      - provider_id: granite
+        provider_type: remote::vllm
+        config:
+          url: ${env.GRANITE_URL}
           max_tokens: 128000
           api_token: fake
           tls_verify: false
@@ -39,7 +46,7 @@ data:
       - provider_id: milvus
         provider_type: inline::milvus
         config:
-          db_path: ${env.MILVUS_DB_PATH}
+         db_path: ${env.MILVUS_DB_PATH}
       safety:
       - provider_id: llama-guard
         provider_type: inline::llama-guard
@@ -92,9 +99,8 @@ data:
         provider_type: inline::meta-reference
         config:
           service_name: ${env.OTEL_SERVICE_NAME:llama-stack}
-          sinks: ${env.TELEMETRY_SINKS:console, otel_trace, otel_metric, sqlite}
+          sinks: ${env.TELEMETRY_SINKS:console, otel_trace, sqlite}
           otel_trace_endpoint: ${env.OTEL_TRACE_ENDPOINT:}
-          otel_metric_endpoint: ${env.OTEL_METRIC_ENDPOINT:}
           sqlite_db_path: ${env.SQLITE_DB_PATH:~/.llama/distributions/remote-vllm/trace_store.db}
       tool_runtime:
       - provider_id: brave-search
@@ -121,8 +127,12 @@ data:
       db_path: ${env.SQLITE_STORE_DIR:~/.llama/distributions/remote-vllm}/registry.db
     models:
     - metadata: {}
-      model_id: ${env.INFERENCE_MODEL}
-      provider_id: vllm-inference
+      model_id: ${env.LLAMA3B_MODEL}
+      provider_id: llama-3b
+      model_type: llm
+    - metadata: {}
+      model_id: ${env.GRANITE_MODEL}
+      provider_id: granite 
       model_type: llm
     - metadata: {}
       model_id: ${env.SAFETY_MODEL}

kubernetes/llama-stack/deployment.yaml

@@ -1,68 +1,75 @@
+kind: Deployment
 apiVersion: apps/v1
-kind: Deployment
 metadata:
   name: llamastack-deployment
 spec:
-  replicas: 1
   selector:
     matchLabels:
       app: llamastack
   template:
     metadata:
+      #annotations:
+      #  sidecar.opentelemetry.io/inject: otelsidecar
       labels:
         app: llamastack
     spec:
+      volumes:
+        - name: run-config-volume
+          configMap:
+            name: run-config
+            defaultMode: 420
+        - name: llama-persist
+          persistentVolumeClaim:
+            claimName: llama-persist
+        - name: cache
+          emptyDir: {}
+        - name: pythain
+          emptyDir: {}
       containers:
-      - args:
-        - --yaml-config
-        - /app-config/config.yaml
-        env:
-        - name: VLLM_MAX_TOKENS
-          value: "128000"
-        - name: INFERENCE_MODEL
-          value: meta-llama/Llama-3.2-3B-Instruct
-        - name: VLLM_URL
-          value: http://vllm:8000/v1
-        - name: VLLM_API_TOKEN
-          value: fake
-        - name:  SAFETY_MODEL
-          value: meta-llama/Llama-Guard-3-8B
-        - name:  SAFETY_VLLM_URL
-          value: http://safety.llama-serve.svc.cluster.local:8000/v1
-        - name: OTEL_TRACE_ENDPOINT
-          value: http://otel-collector-collector.observability-hub.svc.cluster.local:4318/v1/traces
-        - name: OTEL_METRIC_ENDPOINT
-          value: http://otel-collector-collector.observability-hub.svc.cluster.local:4318/v1/metrics
-        - name: MILVUS_DB_PATH
-          value: 'milvus.db'
-        image: quay.io/redhat-et/llama:vllm-0.1.9
-        imagePullPolicy: Always
-        name: llamastack
-        ports:
-        - containerPort: 8321
-          protocol: TCP
-        resources: {}
-        terminationMessagePath: /dev/termination-log
-        terminationMessagePolicy: File
-        volumeMounts:
-        - mountPath: /app-config
-          name: run-config-volume
-        - mountPath: /.llama
-          name: llama-persist
-        - mountPath: /.cache
-          name: cache
-      dnsPolicy: ClusterFirst
-      restartPolicy: Always
-      schedulerName: default-scheduler
+        - resources: {}
+          terminationMessagePath: /dev/termination-log
+          name: llamastack
+          env:
+            - name: MAX_TOKENS
+              value: '128000'
+            - name: VLLM_MAX_TOKENS
+              value: '128000'
+            - name: LLAMA3B_MODEL
+              value: meta-llama/Llama-3.2-3B-Instruct
+            - name: GRANITE_URL
+              value: 'https://granite-8b-llama-serve.apps.ocp-beta-test.nerc.mghpcc.org/v1'
+            - name: GRANITE_MODEL
+              value: ibm-granite/granite-3.2-8b-instruct
+            - name: LLAMA3B_URL
+              value: 'https://llama32-3b-llama-serve.apps.ocp-beta-test.nerc.mghpcc.org/v1'
+            - name: VLLM_API_TOKEN
+              value: fake
+            - name: OTEL_SERVICE_NAME
+              value: om-llamastack
+            - name: OTEL_TRACE_ENDPOINT
+              value: 'http://otel-collector-collector.observability-hub.svc.cluster.local:4318/v1/traces'
+            - name: SAFETY_MODEL
+              value: meta-llama/Llama-Guard-3-8B
+            - name: SAFETY_VLLM_URL
+              value: 'http://safety.llama-serve.svc.cluster.local:8000/v1'
+            - name: MILVUS_DB_PATH
+              value: milvus.db
+          ports:
+            - containerPort: 8321
+              protocol: TCP
+          imagePullPolicy: Always
+          volumeMounts:
+            - name: pythain
+              mountPath: /pythainlp-data
+            - name: run-config-volume
+              mountPath: /app-config
+            - name: llama-persist
+              mountPath: /.llama
+            - name: cache
+              mountPath: /.cache
+          terminationMessagePolicy: File
+          image: 'quay.io/redhat-et/llama:vllm-0.1.9'
+          args:
+            - '--config'
+            - /app-config/config.yaml
       securityContext: {}
-      terminationGracePeriodSeconds: 30
-      volumes:
-      - configMap:
-          defaultMode: 420
-          name: run-config
-        name: run-config-volume
-      - persistentVolumeClaim:
-          claimName: llama-persist
-        name: llama-persist
-      - emptyDir: {}
-        name: cache

kubernetes/llama-stack/template.yaml

@@ -1,32 +1,9 @@
-kind: ConfigMap
+kind: ConfigMap
 apiVersion: v1
 metadata:
   name: template
-  uid: 53e66bb5-bebd-4a5d-8471-54728d3adc18
-  resourceVersion: '602370800'
-  creationTimestamp: '2025-02-20T18:20:10Z'
   labels:
     app: vllm
-  managedFields:
-    - manager: kubectl-create
-      operation: Update
-      apiVersion: v1
-      time: '2025-02-20T18:20:10Z'
-      fieldsType: FieldsV1
-      fieldsV1:
-        'f:data': {}
-        'f:metadata':
-          'f:labels':
-            .: {}
-            'f:app': {}
-    - manager: Mozilla
-      operation: Update
-      apiVersion: v1
-      time: '2025-02-25T18:54:13Z'
-      fieldsType: FieldsV1
-      fieldsV1:
-        'f:data':
-          'f:tool_chat_template_llama3.2_json.jinja': {}
 data:
   tool_chat_template_llama3.2_json.jinja: |
     {{- bos_token }} {%- if custom_tools is defined %}