Update max_completion_tokens and add security context to VLM inferenc… (#370)

Author: bhardwaj-nakul

microservices/vlm-openvino-serving/compose.yaml

@@ -21,6 +21,7 @@ services:
       VLM_DEVICE: ${VLM_DEVICE}
       VLM_SEED: ${VLM_SEED}
       WORKERS: ${WORKERS:-1}
+      VLM_MAX_COMPLETION_TOKENS: ${VLM_MAX_COMPLETION_TOKENS}
     restart: unless-stopped
     devices:
       - /dev/dri:/dev/dri

microservices/vlm-openvino-serving/setup.sh

@@ -40,6 +40,11 @@ export RENDER_GROUP_ID=$(getent group render | awk -F: '{printf "%s\n", $3}')
 export VLM_SERVICE_PORT=9764
 export VLM_SEED=42
 
+# By default, VLM_MAX_COMPLETION_TOKENS is unset (which results in None in Python)
+# To set a specific value, uncomment and modify the following line:
+# export VLM_MAX_COMPLETION_TOKENS=1000
+unset VLM_MAX_COMPLETION_TOKENS
+
 # Check if VLM_MODEL_NAME is not defined or empty
 if [ -z "$VLM_MODEL_NAME" ]; then
     echo -e "ERROR: VLM_MODEL_NAME is not set in your shell environment."

microservices/vlm-openvino-serving/src/utils/common.py

@@ -5,8 +5,9 @@
 import os
 
 from dotenv import load_dotenv
-from pydantic import Field
+from pydantic import Field, field_validator
 from pydantic_settings import BaseSettings
+from typing import Optional, Any
 
 # Configure logger
 logging.basicConfig(
@@ -48,6 +49,20 @@ class Settings(BaseSettings):
     )
     VLM_DEVICE: str = Field(default="CPU", json_schema_extra={"env": "VLM_DEVICE"})
     SEED: int = Field(default=42, json_schema_extra={"env": "SEED"})
+    VLM_MAX_COMPLETION_TOKENS: Optional[int] = Field(
+        default=None,
+        json_schema_extra={"env": "VLM_MAX_COMPLETION_TOKENS"},
+    )
+
+    @field_validator("VLM_MAX_COMPLETION_TOKENS", mode="before")
+    @classmethod
+    def validate_max_completion_tokens(cls, v: Any) -> Optional[int]:
+        if v is None or v == "":
+            return None
+        try:
+            return int(v)
+        except (ValueError, TypeError):
+            return None
 
 
 class ErrorMessages:

microservices/vlm-openvino-serving/src/utils/data_models.py

@@ -5,6 +5,8 @@
 
 from pydantic import BaseModel, Field
 
+from .common import settings
+
 
 class MessageContentText(BaseModel):
     """
@@ -116,7 +118,7 @@ class ChatRequest(BaseModel):
         None, json_schema_extra={"example": 1.15}
     )
     max_completion_tokens: Optional[int] = Field(
-        None, json_schema_extra={"example": 1000}
+        settings.VLM_MAX_COMPLETION_TOKENS, json_schema_extra={"example": 1000}
     )
     temperature: Optional[float] = Field(None, json_schema_extra={"example": 0.3})
     top_p: Optional[float] = Field(None, json_schema_extra={"example": 0.5})

sample-applications/video-search-and-summarization/chart/subchart/vlminference/templates/vlm-inference-deployment.yaml

@@ -14,6 +14,10 @@ spec:
       labels:
         app: {{ .Values.vlminference.name }}
     spec:
+      securityContext:
+        runAsUser: 1000
+        runAsGroup: 1000
+        fsGroup: 1000
       containers:
         - name: {{ .Values.vlminference.name }}
           image: "{{ .Values.vlminference.image.repository }}:{{ .Values.vlminference.image.tag }}"
@@ -48,9 +52,9 @@ spec:
           volumeMounts:
           - name: workspace
             mountPath: /app/ov-model
+          securityContext:
+            readOnlyRootFilesystem: false
       volumes:
         - name: workspace
           persistentVolumeClaim:
-            claimName: {{ .Values.vlminference.volumeMounts.pvcName }}
-
-            
+            claimName: {{ .Values.vlminference.volumeMounts.pvcName }}