From 80e0b5d8da4e89f7853148b96e7f3d352e59952e Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Tue, 10 Mar 2026 11:03:43 -0400 Subject: [PATCH 1/3] fix: use correct GPU node selector labels and improve prod probe timeouts The generated YAML used short GPU names (e.g., "L4") for the nvidia.com/gpu.product node selector, but K8s nodes use labels like "NVIDIA-L4". Add node_selector_label field to GPU catalog and use it in YAML generation. Also increase production liveness probe delay from 120s to 600s to allow time for model loading, and update vLLM image to latest. Signed-off-by: Andre Fredette --- data/configuration/model_catalog.json | 8 ++++++++ src/neuralnav/configuration/generator.py | 13 ++++++++++--- .../templates/kserve-inferenceservice.yaml.j2 | 7 ++++--- src/neuralnav/knowledge_base/model_catalog.py | 2 ++ 4 files changed, 24 insertions(+), 6 deletions(-) diff --git a/data/configuration/model_catalog.json b/data/configuration/model_catalog.json index 1929d6b..7602d6c 100644 --- a/data/configuration/model_catalog.json +++ b/data/configuration/model_catalog.json @@ -1280,6 +1280,7 @@ { "gpu_type": "L4", "aliases": ["NVIDIA-L4", "L4"], + "node_selector_label": "NVIDIA-L4", "memory_gb": 24, "compute_capability": "8.9", "typical_use_cases": ["inference"], @@ -1293,6 +1294,7 @@ { "gpu_type": "A10G", "aliases": ["NVIDIA-A10G", "A10G"], + "node_selector_label": "NVIDIA-A10G", "memory_gb": 24, "compute_capability": "8.6", "typical_use_cases": ["inference"], @@ -1306,6 +1308,7 @@ { "gpu_type": "A100-40", "aliases": ["NVIDIA-A100-40GB", "A100-40", "A100-40GB"], + "node_selector_label": "NVIDIA-A100-SXM4-40GB", "memory_gb": 40, "compute_capability": "8.0", "typical_use_cases": ["inference", "training"], @@ -1319,6 +1322,7 @@ { "gpu_type": "A100-80", "aliases": ["NVIDIA-A100-80GB", "A100-80", "A100-80GB"], + "node_selector_label": "NVIDIA-A100-SXM4-80GB", "memory_gb": 80, "compute_capability": "8.0", "typical_use_cases": ["inference", "training"], @@ -1332,6 +1336,7 @@ { "gpu_type": "H100", "aliases": ["NVIDIA-H100", "H100", "H100-80GB"], + "node_selector_label": "NVIDIA-H100-80GB-HBM3", "memory_gb": 80, "compute_capability": "9.0", "typical_use_cases": ["inference", "training"], @@ -1345,6 +1350,7 @@ { "gpu_type": "H200", "aliases": ["NVIDIA-H200", "H200", "H200-141GB"], + "node_selector_label": "NVIDIA-H200-141GB-HBM3", "memory_gb": 141, "compute_capability": "9.0", "typical_use_cases": ["inference", "training"], @@ -1358,6 +1364,7 @@ { "gpu_type": "B200", "aliases": ["NVIDIA-B200", "B200"], + "node_selector_label": "NVIDIA-B200", "memory_gb": 192, "compute_capability": "10.0", "typical_use_cases": ["inference", "training"], @@ -1371,6 +1378,7 @@ { "gpu_type": "MI300X", "aliases": ["AMD-MI300X", "MI300X", "AMD-Instinct-MI300X"], + "node_selector_label": "AMD-Instinct-MI300X", "memory_gb": 192, "compute_capability": "N/A", "typical_use_cases": ["inference", "training"], diff --git a/src/neuralnav/configuration/generator.py b/src/neuralnav/configuration/generator.py index c2dea93..20d5f41 100644 --- a/src/neuralnav/configuration/generator.py +++ b/src/neuralnav/configuration/generator.py @@ -21,7 +21,7 @@ class DeploymentGenerator: """Generate deployment configurations from recommendations.""" # vLLM version to use - VLLM_VERSION = "v0.6.2" + VLLM_VERSION = "latest" def __init__(self, output_dir: str | None = None, simulator_mode: bool = False): """ @@ -122,9 +122,15 @@ def _prepare_template_context( assert gpu_config is not None, "gpu_config is required for template context" - # Calculate GPU hourly rate from ModelCatalog + # Look up GPU info from ModelCatalog gpu_info = self._catalog.get_gpu_type(gpu_config.gpu_type) - gpu_hourly_rate = gpu_info.cost_per_hour_usd if gpu_info else 1.0 + if gpu_info is None: + raise ValueError( + f"Unknown GPU type '{gpu_config.gpu_type}'. " + f"Add it to the GPU catalog in data/configuration/model_catalog.json." + ) + gpu_hourly_rate = gpu_info.cost_per_hour_usd + gpu_node_selector_label = gpu_info.node_selector_label # Determine resource requests based on GPU type gpu_type = gpu_config.gpu_type @@ -187,6 +193,7 @@ def _prepare_template_context( "simulator_mode": self.simulator_mode, # GPU configuration "gpu_type": gpu_config.gpu_type, + "gpu_node_selector_label": gpu_node_selector_label, "gpu_count": gpu_config.gpu_count, "tensor_parallel": gpu_config.tensor_parallel, "gpus_per_replica": gpu_config.tensor_parallel, # GPUs per pod diff --git a/src/neuralnav/configuration/templates/kserve-inferenceservice.yaml.j2 b/src/neuralnav/configuration/templates/kserve-inferenceservice.yaml.j2 index 7907dee..363eb1e 100644 --- a/src/neuralnav/configuration/templates/kserve-inferenceservice.yaml.j2 +++ b/src/neuralnav/configuration/templates/kserve-inferenceservice.yaml.j2 @@ -78,10 +78,11 @@ spec: {% if simulator_mode %} initialDelaySeconds: 10 {% else %} - initialDelaySeconds: 120 + initialDelaySeconds: 600 {% endif %} periodSeconds: 30 timeoutSeconds: 10 + failureThreshold: 5 readinessProbe: httpGet: path: /health @@ -89,13 +90,13 @@ spec: {% if simulator_mode %} initialDelaySeconds: 5 {% else %} - initialDelaySeconds: 60 + initialDelaySeconds: 120 {% endif %} periodSeconds: 10 timeoutSeconds: 5 {% if not simulator_mode %} nodeSelector: - nvidia.com/gpu.product: {{ gpu_type }} + nvidia.com/gpu.product: {{ gpu_node_selector_label }} tolerations: - key: nvidia.com/gpu operator: Exists diff --git a/src/neuralnav/knowledge_base/model_catalog.py b/src/neuralnav/knowledge_base/model_catalog.py index 9388a80..c4a6f66 100644 --- a/src/neuralnav/knowledge_base/model_catalog.py +++ b/src/neuralnav/knowledge_base/model_catalog.py @@ -54,6 +54,7 @@ class GPUType: def __init__(self, data: dict): self.gpu_type = data["gpu_type"] self.aliases = data.get("aliases", [data["gpu_type"]]) # Default to primary name + self.node_selector_label = data.get("node_selector_label", self.aliases[0]) self.memory_gb = data["memory_gb"] self.compute_capability = data["compute_capability"] self.typical_use_cases = data["typical_use_cases"] @@ -88,6 +89,7 @@ def to_dict(self) -> dict: return { "gpu_type": self.gpu_type, "aliases": self.aliases, + "node_selector_label": self.node_selector_label, "memory_gb": self.memory_gb, "compute_capability": self.compute_capability, "typical_use_cases": self.typical_use_cases, From 520d44cdecf6da7ba2d7efe8113c65c27368a09a Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Tue, 10 Mar 2026 18:33:03 -0400 Subject: [PATCH 2/3] chore: run make format Signed-off-by: Andre Fredette --- ui/app.py | 9 ++++++- ui/components/deployment.py | 12 ++++++--- ui/components/deployment_management.py | 35 ++++++++++++++++---------- ui/components/settings.py | 5 +++- 4 files changed, 43 insertions(+), 18 deletions(-) diff --git a/ui/app.py b/ui/app.py index c1a6707..3b2ab10 100644 --- a/ui/app.py +++ b/ui/app.py @@ -513,7 +513,14 @@ def main(): # Tab-based navigation (6 tabs) tab1, tab2, tab3, tab4, tab5, tab6 = st.tabs( - ["Define Use Case", "Technical Specification", "Recommendations", "Deployment", "Deployment Management", "Configuration"] + [ + "Define Use Case", + "Technical Specification", + "Recommendations", + "Deployment", + "Deployment Management", + "Configuration", + ] ) with tab1: diff --git a/ui/components/deployment.py b/ui/components/deployment.py index 8745060..dbcdec5 100644 --- a/ui/components/deployment.py +++ b/ui/components/deployment.py @@ -185,14 +185,18 @@ def _render_deploy_to_cluster_button(selected_config: dict): use_container_width=True, type="primary", disabled=already_deployed, - help="Already deployed to cluster" if already_deployed else "Deploy to Kubernetes cluster (YAML auto-generated)", + help="Already deployed to cluster" + if already_deployed + else "Deploy to Kubernetes cluster (YAML auto-generated)", key="deploy_to_cluster_btn", ): # Check cluster accessibility when the user clicks with st.spinner("Checking cluster connectivity..."): status = check_cluster_status() if not status.get("accessible", False): - st.error("Kubernetes cluster is not accessible. Please ensure the cluster is running and try again.") + st.error( + "Kubernetes cluster is not accessible. Please ensure the cluster is running and try again." + ) return with st.spinner("Deploying to Kubernetes cluster..."): @@ -208,7 +212,9 @@ def _render_deploy_to_cluster_button(selected_config: dict): st.session_state.deployment_yaml_files = files st.session_state.deployment_yaml_generated = True - st.success(f"Successfully deployed to cluster! Deployment ID: `{result.get('deployment_id')}`") + st.success( + f"Successfully deployed to cluster! Deployment ID: `{result.get('deployment_id')}`" + ) deployment_result = result.get("deployment_result", {}) for applied_file in deployment_result.get("applied_files", []): diff --git a/ui/components/deployment_management.py b/ui/components/deployment_management.py index cf18c0d..3950fd7 100644 --- a/ui/components/deployment_management.py +++ b/ui/components/deployment_management.py @@ -46,12 +46,14 @@ def render_deployment_management_tab(): status = dep.get("status", {}) pods = dep.get("pods", []) ready = status.get("ready", False) - table_data.append({ - "Status": "Ready" if ready else "Pending", - "Name": dep["deployment_id"], - "Pods": len(pods), - "Ready": "Yes" if ready else "No", - }) + table_data.append( + { + "Status": "Ready" if ready else "Pending", + "Name": dep["deployment_id"], + "Pods": len(pods), + "Ready": "Yes" if ready else "No", + } + ) df = pd.DataFrame(table_data) st.dataframe(df, use_container_width=True, hide_index=True) @@ -257,14 +259,21 @@ def _run_inference_test(deployment_id: str, prompt: str, max_tokens: int, temper start_time = time.time() curl_cmd = [ - "curl", "-s", "-X", "POST", + "curl", + "-s", + "-X", + "POST", "http://localhost:8080/v1/completions", - "-H", "Content-Type: application/json", - "-d", json.dumps({ - "prompt": prompt, - "max_tokens": max_tokens, - "temperature": temperature, - }), + "-H", + "Content-Type: application/json", + "-d", + json.dumps( + { + "prompt": prompt, + "max_tokens": max_tokens, + "temperature": temperature, + } + ), ] with st.expander("Debug Info"): diff --git a/ui/components/settings.py b/ui/components/settings.py index 58fd592..417ac86 100644 --- a/ui/components/settings.py +++ b/ui/components/settings.py @@ -33,7 +33,10 @@ def _on_mode_change(): result = update_deployment_mode(new_mode) if result: st.session_state.deployment_mode_selection = st.session_state.deployment_mode_radio - st.session_state["_mode_msg"] = ("success", f"Deployment mode set to **{st.session_state.deployment_mode_radio}**.") + st.session_state["_mode_msg"] = ( + "success", + f"Deployment mode set to **{st.session_state.deployment_mode_radio}**.", + ) else: st.session_state["_mode_msg"] = ("error", "Failed to update deployment mode.") st.session_state["_pending_tab"] = _TAB_INDEX From a27b1d277799845dd76f72340fa01b96c6fd6e18 Mon Sep 17 00:00:00 2001 From: Andre Fredette Date: Tue, 10 Mar 2026 18:40:05 -0400 Subject: [PATCH 3/3] fix: fix lint errors Signed-off-by: Andre Fredette --- ui/components/deployment_management.py | 2 +- ui/components/settings.py | 1 - 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/ui/components/deployment_management.py b/ui/components/deployment_management.py index 3950fd7..9016f6a 100644 --- a/ui/components/deployment_management.py +++ b/ui/components/deployment_management.py @@ -9,7 +9,7 @@ import pandas as pd import streamlit as st -from api_client import delete_deployment, get_k8s_status, load_all_deployments +from api_client import delete_deployment, load_all_deployments def render_deployment_management_tab(): diff --git a/ui/components/settings.py b/ui/components/settings.py index 417ac86..f975f52 100644 --- a/ui/components/settings.py +++ b/ui/components/settings.py @@ -13,7 +13,6 @@ upload_benchmarks, ) - _TAB_INDEX = 5 # Configuration is the 6th tab (0-indexed)