aicr/deployments/aicr-agent/2-job.yaml at e4ec34fa6822f3db9c6499e03104979f85cdf831 · NVIDIA/aicr · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
# Copyright (c) 2025, NVIDIA CORPORATION.  All rights reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

apiVersion: batch/v1
kind: Job
metadata:
  name: aicr
  namespace: gpu-operator
  labels:
    app.kubernetes.io/name: aicr
spec:
  completions: 1                 # total number of one-time Pods to run
  parallelism: 1                 # how many run concurrently (concurrency=1)
  completionMode: NonIndexed     # single pod execution
  backoffLimit: 0                # don't retry (adjust if you want limited retries)
  ttlSecondsAfterFinished: 3600  # auto-cleanup finished Jobs after 1h
  activeDeadlineSeconds: 18000   # hard stop runaway jobs after 5h
  template:
    metadata:
      labels:
        app.kubernetes.io/name: aicr
    spec:
      serviceAccountName: aicr
      restartPolicy: Never
      hostPID: true
      hostNetwork: true
      hostIPC: true
      # Node selection for GPU nodes
      nodeSelector:
        nodeGroup: customer-gpu
      # Tolerations to schedule on tainted GPU nodes
      tolerations:
        - key: dedicated
          operator: Equal
          value: user-workload
          effect: NoSchedule
        - key: dedicated
          operator: Equal
          value: user-workload
          effect: NoExecute
      securityContext:
        runAsUser: 0
        runAsGroup: 0
        fsGroup: 0
        fsGroupChangePolicy: "OnRootMismatch"
      containers:
        - name: aicr
          image: ghcr.io/nvidia/aicr-validator:latest
          command: ["/bin/sh", "-c"]
          env:
            - name: AICR_LOG_PREFIX
              value: agent
            - name: NODE_NAME
              valueFrom:
                fieldRef:
                  fieldPath: spec.nodeName
          args:
            - |
              set -e
              aicr --debug --log-json snapshot -o cm://gpu-operator/aicr-snapshot
          resources:
            requests:
              cpu: "1"
              memory: "4Gi"
              ephemeral-storage: "2Gi"
            limits:
              cpu: "2"
              memory: "8Gi"
              ephemeral-storage: "4Gi"
          securityContext:
            privileged: true
            runAsUser: 0
            runAsGroup: 0
            allowPrivilegeEscalation: true
            capabilities:
              add: ["SYS_ADMIN", "SYS_CHROOT"]
          volumeMounts:
            - name: run-systemd
              mountPath: /run/systemd
              readOnly: true
            - name: tmp
              mountPath: /tmp
      volumes:
        - name: run-systemd
          hostPath:
            path: /run/systemd
            type: Directory
        - name: tmp
          emptyDir: {}