native-submit-plugin/spark-operator-controller.yaml at main · salesforce/native-submit-plugin · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
apiVersion: apps/v1
kind: Deployment
metadata:
  annotations:
    deployment.kubernetes.io/revision: "2"
    meta.helm.sh/release-name: spark-operator
    meta.helm.sh/release-namespace: spark-operator
  creationTimestamp: "2025-05-02T21:41:13Z"
  generation: 2
  labels:
    app.kubernetes.io/component: controller
    app.kubernetes.io/instance: spark-operator
    app.kubernetes.io/managed-by: Helm
    app.kubernetes.io/name: spark-operator
    app.kubernetes.io/version: 2.1.1
    helm.sh/chart: spark-operator-2.1.1
  name: spark-operator-controller
  namespace: spark-operator
  resourceVersion: "2037"
  uid: d7ef3ccf-7e11-4ccf-aa43-adbbb09d4f77
spec:
  progressDeadlineSeconds: 600
  replicas: 1
  revisionHistoryLimit: 10
  selector:
    matchLabels:
      app.kubernetes.io/component: controller
      app.kubernetes.io/instance: spark-operator
      app.kubernetes.io/name: spark-operator
  strategy:
    rollingUpdate:
      maxSurge: 25%
      maxUnavailable: 25%
    type: RollingUpdate
  template:
    metadata:
      annotations:
        prometheus.io/path: /metrics
        prometheus.io/port: "8080"
        prometheus.io/scrape: "true"
      creationTimestamp: null
      labels:
        app.kubernetes.io/component: controller
        app.kubernetes.io/instance: spark-operator
        app.kubernetes.io/name: spark-operator
    spec:
      automountServiceAccountToken: true
      containers:
      - args:
        - controller
        - start
        - --zap-log-level=info
        - --namespaces=default
        - --controller-threads=10
        - --enable-ui-service=true
        - --enable-metrics=true
        - --metrics-bind-address=:8080
        - --metrics-endpoint=/metrics
        - --metrics-prefix=
        - --metrics-labels=app_type
        - --metrics-job-start-latency-buckets=30,60,90,120,150,180,210,240,270,300
        - --leader-election=true
        - --leader-election-lock-name=spark-operator-controller-lock
        - --leader-election-lock-namespace=spark-operator
        - --workqueue-ratelimiter-bucket-qps=50
        - --workqueue-ratelimiter-bucket-size=500
        - --workqueue-ratelimiter-max-delay=6h
        - --driver-pod-creation-grace-period=10s
        - --max-tracked-executor-per-app=1000
        image: ghcr.io/kubeflow/spark-operator/controller:gangav0.0.1
        imagePullPolicy: IfNotPresent
        livenessProbe:
          failureThreshold: 3
          httpGet:
            path: /healthz
            port: 8081
            scheme: HTTP
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        name: spark-operator-controller
        ports:
        - containerPort: 8080
          name: metrics
          protocol: TCP
        readinessProbe:
          failureThreshold: 3
          httpGet:
            path: /readyz
            port: 8081
            scheme: HTTP
          periodSeconds: 10
          successThreshold: 1
          timeoutSeconds: 1
        resources: {}
        securityContext:
          allowPrivilegeEscalation: false
          capabilities:
            drop:
            - ALL
          privileged: false
          readOnlyRootFilesystem: true
          runAsNonRoot: true
          seccompProfile:
            type: RuntimeDefault
        terminationMessagePath: /dev/termination-log
        terminationMessagePolicy: File
        volumeMounts:
        - mountPath: /tmp
          name: tmp
      - name: native-submit
        image: ghcr.io/kubeflow/spark-operator/native-submit:local # Replace with your image
        imagePullPolicy: IfNotPresent
        env:
        - name: NATIVE_SUBMIT_MODE
          value: "rpc"
        ports:
        - containerPort: 12345
      dnsPolicy: ClusterFirst
      restartPolicy: Always
      schedulerName: default-scheduler
      securityContext:
        fsGroup: 185
      serviceAccount: spark-operator-controller
      serviceAccountName: spark-operator-controller
      terminationGracePeriodSeconds: 30
      volumes:
      - emptyDir:
          sizeLimit: 1Gi
        name: tmp
status:
  availableReplicas: 1
  conditions:
  - lastTransitionTime: "2025-05-02T21:41:38Z"
    lastUpdateTime: "2025-05-02T21:41:38Z"
    message: Deployment has minimum availability.
    reason: MinimumReplicasAvailable
    status: "True"
    type: Available
  - lastTransitionTime: "2025-05-02T21:41:13Z"
    lastUpdateTime: "2025-05-02T21:52:41Z"
    message: ReplicaSet "spark-operator-controller-567b4b4897" has successfully progressed.
    reason: NewReplicaSetAvailable
    status: "True"
    type: Progressing
  observedGeneration: 2
  readyReplicas: 1
  replicas: 1
  updatedReplicas: 1