diff --git a/docs/reference/api.md b/docs/reference/api.md index aef1aef4235..17822f9c675 100644 --- a/docs/reference/api.md +++ b/docs/reference/api.md @@ -436,7 +436,8 @@ _Appears in:_ | `activeDeadlineSeconds` _integer_ | ActiveDeadlineSeconds is the duration in seconds that the RayJob may be active before
KubeRay actively tries to terminate the RayJob; value must be positive integer. | | | | `backoffLimit` _integer_ | Specifies the number of retries before marking this job failed.
Each retry creates a new RayCluster. | 0 | | | `rayClusterSpec` _[RayClusterSpec](#rayclusterspec)_ | RayClusterSpec is the cluster template to run the job | | | -| `submitterPodTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | SubmitterPodTemplate is the template for the pod that will run `ray job submit`. | | | +| `submitterPodTemplate` _[PodTemplateSpec](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#podtemplatespec-v1-core)_ | SubmitterPodTemplate is the template for the pod that will run `ray job submit`.
This is used when SubmissionMode is `K8sJobMode`. | | | +| `submitterContainerTemplate` _[Container](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#container-v1-core)_ | SubmitterContainerTemplate is the template for the sidecar submitter container
injected into the Ray head Pod to run `ray job submit`
This is used when SubmissionMode is `SidecarMode`. | | | | `metadata` _object (keys:string, values:string)_ | Refer to Kubernetes API documentation for fields of `metadata`. | | | | `clusterSelector` _object (keys:string, values:string)_ | clusterSelector is used to select running rayclusters by labels | | | | `submitterConfig` _[SubmitterConfig](#submitterconfig)_ | Configurations of submitter k8s job. | | | diff --git a/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml b/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml index c2a3f252a89..030624136da 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml @@ -8536,6 +8536,726 @@ spec: format: int32 type: integer type: object + submitterContainerTemplate: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object submitterPodTemplate: properties: metadata: diff --git a/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml b/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml index f4e0260e101..371942a527f 100644 --- a/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml +++ b/helm-chart/kuberay-operator/crds/ray.io_rayjobs.yaml @@ -8540,6 +8540,726 @@ spec: format: int32 type: integer type: object + submitterContainerTemplate: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object submitterPodTemplate: properties: metadata: diff --git a/ray-operator/apis/ray/v1/rayjob_types.go b/ray-operator/apis/ray/v1/rayjob_types.go index 298da3de91d..b02abc50a32 100644 --- a/ray-operator/apis/ray/v1/rayjob_types.go +++ b/ray-operator/apis/ray/v1/rayjob_types.go @@ -218,8 +218,14 @@ type RayJobSpec struct { // RayClusterSpec is the cluster template to run the job RayClusterSpec *RayClusterSpec `json:"rayClusterSpec,omitempty"` // SubmitterPodTemplate is the template for the pod that will run `ray job submit`. + // This is used when SubmissionMode is `K8sJobMode`. // +optional SubmitterPodTemplate *corev1.PodTemplateSpec `json:"submitterPodTemplate,omitempty"` + // SubmitterContainerTemplate is the template for the sidecar submitter container + // injected into the Ray head Pod to run `ray job submit` + // This is used when SubmissionMode is `SidecarMode`. + // +optional + SubmitterContainerTemplate *corev1.Container `json:"submitterContainerTemplate,omitempty"` // Metadata is data to store along with this job. // +optional Metadata map[string]string `json:"metadata,omitempty"` diff --git a/ray-operator/apis/ray/v1/zz_generated.deepcopy.go b/ray-operator/apis/ray/v1/zz_generated.deepcopy.go index 2adc56438f2..a46c79fd5d5 100644 --- a/ray-operator/apis/ray/v1/zz_generated.deepcopy.go +++ b/ray-operator/apis/ray/v1/zz_generated.deepcopy.go @@ -693,6 +693,11 @@ func (in *RayJobSpec) DeepCopyInto(out *RayJobSpec) { *out = new(corev1.PodTemplateSpec) (*in).DeepCopyInto(*out) } + if in.SubmitterContainerTemplate != nil { + in, out := &in.SubmitterContainerTemplate, &out.SubmitterContainerTemplate + *out = new(corev1.Container) + (*in).DeepCopyInto(*out) + } if in.Metadata != nil { in, out := &in.Metadata, &out.Metadata *out = make(map[string]string, len(*in)) diff --git a/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml b/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml index c2a3f252a89..030624136da 100644 --- a/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml +++ b/ray-operator/config/crd/bases/ray.io_raycronjobs.yaml @@ -8536,6 +8536,726 @@ spec: format: int32 type: integer type: object + submitterContainerTemplate: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object submitterPodTemplate: properties: metadata: diff --git a/ray-operator/config/crd/bases/ray.io_rayjobs.yaml b/ray-operator/config/crd/bases/ray.io_rayjobs.yaml index f4e0260e101..371942a527f 100644 --- a/ray-operator/config/crd/bases/ray.io_rayjobs.yaml +++ b/ray-operator/config/crd/bases/ray.io_rayjobs.yaml @@ -8540,6 +8540,726 @@ spec: format: int32 type: integer type: object + submitterContainerTemplate: + properties: + args: + items: + type: string + type: array + x-kubernetes-list-type: atomic + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + env: + items: + properties: + name: + type: string + value: + type: string + valueFrom: + properties: + configMapKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + fieldRef: + properties: + apiVersion: + type: string + fieldPath: + type: string + required: + - fieldPath + type: object + x-kubernetes-map-type: atomic + fileKeyRef: + properties: + key: + type: string + optional: + default: false + type: boolean + path: + type: string + volumeName: + type: string + required: + - key + - path + - volumeName + type: object + x-kubernetes-map-type: atomic + resourceFieldRef: + properties: + containerName: + type: string + divisor: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + resource: + type: string + required: + - resource + type: object + x-kubernetes-map-type: atomic + secretKeyRef: + properties: + key: + type: string + name: + default: "" + type: string + optional: + type: boolean + required: + - key + type: object + x-kubernetes-map-type: atomic + type: object + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + envFrom: + items: + properties: + configMapRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + prefix: + type: string + secretRef: + properties: + name: + default: "" + type: string + optional: + type: boolean + type: object + x-kubernetes-map-type: atomic + type: object + type: array + x-kubernetes-list-type: atomic + image: + type: string + imagePullPolicy: + type: string + lifecycle: + properties: + postStart: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + preStop: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + sleep: + properties: + seconds: + format: int64 + type: integer + required: + - seconds + type: object + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + type: object + stopSignal: + type: string + type: object + livenessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + name: + type: string + ports: + items: + properties: + containerPort: + format: int32 + type: integer + hostIP: + type: string + hostPort: + format: int32 + type: integer + name: + type: string + protocol: + default: TCP + type: string + required: + - containerPort + type: object + type: array + x-kubernetes-list-map-keys: + - containerPort + - protocol + x-kubernetes-list-type: map + readinessProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + resizePolicy: + items: + properties: + resourceName: + type: string + restartPolicy: + type: string + required: + - resourceName + - restartPolicy + type: object + type: array + x-kubernetes-list-type: atomic + resources: + properties: + claims: + items: + properties: + name: + type: string + request: + type: string + required: + - name + type: object + type: array + x-kubernetes-list-map-keys: + - name + x-kubernetes-list-type: map + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: object + type: object + restartPolicy: + type: string + restartPolicyRules: + items: + properties: + action: + type: string + exitCodes: + properties: + operator: + type: string + values: + items: + format: int32 + type: integer + type: array + x-kubernetes-list-type: set + required: + - operator + type: object + required: + - action + type: object + type: array + x-kubernetes-list-type: atomic + securityContext: + properties: + allowPrivilegeEscalation: + type: boolean + appArmorProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + capabilities: + properties: + add: + items: + type: string + type: array + x-kubernetes-list-type: atomic + drop: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + privileged: + type: boolean + procMount: + type: string + readOnlyRootFilesystem: + type: boolean + runAsGroup: + format: int64 + type: integer + runAsNonRoot: + type: boolean + runAsUser: + format: int64 + type: integer + seLinuxOptions: + properties: + level: + type: string + role: + type: string + type: + type: string + user: + type: string + type: object + seccompProfile: + properties: + localhostProfile: + type: string + type: + type: string + required: + - type + type: object + windowsOptions: + properties: + gmsaCredentialSpec: + type: string + gmsaCredentialSpecName: + type: string + hostProcess: + type: boolean + runAsUserName: + type: string + type: object + type: object + startupProbe: + properties: + exec: + properties: + command: + items: + type: string + type: array + x-kubernetes-list-type: atomic + type: object + failureThreshold: + format: int32 + type: integer + grpc: + properties: + port: + format: int32 + type: integer + service: + default: "" + type: string + required: + - port + type: object + httpGet: + properties: + host: + type: string + httpHeaders: + items: + properties: + name: + type: string + value: + type: string + required: + - name + - value + type: object + type: array + x-kubernetes-list-type: atomic + path: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + scheme: + type: string + required: + - port + type: object + initialDelaySeconds: + format: int32 + type: integer + periodSeconds: + format: int32 + type: integer + successThreshold: + format: int32 + type: integer + tcpSocket: + properties: + host: + type: string + port: + anyOf: + - type: integer + - type: string + x-kubernetes-int-or-string: true + required: + - port + type: object + terminationGracePeriodSeconds: + format: int64 + type: integer + timeoutSeconds: + format: int32 + type: integer + type: object + stdin: + type: boolean + stdinOnce: + type: boolean + terminationMessagePath: + type: string + terminationMessagePolicy: + type: string + tty: + type: boolean + volumeDevices: + items: + properties: + devicePath: + type: string + name: + type: string + required: + - devicePath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - devicePath + x-kubernetes-list-type: map + volumeMounts: + items: + properties: + mountPath: + type: string + mountPropagation: + type: string + name: + type: string + readOnly: + type: boolean + recursiveReadOnly: + type: string + subPath: + type: string + subPathExpr: + type: string + required: + - mountPath + - name + type: object + type: array + x-kubernetes-list-map-keys: + - mountPath + x-kubernetes-list-type: map + workingDir: + type: string + required: + - name + type: object submitterPodTemplate: properties: metadata: diff --git a/ray-operator/controllers/ray/rayjob_controller.go b/ray-operator/controllers/ray/rayjob_controller.go index b5492f93115..cac1bc176b3 100644 --- a/ray-operator/controllers/ray/rayjob_controller.go +++ b/ray-operator/controllers/ray/rayjob_controller.go @@ -579,8 +579,20 @@ func getSubmitterTemplate(rayJobInstance *rayv1.RayJob, rayClusterInstance *rayv } // getSubmitterContainer builds the submitter container for the Ray job Sidecar mode. +// If the user provides a SubmitterContainerTemplate, it is used as the base container. +// Otherwise, a default container is created using GetDefaultSubmitterContainer. func getSubmitterContainer(rayJobInstance *rayv1.RayJob, rayClusterInstance *rayv1.RayCluster) (corev1.Container, error) { - submitterContainer := common.GetDefaultSubmitterContainer(&rayClusterInstance.Spec) + if rayClusterInstance == nil { + return corev1.Container{}, fmt.Errorf("rayClusterInstance must not be nil for SidecarMode") + } + + var submitterContainer corev1.Container + + if rayJobInstance.Spec.SubmitterContainerTemplate != nil { + submitterContainer = *rayJobInstance.Spec.SubmitterContainerTemplate.DeepCopy() + } else { + submitterContainer = common.GetDefaultSubmitterContainer(&rayClusterInstance.Spec) + } if err := configureSubmitterContainer(&submitterContainer, rayJobInstance, rayClusterInstance, rayv1.SidecarMode); err != nil { return corev1.Container{}, err @@ -591,6 +603,20 @@ func getSubmitterContainer(rayJobInstance *rayv1.RayJob, rayClusterInstance *ray // pass the RayCluster instance for cluster selector case func configureSubmitterContainer(container *corev1.Container, rayJobInstance *rayv1.RayJob, rayClusterInstance *rayv1.RayCluster, submissionMode rayv1.JobSubmissionMode) error { + // Always set the container name to the expected submitter container name + container.Name = utils.SubmitterContainerName + + // If the image is not specified, use the image of the Ray head container + if container.Image == "" && rayClusterInstance != nil { + container.Image = rayClusterInstance.Spec.HeadGroupSpec.Template.Spec.Containers[utils.RayContainerIndex].Image + } + + // If resources are not specified, use default resources + if container.Resources.Limits == nil && container.Resources.Requests == nil && rayClusterInstance != nil { + defaultContainer := common.GetDefaultSubmitterContainer(&rayClusterInstance.Spec) + container.Resources = defaultContainer.Resources + } + // If the command in the submitter container manifest isn't set, use the default command. jobCmd, err := common.BuildJobSubmitCommand(rayJobInstance, submissionMode) if err != nil { @@ -598,7 +624,7 @@ func configureSubmitterContainer(container *corev1.Container, rayJobInstance *ra } // K8sJobMode: If the user doesn't specify the command, use the default command. - // SidecarMode: Use the default command. + // SidecarMode: Always use the default command (command is overwritten). if len(container.Command) == 0 || submissionMode == rayv1.SidecarMode { // Without the -e option, the Bash script will continue executing even if a command returns a non-zero exit code. container.Command = utils.GetContainerCommand([]string{"e"}) diff --git a/ray-operator/controllers/ray/rayjob_controller_unit_test.go b/ray-operator/controllers/ray/rayjob_controller_unit_test.go index eb37b1a729a..f473b5482dc 100644 --- a/ray-operator/controllers/ray/rayjob_controller_unit_test.go +++ b/ray-operator/controllers/ray/rayjob_controller_unit_test.go @@ -14,6 +14,7 @@ import ( "go.uber.org/mock/gomock" batchv1 "k8s.io/api/batch/v1" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" @@ -201,6 +202,299 @@ func TestGetSubmitterTemplate(t *testing.T) { assert.Equal(t, "test-job-id", envVar.Value) } +func TestGetSubmitterContainer(t *testing.T) { + // Helper to create a basic RayClusterSpec with a head container + basicRayClusterSpec := func() *rayv1.RayClusterSpec { + return &rayv1.RayClusterSpec{ + HeadGroupSpec: rayv1.HeadGroupSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Image: "rayproject/ray:custom-version", + }, + }, + }, + }, + }, + } + } + + rayClusterInstance := &rayv1.RayCluster{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-raycluster", + Namespace: "default", + }, + Spec: *basicRayClusterSpec(), + } + + t.Run("Default container without SubmitterContainerTemplate", func(t *testing.T) { + rayJobInstance := &rayv1.RayJob{ + Spec: rayv1.RayJobSpec{ + Entrypoint: "python script.py", + SubmitterContainerTemplate: nil, + RayClusterSpec: basicRayClusterSpec(), + }, + Status: rayv1.RayJobStatus{ + DashboardURL: "test-url", + JobId: "test-job-id", + }, + } + + container, err := getSubmitterContainer(rayJobInstance, rayClusterInstance) + require.NoError(t, err) + + // Should use default values + assert.Equal(t, utils.SubmitterContainerName, container.Name) + assert.Equal(t, "rayproject/ray:custom-version", container.Image) + assert.NotEmpty(t, container.Resources.Limits) + assert.NotEmpty(t, container.Resources.Requests) + }) + + t.Run("Custom container with SubmitterContainerTemplate", func(t *testing.T) { + rayJobInstance := &rayv1.RayJob{ + Spec: rayv1.RayJobSpec{ + Entrypoint: "python script.py", + SubmitterContainerTemplate: &corev1.Container{ + Image: "custom/image:v1", + VolumeMounts: []corev1.VolumeMount{ + { + Name: "scripts", + MountPath: "/scripts", + }, + }, + Env: []corev1.EnvVar{ + { + Name: "MY_VAR", + Value: "my_value", + }, + }, + }, + RayClusterSpec: basicRayClusterSpec(), + }, + Status: rayv1.RayJobStatus{ + DashboardURL: "test-url", + JobId: "test-job-id", + }, + } + + container, err := getSubmitterContainer(rayJobInstance, rayClusterInstance) + require.NoError(t, err) + + // Container name should be overwritten + assert.Equal(t, utils.SubmitterContainerName, container.Name) + // Custom image should be preserved + assert.Equal(t, "custom/image:v1", container.Image) + // Volume mounts should be preserved + assert.Len(t, container.VolumeMounts, 1) + assert.Equal(t, "scripts", container.VolumeMounts[0].Name) + assert.Equal(t, "/scripts", container.VolumeMounts[0].MountPath) + // User env vars should be present (before controller env vars) + envVar, found := utils.EnvVarByName("MY_VAR", container.Env) + assert.True(t, found) + assert.Equal(t, "my_value", envVar.Value) + // Controller env vars should also be present + envVar, found = utils.EnvVarByName(utils.RAY_DASHBOARD_ADDRESS, container.Env) + assert.True(t, found) + assert.Equal(t, "test-url", envVar.Value) + }) + + t.Run("Image inheritance when not specified in template", func(t *testing.T) { + rayJobInstance := &rayv1.RayJob{ + Spec: rayv1.RayJobSpec{ + Entrypoint: "python script.py", + SubmitterContainerTemplate: &corev1.Container{ + // Image not specified - should inherit from head container + VolumeMounts: []corev1.VolumeMount{ + { + Name: "data", + MountPath: "/data", + }, + }, + }, + RayClusterSpec: basicRayClusterSpec(), + }, + Status: rayv1.RayJobStatus{ + DashboardURL: "test-url", + JobId: "test-job-id", + }, + } + + container, err := getSubmitterContainer(rayJobInstance, rayClusterInstance) + require.NoError(t, err) + + // Should inherit image from head container + assert.Equal(t, "rayproject/ray:custom-version", container.Image) + // Volume mounts should still be preserved + assert.Len(t, container.VolumeMounts, 1) + assert.Equal(t, "data", container.VolumeMounts[0].Name) + }) + + t.Run("Container name is always overwritten", func(t *testing.T) { + rayJobInstance := &rayv1.RayJob{ + Spec: rayv1.RayJobSpec{ + Entrypoint: "python script.py", + SubmitterContainerTemplate: &corev1.Container{ + Name: "user-specified-name", // Should be overwritten + Image: "custom/image:v1", + }, + RayClusterSpec: basicRayClusterSpec(), + }, + Status: rayv1.RayJobStatus{ + DashboardURL: "test-url", + JobId: "test-job-id", + }, + } + + container, err := getSubmitterContainer(rayJobInstance, rayClusterInstance) + require.NoError(t, err) + + // Name should be overwritten to the standard submitter container name + assert.Equal(t, utils.SubmitterContainerName, container.Name) + }) + + t.Run("Command and Args are always overwritten in SidecarMode", func(t *testing.T) { + rayJobInstance := &rayv1.RayJob{ + Spec: rayv1.RayJobSpec{ + Entrypoint: "python script.py", + SubmitterContainerTemplate: &corev1.Container{ + Image: "custom/image:v1", + Command: []string{"user-command"}, // Should be overwritten + Args: []string{"user-args"}, // Should be overwritten + }, + RayClusterSpec: basicRayClusterSpec(), + }, + Status: rayv1.RayJobStatus{ + DashboardURL: "test-url", + JobId: "test-job-id", + }, + } + + container, err := getSubmitterContainer(rayJobInstance, rayClusterInstance) + require.NoError(t, err) + + // Command should be overwritten to the controller's command + assert.Equal(t, []string{"/bin/bash", "-ce", "--"}, container.Command) + // Args should contain the job submission command + assert.Len(t, container.Args, 1) + assert.Contains(t, container.Args[0], "ray job submit") + }) + + t.Run("Default resources when not specified in template", func(t *testing.T) { + rayJobInstance := &rayv1.RayJob{ + Spec: rayv1.RayJobSpec{ + Entrypoint: "python script.py", + SubmitterContainerTemplate: &corev1.Container{ + Image: "custom/image:v1", + // Resources not specified - should use defaults + }, + RayClusterSpec: basicRayClusterSpec(), + }, + Status: rayv1.RayJobStatus{ + DashboardURL: "test-url", + JobId: "test-job-id", + }, + } + + container, err := getSubmitterContainer(rayJobInstance, rayClusterInstance) + require.NoError(t, err) + + // Should have default resources + assert.NotEmpty(t, container.Resources.Limits) + assert.NotEmpty(t, container.Resources.Requests) + }) + + t.Run("Custom resources are preserved", func(t *testing.T) { + rayJobInstance := &rayv1.RayJob{ + Spec: rayv1.RayJobSpec{ + Entrypoint: "python script.py", + SubmitterContainerTemplate: &corev1.Container{ + Image: "custom/image:v1", + Resources: corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceCPU: resource.MustParse("2"), + corev1.ResourceMemory: resource.MustParse("2Gi"), + }, + }, + }, + RayClusterSpec: basicRayClusterSpec(), + }, + Status: rayv1.RayJobStatus{ + DashboardURL: "test-url", + JobId: "test-job-id", + }, + } + + container, err := getSubmitterContainer(rayJobInstance, rayClusterInstance) + require.NoError(t, err) + + // Custom resources should be preserved + assert.NotEmpty(t, container.Resources.Limits) + assert.Equal(t, resource.MustParse("2"), container.Resources.Limits[corev1.ResourceCPU]) + }) + + t.Run("Security context is preserved", func(t *testing.T) { + runAsUser := int64(1000) + rayJobInstance := &rayv1.RayJob{ + Spec: rayv1.RayJobSpec{ + Entrypoint: "python script.py", + SubmitterContainerTemplate: &corev1.Container{ + Image: "custom/image:v1", + SecurityContext: &corev1.SecurityContext{ + RunAsUser: &runAsUser, + RunAsNonRoot: pointer.Bool(true), + }, + }, + RayClusterSpec: basicRayClusterSpec(), + }, + Status: rayv1.RayJobStatus{ + DashboardURL: "test-url", + JobId: "test-job-id", + }, + } + + container, err := getSubmitterContainer(rayJobInstance, rayClusterInstance) + require.NoError(t, err) + + // Security context should be preserved + assert.NotNil(t, container.SecurityContext) + assert.Equal(t, int64(1000), *container.SecurityContext.RunAsUser) + assert.True(t, *container.SecurityContext.RunAsNonRoot) + }) + + t.Run("EnvFrom is preserved", func(t *testing.T) { + rayJobInstance := &rayv1.RayJob{ + Spec: rayv1.RayJobSpec{ + Entrypoint: "python script.py", + SubmitterContainerTemplate: &corev1.Container{ + Image: "custom/image:v1", + EnvFrom: []corev1.EnvFromSource{ + { + ConfigMapRef: &corev1.ConfigMapEnvSource{ + LocalObjectReference: corev1.LocalObjectReference{ + Name: "my-config", + }, + }, + }, + }, + }, + RayClusterSpec: basicRayClusterSpec(), + }, + Status: rayv1.RayJobStatus{ + DashboardURL: "test-url", + JobId: "test-job-id", + }, + } + + container, err := getSubmitterContainer(rayJobInstance, rayClusterInstance) + require.NoError(t, err) + + // EnvFrom should be preserved + assert.Len(t, container.EnvFrom, 1) + assert.Equal(t, "my-config", container.EnvFrom[0].ConfigMapRef.Name) + }) +} + func TestUpdateStatusToSuspendingIfNeeded(t *testing.T) { newScheme := runtime.NewScheme() _ = rayv1.AddToScheme(newScheme) diff --git a/ray-operator/controllers/ray/utils/validation.go b/ray-operator/controllers/ray/utils/validation.go index 50b795d104a..7dcf3f6507b 100644 --- a/ray-operator/controllers/ray/utils/validation.go +++ b/ray-operator/controllers/ray/utils/validation.go @@ -304,7 +304,7 @@ func ValidateRayJobSpec(rayJob *rayv1.RayJob) error { if rayJob.Spec.SubmissionMode == rayv1.SidecarMode { if rayJob.Spec.SubmitterPodTemplate != nil { - return fmt.Errorf("Currently, SidecarMode doesn't support SubmitterPodTemplate") + return fmt.Errorf("SubmitterPodTemplate is not supported in SidecarMode; use SubmitterContainerTemplate instead") } if rayJob.Spec.SubmitterConfig != nil { @@ -316,6 +316,11 @@ func ValidateRayJobSpec(rayJob *rayv1.RayJob) error { } } + // SubmitterContainerTemplate is only supported in SidecarMode + if rayJob.Spec.SubmissionMode != rayv1.SidecarMode && rayJob.Spec.SubmitterContainerTemplate != nil { + return fmt.Errorf("SubmitterContainerTemplate is only supported in SidecarMode; use SubmitterPodTemplate instead") + } + if rayJob.Spec.RayClusterSpec != nil { if err := ValidateRayClusterSpec(rayJob.Spec.RayClusterSpec, rayJob.Annotations); err != nil { return fmt.Errorf("The RayJob spec is invalid: %w", err) diff --git a/ray-operator/controllers/ray/utils/validation_test.go b/ray-operator/controllers/ray/utils/validation_test.go index 36cadc9b260..a1f33f884cc 100644 --- a/ray-operator/controllers/ray/utils/validation_test.go +++ b/ray-operator/controllers/ray/utils/validation_test.go @@ -1106,6 +1106,42 @@ func TestValidateRayJobSpec(t *testing.T) { }, expectError: true, }, + { + name: "SidecarMode supports SubmitterContainerTemplate", + spec: rayv1.RayJobSpec{ + SubmissionMode: rayv1.SidecarMode, + SubmitterContainerTemplate: createBasicSubmitterContainerTemplate(), + RayClusterSpec: createBasicRayClusterSpec(), + }, + expectError: false, + }, + { + name: "K8sJobMode doesn't support SubmitterContainerTemplate", + spec: rayv1.RayJobSpec{ + SubmissionMode: rayv1.K8sJobMode, + SubmitterContainerTemplate: createBasicSubmitterContainerTemplate(), + RayClusterSpec: createBasicRayClusterSpec(), + }, + expectError: true, + }, + { + name: "HTTPMode doesn't support SubmitterContainerTemplate", + spec: rayv1.RayJobSpec{ + SubmissionMode: rayv1.HTTPMode, + SubmitterContainerTemplate: createBasicSubmitterContainerTemplate(), + RayClusterSpec: createBasicRayClusterSpec(), + }, + expectError: true, + }, + { + name: "InteractiveMode doesn't support SubmitterContainerTemplate", + spec: rayv1.RayJobSpec{ + SubmissionMode: rayv1.InteractiveMode, + SubmitterContainerTemplate: createBasicSubmitterContainerTemplate(), + RayClusterSpec: createBasicRayClusterSpec(), + }, + expectError: true, + }, { name: "SidecarMode doesn't support SubmitterConfig", spec: rayv1.RayJobSpec{ @@ -1910,6 +1946,12 @@ func createBasicRayClusterSpec() *rayv1.RayClusterSpec { } } +func createBasicSubmitterContainerTemplate() *corev1.Container { + return &corev1.Container{ + Image: "rayproject/ray:2.52.0", + } +} + func TestValidateClusterUpgradeOptions(t *testing.T) { tests := []struct { maxSurgePercent *int32 diff --git a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobspec.go b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobspec.go index 60d69094112..b1d5b048cab 100644 --- a/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobspec.go +++ b/ray-operator/pkg/client/applyconfiguration/ray/v1/rayjobspec.go @@ -4,6 +4,7 @@ package v1 import ( rayv1 "github.com/ray-project/kuberay/ray-operator/apis/ray/v1" + apicorev1 "k8s.io/api/core/v1" corev1 "k8s.io/client-go/applyconfigurations/core/v1" ) @@ -21,7 +22,12 @@ type RayJobSpecApplyConfiguration struct { // RayClusterSpec is the cluster template to run the job RayClusterSpec *RayClusterSpecApplyConfiguration `json:"rayClusterSpec,omitempty"` // SubmitterPodTemplate is the template for the pod that will run `ray job submit`. + // This is used when SubmissionMode is `K8sJobMode`. SubmitterPodTemplate *corev1.PodTemplateSpecApplyConfiguration `json:"submitterPodTemplate,omitempty"` + // SubmitterContainerTemplate is the template for the sidecar submitter container + // injected into the Ray head Pod to run `ray job submit` + // This is used when SubmissionMode is `SidecarMode`. + SubmitterContainerTemplate *apicorev1.Container `json:"submitterContainerTemplate,omitempty"` // Metadata is data to store along with this job. Metadata map[string]string `json:"metadata,omitempty"` // clusterSelector is used to select running rayclusters by labels @@ -113,6 +119,14 @@ func (b *RayJobSpecApplyConfiguration) WithSubmitterPodTemplate(value *corev1.Po return b } +// WithSubmitterContainerTemplate sets the SubmitterContainerTemplate field in the declarative configuration to the given value +// and returns the receiver, so that objects can be built by chaining "With" function invocations. +// If called multiple times, the SubmitterContainerTemplate field is set to the value of the last call. +func (b *RayJobSpecApplyConfiguration) WithSubmitterContainerTemplate(value apicorev1.Container) *RayJobSpecApplyConfiguration { + b.SubmitterContainerTemplate = &value + return b +} + // WithMetadata puts the entries into the Metadata field in the declarative configuration // and returns the receiver, so that objects can be build by chaining "With" function invocations. // If called multiple times, the entries provided by each call will be put on the Metadata field, diff --git a/ray-operator/test/e2erayjob/rayjob_sidecar_mode_test.go b/ray-operator/test/e2erayjob/rayjob_sidecar_mode_test.go index 62f6a3c7d33..d3da86b429a 100644 --- a/ray-operator/test/e2erayjob/rayjob_sidecar_mode_test.go +++ b/ray-operator/test/e2erayjob/rayjob_sidecar_mode_test.go @@ -290,4 +290,107 @@ env_vars: LogWithTimestamp(test.T(), "RayJob %s/%s completed successfully with auth token", rayJob.Namespace, rayJob.Name) }) + + test.T().Run("SidecarMode with SubmitterContainerTemplate", func(_ *testing.T) { + // This test verifies that SubmitterContainerTemplate allows users to customize + // the sidecar submitter container with volume mounts and environment variables. + rayJobAC := rayv1ac.RayJob("sidecar-custom-submitter", namespace.Name). + WithSpec(rayv1ac.RayJobSpec(). + WithSubmissionMode(rayv1.SidecarMode). + WithEntrypoint("python /home/ray/jobs/counter.py"). + WithRuntimeEnvYAML(` +env_vars: + counter_name: test_counter +`). + WithShutdownAfterJobFinishes(true). + // Use SubmitterContainerTemplate to customize the submitter container + WithSubmitterContainerTemplate(corev1.Container{ + // Volume mounts for the submitter container + VolumeMounts: []corev1.VolumeMount{ + { + Name: jobs.Name, + MountPath: "/home/ray/jobs", + }, + }, + // Custom environment variable + Env: []corev1.EnvVar{ + { + Name: "CUSTOM_SUBMITTER_VAR", + Value: "submitter_template_test", + }, + }, + }). + // Match the working "Successful RayJob" test pattern + WithRayClusterSpec(rayv1ac.RayClusterSpec(). + WithRayVersion(GetRayVersion()). + WithHeadGroupSpec(rayv1ac.HeadGroupSpec(). + WithRayStartParams(map[string]string{"dashboard-host": "0.0.0.0"}). + WithTemplate(PodTemplateSpecApplyConfiguration(HeadPodTemplateApplyConfiguration(), + MountConfigMap[corev1ac.PodTemplateSpecApplyConfiguration](jobs, "/home/ray/jobs")))))) + + rayJob, err := test.Client().Ray().RayV1().RayJobs(namespace.Name).Apply(test.Ctx(), rayJobAC, TestApplyOptions) + g.Expect(err).NotTo(HaveOccurred()) + LogWithTimestamp(test.T(), "Created RayJob %s/%s with SubmitterContainerTemplate", rayJob.Namespace, rayJob.Name) + + // Wait for RayCluster to be created and ready + LogWithTimestamp(test.T(), "Waiting for RayCluster to be created") + g.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutMedium). + Should(WithTransform(RayJobClusterName, Not(BeEmpty()))) + + rayJob, err = GetRayJob(test, rayJob.Namespace, rayJob.Name) + g.Expect(err).NotTo(HaveOccurred()) + + // Verify the submitter container has the expected configuration + rayCluster, err := GetRayCluster(test, namespace.Name, rayJob.Status.RayClusterName) + g.Expect(err).NotTo(HaveOccurred()) + + headPod, err := GetHeadPod(test, rayCluster) + g.Expect(err).NotTo(HaveOccurred()) + g.Expect(headPod).NotTo(BeNil()) + + // Find the submitter container + var submitterContainer *corev1.Container + for i := range headPod.Spec.Containers { + if headPod.Spec.Containers[i].Name == utils.SubmitterContainerName { + submitterContainer = &headPod.Spec.Containers[i] + break + } + } + g.Expect(submitterContainer).NotTo(BeNil(), "submitter container should be present in head pod") + + // Verify the volume mount is present + hasVolumeMount := false + for _, vm := range submitterContainer.VolumeMounts { + if vm.Name == jobs.Name && vm.MountPath == "/home/ray/jobs" { + hasVolumeMount = true + break + } + } + g.Expect(hasVolumeMount).To(BeTrue(), "submitter container should have the ConfigMap volume mount") + + // Verify the custom environment variable is present + hasCustomEnv := false + for _, env := range submitterContainer.Env { + if env.Name == "CUSTOM_SUBMITTER_VAR" && env.Value == "submitter_template_test" { + hasCustomEnv = true + break + } + } + g.Expect(hasCustomEnv).To(BeTrue(), "submitter container should have the custom environment variable") + + // Wait for the job to complete + LogWithTimestamp(test.T(), "Waiting for RayJob %s/%s to complete", rayJob.Namespace, rayJob.Name) + g.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name), TestTimeoutMedium). + Should(WithTransform(RayJobStatus, Satisfy(rayv1.IsJobTerminal))) + + // Assert the RayJob has completed successfully + g.Expect(GetRayJob(test, rayJob.Namespace, rayJob.Name)). + To(WithTransform(RayJobStatus, Equal(rayv1.JobStatusSucceeded))) + + // And the RayJob deployment status is updated accordingly + g.Eventually(RayJob(test, rayJob.Namespace, rayJob.Name)). + Should(WithTransform(RayJobDeploymentStatus, Equal(rayv1.JobDeploymentStatusComplete))) + + LogWithTimestamp(test.T(), "RayJob %s/%s with SubmitterContainerTemplate completed successfully", rayJob.Namespace, rayJob.Name) + }) }