hub/jobs/async-upload/samples/sample_job_s3_to_oci.yaml at cf2e01b9be2fd74774bf6b3ee268c652ac20ecb9 · kubeflow/hub · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
---
apiVersion: v1
kind: Secret
metadata:
  name: my-s3-credentials
  namespace: default
stringData:
  AWS_ACCESS_KEY_ID: minioadmin
  AWS_SECRET_ACCESS_KEY: minioadmin
  AWS_DEFAULT_REGION: us-east-1
  AWS_S3_ENDPOINT: http://minio.minio.svc.cluster.local:9000
  AWS_S3_BUCKET: default
---
apiVersion: v1
kind: Secret
metadata:
  name: my-oci-credentials
  namespace: default
type: kubernetes.io/dockerconfigjson
stringData:
  .dockerconfigjson: '{"auths": {"distribution-registry-test-service.local-oci-registry-ns.svc.cluster.local:5001": {"auth": "","email": "user@example.com"}}}'
  ACCESS_TYPE: '["Push,Pull"]'
  OCI_HOST: distribution-registry-test-service.local-oci-registry-ns.svc.cluster.local:5001
---
apiVersion: batch/v1
kind: Job
metadata:
  name: my-async-upload-job
  namespace: default
  labels:
    app.kubernetes.io/name: model-registry-async-job
    app.kubernetes.io/component: async-job
    app.kubernetes.io/part-of: model-registry
    app.kubernetes.io/managed-by: kubectl
    component: model-registry-job
    modelregistry.kubeflow.org/job-type: async-upload
    modelregistry.kubeflow.org/model-sync-model-id: "1"
    modelregistry.kubeflow.org/model-sync-model-version-id: "3"
    modelregistry.kubeflow.org/model-sync-model-artifact-id: "6"
  annotations:
    modelregistry.kubeflow.org/description: "Asynchronous job for uploading models to Model Registry"
spec:
  template:
    metadata:
      labels:
        app.kubernetes.io/name: model-registry-async-job
        app.kubernetes.io/component: async-job
        component: model-registry-job
        modelregistry.kubeflow.org/job-type: async-upload
        modelregistry.kubeflow.org/model-sync-model-id: "1"
        modelregistry.kubeflow.org/model-sync-model-version-id: "3"
        modelregistry.kubeflow.org/model-sync-model-artifact-id: "6"
    spec:
      securityContext:
        runAsNonRoot: true
      volumes:
        - name: source-credentials
          secret:
            secretName: my-s3-credentials
        - name: destination-credentials
          secret:
            secretName: my-oci-credentials
            items:
              - key: .dockerconfigjson
                path: .dockerconfigjson
      restartPolicy: Never
      containers:
        - name: async-upload
          image: ghcr.io/kubeflow/hub/job/async-upload:latest
          securityContext:
            allowPrivilegeEscalation: false
            readOnlyRootFilesystem: false
            capabilities:
              drop: ["ALL"]
            seccompProfile:
              type: RuntimeDefault
          resources:
            requests:
              cpu: "100m"
              memory: "128Mi"
            limits:
              memory: "512Mi"
          volumeMounts:
            - name: source-credentials
              readOnly: true
              mountPath: "/opt/creds/source"
            - name: destination-credentials
              readOnly: true
              mountPath: "/opt/creds/destination"
          env:
            - name: HTTP_PROXY
              value: ""
            - name: HTTPS_PROXY
              value: ""
            - name: NO_PROXY
              value: "minio.minio.svc.cluster.local,*.svc.cluster.local"
            # ---- Source ----------
            - name: MODEL_SYNC_SOURCE_TYPE
              value: "s3"
            - name: MODEL_SYNC_SOURCE_AWS_KEY
              value: "my-model"
            - name: MODEL_SYNC_SOURCE_S3_CREDENTIALS_PATH
              value: "/opt/creds/source"

            # ---- Destination -----
            - name: MODEL_SYNC_DESTINATION_TYPE
              value: "oci"
            - name: MODEL_SYNC_DESTINATION_OCI_URI
              value: "distribution-registry-test-service.local-oci-registry-ns.svc.cluster.local:5001/my-model:latest"
            - name: MODEL_SYNC_DESTINATION_OCI_REGISTRY
              value: "distribution-registry-test-service.local-oci-registry-ns.svc.cluster.local:5001"
            - name: MODEL_SYNC_DESTINATION_OCI_CREDENTIALS_PATH
              value: "/opt/creds/destination/.dockerconfigjson"
            - name: MODEL_SYNC_DESTINATION_OCI_BASE_IMAGE
              value: "public.ecr.aws/docker/library/busybox:latest"
            # Uncomment to disable TLS verification when pulling the base image (e.g. disconnected clusters with self-signed certs)
            # - name: MODEL_SYNC_DESTINATION_OCI_BASE_IMAGE_TLS_VERIFY
            #   value: "false"
            # Uncomment to provide auth for pulling the base image from a private/mirrored registry
            # - name: MODEL_SYNC_DESTINATION_OCI_BASE_IMAGE_CREDENTIALS_PATH
            #   value: "/opt/creds/base-image/.dockerconfigjson"
            - name: MODEL_SYNC_DESTINATION_OCI_ENABLE_TLS_VERIFY
              value: "false"

            # ---- Model Params ----
            - name: MODEL_SYNC_MODEL_UPLOAD_INTENT
              value: "update_artifact"
            - name: MODEL_SYNC_MODEL_ARTIFACT_ID
              value: "6"

            # ---- Model Registry Client Params ----
            - name: MODEL_SYNC_REGISTRY_SERVER_ADDRESS
              value: "http://model-registry-service.kubeflow.svc.cluster.local"
            - name: MODEL_SYNC_REGISTRY_PORT
              value: "8080"
            - name: MODEL_SYNC_REGISTRY_AUTHOR
              value: "Alice Bob"
            - name: MODEL_SYNC_REGISTRY_USER_TOKEN
              value: "super-secret"

            # ...