Artifact outputs

examples/multi_backends/argo-train.yml

@@ -16,21 +16,25 @@ spec:
       - gASVSQIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMBWNsZWFulIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjCVleGFtcGxlcy5tdWx0aV9iYWNrZW5kcy5leGFtcGxlOmNsZWFulIwHcnVudGltZZSMCnB5dGhvbjozLjiUjAdjb2RldXJslE6MBWltYWdllIwPbmlsYWJocmEvcGlybGlilHVic4wJZnJhbWV3b3JrlE6MBmNvbmZpZ5R9lIwGaW5wdXRzlF2UaACMBUlucHV0lJOUKYGUfZQoaAWMB2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjAZzb3VyY2WUaACMCkRhdGFTb3VyY2WUk5QpgZR9lCiMB25vZGVfaWSUTowLc3ViZ3JhcGhfaWSUTowJb3V0cHV0X2lklE6MDmdyYXBoX2lucHV0X2lklIwNdHJhaW5fZGF0YXNldJR1YowEbWV0YZRoAIwITWV0YWRhdGGUk5QpgZR9lCiMBG5hbWWUjAdkYXRhc2V0lIwLYW5ub3RhdGlvbnOUTnVidWJhjAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCGMCURJUkVDVE9SWZRoLWgvKYGUfZQoaDKMBnJldHVybpRoNE51YnViYWgtaC8pgZR9lChoMowFY2xlYW6UaDROdWJ1Yi4=
       - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu
       volumeMounts:
-      - name: node-outputs
-        mountPath: /mnt/node_outputs
       - name: train-dataset
         mountPath: /mnt/graph_inputs/train_dataset
+      - name: node-outputs
+        mountPath: /mnt/node_outputs
+    outputs:
+      artifacts:
+      - name: clean
+        path: /mnt/node_outputs/clean/return
     volumes:
-    - name: node-outputs
-      nfs:
-        server: k8s-master.cm.cluster
-        path: /home/nilabhra/pirlib/examples/multi_backends/outputs
-        readOnly: no
     - name: train-dataset
       nfs:
         server: k8s-master.cm.cluster
         path: /home/nilabhra/pirlib/examples/multi_backends/inputs/train_dataset
         readOnly: yes
+    - name: node-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/nilabhra/pirlib/examples/multi_backends/outputs
+        readOnly: no
   - name: train-template
     container:
       image: nilabhra/pirlib
@@ -44,6 +48,10 @@ spec:
       volumeMounts:
       - name: node-outputs
         mountPath: /mnt/node_outputs
+    outputs:
+      artifacts:
+      - name: train
+        path: /mnt/node_outputs/train/return
     volumes:
     - name: node-outputs
       nfs:
@@ -61,21 +69,25 @@ spec:
       - gASV9gIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMCGV2YWx1YXRllIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjChleGFtcGxlcy5tdWx0aV9iYWNrZW5kcy5leGFtcGxlOmV2YWx1YXRllIwHcnVudGltZZSMCnB5dGhvbjozLjiUjAdjb2RldXJslE6MBWltYWdllIwPbmlsYWJocmEvcGlybGlilHVic4wJZnJhbWV3b3JrlE6MBmNvbmZpZ5R9lIwGaW5wdXRzlF2UKGgAjAVJbnB1dJSTlCmBlH2UKGgFjBNrd2FyZ3MudGVzdF9kYXRhc2V0lIwGaW90eXBllIwJRElSRUNUT1JZlIwGc291cmNllGgAjApEYXRhU291cmNllJOUKYGUfZQojAdub2RlX2lklE6MC3N1YmdyYXBoX2lklE6MCW91dHB1dF9pZJROjA5ncmFwaF9pbnB1dF9pZJSMCXNlbnRlbmNlc5R1YowEbWV0YZRoAIwITWV0YWRhdGGUk5QpgZR9lCiMBG5hbWWUjBNrd2FyZ3MudGVzdF9kYXRhc2V0lIwLYW5ub3RhdGlvbnOUTnVidWJoHSmBlH2UKGgFjBJrd2FyZ3MucHJlZGljdGlvbnOUaCGMCURJUkVDVE9SWZRoI2glKYGUfZQoaCiMGGluZmVyX3BpcGVsaW5lLnNlbnRpbWVudJRoKU5oKowGcmV0dXJulGgrTnViaC1oLymBlH2UKGgyjBJrd2FyZ3MucHJlZGljdGlvbnOUaDROdWJ1YmWMB291dHB1dHOUXZRoAIwGT3V0cHV0lJOUKYGUfZQoaAWMBnJldHVybpRoIYwJREFUQUZSQU1FlGgtaC8pgZR9lChoMowGcmV0dXJulGg0TnVidWJhaC1oLymBlH2UKGgyjAhldmFsdWF0ZZRoNE51YnViLg==
       - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu
       volumeMounts:
-      - name: node-outputs
-        mountPath: /mnt/node_outputs
       - name: sentences
         mountPath: /mnt/graph_inputs/sentences
+      - name: node-outputs
+        mountPath: /mnt/node_outputs
+    outputs:
+      artifacts:
+      - name: evaluate
+        path: /mnt/node_outputs/evaluate/return
     volumes:
-    - name: node-outputs
-      nfs:
-        server: k8s-master.cm.cluster
-        path: /home/nilabhra/pirlib/examples/multi_backends/outputs
-        readOnly: no
     - name: sentences
       nfs:
         server: k8s-master.cm.cluster
         path: /home/nilabhra/pirlib/examples/multi_backends/inputs/sentences
         readOnly: yes
+    - name: node-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/nilabhra/pirlib/examples/multi_backends/outputs
+        readOnly: no
   - name: infer-pipeline-translate-1-template
     container:
       image: nilabhra/pirlib
@@ -87,19 +99,18 @@ spec:
       - gASV0wIAAAAAAACMCnBpcmxpYi5waXKUjAROb2RllJOUKYGUfZQojAJpZJSMGmluZmVyX3BpcGVsaW5lLnRyYW5zbGF0ZV8xlIwLZW50cnlwb2ludHOUfZSMBG1haW6UaACMCkVudHJ5cG9pbnSUk5QpgZR9lCiMB3ZlcnNpb26UjAJ2MZSMB2hhbmRsZXKUjClleGFtcGxlcy5tdWx0aV9iYWNrZW5kcy5leGFtcGxlOnRyYW5zbGF0ZZSMB3J1bnRpbWWUjApweXRob246My44lIwHY29kZXVybJROjAVpbWFnZZSMD25pbGFiaHJhL3BpcmxpYpR1YnOMCWZyYW1ld29ya5ROjAZjb25maWeUfZSMA2tleZSMBXZhbHVllHOMBmlucHV0c5RdlChoAIwFSW5wdXSUk5QpgZR9lChoBYwGYXJncy4wlIwGaW90eXBllIwERklMRZSMBnNvdXJjZZRoAIwKRGF0YVNvdXJjZZSTlCmBlH2UKIwHbm9kZV9pZJROjAtzdWJncmFwaF9pZJROjAlvdXRwdXRfaWSUTowOZ3JhcGhfaW5wdXRfaWSUjA90cmFuc2xhdGVfbW9kZWyUdWKMBG1ldGGUaACMCE1ldGFkYXRhlJOUKYGUfZQojARuYW1llIwGYXJncy4wlIwLYW5ub3RhdGlvbnOUTnVidWJoHymBlH2UKGgFjAZhcmdzLjGUaCOMCURJUkVDVE9SWZRoJWgnKYGUfZQoaCpOaCtOaCxOaC2MCXNlbnRlbmNlc5R1YmgvaDEpgZR9lChoNIwGYXJncy4xlGg2TnVidWJljAdvdXRwdXRzlF2UaACMBk91dHB1dJSTlCmBlH2UKGgFjAZyZXR1cm6UaCOMCURJUkVDVE9SWZRoL2gxKYGUfZQoaDSMBnJldHVybpRoNk51YnViYWgvaDEpgZR9lChoNIwLdHJhbnNsYXRlXzGUaDZOdWJ1Yi4=
       - gASVJAEAAAAAAABdlCiMCnBpcmxpYi5waXKUjApHcmFwaElucHV0lJOUKYGUfZQojAJpZJSMDXRyYWluX2RhdGFzZXSUjAZpb3R5cGWUjAlESVJFQ1RPUlmUjARtZXRhlGgBjAhNZXRhZGF0YZSTlCmBlH2UKIwEbmFtZZSMDXRyYWluX2RhdGFzZXSUjAthbm5vdGF0aW9uc5ROdWJ1YmgDKYGUfZQoaAaMD3RyYW5zbGF0ZV9tb2RlbJRoCIwERklMRZRoCmgMKYGUfZQoaA+MD3RyYW5zbGF0ZV9tb2RlbJRoEU51YnViaAMpgZR9lChoBowJc2VudGVuY2VzlGgIjAlESVJFQ1RPUlmUaApoDCmBlH2UKGgPjAlzZW50ZW5jZXOUaBFOdWJ1YmUu
       volumeMounts:
-      - name: node-outputs
-        mountPath: /mnt/node_outputs
       - name: translate-model
         mountPath: /mnt/graph_inputs/translate_model
         subPath: translate_model.txt
       - name: sentences
         mountPath: /mnt/graph_inputs/sentences
+      - name: node-outputs
+        mountPath: /mnt/node_outputs
+    outputs:
+      artifacts:
+      - name: infer-pipeline-translate-1
+        path: /mnt/node_outputs/infer_pipeline.translate_1/return
     volumes:
-    - name: node-outputs
-      nfs:
-        server: k8s-master.cm.cluster
-        path: /home/nilabhra/pirlib/examples/multi_backends/outputs
-        readOnly: no
     - name: translate-model
       nfs:
         server: k8s-master.cm.cluster
@@ -110,6 +121,11 @@ spec:
         server: k8s-master.cm.cluster
         path: /home/nilabhra/pirlib/examples/multi_backends/inputs/sentences
         readOnly: yes
+    - name: node-outputs
+      nfs:
+        server: k8s-master.cm.cluster
+        path: /home/nilabhra/pirlib/examples/multi_backends/outputs
+        readOnly: no
   - name: infer-pipeline-sentiment-template
     container:
       image: nilabhra/pirlib
@@ -123,6 +139,10 @@ spec:
       volumeMounts:
       - name: node-outputs
         mountPath: /mnt/node_outputs
+    outputs:
+      artifacts:
+      - name: infer-pipeline-sentiment
+        path: /mnt/node_outputs/infer_pipeline.sentiment/return
     volumes:
     - name: node-outputs
       nfs:

pirlib/backends/argo_batch.py

@@ -119,14 +119,9 @@ def create_template_from_node(
         graph_inputs_encoded,
     ]
 
-    # Obtain specs for the output volume.
-    op_volume, op_volume_mount = create_nfs_volume_spec(
-        volume_name="node_outputs", is_file=False, is_input=False
-    )
-
     # Define input volumes to be mounted.
-    volumes = [op_volume]
-    volume_mounts = [op_volume_mount]
+    volumes = []
+    volume_mounts = []
     dependencies = []
 
     # If the node has a valid graph input source,
@@ -151,6 +146,21 @@ def create_template_from_node(
         if inp.source.node_id:
             dependencies.append(argo_name(inp.source.node_id))
 
+    # Obtain specs for the output volume.
+    op_volume, op_volume_mount = create_nfs_volume_spec(
+        volume_name="node_outputs", is_file=False, is_input=False
+    )
+    volumes.append(op_volume)
+    volume_mounts.append(op_volume_mount)
+
+    # Create spec for output artifact.
+    artifacts = []
+    for output in node.outputs:
+        # Infer the mount path
+        path = f"{op_volume_mount.get('mountPath')}/{name}/{output.id}"
+        artifacts.append({"name": argo_name(name), "path": path})
+    outputs = {"artifacts": artifacts}
+
     # Create the template dictionary.
     template = {
         "name": argo_name(name),
@@ -159,6 +169,7 @@ def create_template_from_node(
             "command": command,
             "volumeMounts": volume_mounts,
         },
+        "outputs": outputs,
         "volumes": volumes,
         "dependencies": dependencies,
     }