Add node selectors and tolerations to data processing

Relates:
https://issues.redhat.com/browse/RHOAIENG-26219

Signed-off-by: mprahl <mprahl@users.noreply.github.com>

Author: mprahl

pipeline.py

@@ -277,6 +277,11 @@ def ilab_pipeline(
     data_processing_task.after(model_to_pvc_task, sdg_task)
     data_processing_task.set_caching_options(False)
     data_processing_task.set_env_variable("XDG_CACHE_HOME", "/tmp")
+    data_processing_task.set_accelerator_type(eval_gpu_identifier)
+    data_processing_task.set_accelerator_limit(1)
+    add_toleration_json(data_processing_task, train_tolerations)
+    add_node_selector_json(data_processing_task, train_node_selectors)
+
 
     # Upload "skills_processed_data" and "knowledge_processed_data" artifacts to S3 without blocking the rest of the workflow
     skills_processed_data_to_artifact_task = skills_processed_data_to_artifact_op()

pipeline.yaml

@@ -934,6 +934,11 @@ deploymentSpec:
         - name: XDG_CACHE_HOME
           value: /tmp
         image: registry.redhat.io/rhelai1/instructlab-nvidia-rhel9@sha256:3e6eb035c69b204746a44b3a58b2751c20050cfb6af2ba7989ba327809f87c0b
+        resources:
+          accelerator:
+            count: '1'
+            resourceCount: '1'
+            resourceType: '{{$.inputs.parameters[''pipelinechannel--eval_gpu_identifier'']}}'
     exec-deletepvc:
       container:
         image: argostub/deletepvc
@@ -2620,8 +2625,13 @@ root:
         - sdg-op
         inputs:
           parameters:
+            accelerator_type:
+              runtimeValue:
+                constant: '{{$.inputs.parameters[''pipelinechannel--eval_gpu_identifier'']}}'
             max_batch_len:
               componentInputParameter: sdg_max_batch_len
+            pipelinechannel--eval_gpu_identifier:
+              componentInputParameter: eval_gpu_identifier
         taskInfo:
           name: data-processing-op
       deletepvc:
@@ -3395,6 +3405,9 @@ platforms:
     deploymentSpec:
       executors:
         exec-data-processing-op:
+          nodeSelector:
+            nodeSelectorJson:
+              componentInputParameter: train_node_selectors
           pvcMount:
           - mountPath: /model
             pvcNameParameter:
@@ -3412,6 +3425,9 @@ platforms:
             taskOutputParameter:
               outputParameterKey: name
               producerTask: createpvc
+          tolerations:
+          - tolerationJson:
+              componentInputParameter: train_tolerations
         exec-generate-metrics-report-op:
           pvcMount:
           - mountPath: /output