azureml-examples/cli/jobs/pipelines/add-column-and-word-count-using-spark/pipeline.yml at main · YunsongB/azureml-examples · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
$schema: https://azuremlschemas.azureedge.net/latest/pipelineJob.schema.json
type: pipeline

description: 'submit a pipeline with spark job'

inputs:
  iris_data:
    type: uri_file
    path: ./data/iris.csv
    mode: direct

outputs:
  output:
    type: uri_folder
    mode: direct

jobs:
  add_greeting_column:
    type: spark

    inputs:
      file_input: ${{parent.inputs.iris_data}}

    code: ./src
    entry:
      file: add_greeting_column.py # file path of the entry file relative to the code root folder

    py_files:
      - utils.zip

    files:
      - my_files.txt

    args: >-
      --file_input ${{inputs.file_input}}

    compute: azureml:spark31
    conf:
      spark.driver.cores: 2
      spark.driver.memory: "1g"
      spark.executor.cores: 1
      spark.executor.memory: "1g"
      spark.executor.instances: 1
      spark.yarn.appMasterEnv.AZUREML_ENABLE_DATAPATH_RESOLUTION: True
      spark.yarn.appMasterEnv.AZUREML_HADOOP_EXTENSION_URL: https://foobaradrama2.azurefd.net/latest/hadoop-azureml-fs.jar

  count_by_row:
    type: spark

    inputs:
      file_input: ${{parent.inputs.iris_data}}

    outputs:
      output: ${{parent.outputs.output}}

    code: ./src

    entry:
      file: count_by_row.py # file path of the entry file relative to the code root folder

    jars:
      - scalaproj.jar

    files:
      - my_files.txt

    args: >-
      --file_input ${{inputs.file_input}}
      --output ${{outputs.output}}

    compute: azureml:spark31
    conf:
      spark.driver.cores: 2
      spark.driver.memory: "1g"
      spark.executor.cores: 1
      spark.executor.memory: "1g"
      spark.executor.instances: 1
      spark.yarn.appMasterEnv.AZUREML_ENABLE_DATAPATH_RESOLUTION: True
      spark.yarn.appMasterEnv.AZUREML_HADOOP_EXTENSION_URL: https://foobaradrama2.azurefd.net/latest/hadoop-azureml-fs.jar