otter/config.yaml at main · opentargets/otter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
---
work_path: ./work
# release_uri: gs://open-targets-pipeline-runs/jferrer
log_level: TRACE
pool_size: 5

scratchpad: {}

steps:
  simple:
    - name: hello_world simple
      who: simple

  test:
    - name: hello_world test1
      who: test
    - name: hello_world test2
      who: test2
    - name: hello_world test3
      who: test3
    - name: hello_world test with requirements
      requires: [hello_world test1, hello_world test2, hello_world test3]
      who: test4

  explode:
    - name: explode a test
      foreach:
        - alice
        - bob
        - charlie
      do:
        - name: hello_world for ${each}
          who: ${each}

  explode-requires:
    - name: explode some stuff
      foreach:
        - dave
        - eve
        - frank
      do:
        - name: hello_world for ${each} with requires
          who: ${each}
    - name: hello_world final
      requires: [explode some stuff]
      who: final

  inception:
    - name: explode inception
      foreach:
        - gina
        - hank
      each_placeholder: outer
      do:
        - name: explode inner for ${outer}
          foreach:
            - iris
            - jack
          each_placeholder: inner
          do:
            - name: hello_world for ${outer} to ${inner}
              who: says ${outer} to ${inner}
            - name: hello_world for ${inner} to ${outer}
              who: also says ${inner} to ${outer}

  explode-glob:
    - name: explode_glob inception
      glob: explode-glob/*.txt
      do:
        - name: hello_world for ${match_stem} in ${uri}
          who: says hello to ${match_stem} in ${uri}

  copy:
    - name: copy from http
      source: https://www.w3.org/TR/2003/REC-PNG-20031110/iso_8859-1.txt
      destination: destination/copiedfromhttp.txt

    - name: copy from gcp
      source: gs://open-targets-pipeline-runs/jferrer/otterfs/source/copyfromgcp.txt
      destination: destination/copiedfromgcp.txt

  download:
    - name: download from gcp
      source: source/downloadfromgcp.txt
      destination: destination/downloadedfromgcp.txt

  upload:
    - name: upload to gcp
      source: source/uploadtogcp.txt
      destination: destination/uploadedtogcp.txt

  find-latest:
    - name: find_latest cosmic
      source: gs://otar007-cosmic

  copy-many-list:
    - name: copy_many a list of files
      sources:
        - https://www.encodeproject.org/files/ENCFF002CTW/@@download/ENCFF002CTW.bed.gz
        - https://www.encodeproject.org/files/ENCFF557AUI/@@download/ENCFF557AUI.bed.gz
        - https://www.encodeproject.org/files/ENCFF734HLS/@@download/ENCFF734HLS.bed.gz
        - https://www.encodeproject.org/files/ENCFF458LUT/@@download/ENCFF458LUT.bed.gz
        - https://www.encodeproject.org/files/ENCFF813RRM/@@download/ENCFF813RRM.bed.gz
        - https://www.encodeproject.org/files/ENCFF509PMS/@@download/ENCFF509PMS.bed.gz
        - https://www.encodeproject.org/files/ENCFF071ZPG/@@download/ENCFF071ZPG.bed.gz
        - https://www.encodeproject.org/files/ENCFF086JUL/@@download/ENCFF086JUL.bed.gz
      destination: test-copy-many-list/

  copy-many-file:
    - name: copy_many from a source list file
      source_list_file: src.txt
      destination: test-copy-many-file/

  step-fails:
    - name: copy huge file
      source: https://ftp.ensembl.org/pub/release-115/json/homo_sapiens/homo_sapiens.json
      destination: homo_sapiens.json

    - name: copy from non existing source
      source: https://input/will/not/exist/nonexistingfile.txt
      destination: anywhere.txt

    - name: explode some subtasks
      foreach:
        - alice
        - bob
      do:
        - name: hello_world for ${each}
          who: ${each}