Skip to content

Commit ee659e5

Browse files
erikvdbergherikvdbergh
authored
Merge branch 'master' into testing
2 parents 580037c + dbdf45e commit ee659e5

27 files changed

+909
-13
lines changed

‎README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
<img src="docs/TESKlogowfont.png" height="200">
22

33

4-
An implementation of a task execution engine based on the [TES standard](https://github.com/ga4gh/task-execution-schemas) running on Kubernetes.
4+
An implementation of a task execution engine based on the [TES standard](https://github.com/ga4gh/task-execution-schemas) running on Kubernetes. For more details on TES, see the [(very) brief introduction to TES](tesintro.md).
55

66
For organisational reasons, this project is split into 2 repositories: One containing the API and associated docker images ([here](https://github.com/EMBL-EBI-TSI/tesk-api)) and one containing the actual task execution service and associated Docker images (this one). If the API is running on your cluster it will pull the images from our gcr.io repository automatically. In that vein, see below under 'How to install' to get TESK up and running on your Kubernetes cluster.
77

@@ -10,7 +10,8 @@ For organisational reasons, this project is split into 2 repositories: One conta
1010

1111
## How to install
1212
- Clone the repo to your kube-master and cd into the folder
13-
- Edit the following line in `specs/ingress/nginx-ingress-lb.yaml`:
13+
- Find out what is the external IP for the cluster. E.g. with the command `minikube ip`
14+
- Edit the following line in `specs/ingress/nginx-ingress-lb.yaml` with that IP:
1415

1516
```yaml
1617
spec:
@@ -20,7 +21,7 @@ For organisational reasons, this project is split into 2 repositories: One conta
2021

2122
```
2223

23-
- Then create the services necessary to run the API:
24+
- Create the services necessary to run the API:
2425

2526
```
2627
kubectl create -f specs/ingress/

‎examples/cancel/counter.json‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
{
2+
"description": "An example task to test job cancellation. Cancel it please, because it runs forever",
3+
"volumes": [
4+
"/test"
5+
],
6+
"executors": [
7+
{
8+
"image": "alpine",
9+
"command": [
10+
"echo",
11+
"Nothing"
12+
]
13+
},
14+
{
15+
"image": "alpine",
16+
"command": [
17+
"sh",
18+
"-c",
19+
"i=0; while true; do echo \"$i: $(date)\"; i=$((i+1)); sleep 1; done"
20+
]
21+
}
22+
]
23+
}

‎examples/error/error_stops_execution_taskmaster.json‎

Lines changed: 113 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,113 @@
1+
{
2+
"outputs": [],
3+
"inputs": [],
4+
"volumes": [],
5+
"executors": [
6+
{
7+
"apiVersion": "batch/v1",
8+
"kind": "Job",
9+
"metadata": {
10+
"annotations": {},
11+
"labels": {
12+
"job-type": "executor",
13+
"taskmaster-name": "task-ec104c85",
14+
"executor-no": "0"
15+
},
16+
"name": "task-ec104c85-ex-00"
17+
},
18+
"spec": {
19+
"template": {
20+
"metadata": {
21+
"name": "task-ec104c85-ex-00"
22+
},
23+
"spec": {
24+
"containers": [
25+
{
26+
"command": [
27+
"echo",
28+
"You will see this in the logs (stdout)."
29+
],
30+
"image": "alpine",
31+
"name": "task-ec104c85-ex-00",
32+
"resources": {}
33+
}
34+
],
35+
"restartPolicy": "Never"
36+
}
37+
}
38+
}
39+
},
40+
{
41+
"apiVersion": "batch/v1",
42+
"kind": "Job",
43+
"metadata": {
44+
"annotations": {},
45+
"labels": {
46+
"job-type": "executor",
47+
"taskmaster-name": "task-ec104c85",
48+
"executor-no": "1"
49+
},
50+
"name": "task-ec104c85-ex-01"
51+
},
52+
"spec": {
53+
"template": {
54+
"metadata": {
55+
"name": "task-ec104c85-ex-01"
56+
},
57+
"spec": {
58+
"containers": [
59+
{
60+
"command": [
61+
"sh",
62+
"-c",
63+
"exit 1"
64+
],
65+
"image": "alpine",
66+
"name": "task-ec104c85-ex-01",
67+
"resources": {}
68+
}
69+
],
70+
"restartPolicy": "Never"
71+
}
72+
}
73+
}
74+
},
75+
{
76+
"apiVersion": "batch/v1",
77+
"kind": "Job",
78+
"metadata": {
79+
"annotations": {},
80+
"labels": {
81+
"job-type": "executor",
82+
"taskmaster-name": "task-ec104c85",
83+
"executor-no": "2"
84+
},
85+
"name": "task-ec104c85-ex-02"
86+
},
87+
"spec": {
88+
"template": {
89+
"metadata": {
90+
"name": "task-ec104c85-ex-02"
91+
},
92+
"spec": {
93+
"containers": [
94+
{
95+
"command": [
96+
"echo",
97+
"This shouldn't appear in the logs (stdout)."
98+
],
99+
"image": "alpine",
100+
"name": "task-ec104c85-ex-02",
101+
"resources": {}
102+
}
103+
],
104+
"restartPolicy": "Never"
105+
}
106+
}
107+
}
108+
}
109+
],
110+
"resources": {
111+
"disk_gb": 0.1
112+
}
113+
}

‎examples/error/executor_taskmaster.json‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
{
2+
"outputs": [],
3+
"inputs": [],
4+
"volumes": [],
5+
"executors": [
6+
{
7+
"apiVersion": "batch/v1",
8+
"kind": "Job",
9+
"metadata": {
10+
"annotations": {},
11+
"labels": {
12+
"job-type": "executor",
13+
"taskmaster-name": "task-23bf5e64",
14+
"executor-no": "0"
15+
},
16+
"name": "task-23bf5e64-ex-00"
17+
},
18+
"spec": {
19+
"template": {
20+
"metadata": {
21+
"name": "task-23bf5e64-ex-00"
22+
},
23+
"spec": {
24+
"containers": [
25+
{
26+
"command": [
27+
"sh",
28+
"-c",
29+
"exit 1"
30+
],
31+
"image": "alpine",
32+
"name": "task-23bf5e64-ex-00",
33+
"resources": {}
34+
}
35+
],
36+
"restartPolicy": "Never"
37+
}
38+
}
39+
}
40+
}
41+
],
42+
"resources": {
43+
"disk_gb": 0.1
44+
}
45+
}

‎examples/error/input_taskmaster.json‎

Lines changed: 84 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,84 @@
1+
{
2+
"outputs": [],
3+
"inputs": [
4+
{
5+
"url": "http://nonexistent.ebi.ac.uk",
6+
"path": "/somewhere/file",
7+
"type": "FILE"
8+
}
9+
],
10+
"volumes": [],
11+
"executors": [
12+
{
13+
"apiVersion": "batch/v1",
14+
"kind": "Job",
15+
"metadata": {
16+
"annotations": {},
17+
"labels": {
18+
"job-type": "executor",
19+
"taskmaster-name": "task-7ff304bd",
20+
"executor-no": "0"
21+
},
22+
"name": "task-7ff304bd-ex-00"
23+
},
24+
"spec": {
25+
"template": {
26+
"metadata": {
27+
"name": "task-7ff304bd-ex-00"
28+
},
29+
"spec": {
30+
"containers": [
31+
{
32+
"command": [
33+
"echo",
34+
"This shouldn't appear in the logs."
35+
],
36+
"image": "alpine",
37+
"name": "task-7ff304bd-ex-00",
38+
"resources": {}
39+
}
40+
],
41+
"restartPolicy": "Never"
42+
}
43+
}
44+
}
45+
},
46+
{
47+
"apiVersion": "batch/v1",
48+
"kind": "Job",
49+
"metadata": {
50+
"annotations": {},
51+
"labels": {
52+
"job-type": "executor",
53+
"taskmaster-name": "task-7ff304bd",
54+
"executor-no": "1"
55+
},
56+
"name": "task-7ff304bd-ex-01"
57+
},
58+
"spec": {
59+
"template": {
60+
"metadata": {
61+
"name": "task-7ff304bd-ex-01"
62+
},
63+
"spec": {
64+
"containers": [
65+
{
66+
"command": [
67+
"cat",
68+
"/somewhere/file"
69+
],
70+
"image": "alpine",
71+
"name": "task-7ff304bd-ex-01",
72+
"resources": {}
73+
}
74+
],
75+
"restartPolicy": "Never"
76+
}
77+
}
78+
}
79+
}
80+
],
81+
"resources": {
82+
"disk_gb": 0.1
83+
}
84+
}

‎examples/success/input_content.json‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
{
2+
"description": "Demonstrates inlined input",
3+
"inputs": [
4+
{
5+
"content": "ABC TESK and some more text.",
6+
"path": "/tes/volumes/input",
7+
"type": "FILE"
8+
}
9+
],
10+
"executors": [
11+
{
12+
"image": "alpine",
13+
"command": [
14+
"cat",
15+
"/tes/volumes/input"
16+
]
17+
}
18+
]
19+
}

‎examples/success/input_dir_duplicate_file_names.json‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"description": "2 different folders with filename collision, mounted to the same place. The latest one wins (das in our case)",
3+
"inputs": [
4+
{
5+
"url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README",
6+
"path": "/tes/files/rna_README",
7+
"type": "FILE"
8+
},
9+
{
10+
"url": "ftp://ftp.ebi.ac.uk/pub/software/das/README",
11+
"path": "/tes/files/das_README",
12+
"type": "FILE"
13+
},
14+
{
15+
"url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA",
16+
"path": "/tes",
17+
"type": "DIRECTORY"
18+
},
19+
{
20+
"url": "ftp://ftp.ebi.ac.uk/pub/software/das",
21+
"path": "/tes",
22+
"type": "DIRECTORY"
23+
}
24+
],
25+
"executors": [
26+
{
27+
"image": "ubuntu",
28+
"command": [
29+
"sh",
30+
"-c",
31+
"cd /tes; find * -name \"*README\" -print0 | du --files0-from=- -b"
32+
]
33+
}
34+
]
35+
}

‎examples/success/input_dir_file_duplicate_file_names.json‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
{
2+
"description": "Placing a file at a location (/tes/README), that is colliding with a file from directory input. The latest one wins (das in our case)",
3+
"inputs": [
4+
{
5+
"url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README",
6+
"path": "/tes/files/rna_README",
7+
"type": "FILE"
8+
},
9+
{
10+
"url": "ftp://ftp.ebi.ac.uk/pub/software/das/README",
11+
"path": "/tes/files/das_README",
12+
"type": "FILE"
13+
},
14+
{
15+
"url": "ftp://ftp.ebi.ac.uk/pub/databases/16S_RNA/README",
16+
"path": "/tes/README",
17+
"type": "FILE"
18+
},
19+
{
20+
"url": "ftp://ftp.ebi.ac.uk/pub/software/das",
21+
"path": "/tes",
22+
"type": "DIRECTORY"
23+
}
24+
],
25+
"executors": [
26+
{
27+
"image": "ubuntu",
28+
"command": [
29+
"sh",
30+
"-c",
31+
"cd /tes; find * -name \"*README\" -print0 | du --files0-from=- -b"
32+
]
33+
}
34+
]
35+
}

0 commit comments

Comments
 (0)