Skip to content

Commit da6f7a8

Browse files
fallarsfallars
authored
Merge pull request #261 from fallars/E02_auto_mib
Adding workflow to use the example notebook in /e02/notebooks
2 parents 2bdffe3 + ffc9ac8 commit da6f7a8

3 files changed

Lines changed: 195 additions & 0 deletions

File tree

‎e02/schema/notebook_Schema.json‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,36 @@
1+
{
2+
"type": "object",
3+
"description": "inputs for ePSIC notebook which calibrates 4D STEM data",
4+
"properties": {
5+
"sample": {
6+
"default": null,
7+
"maxLength": 128,
8+
"minLength": 1,
9+
"title": "Sample name",
10+
"type": "string"
11+
},
12+
"timestamp": {
13+
"default": null,
14+
"maxLength": 1,
15+
"minLength": 128,
16+
"title": "Time stamp",
17+
"type": "string"
18+
},
19+
"thresh_lower": {
20+
"default": 0.05,
21+
"maximum": 0.95,
22+
"minimum": 0,
23+
"title": "The lower threshold",
24+
"type": "number"
25+
},
26+
"thresh_upper": {
27+
"default": 0.2,
28+
"maximum": 1,
29+
"minimum": 0.05,
30+
"title": "The upper threshold",
31+
"type": "number"
32+
}
33+
},
34+
"title": "Calibration of 4D STEM data notebook"
35+
}
36+

‎e02/schema/notebook_ui.json‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
{
2+
"type": "VerticalLayout",
3+
"options": {
4+
"formWidth": "80%"
5+
},
6+
"elements": [
7+
{
8+
"type": "Group",
9+
"label": "ePSIC notebook which calibrates 4D STEM data",
10+
"elements": [
11+
{
12+
"type": "VerticalLayout",
13+
"elements": [
14+
{
15+
"type": "Control",
16+
"label": "Name of the sample",
17+
"scope": "#/properties/sample"
18+
},
19+
{
20+
"type": "Control",
21+
"label": "Time stamp of the data to process",
22+
"scope": "#/properties/timestamp"
23+
},
24+
{
25+
"type": "Control",
26+
"label": "The lower threshold",
27+
"scope": "#/properties/thresh_lower"
28+
},
29+
{
30+
"type": "Control",
31+
"label": "The upper threshold",
32+
"scope": "#/properties/thresh_upper"
33+
}
34+
]
35+
}
36+
]
37+
}
38+
]
39+
}

‎e02/templates/Notebook.yaml‎

Lines changed: 120 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,120 @@
1+
apiVersion: argoproj.io/v1alpha1
2+
kind: ClusterWorkflowTemplate
3+
metadata:
4+
name: Calibrate4DSTEM
5+
labels:
6+
workflows.diamond.ac.uk/science-group-imaging: "true"
7+
annotations:
8+
workflows.argoproj.io/title: Calibrate 4D STEM data
9+
workflows.argoproj.io/description: |
10+
A utility which takes in a 4D STEM data set and outs the approximate real space pixel size, measured camera length and scan rotation (based off dpc).
11+
workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows"
12+
workflows.diamond.ac.uk/parameter-schema: |
13+
{{- .Files.Get "schema/Calibrate4DSTEMSchema.json" | nindent 6 }}
14+
workflows.diamond.ac.uk/ui-schema: |
15+
{{- .Files.Get "schema/Calibrate4DSTEMUISchema.json" | nindent 6 }}
16+
spec:
17+
entrypoint: mount-files
18+
arguments:
19+
parameters:
20+
- name: visitdir
21+
valueFrom:
22+
configMapKeyRef:
23+
name: sessionspaces
24+
key: data_directory
25+
volumeClaimTemplates:
26+
- metadata:
27+
name: tmp
28+
spec:
29+
accessModes:
30+
- ReadWriteOnce
31+
resources:
32+
requests:
33+
storage: 2Gi
34+
storageClassName: netapp
35+
36+
templates:
37+
- name: calibration-notebook
38+
steps:
39+
- - name: copy-notebook
40+
template: mount-files
41+
- - name: run-notebook
42+
template: notebook
43+
44+
- name: mount-files
45+
script:
46+
image: debian:stable-slim
47+
volumeMounts:
48+
- name: tmp
49+
mountPath: /tmp
50+
- name: session
51+
mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"
52+
command: [bash]
53+
source: |
54+
OUTPUT="{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/
55+
mkdir -p $OUTPUT
56+
echo "something!"
57+
echo '{{ .Files.Get "notebooks/Ptycho_calibration.ipynb" | b64enc }}' | base64 -d > $OUTPUT/Ptycho_calibration.ipynb
58+
59+
- name: notebook
60+
inputs:
61+
parameters:
62+
- name: sample
63+
value: "{{`{{workflow.parameters.sample}}`}}"
64+
- name: timestamp
65+
value: "{{`{{workflow.parameters.timestamp}}`}}"
66+
- name: outpath
67+
value: "{{`{{workflow.parameters.outpath}}`}}"
68+
- name: thresh_lower
69+
value: "{{`{{workflow.parameters.thresh_lower}}`}}"
70+
- name: thresh_upper
71+
value: "{{`{{workflow.parameters.thresh_upper}}`}}"
72+
script:
73+
image: gitlab.diamond.ac.uk:5050/scisoft/ptychography/dimtools/mib2x
74+
volumeMounts:
75+
- name: session
76+
mountPath: "{{`{{ workflows.parameters.visitdir }}`}}"
77+
- name: tmp
78+
mountPath: /tmp
79+
- name: software
80+
mountPath: /dls_sw/e02/medipix_mask
81+
command: [bash]
82+
source: |
83+
ERROR_TXT="raise error"
84+
ERROR_STR="An error occured within the notebook. Please open the output file to check"
85+
OUTPUT="{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/
86+
python -m papermill "{{`{{workflow.parameters.visitdir}}`}}"/processing/workflows/Ptycho_calibration.ipynb $OUTPUT/Calibrate-notebook.ipynb \
87+
-p visit "{{`{{ workflow.parameters.visitdir }}`}}"
88+
-p sample "{{`{{ workflows.parameters.sample }}`}}"
89+
-p timestamp "{{`{{workflows.parameters.timestamp}}`}}"
90+
-p maskpath /software/29042024_12bitmask2.h5
91+
-p outpath "{{`{{workflows.parameters.outpath}}`}}"
92+
-p lower "{{`{{workflows.parameters.thresh_lower}}`}}"
93+
-p upper "{{`{{workflows.parameters.thresh_upper}}`}}"
94+
> $OUTPUT/papermill.log 2>&1
95+
96+
python -m jupyter nbconvert $OUTPUT/Calibrate-notebook.ipynb \
97+
--to html
98+
99+
outputs:
100+
artifacts:
101+
- name: phase-output
102+
path: "{{`{{ inputs.parameters.outpath }}`}}/phase-output.html"
103+
archive:
104+
none: {}
105+
- name: notebook-log
106+
path: "{{`{{ inputs.parameters.outpath }}`}}/papermill.log"
107+
archive:
108+
none: {}
109+
podSpecPatch: |
110+
containers:
111+
- name: main
112+
resources:
113+
requests:
114+
cpu: 10
115+
memory: 30Gi
116+
limits:
117+
cpu: 10
118+
memory: 30Gi
119+
120+

0 commit comments

Comments
 (0)