diff --git a/e02/templates/mib2x-auto.yaml b/e02/templates/mib2x-auto.yaml index 806ce0a..0af7a80 100644 --- a/e02/templates/mib2x-auto.yaml +++ b/e02/templates/mib2x-auto.yaml @@ -8,6 +8,7 @@ metadata: workflows.argoproj.io/title: ePSIC mib automatic conversion workflows.argoproj.io/description: | Convert MIB file to hdf5/hspy files + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/mib2x_auto_Schema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | diff --git a/e02/templates/mib2x.yaml b/e02/templates/mib2x.yaml index 479f37a..de14d6a 100644 --- a/e02/templates/mib2x.yaml +++ b/e02/templates/mib2x.yaml @@ -8,6 +8,7 @@ metadata: workflows.argoproj.io/title: ePSIC mib conversion workflows.argoproj.io/description: | Convert MIB file to hdf5/hspy files + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema.mib_path: | { "type": "string" diff --git a/e02/templates/ptyrex_basic.yaml b/e02/templates/ptyrex_basic.yaml index aabf085..4ebb961 100644 --- a/e02/templates/ptyrex_basic.yaml +++ b/e02/templates/ptyrex_basic.yaml @@ -8,6 +8,7 @@ metadata: workflows.argoproj.io/title: Simple Ptyrex job submission workflows.argoproj.io/description: | Submit a Ptyrex job for a particular scan and projection within a given visit + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/ptyrex_Schema.json" | nindent 6}} workflows.diamond.ac.uk/ui-schema: | diff --git a/httomo/pipelines/visr.json b/httomo/pipelines/visr.json index cdcec11..0d24b95 100644 --- a/httomo/pipelines/visr.json +++ b/httomo/pipelines/visr.json @@ -21,8 +21,7 @@ "module_path": "httomolibgpu.prep.normalize", "parameters": { "flats_multiplier": 1, - "darks_multiplier": 1, - "cutoff": 10 + "darks_multiplier": 1 } }, { diff --git a/httomo/templates/cor-sweep.yaml b/httomo/templates/cor-sweep.yaml index 990c885..675996c 100644 --- a/httomo/templates/cor-sweep.yaml +++ b/httomo/templates/cor-sweep.yaml @@ -2,6 +2,8 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: httomo-cor-sweep + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema.input: | diff --git a/httomo/templates/extract-raw-projections.yaml b/httomo/templates/extract-raw-projections.yaml index ebb1de0..6770c8f 100644 --- a/httomo/templates/extract-raw-projections.yaml +++ b/httomo/templates/extract-raw-projections.yaml @@ -2,6 +2,8 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: extract-raw-projections + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: diff --git a/httomo/templates/httomo-gpu-template-test.yaml b/httomo/templates/httomo-gpu-template-test.yaml index a8be1c9..ae8b649 100644 --- a/httomo/templates/httomo-gpu-template-test.yaml +++ b/httomo/templates/httomo-gpu-template-test.yaml @@ -2,6 +2,8 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: httomo-gpu-job + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: diff --git a/i14/notebooks/ptycho-tomo-phase.ipynb b/i14/notebooks/ptycho-tomo-phase.ipynb index ec0d64d..48a0c6a 100644 --- a/i14/notebooks/ptycho-tomo-phase.ipynb +++ b/i14/notebooks/ptycho-tomo-phase.ipynb @@ -88,6 +88,24 @@ "cell_type": "markdown", "id": "2f61bb2f", "metadata": {}, + "source": [ + "## Set normalise boolean" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3300162-404a-4660-b4ac-136bbd517727", + "metadata": {}, + "outputs": [], + "source": [ + "normalise = True if normalise.lower() == \"true\" else False" + ] + }, + { + "cell_type": "markdown", + "id": "5d23b863-a0da-40f4-aa72-c5b5cc911c00", + "metadata": {}, "source": [ "## Initialise np array" ] diff --git a/i14/notebooks/xanes_autoprocessing0.ipynb b/i14/notebooks/xanes_autoprocessing0.ipynb index ed3f052..2d2eaac 100644 --- a/i14/notebooks/xanes_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_autoprocessing0.ipynb @@ -10,7 +10,7 @@ }, "outputs": [], "source": [ - "# default settings\n", + "# parameters\n", "inpath = \"\" # type: str\n", "outpath = \"\" # type: str\n", "edge_element = \"\" # type: str\n", @@ -169,6 +169,22 @@ "print(f\"The PNG file will be saved as {png_outpath}\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set normalised boolean\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "normalised = True if normalised.lower() == \"true\" else False" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/i14/notebooks/xanes_point_autoprocessing0.ipynb b/i14/notebooks/xanes_point_autoprocessing0.ipynb index 8d7918f..9e70baa 100644 --- a/i14/notebooks/xanes_point_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_point_autoprocessing0.ipynb @@ -84,6 +84,22 @@ " raise FileNotFoundError(msg)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set normalise boolean" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "normalise = True if normalise.lower() == \"true\" else False" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb index 26b7137..074ace9 100644 --- a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb @@ -10,6 +10,7 @@ }, "outputs": [], "source": [ + "# parameters\n", "# outpath will be used to get outpath_nexus, outpath_mantis and outpath_complete for saving data, and outpath itself won't be used\n", "\n", "# default settings\n", @@ -119,6 +120,22 @@ "print(f\"HyperSpy version: {version('hyperspy')}\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set normalised" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "normalised = True if normalised.lower() == \"true\" else False" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -191,14 +208,18 @@ "\n", "for raw_data in file_list:\n", " # windowing happens here for each file\n", - " data = read_raw_data(raw_data)\n", + " data = read_raw_data(raw_data, normalised)\n", " \n", " # window it\n", " w_mca = window_mca(data[\"mca\"], lg_start, lg_end, data[\"scan_shape\"], data[\"scan_model\"])\n", " \n", - " # sum the I0\n", - " I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n", - " \n", + " if normalised:\n", + " # sum the I0\n", + " I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n", + " I0_total.append(I0_t)\n", + " else:\n", + " I0_total.append(np.ones_like(w_mca))\n", + "\n", " # record everything\n", " windowed.append(w_mca)\n", " \n", @@ -206,7 +227,6 @@ " SampleX.append(data[\"x\"])\n", " SampleY.append(data[\"y\"])\n", " scan_shapes.append(data[\"scan_shape\"])\n", - " I0_total.append(I0_t)\n", " \n", "print(f\"Time reading raw data: {(time.perf_counter() - start)/60:.2f} min\")" ] @@ -468,7 +488,8 @@ " tol=tol_residual, \n", " niter_short=num_short_iteration, \n", " niter_final=num_final_iteration, \n", - " verbose=True)" + " verbose=True) \n", + "loop_asd.residuals = np.full(sum(loop_asd.num_iters) + loop_asd.rank_max, np.nan) \n" ] }, { diff --git a/i14/schema/dpcSchema.json b/i14/schema/dpcSchema.json index 9bbd793..1b9c705 100644 --- a/i14/schema/dpcSchema.json +++ b/i14/schema/dpcSchema.json @@ -1,10 +1,13 @@ { "type": "object", "properties": { - "scanNumbers": { - "type": "string", - "title": "Scan Numbers", - "description": "A list containing scan numbers to be processed. Eg. [274317] or [274317, 274318, 274319]" + "multiScan": { + "title": "Scans", + "type": "array", + "items": + { + "type": "object" + } }, "outputFolder": { "type": "string", @@ -12,5 +15,5 @@ "description": "The full path of the output file" } }, - "required": ["scanNumbers", "outputFolder"] + "required": ["multiScan", "outputFolder"] } diff --git a/i14/schema/dpcUISchema.json b/i14/schema/dpcUISchema.json index 662c739..89c5e7a 100644 --- a/i14/schema/dpcUISchema.json +++ b/i14/schema/dpcUISchema.json @@ -3,7 +3,22 @@ "elements": [ { "type": "Control", - "scope": "#/properties/scanNumbers" + "scope": "#/properties/multiScan", + "options": { + "elementLabelProp": "scanRange", + "detail": { + "type": "VerticalLayout", + "elements": [ + { + "type": "Control", + "scope": "#/properties/multiScan/items", + "options": { + "useScanRangeControl": true + } + } + ] + } + } }, { "type": "Control", diff --git a/i14/schema/xanesPointSchema.json b/i14/schema/xanesPointSchema.json index 0dae244..7fc80ae 100644 --- a/i14/schema/xanesPointSchema.json +++ b/i14/schema/xanesPointSchema.json @@ -141,6 +141,7 @@ "normalise": { "type": "boolean", "title": "Normalise", + "default": true, "description": "Whether to normalise the windowed intensity by 'adc_ionCh1a'" } }, diff --git a/i14/schema/xanesSchema.json b/i14/schema/xanesSchema.json index 29758e5..9929b37 100644 --- a/i14/schema/xanesSchema.json +++ b/i14/schema/xanesSchema.json @@ -37,6 +37,7 @@ "normalise": { "type": "boolean", "title": "Normalise", + "default": true, "description": "Whether to normalise the stack by I0" } }, diff --git a/i14/schema/xanesSparseSchema.json b/i14/schema/xanesSparseSchema.json index 310151d..c98083c 100644 --- a/i14/schema/xanesSparseSchema.json +++ b/i14/schema/xanesSparseSchema.json @@ -16,7 +16,8 @@ }, "normalise": { "type": "boolean", - "title": "Normalise" + "title": "Normalise", + "default": true }, "edgeElement": { "type": "string", diff --git a/i14/schema/xrd1dSchema.json b/i14/schema/xrd1dSchema.json index b8f8422..277fe80 100644 --- a/i14/schema/xrd1dSchema.json +++ b/i14/schema/xrd1dSchema.json @@ -1,10 +1,13 @@ { "type": "object", "properties": { - "scanNumbers": { - "type": "string", - "title": "Scan Numbers", - "description": "A list containing scan numbers to be processed. Eg. [274317] or [274317, 274318, 274319]" + "multiScan": { + "title": "Scans", + "type": "array", + "items": + { + "type": "object" + } }, "outputFolder": { "type": "string", @@ -49,7 +52,7 @@ } }, "required": [ - "scanNumbers", + "multiScan", "outputFolder", "xrdProcessConfigUpload" ] diff --git a/i14/schema/xrd1dUISchema.json b/i14/schema/xrd1dUISchema.json index 9c54298..0d65740 100644 --- a/i14/schema/xrd1dUISchema.json +++ b/i14/schema/xrd1dUISchema.json @@ -3,7 +3,22 @@ "elements": [ { "type": "Control", - "scope": "#/properties/scanNumbers" + "scope": "#/properties/multiScan", + "options": { + "elementLabelProp": "scanRange", + "detail": { + "type": "VerticalLayout", + "elements": [ + { + "type": "Control", + "scope": "#/properties/multiScan/items", + "options": { + "useScanRangeControl": true + } + } + ] + } + } }, { "type": "Control", diff --git a/i14/schema/xrd2dSchema.json b/i14/schema/xrd2dSchema.json index 99b6760..e750a0f 100644 --- a/i14/schema/xrd2dSchema.json +++ b/i14/schema/xrd2dSchema.json @@ -1,10 +1,13 @@ { "type": "object", "properties": { - "scanNumbers": { - "type": "string", - "title": "Scan Numbers", - "description": "A list containing scan numbers to be processed. Eg. [274317] or [274317, 274318, 274319]" + "multiScan": { + "title": "Scans", + "type": "array", + "items": + { + "type": "object" + } }, "outputFolder": { "type": "string", @@ -28,7 +31,7 @@ } }, "required": [ - "scanNumbers", + "multiScan", "outputFolder", "xrdProcessConfigUpload" ] diff --git a/i14/schema/xrd2dUISchema.json b/i14/schema/xrd2dUISchema.json index 6b631c5..2467e73 100644 --- a/i14/schema/xrd2dUISchema.json +++ b/i14/schema/xrd2dUISchema.json @@ -3,7 +3,22 @@ "elements": [ { "type": "Control", - "scope": "#/properties/scanNumbers" + "scope": "#/properties/multiScan", + "options": { + "elementLabelProp": "scanRange", + "detail": { + "type": "VerticalLayout", + "elements": [ + { + "type": "Control", + "scope": "#/properties/multiScan/items", + "options": { + "useScanRangeControl": true + } + } + ] + } + } }, { "type": "Control", diff --git a/i14/schema/xrfTomoSchema.json b/i14/schema/xrfTomoSchema.json index 2441a12..d7417ec 100644 --- a/i14/schema/xrfTomoSchema.json +++ b/i14/schema/xrfTomoSchema.json @@ -52,6 +52,7 @@ "normalise": { "type": "boolean", "title": "Normalise", + "default": true, "description": "Whether to normalise the stack using the total mass of each projection." }, "cor": { diff --git a/i14/templates/align-template.yaml b/i14/templates/align-template.yaml index bb0f233..5c50830 100644 --- a/i14/templates/align-template.yaml +++ b/i14/templates/align-template.yaml @@ -2,10 +2,14 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: align + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: workflows.argoproj.io/title: Align data workflows.argoproj.io/description: | A utility which takes in a file path, a mask, aligns data according to vertical mass or phase correlation between images, and saves the results. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/alignSchema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | @@ -65,13 +69,25 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outpath }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb $OUTPUT/align-notebook.ipynb \ -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \ -p maskpath "{{`{{ inputs.parameters.maskpath }}`}}" \ -p complexpath "{{`{{ inputs.parameters.complexpath }}`}}" \ - -p outpath "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output align-notebook --output-dir "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" --to html --output align-output --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + -p outpath $OUTPUT \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/align-notebook.ipynb \ + --to html --output align-output \ + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -83,6 +99,10 @@ spec: path: "{{`{{ inputs.parameters.outpath }}`}}/align-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outpath }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/dpc-batch-template.yaml b/i14/templates/dpc-batch-template.yaml index f23c5e7..f5ab2c2 100644 --- a/i14/templates/dpc-batch-template.yaml +++ b/i14/templates/dpc-batch-template.yaml @@ -51,6 +51,12 @@ spec: mountPath: /tmp - name: dpc-notebook steps: + - - name: gen-multi-range + template: gen-multi-range-tmp + arguments: + parameters: + - name: multiScan + value: "{{`{{ workflow.parameters.multiScan }}`}}" - - name: dpc template: gen-notebook arguments: @@ -59,7 +65,29 @@ spec: value: "{{`{{ workflow.parameters.outputFolder }}`}}" - name: scanNumber value: "{{`{{ item }}`}}" - withParam: "{{`{{ workflow.parameters.scanNumbers }}`}}" + withParam: "{{`{{ steps.gen-multi-range.outputs.result }}`}}" + - name: gen-multi-range-tmp + inputs: + parameters: + - name: multiScan + script: + image: python:3.10 + command: [python] + source: | + import json, sys + file_number_list = [] + multiscan_list = {{`{{ inputs.parameters.multiScan }}`}} + + for multiscan in multiscan_list: + for range_ in multiscan.values(): + excluded = range_['excluded'] + assert isinstance(excluded, list) + assert range_['end'] >= range_['start'] + scans = [x for x in range(range_['start'], range_['end'] + 1) if x not in excluded] + file_number_list.append(scans) + + file_numbers = [n for _list in file_number_list for n in _list] + json.dump(file_numbers, sys.stdout) - name: gen-notebook inputs: parameters: @@ -72,13 +100,27 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb \ + $OUTPUT/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p process_configfile /tmp/config.yaml \ -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}_dpc-phase.nxs" \ - -p auto_processing False - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" --to html --no-input --output dpc-output-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + -p auto_processing False \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb \ + --to html --no-input \ + --output dpc-output-{{`{{ inputs.parameters.scanNumber }}`}} \ + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -90,6 +132,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-output-{{`{{ inputs.parameters.scanNumber }}`}}.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/mask-template.yaml b/i14/templates/mask-template.yaml index 8a56429..f1ffbc9 100644 --- a/i14/templates/mask-template.yaml +++ b/i14/templates/mask-template.yaml @@ -2,10 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: mask + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.argoproj.io/title: Generate a Mask workflows.argoproj.io/description: | A utility which takes in a file, such as one with complex or phase information, creates a mask, and saves the results. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/maskSchema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | @@ -73,15 +76,27 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outpath }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb $OUTPUT/mask-notebook.ipynb \ -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \ -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ -p padding "{{`{{ inputs.parameters.padding }}`}}" \ -p min_size "{{`{{ inputs.parameters.minSize }}`}}" \ -p erosion_radius "{{`{{ inputs.parameters.erosionRadius }}`}}" \ - -p dilation_radius "{{`{{ inputs.parameters.dilationRadius }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output mask-notebook --output-dir "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" --to html --output mask-output --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + -p dilation_radius "{{`{{ inputs.parameters.dilationRadius }}`}}" \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/mask-notebook.ipynb \ + --to html --output mask-output \ + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -97,6 +112,10 @@ spec: path: "{{`{{ inputs.parameters.outpath }}`}}/mask-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outpath }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/phase-template.yaml b/i14/templates/phase-template.yaml index dcdca73..842ae3e 100644 --- a/i14/templates/phase-template.yaml +++ b/i14/templates/phase-template.yaml @@ -2,10 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: phase + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.argoproj.io/title: Create phase data workflows.argoproj.io/description: | A utility which takes in a complex file, a mask, creates the phase data, and saves the results. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/phaseSchema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | @@ -24,7 +27,7 @@ spec: - name: maskpath value: "" - name: normalise - value: False + value: false - name: outpath value: "" volumeClaimTemplates: @@ -65,13 +68,26 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outpath }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb $OUTPUT/phase-notebook.ipynb \ -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \ -p maskpath "{{`{{ inputs.parameters.maskpath }}`}}" \ -p normalise "{{`{{ inputs.parameters.normalise }}`}}" \ - -p outpath "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output phase-notebook --output-dir "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" --to html --output phase-output --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/phase-notebook.ipynb \ + --to html \ + --output phase-output \ + --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -83,6 +99,10 @@ spec: path: "{{`{{ inputs.parameters.outpath }}`}}/phase-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outpath }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/tomo-job.yaml b/i14/templates/tomo-job.yaml index fffee59..fd93368 100644 --- a/i14/templates/tomo-job.yaml +++ b/i14/templates/tomo-job.yaml @@ -2,6 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptycho-tomo-job + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: entrypoint: ptycho-tomo-run templates: diff --git a/i14/templates/xanes-auto-template.yaml b/i14/templates/xanes-auto-template.yaml index baa9eb2..a8b076f 100644 --- a/i14/templates/xanes-auto-template.yaml +++ b/i14/templates/xanes-auto-template.yaml @@ -40,7 +40,7 @@ spec: - name: transitionToAlign value: None - name: normalise - value: "True" + value: true volumeClaimTemplates: - metadata: name: tmp @@ -128,7 +128,13 @@ spec: value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/ command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb \ + $OUTPUT/xanes-notebook.ipynb \ -p first "{{`{{ inputs.parameters.first }}`}}" \ -p last "{{`{{ inputs.parameters.last }}`}}" \ -p exclude "{{`{{ inputs.parameters.exclude }}`}}" \ @@ -142,9 +148,16 @@ spec: -p sztol 0.9 \ -p ref_index None \ -p max_fractional_shift 0.2 \ - -p auto_processing False - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xanes-notebook --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" --to html --output xanes-output --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" + -p auto_processing False \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/xanes-notebook.ipynb \ + --to html --output xanes-output \ + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -156,6 +169,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/xanes-point-template.yaml b/i14/templates/xanes-point-template.yaml index 57dfd21..4ea16c1 100644 --- a/i14/templates/xanes-point-template.yaml +++ b/i14/templates/xanes-point-template.yaml @@ -33,7 +33,7 @@ spec: - name: edgeTransition value: "Ka" - name: normalise - value: "True" + value: true volumeClaimTemplates: - metadata: name: tmp @@ -72,14 +72,26 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb $OUTPUT/xanes-point-notebook.ipynb \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ - -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}xanes_point.txt" \ + -p outpath "$OUTPUT/i14-{{`{{ inputs.parameters.scanNumber }}`}}xanes_point.txt" \ -p edge_element "{{`{{ inputs.parameters.edgeElement }}`}}" \ -p normalise "{{`{{ inputs.parameters.normalise }}`}}" \ - -p auto_processing False - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xanes-point-notebook --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" --to html --output xanes-point-output --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + -p auto_processing False \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/xanes-point-notebook.ipynb \ + --to html --output xanes-point-output \ + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -91,6 +103,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/xanes-sparse-template.yaml b/i14/templates/xanes-sparse-template.yaml index e6975d0..12d6129 100644 --- a/i14/templates/xanes-sparse-template.yaml +++ b/i14/templates/xanes-sparse-template.yaml @@ -34,7 +34,7 @@ spec: - name: edgeTransition value: "Ka" - name: normalise - value: "True" + value: true volumeClaimTemplates: - metadata: name: tmp @@ -48,7 +48,7 @@ spec: templates: - name: xanes-sparse-mount-files script: - image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.2 command: [bash] source: | echo '{{ .Files.Get "notebooks/xanes_sparse_stack_autoprocessing0.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb @@ -87,7 +87,7 @@ spec: - name: edgeElement - name: normalise script: - image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.2 env: - name: HOME value: /tmp @@ -95,7 +95,13 @@ spec: value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/ command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb \ + $OUTPUT/xanes-sparse-notebook.ipynb \ -p first "{{`{{ inputs.parameters.first }}`}}" \ -p last "{{`{{ inputs.parameters.last }}`}}" \ -p exclude "{{`{{ inputs.parameters.exclude }}`}}" \ @@ -108,9 +114,15 @@ spec: -p num_short_iteration 75 \ -p num_final_iteration 2000 \ -p normalised "{{`{{ inputs.parameters.normalise }}`}}" \ - -p auto_processing False - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xanes-sparse-notebook --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" --to html --output xanes-sparse-output --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" + -p auto_processing False \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/xanes-sparse-notebook.ipynb \ + --to html --output xanes-sparse-output + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -122,6 +134,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/xrd1d-batch-template.yaml b/i14/templates/xrd1d-batch-template.yaml index a83d2f1..2f1d7eb 100644 --- a/i14/templates/xrd1d-batch-template.yaml +++ b/i14/templates/xrd1d-batch-template.yaml @@ -61,6 +61,12 @@ spec: mountPath: /tmp - name: xrd1d-notebook steps: + - - name: gen-multi-range + template: gen-multi-range-tmp + arguments: + parameters: + - name: multiScan + value: "{{`{{ workflow.parameters.multiScan }}`}}" - - name: xrd1d template: gen-notebook arguments: @@ -81,7 +87,29 @@ spec: value: "{{`{{ workflow.parameters.backgroundType }}`}}" - name: polynomialOrder value: "{{`{{ workflow.parameters.polynomialOrder }}`}}" - withParam: "{{`{{ workflow.parameters.scanNumbers }}`}}" + withParam: "{{`{{ steps.gen-multi-range.outputs.result }}`}}" + - name: gen-multi-range-tmp + inputs: + parameters: + - name: multiScan + script: + image: python:3.10 + command: [python] + source: | + import json, sys + file_number_list = [] + multiscan_list = {{`{{ inputs.parameters.multiScan }}`}} + + for multiscan in multiscan_list: + for range_ in multiscan.values(): + excluded = range_['excluded'] + assert isinstance(excluded, list) + assert range_['end'] >= range_['start'] + scans = [x for x in range(range_['start'], range_['end'] + 1) if x not in excluded] + file_number_list.append(scans) + + file_numbers = [n for _list in file_number_list for n in _list] + json.dump(file_numbers, sys.stdout) - name: gen-notebook inputs: parameters: @@ -100,17 +128,33 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb \ + "$OUTPUT/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p process_configfile "{{`{{ inputs.parameters.xrdProcessConfigUpload }}`}}" \ - -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd1d.nxs" \ + -p outpath "$OUTPUT/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd1d.nxs" \ -p calibration_path "{{`{{ inputs.parameters.calibrationPath }}`}}" \ -p mask_path "{{`{{ inputs.parameters.maskPath }}`}}" \ -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" \ -p background_type "{{`{{ inputs.parameters.backgroundType }}`}}" \ - -p polynomial_order "{{`{{ inputs.parameters.polynomialOrder }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" --to html --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + -p polynomial_order "{{`{{ inputs.parameters.polynomialOrder }}`}}" \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert "$OUTPUT/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + --to html --output xanes-sparse-output + + python -m jupyter nbconvert "$OUTPUT/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + --to html --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ + --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -122,6 +166,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}}.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/xrd2d-batch-template.yaml b/i14/templates/xrd2d-batch-template.yaml index 69e74cd..cb3e5b4 100644 --- a/i14/templates/xrd2d-batch-template.yaml +++ b/i14/templates/xrd2d-batch-template.yaml @@ -56,6 +56,12 @@ spec: mountPath: /tmp - name: xrd2d-notebook steps: + - - name: gen-multi-range + template: gen-multi-range-tmp + arguments: + parameters: + - name: multiScan + value: "{{`{{ workflow.parameters.multiScan }}`}}" - - name: xrd2d template: gen-notebook arguments: @@ -72,7 +78,29 @@ spec: value: "{{`{{ workflow.parameters.maskPath }}`}}" - name: flatPath value: "{{`{{ workflow.parameters.flatPath }}`}}" - withParam: "{{`{{ workflow.parameters.scanNumbers }}`}}" + withParam: "{{`{{ steps.gen-multi-range.outputs.result }}`}}" + - name: gen-multi-range-tmp + inputs: + parameters: + - name: multiScan + script: + image: python:3.10 + command: [python] + source: | + import json, sys + file_number_list = [] + multiscan_list = {{`{{ inputs.parameters.multiScan }}`}} + + for multiscan in multiscan_list: + for range_ in multiscan.values(): + excluded = range_['excluded'] + assert isinstance(excluded, list) + assert range_['end'] >= range_['start'] + scans = [x for x in range(range_['start'], range_['end'] + 1) if x not in excluded] + file_number_list.append(scans) + + file_numbers = [n for _list in file_number_list for n in _list] + json.dump(file_numbers, sys.stdout) - name: gen-notebook inputs: parameters: @@ -89,15 +117,29 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb \ + "$OUTPUT/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p process_configfile "{{`{{ inputs.parameters.xrdProcessConfigUpload }}`}}" \ - -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd2d.nxs" \ + -p outpath "$OUTPUT/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd2d.nxs" \ -p calibration_path "{{`{{ inputs.parameters.calibrationPath }}`}}" \ -p mask_path "{{`{{ inputs.parameters.maskPath }}`}}" \ - -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" --to html --no-input --output xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert "$OUTPUT/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + --to html --no-input \ + --output xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -109,6 +151,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}}.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/nxstacker/schema/nxstackerSchema.json b/nxstacker/schema/nxstackerSchema.json index b65fd15..c9c3351 100644 --- a/nxstacker/schema/nxstackerSchema.json +++ b/nxstacker/schema/nxstackerSchema.json @@ -14,11 +14,13 @@ "quiet": { "type": "boolean", "title": "Quiet", + "default": false, "description": "Suppress log messages" }, "dryRun": { "type": "boolean", - "title": "Perform a dry run" + "title": "Perform a dry run", + "default": false }, "projDir": { "type": "string", @@ -93,21 +95,25 @@ "sortByAngle": { "type": "boolean", "title": "Sort by angle", + "default": false, "description": "Sort the projections by their rotation angles" }, "padToMax": { "type": "boolean", "title": "Pad to max", + "default": false, "description": "Pad projection to the maximum size of the stack" }, "compress": { "type": "boolean", "title": "Compress", + "default": false, "description": "Compress the NXtomo file" }, "skipCheck": { "type": "boolean", "title": "Skip check", + "default": false, "description": "Skip the validation of the projection file" }, "retrievalMethod": { @@ -251,36 +257,44 @@ }, "saveComplex": { "type": "boolean", - "title": "Save complex" + "title": "Save complex", + "default": false }, "saveModulus": { "type": "boolean", - "title": "Save modulus" + "title": "Save modulus", + "default": false }, "savePhase": { "type": "boolean", - "title": "Save phase" + "title": "Save phase", + "default": false }, "removeRamp": { "type": "boolean", - "title": "Remove ramp" + "title": "Remove ramp", + "default": false }, "medianNorm": { "type": "boolean", "title": "Median norm", - "description": "Normalise the phase by shifting its median" + "description": "Normalise the phase by shifting its median", + "default": false }, "unwrapPhase": { "type": "boolean", - "title": "Unwrap phase" + "title": "Unwrap phase", + "default": false }, "rescale": { "type": "boolean", - "title": "Rescale" + "title": "Rescale", + "default": false }, "ignoreRaw": { "type": "boolean", - "title": "Ignore Raw" + "title": "Ignore Raw", + "default": false } }, "required": ["experimentType"] diff --git a/nxstacker/templates/nxstacker-template.yaml b/nxstacker/templates/nxstacker-template.yaml index ab05f4f..14fb331 100644 --- a/nxstacker/templates/nxstacker-template.yaml +++ b/nxstacker/templates/nxstacker-template.yaml @@ -2,10 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: nxstacker + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.argoproj.io/title: nxstacker workflows.argoproj.io/description: | nxstacker is an utility to stack projections from different types of experiments to produce NeXus-compliance file(s). + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/nxstacker" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/nxstackerSchema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | diff --git a/ptypy/config/p99_dm.yaml b/ptypy/config/p99_dm.yaml new file mode 100755 index 0000000..aca07b4 --- /dev/null +++ b/ptypy/config/p99_dm.yaml @@ -0,0 +1,82 @@ +base_file: null +visit_path: +parameter_tree: + run: '' + verbose_level: 3 + frames_per_block: 1000 + data_type: single + dry_run: false + engines: + engine0: + name: DM + numiter: 100 + numiter_contiguous: 10 + alpha: 0.95 + probe_support: null + probe_update_start: 0 + update_object_first: false + object_inertia: 0.001 + probe_inertia: 0.001 + overlap_converge_factor: 0.01 + overlap_max_iterations: 10 + obj_smooth_std: 20 + fourier_power_bound: 0.25 + io: + autoplot: + active: false + autosave: + active: false + interval: 100 + rformat: dls + scans: + scan_00: + name: BlockFull + coherence: + num_object_modes: 1 + num_probe_modes: 1 + sample: + diversity: null + model: null + process: null + illumination: + model: null + photons: null + aperture: + form: circ + size: 0.000333 + propagation: + antialiasing: 1 + focussed: 0.013725 + parallel: 50.e-06 + diversity: + noise: + - 0.5 + - 1.0 + power: 0.1 + data: + name: Hdf5Loader + orientation: 0 + psize: 11.e-06 + rebin: 1 + shape: + - 128 + - 128 + auto_center: true + save: null + intensities: + file: /%(visit_path)s/p99-%(run)s.nxs + key: /entry/andor2_det/andor2_det + positions: + file: /%(visit_path)s/p99-%(run)s.nxs + fast_key: /entry/andor2_det/sample_stage-x + fast_multiplier: 0.001 + slow_key: /entry/andor2_det/sample_stage-y + slow_multiplier: 0.001 + recorded_energy: + file: /%(visit_path)s/p99-%(run)s.nxs + key: /entry/plan_metadata/energy/value + multiplier: 0.001 + recorded_distance: + file: /%(visit_path)s/p99-%(run)s.nxs + key: /entry/plan_metadata/detector_dist/value + multiplier: 0.001 diff --git a/ptypy/notebooks/ptypy_plotting.ipynb b/ptypy/notebooks/ptypy_plotting.ipynb new file mode 100644 index 0000000..882058d --- /dev/null +++ b/ptypy/notebooks/ptypy_plotting.ipynb @@ -0,0 +1,359 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "d36dc2c0", + "metadata": { + "tags": [ + "parameters" + ] + }, + "outputs": [], + "source": [ + "recon_file = \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59a4658d", + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "import h5py\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import ptypy" + ] + }, + { + "cell_type": "markdown", + "id": "be4c5e89", + "metadata": {}, + "source": [ + "### Set locations for artefacts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c8d26f8", + "metadata": {}, + "outputs": [], + "source": [ + "ARTEFACTS = {\n", + " 'amplitude': '/tmp/amplitude_plot.png',\n", + " 'phase': '/tmp/phase_plot.png',\n", + " 'probe_amp': '/tmp/probe_amplitude_plot.png',\n", + " 'probe_phase': '/tmp/probe_phase_plot.png',\n", + " 'errors': '/tmp/error_plot.png',\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24d4d677", + "metadata": {}, + "outputs": [], + "source": [ + "if not os.path.exists(recon_file):\n", + " raise ValueError(f\"{recon_file} does not exist!\")" + ] + }, + { + "cell_type": "markdown", + "id": "8013201f", + "metadata": {}, + "source": [ + "## Load reconstruction from ptyr file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca78c318", + "metadata": {}, + "outputs": [], + "source": [ + "with h5py.File(recon_file, \"r\") as f:\n", + " pixelsize = f['content/obj/Sscan_00G00/_psize'][0]*1e9\n", + " probe = np.array(f['content/probe/Sscan_00G00/data'][:])\n", + " obj = np.array(f['content/obj/Sscan_00G00/data'][0,:,:])\n", + " Niter = len(f['content/runtime/iter_info/'])\n", + " errors = np.array([f[\"content/runtime/iter_info/%05d/error\" %i][:] for i in range(Niter)])\n", + " niters = np.array([f[\"content/runtime/iter_info/%05d/iteration\" %i][...] for i in range(Niter)])\n", + " energy = f[\"content/probe/Sscan_00G00/_energy\"][...]\n", + " wavelength = 1.239/energy\n", + "e, v = ptypy.utils.ortho(probe)\n", + "print(\"Pixelsize = %.3f nm\" %(pixelsize))\n", + "print(\"Photon energy = %f\" %(energy*1000))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b155ec18", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(8,4), dpi=100)\n", + "plt.plot(niters, errors[:,1])\n", + "plt.ylabel(\"Log-likelihood\")\n", + "plt.xlabel(\"Nr. of iterations\")\n", + "plt.savefig(ARTEFACTS['errors'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "b7f5117a", + "metadata": {}, + "source": [ + "## Plot amplitude and phase histograms" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e39d83e", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(np.abs(obj).flatten(), bins=100)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "369868bc", + "metadata": {}, + "outputs": [], + "source": [ + "plt.hist(np.angle(obj).flatten(), bins=100)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "6bca57ea", + "metadata": {}, + "source": [ + "## Plot amplitude" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "611cf3c1", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(6,5),dpi=100)\n", + "plt.axis(\"off\")\n", + "plt.imshow(np.abs(obj), cmap=\"gray\", interpolation='none',vmin=0, vmax=5)\n", + "plt.gca().add_patch(plt.Rectangle((10,10),1000/pixelsize,5, color='w'))\n", + "plt.gca().text(10+1000/pixelsize/2,20,r\"%d um\" %(1000/1e3), color='w', va='top', ha='center', fontsize=15)\n", + "plt.colorbar()\n", + "plt.savefig(ARTEFACTS['amplitude'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "43da45cb", + "metadata": {}, + "source": [ + "## Plot phase" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "beaacd77", + "metadata": {}, + "outputs": [], + "source": [ + "plt.figure(figsize=(6,5), dpi=100)\n", + "plt.axis(\"off\")\n", + "plt.imshow(np.angle(obj), cmap=\"gray\", vmin=-1*np.pi, vmax=1*np.pi, interpolation='none')\n", + "plt.gca().add_patch(plt.Rectangle((10,10),1000/pixelsize,5, color='w'))\n", + "plt.gca().text(10+1000/pixelsize/2,20,r\"%d μm\" %(1000/1e3), color='w', va='top', ha='center', fontsize=15)\n", + "plt.colorbar()\n", + "plt.savefig(ARTEFACTS['phase'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "0e21559c", + "metadata": {}, + "source": [ + "## Plot probe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "936aca8a", + "metadata": {}, + "outputs": [], + "source": [ + "m = 0\n", + "fig = plt.figure(figsize=(5,5))\n", + "ax = ptypy.utils.PtyAxis(fig.add_subplot(111), channel='a', vmin=None, vmax=None)\n", + "ax.set_data(np.abs(v[m]))\n", + "ax.ax.axis(\"off\")\n", + "ax.ax.add_patch(plt.Rectangle((5,5),1000/pixelsize,10, color='w'))\n", + "ax.ax.text(5+1000/pixelsize/2,20,r\"%d μm\" %(1000/1e3), color='w', va='top', ha='center', fontsize=20)\n", + "ax.ax.set_title(f\"Probe Amplitude | {e[m]*100:.1f} %\")\n", + "fig.savefig(ARTEFACTS['probe_amp'])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "63c74baa", + "metadata": {}, + "outputs": [], + "source": [ + "m = 0\n", + "fig = plt.figure(figsize=(5,5))\n", + "ax = ptypy.utils.PtyAxis(fig.add_subplot(111), channel='a', vmin=None, vmax=None)\n", + "ax.set_data(np.angle(v[m]))\n", + "ax.ax.set_title(f\"Probe Phase | {e[m]*100:.1f}\")\n", + "ax.ax.axis(\"off\")\n", + "fig.savefig(ARTEFACTS['probe_phase'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "7ab09476", + "metadata": {}, + "source": [ + "## Plot propagation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0aa87921", + "metadata": {}, + "outputs": [], + "source": [ + "import ptypy.utils as u\n", + "from ptypy.core import geometry\n", + "\n", + "def nfprop(energy, distance, resolution, shape):\n", + " g = u.Param()\n", + " g.energy = energy\n", + " g.distance = distance\n", + " g.psize = resolution\n", + " g.shape = shape\n", + " g.propagation = \"nearfield\"\n", + " G = geometry.Geo(owner=None, pars=g)\n", + " return G.propagator.fw\n", + "\n", + "def propagated(probe, distances, energy, resolution, eps=1e-9):\n", + " prop_array = np.zeros((len(distances),)+probe.shape, dtype=probe.dtype)\n", + " for i,d in enumerate(distances):\n", + " if np.abs(d) < eps:\n", + " prop_array[i] = probe.copy()\n", + " else:\n", + " prop_array[i] = nfprop(energy, d, resolution, probe.shape)(probe)\n", + " return prop_array" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad7d8878", + "metadata": {}, + "outputs": [], + "source": [ + "prb = v[0]\n", + "full_defocus_range = np.linspace(-50e-6, 50e-6, 201)\n", + "main_probe = propagated(prb, full_defocus_range, energy, pixelsize*1e-9)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7ade75fc", + "metadata": {}, + "outputs": [], + "source": [ + "cz,cy,cx = np.unravel_index(np.abs(main_probe).argmax(),main_probe.shape)\n", + "ny,nx = prb.shape\n", + "fz,fy,fx = 100,cy,cx" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6fa9ef6", + "metadata": {}, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(12,4),dpi=100)\n", + "axy = fig.add_subplot(211)\n", + "axx = fig.add_subplot(212)\n", + "axz = fig.add_axes([-0.2,0.025, 0.3,0.85])\n", + "\n", + "Axz = ptypy.utils.PtyAxis(axz, channel='a', vmax=0.25*np.abs(main_probe).max())\n", + "Axz.set_data(main_probe[fz])\n", + "axz.axvline(fx, color='w', ls=':')\n", + "axz.axhline(fy, color='w', ls=':')\n", + "axz.axis('off')\n", + "axz.add_patch(plt.Rectangle((5,5),1000/pixelsize,10, color='w'))\n", + "axz.text(5+1000/pixelsize/2,20,r\"%d μm\" %(1000/1e3), color='w', va='top', ha='center', fontsize=10)\n", + "\n", + "Axy = ptypy.utils.PtyAxis(axy, channel='a', vmax=0.5*np.abs(main_probe).max())\n", + "Axy.set_data(main_probe[:,fy].T)\n", + "axy.set_aspect('auto')\n", + "axy.axvline(fz, color='w', ls=':')\n", + "axy.axvline(cz, color='r', ls=':')\n", + "axy.set_xticks([])\n", + "axy.set_yticks(range(ny)[5::50])\n", + "axy.set_yticklabels(np.round(np.linspace(-cy*pixelsize/1e3, cy*pixelsize/1e3,nx)[5::50],1))\n", + "axy.tick_params('y', right=True, left=False, labelright=True, labelleft=False)\n", + "axy.set_ylabel(\"horiz. position [μm]\")\n", + "axy.yaxis.set_label_position(\"right\")\n", + "\n", + "Axx = ptypy.utils.PtyAxis(axx, channel='a', vmax=0.5*np.abs(main_probe).max())\n", + "Axx.set_data(main_probe[:,:,fx].T)\n", + "axx.axvline(fz, color='w', ls=':')\n", + "axx.axvline(cz, color='r', ls=':')\n", + "axx.set_aspect('auto')\n", + "axx.set_xticks(range(len(full_defocus_range))[::20])\n", + "axx.set_xticklabels(np.round(full_defocus_range[::20] * 1e6).astype(int))\n", + "axx.set_yticks(range(ny)[5::50])\n", + "axx.set_yticklabels(np.round(np.linspace(-cx*pixelsize/1e3, cx*pixelsize/1e3,nx)[5::50],1))\n", + "axx.tick_params('y', right=True, left=False, labelright=True, labelleft=False)\n", + "axx.set_xlabel(\"Defocus [μm]\")\n", + "axx.set_ylabel(\"vert. position [μm]\")\n", + "axx.yaxis.set_label_position(\"right\")\n", + "\n", + "plt.show()" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cuda12.6", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "version": "3.11.10" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ptypy/templates/ptypy-cpu-moonflower-test.yaml b/ptypy/templates/ptypy-cpu-moonflower-test.yaml index b780ad7..d0a6ff4 100644 --- a/ptypy/templates/ptypy-cpu-moonflower-test.yaml +++ b/ptypy/templates/ptypy-cpu-moonflower-test.yaml @@ -2,6 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptypy-cpu-moonflower-benchmark + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: entrypoint: ptypy-run volumeClaimTemplates: diff --git a/ptypy/templates/ptypy-cpu-workflow-template.yaml b/ptypy/templates/ptypy-cpu-workflow-template.yaml index 6f3ea2f..9f2defa 100644 --- a/ptypy/templates/ptypy-cpu-workflow-template.yaml +++ b/ptypy/templates/ptypy-cpu-workflow-template.yaml @@ -2,10 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptypy-cpu-job + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.argoproj.io/title: PtyPy CPU Job Submitter workflows.argoproj.io/description: | Runs a PtyPy reconstruction job inside a container. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema.config: | {{- .Files.Get "schema/ptypy_moonflower.json" | nindent 6 }} workflows.diamond.ac.uk/parameter-schema.id: | diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml new file mode 100644 index 0000000..e95f315 --- /dev/null +++ b/ptypy/templates/ptypy-generic.yaml @@ -0,0 +1,153 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: ptypy-generic + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" +spec: + entrypoint: ptypy-gpu-run + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + storageClassName: netapp + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory + templates: + - name: ptypy-gpu-run + inputs: + parameters: + - name: config + - name: id + - name: outpath + - name: nprocs + value: 1 + - name: memory + value: 20Gi + script: + image: ghcr.io/ptycho/ptypy_cupy_openmpi:0.9.1 + env: + - name: CUPY_CACHE_DIR + value: /tmp/.cupy/kernel_cache + - name: MKL_NUM_THREADS + value: "1" + - name: NUMEXPR_NUM_THREADS + value: "1" + - name: OMP_NUM_THREADS + value: "1" + command: [bash] + source: | + mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ + ptypy.cli \ + -f {{`{{ inputs.parameters.config }}`}} \ + -i {{`{{ inputs.parameters.id }}`}} \ + -o {{`{{ inputs.parameters.outpath }}`}} \ + -s hdf5_loader \ + -b cupy + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmpdir + mountPath: /tmp + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: "{{`{{ inputs.parameters.nprocs }}`}}" + memory: "{{`{{ inputs.parameters.memory }}`}}" + nvidia.com/gpu: "{{`{{ inputs.parameters.nprocs }}`}}" + limits: + cpu: "{{`{{ inputs.parameters.nprocs }}`}}" + memory: "{{`{{ inputs.parameters.memory }}`}}" + nvidia.com/gpu: "{{`{{ inputs.parameters.nprocs }}`}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: nodetype + operator: Equal + value: gpu + effect: NoSchedule + - key: nodegroup + operator: Equal + value: workflows + effect: NoSchedule + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory + - name: ptypy-cpu-run + inputs: + parameters: + - name: config + - name: id + - name: outpath + - name: nprocs + value: 1 + - name: memory + value: 20Gi + script: + image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 + env: + - name: MKL_NUM_THREADS + value: "1" + - name: NUMEXPR_NUM_THREADS + value: "1" + - name: OMP_NUM_THREADS + value: "1" + command: [bash] + source: | + mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ + ptypy.cli \ + -f {{`{{ inputs.parameters.config }}`}} \ + -i {{`{{ inputs.parameters.id }}`}} \ + -o {{`{{ inputs.parameters.outpath }}`}} \ + -s hdf5_loader + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmpdir + mountPath: /tmp + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: "{{`{{ inputs.parameters.nprocs }}`}}" + memory: "{{`{{ inputs.parameters.memory }}`}}" + limits: + cpu: "{{`{{ inputs.parameters.nprocs }}`}}" + memory: "{{`{{ inputs.parameters.memory }}`}}" + tolerations: + - key: nodegroup + operator: Equal + value: workflows + effect: NoSchedule + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: nodegroup + operator: In + values: + - workflows + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory diff --git a/ptypy/templates/ptypy-gpu-moonflower-test.yaml b/ptypy/templates/ptypy-gpu-moonflower-test.yaml index 8592c9b..9862391 100644 --- a/ptypy/templates/ptypy-gpu-moonflower-test.yaml +++ b/ptypy/templates/ptypy-gpu-moonflower-test.yaml @@ -2,6 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptypy-gpu-moonflower-benchmark + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: entrypoint: ptypy-run volumeClaimTemplates: diff --git a/ptypy/templates/ptypy-gpu-workflow-template.yaml b/ptypy/templates/ptypy-gpu-workflow-template.yaml index 3f074d5..629fc73 100644 --- a/ptypy/templates/ptypy-gpu-workflow-template.yaml +++ b/ptypy/templates/ptypy-gpu-workflow-template.yaml @@ -2,6 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptypy-gpu-job + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: entrypoint: ptypy-run volumeClaimTemplates: diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml new file mode 100644 index 0000000..bab3b5e --- /dev/null +++ b/ptypy/templates/ptypy-p99.yaml @@ -0,0 +1,199 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: ptypy-p99-from-config + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.argoproj.io/title: PtyPy Reconstruction for P99 + workflows.argoproj.io/description: | + Runs a PtyPy reconstruction job inside a container based on data collected at P99. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" + workflows.diamond.ac.uk/parameter-schema.id: | + { + "title": "Scan Number", + "type": "integer" + } + workflows.diamond.ac.uk/parameter-schema.inpath: | + { + "title": "Path to raw data folder", + "type": "string", + "default": "processing/writenData/reconstruction_test_data" + } + workflows.diamond.ac.uk/parameter-schema.outpath: | + { + "title": "Path to output folder", + "type": "string", + "default": "processing/workflows/ptypy" + } + workflows.diamond.ac.uk/parameter-schema.nprocs: | + { + "title": "Nr. of processes", + "type": "integer", + "default": 1 + } + workflows.diamond.ac.uk/parameter-schema.memory: | + { + "title": "Requested CPU memory", + "type": "string", + "pattern": "^[0-9]+[GMK]i$", + "default": "20Gi" + } + workflows.diamond.ac.uk/parameter-schema.usegpu: | + { + "title": "Use GPU", + "type": "boolean", + "default": false + } +spec: + entrypoint: ptypy-pipeline + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory + - name: id + value: 7 + - name: inpath + value: "processing/writenData/reconstruction_test_data" + - name: outpath + value: "processing/workflows/ptypy" + - name: nprocs + value: 1 + - name: memory + value: 20Gi + - name: usegpu + value: false + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + storageClassName: netapp + + templates: + - name: ptypy-setup + script: + image: python:3.14 + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmpdir + mountPath: /tmp + command: [bash] + source: | + OUTPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.outpath }}`}}" + mkdir -p $OUTPUT + INPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.inpath }}`}}" + echo '{{ .Files.Get "config/p99_dm.yaml" | b64enc }}' | base64 -d > $OUTPUT/ptypy_config.yaml + sed -i "s;;$INPUT;" $OUTPUT/ptypy_config.yaml + "{{`{{ workflow.parameters.usegpu }}`}}" && sed -i "s;DM;DM_cupy;" $OUTPUT/ptypy_config.yaml + cat $OUTPUT/ptypy_config.yaml + outputs: + parameters: + - name: config + value: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/ptypy_config.yaml" + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory + + - name: get-filepath + inputs: + parameters: + - name: config + script: + image: python:3.14 + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmpdir + mountPath: /tmp + command: [python] + source: | + import re + OUTPATH="{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}" + SCAN="{{`{{ workflow.parameters.id }}`}}" + CONFIG="{{`{{ inputs.parameters.config }}`}}" + with open("/tmp/out.txt", "w") as o: + with open(CONFIG, 'r') as f: + text = f.read() + ENGINE = re.findall(" engine0:\n name: (.*)", text)[0] + NITER = int(re.findall(" numiter: (.*)", text)[0]) + o.write(f"{OUTPATH}/scan_{SCAN}/{SCAN}_{ENGINE}_{NITER:04d}.ptyr") + outputs: + parameters: + - name: filepath + valueFrom: + path: /tmp/out.txt + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory + - name: ptypy-pipeline + dag: + tasks: + - name: prepare + template: ptypy-setup + - name: reconstruct-with-gpu + when: "{{`{{ workflow.parameters.usegpu }}`}} == true" + templateRef: + name: ptypy-generic + template: ptypy-gpu-run + clusterScope: true + depends: "prepare" + arguments: + parameters: + - name: config + value: "{{`{{ tasks.prepare.outputs.parameters.config }}`}}" + - name: id + value: "{{`{{ workflow.parameters.id }}`}}" + - name: outpath + value: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}" + - name: nprocs + value: "{{`{{ workflow.parameters.nprocs }}`}}" + - name: memory + value: "{{`{{ workflow.parameters.memory }}`}}" + - name: reconstruct-with-cpu + when: "{{`{{ workflow.parameters.usegpu }}`}} == false" + templateRef: + name: ptypy-generic + template: ptypy-cpu-run + clusterScope: true + depends: "prepare" + arguments: + parameters: + - name: config + value: "{{`{{ tasks.prepare.outputs.parameters.config }}`}}" + - name: id + value: "{{`{{ workflow.parameters.id }}`}}" + - name: outpath + value: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}" + - name: nprocs + value: "{{`{{ workflow.parameters.nprocs }}`}}" + - name: memory + value: "{{`{{ workflow.parameters.memory }}`}}" + - name: get-filepath + template: get-filepath + depends: "reconstruct-with-gpu || reconstruct-with-cpu" + arguments: + parameters: + - name: config + value: "{{`{{ tasks.prepare.outputs.parameters.config }}`}}" + - name: plot + templateRef: + name: ptypy-utils + template: ptypy-plotting + clusterScope: true + depends: "get-filepath" + arguments: + parameters: + - name: filepath + value: "{{`{{tasks.get-filepath.outputs.parameters.filepath}}`}}" diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml new file mode 100644 index 0000000..b42ffe8 --- /dev/null +++ b/ptypy/templates/ptypy-utils.yaml @@ -0,0 +1,109 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: ptypy-utils + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" +spec: + entrypoint: ptypy-plotting + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + storageClassName: netapp + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory + templates: + - name: ptypy-plotting + inputs: + parameters: + - name: filepath + script: + image: ghcr.io/diamondlightsource/imaging_toolbox/ptypy_workflows:v0.3.3 + env: + - name: MKL_NUM_THREADS + value: "1" + - name: NUMEXPR_NUM_THREADS + value: "1" + - name: OMP_NUM_THREADS + value: "1" + command: [bash] + source: | + # mount notebook + echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb + # run notebook + python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb \ + -p recon_file "{{`{{ inputs.parameters.filepath }}`}}" + python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmpdir + mountPath: /tmp + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + - name: amplitude + path: /tmp/amplitude_plot.png + archive: + none: {} + - name: phase + path: /tmp/phase_plot.png + archive: + none: {} + - name: probe-amplitude + path: /tmp/probe_amplitude_plot.png + archive: + none: {} + - name: probe-phase + path: /tmp/probe_phase_plot.png + archive: + none: {} + - name: errors + path: /tmp/error_plot.png + archive: + none: {} + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: 1 + memory: 20Gi + limits: + cpu: 1 + memory: 20Gi + tolerations: + - key: nodegroup + operator: Equal + value: workflows + effect: NoSchedule + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: nodegroup + operator: In + values: + - workflows + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory