From 4b4a73265ff14071597fb000ea1d9bb7e67db28a Mon Sep 17 00:00:00 2001 From: David Hadley Date: Wed, 13 May 2026 08:59:15 +0100 Subject: [PATCH 01/51] fix: workflow linter errors --- e02/templates/mib2x-auto.yaml | 1 + e02/templates/mib2x.yaml | 1 + e02/templates/ptyrex_basic.yaml | 1 + httomo/templates/cor-sweep.yaml | 2 ++ httomo/templates/extract-raw-projections.yaml | 2 ++ httomo/templates/httomo-gpu-template-test.yaml | 2 ++ i14/templates/align-template.yaml | 4 ++++ i14/templates/mask-template.yaml | 3 +++ i14/templates/phase-template.yaml | 3 +++ i14/templates/tomo-job.yaml | 4 ++++ nxstacker/templates/nxstacker-template.yaml | 3 +++ ptypy/templates/ptypy-cpu-moonflower-test.yaml | 4 ++++ ptypy/templates/ptypy-cpu-workflow-template.yaml | 3 +++ ptypy/templates/ptypy-gpu-moonflower-test.yaml | 4 ++++ ptypy/templates/ptypy-gpu-workflow-template.yaml | 4 ++++ 15 files changed, 41 insertions(+) diff --git a/e02/templates/mib2x-auto.yaml b/e02/templates/mib2x-auto.yaml index 806ce0a..0af7a80 100644 --- a/e02/templates/mib2x-auto.yaml +++ b/e02/templates/mib2x-auto.yaml @@ -8,6 +8,7 @@ metadata: workflows.argoproj.io/title: ePSIC mib automatic conversion workflows.argoproj.io/description: | Convert MIB file to hdf5/hspy files + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/mib2x_auto_Schema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | diff --git a/e02/templates/mib2x.yaml b/e02/templates/mib2x.yaml index 479f37a..de14d6a 100644 --- a/e02/templates/mib2x.yaml +++ b/e02/templates/mib2x.yaml @@ -8,6 +8,7 @@ metadata: workflows.argoproj.io/title: ePSIC mib conversion workflows.argoproj.io/description: | Convert MIB file to hdf5/hspy files + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema.mib_path: | { "type": "string" diff --git a/e02/templates/ptyrex_basic.yaml b/e02/templates/ptyrex_basic.yaml index aabf085..4ebb961 100644 --- a/e02/templates/ptyrex_basic.yaml +++ b/e02/templates/ptyrex_basic.yaml @@ -8,6 +8,7 @@ metadata: workflows.argoproj.io/title: Simple Ptyrex job submission workflows.argoproj.io/description: | Submit a Ptyrex job for a particular scan and projection within a given visit + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/ptyrex_Schema.json" | nindent 6}} workflows.diamond.ac.uk/ui-schema: | diff --git a/httomo/templates/cor-sweep.yaml b/httomo/templates/cor-sweep.yaml index 990c885..675996c 100644 --- a/httomo/templates/cor-sweep.yaml +++ b/httomo/templates/cor-sweep.yaml @@ -2,6 +2,8 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: httomo-cor-sweep + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema.input: | diff --git a/httomo/templates/extract-raw-projections.yaml b/httomo/templates/extract-raw-projections.yaml index ebb1de0..6770c8f 100644 --- a/httomo/templates/extract-raw-projections.yaml +++ b/httomo/templates/extract-raw-projections.yaml @@ -2,6 +2,8 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: extract-raw-projections + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: diff --git a/httomo/templates/httomo-gpu-template-test.yaml b/httomo/templates/httomo-gpu-template-test.yaml index a8be1c9..ae8b649 100644 --- a/httomo/templates/httomo-gpu-template-test.yaml +++ b/httomo/templates/httomo-gpu-template-test.yaml @@ -2,6 +2,8 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: httomo-gpu-job + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: diff --git a/i14/templates/align-template.yaml b/i14/templates/align-template.yaml index bb0f233..c7dcfe3 100644 --- a/i14/templates/align-template.yaml +++ b/i14/templates/align-template.yaml @@ -2,10 +2,14 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: align + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: workflows.argoproj.io/title: Align data workflows.argoproj.io/description: | A utility which takes in a file path, a mask, aligns data according to vertical mass or phase correlation between images, and saves the results. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/alignSchema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | diff --git a/i14/templates/mask-template.yaml b/i14/templates/mask-template.yaml index 8a56429..962685a 100644 --- a/i14/templates/mask-template.yaml +++ b/i14/templates/mask-template.yaml @@ -2,10 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: mask + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.argoproj.io/title: Generate a Mask workflows.argoproj.io/description: | A utility which takes in a file, such as one with complex or phase information, creates a mask, and saves the results. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/maskSchema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | diff --git a/i14/templates/phase-template.yaml b/i14/templates/phase-template.yaml index dcdca73..46b0ad2 100644 --- a/i14/templates/phase-template.yaml +++ b/i14/templates/phase-template.yaml @@ -2,10 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: phase + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.argoproj.io/title: Create phase data workflows.argoproj.io/description: | A utility which takes in a complex file, a mask, creates the phase data, and saves the results. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/phaseSchema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | diff --git a/i14/templates/tomo-job.yaml b/i14/templates/tomo-job.yaml index fffee59..fd93368 100644 --- a/i14/templates/tomo-job.yaml +++ b/i14/templates/tomo-job.yaml @@ -2,6 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptycho-tomo-job + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: entrypoint: ptycho-tomo-run templates: diff --git a/nxstacker/templates/nxstacker-template.yaml b/nxstacker/templates/nxstacker-template.yaml index ab05f4f..14fb331 100644 --- a/nxstacker/templates/nxstacker-template.yaml +++ b/nxstacker/templates/nxstacker-template.yaml @@ -2,10 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: nxstacker + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.argoproj.io/title: nxstacker workflows.argoproj.io/description: | nxstacker is an utility to stack projections from different types of experiments to produce NeXus-compliance file(s). + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/nxstacker" workflows.diamond.ac.uk/parameter-schema: | {{- .Files.Get "schema/nxstackerSchema.json" | nindent 6 }} workflows.diamond.ac.uk/ui-schema: | diff --git a/ptypy/templates/ptypy-cpu-moonflower-test.yaml b/ptypy/templates/ptypy-cpu-moonflower-test.yaml index b780ad7..d0a6ff4 100644 --- a/ptypy/templates/ptypy-cpu-moonflower-test.yaml +++ b/ptypy/templates/ptypy-cpu-moonflower-test.yaml @@ -2,6 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptypy-cpu-moonflower-benchmark + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: entrypoint: ptypy-run volumeClaimTemplates: diff --git a/ptypy/templates/ptypy-cpu-workflow-template.yaml b/ptypy/templates/ptypy-cpu-workflow-template.yaml index 6f3ea2f..9f2defa 100644 --- a/ptypy/templates/ptypy-cpu-workflow-template.yaml +++ b/ptypy/templates/ptypy-cpu-workflow-template.yaml @@ -2,10 +2,13 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptypy-cpu-job + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" annotations: workflows.argoproj.io/title: PtyPy CPU Job Submitter workflows.argoproj.io/description: | Runs a PtyPy reconstruction job inside a container. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" workflows.diamond.ac.uk/parameter-schema.config: | {{- .Files.Get "schema/ptypy_moonflower.json" | nindent 6 }} workflows.diamond.ac.uk/parameter-schema.id: | diff --git a/ptypy/templates/ptypy-gpu-moonflower-test.yaml b/ptypy/templates/ptypy-gpu-moonflower-test.yaml index 8592c9b..9862391 100644 --- a/ptypy/templates/ptypy-gpu-moonflower-test.yaml +++ b/ptypy/templates/ptypy-gpu-moonflower-test.yaml @@ -2,6 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptypy-gpu-moonflower-benchmark + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: entrypoint: ptypy-run volumeClaimTemplates: diff --git a/ptypy/templates/ptypy-gpu-workflow-template.yaml b/ptypy/templates/ptypy-gpu-workflow-template.yaml index 3f074d5..629fc73 100644 --- a/ptypy/templates/ptypy-gpu-workflow-template.yaml +++ b/ptypy/templates/ptypy-gpu-workflow-template.yaml @@ -2,6 +2,10 @@ apiVersion: argoproj.io/v1alpha1 kind: ClusterWorkflowTemplate metadata: name: ptypy-gpu-job + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" spec: entrypoint: ptypy-run volumeClaimTemplates: From 6f880d1971a9d7e3d30df2d9c4ebd886f7e50111 Mon Sep 17 00:00:00 2001 From: Jessica V Date: Mon, 18 May 2026 13:19:29 +0100 Subject: [PATCH 02/51] Update default boolean values to be consistent --- i14/templates/phase-template.yaml | 2 +- i14/templates/xanes-auto-template.yaml | 2 +- i14/templates/xanes-point-template.yaml | 2 +- i14/templates/xanes-sparse-template.yaml | 2 +- 4 files changed, 4 insertions(+), 4 deletions(-) diff --git a/i14/templates/phase-template.yaml b/i14/templates/phase-template.yaml index 46b0ad2..2e56716 100644 --- a/i14/templates/phase-template.yaml +++ b/i14/templates/phase-template.yaml @@ -27,7 +27,7 @@ spec: - name: maskpath value: "" - name: normalise - value: False + value: false - name: outpath value: "" volumeClaimTemplates: diff --git a/i14/templates/xanes-auto-template.yaml b/i14/templates/xanes-auto-template.yaml index baa9eb2..8c89469 100644 --- a/i14/templates/xanes-auto-template.yaml +++ b/i14/templates/xanes-auto-template.yaml @@ -40,7 +40,7 @@ spec: - name: transitionToAlign value: None - name: normalise - value: "True" + value: true volumeClaimTemplates: - metadata: name: tmp diff --git a/i14/templates/xanes-point-template.yaml b/i14/templates/xanes-point-template.yaml index 57dfd21..7a0b81a 100644 --- a/i14/templates/xanes-point-template.yaml +++ b/i14/templates/xanes-point-template.yaml @@ -33,7 +33,7 @@ spec: - name: edgeTransition value: "Ka" - name: normalise - value: "True" + value: true volumeClaimTemplates: - metadata: name: tmp diff --git a/i14/templates/xanes-sparse-template.yaml b/i14/templates/xanes-sparse-template.yaml index e6975d0..8f6c74f 100644 --- a/i14/templates/xanes-sparse-template.yaml +++ b/i14/templates/xanes-sparse-template.yaml @@ -34,7 +34,7 @@ spec: - name: edgeTransition value: "Ka" - name: normalise - value: "True" + value: true volumeClaimTemplates: - metadata: name: tmp From 6565feb334eaa03b21e608a4b87cebcc3a199b8b Mon Sep 17 00:00:00 2001 From: Jessica V Date: Mon, 18 May 2026 13:20:10 +0100 Subject: [PATCH 03/51] Set default boolean values for json files --- i14/schema/xanesPointSchema.json | 1 + i14/schema/xanesSchema.json | 1 + i14/schema/xanesSparseSchema.json | 3 ++- i14/schema/xrfTomoSchema.json | 1 + nxstacker/schema/nxstackerSchema.json | 32 +++++++++++++++++++-------- 5 files changed, 28 insertions(+), 10 deletions(-) diff --git a/i14/schema/xanesPointSchema.json b/i14/schema/xanesPointSchema.json index 0dae244..7fc80ae 100644 --- a/i14/schema/xanesPointSchema.json +++ b/i14/schema/xanesPointSchema.json @@ -141,6 +141,7 @@ "normalise": { "type": "boolean", "title": "Normalise", + "default": true, "description": "Whether to normalise the windowed intensity by 'adc_ionCh1a'" } }, diff --git a/i14/schema/xanesSchema.json b/i14/schema/xanesSchema.json index 29758e5..9929b37 100644 --- a/i14/schema/xanesSchema.json +++ b/i14/schema/xanesSchema.json @@ -37,6 +37,7 @@ "normalise": { "type": "boolean", "title": "Normalise", + "default": true, "description": "Whether to normalise the stack by I0" } }, diff --git a/i14/schema/xanesSparseSchema.json b/i14/schema/xanesSparseSchema.json index 310151d..c98083c 100644 --- a/i14/schema/xanesSparseSchema.json +++ b/i14/schema/xanesSparseSchema.json @@ -16,7 +16,8 @@ }, "normalise": { "type": "boolean", - "title": "Normalise" + "title": "Normalise", + "default": true }, "edgeElement": { "type": "string", diff --git a/i14/schema/xrfTomoSchema.json b/i14/schema/xrfTomoSchema.json index 2441a12..d7417ec 100644 --- a/i14/schema/xrfTomoSchema.json +++ b/i14/schema/xrfTomoSchema.json @@ -52,6 +52,7 @@ "normalise": { "type": "boolean", "title": "Normalise", + "default": true, "description": "Whether to normalise the stack using the total mass of each projection." }, "cor": { diff --git a/nxstacker/schema/nxstackerSchema.json b/nxstacker/schema/nxstackerSchema.json index b65fd15..c9c3351 100644 --- a/nxstacker/schema/nxstackerSchema.json +++ b/nxstacker/schema/nxstackerSchema.json @@ -14,11 +14,13 @@ "quiet": { "type": "boolean", "title": "Quiet", + "default": false, "description": "Suppress log messages" }, "dryRun": { "type": "boolean", - "title": "Perform a dry run" + "title": "Perform a dry run", + "default": false }, "projDir": { "type": "string", @@ -93,21 +95,25 @@ "sortByAngle": { "type": "boolean", "title": "Sort by angle", + "default": false, "description": "Sort the projections by their rotation angles" }, "padToMax": { "type": "boolean", "title": "Pad to max", + "default": false, "description": "Pad projection to the maximum size of the stack" }, "compress": { "type": "boolean", "title": "Compress", + "default": false, "description": "Compress the NXtomo file" }, "skipCheck": { "type": "boolean", "title": "Skip check", + "default": false, "description": "Skip the validation of the projection file" }, "retrievalMethod": { @@ -251,36 +257,44 @@ }, "saveComplex": { "type": "boolean", - "title": "Save complex" + "title": "Save complex", + "default": false }, "saveModulus": { "type": "boolean", - "title": "Save modulus" + "title": "Save modulus", + "default": false }, "savePhase": { "type": "boolean", - "title": "Save phase" + "title": "Save phase", + "default": false }, "removeRamp": { "type": "boolean", - "title": "Remove ramp" + "title": "Remove ramp", + "default": false }, "medianNorm": { "type": "boolean", "title": "Median norm", - "description": "Normalise the phase by shifting its median" + "description": "Normalise the phase by shifting its median", + "default": false }, "unwrapPhase": { "type": "boolean", - "title": "Unwrap phase" + "title": "Unwrap phase", + "default": false }, "rescale": { "type": "boolean", - "title": "Rescale" + "title": "Rescale", + "default": false }, "ignoreRaw": { "type": "boolean", - "title": "Ignore Raw" + "title": "Ignore Raw", + "default": false } }, "required": ["experimentType"] From 62bceaf028718f8265b1faf625541fde3c538f51 Mon Sep 17 00:00:00 2001 From: Jessica V Date: Mon, 18 May 2026 13:20:42 +0100 Subject: [PATCH 04/51] Set normalised boolean within notebooks --- i14/notebooks/ptycho-tomo-phase.ipynb | 18 ++++++++++++++++++ i14/notebooks/xanes_autoprocessing0.ipynb | 16 ++++++++++++++++ .../xanes_point_autoprocessing0.ipynb | 16 ++++++++++++++++ .../xanes_sparse_stack_autoprocessing0.ipynb | 16 ++++++++++++++++ 4 files changed, 66 insertions(+) diff --git a/i14/notebooks/ptycho-tomo-phase.ipynb b/i14/notebooks/ptycho-tomo-phase.ipynb index ec0d64d..48a0c6a 100644 --- a/i14/notebooks/ptycho-tomo-phase.ipynb +++ b/i14/notebooks/ptycho-tomo-phase.ipynb @@ -88,6 +88,24 @@ "cell_type": "markdown", "id": "2f61bb2f", "metadata": {}, + "source": [ + "## Set normalise boolean" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e3300162-404a-4660-b4ac-136bbd517727", + "metadata": {}, + "outputs": [], + "source": [ + "normalise = True if normalise.lower() == \"true\" else False" + ] + }, + { + "cell_type": "markdown", + "id": "5d23b863-a0da-40f4-aa72-c5b5cc911c00", + "metadata": {}, "source": [ "## Initialise np array" ] diff --git a/i14/notebooks/xanes_autoprocessing0.ipynb b/i14/notebooks/xanes_autoprocessing0.ipynb index ed3f052..53ba234 100644 --- a/i14/notebooks/xanes_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_autoprocessing0.ipynb @@ -169,6 +169,22 @@ "print(f\"The PNG file will be saved as {png_outpath}\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set normalised boolean\n" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "normalised = True if normalised.lower() == \"true\" else False" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/i14/notebooks/xanes_point_autoprocessing0.ipynb b/i14/notebooks/xanes_point_autoprocessing0.ipynb index 8d7918f..9e70baa 100644 --- a/i14/notebooks/xanes_point_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_point_autoprocessing0.ipynb @@ -84,6 +84,22 @@ " raise FileNotFoundError(msg)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set normalise boolean" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "normalise = True if normalise.lower() == \"true\" else False" + ] + }, { "cell_type": "markdown", "metadata": {}, diff --git a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb index 26b7137..0d62c66 100644 --- a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb @@ -119,6 +119,22 @@ "print(f\"HyperSpy version: {version('hyperspy')}\")" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Set normalised" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "normalised = True if normalised.lower() == \"true\" else False" + ] + }, { "cell_type": "markdown", "metadata": {}, From c1407af2fb36c34ca41500ce65adacdb96f7a9fb Mon Sep 17 00:00:00 2001 From: Jessica V Date: Thu, 14 May 2026 15:01:48 +0100 Subject: [PATCH 05/51] Alter papermill line to prevent notebook execution. Remove redundant nbconvert. --- i14/templates/align-template.yaml | 10 ++++++---- i14/templates/dpc-batch-template.yaml | 12 ++++++++---- i14/templates/mask-template.yaml | 10 ++++++---- i14/templates/phase-template.yaml | 11 +++++++---- i14/templates/xanes-auto-template.yaml | 11 +++++++---- i14/templates/xanes-point-template.yaml | 10 ++++++---- i14/templates/xanes-sparse-template.yaml | 11 +++++++---- i14/templates/xrd1d-batch-template.yaml | 11 +++++++---- i14/templates/xrd2d-batch-template.yaml | 12 ++++++++---- 9 files changed, 62 insertions(+), 36 deletions(-) diff --git a/i14/templates/align-template.yaml b/i14/templates/align-template.yaml index c7dcfe3..86c14e4 100644 --- a/i14/templates/align-template.yaml +++ b/i14/templates/align-template.yaml @@ -69,13 +69,15 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" \ -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \ -p maskpath "{{`{{ inputs.parameters.maskpath }}`}}" \ -p complexpath "{{`{{ inputs.parameters.complexpath }}`}}" \ - -p outpath "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output align-notebook --output-dir "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" --to html --output align-output --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ + --prepare-only + python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" \ + --to html --execute --allow-errors --output align-output \ + --output-dir "{{`{{ inputs.parameters.outpath }}`}}" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/i14/templates/dpc-batch-template.yaml b/i14/templates/dpc-batch-template.yaml index f23c5e7..2bf5e40 100644 --- a/i14/templates/dpc-batch-template.yaml +++ b/i14/templates/dpc-batch-template.yaml @@ -72,13 +72,17 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + python -m papermill /tmp/notebook.ipynb \ + "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p process_configfile /tmp/config.yaml \ -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}_dpc-phase.nxs" \ - -p auto_processing False - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" --to html --no-input --output dpc-output-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + -p auto_processing False \ + --prepare-only + python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + --to html --execute --allow-errors --no-input \ + --output dpc-output-{{`{{ inputs.parameters.scanNumber }}`}} \ + --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/i14/templates/mask-template.yaml b/i14/templates/mask-template.yaml index 962685a..e8dd1b3 100644 --- a/i14/templates/mask-template.yaml +++ b/i14/templates/mask-template.yaml @@ -76,15 +76,17 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" \ -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \ -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ -p padding "{{`{{ inputs.parameters.padding }}`}}" \ -p min_size "{{`{{ inputs.parameters.minSize }}`}}" \ -p erosion_radius "{{`{{ inputs.parameters.erosionRadius }}`}}" \ - -p dilation_radius "{{`{{ inputs.parameters.dilationRadius }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output mask-notebook --output-dir "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" --to html --output mask-output --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + -p dilation_radius "{{`{{ inputs.parameters.dilationRadius }}`}}" \ + --prepare-only + python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" \ + --to html --execute --allow-errors --output mask-output \ + --output-dir "{{`{{ inputs.parameters.outpath }}`}}" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/i14/templates/phase-template.yaml b/i14/templates/phase-template.yaml index 2e56716..667bb96 100644 --- a/i14/templates/phase-template.yaml +++ b/i14/templates/phase-template.yaml @@ -68,13 +68,16 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" \ -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \ -p maskpath "{{`{{ inputs.parameters.maskpath }}`}}" \ -p normalise "{{`{{ inputs.parameters.normalise }}`}}" \ - -p outpath "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output phase-notebook --output-dir "{{`{{ inputs.parameters.outpath }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" --to html --output phase-output --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ + --prepare-only + python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" \ + --to html --execute --allow-errors \ + --output phase-output \ + --output-dir "{{`{{ inputs.parameters.outpath }}`}}" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/i14/templates/xanes-auto-template.yaml b/i14/templates/xanes-auto-template.yaml index 8c89469..bfc46df 100644 --- a/i14/templates/xanes-auto-template.yaml +++ b/i14/templates/xanes-auto-template.yaml @@ -128,7 +128,8 @@ spec: value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/ command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + python -m papermill /tmp/notebook.ipynb \ + "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" \ -p first "{{`{{ inputs.parameters.first }}`}}" \ -p last "{{`{{ inputs.parameters.last }}`}}" \ -p exclude "{{`{{ inputs.parameters.exclude }}`}}" \ @@ -142,9 +143,11 @@ spec: -p sztol 0.9 \ -p ref_index None \ -p max_fractional_shift 0.2 \ - -p auto_processing False - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xanes-notebook --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" --to html --output xanes-output --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" + -p auto_processing False \ + --prepare-only + python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" \ + --to html --execute --allow-errors --output xanes-output \ + --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/i14/templates/xanes-point-template.yaml b/i14/templates/xanes-point-template.yaml index 7a0b81a..8acaca7 100644 --- a/i14/templates/xanes-point-template.yaml +++ b/i14/templates/xanes-point-template.yaml @@ -72,14 +72,16 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}xanes_point.txt" \ -p edge_element "{{`{{ inputs.parameters.edgeElement }}`}}" \ -p normalise "{{`{{ inputs.parameters.normalise }}`}}" \ - -p auto_processing False - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xanes-point-notebook --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" --to html --output xanes-point-output --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + -p auto_processing False \ + --prepare-only + python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" \ + --to html --execute --allow-errors --output xanes-point-output \ + --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/i14/templates/xanes-sparse-template.yaml b/i14/templates/xanes-sparse-template.yaml index 8f6c74f..03242db 100644 --- a/i14/templates/xanes-sparse-template.yaml +++ b/i14/templates/xanes-sparse-template.yaml @@ -95,7 +95,8 @@ spec: value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/ command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + python -m papermill /tmp/notebook.ipynb \ + "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" \ -p first "{{`{{ inputs.parameters.first }}`}}" \ -p last "{{`{{ inputs.parameters.last }}`}}" \ -p exclude "{{`{{ inputs.parameters.exclude }}`}}" \ @@ -108,9 +109,11 @@ spec: -p num_short_iteration 75 \ -p num_final_iteration 2000 \ -p normalised "{{`{{ inputs.parameters.normalise }}`}}" \ - -p auto_processing False - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xanes-sparse-notebook --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" --to html --output xanes-sparse-output --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" + -p auto_processing False \ + --prepare-only + python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" \ + --to html --execute --allow-errors --output xanes-sparse-output \ + --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/i14/templates/xrd1d-batch-template.yaml b/i14/templates/xrd1d-batch-template.yaml index a83d2f1..27197ce 100644 --- a/i14/templates/xrd1d-batch-template.yaml +++ b/i14/templates/xrd1d-batch-template.yaml @@ -100,7 +100,8 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + python -m papermill /tmp/notebook.ipynb \ + "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p process_configfile "{{`{{ inputs.parameters.xrdProcessConfigUpload }}`}}" \ -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd1d.nxs" \ @@ -108,9 +109,11 @@ spec: -p mask_path "{{`{{ inputs.parameters.maskPath }}`}}" \ -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" \ -p background_type "{{`{{ inputs.parameters.backgroundType }}`}}" \ - -p polynomial_order "{{`{{ inputs.parameters.polynomialOrder }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" --to html --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + -p polynomial_order "{{`{{ inputs.parameters.polynomialOrder }}`}}" \ + --prepare-only + python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + --to html --execute --allow-errors --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ + --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/i14/templates/xrd2d-batch-template.yaml b/i14/templates/xrd2d-batch-template.yaml index 69e74cd..8a95f44 100644 --- a/i14/templates/xrd2d-batch-template.yaml +++ b/i14/templates/xrd2d-batch-template.yaml @@ -89,15 +89,19 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb /tmp/notebook-parametrized.ipynb \ + python -m papermill /tmp/notebook.ipynb \ + "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p process_configfile "{{`{{ inputs.parameters.xrdProcessConfigUpload }}`}}" \ -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd2d.nxs" \ -p calibration_path "{{`{{ inputs.parameters.calibrationPath }}`}}" \ -p mask_path "{{`{{ inputs.parameters.maskPath }}`}}" \ - -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" - python -m jupyter nbconvert /tmp/notebook-parametrized.ipynb --to notebook --execute --allow-errors --output xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" --to html --no-input --output xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}} --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" \ + --prepare-only + python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + --to html --execute --allow-errors --no-input \ + --output xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ + --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From ec06a61452f679e89da1efc0a8e14c6ac33e2bfd Mon Sep 17 00:00:00 2001 From: Jessica V Date: Mon, 18 May 2026 14:58:30 +0100 Subject: [PATCH 06/51] Remove nbconvert --allow-errors, so that notebooks stop running at first failed cell --- i14/templates/align-template.yaml | 2 +- i14/templates/dpc-batch-template.yaml | 2 +- i14/templates/mask-template.yaml | 2 +- i14/templates/phase-template.yaml | 2 +- i14/templates/xanes-auto-template.yaml | 2 +- i14/templates/xanes-point-template.yaml | 2 +- i14/templates/xanes-sparse-template.yaml | 2 +- i14/templates/xrd1d-batch-template.yaml | 2 +- i14/templates/xrd2d-batch-template.yaml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/i14/templates/align-template.yaml b/i14/templates/align-template.yaml index 86c14e4..e2dc2e0 100644 --- a/i14/templates/align-template.yaml +++ b/i14/templates/align-template.yaml @@ -76,7 +76,7 @@ spec: -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ --prepare-only python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" \ - --to html --execute --allow-errors --output align-output \ + --to html --execute --output align-output \ --output-dir "{{`{{ inputs.parameters.outpath }}`}}" volumeMounts: - name: session diff --git a/i14/templates/dpc-batch-template.yaml b/i14/templates/dpc-batch-template.yaml index 2bf5e40..bb4ac52 100644 --- a/i14/templates/dpc-batch-template.yaml +++ b/i14/templates/dpc-batch-template.yaml @@ -80,7 +80,7 @@ spec: -p auto_processing False \ --prepare-only python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ - --to html --execute --allow-errors --no-input \ + --to html --execute --no-input \ --output dpc-output-{{`{{ inputs.parameters.scanNumber }}`}} \ --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" volumeMounts: diff --git a/i14/templates/mask-template.yaml b/i14/templates/mask-template.yaml index e8dd1b3..b9b0b49 100644 --- a/i14/templates/mask-template.yaml +++ b/i14/templates/mask-template.yaml @@ -85,7 +85,7 @@ spec: -p dilation_radius "{{`{{ inputs.parameters.dilationRadius }}`}}" \ --prepare-only python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" \ - --to html --execute --allow-errors --output mask-output \ + --to html --execute --output mask-output \ --output-dir "{{`{{ inputs.parameters.outpath }}`}}" volumeMounts: - name: session diff --git a/i14/templates/phase-template.yaml b/i14/templates/phase-template.yaml index 667bb96..7502f28 100644 --- a/i14/templates/phase-template.yaml +++ b/i14/templates/phase-template.yaml @@ -75,7 +75,7 @@ spec: -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ --prepare-only python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" \ - --to html --execute --allow-errors \ + --to html --execute \ --output phase-output \ --output-dir "{{`{{ inputs.parameters.outpath }}`}}" volumeMounts: diff --git a/i14/templates/xanes-auto-template.yaml b/i14/templates/xanes-auto-template.yaml index bfc46df..e5cc7a8 100644 --- a/i14/templates/xanes-auto-template.yaml +++ b/i14/templates/xanes-auto-template.yaml @@ -146,7 +146,7 @@ spec: -p auto_processing False \ --prepare-only python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" \ - --to html --execute --allow-errors --output xanes-output \ + --to html --execute --output xanes-output \ --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" volumeMounts: - name: session diff --git a/i14/templates/xanes-point-template.yaml b/i14/templates/xanes-point-template.yaml index 8acaca7..0873b58 100644 --- a/i14/templates/xanes-point-template.yaml +++ b/i14/templates/xanes-point-template.yaml @@ -80,7 +80,7 @@ spec: -p auto_processing False \ --prepare-only python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" \ - --to html --execute --allow-errors --output xanes-point-output \ + --to html --execute --output xanes-point-output \ --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" volumeMounts: - name: session diff --git a/i14/templates/xanes-sparse-template.yaml b/i14/templates/xanes-sparse-template.yaml index 03242db..b1d220c 100644 --- a/i14/templates/xanes-sparse-template.yaml +++ b/i14/templates/xanes-sparse-template.yaml @@ -112,7 +112,7 @@ spec: -p auto_processing False \ --prepare-only python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" \ - --to html --execute --allow-errors --output xanes-sparse-output \ + --to html --execute --output xanes-sparse-output \ --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" volumeMounts: - name: session diff --git a/i14/templates/xrd1d-batch-template.yaml b/i14/templates/xrd1d-batch-template.yaml index 27197ce..1ad4647 100644 --- a/i14/templates/xrd1d-batch-template.yaml +++ b/i14/templates/xrd1d-batch-template.yaml @@ -112,7 +112,7 @@ spec: -p polynomial_order "{{`{{ inputs.parameters.polynomialOrder }}`}}" \ --prepare-only python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ - --to html --execute --allow-errors --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ + --to html --execute --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" volumeMounts: - name: session diff --git a/i14/templates/xrd2d-batch-template.yaml b/i14/templates/xrd2d-batch-template.yaml index 8a95f44..e17ca05 100644 --- a/i14/templates/xrd2d-batch-template.yaml +++ b/i14/templates/xrd2d-batch-template.yaml @@ -99,7 +99,7 @@ spec: -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" \ --prepare-only python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ - --to html --execute --allow-errors --no-input \ + --to html --execute --no-input \ --output xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" volumeMounts: From 1251aee0397ce6c5f8ca435e8b3a7d2a0696f202 Mon Sep 17 00:00:00 2001 From: Jessica V Date: Wed, 20 May 2026 18:13:38 +0100 Subject: [PATCH 07/51] Add parameter tag to notebooks --- i14/notebooks/xanes_autoprocessing0.ipynb | 2 +- i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/i14/notebooks/xanes_autoprocessing0.ipynb b/i14/notebooks/xanes_autoprocessing0.ipynb index 53ba234..2d2eaac 100644 --- a/i14/notebooks/xanes_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_autoprocessing0.ipynb @@ -10,7 +10,7 @@ }, "outputs": [], "source": [ - "# default settings\n", + "# parameters\n", "inpath = \"\" # type: str\n", "outpath = \"\" # type: str\n", "edge_element = \"\" # type: str\n", diff --git a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb index 0d62c66..032fc4d 100644 --- a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb @@ -10,6 +10,7 @@ }, "outputs": [], "source": [ + "# parameters\n", "# outpath will be used to get outpath_nexus, outpath_mantis and outpath_complete for saving data, and outpath itself won't be used\n", "\n", "# default settings\n", From 41012e383bac92602731cd34611f72cc25984d36 Mon Sep 17 00:00:00 2001 From: Jessica V Date: Wed, 20 May 2026 18:15:21 +0100 Subject: [PATCH 08/51] Create directory before papermill line --- i14/templates/xanes-auto-template.yaml | 9 ++++++--- i14/templates/xanes-sparse-template.yaml | 9 ++++++--- 2 files changed, 12 insertions(+), 6 deletions(-) diff --git a/i14/templates/xanes-auto-template.yaml b/i14/templates/xanes-auto-template.yaml index e5cc7a8..cc31ea7 100644 --- a/i14/templates/xanes-auto-template.yaml +++ b/i14/templates/xanes-auto-template.yaml @@ -128,8 +128,11 @@ spec: value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/ command: [bash] source: | + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" + mkdir -p $OUTPUT + python -m papermill /tmp/notebook.ipynb \ - "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" \ + $OUTPUT/xanes-notebook.ipynb \ -p first "{{`{{ inputs.parameters.first }}`}}" \ -p last "{{`{{ inputs.parameters.last }}`}}" \ -p exclude "{{`{{ inputs.parameters.exclude }}`}}" \ @@ -145,9 +148,9 @@ spec: -p max_fractional_shift 0.2 \ -p auto_processing False \ --prepare-only - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-notebook.ipynb" \ + python -m jupyter nbconvert $OUTPUT/xanes-notebook.ipynb \ --to html --execute --output xanes-output \ - --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" + --output-dir $OUTPUT volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/i14/templates/xanes-sparse-template.yaml b/i14/templates/xanes-sparse-template.yaml index b1d220c..5822029 100644 --- a/i14/templates/xanes-sparse-template.yaml +++ b/i14/templates/xanes-sparse-template.yaml @@ -95,8 +95,11 @@ spec: value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/ command: [bash] source: | + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" + mkdir -p $OUTPUT + python -m papermill /tmp/notebook.ipynb \ - "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" \ + $OUTPUT/xanes-sparse-notebook.ipynb \ -p first "{{`{{ inputs.parameters.first }}`}}" \ -p last "{{`{{ inputs.parameters.last }}`}}" \ -p exclude "{{`{{ inputs.parameters.exclude }}`}}" \ @@ -111,9 +114,9 @@ spec: -p normalised "{{`{{ inputs.parameters.normalise }}`}}" \ -p auto_processing False \ --prepare-only - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-notebook.ipynb" \ + python -m jupyter nbconvert $OUTPUT/xanes-sparse-notebook.ipynb \ --to html --execute --output xanes-sparse-output \ - --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" + --output-dir $OUTPUT volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From 1d47015fd322a011f79f8bc8d6eeba2150594ef7 Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Thu, 4 Jun 2026 14:56:12 +0100 Subject: [PATCH 09/51] setup ptypy recons for P99 --- ptypy/config/p99_dm.yaml | 81 ++++++++++++++ ptypy/templates/ptypy-generic.yaml | 166 +++++++++++++++++++++++++++++ ptypy/templates/ptypy-p99.yaml | 156 +++++++++++++++++++++++++++ ptypy/templates/ptypy-utils.yaml | 79 ++++++++++++++ 4 files changed, 482 insertions(+) create mode 100755 ptypy/config/p99_dm.yaml create mode 100644 ptypy/templates/ptypy-generic.yaml create mode 100644 ptypy/templates/ptypy-p99.yaml create mode 100644 ptypy/templates/ptypy-utils.yaml diff --git a/ptypy/config/p99_dm.yaml b/ptypy/config/p99_dm.yaml new file mode 100755 index 0000000..5f8401c --- /dev/null +++ b/ptypy/config/p99_dm.yaml @@ -0,0 +1,81 @@ +visit_path: +parameter_tree: + run: '' + verbose_level: 3 + frames_per_block: 1000 + data_type: single + dry_run: false + engines: + engine0: + name: DM + numiter: 100 + numiter_contiguous: 10 + alpha: 0.95 + probe_support: null + probe_update_start: 0 + update_object_first: false + object_inertia: 0.001 + probe_inertia: 0.001 + overlap_converge_factor: 0.01 + overlap_max_iterations: 10 + obj_smooth_std: 20 + fourier_power_bound: 0.25 + io: + autoplot: + active: false + autosave: + active: false + interval: 100 + rformat: dls + scans: + scan_00: + name: BlockFull + coherence: + num_object_modes: 1 + num_probe_modes: 1 + sample: + diversity: null + model: null + process: null + illumination: + model: null + photons: null + aperture: + form: circ + size: 0.000333 + propagation: + antialiasing: 1 + focussed: 0.013725 + parallel: 50.e-06 + diversity: + noise: + - 0.5 + - 1.0 + power: 0.1 + data: + name: Hdf5Loader + orientation: 0 + psize: 11.e-06 + rebin: 1 + shape: + - 128 + - 128 + auto_center: true + save: null + intensities: + file: /%(visit_path)s/p99-%(run)s.nxs + key: /entry/andor2_det/andor2_det + positions: + file: /%(visit_path)s/p99-%(run)s.nxs + fast_key: /entry/andor2_det/sample_stage-x + fast_multiplier: 0.001 + slow_key: /entry/andor2_det/sample_stage-y + slow_multiplier: 0.001 + recorded_energy: + file: /%(visit_path)s/p99-%(run)s.nxs + key: /entry/plan_metadata/energy/value + multiplier: 0.001 + recorded_distance: + file: /%(visit_path)s/p99-%(run)s.nxs + key: /entry/plan_metadata/detector_dist/value + multiplier: 0.001 diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml new file mode 100644 index 0000000..c21e51c --- /dev/null +++ b/ptypy/templates/ptypy-generic.yaml @@ -0,0 +1,166 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: ptypy-generic + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" +spec: + entrypoint: ptypy-gpu-run + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + storageClassName: netapp + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory + templates: + - name: ptypy-gpu-run + inputs: + parameters: + - name: config + - name: id + - name: output + - name: nprocs + value: 1 + - name: memory + value: 20Gi + container: + image: ghcr.io/ptycho/ptypy_cupy_openmpi:v0.9.0 + env: + - name: CUPY_CACHE_DIR + value: /tmp/.cupy/kernel_cache + - name: MKL_NUM_THREADS + value: "1" + - name: NUMEXPR_NUM_THREADS + value: "1" + - name: OMP_NUM_THREADS + value: "1" + command: + - mpirun + args: + - "-n" + - "{{`{{ inputs.parameters.nprocs }}`}}" + - "ptypy.cli" + - "-f" + - "{{`{{ inputs.parameters.config }}`}}" + - "-i" + - "{{`{{ inputs.parameters.id }}`}}" + - "-o" + - "{{`{{ inputs.parameters.output }}`}}" + - "-s" + - "hdf5_loader" + - "-b" + - "cupy" + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmpdir + mountPath: /tmp + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: "{{`{{ inputs.parameters.nprocs }}`}}" + memory: "{{`{{ inputs.parameters.memory }}`}}" + nvidia.com/gpu: "{{`{{ inputs.parameters.nprocs }}`}}" + limits: + cpu: "{{`{{ inputs.parameters.nprocs }}`}}" + memory: "{{`{{ inputs.parameters.memory }}`}}" + nvidia.com/gpu: "{{`{{ inputs.parameters.nprocs }}`}}" + tolerations: + - key: nvidia.com/gpu + operator: Exists + effect: NoSchedule + - key: nodetype + operator: Equal + value: gpu + effect: NoSchedule + - key: nodegroup + operator: Equal + value: workflows + effect: NoSchedule + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory + - name: ptypy-cpu-run + inputs: + parameters: + - name: config + - name: id + - name: output + - name: nprocs + value: 1 + - name: memory + value: 20Gi + container: + image: ghcr.io/ptycho/ptypy_full_openmpi:v0.9.0 + env: + - name: MKL_NUM_THREADS + value: "1" + - name: NUMEXPR_NUM_THREADS + value: "1" + - name: OMP_NUM_THREADS + value: "1" + command: + - mpirun + args: + - "-n" + - "{{`{{ inputs.parameters.nprocs }}`}}" + - "ptypy.cli" + - "-f" + - "{{`{{ inputs.parameters.config }}`}}" + - "-i" + - "{{`{{ inputs.parameters.id }}`}}" + - "-o" + - "{{`{{ inputs.parameters.output }}`}}" + - "-s" + - "hdf5_loader" + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmpdir + mountPath: /tmp + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: "{{`{{ inputs.parameters.nprocs }}`}}" + memory: "{{`{{ inputs.parameters.memory }}`}}" + limits: + cpu: "{{`{{ inputs.parameters.nprocs }}`}}" + memory: "{{`{{ inputs.parameters.memory }}`}}" + tolerations: + - key: nodegroup + operator: Equal + value: workflows + effect: NoSchedule + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: nodegroup + operator: In + values: + - workflows + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml new file mode 100644 index 0000000..0d288b1 --- /dev/null +++ b/ptypy/templates/ptypy-p99.yaml @@ -0,0 +1,156 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: ptypy-p99-from-config + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.argoproj.io/title: PtyPy Reconstruction for P99 + workflows.argoproj.io/description: | + Runs a PtyPy reconstruction job inside a container based on data collected at P99. + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" + workflows.diamond.ac.uk/parameter-schema.id: | + { + "title": "Scan Number", + "type": "integer" + } + workflows.diamond.ac.uk/parameter-schema.inpath: | + { + "title": "Path to raw data folder", + "type": "string", + "default": "processing/writenData/reconstruction_test_data" + } + workflows.diamond.ac.uk/parameter-schema.outpath: | + { + "title": "Path to output folder", + "type": "string", + "default": "processing/workflows/ptypy" + } + workflows.diamond.ac.uk/parameter-schema.nprocs: | + { + "title": "Nr. of processes", + "type": "integer", + "default": 1 + } + workflows.diamond.ac.uk/parameter-schema.memory: | + { + "title": "Requested CPU memory", + "type": "string", + "pattern": "^[0-9]+[GMK]i$", + "default": "20Gi" + } + workflows.diamond.ac.uk/parameter-schema.usegpu: | + { + "title": "Use GPU", + "type": "boolean", + "default": false + } +spec: + entrypoint: ptypy-pipeline + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory + - name: id + value: 7 + - name: inpath + value: "processing/writenData/reconstruction_test_data" + - name: outpath + value: "processing/workflows/ptypy" + - name: nprocs + value: 1 + - name: memory + value: 20Gi + - name: usegpu + value: "false" + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + storageClassName: netapp + + templates: + - name: ptypy-setup + script: + image: python:3.14 + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmp + mountPath: /tmp + command: [bash] + source: | + OUTPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.outpath }}`}}" + mkdir -p $OUTPUT + INPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.inpath }}`}}" + echo '{{ .Files.Get "config/p99_dm.yaml" | b64enc }}' | base64 -d > $OUTPUT/ptypy_config.yaml + sed -i 's;;$INPUT;' $OUTPUT/ptypy_config.yaml + outputs: + parameters: + - name: config + valueFrom: + path: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/ptypy_config.yaml" + - name: reconfile + valueFrom: + path: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/test.ptyr" + + - name: ptypy-pipeline + dag: + tasks: + - name: prepare + template: ptypy-setup + - name: reconstruct-with-gpu + when: "{{`{{ workflows.parameters.usegpu }} == true`}}" + templateRef: + name: ptypy-generic + template: ptypy-gpu-run + clusterScope: true + depends: "prepare" + arguments: + parameters: + - name: config + value: "{{`{{ tasks.prepare.output.parameters.config }}`}}" + - name: id + value: "{{`{{ workflows.parameters.id }}`}}" + - name: outpath + value: "{{`{{ workflows.parameters.outpath }}`}}" + - name: nprocs + value: "{{`{{ workflows.parameters.nprocs }}`}}" + - name: memory + value: "{{`{{ workflows.parameters.memory }}`}}" + - name: reconstruct-with-cpu + when: "{{`{{ workflows.parameters.usegpu }} == false `}}" + templateRef: + name: ptypy-generic + template: ptypy-cpu-run + clusterScope: true + depends: "prepare" + arguments: + parameters: + - name: config + value: "{{`{{ tasks.prepare.output.parameters.config }}`}}" + - name: id + value: "{{`{{ workflows.parameters.id }}`}}" + - name: outpath + value: "{{`{{ workflows.parameters.outpath }}`}}" + - name: nprocs + value: "{{`{{ workflows.parameters.nprocs }}`}}" + - name: memory + value: "{{`{{ workflows.parameters.memory }}`}}" + - name: plot + templateRef: + name: ptypy-utils + template: ptypy-plotting + clusterScope: true + depends: "reconstruct-with-gpu || reconstruct-with-cpu" + arguments: + parameters: + - name: recon + value: "{{`{{ tasks.prepare.outputs.parameters.reconfile }}`}}" diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml new file mode 100644 index 0000000..c03c1ec --- /dev/null +++ b/ptypy/templates/ptypy-utils.yaml @@ -0,0 +1,79 @@ +apiVersion: argoproj.io/v1alpha1 +kind: ClusterWorkflowTemplate +metadata: + name: ptypy-utils + labels: + workflows.diamond.ac.uk/science-group-imaging: "true" + annotations: + workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows" +spec: + entrypoint: ptypy-plotting + volumeClaimTemplates: + - metadata: + name: tmpdir + spec: + accessModes: [ "ReadWriteOnce" ] + resources: + requests: + storage: 1Gi + storageClassName: netapp + arguments: + parameters: + - name: visitdir + valueFrom: + configMapKeyRef: + name: sessionspaces + key: data_directory + templates: + - name: ptypy-plotting + inputs: + parameters: + - name: recon + script: + image: ghcr.io/ptycho/ptypy_full_openmpi:v0.9.0 + env: + - name: MKL_NUM_THREADS + value: "1" + - name: NUMEXPR_NUM_THREADS + value: "1" + - name: OMP_NUM_THREADS + value: "1" + command: [bash] + source: | + echo "{{`{{ inputs.parameters.recon }}`}}" + + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmpdir + mountPath: /tmp + podSpecPatch: | + containers: + - name: main + resources: + requests: + cpu: 1 + memory: 20Gi + limits: + cpu: 1 + memory: 20Gi + tolerations: + - key: nodegroup + operator: Equal + value: workflows + effect: NoSchedule + affinity: + nodeAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 100 + preference: + matchExpressions: + - key: nodegroup + operator: In + values: + - workflows + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory From 0e6ba1909c348ea4b6393f8960270224df085d41 Mon Sep 17 00:00:00 2001 From: Jessica V <55289415+jessicavers@users.noreply.github.com> Date: Tue, 9 Jun 2026 10:40:21 +0100 Subject: [PATCH 10/51] Update nb execution and error handling for i14 workflows (#205) * Add line to fail workflow task if notebook fails * Use papermill for notebook execution --- i14/templates/align-template.yaml | 26 +++++++++++++++++------ i14/templates/dpc-batch-template.yaml | 24 ++++++++++++++++----- i14/templates/mask-template.yaml | 24 ++++++++++++++++----- i14/templates/phase-template.yaml | 22 +++++++++++++++---- i14/templates/xanes-auto-template.yaml | 15 +++++++++++-- i14/templates/xanes-point-template.yaml | 26 +++++++++++++++++------ i14/templates/xanes-sparse-template.yaml | 16 +++++++++++--- i14/templates/xrd1d-batch-template.yaml | 27 +++++++++++++++++++----- i14/templates/xrd2d-batch-template.yaml | 26 +++++++++++++++++------ 9 files changed, 164 insertions(+), 42 deletions(-) diff --git a/i14/templates/align-template.yaml b/i14/templates/align-template.yaml index e2dc2e0..5c50830 100644 --- a/i14/templates/align-template.yaml +++ b/i14/templates/align-template.yaml @@ -69,15 +69,25 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outpath }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb $OUTPUT/align-notebook.ipynb \ -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \ -p maskpath "{{`{{ inputs.parameters.maskpath }}`}}" \ -p complexpath "{{`{{ inputs.parameters.complexpath }}`}}" \ - -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ - --prepare-only - python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/align-notebook.ipynb" \ - --to html --execute --output align-output \ - --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + -p outpath $OUTPUT \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/align-notebook.ipynb \ + --to html --output align-output \ + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -89,6 +99,10 @@ spec: path: "{{`{{ inputs.parameters.outpath }}`}}/align-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outpath }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/dpc-batch-template.yaml b/i14/templates/dpc-batch-template.yaml index bb4ac52..759f872 100644 --- a/i14/templates/dpc-batch-template.yaml +++ b/i14/templates/dpc-batch-template.yaml @@ -72,17 +72,27 @@ spec: value: /tmp command: [bash] source: | + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}" + mkdir -p $OUTPUT + python -m papermill /tmp/notebook.ipynb \ - "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + $OUTPUT/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p process_configfile /tmp/config.yaml \ -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}_dpc-phase.nxs" \ -p auto_processing False \ - --prepare-only - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ - --to html --execute --no-input \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/dpc-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb \ + --to html --no-input \ --output dpc-output-{{`{{ inputs.parameters.scanNumber }}`}} \ - --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -94,6 +104,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/dpc-output-{{`{{ inputs.parameters.scanNumber }}`}}.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/mask-template.yaml b/i14/templates/mask-template.yaml index b9b0b49..f1ffbc9 100644 --- a/i14/templates/mask-template.yaml +++ b/i14/templates/mask-template.yaml @@ -76,17 +76,27 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outpath }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb $OUTPUT/mask-notebook.ipynb \ -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \ -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ -p padding "{{`{{ inputs.parameters.padding }}`}}" \ -p min_size "{{`{{ inputs.parameters.minSize }}`}}" \ -p erosion_radius "{{`{{ inputs.parameters.erosionRadius }}`}}" \ -p dilation_radius "{{`{{ inputs.parameters.dilationRadius }}`}}" \ - --prepare-only - python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/mask-notebook.ipynb" \ - --to html --execute --output mask-output \ - --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/mask-notebook.ipynb \ + --to html --output mask-output \ + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -102,6 +112,10 @@ spec: path: "{{`{{ inputs.parameters.outpath }}`}}/mask-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outpath }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/phase-template.yaml b/i14/templates/phase-template.yaml index 7502f28..842ae3e 100644 --- a/i14/templates/phase-template.yaml +++ b/i14/templates/phase-template.yaml @@ -68,16 +68,26 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outpath }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb $OUTPUT/phase-notebook.ipynb \ -p inpath "{{`{{ inputs.parameters.inpath }}`}}" \ -p maskpath "{{`{{ inputs.parameters.maskpath }}`}}" \ -p normalise "{{`{{ inputs.parameters.normalise }}`}}" \ -p outpath "{{`{{ inputs.parameters.outpath }}`}}" \ - --prepare-only - python -m jupyter nbconvert "{{`{{ inputs.parameters.outpath }}`}}/phase-notebook.ipynb" \ - --to html --execute \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/phase-notebook.ipynb \ + --to html \ --output phase-output \ --output-dir "{{`{{ inputs.parameters.outpath }}`}}" + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -89,6 +99,10 @@ spec: path: "{{`{{ inputs.parameters.outpath }}`}}/phase-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outpath }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/xanes-auto-template.yaml b/i14/templates/xanes-auto-template.yaml index cc31ea7..a8b076f 100644 --- a/i14/templates/xanes-auto-template.yaml +++ b/i14/templates/xanes-auto-template.yaml @@ -128,6 +128,8 @@ spec: value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/ command: [bash] source: | + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}" mkdir -p $OUTPUT @@ -147,10 +149,15 @@ spec: -p ref_index None \ -p max_fractional_shift 0.2 \ -p auto_processing False \ - --prepare-only + > $OUTPUT/papermill.log 2>&1 + python -m jupyter nbconvert $OUTPUT/xanes-notebook.ipynb \ - --to html --execute --output xanes-output \ + --to html --output xanes-output \ --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -162,6 +169,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/xanes-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-{{`{{ inputs.parameters.first }}`}}-{{`{{ inputs.parameters.last }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/xanes-point-template.yaml b/i14/templates/xanes-point-template.yaml index 0873b58..4ea16c1 100644 --- a/i14/templates/xanes-point-template.yaml +++ b/i14/templates/xanes-point-template.yaml @@ -72,16 +72,26 @@ spec: value: /tmp command: [bash] source: | - python -m papermill /tmp/notebook.ipynb "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" \ + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}" + mkdir -p $OUTPUT + + python -m papermill /tmp/notebook.ipynb $OUTPUT/xanes-point-notebook.ipynb \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ - -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}xanes_point.txt" \ + -p outpath "$OUTPUT/i14-{{`{{ inputs.parameters.scanNumber }}`}}xanes_point.txt" \ -p edge_element "{{`{{ inputs.parameters.edgeElement }}`}}" \ -p normalise "{{`{{ inputs.parameters.normalise }}`}}" \ -p auto_processing False \ - --prepare-only - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-notebook.ipynb" \ - --to html --execute --output xanes-point-output \ - --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert $OUTPUT/xanes-point-notebook.ipynb \ + --to html --output xanes-point-output \ + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -93,6 +103,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes-point-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/xanes-sparse-template.yaml b/i14/templates/xanes-sparse-template.yaml index 5822029..2b8e9ab 100644 --- a/i14/templates/xanes-sparse-template.yaml +++ b/i14/templates/xanes-sparse-template.yaml @@ -95,6 +95,8 @@ spec: value: /opt/venv/lib64/python3.12/site-packages/hdf5plugin/plugins/ command: [bash] source: | + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}" mkdir -p $OUTPUT @@ -113,10 +115,14 @@ spec: -p num_final_iteration 2000 \ -p normalised "{{`{{ inputs.parameters.normalise }}`}}" \ -p auto_processing False \ - --prepare-only + > $OUTPUT/papermill.log 2>&1 + python -m jupyter nbconvert $OUTPUT/xanes-sparse-notebook.ipynb \ - --to html --execute --output xanes-sparse-output \ - --output-dir $OUTPUT + --to html --output xanes-sparse-output + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -128,6 +134,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/xanes-sparse-output.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/xanes_sparse_{{`{{ inputs.parameters.first }}`}}_{{`{{ inputs.parameters.last }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/xrd1d-batch-template.yaml b/i14/templates/xrd1d-batch-template.yaml index 1ad4647..ae19452 100644 --- a/i14/templates/xrd1d-batch-template.yaml +++ b/i14/templates/xrd1d-batch-template.yaml @@ -100,20 +100,33 @@ spec: value: /tmp command: [bash] source: | + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}" + mkdir -p $OUTPUT + python -m papermill /tmp/notebook.ipynb \ - "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + "$OUTPUT/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p process_configfile "{{`{{ inputs.parameters.xrdProcessConfigUpload }}`}}" \ - -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd1d.nxs" \ + -p outpath "$OUTPUT/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd1d.nxs" \ -p calibration_path "{{`{{ inputs.parameters.calibrationPath }}`}}" \ -p mask_path "{{`{{ inputs.parameters.maskPath }}`}}" \ -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" \ -p background_type "{{`{{ inputs.parameters.backgroundType }}`}}" \ -p polynomial_order "{{`{{ inputs.parameters.polynomialOrder }}`}}" \ - --prepare-only - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ - --to html --execute --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert "$OUTPUT/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + --to html --output xanes-sparse-output + + python -m jupyter nbconvert "$OUTPUT/xrd1d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + --to html --no-input --output xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -125,6 +138,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xrd1d-output-{{`{{ inputs.parameters.scanNumber }}`}}.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main diff --git a/i14/templates/xrd2d-batch-template.yaml b/i14/templates/xrd2d-batch-template.yaml index e17ca05..0b80372 100644 --- a/i14/templates/xrd2d-batch-template.yaml +++ b/i14/templates/xrd2d-batch-template.yaml @@ -89,19 +89,29 @@ spec: value: /tmp command: [bash] source: | + ERROR_TXT="raise error" + ERROR_STR="An error arose within the executed notebook. Please open the output file to check." + OUTPUT="{{`{{ inputs.parameters.outputFolder }}`}}" + mkdir -p $OUTPUT + python -m papermill /tmp/notebook.ipynb \ - "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + "$OUTPUT/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ -p inpath "{{`{{ workflow.parameters.visitdir }}`}}/scan/i14-{{`{{ inputs.parameters.scanNumber }}`}}.nxs" \ -p process_configfile "{{`{{ inputs.parameters.xrdProcessConfigUpload }}`}}" \ - -p outpath "{{`{{ inputs.parameters.outputFolder }}`}}/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd2d.nxs" \ + -p outpath "$OUTPUT/i14-{{`{{ inputs.parameters.scanNumber }}`}}-xrd2d.nxs" \ -p calibration_path "{{`{{ inputs.parameters.calibrationPath }}`}}" \ -p mask_path "{{`{{ inputs.parameters.maskPath }}`}}" \ -p flat_path "{{`{{ inputs.parameters.flatPath }}`}}" \ - --prepare-only - python -m jupyter nbconvert "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ - --to html --execute --no-input \ + > $OUTPUT/papermill.log 2>&1 + + python -m jupyter nbconvert "$OUTPUT/xrd2d-notebook-{{`{{ inputs.parameters.scanNumber }}`}}.ipynb" \ + --to html --no-input \ --output xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}} \ - --output-dir "{{`{{ inputs.parameters.outputFolder }}`}}" + --output-dir $OUTPUT + + grep -q "$ERROR_TXT" "$OUTPUT/papermill.log" && { echo "$ERROR_STR" >&2; exit 1; } || exit 0 + # Raise an exit code to display failure of task if error text found in log + volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -113,6 +123,10 @@ spec: path: "{{`{{ inputs.parameters.outputFolder }}`}}/xrd2d-output-{{`{{ inputs.parameters.scanNumber }}`}}.html" archive: none: {} + - name: notebook-log + path: "{{`{{ inputs.parameters.outputFolder }}`}}/papermill.log" + archive: + none: {} podSpecPatch: | containers: - name: main From 7fe41816237daf19d23a80d9e42c67bd3f94f720 Mon Sep 17 00:00:00 2001 From: Jessica V <55289415+jessicavers@users.noreply.github.com> Date: Tue, 9 Jun 2026 15:29:44 +0100 Subject: [PATCH 11/51] Extended length of residuals array to fix shape mismatch (#206) --- i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb index 032fc4d..c225416 100644 --- a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb @@ -485,7 +485,8 @@ " tol=tol_residual, \n", " niter_short=num_short_iteration, \n", " niter_final=num_final_iteration, \n", - " verbose=True)" + " verbose=True) \n", + "loop_asd.residuals = np.full(sum(loop_asd.num_iters) + loop_asd.rank_max, np.nan) \n" ] }, { From 60a023115bc8c16db85940390ea0a7d2199a2dba Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Tue, 9 Jun 2026 15:48:15 +0100 Subject: [PATCH 12/51] Fixing ptypy P99 template --- ptypy/templates/ptypy-generic.yaml | 8 ++++---- ptypy/templates/ptypy-p99.yaml | 23 ++++++++++++++--------- 2 files changed, 18 insertions(+), 13 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index c21e51c..9380195 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -30,7 +30,7 @@ spec: parameters: - name: config - name: id - - name: output + - name: outpath - name: nprocs value: 1 - name: memory @@ -57,7 +57,7 @@ spec: - "-i" - "{{`{{ inputs.parameters.id }}`}}" - "-o" - - "{{`{{ inputs.parameters.output }}`}}" + - "{{`{{ inputs.parameters.outpath }}`}}" - "-s" - "hdf5_loader" - "-b" @@ -101,7 +101,7 @@ spec: parameters: - name: config - name: id - - name: output + - name: outpath - name: nprocs value: 1 - name: memory @@ -126,7 +126,7 @@ spec: - "-i" - "{{`{{ inputs.parameters.id }}`}}" - "-o" - - "{{`{{ inputs.parameters.output }}`}}" + - "{{`{{ inputs.parameters.outpath }}`}}" - "-s" - "hdf5_loader" volumeMounts: diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 0d288b1..bd8f6fd 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -1,5 +1,5 @@ apiVersion: argoproj.io/v1alpha1 -kind: ClusterWorkflowTemplate +kind: WorkflowTemplate metadata: name: ptypy-p99-from-config labels: @@ -86,12 +86,12 @@ spec: - name: tmp mountPath: /tmp command: [bash] - source: | - OUTPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.outpath }}`}}" - mkdir -p $OUTPUT - INPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.inpath }}`}}" - echo '{{ .Files.Get "config/p99_dm.yaml" | b64enc }}' | base64 -d > $OUTPUT/ptypy_config.yaml - sed -i 's;;$INPUT;' $OUTPUT/ptypy_config.yaml + source: | + OUTPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.outpath }}`}}" + mkdir -p $OUTPUT + INPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.inpath }}`}}" + echo '{{ .Files.Get "config/p99_dm.yaml" | b64enc }}' | base64 -d > $OUTPUT/ptypy_config.yaml + sed -i 's;;$INPUT;' $OUTPUT/ptypy_config.yaml outputs: parameters: - name: config @@ -100,6 +100,11 @@ spec: - name: reconfile valueFrom: path: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/test.ptyr" + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory - name: ptypy-pipeline dag: @@ -116,7 +121,7 @@ spec: arguments: parameters: - name: config - value: "{{`{{ tasks.prepare.output.parameters.config }}`}}" + value: "{{`{{ tasks.prepare.outputs.parameters.config }}`}}" - name: id value: "{{`{{ workflows.parameters.id }}`}}" - name: outpath @@ -135,7 +140,7 @@ spec: arguments: parameters: - name: config - value: "{{`{{ tasks.prepare.output.parameters.config }}`}}" + value: "{{`{{ tasks.prepare.outputs.parameters.config }}`}}" - name: id value: "{{`{{ workflows.parameters.id }}`}}" - name: outpath From 9cfb8dd2efd2a6a3db53e225d24b543f7e5f4936 Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Wed, 10 Jun 2026 15:53:11 +0100 Subject: [PATCH 13/51] Get p99 ptypy workflow to work --- ptypy/config/p99_dm.yaml | 1 + ptypy/templates/ptypy-p99.yaml | 33 ++++++++++++++++----------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/ptypy/config/p99_dm.yaml b/ptypy/config/p99_dm.yaml index 5f8401c..aca07b4 100755 --- a/ptypy/config/p99_dm.yaml +++ b/ptypy/config/p99_dm.yaml @@ -1,3 +1,4 @@ +base_file: null visit_path: parameter_tree: run: '' diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index bd8f6fd..ac7c9f3 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -1,5 +1,5 @@ apiVersion: argoproj.io/v1alpha1 -kind: WorkflowTemplate +kind: ClusterWorkflowTemplate metadata: name: ptypy-p99-from-config labels: @@ -65,7 +65,7 @@ spec: - name: memory value: 20Gi - name: usegpu - value: "false" + value: false volumeClaimTemplates: - metadata: name: tmpdir @@ -83,7 +83,7 @@ spec: volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" - - name: tmp + - name: tmpdir mountPath: /tmp command: [bash] source: | @@ -92,14 +92,13 @@ spec: INPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.inpath }}`}}" echo '{{ .Files.Get "config/p99_dm.yaml" | b64enc }}' | base64 -d > $OUTPUT/ptypy_config.yaml sed -i 's;;$INPUT;' $OUTPUT/ptypy_config.yaml + cat $OUTPUT/ptypy_config.yaml outputs: parameters: - name: config - valueFrom: - path: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/ptypy_config.yaml" + value: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/ptypy_config.yaml" - name: reconfile - valueFrom: - path: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/test.ptyr" + value: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/test.ptyr" volumes: - name: session hostPath: @@ -112,7 +111,7 @@ spec: - name: prepare template: ptypy-setup - name: reconstruct-with-gpu - when: "{{`{{ workflows.parameters.usegpu }} == true`}}" + when: "{{`{{ workflow.parameters.usegpu }}`}} == true" templateRef: name: ptypy-generic template: ptypy-gpu-run @@ -123,15 +122,15 @@ spec: - name: config value: "{{`{{ tasks.prepare.outputs.parameters.config }}`}}" - name: id - value: "{{`{{ workflows.parameters.id }}`}}" + value: "{{`{{ workflow.parameters.id }}`}}" - name: outpath - value: "{{`{{ workflows.parameters.outpath }}`}}" + value: "{{`{{ workflow.parameters.outpath }}`}}" - name: nprocs - value: "{{`{{ workflows.parameters.nprocs }}`}}" + value: "{{`{{ workflow.parameters.nprocs }}`}}" - name: memory - value: "{{`{{ workflows.parameters.memory }}`}}" + value: "{{`{{ workflow.parameters.memory }}`}}" - name: reconstruct-with-cpu - when: "{{`{{ workflows.parameters.usegpu }} == false `}}" + when: "{{`{{ workflow.parameters.usegpu }}`}} == false" templateRef: name: ptypy-generic template: ptypy-cpu-run @@ -142,13 +141,13 @@ spec: - name: config value: "{{`{{ tasks.prepare.outputs.parameters.config }}`}}" - name: id - value: "{{`{{ workflows.parameters.id }}`}}" + value: "{{`{{ workflow.parameters.id }}`}}" - name: outpath - value: "{{`{{ workflows.parameters.outpath }}`}}" + value: "{{`{{ workflow.parameters.outpath }}`}}" - name: nprocs - value: "{{`{{ workflows.parameters.nprocs }}`}}" + value: "{{`{{ workflow.parameters.nprocs }}`}}" - name: memory - value: "{{`{{ workflows.parameters.memory }}`}}" + value: "{{`{{ workflow.parameters.memory }}`}}" - name: plot templateRef: name: ptypy-utils From a2225f0f184d5545e28ba46b07ca0e2951e563d2 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Wed, 10 Jun 2026 16:22:15 +0100 Subject: [PATCH 14/51] fix input path injection in p99 ptypy --- ptypy/templates/ptypy-p99.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index ac7c9f3..ec72339 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -91,7 +91,7 @@ spec: mkdir -p $OUTPUT INPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.inpath }}`}}" echo '{{ .Files.Get "config/p99_dm.yaml" | b64enc }}' | base64 -d > $OUTPUT/ptypy_config.yaml - sed -i 's;;$INPUT;' $OUTPUT/ptypy_config.yaml + sed -i "s;;$INPUT;" $OUTPUT/ptypy_config.yaml cat $OUTPUT/ptypy_config.yaml outputs: parameters: From af8e0dae9dbf7502017a0878d243979d6c7a2690 Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Thu, 11 Jun 2026 11:22:36 +0100 Subject: [PATCH 15/51] inject cupy engine for GPU workflow --- ptypy/templates/ptypy-generic.yaml | 4 ++-- ptypy/templates/ptypy-p99.yaml | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 9380195..5624deb 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -36,7 +36,7 @@ spec: - name: memory value: 20Gi container: - image: ghcr.io/ptycho/ptypy_cupy_openmpi:v0.9.0 + image: ghcr.io/ptycho/ptypy_cupy_openmpi:0.9.1 env: - name: CUPY_CACHE_DIR value: /tmp/.cupy/kernel_cache @@ -107,7 +107,7 @@ spec: - name: memory value: 20Gi container: - image: ghcr.io/ptycho/ptypy_full_openmpi:v0.9.0 + image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 env: - name: MKL_NUM_THREADS value: "1" diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index ec72339..eeef312 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -92,6 +92,7 @@ spec: INPUT="{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.inpath }}`}}" echo '{{ .Files.Get "config/p99_dm.yaml" | b64enc }}' | base64 -d > $OUTPUT/ptypy_config.yaml sed -i "s;;$INPUT;" $OUTPUT/ptypy_config.yaml + "{{`{{ workflow.parameters.usegpu }}`}}" && sed -i "s;DM;DM_cupy;" $OUTPUT/ptypy_config.yaml cat $OUTPUT/ptypy_config.yaml outputs: parameters: From 35deaf62460805ef1a4da4e81f346143aad4b4d8 Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Thu, 11 Jun 2026 15:09:37 +0100 Subject: [PATCH 16/51] Remove reconfile argument --- ptypy/templates/ptypy-p99.yaml | 7 ------- ptypy/templates/ptypy-utils.yaml | 3 ++- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index eeef312..8b5ba54 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -93,13 +93,10 @@ spec: echo '{{ .Files.Get "config/p99_dm.yaml" | b64enc }}' | base64 -d > $OUTPUT/ptypy_config.yaml sed -i "s;;$INPUT;" $OUTPUT/ptypy_config.yaml "{{`{{ workflow.parameters.usegpu }}`}}" && sed -i "s;DM;DM_cupy;" $OUTPUT/ptypy_config.yaml - cat $OUTPUT/ptypy_config.yaml outputs: parameters: - name: config value: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/ptypy_config.yaml" - - name: reconfile - value: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}/test.ptyr" volumes: - name: session hostPath: @@ -155,7 +152,3 @@ spec: template: ptypy-plotting clusterScope: true depends: "reconstruct-with-gpu || reconstruct-with-cpu" - arguments: - parameters: - - name: recon - value: "{{`{{ tasks.prepare.outputs.parameters.reconfile }}`}}" diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index c03c1ec..49cae02 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -29,8 +29,9 @@ spec: inputs: parameters: - name: recon + value: test.ptyr script: - image: ghcr.io/ptycho/ptypy_full_openmpi:v0.9.0 + image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 env: - name: MKL_NUM_THREADS value: "1" From b3e8b2f2fbb3c6284b7798c830091ec5eeba5489 Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Thu, 11 Jun 2026 15:29:29 +0100 Subject: [PATCH 17/51] temporarily pull image always --- ptypy/templates/ptypy-generic.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 5624deb..35847bd 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -37,6 +37,7 @@ spec: value: 20Gi container: image: ghcr.io/ptycho/ptypy_cupy_openmpi:0.9.1 + imagePullPolicy: Always env: - name: CUPY_CACHE_DIR value: /tmp/.cupy/kernel_cache From 8cc593c8e204882763ed48d6bfc3764633f6c952 Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Thu, 11 Jun 2026 15:37:18 +0100 Subject: [PATCH 18/51] Provide complete path to output --- ptypy/templates/ptypy-p99.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 8b5ba54..952548b 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -122,7 +122,7 @@ spec: - name: id value: "{{`{{ workflow.parameters.id }}`}}" - name: outpath - value: "{{`{{ workflow.parameters.outpath }}`}}" + value: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}" - name: nprocs value: "{{`{{ workflow.parameters.nprocs }}`}}" - name: memory @@ -141,7 +141,7 @@ spec: - name: id value: "{{`{{ workflow.parameters.id }}`}}" - name: outpath - value: "{{`{{ workflow.parameters.outpath }}`}}" + value: "{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}" - name: nprocs value: "{{`{{ workflow.parameters.nprocs }}`}}" - name: memory From 23113c03f5194e28d5e1a883ca161a3a90316ca7 Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Thu, 11 Jun 2026 15:43:59 +0100 Subject: [PATCH 19/51] Revert image pulling policy change --- ptypy/templates/ptypy-generic.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 35847bd..5624deb 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -37,7 +37,6 @@ spec: value: 20Gi container: image: ghcr.io/ptycho/ptypy_cupy_openmpi:0.9.1 - imagePullPolicy: Always env: - name: CUPY_CACHE_DIR value: /tmp/.cupy/kernel_cache From 08f001b5e6ef08fa37b2c92b9308acbe9aaba8f6 Mon Sep 17 00:00:00 2001 From: Benedikt Daurer Date: Thu, 11 Jun 2026 15:49:48 +0100 Subject: [PATCH 20/51] Fix p99 setup --- ptypy/templates/ptypy-p99.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 952548b..5776bcc 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -93,6 +93,7 @@ spec: echo '{{ .Files.Get "config/p99_dm.yaml" | b64enc }}' | base64 -d > $OUTPUT/ptypy_config.yaml sed -i "s;;$INPUT;" $OUTPUT/ptypy_config.yaml "{{`{{ workflow.parameters.usegpu }}`}}" && sed -i "s;DM;DM_cupy;" $OUTPUT/ptypy_config.yaml + cat $OUTPUT/ptypy_config.yaml outputs: parameters: - name: config From cdf9ef82360ca810e5752ddd93f6d2e696ad5774 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Tue, 16 Jun 2026 16:27:20 +0100 Subject: [PATCH 21/51] add plotting to ptypy workflows --- ptypy/notebooks/ptypy_plotting.ipynb | 416 +++++++++++++++++++++++++++ ptypy/templates/ptypy-generic.yaml | 6 + ptypy/templates/ptypy-utils.yaml | 39 ++- 3 files changed, 455 insertions(+), 6 deletions(-) create mode 100644 ptypy/notebooks/ptypy_plotting.ipynb diff --git a/ptypy/notebooks/ptypy_plotting.ipynb b/ptypy/notebooks/ptypy_plotting.ipynb new file mode 100644 index 0000000..1e25275 --- /dev/null +++ b/ptypy/notebooks/ptypy_plotting.ipynb @@ -0,0 +1,416 @@ +{ + "cells": [ + { + "cell_type": "code", + "execution_count": null, + "id": "d36dc2c0", + "metadata": { + "tags": [ + "parameters" + ], + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "recon_file = \"\"" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "59a4658d", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "import os\n", + "import h5py\n", + "import numpy as np\n", + "import matplotlib.pyplot as plt\n", + "import ptypy" + ] + }, + { + "cell_type": "markdown", + "id": "be4c5e89", + "metadata": {}, + "source": [ + "### Set locations for artefacts" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "2c8d26f8", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "ARTEFACTS = {\n", + " 'amplitude': '/tmp/amplitude_plot.png',\n", + " 'phase': '/tmp/phase_plot.png',\n", + " 'probe_amp': '/tmp/probe_amplitude_plot.png',\n", + " 'probe_phase': '/tmp/probe_phase_plot.png',\n", + " 'errors': '/tmp/error_plot.png',\n", + "}" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "24d4d677", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "if not os.path.exists(recon_file):\n", + " raise ValueError(f\"{recon_file} does not exist!\")" + ] + }, + { + "cell_type": "markdown", + "id": "8013201f", + "metadata": {}, + "source": [ + "## Load reconstruction from ptyr file" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ca78c318", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "with h5py.File(recon_file, \"r\") as f:\n", + " pixelsize = f['content/obj/Sscan_00G00/_psize'][0]*1e9\n", + " probe = np.array(f['content/probe/Sscan_00G00/data'][:])\n", + " obj = np.array(f['content/obj/Sscan_00G00/data'][0,:,:])\n", + " Niter = len(f['content/runtime/iter_info/'])\n", + " errors = np.array([f[\"content/runtime/iter_info/%05d/error\" %i][:] for i in range(Niter)])\n", + " niters = np.array([f[\"content/runtime/iter_info/%05d/iteration\" %i][...] for i in range(Niter)])\n", + " energy = f[\"content/probe/Sscan_00G00/_energy\"][...]\n", + " wavelength = 1.239/energy\n", + "e, v = ptypy.utils.ortho(probe)\n", + "print(\"Pixelsize = %.3f nm\" %(pixelsize))\n", + "print(\"Photon energy = %f\" %(energy*1000))" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b155ec18", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(8,4), dpi=100)\n", + "plt.plot(niters, errors[:,1])\n", + "plt.ylabel(\"Log-likelihood\")\n", + "plt.xlabel(\"Nr. of iterations\")\n", + "plt.savefig(ARTEFACTS['errors'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "b7f5117a", + "metadata": {}, + "source": [ + "## Plot amplitude and phase histograms" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0e39d83e", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "plt.hist(np.abs(obj).flatten(), bins=100)\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "369868bc", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "plt.hist(np.angle(obj).flatten(), bins=100)\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "6bca57ea", + "metadata": {}, + "source": [ + "## Plot amplitude" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "611cf3c1", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(6,5),dpi=100)\n", + "plt.axis(\"off\")\n", + "plt.imshow(np.abs(obj), cmap=\"gray\", interpolation='none',vmin=0, vmax=5)\n", + "plt.gca().add_patch(plt.Rectangle((10,10),1000/pixelsize,5, color='w'))\n", + "plt.gca().text(10+1000/pixelsize/2,20,r\"%d um\" %(1000/1e3), color='w', va='top', ha='center', fontsize=15)\n", + "plt.colorbar()\n", + "plt.savefig(ARTEFACTS['amplitude'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "43da45cb", + "metadata": {}, + "source": [ + "## Plot phase" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "beaacd77", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "plt.figure(figsize=(6,5), dpi=100)\n", + "plt.axis(\"off\")\n", + "plt.imshow(np.angle(obj), cmap=\"gray\", vmin=-1*np.pi, vmax=1*np.pi, interpolation='none')\n", + "plt.gca().add_patch(plt.Rectangle((10,10),1000/pixelsize,5, color='w'))\n", + "plt.gca().text(10+1000/pixelsize/2,20,r\"%d μm\" %(1000/1e3), color='w', va='top', ha='center', fontsize=15)\n", + "plt.colorbar()\n", + "plt.savefig(ARTEFACTS['phase'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "0e21559c", + "metadata": {}, + "source": [ + "## Plot probe" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "936aca8a", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "m = 0\n", + "fig = plt.figure(figsize=(5,5))\n", + "ax = ptypy.utils.PtyAxis(fig.add_subplot(111), channel='a', vmin=None, vmax=None)\n", + "ax.set_data(np.abs(v[m]))\n", + "ax.ax.axis(\"off\")\n", + "ax.ax.add_patch(plt.Rectangle((5,5),1000/pixelsize,10, color='w'))\n", + "ax.ax.text(5+1000/pixelsize/2,20,r\"%d μm\" %(1000/1e3), color='w', va='top', ha='center', fontsize=20)\n", + "ax.ax.set_title(f\"Probe Amplitude | {e[m]*100:.1f} %\")\n", + "fig.savefig(ARTEFACTS['probe_amp'])\n", + "plt.show()" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "63c74baa", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "m = 0\n", + "fig = plt.figure(figsize=(5,5))\n", + "ax = ptypy.utils.PtyAxis(fig.add_subplot(111), channel='a', vmin=None, vmax=None)\n", + "ax.set_data(np.angle(v[m]))\n", + "ax.ax.set_title(f\"Probe Phase | {e[m]*100:.1f}\")\n", + "ax.ax.axis(\"off\")\n", + "fig.savefig(ARTEFACTS['probe_phase'])\n", + "plt.show()" + ] + }, + { + "cell_type": "markdown", + "id": "7ab09476", + "metadata": {}, + "source": [ + "## Plot propagation" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "0aa87921", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "import ptypy.utils as u\n", + "from ptypy.core import geometry\n", + "\n", + "def nfprop(energy, distance, resolution, shape):\n", + " g = u.Param()\n", + " g.energy = energy\n", + " g.distance = distance\n", + " g.psize = resolution\n", + " g.shape = shape\n", + " g.propagation = \"nearfield\"\n", + " G = geometry.Geo(owner=None, pars=g)\n", + " return G.propagator.fw\n", + "\n", + "def propagated(probe, distances, energy, resolution, eps=1e-9):\n", + " prop_array = np.zeros((len(distances),)+probe.shape, dtype=probe.dtype)\n", + " for i,d in enumerate(distances):\n", + " if np.abs(d) < eps:\n", + " prop_array[i] = probe.copy()\n", + " else:\n", + " prop_array[i] = nfprop(energy, d, resolution, probe.shape)(probe)\n", + " return prop_array" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "ad7d8878", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "prb = v[0]\n", + "full_defocus_range = np.linspace(-50e-6, 50e-6, 201)\n", + "main_probe = propagated(prb, full_defocus_range, energy, pixelsize*1e-9)" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "7ade75fc", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "cz,cy,cx = np.unravel_index(np.abs(main_probe).argmax(),main_probe.shape)\n", + "ny,nx = prb.shape\n", + "fz,fy,fx = 100,cy,cx" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "e6fa9ef6", + "metadata": { + "vscode": { + "languageId": "plaintext" + } + }, + "outputs": [], + "source": [ + "fig = plt.figure(figsize=(12,4),dpi=100)\n", + "axy = fig.add_subplot(211)\n", + "axx = fig.add_subplot(212)\n", + "axz = fig.add_axes([-0.2,0.025, 0.3,0.85])\n", + "\n", + "Axz = ptypy.utils.PtyAxis(axz, channel='a', vmax=0.25*np.abs(main_probe).max())\n", + "Axz.set_data(main_probe[fz])\n", + "axz.axvline(fx, color='w', ls=':')\n", + "axz.axhline(fy, color='w', ls=':')\n", + "axz.axis('off')\n", + "axz.add_patch(plt.Rectangle((5,5),1000/pixelsize,10, color='w'))\n", + "axz.text(5+1000/pixelsize/2,20,r\"%d μm\" %(1000/1e3), color='w', va='top', ha='center', fontsize=10)\n", + "\n", + "Axy = ptypy.utils.PtyAxis(axy, channel='a', vmax=0.5*np.abs(main_probe).max())\n", + "Axy.set_data(main_probe[:,fy].T)\n", + "axy.set_aspect('auto')\n", + "axy.axvline(fz, color='w', ls=':')\n", + "axy.axvline(cz, color='r', ls=':')\n", + "axy.set_xticks([])\n", + "axy.set_yticks(range(ny)[5::50])\n", + "axy.set_yticklabels(np.round(np.linspace(-cy*pixelsize/1e3, cy*pixelsize/1e3,nx)[5::50],1))\n", + "axy.tick_params('y', right=True, left=False, labelright=True, labelleft=False)\n", + "axy.set_ylabel(\"horiz. position [μm]\")\n", + "axy.yaxis.set_label_position(\"right\")\n", + "\n", + "Axx = ptypy.utils.PtyAxis(axx, channel='a', vmax=0.5*np.abs(main_probe).max())\n", + "Axx.set_data(main_probe[:,:,fx].T)\n", + "axx.axvline(fz, color='w', ls=':')\n", + "axx.axvline(cz, color='r', ls=':')\n", + "axx.set_aspect('auto')\n", + "axx.set_xticks(range(len(full_defocus_range))[::20])\n", + "axx.set_xticklabels(np.round(full_defocus_range[::20] * 1e6).astype(int))\n", + "axx.set_yticks(range(ny)[5::50])\n", + "axx.set_yticklabels(np.round(np.linspace(-cx*pixelsize/1e3, cx*pixelsize/1e3,nx)[5::50],1))\n", + "axx.tick_params('y', right=True, left=False, labelright=True, labelleft=False)\n", + "axx.set_xlabel(\"Defocus [μm]\")\n", + "axx.set_ylabel(\"vert. position [μm]\")\n", + "axx.yaxis.set_label_position(\"right\")\n", + "\n", + "plt.show()" + ] + } + ], + "metadata": { + "language_info": { + "name": "python" + } + }, + "nbformat": 4, + "nbformat_minor": 5 +} diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 5624deb..aca5236 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -62,6 +62,9 @@ spec: - "hdf5_loader" - "-b" - "cupy" + - "|" + - "tee" + - "/tmp/ptypy_log.txt" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -129,6 +132,9 @@ spec: - "{{`{{ inputs.parameters.outpath }}`}}" - "-s" - "hdf5_loader" + - "|" + - "tee" + - "/tmp/ptypy_log.txt" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index 49cae02..5cb629e 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -26,10 +26,6 @@ spec: key: data_directory templates: - name: ptypy-plotting - inputs: - parameters: - - name: recon - value: test.ptyr script: image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 env: @@ -41,13 +37,44 @@ spec: value: "1" command: [bash] source: | - echo "{{`{{ inputs.parameters.recon }}`}}" - + pip install jupyter papermill nbconvert + file_path=$(sed -rn "s/^Saving to (.*)$/\1/p" /tmp/ptypy_log.txt) + # mount notebook + echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb + # run notebook + python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb -p recon_file "$file_path" + python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" - name: tmpdir mountPath: /tmp + outputs: + artifacts: + - name: notebook + path: /tmp/notebook.html + archive: + none: {} + - name: amplitude + path: /tmp/amplitude_plot.png + archive: + none: {} + - name: phase + path: /tmp/phase_plot.png + archive: + none: {} + - name: probe-amplitude + path: /tmp/probe_amplitude_plot.png + archive: + none: {} + - name: probe-phase + path: /tmp/probe_phase_plot.png + archive: + none: {} + - name: errors + path: /tmp/error_plot.png + archive: + none: {} podSpecPatch: | containers: - name: main From 58a77d285939092e279806ae2413761afd14a6a9 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Wed, 17 Jun 2026 15:39:18 +0100 Subject: [PATCH 22/51] update ptypy-utils --- ptypy/templates/ptypy-utils.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index 5cb629e..ce56d5d 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -37,7 +37,7 @@ spec: value: "1" command: [bash] source: | - pip install jupyter papermill nbconvert + sudo -H pip install jupyter papermill nbconvert file_path=$(sed -rn "s/^Saving to (.*)$/\1/p" /tmp/ptypy_log.txt) # mount notebook echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb From 9f2ed9e1664f2e51d120837fe3217f43eddb5ed1 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Thu, 18 Jun 2026 10:33:11 +0100 Subject: [PATCH 23/51] update ptypy-utils --- ptypy/templates/ptypy-utils.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index ce56d5d..43f141b 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -37,7 +37,7 @@ spec: value: "1" command: [bash] source: | - sudo -H pip install jupyter papermill nbconvert + mamba install jupyter papermill nbconvert file_path=$(sed -rn "s/^Saving to (.*)$/\1/p" /tmp/ptypy_log.txt) # mount notebook echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb From 5777384c798cffaaccedd9d733f5910f91796f0b Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Thu, 18 Jun 2026 10:55:00 +0100 Subject: [PATCH 24/51] update ptypy-generic --- ptypy/templates/ptypy-generic.yaml | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index aca5236..03ce718 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -47,8 +47,9 @@ spec: - name: OMP_NUM_THREADS value: "1" command: - - mpirun + - bash args: + - "mpirun" - "-n" - "{{`{{ inputs.parameters.nprocs }}`}}" - "ptypy.cli" @@ -119,8 +120,9 @@ spec: - name: OMP_NUM_THREADS value: "1" command: - - mpirun + - bash args: + - "mpirun" - "-n" - "{{`{{ inputs.parameters.nprocs }}`}}" - "ptypy.cli" From b62857113f7df908464b1491ee29ba52c901dc01 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Thu, 18 Jun 2026 11:53:48 +0100 Subject: [PATCH 25/51] update ptypy-utils --- ptypy/templates/ptypy-utils.yaml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index 43f141b..7248950 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -37,13 +37,14 @@ spec: value: "1" command: [bash] source: | - mamba install jupyter papermill nbconvert + python -m venv --system-site-packages /tmp/venv + /tmp/venv/bin/pip install jupyter papermill nbconvert file_path=$(sed -rn "s/^Saving to (.*)$/\1/p" /tmp/ptypy_log.txt) # mount notebook echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb # run notebook - python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb -p recon_file "$file_path" - python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb + /tmp/venv/bin/python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb -p recon_file "$file_path" + /tmp/venv/bin/python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From 5d086a20cdaab3adb8672223f44292b34921884c Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Thu, 18 Jun 2026 13:31:37 +0100 Subject: [PATCH 26/51] update ptypy_plotting notebook --- ptypy/notebooks/ptypy_plotting.ipynb | 103 ++++++--------------------- 1 file changed, 23 insertions(+), 80 deletions(-) diff --git a/ptypy/notebooks/ptypy_plotting.ipynb b/ptypy/notebooks/ptypy_plotting.ipynb index 1e25275..882058d 100644 --- a/ptypy/notebooks/ptypy_plotting.ipynb +++ b/ptypy/notebooks/ptypy_plotting.ipynb @@ -7,10 +7,7 @@ "metadata": { "tags": [ "parameters" - ], - "vscode": { - "languageId": "plaintext" - } + ] }, "outputs": [], "source": [ @@ -21,11 +18,7 @@ "cell_type": "code", "execution_count": null, "id": "59a4658d", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "import os\n", @@ -47,11 +40,7 @@ "cell_type": "code", "execution_count": null, "id": "2c8d26f8", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "ARTEFACTS = {\n", @@ -67,11 +56,7 @@ "cell_type": "code", "execution_count": null, "id": "24d4d677", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "if not os.path.exists(recon_file):\n", @@ -90,11 +75,7 @@ "cell_type": "code", "execution_count": null, "id": "ca78c318", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "with h5py.File(recon_file, \"r\") as f:\n", @@ -115,11 +96,7 @@ "cell_type": "code", "execution_count": null, "id": "b155ec18", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "plt.figure(figsize=(8,4), dpi=100)\n", @@ -142,11 +119,7 @@ "cell_type": "code", "execution_count": null, "id": "0e39d83e", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "plt.hist(np.abs(obj).flatten(), bins=100)\n", @@ -157,11 +130,7 @@ "cell_type": "code", "execution_count": null, "id": "369868bc", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "plt.hist(np.angle(obj).flatten(), bins=100)\n", @@ -180,11 +149,7 @@ "cell_type": "code", "execution_count": null, "id": "611cf3c1", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "plt.figure(figsize=(6,5),dpi=100)\n", @@ -209,11 +174,7 @@ "cell_type": "code", "execution_count": null, "id": "beaacd77", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "plt.figure(figsize=(6,5), dpi=100)\n", @@ -238,11 +199,7 @@ "cell_type": "code", "execution_count": null, "id": "936aca8a", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "m = 0\n", @@ -261,11 +218,7 @@ "cell_type": "code", "execution_count": null, "id": "63c74baa", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "m = 0\n", @@ -290,11 +243,7 @@ "cell_type": "code", "execution_count": null, "id": "0aa87921", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "import ptypy.utils as u\n", @@ -324,11 +273,7 @@ "cell_type": "code", "execution_count": null, "id": "ad7d8878", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "prb = v[0]\n", @@ -340,11 +285,7 @@ "cell_type": "code", "execution_count": null, "id": "7ade75fc", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "cz,cy,cx = np.unravel_index(np.abs(main_probe).argmax(),main_probe.shape)\n", @@ -356,11 +297,7 @@ "cell_type": "code", "execution_count": null, "id": "e6fa9ef6", - "metadata": { - "vscode": { - "languageId": "plaintext" - } - }, + "metadata": {}, "outputs": [], "source": [ "fig = plt.figure(figsize=(12,4),dpi=100)\n", @@ -407,8 +344,14 @@ } ], "metadata": { + "kernelspec": { + "display_name": "cuda12.6", + "language": "python", + "name": "python3" + }, "language_info": { - "name": "python" + "name": "python", + "version": "3.11.10" } }, "nbformat": 4, From 570a3cea95b3225894630114364eaaa229ee8b39 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Thu, 18 Jun 2026 14:03:02 +0100 Subject: [PATCH 27/51] update ptypy-generic --- ptypy/templates/ptypy-generic.yaml | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 03ce718..dc36a7d 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -63,8 +63,7 @@ spec: - "hdf5_loader" - "-b" - "cupy" - - "|" - - "tee" + - ">" - "/tmp/ptypy_log.txt" volumeMounts: - name: session @@ -134,8 +133,7 @@ spec: - "{{`{{ inputs.parameters.outpath }}`}}" - "-s" - "hdf5_loader" - - "|" - - "tee" + - ">" - "/tmp/ptypy_log.txt" volumeMounts: - name: session From 585eac834ea541414cbe18d47737549cc0c56ee5 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Thu, 18 Jun 2026 14:22:29 +0100 Subject: [PATCH 28/51] update ptypy-generic --- ptypy/templates/ptypy-generic.yaml | 20 ++++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index dc36a7d..fcef484 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -35,6 +35,12 @@ spec: value: 1 - name: memory value: 20Gi + outputs: + artifacts: + - name: ptypy-gpu-run-log + path: /tmp/ptypy_log.txt + archive: + none: {} container: image: ghcr.io/ptycho/ptypy_cupy_openmpi:0.9.1 env: @@ -50,6 +56,8 @@ spec: - bash args: - "mpirun" + - "--output-filename" + - "/tmp/ptypy_log.txt" - "-n" - "{{`{{ inputs.parameters.nprocs }}`}}" - "ptypy.cli" @@ -63,8 +71,6 @@ spec: - "hdf5_loader" - "-b" - "cupy" - - ">" - - "/tmp/ptypy_log.txt" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -109,6 +115,12 @@ spec: value: 1 - name: memory value: 20Gi + outputs: + artifacts: + - name: ptypy-cpu-run-log + path: /tmp/ptypy_log.txt + archive: + none: {} container: image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 env: @@ -122,6 +134,8 @@ spec: - bash args: - "mpirun" + - "--output-filename" + - "/tmp/ptypy_log.txt" - "-n" - "{{`{{ inputs.parameters.nprocs }}`}}" - "ptypy.cli" @@ -133,8 +147,6 @@ spec: - "{{`{{ inputs.parameters.outpath }}`}}" - "-s" - "hdf5_loader" - - ">" - - "/tmp/ptypy_log.txt" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From 4bab75717536d7c19a1b310a2b613bb9bcf6652f Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Thu, 18 Jun 2026 15:09:39 +0100 Subject: [PATCH 29/51] update ptypy-generic --- ptypy/templates/ptypy-generic.yaml | 14 ++++---------- 1 file changed, 4 insertions(+), 10 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index fcef484..f169ca6 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -35,12 +35,6 @@ spec: value: 1 - name: memory value: 20Gi - outputs: - artifacts: - - name: ptypy-gpu-run-log - path: /tmp/ptypy_log.txt - archive: - none: {} container: image: ghcr.io/ptycho/ptypy_cupy_openmpi:0.9.1 env: @@ -56,8 +50,6 @@ spec: - bash args: - "mpirun" - - "--output-filename" - - "/tmp/ptypy_log.txt" - "-n" - "{{`{{ inputs.parameters.nprocs }}`}}" - "ptypy.cli" @@ -71,6 +63,8 @@ spec: - "hdf5_loader" - "-b" - "cupy" + - ">" + - "/tmp/ptypy_log.txt" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -134,8 +128,6 @@ spec: - bash args: - "mpirun" - - "--output-filename" - - "/tmp/ptypy_log.txt" - "-n" - "{{`{{ inputs.parameters.nprocs }}`}}" - "ptypy.cli" @@ -147,6 +139,8 @@ spec: - "{{`{{ inputs.parameters.outpath }}`}}" - "-s" - "hdf5_loader" + - ">" + - "/tmp/ptypy_log.txt" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From 299a47c5df205683ba323c06bb1391c9ab19bf5f Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Fri, 19 Jun 2026 08:48:49 +0100 Subject: [PATCH 30/51] update ptypy-generic --- ptypy/templates/ptypy-generic.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index f169ca6..14496db 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -64,7 +64,9 @@ spec: - "-b" - "cupy" - ">" - - "/tmp/ptypy_log.txt" + - ">(tee -a /tmp/ptypy_log.txt)" + - "2>" + - ">(tee -a /tmp/ptypy_log.txt)" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -140,7 +142,9 @@ spec: - "-s" - "hdf5_loader" - ">" - - "/tmp/ptypy_log.txt" + - ">(tee -a /tmp/ptypy_log.txt)" + - "2>" + - ">(tee -a /tmp/ptypy_log.txt)" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From 588a5841f9a8e7ee0b598675a1517d24f2a14f19 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Fri, 19 Jun 2026 09:22:23 +0100 Subject: [PATCH 31/51] check /tmp for ptypy --- ptypy/templates/ptypy-generic.yaml | 4 ++++ ptypy/templates/ptypy-utils.yaml | 13 +++++++------ 2 files changed, 11 insertions(+), 6 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 14496db..992a4fe 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -67,6 +67,8 @@ spec: - ">(tee -a /tmp/ptypy_log.txt)" - "2>" - ">(tee -a /tmp/ptypy_log.txt)" + - ";" + - "ls /tmp" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -145,6 +147,8 @@ spec: - ">(tee -a /tmp/ptypy_log.txt)" - "2>" - ">(tee -a /tmp/ptypy_log.txt)" + - ";" + - "ls /tmp" volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index 7248950..b005817 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -37,14 +37,15 @@ spec: value: "1" command: [bash] source: | - python -m venv --system-site-packages /tmp/venv - /tmp/venv/bin/pip install jupyter papermill nbconvert - file_path=$(sed -rn "s/^Saving to (.*)$/\1/p" /tmp/ptypy_log.txt) + ls /tmp + #python -m venv --system-site-packages /tmp/venv + #/tmp/venv/bin/pip install jupyter papermill nbconvert + #file_path=$(sed -rn "s/^Saving to (.*)$/\1/p" /tmp/ptypy_log.txt) # mount notebook - echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb + #echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb # run notebook - /tmp/venv/bin/python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb -p recon_file "$file_path" - /tmp/venv/bin/python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb + #/tmp/venv/bin/python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb -p recon_file "$file_path" + #/tmp/venv/bin/python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From 851c4acbf21231a29188eccf3e591e8c94bf102b Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Mon, 22 Jun 2026 10:59:51 +0100 Subject: [PATCH 32/51] try adding brackets --- ptypy/templates/ptypy-generic.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 992a4fe..2b1ee0f 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -131,6 +131,7 @@ spec: command: - bash args: + - "(" - "mpirun" - "-n" - "{{`{{ inputs.parameters.nprocs }}`}}" @@ -143,6 +144,7 @@ spec: - "{{`{{ inputs.parameters.outpath }}`}}" - "-s" - "hdf5_loader" + - ")" - ">" - ">(tee -a /tmp/ptypy_log.txt)" - "2>" From 630647a38e2ff75feaba0d9fd91aa4fa8fa8c56b Mon Sep 17 00:00:00 2001 From: Jessica V <55289415+jessicavers@users.noreply.github.com> Date: Mon, 22 Jun 2026 14:09:40 +0100 Subject: [PATCH 33/51] Add normalisation parameter to raw data reading (#226) * Handle the reading raw data including the normalisation parameter * Update the image version --- i14/notebooks/xanes_autoprocessing0.ipynb | 13 ++++++++----- .../xanes_sparse_stack_autoprocessing0.ipynb | 13 ++++++++----- i14/templates/xanes-sparse-template.yaml | 4 ++-- 3 files changed, 18 insertions(+), 12 deletions(-) diff --git a/i14/notebooks/xanes_autoprocessing0.ipynb b/i14/notebooks/xanes_autoprocessing0.ipynb index 2d2eaac..286eaa0 100644 --- a/i14/notebooks/xanes_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_autoprocessing0.ipynb @@ -248,7 +248,7 @@ "\n", "for raw_data in file_list:\n", " # windowing happens here for each file\n", - " data = read_raw_data(raw_data)\n", + " data = read_raw_data(raw_data, normalised)\n", " \n", " # window it\n", " windowed_align = window_mca(data[\"mca\"], start_align, end_align, data[\"scan_shape\"], data[\"scan_model\"])\n", @@ -257,9 +257,13 @@ " else:\n", " windowed_edge = windowed_align\n", " \n", - " # sum the I0\n", - " I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n", - " \n", + " if normalised:\n", + " # sum the I0\n", + " I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n", + " I0_total.append(I0_t)\n", + " else:\n", + " I0_total.append(np.ones_like(windowed_align))\n", + "\n", " # record everything\n", " stack_element_align.append(windowed_align)\n", " stack_edge_element.append(windowed_edge)\n", @@ -268,7 +272,6 @@ " SampleX.append(data[\"x\"])\n", " SampleY.append(data[\"y\"])\n", " scan_shapes.append(data[\"scan_shape\"])\n", - " I0_total.append(I0_t)\n", " \n", "print(f\"Time reading raw data: {(time.perf_counter() - start)/60:.2f} min\")" ] diff --git a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb index c225416..074ace9 100644 --- a/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb @@ -208,14 +208,18 @@ "\n", "for raw_data in file_list:\n", " # windowing happens here for each file\n", - " data = read_raw_data(raw_data)\n", + " data = read_raw_data(raw_data, normalised)\n", " \n", " # window it\n", " w_mca = window_mca(data[\"mca\"], lg_start, lg_end, data[\"scan_shape\"], data[\"scan_model\"])\n", " \n", - " # sum the I0\n", - " I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n", - " \n", + " if normalised:\n", + " # sum the I0\n", + " I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n", + " I0_total.append(I0_t)\n", + " else:\n", + " I0_total.append(np.ones_like(w_mca))\n", + "\n", " # record everything\n", " windowed.append(w_mca)\n", " \n", @@ -223,7 +227,6 @@ " SampleX.append(data[\"x\"])\n", " SampleY.append(data[\"y\"])\n", " scan_shapes.append(data[\"scan_shape\"])\n", - " I0_total.append(I0_t)\n", " \n", "print(f\"Time reading raw data: {(time.perf_counter() - start)/60:.2f} min\")" ] diff --git a/i14/templates/xanes-sparse-template.yaml b/i14/templates/xanes-sparse-template.yaml index 2b8e9ab..12d6129 100644 --- a/i14/templates/xanes-sparse-template.yaml +++ b/i14/templates/xanes-sparse-template.yaml @@ -48,7 +48,7 @@ spec: templates: - name: xanes-sparse-mount-files script: - image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.2 command: [bash] source: | echo '{{ .Files.Get "notebooks/xanes_sparse_stack_autoprocessing0.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb @@ -87,7 +87,7 @@ spec: - name: edgeElement - name: normalise script: - image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.2 env: - name: HOME value: /tmp From 553e85e1d128ab66dfc10948626c85ef164a1ded Mon Sep 17 00:00:00 2001 From: Jessica V <55289415+jessicavers@users.noreply.github.com> Date: Mon, 22 Jun 2026 14:41:12 +0100 Subject: [PATCH 34/51] Update xanes auto image (#229) --- i14/templates/xanes-auto-template.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/i14/templates/xanes-auto-template.yaml b/i14/templates/xanes-auto-template.yaml index a8b076f..fb77914 100644 --- a/i14/templates/xanes-auto-template.yaml +++ b/i14/templates/xanes-auto-template.yaml @@ -54,7 +54,7 @@ spec: templates: - name: xanes-mount-files script: - image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.2 command: [bash] source: | echo '{{ .Files.Get "notebooks/xanes_autoprocessing0.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb @@ -120,7 +120,7 @@ spec: - name: method - name: normalise script: - image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.2 env: - name: HOME value: /tmp From 4a5ad6d3f37030f442aedf71030735c888ece54f Mon Sep 17 00:00:00 2001 From: Jessica V <55289415+jessicavers@users.noreply.github.com> Date: Mon, 22 Jun 2026 14:42:53 +0100 Subject: [PATCH 35/51] Allow scan range input for batch workflows (#228) * Update schemas to use multiScan * Allow scan range input for batch workflows --- i14/schema/dpcSchema.json | 13 ++++++----- i14/schema/dpcUISchema.json | 17 +++++++++++++- i14/schema/xrd1dSchema.json | 13 ++++++----- i14/schema/xrd1dUISchema.json | 17 +++++++++++++- i14/schema/xrd2dSchema.json | 13 ++++++----- i14/schema/xrd2dUISchema.json | 17 +++++++++++++- i14/templates/dpc-batch-template.yaml | 30 ++++++++++++++++++++++++- i14/templates/xrd1d-batch-template.yaml | 30 ++++++++++++++++++++++++- i14/templates/xrd2d-batch-template.yaml | 30 ++++++++++++++++++++++++- 9 files changed, 159 insertions(+), 21 deletions(-) diff --git a/i14/schema/dpcSchema.json b/i14/schema/dpcSchema.json index 9bbd793..1b9c705 100644 --- a/i14/schema/dpcSchema.json +++ b/i14/schema/dpcSchema.json @@ -1,10 +1,13 @@ { "type": "object", "properties": { - "scanNumbers": { - "type": "string", - "title": "Scan Numbers", - "description": "A list containing scan numbers to be processed. Eg. [274317] or [274317, 274318, 274319]" + "multiScan": { + "title": "Scans", + "type": "array", + "items": + { + "type": "object" + } }, "outputFolder": { "type": "string", @@ -12,5 +15,5 @@ "description": "The full path of the output file" } }, - "required": ["scanNumbers", "outputFolder"] + "required": ["multiScan", "outputFolder"] } diff --git a/i14/schema/dpcUISchema.json b/i14/schema/dpcUISchema.json index 662c739..89c5e7a 100644 --- a/i14/schema/dpcUISchema.json +++ b/i14/schema/dpcUISchema.json @@ -3,7 +3,22 @@ "elements": [ { "type": "Control", - "scope": "#/properties/scanNumbers" + "scope": "#/properties/multiScan", + "options": { + "elementLabelProp": "scanRange", + "detail": { + "type": "VerticalLayout", + "elements": [ + { + "type": "Control", + "scope": "#/properties/multiScan/items", + "options": { + "useScanRangeControl": true + } + } + ] + } + } }, { "type": "Control", diff --git a/i14/schema/xrd1dSchema.json b/i14/schema/xrd1dSchema.json index b8f8422..277fe80 100644 --- a/i14/schema/xrd1dSchema.json +++ b/i14/schema/xrd1dSchema.json @@ -1,10 +1,13 @@ { "type": "object", "properties": { - "scanNumbers": { - "type": "string", - "title": "Scan Numbers", - "description": "A list containing scan numbers to be processed. Eg. [274317] or [274317, 274318, 274319]" + "multiScan": { + "title": "Scans", + "type": "array", + "items": + { + "type": "object" + } }, "outputFolder": { "type": "string", @@ -49,7 +52,7 @@ } }, "required": [ - "scanNumbers", + "multiScan", "outputFolder", "xrdProcessConfigUpload" ] diff --git a/i14/schema/xrd1dUISchema.json b/i14/schema/xrd1dUISchema.json index 9c54298..0d65740 100644 --- a/i14/schema/xrd1dUISchema.json +++ b/i14/schema/xrd1dUISchema.json @@ -3,7 +3,22 @@ "elements": [ { "type": "Control", - "scope": "#/properties/scanNumbers" + "scope": "#/properties/multiScan", + "options": { + "elementLabelProp": "scanRange", + "detail": { + "type": "VerticalLayout", + "elements": [ + { + "type": "Control", + "scope": "#/properties/multiScan/items", + "options": { + "useScanRangeControl": true + } + } + ] + } + } }, { "type": "Control", diff --git a/i14/schema/xrd2dSchema.json b/i14/schema/xrd2dSchema.json index 99b6760..e750a0f 100644 --- a/i14/schema/xrd2dSchema.json +++ b/i14/schema/xrd2dSchema.json @@ -1,10 +1,13 @@ { "type": "object", "properties": { - "scanNumbers": { - "type": "string", - "title": "Scan Numbers", - "description": "A list containing scan numbers to be processed. Eg. [274317] or [274317, 274318, 274319]" + "multiScan": { + "title": "Scans", + "type": "array", + "items": + { + "type": "object" + } }, "outputFolder": { "type": "string", @@ -28,7 +31,7 @@ } }, "required": [ - "scanNumbers", + "multiScan", "outputFolder", "xrdProcessConfigUpload" ] diff --git a/i14/schema/xrd2dUISchema.json b/i14/schema/xrd2dUISchema.json index 6b631c5..2467e73 100644 --- a/i14/schema/xrd2dUISchema.json +++ b/i14/schema/xrd2dUISchema.json @@ -3,7 +3,22 @@ "elements": [ { "type": "Control", - "scope": "#/properties/scanNumbers" + "scope": "#/properties/multiScan", + "options": { + "elementLabelProp": "scanRange", + "detail": { + "type": "VerticalLayout", + "elements": [ + { + "type": "Control", + "scope": "#/properties/multiScan/items", + "options": { + "useScanRangeControl": true + } + } + ] + } + } }, { "type": "Control", diff --git a/i14/templates/dpc-batch-template.yaml b/i14/templates/dpc-batch-template.yaml index 759f872..f5ab2c2 100644 --- a/i14/templates/dpc-batch-template.yaml +++ b/i14/templates/dpc-batch-template.yaml @@ -51,6 +51,12 @@ spec: mountPath: /tmp - name: dpc-notebook steps: + - - name: gen-multi-range + template: gen-multi-range-tmp + arguments: + parameters: + - name: multiScan + value: "{{`{{ workflow.parameters.multiScan }}`}}" - - name: dpc template: gen-notebook arguments: @@ -59,7 +65,29 @@ spec: value: "{{`{{ workflow.parameters.outputFolder }}`}}" - name: scanNumber value: "{{`{{ item }}`}}" - withParam: "{{`{{ workflow.parameters.scanNumbers }}`}}" + withParam: "{{`{{ steps.gen-multi-range.outputs.result }}`}}" + - name: gen-multi-range-tmp + inputs: + parameters: + - name: multiScan + script: + image: python:3.10 + command: [python] + source: | + import json, sys + file_number_list = [] + multiscan_list = {{`{{ inputs.parameters.multiScan }}`}} + + for multiscan in multiscan_list: + for range_ in multiscan.values(): + excluded = range_['excluded'] + assert isinstance(excluded, list) + assert range_['end'] >= range_['start'] + scans = [x for x in range(range_['start'], range_['end'] + 1) if x not in excluded] + file_number_list.append(scans) + + file_numbers = [n for _list in file_number_list for n in _list] + json.dump(file_numbers, sys.stdout) - name: gen-notebook inputs: parameters: diff --git a/i14/templates/xrd1d-batch-template.yaml b/i14/templates/xrd1d-batch-template.yaml index ae19452..2f1d7eb 100644 --- a/i14/templates/xrd1d-batch-template.yaml +++ b/i14/templates/xrd1d-batch-template.yaml @@ -61,6 +61,12 @@ spec: mountPath: /tmp - name: xrd1d-notebook steps: + - - name: gen-multi-range + template: gen-multi-range-tmp + arguments: + parameters: + - name: multiScan + value: "{{`{{ workflow.parameters.multiScan }}`}}" - - name: xrd1d template: gen-notebook arguments: @@ -81,7 +87,29 @@ spec: value: "{{`{{ workflow.parameters.backgroundType }}`}}" - name: polynomialOrder value: "{{`{{ workflow.parameters.polynomialOrder }}`}}" - withParam: "{{`{{ workflow.parameters.scanNumbers }}`}}" + withParam: "{{`{{ steps.gen-multi-range.outputs.result }}`}}" + - name: gen-multi-range-tmp + inputs: + parameters: + - name: multiScan + script: + image: python:3.10 + command: [python] + source: | + import json, sys + file_number_list = [] + multiscan_list = {{`{{ inputs.parameters.multiScan }}`}} + + for multiscan in multiscan_list: + for range_ in multiscan.values(): + excluded = range_['excluded'] + assert isinstance(excluded, list) + assert range_['end'] >= range_['start'] + scans = [x for x in range(range_['start'], range_['end'] + 1) if x not in excluded] + file_number_list.append(scans) + + file_numbers = [n for _list in file_number_list for n in _list] + json.dump(file_numbers, sys.stdout) - name: gen-notebook inputs: parameters: diff --git a/i14/templates/xrd2d-batch-template.yaml b/i14/templates/xrd2d-batch-template.yaml index 0b80372..cb3e5b4 100644 --- a/i14/templates/xrd2d-batch-template.yaml +++ b/i14/templates/xrd2d-batch-template.yaml @@ -56,6 +56,12 @@ spec: mountPath: /tmp - name: xrd2d-notebook steps: + - - name: gen-multi-range + template: gen-multi-range-tmp + arguments: + parameters: + - name: multiScan + value: "{{`{{ workflow.parameters.multiScan }}`}}" - - name: xrd2d template: gen-notebook arguments: @@ -72,7 +78,29 @@ spec: value: "{{`{{ workflow.parameters.maskPath }}`}}" - name: flatPath value: "{{`{{ workflow.parameters.flatPath }}`}}" - withParam: "{{`{{ workflow.parameters.scanNumbers }}`}}" + withParam: "{{`{{ steps.gen-multi-range.outputs.result }}`}}" + - name: gen-multi-range-tmp + inputs: + parameters: + - name: multiScan + script: + image: python:3.10 + command: [python] + source: | + import json, sys + file_number_list = [] + multiscan_list = {{`{{ inputs.parameters.multiScan }}`}} + + for multiscan in multiscan_list: + for range_ in multiscan.values(): + excluded = range_['excluded'] + assert isinstance(excluded, list) + assert range_['end'] >= range_['start'] + scans = [x for x in range(range_['start'], range_['end'] + 1) if x not in excluded] + file_number_list.append(scans) + + file_numbers = [n for _list in file_number_list for n in _list] + json.dump(file_numbers, sys.stdout) - name: gen-notebook inputs: parameters: From d5f3c6afbf283c5d0749ed48216953a6911500fd Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Tue, 23 Jun 2026 10:14:46 +0100 Subject: [PATCH 36/51] update ptypy-generic --- ptypy/templates/ptypy-generic.yaml | 69 +++++++++--------------------- 1 file changed, 21 insertions(+), 48 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 2b1ee0f..0267d87 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -35,7 +35,7 @@ spec: value: 1 - name: memory value: 20Gi - container: + script: image: ghcr.io/ptycho/ptypy_cupy_openmpi:0.9.1 env: - name: CUPY_CACHE_DIR @@ -46,29 +46,16 @@ spec: value: "1" - name: OMP_NUM_THREADS value: "1" - command: - - bash - args: - - "mpirun" - - "-n" - - "{{`{{ inputs.parameters.nprocs }}`}}" - - "ptypy.cli" - - "-f" - - "{{`{{ inputs.parameters.config }}`}}" - - "-i" - - "{{`{{ inputs.parameters.id }}`}}" - - "-o" - - "{{`{{ inputs.parameters.outpath }}`}}" - - "-s" - - "hdf5_loader" - - "-b" - - "cupy" - - ">" - - ">(tee -a /tmp/ptypy_log.txt)" - - "2>" - - ">(tee -a /tmp/ptypy_log.txt)" - - ";" - - "ls /tmp" + command: [bash] + source: + (mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ + ptypy.cli \ + -f {{`{{ inputs.parameters.config }}`}} \ + -i {{`{{ inputs.parameters.id }}`}} \ + -o {{`{{ inputs.parameters.outpath }}`}} \ + -s hdf5_loader \ + -b cupy \ + ) > >(tee -a /tmp/ptypy_log.txt) volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -119,7 +106,7 @@ spec: path: /tmp/ptypy_log.txt archive: none: {} - container: + script: image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 env: - name: MKL_NUM_THREADS @@ -128,29 +115,15 @@ spec: value: "1" - name: OMP_NUM_THREADS value: "1" - command: - - bash - args: - - "(" - - "mpirun" - - "-n" - - "{{`{{ inputs.parameters.nprocs }}`}}" - - "ptypy.cli" - - "-f" - - "{{`{{ inputs.parameters.config }}`}}" - - "-i" - - "{{`{{ inputs.parameters.id }}`}}" - - "-o" - - "{{`{{ inputs.parameters.outpath }}`}}" - - "-s" - - "hdf5_loader" - - ")" - - ">" - - ">(tee -a /tmp/ptypy_log.txt)" - - "2>" - - ">(tee -a /tmp/ptypy_log.txt)" - - ";" - - "ls /tmp" + command: [bash] + source: | + (mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ + ptypy.cli \ + -f {{`{{ inputs.parameters.config }}`}} \ + -i {{`{{ inputs.parameters.id }}`}} \ + -o {{`{{ inputs.parameters.outpath }}`}} \ + -s hdf5_loader \ + ) > >(tee -a /tmp/ptypy_log.txt) volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From ba0bfa3ad3ae062862994f5dfab757c65a9963a5 Mon Sep 17 00:00:00 2001 From: Yousef Moazzam Date: Tue, 23 Jun 2026 12:16:21 +0100 Subject: [PATCH 37/51] Remove `cutoff` param in dark/flat field correction method --- httomo/pipelines/visr.json | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/httomo/pipelines/visr.json b/httomo/pipelines/visr.json index cdcec11..0d24b95 100644 --- a/httomo/pipelines/visr.json +++ b/httomo/pipelines/visr.json @@ -21,8 +21,7 @@ "module_path": "httomolibgpu.prep.normalize", "parameters": { "flats_multiplier": 1, - "darks_multiplier": 1, - "cutoff": 10 + "darks_multiplier": 1 } }, { From 440326fbf0a1ec5b3b8aa9d0300ccc9cc01dc8c4 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Tue, 23 Jun 2026 14:01:57 +0100 Subject: [PATCH 38/51] update ptypy workflows --- ptypy/templates/ptypy-generic.yaml | 16 ++++------------ ptypy/templates/ptypy-p99.yaml | 4 ++++ ptypy/templates/ptypy-utils.yaml | 8 ++++++-- 3 files changed, 14 insertions(+), 14 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 0267d87..b9db77d 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -48,14 +48,13 @@ spec: value: "1" command: [bash] source: - (mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ + mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ ptypy.cli \ -f {{`{{ inputs.parameters.config }}`}} \ -i {{`{{ inputs.parameters.id }}`}} \ -o {{`{{ inputs.parameters.outpath }}`}} \ -s hdf5_loader \ - -b cupy \ - ) > >(tee -a /tmp/ptypy_log.txt) + -b cupy volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -100,12 +99,6 @@ spec: value: 1 - name: memory value: 20Gi - outputs: - artifacts: - - name: ptypy-cpu-run-log - path: /tmp/ptypy_log.txt - archive: - none: {} script: image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 env: @@ -117,13 +110,12 @@ spec: value: "1" command: [bash] source: | - (mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ + mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ ptypy.cli \ -f {{`{{ inputs.parameters.config }}`}} \ -i {{`{{ inputs.parameters.id }}`}} \ -o {{`{{ inputs.parameters.outpath }}`}} \ - -s hdf5_loader \ - ) > >(tee -a /tmp/ptypy_log.txt) + -s hdf5_loader volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 5776bcc..310adfa 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -153,3 +153,7 @@ spec: template: ptypy-plotting clusterScope: true depends: "reconstruct-with-gpu || reconstruct-with-cpu" + arguments: + artifacts: + - name: reconstruction-log + fromExpression: "{{`{{ workflow.parameters.usegpu }}`}} == true ? tasks.reconstruct-with-gpu.outputs.result : tasks.reconstruct-with-cpu.outputs.result" diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index b005817..70e9ae7 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -26,6 +26,10 @@ spec: key: data_directory templates: - name: ptypy-plotting + inputs: + artifacts: + - name: reconstruction-log + path: /tmp/ptypy_log.txt script: image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 env: @@ -37,10 +41,10 @@ spec: value: "1" command: [bash] source: | - ls /tmp #python -m venv --system-site-packages /tmp/venv #/tmp/venv/bin/pip install jupyter papermill nbconvert - #file_path=$(sed -rn "s/^Saving to (.*)$/\1/p" /tmp/ptypy_log.txt) + file_path=$(sed -rn "s/^Saving to (.*)$/\1/p" /tmp/ptypy_log.txt) + echo $file_path # mount notebook #echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb # run notebook From 02583c4e25b215b4569612563dc79743ddb568c0 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Wed, 24 Jun 2026 15:37:21 +0100 Subject: [PATCH 39/51] update ptypy-p99 and ptypy-generic --- ptypy/templates/ptypy-generic.yaml | 8 +++++--- ptypy/templates/ptypy-p99.yaml | 5 +---- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index b9db77d..4b68510 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -47,14 +47,15 @@ spec: - name: OMP_NUM_THREADS value: "1" command: [bash] - source: + source: | mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ ptypy.cli \ -f {{`{{ inputs.parameters.config }}`}} \ -i {{`{{ inputs.parameters.id }}`}} \ -o {{`{{ inputs.parameters.outpath }}`}} \ -s hdf5_loader \ - -b cupy + -b cupy \ + | tee -a /tmp/ptypy_log.txt volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -115,7 +116,8 @@ spec: -f {{`{{ inputs.parameters.config }}`}} \ -i {{`{{ inputs.parameters.id }}`}} \ -o {{`{{ inputs.parameters.outpath }}`}} \ - -s hdf5_loader + -s hdf5_loader \ + | tee -a /tmp/ptypy_log.txt volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 310adfa..0f86f8f 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -153,7 +153,4 @@ spec: template: ptypy-plotting clusterScope: true depends: "reconstruct-with-gpu || reconstruct-with-cpu" - arguments: - artifacts: - - name: reconstruction-log - fromExpression: "{{`{{ workflow.parameters.usegpu }}`}} == true ? tasks.reconstruct-with-gpu.outputs.result : tasks.reconstruct-with-cpu.outputs.result" + From d2f24e73167e9252bd9a8b02b8a3d8d8f3706c05 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Wed, 24 Jun 2026 15:46:24 +0100 Subject: [PATCH 40/51] update ptypy-utils --- ptypy/templates/ptypy-utils.yaml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index 70e9ae7..c3abd6b 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -26,10 +26,6 @@ spec: key: data_directory templates: - name: ptypy-plotting - inputs: - artifacts: - - name: reconstruction-log - path: /tmp/ptypy_log.txt script: image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 env: From 6bff9925e2435be3ec03ce8532b92a25e0911c88 Mon Sep 17 00:00:00 2001 From: Jessica V <55289415+jessicavers@users.noreply.github.com> Date: Thu, 25 Jun 2026 11:35:31 +0100 Subject: [PATCH 41/51] Revert image to v0.1 for xanes auto (#236) --- i14/notebooks/xanes_autoprocessing0.ipynb | 13 +++++-------- i14/templates/xanes-auto-template.yaml | 4 ++-- 2 files changed, 7 insertions(+), 10 deletions(-) diff --git a/i14/notebooks/xanes_autoprocessing0.ipynb b/i14/notebooks/xanes_autoprocessing0.ipynb index 286eaa0..2d2eaac 100644 --- a/i14/notebooks/xanes_autoprocessing0.ipynb +++ b/i14/notebooks/xanes_autoprocessing0.ipynb @@ -248,7 +248,7 @@ "\n", "for raw_data in file_list:\n", " # windowing happens here for each file\n", - " data = read_raw_data(raw_data, normalised)\n", + " data = read_raw_data(raw_data)\n", " \n", " # window it\n", " windowed_align = window_mca(data[\"mca\"], start_align, end_align, data[\"scan_shape\"], data[\"scan_model\"])\n", @@ -257,13 +257,9 @@ " else:\n", " windowed_edge = windowed_align\n", " \n", - " if normalised:\n", - " # sum the I0\n", - " I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n", - " I0_total.append(I0_t)\n", - " else:\n", - " I0_total.append(np.ones_like(windowed_align))\n", - "\n", + " # sum the I0\n", + " I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n", + " \n", " # record everything\n", " stack_element_align.append(windowed_align)\n", " stack_edge_element.append(windowed_edge)\n", @@ -272,6 +268,7 @@ " SampleX.append(data[\"x\"])\n", " SampleY.append(data[\"y\"])\n", " scan_shapes.append(data[\"scan_shape\"])\n", + " I0_total.append(I0_t)\n", " \n", "print(f\"Time reading raw data: {(time.perf_counter() - start)/60:.2f} min\")" ] diff --git a/i14/templates/xanes-auto-template.yaml b/i14/templates/xanes-auto-template.yaml index fb77914..a8b076f 100644 --- a/i14/templates/xanes-auto-template.yaml +++ b/i14/templates/xanes-auto-template.yaml @@ -54,7 +54,7 @@ spec: templates: - name: xanes-mount-files script: - image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.2 + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 command: [bash] source: | echo '{{ .Files.Get "notebooks/xanes_autoprocessing0.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb @@ -120,7 +120,7 @@ spec: - name: method - name: normalise script: - image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.2 + image: gitlab.diamond.ac.uk:5050/i14/i14_utility/xanes:v0.1 env: - name: HOME value: /tmp From 5af8a569230c10c8b6d5edacad566803a1ac556c Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Thu, 25 Jun 2026 12:00:18 +0100 Subject: [PATCH 42/51] update ptypy-generic --- ptypy/templates/ptypy-generic.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index 4b68510..ddfe7ca 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -111,13 +111,13 @@ spec: value: "1" command: [bash] source: | + exec &> >(tee /tmp/ptypy_log.txt) mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ ptypy.cli \ -f {{`{{ inputs.parameters.config }}`}} \ -i {{`{{ inputs.parameters.id }}`}} \ -o {{`{{ inputs.parameters.outpath }}`}} \ - -s hdf5_loader \ - | tee -a /tmp/ptypy_log.txt + -s hdf5_loader volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From 4eb83ab1b49f1dad7de73882771f8372fc1e5aa6 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Fri, 26 Jun 2026 13:35:52 +0100 Subject: [PATCH 43/51] update ptypy workflows --- ptypy/templates/ptypy-generic.yaml | 4 +-- ptypy/templates/ptypy-p99.yaml | 41 ++++++++++++++++++++++++++++-- ptypy/templates/ptypy-utils.yaml | 16 +++++++----- 3 files changed, 49 insertions(+), 12 deletions(-) diff --git a/ptypy/templates/ptypy-generic.yaml b/ptypy/templates/ptypy-generic.yaml index ddfe7ca..e95f315 100644 --- a/ptypy/templates/ptypy-generic.yaml +++ b/ptypy/templates/ptypy-generic.yaml @@ -54,8 +54,7 @@ spec: -i {{`{{ inputs.parameters.id }}`}} \ -o {{`{{ inputs.parameters.outpath }}`}} \ -s hdf5_loader \ - -b cupy \ - | tee -a /tmp/ptypy_log.txt + -b cupy volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" @@ -111,7 +110,6 @@ spec: value: "1" command: [bash] source: | - exec &> >(tee /tmp/ptypy_log.txt) mpirun -n {{`{{ inputs.parameters.nprocs }}`}} \ ptypy.cli \ -f {{`{{ inputs.parameters.config }}`}} \ diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 0f86f8f..46ff436 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -102,7 +102,31 @@ spec: - name: session hostPath: path: "{{`{{ workflow.parameters.visitdir }}`}}" - type: Directory + type: Directory + + - name: parse-filepath + inputs: + parameters: + - name: gpu-output + default: "no output" + - name: cpu-output + default: "no output" + script: + image: alpine:3.23.5 + command: [bash] + source: | + cpu="{{`{{ inputs.parameters.cpu-output }}`}}" + gpu="{{`{{ inputs.parameters.gpu-output }}`}}" + if [[ $cpu == "no output" && $gpu == "no output" ]]; then + echo "No inputs from either gpu or cpu reconstructions" >&2 + exit 1 + fi + + if [[ $cpu != "no output" ]]; then + echo "$cpu" | sed -rn "s/^Saving to (.*)$/\1/p" + else + echo "$gpu" | sed -rn "s/^Saving to (.*)$/\1/p" + fi - name: ptypy-pipeline dag: @@ -147,10 +171,23 @@ spec: value: "{{`{{ workflow.parameters.nprocs }}`}}" - name: memory value: "{{`{{ workflow.parameters.memory }}`}}" + - name: parse-recon-output + template: parse-filepath + depends: "reconstruct-with-gpu || reconstruct-with-cpu" + arguments: + parameters: + - name: cpu-output + value: "{{`{{tasks.reconstruct-with-cpu.outputs.result}}`}}" + - name: gpu-output + value: "{{`{{tasks.reconstruct-with-gpu.outputs.result}}`}}" - name: plot templateRef: name: ptypy-utils template: ptypy-plotting clusterScope: true - depends: "reconstruct-with-gpu || reconstruct-with-cpu" + depends: "parse-recon-output" + arguments: + parameters: + - name: filepath + value: "{{`{{tasks.parse-recon-output.outputs.result}}`}}" diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index c3abd6b..580d3ec 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -26,6 +26,9 @@ spec: key: data_directory templates: - name: ptypy-plotting + inputs: + parameters: + - name: filepath script: image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 env: @@ -37,15 +40,14 @@ spec: value: "1" command: [bash] source: | - #python -m venv --system-site-packages /tmp/venv - #/tmp/venv/bin/pip install jupyter papermill nbconvert - file_path=$(sed -rn "s/^Saving to (.*)$/\1/p" /tmp/ptypy_log.txt) - echo $file_path + python -m venv --system-site-packages /tmp/venv + /tmp/venv/bin/pip install jupyter papermill nbconvert # mount notebook - #echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb + echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb # run notebook - #/tmp/venv/bin/python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb -p recon_file "$file_path" - #/tmp/venv/bin/python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb + /tmp/venv/bin/python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb \ + -p recon_file "{{`{{ inputs.parameters.filepath }}`}}" + /tmp/venv/bin/python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From b58fb1c1e3df8bff8a748b499ee621938ec1782a Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Fri, 26 Jun 2026 16:25:28 +0100 Subject: [PATCH 44/51] update ptypy-p99 --- ptypy/templates/ptypy-p99.yaml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 46ff436..2959cc4 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -112,7 +112,7 @@ spec: - name: cpu-output default: "no output" script: - image: alpine:3.23.5 + image: docker.io/alpine:3.23.5 command: [bash] source: | cpu="{{`{{ inputs.parameters.cpu-output }}`}}" @@ -190,4 +190,3 @@ spec: parameters: - name: filepath value: "{{`{{tasks.parse-recon-output.outputs.result}}`}}" - From bf91189c1506132f89a689a2603543fc8889372d Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Mon, 29 Jun 2026 09:50:43 +0100 Subject: [PATCH 45/51] update ptypy-p99 --- ptypy/templates/ptypy-p99.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 2959cc4..57c4684 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -177,9 +177,13 @@ spec: arguments: parameters: - name: cpu-output - value: "{{`{{tasks.reconstruct-with-cpu.outputs.result}}`}}" + valueFrom: + parameter: "{{`{{tasks.reconstruct-with-cpu.outputs.result}}`}}" + default: "no output" - name: gpu-output - value: "{{`{{tasks.reconstruct-with-gpu.outputs.result}}`}}" + valueFrom: + parameter: "{{`{{tasks.reconstruct-with-gpu.outputs.result}}`}}" + default: "no output" - name: plot templateRef: name: ptypy-utils From 63193a36d81e6c8e1cd38cdbab76a8465a49a6d4 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Wed, 1 Jul 2026 08:45:56 +0100 Subject: [PATCH 46/51] update ptypy-p99 --- ptypy/templates/ptypy-p99.yaml | 57 +++++++++++++++------------------- 1 file changed, 25 insertions(+), 32 deletions(-) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 57c4684..23a759a 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -104,30 +104,29 @@ spec: path: "{{`{{ workflow.parameters.visitdir }}`}}" type: Directory - - name: parse-filepath + - name: get-filepath inputs: parameters: - - name: gpu-output - default: "no output" - - name: cpu-output - default: "no output" + - name: config script: - image: docker.io/alpine:3.23.5 - command: [bash] + image: python:3.14 + command: [python] source: | - cpu="{{`{{ inputs.parameters.cpu-output }}`}}" - gpu="{{`{{ inputs.parameters.gpu-output }}`}}" - if [[ $cpu == "no output" && $gpu == "no output" ]]; then - echo "No inputs from either gpu or cpu reconstructions" >&2 - exit 1 - fi - - if [[ $cpu != "no output" ]]; then - echo "$cpu" | sed -rn "s/^Saving to (.*)$/\1/p" - else - echo "$gpu" | sed -rn "s/^Saving to (.*)$/\1/p" - fi - + import re + OUTPATH='"{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.outpath }}`}}"' + SCAN="{{`{{ workflow.parameters.id }}`}}" + CONFIG='"{{`{{ inputs.parameters.config }}`}}"' + with open("./out.txt", "w") as o: + with open(CONFIG, 'r') as f: + text = f.read() + ENGINE = re.findall(" engine0:\n name: (.*)", text)[0] + NITER = int(re.findall(" numiter: (.*)", text)[0]) + o.write(f"{OUTPATH}/scan_{SCAN}/{SCAN}_{ENGINE}_{NITER:04d}.ptyr") + outputs: + parameters: + - name: filepath + valueFrom: + path: /tmp/out.txt - name: ptypy-pipeline dag: tasks: @@ -171,26 +170,20 @@ spec: value: "{{`{{ workflow.parameters.nprocs }}`}}" - name: memory value: "{{`{{ workflow.parameters.memory }}`}}" - - name: parse-recon-output - template: parse-filepath + - name: get-filepath + template: get-filepath depends: "reconstruct-with-gpu || reconstruct-with-cpu" arguments: parameters: - - name: cpu-output - valueFrom: - parameter: "{{`{{tasks.reconstruct-with-cpu.outputs.result}}`}}" - default: "no output" - - name: gpu-output - valueFrom: - parameter: "{{`{{tasks.reconstruct-with-gpu.outputs.result}}`}}" - default: "no output" + - name: config + value: "{{`{{ tasks.prepare.outputs.parameters.config }}`}}" - name: plot templateRef: name: ptypy-utils template: ptypy-plotting clusterScope: true - depends: "parse-recon-output" + depends: "get-filepath" arguments: parameters: - name: filepath - value: "{{`{{tasks.parse-recon-output.outputs.result}}`}}" + value: "{{`{{tasks.get-filepath.outputs.parameters.filepath}}`}}" From 5262b5baf11c23a75dd66dc42cde680618d93a75 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Wed, 1 Jul 2026 09:11:37 +0100 Subject: [PATCH 47/51] update ptypy-p99 --- ptypy/templates/ptypy-p99.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 23a759a..1fc211f 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -116,7 +116,7 @@ spec: OUTPATH='"{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.outpath }}`}}"' SCAN="{{`{{ workflow.parameters.id }}`}}" CONFIG='"{{`{{ inputs.parameters.config }}`}}"' - with open("./out.txt", "w") as o: + with open("/tmp/out.txt", "w") as o: with open(CONFIG, 'r') as f: text = f.read() ENGINE = re.findall(" engine0:\n name: (.*)", text)[0] From 7a11cbf92c65dac5b99558a251606f1dcec109f5 Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Wed, 1 Jul 2026 09:52:55 +0100 Subject: [PATCH 48/51] update ptypy-p99 --- ptypy/templates/ptypy-p99.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 1fc211f..2418b61 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -110,6 +110,11 @@ spec: - name: config script: image: python:3.14 + volumeMounts: + - name: session + mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" + - name: tmpdir + mountPath: /tmp command: [python] source: | import re @@ -127,6 +132,11 @@ spec: - name: filepath valueFrom: path: /tmp/out.txt + volumes: + - name: session + hostPath: + path: "{{`{{ workflow.parameters.visitdir }}`}}" + type: Directory - name: ptypy-pipeline dag: tasks: From ecf81bce60408e7a2b68362f51b7cef5abc51d5b Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Wed, 1 Jul 2026 10:02:51 +0100 Subject: [PATCH 49/51] update ptypy-p99 --- ptypy/templates/ptypy-p99.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ptypy/templates/ptypy-p99.yaml b/ptypy/templates/ptypy-p99.yaml index 2418b61..bab3b5e 100644 --- a/ptypy/templates/ptypy-p99.yaml +++ b/ptypy/templates/ptypy-p99.yaml @@ -118,9 +118,9 @@ spec: command: [python] source: | import re - OUTPATH='"{{`{{ workflow.parameters.visitdir }}`}}"/"{{`{{ workflow.parameters.outpath }}`}}"' + OUTPATH="{{`{{ workflow.parameters.visitdir }}`}}/{{`{{ workflow.parameters.outpath }}`}}" SCAN="{{`{{ workflow.parameters.id }}`}}" - CONFIG='"{{`{{ inputs.parameters.config }}`}}"' + CONFIG="{{`{{ inputs.parameters.config }}`}}" with open("/tmp/out.txt", "w") as o: with open(CONFIG, 'r') as f: text = f.read() From 64bf3c8d97242a887fae124810b41eec7f4050cf Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Tue, 7 Jul 2026 14:33:02 +0100 Subject: [PATCH 50/51] update ptypy-utils to use new image --- ptypy/templates/ptypy-utils.yaml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index 580d3ec..859447e 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -30,7 +30,7 @@ spec: parameters: - name: filepath script: - image: ghcr.io/ptycho/ptypy_full_openmpi:0.9.1 + image: ghcr.io/diamondlightsource/imaging_toolbox/ptypy_workflows:0.3.3 env: - name: MKL_NUM_THREADS value: "1" @@ -40,14 +40,12 @@ spec: value: "1" command: [bash] source: | - python -m venv --system-site-packages /tmp/venv - /tmp/venv/bin/pip install jupyter papermill nbconvert # mount notebook echo '{{ .Files.Get "notebooks/ptypy_plotting.ipynb" | b64enc }}' | base64 -d > /tmp/notebook.ipynb # run notebook - /tmp/venv/bin/python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb \ + python -m papermill /tmp/notebook.ipynb /tmp/notebook.ipynb \ -p recon_file "{{`{{ inputs.parameters.filepath }}`}}" - /tmp/venv/bin/python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb + python -m nbconvert --to html --output notebook --output-dir /tmp /tmp/notebook.ipynb volumeMounts: - name: session mountPath: "{{`{{ workflow.parameters.visitdir }}`}}" From a2c52ce3c4676629357249a50680b674c3c8902e Mon Sep 17 00:00:00 2001 From: NatLeung96 Date: Tue, 7 Jul 2026 16:11:21 +0100 Subject: [PATCH 51/51] fix ptypy_utils image version --- ptypy/templates/ptypy-utils.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ptypy/templates/ptypy-utils.yaml b/ptypy/templates/ptypy-utils.yaml index 859447e..b42ffe8 100644 --- a/ptypy/templates/ptypy-utils.yaml +++ b/ptypy/templates/ptypy-utils.yaml @@ -30,7 +30,7 @@ spec: parameters: - name: filepath script: - image: ghcr.io/diamondlightsource/imaging_toolbox/ptypy_workflows:0.3.3 + image: ghcr.io/diamondlightsource/imaging_toolbox/ptypy_workflows:v0.3.3 env: - name: MKL_NUM_THREADS value: "1"