Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
85 commits
Select commit Hold shift + click to select a range
fa9205d
Merge pull request #200 from DiamondLightSource/E02_auto_mib
fallars May 7, 2026
4b4a732
fix: workflow linter errors
davehadley May 13, 2026
99a01bf
Merge pull request #201 from DiamondLightSource/drh/fix-linter-errors
daurer May 15, 2026
6f880d1
Update default boolean values to be consistent
jessicavers May 18, 2026
6565feb
Set default boolean values for json files
jessicavers May 18, 2026
62bceaf
Set normalised boolean within notebooks
jessicavers May 18, 2026
23baeba
Merge pull request #203 from DiamondLightSource/boolean-update
jessicavers May 21, 2026
c1407af
Alter papermill line to prevent notebook execution. Remove redundant …
jessicavers May 14, 2026
ec06a61
Remove nbconvert --allow-errors, so that notebooks stop running at fi…
jessicavers May 18, 2026
1251aee
Add parameter tag to notebooks
jessicavers May 20, 2026
41012e3
Create directory before papermill line
jessicavers May 20, 2026
0bb49e2
Merge pull request #202 from DiamondLightSource/update-workflows
jessicavers May 21, 2026
1d47015
setup ptypy recons for P99
daurer Jun 4, 2026
a770903
Merge pull request #204 from DiamondLightSource/p99-ptypy-template
daurer Jun 5, 2026
0e6ba19
Update nb execution and error handling for i14 workflows (#205)
jessicavers Jun 9, 2026
7fe4181
Extended length of residuals array to fix shape mismatch (#206)
jessicavers Jun 9, 2026
60a0231
Fixing ptypy P99 template
daurer Jun 9, 2026
bf4db48
Merge pull request #207 from DiamondLightSource/working-on-p99-template
daurer Jun 9, 2026
9cfb8dd
Get p99 ptypy workflow to work
daurer Jun 10, 2026
cb6f671
Merge pull request #208 from DiamondLightSource/ptypy-p99
daurer Jun 10, 2026
a2225f0
fix input path injection in p99 ptypy
NatLeung96 Jun 10, 2026
aec2756
Merge pull request #209 from NatLeung96/nl/fix-sed-cmd-in-p99-ptypy
daurer Jun 11, 2026
af8e0da
inject cupy engine for GPU workflow
daurer Jun 11, 2026
a171d67
Merge pull request #210 from DiamondLightSource/p99-ptypy-gpu-switch
daurer Jun 11, 2026
35deaf6
Remove reconfile argument
daurer Jun 11, 2026
71ffae3
Merge pull request #211 from DiamondLightSource/p99-ptypy-output-folder
daurer Jun 11, 2026
b3e8b2f
temporarily pull image always
daurer Jun 11, 2026
7bc0815
Merge pull request #212 from DiamondLightSource/p99-image-pull
daurer Jun 11, 2026
8cc593c
Provide complete path to output
daurer Jun 11, 2026
9e7d6a2
Merge pull request #213 from DiamondLightSource/p99-fix-folder-name
daurer Jun 11, 2026
23113c0
Revert image pulling policy change
daurer Jun 11, 2026
b07d09c
Merge pull request #214 from DiamondLightSource/p99_container
daurer Jun 11, 2026
08f001b
Fix p99 setup
daurer Jun 11, 2026
d6b8bf1
Merge pull request #215 from DiamondLightSource/p99-setup-fix
daurer Jun 11, 2026
cdf9ef8
add plotting to ptypy workflows
NatLeung96 Jun 16, 2026
b3dec00
Merge pull request #216 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 17, 2026
58a77d2
update ptypy-utils
NatLeung96 Jun 17, 2026
7136cbf
Merge pull request #217 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 17, 2026
9f2ed9e
update ptypy-utils
NatLeung96 Jun 18, 2026
8abc90b
Merge pull request #218 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 18, 2026
5777384
update ptypy-generic
NatLeung96 Jun 18, 2026
b628571
update ptypy-utils
NatLeung96 Jun 18, 2026
dc0a4a8
Merge pull request #219 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 18, 2026
5d086a2
update ptypy_plotting notebook
NatLeung96 Jun 18, 2026
b4053bb
Merge pull request #220 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 18, 2026
570a3ce
update ptypy-generic
NatLeung96 Jun 18, 2026
9d2ba97
Merge pull request #221 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 18, 2026
585eac8
update ptypy-generic
NatLeung96 Jun 18, 2026
dbbd0e2
Merge pull request #222 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 18, 2026
4bab757
update ptypy-generic
NatLeung96 Jun 18, 2026
b10a486
Merge pull request #223 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 18, 2026
299a47c
update ptypy-generic
NatLeung96 Jun 19, 2026
69484a0
Merge pull request #224 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 19, 2026
588a584
check /tmp for ptypy
NatLeung96 Jun 19, 2026
d468b3d
Merge pull request #225 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 19, 2026
851c4ac
try adding brackets
NatLeung96 Jun 22, 2026
917e96b
Merge pull request #227 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 22, 2026
630647a
Add normalisation parameter to raw data reading (#226)
jessicavers Jun 22, 2026
553e85e
Update xanes auto image (#229)
jessicavers Jun 22, 2026
4a5ad6d
Allow scan range input for batch workflows (#228)
jessicavers Jun 22, 2026
d5f3c6a
update ptypy-generic
NatLeung96 Jun 23, 2026
aae19f3
Merge pull request #230 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 23, 2026
ba0bfa3
Remove `cutoff` param in dark/flat field correction method
yousefmoazzam Jun 23, 2026
31c6654
Merge pull request #231 from DiamondLightSource/update-visr-httomo-pi…
yousefmoazzam Jun 23, 2026
440326f
update ptypy workflows
NatLeung96 Jun 23, 2026
13484bb
Merge pull request #233 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 23, 2026
02583c4
update ptypy-p99 and ptypy-generic
NatLeung96 Jun 24, 2026
1477f6c
Merge pull request #234 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 24, 2026
d2f24e7
update ptypy-utils
NatLeung96 Jun 24, 2026
f2d2d1b
Merge pull request #235 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 24, 2026
6bff992
Revert image to v0.1 for xanes auto (#236)
jessicavers Jun 25, 2026
5af8a56
update ptypy-generic
NatLeung96 Jun 25, 2026
2b5840b
Merge pull request #237 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 25, 2026
4eb83ab
update ptypy workflows
NatLeung96 Jun 26, 2026
b08780f
Merge pull request #238 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 26, 2026
b58fb1c
update ptypy-p99
NatLeung96 Jun 26, 2026
6d71820
Merge pull request #239 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 26, 2026
bf91189
update ptypy-p99
NatLeung96 Jun 29, 2026
2178a25
Merge pull request #241 from DiamondLightSource/nl/add-plotting-to-pt…
NatLeung96 Jun 29, 2026
63193a3
update ptypy-p99
NatLeung96 Jul 1, 2026
5262b5b
update ptypy-p99
NatLeung96 Jul 1, 2026
7a11cbf
update ptypy-p99
NatLeung96 Jul 1, 2026
ecf81bc
update ptypy-p99
NatLeung96 Jul 1, 2026
64bf3c8
update ptypy-utils to use new image
NatLeung96 Jul 7, 2026
a2c52ce
fix ptypy_utils image version
NatLeung96 Jul 7, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions e02/templates/mib2x-auto.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ metadata:
workflows.argoproj.io/title: ePSIC mib automatic conversion
workflows.argoproj.io/description: |
Convert MIB file to hdf5/hspy files
workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows"
workflows.diamond.ac.uk/parameter-schema: |
{{- .Files.Get "schema/mib2x_auto_Schema.json" | nindent 6 }}
workflows.diamond.ac.uk/ui-schema: |
Expand Down
1 change: 1 addition & 0 deletions e02/templates/mib2x.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ metadata:
workflows.argoproj.io/title: ePSIC mib conversion
workflows.argoproj.io/description: |
Convert MIB file to hdf5/hspy files
workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows"
workflows.diamond.ac.uk/parameter-schema.mib_path: |
{
"type": "string"
Expand Down
1 change: 1 addition & 0 deletions e02/templates/ptyrex_basic.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ metadata:
workflows.argoproj.io/title: Simple Ptyrex job submission
workflows.argoproj.io/description: |
Submit a Ptyrex job for a particular scan and projection within a given visit
workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows"
workflows.diamond.ac.uk/parameter-schema: |
{{- .Files.Get "schema/ptyrex_Schema.json" | nindent 6}}
workflows.diamond.ac.uk/ui-schema: |
Expand Down
3 changes: 1 addition & 2 deletions httomo/pipelines/visr.json
Original file line number Diff line number Diff line change
Expand Up @@ -21,8 +21,7 @@
"module_path": "httomolibgpu.prep.normalize",
"parameters": {
"flats_multiplier": 1,
"darks_multiplier": 1,
"cutoff": 10
"darks_multiplier": 1
}
},
{
Expand Down
2 changes: 2 additions & 0 deletions httomo/templates/cor-sweep.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ apiVersion: argoproj.io/v1alpha1
kind: ClusterWorkflowTemplate
metadata:
name: httomo-cor-sweep
labels:
workflows.diamond.ac.uk/science-group-imaging: "true"
annotations:
workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows"
workflows.diamond.ac.uk/parameter-schema.input: |
Expand Down
2 changes: 2 additions & 0 deletions httomo/templates/extract-raw-projections.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ apiVersion: argoproj.io/v1alpha1
kind: ClusterWorkflowTemplate
metadata:
name: extract-raw-projections
labels:
workflows.diamond.ac.uk/science-group-imaging: "true"
annotations:
workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows"
spec:
Expand Down
2 changes: 2 additions & 0 deletions httomo/templates/httomo-gpu-template-test.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ apiVersion: argoproj.io/v1alpha1
kind: ClusterWorkflowTemplate
metadata:
name: httomo-gpu-job
labels:
workflows.diamond.ac.uk/science-group-imaging: "true"
annotations:
workflows.diamond.ac.uk/repository: "https://github.com/DiamondLightSource/imaging-workflows"
spec:
Expand Down
18 changes: 18 additions & 0 deletions i14/notebooks/ptycho-tomo-phase.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,24 @@
"cell_type": "markdown",
"id": "2f61bb2f",
"metadata": {},
"source": [
"## Set normalise boolean"
]
},
{
"cell_type": "code",
"execution_count": null,
"id": "e3300162-404a-4660-b4ac-136bbd517727",
"metadata": {},
"outputs": [],
"source": [
"normalise = True if normalise.lower() == \"true\" else False"
]
},
{
"cell_type": "markdown",
"id": "5d23b863-a0da-40f4-aa72-c5b5cc911c00",
"metadata": {},
"source": [
"## Initialise np array"
]
Expand Down
18 changes: 17 additions & 1 deletion i14/notebooks/xanes_autoprocessing0.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@
},
"outputs": [],
"source": [
"# default settings\n",
"# parameters\n",
"inpath = \"\" # type: str\n",
"outpath = \"\" # type: str\n",
"edge_element = \"\" # type: str\n",
Expand Down Expand Up @@ -169,6 +169,22 @@
"print(f\"The PNG file will be saved as {png_outpath}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Set normalised boolean\n"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"normalised = True if normalised.lower() == \"true\" else False"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down
16 changes: 16 additions & 0 deletions i14/notebooks/xanes_point_autoprocessing0.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,22 @@
" raise FileNotFoundError(msg)"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Set normalise boolean"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"normalise = True if normalise.lower() == \"true\" else False"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down
33 changes: 27 additions & 6 deletions i14/notebooks/xanes_sparse_stack_autoprocessing0.ipynb
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@
},
"outputs": [],
"source": [
"# parameters\n",
"# outpath will be used to get outpath_nexus, outpath_mantis and outpath_complete for saving data, and outpath itself won't be used\n",
"\n",
"# default settings\n",
Expand Down Expand Up @@ -119,6 +120,22 @@
"print(f\"HyperSpy version: {version('hyperspy')}\")"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"### Set normalised"
]
},
{
"cell_type": "code",
"execution_count": null,
"metadata": {},
"outputs": [],
"source": [
"normalised = True if normalised.lower() == \"true\" else False"
]
},
{
"cell_type": "markdown",
"metadata": {},
Expand Down Expand Up @@ -191,22 +208,25 @@
"\n",
"for raw_data in file_list:\n",
" # windowing happens here for each file\n",
" data = read_raw_data(raw_data)\n",
" data = read_raw_data(raw_data, normalised)\n",
" \n",
" # window it\n",
" w_mca = window_mca(data[\"mca\"], lg_start, lg_end, data[\"scan_shape\"], data[\"scan_model\"])\n",
" \n",
" # sum the I0\n",
" I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n",
" \n",
" if normalised:\n",
" # sum the I0\n",
" I0_t = np.squeeze(data[\"I0_1\"] + data[\"I0_2\"] + data[\"I0_3\"] + data[\"I0_4\"])\n",
" I0_total.append(I0_t)\n",
" else:\n",
" I0_total.append(np.ones_like(w_mca))\n",
"\n",
" # record everything\n",
" windowed.append(w_mca)\n",
" \n",
" energy_data.append(data[\"energy\"])\n",
" SampleX.append(data[\"x\"])\n",
" SampleY.append(data[\"y\"])\n",
" scan_shapes.append(data[\"scan_shape\"])\n",
" I0_total.append(I0_t)\n",
" \n",
"print(f\"Time reading raw data: {(time.perf_counter() - start)/60:.2f} min\")"
]
Expand Down Expand Up @@ -468,7 +488,8 @@
" tol=tol_residual, \n",
" niter_short=num_short_iteration, \n",
" niter_final=num_final_iteration, \n",
" verbose=True)"
" verbose=True) \n",
"loop_asd.residuals = np.full(sum(loop_asd.num_iters) + loop_asd.rank_max, np.nan) \n"
]
},
{
Expand Down
13 changes: 8 additions & 5 deletions i14/schema/dpcSchema.json
Original file line number Diff line number Diff line change
@@ -1,16 +1,19 @@
{
"type": "object",
"properties": {
"scanNumbers": {
"type": "string",
"title": "Scan Numbers",
"description": "A list containing scan numbers to be processed. Eg. [274317] or [274317, 274318, 274319]"
"multiScan": {
"title": "Scans",
"type": "array",
"items":
{
"type": "object"
}
},
"outputFolder": {
"type": "string",
"title": "Output Folder",
"description": "The full path of the output file"
}
},
"required": ["scanNumbers", "outputFolder"]
"required": ["multiScan", "outputFolder"]
}
17 changes: 16 additions & 1 deletion i14/schema/dpcUISchema.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,22 @@
"elements": [
{
"type": "Control",
"scope": "#/properties/scanNumbers"
"scope": "#/properties/multiScan",
"options": {
"elementLabelProp": "scanRange",
"detail": {
"type": "VerticalLayout",
"elements": [
{
"type": "Control",
"scope": "#/properties/multiScan/items",
"options": {
"useScanRangeControl": true
}
}
]
}
}
},
{
"type": "Control",
Expand Down
1 change: 1 addition & 0 deletions i14/schema/xanesPointSchema.json
Original file line number Diff line number Diff line change
Expand Up @@ -141,6 +141,7 @@
"normalise": {
"type": "boolean",
"title": "Normalise",
"default": true,
"description": "Whether to normalise the windowed intensity by 'adc_ionCh1a'"
}
},
Expand Down
1 change: 1 addition & 0 deletions i14/schema/xanesSchema.json
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@
"normalise": {
"type": "boolean",
"title": "Normalise",
"default": true,
"description": "Whether to normalise the stack by I0"
}
},
Expand Down
3 changes: 2 additions & 1 deletion i14/schema/xanesSparseSchema.json
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,8 @@
},
"normalise": {
"type": "boolean",
"title": "Normalise"
"title": "Normalise",
"default": true
},
"edgeElement": {
"type": "string",
Expand Down
13 changes: 8 additions & 5 deletions i14/schema/xrd1dSchema.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
{
"type": "object",
"properties": {
"scanNumbers": {
"type": "string",
"title": "Scan Numbers",
"description": "A list containing scan numbers to be processed. Eg. [274317] or [274317, 274318, 274319]"
"multiScan": {
"title": "Scans",
"type": "array",
"items":
{
"type": "object"
}
},
"outputFolder": {
"type": "string",
Expand Down Expand Up @@ -49,7 +52,7 @@
}
},
"required": [
"scanNumbers",
"multiScan",
"outputFolder",
"xrdProcessConfigUpload"
]
Expand Down
17 changes: 16 additions & 1 deletion i14/schema/xrd1dUISchema.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,22 @@
"elements": [
{
"type": "Control",
"scope": "#/properties/scanNumbers"
"scope": "#/properties/multiScan",
"options": {
"elementLabelProp": "scanRange",
"detail": {
"type": "VerticalLayout",
"elements": [
{
"type": "Control",
"scope": "#/properties/multiScan/items",
"options": {
"useScanRangeControl": true
}
}
]
}
}
},
{
"type": "Control",
Expand Down
13 changes: 8 additions & 5 deletions i14/schema/xrd2dSchema.json
Original file line number Diff line number Diff line change
@@ -1,10 +1,13 @@
{
"type": "object",
"properties": {
"scanNumbers": {
"type": "string",
"title": "Scan Numbers",
"description": "A list containing scan numbers to be processed. Eg. [274317] or [274317, 274318, 274319]"
"multiScan": {
"title": "Scans",
"type": "array",
"items":
{
"type": "object"
}
},
"outputFolder": {
"type": "string",
Expand All @@ -28,7 +31,7 @@
}
},
"required": [
"scanNumbers",
"multiScan",
"outputFolder",
"xrdProcessConfigUpload"
]
Expand Down
17 changes: 16 additions & 1 deletion i14/schema/xrd2dUISchema.json
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,22 @@
"elements": [
{
"type": "Control",
"scope": "#/properties/scanNumbers"
"scope": "#/properties/multiScan",
"options": {
"elementLabelProp": "scanRange",
"detail": {
"type": "VerticalLayout",
"elements": [
{
"type": "Control",
"scope": "#/properties/multiScan/items",
"options": {
"useScanRangeControl": true
}
}
]
}
}
},
{
"type": "Control",
Expand Down
1 change: 1 addition & 0 deletions i14/schema/xrfTomoSchema.json
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
"normalise": {
"type": "boolean",
"title": "Normalise",
"default": true,
"description": "Whether to normalise the stack using the total mass of each projection."
},
"cor": {
Expand Down
Loading