Merge pull request #283 from DiamondLightSource/improve-xrf-tomo

Improve xrf tomo

Author: daurer

i14/configs/httomo_config_fbp.yaml i14/configs/httomo_config_FBP.yamli14/configs/httomo_config_fbp.yaml renamed to i14/configs/httomo_config_FBP.yaml

@@ -23,21 +23,18 @@
     dif: 1000 # A difference between the outlier value and the median value of neighbouring pixels.
   save_result: true
 # --- Center of Rotation auto-finding. Required for reconstruction bellow. ---
-- method: find_center_vo
-  module_path: httomolibgpu.recon.rotation
+- method: find_center
+  module_path: tomopy.recon.rotation
   parameters:
-    ind: null  # A vertical slice (sinogram) index to calculate CoR, `mid` can be used for middle
-    average_radius: 0 # Average several sinograms to improve SNR, one can try 3-5 range
-    cor_initialisation_value: null  # Use if an approximate CoR is known
-    smin: -50
-    smax: 50
-    srad: 6.0
-    step: 0.5
-    ratio: 0.5
-    drop: 20
+    ind: null
+    init: <INITCOR>
+    tol: 0.5
+    mask: true
+    ratio: 1.0
+    sinogram_order: false
   id: centering
   side_outputs:
-    cor: centre_of_rotation  # A side output of the method, here a CoR scalar value
+    cor: centre_of_rotation
 # --- Reconstruction method. ---
 - method: FBP2d_astra
   module_path: httomolibgpu.recon.algorithm

i14/configs/httomo_config_OLEM.yaml

@@ -0,0 +1,74 @@
+# --- Standard tomography loader for NeXus files. ---
+- method: standard_tomo
+  module_path: httomo.data.hdf.loaders
+  parameters:
+    data_path: auto
+    image_key_path: auto
+    rotation_angles: auto
+    preview:
+      detector_x:   # horizontal data previewing/cropping.
+        # when null, the full data dimension is used, i.e., no previewing
+        start: null
+        stop: null
+      detector_y:   # vertical data previewing/cropping.
+        # when null, the full data dimension is used, i.e., no previewing
+        start: null
+        stop: null
+    darks: null
+    flats: null
+- method: remove_outlier
+  module_path: httomolibgpu.misc.corr
+  parameters:
+    kernel_size: 3  # The size of the 3D neighbourhood surrounding the voxel. Odd integer.
+    dif: 1000 # A difference between the outlier value and the median value of neighbouring pixels.
+  save_result: true
+- method: find_center
+  module_path: tomopy.recon.rotation
+  parameters:
+    ind: null
+    init: <INITCOR>
+    tol: 0.5
+    mask: true
+    ratio: 1.0
+    sinogram_order: false
+  id: centering
+  side_outputs:
+    cor: centre_of_rotation
+# --- Reconstruction method. ---
+- method: OSEM3d_tomobar
+  module_path: httomolibgpu.recon.algorithm
+  parameters:
+    center: ${{centering.side_outputs.centre_of_rotation}}
+    detector_pad: false
+    recon_size: null
+    recon_mask_radius: 0.95
+    iterations: 20
+    subsets_number: 12
+    regularisation_type: PD_TV
+    regularisation_parameter: 0.01
+    regularisation_iterations: 30
+    regularisation_half_precision: true
+    nonnegativity: true
+# --- Calculate global statistics on the reconstructed volume, required for data rescaling. ---
+- method: calculate_stats
+  module_path: httomo.methods
+  parameters: {}
+  id: statistics
+  side_outputs:
+    glob_stats: glob_stats
+# --- Rescaling the data using min/max obtained from `calculate_stats`. ---
+- method: rescale_to_int
+  module_path: httomolibgpu.misc.rescale
+  parameters:
+    perc_range_min: 0.0
+    perc_range_max: 100.0
+    bits: 32
+    glob_stats: ${{statistics.side_outputs.glob_stats}}
+# --- Saving data into images. ---
+- method: save_to_images
+  module_path: httomolib.misc.images
+  parameters:
+    subfolder_name: images
+    axis: auto
+    file_format: tif  # `tif` or `jpeg` can be used.
+    asynchronous: true

i14/notebooks/xrf_tomo_align_initial.ipynb

@@ -25,6 +25,7 @@
     "inpath = \"\"\n",
     "outpath_nexus = None\n",
     "outpath_shifts = None\n",
+    "outpath_cor = None\n",
     "alignment_section = \"all\"\n",
     "alignment_band = 15\n",
     "cropping = \"auto\"\n",
@@ -895,6 +896,24 @@
     "np.savetxt(outpath_shifts, shifts+delta_shifts, delimiter=\" \")\n",
     "np.savetxt(outpath_limits, [cmin, cmax], delimiter=\" \")"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "id": "16c977de-ca7b-4afb-a564-780df133334b",
+   "metadata": {},
+   "source": [
+    "## Saving estimated centre of rotation to a TXT file"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "id": "a1784c75-6b2a-4e48-84d3-315d70311562",
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "np.savetxt(outpath_cor, np.array([(cmax-cmin)/2]))"
+   ]
   }
  ],
  "metadata": {

i14/schema/xrfTomoSchema.json

@@ -40,7 +40,7 @@
     },
     "alignmentSection": {
       "type": "string",
-        "enum": ["all", "top", "middle", "bottom"],
+      "enum": ["all", "top", "middle", "bottom"],
       "title": "Section for alignment",
       "description": "The section of the projections to be used for the initial alignment."
     },
@@ -59,6 +59,12 @@
       "type": "number",
       "title": "Centre of rotation",
       "description": "Optional value for centre of rotation for the tomographic reconstruction, using Vo centering method if not provided."
+    },
+    "reconsMethod" : {
+      "type": "string",
+      "enum": ["FBP", "OLEM"],
+      "title": "Reconstruction Method",
+      "description": "The method to be used for reconstruction. FBP is default. OLEM might provide smoother results."
     }
   },
   "required": ["outputFolder", "multiScan", "multiEdge"],

i14/schema/xrfTomoUISchema.json

@@ -71,9 +71,18 @@
       ]
     },
     {
-      "type": "Control",
-      "scope": "#/properties/cor"
-    },      
+      "type": "HorizontalLayout",
+      "elements": [
+        {
+          "type": "Control",
+          "scope": "#/properties/cor"
+        },
+        {
+          "type": "Control",
+          "scope": "#/properties/reconsMethod"
+        }
+      ]
+    },
     {
       "type": "Control",
       "scope": "#/properties/outputFolder"

i14/templates/xrf-tomo-template.yaml

@@ -25,11 +25,11 @@ spec:
     - name: outputFolder
       value: "testing"
     - name: multiScan
-      value: '[{"multiScan":{"start":423844,"end":423947,"excluded":[423908,423909,423911,423916,423917,423935,423936,423937,423940,423942,423943,423944,423945]}}]'
+      value: '[{"multiScan":{"start":436147,"end":436230,"excluded":[]}}]'
     - name: multiEdge
-      value: '[{"edgeElement":"Pb","edgeTransition":"La"},{"edgeElement":"Br","edgeTransition":"Ka"}]'
+      value: '[{"edgeElement":"Tl","edgeTransition":"La"},{"edgeElement":"Ga","edgeTransition":"Ka"}]'
     - name: elementToAlign
-      value: "Pb"
+      value: "Tl"
     - name: transitionToAlign
       value: "La"
     - name: alignmentSection
@@ -40,6 +40,8 @@ spec:
       value: true
     - name: cor
       value: "null"
+    - name: reconsMethod
+      value: "FBP"
   volumeClaimTemplates:
   - metadata:
       name: tmp
@@ -57,16 +59,24 @@ spec:
       volumeMounts:
       - name: session
         mountPath: "{{`{{ workflow.parameters.visitdir }}`}}"
-      - name: tmp
-        mountPath: /tmp
       command: [bash]
       source: |
         OUTPUT="{{`{{ workflow.parameters.visitdir }}`}}"/processing/workflows/"{{`{{ workflow.parameters.outputFolder }}`}}"
+        METHOD="{{`{{ workflow.parameters.reconsMethod }}`}}"
         mkdir -p $OUTPUT
         echo '{{ .Files.Get "notebooks/xrf_tomo_align_initial.ipynb" | b64enc }}' | base64 -d > $OUTPUT/align_initial.ipynb
         echo '{{ .Files.Get "notebooks/xrf_tomo_align_and_normalise.ipynb" | b64enc }}' | base64 -d > $OUTPUT/align_and_normalise.ipynb
         echo '{{ .Files.Get "notebooks/xrf_tomo_plotting.ipynb" | b64enc }}' | base64 -d > $OUTPUT/plotting.ipynb
-        echo '{{ .Files.Get "configs/httomo_config_fbp.yaml" | b64enc }}' | base64 -d > $OUTPUT/httomo.yaml
+        echo '{{ .Files.Get "configs/httomo_config_OLEM.yaml" | b64enc }}' | base64 -d > $OUTPUT/httomo_OLEM.yaml
+        echo '{{ .Files.Get "configs/httomo_config_FBP.yaml" | b64enc }}' | base64 -d > $OUTPUT/httomo_FBP.yaml
+        if [ "$METHOD" = "OLEM" ]; then
+           mv $OUTPUT/httomo_OLEM.yaml $OUTPUT/httomo.yaml
+           rm $OUTPUT/httomo_FBP.yaml
+        fi
+        if [ "$METHOD" = "FBP" ]; then
+           mv $OUTPUT/httomo_FBP.yaml $OUTPUT/httomo.yaml
+           rm $OUTPUT/httomo_OLEM.yaml
+        fi
     volumes:
     - name: session
       hostPath:
@@ -138,6 +148,7 @@ spec:
           -p inpath "tomo_xrf_${START}_${END}_${EDGE}.nxs" \
           -p outpath_nexus "${EDGE}_aligned_initial.nxs" \
           -p outpath_shifts "${EDGE}_shifts.txt" \
+          -p outpath_cor "${EDGE}_cor.txt" \
           -p alignment_section "{{`{{ workflow.parameters.alignmentSection }}`}}" \
           -p alignment_band "{{`{{ workflow.parameters.alignmentBand }}`}}"
           --prepare-only
@@ -149,6 +160,10 @@ spec:
       - name: tmp
         mountPath: /tmp
     outputs:
+      parameters:
+      - name: initcor
+        valueFrom:
+          path: "{{`{{ workflow.parameters.visitdir }}`}}/processing/workflows/{{`{{ workflow.parameters.outputFolder }}`}}/{{`{{ workflow.parameters.elementToAlign }}`}}-{{`{{ workflow.parameters.transitionToAlign }}`}}_cor.txt"
       artifacts:
       - name: align_initial
         path: "{{`{{ workflow.parameters.visitdir }}`}}/processing/workflows/{{`{{ workflow.parameters.outputFolder }}`}}/_assets/align_initial.html"
@@ -265,8 +280,9 @@ spec:
     inputs:
       parameters:
         - name: edge
+        - name: initcor
     script:
-      image: ghcr.io/diamondlightsource/httomo:2.5
+      image: ghcr.io/diamondlightsource/httomo:3.2.1
       env:
       - name: CUPY_CACHE_DIR
         value: /tmp/.cupy/kernel_cache
@@ -284,8 +300,10 @@ spec:
         TRANSITION="{{`{{= sprig.mustFromJson(inputs.parameters.edge).edgeTransition }}`}}"
         mkdir -p $OUTPUT/httomo_output
 
+        COR_ESTIMATE="{{`{{ inputs.parameters.initcor }}`}}"
+        [[ $COR_ESTIMATE != "null" ]] && sed -i 's/<INITCOR>/'"${COR_ESTIMATE}"'/' $OUTPUT/httomo.yaml
         COR="{{`{{ workflow.parameters.cor }}`}}"
-        [[ $COR != "null" ]] && sed -i 's/${{`{{centering.side_outputs.centre_of_rotation}}`}}/'"${COR}"'/' $OUTPUT/httomo.yaml 
+        [[ $COR != "null" ]] && sed -i 's/${{`{{centering.side_outputs.centre_of_rotation}}`}}/'"${COR}"'/' $OUTPUT/httomo.yaml
 
         cmd=(/opt/conda/bin/mpirun -n 1 /opt/conda/bin/python -m httomo run)
         cmd+=("--output-folder-name=${ELEMENT}-${TRANSITION}")
@@ -418,6 +436,7 @@ spec:
     inputs:
       parameters:
         - name: edge
+        - name: initcor
     dag:
       tasks:
       - name: alignment
@@ -432,6 +451,8 @@ spec:
           parameters:
             - name: edge
               value: "{{`{{ inputs.parameters.edge }}`}}"
+            - name: initcor
+              value: "{{`{{ inputs.parameters.initcor }}`}}"
         dependencies: [alignment]
 
   - name: xrf-tomo-pipeline
@@ -463,6 +484,8 @@ spec:
           parameters:
             - name: edge
               value: "{{`{{ item }}`}}"
+            - name: initcor
+              value: "{{`{{ tasks.generate-shifts.outputs.parameters.initcor }}`}}"
         withParam: "{{`{{ workflow.parameters.multiEdge }}`}}"
 
       - name: show-results