feat: Migrating existing configs to include frame & ctf metadata

ingestion_tools/dataset_configs/10426.yaml

@@ -1,202 +1,191 @@
 alignments:
-- metadata:
-    format: IMOD
-    method_type: fiducial_based
-  sources:
-  - source_multi_glob:
-      list_globs:
-      - alignmentfiles/{run_name}_binalign8_fid.tlt
-      - alignmentfiles/{run_name}_binalign8_fid.xf
-
-annotations: []
-
+  - metadata:
+      format: IMOD
+      method_type: fiducial_based
+    sources:
+      - source_multi_glob:
+          list_globs:
+            - alignmentfiles/{run_name}_binalign8_fid.tlt
+            - alignmentfiles/{run_name}_binalign8_fid.xf
+annotations: [ ]
 collection_metadata:
-- sources:
-  - source_glob:
-      list_glob: 'mdocs/{mdoc_name}'
-
+  - sources:
+      - source_glob:
+          list_glob: mdocs/{mdoc_name}
 datasets:
-- metadata:
-    authors: &id001
-    - name: Willy W. Sun
-      primary_author_status: true
-    - name: Dennis J Michalak
-      primary_author_status: true
-    - ORCID: 0000-0003-1160-4558
-      corresponding_author_status: true
-      name: Kem A. Sochacki
-      primary_author_status: true
-    - ORCID: 0000-0002-5130-5061
-      corresponding_author_status: true
-      name: Jenny E. Hinshaw
-      primary_author_status: false
-    - ORCID: 0000-0001-5355-9535
-      corresponding_author_status: true
-      name: Justin W. Taraska
-      primary_author_status: false
-    cell_strain:
-      id: CLO:0051569
-      name: RCB1975 cell
-    cell_type:
-      id: CL:0000010
-      name: cultured cell
-    cross_references: &id002
-      publications: 10.1038/s41467-025-56045-z, 10.1101/2024.06.27.600657
-      related_database_entries: EMD-44921, EMD-44909, EMD-44922, EMD-46973
-    dataset_description: Michalak, Sun, Sochacki et al. 2024 unroofing manuscript
-      contains 10 grids. This is grid 1 of 10 and contains basal membranes of HSC3
-      cells.  The cell line expresses EGFR-GFP as described in Panilla-Macua et al.
-      Elife 2017 and has been serum starved.  The cells were grown on Quantifoil Au
-      300 R2/2, 2 nm carbon grids prior to removing the apical membrane and cytoplasm
-      with 0.7 bar of paraformaldehyde prior to vitrification.
-    dataset_identifier: 10426
-    dataset_title: Plasma membranes isolated by unroofing - Grid 01
-    dates: &dates
-      deposition_date: '2024-07-01'
-      last_modified_date: '2025-01-08'
-      release_date: '2025-01-08'
-    funding:
-    - funding_agency_name: Chan Zuckerberg Initiative
-      grant_id: 2021-234544
-    - funding_agency_name: NIH
-      grant_id: 1ZIAHL006098
-    - funding_agency_name: NIH
-      grant_id: 1ZIADK060100
-    grid_preparation: plasma discharge 30 sec + UV treatment 20 min + 125 ng/ul fibronectin,
-      20 minutes + water rinse + seeded with 15000 cells
-    organism:
-      name: Homo sapiens
-      taxonomy_id: 9606
-    sample_preparation: serum starved and grown on Quantifoil Au 300 R2/2, 2 nm carbon
-      grids
-    sample_type: organelle
-    cell_component:
-      name: plasma membrane
-      id: GO:0005886
-  sources:
-  - literal:
-      value:
-      - '10426'
-
-depositions:
-- metadata:
-    authors: *id001
-    cross_references: *id002
-    dates: *dates
-    deposition_description: Cryo-electron tomography datasets used in the development
-      and evaluation of the unroofing method for preparation of plasma membranes and
-      associated proteins.
-    deposition_identifier: 10306
-    deposition_title: Human plasma membranes isolated using the unroofing method
-    deposition_types:
-    - dataset
-  sources:
-  - literal:
-      value:
-      - 10306
-
+  - metadata:
+      authors: &id001
+        - name: Willy W. Sun
+          primary_author_status: true
+        - name: Dennis J Michalak
+          primary_author_status: true
+        - ORCID: 0000-0003-1160-4558
+          corresponding_author_status: true
+          name: Kem A. Sochacki
+          primary_author_status: true
+        - ORCID: 0000-0002-5130-5061
+          corresponding_author_status: true
+          name: Jenny E. Hinshaw
+          primary_author_status: false
+        - ORCID: 0000-0001-5355-9535
+          corresponding_author_status: true
+          name: Justin W. Taraska
+          primary_author_status: false
+      cell_component:
+        id: GO:0005886
+        name: plasma membrane
+      cell_strain:
+        id: CLO:0051569
+        name: RCB1975 cell
+      cell_type:
+        id: CL:0000010
+        name: cultured cell
+      cross_references: &id002
+        publications: 10.1038/s41467-025-56045-z, 10.1101/2024.06.27.600657
+        related_database_entries: EMD-44921, EMD-44909, EMD-44922, EMD-46973
+      dataset_description: Michalak, Sun, Sochacki et al. 2024 unroofing manuscript
+        contains 10 grids. This is grid 1 of 10 and contains basal membranes of HSC3
+        cells.  The cell line expresses EGFR-GFP as described in Panilla-Macua et al.
+        Elife 2017 and has been serum starved.  The cells were grown on Quantifoil Au
+        300 R2/2, 2 nm carbon grids prior to removing the apical membrane and cytoplasm
+        with 0.7 bar of paraformaldehyde prior to vitrification.
+      dataset_identifier: 10426
+      dataset_title: Plasma membranes isolated by unroofing - Grid 01
+      dates: &id003
+        deposition_date: '2024-07-01'
+        last_modified_date: '2025-01-08'
+        release_date: '2025-01-08'
+      funding:
+        - funding_agency_name: Chan Zuckerberg Initiative
+          grant_id: 2021-234544
+        - funding_agency_name: NIH
+          grant_id: 1ZIAHL006098
+        - funding_agency_name: NIH
+          grant_id: 1ZIADK060100
+      grid_preparation: plasma discharge 30 sec + UV treatment 20 min + 125 ng/ul fibronectin,
+        20 minutes + water rinse + seeded with 15000 cells
+      organism:
+        name: Homo sapiens
+        taxonomy_id: 9606
+      sample_preparation: serum starved and grown on Quantifoil Au 300 R2/2, 2 nm carbon
+        grids
+      sample_type: organelle
+    sources:
+      - literal:
+          value:
+            - '10426'
 deposition_keyphotos:
-- sources:
-  - literal:
-      value:
-        snapshot: "cryoetportal-rawdatasets-dev/deposition_key_photos/deposition_10306_snapshot.png"
-        thumbnail: "cryoetportal-rawdatasets-dev/deposition_key_photos/deposition_10306_thumbnail.png"
-
+  - sources:
+      - literal:
+          value:
+            snapshot: cryoetportal-rawdatasets-dev/deposition_key_photos/deposition_10306_snapshot.png
+            thumbnail: cryoetportal-rawdatasets-dev/deposition_key_photos/deposition_10306_thumbnail.png
+depositions:
+  - metadata:
+      authors: *id001
+      cross_references: *id002
+      dates: *id003
+      deposition_description: Cryo-electron tomography datasets used in the development
+        and evaluation of the unroofing method for preparation of plasma membranes and
+        associated proteins.
+      deposition_identifier: 10306
+      deposition_title: Human plasma membranes isolated using the unroofing method
+      deposition_types:
+        - dataset
+    sources:
+      - literal:
+          value:
+            - 10306
 frames:
-- sources:
-  - source_glob:
-      list_glob: 'frames/{frame_prefix}*.mrc'
-
-gains: []
-
+  - metadata:
+      dose_rate: float {dose_rate}
+      is_gain_corrected: true
+    sources:
+      - source_glob:
+          list_glob: frames/{frame_prefix}*.mrc
+gains: [ ]
 rawtilts:
   - sources:
-    - source_glob:
-        list_glob: 'rawtilts/{run_name}.rawtlt'
-
+      - source_glob:
+          list_glob: rawtilts/{run_name}.rawtlt
 runs:
-- sources:
-  - source_glob:
-      list_glob: tomograms_bin8/*
-      match_regex: \.rec$
-      name_regex: (.*)_binalign8.rec
-
+  - sources:
+      - source_glob:
+          list_glob: tomograms_bin8/*
+          match_regex: \.rec$
+          name_regex: (.*)_binalign8.rec
 standardization_config:
   deposition_id: 10306
-  run_data_map_file: run_to_data_map.tsv
+  run_data_map_file: run_to_data_map_vali.tsv
   source_prefix: kem_sochacki_06_2024/grid01/
-
 tiltseries:
-- metadata:
-    acceleration_voltage: 300000
-    binning_from_frames: 1
-    camera:
-      manufacturer: Gatan
-      model: K3
-    data_acquisition_software: SerialEM
-    is_aligned: false
-    microscope:
-      manufacturer: FEI
-      model: TITAN KRIOS
-    microscope_optical_setup:
-      energy_filter: Gatan Bioquantum
-    pixel_spacing: 1.0825
-    spherical_aberration_constant: 2.7
-    tilt_axis: 0.0
-    tilt_range:
-      max: float {tilt_series_max_tilt}
-      min: float {tilt_series_min_tilt}
-    tilt_series_quality: 5
-    tilt_step: 2.0
-    tilting_scheme: dose-symmetric
-    total_flux: float {tilt_series_total_flux}
-  sources:
-  - source_glob:
-      list_glob: 'tilt_series/{run_name}_binalign8.st'
-      match_regex: ^.*\.st$
-
+  - metadata:
+      acceleration_voltage: 300000
+      binning_from_frames: 8
+      camera:
+        manufacturer: Gatan
+        model: K3
+      data_acquisition_software: SerialEM
+      is_aligned: false
+      microscope:
+        manufacturer: FEI
+        model: TITAN KRIOS
+      microscope_optical_setup:
+        energy_filter: Gatan Bioquantum
+      pixel_spacing: 8.66
+      spherical_aberration_constant: 2.7
+      tilt_axis: 177.0
+      tilt_range:
+        max: float {tilt_series_max_tilt}
+        min: float {tilt_series_min_tilt}
+      tilt_series_quality: 5
+      tilt_step: 2.0
+      tilting_scheme: dose-symmetric
+      total_flux: float {tilt_series_total_flux}
+    sources:
+      - source_glob:
+          list_glob: tilt_series/{run_name}_binalign8.st
+          match_regex: ^.*\.st$
 tomograms:
-- metadata:
-    affine_transformation_matrix:
-    - - 1
-      - 0
-      - 0
-      - 0
-    - - 0
-      - 1
-      - 0
-      - 0
-    - - 0
-      - 0
-      - 1
-      - 0
-    - - 0
-      - 0
-      - 0
-      - 1
-    authors: *id001
-    ctf_corrected: false
-    dates: *dates
-    fiducial_alignment_status: FIDUCIAL
-    is_visualization_default: true
-    offset:
-      x: 0
-      y: 0
-      z: 0
-    processing: raw
-    reconstruction_method: SIRT
-    reconstruction_software: IMOD
-    tomogram_version: 1
-    voxel_spacing: 8.66
-  sources:
-  - source_glob:
-      list_glob: 'tomograms_bin8/{run_name}_binalign8.rec'
-      match_regex: (.*)\.rec
-
+  - metadata:
+      affine_transformation_matrix:
+        - - 1
+          - 0
+          - 0
+          - 0
+        - - 0
+          - 1
+          - 0
+          - 0
+        - - 0
+          - 0
+          - 1
+          - 0
+        - - 0
+          - 0
+          - 0
+          - 1
+      authors: *id001
+      ctf_corrected: false
+      dates: *id003
+      fiducial_alignment_status: FIDUCIAL
+      is_visualization_default: true
+      offset:
+        x: 0
+        y: 0
+        z: 0
+      processing: raw
+      reconstruction_method: SIRT
+      reconstruction_software: IMOD
+      tomogram_version: 1
+      voxel_spacing: 8.66
+    sources:
+      - source_glob:
+          list_glob: tomograms_bin8/{run_name}_binalign8.rec
+          match_regex: (.*)\.rec
 version: 1.1.0
-
 voxel_spacings:
-- sources:
-  - literal:
-      value:
-      - 8.66
+  - sources:
+      - literal:
+          value:
+            - 8.66