feat: Migrating existing configs to include frame & ctf metadata

ingestion_tools/dataset_configs/10301.yaml

@@ -7,13 +7,6 @@ alignments:
       list_globs:
       - '{run_name}/AreTomo/{run_name}_dose-filt.tlt'
       - '{run_name}/AreTomo/{run_name}_dose-filt.xf'
-- metadata:
-    format: ARETOMO3
-    method_type: patch_tracking
-  sources:
-  - source_multi_glob:
-      list_globs:
-      - '{run_name}/ctffind4/ctfphaseflip_ctffind4.txt'
 annotations:
 - metadata:
     annotation_method: Template matching + manual filtering + 3D classification filtering
@@ -198,6 +191,12 @@ collection_metadata:
   - source_multi_glob:
       list_globs:
       - '{run_name}/{run_name}.mdoc'
+ctf:
+- metadata:
+    format: CTFFIND
+  sources:
+  - source_glob:
+      list_glob: '{run_name}/ctffind4/ctfphaseflip_ctffind4.txt'
 dataset_keyphotos:
 - sources:
   - literal:

ingestion_tools/dataset_configs/10302.yaml

@@ -7,19 +7,18 @@ alignments:
       list_globs:
       - chlamy_visual_proteomics/{run_name}/AreTomo/{run_name}_dose-filt.tlt
       - chlamy_visual_proteomics/{run_name}/AreTomo/{run_name}_dose-filt.xf
-- metadata:
-    format: ARETOMO3
-    method_type: patch_tracking
-  sources:
-  - source_multi_glob:
-      list_globs:
-      - chlamy_visual_proteomics/{run_name}/ctffind4/ctfphaseflip_ctffind4.txt
 annotations: []
 collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
       - chlamy_visual_proteomics/{run_name}/{run_name}.mdoc
+ctf:
+- metadata:
+    format: CTFFIND
+  sources:
+  - source_glob:
+      list_glob: 'chlamy_visual_proteomics/{run_name}/ctffind4/ctfphaseflip_ctffind4.txt'
 dataset_keyphotos:
 - sources:
   - literal:

ingestion_tools/dataset_configs/gjensen/10014.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -68,11 +69,20 @@ depositions:
   - literal:
       value:
       - 10014
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/{mapped_frame_name}/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10015.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -68,11 +69,20 @@ depositions:
   - literal:
       value:
       - 10014
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/3dimage_*/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10016.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -68,11 +69,20 @@ depositions:
   - literal:
       value:
       - 10014
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/3dimage_*/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10017.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -65,11 +66,20 @@ depositions:
   - literal:
       value:
       - 10014
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/3dimage_*/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10018.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -64,11 +65,20 @@ depositions:
   - literal:
       value:
       - 10014
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/3dimage_*/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10019.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -63,11 +64,20 @@ depositions:
   - literal:
       value:
       - 10014
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/3dimage_*/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10020.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -68,11 +69,20 @@ depositions:
   - literal:
       value:
       - 10014
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/3dimage_*/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10021.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -63,11 +64,20 @@ depositions:
   - literal:
       value:
       - 10049
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/3dimage_*/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10022.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -61,11 +62,20 @@ depositions:
   - literal:
       value:
       - 10018
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/3dimage_*/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10023.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -63,11 +64,20 @@ depositions:
   - literal:
       value:
       - 10018
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/{mapped_frame_name}/*.rawtlt'
 runs:
 - sources:

ingestion_tools/dataset_configs/gjensen/10024.yaml

@@ -3,7 +3,8 @@ collection_metadata:
 - sources:
   - source_multi_glob:
       list_globs:
-      - '{run_name}/rawdata/*.mdoc'
+      - '{run_name}/*/*.mdoc'
+      - '{run_name}/generated/*.mdoc'
 dataset_keyphotos:
 - sources:
   - literal:
@@ -62,11 +63,20 @@ depositions:
   - literal:
       value:
       - 10018
+frames:
+- metadata:
+    dose_rate: 0.0
+    is_gain_corrected: false
+  sources:
+  - literal:
+      value:
+      - default
 rawtilts:
 - sources:
   - source_multi_glob:
       list_globs:
       - '{run_name}/file_*/*.rawtlt'
+      - '{run_name}/generated/*.rawtlt'
       - '{run_name}/3dimage_*/*.rawtlt'
 runs:
 - sources: