bug: filter clinvar missing conditions (#1083)

Author: bpblanken

Diff for: v03_pipeline/lib/reference_datasets/clinvar.py

@@ -154,10 +154,13 @@ def select_fields(ht):
         conflictingPathogenicities=parsed_and_mapped_clnsigconf(ht),
         goldStars=CLINVAR_GOLD_STARS_LOOKUP.get(hl.delimit(ht.info.CLNREVSTAT)),
         submitters=ht.submitters,
-        # assumes the format 'MedGen#:condition', e.g.'C0023264:Leigh syndrome'
-        conditions=hl.map(
-            lambda p: p.split(r':')[1],
-            ht.conditions,
+        # assumes the format 'MedGen#:condition;MedGen#:condition', e.g.'C0023264:Leigh syndrome'
+        conditions=hl.filter(
+            hl.is_defined,
+            hl.flatmap(
+                lambda p: p.split(';'),
+                ht.conditions,
+            ).map(lambda p: p.split(':')[1]),
         ),
     )
 

Diff for: v03_pipeline/lib/reference_datasets/clinvar_test.py

@@ -169,3 +169,73 @@ def test_get_ht(self):
                     ),
                 ],
             )
+
+            # VariationID 9 tests Conditions parsing
+            self.assertListEqual(
+                ht.collect()[8].submitters,
+                [
+                    'Hemochromatosis type 1',
+                    'Hereditary cancer-predisposing syndrome',
+                    'HFE-related disorder',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Bronze diabetes',
+                    'Hemochromatosis type 1',
+                    'HFE-related disorder',
+                    'Hemochromatosis type 1',
+                    'Abdominal pain',
+                    'Atypical behavior',
+                    'Pain',
+                    'Peripheral neuropathy',
+                    'Abnormality of the nervous system',
+                    'Abnormality of the male genitalia',
+                    'Abnormal peripheral nervous system morphology',
+                    'Hereditary hemochromatosis',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'Hereditary hemochromatosis',
+                    'not provided',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'not provided',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hereditary hemochromatosis',
+                    'Cardiomyopathy',
+                    'not provided',
+                    'Juvenile hemochromatosis',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'not provided',
+                    'Inborn genetic diseases',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'Porphyrinuria',
+                    'Cutaneous photosensitivity',
+                    'Hemochromatosis type 1',
+                    'Hereditary hemochromatosis',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'Hemochromatosis type 1',
+                    'Variegate porphyria',
+                    'Familial porphyria cutanea tarda',
+                    'Alzheimer disease type 1',
+                    'Microvascular complications of diabetes, susceptibility to, 7',
+                    'Transferrin serum level quantitative trait locus 2',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                ],
+            )