feat: filter long alleles (#1080)

* feat: filter long alleles

* trailing comma

* Update validate_callset.py

* bug: filter clinvar missing conditions (#1083)

Author: bpblanken

Diff for: v03_pipeline/lib/model/dataset_type.py

@@ -373,6 +373,10 @@ def should_send_to_allele_registry(self):
     def requires_dataproc(self):
         return self == DatasetType.SNV_INDEL
 
+    @property
+    def filter_invalid_sites(self):
+        return self == DatasetType.SNV_INDEL
+
     @property
     def should_export_to_vcf(self):
         return self == DatasetType.SV

Diff for: v03_pipeline/lib/reference_datasets/clinvar.py

@@ -154,10 +154,13 @@ def select_fields(ht):
         conflictingPathogenicities=parsed_and_mapped_clnsigconf(ht),
         goldStars=CLINVAR_GOLD_STARS_LOOKUP.get(hl.delimit(ht.info.CLNREVSTAT)),
         submitters=ht.submitters,
-        # assumes the format 'MedGen#:condition', e.g.'C0023264:Leigh syndrome'
-        conditions=hl.map(
-            lambda p: p.split(r':')[1],
-            ht.conditions,
+        # assumes the format 'MedGen#:condition;MedGen#:condition', e.g.'C0023264:Leigh syndrome'
+        conditions=hl.filter(
+            hl.is_defined,
+            hl.flatmap(
+                lambda p: p.split(';'),
+                ht.conditions,
+            ).map(lambda p: p.split(':')[1]),
         ),
     )
 

Diff for: v03_pipeline/lib/reference_datasets/clinvar_test.py

@@ -169,3 +169,73 @@ def test_get_ht(self):
                     ),
                 ],
             )
+
+            # VariationID 9 tests Conditions parsing
+            self.assertListEqual(
+                ht.collect()[8].submitters,
+                [
+                    'Hemochromatosis type 1',
+                    'Hereditary cancer-predisposing syndrome',
+                    'HFE-related disorder',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Bronze diabetes',
+                    'Hemochromatosis type 1',
+                    'HFE-related disorder',
+                    'Hemochromatosis type 1',
+                    'Abdominal pain',
+                    'Atypical behavior',
+                    'Pain',
+                    'Peripheral neuropathy',
+                    'Abnormality of the nervous system',
+                    'Abnormality of the male genitalia',
+                    'Abnormal peripheral nervous system morphology',
+                    'Hereditary hemochromatosis',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'Hereditary hemochromatosis',
+                    'not provided',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'not provided',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hereditary hemochromatosis',
+                    'Cardiomyopathy',
+                    'not provided',
+                    'Juvenile hemochromatosis',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'not provided',
+                    'Inborn genetic diseases',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'Porphyrinuria',
+                    'Cutaneous photosensitivity',
+                    'Hemochromatosis type 1',
+                    'Hereditary hemochromatosis',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                    'not provided',
+                    'Hemochromatosis type 1',
+                    'Variegate porphyria',
+                    'Familial porphyria cutanea tarda',
+                    'Alzheimer disease type 1',
+                    'Microvascular complications of diabetes, susceptibility to, 7',
+                    'Transferrin serum level quantitative trait locus 2',
+                    'Hemochromatosis type 1',
+                    'Hemochromatosis type 1',
+                ],
+            )

Diff for: v03_pipeline/lib/tasks/validate_callset.py

@@ -26,6 +26,8 @@
     WriteValidationErrorsForRunTask,
 )
 
+MAX_SNV_INDEL_ALLELE_LENGTH = 500
+
 
 @luigi.util.inherits(BaseLoadingRunParams)
 class ValidateCallsetTask(BaseUpdateTask):
@@ -71,14 +73,20 @@ def update_table(self, mt: hl.MatrixTable) -> hl.MatrixTable:
                 self.callset_path,
             ),
         )
-        if self.dataset_type.can_run_validation:
-            # Rather than throwing an error, we silently remove invalid contigs.
-            # This happens fairly often for AnVIL requests.
+        if self.dataset_type.filter_invalid_sites:
             mt = mt.filter_rows(
-                hl.set(self.reference_genome.standard_contigs).contains(
-                    mt.locus.contig,
+                (
+                    # Rather than throwing an error, we silently remove invalid contigs.
+                    # This happens fairly often for AnVIL requests.
+                    hl.set(self.reference_genome.standard_contigs).contains(
+                        mt.locus.contig,
+                    )
+                    # DRAGEN callsets produce long alternate alleles
+                    # that aren't particularly analyzable as INDELs.
+                    & (hl.len(mt.alleles[1]) < MAX_SNV_INDEL_ALLELE_LENGTH)
                 ),
             )
+
         validation_exceptions = []
         if self.skip_validation or not self.dataset_type.can_run_validation:
             return mt.select_globals(