DSERV-1291 Scorpion adapter

data/active_adapters.py

@@ -39,6 +39,7 @@
 from adapters.clingen_variant_disease_adapter import ClinGen
 from adapters.gencode_gene_structure_adapter import GencodeStructure
 from adapters.VAMP_coding_variant_scores_adapter import VAMPAdapter
+from adapters.scorpion_adapter import ScorpionAdapter
 from adapters.SEM_motif_adapter import SEMMotif
 from adapters.SEM_prediction_adapter import SEMPred
 from adapters.BlueSTARR_variants_biosamples_adapter import BlueSTARRVariantBiosample
@@ -131,6 +132,7 @@
     'starr_seq_variant_biosample': STARRseqVariantBiosample,
     'vamp_coding_variant_phenotype': VAMPAdapter,
     'ontology': Ontology,
+    'scorpion': ScorpionAdapter,
     'SEM_motif': SEMMotif,
     'SEM_motif_protein': SEMMotif,
     'SEM_variant_protein': SEMPred,

data/adapters/helpers.py

@@ -807,3 +807,28 @@ def get_gene_map_from_arangodb(field):
         else:
             gene_map[gval].append(gkey)
     return gene_map
+
+
+def gene_synonym_to_ensembl_id(identifier):
+    # special edge cases where ENSG00000147996 has both in the synonyms list
+    if identifier == 'CBWD5':
+        return 'ENSG00000147996'
+    if identifier == 'CBWD3':
+        return 'ENSG00000196873'
+
+    db = ArangoDB().get_igvf_connection()
+
+    if identifier.startswith('HGNC'):
+        cursor = db.aql.execute(
+            f'FOR gene IN genes FILTER gene.hgnc == "{identifier}" RETURN {{ key: gene._key }}'
+        )
+        for gene in cursor:
+            return gene['key']
+
+    cursor = db.aql.execute(
+        f'FOR gene IN genes FILTER "{identifier}" in gene.synonyms[*] RETURN {{ key: gene._key }}'
+    )
+    for gene in cursor:
+        return gene['key']
+
+    return None

data/adapters/scorpion_adapter.py

@@ -0,0 +1,91 @@
+import csv
+import requests
+import gzip
+import json
+import math
+from typing import Optional
+
+from adapters.helpers import get_file_fileset_by_accession_in_arangodb, gene_synonym_to_ensembl_id
+from adapters.base import BaseAdapter
+from adapters.writer import Writer
+
+# Sample data:
+# tf,tf_ensembl,target,target_ensembl,beta,P,FDR
+# ZNF250,ENSG00000196150,WNK1,ENSG00000060237,2.79614395634379,2.23950511461299e-22,2.2770944348533403e-19
+# ZNF770,ENSG00000198146,IL32,ENSG00000008517,2.68504180081855,1.80943850543389e-22,1.8921041491990902e-19
+# ZNF250,ENSG00000196150,RAB3GAP1,ENSG00000115839,2.60846182810368,6.8583125999997006e-15,4.46018704732234e-13
+# ZNF770,ENSG00000198146,FXYD5,ENSG00000089327,2.47636665757162,2.6083752753581298e-30,2.82424470955305e-26
+# ZNF250,ENSG00000196150,ECD,ENSG00000122882,2.47274455661664,1.0652979856438198e-11,2.14332529077824e-10
+# SP2,ENSG00000167182,IL32,ENSG00000008517,2.46075901773533,4.1256177738298497e-22,3.83064494763081e-19
+# KLF5,ENSG00000102554,RGS1,ENSG00000090104,-2.45832343792633,9.80241382630337e-12,1.9972107166922901e-10
+# ZNF250,ENSG00000196150,SECISBP2L,ENSG00000138593,2.44734259208731,8.10284619583455e-10,8.828469446648141e-09
+
+
+class ScorpionAdapter(BaseAdapter):
+    SOURCE = 'IGVF'
+    LABEL = 'predicted gene regulatory networks'
+
+    def __init__(self, filepath=None, writer: Optional[Writer] = None, validate=False, **kwargs):
+        self.filepath = filepath
+        self.file_accession = self.filepath.split('/')[-1].split('.')[0]
+        self.source_url = 'https://data.igvf.org/tabular-files/' + self.file_accession
+        super().__init__(filepath, '', writer, validate)
+
+    def process_file(self) -> Optional[dict]:
+        self.writer.open()
+        file_fileset_obj = get_file_fileset_by_accession_in_arangodb(
+            self.file_accession)
+        self.method = file_fileset_obj['method']
+        self.collection_class = file_fileset_obj['class']
+
+        with gzip.open(self.filepath, 'rt') as data_file:
+            data_csv = csv.DictReader(data_file)
+
+            skip = False
+            for row in data_csv:
+                if row['tf_ensembl'] == '':
+                    if row['tf'] != '':
+                        tf_ensembl = gene_synonym_to_ensembl_id(row['tf'])
+                        if tf_ensembl is not None:
+                            row['tf_ensembl'] = tf_ensembl
+                        else:
+                            skip = True
+                if row['target_ensembl'] == '':
+                    if row['target'] != '':
+                        target_ensembl = gene_synonym_to_ensembl_id(
+                            row['target'])
+                        if target_ensembl is not None:
+                            row['target_ensembl'] = target_ensembl
+                        else:
+                            skip = True
+
+                if skip:
+                    print('Row is invalid or gene name resolution failed, skipping:')
+                    print(row)
+                    skip = False
+                    continue
+
+                props = {
+                    '_key': row['tf_ensembl'] + '_' + row['target_ensembl'] + '_' + self.LABEL.replace(' ', '_'),
+                    '_from': 'genes/' + row['tf_ensembl'],
+                    '_to': 'genes/' + row['target_ensembl'],
+                    'effect_size': float(row['beta']) if row['beta'] != '' else None,
+                    'p_value': float(row['P']) if row['P'] != '' else None,
+                    'p_value_adj': float(row['FDR']) if row['FDR'] != '' else None,
+                    'neg_log10_p_value': -math.log10(float(row['P'])) if row['P'] != '' else None,
+                    'name': 'regulates',
+                    'inverse_name': 'is regulated by',
+                    'files_filesets': 'files_filesets/' + self.file_accession,
+                    'class': self.collection_class,
+                    'method': self.method,
+                    'label': self.LABEL,
+                    'source': self.SOURCE,
+                    'source_url': self.source_url
+
+                }
+                if self.validate:
+                    self.validate_doc(props)
+
+                self.writer.write(json.dumps(props))
+                self.writer.write('\n')
+        self.writer.close()

data/data_sources/data_sources.yaml

@@ -1497,6 +1497,13 @@ mouse gene to gene interaction:
   source: BioGRID
   molecular_function: ontology_terms/GO_0005515
 
+# Example: pypy3 data_parser.py --adapter scorpion --output-bucket igvf-catalog-parsed-collections --filepath ~/dataset/IGVFFI3035IXUW.csv.gz --output-bucket-key genes_genes/genes_genes_scorpion.jsonl
+gene to gene scorpion:
+  collection: genes_genes
+  command: pypy3 data_parser.py --adapter scorpion --output-bucket igvf-catalog-parsed-collections --filepath {filepath} --output-bucket-key genes_genes/genes_genes_scorpion.jsonl
+  datafiles:
+    - https://api.data.igvf.org/tabular-files/IGVFFI3035IXUW/@@download/IGVFFI3035IXUW.csv.gz
+
 # Example: pypy3 data_parser.py --adapter coxpresdb --output-bucket igvf-catalog-parsed-collections --filepath ~/dataset/Hsa-r.v22-05.G16651-S235187.combat_pca.subagging.z.d/ --output-bucket-key genes_genes/genes_genes_coxpresdb.jsonl
 gene to gene coexpression association:
   collection: genes_genes

data/data_sources/data_sources_file_fileset.yaml

@@ -29,6 +29,8 @@ igvf donor:
     --replace
 
 igvf_file_accessions:
+  Scorpion:
+    - IGVFFI3035IXUW
   BlueSTARR:
     - IGVFFI1663LKVQ
     - IGVFFI5288RAAV