Pyoxigraph and test improvements (#590)

This PR upgrades [pyoxigraph v0.2.5 to v0.4.11](https://pyoxigraph.readthedocs.io/en/stable/migration.html#from-0-2-to-0-3), which requires the interface to be updated.

It also replaces the dodgy/outdated [python-Levenshtein-wheels](https://pypi.org/project/python-Levenshtein-wheels/) with the up-to-date [python-Levenshtein](https://pypi.org/project/python-Levenshtein/).

Adapted from PR #588.

Author: gaurav

requirements.txt

@@ -1,14 +1,15 @@
 biopython
 bmt
-datrie
 jsonlines
 pandas
 more-itertools
-pyoxigraph~=0.2.5
+#pyoxigraph~=0.2.5
+pyoxigraph~=0.4.11
 psycopg2-binary
 pytest
 pytest-cov
-python-Levenshtein-wheels
+#python-Levenshtein-wheels
+python-levenshtein
 pyyaml
 requests
 snakemake

src/datahandlers/chembl.py

@@ -1,3 +1,6 @@
+import os.path
+import pathlib
+
 from src.prefixes import CHEMBLCOMPOUND
 from src.babel_utils import pull_via_ftp, make_local_name
 import ftplib
@@ -48,11 +51,11 @@ def __init__(self,ifname,ccofile):
         from datetime import datetime as dt
         print('loading chembl')
         start = dt.now()
-        self.m= pyoxigraph.MemoryStore()
+        self.m= pyoxigraph.Store()
         with open(ccofile,'rb') as inf:
-            self.m.load(inf,'application/turtle')
+            self.m.bulk_load(input=inf, format=pyoxigraph.RdfFormat.TURTLE)
         with open(ifname,'rb') as inf:
-            self.m.load(inf,'application/turtle')
+            self.m.bulk_load(input=inf, format=pyoxigraph.RdfFormat.TURTLE)
         end = dt.now()
         print('loading complete')
         print(f'took {end-start}')

src/datahandlers/clo.py

@@ -24,9 +24,9 @@ def __init__(self,ifname):
         from datetime import datetime as dt
         print('loading CLO')
         start = dt.now()
-        self.m= pyoxigraph.MemoryStore()
+        self.m= pyoxigraph.Store()
         with open(ifname,'rb') as inf:
-            self.m.load(inf,'application/rdf+xml',base_iri='http://example.org/')
+            self.m.bulk_load(input=inf, format=pyoxigraph.RdfFormat.RDF_XML, base_iri='http://example.org/')
         end = dt.now()
         print('loading complete')
         print(f'took {end-start}')

src/datahandlers/datacollect.py

@@ -1,5 +1,5 @@
 from src.ubergraph import UberGraph
-from src.babel_utils import make_local_name, pull_via_ftp
+from src.babel_utils import make_local_name, pull_via_ftp, pull_via_urllib
 from collections import defaultdict
 import os, gzip
 from json import loads,dumps

src/datahandlers/ec.py

@@ -23,9 +23,9 @@ def __init__(self):
         from datetime import datetime as dt
         print('loading EC')
         start = dt.now()
-        self.m= pyoxigraph.MemoryStore()
+        self.m= pyoxigraph.Store()
         with open(ifname,'rb') as inf:
-            self.m.load(inf,'application/rdf+xml',base_iri='http://example.org/')
+            self.m.bulk_load(input=inf,format=pyoxigraph.RdfFormat.RDF_XML,base_iri='http://example.org/')
         end = dt.now()
         print('loading complete')
         print(f'took {end-start}')

src/datahandlers/efo.py

@@ -27,9 +27,9 @@ def __init__(self):
         from datetime import datetime as dt
         print('loading EFO')
         start = dt.now()
-        self.m= pyoxigraph.MemoryStore()
+        self.m= pyoxigraph.Store()
         with open(ifname,'rb') as inf:
-            self.m.load(inf,'application/rdf+xml',base_iri='http://example.org/')
+            self.m.bulk_load(input=inf,format=pyoxigraph.RdfFormat.RDF_XML,base_iri='http://example.org/')
         end = dt.now()
         print('loading complete')
         print(f'took {end-start}')
@@ -120,6 +120,7 @@ def get_exacts(self, iri, outfile):
             outfile.write(f"{iri}\tskos:exactMatch\t{otherid}\n")
             nwrite += 1
         return nwrite
+
     def get_xrefs(self, iri, outfile):
         query = f"""
          prefix rdfs: <http://www.w3.org/2000/01/rdf-schema#>

src/datahandlers/hgncfamily.py

@@ -1,3 +1,5 @@
+import csv
+
 from pronto.utils.io import decompress
 
 from src.babel_utils import make_local_name, pull_via_ftp, pull_via_urllib
@@ -11,19 +13,18 @@ def pull_hgncfamily():
                     decompress=False,
                     subpath=HGNCFAMILY)
 
-def pull_labels(infile,outfile, metadata_yaml):
-    with open(infile,'r') as inf:
-        data = inf.read().strip()
-    lines = data.split('\n')
-    with open(outfile,'w') as outf:
-        #skip header
-        for line in lines[1:]:
-            parts = line.split(',')
-            if len(parts) < 10:
-                continue
-            i = f"{HGNCFAMILY}:{parts[0][1:-1]}"
-            l = parts[2][1:-1]
-            outf.write(f'{i}\t{l}\n')
+def pull_labels(infile, labelsfile, descriptionsfile, metadata_yaml):
+    with open(infile, 'r') as inf, open(labelsfile, 'w') as labelsf, open(descriptionsfile, 'w') as descriptionsf:
+        reader = csv.DictReader(inf)
+        for row in reader:
+            curie = f"{HGNCFAMILY}:{row['id']}"
+            name = row['name']
+            description = row['desc_comment']
+            # There is also a 'desc_label' field, but this seems to be pretty similar to 'name'.
+            labelsf.write(f'{curie}\t{name}\n')
+
+            if description and description != "NULL":
+                descriptionsf.write(f'{curie}\t{description}\n')
 
     write_metadata(
         metadata_yaml,

src/datahandlers/mesh.py

@@ -13,9 +13,9 @@ def __init__(self):
         from datetime import datetime as dt
         print('loading mesh.nt')
         start = dt.now()
-        self.m= pyoxigraph.MemoryStore()
+        self.m= pyoxigraph.Store()
         with open(ifname,'rb') as inf:
-            self.m.load(inf,'application/n-triples')
+            self.m.bulk_load(input=inf,format=pyoxigraph.RdfFormat.N_TRIPLES)
         end = dt.now()
         print('loading complete')
         print(f'took {end-start}')

src/datahandlers/mods.py

@@ -10,7 +10,15 @@
 def pull_mods():
     for mod in mods:
         subp = modmap[mod]
+
+        # We get the downloads from https://www.alliancegenome.org/downloads#gene-descriptions.
+        # They are also available at:
+        # - https://download.alliancegenome.org/8.1.0/GENE-DESCRIPTION-JSON/SGD/GENE-DESCRIPTION-JSON_SGD_9.json.gz
+        # - origname = pull_via_urllib(f"https://download.alliancegenome.org/8.1.0/GENE-DESCRIPTION-JSON/{mod}/",f'GENE-DESCRIPTION-JSON_{mod}_9.json.gz', subpath=subp)
+        #
+        # However, the following URL returns the latest version of this file for each model organism.
         origname = pull_via_urllib('https://fms.alliancegenome.org/download/',f'GENE-DESCRIPTION-JSON_{mod}.json.gz',subpath=subp)
+
         #This should be fine.  But for the makefile it's nice if the directory in which this goes is the same as the {mod} in the filename.
         # And we'd like it to be the names of the prefixes
         if mod != modmap[mod]:

src/datahandlers/pantherfamily.py

@@ -9,24 +9,20 @@ def pull_pantherfamily():
     # - http://data.pantherdb.org/ftp/sequence_classifications/current_release/PANTHER_Sequence_Classification_files/
 
 def pull_labels(infile,outfile, metadata_yaml):
-    with open(infile,'r') as inf:
-        data = inf.read()
-    lines = data.strip().split('\n')
     SUBFAMILY_COLUMN = 3
     MAINFAMILY_NAME_COLUMN = 4
     SUBFAMILY_NAME_COLUMN = 5
-    panther_families=[]
-    labels = {}
     done = set()
-    with open(outfile,'w') as labelf:
-        for line in lines[1:]:
+    with open(infile,'r') as inf, open(outfile,'w') as labelf:
+        for raw_line in inf:
+            line = raw_line.strip()
             parts = line.split('\t')
             if len(parts) < 5:
                 continue
             sf = parts[SUBFAMILY_COLUMN]
-            mf = sf.split(':')[0]
-            mfname = parts[MAINFAMILY_NAME_COLUMN]
-            sfname = parts[SUBFAMILY_NAME_COLUMN]
+            mf = sf.split(':')[0] # PTHR10845:SF155 -> PTHR10845
+            mfname = parts[MAINFAMILY_NAME_COLUMN] # REGULATOR OF G PROTEIN SIGNALING
+            sfname = parts[SUBFAMILY_NAME_COLUMN] # REGULATOR OF G-PROTEIN SIGNALING 18
             if mf not in done:
                 main_family = f'{PANTHERFAMILY}:{mf}'
                 #panther_families.append(main_family)