Final 7.0.3.2. Fixed lis-phylotree for new files.

Author: Sam Hokin

lis-annotation/src/main/java/org/intermine/bio/dataconversion/AnnotationFileConverter.java

@@ -58,7 +58,8 @@
 public class AnnotationFileConverter extends DatastoreFileConverter {
 
     private static final Logger LOG = Logger.getLogger(AnnotationFileConverter.class);
-    private static final String TEMPFILE = "/tmp/annotation.gff3";
+    private static final String TEMPGENEFILE = "/tmp/gene_models_main.gff3";
+    private static final String TEMPIPRSCANFILE = "/tmp/iprscan.gff3";
 
     // GFF sourced
     Map<String,Item> chromosomes = new HashMap<>();
@@ -142,9 +143,6 @@ public void process(Reader reader) throws IOException {
             System.out.println("## Processing "+getCurrentFile().getName());
             processGFAFile();
             gfaFileExists = true;            
-        } else if (getCurrentFile().getName().endsWith(".pathway.tsv.gz")) {
-            System.out.println("## Processing "+getCurrentFile().getName());
-            processPathwayFile();
 	} else if (getCurrentFile().getName().endsWith(".protein.faa.gz") || getCurrentFile().getName().endsWith(".protein_primary.faa.gz")) {
             System.out.println("## Processing "+getCurrentFile().getName());
             processProteinFasta();
@@ -157,8 +155,14 @@ public void process(Reader reader) throws IOException {
             System.out.println("## Processing "+getCurrentFile().getName());
             processMRNAFasta();
             mrnaFileExists = true;
+        } else if (getCurrentFile().getName().endsWith(".pathway.tsv.gz")) {
+            System.out.println("## Processing "+getCurrentFile().getName());
+            processPathwayFile();
+        } else if (getCurrentFile().getName().endsWith(".iprscan.gff3.gz")) {
+            System.out.println("## Processing "+getCurrentFile().getName());
+            processIPRScanGFF3();
         } else {
-            System.out.println("## Skipping file "+getCurrentFile().getName());
+            System.out.println("## - Skipping "+getCurrentFile().getName());
         }
     }
 
@@ -170,12 +174,11 @@ public void close() throws ObjectStoreException, RuntimeException {
         if (readme==null) {
             throw new RuntimeException("README file not found. Aborting.");
         }
-        if (!cdsFileExists) System.err.println("ERROR: cds FASTA file is missing.");
-        if (!mrnaFileExists) System.err.println("ERROR: mrna FASTA file is missing.");
+        if (!cdsFileExists && !mrnaFileExists) System.err.println("ERROR: neither mrna nor cds FASTA file is present. One must be.");
         if (!proteinFileExists) System.err.println("ERROR: protein FASTA file is missing.");
         if (!gfaFileExists) System.err.println("ERROR: gfa file is missing.");
         if (!gff3FileExists) System.err.println("ERROR: GFF3 file is missing.");
-        if (!cdsFileExists || !mrnaFileExists || !proteinFileExists || !gfaFileExists || !gff3FileExists) {
+        if ((!cdsFileExists && !mrnaFileExists) || !proteinFileExists || !gfaFileExists || !gff3FileExists) {
             throw new RuntimeException("Missing required annotation file(s). Aborting.");
         }
         // set references and collections for objects loaded from FASTAs based on matching identifiers
@@ -376,7 +379,6 @@ void processGFAFile() throws IOException, RuntimeException {
             geneFamily.addToCollection("genes", gene);
             geneFamily.addToCollection("proteins", protein);
         }
-        br.close();
     }
 
     /**
@@ -411,7 +413,7 @@ void processGFF3File() throws IOException, RuntimeException {
             throw new RuntimeException("README not read before "+getCurrentFile().getName()+". Aborting.");
         }
         // uncompress the gff3.gz file to a temp file
-        File tempfile = new File(TEMPFILE);
+        File tempfile = new File(TEMPGENEFILE);
         tempfile.delete();
         BufferedWriter writer = new BufferedWriter(new FileWriter(tempfile));
         BufferedReader reader = GZIPBufferedReader.getReader(getCurrentFile());
@@ -422,7 +424,7 @@ void processGFF3File() throws IOException, RuntimeException {
         }
         writer.close();
         // now load the uncompressed GFF
-        FeatureList featureList = GFF3Reader.read(TEMPFILE);
+        FeatureList featureList = GFF3Reader.read(TEMPGENEFILE);
         for (FeatureI featureI : featureList) {
             String seqname = featureI.seqname();
             Location location = featureI.location();
@@ -535,6 +537,61 @@ void processGFF3File() throws IOException, RuntimeException {
         }
     }
 
+    /**
+     * Process an IPRScan GFF file which has Proteins in the sequence column.
+     * 0     1            2    3    4    5       6    7
+     * medsa.XinJiangDaYe.gnm1.ann1.RKB9.iprscan.gff3.gz
+     *
+     * medsa.XinJiangDaYe.gnm1.ann1.MS_gene000000.t1 PANTHER protein_match 1 463 . + . Name=PTHR10178:SF14;status=T;ID=match$1047424_1_463;date=04-02-2021
+     * medsa.XinJiangDaYe.gnm1.ann1.MS_gene000008.t1 ProSiteProfiles protein_match  10 264   . + . Name=PS50294;status=T;ID=match$461640_10_264;date=03-02-2021;
+     *    signature_desc=Trp-Asp (WD) repeats circular profile.
+     * medsa.XinJiangDaYe.gnm1.ann1.MS_gene000000.t1 Pfam    protein_hmm_match 3 88 4.8E-11 + . Name=PF03732;status=T;ID=match$1047423_3_88;date=04-02-2021;
+     *    signature_desc=Retrotransposon gag protein;Target=PF03732 6 96;
+     */
+    void processIPRScanGFF3() throws IOException {
+        // uncompress the gff3.gz file to a temp file
+        File tempfile = new File(TEMPIPRSCANFILE);
+        tempfile.delete();
+        BufferedWriter writer = new BufferedWriter(new FileWriter(tempfile));
+        BufferedReader reader = GZIPBufferedReader.getReader(getCurrentFile());
+        String line = null;
+        while ( (line=reader.readLine())!=null ) {
+            writer.write(line);
+            writer.newLine();
+        }
+        writer.close();
+        // now load the uncompressed GFF
+        FeatureList featureList = GFF3Reader.read(TEMPIPRSCANFILE);
+        for (FeatureI featureI : featureList) {
+            String seqname = featureI.seqname();
+            Location location = featureI.location();
+            String type = featureI.type();
+            // feature
+            Item feature = getFeatureOnProtein(type, seqname, location);
+            String id = getAttribute(featureI, "ID");
+            feature.setAttribute("primaryIdentifier", id);
+            features.put(id, feature);
+            // source isn't supplied by FeatureI
+            // feature.setAttribute("source", rec.getSource());
+            // attributes
+            String name = getAttribute(featureI, "Name");
+            String status = getAttribute(featureI, "status");
+            String date = getAttribute(featureI, "date");
+            String target = getAttribute(featureI, "Target");
+            String signatureDesc = getAttribute(featureI, "signature_desc");
+            // accession=Name
+            feature.setAttribute("accession", name);
+            // status
+            if (status!=null) feature.setAttribute("status", status);
+            // date
+            if (date!=null) feature.setAttribute("date", date);
+            // target
+            if (target!=null) feature.setAttribute("target", target);
+            // signatureDesc
+            if (signatureDesc!=null) feature.setAttribute("signatureDesc", signatureDesc);
+        }
+    }
+
     /**
      * Add an OntologyAnnotation with the given identifier to the given feature's collection
      * NOTE: GO terms are GOTerm objects.
@@ -625,6 +682,31 @@ void placeFeatureOnSequence(Item feature, String seqname, Location location) thr
         }
     }
 
+    /**
+     * Place a feature on a protein.
+     */
+    Item getFeatureOnProtein(String type, String seqname, Location location) throws RuntimeException {
+        Item protein = getProtein(seqname);
+        Item feature = null;
+        if (type.equals("protein_match")) {
+            feature = createItem("ProteinMatch");
+        } else if (type.equals("protein_hmm_match")) {
+            feature = createItem("ProteinHmmMatch");
+        } else {
+            throw new RuntimeException("IPRSCAN GFF record type "+type+" is not supported by this loader.");
+        }
+        feature.setReference("protein", protein);
+        // reference feature on new IM Location
+        Item proteinLocation = createItem("Location");
+        proteinLocation.setReference("feature", feature);
+        proteinLocation.setAttribute("start", String.valueOf(location.bioStart()));
+        proteinLocation.setAttribute("end", String.valueOf(location.bioEnd()));
+        proteinLocation.setReference("locatedOn", protein);
+        locations.add(proteinLocation);
+        feature.setReference("location", proteinLocation);
+        return feature;
+    }
+    
     /**
      * Get/add a Gene Item, keyed by primaryIdentifier
      */

lis-annotation/src/main/resources/lis-annotation_keys.properties

@@ -1,5 +1,5 @@
 ##
-## lis-gfa keys
+## lis-annotation keys
 ##
 
 Ontology.key_name=name
@@ -8,26 +8,25 @@ SOTerm.key_name=name
 OntologyAnnotation.key_subject_term=subject,ontologyTerm
 DataSource.key_name=name
 DataSet.key_name=name
-
 Organism.key_taxonid=taxonId
 Strain.key_identifier=identifier
-
 Publication.key_doi=doi
 
-## BioEntity
-Protein.key_primaryidentifier=primaryIdentifier
-ProteinDomain.key_primaryidentifier=primaryIdentifier
-
-## SequenceFeature
+## BioEntity (and SequenceFeature)
 Chromosome.key_primaryidentifier=primaryIdentifier
 Supercontig.key_primaryidentifier=primaryIdentifier
 Gene.key_primaryidentifier=primaryIdentifier
 CDS.key_primaryidentifier=primaryIdentifier
 CDSRegion.key_primaryidentifier=primaryIdentifier
 Exon.key_primaryidentifier=primaryIdentifier
 MRNA.key_primaryidentifier=primaryIdentifier
+Protein.key_primaryidentifier=primaryIdentifier
+ProteinDomain.key_primaryidentifier=primaryIdentifier
+ProteinMatch.key_primaryidentifier=primaryIdentifier
+ProteinHmmMatch.key_primaryidentifier=primaryIdentifier
 Transcript.key_primaryidentifier=primaryIdentifier
 NCRNA.key_primaryidentifier=primaryIdentifier
+ThreePrimeUTR.key_primaryidentifier=primaryIdentifier
 FivePrimeUTR.key_primaryidentifier=primaryIdentifier
 LncRNA.key_primaryidentifier=primaryIdentifier
 Pseudogene.key_primaryidentifier=primaryIdentifier

lis-genefamily/src/main/java/org/intermine/bio/dataconversion/GeneFamilyFileConverter.java

@@ -20,6 +20,8 @@
 import org.intermine.objectstore.ObjectStoreException;
 import org.intermine.xml.full.Item;
 
+import org.ncgr.zip.GZIPBufferedReader;
+
 /**
  * Load gene family data from LIS datastore files.
  *
@@ -39,6 +41,8 @@ public class GeneFamilyFileConverter extends DatastoreFileConverter {
     Map<String,Item> proteinDomains = new HashMap<>();
     Map<String,Item> geneFamilies = new HashMap<>();
 
+    boolean ahrdFileFound = false;
+
     /**
      * Create a new GeneFamilyFileConverter
      * @param writer the ItemWriter to write out new items
@@ -69,8 +73,9 @@ public void process(Reader reader) throws IOException {
         }
         dataSet.setReference("dataSource", dataSource);
         // process files
-        if (getCurrentFile().getName().endsWith(".info_annot_ahrd.tsv")) {
-            processInfoAnnotAhrdFile(reader);
+        if (getCurrentFile().getName().endsWith(".info_annot_ahrd.tsv.gz")) {
+            processInfoAnnotAhrdFile();
+            ahrdFileFound = true;
 	}
     }
 
@@ -79,6 +84,9 @@ public void process(Reader reader) throws IOException {
      */
     @Override
     public void close() throws Exception {
+        if (!ahrdFileFound) {
+            throw new RuntimeException("File ending in .info_annot_ahrd.tsv.gz not found. Aborting");
+        }
         store(dataSource);
         store(dataSet);
 	store(geneFamilies.values());
@@ -164,9 +172,9 @@ public Item getProteinDomain(String identifier) {
     }
 
     /**
-     * Process an info_annot_ahrd.tsv file which contains gene families and semi-colon separated groups of ontology terms.
-     * 0      1       2    3    4               5
-     * lis.genefam.fam1.M65K.info_annot_ahrd.tsv
+     * Process an info_annot_ahrd.tsv.gz file which contains gene families and semi-colon separated groups of ontology terms.
+     * 0   1       2    3    4               5   6
+     * lis.genefam.fam1.M65K.info_annot_ahrd.tsv.gz
      * legfed_v1_0.L_LFXSXJ splicing factor 3B subunit 3-like isoform X2 [Glycine max];
      *                      IPR004871 (Cleavage/polyadenylation specificity factor, A subunit, C-terminal); 
      *                      GO:0003676 (nucleic acid binding), GO:0005634 (nucleus)
@@ -182,9 +190,9 @@ public Item getProteinDomain(String identifier) {
      * SIASPGRGILAIDESNATCGKRLASIGLDNTEVNRQAYRQLLLTTPGLGEYISGAILFEE
      * ...
      */
-    void processInfoAnnotAhrdFile(Reader reader) throws IOException {
+    void processInfoAnnotAhrdFile() throws IOException {
         // spin through the AHRD file lines
-        BufferedReader br = new BufferedReader(reader);
+        BufferedReader br = GZIPBufferedReader.getReader(getCurrentFile());
         String line = null;
         while ((line=br.readLine())!=null) {
             if (line.startsWith("#") || line.trim().length()==0) continue; // comment line

lis-phylotree/libs/bioinfweb-commons-java-bio-3.0.0.jar

@@ -64,24 +64,25 @@ public PhylotreeFileConverter(ItemWriter writer, Model model) throws ObjectStore
      */
     @Override
     public void process(Reader reader) {
-        // there is no README (yet)
-        // DataSet
-        if (dataSetName==null || dataSetUrl==null || dataSetDescription==null) {
-            throw new RuntimeException("ERROR: dataSetName, dataSetUrl, and dataSetDescription must be set in project.xml.");
-        }
-        dataSet = createItem("DataSet");
-        dataSet.setAttribute("name", dataSetName);
-        dataSet.setAttribute("url", dataSetUrl);
-        dataSet.setAttribute("description", dataSetDescription);
-        if (dataSetLicence!=null) {
-            dataSet.setAttribute("licence", dataSetLicence);
-        } else {
-            dataSet.setAttribute("licence", DatastoreFileConverter.DEFAULT_DATASET_LICENCE);
+        // there is no README (yet) so hand-roll DataSet
+        if (dataSet==null) {
+            if (dataSetName==null || dataSetUrl==null || dataSetDescription==null) {
+                throw new RuntimeException("ERROR: dataSetName, dataSetUrl, and dataSetDescription must be set in project.xml.");
+            }
+            dataSet = createItem("DataSet");
+            dataSet.setAttribute("name", dataSetName);
+            dataSet.setAttribute("url", dataSetUrl);
+            dataSet.setAttribute("description", dataSetDescription);
+            if (dataSetLicence!=null) {
+                dataSet.setAttribute("licence", dataSetLicence);
+            } else {
+                dataSet.setAttribute("licence", DatastoreFileConverter.DEFAULT_DATASET_LICENCE);
+            }
+            dataSet.setReference("dataSource", dataSource);
         }
-        dataSet.setReference("dataSource", dataSource);
         try {
             processTreeFile();
-        } catch (IOException ex) {
+        } catch (Exception ex) {
             throw new RuntimeException(ex);
         }
     }
@@ -125,9 +126,11 @@ Item getPhylonode(Node node, String name) {
         if (node.isFeature()) {
             phylonode.setAttribute("isLeaf", "true");
             phylonode.setAttribute("name", node.label);
-            // assume Proteins
-            Item protein = getProtein(node.label);
-            phylonode.setReference("protein", protein);
+            // some nodes hold proteins
+            if (isFullYuck(node.label)) {
+                Item protein = getProtein(node.label);
+                phylonode.setReference("protein", protein);
+            }
         }
         return phylonode;
     }
@@ -170,6 +173,7 @@ void processEdge(Edge e, String identifier) {
 
     /**
      * Process a file in the phylotree subdirectory, in Newick format.
+     * legume.genefam.fam1.M65K.trees_ML_rooted/legfed_v1_0.L_6W30ML
      */
     void processTreeFile() throws IOException {
         JPhyloIOReaderWriterFactory factory = new JPhyloIOReaderWriterFactory();
@@ -184,13 +188,20 @@ void processTreeFile() throws IOException {
             throw new RuntimeException(ex);
         }
         // create this Phylotree and GeneFamily
-        String name = getCurrentFile().getName();
+        // 0           1
+        // legfed_v1_0.L_6W30ML
+        String identifier = getCurrentFile().getName();
+        String[] parts = identifier.split("\\.");
+        String name = parts[1];
         Item phylotree = createItem("Phylotree");
         phylotrees.add(phylotree);
+        // TODO: 5.0.7.3 switch to identifier
+        // phylotree.setAttribute("identifier", identifier);
         phylotree.setAttribute("name", name);
         phylotree.setReference("dataSet", dataSet);
         Item geneFamily = createItem("GeneFamily");
         geneFamilies.add(geneFamily);
+        // TODO: 5.0.7.3 switch to identifier
         geneFamily.setAttribute("name", name);
         geneFamily.setReference("phylotree", phylotree);
         phylotree.setReference("geneFamily", geneFamily);
@@ -225,6 +236,7 @@ void processTreeFile() throws IOException {
                     // Indicates a node in a phylogenetic tree or network.
                     Node n = new Node(event.asNodeEvent());
                     Item phylonode = getPhylonode(n, name);
+                    if (phylonode==null) return;
                     phylonode.setReference("tree", phylotree);
                     phylotree.addToCollection("nodes", phylonode);
                     if (n.isFeature()) numLeaves++;
@@ -263,15 +275,17 @@ void processTreeFile() throws IOException {
         newick.setAttribute("contents", contents);
     }
 
+    /**
+     * Return true if the given identifier is in LIS full yuck format.
+     */
+    boolean isFullYuck(String primaryIdentifier) {
+        return primaryIdentifier.split("\\.").length>=5;
+    }
+        
     /**
      * Get/add a Protein Item, keyed by primaryIdentifier
      */
     Item getProtein(String primaryIdentifier) {
-        // check that we've got a full-yuck identifier
-        String[] parts = primaryIdentifier.split("\\.");
-        if (parts.length<5) {
-            throw new RuntimeException("Protein primary identifier in "+getCurrentFile().getName()+" is not LIS format:"+primaryIdentifier);
-        }
         if (proteins.containsKey(primaryIdentifier)) {
             return proteins.get(primaryIdentifier);
         } else {