Merge pull request #2329 from broadinstitute/jb-ols-taxon-fallback

Using OLS validation as fallback for extended taxon IDs (SCP-6070)

Author: bistline

app/javascript/lib/validation/ontology-validation.js

@@ -17,7 +17,7 @@ import { metadataSchema } from './shared-validation'
 // TODO: Replace "development" with "main" after next ingest release
 const ONTOLOGY_BASE_URL =
   'https://raw.githubusercontent.com/broadinstitute/scp-ingest-pipeline/' +
-  'development/ingest/validation/ontologies/'
+  'main/ingest/validation/ontologies/'
 
 /** Quickly retrieve current version cache key for ontologies */
 async function fetchOntologyCacheVersion() {
@@ -169,7 +169,7 @@ export function getOntologyBasedProps() {
  */
 export function getAcceptedOntologies(key, metadataSchema) {
   // E.g. "ontology_browser_url": "https://www.ebi.ac.uk/ols/ontologies/mondo,https://www.ebi.ac.uk/ols/ontologies/pato"
-  const olsUrls = metadataSchema.properties[key].ontology
+  const olsUrls = metadataSchema.properties[key]?.ontology
 
   const acceptedOntologies =
     olsUrls?.split(',').map(url => url.split('/').slice(-1)[0].toUpperCase())
@@ -180,3 +180,31 @@ export function getAcceptedOntologies(key, metadataSchema) {
 
   return acceptedOntologies
 }
+
+/**
+ * fetch a remote ontology term from OLS for NCBI taxon IDs
+ * @param termId {String} ontology term ID, e.g. "NCBITaxon_9606"
+ * @returns {Object, null} JSON of ontology term, if found
+ */
+export async function fetchOlsOntologyTerm(termId) {
+  const noMatch = {}
+  noMatch[termId] = 'Not found'
+  try {
+    const ontologyName = termId.split('_')[0].toLowerCase()
+    const purlIri = `http://purl.obolibrary.org/obo/${termId}`
+    // purl IRI values must be double-encoded, to match behavior in:
+    // https://github.com/broadinstitute/scp-ingest-pipeline/blob/development/ingest/validation/validate_metadata.py#L348
+    const termUrl = `https://www.ebi.ac.uk/ols4/api/ontologies/${ontologyName}/` +
+                            `terms/${encodeURIComponent(encodeURIComponent(purlIri))}?lang=en`
+    console.debug(`termUrl: ${termUrl}`)
+    const rawTerm = await fetch(termUrl)
+    console.debug(`rawTerm: ${JSON.stringify(rawTerm)}`)
+    if (rawTerm.ok) {
+      return rawTerm.json()
+    } else {
+      return noMatch
+    }
+  } catch (error) {
+    return noMatch
+  }
+}

app/javascript/lib/validation/shared-validation.js

@@ -4,6 +4,7 @@
 
 // Ultimately sourced from: scp-ingest-pipeline/schemas
 import * as _schema from 'lib/assets/metadata_schemas/alexandria_convention/alexandria_convention_schema.json';
+import { fetchOlsOntologyTerm } from '~/lib/validation/ontology-validation'
 
 export const metadataSchema = _schema
 export const REQUIRED_CONVENTION_COLUMNS = metadataSchema.required.filter(c => c !== 'CellID')
@@ -323,3 +324,43 @@ export function getLabelSuffixForOntology(indentifier) {
   const shortName = getOntologyShortNameLc(indentifier)
   return shortName === 'uo' ? '_label' : '__ontology_label'
 }
+
+/** attempt to rectify invalid taxon ID issues, if present */
+export async function fixTaxonIdIssues(issues) {
+  // fallback for any taxon IDs not found
+  const invalidIds = findInvalidIds(issues)
+  if (Object.keys(invalidIds).length > 0) {
+    let filteredIssues = issues
+    const promises = []
+    Object.entries(invalidIds).map(async ([termId, label]) => {
+      const promise = fetchOlsOntologyTerm(termId)
+      promises.push(promise)
+    })
+    const results= await Promise.all(promises)
+    results.map(olsTerm => {
+      const termId = olsTerm?.short_form || Object.keys(olsTerm)[0]
+      const matchingIssue = issues.find(issue => {return issue[3]?.id === termId})
+      const label = matchingIssue[3].label
+      if (olsTerm && olsTerm.label === label) {
+        filteredIssues = filteredIssues.filter(issue => {return issue[3]?.id !== termId})
+      }
+    })
+    return filteredIssues
+  } else {
+    return issues
+  }
+}
+
+
+/** find instances of 'ontology:invalid-id' errors */
+export function findInvalidIds(issues) {
+  const invalidIds = {}
+  for (let i = 0; i < issues.length; i++) {
+    const issue = issues[i]
+    if (issue[3]?.subtype === 'ontology:invalid-id' && issue[3].id.toLowerCase().startsWith('ncbi')) {
+      const invalidId = issue[3].id
+      invalidIds[invalidId] = issue[3].label
+    }
+  }
+  return invalidIds
+}

app/javascript/lib/validation/validate-anndata.js

@@ -4,7 +4,7 @@ import { getOAuthToken } from '~/lib/scp-api'
 import {
   validateUnique, validateRequiredMetadataColumns,
   validateAlphanumericAndUnderscores, getOntologyShortNameLc,
-  metadataSchema, REQUIRED_CONVENTION_COLUMNS
+  metadataSchema, REQUIRED_CONVENTION_COLUMNS, fixTaxonIdIssues
 } from './shared-validation'
 import { fetchOntologies, getOntologyBasedProps, getAcceptedOntologies } from './ontology-validation'
 
@@ -131,7 +131,6 @@ export function checkOntologyIdFormat(key, ontologyIds) {
 /** Validate author's annotation labels and IDs match those in ontologies */
 export async function checkOntologyLabelsAndIds(key, ontologies, groups) {
   const [ids, idIndexes, labels, labelIndexes] = groups
-
   const issues = []
 
   // Determine unique (ontology ID, ontology label) pairs
@@ -158,7 +157,7 @@ export async function checkOntologyLabelsAndIds(key, ontologies, groups) {
       const msg = `Invalid ontology ID: ${id}`
       issues.push([
         'error', 'ontology:label-lookup-error', msg,
-        { subtype: 'ontology:invalid-id' }
+        { subtype: 'ontology:invalid-id', id, label }
       ])
     } else {
       const validLabels = ontology[id]
@@ -252,7 +251,7 @@ async function validateOntologyLabelsAndIds(hdf5File) {
     }
   }
 
-  return issues
+  return await fixTaxonIdIssues(issues)
 }
 
 

app/javascript/lib/validation/validate-file-content.js

@@ -18,12 +18,12 @@ import {
   getParsedHeaderLines, parseLine, ParseException,
   validateUniqueCellNamesWithinFile, validateMetadataLabelMatches,
   validateGroupColumnCounts, timeOutCSFV, validateUnique,
-  validateRequiredMetadataColumns, validateAlphanumericAndUnderscores
+  validateRequiredMetadataColumns, validateAlphanumericAndUnderscores,
+  getOntologyShortNameLc, getLabelSuffixForOntology, fixTaxonIdIssues
 } from './shared-validation'
 import { parseDifferentialExpressionFile } from './validate-differential-expression'
 import { parseAnnDataFile } from './validate-anndata'
 import { fetchOntologies, getOntologyBasedProps } from '~/lib/validation/ontology-validation'
-import { getOntologyShortNameLc, getLabelSuffixForOntology } from './shared-validation'
 
 /**
  * Gzip decompression requires reading the whole file, given the current
@@ -243,7 +243,8 @@ export async function parseMetadataFile(chunker, mimeType, fileOptions) {
     // add other line-by-line validations here
     }
   })
-  return { issues, delimiter, numColumns: headers[0].length }
+  const filteredIssues = await fixTaxonIdIssues(issues)
+  return { issues: filteredIssues, delimiter, numColumns: headers[0].length }
 }
 
 /** validate all ontology-based convention terms in a given line */
@@ -293,7 +294,7 @@ export function validateOntologyTerm(prop, ontologyId, label, ontologies, knownE
     errorIdentifier = `${ontologyId}-invalid-id`
     issue = [
       'error', 'ontology:label-lookup-error', msg,
-      { subtype: 'ontology:invalid-id' }
+      { subtype: 'ontology:invalid-id', id: ontologyId, label }
     ]
   } else {
     const validLabels = ontology[ontologyId]

config/application.rb

@@ -29,7 +29,7 @@ class Application < Rails::Application
     config.middleware.use Rack::Brotli
 
     # Docker image for file parsing via scp-ingest-pipeline
-    config.ingest_docker_image = 'gcr.io/broad-singlecellportal-staging/scp-ingest-pipeline:1.43.0'
+    config.ingest_docker_image = 'gcr.io/broad-singlecellportal-staging/scp-ingest-pipeline:1.43.1'
 
     # Docker image for image pipeline jobs
     config.image_pipeline_docker_image = 'gcr.io/broad-singlecellportal-staging/image-pipeline:0.1.0_c2b090043'

config/initializers/content_security_policy.rb

@@ -57,7 +57,7 @@
                              'https://www.googleapis.com', 'https://storage.googleapis.com', 'https://s3.amazonaws.com', 'https://data.broadinstitute.org', 'https://portals.broadinstitute.org',
                              'https://accounts.google.com', 'https://terra-bard-dev.appspot.com', 'https://terra-bard-alpha.appspot.com', 'https://terra-bard-prod.appspot.com',
                              'https://rest.ensembl.org', 'https://eutils.ncbi.nlm.nih.gov', 'https://mygene.info', 'https://webservice.wikipathways.org', 'https://o54426.ingest.sentry.io',
-                             'https://igv-genepattern-org.s3.amazonaws.com', 'https://*.soe.ucsc.edu', 'https://raw.githubusercontent.com'
+                             'https://igv-genepattern-org.s3.amazonaws.com', 'https://*.soe.ucsc.edu', 'https://raw.githubusercontent.com', 'https://www.ebi.ac.uk'
                             ]
   if ENV['NOT_DOCKERIZED']
     # enable connections to live reload server

test/js/lib/validate-file-content.test.js

@@ -4,10 +4,10 @@ import { render, screen } from '@testing-library/react'
 import '@testing-library/jest-dom/extend-expect'
 
 import ValidateFile from 'lib/validation/validate-file'
-import { fetchOntologies } from 'lib/validation/ontology-validation'
+import { fetchOntologies, fetchOlsOntologyTerm } from 'lib/validation/ontology-validation'
 import { validateConventionTerms, validateOntologyTerm } from 'lib/validation/validate-file-content'
 import {
-  REQUIRED_CONVENTION_COLUMNS, getOntologyShortNameLc, getLabelSuffixForOntology
+  REQUIRED_CONVENTION_COLUMNS, getOntologyShortNameLc, getLabelSuffixForOntology, fixTaxonIdIssues
 } from 'lib/validation/shared-validation'
 import { getLogProps } from 'lib/validation/log-validation'
 import ValidationMessage from 'components/validation/ValidationMessage'
@@ -642,4 +642,25 @@ describe('validates file contents against minified ontologies', () => {
     const uoId = "UO_0000036"
     expect("_label").toEqual(getLabelSuffixForOntology(uoId))
   })
+
+  it('gets remote term from OLS', async () => {
+    const termId = "NCBITaxon_197152"
+    const olsTerm = await fetchOlsOntologyTerm(termId)
+    expect(olsTerm.label).toBe('Cloeon dipterum')
+    const missingTerm = await fetchOlsOntologyTerm('NCBITaxon_foo')
+    expect(missingTerm).toMatchObject({ 'NCBITaxon_foo': 'Not found' })
+    const missingOntology = await fetchOlsOntologyTerm('foo_197152')
+    expect(missingOntology).toMatchObject({ 'foo_197152': 'Not found' })
+  })
+
+  it('filters out extended ontolgy issues', async () => {
+    const issues = [
+      [
+        'error', 'ontology:label-lookup-error', 'Invalid ontology ID: NCBITaxon_197152',
+        { id: 'NCBITaxon_197152', label: 'Cloeon dipterum', subtype: 'ontology:invalid-id' }
+      ]
+    ]
+    const filtered = await fixTaxonIdIssues(issues)
+    expect(filtered).toHaveLength(0)
+  })
 })