feature/KBDEV-1502 cosmic loader update

src/cosmic/README.md

@@ -10,34 +10,38 @@ First the data must be downloaded. This requires an account
 AUTH=$( echo "$COSMIC_EMAIL:$COSMIC_PASSWORD" | base64 )
 
 # Download resistance mutations
-resp=$( curl -H "Authorization: Basic $AUTH" https://cancer.sanger.ac.uk/cosmic/file_download/GRCh38/cosmic/v92/CosmicResistanceMutations.tsv.gz );
+resp=$( curl -H "Authorization: Basic $AUTH" "https://cancer.sanger.ac.uk/api/mono/products/v1/downloads/scripted?path=grch38/cosmic/v103/Cosmic_ResistanceMutations_Tsv_v103_GRCh38.tar&bucket=downloads" );
 echo $resp
 url=$( node  -e "var resp = $resp; console.log(resp.url);" );
-curl "$url" -o CosmicResistanceMutations.tsv.gz
-gunzip CosmicResistanceMutations.tsv.gz
+curl "$url" -o Cosmic_ResistanceMutations_Tsv_v103_GRCh38.tar
+tar -xf Cosmic_ResistanceMutations_Tsv_v103_GRCh38.tar
+gunzip Cosmic_ResistanceMutations_v103_GRCh38.tsv.gz
 
 # Download disease mappings
-resp=$( curl -H "Authorization: Basic $AUTH" https://cancer.sanger.ac.uk/cosmic/file_download/GRCh38/cosmic/v92/classification.csv );
+resp=$( curl -H "Authorization: Basic $AUTH" "https://cancer.sanger.ac.uk/api/mono/products/v1/downloads/scripted?path=grch38/cosmic/v103/Cosmic_Classification_Tsv_v103_GRCh38.tar&bucket=downloads" );
 echo $resp
 url=$( node  -e "var resp = $resp; console.log(resp.url);" );
-curl "$url" -o classification.csv
+curl "$url" -o Cosmic_Classification_Tsv_v103_GRCh38.tar
+tar -xf Cosmic_Classification_Tsv_v103_GRCh38.tar
+gunzip Cosmic_Classification_v103_GRCh38.tsv.gz
 
 # Download fusion files
-resp=$( curl -H "Authorization: Basic $AUTH" https://cancer.sanger.ac.uk/cosmic/file_download/GRCh38/cosmic/v92/CosmicFusionExport.tsv.gz );
+resp=$( curl -H "Authorization: Basic $AUTH" "https://cancer.sanger.ac.uk/api/mono/products/v1/downloads/scripted?path=grch38/cosmic/v103/Cosmic_Fusion_Tsv_v103_GRCh38.tar&bucket=downloads" );
 echo $resp
 url=$( node  -e "var resp = $resp; console.log(resp.url);" );
-curl "$url" -o CosmicFusionExport.tsv.gz
-gunzip CosmicFusionExport.tsv.gz
+curl "$url" -o Cosmic_Fusion_Tsv_v103_GRCh38.tar
+tar -xf Cosmic_Fusion_Tsv_v103_GRCh38.tar
+gunzip Cosmic_Fusion_v103_GRCh38.tsv.gz
 ```
 
 Since this loader requires 2 files, it is separate from the other more general loaders
 
 ```bash
-node bin/load.js cosmic resistance CosmicResistanceMutations.tsv classification.csv
+node bin/load.js cosmic resistance Cosmic_ResistanceMutations_v103_GRCh38.tsv Cosmic_Classification_v103_GRCh38.tsv
 ```
 
 And then to load the fusions (Will create recurrency statements)
 
 ```bash
-node bin/load.js cosmic fusions CosmicFusionExport.tsv classification.csv
+node bin/load.js cosmic fusions Cosmic_Fusion_v103_GRCh38.tsv Cosmic_Classification_v103_GRCh38.tsv
 ```

src/cosmic/fusions.js

@@ -19,29 +19,56 @@ const { logger } = require('../logging');
 const { cosmic: SOURCE_DEFN } = require('../sources');
 
 const RECURRENCE_THRESHOLD = 3;
+const cache = {};
 
 const HEADER = {
-    disease: 'HISTOLOGY_SUBTYPE_1',
-    diseaseFamily: 'PRIMARY_HISTOLOGY',
-    exon1: '5\'_LAST_OBSERVED_EXON',
-    exon2: '3\'_FIRST_OBSERVED_EXON',
-    fusionId: 'FUSION_ID',
-    fusionName: 'TRANSLOCATION_NAME',
-    gene1: '5\'_GENE_NAME',
-    gene2: '3\'_GENE_NAME',
+    diseaseId: 'COSMIC_PHENOTYPE_ID',
+    exon1: 'FIVE_PRIME_LAST_OBSERVE_EXON',
+    exon2: 'THREE_PRIME_FIRST_OBSERVE_EXON',
+    fusionId: 'COSMIC_FUSION_ID',
+    fusionName: 'FUSION_SYNTAX',
+    gene1: 'FIVE_PRIME_CHROMOSOME',
+    gene2: 'THREE_PRIME_CHROMOSOME',
     pubmed: 'PUBMED_PMID',
-    sampleId: 'SAMPLE_ID',
+    sampleId: 'COSMIC_SAMPLE_ID',
+};
+
+const fetchChromosome = async (conn, sourceId) => {
+    if (cache[sourceId]) {
+        return cache[sourceId];
+    }
+    const record = await conn.getUniqueRecordBy({
+        filters: {
+            AND: [
+                { sourceId },
+                { biotype: 'chromosome' },
+            ],
+        },
+        sort: orderPreferredOntologyTerms,
+        target: 'Feature',
+    });
+    cache[sourceId] = record;
+    return record;
 };
 
 
 const processVariants = async ({
     conn, record, variantType, exonSpecific,
 }) => {
     // fetch the features
-    const [gene1] = await _gene.fetchAndLoadBySymbol(conn, record.gene1);
-    const [gene2] = await _gene.fetchAndLoadBySymbol(conn, record.gene2);
+    let [gene1] = await _gene.fetchAndLoadBySymbol(conn, record.gene1),
+        [gene2] = await _gene.fetchAndLoadBySymbol(conn, record.gene2);
 
     // create the variants
+    if (!gene1) {
+        gene1 = await fetchChromosome(conn, record.gene1);
+    }
+    if (!gene2) {
+        gene2 = await fetchChromosome(conn, record.gene2);
+    }
+    if (!gene1 || !gene2) {
+        throw new Error(`unable to find genes for record ${record.id}: ${record.gene1}, ${record.gene2}`);
+    }
     const general = await conn.addVariant({
         content: {
             reference1: rid(gene1),
@@ -93,7 +120,6 @@ const processRecordGroup = async ({
         conn, exonSpecific, record: records[0], variantType,
     });
 
-
     // create the recurrence statement
     await conn.addRecord({
         content: {
@@ -138,7 +164,7 @@ const uploadFile = async ({
     const variantType = rid(await conn.getVocabularyTerm('fusion'));
     const cancer = rid(await conn.getUniqueRecordBy({ filters: { name: 'cancer' }, sort: orderPreferredOntologyTerms, target: 'Disease' }));
 
-    await _pubmed.fetchAndLoadByIds(conn, records.map(rec => rec.pumbed));
+    await _pubmed.fetchAndLoadByIds(conn, records.map(rec => rec.pubmed));
 
     const addPropertyMatch = (histogram, object, properties) => {
         const hashId = hashRecordToId(object, properties);
@@ -163,7 +189,9 @@ const uploadFile = async ({
     // pre-process/clean records
     for (const record of records) {
         record.id = hashRecordToId(record, ['fusionId', 'sampleId']);
-        record.ncit = (mapping[record.diseaseFamily] || {})[record.disease];
+        record.ncit = mapping[record.diseaseId].ncit || '';
+        record.disease = mapping[record.diseaseId].disease || '';
+        record.diseaseFamily = mapping[record.diseaseId].diseaseFamily || '';
 
         record.disease = record.disease.toUpperCase() === 'NS'
             ? ''
@@ -183,7 +211,9 @@ const uploadFile = async ({
         record.variant = `(${record.gene1},${record.gene2}).fus(e.${record.exon1},e.${record.exon2})`;
         record.nonSpecificVariant = `(${record.gene1},${record.gene2}).fus(e.?,e.?)`;
 
-        recurrentProperties.forEach((plist, index) => addPropertyMatch(recurrenceCounts[index], record, plist));
+        recurrentProperties.forEach(
+            (plist, index) => addPropertyMatch(recurrenceCounts[index], record, plist),
+        );
     }
 
     const getSampleCount = group => (new Set(group.map(row => row.sampleId))).size;
@@ -192,7 +222,7 @@ const uploadFile = async ({
     const processed = new Set();
 
     // disease-specific, exon-specific, fusions
-    for (let index = 0; index <= recurrenceCounts.length; index++) {
+    for (let index = 0; index < recurrenceCounts.length; index++) {
         const recurrencyLevel = recurrenceCounts[index];
 
         for (const [groupId, group] of Object.entries(recurrencyLevel)) {

src/cosmic/resistance.js

@@ -24,22 +24,20 @@ const { cosmic: SOURCE_DEFN } = require('../sources');
 
 const HEADER = {
     cds: 'HGVSC',
-    disease: 'Histology Subtype 1',
-    diseaseFamily: 'Histology',
-    gene: 'Gene Name',
+    diseaseId: 'COSMIC_PHENOTYPE_ID',
+    gene: 'GENE_SYMBOL',
     genomic: 'HGVSG',
     mutationId: 'LEGACY_MUTATION_ID',
     protein: 'HGVSP',
-    pubmed: 'Pubmed Id',
-    sampleId: 'Sample ID',
-    sampleName: 'Sample Name',
-    therapy: 'Drug Name',
-    transcript: 'Transcript',
+    pubmed: 'PUBMED_PMID',
+    sampleName: 'SAMPLE_NAME',
+    therapy: 'DRUG_NAME',
+    transcript: 'TRANSCRIPT_ACCESSION',
 };
 
 
 /**
- * Create and link the variant defuinitions for a single row/record
+ * Create and link the variant definitions for a single row/record
  */
 const processVariants = async ({ conn, record, source }) => {
     let protein,
@@ -71,43 +69,45 @@ const processVariants = async ({ conn, record, source }) => {
                 throw Error(`failed to find the HGNC gene for ${record.gene}`);
             }
         } catch (err) {
-            logger.error(err);
+            logger.warn(err);
         }
     }
 
-    try {
-        // add the protein variant with its protein translation
-        const variant = jsonifyVariant(parseVariant(record.protein, false));
-        variant.type = rid(await conn.getVocabularyTerm(variant.type));
-
-        const reference1 = rid(await _ensembl.fetchAndLoadById(
-            conn,
-            { biotype: 'protein', sourceId: variant.reference1 },
-        ));
-        protein = rid(await conn.addVariant({
-            content: { ...variant, reference1 },
-            existsOk: true,
-            target: 'PositionalVariant',
-        }));
-
-        if (gene) {
-            // add the same protein varaint with the gene notation
-            generalProtein = rid(await conn.addVariant({
-                content: { ...variant, reference1: gene },
+    if (record.protein && record.protein.trim()) {
+        try {
+            // add the protein variant with its protein translation
+            const variant = jsonifyVariant(parseVariant(record.protein, false));
+            variant.type = rid(await conn.getVocabularyTerm(variant.type));
+
+            const reference1 = rid(await _ensembl.fetchAndLoadById(
+                conn,
+                { biotype: 'protein', sourceId: variant.reference1 },
+            ));
+            protein = rid(await conn.addVariant({
+                content: { ...variant, reference1 },
                 existsOk: true,
                 target: 'PositionalVariant',
             }));
 
-            // link the translation version to the gene version
-            await conn.addRecord({
-                content: { in: generalProtein, out: protein },
-                existsOk: true,
-                fetchExisting: false,
-                target: 'Infers',
-            });
+            if (gene) {
+                // add the same protein variant with the gene notation
+                generalProtein = rid(await conn.addVariant({
+                    content: { ...variant, reference1: gene },
+                    existsOk: true,
+                    target: 'PositionalVariant',
+                }));
+
+                // link the translation version to the gene version
+                await conn.addRecord({
+                    content: { in: generalProtein, out: protein },
+                    existsOk: true,
+                    fetchExisting: false,
+                    target: 'Infers',
+                });
+            }
+        } catch (err) {
+            logger.error(err);
         }
-    } catch (err) {
-        logger.error(err);
     }
 
     // create the cds variant
@@ -299,17 +299,20 @@ const processCosmicRecord = async (conn, record, source) => {
  * Disease mappings
  */
 const loadClassifications = async (filename) => {
-    const classifications = await loadDelimToJson(filename, { delim: ',' });
+    const classifications = await loadDelimToJson(filename);
     const mapping = {};
 
     for (const row of classifications) {
-        const disease = row.HISTOLOGY_COSMIC;
-        const subdisease = row.HIST_SUBTYPE1_COSMIC;
+        const diseaseId = row.COSMIC_PHENOTYPE_ID;
 
-        if (!mapping[disease]) {
-            mapping[disease] = {};
+        if (!mapping[diseaseId]) {
+            mapping[diseaseId] = {};
         }
-        mapping[disease][subdisease] = row.NCI_CODE;
+        mapping[diseaseId] = {
+            disease: row.HISTOLOGY_SUBTYPE_1,
+            diseaseFamily: row.PRIMARY_HISTOLOGY,
+            ncit: row.NCI_CODE,
+        };
     }
     return mapping;
 };
@@ -337,7 +340,7 @@ const uploadFile = async ({
         filters: [
             { source },
             { relevance },
-            { createdBy: { filters: { name: conn.username }, target: 'User' } },
+            { createdBy: { filters: { name: 'graphkb_importer' }, target: 'User' } },
         ],
         returnProperties: ['@rid'],
         target: 'Statement',
@@ -350,6 +353,7 @@ const uploadFile = async ({
     const errorList = [];
     logger.info(`Processing ${jsonList.length} records`);
     // Upload the list of pubmed IDs
+    await _pubmed.preLoadCache(conn);
     await _pubmed.fetchAndLoadByIds(conn, jsonList.map(rec => rec[HEADER.pubmed]), { upsert: true });
 
     for (let index = 0; index < jsonList.length; index++) {
@@ -367,7 +371,10 @@ const uploadFile = async ({
         }
 
         try {
-            record.ncit = (mapping[record.diseaseFamily] || {})[record.disease];
+            const diseaseMapping = mapping[record.diseaseId] || {};
+            record.ncit = diseaseMapping.ncit || '';
+            record.disease = diseaseMapping.disease || '';
+            record.diseaseFamily = diseaseMapping.diseaseFamily || '';
             record.publication = rid((await _pubmed.fetchAndLoadByIds(conn, [record.pubmed]))[0]);
             const statement = await processCosmicRecord(conn, record, source);
 

src/entrez/pubmed.js

@@ -71,9 +71,17 @@ const createDisplayName = sourceId => `pmid:${sourceId}`;
  * @param {Array.<string>} idList list of pubmed IDs
  */
 const fetchAndLoadByIds = async (api, idListIn, opt = {}) => {
-    const pmcIds = idListIn.filter(id => /^pmc\d+$/i.exec(id)).map(id => id.replace(/^pmc/i, ''));
+    // hardcoded fix for a record that was deleted from pubmed after being cited in a paper
+    // https://pubmed.ncbi.nlm.nih.gov/21656749/
+    const idList = idListIn.map((id) => {
+        if (id === '21225871') {
+            return '21656749';
+        }
+        return id;
+    });
+    const pmcIds = idList.filter(id => /^pmc\d+$/i.exec(id)).map(id => id.replace(/^pmc/i, ''));
     const records = await fetchByIdList(
-        idListIn.filter(id => !/^pmc\d+$/i.exec(id)),
+        idList.filter(id => !/^pmc\d+$/i.exec(id)),
         {
             cache: CACHE, db: DB_NAME, parser: parseRecord,
         },

src/entrez/util.js

@@ -91,6 +91,11 @@ const fetchByIdList = async (rawIdList, opt) => {
         Object.keys(result).filter(k => k !== 'uids').forEach((key) => {
             const rec = result[key];
 
+            if (rec.error) {
+                logger.error(`error fetching record for id ${key}: ${rec.error}`);
+                return;
+            }
+
             try {
                 records.push(parser(rec));
             } catch (err) {

src/hgnc/index.js

@@ -218,6 +218,10 @@ const fetchAndLoadBySymbol = async ({
         uri,
     });
 
+    if (!docs || docs.length === 0) {
+        throw new Error(`No HGNC record found for ${paramType}: ${symbol}`);
+    }
+
     for (const record of docs) {
         checkSpec(validateHgncSpec, record, rec => rec.hgnc_id);
     }
@@ -238,7 +242,9 @@ const fetchAndLoadBySymbol = async ({
             filters: { name: ensemblSourceName },
             target: 'Source',
         });
-    } catch (err) { }
+    } catch (err) {
+        logger.info('Unable to fetch ensembl source for linking records');
+    }
     const result = await uploadRecord({
         conn,
         deprecated: (