Feat/external deployment merge from dev

Dockerfile.snakemake

@@ -0,0 +1,38 @@
+FROM node:16 as node_stage
+WORKDIR /usr/src/app
+# Bundle app source
+COPY package*.json ./
+RUN npm ci --only=production
+# COPY everything not in dockerignore file
+COPY . .
+# set to avoid errors when singularity overloads working dir
+ENV NODE_PATH=/usr/src/app/node_modules
+
+FROM python:3.12
+WORKDIR /usr/src/app
+COPY --from=node_stage /usr/src/app/package*.json ./
+COPY --from=node_stage /usr/src/app/node_modules ./node_modules
+
+RUN apt-get update && \
+    apt-get install -y curl && \
+    curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \
+    apt-get install -y nodejs
+
+# Verify Node.js version
+RUN node -v
+
+RUN pip3 install --upgrade setuptools pip wheel && \
+    pip3 install "pulp<2.3.1" && \
+    pip3 install snakemake
+
+RUN mkdir /usr/src/app/snakemake_data
+RUN mkdir /usr/src/app/snakemake_logs
+RUN chmod 777 /usr/src/app/snakemake_data
+RUN chmod 777 /usr/src/app/snakemake_logs
+
+# Copy the Snakefile to the working directory (assuming it's already in the build context)
+COPY . .
+
+# Run the Snakefile using Snakemake
+CMD [ "sh", "-c", "snakemake --debug -j 1 --config gkb_url=$GKB_URL gkb_user=$GKB_USER gkb_pass=$GKB_PASS --until $GKB_LOADER"]
+

Snakefile

@@ -11,6 +11,16 @@ if not os.path.exists(DATA_DIR):
 if not os.path.exists(LOGS_DIR):
     os.mkdir(LOGS_DIR)
 
+def build_docker_image(tag):
+    docker_image_name = f"pori-graphkb-loader:{tag}"
+    shell(f"docker build -t {docker_image_name} .")
+    return docker_image_name
+
+if config.get('build_docker'):
+    containerchoice = build_docker_image('test')
+else:
+    containerchoice = CONTAINER
+
 
 LOADER_COMMAND = 'node bin/load.js ' + ' '.join([f'--{k} {v}' for k, v in {
     'username': config.get('gkb_user') or os.environ.get('GKB_USER'),
@@ -33,12 +43,12 @@ rule all:
     input: f'{DATA_DIR}/civic.COMPLETE',
         f'{DATA_DIR}/cgi.COMPLETE',
         f'{DATA_DIR}/docm.COMPLETE',
-        f'{DATA_DIR}/dgidb.COMPLETE',
+        #f'{DATA_DIR}/dgidb.COMPLETE',
         f'{DATA_DIR}/PMC4468049.COMPLETE',
         f'{DATA_DIR}/PMC4232638.COMPLETE',
         f'{DATA_DIR}/uberon.COMPLETE',
         f'{DATA_DIR}/fdaApprovals.COMPLETE',
-        f'{DATA_DIR}/cancerhotspots.COMPLETE',
+        #f'{DATA_DIR}/cancerhotspots.COMPLETE',
         f'{DATA_DIR}/moa.COMPLETE',
         f'{DATA_DIR}/ncitFdaXref.COMPLETE',
         *([f'{DATA_DIR}/clinicaltrialsgov.COMPLETE'] if BACKFILL_TRIALS else []),
@@ -205,7 +215,7 @@ rule download_cosmic_fusions:
 
 rule load_local:
     input: f'{DATA_DIR}/local/{{local}}.json'
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/local-{{local}}.logs.txt'
     output: f'{DATA_DIR}/local-{{local}}.COMPLETE'
     shell: LOADER_COMMAND + ' file ontology {input} &> {log}; cp {log} {output}'
@@ -214,7 +224,7 @@ rule load_local:
 rule load_ncit:
     input: expand(rules.load_local.output, local=['vocab']),
         data=rules.download_ncit.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/ncit.logs.txt'
     output: f'{DATA_DIR}/ncit.COMPLETE'
     shell: LOADER_COMMAND + ' file ncit {input.data} &> {log}; cp {log} {output}'
@@ -224,7 +234,7 @@ rule load_fda_srs:
     input: expand(rules.load_local.output, local=['vocab']),
         f'{DATA_DIR}/ncit.COMPLETE',
         data=f'{DATA_DIR}/fda/UNII_Records.txt'
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/fdaSrs.logs.txt'
     output: f'{DATA_DIR}/fdaSrs.COMPLETE'
     shell: LOADER_COMMAND + ' file fdaSrs {input.data} &> {log}; cp {log} {output}'
@@ -234,7 +244,7 @@ rule load_ncit_fda:
     input: rules.load_ncit.output,
         rules.load_fda_srs.output,
         data=rules.download_ncit_fda.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/ncitFdaXref.logs.txt'
     output: f'{DATA_DIR}/ncitFdaXref.COMPLETE'
     shell: LOADER_COMMAND + ' file ncitFdaXref {input.data} &> {log}; cp {log} {output}'
@@ -243,7 +253,7 @@ rule load_ncit_fda:
 rule load_refseq:
     input: expand(rules.load_local.output, local=['vocab']),
         data=rules.download_refseq.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/refseq.logs.txt'
     output: f'{DATA_DIR}/refseq.COMPLETE'
     shell: LOADER_COMMAND + ' file refseq {input.data} &> {log}; cp {log} {output}'
@@ -252,7 +262,7 @@ rule load_refseq:
 rule load_ensembl:
     input: rules.load_refseq.output,
         data=rules.download_ensembl.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/ensembl.logs.txt'
     output: f'{DATA_DIR}/ensembl.COMPLETE'
     shell: LOADER_COMMAND + ' file ensembl {input.data} &> {log}; cp {log} {output}'
@@ -261,7 +271,7 @@ rule load_ensembl:
 rule load_do:
     input: rules.load_ncit.output,
         data=rules.download_do.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/do.logs.txt'
     output: f'{DATA_DIR}/do.COMPLETE'
     shell: LOADER_COMMAND + ' file diseaseOntology {input.data} &> {log}; cp {log} {output}'
@@ -270,7 +280,7 @@ rule load_do:
 rule load_uberon:
     input: rules.load_ncit.output,
         data=rules.download_uberon.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/uberon.logs.txt'
     output: f'{DATA_DIR}/uberon.COMPLETE'
     shell: LOADER_COMMAND + ' file uberon {input.data} &> {log}; cp {log} {output}'
@@ -279,15 +289,15 @@ rule load_uberon:
 rule load_drugbank:
     input: rules.load_fda_srs.output,
         data=rules.download_drugbank.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/drugbank.logs.txt'
     output: f'{DATA_DIR}/drugbank.COMPLETE'
     shell: LOADER_COMMAND + ' file drugbank {input.data} &> {log}; cp {log} {output}'
 
 
 rule load_oncotree:
     input: rules.load_ncit.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/oncotree.logs.txt'
     output: f'{DATA_DIR}/oncotree.COMPLETE'
     shell: LOADER_COMMAND + ' api oncotree &> {log}; cp {log} {output}'
@@ -296,15 +306,15 @@ rule load_oncotree:
 def get_drug_inputs(wildcards):
     inputs = [*rules.load_ncit.output]
     inputs.extend(rules.load_fda_srs.output)
-    container: CONTAINER
+    containerized: containerchoice
     if USE_DRUGBANK:
         inputs.append(*rules.load_drugbank.output)
     return inputs
 
 
 rule all_drugs:
     input: lambda wildcards: get_drug_inputs(wildcards)
-    container: CONTAINER
+    containerized: containerchoice
     output: f'{DATA_DIR}/all_drugs.COMPLETE'
     shell: 'touch {output}'
 
@@ -313,22 +323,22 @@ rule all_diseases:
     input: rules.load_do.output,
         rules.load_ncit.output,
         rules.load_oncotree.output
-    container: CONTAINER
+    containerized: containerchoice
     output: f'{DATA_DIR}/all_diseases.COMPLETE'
     shell: 'touch {output}'
 
 
 rule all_local:
     input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes', 'evidenceLevels', 'aacr', 'asco']),
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/all_local.logs.txt'
     output: f'{DATA_DIR}/all_local.COMPLETE'
     shell: 'touch {output}'
 
 
 rule load_dgidb:
     input: rules.all_local.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/dgidb.logs.txt'
     output: f'{DATA_DIR}/dgidb.COMPLETE'
     shell: LOADER_COMMAND + ' api dgidb &> {log}; cp {log} {output}'
@@ -339,7 +349,7 @@ rule load_cancerhotspots:
         rules.load_oncotree.output,
         rules.load_ensembl.output,
         data=rules.download_cancerhotspots.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/cancerhotspots.logs.txt'
     output: f'{DATA_DIR}/cancerhotspots.COMPLETE'
     shell: LOADER_COMMAND + ' file cancerhotspots {input.data} &> {log}; cp {log} {output}'
@@ -348,7 +358,7 @@ rule load_cancerhotspots:
 rule load_PMC4232638:
     input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes']),
         data=rules.download_PMC4232638.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/PMC4232638.logs.txt'
     output: f'{DATA_DIR}/PMC4232638.COMPLETE'
     shell: LOADER_COMMAND + ' file PMC4232638 {input.data} &> {log}; cp {log} {output}'
@@ -358,7 +368,7 @@ rule load_PMC4468049:
     input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes']),
         rules.all_diseases.output,
         data=rules.download_PMC4468049.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/PMC4468049.logs.txt'
     output: f'{DATA_DIR}/PMC4468049.COMPLETE'
     shell: LOADER_COMMAND + ' file PMC4468049 {input.data} &> {log}; cp {log} {output}'
@@ -368,7 +378,7 @@ rule load_civic:
     input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes', 'evidenceLevels', 'aacr', 'asco']),
         rules.load_ncit.output,
         rules.load_do.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/civic.logs.txt'
     output: f'{DATA_DIR}/civic.COMPLETE'
     shell: LOADER_COMMAND + ' civic &> {log}; cp {log} {output}'
@@ -379,7 +389,7 @@ rule load_cgi:
         rules.all_diseases.output,
         rules.all_drugs.output,
         data=rules.download_cgi.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/cgi.logs.txt'
     output: f'{DATA_DIR}/cgi.COMPLETE'
     shell: LOADER_COMMAND + ' file cgi {input.data} &> {log}; cp {log} {output}'
@@ -389,15 +399,15 @@ rule load_docm:
     input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes']),
         rules.load_ncit.output,
         rules.load_do.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/docm.logs.txt'
     output: f'{DATA_DIR}/docm.COMPLETE'
     shell: LOADER_COMMAND + ' api docm &> {log}; cp {log} {output}'
 
 
 rule load_approvals:
     input:
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/fdaApprovals.logs.txt'
     output: f'{DATA_DIR}/fdaApprovals.COMPLETE'
     shell: LOADER_COMMAND + ' api fdaApprovals &> {log}; cp {log} {output}'
@@ -407,7 +417,7 @@ rule load_clinicaltrialsgov:
     input: expand(rules.load_local.output, local=['vocab']),
         rules.all_diseases.output,
         rules.all_drugs.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/clinicaltrialsgov.logs.txt'
     output: f'{DATA_DIR}/clinicaltrialsgov.COMPLETE'
     shell: LOADER_COMMAND + ' clinicaltrialsgov &> {log}; cp {log} {output}'
@@ -419,7 +429,7 @@ rule load_cosmic_resistance:
         rules.all_drugs.output,
         main=rules.download_cosmic_resistance.output,
         supp=rules.download_cosmic_diseases.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/cosmic_resistance.logs.txt'
     output: f'{DATA_DIR}/cosmic_resistance.COMPLETE'
     shell: LOADER_COMMAND + ' cosmic resistance {input.main} {input.supp} &> {log}; cp {log} {output}'
@@ -429,7 +439,7 @@ rule load_cosmic_fusions:
     input: rules.all_diseases.output,
         main=rules.download_cosmic_fusions.output,
         supp=rules.download_cosmic_diseases.output
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/cosmic_fusions.logs.txt'
     output: f'{DATA_DIR}/cosmic_fusions.COMPLETE'
     shell: LOADER_COMMAND + ' cosmic fusions {input.main} {input.supp} &> {log}; cp {log} {output}'
@@ -438,7 +448,7 @@ rule load_cosmic_fusions:
 rule load_moa:
     input: rules.load_oncotree.output,
         expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes', 'evidenceLevels', 'aacr', 'asco'])
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/load_moa.logs.txt'
     output: f'{DATA_DIR}/moa.COMPLETE'
     shell: LOADER_COMMAND + ' api moa  &> {log}; cp {log} {output}'
@@ -454,7 +464,7 @@ rule download_sources:
 
 rule load_sources:
     input: f'{DATA_DIR}/local/sources.json'
-    container: CONTAINER
+    containerized: containerchoice
     log: f'{LOGS_DIR}/sources.logs.txt'
     output: f'{DATA_DIR}/sources.COMPLETE'
     shell: LOADER_COMMAND + ' file sources {input} &> {log}; cp {log} {output}'
@@ -472,7 +482,7 @@ rule all_ontologies:
         rules.load_sources.output,
         rules.load_fda_srs.output,
         rules.load_ncit_fda.output,
-        rules.load_dgidb.output
-    container: CONTAINER
+        #rules.load_dgidb.output
+    containerized: containerchoice
     output: f'{DATA_DIR}/all_ontologies.COMPLETE'
     shell: 'touch {output}'