diff --git a/Dockerfile.snakemake b/Dockerfile.snakemake new file mode 100644 index 00000000..d1593bbd --- /dev/null +++ b/Dockerfile.snakemake @@ -0,0 +1,38 @@ +FROM node:16 as node_stage +WORKDIR /usr/src/app +# Bundle app source +COPY package*.json ./ +RUN npm ci --only=production +# COPY everything not in dockerignore file +COPY . . +# set to avoid errors when singularity overloads working dir +ENV NODE_PATH=/usr/src/app/node_modules + +FROM python:3.12 +WORKDIR /usr/src/app +COPY --from=node_stage /usr/src/app/package*.json ./ +COPY --from=node_stage /usr/src/app/node_modules ./node_modules + +RUN apt-get update && \ + apt-get install -y curl && \ + curl -fsSL https://deb.nodesource.com/setup_16.x | bash - && \ + apt-get install -y nodejs + +# Verify Node.js version +RUN node -v + +RUN pip3 install --upgrade setuptools pip wheel && \ + pip3 install "pulp<2.3.1" && \ + pip3 install snakemake + +RUN mkdir /usr/src/app/snakemake_data +RUN mkdir /usr/src/app/snakemake_logs +RUN chmod 777 /usr/src/app/snakemake_data +RUN chmod 777 /usr/src/app/snakemake_logs + +# Copy the Snakefile to the working directory (assuming it's already in the build context) +COPY . . + +# Run the Snakefile using Snakemake +CMD [ "sh", "-c", "snakemake --debug -j 1 --config gkb_url=$GKB_URL gkb_user=$GKB_USER gkb_pass=$GKB_PASS --until $GKB_LOADER"] + diff --git a/Snakefile b/Snakefile index 475f39b6..bb9c5ab0 100644 --- a/Snakefile +++ b/Snakefile @@ -11,6 +11,16 @@ if not os.path.exists(DATA_DIR): if not os.path.exists(LOGS_DIR): os.mkdir(LOGS_DIR) +def build_docker_image(tag): + docker_image_name = f"pori-graphkb-loader:{tag}" + shell(f"docker build -t {docker_image_name} .") + return docker_image_name + +if config.get('build_docker'): + containerchoice = build_docker_image('test') +else: + containerchoice = CONTAINER + LOADER_COMMAND = 'node bin/load.js ' + ' '.join([f'--{k} {v}' for k, v in { 'username': config.get('gkb_user') or os.environ.get('GKB_USER'), @@ -33,12 +43,12 @@ rule all: input: f'{DATA_DIR}/civic.COMPLETE', f'{DATA_DIR}/cgi.COMPLETE', f'{DATA_DIR}/docm.COMPLETE', - f'{DATA_DIR}/dgidb.COMPLETE', + #f'{DATA_DIR}/dgidb.COMPLETE', f'{DATA_DIR}/PMC4468049.COMPLETE', f'{DATA_DIR}/PMC4232638.COMPLETE', f'{DATA_DIR}/uberon.COMPLETE', f'{DATA_DIR}/fdaApprovals.COMPLETE', - f'{DATA_DIR}/cancerhotspots.COMPLETE', + #f'{DATA_DIR}/cancerhotspots.COMPLETE', f'{DATA_DIR}/moa.COMPLETE', f'{DATA_DIR}/ncitFdaXref.COMPLETE', *([f'{DATA_DIR}/clinicaltrialsgov.COMPLETE'] if BACKFILL_TRIALS else []), @@ -205,7 +215,7 @@ rule download_cosmic_fusions: rule load_local: input: f'{DATA_DIR}/local/{{local}}.json' - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/local-{{local}}.logs.txt' output: f'{DATA_DIR}/local-{{local}}.COMPLETE' shell: LOADER_COMMAND + ' file ontology {input} &> {log}; cp {log} {output}' @@ -214,7 +224,7 @@ rule load_local: rule load_ncit: input: expand(rules.load_local.output, local=['vocab']), data=rules.download_ncit.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/ncit.logs.txt' output: f'{DATA_DIR}/ncit.COMPLETE' shell: LOADER_COMMAND + ' file ncit {input.data} &> {log}; cp {log} {output}' @@ -224,7 +234,7 @@ rule load_fda_srs: input: expand(rules.load_local.output, local=['vocab']), f'{DATA_DIR}/ncit.COMPLETE', data=f'{DATA_DIR}/fda/UNII_Records.txt' - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/fdaSrs.logs.txt' output: f'{DATA_DIR}/fdaSrs.COMPLETE' shell: LOADER_COMMAND + ' file fdaSrs {input.data} &> {log}; cp {log} {output}' @@ -234,7 +244,7 @@ rule load_ncit_fda: input: rules.load_ncit.output, rules.load_fda_srs.output, data=rules.download_ncit_fda.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/ncitFdaXref.logs.txt' output: f'{DATA_DIR}/ncitFdaXref.COMPLETE' shell: LOADER_COMMAND + ' file ncitFdaXref {input.data} &> {log}; cp {log} {output}' @@ -243,7 +253,7 @@ rule load_ncit_fda: rule load_refseq: input: expand(rules.load_local.output, local=['vocab']), data=rules.download_refseq.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/refseq.logs.txt' output: f'{DATA_DIR}/refseq.COMPLETE' shell: LOADER_COMMAND + ' file refseq {input.data} &> {log}; cp {log} {output}' @@ -252,7 +262,7 @@ rule load_refseq: rule load_ensembl: input: rules.load_refseq.output, data=rules.download_ensembl.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/ensembl.logs.txt' output: f'{DATA_DIR}/ensembl.COMPLETE' shell: LOADER_COMMAND + ' file ensembl {input.data} &> {log}; cp {log} {output}' @@ -261,7 +271,7 @@ rule load_ensembl: rule load_do: input: rules.load_ncit.output, data=rules.download_do.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/do.logs.txt' output: f'{DATA_DIR}/do.COMPLETE' shell: LOADER_COMMAND + ' file diseaseOntology {input.data} &> {log}; cp {log} {output}' @@ -270,7 +280,7 @@ rule load_do: rule load_uberon: input: rules.load_ncit.output, data=rules.download_uberon.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/uberon.logs.txt' output: f'{DATA_DIR}/uberon.COMPLETE' shell: LOADER_COMMAND + ' file uberon {input.data} &> {log}; cp {log} {output}' @@ -279,7 +289,7 @@ rule load_uberon: rule load_drugbank: input: rules.load_fda_srs.output, data=rules.download_drugbank.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/drugbank.logs.txt' output: f'{DATA_DIR}/drugbank.COMPLETE' shell: LOADER_COMMAND + ' file drugbank {input.data} &> {log}; cp {log} {output}' @@ -287,7 +297,7 @@ rule load_drugbank: rule load_oncotree: input: rules.load_ncit.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/oncotree.logs.txt' output: f'{DATA_DIR}/oncotree.COMPLETE' shell: LOADER_COMMAND + ' api oncotree &> {log}; cp {log} {output}' @@ -296,7 +306,7 @@ rule load_oncotree: def get_drug_inputs(wildcards): inputs = [*rules.load_ncit.output] inputs.extend(rules.load_fda_srs.output) - container: CONTAINER + containerized: containerchoice if USE_DRUGBANK: inputs.append(*rules.load_drugbank.output) return inputs @@ -304,7 +314,7 @@ def get_drug_inputs(wildcards): rule all_drugs: input: lambda wildcards: get_drug_inputs(wildcards) - container: CONTAINER + containerized: containerchoice output: f'{DATA_DIR}/all_drugs.COMPLETE' shell: 'touch {output}' @@ -313,14 +323,14 @@ rule all_diseases: input: rules.load_do.output, rules.load_ncit.output, rules.load_oncotree.output - container: CONTAINER + containerized: containerchoice output: f'{DATA_DIR}/all_diseases.COMPLETE' shell: 'touch {output}' rule all_local: input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes', 'evidenceLevels', 'aacr', 'asco']), - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/all_local.logs.txt' output: f'{DATA_DIR}/all_local.COMPLETE' shell: 'touch {output}' @@ -328,7 +338,7 @@ rule all_local: rule load_dgidb: input: rules.all_local.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/dgidb.logs.txt' output: f'{DATA_DIR}/dgidb.COMPLETE' shell: LOADER_COMMAND + ' api dgidb &> {log}; cp {log} {output}' @@ -339,7 +349,7 @@ rule load_cancerhotspots: rules.load_oncotree.output, rules.load_ensembl.output, data=rules.download_cancerhotspots.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/cancerhotspots.logs.txt' output: f'{DATA_DIR}/cancerhotspots.COMPLETE' shell: LOADER_COMMAND + ' file cancerhotspots {input.data} &> {log}; cp {log} {output}' @@ -348,7 +358,7 @@ rule load_cancerhotspots: rule load_PMC4232638: input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes']), data=rules.download_PMC4232638.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/PMC4232638.logs.txt' output: f'{DATA_DIR}/PMC4232638.COMPLETE' shell: LOADER_COMMAND + ' file PMC4232638 {input.data} &> {log}; cp {log} {output}' @@ -358,7 +368,7 @@ rule load_PMC4468049: input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes']), rules.all_diseases.output, data=rules.download_PMC4468049.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/PMC4468049.logs.txt' output: f'{DATA_DIR}/PMC4468049.COMPLETE' shell: LOADER_COMMAND + ' file PMC4468049 {input.data} &> {log}; cp {log} {output}' @@ -368,7 +378,7 @@ rule load_civic: input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes', 'evidenceLevels', 'aacr', 'asco']), rules.load_ncit.output, rules.load_do.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/civic.logs.txt' output: f'{DATA_DIR}/civic.COMPLETE' shell: LOADER_COMMAND + ' civic &> {log}; cp {log} {output}' @@ -379,7 +389,7 @@ rule load_cgi: rules.all_diseases.output, rules.all_drugs.output, data=rules.download_cgi.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/cgi.logs.txt' output: f'{DATA_DIR}/cgi.COMPLETE' shell: LOADER_COMMAND + ' file cgi {input.data} &> {log}; cp {log} {output}' @@ -389,7 +399,7 @@ rule load_docm: input: expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes']), rules.load_ncit.output, rules.load_do.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/docm.logs.txt' output: f'{DATA_DIR}/docm.COMPLETE' shell: LOADER_COMMAND + ' api docm &> {log}; cp {log} {output}' @@ -397,7 +407,7 @@ rule load_docm: rule load_approvals: input: - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/fdaApprovals.logs.txt' output: f'{DATA_DIR}/fdaApprovals.COMPLETE' shell: LOADER_COMMAND + ' api fdaApprovals &> {log}; cp {log} {output}' @@ -407,7 +417,7 @@ rule load_clinicaltrialsgov: input: expand(rules.load_local.output, local=['vocab']), rules.all_diseases.output, rules.all_drugs.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/clinicaltrialsgov.logs.txt' output: f'{DATA_DIR}/clinicaltrialsgov.COMPLETE' shell: LOADER_COMMAND + ' clinicaltrialsgov &> {log}; cp {log} {output}' @@ -419,7 +429,7 @@ rule load_cosmic_resistance: rules.all_drugs.output, main=rules.download_cosmic_resistance.output, supp=rules.download_cosmic_diseases.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/cosmic_resistance.logs.txt' output: f'{DATA_DIR}/cosmic_resistance.COMPLETE' shell: LOADER_COMMAND + ' cosmic resistance {input.main} {input.supp} &> {log}; cp {log} {output}' @@ -429,7 +439,7 @@ rule load_cosmic_fusions: input: rules.all_diseases.output, main=rules.download_cosmic_fusions.output, supp=rules.download_cosmic_diseases.output - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/cosmic_fusions.logs.txt' output: f'{DATA_DIR}/cosmic_fusions.COMPLETE' shell: LOADER_COMMAND + ' cosmic fusions {input.main} {input.supp} &> {log}; cp {log} {output}' @@ -438,7 +448,7 @@ rule load_cosmic_fusions: rule load_moa: input: rules.load_oncotree.output, expand(rules.load_local.output, local=['vocab', 'signatures', 'chromosomes', 'evidenceLevels', 'aacr', 'asco']) - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/load_moa.logs.txt' output: f'{DATA_DIR}/moa.COMPLETE' shell: LOADER_COMMAND + ' api moa &> {log}; cp {log} {output}' @@ -454,7 +464,7 @@ rule download_sources: rule load_sources: input: f'{DATA_DIR}/local/sources.json' - container: CONTAINER + containerized: containerchoice log: f'{LOGS_DIR}/sources.logs.txt' output: f'{DATA_DIR}/sources.COMPLETE' shell: LOADER_COMMAND + ' file sources {input} &> {log}; cp {log} {output}' @@ -472,7 +482,7 @@ rule all_ontologies: rules.load_sources.output, rules.load_fda_srs.output, rules.load_ncit_fda.output, - rules.load_dgidb.output - container: CONTAINER + #rules.load_dgidb.output + containerized: containerchoice output: f'{DATA_DIR}/all_ontologies.COMPLETE' shell: 'touch {output}'