config.yaml

---
work_path: ../pipeline_work

log_level: DEBUG
pool_size: 4

scratchpad:
  chembl_version: '36'
  efo_version: '3.83.0'
  ensembl_version: '115'
  gencode_version: '49'
  depmap_version: '2025Q2'
  hpo_version: '2025-01-16'
  mondo_version: '2025-02-04'
  ot_curation: '25.12.2'
  probes_drugs_version: '02_2025'
  gnomad_version: '4.1'
  gtex_version: '10'

steps:
  ##################################################################################################
  #: BASELINE EXPRESSION STEP :#####################################################################
  baseline_expression_raw:
    - name: copy GTEx TPM matrix
      source: https://storage.googleapis.com/adult-gtex/bulk-gex/v${gtex_version}/rna-seq/GTEx_Analysis_v${gtex_version}_RNASeQCv2.4.2_gene_tpm.gct.gz
      destination: input/baseline_expression/bulkRNAseq/GTEx/gene_tpm.gct.gz

    - name: copy GTEx sample attributes
      source: https://storage.googleapis.com/adult-gtex/annotations/v${gtex_version}/metadata-files/GTEx_Analysis_v${gtex_version}_Annotations_SampleAttributesDS.txt
      destination: input/baseline_expression/bulkRNAseq/GTEx/sample_attributes.txt

    - name: copy GTEx subject phenotypes
      source: https://storage.googleapis.com/adult-gtex/annotations/v${gtex_version}/metadata-files/GTEx_Analysis_v${gtex_version}_Annotations_SubjectPhenotypesDS.txt
      destination: input/baseline_expression/bulkRNAseq/GTEx/subject_phenotypes.txt

    - name: explode DICE expression data
      foreach:
        - B_CELL_NAIVE_TPM.csv
        - CD4_NAIVE_TPM.csv
        - CD4_STIM_TPM.csv
        - CD8_NAIVE_TPM.csv
        - CD8_STIM_TPM.csv
        - M2_TPM.csv
        - MONOCYTES_TPM.csv
        - NK_TPM.csv
        - TFH_TPM.csv
        - TH1_TPM.csv
        - TH17_TPM.csv
        - TH2_TPM.csv
        - THSTAR_TPM.csv
        - TREG_MEM_TPM.csv
        - TREG_NAIVE_TPM.csv
      do:
        - name: copy DICE ${each}
          source: https://dice-database.org/download/${each}
          destination: input/baseline_expression/bulkRNAseq/DICE/${each}

    - name: copy DICE ontology mapping
      source: gs://ot-team/tobi/baseline_expression/data/DICE/DICE_ontology_mapping.txt
      destination: input/baseline_expression/bulkRNAseq/DICE/DICE_ontology_mapping.txt

    - name: copy Tabula Sapiens data
      source: https://datasets.cellxgene.cziscience.com/5a495302-b7cd-4bf9-853e-95627b00bb03.h5ad
      destination: input/baseline_expression/scRNAseq/CxG-tabula_sapiens.h5ad

    - name: copy PRIDE tissue ontology mapping
      source: gs://ot-team/tobi/baseline_expression/data/PRIDE/tissue-complete_mapping.tsv
      destination: input/baseline_expression/MSproteomics/PRIDE/tissue-complete_mapping.tsv

    - name: explode PRIDE proteomics data
      foreach:
        - PXD000561
        - PXD000865
        - PXD016999
        - PXD020192
      do:
        - name: copy PRIDE ${each} ppb.json
          source: gs://ot-team/tobi/baseline_expression/data/PRIDE/${each}/${each}_OpenTargets_ppb.json
          destination: input/baseline_expression/MSproteomics/PRIDE/${each}/${each}_OpenTargets_ppb.json
        - name: copy PRIDE ${each} ppb.txt
          source: gs://ot-team/tobi/baseline_expression/data/PRIDE/${each}/${each}_OpenTargets_ppb.txt
          destination: input/baseline_expression/MSproteomics/PRIDE/${each}/${each}_OpenTargets_ppb.txt
        - name: copy PRIDE ${each} sdrf.json
          source: gs://ot-team/tobi/baseline_expression/data/PRIDE/${each}/${each}_OpenTargets_sdrf.json
          destination: input/baseline_expression/MSproteomics/PRIDE/${each}/${each}_OpenTargets_sdrf.json
        - name: copy PRIDE ${each} individuals.txt
          source: gs://ot-team/tobi/baseline_expression/data/PRIDE/${each}/${each}_individuals.txt
          destination: input/baseline_expression/MSproteomics/PRIDE/${each}/${each}_individuals.txt
        - name: copy PRIDE ${each} tissues.txt
          source: gs://ot-team/tobi/baseline_expression/data/PRIDE/${each}/${each}_tissues.txt
          destination: input/baseline_expression/MSproteomics/PRIDE/${each}/${each}_tissues.txt

  baseline_expression:
    - name: explode processed baseline expression datasets
      each_placeholder: dataset
      foreach:
        - GTEx/tissue
        - PRIDE/tissue
        - tabula_sapiens/celltype
        - tabula_sapiens/tissue
        - tabula_sapiens/tissue_celltype
        - DICE/celltype
      do:
        - name: copy_many processed ${dataset} aggregated expression
          sources: gs://baseline_expression/processed/aggregated/${dataset}/parquet/*
          destination: intermediate/baseline_expression/aggregated/${dataset}/parquet
        # - name: copy processed ${dataset} unaggregated expression
        #   source: gs://baseline_expression/processed/unaggregated/${dataset}/parquet
        #   destination: intermediate/baseline_expression/unaggregated/${dataset}/parquet

    - name: explode parent mapping
      foreach:
        - celltype
        - tissue
      do:
        - name: copy parent mapping for ${each}
          source: gs://baseline_expression/parent_mappings/${each}-parent_mapping.tsv
          destination: input/baseline_expression/${each}-parent_mapping.tsv

  ##################################################################################################

  #: BIOSAMPLE STEP :###############################################################################
  biosample:
    - name: copy cell ontology
      source: https://github.com/obophenotype/cell-ontology/releases/latest/download/cl.json
      destination: input/biosample/cl.json

    - name: copy uberon
      source: https://github.com/obophenotype/uberon/releases/latest/download/uberon.json
      destination: input/biosample/uberon.json

    - name: copy efo
      source: https://github.com/EBISPOT/efo/releases/download/v${efo_version}/efo.json
      destination: input/biosample/efo.json

  #################################################################################################

  #: CLINICAL_REPORT STEP :########################################################################
  clinical_report:
    - name: explode_glob clinical_report
      glob: gs://ot-team/irene/clinical_mining/2026-05-27/input/clinical_report/**/*
      do:
        - name: copy clinical_report ${match_stem} ${uuid}
          source: ${uri}
          destination: input/clinical_report/${match_path}/${match_stem}.${match_ext}
  ##################################################################################################

  #: DISEASE STEP :#################################################################################
  disease:
    - name: copy efo otar_slim
      source: https://github.com/EBISPOT/efo/releases/download/v${efo_version}/efo_otar_slim.json
      destination: input/disease/efo_otar_slim.json

    - name: copy hpo
      source: https://github.com/obophenotype/human-phenotype-ontology/releases/download/v${hpo_version}/hp-full.json
      destination: input/disease/hp-full.json

    - name: copy hpo phenotypes
      source: http://purl.obolibrary.org/obo/hp/hpoa/phenotype.hpoa
      destination: input/disease/phenotype.hpoa

    - name: copy mondo
      # we were using v2021-12-30 release until 25.03
      source: https://github.com/monarch-initiative/mondo/releases/download/v${mondo_version}/mondo.json
      destination: input/disease/mondo.json
  ##################################################################################################

  #: DRUG STEP :####################################################################################
  drug:
    - name: copy chemical probes
      source: https://www.probes-drugs.org/media/download/probes/pd_export_${probes_drugs_version}_probes_standardized.xlsx
      destination: input/target/chemicalprobes/probes.xlsx

    - name: copy drugbank annotation
      source: https://ftp.ebi.ac.uk/pub/databases/chembl/UniChem/data/wholeSourceMapping/src_id1/src1src2.txt.gz
      destination: input/drug/drugbank.csv.gz

    # AACT clinical-trial batch extraction — a standalone source consumed by
    # chembl_molecule (and clinical_report). Downloaded in the drug step so
    # chembl_molecule's existing pis_drug dependency covers it.
    - name: copy_many aact extraction batch results
      sources: gs://ot-team/irene/clinical_mining/aact_extraction_batch_result/output/*
      destination: input/clinical_report/aact_extraction_batch_results

    - name: elasticsearch chembl drug warning
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: input/drug/chembl_drug_warning.jsonl
      index: chembl_${chembl_version}_drug_warning
      fields:
        - _metadata
        - molecule_chembl_id
        - parent_molecule_chembl_id
        - efo_id
        - efo_term
        - efo_id_for_warning_class
        - warning_class
        - warning_country
        - warning_description
        - warning_id
        - warning_refs
        - warning_type
        - warning_year
    - name: elasticsearch chembl mechanism of action
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: input/drug/chembl_mechanism.jsonl
      index: chembl_${chembl_version}_mechanism
      fields:
        - _metadata
        - molecule_chembl_id
        - target_chembl_id
        - action_type
        - mechanism_of_action
        - mechanism_refs
        - record_id
        - parent_molecule_chembl_id
    - name: elasticsearch chembl molecule
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: input/drug/chembl_molecule.jsonl
      index: chembl_${chembl_version}_molecule
      fields:
        - molecule_chembl_id
        - molecule_hierarchy
        - molecule_type
        - pref_name
        - first_approval
        - max_phase
        - withdrawn_flag
        - black_box_warning
        - molecule_synonyms
        - cross_references
        - molecule_structures
    - name: elasticsearch chembl target
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: input/drug/chembl_target.jsonl
      index: chembl_${chembl_version}_target
      fields:
        - _metadata
        - target_chembl_id
        - target_components
        - target_type
        - pref_name
  ##################################################################################################

  #: IMPC STEP :####################################################################################
  impc:
    - name: solr fetch gene_gene
      destination: input/impc
      url: http://www.ebi.ac.uk/mi/impc/solr/phenodigm/select
      data_type: gene_gene
      fields:
        - gene_id
        - hgnc_gene_id

    - name: solr fetch ontology_ontology
      destination: input/impc
      url: http://www.ebi.ac.uk/mi/impc/solr/phenodigm/select
      data_type: ontology_ontology
      fields:
        - mp_id
        - hp_id
      strict: False

    - name: solr fetch mouse_model
      destination: input/impc
      url: http://www.ebi.ac.uk/mi/impc/solr/phenodigm/select
      queries:
      data_type: mouse_model
      fields:
        - model_id
        - model_phenotypes

    - name: solr fetch disease
      destination: input/impc
      url: http://www.ebi.ac.uk/mi/impc/solr/phenodigm/select
      data_type: disease
      fields:
        - disease_id
        - disease_phenotypes

    - name: solr fetch disease_model_summary
      destination: input/impc
      url: http://www.ebi.ac.uk/mi/impc/solr/phenodigm/select
      batch_size: 100000000
      data_type: disease_model_summary
      fields:
        - model_id
        - model_genetic_background
        - model_description
        - disease_id
        - disease_term
        - disease_model_avg_norm
        - disease_model_max_norm
        - marker_id

    - name: solr fetch ontology
      destination: input/impc
      url: http://www.ebi.ac.uk/mi/impc/solr/phenodigm/select
      data_type: ontology
      fields:
        - ontology
        - phenotype_id
        - phenotype_term

    - name: explode impc metadata
      foreach:
        - MGI_Gene_Model_Coord.rpt # Mouse gene mappings
        - MGI_PhenoGenoMP.rpt # Mouse PubMed references
        - mp.json # Mammalian Phenotype ontology
      do:
        - name: copy ${each}
          source: http://www.informatics.jax.org/downloads/reports/${each}
          destination: input/impc/${each}

    - name: copy gene name
      source: https://storage.googleapis.com/public-download-files/hgnc/tsv/tsv/hgnc_complete_set.txt
      destination: input/target/genenames/hgnc_complete_set.tsv
  ##################################################################################################

  #: EVIDENCE_EXPRESSION_ATLAS STEP :###############################################################
  evidence_expression_atlas:
    - name: find_latest atlas
      source: gs://otar010-atlas
      scratchpad_key: 'latest_atlas'
    - name: copy atlas
      requires:
        - find_latest atlas
      source: ${latest_atlas}
      destination: input/evidence/atlas.json.bz2
  ##################################################################################################

  #: EVIDENCE_CANCER_BIOMARKERS STEP :##############################################################
  evidence_cancer_biomarkers:
    - name: copy cancer biomarkers associations
      source: gs://otar000-evidence_input/CancerBiomarkers/data_files/cancerbiomarkers-2018-05-01.tsv
      destination: input/evidence/cancerbiomarkers/associations.tsv

    - name: copy cancer biomarkers sources
      source: gs://otar000-evidence_input/CancerBiomarkers/data_files/cancer_biomarker_source.jsonl
      destination: input/evidence/cancerbiomarkers/sources.jsonl

    - name: copy cancer biomarkers diseases
      source: gs://otar000-evidence_input/CancerBiomarkers/data_files/cancer_biomarker_disease.jsonl
      destination: input/evidence/cancerbiomarkers/diseases.jsonl
  ##################################################################################################

  #: EVIDENCE_CLINGEN STEP :########################################################################
  evidence_clingen:
    - name: copy clingen
      source: https://search.clinicalgenome.org/kb/gene-validity/download
      destination: input/evidence/clingen.csv
  ##################################################################################################

  #: EVIDENCE_COSMIC STEP :#########################################################################
  evidence_cosmic:
    - name: find_latest cosmic
      source: gs://otar007-cosmic
      scratchpad_key: 'latest_cosmic'
    - name: copy cosmic
      requires:
        - find_latest cosmic
      source: ${latest_cosmic}
      destination: input/evidence/cosmic.json.gz
  ##################################################################################################

  #: EVIDENCE_PROJECT_SCORE STEP :##################################################################
  evidence_project_score:
    - name: copy sanger cell passport
      source: https://cog.sanger.ac.uk/cmp/download/model_list_latest.csv.gz
      destination: input/evidence/cell_passport.csv.gz

    - name: copy project_score cell types
      source: gs://otar000-evidence_input/CRISPR/data_files/project_score_v2/cell_types.tsv
      destination: input/evidence/project_score/cell_types.tsv

    - name: copy project_score gene scores
      source: gs://otar000-evidence_input/CRISPR/data_files/project_score_v2/mapped_diseases.tsv
      destination: input/evidence/project_score/mapped_diseases.tsv

    - name: copy ot_curation depmap_tissue_mapping
      source: https://raw.githubusercontent.com/opentargets/curation/${ot_curation}/mappings/biosystem/depmap_uberon_mapping.csv
      destination: input/ot_curation/mappings/biosystem/depmap_uberon_mapping.csv
  ##################################################################################################

  #: EVIDENCE_CRISPR_SCREENS STEP :#################################################################
  evidence_crispr_screens:
    - name: crispr_brain fetch
      destination: input/evidence/crispr_screens
      api_url: https://crisprbrain.org/api/screens
      client_id: 92f263647c43525d3e4f181aa7e348f26e32129c0f827321d9261cc9765c56c0
      study_prefix_url: https://storage.googleapis.com/crisprbrain.appspot.com/api-data/
      validate_version: 1

    - name: copy crispr_brain mappings
      source: https://raw.githubusercontent.com/opentargets/curation/refs/tags/${ot_curation}/mappings/disease/brain_crispr_studies.tsv
      destination: input/ot_curation/mappings/disease/brain_crispr_studies.tsv
  ##################################################################################################

  #: EVIDENCE_EVA STEP :############################################################################
  evidence_eva:
    - name: find_latest eva
      source: gs://otar012-eva/disease-target-evidence/*
      scratchpad_key: 'latest_eva'
    - name: copy eva
      requires:
        - find_latest eva
      source: ${latest_eva}
      destination: input/evidence/eva.json.gz
  ##################################################################################################

  #: EVIDENCE_GENE_BURDEN STEP :####################################################################
  evidence_gene_burden:
    - name: copy gene_burden cvdi
      source: https://static-content.springer.com/esm/art%3A10.1038%2Fs41588-024-01894-5/MediaObjects/41588_2024_1894_MOESM4_ESM.xlsx
      destination: input/evidence/gene_burden/cvdi.xlsx

    - name: explode_glob download genebass
      glob: 'gs://otar000-evidence_input/GeneBurden/data_files/genebass_entries/*.parquet'
      do:
        - name: copy genebass ${match_stem}
          source: ${uri}
          destination: input/evidence/gene_burden/genebass/${match_stem}.${match_ext}

    - name: copy gene_burden finngen
      source: gs://finngen-public-data-r12/lof/data/finngen_R12_lof.txt.gz
      destination: input/evidence/gene_burden/finngen/finngen.txt.gz

    - name: copy gene_burden finngen phenos
      source: https://r12.finngen.fi/api/phenos
      destination: input/evidence/gene_burden/finngen/phenos

    - name: copy_many AZ parquet (quantitative)
      sources: gs://otar000-evidence_input/GeneBurden/data_files/azphewas-com-470k-phewas-quantitative/*.parquet
      destination: input/evidence/gene_burden/astrazeneca/azphewas-com-470k-phewas-quantitative

    - name: copy_many AZ parquet (binary)
      sources: gs://otar000-evidence_input/GeneBurden/data_files/azphewas-com-470k-phewas-binary/*.parquet
      destination: input/evidence/gene_burden/astrazeneca/azphewas-com-470k-phewas-binary

    - name: explode AZ CSVs
      foreach:
        - azphewas_com_genes_UK_Biobank_470k.csv
        - azphewas_com_phenotypes_UK_Biobank_470k.csv
      do:
        - name: copy ${each}
          source: gs://otar000-evidence_input/GeneBurden/data_files/${each}
          destination: input/evidence/gene_burden/astrazeneca/${each}

    - name: explode ot_curation files
      foreach:
        - mappings/disease/manual_string.tsv
        - gene_burden/curated_evidence.tsv
      do:
        - name: copy ${each}
          source: https://raw.githubusercontent.com/opentargets/curation/refs/tags/${ot_curation}/${each}
          destination: input/ot_curation/${each}
  ##################################################################################################

  #: EVIDENCE_GENE2PHENOTYPE STEP :#################################################################
  evidence_gene2phenotype:
    - name: copy gene2phenotype cancer
      source: https://www.ebi.ac.uk/gene2phenotype/api/panel/Cancer/download
      destination: input/evidence/gene2phenotype/cancer.csv

    - name: copy gene2phenotype cardiac
      source: https://www.ebi.ac.uk/gene2phenotype/api/panel/Cardiac/download
      destination: input/evidence/gene2phenotype/cardiac.csv

    - name: copy gene2phenotype dd
      source: https://www.ebi.ac.uk/gene2phenotype/api/panel/DD/download
      destination: input/evidence/gene2phenotype/dd.csv

    - name: copy gene2phenotype ear
      source: https://www.ebi.ac.uk/gene2phenotype/api/panel/Ear/download
      destination: input/evidence/gene2phenotype/ear.csv

    - name: copy gene2phenotype eye
      source: https://www.ebi.ac.uk/gene2phenotype/api/panel/Eye/download
      destination: input/evidence/gene2phenotype/eye.csv

    - name: copy gene2phenotype skeletal
      source: https://www.ebi.ac.uk/gene2phenotype/api/panel/Skeletal/download
      destination: input/evidence/gene2phenotype/skeletal.csv

    - name: copy gene2phenotype skin
      source: https://www.ebi.ac.uk/gene2phenotype/api/panel/Skin/download
      destination: input/evidence/gene2phenotype/skin.csv
  ##################################################################################################

  #: EVIDENCE_PANEL_APP STEP :######################################################################
  evidence_panel_app:
    - name: copy panel_app
      source: gs://otar000-evidence_input/GenomicsEngland/data_files/All_genes_20250525_public_v1plus_genes_all_ratings.tsv
      destination: input/evidence/panelapp/panel_app.tsv
    - name: panel_app fetch
      requires:
        - copy panel_app
      evidence_file: input/evidence/panelapp/panel_app.tsv
      api_url: https://panelapp.genomicsengland.co.uk/api/v1/panels
      destination: input/evidence/panelapp/panels.jsonl
  ##################################################################################################

  #: EVIDENCE_ORPHANET STEP :#######################################################################
  evidence_orphanet:
    - name: copy orphanet
      source: http://www.orphadata.org/data/xml/en_product6.xml
      destination: input/evidence/orphanet.xml
  ##################################################################################################

  #: EVIDENCE_REACTOME STEP :#######################################################################
  evidence_reactome:
    - name: find_latest reactome
      source: gs://otar006-reactome
      scratchpad_key: 'latest_reactome'
    - name: copy reactome
      requires:
        - find_latest reactome
      source: ${latest_reactome}
      destination: input/evidence/reactome.json.gz
  ##################################################################################################

  #: EVIDENCE_UNIPROT_LITERATURE STEP :#############################################################
  evidence_uniprot_literature:
    - name: find_latest uniprot literature
      source: gs://otar000-evidence_input/Uniprot_literature/json
      scratchpad_key: 'latest_uniprot_literature'
    - name: copy uniprot_literature
      requires:
        - find_latest uniprot literature
      source: ${latest_uniprot_literature}
      destination: input/evidence/uniprot_literature.json.gz
  ##################################################################################################

  #: EVIDENCE_UNIPROT_VARIANTS STEP :###############################################################
  evidence_uniprot_variants:
    - name: find_latest uniprot_variants
      source: gs://otar000-evidence_input/Uniprot_variants/json
      scratchpad_key: 'latest_uniprot_variants'
    - name: copy uniprot_variants
      requires:
        - find_latest uniprot_variants
      source: ${latest_uniprot_variants}
      destination: input/evidence/uniprot_variants.json.gz
  ##################################################################################################

  #: EVIDENCE_INTOGEN STEP :#######################################################################
  evidence_intogen:
    - name: explode intogen files
      foreach:
        - Compendium_Cancer_Genes.tsv
        - cohorts.tsv
      do:
        - name: copy ${each}
          source: gs://otar000-evidence_input/IntOgen/data_files/2024.06/${each}
          destination: input/evidence/intogen/${each}

    - name: copy ot_curation disease_mapping
      source: https://raw.githubusercontent.com/opentargets/curation/${ot_curation}/mappings/disease/cancer2EFO_mappings.tsv
      destination: input/ot_curation/mappings/disease/cancer2EFO_mappings.tsv
  ##################################################################################################

  #: EVIDENCE_PPP STEP :############################################################################
  evidence_ppp:
    # Validation lab => Get latest evidence.
    - name: find_latest validation_lab
      source: gs://otar013-ppp/validation_lab
      scratchpad_key: latest_validation_lab
    - name: copy validation_lab
      requires:
        - find_latest validation_lab
      source: ${latest_validation_lab}
      destination: input/evidence/validation_lab.json.gz

    # Encore => Get latest evidence.
    - name: find_latest encore
      source: gs://otar013-ppp/encore
      scratchpad_key: latest_encore
    - name: copy encore
      requires:
        - find_latest encore
      source: ${latest_encore}
      destination: input/evidence/encore.json.gz

    # OT_crispr => generate evience.
    - name: spreadsheet_download otar crispr study spreadsheet
      sheet_id: 1eow_rTizGY_Vda0Q-8sFNWCOyFkKRSUen1Em1ysPSiA
      gid: '0'
      destination: input/evidence/ot_crispr/config.tsv

    - name: explode_glob ot_crispr files
      glob: 'gs://otar013-ppp/ot_crispr/input_data_filtered/**/*'
      do:
        - name: copy ot_crispr ${match_stem} ${uuid}
          source: ${uri}
          destination: input/evidence/ot_crispr/data/${match_path}/${match_stem}.${match_ext}

  ##################################################################################################

  #: GO STEP :######################################################################################
  go:
    - name: copy gene ontology
      source: http://purl.obolibrary.org/obo/go.obo
      destination: input/go/go.obo
  ##################################################################################################

  #: INTERACTION STEP :#############################################################################
  interaction:
    - name: copy ensembl interactions grch38
      source: https://ftp.ensembl.org/pub/release-${ensembl_version}/gtf/homo_sapiens/Homo_sapiens.GRCh38.${ensembl_version}.chr.gtf.gz
      destination: input/interaction/Homo_sapiens.GRCh38.chr.gtf.gz

    - name: copy human 9606 idmapping
      source: https://ftp.ebi.ac.uk/pub/databases/uniprot/current_release/knowledgebase/idmapping/by_organism/HUMAN_9606_idmapping.dat.gz
      destination: input/interaction/HUMAN_9606_idmapping.dat.gz

    - name: copy intact interactors
      source: https://ftp.ebi.ac.uk/pub/databases/intact/various/ot_graphdb/current/data/interactor_pair_interactions.json
      destination: input/interaction/intact-interactors.json

    - name: copy rna central
      source: https://ftp.ebi.ac.uk/pub/databases/RNAcentral/current_release/id_mapping/database_mappings/ensembl.tsv
      destination: input/interaction/rna_central_ensembl.tsv

    - name: find_latest string interactions
      source: gs://otar001-core/stringInteractions
      scratchpad_key: latest_string_interactions
    - name: copy string interactions
      requires:
        - find_latest string interactions
      source: ${latest_string_interactions}
      destination: input/interaction/string-interactions.txt.gz
  ##################################################################################################

  #: ENCODE STEP :##################################################################################
  encode:
    - name: copy chromosome contigs
      source: gs://vep_cache_data/vep/cache/contig.parquet
      destination: input/chromosome/chromosome.parquet

    - name: copy encode rE2G biosample mapping
      source: https://raw.githubusercontent.com/opentargets/curation/refs/tags/${ot_curation}/genetics/E2G_biosample_mapping.csv
      destination: input/encode/rE2G/biosample_mapping.csv

    - name: copy encode rE2G manifest
      # NOTE: excluded samples from query have divergent schemas that do not conform to expected rE2G format
      source: https://www.encodeproject.org/report.tsv?type=File&searchTerm=encode-re2g+thresholded&output_type=thresholded+element+gene+links&status=released&limit=all&accession!=ENCFF094BVF&accession!=ENCFF269DKY
      destination: input/encode/rE2G/download_manifest.tsv

    - name: crawl_encode rE2G manifest
      requires:
        - copy encode rE2G manifest
      source: input/encode/rE2G/download_manifest.tsv
      destination: input/encode/rE2G/source_list_file.txt

    - name: copy_many encode rE2G datasets
      requires:
        - crawl_encode rE2G manifest
      source_list_file: input/encode/rE2G/source_list_file.txt
      destination: input/encode/rE2G/datasets
  ##################################################################################################

  #: GWAS STEP :######################################################################################
  gwas:
    - name: copy_many GCCA study
      sources: gs://gwas_catalog_top_hits/study_index/*.parquet
      destination: input/gwas/study/gcca

    - name: copy_many GCCA credible sets
      sources: gs://gwas_catalog_top_hits/credible_sets/*.parquet
      destination: input/gwas/credible_set/gcca

    - name: copy_many GCSS pics study
      sources: gs://gwas_catalog_sumstats_pics/study_index/*.parquet
      destination: input/gwas/study/gcss_pics

    - name: copy_many GCSS pics credible sets
      sources: gs://gwas_catalog_sumstats_pics/credible_sets/*.parquet
      destination: input/gwas/credible_set/gcss_pics

    - name: copy_many GCSS susie study
      sources: gs://gwas_catalog_sumstats_susie/study_index/*.parquet
      destination: input/gwas/study/gcss_susie

    - name: copy_many GCSS susie credible sets
      sources: gs://gwas_catalog_sumstats_susie/credible_set_clean/20260130/*.parquet
      destination: input/gwas/credible_set/gcss_susie

    - name: copy_many finngen study
      sources: gs://finngen_data/r12/study_index/*.parquet
      destination: input/gwas/study/finngen

    - name: copy_many finngen credible sets
      sources: gs://finngen_data/r12/credible_set_datasets/susie/*.parquet
      destination: input/gwas/credible_set/finngen
  ##################################################################################################

  #: GWAS PPP STEP :################################################################################
  gwas_ppp:
    - name: copy_many finngen-UKBB-MVP study
      sources: gs://finngen_ukb_mvp_meta_data/study_index/*.parquet
      destination: input/gwas/study/finngen_ukb_mvp

    - name: copy_many finngen-UKBB-MVP credible sets
      sources: gs://finngen_ukb_mvp_meta_data/credible_sets/*.parquet
      destination: input/gwas/credible_set/finngen_ukb_mvp
  ##################################################################################################

  #: L2G STEP :#####################################################################################
  l2g:
    - name: copy_many l2g gold standard
      sources: gs://genetics-portal-dev-analysis/yt4/2506_release/training_set/20250625_gentropy_paper_v1.json/*.json
      destination: input/l2g/gold_standard.json
  ##################################################################################################

  #: LITERATURE STEP :##############################################################################
  literature:
    - name: copy literature
      source: https://ftp.ebi.ac.uk/pub/databases/pmc/DOI/PMID_PMCID_DOI.csv.gz
      destination: input/literature/PMID_PMCID_DOI.csv.gz

    - name: explode_glob literature dating lut
      glob: gs://otar000-evidence_input/literature_export/2026.03/*.json.gz
      do:
        - name: copy literature dating lut ${match_stem}
          source: ${match_prefix}/${match_path}/${match_stem}.${match_ext}
          destination: input/literature/literature_export/${uuid}.json.gz

  ##################################################################################################

  #: molQTL STEP: ##############################################################################
  molqtl:
    - name: copy_many eQTL Catalogue study
      sources: gs://eqtl_catalogue_data/study_index/*.parquet
      destination: input/molqtl/study/eqtl_catalogue

    - name: copy_many eQTL Catalogue credible sets
      sources: gs://eqtl_catalogue_data/credible_set_datasets/eqtl_catalogue_susie_patched_v2/*.parquet
      destination: input/molqtl/credible_set/eqtl_catalogue

    - name: copy_many UKBB PPP EUR study
      sources: gs://ukb_ppp_eur_data/study_index/*.parquet
      destination: input/molqtl/study/ukb_ppp_eur

    - name: copy_many UKBB PPP EUR credible sets
      sources: gs://ukb_ppp_eur_data/credible_set_clean/20250129/*.parquet
      destination: input/molqtl/credible_set/ukb_ppp_eur
  ##################################################################################################

  #: molQTL PPP STEP: ##############################################################################
  molqtl_ppp:
    - name: explode otar2077 molqtl
      foreach:
        - microglia
        - MAGE_OTAR
        - IBDverse
      do:
        - name: copy_many molqtl ${each} study
          sources: gs://eqtl_catalogue_processed_otar_internal_data/otar2077/${each}/gentropy/study_index/*/*.parquet
          destination: input/molqtl/study/otar2077/${each}

        - name: copy_many molqtl ${each} credible sets
          sources: gs://eqtl_catalogue_processed_otar_internal_data/otar2077/${each}/gentropy/study_locus/*/*.parquet
          destination: input/molqtl/credible_set/otar2077/${each}

    - name: explode eQTLGen molqtl
      foreach:
        - single_cell_eqtl
        - coeqtl
      do:
        - name: copy_many molqtl eQTLGen ${each} study
          sources: gs://eqtlgen_data/${each}/260326/study/*.parquet
          destination: input/molqtl/study/eQTLGen/${each}

        - name: copy_many molqtl eQTLGen ${each} credible sets
          sources: gs://eqtlgen_data/${each}/260326/credible_set/*.parquet
          destination: input/molqtl/credible_set/eQTLGen/${each}

    - name: copy_many molqtl eQTL Catalogue GTEx v10 study
      sources: gs://eqtl_catalogue_processed_otar_internal_data/otar2077/GTEx_v10_majiq/gentropy/output/study_index/*/*.parquet
      destination: input/molqtl/study/GTEx_v10

    - name: copy_many molqtl eQTL Catalogue GTEx v10 credible sets
      sources: gs://eqtl_catalogue_processed_otar_internal_data/otar2077/GTEx_v10_majiq/gentropy/output/study_locus/*/*.parquet
      destination: input/molqtl/credible_set/GTEx_v10
  ##################################################################################################

  #: ONTOMA STEP :##################################################################################
  ontoma:
    - name: copy ot disease curation
      source: https://raw.githubusercontent.com/opentargets/curation/refs/tags/${ot_curation}/mappings/disease/manual_string.tsv
      destination: input/ontoma/ot_disease_curation.tsv

    - name: copy eva clinvar
      source: https://ftp.ebi.ac.uk/pub/databases/eva/ClinVar/latest/eva_clinvar.txt
      destination: input/ontoma/eva_clinvar.txt

    - name: copy clinvar xrefs
      source: https://ftp.ebi.ac.uk/pub/databases/eva/ClinVar/latest/clinvar_xrefs.txt
      destination: input/ontoma/clinvar_xrefs.txt
  ##################################################################################################

  #: OPENFDA STEP :#################################################################################
  # in case we want to test with only a few files, we can use the jq_filter:
  # '.results.drug.event.partitions[0:10] | map(.file)[]'
  openfda:
    - name: copy blacklisted event
      source: https://raw.githubusercontent.com/opentargets/platform-etl-backend/master/src/main/resources/blacklisted_events.txt
      destination: input/openfda/blacklisted_events.txt

    - name: copy openfda archive manifest
      source: https://api.fda.gov/download.json
      destination: input/openfda/fda_events.json
    - name: explode_openfda openfda archive
      requires:
        - copy openfda archive manifest
      source: input/openfda/fda_events.json
      jq_filter: '.results.drug.event.partitions[].file'
      prefix: 'https://download.open.fda.gov/drug/event/'
      do:
        - name: copy fda event ${each}
          source: ${prefix}${each}
          destination: input/openfda/${each}
  ##################################################################################################

  #: OTAR STEP :####################################################################################
  otar:
    - name: spreadsheet_download otar meta spreadsheet
      sheet_id: 1CV_shXJy1ACM09HZBB_-3Nl6l_dfkrA26elMAF0ttHs
      gid: '1179867447'
      destination: input/otar/otar_meta.csv
    - name: spreadsheet_download otar project to efo spreadsheet
      sheet_id: 1CV_shXJy1ACM09HZBB_-3Nl6l_dfkrA26elMAF0ttHs
      gid: '72910172'
      destination: input/otar/otar_project_to_efo.csv
  ##################################################################################################

  #: PHARMACOGENETICS STEP :########################################################################
  pharmacogenetics:
    - name: find_latest clinpgx annotation
      source: gs://otar012-eva/pharmacogenomics/
      scratchpad_key: latest_clinpgx_annotation
    - name: copy clinpgx annotation
      requires:
        - find_latest clinpgx annotation
      source: ${latest_clinpgx_annotation}
      destination: input/pharmacogenetics/clinpgx_annotation.json.gz

    - name: copy ot_curation extracted phenotypes
      source: https://raw.githubusercontent.com/opentargets/curation/refs/tags/${ot_curation}/pharmacogenetics/pgx_phenotypes.json
      destination: input/pharmacogenetics/phenotypes.json
  ##################################################################################################

  #: REACTOME STEP :################################################################################
  reactome:
    - name: copy reactome pathways
      source: https://reactome.org/download/current/ReactomePathways.txt
      destination: input/reactome/ReactomePathways.txt

    - name: copy reactome pathways relation
      source: https://reactome.org/download/current/ReactomePathwaysRelation.txt
      destination: input/reactome/ReactomePathwaysRelation.txt
  ##################################################################################################

  #: SO STEP :######################################################################################
  so:
    - name: copy sequence ontology
      source: https://raw.githubusercontent.com/The-Sequence-Ontology/SO-Ontologies/master/Ontology_Files/so.json
      destination: input/so/so.json
  ##################################################################################################

  #: TARGET STEP :##################################################################################
  target:
    - name: elasticsearch chembl target
      url: https://www.ebi.ac.uk/chembl/elk/es
      destination: input/target/chembl/chembl_target.jsonl
      index: chembl_${chembl_version}_target
      fields:
        - _metadata
        - target_chembl_id
        - target_components
        - target_type
        - pref_name

    - name: copy chemical probes
      source: https://www.probes-drugs.org/media/download/probes/pd_export_${probes_drugs_version}_probes_standardized.xlsx
      destination: input/target/chemicalprobes/probes.xlsx

    - name: copy chemical probes links
      source: https://www.probes-drugs.org/media/download/id_mapping/pd_export_${probes_drugs_version}_links_standardized.csv
      destination: input/target/chemicalprobes/probes_links.csv

    - name: copy ensembl
      source: https://ftp.ensembl.org/pub/release-${ensembl_version}/json/homo_sapiens/homo_sapiens.json
      destination: input/target/ensembl/homo_sapiens.json

    - name: copy gencode
      source: https://ftp.ebi.ac.uk/pub/databases/gencode/Gencode_human/release_${gencode_version}/gencode.v${gencode_version}.annotation.gff3.gz
      destination: input/target/gencode/gencode.gff3.gz

    - name: explode_glob download essentiality files
      glob: 'gs://otar000-evidence_input/Essentiality/DepMap/${depmap_version}/*.csv'
      do:
        - name: copy depmap ${match_stem}
          source: ${uri}
          destination: input/target/gene-essentiality/depmap/${match_stem}.${match_ext}

    - name: copy ot_curation depmap_tissue_mapping
      source: https://raw.githubusercontent.com/opentargets/curation/refs/tags/${ot_curation}/mappings/biosystem/depmap_uberon_mapping.csv
      destination: input/ot_curation/mappings/biosystem/depmap_uberon_mapping.csv

    - name: copy gene name tsv
      source: https://storage.googleapis.com/public-download-files/hgnc/tsv/tsv/hgnc_complete_set.txt
      destination: input/target/genenames/hgnc_complete_set.tsv

    - name: copy gene name json
      source: https://storage.googleapis.com/public-download-files/hgnc/json/json/hgnc_complete_set.json
      destination: input/target/genenames/hgnc_complete_set.json

    - name: copy gnomad
      source: https://storage.googleapis.com/gcp-public-data--gnomad/release/${gnomad_version}/constraint/gnomad.v${gnomad_version}.constraint_metrics.tsv
      destination: input/target/gnomad/gnomad_constraint_metrics.tsv

    - name: copy go annotation human eco gpa
      source: https://ftp.ebi.ac.uk/pub/databases/GO/goa/HUMAN/goa_human.gpa.gz
      destination: input/target/go/goa_human_eco.gpa.gz

    - name: copy go annotation human gaf
      source: https://ftp.ebi.ac.uk/pub/databases/GO/goa/HUMAN/goa_human.gaf.gz
      destination: input/target/go/goa_human.gaf.gz

    - name: copy go annotation human rna gaf
      source: https://ftp.ebi.ac.uk/pub/databases/GO/goa/HUMAN/goa_human_rna.gaf.gz
      destination: input/target/go/goa_human_rna.gaf.gz

    - name: copy go ensembl
      source: https://ftp.ebi.ac.uk/pub/databases/RNAcentral/current_release/id_mapping/database_mappings/ensembl.tsv
      destination: input/target/go/ensembl.tsv

    - name: find_latest cosmic hallmark
      source: gs://otar007-cosmic/hallmarks
      scratchpad_key: 'latest_cosmic_hallmark'
    - name: copy cosmic
      requires:
        - find_latest cosmic hallmark
      source: ${latest_cosmic_hallmark}
      destination: input/target/hallmarks/cosmic-hallmarks.tsv.gz

    - name: explode homology
      foreach:
        - caenorhabditis_elegans
        - canis_lupus_familiaris
        - cavia_porcellus
        - danio_rerio
        - drosophila_melanogaster
        - macaca_mulatta
        - mus_musculus
        - oryctolagus_cuniculus
        - pan_troglodytes
        - rattus_norvegicus
        - sus_scrofa
        - xenopus_tropicalis
        - homo_sapiens
      do:
        - name: copy ${each} gene dictionary
          source: https://ftp.ensembl.org/pub/release-${ensembl_version}/json/${each}/${each}.json
          destination: input/target/homologue/gene_dictionary/${each}.json
        - name: copy ${each} protein homology
          source: https://ftp.ensembl.org/pub/release-${ensembl_version}/tsv/ensembl-compara/homologies/${each}/Compara.${ensembl_version}.protein_default.homologies.tsv.gz
          destination: input/target/homologue/homologies/protein-${each}.tsv.gz
        - name: copy ${each} ncrna homology
          source: https://ftp.ensembl.org/pub/release-${ensembl_version}/tsv/ensembl-compara/homologies/${each}/Compara.${ensembl_version}.ncrna_default.homologies.tsv.gz
          destination: input/target/homologue/homologies/ncrna-${each}.tsv.gz

    - name: copy homology ensembl vertebrates
      source: https://ftp.ensembl.org/pub/release-${ensembl_version}/species_EnsemblVertebrates.txt
      destination: input/target/homologue/species_EnsemblVertebrates.txt

    - name: copy hpa subcellular location
      source: https://www.proteinatlas.org/download/tsv/subcellular_location.tsv.zip
      destination: input/target/hpa/subcellular_location.tsv.zip

    - name: find_latest hpa subcellular location ssl
      source: gs://otar001-core/subcellularLocations
      scratchpad_key: latest_hpa_subcellular_location_ssl
    - name: copy subcellular location
      requires:
        - find_latest hpa subcellular location ssl
      source: ${latest_hpa_subcellular_location_ssl}
      destination: input/target/hpa/subcellular_locations_ssl.tsv

    - name: copy ncbi homosapiens gene info
      source: https://ftp.ncbi.nlm.nih.gov/gene/DATA/GENE_INFO/Mammalia/Homo_sapiens.gene_info.gz
      destination: input/target/ncbi/Homo_sapiens.gene_info.gz

    - name: copy reactome ensembl2reactome
      source: https://reactome.org/download/current/Ensembl2Reactome.txt
      destination: input/target/reactome/Ensembl2Reactome.txt

    - name: explode ot_curation target safety
      foreach:
        - adverse_effects.tsv
        - safety_risks.tsv
        - secondary_pharmacology.json
      do:
        - name: copy ${each}
          source: https://raw.githubusercontent.com/opentargets/curation/refs/tags/${ot_curation}/target_safety/${each}
          destination: input/target/safety/${each}

    - name: find_latest toxcast
      source: gs://otar001-core/TargetSafety/data_files/toxcast
      scratchpad_key: latest_toxcast
    - name: copy toxcast
      requires:
        - find_latest toxcast
      source: ${latest_toxcast}
      destination: input/target/safety/toxcast.tsv

    - name: find_latest aopwiki
      source: gs://otar001-core/TargetSafety/data_files/aopwiki
      scratchpad_key: latest_aopwiki
    - name: copy aopwiki
      requires:
        - find_latest aopwiki
      source: ${latest_aopwiki}
      destination: input/target/safety/aopwiki.json

    - name: find_latest tep
      source: gs://otar001-core/TEPs
      scratchpad_key: latest_tep
    - name: copy tep
      requires:
        - find_latest tep
      source: ${latest_tep}
      destination: input/target/tep/tep.json.gz

    - name: find_latest tractability
      source: gs://otar001-core/Tractability
      scratchpad_key: latest_tractability
    - name: copy tractability
      requires:
        - find_latest tractability
      source: ${latest_tractability}
      destination: input/target/tractability/tractability.tsv

    - name: copy uniprot
      source: https://rest.uniprot.org/uniprotkb/stream?compressed=true&format=txt&query=%28%28reviewed%3Atrue%29%29%20AND%20%28model_organism%3A9606%29
      destination: input/target/uniprot/uniprot.txt.gz

    - name: copy uniprot-ssl
      source: https://rest.uniprot.org/locations/stream?compressed=true&fields=id%2Cname%2Ccategory&format=tsv&query=%28%2A%29
      destination: input/target/uniprot/uniprot-ssl.tsv.gz
  ##################################################################################################

  #: TARGET_PRIORITISATION STEP :###################################################################
  target_prioritisation:
    - name: copy protein atlas
      source: https://www.proteinatlas.org/download/proteinatlas.json.gz
      destination: input/target_prioritisation/proteinatlas.json.gz

    - name: copy uniprot locations
      source: https://rest.uniprot.org/locations/stream?compressed=true&fields=id%2Cname%2Ccategory%2Cgene_ontologies%2Cpart_of%2Cis_a&format=tsv&query=%28%2A%29
      destination: input/target_prioritisation/uniprot_locations.tsv.gz

    - name: copy mouse phenotype scores
      source: https://raw.githubusercontent.com/opentargets/target_engine/main/src/data_flow/phenotypeScores/20230825_mousePheScores.csv
      destination: input/target_prioritisation/mouse_pheno_scores.csv
  ##################################################################################################