Merge branch 'release-16.4.0'

rxu17 · rxu17 · commit 2789838825c3 · 2024-07-23T05:01:58.000-07:00
diff --git a/.github/workflows/build_docs.yml b/.github/workflows/build_docs.yml
@@ -0,0 +1,36 @@
+name: Build Github Pages Docs
+on:
+  push:
+    branches:
+      - main
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: 3.9
+
+      - name: Cache dependences
+        uses: actions/cache@v3
+        with:
+          path: ~/.cache/pip
+          # Unique cache key based on requirements.txt in docs directory
+          key: ${{ runner.os }}-pip-${{ hashFiles('docs/requirements.txt') }}-mkdocs-deps 
+          # Restore partial cache if exact match is not found
+          restore-keys: |
+            ${{ runner.os }}-pip-  
+
+      - name: Install mkdocs and dependencies
+        run: |
+            pip install mkdocs mkdocs-material mkdocstrings mkdocstrings-python mkdocs-open-in-new-tab
+
+      - name: Build and deploy mkdocs site to GitHub Pages
+        env:
+            GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: mkdocs gh-deploy --force
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -5,15 +5,18 @@ name: build
 
 on:
   push:
-    branches:
-      - develop
+    branches: [main, develop, 'GEN*', 'gen*']
 
   pull_request:
 
   release:
     types:
       - created
 
+env:
+  REGISTRY: ghcr.io
+  IMAGE_NAME: ${{ github.repository }}
+
 jobs:
 
   test:
@@ -22,9 +25,9 @@ jobs:
       matrix:
         python-version: [3.8, 3.9]
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Install dependencies
@@ -44,7 +47,7 @@ jobs:
       run: |
         pytest tests/ --cov=genie --cov=genie_registry --cov-report=html
     - name: Upload pytest test results
-      uses: actions/upload-artifact@v2
+      uses: actions/upload-artifact@v4
       with:
         name: pytest-results-${{ matrix.python-version }}
         path: htmlcov
@@ -54,7 +57,7 @@ jobs:
   lint:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - uses: psf/black@stable
 
   deploy:
@@ -64,9 +67,9 @@ jobs:
     permissions:
       id-token: write
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v4
     - name: Set up Python
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: '3.x'
     - name: Install dependencies
@@ -77,3 +80,51 @@ jobs:
       run: python -m build
     - name: Publish to pypi
       uses: pypa/gh-action-pypi-publish@release/v1
+
+
+  build-container:
+    needs: [test, lint]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: read
+      packages: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Setup Docker buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Log into registry ${{ env.REGISTRY }}
+        uses: docker/login-action@v3
+        with:
+          registry: ${{ env.REGISTRY }}
+          username: ${{ github.actor }}
+          password: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Extract Docker metadata
+        id: meta
+        uses: docker/metadata-action@v5
+        with:
+          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
+      
+      - name: Format tags as registry refs
+        id: registry_refs
+        env:
+          TAGS: ${{ steps.meta.outputs.json }}
+        run: |
+          echo tags=$(echo $TAGS | jq '.tags[] | "type=registry,ref=" + . + "_cache"| @text') >> $GITHUB_OUTPUT
+
+      - name: Build and push Docker image
+        id: build-and-push
+        uses: docker/build-push-action@v5
+        if: github.event_name != 'pull_request'
+        with:
+          context: .
+          push: true
+          provenance: false
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+          cache-from: ${{ steps.registry_refs.outputs.tags }},mode=max
+          cache-to: ${{ steps.registry_refs.outputs.tags }},mode=max
diff --git a/.gitignore b/.gitignore
@@ -26,3 +26,6 @@ data_guide.out
 Genie.Rproj
 build/
 dist/
+
+# MkDocs output
+docs_site/
diff --git a/Dockerfile b/Dockerfile
@@ -23,8 +23,8 @@ RUN apt-get update && apt-get install -y --allow-unauthenticated --no-install-re
 		python3-pip \
 		python3-dev \
 		git \
-		r-base \
-		r-base-dev \
+		r-base-core=4.3.3-1.2004.0 \
+		r-base-dev=4.3.3-1.2004.0 \
 		cmake \
 		curl \
 		# synapser client dependencies
diff --git a/README.md b/README.md
@@ -6,11 +6,14 @@
 [![Docker Automated](https://img.shields.io/docker/automated/sagebionetworks/genie.svg?style=for-the-badge&logo=docker)](https://hub.docker.com/r/sagebionetworks/genie)
 [![GitHub CI](https://img.shields.io/github/actions/workflow/status/Sage-Bionetworks/Genie/ci.yml?branch=develop&style=for-the-badge&logo=github)](https://github.com/Sage-Bionetworks/Genie)
 
-
 ## Introduction
 
 This repository documents code used to gather, QC, standardize, and analyze data uploaded by institutes participating in AACR's Project GENIE (Genomics, Evidence, Neoplasia, Information, Exchange).
 
+## Documentation
+
+For more information about the AACR genie repository, [visit the GitHub Pages site.](https://sage-bionetworks.github.io/Genie/)
+
 ## Dependencies
 
 This package contains both R, Python and cli tools.  These are tools or packages you will need, to be able to reproduce these results:
diff --git a/docs/getting_started.md b/docs/getting_started.md
@@ -0,0 +1,74 @@
+# Getting Started
+
+## Dependencies
+
+This package contains both R, Python and cli tools.  These are tools or packages you will need, to be able to reproduce these results:
+
+- Python >=3.8 or <3.10
+    - `pip install -r requirements.txt`
+- [bedtools](https://bedtools.readthedocs.io/en/latest/content/installation.html)
+- R 4.2.2
+    - `renv::install()`
+    - Follow instructions [here](https://r-docs.synapse.org/#note-for-windows-and-mac-users) to install synapser
+- [Java > 8](https://www.java.com/en/download/)
+    - For mac users, it seems to work better to run `brew install java`
+- [wget](https://www.gnu.org/software/wget/)
+    - For mac users, have to run `brew install wget`
+
+## Installation
+
+### PyPi
+
+The [aacrgenie](https://pypi.org/project/aacrgenie/) package is available from PyPI. It can be installed or upgraded with pip.
+
+```
+pip install aacrgenie
+```
+
+### Local
+
+Source code and development versions are available on Github. Installing from source:
+
+```
+git clone https://github.com/Sage-Bionetworks/Genie.git
+cd Genie
+```
+
+Install the packages locally.
+
+```
+pip install -e .
+pip install -r requirements.txt
+pip install -r requirements-dev.txt
+```
+
+You can stay on the main branch to get the latest stable release or check out the develop branch or a tagged revision:
+
+```
+git checkout <branch or tag>
+```
+
+
+## Configuration
+
+Configure the Synapse client to authenticate to Synapse.
+
+1. Create a Synapse [Personal Access token (PAT)](https://help.synapse.org/docs/Managing-Your-Account.2055405596.html#ManagingYourAccount-PersonalAccessTokens).
+1. Add a `~/.synapseConfig` file with the following information:
+
+    ```
+    [authentication]
+    authtoken = <PAT here>
+    ```
+
+1. OR set an environmental variable
+
+    ```
+    export SYNAPSE_AUTH_TOKEN=<PAT here>
+    ```
+
+1. Confirm you can log in to synapse in your terminal.
+
+    ```shell
+    synapse login
+    ```
diff --git a/docs/index.md b/docs/index.md
@@ -0,0 +1,15 @@
+![genie banner](https://raw.githubusercontent.com/Sage-Bionetworks/Genie/master/genie_banner.png)
+
+# AACR Project GENIE
+
+[![PyPi](https://img.shields.io/pypi/v/aacrgenie.svg?style=for-the-badge&label=PyPi&logo=PyPi)](https://pypi.org/project/aacrgenie)
+[![Docker Automated](https://img.shields.io/docker/automated/sagebionetworks/genie.svg?style=for-the-badge&logo=docker)](https://hub.docker.com/r/sagebionetworks/genie)
+[![GitHub CI](https://img.shields.io/github/actions/workflow/status/Sage-Bionetworks/Genie/ci.yml?branch=develop&style=for-the-badge&logo=github)](https://github.com/Sage-Bionetworks/Genie)
+
+## Introduction
+
+This repository documents code used to gather, QC, standardize, and analyze data uploaded by institutes participating in AACR's Project GENIE (Genomics, Evidence, Neoplasia, Information, Exchange).
+
+## Additional Documentation
+
+Visit the [Project Genie Data Portal](https://genie.synapse.org/) to learn more about our data
diff --git a/docs/reference/fileformats/assay.md b/docs/reference/fileformats/assay.md
@@ -0,0 +1 @@
+::: genie_registry.assay
diff --git a/docs/reference/fileformats/bed.md b/docs/reference/fileformats/bed.md
@@ -0,0 +1 @@
+::: genie_registry.bed
diff --git a/docs/reference/fileformats/clinical.md b/docs/reference/fileformats/clinical.md
@@ -0,0 +1 @@
+::: genie_registry.clinical
diff --git a/docs/reference/fileformats/cna.md b/docs/reference/fileformats/cna.md
@@ -0,0 +1 @@
+::: genie_registry.cna
diff --git a/docs/reference/fileformats/fileformat.md b/docs/reference/fileformats/fileformat.md
@@ -0,0 +1 @@
+::: genie.example_filetype_format
diff --git a/docs/reference/fileformats/maf.md b/docs/reference/fileformats/maf.md
@@ -0,0 +1 @@
+::: genie_registry.maf
diff --git a/docs/reference/fileformats/mutations_in_cis.md b/docs/reference/fileformats/mutations_in_cis.md
@@ -0,0 +1 @@
+::: genie_registry.mutationsInCis
diff --git a/docs/reference/fileformats/patient_retraction.md b/docs/reference/fileformats/patient_retraction.md
@@ -0,0 +1 @@
+::: genie_registry.patientRetraction
diff --git a/docs/reference/fileformats/sample_retraction.md b/docs/reference/fileformats/sample_retraction.md
@@ -0,0 +1 @@
+::: genie_registry.sampleRetraction
diff --git a/docs/reference/fileformats/seg.md b/docs/reference/fileformats/seg.md
@@ -0,0 +1 @@
+::: genie_registry.seg
diff --git a/docs/reference/fileformats/structural_variant.md b/docs/reference/fileformats/structural_variant.md
@@ -0,0 +1 @@
+::: genie_registry.structural_variant
diff --git a/docs/reference/fileformats/vcf.md b/docs/reference/fileformats/vcf.md
@@ -0,0 +1 @@
+::: genie_registry.vcf
diff --git a/docs/reference/fileformats/workflow.md b/docs/reference/fileformats/workflow.md
@@ -0,0 +1 @@
+::: genie_registry.workflow
diff --git a/docs/reference/helper_modules/create_case_lists.md b/docs/reference/helper_modules/create_case_lists.md
@@ -0,0 +1 @@
+::: genie.create_case_lists
diff --git a/docs/reference/helper_modules/dashboard_table_updater.md b/docs/reference/helper_modules/dashboard_table_updater.md
@@ -0,0 +1 @@
+::: genie.dashboard_table_updater
diff --git a/docs/reference/helper_modules/extract.md b/docs/reference/helper_modules/extract.md
@@ -0,0 +1 @@
+::: genie.extract
diff --git a/docs/reference/helper_modules/load.md b/docs/reference/helper_modules/load.md
@@ -0,0 +1 @@
+::: genie.load
diff --git a/docs/reference/helper_modules/process_functions.md b/docs/reference/helper_modules/process_functions.md
@@ -0,0 +1 @@
+::: genie.process_functions
diff --git a/docs/reference/helper_modules/transform.md b/docs/reference/helper_modules/transform.md
@@ -0,0 +1 @@
+::: genie.transform
diff --git a/docs/reference/helper_modules/validate.md b/docs/reference/helper_modules/validate.md
@@ -0,0 +1 @@
+::: genie.validate
diff --git a/docs/reference/main_pipeline_commands/consortium_to_public.md b/docs/reference/main_pipeline_commands/consortium_to_public.md
@@ -0,0 +1 @@
+::: bin.consortium_to_public
diff --git a/docs/reference/main_pipeline_commands/database_to_staging.md b/docs/reference/main_pipeline_commands/database_to_staging.md
@@ -0,0 +1 @@
+::: bin.database_to_staging
diff --git a/docs/reference/main_pipeline_commands/input_to_database.md b/docs/reference/main_pipeline_commands/input_to_database.md
@@ -0,0 +1 @@
+::: bin.input_to_database
diff --git a/docs/reference/main_pipeline_steps/consortium_to_public.md b/docs/reference/main_pipeline_steps/consortium_to_public.md
@@ -0,0 +1 @@
+::: genie.consortium_to_public
diff --git a/docs/reference/main_pipeline_steps/database_to_staging.md b/docs/reference/main_pipeline_steps/database_to_staging.md
@@ -0,0 +1 @@
+::: genie.database_to_staging
diff --git a/docs/reference/main_pipeline_steps/input_to_database/input_to_database.md b/docs/reference/main_pipeline_steps/input_to_database/input_to_database.md
@@ -0,0 +1 @@
+::: genie.input_to_database
diff --git a/docs/reference/main_pipeline_steps/input_to_database/process_mutation.md b/docs/reference/main_pipeline_steps/input_to_database/process_mutation.md
@@ -0,0 +1 @@
+::: genie.process_mutation
diff --git a/docs/reference/main_pipeline_steps/input_to_database/to_retract.md b/docs/reference/main_pipeline_steps/input_to_database/to_retract.md
@@ -0,0 +1 @@
+::: genie.toRetract
diff --git a/docs/reference/main_pipeline_steps/input_to_database/write_invalid_reasons.md b/docs/reference/main_pipeline_steps/input_to_database/write_invalid_reasons.md
@@ -0,0 +1 @@
+::: genie.write_invalid_reasons
diff --git a/docs/requirements.txt b/docs/requirements.txt
@@ -0,0 +1,5 @@
+mkdocs<=1.6.0
+mkdocs-material<=9.5.23
+mkdocs-open-in-new-tab<=1.0.3
+mkdocstrings<=0.25.1
+mkdocstrings-python<=1.10.2
diff --git a/docs/tutorials/local_file_validation.md b/docs/tutorials/local_file_validation.md
@@ -0,0 +1,20 @@
+# Local File Validation
+
+One of the features of the `aacrgenie` package is that is provides a local validation tool that GENIE data contributors and install and use to validate their files locally prior to uploading to Synapse.
+
+```
+pip install aacrgenie
+genie -v
+```
+
+This will install all the necessary components for you to run the validator locally on all of your files, including the Synapse client.  Please view the help to see how to run to validator.
+
+```
+genie validate -h
+```
+
+Validate a file
+
+```
+genie validate data_clinical_supp_SAGE.txt SAGE
+```
diff --git a/genie/__init__.py b/genie/__init__.py
diff --git a/genie/consortium_to_public.py b/genie/consortium_to_public.py
diff --git a/genie/database_to_staging.py b/genie/database_to_staging.py
diff --git a/genie/extract.py b/genie/extract.py
diff --git a/genie/load.py b/genie/load.py
diff --git a/genie/process_functions.py b/genie/process_functions.py
diff --git a/genie/process_mutation.py b/genie/process_mutation.py
diff --git a/genie_registry/bed.py b/genie_registry/bed.py
diff --git a/genie_registry/clinical.py b/genie_registry/clinical.py
diff --git a/mkdocs.yml b/mkdocs.yml
diff --git a/setup.cfg b/setup.cfg
diff --git a/tests/conftest.py b/tests/conftest.py
diff --git a/tests/test_bed.py b/tests/test_bed.py
diff --git a/tests/test_clinical.py b/tests/test_clinical.py
diff --git a/tests/test_extract.py b/tests/test_extract.py
diff --git a/tests/test_load.py b/tests/test_load.py
diff --git a/tests/test_process_functions.py b/tests/test_process_functions.py
diff --git a/tests/test_process_mutation.py b/tests/test_process_mutation.py
