Skip to content

Commit 5840c9d

Browse files
committed
First version of dropseq_terra_utils with an email_outputs
1 parent 9c6d273 commit 5840c9d

18 files changed

+1707
-0
lines changed
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: Python Package using Conda
2+
3+
on:
4+
push:
5+
branches: [ "master" ]
6+
pull_request:
7+
branches: [ "master" ]
8+
paths:
9+
- 'src/python/dropseq_terra_utils/**'
10+
11+
12+
jobs:
13+
build-linux:
14+
runs-on: ubuntu-latest
15+
strategy:
16+
max-parallel: 5
17+
18+
steps:
19+
- uses: actions/checkout@v4
20+
- name: Set up Python
21+
uses: actions/setup-python@v3
22+
with:
23+
python-version: '3.12'
24+
- name: Add conda to system path
25+
run: |
26+
# $CONDA is an environment variable pointing to the root of the miniconda directory
27+
# https://github.com/actions/runner-images/blob/ff9acc6/images/ubuntu/Ubuntu2204-Readme.md#environment-variables
28+
echo $CONDA/bin >> $GITHUB_PATH
29+
- name: Modify the conda configuration to use conda-forge
30+
run: |
31+
conda config --add channels conda-forge
32+
conda config --remove channels defaults || true
33+
34+
# There are many issues filed regarding:
35+
# > warning libmamba Problem type not implemented SOLVER_RULE_STRICT_REPO_PRIORITY
36+
# when using the recommended strict channel priority and creating environments from YAML files,
37+
# but the FOSS conda/mamba community has not been able to fix the issue.
38+
# https://github.com/mamba-org/mamba/issues/2810#issuecomment-1910011988
39+
conda config --set channel_priority flexible
40+
41+
# Disable lock files to avoid bugs in mamba and libmamba
42+
# https://mamba.readthedocs.io/en/latest/user_guide/troubleshooting.html#hangs-during-package-installation-on-nfs-network-file-systems
43+
# https://github.com/mamba-org/mamba/issues/1993#issuecomment-1268397084
44+
echo "use_lockfiles: false" >> ~/.mambarc
45+
- name: Install dependencies
46+
run: |
47+
cd src/python/dropseq_terra_utils
48+
conda env update --file environment.yml --name base
49+
- name: Lint with flake8
50+
run: |
51+
cd src/python/dropseq_terra_utils
52+
53+
# Explicitly using the classic solver to avoid:
54+
# a) "libarchive.so.20: cannot open shared object", and
55+
# b) switching from the miniconda installed in the "ubuntu-latest" image
56+
# https://github.com/actions/runner-images/blob/ff9acc6/images/ubuntu/Ubuntu2204-Readme.md#package-management
57+
# c) upgrading to a working version using the below takes minutes while the tests themselves take seconds
58+
# https://stackoverflow.com/questions/77617946/solve-conda-libmamba-solver-libarchive-so-19-error-after-updating-conda-to-23#answer-78293971
59+
conda install --solver=classic --override-channels --channel conda-forge flake8
60+
61+
# stop the build if there are Python syntax errors or undefined names
62+
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
63+
64+
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
65+
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
66+
- name: Test with unittest
67+
run: |
68+
cd src/python/dropseq_terra_utils
69+
PYTHONPATH=src python -m unittest discover -s tests

.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -10,3 +10,4 @@ jacoco_reports
1010

1111
.Rproj.user
1212
.Rhistory
13+
.DS_Store

src/python/.gitignore

+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
dist
2+
__pycache__
+19
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
Copyright 2025 Broad Institute
2+
3+
Permission is hereby granted, free of charge, to any person obtaining a copy
4+
of this software and associated documentation files (the "Software"), to deal
5+
in the Software without restriction, including without limitation the rights
6+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
7+
copies of the Software, and to permit persons to whom the Software is
8+
furnished to do so, subject to the following conditions:
9+
10+
The above copyright notice and this permission notice shall be included in all
11+
copies or substantial portions of the Software.
12+
13+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
14+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
15+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
16+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
17+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
18+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
19+
SOFTWARE.
+13
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
# Simple python tools for interacting with workflows in Terra.bio
2+
3+
## Installation
4+
5+
Requires python >= 3.12
6+
7+
```
8+
pip install 'git+https://github.com/broadinstitute/Drop-seq.git#egg=dropseq_terra_utils&subdirectory=src/python/dropseq_terra_utils'
9+
```
10+
11+
## Usage
12+
13+
Run `dropseq_terra_utils -h` for usage information.
+22
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env python3
2+
# MIT License
3+
#
4+
# Copyright 2025 Broad Institute
5+
#
6+
# Permission is hereby granted, free of charge, to any person obtaining a copy
7+
# of this software and associated documentation files (the "Software"), to deal
8+
# in the Software without restriction, including without limitation the rights
9+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
# copies of the Software, and to permit persons to whom the Software is
11+
# furnished to do so, subject to the following conditions:
12+
#
13+
# The above copyright notice and this permission notice shall be included in all
14+
# copies or substantial portions of the Software.
15+
#
16+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
# SOFTWARE.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
# MIT License
2+
#
3+
# Copyright 2025 Broad Institute
4+
#
5+
# Permission is hereby granted, free of charge, to any person obtaining a copy
6+
# of this software and associated documentation files (the "Software"), to deal
7+
# in the Software without restriction, including without limitation the rights
8+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
# copies of the Software, and to permit persons to whom the Software is
10+
# furnished to do so, subject to the following conditions:
11+
#
12+
# The above copyright notice and this permission notice shall be included in all
13+
# copies or substantial portions of the Software.
14+
#
15+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
# SOFTWARE.
22+
23+
name: dropseq_terra_utils
24+
channels:
25+
- conda-forge
26+
- nodefaults
27+
dependencies:
28+
- google-auth
29+
- google-cloud-storage
30+
- pandas
31+
- python=3.12
32+
- python-dateutil
33+
- requests
34+
- urllib3
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,51 @@
1+
# MIT License
2+
#
3+
# Copyright 2025 Broad Institute
4+
#
5+
# Permission is hereby granted, free of charge, to any person obtaining a copy
6+
# of this software and associated documentation files (the "Software"), to deal
7+
# in the Software without restriction, including without limitation the rights
8+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9+
# copies of the Software, and to permit persons to whom the Software is
10+
# furnished to do so, subject to the following conditions:
11+
#
12+
# The above copyright notice and this permission notice shall be included in all
13+
# copies or substantial portions of the Software.
14+
#
15+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21+
# SOFTWARE.
22+
23+
[build-system]
24+
requires = ["hatchling"]
25+
build-backend = "hatchling.build"
26+
[project]
27+
name = "dropseq_terra_utils"
28+
version = "0.0.1"
29+
dependencies = [
30+
"google-auth",
31+
"google-cloud-storage",
32+
"pandas",
33+
"python-dateutil",
34+
"requests",
35+
"urllib3",
36+
]
37+
description = "Simple python tools for interacting with Drop-seq workflows in Terra.bio."
38+
readme = "README.md"
39+
requires-python = ">=3.12"
40+
classifiers = [
41+
"Programming Language :: Python :: 3",
42+
"License :: OSI Approved :: MIT License",
43+
"Operating System :: OS Independent",
44+
]
45+
46+
[project.urls]
47+
Homepage = "https://github.com/broadinstitute/Drop-seq/"
48+
Issues = "https://github.com/broadinstitute/Drop-seq/issues"
49+
50+
[project.scripts]
51+
dropseq_terra_utils = "dropseq_terra_utils.cli:main"
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env python3
2+
# MIT License
3+
#
4+
# Copyright 2025 Broad Institute
5+
#
6+
# Permission is hereby granted, free of charge, to any person obtaining a copy
7+
# of this software and associated documentation files (the "Software"), to deal
8+
# in the Software without restriction, including without limitation the rights
9+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
# copies of the Software, and to permit persons to whom the Software is
11+
# furnished to do so, subject to the following conditions:
12+
#
13+
# The above copyright notice and this permission notice shall be included in all
14+
# copies or substantial portions of the Software.
15+
#
16+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
# SOFTWARE.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
#!/usr/bin/env python3
2+
# MIT License
3+
#
4+
# Copyright 2025 Broad Institute
5+
#
6+
# Permission is hereby granted, free of charge, to any person obtaining a copy
7+
# of this software and associated documentation files (the "Software"), to deal
8+
# in the Software without restriction, including without limitation the rights
9+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
# copies of the Software, and to permit persons to whom the Software is
11+
# furnished to do so, subject to the following conditions:
12+
#
13+
# The above copyright notice and this permission notice shall be included in all
14+
# copies or substantial portions of the Software.
15+
#
16+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
# SOFTWARE.
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
#!/usr/bin/env python3
2+
# MIT License
3+
#
4+
# Copyright 2025 Broad Institute
5+
#
6+
# Permission is hereby granted, free of charge, to any person obtaining a copy
7+
# of this software and associated documentation files (the "Software"), to deal
8+
# in the Software without restriction, including without limitation the rights
9+
# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
10+
# copies of the Software, and to permit persons to whom the Software is
11+
# furnished to do so, subject to the following conditions:
12+
#
13+
# The above copyright notice and this permission notice shall be included in all
14+
# copies or substantial portions of the Software.
15+
#
16+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
17+
# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
18+
# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
19+
# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
20+
# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
21+
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
22+
# SOFTWARE.
23+
"""
24+
Simple python tools for interacting with workflows in Terra.bio.
25+
"""
26+
import argparse
27+
import logging
28+
import sys
29+
from pathlib import Path
30+
31+
try:
32+
from . import email_outputs
33+
except ImportError:
34+
import email_outputs
35+
36+
# I cannot believe I need to do this to cause logger to write to stderr.
37+
logging.basicConfig(
38+
level=logging.INFO, # Set the logging level (DEBUG, INFO, WARNING, ERROR, CRITICAL)
39+
format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
40+
handlers=[logging.StreamHandler()] # StreamHandler writes to sys.stderr by default
41+
)
42+
logger = logging.getLogger(Path(__file__).stem)
43+
44+
dctLogLevel = {
45+
"DEBUG": logging.DEBUG,
46+
"INFO": logging.INFO,
47+
"WARNING": logging.WARNING,
48+
"ERROR": logging.ERROR,
49+
"CRITICAL": logging.CRITICAL
50+
}
51+
52+
53+
def main(args=None):
54+
parser = argparse.ArgumentParser(prog="dropseq_terra_utils", description=__doc__)
55+
parser.add_argument("--log-level", "-l", default="INFO", choices=dctLogLevel.keys(),
56+
help="Set the logging level. (default: %(default)s)")
57+
subparsers = parser.add_subparsers(
58+
title="sub-commands",
59+
description="valid commands",
60+
dest="tool")
61+
email_outputs.add_subparser(subparsers)
62+
63+
if args is None:
64+
args = sys.argv[1:]
65+
if len(args) == 0:
66+
parser.print_help()
67+
return 1
68+
else:
69+
options = parser.parse_args(args)
70+
logger.setLevel(dctLogLevel[options.log_level])
71+
if options.tool == "email_outputs":
72+
return email_outputs.main(options)
73+
else:
74+
# should be unpossible because parse_args will complain
75+
raise ValueError(f"Unrecognized tool: {options.tool}")
76+
77+
78+
if __name__ == "__main__":
79+
sys.exit(main())

0 commit comments

Comments
 (0)