Skip to content

Commit 542106f

Browse files
jbohnslavjbohnslav
committed
script to build and test docker images
1 parent a77c77c commit 542106f

File tree

12 files changed

+215
-93
lines changed

12 files changed

+215
-93
lines changed

‎.github/workflows/gpu.yml‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -33,12 +33,10 @@
3333
# run: |
3434
# pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 -f https://download.pytorch.org/whl/torch_stable.html
3535
#
36-
# - name: Install package and test dependencies
36+
# - name: Install dependencies
3737
# run: |
38-
# python -m pip install --upgrade "pip<24.0"
39-
# pip install -r requirements.txt
40-
# pip install pytest pytest-cov
41-
# python setup.py develop
38+
# python -m pip install --upgrade pip
39+
# pip install -e .
4240
#
4341
# - name: Setup test data
4442
# run: |

‎.github/workflows/main.yml‎

Lines changed: 3 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -48,12 +48,10 @@ jobs:
4848
run: |
4949
pip install torch torchvision --index-url https://download.pytorch.org/whl/cpu
5050
51-
- name: Install package and test dependencies
51+
- name: Install dependencies
5252
run: |
53-
python -m pip install --upgrade "pip<24.0"
54-
pip install -r requirements.txt
55-
pip install pytest pytest-cov
56-
python setup.py develop
53+
python -m pip install --upgrade pip
54+
pip install .
5755
5856
- name: Setup test data
5957
run: |

‎docker/Dockerfile-full‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -31,10 +31,9 @@ RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39
3131

3232
# install
3333
RUN conda install python=3.7 -y
34-
RUN pip install setuptools --upgrade && pip install --upgrade "pip<24.0"
35-
RUN pip install torch==1.11.0+cu115 torchvision==0.12.0+cu115 -f https://download.pytorch.org/whl/torch_stable.html
36-
34+
RUN pip install --upgrade pip
35+
# Install deepethogram with dev dependencies
3736
ADD . /app/deepethogram
3837
WORKDIR /app/deepethogram
3938
ENV DEG_VERSION='full'
40-
RUN pip install -e .
39+
RUN pip install -e ".[dev]"

‎docker/Dockerfile-gui‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -35,13 +35,13 @@ RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39
3535

3636
# install
3737
RUN conda install python=3.7 -y
38-
RUN pip install setuptools --upgrade && pip install --upgrade pip
38+
RUN pip install --upgrade pip
3939

4040
# TODO: REFACTOR CODE SO IT'S POSSIBLE TO RUN GUI WITHOUT TORCH
4141
RUN conda install pytorch cpuonly -c pytorch
4242

43-
# # needed for pandas for some reason
43+
# Install deepethogram with dev dependencies
4444
ADD . /app/deepethogram
4545
WORKDIR /app/deepethogram
4646
ENV DEG_VERSION='gui'
47-
RUN pip install -e .
47+
RUN pip install -e ".[dev]"

‎docker/Dockerfile-headless‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -33,11 +33,11 @@ RUN curl -sLo ~/miniconda.sh https://repo.continuum.io/miniconda/Miniconda3-py39
3333

3434
# install
3535
RUN conda install python=3.7 -y
36-
RUN pip install setuptools --upgrade && pip install --upgrade pip
37-
RUN conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
36+
RUN pip install --upgrade pip
37+
# RUN conda install pytorch torchvision torchaudio cudatoolkit=11.3 -c pytorch
3838

39-
# # needed for pandas for some reason
39+
# Install deepethogram with dev dependencies
4040
ADD . /app/deepethogram
4141
WORKDIR /app/deepethogram
4242
ENV DEG_VERSION='headless'
43-
RUN pip install -e .
43+
RUN pip install -e ".[dev]"

‎docker/build_and_test.sh‎

Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
#!/bin/bash
2+
3+
set -e # Exit on any error
4+
5+
# Colors for output
6+
GREEN='\033[0;32m'
7+
RED='\033[0;31m'
8+
NC='\033[0m' # No Color
9+
BLUE='\033[0;34m'
10+
YELLOW='\033[1;33m'
11+
12+
# Function to print section headers
13+
print_header() {
14+
echo -e "\n${BLUE}=== $1 ===${NC}\n"
15+
}
16+
17+
# Function to echo command before running
18+
echo_run() {
19+
echo -e "${YELLOW}Running: $@${NC}"
20+
"$@"
21+
}
22+
23+
# Function to build an image
24+
build_image() {
25+
local type=$1
26+
print_header "Building $type image"
27+
echo_run docker build -t deepethogram:$type -f docker/Dockerfile-$type .
28+
}
29+
30+
# Function to verify GPU in container
31+
verify_gpu() {
32+
local gpu_flag=$1
33+
local type=$2
34+
echo "Verifying GPU access in container..."
35+
echo -e "${YELLOW}Running: docker run $gpu_flag --rm deepethogram:$type nvidia-smi${NC}"
36+
if ! docker run $gpu_flag --rm deepethogram:$type nvidia-smi; then
37+
echo -e "${RED}Failed to access GPU in container${NC}"
38+
return 1
39+
fi
40+
echo -e "${YELLOW}Running: docker run $gpu_flag --rm deepethogram:$type python -c \"import torch; print('CUDA available:', torch.cuda.is_available())\"${NC}"
41+
if ! docker run $gpu_flag --rm deepethogram:$type python -c "import torch; print('CUDA available:', torch.cuda.is_available())" | grep -q "CUDA available: True"; then
42+
echo -e "${RED}Failed to access GPU through PyTorch${NC}"
43+
return 1
44+
fi
45+
return 0
46+
}
47+
48+
# Function to run tests in container
49+
test_container() {
50+
local type=$1
51+
local gpu_flag=$2
52+
local has_gpu=$3
53+
54+
print_header "Testing $type container"
55+
56+
# Test basic import
57+
echo "Testing Python import..."
58+
echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type python -c \"import deepethogram\"${NC}"
59+
docker run $gpu_flag -it deepethogram:$type python -c "import deepethogram" && \
60+
echo -e "${GREEN}✓ Import test passed${NC}" || \
61+
(echo -e "${RED}✗ Import test failed${NC}" && exit 1)
62+
63+
# For containers that should support tests
64+
if [ "$type" = "full" ] || [ "$type" = "headless" ]; then
65+
echo "Running CPU tests..."
66+
echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type pytest -v -m \"not gpu\" tests/${NC}"
67+
docker run $gpu_flag -it deepethogram:$type pytest -v -m "not gpu" tests/ && \
68+
echo -e "${GREEN}✓ CPU tests passed${NC}" || \
69+
(echo -e "${RED}✗ CPU tests failed${NC}" && exit 1)
70+
71+
# Run GPU tests if GPU is available
72+
if [ "$has_gpu" = true ] && [ "$type" != "gui" ]; then
73+
echo "Running GPU tests..."
74+
# First verify CUDA is accessible
75+
echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type python -c \"import torch; assert torch.cuda.is_available(), 'CUDA not available'; print('CUDA is available')\"${NC}"
76+
docker run $gpu_flag -it deepethogram:$type python -c "import torch; assert torch.cuda.is_available(), 'CUDA not available'; print('CUDA is available')"
77+
# Run the actual GPU tests
78+
echo -e "${YELLOW}Running: docker run $gpu_flag -it deepethogram:$type bash -c \"export CUDA_VISIBLE_DEVICES=0 && pytest -v -m gpu tests/\"${NC}"
79+
docker run $gpu_flag -it deepethogram:$type \
80+
bash -c "export CUDA_VISIBLE_DEVICES=0 && pytest -v -m gpu tests/" && \
81+
echo -e "${GREEN}✓ GPU tests passed${NC}" || \
82+
(echo -e "${RED}✗ GPU tests failed${NC}" && exit 1)
83+
fi
84+
fi
85+
86+
# For containers that should support GUI
87+
if [ "$type" = "full" ] || [ "$type" = "gui" ]; then
88+
echo "Testing GUI import..."
89+
echo -e "${YELLOW}Running: docker run $gpu_flag -e DISPLAY=$DISPLAY -v /tmp/.X11-unix:/tmp/.X11-unix:rw -it deepethogram:$type python -c \"from deepethogram.gui import main\"${NC}"
90+
docker run $gpu_flag -e DISPLAY=$DISPLAY \
91+
-v /tmp/.X11-unix:/tmp/.X11-unix:rw \
92+
-it deepethogram:$type python -c "from deepethogram.gui import main" && \
93+
echo -e "${GREEN}✓ GUI import test passed${NC}" || \
94+
(echo -e "${RED}✗ GUI import test failed${NC}" && exit 1)
95+
fi
96+
}
97+
98+
# Main execution
99+
main() {
100+
# Ensure we're in the project root
101+
if [[ ! -f "pyproject.toml" ]]; then
102+
echo -e "${RED}Error: Must run from project root directory (where pyproject.toml is located)${NC}"
103+
exit 1
104+
fi
105+
106+
# Check if GPU is available by testing nvidia-smi
107+
local has_gpu=false
108+
if command -v nvidia-smi &> /dev/null && nvidia-smi &> /dev/null; then
109+
GPU_FLAG="--gpus all"
110+
has_gpu=true
111+
echo -e "${GREEN}NVIDIA GPU detected, will use GPUs and run GPU tests${NC}"
112+
else
113+
GPU_FLAG=""
114+
echo -e "${RED}No NVIDIA GPU detected, running without GPU${NC}"
115+
fi
116+
117+
# Build and test each image type
118+
for type in "headless" "gui" "full"; do
119+
build_image $type
120+
# Verify GPU access after building if we have a GPU
121+
if [ "$has_gpu" = true ] && [ "$type" != "gui" ]; then
122+
if ! verify_gpu "$GPU_FLAG" "$type"; then
123+
echo -e "${RED}GPU detected on host but not accessible in container. Please check nvidia-docker installation.${NC}"
124+
exit 1
125+
fi
126+
fi
127+
test_container $type "$GPU_FLAG" $has_gpu
128+
done
129+
130+
print_header "All builds and tests completed successfully!"
131+
}
132+
133+
# Execute main function
134+
main

‎docs/installation.md‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@
1414
* `conda env create -f environment.yml`
1515
* Be prepared to wait a long time!! On mechanical hard drives, this may take 5-10 minutes (or more). Interrupting here will cause installation to fail.
1616
* `conda activate deg`
17-
* `python setup.py develop`
17+
* `pip install -e .`
1818

1919
### Installing Anaconda
2020
For instructions on installing anaconda,
@@ -62,3 +62,9 @@ environment with `conda create --name deg python=3.8` before installation.
6262
* This is an issue where Shiboken and PySide2 are not playing nicely together. Please `pip uninstall pyside2` and `conda remove pyside2`. Don't manually install these packages; instead, let DeepEthogram install it for you via pip. Therefore, `pip uninstall deepethogram` and `pip install deepethogram`.
6363
* `qt.qpa.plugin: Could not load the Qt platform plugin "xcb" in ".../python3.8/site-packages/cv2/qt/plugins" even though it was found. This application failed to start because no Qt platform plugin could be initialized. Reinstalling the application may fix this problem.`
6464
* This is an issue with a recent version of `opencv-python` not working well with Qt. Please do `pip install --force-reinstall opencv-python-headless==4.1.2.30`
65+
66+
# Beta: Using UV
67+
68+
* install astral's UV on your system: `pip install uv`
69+
* `uv venv --python 3.7`: make a virtual environment
70+
* `uv pip install -e .`

‎environment.yml‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,9 +4,10 @@ channels:
44
- conda-forge
55
- defaults
66
dependencies:
7+
- python=3.7
8+
- pytorch
9+
- torchvision
710
- pip
8-
- conda-forge::pyside2=5.13.2
9-
- python>3.7, <3.9
10-
- pytorch::pytorch
1111
- pip:
12-
- -r requirements.txt
12+
- -e .
13+
- conda-forge::pyside2=5.13.2

‎pyproject.toml‎

Lines changed: 50 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,53 @@
1+
[build-system]
2+
requires = ["setuptools>=42", "wheel"]
3+
build-backend = "setuptools.build_meta"
4+
5+
[project]
6+
name = "deepethogram"
7+
version = "0.2.0"
8+
description = "Temporal action detection for biology"
9+
readme = "README.md"
10+
authors = [
11+
{name = "Jim Bohnslav", email = "[email protected]"},
12+
]
13+
requires-python = ">=3.7,<3.8"
14+
classifiers = [
15+
"Programming Language :: Python :: 3",
16+
"Operating System :: OS Independent",
17+
]
18+
dependencies = [
19+
"chardet<4.0",
20+
"h5py",
21+
"kornia>=0.5",
22+
"matplotlib",
23+
"numpy",
24+
"omegaconf>=2",
25+
"opencv-python-headless",
26+
"opencv-transforms",
27+
"pandas<1.4",
28+
"PySide2==5.13.2",
29+
"scikit-learn<1.1",
30+
"scipy<1.8",
31+
"tqdm",
32+
"vidio",
33+
"pytorch_lightning==1.6.5",
34+
]
35+
36+
[project.optional-dependencies]
37+
dev = [
38+
"ruff>=0.1.0",
39+
"pre-commit>=2.20.0,<3.0.0",
40+
"pytest",
41+
"pytest-cov",
42+
"gdown",
43+
]
44+
45+
[project.scripts]
46+
deepethogram = "deepethogram.gui.main:entry"
47+
48+
[tool.setuptools]
49+
packages = ["deepethogram"]
50+
151
[tool.ruff]
252
# Python version compatibility
353
target-version = "py37"
@@ -33,7 +83,6 @@ exclude = [
3383
]
3484

3585
[tool.ruff.lint]
36-
3786
# Allow autofix for all enabled rules (when `--fix`) is provided.
3887
fixable = ["ALL"]
3988
unfixable = []

‎requirements.txt‎

Lines changed: 0 additions & 20 deletions
This file was deleted.

0 commit comments

Comments
 (0)