Skip to content

Commit 9cbc62b

Browse files
kaylodekaylode
authored
Merge pull request #50 from kaylode/dev
Another big update
2 parents c4623d9 + 65f3cd0 commit 9cbc62b

File tree

125 files changed

+1739
-1982
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

125 files changed

+1739
-1982
lines changed

‎.dvc/.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
/config.local
2+
/tmp
3+
/cache

‎.dvc/config‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
[core]
2+
remote = gdrive
3+
['remote "gdrive"']
4+
url = gdrive://155tBftKDG8VSAWojOWT3exax3hz0Xuwg

‎.dvcignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# Add patterns of files dvc should ignore, which could improve
2+
# the performance. Learn more at
3+
# https://dvc.org/doc/user-guide/dvcignore

‎.github/workflows/clf.yml‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@ on:
44
branches:
55
- master
66
- 'V**'
7-
pull_request:
8-
branches:
9-
- master
10-
- 'V**'
7+
- dev
8+
env:
9+
GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
10+
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
11+
1112
jobs:
1213
build:
1314
runs-on: ubuntu-latest
@@ -18,13 +19,40 @@ jobs:
1819
with:
1920
python-version: '3.8'
2021
architecture: 'x64'
22+
- uses: actions/setup-node@v3
23+
with:
24+
node-version: '16'
25+
- uses: iterative/setup-cml@v1
2126
- name: Display Python version
2227
run: python -c "import sys; print(sys.version)"
2328
- name: Install dependencies
2429
run: |
2530
python -m pip install --upgrade pip
2631
pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu
27-
pip install -e '.[cv,cv_classification]'
32+
pip install -e '.[dev,cv,cv_classification]'
33+
pip install dvc dvc-gdrive
34+
npm install -g markdown-table-cli
35+
- name: Download data
36+
run: |
37+
echo "$GDRIVE_CREDENTIALS_DATA" > credentials.json
38+
dvc remote modify gdrive --local gdrive_user_credentials_file ./credentials.json
39+
dvc pull
40+
ls -la samples
2841
- name: Test with pytest
2942
run: |
3043
pytest tests/classification --capture=no
44+
- name: Display report
45+
run: |
46+
echo "# Classification Report" > report.md
47+
echo "#### Metrics" >> report.md
48+
cat runs/pytest_clf/Validation/metrics.json | md-table >> report.md
49+
echo "#### Confusion Matrix" >> report.md
50+
echo "![Confusion Matrix](runs/pytest_clf/Validation/cfm.png)" >> report.md
51+
echo "#### Errorcases" >> report.md
52+
echo "![Confusion Matrix](runs/pytest_clf/Validation/errorcases.png)" >> report.md
53+
echo "------------------" >> report.md
54+
echo "#### Hyperparameters Tuning" >> report.md
55+
echo "Leaderboard" >> report.md
56+
cat runs/optuna/clf/overview/leaderboard.json | md-table >> report.md
57+
echo >> report.md
58+
cml comment create report.md

‎.github/workflows/docker.yml‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
name: autobuild_docker
2+
on:
3+
pull_request:
4+
branches:
5+
- master
6+
- 'V**'
7+
- dev
8+
9+
env:
10+
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
11+
GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
12+
DOCKERHUB_TAG: kaylode/theseus:latest
13+
14+
jobs:
15+
run:
16+
runs-on: ubuntu-latest
17+
steps:
18+
- uses: actions/checkout@v3
19+
- name: Set up Python 3.8
20+
uses: actions/setup-python@v4
21+
with:
22+
python-version: '3.8'
23+
architecture: 'x64'
24+
- uses: iterative/setup-cml@v1
25+
- uses: iterative/setup-dvc@v1
26+
- name: Set up QEMU
27+
uses: docker/setup-qemu-action@v2
28+
- name: Set up Docker Buildx
29+
uses: docker/setup-buildx-action@v2
30+
- name: Create credentials file
31+
run: |
32+
echo "$GDRIVE_CREDENTIALS_DATA" > credentials.json
33+
- name: Build Dockerfile
34+
uses: docker/build-push-action@v4
35+
with:
36+
context: .
37+
tags: ${{ env.DOCKERHUB_TAG }}
38+
outputs: type=docker,dest=${{ github.workspace }}/image.tar
39+
secret-files: |
40+
credentials=./credentials.json
41+
42+
- name: Free up space
43+
run: |
44+
docker system prune -f
45+
sudo rm -rf "/usr/local/share/boost"
46+
sudo rm -rf "$AGENT_TOOLSDIRECTORY"
47+
48+
- name: Docker name
49+
run: |
50+
docker load -i ${{ github.workspace }}/image.tar
51+
docker image inspect ${{ env.DOCKERHUB_TAG }}
52+
53+
- name: Create outputs folder
54+
run: |
55+
mkdir -p outputs
56+
57+
- name: Test Docker
58+
uses: addnab/docker-run-action@v3
59+
with:
60+
image: ${{ env.DOCKERHUB_TAG }}
61+
options: --rm -v ${{ github.workspace }}/outputs/:/workspace/runs/
62+
run: |
63+
pytest tests --capture=no
64+
65+
- name: where am I
66+
run: |
67+
pwd
68+
ls -la
69+
ls -la outputs

‎.github/workflows/segm.yml‎

Lines changed: 27 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@ on:
44
branches:
55
- master
66
- 'V**'
7-
pull_request:
8-
branches:
9-
- master
10-
- 'V**'
7+
- dev
8+
env:
9+
GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
10+
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
11+
1112
jobs:
1213
build:
1314
runs-on: ubuntu-latest
@@ -18,13 +19,34 @@ jobs:
1819
with:
1920
python-version: '3.8'
2021
architecture: 'x64'
22+
- uses: actions/setup-node@v3
23+
with:
24+
node-version: '16'
25+
- uses: iterative/setup-cml@v1
2126
- name: Display Python version
2227
run: python -c "import sys; print(sys.version)"
2328
- name: Install dependencies
2429
run: |
2530
python -m pip install --upgrade pip
2631
pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu
27-
pip install -e '.[cv,cv_semantic]'
32+
pip install -e '.[dev,cv,cv_semantic]'
33+
pip install dvc dvc-gdrive
34+
npm install -g markdown-table-cli
35+
- name: Download data
36+
run: |
37+
echo "$GDRIVE_CREDENTIALS_DATA" > credentials.json
38+
dvc remote modify gdrive --local gdrive_user_credentials_file ./credentials.json
39+
dvc pull
40+
ls -la samples
2841
- name: Test with pytest
2942
run: |
3043
pytest tests/semantic --capture=no
44+
- name: Display report
45+
run: |
46+
echo "# Semantic Report" > report.md
47+
echo "#### Metrics" >> report.md
48+
cat runs/pytest_segm/Validation/metrics.json | md-table >> report.md
49+
echo >> report.md
50+
echo "#### Prediction" >> report.md
51+
echo "![Prediction](runs/pytest_segm/Validation/prediction.png)" >> report.md
52+
cml comment create report.md

‎.github/workflows/tablr.yml‎

Lines changed: 41 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -4,10 +4,11 @@ on:
44
branches:
55
- master
66
- 'V**'
7-
pull_request:
8-
branches:
9-
- master
10-
- 'V**'
7+
- dev
8+
env:
9+
GDRIVE_CREDENTIALS_DATA: ${{ secrets.GDRIVE_CREDENTIALS_DATA }}
10+
REPO_TOKEN: ${{ secrets.GITHUB_TOKEN }}
11+
1112
jobs:
1213
build:
1314
runs-on: ubuntu-latest
@@ -18,13 +19,48 @@ jobs:
1819
with:
1920
python-version: '3.8'
2021
architecture: 'x64'
22+
- uses: actions/setup-node@v3
23+
with:
24+
node-version: '16'
25+
- uses: iterative/setup-cml@v1
2126
- name: Display Python version
2227
run: python -c "import sys; print(sys.version)"
2328
- name: Install dependencies
2429
run: |
2530
python -m pip install --upgrade pip
2631
pip install torch torchvision --extra-index-url https://download.pytorch.org/whl/cpu
27-
pip install -e '.[tabular, tabular_classification]'
32+
pip install -e '.[dev,tabular,tabular_classification]'
33+
pip install dvc dvc-gdrive
34+
npm install -g markdown-table-cli
35+
- name: Download data
36+
run: |
37+
echo "$GDRIVE_CREDENTIALS_DATA" > credentials.json
38+
dvc remote modify gdrive --local gdrive_user_credentials_file ./credentials.json
39+
dvc pull
40+
ls -la samples
2841
- name: Test with pytest
2942
run: |
3043
pytest tests/tabular --capture=no
44+
- name: Display report
45+
run: |
46+
echo "# Tabular Classification Report" > report.md
47+
echo "#### Metrics" >> report.md
48+
cat runs/pytest_tablr/Validation/metrics.json | md-table >> report.md
49+
echo "#### SHAP train" >> report.md
50+
echo "![SHAP](runs/pytest_tablr/Importance/SHAP/train.png)" >> report.md
51+
echo "#### SHAP val" >> report.md
52+
echo "![SHAP](runs/pytest_tablr/Importance/SHAP/val.png)" >> report.md
53+
echo "------------------" >> report.md
54+
55+
echo "#### Hyperparameters Tuning" >> report.md
56+
echo "Leaderboard" >> report.md
57+
cat runs/optuna/tablr/overview/leaderboard.json | md-table >> report.md
58+
echo >> report.md
59+
echo "Figures" >> report.md
60+
echo "![History](runs/optuna/tablr/overview/history.png)" >> report.md
61+
echo "![Contour plot](runs/optuna/tablr/overview/contour.png)" >> report.md
62+
echo "![Parallel](runs/optuna/tablr/overview/parallel_coordinate.png)" >> report.md
63+
echo "![Importance](runs/optuna/tablr/overview/param_importances.png)" >> report.md
64+
echo "![Slice](runs/optuna/tablr/overview/slice.png)" >> report.md
65+
66+
cml comment create report.md

‎Dockerfile‎

Lines changed: 38 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -10,40 +10,62 @@
1010
ARG BASE_IMAGE=ubuntu:18.04
1111

1212
# Instal basic utilities
13-
ENV DEBIAN_FRONTEND noninteractiveee
14-
RUN --mount=type=cache,id=apt-dev,target=/var/cache/apt \
15-
apt-get update && apt-get install -y --no-install-recommends \
13+
FROM ${BASE_IMAGE} as dev-base
14+
RUN apt-get clean && apt-get update && apt-get upgrade && apt-get install -y --no-install-recommends \
1615
build-essential \
1716
ca-certificates \
1817
ccache \
1918
cmake \
2019
curl \
2120
git \
2221
gcc \
22+
wget \
2323
libjpeg-dev \
24+
zip \
25+
swig python3-dev \
2426
unzip bzip2 ffmpeg libsm6 libxext6 \
2527
libpng-dev && \
2628
rm -rf /var/lib/apt/lists/*
27-
28-
RUN curl -fsSL -v -o ~/miniconda.sh -O https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh && \
29-
chmod +x ~/miniconda.sh && \
30-
~/miniconda.sh -b -p /opt/conda && \
31-
rm ~/miniconda.sh && \
32-
/opt/conda/bin/conda install -c pytorch -c nvidia -y \
33-
python=${PYTHON_VERSION} \
34-
pytorch=${PYTORCH_VERSION} torchvision "pytorch-cuda=${CUDA_VERSION}" && \
35-
/opt/conda/bin/conda clean -ya
36-
3729
RUN /usr/sbin/update-ccache-symlinks
3830
RUN mkdir /opt/ccache && ccache --set-config=cache_dir=/opt/ccache
39-
4031
ENV PATH /opt/conda/bin:$PATH
32+
33+
# Instal environment
34+
FROM dev-base as conda-installs
35+
ARG PYTHON_VERSION=3.9
36+
ARG CUDA_VERSION=11.3
37+
ARG PYTORCH_VERSION=1.12.1
38+
ARG CUDA_CHANNEL=nvidia
39+
ARG INSTALL_CHANNEL=pytorch
40+
ENV CONDA_OVERRIDE_CUDA=${CUDA_VERSION}
41+
RUN curl -fsSL -v -o ~/mambaforge.sh -O https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-Linux-x86_64.sh && \
42+
chmod +x ~/mambaforge.sh && \
43+
~/mambaforge.sh -b -p /opt/mamba && \
44+
rm ~/mambaforge.sh && \
45+
/opt/mamba/bin/mamba install -c "${INSTALL_CHANNEL}" -c "${CUDA_CHANNEL}" -y \
46+
python=${PYTHON_VERSION} \
47+
pytorch=${PYTORCH_VERSION} torchvision "cudatoolkit=${CUDA_VERSION}" && \
48+
/opt/mamba/bin/mamba clean -ya
49+
50+
ENV PATH /opt/mamba/bin:$PATH
4151
ENV NVIDIA_VISIBLE_DEVICES all
4252
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
4353
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64
4454
ENV PYTORCH_VERSION ${PYTORCH_VERSION}
4555

46-
# Install
56+
# Install dependencies
4757
COPY ./ /workspace/
4858
WORKDIR /workspace/
49-
RUN /opt/conda/bin/python -m pip install -e .
59+
RUN /opt/mamba/bin/python -m pip install --upgrade pip && \
60+
/opt/mamba/bin/python -m pip install -e .[cv,cv_classification,cv_semantic,cv_detection,nlp,nlp_retrieval,tabular,tabular_classification,dev] && \
61+
/opt/mamba/bin/python -m pip install dvc dvc-gdrive && \
62+
/opt/mamba/bin/python -m pip install -U timm
63+
64+
# Pull data from GDrive
65+
RUN --mount=type=secret,id=credentials \
66+
CREDENTIALS=$(cat /run/secrets/credentials) \
67+
&& echo "$CREDENTIALS" > /workspace/credentials.json
68+
RUN dvc remote modify gdrive --local gdrive_user_credentials_file /workspace/credentials.json
69+
RUN dvc pull
70+
71+
ENTRYPOINT ["/bin/bash"]

‎configs/base/globals.yaml‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
global:
2+
exp_name: null
3+
exist_ok: false
4+
debug: false
5+
save_dir: runs
6+
device: cuda:0
7+
pretrained: null
8+
resume: null
9+
trainer:
10+
name: SupervisedTrainer
11+
args:
12+
num_iterations: 10000
13+
clip_grad: 1.0
14+
evaluate_interval: 1
15+
use_fp16: true

‎configs/base/optimizer.yaml‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
optimizer:
2+
name: AdamW
3+
args:
4+
lr: 0.001
5+
weight_decay: 0.0005
6+
betas:
7+
- 0.937
8+
- 0.999
9+
scheduler:
10+
name: SchedulerWrapper
11+
args:
12+
scheduler_name: cosine2
13+
t_initial: 7
14+
t_mul: 0.9
15+
eta_mul: 0.9
16+
eta_min: 1.0e-06

0 commit comments

Comments
 (0)