Skip to content

Commit 24afdf5

Browse files
cmccullycmccully
authored
Merge pull request #405 from LCOGT/bugfix/cals-by-block-migrate
Fixes to migration from cals by block.
2 parents 6680bbc + 01f29ce commit 24afdf5

30 files changed

+4175
-1183
lines changed

Diff for: ‎.dockerignore

+4
Original file line numberDiff line numberDiff line change
@@ -73,3 +73,7 @@ distribute-*.tar.gz
7373
*.xml
7474

7575
*.iml
76+
77+
.tmp
78+
79+
.tox

Diff for: ‎.github/workflows/codestyle.yaml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: Code Style
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest # Or macos-latest if needed
12+
13+
steps:
14+
- uses: actions/checkout@v3
15+
- name: Set up Python
16+
uses: actions/setup-python@v4
17+
with:
18+
python-version: '3.12' # Specify your desired version
19+
- name: Install Poetry
20+
run: pip install poetry
21+
- name: Set up project
22+
run: poetry install -E cpu -E style
23+
- name: Code Style
24+
run: poetry run pycodestyle --count --max-line-length=120

Diff for: ‎.github/workflows/coverage.yaml

+24
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
name: Coverage
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest # Or macos-latest if needed
12+
13+
steps:
14+
- uses: actions/checkout@v3
15+
- name: Set up
16+
uses: actions/setup-python@v4
17+
with:
18+
python-version: '3.12' # Specify your desired version
19+
- name: Install Poetry
20+
run: pip install poetry
21+
- name: Set up project
22+
run: poetry install -E cpu -E test
23+
- name: Run coverage
24+
run: poetry run pytest --pyargs banzai.tests -m "not e2e" --cov banzai --cov-config="setup.cfg"

Diff for: ‎.github/workflows/docs.yaml

+25
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,25 @@
1+
name: Docs
2+
3+
on:
4+
push:
5+
branches: [ main ]
6+
pull_request:
7+
branches: [ main ]
8+
9+
jobs:
10+
build:
11+
runs-on: ubuntu-latest # Or macos-latest if needed
12+
13+
steps:
14+
- uses: actions/checkout@v3
15+
- name: Set up Python
16+
uses: actions/setup-python@v4
17+
with:
18+
python-version: '3.12' # Specify your desired version
19+
- name: Install dependencies
20+
run: pip install poetry
21+
- name: Set up project
22+
run: poetry install -E cpu -E docs
23+
- name: Build Docs
24+
run: poetry run sphinx-build -W -b html docs docs/_build/html
25+

Diff for: ‎.github/workflows/e2e.yaml

+18-10
Original file line numberDiff line numberDiff line change
@@ -35,24 +35,25 @@ jobs:
3535
3636
- name: Start banzai
3737
run: |
38-
cat banzai/tests/e2e-k8s.yaml
38+
kubectl create secret generic auth-token-secret --from-literal=auth-token=${{ secrets.ArchiveAuthToken }}
3939
4040
# Deploy banzai stack
41-
kubectl apply -f banzai/tests/e2e-k8s.yaml
41+
kubectl apply -k banzai/tests/k8s
4242
4343
# Wait for banzai to be ready
4444
kubectl wait --for=condition=Ready --timeout=60m pod/banzai-e2e-test
45-
45+
4646
- name: Test Super Bias Creation
4747
run: |
4848
set +e
4949
export START=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
50-
5150
kubectl exec banzai-e2e-test -c banzai-listener -- pytest -s --pyargs banzai --durations=0 --junitxml=/archive/engineering/pytest-master-bias.xml -m master_bias
5251
EXIT_CODE=$?
5352
5453
# Always print logs even if (especially if?) the reduction fails
55-
kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
54+
kubectl logs banzai-e2e-test -c banzai-listener --since-time=$START --prefix=true
55+
kubectl logs banzai-e2e-test -c banzai-celery-workers --since-time=$START --prefix=true
56+
kubectl logs banzai-e2e-test -c banzai-large-celery-workers --since-time=$START --prefix=true
5657
5758
# Exit with the captured status so the job properly fails or succeeds
5859
exit $EXIT_CODE
@@ -66,8 +67,10 @@ jobs:
6667
EXIT_CODE=$?
6768
6869
# Always print logs even if (especially if?) the reduction fails
69-
kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
70-
70+
kubectl logs banzai-e2e-test -c banzai-listener --since-time=$START --prefix=true
71+
kubectl logs banzai-e2e-test -c banzai-celery-workers --since-time=$START --prefix=true
72+
kubectl logs banzai-e2e-test -c banzai-large-celery-workers --since-time=$START --prefix=true
73+
7174
# Exit with the captured status so the job properly fails or succeeds
7275
exit $EXIT_CODE
7376
@@ -80,7 +83,9 @@ jobs:
8083
EXIT_CODE=$?
8184
8285
# Always print logs even if (especially if?) the reduction fails
83-
kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
86+
kubectl logs banzai-e2e-test -c banzai-listener --since-time=$START --prefix=true
87+
kubectl logs banzai-e2e-test -c banzai-celery-workers --since-time=$START --prefix=true
88+
kubectl logs banzai-e2e-test -c banzai-large-celery-workers --since-time=$START --prefix=true
8489
8590
# Exit with the captured status so the job properly fails or succeeds
8691
exit $EXIT_CODE
@@ -94,11 +99,14 @@ jobs:
9499
EXIT_CODE=$?
95100
96101
# Always print logs even if (especially if?) the reduction fails
97-
kubectl logs banzai-e2e-test --since-time=$START --all-containers --prefix=true
102+
kubectl logs banzai-e2e-test -c banzai-listener --since-time=$START --prefix=true
103+
kubectl logs banzai-e2e-test -c banzai-celery-workers --since-time=$START --prefix=true
104+
kubectl logs banzai-e2e-test -c banzai-large-celery-workers --since-time=$START --prefix=true
98105
99106
# Exit with the captured status so the job properly fails or succeeds
100107
exit $EXIT_CODE
101108
102109
- name: Cleanup
103110
run: |
104-
kubectl delete pod banzai-e2e-test
111+
kubectl delete service --selector=group=banzai-e2e-test
112+
kubectl delete deployment --selector=group=banzai-e2e-test

Diff for: ‎.github/workflows/unit-tests.yml

+32-49
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,41 @@
11
name: Build/Test
22

33
on:
4-
# Run this workflow for pushes on all branches
54
push:
5+
branches: [main]
6+
pull_request:
7+
branches: [main]
68

79
jobs:
8-
tests:
9-
name: ${{ matrix.name }} (${{ matrix.os }}, ${{ matrix.toxenv }})
10-
runs-on: ${{ matrix.os }}
10+
linuxtest:
11+
runs-on: ubuntu-latest
1112
strategy:
12-
fail-fast: false
1313
matrix:
14-
include:
15-
16-
- name: Documentation build
17-
os: ubuntu-latest
18-
python-version: 3.9
19-
toxenv: build_docs
20-
21-
- name: Python 3.9 with minimal dependencies
22-
os: ubuntu-latest
23-
python-version: '3.9'
24-
toxenv: py39-test
25-
26-
- name: Python 3.10 with minimal dependencies
27-
os: ubuntu-latest
28-
python-version: '3.10'
29-
toxenv: py310-test
30-
31-
- name: Python 3.11 with minimal dependencies
32-
os: ubuntu-latest
33-
python-version: '3.11'
34-
toxenv: py311-test
35-
36-
- name: Code style checks
37-
os: ubuntu-latest
38-
python-version: 3.9
39-
toxenv: codestyle
40-
14+
python-version: ['3.10', '3.11', '3.12', '3.13']
15+
steps:
16+
- uses: actions/checkout@v3
17+
- uses: actions/setup-python@v4
18+
with:
19+
python-version: ${{ matrix.python-version }}
20+
- name: Install Poetry
21+
run: pip install poetry
22+
- name: Set up project
23+
run: poetry install -E cpu -E test
24+
- name: Run tests
25+
run: poetry run pytest --pyargs banzai.tests -m "not e2e"
26+
macos: # New macOS job
27+
runs-on: macos-latest
28+
strategy:
29+
matrix:
30+
python-version: ['3.12']
4131
steps:
42-
- uses: actions/checkout@v2
43-
with:
44-
fetch-depth: 0
45-
- name: Set up Python ${{ matrix.python-version }}
46-
uses: actions/setup-python@v2
47-
with:
48-
python-version: ${{ matrix.python-version }}
49-
- name: Install dependencies
50-
run: |
51-
python -m pip install --upgrade pip
52-
python -m pip install tox
53-
- name: Install graphviz dependency
54-
if: "endsWith(matrix.toxenv, 'build_docs')"
55-
run: sudo apt-get -y install graphviz
56-
- name: Run tests
57-
run: |
58-
tox -e ${{ matrix.toxenv }}
32+
- uses: actions/checkout@v3
33+
- uses: actions/setup-python@v4
34+
with:
35+
python-version: ${{ matrix.python-version }}
36+
- name: Install Poetry
37+
run: pip install poetry
38+
- name: Set up project
39+
run: poetry install -E cpu -E test
40+
- name: Run tests
41+
run: poetry run pytest --pyargs banzai.tests -m "not e2e"

Diff for: ‎.gitignore

+2-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ a.out
1919
*/version.py
2020
*/cython_version.py
2121
htmlcov
22-
.coverage
22+
.coverage*
2323
MANIFEST
2424
.ipynb_checkpoints
2525
env
@@ -75,3 +75,4 @@ distribute-*.tar.gz
7575
*.iml
7676

7777
.coverage.subprocess
78+
test.db

Diff for: ‎.rtd-environment.yml

-12
This file was deleted.

Diff for: ‎CHANGES.md

+6
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,9 @@
1+
1.20.2 (2025-02-10)
2+
-------------------
3+
- Bugfix to logging on reduction stopped
4+
- Updated the alembic scripts because the migration was taking too long
5+
- Dependencies and package build are now managed by poetry
6+
17
1.20.1 (2025-02-05)
28
-------------------
39
- Added tso proposals to be instantly public

Diff for: ‎Dockerfile

+17-15
Original file line numberDiff line numberDiff line change
@@ -1,28 +1,30 @@
1-
FROM continuumio/miniconda3:23.5.2-0
2-
3-
# In principle I could remove the gcc to shrink the image, but pytorch is already so large it doesn't make much difference
4-
RUN apt-get -y update && apt-get -y install gcc && \
5-
apt-get autoclean && \
6-
rm -rf /var/lib/apt/lists/*
1+
FROM python:3.12-slim
72

3+
# Make a non-privledged user to run the pipeline
84
RUN mkdir /home/archive && /usr/sbin/groupadd -g 10000 "domainusers" \
95
&& /usr/sbin/useradd -g 10000 -d /home/archive -M -N -u 10087 archive \
106
&& chown -R archive:domainusers /home/archive
117

12-
USER archive
13-
14-
ENV HOME=/home/archive
8+
RUN pip install poetry --no-cache
159

16-
WORKDIR /home/archive
10+
RUN apt-get -y update && apt-get -y install gcc procps && \
11+
apt-get autoclean && \
12+
rm -rf /var/lib/apt/lists/*
1713

18-
COPY environment.yaml .
14+
RUN poetry config virtualenvs.create false
1915

20-
RUN . /opt/conda/etc/profile.d/conda.sh && conda config --set remote_read_timeout_secs 900 && conda env create -p /home/archive/envs/banzai -f environment.yaml --solver=libmamba
16+
COPY pyproject.toml poetry.lock /lco/banzai/
2117

22-
COPY --chown=10087:10000 . /lco/banzai
18+
RUN poetry install --directory=/lco/banzai -E cpu --no-root --no-cache
2319

24-
ENV PATH=/home/archive/envs/banzai/bin:$PATH
20+
COPY . /lco/banzai
2521

26-
RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/
22+
RUN poetry install --directory /lco/banzai -E cpu --no-cache
2723

2824
RUN cp /lco/banzai/pytest.ini /home/archive/pytest.ini
25+
26+
USER archive
27+
28+
ENV HOME=/home/archive
29+
30+
WORKDIR /home/archive

0 commit comments

Comments
 (0)