Fixes to migration from cals by block.

.dockerignore

@@ -73,3 +73,7 @@ distribute-*.tar.gz
 *.xml
 
 *.iml
+
+.tmp
+
+.tox

.github/workflows/codestyle.yaml

@@ -0,0 +1,24 @@
+name: Code Style
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest  # Or macos-latest if needed
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12' # Specify your desired version
+      - name: Install Poetry
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E style
+      - name: Code Style
+        run: poetry run pycodestyle --count --max-line-length=120

.github/workflows/coverage.yaml

@@ -0,0 +1,24 @@
+name: Coverage
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest  # Or macos-latest if needed
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up 
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12' # Specify your desired version
+      - name: Install Poetry
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E test
+      - name: Run coverage
+        run: poetry run pytest --pyargs banzai.tests -m "not e2e" --cov banzai --cov-config="setup.cfg"

.github/workflows/docs.yaml

@@ -0,0 +1,25 @@
+name: Docs
+
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest  # Or macos-latest if needed
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '3.12' # Specify your desired version
+      - name: Install dependencies
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E docs
+      - name: Build Docs
+        run: poetry run sphinx-build -W -b html docs docs/_build/html
+

.github/workflows/unit-tests.yml

@@ -1,58 +1,41 @@
 name: Build/Test
 
 on:
-  # Run this workflow for pushes on all branches
   push:
+    branches: [main]
+  pull_request:
+    branches: [main]
 
 jobs:
-  tests:
-    name: ${{ matrix.name }} (${{ matrix.os }}, ${{ matrix.toxenv }})
-    runs-on: ${{ matrix.os }}
+  linuxtest:
+    runs-on: ubuntu-latest
     strategy:
-      fail-fast: false
       matrix:
-        include:
-
-          - name: Documentation build
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: build_docs
-
-          - name: Python 3.9 with minimal dependencies
-            os: ubuntu-latest
-            python-version: '3.9'
-            toxenv: py39-test
-
-          - name: Python 3.10 with minimal dependencies
-            os: ubuntu-latest
-            python-version: '3.10'
-            toxenv: py310-test
-
-          - name: Python 3.11 with minimal dependencies
-            os: ubuntu-latest
-            python-version: '3.11'
-            toxenv: py311-test
-
-          - name: Code style checks
-            os: ubuntu-latest
-            python-version: 3.9
-            toxenv: codestyle
-
+        python-version: ['3.10', '3.11', '3.12', '3.13']
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E test
+      - name: Run tests
+        run: poetry run pytest --pyargs banzai.tests -m "not e2e"
+  macos:  # New macOS job
+    runs-on: macos-latest
+    strategy:
+      matrix:
+        python-version: ['3.12']  
     steps:
-    - uses: actions/checkout@v2
-      with:
-        fetch-depth: 0
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      with:
-        python-version: ${{ matrix.python-version }}
-    - name: Install dependencies
-      run: | 
-       python -m pip install --upgrade pip
-       python -m pip install tox
-    - name: Install graphviz dependency
-      if: "endsWith(matrix.toxenv, 'build_docs')"
-      run: sudo apt-get -y install graphviz
-    - name: Run tests
-      run: |
-        tox -e ${{ matrix.toxenv }}
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install Poetry
+        run: pip install poetry
+      - name: Set up project
+        run: poetry install -E cpu -E test
+      - name: Run tests
+        run: poetry run pytest --pyargs banzai.tests -m "not e2e"

.gitignore

@@ -19,7 +19,7 @@ a.out
 */version.py
 */cython_version.py
 htmlcov
-.coverage
+.coverage*
 MANIFEST
 .ipynb_checkpoints
 env
@@ -75,3 +75,4 @@ distribute-*.tar.gz
 *.iml
 
 .coverage.subprocess
+test.db

CHANGES.md

@@ -1,3 +1,8 @@
+1.20.2 (2025-02-10)
+-------------------
+- Bugfix to logging on reduction stopped
+- Updated the alembic scripts because the migration was taking too long
+
 1.20.1 (2025-02-05)
 -------------------
 - Added tso proposals to be instantly public

Dockerfile

@@ -1,28 +1,29 @@
-FROM continuumio/miniconda3:23.5.2-0
-
-#  In principle I could remove the gcc to shrink the image, but pytorch is already so large it doesn't make much difference
-RUN apt-get -y update && apt-get -y install gcc && \
-        apt-get autoclean && \
-        rm -rf /var/lib/apt/lists/*
+FROM python:3.12-slim
 
 RUN mkdir /home/archive && /usr/sbin/groupadd -g 10000 "domainusers" \
         && /usr/sbin/useradd -g 10000 -d /home/archive -M -N -u 10087 archive \
         && chown -R archive:domainusers /home/archive
 
-USER archive
+RUN pip install poetry --no-cache
 
-ENV HOME=/home/archive
+RUN apt-get -y update && apt-get -y install gcc procps && \
+        apt-get autoclean && \
+        rm -rf /var/lib/apt/lists/*
 
-WORKDIR /home/archive
+RUN poetry config virtualenvs.create false
 
-COPY environment.yaml .
+COPY --chown=10087:10000 pyproject.toml poetry.lock /lco/banzai/
 
-RUN  . /opt/conda/etc/profile.d/conda.sh && conda config --set remote_read_timeout_secs 900 && conda env create -p /home/archive/envs/banzai -f environment.yaml --solver=libmamba
+RUN poetry install --directory=/lco/banzai -E cpu --no-root --no-cache
 
 COPY --chown=10087:10000 . /lco/banzai
 
-ENV PATH=/home/archive/envs/banzai/bin:$PATH
-
-RUN /home/archive/envs/banzai/bin/pip install --no-cache-dir /lco/banzai/ 
+RUN poetry install --directory /lco/banzai -E cpu --no-cache
 
 RUN cp /lco/banzai/pytest.ini /home/archive/pytest.ini
+
+USER archive
+
+ENV HOME=/home/archive
+
+WORKDIR /home/archive

alembic/migrations/versions/51b98e26cc50_migration_of_past_data_for_blockid_.py

@@ -0,0 +1,70 @@
+"""Migration of past data for blockid, public-date, and proposalid
+
+Revision ID: 51b98e26cc50
+Revises: 8e35c490a971
+Create Date: 2025-02-10 11:08:49.733571
+
+"""
+from typing import Sequence, Union
+
+from alembic import op
+from sqlalchemy.sql import text
+import os
+import requests
+import datetime
+
+# revision identifiers, used by Alembic.
+revision: str = '51b98e26cc50'
+down_revision: Union[str, None] = '8e35c490a971'
+branch_labels: Union[str, Sequence[str], None] = None
+depends_on: Union[str, Sequence[str], None] = '8e35c490a971'
+
+
+def parse_date(date_to_parse):
+    date_formats = ['%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%SZ', '%Y-%m-%d %H:%M:%S.%f']
+    for date_format in date_formats:
+        try:
+            parsed_date = datetime.datetime.strptime(date_to_parse, date_format)
+            break
+        except ValueError:
+            continue
+    return parsed_date
+
+
+def upgrade() -> None:
+    if os.getenv("AUTH_TOKEN") is not None:
+        auth_header = {'Authorization': f'Token {os.environ["AUTH_TOKEN"]}'}
+    else:
+        auth_header = None
+
+    connection = op.get_bind()
+
+    query_str = "SELECT id, frameid, filename,type, dateobs FROM calimages"
+    # Get all of the frames from all of the cameras for all obstypes in this batch
+    rows = connection.execute(text(query_str)).fetchall()
+    for row in rows:
+        basename = row.filename.replace('.fits', '').replace('.fz', '')
+        if row.frameid is not None:
+            request_results = requests.get(f'https://archive-api.lco.global/frames/{row.frameid}', headers=auth_header)
+            request_results = request_results.json()
+        else:
+            params = {'basename_exact': basename, 'start': parse_date(row.dateobs) - datetime.timedelta(days=1),
+                      'end': parse_date(row.dateobs) + datetime.timedelta(days=1)}
+            request_results = requests.get('https://archive-api.lco.global/frames/', params=params, headers=auth_header)
+            if len(request_results.json()['results']) == 0:
+                continue
+            request_results = request_results.json()['results'][0]
+
+        values = {'id': row.id,
+                  'public_date': parse_date(request_results['public_date']),
+                  'proposal': request_results['proposal_id'],
+                  'blockid': request_results['BLKUID']}
+        query_str = 'blockid = :blockid, proposal = :proposal, public_date = :public_date'
+        if row.frameid is None:
+            query_str += ', frameid = :frameid'
+            values['frameid'] = request_results['id']
+        connection.execute(text(f"UPDATE calimages SET {query_str} WHERE id = :id"), values)
+
+
+def downgrade() -> None:
+    pass

alembic/migrations/versions/8e35c490a971_add_calibrations_block_info_proposal_id_.py

@@ -1,7 +1,7 @@
 """Add calibrations block info, proposal id, and public date to CalibrationImage
 
 Revision ID: 8e35c490a971
-Revises: 
+Revises:
 Create Date: 2025-01-31 12:19:03.407606
 
 """
@@ -10,10 +10,6 @@
 from alembic import op
 import sqlalchemy as sa
 from sqlalchemy import inspect
-from sqlalchemy.sql import text
-import requests
-import os
-import datetime
 
 
 # revision identifiers, used by Alembic.
@@ -34,38 +30,6 @@ def upgrade() -> None:
         op.add_column('calimages', sa.Column('proposal', sa.String(50), nullable=True))
     if 'public_date' not in columns:
         op.add_column('calimages', sa.Column('public_date', sa.DateTime(), nullable=True))
-    if os.getenv("AUTH_TOKEN") is not None:
-        auth_header = {'Authorization': f'Token {os.environ["AUTH_TOKEN"]}'}
-    else:
-        auth_header = None
-
-    connection = op.get_bind()
-    rows = connection.execute(text("SELECT id, frameid, filename FROM calimages"))
-    for row in rows:
-        params = {'basename_exact': row.filename.replace('.fz', '').replace('.fits', '')}
-        request_results = requests.get('https://archive-api.lco.global/frames/', params=params, headers=auth_header)
-        request_results = request_results.json()['results']
-        if len(request_results) == 0:
-            continue
-
-        blockid = request_results[0]['BLKUID']
-        proposal = request_results[0]['proposal_id']
-        date_formats = ['%Y-%m-%dT%H:%M:%S.%fZ', '%Y-%m-%dT%H:%M:%SZ']
-        for date_format in date_formats:
-            try:
-                public_date = datetime.datetime.strptime(request_results[0]['public_date'], date_format)
-                break
-            except ValueError:
-                continue
-        values = {'id': row.id, 'public_date': public_date, 'proposal': proposal, 'blockid': blockid}
-        query_str = 'blockid = :blockid, proposal = :proposal, public_date = :public_date'
-        if row.frameid is None:
-            query_str += ', frameid = :frameid'
-            values['frameid'] = request_results[0]['id']
-        connection.execute(
-            text(f"UPDATE calimages SET {query_str} WHERE id = :id"),
-            values
-        )
 
 
 def downgrade() -> None: