Merge branch 'master' into resurrect_systematics

Author: lgray

Diff for: .github/dependabot.yml

@@ -5,3 +5,6 @@ updates:
     directory: "/"
     schedule:
       interval: "daily"
+    commit-message:
+      # Prefix all commit messages with "npm: "
+      prefix: "ci: "

Diff for: .github/workflows/ci.yml

@@ -26,8 +26,8 @@ jobs:
     name: pre-commit
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-python@v4
+    - uses: actions/checkout@v4
+    - uses: actions/setup-python@v5
     - uses: pre-commit/[email protected]
       with:
         extra_args: --hook-stage manual --all-files
@@ -45,13 +45,13 @@ jobs:
     name: test coffea (${{ matrix.os }}) - python ${{ matrix.python-version }}, JDK${{ matrix.java-version }}
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Set up JDK ${{ matrix.java-distribution }}/${{ matrix.java-version }}
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         java-version: ${{ matrix.java-version }}
         distribution: ${{ matrix.java-distribution }}
@@ -67,9 +67,9 @@ jobs:
         # mltool installs
         python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
         python -m pip install xgboost
-        python -m pip install tritonclient[grpc,http]
+        python -m pip install 'tritonclient[grpc,http]!=2.41.0'
         # install checked out coffea
-        python -m pip install -q -e '.[dev,parsl,dask,spark]'
+        python -m pip install -q -e '.[dev,parsl,dask,spark]' --upgrade --upgrade-strategy eager
         python -m pip list
         java -version
     - name: Install dependencies (MacOS)
@@ -80,7 +80,7 @@ jobs:
         python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
         python -m pip install xgboost
         # install checked out coffea
-        python -m pip install -q -e '.[dev,dask,spark]'
+        python -m pip install -q -e '.[dev,dask,spark]' --upgrade --upgrade-strategy eager
         python -m pip list
         java -version
     - name: Install dependencies (Windows)
@@ -91,18 +91,18 @@ jobs:
         python -m pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu
         python -m pip install xgboost
         # install checked out coffea
-        python -m pip install -q -e '.[dev,dask]'
+        python -m pip install -q -e '.[dev,dask]' --upgrade --upgrade-strategy eager
         python -m pip list
         java -version
 
     - name: Start triton server with example model
       if: matrix.os == 'ubuntu-latest'
       run: |
-        docker run -d --rm -p 8000:8000 -p 8001:8001 -p 8002:8002 -v ${{ github.workspace }}/tests/samples/triton_models_test:/models nvcr.io/nvidia/tritonserver:23.04-py3 tritonserver --model-repository=/models
+        docker run -d --rm -p 8000:8000 -p 8001:8001 -p 8002:8002 -v ${{ github.workspace }}/tests/samples/triton_models_test:/models nvcr.io/nvidia/tritonserver:23.04-pyt-python-py3 tritonserver --model-repository=/models
 
     - name: Test with pytest
       run: |
-        pytest --cov-report=xml --cov=coffea tests
+        pytest --cov-report=xml --cov=coffea --deselect=test_taskvine
     - name: Upload codecov
       if: matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11
       uses: codecov/codecov-action@v3
@@ -119,48 +119,48 @@ jobs:
         touch build/html/.nojekyll
     - name: Deploy documentation
       if: github.event_name == 'push' && matrix.os == 'ubuntu-latest' && matrix.python-version == 3.11
-      uses: crazy-max/ghaction-github-pages@v3
+      uses: crazy-max/ghaction-github-pages@v4
       with:
         target_branch: gh-pages
         build_dir: docs/build/html
       env:
         GH_PAT: ${{ secrets.GITHUB_OAUTH }}
 
-  testwq:
+  test-vine:
     runs-on: ubuntu-latest
     needs: pre-commit
     strategy:
       matrix:
         python-version: ["3.11"]
-    name: test coffea-workqueue
-
+    name: test coffea-taskvine
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Conda
-      uses: conda-incubator/setup-miniconda@v2
+      uses: conda-incubator/setup-miniconda@v3
       env:
         ACTIONS_ALLOW_UNSECURE_COMMANDS: true
       with:
         auto-update-conda: true
         python-version: ${{ matrix.python-version }}
         channels: conda-forge
-    - name: Test work_queue
+    - name: Construct conda environment
       shell: bash -l {0}
       run: |
-        conda create --yes --name coffea-env -c conda-forge python=${{ matrix.python-version }} ndcctools dill conda-pack conda
-        conda activate coffea-env
-        python -m pip install --ignore-installed .
-        cd tests
-        conda-pack --output coffea-env.tar.gz
-        python wq.py coffea-env.tar.gz
+        conda create --yes --prefix ./coffea-conda-test-env -c conda-forge python=${{ matrix.python-version }} ndcctools
+        conda activate ./coffea-conda-test-env
+        python -m pip install .
+        python -m pip install pytest
+    - name: Test with pytest
+      run: |
+        conda run --prefix ./coffea-conda-test-env pytest tests/test_taskvine.py
 
 #  testskyhookjob:
 #    runs-on: ubuntu-latest
 #    needs: pre-commit
 #    name: test coffea-skyhook-job
 #
 #    steps:
-#    - uses: actions/checkout@3.2.0
+#    - uses: actions/checkout@3
 #    - name: Test Coffea Skyhook Bindings
 #      shell: bash -l {0}
 #      run: |
@@ -178,30 +178,30 @@ jobs:
   release:
     if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/v')
     runs-on: ubuntu-latest
-    needs: [test, testwq]
+    needs: [test, test-vine]
     strategy:
       matrix:
         python-version: ["3.11"]
     name: deploy release
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
+      uses: actions/setup-python@v5
       with:
         python-version: ${{ matrix.python-version }}
     - name: Build package for PyPI
       run: |
         python -m pip install pip hatch --upgrade
         python -m hatch build -t sdist -t wheel
     - name: Publish package to PyPI
-      uses: pypa/[email protected].8
+      uses: pypa/[email protected].11
       with:
         user: __token__
         password: ${{ secrets.PYPI_TOKEN }}
 
   pass:
-    needs: [test, testwq]
+    needs: [test, test-vine]
     runs-on: ubuntu-latest
     steps:
       - run: echo "All jobs passed"

Diff for: .github/workflows/pr.yml

@@ -17,6 +17,6 @@ jobs:
     name: Validate PR title
     runs-on: ubuntu-latest
     steps:
-      - uses: amannn/action-semantic-pull-request@v5.2.0
+      - uses: amannn/action-semantic-pull-request@v5
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

Diff for: .pre-commit-config.yaml

@@ -5,26 +5,26 @@ ci:
         for more information, see https://pre-commit.ci
     autofix_prs: true
     autoupdate_branch: ''
-    autoupdate_commit_msg: '[pre-commit.ci] pre-commit autoupdate'
+    autoupdate_commit_msg: 'ci(pre-commit): pre-commit autoupdate'
     autoupdate_schedule: weekly
     skip: []
     submodules: false
 
 repos:
 - repo: https://github.com/psf/black
-  rev: 23.7.0
+  rev: 23.12.1
   hooks:
   - id: black
 
 - repo: https://github.com/PyCQA/isort
-  rev: 5.12.0
+  rev: 5.13.2
   hooks:
   - id: isort
     language_version: python3
     args: ["--profile", "black", "--filter-files"]
 
 - repo: https://github.com/pre-commit/pre-commit-hooks
-  rev: v4.4.0
+  rev: v4.5.0
   hooks:
   - id: check-case-conflict
   - id: check-merge-conflict
@@ -37,24 +37,24 @@ repos:
   - id: trailing-whitespace
 
 - repo: https://github.com/asottile/pyupgrade
-  rev: v3.9.0
+  rev: v3.15.0
   hooks:
   - id: pyupgrade
     args: ["--py38-plus"]
 
 - repo: https://github.com/asottile/setup-cfg-fmt
-  rev: v2.4.0
+  rev: v2.5.0
   hooks:
   - id: setup-cfg-fmt
 
 - repo: https://github.com/pycqa/flake8
-  rev: 6.0.0
+  rev: 7.0.0
   hooks:
   - id: flake8
     exclude: coffea/processor/templates
 
 - repo: https://github.com/codespell-project/codespell
-  rev: v2.2.5
+  rev: v2.2.6
   hooks:
   - id: codespell
-    args: ["--skip=*.ipynb","-L hist,Hist,nd,SubJet,subjet,Subjet,PTD,ptd,fPt,fpt"]
+    args: ["--skip=*.ipynb","-L hist,Hist,nd,SubJet,subjet,Subjet,PTD,ptd,fPt,fpt,Ser,ser,hda"]

Diff for: CITATION.cff

@@ -0,0 +1,47 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+authors:
+- family-names: "Gray"
+  given-names: "Lindsey"
+  affiliation: "Fermilab"
+  orcid: "https://orcid.org/0000-0002-6408-4288"
+  email: "[email protected]"
+- family-names: "Smith"
+  given-names: "Nicholas"
+  affiliation: "Fermilab"
+  orcid: "https://orcid.org/0000-0002-0324-3054"
+  email: "[email protected]"
+- family-names: "Novak"
+  given-names: "Andrzej"
+  affiliation: "RWTH Aachen"
+  orcid: "https://orcid.org/0000-0002-0389-5896"
+- family-names: "Fackeldey"
+  given-names: "Peter"
+  affiliation: "RWTH Aachen University"
+  orcid: "https://orcid.org/0000-0003-4932-7162"
+  email: "[email protected]"
+- family-names: "Tovar"
+  given-names: "Benjamin"
+  affiliation: "University of Notre Dame"
+  orcid: "https://orcid.org/0000-0002-5294-2281"
+  email: "[email protected]"
+- family-names: "Chen"
+  given-names: "Yi-Mu"
+  affiliation: "University of Maryland, College Park"
+  orcid: "https://orcid.org/0000-0002-5795-4783"
+  email: "[email protected]"
+- family-names: "Watts"
+  given-names: "Gordon"
+  affiliation: "University of Washington"
+  orcid: "https://orcid.org/0000-0002-0753-7308"
+  email: "[email protected]"
+- family-names: "Krommydas"
+  given-names: "Iason"
+  affiliation: "Rice University"
+  orcid: "https://orcid.org/0000-0001-7849-8863"
+  email: "[email protected]"
+title: "coffea"
+version: 0.7.21
+doi: 10.5281/zenodo.7733568
+date-released: 2023-03-14
+url: "https://github.com/CoffeaTeam/coffea"

Diff for: README.rst

@@ -20,7 +20,7 @@ coffea - Columnar Object Framework For Effective Analysis
     :target: https://anaconda.org/conda-forge/coffea
 
 .. image:: https://badges.gitter.im/CoffeaTeam/coffea.svg
-    :target: https://gitter.im/coffea-hep
+    :target: https://matrix.to/#/#coffea-hep_community:gitter.im
 
 .. image:: https://mybinder.org/badge_logo.svg
    :target: https://mybinder.org/v2/gh/CoffeaTeam/coffea/master?filepath=binder/
@@ -81,10 +81,17 @@ The following are installed automatically when you install coffea with pip:
 - `numba <https://numba.pydata.org/>`__ just-in-time compilation of python functions;
 - `scipy <https://scipy.org/scipylib/index.html>`__ for many statistical functions;
 - `matplotlib <https://matplotlib.org/>`__ as a plotting backend;
-- and other utility packages, as enumerated in ``setup.py``.
+- and other utility packages, as enumerated in ``pyproject.toml``.
 
 .. inclusion-marker-3-do-not-remove
 
 Documentation
 =============
 All documentation is hosted at https://coffeateam.github.io/coffea/
+
+Citation
+========
+If you would like to cite this code in your work, you can use the zenodo DOI indicated in ``CITATION.cff``, or the `latest DOI <https://zenodo.org/badge/latestdoi/159673139>`__. You may also cite the proceedings:
+
+- "N. Smith et al 2020 EPJ Web Conf. 245 06012"
+- "L. Gray et al 2023 J. Phys.: Conf. Ser. 2438 012033"