Update Python support

.github/workflows/build_workflow.yml

@@ -19,21 +19,24 @@ jobs:
       - name: Checkout Code Repository
         uses: actions/checkout@v3
 
-      - name: Set up Python 3.9
+      - name: Set up Python 3.13
         uses: actions/setup-python@v4
         with:
-          python-version: 3.9
+          python-version: "3.13"
 
       # Run all pre-commit hooks on all the files.
       # Getting only staged files can be tricky in case a new PR is opened
       # since the action is run on a branch in detached head state.
       # This is the equivalent of running "pre-commit run --all-files" locally.
       # If you commit with the `--no-verify` flag, this check may fail.
       - name: Install and Run Pre-commit
-        uses: pre-commit/[email protected].0
+        uses: pre-commit/[email protected].1
 
   build:
     runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ["3.11", "3.12", "3.13"]
     defaults:
       run:
         shell: bash -l {0}
@@ -44,11 +47,11 @@ jobs:
       - name: Cache Conda
         uses: actions/cache@v3
         env:
-          CACHE_NUMBER: 0
+          CACHE_NUMBER: 1  # Increment this to invalidate cache
         with:
           path: ~/conda_pkgs_dir
           key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
-            hashFiles('conda/dev.yml') }}
+            hashFiles('conda/dev.yml') }}-python${{ matrix.python-version }}
 
       - name: Build Conda Environment
         uses: conda-incubator/setup-miniconda@v3
@@ -57,13 +60,27 @@ jobs:
           miniforge-variant: Miniforge3
           miniforge-version: latest
           environment-file: conda/dev.yml
-          channel-priority: strict
+          channel-priority: flexible  # Changed from strict to flexible
           auto-update-conda: true
+          python-version: ${{ matrix.python-version }}
+          channels: conda-forge
+          use-only-tar-bz2: true
+
+      - name: Verify Environment and Fix Dependencies
+        run: |
+          conda info
+          conda list
+          # Ensure we have the right Python version
+          python --version
+          # Fix pip issues for Python 3.12+
+          if [[ "${{ matrix.python-version }}" == "3.12" ]] || [[ "${{ matrix.python-version }}" == "3.13" ]]; then
+            python -m ensurepip --upgrade || true
+            python -m pip install --upgrade --force-reinstall pip setuptools wheel
+          fi
 
       - name: Install `zstash` Package
         run: |
-          python -m pip install --upgrade pip
-          pip install .
+          python -m pip install .
 
       - name: Run Tests
         run: |
@@ -77,7 +94,7 @@ jobs:
     defaults:
       run:
         shell: bash -l {0}
-    timeout-minutes: 5
+    timeout-minutes: 10  # Increased timeout for docs
     steps:
       - uses: actions/checkout@v3
         with:
@@ -87,11 +104,11 @@ jobs:
       - name: Cache Conda
         uses: actions/cache@v3
         env:
-          CACHE_NUMBER: 0
+          CACHE_NUMBER: 1  # Match the build job cache number
         with:
           path: ~/conda_pkgs_dir
           key: ${{ runner.os }}-conda-${{ env.CACHE_NUMBER }}-${{
-            hashFiles('conda/dev.yml') }}
+            hashFiles('conda/dev.yml') }}-docs
 
       - name: Build Conda Environment
         uses: conda-incubator/setup-miniconda@v3
@@ -100,8 +117,9 @@ jobs:
           miniforge-variant: Miniforge3
           miniforge-version: latest
           environment-file: conda/dev.yml
-          channel-priority: strict
+          channel-priority: flexible  # Changed from strict to flexible
           auto-update-conda: true
+          python-version: "3.11"  # Use stable Python version for docs
 
       # sphinx-multiversion allows for version docs.
       - name: Build Sphinx Docs

conda/dev.yml

@@ -1,31 +1,30 @@
 name: zstash_dev
 channels:
   - conda-forge
-  - defaults
 dependencies:
   # Base
   # =================
-  - pip=22.2.2
-  - python=3.9.13
-  - six=1.16.0
-  - globus-sdk=3.15.0
+  - pip
+  - python >=3.11,<3.14
+  - sqlite
+  - six >=1.16.0
+  - globus-sdk >=3.15.0
   # Developer Tools
   # =================
   # If versions are updated, also update 'rev' in `.pre-commit.config.yaml`
-  - black=24.10.0
-  - flake8=7.1.1
-  - flake8-isort=6.1.1
-  - mypy=1.11.2
-  - pre-commit=4.0.1
-  - tbump=6.9.0
+  - black >=24.0.0
+  - flake8 >=7.0.0
+  - flake8-isort >=6.0.0
+  - mypy >=1.11.0
+  - pre-commit >=4.0.0
+  - tbump >=6.9.0
   # Documentation
   # =================
   # If versions are updated, also update in `.github/workflows/workflow.yml`
-  - jinja2<3.1
-  - sphinx=5.2.3
-  - sphinx-multiversion=0.2.4
-  - sphinx_rtd_theme=1.0.0
+  - jinja2 <3.1
+  - sphinx >=5.2.0
+  - sphinx-multiversion >=0.2.4
+  - sphinx_rtd_theme >=1.0.0
   # Need to pin docutils because 0.17 has a bug with unordered lists
   # https://github.com/readthedocs/sphinx_rtd_theme/issues/1115
-  - docutils=0.16
-prefix: /opt/miniconda3/envs/zstash_dev
+  - docutils >=0.16,<0.17

conda/meta.yaml

@@ -16,14 +16,14 @@ build:
 
 requirements:
   host:
-    - python >=3.9
+    - python >=3.11,<3.14
     - pip
 
   run:
-    - python >=3.9
-    - fair-research-login >=0.2.6,<0.3.0
-    - globus-sdk >=3.0.0,<4.0.0
-    - six
+    - python >=3.11,<3.14
+    - globus-sdk=3.15.0
+    - six=1.16.0
+    - sqlite
 
 test:
   imports:

setup.cfg

@@ -46,7 +46,7 @@ exclude =
     venv
 
 [mypy]
-python_version = 3.9
+python_version = 3.13
 check_untyped_defs = True
 ignore_missing_imports = True
 warn_unused_ignores = True

setup.py

@@ -7,6 +7,6 @@
     author_email="[email protected], [email protected], [email protected]",
     description="Long term HPSS archiving software for E3SM",
     packages=find_packages(include=["zstash", "zstash.*"]),
-    python_requires=">=3.9",
+    python_requires=">=3.11,<3.14",
     entry_points={"console_scripts": ["zstash=zstash.main:main"]},
 )

zstash/globus.py

@@ -211,7 +211,7 @@ def globus_block_wait(
         try:
             # Wait for the task to complete
             logger.info(
-                f"{ts_utc()}: on task_wait try {retry_count+1} out of {max_retries}"
+                f"{ts_utc()}: on task_wait try {retry_count + 1} out of {max_retries}"
             )
             transfer_client.task_wait(
                 task_id, timeout=wait_timeout, polling_interval=10

zstash/hpss_utils.py

@@ -4,6 +4,7 @@
 import os
 import os.path
 import sqlite3
+import sys
 import tarfile
 import traceback
 from datetime import datetime
@@ -281,43 +282,67 @@ def add_file(
     # Change the size of any hardlinks from 0 to the size of the actual file
     if tarinfo.islnk():
         tarinfo.size = os.path.getsize(file_name)
-    # Add the file to the tar
-    tar.addfile(tarinfo)
+
+    # Add the file to the tar - ONLY change this line for Python 3.13+
+    if (
+        sys.version_info >= (3, 13)
+        and (tarinfo.isfile() or tarinfo.islnk())
+        and tarinfo.size > 0
+    ):
+        # Python 3.13+ requires fileobj for non-empty regular files
+        with open(file_name, "rb") as fileobj:
+            tar.addfile(tarinfo, fileobj)
+    else:
+        # Original code - unchanged
+        tar.addfile(tarinfo)
 
     md5: Optional[str] = None
     # Only add files or hardlinks.
     # (So don't add directories or softlinks.)
     if tarinfo.isfile() or tarinfo.islnk():
         f: _io.TextIOWrapper = open(file_name, "rb")
         hash_md5: _hashlib.HASH = hashlib.md5()
-        if tar.fileobj is not None:
-            fileobj: _io.BufferedWriter = tar.fileobj
+
+        # For Python 3.13+, addfile() already wrote the content, so we only calculate MD5
+        if sys.version_info >= (3, 13) and tarinfo.size > 0:
+            # Just calculate MD5, don't write to tar (already done by addfile)
+            while True:
+                data: bytes = f.read(BLOCK_SIZE)
+                if len(data) > 0:
+                    hash_md5.update(data)
+                if len(data) < BLOCK_SIZE:
+                    break
+            md5 = hash_md5.hexdigest()
         else:
-            raise TypeError("Invalid tar.fileobj={}".format(tar.fileobj))
-        while True:
-            s: str = f.read(BLOCK_SIZE)
-            if len(s) > 0:
-                # If the block read in is non-empty, write it to fileobj and update the hash
-                fileobj.write(s)
-                hash_md5.update(s)
-            if len(s) < BLOCK_SIZE:
-                # If the block read in is smaller than BLOCK_SIZE,
-                # then we have reached the end of the file.
-                # blocks = how many blocks of tarfile.BLOCKSIZE fit in tarinfo.size
-                # remainder = how much more content is required to reach tarinfo.size
-                blocks: int
-                remainder: int
-                blocks, remainder = divmod(tarinfo.size, tarfile.BLOCKSIZE)
-                if remainder > 0:
-                    null_bytes: bytes = tarfile.NUL
-                    # Write null_bytes to get the last block to tarfile.BLOCKSIZE
-                    fileobj.write(null_bytes * (tarfile.BLOCKSIZE - remainder))
-                    blocks += 1
-                # Increase the offset by the amount already saved to the tar
-                tar.offset += blocks * tarfile.BLOCKSIZE
-                break
+            # Original logic for Python < 3.13 or empty files
+            if tar.fileobj is not None:
+                tar_fileobj: _io.BufferedWriter = tar.fileobj
+            else:
+                raise TypeError("Invalid tar.fileobj={}".format(tar.fileobj))
+            while True:
+                s: bytes = f.read(BLOCK_SIZE)
+                if len(s) > 0:
+                    # If the block read in is non-empty, write it to fileobj and update the hash
+                    tar_fileobj.write(s)
+                    hash_md5.update(s)
+                if len(s) < BLOCK_SIZE:
+                    # If the block read in is smaller than BLOCK_SIZE,
+                    # then we have reached the end of the file.
+                    # blocks = how many blocks of tarfile.BLOCKSIZE fit in tarinfo.size
+                    # remainder = how much more content is required to reach tarinfo.size
+                    blocks: int
+                    remainder: int
+                    blocks, remainder = divmod(tarinfo.size, tarfile.BLOCKSIZE)
+                    if remainder > 0:
+                        null_bytes: bytes = tarfile.NUL
+                        # Write null_bytes to get the last block to tarfile.BLOCKSIZE
+                        tar_fileobj.write(null_bytes * (tarfile.BLOCKSIZE - remainder))
+                        blocks += 1
+                    # Increase the offset by the amount already saved to the tar
+                    tar.offset += blocks * tarfile.BLOCKSIZE
+                    break
+            md5 = hash_md5.hexdigest()
         f.close()
-        md5 = hash_md5.hexdigest()
     size: int = tarinfo.size
     mtime: datetime = datetime.utcfromtimestamp(tarinfo.mtime)
     return offset, size, mtime, md5