merge develop into feature branch

Author: danielfromearth

.github/workflows/run_tests.yml

@@ -10,21 +10,25 @@ on:
     secrets:
       codecov_token:
         required: true
+      EARTHDATA_USERNAME:
+        description: 'NASA Earthdata username'
+        required: true
+      EARTHDATA_PASSWORD:
+        description: 'NASA Earthdata password'
+        required: true
   workflow_dispatch:
 
 jobs:
   build_and_test:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.10', '3.11', '3.12', '3.13' ]
+        python-version: [ '3.11', '3.12', '3.13', '3.14' ]
 
     name: Python ${{ matrix.python-version }} tests
     steps:
       - name: Checkout repository with LFS
         uses: actions/checkout@v5
-        with:
-          lfs: 'true'
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -42,12 +46,27 @@ jobs:
       - name: Run linting
         run: uv run ruff check ncompare
 
-      - name: Run tests and collect coverage
+      - name: Run unit tests with coverage
+        run: |
+          uv run pytest --python ${{ matrix.python-version }} -v -m "not integration" --cov=ncompare --cov-report=
+
+      - name: Run integration tests with coverage
+        env:
+          EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }}
+          EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
+        run: |
+          uv run pytest --python ${{ matrix.python-version }} -v -m integration --cov=ncompare --cov-append --cov-report=
+
+      - name: Generate combined coverage report
         run: |
-          uv run pytest --cov=ncompare --cov-report=xml:build/reports/coverage${{ matrix.python-version }}.xml
+          uv run coverage xml -o build/reports/coverage${{ matrix.python-version }}.xml
+          uv run coverage report
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+          files: build/reports/coverage${{ matrix.python-version }}.xml
+          flags: python-${{ matrix.python-version }}
+          name: python-${{ matrix.python-version }}
           verbose: true

.pre-commit-config.yaml

@@ -5,11 +5,11 @@ ci:
   autofix_prs: false # Comment "pre-commit.ci autofix" on a PR to trigger
 
 default_language_version:
-  python: python3.10
+  python: python3.11
 
 repos:
   - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.28.0
+    rev: v8.30.0
     hooks:
       - id: gitleaks
 
@@ -41,14 +41,14 @@ repos:
         exclude_types: ["jupyter", "text"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.13.3
+    rev: v0.14.10
     hooks:
       - id: ruff
         args: ["--fix", "--exit-non-zero-on-fix"]
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.18.2
+    rev: v1.19.1
     hooks:
       - id: mypy
 

CHANGELOG.md

@@ -10,6 +10,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 - update the default python version from 3.9 to 3.10 ([#323](https://github.com/nasa/stitchee/pull/323))([**@danielfromearth**](https://github.com/danielfromearth))
 - Implement uv dependency management and versioning with bump-my-version ([#???](https://github.com/nasa/stitchee/issues/???))([**@danielfromearth**](https://github.com/danielfromearth))
+- Download ICESat-2 ATL06 granules for integration testing instead of using git lfs ([#327](https://github.com/nasa/stitchee/pull/327))([**@danielfromearth**](https://github.com/danielfromearth))
 
 ## [1.13.1] - 2025-01-08
 

pyproject.toml

@@ -4,13 +4,13 @@ build-backend = "hatchling.build"
 
 [project]
 name = "ncompare"
-version = "1.14.0a26"
+version = "1.14.0a27"
 description = "Compare the structure of two NetCDF files at the command line"
 authors = [
     { name = "Daniel Kaufman", email = "daniel.kaufman@nasa.gov" }
 ]
 readme = "README.md"
-requires-python = ">=3.10,<4"
+requires-python = ">=3.11,<4"
 repository = "https://github.com/nasa/ncompare"
 license = "Apache-2.0"
 classifiers = [
@@ -19,10 +19,10 @@ classifiers = [
     "Operating System :: Microsoft :: Windows",
     "Operating System :: MacOS",
     "Operating System :: POSIX :: Linux",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 dependencies = [
@@ -52,6 +52,15 @@ dev = [
     "mkdocs-material>=9.5.12"
 ]
 
+[tool.pytest.ini_options]
+markers = [
+    "integration: marks tests as integration tests (downloads data, requires auth)",
+    "slow: marks tests as slow running",
+]
+# Skip integration tests by default
+addopts = "-v -m 'not integration'"
+testpaths = ["tests"]
+
 [[tool.mypy.overrides]]
 module = [
   "colorama.*",
@@ -70,7 +79,7 @@ exclude = [
 ]
 # Allow lines to be as long as 100.
 line-length = 100
-target-version = "py310"
+target-version = "py311"
 
 [tool.ruff.lint]
 # E402: module level import not at top of file

tests/conftest.py

@@ -23,8 +23,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+import os
 from pathlib import Path
 
+import earthaccess
 import netCDF4 as nC
 import numpy as np
 import pytest
@@ -33,6 +35,89 @@
 from ncompare.printing import Outputter
 
 
+@pytest.fixture(scope="session")
+def icesat2_cache_dir():
+    """Persistent cache directory for ICESat-2 test data."""
+    cache_dir = Path.home() / ".cache" / "icesat2_test_data"
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return cache_dir
+
+
+@pytest.fixture(scope="session")
+def earthdata_auth():
+    """
+    Authenticate with NASA Earthdata.
+    Uses credentials from environment variables or .netrc file.
+    """
+    # Check for environment variables (used in CI)
+    username = os.getenv("EARTHDATA_USERNAME")
+    password = os.getenv("EARTHDATA_PASSWORD")
+
+    if username and password:
+        earthaccess.login(strategy="environment")
+    else:
+        # Use .netrc or prompt for local testing
+        earthaccess.login()
+
+    return True
+
+
+@pytest.fixture(scope="session")
+def icesat2_atl06_granule_1(icesat2_cache_dir, earthdata_auth):
+    """
+    Download or use cached ICESat-2 ATL06 granule #1 for comparison tests.
+    Temporal range: 2023-08-16 16:16:15 to 2023-08-16 16:25:00
+    """
+    # Check if already cached
+    cached_files = list(icesat2_cache_dir.glob("ATL06_20230816161508_*.h5"))
+    if cached_files:
+        return str(cached_files[0])
+
+    # Download if not cached
+    results = earthaccess.search_data(
+        short_name="ATL06", temporal=("2023-08-16 16:16:15", "2023-08-16 16:25:00"), count=1
+    )
+
+    if not results:
+        pytest.skip("ICESat-2 granule #1 not found for test")
+
+    # Download the data
+    files = earthaccess.download(results, str(icesat2_cache_dir))
+
+    if not files:
+        pytest.skip("Failed to download ICESat-2 granule #1")
+
+    return files[0]
+
+
+@pytest.fixture(scope="session")
+def icesat2_atl06_granule_2(icesat2_cache_dir, earthdata_auth):
+    """
+    Download or use cached ICESat-2 ATL06 granule #2 for comparison tests.
+    Temporal range: 2023-08-16 23:46:00 to 2023-08-16 23:48:00
+    """
+    # Check if already cached
+    cached_files = list(icesat2_cache_dir.glob("ATL06_20230816234629_*.h5"))
+    if cached_files:
+        return str(cached_files[0])
+
+    # Download if not cached
+    results = earthaccess.search_data(
+        short_name="ATL06", temporal=("2023-08-16 23:46:00", "2023-08-16 23:48:00"), count=1
+    )
+
+    if not results:
+        pytest.skip("ICESat-2 granule #2 not found for test")
+
+    # Download the data
+    files = earthaccess.download(results, str(icesat2_cache_dir))
+
+    if not files:
+        pytest.skip("Failed to download ICESat-2 granule #2")
+
+    return files[0]
+
+
 @pytest.fixture(scope="session")
 def temp_data_dir(tmpdir_factory) -> Path:
     return Path(tmpdir_factory.mktemp("data"))

tests/test_core.py

@@ -74,18 +74,23 @@ def test_zero_for_comparison_with_no_differences(ds_3dims_3vars_4coords_1subgrou
     assert compare(ds_3dims_3vars_4coords_1subgroup, ds_3dims_3vars_4coords_1subgroup) == 0
 
 
-def test_icesat(temp_data_dir):
+@pytest.mark.integration
+def test_icesat(temp_data_dir, icesat2_atl06_granule_1, icesat2_atl06_granule_2):
     # Compare the `ncompare` output when testing ICESat
     out_path = temp_data_dir / "output_file_icesat-2-atl06.txt"
 
     num_differences = compare(
-        data_for_tests_dir / "icesat-2-ATL06" / "ATL06_20230816161508_08782002_006_02.h5",
-        data_for_tests_dir / "icesat-2-ATL06" / "ATL06_20230816234629_08822013_006_01.h5",
+        icesat2_atl06_granule_1,
+        icesat2_atl06_granule_2,
         show_chunks=True,
         show_attributes=True,
         file_text=str(out_path),
     )
 
+    # Verify that differences were found and output was written
+    assert num_differences > 0, "Expected to find differences between granules"
+    assert out_path.exists(), "Output file was not created"
+
     assert num_differences == 5280