Merge pull request #327 from nasa/feature/issue-326-programmatically-download-test-files-for-comparisons

Programmatically download IceSAT-2 test files instead of git lfs

Author: danielfromearth

.github/workflows/build-pipeline.yml

@@ -11,13 +11,15 @@ on:
 
 env:
   POETRY_VERSION: "2.1.3"
-  PYTHON_VERSION: "3.10"
+  PYTHON_VERSION: "3.11"
 
 jobs:
   run_tests:
     uses: ./.github/workflows/reusable_run_tests.yml
     secrets:
       codecov_token: ${{ secrets.CODECOV_TOKEN }}
+      EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }}
+      EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
 
   bump_version:
     needs: run_tests

.github/workflows/release_created.yml

@@ -6,7 +6,7 @@ on:
 
 env:
   POETRY_VERSION: "2.1.3"
-  PYTHON_VERSION: "3.10"
+  PYTHON_VERSION: "3.11"
 
 jobs:
   # First job in the workflow builds and verifies the software artifacts

.github/workflows/reusable_run_tests.yml

@@ -10,6 +10,12 @@ on:
     secrets:
       codecov_token:
         required: true
+      EARTHDATA_USERNAME:
+        description: 'NASA Earthdata username'
+        required: true
+      EARTHDATA_PASSWORD:
+        description: 'NASA Earthdata password'
+        required: true
   workflow_dispatch:
 
 env:
@@ -20,13 +26,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        python-version: [ '3.10', '3.11', '3.12', '3.13' ]
+        python-version: [ '3.11', '3.12', '3.13', '3.14' ]
 
     name: Python ${{ matrix.python-version }} tests
     steps:
       - uses: actions/checkout@v5
-        with:
-          lfs: 'true'
 
       - name: Set up Python
         uses: actions/setup-python@v5
@@ -45,12 +49,27 @@ jobs:
         run: |
           poetry run ruff check ncompare
 
-      - name: Run tests with coverage
+      - name: Run unit tests with coverage
+        run: |
+          poetry run pytest -v -m "not integration" --cov=ncompare --cov-report=
+
+      - name: Run integration tests with coverage
+        env:
+          EARTHDATA_USERNAME: ${{ secrets.EARTHDATA_USERNAME }}
+          EARTHDATA_PASSWORD: ${{ secrets.EARTHDATA_PASSWORD }}
+        run: |
+          poetry run pytest -v -m integration --cov=ncompare --cov-append --cov-report=
+
+      - name: Generate combined coverage report
         run: |
-          poetry run pytest --cov=ncompare --cov-report=xml:build/reports/coverage${{ matrix.python-version }}.xml
+          poetry run coverage xml -o build/reports/coverage${{ matrix.python-version }}.xml
+          poetry run coverage report
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5
         with:
           token: ${{ secrets.CODECOV_TOKEN }}
+          files: build/reports/coverage${{ matrix.python-version }}.xml
+          flags: python-${{ matrix.python-version }}
+          name: python-${{ matrix.python-version }}
           verbose: true

.pre-commit-config.yaml

@@ -5,11 +5,11 @@ ci:
   autofix_prs: false # Comment "pre-commit.ci autofix" on a PR to trigger
 
 default_language_version:
-  python: python3.10
+  python: python3.11
 
 repos:
   - repo: https://github.com/gitleaks/gitleaks
-    rev: v8.28.0
+    rev: v8.30.0
     hooks:
       - id: gitleaks
 
@@ -41,14 +41,14 @@ repos:
         exclude_types: ["jupyter", "text"]
 
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.13.3
+    rev: v0.14.10
     hooks:
       - id: ruff
         args: ["--fix", "--exit-non-zero-on-fix"]
       - id: ruff-format
 
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.18.2
+    rev: v1.19.1
     hooks:
       - id: mypy
 

CHANGELOG.md

@@ -9,6 +9,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - update the default python version from 3.9 to 3.10 ([#323](https://github.com/nasa/stitchee/pull/323))([**@danielfromearth**](https://github.com/danielfromearth))
+- Download ICESat-2 ATL06 granules for integration testing instead of using git lfs ([#327](https://github.com/nasa/stitchee/pull/327))([**@danielfromearth**](https://github.com/danielfromearth))
 
 ## [1.13.1] - 2025-01-08
 

poetry.lock

@@ -12,25 +12,26 @@ classifiers = [
     "Operating System :: Microsoft :: Windows",
     "Operating System :: MacOS",
     "Operating System :: POSIX :: Linux",
-    "Programming Language :: Python :: 3.10",
     "Programming Language :: Python :: 3.11",
     "Programming Language :: Python :: 3.12",
     "Programming Language :: Python :: 3.13",
+    "Programming Language :: Python :: 3.14",
     "Topic :: Software Development :: Libraries :: Python Modules",
 ]
 
 [tool.poetry.scripts]
 ncompare = "ncompare.console:main"
 
 [tool.poetry.dependencies]
-python = "^3.10"
+python = "^3.11"
 numpy = ">=1.26.0"
 netCDF4 = ">=1.6.4"
 xarray = ">=2023.9"
 colorama = ">=0.4.6"
 openpyxl = ">=3.1.2"
 h5py = ">=3.12.1"
 h5netcdf = ">=1.4.1"
+earthaccess = ">=0.15.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.2,<9.0.0"
@@ -48,6 +49,15 @@ mkdocs-material = ">=9.5.12"
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
+[tool.pytest.ini_options]
+markers = [
+    "integration: marks tests as integration tests (downloads data, requires auth)",
+    "slow: marks tests as slow running",
+]
+# Skip integration tests by default
+addopts = "-v -m 'not integration'"
+testpaths = ["tests"]
+
 [[tool.mypy.overrides]]
 module = [
   "colorama.*",
@@ -66,7 +76,7 @@ exclude = [
 ]
 # Allow lines to be as long as 100.
 line-length = 100
-target-version = "py310"
+target-version = "py311"
 
 [tool.ruff.lint]
 # E402: module level import not at top of file

pyproject.toml

@@ -23,8 +23,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+import os
 from pathlib import Path
 
+import earthaccess
 import netCDF4 as nC
 import numpy as np
 import pytest
@@ -33,6 +35,89 @@
 from ncompare.printing import Outputter
 
 
+@pytest.fixture(scope="session")
+def icesat2_cache_dir():
+    """Persistent cache directory for ICESat-2 test data."""
+    cache_dir = Path.home() / ".cache" / "icesat2_test_data"
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return cache_dir
+
+
+@pytest.fixture(scope="session")
+def earthdata_auth():
+    """
+    Authenticate with NASA Earthdata.
+    Uses credentials from environment variables or .netrc file.
+    """
+    # Check for environment variables (used in CI)
+    username = os.getenv("EARTHDATA_USERNAME")
+    password = os.getenv("EARTHDATA_PASSWORD")
+
+    if username and password:
+        earthaccess.login(strategy="environment")
+    else:
+        # Use .netrc or prompt for local testing
+        earthaccess.login()
+
+    return True
+
+
+@pytest.fixture(scope="session")
+def icesat2_atl06_granule_1(icesat2_cache_dir, earthdata_auth):
+    """
+    Download or use cached ICESat-2 ATL06 granule #1 for comparison tests.
+    Temporal range: 2023-08-16 16:16:15 to 2023-08-16 16:25:00
+    """
+    # Check if already cached
+    cached_files = list(icesat2_cache_dir.glob("ATL06_20230816161508_*.h5"))
+    if cached_files:
+        return str(cached_files[0])
+
+    # Download if not cached
+    results = earthaccess.search_data(
+        short_name="ATL06", temporal=("2023-08-16 16:16:15", "2023-08-16 16:25:00"), count=1
+    )
+
+    if not results:
+        pytest.skip("ICESat-2 granule #1 not found for test")
+
+    # Download the data
+    files = earthaccess.download(results, str(icesat2_cache_dir))
+
+    if not files:
+        pytest.skip("Failed to download ICESat-2 granule #1")
+
+    return files[0]
+
+
+@pytest.fixture(scope="session")
+def icesat2_atl06_granule_2(icesat2_cache_dir, earthdata_auth):
+    """
+    Download or use cached ICESat-2 ATL06 granule #2 for comparison tests.
+    Temporal range: 2023-08-16 23:46:00 to 2023-08-16 23:48:00
+    """
+    # Check if already cached
+    cached_files = list(icesat2_cache_dir.glob("ATL06_20230816234629_*.h5"))
+    if cached_files:
+        return str(cached_files[0])
+
+    # Download if not cached
+    results = earthaccess.search_data(
+        short_name="ATL06", temporal=("2023-08-16 23:46:00", "2023-08-16 23:48:00"), count=1
+    )
+
+    if not results:
+        pytest.skip("ICESat-2 granule #2 not found for test")
+
+    # Download the data
+    files = earthaccess.download(results, str(icesat2_cache_dir))
+
+    if not files:
+        pytest.skip("Failed to download ICESat-2 granule #2")
+
+    return files[0]
+
+
 @pytest.fixture(scope="session")
 def temp_data_dir(tmpdir_factory) -> Path:
     return Path(tmpdir_factory.mktemp("data"))

tests/conftest.py

@@ -74,18 +74,23 @@ def test_zero_for_comparison_with_no_differences(ds_3dims_3vars_4coords_1subgrou
     assert compare(ds_3dims_3vars_4coords_1subgroup, ds_3dims_3vars_4coords_1subgroup) == 0
 
 
-def test_icesat(temp_data_dir):
+@pytest.mark.integration
+def test_icesat(temp_data_dir, icesat2_atl06_granule_1, icesat2_atl06_granule_2):
     # Compare the `ncompare` output when testing ICESat
     out_path = temp_data_dir / "output_file_icesat-2-atl06.txt"
 
     num_differences = compare(
-        data_for_tests_dir / "icesat-2-ATL06" / "ATL06_20230816161508_08782002_006_02.h5",
-        data_for_tests_dir / "icesat-2-ATL06" / "ATL06_20230816234629_08822013_006_01.h5",
+        icesat2_atl06_granule_1,
+        icesat2_atl06_granule_2,
         show_chunks=True,
         show_attributes=True,
         file_text=str(out_path),
     )
 
+    # Verify that differences were found and output was written
+    assert num_differences > 0, "Expected to find differences between granules"
+    assert out_path.exists(), "Output file was not created"
+
     assert num_differences == 5280