Replace git LFS test data with earthaccess integration tests

Add earthaccess-based fixtures to download ICESat-2 ATL06 granules
for integration testing. Granules are cached locally to avoid repeated
downloads. Update CI workflow to support integration tests with NASA
Earthdata credentials.

Author: danielfromearth

.github/workflows/reusable_run_tests.yml

@@ -10,6 +10,12 @@ on:
     secrets:
       codecov_token:
         required: true
+      EARTHDATA_USERNAME:
+        description: 'NASA Earthdata username'
+        required: false
+      EARTHDATA_PASSWORD:
+        description: 'NASA Earthdata password'
+        required: false
   workflow_dispatch:
 
 env:
@@ -45,9 +51,13 @@ jobs:
         run: |
           poetry run ruff check ncompare
 
-      - name: Run tests with coverage
+      - name: Run unit tests with coverage
         run: |
-          poetry run pytest --cov=ncompare --cov-report=xml:build/reports/coverage${{ matrix.python-version }}.xml
+          poetry run pytest -v -m "not integration" --cov=ncompare --cov-report=xml:build/reports/coverage${{ matrix.python-version }}.xml
+
+      - name: Run integration tests with coverage
+        run: |
+          poetry run pytest -v -m integration --cov=ncompare --cov-report=xml:build/reports/coverage${{ matrix.python-version }}.xml
 
       - name: Upload coverage reports to Codecov
         uses: codecov/codecov-action@v5

poetry.lock

@@ -31,6 +31,7 @@ colorama = ">=0.4.6"
 openpyxl = ">=3.1.2"
 h5py = ">=3.12.1"
 h5netcdf = ">=1.4.1"
+earthaccess = ">=0.15.1"
 
 [tool.poetry.group.dev.dependencies]
 pytest = ">=7.4.2,<9.0.0"
@@ -48,6 +49,15 @@ mkdocs-material = ">=9.5.12"
 requires = ["poetry-core>=1.0.0"]
 build-backend = "poetry.core.masonry.api"
 
+[tool.pytest.ini_options]
+markers = [
+    "integration: marks tests as integration tests (downloads data, requires auth)",
+    "slow: marks tests as slow running",
+]
+# Skip integration tests by default
+addopts = "-v -m 'not integration'"
+testpaths = ["tests"]
+
 [[tool.mypy.overrides]]
 module = [
   "colorama.*",

pyproject.toml

@@ -23,8 +23,10 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and limitations under the License.
 
+import os
 from pathlib import Path
 
+import earthaccess
 import netCDF4 as nC
 import numpy as np
 import pytest
@@ -33,6 +35,89 @@
 from ncompare.printing import Outputter
 
 
+@pytest.fixture(scope="session")
+def icesat2_cache_dir():
+    """Persistent cache directory for ICESat-2 test data."""
+    cache_dir = Path.home() / ".cache" / "icesat2_test_data"
+    cache_dir.mkdir(parents=True, exist_ok=True)
+    return cache_dir
+
+
+@pytest.fixture(scope="session")
+def earthdata_auth():
+    """
+    Authenticate with NASA Earthdata.
+    Uses credentials from environment variables or .netrc file.
+    """
+    # Check for environment variables (used in CI)
+    username = os.getenv("EARTHDATA_USERNAME")
+    password = os.getenv("EARTHDATA_PASSWORD")
+
+    if username and password:
+        earthaccess.login(strategy="environment")
+    else:
+        # Use .netrc or prompt for local testing
+        earthaccess.login()
+
+    return True
+
+
+@pytest.fixture(scope="session")
+def icesat2_atl06_granule_1(icesat2_cache_dir, earthdata_auth):
+    """
+    Download or use cached ICESat-2 ATL06 granule #1 for comparison tests.
+    Temporal range: 2023-08-16 16:16:15 to 2023-08-16 16:25:00
+    """
+    # Check if already cached
+    cached_files = list(icesat2_cache_dir.glob("ATL06_20230816161508_*.h5"))
+    if cached_files:
+        return str(cached_files[0])
+
+    # Download if not cached
+    results = earthaccess.search_data(
+        short_name="ATL06", temporal=("2023-08-16 16:16:15", "2023-08-16 16:25:00"), count=1
+    )
+
+    if not results:
+        pytest.skip("ICESat-2 granule #1 not found for test")
+
+    # Download the data
+    files = earthaccess.download(results, str(icesat2_cache_dir))
+
+    if not files:
+        pytest.skip("Failed to download ICESat-2 granule #1")
+
+    return files[0]
+
+
+@pytest.fixture(scope="session")
+def icesat2_atl06_granule_2(icesat2_cache_dir, earthdata_auth):
+    """
+    Download or use cached ICESat-2 ATL06 granule #2 for comparison tests.
+    Temporal range: 2023-08-16 23:46:00 to 2023-08-16 23:48:00
+    """
+    # Check if already cached
+    cached_files = list(icesat2_cache_dir.glob("ATL06_20230816234629_*.h5"))
+    if cached_files:
+        return str(cached_files[0])
+
+    # Download if not cached
+    results = earthaccess.search_data(
+        short_name="ATL06", temporal=("2023-08-16 23:46:00", "2023-08-16 23:48:00"), count=1
+    )
+
+    if not results:
+        pytest.skip("ICESat-2 granule #2 not found for test")
+
+    # Download the data
+    files = earthaccess.download(results, str(icesat2_cache_dir))
+
+    if not files:
+        pytest.skip("Failed to download ICESat-2 granule #2")
+
+    return files[0]
+
+
 @pytest.fixture(scope="session")
 def temp_data_dir(tmpdir_factory) -> Path:
     return Path(tmpdir_factory.mktemp("data"))

tests/conftest.py

@@ -74,18 +74,23 @@ def test_zero_for_comparison_with_no_differences(ds_3dims_3vars_4coords_1subgrou
     assert compare(ds_3dims_3vars_4coords_1subgroup, ds_3dims_3vars_4coords_1subgroup) == 0
 
 
-def test_icesat(temp_data_dir):
+@pytest.mark.integration
+def test_icesat(temp_data_dir, icesat2_atl06_granule_1, icesat2_atl06_granule_2):
     # Compare the `ncompare` output when testing ICESat
     out_path = temp_data_dir / "output_file_icesat-2-atl06.txt"
 
     num_differences = compare(
-        data_for_tests_dir / "icesat-2-ATL06" / "ATL06_20230816161508_08782002_006_02.h5",
-        data_for_tests_dir / "icesat-2-ATL06" / "ATL06_20230816234629_08822013_006_01.h5",
+        icesat2_atl06_granule_1,
+        icesat2_atl06_granule_2,
         show_chunks=True,
         show_attributes=True,
         file_text=str(out_path),
     )
 
+    # Verify that differences were found and output was written
+    assert num_differences > 0, "Expected to find differences between granules"
+    assert out_path.exists(), "Output file was not created"
+
     assert num_differences == 5280