Merge pull request #2924 from ASFHyP3/develop

jtherrmann · web-flow · commit 38fb80078d4d · 2025-10-02T10:03:42.000-08:00
Release v10.11.9
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,11 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [10.11.9]
+
+### Fixed
+- CMR outages no longer block jobs from being submitted, except for the `OPERA_RTC_S1` and `OPERA_RTC_S1_SLC` job types. Fixes https://github.com/ASFHyP3/hyp3/issues/2761
+
 ## [10.11.8]
 
 ### Changed
diff --git a/apps/api/src/hyp3_api/handlers.py b/apps/api/src/hyp3_api/handlers.py
@@ -1,6 +1,5 @@
 from http.client import responses
 
-import requests
 from flask import Response, abort, jsonify, request
 
 import dynamo
@@ -13,7 +12,7 @@
 )
 from hyp3_api import util
 from hyp3_api.multi_burst_validation import MultiBurstValidationError
-from hyp3_api.validation import ValidationError, validate_jobs
+from hyp3_api.validation import CmrError, ValidationError, validate_jobs
 
 
 def problem_format(status: int, message: str) -> Response:
@@ -28,9 +27,8 @@ def post_jobs(body: dict, user: str) -> dict:
 
     try:
         validate_jobs(body['jobs'])
-    except requests.HTTPError as e:
-        print(f'CMR search failed: {e}')
-        abort(problem_format(503, 'Could not submit jobs due to a CMR error. Please try again later.'))
+    except CmrError as e:
+        abort(problem_format(503, str(e)))
     except (ValidationError, MultiBurstValidationError) as e:
         abort(problem_format(400, str(e)))
 
diff --git a/apps/api/src/hyp3_api/validation.py b/apps/api/src/hyp3_api/validation.py
@@ -26,6 +26,12 @@ class ValidationError(Exception):
     pass
 
 
+class CmrError(Exception):
+    """Raised when the CMR query has failed and is required for the current job type."""
+
+    pass
+
+
 with (Path(__file__).parent / 'job_validation_map.yml').open() as job_validation_map_file:
     JOB_VALIDATION_MAP = yaml.safe_load(job_validation_map_file.read())
 
@@ -39,6 +45,9 @@ def _has_sufficient_coverage(granule: Polygon) -> bool:
 
 
 def _get_cmr_metadata(granules: Iterable[str]) -> list[dict]:
+    if not granules:
+        return []
+
     cmr_parameters = {
         'granule_ur': [f'{granule}*' for granule in granules],
         'options[granule_ur][pattern]': 'true',
@@ -61,15 +70,25 @@ def _get_cmr_metadata(granules: Iterable[str]) -> list[dict]:
         'page_size': 2000,
     }
     response = requests.post(CMR_URL, data=cmr_parameters)
-    response.raise_for_status()
-    return [
+
+    try:
+        response.raise_for_status()
+    except requests.HTTPError as e:
+        print(f'CMR search failed: {e}')
+        return []
+
+    granule_metadata = [
         {
             'name': entry.get('producer_granule_id', entry.get('title')),
             'polygon': Polygon(_format_points(entry['polygons'][0][0])),
         }
         for entry in response.json()['feed']['entry']
     ]
 
+    _make_sure_granules_exist(granules, granule_metadata)
+
+    return granule_metadata
+
 
 def _is_third_party_granule(granule: str) -> bool:
     return granule.startswith('S2') or granule.startswith('L')
@@ -83,6 +102,13 @@ def _make_sure_granules_exist(granules: Iterable[str], granule_metadata: list[di
         raise ValidationError(f'Some requested scenes could not be found: {", ".join(not_found_granules)}')
 
 
+def check_cmr_query_succeeded(job: dict, granule_metadata: list[dict]) -> None:
+    if not granule_metadata:
+        raise CmrError(
+            f'Cannot validate job(s) of type {job["job_type"]} because CMR query failed. Please try again later.'
+        )
+
+
 def check_dem_coverage(_, granule_metadata: list[dict]) -> None:
     bad_granules = [g['name'] for g in granule_metadata if not _has_sufficient_coverage(g['polygon'])]
     if bad_granules:
@@ -279,12 +305,7 @@ def check_opera_rtc_s1_date(job: dict, _) -> None:
 
 def validate_jobs(jobs: list[dict]) -> None:
     granules = get_granules(jobs)
-
-    if granules:
-        granule_metadata = _get_cmr_metadata(granules)
-        _make_sure_granules_exist(granules, granule_metadata)
-    else:
-        granule_metadata = []
+    granule_metadata = _get_cmr_metadata(granules)
 
     for job in jobs:
         for validator_name in JOB_VALIDATION_MAP[job['job_type']]:
diff --git a/job_spec/OPERA_RTC_S1.yml b/job_spec/OPERA_RTC_S1.yml
@@ -24,6 +24,7 @@ OPERA_RTC_S1:
       cost: 1.0
   validators:
     - check_opera_rtc_s1_date
+    - check_cmr_query_succeeded
     - check_opera_rtc_s1_bounds
   steps:
     - name: ''
diff --git a/job_spec/OPERA_RTC_S1_SLC.yml b/job_spec/OPERA_RTC_S1_SLC.yml
@@ -23,6 +23,7 @@ OPERA_RTC_S1_SLC:
     DEFAULT:
       cost: 1.0
   validators:
+    - check_cmr_query_succeeded
     - check_opera_rtc_s1_bounds
   steps:
     - name: ''
diff --git a/requirements-all.txt b/requirements-all.txt
@@ -5,15 +5,15 @@
 -r requirements-apps-disable-private-dns.txt
 -r requirements-apps-search-archive.txt
 -r requirements-apps-update-db.txt
-boto3==1.40.35
+boto3==1.40.40
 jinja2==3.1.6
 moto[dynamodb]==5.1.13
 pytest==8.4.2
-PyYAML==6.0.2
+PyYAML==6.0.3
 responses==0.25.8
-ruff==0.13.1
+ruff==0.13.2
 mypy==1.18.2
 setuptools==80.9.0
 setuptools_scm==9.2.0
 openapi-spec-validator==0.7.2
-cfn-lint==1.39.1
+cfn-lint==1.40.0
diff --git a/requirements-apps-api.txt b/requirements-apps-api.txt
@@ -6,6 +6,6 @@ prance==25.4.8.0
 PyJWT==2.10.1
 requests==2.32.5
 serverless_wsgi==3.1.0
-shapely==2.1.1
+shapely==2.1.2
 strict-rfc3339==0.7
 ./lib/dynamo/
diff --git a/requirements-apps-disable-private-dns.txt b/requirements-apps-disable-private-dns.txt
@@ -1 +1 @@
-boto3==1.40.35
+boto3==1.40.40
diff --git a/requirements-apps-get-files.txt b/requirements-apps-get-files.txt
@@ -1,2 +1,2 @@
-boto3==1.40.35
+boto3==1.40.40
 ./lib/dynamo/
diff --git a/requirements-apps-start-execution.txt b/requirements-apps-start-execution.txt
@@ -1,3 +1,3 @@
-boto3==1.40.35
+boto3==1.40.40
 ./lib/dynamo/
 ./lib/lambda_logging/
diff --git a/tests/test_api/test_opera_rtc_s1.py b/tests/test_api/test_opera_rtc_s1.py
@@ -2,9 +2,10 @@
 from unittest.mock import MagicMock
 
 import pytest
+import responses
 
 import hyp3_api.validation
-from test_api.conftest import JOBS_URI, login
+from test_api.conftest import CMR_URL_RE, JOBS_URI, login
 
 
 @pytest.mark.network
@@ -284,3 +285,29 @@ def test_opera_rtc_s1_nonexistent_granule(client, tables, approved_user):
         == 'Some requested scenes could not be found: S1_157213_IW3_20190210T182738_HH_D6C7-BURST'
     )
     assert len(tables.jobs_table.scan()['Items']) == 0
+
+
+@responses.activate
+def test_opera_rtc_s1_cmr_error(client, tables, approved_user):
+    login(client, username=approved_user)
+
+    responses.post(url=CMR_URL_RE, status=500)
+
+    response = client.post(
+        JOBS_URI,
+        json={
+            'jobs': [
+                {
+                    'job_type': 'OPERA_RTC_S1',
+                    'job_parameters': {'granules': ['S1_018641_IW1_20180307T081710_HH_20C9-BURST']},
+                }
+            ],
+        },
+    )
+
+    assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
+    assert (
+        response.json['detail']
+        == 'Cannot validate job(s) of type OPERA_RTC_S1 because CMR query failed. Please try again later.'
+    )
+    assert len(tables.jobs_table.scan()['Items']) == 0
diff --git a/tests/test_api/test_submit_job.py b/tests/test_api/test_submit_job.py
@@ -6,7 +6,7 @@
 import hyp3_api.util
 from dynamo.user import APPLICATION_PENDING
 from dynamo.util import current_utc_time
-from test_api.conftest import CMR_URL_RE, JOBS_URI, login, setup_mock_cmr_response_for_polygons
+from test_api.conftest import JOBS_URI, login, setup_mock_cmr_response_for_polygons
 
 
 def make_job(granules=None, name='someName', job_type='RTC_GAMMA', parameters=None):
@@ -485,14 +485,3 @@ def test_submit_validate_only(client, tables, approved_user):
     assert response.status_code == HTTPStatus.OK
     jobs = tables.jobs_table.scan()['Items']
     assert len(jobs) == 2
-
-
-@responses.activate
-def test_cmr_error(client, tables, approved_user):
-    login(client, username=approved_user)
-    responses.post(url=CMR_URL_RE, status=500)
-
-    response = submit_batch(client, [make_job()])
-    assert response.status_code == HTTPStatus.SERVICE_UNAVAILABLE
-    assert response.json['detail'] == 'Could not submit jobs due to a CMR error. Please try again later.'
-    assert len(tables.jobs_table.scan()['Items']) == 0
diff --git a/tests/test_api/test_validation.py b/tests/test_api/test_validation.py
@@ -28,6 +28,8 @@ def test_not_antimeridian():
     with pytest.raises(validation.ValidationError, match=r'.*crosses the antimeridian.*'):
         validation.check_not_antimeridian({}, bad_granules)
 
+    validation.check_not_antimeridian({}, [])
+
 
 def test_has_sufficient_coverage():
     # Wyoming
@@ -221,6 +223,9 @@ def test_make_sure_granules_exist():
     validation._make_sure_granules_exist(['scene1'], granule_metadata)
     validation._make_sure_granules_exist(['scene1', 'scene2'], granule_metadata)
 
+    with pytest.raises(validation.ValidationError, match=r'.*scene1$'):
+        validation._make_sure_granules_exist(['scene1'], [])
+
     with pytest.raises(validation.ValidationError) as e:
         validation._make_sure_granules_exist(
             ['scene1', 'scene2', 'scene3', 'scene4', 'S2_foo', 'LC08_bar', 'LC09_bar'], granule_metadata
@@ -250,6 +255,9 @@ def test_is_third_party_granule():
 
 @responses.activate
 def test_get_cmr_metadata():
+    assert validation._get_cmr_metadata([]) == []
+    assert validation._get_cmr_metadata(set()) == []
+
     response_payload = {
         'feed': {
             'entry': [
@@ -266,7 +274,7 @@ def test_get_cmr_metadata():
     }
     responses.post(CMR_URL, json=response_payload)
 
-    assert validation._get_cmr_metadata(['foo', 'bar', 'hello']) == [
+    assert validation._get_cmr_metadata(['foo', 'bar']) == [
         {
             'name': 'foo',
             'polygon': Polygon([[25.0, -31.4], [25.5, -29.7], [24.6, -29.5], [24.1, -31.2]]),
@@ -277,6 +285,12 @@ def test_get_cmr_metadata():
         },
     ]
 
+    with pytest.raises(validation.ValidationError, match=r'.*could not be found: hello$'):
+        validation._get_cmr_metadata(['foo', 'bar', 'hello'])
+
+    responses.post(CMR_URL, status=500)
+    assert validation._get_cmr_metadata(['foo', 'bar']) == []
+
 
 @responses.activate
 def test_validate_jobs():
@@ -358,6 +372,9 @@ def test_validate_jobs():
     with pytest.raises(validation.ValidationError):
         validation.validate_jobs(jobs)
 
+    responses.post(CMR_URL, status=500)
+    validation.validate_jobs(jobs)
+
 
 def test_all_validators_have_correct_signature():
     validators = [getattr(validation, attr) for attr in dir(validation) if attr.startswith('check_')]
@@ -433,6 +450,8 @@ def test_check_granules_intersecting_bounds():
     with pytest.raises(validation.ValidationError, match=error_pattern):
         validation.check_granules_intersecting_bounds(job_with_specified_bounds, invalid_granule_metadata)
 
+    validation.check_granules_intersecting_bounds(job_with_specified_bounds, [])
+
 
 def test_check_same_relative_orbits():
     valid_granule_metadata = [
@@ -443,11 +462,15 @@ def test_check_same_relative_orbits():
     ]
     invalid_granule_metadata = valid_granule_metadata.copy()
     invalid_granule_metadata.append({'name': 'S1B_IW_RAW__0SDV_20200623T161535_20200623T161607_012345_02A10F_7FD6'})
+
     validation.check_same_relative_orbits({}, valid_granule_metadata)
+
     error_pattern = r'.*69 is not 87.*'
     with pytest.raises(validation.ValidationError, match=error_pattern):
         validation.check_same_relative_orbits({}, invalid_granule_metadata)
 
+    validation.check_same_relative_orbits({}, [])
+
 
 def test_check_bounding_box_size():
     job = {'job_parameters': {'bounds': [0, 0, 10, 10]}}
@@ -589,3 +612,13 @@ def test_check_opera_rtc_s1_date_max_configurable(monkeypatch):
 def test_check_aria_s1_gunw_dates(job_parameters, error):
     with error:
         validation.check_aria_s1_gunw_dates({'job_parameters': job_parameters}, None)
+
+
+def test_check_cmr_query_succeeded():
+    validation.check_cmr_query_succeeded({'job_type': 'FOO'}, [{}])
+
+    with pytest.raises(
+        validation.CmrError,
+        match=r'^Cannot validate job\(s\) of type FOO because CMR query failed\. Please try again later\.$',
+    ):
+        validation.check_cmr_query_succeeded({'job_type': 'FOO'}, [])

Original file line number	Diff line number	Diff line change
`@@ -1,2 +1,2 @@`
`1`		`-boto3==1.40.35`
	`1`	`+boto3==1.40.40`
`2`	`2`	`./lib/dynamo/`