Merge pull request #2810 from ASFHyP3/develop

Release v10.8.0

Author: asjohnston-asf

CHANGELOG.md

@@ -4,6 +4,19 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [10.8.0]
+
+### Added
+- `cloudformation:DeleteStack` permissions to the [HyP3 deployment policy](cicd-stacks/JPL-deployment-policy-cf.yml) for JPL accounts
+
+### Changed
+- `ARIA_S1_GUNW` now takes `reference_date` and `secondary_date` as inputs instead of `reference` and `secondary` granule lists
+- `ARIA_S1_GUNW` jobs now enforce minimum frame coverage of `0.9`.
+- `OPERA_RTC_S1` processing bounds have been expanded to scenes north of (or intersecting) -60 degrees latitude.
+
+### Removed
+- DEM bounds check for OPERA_RTC_S1 job type since it uses a different DEM.
+
 ## [10.7.0]
 
 ### Added
@@ -82,7 +95,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 - `render-cf.py` now determines the version number to report in the API from the git history and tags using [`setuptools_scm`](https://pypi.org/project/setuptools-scm/).
-  
+
 > [!WARNING]
 > In CI/CD pipelines, to dynamically calculate the version number you must now check out the full history and tags (no shallow clones). In GitHub Actions, this usually looks like specifying `fetch-depth: 0` with `actions/checkout`. For pipelines where you *do not* care about an accurate version number, you can still use a shallow clone by setting the `SETUPTOOLS_SCM_PRETEND_VERSION_FOR_HYP3` environment variable, see: <http://setuptools-scm.readthedocs.io/en/latest/overrides/>.
 

apps/api/src/hyp3_api/handlers.py

@@ -13,7 +13,7 @@
 )
 from hyp3_api import util
 from hyp3_api.multi_burst_validation import MultiBurstValidationError
-from hyp3_api.validation import BoundsValidationError, GranuleValidationError, validate_jobs
+from hyp3_api.validation import ValidationError, validate_jobs
 
 
 def problem_format(status: int, message: str) -> Response:
@@ -31,7 +31,7 @@ def post_jobs(body: dict, user: str) -> dict:
     except requests.HTTPError as e:
         print(f'CMR search failed: {e}')
         abort(problem_format(503, 'Could not submit jobs due to a CMR error. Please try again later.'))
-    except (BoundsValidationError, GranuleValidationError, MultiBurstValidationError) as e:
+    except (ValidationError, MultiBurstValidationError) as e:
         abort(problem_format(400, str(e)))
 
     try:

apps/api/src/hyp3_api/validation.py

@@ -7,7 +7,7 @@
 
 import requests
 import yaml
-from shapely.geometry import MultiPolygon, Polygon, shape
+from shapely.geometry import MultiPolygon, Polygon, box, shape
 
 from hyp3_api import CMR_URL, multi_burst_validation
 from hyp3_api.util import get_granules
@@ -22,11 +22,7 @@ class InternalValidationError(Exception):
     pass
 
 
-class GranuleValidationError(Exception):
-    pass
-
-
-class BoundsValidationError(Exception):
+class ValidationError(Exception):
     pass
 
 
@@ -84,13 +80,13 @@ def _make_sure_granules_exist(granules: Iterable[str], granule_metadata: list[di
     not_found_granules = set(granules) - set(found_granules)
     not_found_granules = {granule for granule in not_found_granules if not _is_third_party_granule(granule)}
     if not_found_granules:
-        raise GranuleValidationError(f'Some requested scenes could not be found: {", ".join(not_found_granules)}')
+        raise ValidationError(f'Some requested scenes could not be found: {", ".join(not_found_granules)}')
 
 
 def check_dem_coverage(_, granule_metadata: list[dict]) -> None:
     bad_granules = [g['name'] for g in granule_metadata if not _has_sufficient_coverage(g['polygon'])]
     if bad_granules:
-        raise GranuleValidationError(f'Some requested scenes do not have DEM coverage: {", ".join(bad_granules)}')
+        raise ValidationError(f'Some requested scenes do not have DEM coverage: {", ".join(bad_granules)}')
 
 
 def check_multi_burst_pairs(job: dict, _) -> None:
@@ -114,17 +110,17 @@ def check_single_burst_pair(job: dict, _) -> None:
     granule2_id = '_'.join(granule2.split('_')[1:3])
 
     if granule1_id != granule2_id:
-        raise GranuleValidationError(f'Burst IDs do not match for {granule1} and {granule2}.')
+        raise ValidationError(f'Burst IDs do not match for {granule1} and {granule2}.')
 
     granule1_pol = granule1.split('_')[4]
     granule2_pol = granule2.split('_')[4]
 
     if granule1_pol != granule2_pol:
-        raise GranuleValidationError(
+        raise ValidationError(
             f'The requested scenes need to have the same polarization, got: {", ".join([granule1_pol, granule2_pol])}'
         )
     if granule1_pol not in ['VV', 'HH']:
-        raise GranuleValidationError(f'Only VV and HH polarizations are currently supported, got: {granule1_pol}')
+        raise ValidationError(f'Only VV and HH polarizations are currently supported, got: {granule1_pol}')
 
 
 def check_not_antimeridian(_, granule_metadata: list[dict]) -> None:
@@ -135,7 +131,7 @@ def check_not_antimeridian(_, granule_metadata: list[dict]) -> None:
                 f'Granule {granule["name"]} crosses the antimeridian.'
                 ' Processing across the antimeridian is not currently supported.'
             )
-            raise GranuleValidationError(msg)
+            raise ValidationError(msg)
 
 
 def _format_points(point_string: str) -> list:
@@ -155,10 +151,10 @@ def _get_multipolygon_from_geojson(input_file: str) -> MultiPolygon:
 def check_bounds_formatting(job: dict, _) -> None:
     bounds = job['job_parameters']['bounds']
     if bounds == [0.0, 0.0, 0.0, 0.0]:
-        raise BoundsValidationError('Invalid bounds. Bounds cannot be [0, 0, 0, 0].')
+        raise ValidationError('Invalid bounds. Bounds cannot be [0, 0, 0, 0].')
 
     if bounds[0] >= bounds[2] or bounds[1] >= bounds[3]:
-        raise BoundsValidationError(
+        raise ValidationError(
             'Invalid order for bounds. Bounds should be ordered [min lon, min lat, max lon, max lat].'
         )
 
@@ -169,15 +165,15 @@ def bad_lon(lon: float) -> bool:
         return lon > 180 or lon < -180
 
     if any([bad_lon(bounds[0]), bad_lon(bounds[2]), bad_lat(bounds[1]), bad_lat(bounds[3])]):
-        raise BoundsValidationError(
+        raise ValidationError(
             'Invalid lon/lat value(s) in bounds. Bounds should be ordered [min lon, min lat, max lon, max lat].'
         )
 
 
 def check_granules_intersecting_bounds(job: dict, granule_metadata: list[dict]) -> None:
     bounds = job['job_parameters']['bounds']
     if bounds == [0.0, 0.0, 0.0, 0.0]:
-        raise BoundsValidationError('Invalid bounds. Bounds cannot be [0, 0, 0, 0].')
+        raise ValidationError('Invalid bounds. Bounds cannot be [0, 0, 0, 0].')
 
     bounds = Polygon.from_bounds(*bounds)
     bad_granules = []
@@ -186,7 +182,7 @@ def check_granules_intersecting_bounds(job: dict, granule_metadata: list[dict])
         if not bbox.intersection(bounds):
             bad_granules.append(granule['name'])
     if bad_granules:
-        raise GranuleValidationError(f'The following granules do not intersect the provided bounds: {bad_granules}.')
+        raise ValidationError(f'The following granules do not intersect the provided bounds: {bad_granules}.')
 
 
 def check_same_relative_orbits(_, granule_metadata: list[dict]) -> None:
@@ -200,7 +196,7 @@ def check_same_relative_orbits(_, granule_metadata: list[dict]) -> None:
         if not previous_relative_orbit:
             previous_relative_orbit = relative_orbit
         if relative_orbit != previous_relative_orbit:
-            raise GranuleValidationError(
+            raise ValidationError(
                 f'Relative orbit number for {granule["name"]} does not match that of the previous granules: '
                 f'{relative_orbit} is not {previous_relative_orbit}.'
             )
@@ -212,32 +208,43 @@ def check_bounding_box_size(job: dict, _, max_bounds_area: float = 4.5) -> None:
     bounds_area = (bounds[3] - bounds[1]) * (bounds[2] - bounds[0])
 
     if bounds_area > max_bounds_area:
-        raise BoundsValidationError(
+        raise ValidationError(
             f'Bounds must be smaller than {max_bounds_area} degrees squared. Box provided was {bounds_area:.2f}'
         )
 
 
-def _has_opera_rtc_s1_static_coverage(granule_name: str) -> bool:
-    burst_number, swath = granule_name.split('_')[1:3]
-    params = {
-        'short_name': 'OPERA_L2_RTC-S1-STATIC_V1',
-        'granule_ur': f'OPERA_L2_RTC-S1-STATIC_T*-{burst_number}-{swath}_*',
-        'options[granule_ur][pattern]': 'true',
-    }
-    response = requests.get(CMR_URL, params=params)
-    response.raise_for_status()
-    return bool(response.json()['feed']['entry'])
+def check_opera_rtc_s1_bounds(_, granule_metadata: list[dict]) -> None:
+    opera_rtc_s1_bounds = box(-180, -60, 180, 90)
+    for granule in granule_metadata:
+        if not granule['polygon'].intersects(opera_rtc_s1_bounds):
+            raise ValidationError(
+                f'Granule {granule["name"]} is south of -60 degrees latitude and outside the valid processing extent '
+                f'for OPERA RTC-S1 products.'
+            )
 
 
-def check_opera_rtc_s1_static_coverage(job: dict, _) -> None:
-    granules = job['job_parameters']['granules']
-    if len(granules) != 1:
-        raise InternalValidationError(f'Expected 1 granule, got {granules}')
+def check_aria_s1_gunw_dates(job: dict, _) -> None:
+    def format_date(key: str) -> date:
+        return date.fromisoformat(job['job_parameters'][key])
 
-    granule = granules[0]
-    if not _has_opera_rtc_s1_static_coverage(granule):
-        raise GranuleValidationError(
-            f'Granule {granule} is outside the valid processing extent for OPERA RTC-S1 products.'
+    reference, secondary = format_date('reference_date'), format_date('secondary_date')
+    _validate_date_during_s1('reference_date', reference)
+    _validate_date_during_s1('secondary_date', secondary)
+
+    if secondary >= reference:
+        raise ValidationError('secondary date must be earlier than reference date.')
+
+
+def _validate_date_during_s1(date_name: str, date_value: date) -> None:
+    s1_start_date = date(2014, 6, 15)
+    todays_date = date.today()
+
+    if date_value > todays_date:
+        raise ValidationError(f'"{date_name}" is {date_value} which is a date in the future.')
+
+    if date_value < s1_start_date:
+        raise ValidationError(
+            f'"{date_name}" is {date_value} which is before the start of the sentinel 1 mission ({s1_start_date}).'
         )
 
 
@@ -252,7 +259,7 @@ def check_opera_rtc_s1_date(job: dict, _) -> None:
     # Disallow IPF version < 002.70 according to the dates given at https://sar-mpc.eu/processor/ipf/
     # Also see https://github.com/ASFHyP3/hyp3/issues/2739
     if granule_date < date(2016, 4, 14):
-        raise GranuleValidationError(
+        raise ValidationError(
             f'Granule {granule} was acquired before 2016-04-14 '
             'and is not available for On-Demand OPERA RTC-S1 processing.'
         )
@@ -263,7 +270,7 @@ def check_opera_rtc_s1_date(job: dict, _) -> None:
 
     end_date = datetime.strptime(end_date_str, '%Y-%m-%d').date()
     if granule_date >= end_date:
-        raise GranuleValidationError(
+        raise ValidationError(
             f'Granule {granule} was acquired on or after {end_date_str} '
             'and is not available for On-Demand OPERA RTC-S1 processing. '
             'You can download the product from the ASF DAAC archive.'
@@ -272,8 +279,13 @@ def check_opera_rtc_s1_date(job: dict, _) -> None:
 
 def validate_jobs(jobs: list[dict]) -> None:
     granules = get_granules(jobs)
-    granule_metadata = _get_cmr_metadata(granules)
-    _make_sure_granules_exist(granules, granule_metadata)
+
+    if granules:
+        granule_metadata = _get_cmr_metadata(granules)
+        _make_sure_granules_exist(granules, granule_metadata)
+    else:
+        granule_metadata = []
+
     for job in jobs:
         for validator_name in JOB_VALIDATION_MAP[job['job_type']]:
             job_granule_metadata = [granule for granule in granule_metadata if granule['name'] in get_granules([job])]

cicd-stacks/JPL-deployment-policy-cf.yml

@@ -51,6 +51,7 @@ Resources:
               - cloudformation:SetStackPolicy
               - cloudformation:CreateStack
               - cloudformation:UpdateStack
+              - cloudformation:DeleteStack
               - cloudformation:CreateChangeSet
               - cloudformation:DescribeChangeSet
               - cloudformation:ExecuteChangeSet

job_spec/ARIA_S1_GUNW.yml

@@ -1,39 +1,21 @@
 ARIA_S1_GUNW:
   required_parameters:
-    - reference
-    - secondary
+    - reference_date
+    - secondary_date
     - frame_id
   parameters:
-    reference:
+    reference_date:
       api_schema:
-        type: array
-        minItems: 1
-        maxItems: 3
-        example:
-          - S1A_IW_SLC__1SDV_20191231T135206_20191231T135234_030593_03813A_D336
-          - S1A_IW_SLC__1SDV_20191231T135141_20191231T135208_030593_03813A_837F
-        items:
-          description: The names of the Sentinel-1 SLC granules to use as reference scenes for InSAR processing
-          type: string
-          pattern: "^S1[AB]_IW_SLC__1S[SD]V"
-          minLength: 67
-          maxLength: 67
-          example: S1A_IW_SLC__1SDV_20191231T135206_20191231T135234_030593_03813A_D336
-    secondary:
+        description: The date to find Sentinel-1 SLC granules to use as reference scenes for InSAR processing
+        type: string
+        format: date
+        example: "2019-12-31"
+    secondary_date:
       api_schema:
-        type: array
-        minItems: 1
-        maxItems: 3
-        example:
-          - S1A_IW_SLC__1SDV_20181212T135155_20181212T135222_024993_02C174_2111
-          - S1A_IW_SLC__1SDV_20181212T135130_20181212T135157_024993_02C174_57AC
-        items:
-          description: The names of the Sentinel-1 SLC granules to use as secondary scenes for InSAR processing
-          type: string
-          pattern: "^S1[AB]_IW_SLC__1S[SD]V"
-          minLength: 67
-          maxLength: 67
-          example: S1A_IW_SLC__1SDV_20181212T135155_20181212T135222_024993_02C174_2111
+        description: The date to find Sentinel-1 SLC granules to use as secondary scenes for InSAR processing
+        type: string
+        format: date
+        example: "2018-12-12"
     frame_id:
       api_schema:
         description: Subset GUNW products to this frame.
@@ -47,7 +29,7 @@ ARIA_S1_GUNW:
     DEFAULT:
       cost: 1.0
   validators:
-    - check_dem_coverage
+    - check_aria_s1_gunw_dates
   steps:
     - name: ''
       image: ghcr.io/access-cloud-based-insar/dockerizedtopsapp
@@ -58,14 +40,14 @@ ARIA_S1_GUNW:
         - '!Ref Bucket'
         - --bucket-prefix
         - Ref::job_id
-        - --reference-scenes
-        - Ref::reference
-        - --secondary-scenes
-        - Ref::secondary
+        - --reference-date
+        - Ref::reference_date
+        - --secondary-date
+        - Ref::secondary_date
         - --frame-id
         - Ref::frame_id
         - --min-frame-coverage
-        - '0.01'
+        - '0.9'
       timeout: 21600  # 6 hr
       compute_environment: AriaS1Gunw
       vcpu: 1

job_spec/OPERA_RTC_S1.yml

@@ -24,9 +24,7 @@ OPERA_RTC_S1:
       cost: 1.0
   validators:
     - check_opera_rtc_s1_date
-    - check_dem_coverage
-    # Static coverage check is the slowest because it involves a CMR query, so should come last
-    - check_opera_rtc_s1_static_coverage
+    - check_opera_rtc_s1_bounds
   steps:
     - name: ''
       image: ghcr.io/asfhyp3/hyp3-opera-rtc

requirements-all.txt

@@ -4,15 +4,15 @@
 -r requirements-apps-start-execution.txt
 -r requirements-apps-disable-private-dns.txt
 -r requirements-apps-update-db.txt
-boto3==1.38.32
+boto3==1.38.37
 jinja2==3.1.6
-moto[dynamodb]==5.1.5
+moto[dynamodb]==5.1.6
 pytest==8.4.0
 PyYAML==6.0.2
 responses==0.25.7
 ruff==0.11.13
-mypy==1.16.0
+mypy==1.16.1
 setuptools==80.9.0
 setuptools_scm==8.3.1
 openapi-spec-validator==0.7.2
-cfn-lint==1.35.4
+cfn-lint==1.36.0

requirements-apps-api-binary.txt

@@ -1 +1 @@
-cryptography==45.0.3
+cryptography==45.0.4

requirements-apps-api.txt

@@ -1,11 +1,11 @@
 flask==3.1.1
-Flask-Cors==6.0.0
+Flask-Cors==6.0.1
 jsonschema==4.24.0
 openapi-core==0.19.5
 prance==25.4.8.0
 PyJWT==2.10.1
 requests==2.32.4
-serverless_wsgi==3.0.5
+serverless_wsgi==3.1.0
 shapely==2.1.1
 strict-rfc3339==0.7
 ./lib/dynamo/