Merge pull request #245 from ASFHyP3/develop

Release: v0.6.0

Author: asjohnston-asf

.github/workflows/deploy.yml

@@ -10,6 +10,7 @@ env:
   AWS_REGION: us-west-2
   TEMPLATE_BUCKET: cf-templates-aubvn3i9olmk-us-west-2
   RTC_GAMMA_REPOSITORY: 626226570674.dkr.ecr.us-east-1.amazonaws.com/hyp3-rtc-gamma
+  INSAR_GAMMA_REPOSITORY: 626226570674.dkr.ecr.us-east-1.amazonaws.com/hyp3-insar-gamma
 
   VPC_ID: ${{ secrets.VPC_ID }}
   SUBNET_IDS: ${{ secrets.SUBNET_IDS }}
@@ -77,6 +78,7 @@ jobs:
                 EDLUsername="${EDL_USERNAME}" \
                 EDLPassword="${EDL_PASSWORD}" \
                 RtcGammaImage="${RTC_GAMMA_REPOSITORY}:${IMAGE_TAG}" \
+                InsarGammaImage="${INSAR_GAMMA_REPOSITORY}:${IMAGE_TAG}" \
                 DomainName="${DOMAIN_NAME}" \
                 CertificateArn="${CERTIFICATE_ARN}"
 

CHANGELOG.md

@@ -4,6 +4,18 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
 and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
+## [0.6.0]
+### Added
+- Added a new `INSAR_GAMMA` job type for producing an interferogram from a pair of Sentinel-1 SLC IW scenes using [GAMMA](https://www.gamma-rs.ch/software).  For details, refer to the [hyp3-insar-gamma](https://github.com/ASFHyP3/hyp3-insar-gamma) plugin repository.
+
+### Changed
+- All job types requiring one or more granules now expose a single `granules` job parameter, formatted as a list of granule names:
+  - `"granules": ["myGranule"]` for `RTC_GAMMA` jobs
+  - `"granules": ["granule1", "granule2"]` for `INSAR_GAMMA` jobs
+
+  Note this is a breaking change for `RTC_GAMMA` jobs.
+- Browse and thumbnail URLs for `RTC_GAMMA` jobs will now be sorted with the amplitude image first, followed by the rgb image, in `GET /jobs` responses.
+
 ## [0.5.1]
 ### Fixed
 - Resolved HTTP 500 error when submitting jobs with a resolution with decimal precision (e.g. `30.0`)

api/src/hyp3_api/handlers.py

@@ -11,7 +11,7 @@
 from flask_cors import CORS
 
 from hyp3_api import DYNAMODB_RESOURCE, connexion_app
-from hyp3_api.util import convert_floats_to_decimals, format_time, get_remaining_jobs_for_user, \
+from hyp3_api.util import convert_floats_to_decimals, format_time, get_granules, get_remaining_jobs_for_user, \
     get_request_time_expression
 from hyp3_api.validation import GranuleValidationError, validate_granules
 
@@ -47,7 +47,7 @@ def post_jobs(body, user):
         return problem(400, 'Bad Request', message)
 
     try:
-        granules = [job['job_parameters']['granule'] for job in body['jobs']]
+        granules = get_granules(body['jobs'])
         validate_granules(granules)
     except requests.HTTPError as e:
         print(f'WARN: CMR search failed: {e}')

api/src/hyp3_api/openapi-spec.yml

@@ -136,6 +136,13 @@ components:
         - job_type
         - job_parameters
       additionalProperties: false
+      example:
+        job_parameters:
+          granules:
+            - S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8
+        job_type: RTC_GAMMA
+        name: Job Name
+
       properties:
         job_type:
           $ref: "#/components/schemas/job_type"
@@ -205,16 +212,39 @@ components:
       example: RTC_GAMMA
       enum:
         - RTC_GAMMA
+        - INSAR_GAMMA
 
     job_parameters:
+      oneOf:
+        - $ref: "#/components/schemas/rtc_gamma_job_parameters"
+        - $ref: "#/components/schemas/insar_gamma_job_parameters"
+
+    insar_gamma_job_parameters:
+      description: Options and parameters for running the job process.
+      type: object
+      required:
+        - granules
+      additionalProperties: false
+      properties:
+        granules:
+          $ref: "#/components/schemas/insar_gamma_granules"
+        include_inc_map:
+          $ref: "#/components/schemas/include_inc_map"
+        looks:
+          $ref: "#/components/schemas/looks"
+        include_los_displacement:
+          $ref: "#/components/schemas/los_displacement"
+
+
+    rtc_gamma_job_parameters:
       description: Options and parameters for running the job process.
       type: object
       required:
-        - granule
+        - granules
       additionalProperties: false
       properties:
-        granule:
-          $ref: "#/components/schemas/granule"
+        granules:
+          $ref: "#/components/schemas/rtc_gamma_granules"
         resolution:
           $ref: "#/components/schemas/resolution"
         radiometry:
@@ -230,10 +260,35 @@ components:
         include_inc_map:
           $ref: "#/components/schemas/include_inc_map"
 
-    granule:
-      description: The name of the Sentinel-1 granule to process.  Only SLC and GRDH products with beam mode IW are supported.
+    insar_gamma_granules:
+      type: array
+      minItems: 2
+      maxItems: 2
+      items:
+        $ref: "#/components/schemas/granule_slc"
+
+    rtc_gamma_granules:
+      type: array
+      minItems: 1
+      maxItems: 1
+      items:
+        anyOf:
+          - $ref: "#/components/schemas/granule_grdh"
+          - $ref: "#/components/schemas/granule_slc"
+
+
+    granule_grdh:
+      description: The name of the Sentinel-1 GRDH granule to process.
       type: string
-      pattern: "^S1[AB]_IW_(SLC_|GRDH)_*"
+      pattern: "^S1[AB]_IW_GRDH_*"
+      minLength: 67
+      maxLength: 67
+      example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8
+
+    granule_slc:
+      description: The name of the Sentinel-1 SLC granule to process.
+      type: string
+      pattern: "^S1[AB]_IW_SLC__*"
       minLength: 67
       maxLength: 67
       example: S1A_IW_SLC__1SSV_20150621T120220_20150621T120232_006471_008934_72D8
@@ -278,6 +333,18 @@ components:
       default: false
       type: boolean
 
+    looks:
+      description: Number of looks to take in range and azimuth
+      type: string
+      enum:
+        - 20x4
+        - 10x2
+
+    los_displacement:
+      description: Include a GeoTIFF in the product package containing the displacements along the Line-Of-Sight (LOS)
+      default: false
+      type: boolean
+
     datetime:
       description: Date and time object formatted according to ISO 8601
       type: string

api/src/hyp3_api/util.py

@@ -8,6 +8,14 @@
 from hyp3_api import handlers
 
 
+def get_granules(jobs):
+    granules = set()
+    for job in jobs:
+        for granule in job['job_parameters']['granules']:
+            granules.add(granule)
+    return granules
+
+
 def format_time(time: datetime):
     if time.tzinfo is None:
         raise ValueError(f'missing tzinfo for datetime {time}')

api/tests/conftest.py

@@ -8,6 +8,7 @@
 from moto import mock_dynamodb2
 
 from hyp3_api import CMR_URL, DYNAMODB_RESOURCE, auth, connexion_app  # noqa hyp3 must be imported here
+from hyp3_api.util import get_granules
 
 AUTH_COOKIE = 'asf-urs'
 JOBS_URI = '/jobs'
@@ -45,14 +46,15 @@ def get_table_properties_from_template():
     return table_properties
 
 
-def make_job(granule='S1B_IW_SLC__1SDV_20200604T082207_20200604T082234_021881_029874_5E38',
+def make_job(granules=['S1B_IW_SLC__1SDV_20200604T082207_20200604T082234_021881_029874_5E38'],
              name='someName',
-             job_type='RTC_GAMMA'):
+             job_type='RTC_GAMMA',
+             parameters={},):
     job = {
         'job_type': job_type,
         'job_parameters': {
-            'granule': granule,
-            'resolution': 30.0,
+            'granules': granules,
+            **parameters
         }
     }
     if name is not None:
@@ -73,7 +75,7 @@ def submit_batch(client, batch=None, validate_only=None):
 
 
 def make_db_record(job_id,
-                   granule='S1A_IW_SLC__1SDV_20200610T173646_20200610T173704_032958_03D14C_5F2B',
+                   granules=['S1A_IW_SLC__1SDV_20200610T173646_20200610T173704_032958_03D14C_5F2B'],
                    job_type='RTC_GAMMA',
                    user_id=DEFAULT_USERNAME,
                    request_time='2019-12-31T15:00:00+00:00',
@@ -88,7 +90,7 @@ def make_db_record(job_id,
         'user_id': user_id,
         'job_type': job_type,
         'job_parameters': {
-            'granule': granule,
+            'granules': granules,
         },
         'request_time': request_time,
         'status_code': status_code,
@@ -107,7 +109,7 @@ def make_db_record(job_id,
 
 
 def setup_requests_mock(batch):
-    granules = [job['job_parameters']['granule'] for job in batch]
+    granules = get_granules(batch)
     cmr_response = {
         'feed': {
             'entry': [

api/tests/test_submit_job.py

@@ -17,6 +17,32 @@ def test_submit_one_job(client, table):
     assert jobs[0]['user_id'] == DEFAULT_USERNAME
 
 
+def test_submit_insar_gamma(client, table):
+    login(client)
+    job = make_job(
+        [
+            'S1A_IW_SLC__1SDV_20200720T172109_20200720T172128_033541_03E2FB_341F',
+            'S1A_IW_SLC__1SDV_20200813T172110_20200813T172129_033891_03EE3F_2C3E'
+        ],
+        job_type='INSAR_GAMMA'
+    )
+    response = submit_batch(client, batch=[job])
+    assert response.status_code == status.HTTP_200_OK
+
+
+def test_submit_multiple_job_types(client, table):
+    login(client)
+    job = make_job(
+        [
+            'S1A_IW_SLC__1SDV_20200720T172109_20200720T172128_033541_03E2FB_341F',
+            'S1A_IW_SLC__1SDV_20200813T172110_20200813T172129_033891_03EE3F_2C3E'
+        ],
+        job_type='INSAR_GAMMA'
+    )
+    response = submit_batch(client, batch=[job, make_job()])
+    assert response.status_code == status.HTTP_200_OK
+
+
 def test_submit_many_jobs(client, table):
     max_jobs = 25
     login(client)
@@ -96,11 +122,11 @@ def test_submit_job_with_long_name(client):
 
 def test_submit_job_granule_does_not_exist(client, table):
     batch = [
-        make_job('S1B_IW_SLC__1SDV_20200604T082207_20200604T082234_021881_029874_5E38'),
-        make_job('S1A_IW_SLC__1SDV_20200610T173646_20200610T173704_032958_03D14C_5F2B')
+        make_job(['S1B_IW_SLC__1SDV_20200604T082207_20200604T082234_021881_029874_5E38']),
+        make_job(['S1A_IW_SLC__1SDV_20200610T173646_20200610T173704_032958_03D14C_5F2B'])
     ]
     setup_requests_mock(batch)
-    batch.append(make_job('S1A_IW_SLC__1SDV_20200610T173646_20200610T173704_032958_03D14C_5F2A'))
+    batch.append(make_job(['S1A_IW_SLC__1SDV_20200610T173646_20200610T173704_032958_03D14C_5F2A']))
 
     login(client)
     response = submit_batch(client, batch)
@@ -120,7 +146,7 @@ def test_submit_good_granule_names(client, table):
     ]
     for granule in good_granule_names:
         batch = [
-            make_job(granule),
+            make_job([granule]),
         ]
         setup_requests_mock(batch)
         response = submit_batch(client, batch)
@@ -155,6 +181,19 @@ def test_submit_bad_granule_names(client):
         assert response.status_code == status.HTTP_400_BAD_REQUEST
 
 
+def test_float_input(client, table):
+    login(client)
+    job = make_job(parameters={'resolution': 30.0})
+    response = submit_batch(client, batch=[job])
+    assert response.status_code == status.HTTP_200_OK
+    assert isinstance(response.json['jobs'][0]['job_parameters']['resolution'], float)
+
+    job = make_job(parameters={'resolution': 30})
+    response = submit_batch(client, batch=[job])
+    assert response.status_code == status.HTTP_200_OK
+    assert isinstance(response.json['jobs'][0]['job_parameters']['resolution'], int)
+
+
 def test_submit_validate_only(client, table):
     login(client)