Update readme and unit tests (#88)

frankinspace · web-flow · commit 335cc0b6a6fd · 2025-07-29T14:40:57.000-07:00
* Updated the test for the subdaily changes. Added subdaily key to readme

* Example file should be png not tiff
diff --git a/README.md b/README.md
@@ -114,9 +114,10 @@ The contents of the configuration file should be a valid json object with the fo
 | imgVariables       | list(object) | List of JSON objects with at least one attribute called `id` whose value is the name of a variable to generate an image for. `all` can be used in cases where the collection does not have variables or all variables in the collection should have images generated |
 | height             | int          | Controls the height of the output image from Harmony (see https://github.com/nasa/harmony-browse-image-generator?tab=readme-ov-file#dimensions--scale-sizes)                                                                                                         |
 | width              | int          | Controls the width of the output image from Harmony (see https://github.com/nasa/harmony-browse-image-generator?tab=readme-ov-file#dimensions--scale-sizes)                                                                                                          |
-| dataDayStrategy    | string       | [OPTIONAL] If this keyword is set to "single_day_of_year", bignbit will override the date from the granule metadata with the one specified in "singleDayNumber"                                                                                                      |
-| singleDayNumber    | string       | [OPTIONAL] If using the "dataDayStrategy" keyword, all granules in this dataset will use the day of year specified in this keyword. ex: "001" for January 1st                                                                                                        |
-| concept_id         | string       | [OPTIONAL] Overrides the concept id derived from the granule metadata with this value. ex: "C1996881146-POCLOUD"                                                                                                                                                     |
+| dataDayStrategy    | string       | [OPTIONAL] (Default: "") If this keyword is set to "single_day_of_year", bignbit will override the date from the granule metadata with the one specified in "singleDayNumber"                                                                                        |
+| singleDayNumber    | string       | [OPTIONAL] (Default: "") If using the "dataDayStrategy" keyword, all granules in this dataset will use the day of year specified in this keyword. ex: "001" for January 1st                                                                                          |
+| subdaily           | boolean      | [OPTIONAL] (Default: False) Set to true if granules contain subdaily data. This will send `DataDateTime` metadata to GIBS as described in the GIBS ICD                                                                                                               |
+| concept_id         | string       | [OPTIONAL] (Default: "") Overrides the concept id derived from the granule metadata with this value. ex: "C1996881146-POCLOUD"                                                                                                                                       |
 
 A few example configurations can be found in the [podaac/bignbit-config](https://github.com/podaac/bignbit-config) repository. NOTE: some of the example configurations have other options specified (e.g. `variables`, `latVar`, `lonVar`, etc...) that are no longer supported by this module. The table above are the attributes that are still in use.
 
diff --git a/tests/sample_messages/generate_image_metadata/cma.uat.input.TEMPO_NO2_L3.json b/tests/sample_messages/generate_image_metadata/cma.uat.input.TEMPO_NO2_L3.json
@@ -29,7 +29,7 @@
                 }
             },
             "payload":{
-                "granules":{
+                "granules":[{
                     "files": [
                         {
                             "bucket": "podaac-sit-svc-private",
@@ -40,7 +40,7 @@
                             "cmrConceptId":"G1273455903-LARC_CLOUD"
                         }
                     ]
-                },
+                }],
                 "granule_umm_json": {
                     "PGEVersionClass": {
                         "PGEVersion": "1.0.0"
@@ -391,9 +391,9 @@
                 "big": [
                     {
                         "bucket": "podaac-sit-svc-private",
-                        "fileName": "TEMPO_NO2_L3_V03_20250422T114702Z_S003.nc",
+                        "fileName": "TEMPO_NO2_L3_V03_20250422T114702Z_S003.png",
                         "type": "data",
-                        "key": "TEMPO_NO2_L3/TEMPO_NO2_L3_V03_20250422T114702Z_S003.nc"
+                        "key": "TEMPO_NO2_L3/TEMPO_NO2_L3_V03_20250422T114702Z_S003.png"
                     }
                 ],
                 "datasetConfigurationForBIG": {
diff --git a/tests/test_generate_image_metadata.py b/tests/test_generate_image_metadata.py
@@ -2,20 +2,19 @@
 import os
 import xml.etree.ElementTree as ET
 
+import pytest
+
 import bignbit.generate_image_metadata
 from moto import mock_s3
 
 import boto3
 import botocore
 
 
-@mock_s3
-def test_process_opera_input():
-    test_dir = os.path.dirname(os.path.realpath(__file__))
-    cma_json = json.load(open(os.path.join(
-        test_dir, 'sample_messages', 'generate_image_metadata',
-        'cma.uat.input.OPERA_L3_DSWx-HLS_T48SUE_20190302T034350Z_20230131T222341Z_L8_30_v0.0.json')))
-
+def create_mock_buckets(cma_json):
+    """
+    Create mock S3 buckets for testing purposes.
+    """
     buckets_to_create = set([b['name'] for _, b in cma_json['cma']['event']['meta']['buckets'].items()])
     buckets_to_create = buckets_to_create.union(
         set([granule_file['bucket'] for granule in cma_json['cma']['event']['payload']['granules'] for granule_file in
@@ -27,6 +26,15 @@ def test_process_opera_input():
         aws_s3 = boto3.resource('s3', region_name='us-east-1')
         aws_s3.create_bucket(Bucket=bucket_name)
 
+@mock_s3
+def test_process_opera_input():
+    test_dir = os.path.dirname(os.path.realpath(__file__))
+    cma_json = json.load(open(os.path.join(
+        test_dir, 'sample_messages', 'generate_image_metadata',
+        'cma.uat.input.OPERA_L3_DSWx-HLS_T48SUE_20190302T034350Z_20230131T222341Z_L8_30_v0.0.json')))
+
+    create_mock_buckets(cma_json)
+
     result = bignbit.generate_image_metadata.lambda_handler(cma_json, {})
 
     assert result
@@ -35,23 +43,15 @@ def test_process_opera_input():
     assert any([r['subtype'] == 'ImageMetadata-v1.2' for r in result['payload']['big']])
     assert any([r['subtype'] == 'geotiff' for r in result['payload']['big']])
 
+
 @mock_s3
 def test_process_static_data_day():
     test_dir = os.path.dirname(os.path.realpath(__file__))
     cma_json = json.load(open(os.path.join(
         test_dir, 'sample_messages', 'generate_image_metadata',
         'cma.uat.input.OPERA_L3_DIST-ANN_mock.json')))
 
-    buckets_to_create = set([b['name'] for _, b in cma_json['cma']['event']['meta']['buckets'].items()])
-    buckets_to_create = buckets_to_create.union(
-        set([granule_file['bucket'] for granule in cma_json['cma']['event']['payload']['granules'] for granule_file in
-             granule['files']]))
-
-    for bucket_name in buckets_to_create:
-        if "*" in bucket_name:
-            continue
-        aws_s3 = boto3.resource('s3', region_name='us-east-1')
-        aws_s3.create_bucket(Bucket=bucket_name)
+    create_mock_buckets(cma_json)
 
     result = bignbit.generate_image_metadata.lambda_handler(cma_json, {})
     s3_mock = boto3.client('s3')
@@ -82,37 +82,45 @@ def test_process_static_data_day():
 
 
 @mock_s3
-def test_process_subdaily():
+@pytest.mark.parametrize("subdaily", [True, False])
+def test_process_subdaily(subdaily: bool):
     test_dir = os.path.dirname(os.path.realpath(__file__))
     cma_json = json.load(open(os.path.join(
         test_dir, 'sample_messages', 'generate_image_metadata',
         'cma.uat.input.TEMPO_NO2_L3.json')))
 
+    cma_json['cma']['event']['payload']['datasetConfigurationForBIG']['config']['subdaily'] = subdaily
+    expected_xml_tags = [('DataStartDateTime', '2025-04-22T11:47:02.000000Z'),
+                         ('DataMidDateTime', '2025-04-22T12:06:56.500000Z'),
+                         ('DataEndDateTime', '2025-04-22T12:26:51.000000Z'),
+                         ('DataDateTime', '2025-04-22T11:47:02.000000Z')]
+    if not subdaily:
+        # If subdaily is False, we expect the DataDateTime to not be present in the metadata XML
+        expected_xml_tags = list(filter(lambda t: t[0] != 'DataDateTime', expected_xml_tags))
+
+    create_mock_buckets(cma_json)
     result = bignbit.generate_image_metadata.lambda_handler(cma_json, {})
-    s3_mock = boto3.client('s3')
-    metadata_bucket = 'podaac-uat-cumulus-private'
 
     assert result
-    # Assert one outputs, the image metadata xml
-    assert len(result['payload']['big']) == 1
-    for r in result['payload']['big']:
-        metadata_xml = r['key']
-        try:
-            s3_mock.download_file(metadata_bucket, metadata_xml, r['fileName'])
-        except botocore.exceptions.ClientError:
-            print(f"could not stat s3://{metadata_bucket}/{metadata_xml}")
-            continue
-        md_tree = ET.parse(r['fileName'])
-        md_root = md_tree.getroot()
-        try:
-            for child in md_root:
-                if child.tag == "DataStartDateTime":
-                    assert child.text == "2025-04-22T11:47:02.000000Z"
-                elif child.tag == "DataMidDateTime":
-                    assert child.text == "2025-04-22T12:06:56.500000Z"
-                elif child.tag == "DataEndDateTime":
-                    assert child.text == "2025-04-22T12:26:51.000000Z"
-                elif child.tag == "DataDateTime":
-                    assert child.text == "2025-04-22T11:47:02.000000Z"
-        finally:
-            os.remove(r['fileName'])
+    # Assert two outputs, the image and the image metadata xml
+    assert len(result['payload']['big']) == 2
+    assert 'metadata' in [r['type'] for r in result['payload']['big']]
+
+    # Download the metadata XML file and check the contents are as expected
+    result_metadata_xml = next(filter(lambda r: r['type'] == 'metadata', result['payload']['big']))
+    result_metadata_xml_filename = result_metadata_xml['fileName']
+    s3_mock = boto3.client('s3')
+    s3_mock.download_file(result_metadata_xml['bucket'], result_metadata_xml['key'], result_metadata_xml_filename)
+    actual_xml_tags = []
+    md_tree = ET.parse(result_metadata_xml_filename)
+    md_root = md_tree.getroot()
+    try:
+        for child in md_root:
+            for tag, text in expected_xml_tags:
+                if child.tag == tag:
+                    assert child.text == text
+                    actual_xml_tags.append((child.tag, child.text))
+    finally:
+        os.remove(result_metadata_xml_filename)
+
+    assert expected_xml_tags == actual_xml_tags

Original file line number	Diff line number	Diff line change
`@@ -29,7 +29,7 @@`
`29`	`29`	`}`
`30`	`30`	`},`
`31`	`31`	`"payload":{`
`32`		`- "granules":{`
	`32`	`+ "granules":[{`
`33`	`33`	`"files": [`
`34`	`34`	`{`
`35`	`35`	`"bucket": "podaac-sit-svc-private",`
`@@ -40,7 +40,7 @@`
`40`	`40`	`"cmrConceptId":"G1273455903-LARC_CLOUD"`
`41`	`41`	`}`
`42`	`42`	`]`
`43`		`- },`
	`43`	`+ }],`
`44`	`44`	`"granule_umm_json": {`
`45`	`45`	`"PGEVersionClass": {`
`46`	`46`	`"PGEVersion": "1.0.0"`
`@@ -391,9 +391,9 @@`
`391`	`391`	`"big": [`
`392`	`392`	`{`
`393`	`393`	`"bucket": "podaac-sit-svc-private",`
`394`		`- "fileName": "TEMPO_NO2_L3_V03_20250422T114702Z_S003.nc",`
	`394`	`+ "fileName": "TEMPO_NO2_L3_V03_20250422T114702Z_S003.png",`
`395`	`395`	`"type": "data",`
`396`		`- "key": "TEMPO_NO2_L3/TEMPO_NO2_L3_V03_20250422T114702Z_S003.nc"`
	`396`	`+ "key": "TEMPO_NO2_L3/TEMPO_NO2_L3_V03_20250422T114702Z_S003.png"`
`397`	`397`	`}`
`398`	`398`	`],`
`399`	`399`	`"datasetConfigurationForBIG": {`