Skip to content

Commit 329a2e0

Browse files
frankinspacefrankinspace
committed
Updated the test for the subdaily changes. Added subdaily key to readme
1 parent 366065a commit 329a2e0

3 files changed

Lines changed: 60 additions & 51 deletions

File tree

‎README.md‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -114,9 +114,10 @@ The contents of the configuration file should be a valid json object with the fo
114114
| imgVariables | list(object) | List of JSON objects with at least one attribute called `id` whose value is the name of a variable to generate an image for. `all` can be used in cases where the collection does not have variables or all variables in the collection should have images generated |
115115
| height | int | Controls the height of the output image from Harmony (see https://github.com/nasa/harmony-browse-image-generator?tab=readme-ov-file#dimensions--scale-sizes) |
116116
| width | int | Controls the width of the output image from Harmony (see https://github.com/nasa/harmony-browse-image-generator?tab=readme-ov-file#dimensions--scale-sizes) |
117-
| dataDayStrategy | string | [OPTIONAL] If this keyword is set to "single_day_of_year", bignbit will override the date from the granule metadata with the one specified in "singleDayNumber" |
118-
| singleDayNumber | string | [OPTIONAL] If using the "dataDayStrategy" keyword, all granules in this dataset will use the day of year specified in this keyword. ex: "001" for January 1st |
119-
| concept_id | string | [OPTIONAL] Overrides the concept id derived from the granule metadata with this value. ex: "C1996881146-POCLOUD" |
117+
| dataDayStrategy | string | [OPTIONAL] (Default: "") If this keyword is set to "single_day_of_year", bignbit will override the date from the granule metadata with the one specified in "singleDayNumber" |
118+
| singleDayNumber | string | [OPTIONAL] (Default: "") If using the "dataDayStrategy" keyword, all granules in this dataset will use the day of year specified in this keyword. ex: "001" for January 1st |
119+
| subdaily | boolean | [OPTIONAL] (Default: False) Set to true if granules contain subdaily data. This will send `DataDateTime` metadata to GIBS as described in the GIBS ICD |
120+
| concept_id | string | [OPTIONAL] (Default: "") Overrides the concept id derived from the granule metadata with this value. ex: "C1996881146-POCLOUD" |
120121

121122
A few example configurations can be found in the [podaac/bignbit-config](https://github.com/podaac/bignbit-config) repository. NOTE: some of the example configurations have other options specified (e.g. `variables`, `latVar`, `lonVar`, etc...) that are no longer supported by this module. The table above are the attributes that are still in use.
122123

‎tests/sample_messages/generate_image_metadata/cma.uat.input.TEMPO_NO2_L3.json‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -29,18 +29,18 @@
2929
}
3030
},
3131
"payload":{
32-
"granules":{
32+
"granules":[{
3333
"files": [
3434
{
3535
"bucket": "podaac-sit-svc-private",
36-
"fileName": "TEMPO_NO2_L3_V03_20250422T114702Z_S003.nc",
36+
"fileName": "TEMPO_NO2_L3_V03_20250422T114702Z_S003.tiff",
3737
"granuleId":"TEMPO_NO2_L3_V03_20250422T114702Z_S003",
3838
"provider":"larc",
3939
"cmrLink":"https://cmr.uat.earthdata.nasa.gov/search/concepts/G1273455903-LARC_CLOUD.umm_json",
4040
"cmrConceptId":"G1273455903-LARC_CLOUD"
4141
}
4242
]
43-
},
43+
}],
4444
"granule_umm_json": {
4545
"PGEVersionClass": {
4646
"PGEVersion": "1.0.0"
@@ -391,9 +391,9 @@
391391
"big": [
392392
{
393393
"bucket": "podaac-sit-svc-private",
394-
"fileName": "TEMPO_NO2_L3_V03_20250422T114702Z_S003.nc",
394+
"fileName": "TEMPO_NO2_L3_V03_20250422T114702Z_S003.tiff",
395395
"type": "data",
396-
"key": "TEMPO_NO2_L3/TEMPO_NO2_L3_V03_20250422T114702Z_S003.nc"
396+
"key": "TEMPO_NO2_L3/TEMPO_NO2_L3_V03_20250422T114702Z_S003.tiff"
397397
}
398398
],
399399
"datasetConfigurationForBIG": {

‎tests/test_generate_image_metadata.py‎

Lines changed: 51 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -2,20 +2,19 @@
22
import os
33
import xml.etree.ElementTree as ET
44

5+
import pytest
6+
57
import bignbit.generate_image_metadata
68
from moto import mock_s3
79

810
import boto3
911
import botocore
1012

1113

12-
@mock_s3
13-
def test_process_opera_input():
14-
test_dir = os.path.dirname(os.path.realpath(__file__))
15-
cma_json = json.load(open(os.path.join(
16-
test_dir, 'sample_messages', 'generate_image_metadata',
17-
'cma.uat.input.OPERA_L3_DSWx-HLS_T48SUE_20190302T034350Z_20230131T222341Z_L8_30_v0.0.json')))
18-
14+
def create_mock_buckets(cma_json):
15+
"""
16+
Create mock S3 buckets for testing purposes.
17+
"""
1918
buckets_to_create = set([b['name'] for _, b in cma_json['cma']['event']['meta']['buckets'].items()])
2019
buckets_to_create = buckets_to_create.union(
2120
set([granule_file['bucket'] for granule in cma_json['cma']['event']['payload']['granules'] for granule_file in
@@ -27,6 +26,15 @@ def test_process_opera_input():
2726
aws_s3 = boto3.resource('s3', region_name='us-east-1')
2827
aws_s3.create_bucket(Bucket=bucket_name)
2928

29+
@mock_s3
30+
def test_process_opera_input():
31+
test_dir = os.path.dirname(os.path.realpath(__file__))
32+
cma_json = json.load(open(os.path.join(
33+
test_dir, 'sample_messages', 'generate_image_metadata',
34+
'cma.uat.input.OPERA_L3_DSWx-HLS_T48SUE_20190302T034350Z_20230131T222341Z_L8_30_v0.0.json')))
35+
36+
create_mock_buckets(cma_json)
37+
3038
result = bignbit.generate_image_metadata.lambda_handler(cma_json, {})
3139

3240
assert result
@@ -35,23 +43,15 @@ def test_process_opera_input():
3543
assert any([r['subtype'] == 'ImageMetadata-v1.2' for r in result['payload']['big']])
3644
assert any([r['subtype'] == 'geotiff' for r in result['payload']['big']])
3745

46+
3847
@mock_s3
3948
def test_process_static_data_day():
4049
test_dir = os.path.dirname(os.path.realpath(__file__))
4150
cma_json = json.load(open(os.path.join(
4251
test_dir, 'sample_messages', 'generate_image_metadata',
4352
'cma.uat.input.OPERA_L3_DIST-ANN_mock.json')))
4453

45-
buckets_to_create = set([b['name'] for _, b in cma_json['cma']['event']['meta']['buckets'].items()])
46-
buckets_to_create = buckets_to_create.union(
47-
set([granule_file['bucket'] for granule in cma_json['cma']['event']['payload']['granules'] for granule_file in
48-
granule['files']]))
49-
50-
for bucket_name in buckets_to_create:
51-
if "*" in bucket_name:
52-
continue
53-
aws_s3 = boto3.resource('s3', region_name='us-east-1')
54-
aws_s3.create_bucket(Bucket=bucket_name)
54+
create_mock_buckets(cma_json)
5555

5656
result = bignbit.generate_image_metadata.lambda_handler(cma_json, {})
5757
s3_mock = boto3.client('s3')
@@ -82,37 +82,45 @@ def test_process_static_data_day():
8282

8383

8484
@mock_s3
85-
def test_process_subdaily():
85+
@pytest.mark.parametrize("subdaily", [True, False])
86+
def test_process_subdaily(subdaily: bool):
8687
test_dir = os.path.dirname(os.path.realpath(__file__))
8788
cma_json = json.load(open(os.path.join(
8889
test_dir, 'sample_messages', 'generate_image_metadata',
8990
'cma.uat.input.TEMPO_NO2_L3.json')))
9091

92+
cma_json['cma']['event']['payload']['datasetConfigurationForBIG']['config']['subdaily'] = subdaily
93+
expected_xml_tags = [('DataStartDateTime', '2025-04-22T11:47:02.000000Z'),
94+
('DataMidDateTime', '2025-04-22T12:06:56.500000Z'),
95+
('DataEndDateTime', '2025-04-22T12:26:51.000000Z'),
96+
('DataDateTime', '2025-04-22T11:47:02.000000Z')]
97+
if not subdaily:
98+
# If subdaily is False, we expect the DataDateTime to not be present in the metadata XML
99+
expected_xml_tags = list(filter(lambda t: t[0] != 'DataDateTime', expected_xml_tags))
100+
101+
create_mock_buckets(cma_json)
91102
result = bignbit.generate_image_metadata.lambda_handler(cma_json, {})
92-
s3_mock = boto3.client('s3')
93-
metadata_bucket = 'podaac-uat-cumulus-private'
94103

95104
assert result
96-
# Assert one outputs, the image metadata xml
97-
assert len(result['payload']['big']) == 1
98-
for r in result['payload']['big']:
99-
metadata_xml = r['key']
100-
try:
101-
s3_mock.download_file(metadata_bucket, metadata_xml, r['fileName'])
102-
except botocore.exceptions.ClientError:
103-
print(f"could not stat s3://{metadata_bucket}/{metadata_xml}")
104-
continue
105-
md_tree = ET.parse(r['fileName'])
106-
md_root = md_tree.getroot()
107-
try:
108-
for child in md_root:
109-
if child.tag == "DataStartDateTime":
110-
assert child.text == "2025-04-22T11:47:02.000000Z"
111-
elif child.tag == "DataMidDateTime":
112-
assert child.text == "2025-04-22T12:06:56.500000Z"
113-
elif child.tag == "DataEndDateTime":
114-
assert child.text == "2025-04-22T12:26:51.000000Z"
115-
elif child.tag == "DataDateTime":
116-
assert child.text == "2025-04-22T11:47:02.000000Z"
117-
finally:
118-
os.remove(r['fileName'])
105+
# Assert two outputs, the image and the image metadata xml
106+
assert len(result['payload']['big']) == 2
107+
assert 'metadata' in [r['type'] for r in result['payload']['big']]
108+
109+
# Download the metadata XML file and check the contents are as expected
110+
result_metadata_xml = next(filter(lambda r: r['type'] == 'metadata', result['payload']['big']))
111+
result_metadata_xml_filename = result_metadata_xml['fileName']
112+
s3_mock = boto3.client('s3')
113+
s3_mock.download_file(result_metadata_xml['bucket'], result_metadata_xml['key'], result_metadata_xml_filename)
114+
actual_xml_tags = []
115+
md_tree = ET.parse(result_metadata_xml_filename)
116+
md_root = md_tree.getroot()
117+
try:
118+
for child in md_root:
119+
for tag, text in expected_xml_tags:
120+
if child.tag == tag:
121+
assert child.text == text
122+
actual_xml_tags.append((child.tag, child.text))
123+
finally:
124+
os.remove(result_metadata_xml_filename)
125+
126+
assert expected_xml_tags == actual_xml_tags

0 commit comments

Comments
 (0)