fixed test merge conflicts

Author: DavidVWoodSr

CHANGELOG.md

@@ -12,6 +12,17 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Removed
 ### Fixed
 ### Security
+- [issues/151](https://github.com/podaac/bignbit/issues/151): Added ExpectedBucketOwner parameter when making S3 requests.
+
+## [0.7.2]
+### Added
+### Changed
+- increased timeout for "Handle BIG Result" lambda
+### Deprecated
+### Removed
+### Fixed
+- [issues/168](https://github.com/podaac/bignbit/issues/164): Added `reprojection` keyword to allow overriding default behavior for EPSG:4326
+### Security
 
 ## [0.7.1]
 ### Added

README.md

@@ -175,6 +175,7 @@ The contents of the configuration file should be a valid json object with the fo
 | singleDayNumber    | string       | [OPTIONAL] (Default: "") If using the "dataDayStrategy" keyword, all granules in this dataset will use the day of year specified in this keyword. ex: "001" for January 1st                                                                                          |
 | subdaily           | boolean      | [OPTIONAL] (Default: False) Set to true if granules contain subdaily data. This will send `DataDateTime` metadata to GIBS as described in the GIBS ICD                                                                                                               |
 | outputCrs          | list(string) | [OPTIONAL] (Default: ["EPSG:4326"]) Specifies a list of output projections or coordinate reference systems for which to produce browse images. Applies to all variables in granule. GIBS-compatible values are EPSG:4326, EPSG:3413, or EPSG:3031                    |
+| reprojection       | boolean      | [OPTIONAL] (Default: False) Set to true to force Harmony jobs to use the outputCrs parameter. If the service (chain) used by the job does not support reprojection, then the job will fail.                                                                          |
 | concept_id         | string       | [OPTIONAL] (Default: "") Overrides the concept id derived from the granule metadata with this value. ex: "C1996881146-POCLOUD"                                                                                                                                       |
 
 A few example configurations can be found in the [podaac/bignbit-config](https://github.com/podaac/bignbit-config) repository. NOTE: some of the example configurations have other options specified (e.g. `variables`, `latVar`, `lonVar`, etc...) that are no longer supported by this module. The table above are the attributes that are still in use.

bignbit/get_dataset_configuration.py

@@ -9,6 +9,8 @@
 from cumulus_logger import CumulusLogger
 from cumulus_process import Process
 
+from bignbit import utils
+
 CUMULUS_LOGGER = CumulusLogger('get_dataset_configuration')
 
 
@@ -69,7 +71,8 @@ def get_collection_config(config_bucket_name: str, config_key_name: str) -> dict
     s3_client = boto3.client('s3')
 
     try:
-        object_result = s3_client.get_object(Bucket=config_bucket_name, Key=config_key_name)
+        object_result = s3_client.get_object(Bucket=config_bucket_name, Key=config_key_name,
+                                             ExpectedBucketOwner=utils.get_aws_account_id())
     except s3_client.exceptions.NoSuchKey as ex:
         raise MissingDatasetConfiguration(
             f"Dataset configuration not found s3://{config_bucket_name}/{config_key_name}") from ex

bignbit/handle_big_result.py

@@ -216,7 +216,8 @@ def process_harmony_results(harmony_job: dict[str, str], cmr_env: str) -> list[d
     for url in result_urls:
         bucket, key = urlparse(url).netloc, urlparse(url).path.lstrip('/')
 
-        response = s3_client.get_object(Bucket=bucket, Key=key)
+        response = s3_client.get_object(Bucket=bucket, Key=key,
+                                        ExpectedBucketOwner=utils.get_aws_account_id())
         md5_hash = hashlib.new('md5')
         for chunk in response['Body'].iter_chunks(chunk_size=100 * 1024 * 1024):  # 100 MB chunk size
             md5_hash.update(chunk)

bignbit/send_to_gitc.py

@@ -9,6 +9,8 @@
 from cumulus_logger import CumulusLogger
 from cumulus_process import Process
 
+from bignbit import utils
+
 CUMULUS_LOGGER = CumulusLogger('send_to_gitc')
 
 GIBS_REGION_ENV_NAME = "GIBS_REGION"
@@ -72,7 +74,8 @@ def read_cnm(cnm_bucket: str, cnm_key: str) -> str:
       Key within the bucket pointing to CNM JSON
     """
     s3_client = boto3.client('s3')
-    response = s3_client.get_object(Bucket=cnm_bucket, Key=cnm_key)
+    response = s3_client.get_object(Bucket=cnm_bucket, Key=cnm_key,
+                                    ExpectedBucketOwner=utils.get_aws_account_id())
     return response['Body'].read().decode('utf-8')
 
 

bignbit/submit_harmony_job.py

@@ -111,7 +111,7 @@ def generate_harmony_request(collection_concept_id, granule_concept_id, variable
     # equirectangular projection through the reproject service.
     # Avoids unnecessary processing and errors for some collections
     # that do not support reprojection.
-    if output_crs.upper() != 'EPSG:4326':
+    if output_crs.upper() != 'EPSG:4326' or big_config['config'].get('reprojection', False):
         kwargs['crs'] = output_crs
         # Use the scaleExtent either from datasetConfig or use the
         # default values from GIBS
@@ -152,7 +152,7 @@ def lambda_handler(event, context):
     }
 
     logging_level = os.environ.get('LOGGING_LEVEL', 'info')
-    CUMULUS_LOGGER.logger.level = levels.get(logging_level, 'info')
+    CUMULUS_LOGGER.logger.setLevel(levels.get(logging_level, 'info'))
     CUMULUS_LOGGER.setMetadata(event, context)
 
     return CMA.cumulus_handler(event, context=context)

bignbit/utils.py

@@ -16,10 +16,27 @@
 ED_USER = ED_PASS = None
 EDL_USER_TOKEN: dict[str, str] = {}
 HARMONY_CLIENT: Client | None = None
+AWS_ACCOUNT_ID: str | None = None
 
 HARMONY_SHOULD_VALIDATE_AUTH = os.environ.get('HARMONY_SHOULD_VALIDATE_AUTH', default='False').upper() == 'TRUE'
 
 
+def get_aws_account_id() -> str:
+    """
+    Get and cache the current AWS account ID via STS.
+
+    Returns
+    -------
+    str
+        The AWS account ID for the current caller identity
+    """
+    global AWS_ACCOUNT_ID  # pylint: disable=W0603
+    if not AWS_ACCOUNT_ID:
+        sts_client = boto3.client('sts')
+        AWS_ACCOUNT_ID = sts_client.get_caller_identity()['Account']
+    return AWS_ACCOUNT_ID
+
+
 def get_edl_creds() -> tuple[str, str]:
     """
     Get EDL username and password from SSM.
@@ -216,7 +233,8 @@ def upload_string_as_object(bucket_name: str, key_name: str, object_content: str
     s3_client.put_object(
         Body=object_content.encode(),
         Bucket=bucket_name,
-        Key=key_name
+        Key=key_name,
+        ExpectedBucketOwner=get_aws_account_id()
     )
     return f's3://{bucket_name}/{key_name}'
 
@@ -236,6 +254,7 @@ def upload_object(
         Key=key,
         Body=body_content,
         ContentType=content_type,
+        ExpectedBucketOwner=get_aws_account_id()
     )
     return f's3://{bucket}/{key}'
 
@@ -259,7 +278,8 @@ def upload_to_s3(filepath: pathlib.Path, bucket_name: str, object_key: str):
       s3 uri of new object
     """
     s3_client = boto3.client('s3')
-    s3_client.upload_file(str(filepath), bucket_name, object_key)
+    s3_client.upload_file(str(filepath), bucket_name, object_key,
+                          ExtraArgs={'ExpectedBucketOwner': get_aws_account_id()})
 
     return f's3://{bucket_name}/{object_key}'
 

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "bignbit"
-version = "0.8.0a4"
+version = "0.8.0a7"
 description = "Browse image generation and transfer"
 authors = ["PO.DAAC <podaac@jpl.nasa.gov>"]
 license = "Apache 2.0"

scripts/create_dataset_config.py

@@ -399,6 +399,15 @@ def cli() -> argparse.Namespace:
             'EPSG:4326, EPSG:3413, or EPSG:3031'
         )
     )
+    parser.add_argument(
+        '--reprojection',
+        action='store_true',
+        help=(
+            'Set to true to force reprojection for all granules. This will send the `outputCrs` '
+            'parameter in Harmony API calls unconditionally. Warning: if the Harmony service used '
+            'to generate browse images does not support reprojection, jobs will fail.'
+        )
+    )
     parser.add_argument(
         '--s3Destination',
         type=str,

terraform/lambda_functions.tf

@@ -244,8 +244,8 @@ resource "aws_lambda_function" "handle_big_result" {
   }
   function_name = local.handle_big_result_function_name
   role          = aws_iam_role.bignbit_lambda_role.arn
-  timeout       = 180
-  memory_size   = 512
+  timeout       = 300
+  memory_size   = 1024
 
   environment {
     variables = {