Merge pull request #86 from GeoscienceAustralia/upgrades/s3_path

Improvements to S3 paths

Author: abradley60

.gitignore

@@ -21,6 +21,9 @@ tests/sar_pipeline/data/**/*.json
 !tests/sar_pipeline/data/**/*.tif
 !tests/sar_pipeline/data/**/*.vrt
 
+# ignore create test data
+tests/sar_pipeline/data/isce3_rtc/results/TMP/*
+
 # ignore secret files
 *.secret
 

docs/workflows/aws.md

@@ -0,0 +1,7 @@
+EARTHDATA_LOGIN=
+EARTHDATA_PASSWORD=
+AWS_ACCESS_KEY_ID=
+AWS_SECRET_ACCESS_KEY=
+AWS_DEFAULT_REGION=
+CDSE_LOGIN=
+CDSE_PASSWORD=

env.secret.example

@@ -23,7 +23,6 @@
 from sar_pipeline.aws.metadata.stac import BurstH5toStacManager
 from sar_pipeline.aws.metadata.odc import (
     make_static_layer_base_url,
-    get_collection_number,
 )
 from sar_pipeline.utils.s3upload import push_files_in_folder_to_s3
 from sar_pipeline.utils.general import log_timing
@@ -105,11 +104,10 @@
     help="project folder in the s3 bucket",
 )
 @click.option(
-    "--collection",
+    "--collection-number",
     required=True,
-    type=str,
-    help="collection associated with product. e.g. s1_rtc_c1. Must end in 'cX' where X is an "
-    "integer number referring to the collection.",
+    type=int,
+    help="The collection number of the product.",
 )
 @click.option(
     "--download-folder",
@@ -149,7 +147,7 @@
     is_flag=True,
     default=False,
     help="If static layers should be linked to RTC_S1 products in the"
-    "STAC metadata. A url to the static layer collection will be added"
+    "STAC metadata. A url to the static layers will be added"
     "to the run config file.",
 )
 @click.option(
@@ -159,17 +157,16 @@
     help="S3 bucket containing the RTC_S1_STATIC data that will be linked to the RTC_S1 bursts.",
 )
 @click.option(
-    "--linked-static-layers-collection",
+    "--linked-static-layers-collection-number",
     required=False,
     type=str,
-    help="Collection of RTC_S1_STATIC data that will be linked to the RTC_S1 bursts.",
+    help="Collection number of RTC_S1_STATIC data that will be linked to the RTC_S1 bursts.",
 )
 @click.option(
     "--linked-static-layers-s3-project-folder",
     required=False,
     type=str,
-    help="Project folder containing the RTC_S1_STATIC data that will be linked to the RTC_S1 bursts. "
-    "Expected for linked files path is : s3_bucket/s3_project_folder/collection/burst_id/*files",
+    help="Project folder containing the RTC_S1_STATIC data that will be linked to the RTC_S1 bursts. ",
 )
 @click.option(
     "--scene-data-source",
@@ -204,15 +201,15 @@ def get_data_for_scene_and_make_run_config(
     backscatter_convention,
     s3_bucket,
     s3_project_folder,
-    collection,
+    collection_number,
     download_folder,
     scratch_folder,
     out_folder,
     run_config_save_path,
     make_existing_products,
     link_static_layers,
     linked_static_layers_s3_bucket,
-    linked_static_layers_collection,
+    linked_static_layers_collection_number,
     linked_static_layers_s3_project_folder,
     scene_data_source,
     orbit_data_source,
@@ -242,10 +239,6 @@ def get_data_for_scene_and_make_run_config(
     if backscatter_convention not in ["gamma0", "sigma0", "beta0"]:
         raise ValueError("backscatter_convention must be one of gamma0, sigma0, beta0")
 
-    # ensure the collection ends with cX, where X is a positive integer.
-    # Raise error for invalid naming
-    _ = get_collection_number(collection)
-
     # sub-folders for downloads
     orbit_folder = download_folder / "orbits"
     scene_folder = download_folder / "scenes"
@@ -305,7 +298,7 @@ def get_data_for_scene_and_make_run_config(
         burst_polarisations=burst_pols,
         s3_bucket=s3_bucket,
         s3_project_folder=s3_project_folder,
-        collection=collection,
+        collection_number=collection_number,
         make_existing_products=make_existing_products,
         early_exit=True,
         early_exit_code=100,
@@ -322,7 +315,7 @@ def get_data_for_scene_and_make_run_config(
             scene=scene,
             burst_id_list=burst_id_list_to_process,
             static_layers_s3_bucket=linked_static_layers_s3_bucket,
-            static_layers_collection=linked_static_layers_collection,
+            static_layers_collection_number=linked_static_layers_collection_number,
             static_layers_s3_project_folder=linked_static_layers_s3_project_folder,
             early_exit_code=101,
         )
@@ -493,7 +486,7 @@ def get_data_for_scene_and_make_run_config(
         # add the static layer base url
         static_layer_base_url = make_static_layer_base_url(
             linked_static_layers_s3_bucket,
-            linked_static_layers_collection,
+            linked_static_layers_collection_number,
             linked_static_layers_s3_project_folder,
         )
         logger.info(f"static layer base url : {static_layer_base_url}")
@@ -550,11 +543,10 @@ def get_data_for_scene_and_make_run_config(
     help="Backscatter convention of the product to be made (gamma0, sigma0 or beta0)",
 )
 @click.option(
-    "--collection",
+    "--collection-number",
     required=True,
-    type=str,
-    help="collection associated with product. e.g. s1_rtc_c1. Must end in 'cX' where X is an "
-    "integer number referring to the collection.",
+    type=int,
+    help="The collection number of the product.",
 )
 @click.option(
     "--s3-bucket", required=True, type=str, help="The bucket to upload the files"
@@ -605,7 +597,7 @@ def make_rtc_opera_stac_and_upload_bursts(
     run_config_path,
     product,
     backscatter_convention,
-    collection,
+    collection_number,
     s3_bucket,
     s3_project_folder,
     skip_upload_to_s3,
@@ -615,7 +607,10 @@ def make_rtc_opera_stac_and_upload_bursts(
 ):
     """makes STAC metadata for opera-rtc and uploads them to a desired s3 bucket.
     The final path in s3 will follow the following pattern:
-    s3_bucket/s3_folder/collection/burst_id/burst_year/burst_month/burst_day/*files
+    product = RTC_S1:
+        s3_bucket/s3_folder/odc_product_name/burst_id/burst_year/burst_month/burst_day/*files
+    product = RTC_S1_STATIC:
+        s3_bucket/s3_folder/odc_product_name/burst_id/*files
     """
 
     # iterate through the burst directory and create STAC metadata
@@ -641,7 +636,7 @@ def make_rtc_opera_stac_and_upload_bursts(
             h5_filepath=burst_h5_filepath,
             product=product,
             backscatter_convention=backscatter_convention,
-            collection=collection,
+            collection_number=collection_number,
             s3_bucket=s3_bucket,
             s3_project_folder=s3_project_folder,
         )
@@ -703,7 +698,7 @@ def make_rtc_opera_stac_and_upload_bursts(
                 burst_polarisations=burst_stac_manager.polarisations,
                 s3_bucket=s3_bucket,
                 s3_project_folder=s3_project_folder,
-                collection=collection,
+                collection_number=collection_number,
                 make_existing_products=make_existing_products,
                 early_exit=False,  # don't exit early, move to next burst
             )

sar_pipeline/aws/cli.py

@@ -1,72 +1,39 @@
-import re
-
-
-def get_collection_number(collection: str) -> int:
-    """Get the collection number from the collection string
-
-    Parameters
-    ----------
-    collection : str
-        Collection of the product. e.g. s1_rtc_c1. The collection MUST end in cX where X
-        is an integer associated with the collection. E.g. rtc_s1_c1.
-
-    Returns
-    -------
-    int
-        number. e.g. s1_rtc_c1 -> 1
-
-    Raises
-    ------
-    ValueError
-        Invalid collection name.
-    """
-
-    # ensure the collection ends with cX, where X is a positive integer
-    match = re.search(r"c(\d+)$", collection)
-    if not match:
-        raise ValueError(
-            f"Invalid collection name. The collection MUST end in cX where X"
-            " is an integer associated with the collection. E.g. rtc_s1_c1."
-        )
-    return int(match.group(1))
-
-
 def get_odc_product_name(product, collection_number, polarisations):
-    """set the odc:product value. WARNING this must align with
+    """get the odc product name. WARNING this must align with
     the DEA product name at indexing into the datacube.
     These are hard-coded and set by the provided `collection_number`.
     """
     if product == "RTC_S1":
         if all([pol in polarisations for pol in ["VV", "VH"]]):
-            return f"ga_s1_iw_vv_vh_c{collection_number}"
+            return f"ga_s1_nrb_iw_vv_vh_c{collection_number}"
         elif all([pol in polarisations for pol in ["HH", "HV"]]):
-            return f"ga_s1_iw_hh_hv_c{collection_number}"
+            return f"ga_s1_nrb_iw_hh_hv_c{collection_number}"
         elif polarisations == ["VV"]:
-            return f"ga_s1_iw_vv_c{collection_number}"
+            return f"ga_s1_nrb_iw_vv_c{collection_number}"
         elif polarisations == ["HH"]:
-            return f"ga_s1_iw_hh_c{collection_number}"
+            return f"ga_s1_nrb_iw_hh_c{collection_number}"
     elif product == "RTC_S1_STATIC":
-        return f"ga_s1_iw_static_c{collection_number}"
+        return f"ga_s1_nrb_iw_static_c{collection_number}"
 
 
 def make_rtc_s1_s3_subpath(
     s3_project_folder: str,
-    collection: str,
+    collection_number: int,
     burst_polarisations: list,
     burst_id: str,
     year: str,
     month: str,
     day: str,
 ):
-    """Structure for the rtc_s1 product sub-folders. These include
+    """Structure for the RTC_S1 product sub-folders. These include
     information about when the burst was acquired.
 
     Parameters
     ----------
     s3_project_folder : str
         s3 project folder
-    collection : str
-        collection. e.g. rtc_s1_static_c1
+    collection_number : int
+        collection number as an integer
     burst_polarisations: list
         list of burst polarisations
     burst_id : str
@@ -82,18 +49,18 @@ def make_rtc_s1_s3_subpath(
     -------
     str
         path to the s3 bucket subfolder
-        e.g. s3_project_folder/c1/s1_rtc_c1/ga_s1_iw_vv_c1/t028_059507_iw2/2022/01/01
+        e.g. s3_project_folder/ga_s1_nrb_iw_vv_c1/t028_059507_iw2/2022/01/01
     """
-    # get collection name and number from input collection
-    c_number = get_collection_number(collection)
-    # get the odc product name which includes the collection
-    odc_product_name = get_odc_product_name("RTC_S1", c_number, burst_polarisations)
-    return f"{s3_project_folder}/c{c_number}/{collection}/{odc_product_name}/{burst_id}/{year}/{month}/{day}"
+    # get the odc product name which includes the collection number
+    odc_product_name = get_odc_product_name(
+        "RTC_S1", collection_number, burst_polarisations
+    )
+    return f"{s3_project_folder}/{odc_product_name}/{burst_id}/{year}/{month}/{day}"
 
 
 def make_rtc_s1_static_s3_subpath(
     s3_project_folder: str,
-    collection: str,
+    collection_number: int,
     burst_id: str,
 ) -> str:
     """Structure for the bucket subpath for static layers
@@ -102,27 +69,25 @@ def make_rtc_s1_static_s3_subpath(
     ----------
     s3_project_folder : str
         s3 project folder
-    collection : str
-        collection. e.g. rtc_s1_static_c1
+    collection_number : int
+        collection number as an integer
     burst_id : str
         burst_id. e.g. t028_059507_iw2
 
     Returns
     -------
     str
         path to the s3 bucket subfolder
-        e.g. s3_project_folder/c1/s1_rtc_static_c1/ga_s1_iw_static_c1/t028_059507_iw2
+        e.g. s3_project_folder/ga_s1_nrb_iw_static_c1/t028_059507_iw2
     """
-    # get collection name and number from input collection
-    c_number = get_collection_number(collection)
-    # get the odc product name which includes the collection
-    odc_product_name = get_odc_product_name("RTC_S1_STATIC", c_number, [])
-    return f"{s3_project_folder}/c{c_number}/{collection}/{odc_product_name}/{burst_id}"
+    # get the odc product name which includes the collection number
+    odc_product_name = get_odc_product_name("RTC_S1_STATIC", collection_number, [])
+    return f"{s3_project_folder}/{odc_product_name}/{burst_id}"
 
 
 def make_static_layer_base_url(
     static_layers_s3_bucket: str,
-    static_layers_collection: str,
+    static_layers_collection_number: int,
     static_layers_s3_project_folder: str,
     s3_region: str = "ap-southeast-2",
 ) -> str:
@@ -132,8 +97,8 @@ def make_static_layer_base_url(
     ----------
     static_layers_s3_bucket : str
         Bucket containing static layer
-    static_layers_collection : str
-        collection static layers belong to
+    static_layers_collection_number : int
+        collection number of the static layers
     static_layers_s3_project_folder : str
         project folder within bucket if exists
     s3_region : str, optional
@@ -147,7 +112,7 @@ def make_static_layer_base_url(
     """
     root_static_layer_path = make_rtc_s1_static_s3_subpath(
         s3_project_folder=static_layers_s3_project_folder,
-        collection=static_layers_collection,
+        collection_number=static_layers_collection_number,
         burst_id="",
     )
     return (