Merge pull request #91 from GeoscienceAustralia/upgrade/product_formatting

STAC and product name updates

Author: abradley60

docs/workflows/aws_isce3_rtc.md

@@ -83,7 +83,7 @@ The AWS pipeline runs using a docker container. At runtime, the script [run_aws_
 --backscatter_convention=gamma0 # gamma0, sigma0 or beta0
 --s3_bucket="deant-data-public-dev"
 --s3_project_folder="baseline"
---collection_number=1
+--collection_number=0
 --make_existing_products=false
 --skip_upload_to_s3=false
 --scene_data_source=("AUS_COP_HUB" "ASF" "CDSE")
@@ -94,7 +94,7 @@ The AWS pipeline runs using a docker container. At runtime, the script [run_aws_
 --link_static_layers=false           
 --linked_static_layers_s3_bucket="deant-data-public-dev"
 --linked_static_layers_s3_project_folder="baseline" 
---linked_static_layers_collection_number="s1_rtc_static_c1" 
+--linked_static_layers_collection_number=0 
 ```
 - `scene` -> A valid sentinel-1 IW scene (e.g. S1A_IW_SLC__1SSH_20220101T124744_20220101T124814_041267_04E7A2_1DAD)
 - `burst_id_list` -> A list of burst id's corresponding to the scene. If not provided, all will be processed. Can be space separated list or line separated .txt file.

sar_pipeline/aws/cli.py

@@ -501,7 +501,7 @@ def get_data_for_scene_and_make_run_config(
     # check if the collection_number passed in aligns with the
     # product_version specified in the config
     prod_version = RTC_RUN_CONFIG.get(f"{gk}.product_group.product_version")
-    prod_version_collection_number = int(prod_version.split(".")[0])
+    prod_version_collection_number = int(re.split(r"[-.]", prod_version)[0])
     if prod_version_collection_number != collection_number:
         logger.warning(
             f"The specified collection_number is {collection_number}. However, "
@@ -649,11 +649,26 @@ def make_rtc_opera_stac_and_upload_bursts(
     """
 
     # iterate through the burst directory and create STAC metadata
+    logger.info(
+        f"Iterating through the burst folders to reformat files and create STAC metadata"
+    )
     burst_folders = [x for x in results_folder.iterdir() if x.is_dir()]
+
     for i, burst_folder in enumerate(burst_folders):
         logger.info(
             f"Making STAC metadata for burst {i+1} of {len(burst_folders)} : {burst_folder}"
         )
+
+        logger.info(f"Renaming all files so 'v' is not in the product version number ")
+        for product_file in burst_folder.iterdir():
+            if product_file.is_file():
+                new_path = product_file.with_name(
+                    re.sub(r"v(?=\d)", "", product_file.name)
+                )
+                if new_path != product_file:
+                    logger.info(f"Renaming: {product_file.name} -> {new_path.name}")
+                    product_file.rename(new_path)
+
         # load in the .h5 file containing metadata for each burst
         burst_h5_files = list(burst_folder.glob("*.h5"))
         if len(burst_h5_files) != 1:
@@ -665,15 +680,15 @@ def make_rtc_opera_stac_and_upload_bursts(
 
         # get the product name common to files from the .h5
         # e.g. ga_s1_nrb-static_v0.1.0_T070-149815-IW3_20140403
-        burst_product_name = burst_h5_filepath.stem.replace(".metadata", "")
+        burst_product_name = burst_h5_filepath.stem.replace("_metadata", "")
 
         # rename the .h5 file to conform with ga naming conventions
         burst_h5_filepath = burst_h5_filepath.rename(
-            burst_folder / f"{burst_product_name}.metadata.h5"
+            burst_folder / f"{burst_product_name}_metadata.h5"
         )
         # copy the run config file to the burst folder and rename to conform with ga naming conventions
         burst_run_config_filepath = (
-            burst_folder / f"{burst_product_name}.proc-config.yaml"
+            burst_folder / f"{burst_product_name}_proc-config.yaml"
         )
         shutil.copy(run_config_path, burst_run_config_filepath)
 
@@ -711,7 +726,7 @@ def make_rtc_opera_stac_and_upload_bursts(
             burst_stac_manager.add_linked_static_layers_as_assets_to_stac()
         # add additional links to metadata files. e.g. stac, xml, proc config
         logging.info(f"Adding links to metadata files and self")
-        stac_filepath = burst_folder / f"{burst_product_name}.stac-item.json"
+        stac_filepath = burst_folder / f"{burst_product_name}_stac-item.json"
         burst_stac_manager.add_metadata_links(
             stac_filepath=stac_filepath,
             h5_filepath=burst_h5_filepath,

sar_pipeline/aws/metadata/odc.py

@@ -65,7 +65,7 @@ def make_rtc_s1_s3_subpath(
     odc_product_name = get_odc_product_name(
         "RTC_S1", collection_number, burst_polarisations
     )
-    return f"{s3_project_folder}/{odc_product_name}/{burst_id}/{year}/{month}/{day}"
+    return f"{s3_project_folder}/{odc_product_name}/{burst_id}/{year}/{month:02d}/{day:02d}"
 
 
 def make_rtc_s1_static_s3_subpath(

sar_pipeline/aws/metadata/stac.py

@@ -322,7 +322,9 @@ def add_properties_from_h5(self):
         self.item.properties["processing:software"] = {
             "isce3": self.h5.search_value("algorithms/isce3Version"),
             "s1Reader": self.h5.search_value("algorithms/s1ReaderVersion"),
-            "OPERA-adt/RTC": self.h5.search_value("algorithms/softwareVersion"),
+            "GeoscienceAustralia/RTC": self.h5.search_value(
+                "algorithms/softwareVersion"
+            ),
             "sar-pipeline": sar_pipeline.__version__,
             "dem-handler": dem_handler.__version__,
         }
@@ -399,12 +401,11 @@ def add_properties_from_h5(self):
 
         # add the storage stac extension properties
         self.item.properties["storage:schemes"] = {
-            "aws-std": {
+            "aws": {
                 "type": "aws-s3",
                 "platform": "https://{bucket}.s3.{region}.amazonaws.com",
                 "bucket": f"{self.s3_bucket}",
                 "region": f"{self.s3_region}",
-                "requester_pays": True,
             }
         }
 
@@ -416,6 +417,7 @@ def add_fixed_links(self):
             pystac.Link(
                 rel="geoid-source",
                 target="https://aria-geoid.s3.us-west-2.amazonaws.com/us_nga_egm2008_1_4326__agisoft.tif",
+                media_type=pystac.media_type.MediaType.GEOTIFF,
             )
         )
 
@@ -446,6 +448,7 @@ def add_dynamic_links_from_h5(self):
             pystac.Link(
                 rel="dem-source",
                 target=self._extract_http_link(self.h5.search_value("demSource")),
+                media_type=pystac.media_type.MediaType.HTML,
             )
         )
 
@@ -458,6 +461,7 @@ def add_dynamic_links_from_h5(self):
                 pystac.Link(
                     rel="rtc-algorithm",
                     target=self._extract_doi_link(ref_text),
+                    media_type=pystac.media_type.MediaType.HTML,
                 )
             )
 
@@ -467,6 +471,7 @@ def add_dynamic_links_from_h5(self):
             pystac.Link(
                 rel="geocoding-algorithm",
                 target=self._extract_doi_link(ref_text),
+                media_type=pystac.media_type.MediaType.HTML,
             )
         )
 
@@ -477,6 +482,7 @@ def add_dynamic_links_from_h5(self):
                 pystac.Link(
                     rel="noise-correction",
                     target=self._extract_http_link(ref_text),
+                    media_type=pystac.media_type.MediaType.PDF,
                 )
             )
 
@@ -524,7 +530,7 @@ def add_metadata_links(
             pystac.Link(
                 rel="processing-config",
                 target=f"{self.base_href}/{Path(runconfig_filepath).name}",
-                media_type=pystac.media_type.MediaType.HDF5,
+                media_type="application/yaml",
             )
         )
 
@@ -533,7 +539,7 @@ def add_metadata_links(
             pystac.Link(
                 rel="browse",
                 target=f"{self.browse_href}",
-                media_type=pystac.media_type.MediaType.JSON,
+                media_type=pystac.media_type.MediaType.HTML,
             )
         )
 
@@ -543,7 +549,7 @@ def add_metadata_links(
             pystac.Link(
                 rel="self",
                 target=f"{self.base_href}/{Path(stac_filepath).name}",
-                media_type=pystac.media_type.MediaType.JSON,
+                media_type=pystac.media_type.STAC_JSON,
             )
         )
 
@@ -583,7 +589,7 @@ def add_collection_link(
             pystac.Link(
                 rel="collection",
                 target=stac_href,
-                media_type=pystac.media_type.MediaType.JSON,
+                media_type=pystac.media_type.STAC_JSON,
             )
         )
 
@@ -821,6 +827,7 @@ def add_linked_static_layers_as_assets_to_stac(
             pystac.Link(
                 rel="static-layers-stac-item",
                 target=burst_static_layer_stac_url,
+                media_type=pystac.media_type.STAC_JSON,
             )
         )
 
@@ -834,6 +841,7 @@ def add_linked_static_layers_as_assets_to_stac(
             pystac.Link(
                 rel="static-layers-browse",
                 target=static_layer_browse_url,
+                media_type=pystac.media_type.MediaType.HTML,
             )
         )
 

sar_pipeline/configs/ISCE3-RTC/S1_RTC.yaml

@@ -42,7 +42,7 @@ runconfig:
             # <collection_number>.<minor_version>.<patch_version>
             # note the collection_number is set by the pipeline script
             # at runtime and should align to this value
-            product_version: 0.1.0
+            product_version: 0-1-0
 
             # Directory where PGE will place results
             product_path: .

sar_pipeline/configs/ISCE3-RTC/S1_RTC_STATIC.yaml

@@ -43,7 +43,7 @@ runconfig:
             # <collection_number>.<minor_version>.<patch_version>
             # note the collection_number is set by the pipeline script
             # at runtime and should align to this value
-            product_version: 0.1.0
+            product_version: 0-1-0
 
             # Directory where PGE will place results
             product_path: .