Merge pull request #118 from aodn/AwsRegistry

Feat: add function to retrieve info from CSV to populate AWS Registry

Author: lbesnard

.github/workflows/build.yml

@@ -79,7 +79,7 @@ jobs:
       - name: Upload coverage report
         uses: actions/upload-artifact@v4
         with:
-          name: coverage-report-${{ github.job }}-${{ github.run_number }}
+          name: coverage-report-${{ github.job }}-${{ github.run_number }}-${{ matrix.python-version }}
           path: coverage.xml
 
       - name: Build package

aodn_cloud_optimised/bin/create_aws_registry_dataset.py

@@ -27,21 +27,23 @@
 import io
 import json
 import os
+import sys
 import tempfile
 from argparse import RawTextHelpFormatter
 from importlib.resources import files
 
+import pandas as pd
 import requests
 import xmltodict
-from colorama import init, Fore, Style
+from colorama import Fore, Style, init
 
+from aodn_cloud_optimised.lib.common import list_json_files
 from aodn_cloud_optimised.lib.CommonHandler import CommonHandler
 from aodn_cloud_optimised.lib.config import (
-    load_dataset_config,
     load_config,
+    load_dataset_config,
     load_variable_from_config,
 )
-from aodn_cloud_optimised.lib.common import list_json_files
 
 
 def retrieve_geonetwork_metadata(
@@ -192,6 +194,63 @@ def update_nested_dict_key(dataset_config, keys, new_value):
     return dataset_config
 
 
+def populate_dataset_config_with_metadata_from_csv(json_file, csv_path):
+    json_path = str(files("aodn_cloud_optimised.config.dataset").joinpath(json_file))
+    dataset_config = load_dataset_config(json_path)
+
+    csv_data = pd.read_csv(
+        csv_path,
+        index_col="Cloud_Optimised_Collection_Name",
+        encoding="ISO-8859-1",
+        na_filter=False,
+    )
+
+    dataset_name = dataset_config["dataset_name"]
+    try:
+        csv_dataset = csv_data.loc[dataset_name]
+    except Exception as err:
+        print(f"{dataset_name} NOT FOUND in CSV file")
+        return
+
+    if not csv_dataset["AWS_Title"] == "":
+        dataset_config = update_nested_dict_key(
+            dataset_config,
+            ["aws_opendata_registry", "Name"],
+            csv_dataset["AWS_Title"],
+        )
+    else:
+        Warning(f"AWS_Title for {dataset_name} is missing from {csv_path}")
+
+    if not csv_dataset["AWS_Tags"] == "":
+        aws_tags = [keyword.strip() for keyword in csv_dataset["AWS_Tags"].split(";")]
+        dataset_config = update_nested_dict_key(
+            dataset_config, ["aws_opendata_registry", "Tags"], aws_tags
+        )
+    else:
+        Warning(f"AWS_Tags for {dataset_name} is missing from {csv_path}")
+
+    if not csv_dataset["AWS_Citation"] == "":
+        dataset_config = update_nested_dict_key(
+            dataset_config,
+            ["aws_opendata_registry", "Citation"],
+            csv_dataset["AWS_Citation"],
+        )
+    else:
+        Warning(f"AWS_Citation for {dataset_name} is missing from {csv_path}")
+
+    # dataset config coming from load_dataset_config is the result of parent and child configuration. When writing back
+    # the configuration, we only want to write the child data back
+    dataset_config_child = load_config(json_path)
+    # Overwrite the original JSON file with the modified dataset_config
+    with open(json_path, "w") as f:
+        dataset_config_child["aws_opendata_registry"] = dataset_config[
+            "aws_opendata_registry"
+        ]
+        json.dump(dataset_config_child, f, indent=2)
+
+    print(f"Updated JSON file saved at: {json_path}")
+
+
 def populate_dataset_config_with_geonetwork_metadata(json_file):
     """ """
 
@@ -286,7 +345,7 @@ def populate_dataset_config_with_geonetwork_metadata(json_file):
         dataset_config, ["aws_opendata_registry", "Resources"], dataset_location
     )
 
-    # dataset confi coming from load_dataset_config is the result of parent and child configuration. When writing back
+    # dataset config coming from load_dataset_config is the result of parent and child configuration. When writing back
     # the configuration, we only want to write the child data back
     dataset_config_child = load_config(json_path)
     # Overwrite the original JSON file with the modified dataset_config
@@ -317,29 +376,7 @@ def convert_to_opendata_registry(json_file, output_directory):
     handler.create_metadata_aws_registry(target_directory=output_directory)
 
 
-def main():
-    """
-    Main function to convert JSON files to AWS OpenData Registry format.
-
-    The script can be run in different ways:
-
-    1. Convert a specific JSON file to AWS OpenData Registry format.
-    2. Convert all JSON files in the directory.
-    3. Run interactively to list all available JSON files and prompt the user to choose one to convert.
-
-    Important:
-        If the -g option is provided, the script will download metadata from the GeoNetwork metadata
-        record and prompt the user to choose to replace existing values or not.
-
-
-    Args (optional):
-        -f, --file (str): Name of a specific JSON file to convert.
-        -d, --directory (str): Output directory to save converted YAML files.
-        -a, --all: Convert all JSON files in the directory.
-        -g, --geonetwork: Retrieve metadata fields from GeoNetwork3 metadata record
-
-    If the directory is not specified, a temporary directory is created.
-    """
+def parse_args(arg_list: list[str] | None):
     parser = argparse.ArgumentParser(
         description="""
         Create AWS OpenData Registry YAML files from the dataset configuration, ready to be added to the OpenData Github
@@ -371,24 +408,71 @@ def main():
         help="Retrieve metadata from Geonetwork instance to populate OpenData Registry format. Interactive mode",
     )
 
+    parser.add_argument(
+        "-c",
+        "--csv-path",
+        help="Add specific metadata from an external CSV file",
+    )
+
     args = parser.parse_args()
 
+    return args
+
+
+def main(arg_list: list[str] | None = None):
+    """
+    Main function to convert JSON files to AWS OpenData Registry format.
+
+    The script can be run in different ways:
+
+    1. Convert a specific JSON file to AWS OpenData Registry format.
+    2. Convert all JSON files in the directory.
+    3. Run interactively to list all available JSON files and prompt the user to choose one to convert.
+
+    Important:
+        If the -g option is provided, the script will download metadata from the GeoNetwork metadata
+        record and prompt the user to choose to replace existing values or not.
+
+
+    Args (optional):
+        -f, --file (str): Name of a specific JSON file to convert.
+        -d, --directory (str): Output directory to save converted YAML files.
+        -a, --all: Convert all JSON files in the directory.
+        -g, --geonetwork: Retrieve metadata fields from GeoNetwork3 metadata record
+
+    If the directory is not specified, a temporary directory is created.
+    """
     json_directory = str(files("aodn_cloud_optimised.config.dataset")._paths[0])
 
+    args = parse_args(sys.argv[1:])
+
     if args.all:
         json_files = list_json_files(json_directory)
         if json_files:
             output_dir = args.directory or tempfile.mkdtemp()
             for file in json_files:
                 if args.geonetwork:
                     populate_dataset_config_with_geonetwork_metadata(file)
+
+                if args.csv_path:
+                    if os.path.exists(args.csv_path):
+                        populate_dataset_config_with_metadata_from_csv(
+                            file, args.csv_path
+                        )
+                    else:
+                        raise ValueError(f"{args.csv_path} does not exist")
                 convert_to_opendata_registry(file, output_dir)
         else:
             print(f"No JSON files found in {json_directory}.")
     elif args.file:
         output_dir = args.directory or tempfile.mkdtemp()
         if args.geonetwork:
             populate_dataset_config_with_geonetwork_metadata(args.file)
+        if args.csv_path:
+            if os.path.exists(args.csv_path):
+                populate_dataset_config_with_metadata_from_csv(args.file, args.csv_path)
+            else:
+                raise ValueError(f"{args.csv_path} does not exist")
 
         convert_to_opendata_registry(args.file, output_dir)
     else:
@@ -406,6 +490,15 @@ def main():
                         populate_dataset_config_with_geonetwork_metadata(
                             json_files[choice_idx]
                         )
+
+                    if args.csv_path:
+                        if os.path.exists(args.csv_path):
+                            populate_dataset_config_with_metadata_from_csv(
+                                json_files[choice_idx], args.csv_path
+                            )
+                        else:
+                            raise ValueError(f"{args.csv_path} does not exist")
+
                     convert_to_opendata_registry(json_files[choice_idx], output_dir)
                 else:
                     print("Invalid choice. Aborting.")

aodn_cloud_optimised/config/dataset/animal_acoustic_tracking_delayed_qc.json

@@ -201,7 +201,8 @@
     "UpdateFrequency": "As Needed",
     "Tags": [
       "oceans",
-      "marine mammals"
+      "marine mammals",
+      "biology"
     ],
     "License": "http://creativecommons.org/licenses/by/4.0/",
     "Resources": [
@@ -230,6 +231,6 @@
         }
       ]
     },
-    "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]"
+    "Citation": "The citation in a list of references is: \"IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access].\""
   }
 }

aodn_cloud_optimised/config/dataset/animal_ctd_satellite_relay_tagging_delayed_qc.json

@@ -432,14 +432,18 @@
   },
   "force_old_pq_del": false,
   "aws_opendata_registry": {
-    "Name": "Satellite Relay Tagging Program - Southern Ocean - MEOP Quality Controlled CTD Profiles",
+    "Name": "Marine Animal - Satellite Relay Tagging -  Quality controlled profiles",
     "Description": "CTD (Conductivity-Temperature_Depth)-Satellite Relay Data Loggers (CTD-SRDLs) are used to explore how marine animal behaviour relates to their oceanic environment. Loggers developed at the University of St Andrews Sea Mammal Research Unit transmit data in near real-time via the Argo satellite system. Data represented here was collected in the Southern Ocean, from elephant, fur and Weddell Seals. In 2024 data was added from flatback and olive ridley turtles, from a pilot study co-funded by the Royal Australian Navy in collaboration with the Australian Institute of Marine Science and Indigenous Ranger groups.\n\nData parameters measured by the instruments include time, conductivity (salinity), temperature, pressure and depth. The data represented by this record have been Qc'd and are the Australian subset of the MEOP-CTD database (MEOP: Marine Mammals Exploring the Oceans Pole to Pole), complemented with the most recent Southern Ocean deployment data. This Australian subset of the Southern Ocean database represents about one quarter of the entire MEOP-CTD database, which currently is about 52,000 profiles obtained from 275 CTD-SRDL tag deployments. The Australian dataset originated in 2004, and was initially collected by Mark Hindell's team based at the University of Tasmania, and in later years his data has formed part of the Animal Tracking Facility of Integrated Marine Observing System (IMOS).",
     "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/95d6314c-cfc7-40ae-b439-85f14541db71",
     "Contact": "info@aodn.org.au",
     "ManagedBy": "AODN",
     "UpdateFrequency": "As Needed",
     "Tags": [
-      "FILL UP MANUALLY - CHECK DOCUMENTATION"
+      "oceans",
+      "marine mammals",
+      "biology",
+      "chemistry",
+      "chemical biology"
     ],
     "License": "http://creativecommons.org/licenses/by/4.0/",
     "Resources": [
@@ -468,6 +472,6 @@
         }
       ]
     },
-    "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]"
+    "Citation": "The citation in a list of references is: \"Integrated Marine Observing System (IMOS); Hindell, Mark [year-of-data-download], [Title], Antarctic Climate and Ecosystem Cooperative Research Centre (ACE CRC); Institute for Marine and Antarctic Studies (IMAS), University of Tasmania (UTAS), [data-access-URL], accessed [date-of-access].\" If data includes turtle data between January 2022 - June 2024, please cite as follows:\n\"Integrated Marine Observing System (IMOS); Hindell, Mark; Department of Defence [year-of-data-download],  [Title], Antarctic Climate and Ecosystem Cooperative Research Centre (ACE CRC); Institute for Marine and Antarctic Studies (IMAS), University of Tasmania (UTAS), [data-access-URL], accessed [date-of-access].\""
   }
 }

aodn_cloud_optimised/config/dataset/autonomous_underwater_vehicle.json

@@ -179,14 +179,16 @@
   },
   "force_old_pq_del": false,
   "aws_opendata_registry": {
-    "Name": "IMOS - Autonomous Underwater Vehicle (AUV) Facility",
+    "Name": "Autonomous underwater vehicles",
     "Description": "The IMOS Autonomous Underwater Vehicle (AUV) Facility operates multiple ocean-going AUVs capable of undertaking high resolution, geo-referenced survey work. AUV Sirius is a modified version of a mid-size robotic vehicle Seabed built at the Woods Hole Oceanographic Institution. This class of AUV has been designed specifically for low speed, high resolution imaging and is passively stable in pitch and roll. AUV Nimbus is a custom design with the aim of deploying off smaller vessels than Sirius whilst remaining able to operate in similar environments. AUV Iver is an extended Ocean Server Iver2 class platform with additional navigation and imaging sensors but lacks the ability to work over complex underwater terrain. Based on its small size, it can be deployed from RHIBs and other small vessels. The main objective of the IMOS AUV Facility is to support sustained observations of the benthos, in support of the IMOS integrated benthic monitoring program. . The AUV facility is based at the Australian Centre for Field Robotics (ACFR) within the School of Aerospace, Mechanical and Mechatronic Engineering at the University of Sydney.\n\nThis IMOS Facility finished in June 2023, with data still available through the AODN Portal.",
     "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/af5d0ff9-bb9c-4b7c-a63c-854a630b6984",
     "Contact": "info@aodn.org.au",
     "ManagedBy": "AODN",
     "UpdateFrequency": "As Needed",
     "Tags": [
-      "FILL UP MANUALLY - CHECK DOCUMENTATION"
+      "oceans",
+      "chemistry",
+      "chemical biology"
     ],
     "License": "http://creativecommons.org/licenses/by/4.0/",
     "Resources": [
@@ -215,6 +217,6 @@
         }
       ]
     },
-    "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]"
+    "Citation": "The citation in a list of references is: \"IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access].\""
   }
 }

aodn_cloud_optimised/config/dataset/model_sea_level_anomaly_gridded_realtime.json

@@ -170,6 +170,6 @@
         }
       ]
     },
-    "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]"
+    "Citation": "The citation in a list of references is: \"IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access].\""
   }
 }

aodn_cloud_optimised/config/dataset/mooring_hourly_timeseries_delayed_qc.json

@@ -813,6 +813,6 @@
         }
       ]
     },
-    "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]. If using data from the Ningaloo (TAN100) mooring, please add to the citation - \"Department of Jobs, Tourism, Science and Innovation (DJTSI), Western Australian Government\". "
+    "Citation": "The citation in a list of references is: \"IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access].\" If using data from the Ningaloo (TAN100) mooring, please add to the citation - \"Department of Jobs, Tourism, Science and Innovation (DJTSI), Western Australian Government\". "
   }
 }

aodn_cloud_optimised/config/dataset/mooring_satellite_altimetry_calibration_validation.json

@@ -302,14 +302,16 @@
   },
   "force_old_pq_del": false,
   "aws_opendata_registry": {
-    "Name": "IMOS - SRS Satellite Altimetry Calibration and Validation Sub-Facility",
+    "Name": "Satellite - Altimetry calibration and validation",
     "Description": "High precision satellite altimeter missions including TOPEX/Poseidon (T/P), Jason-1 and now OSTM/Jason-2, have contributed fundamental advances in our understanding of regional and global ocean circulation and its role in the Earth's climate and regional applications. These altimeter satellites essentially observe the height of the global oceans \u2013 as such, they have become the tool of choice for scientists to measure sea level rise \u2013 both at regional and global scales as well as giving information about ocean currents and large- and small-scale variability. The determination of changes in global mean sea level is of fundamental importance in understanding the response of the ocean to a continuing warming climate \u2013 both through thermal expansion of the ocean, melting of the major ice sheets of Greenland and Antarctica, and mountain glaciers, and redistribution of water over the continents and atmosphere. As with all scientific observations, it is vital that the measurement tool is operating within its specifications \u2013 calibration and validation is therefore an important component in this regard. \n\nThis IMOS sub-facility provides the sole southern hemisphere in situ calibration site to provide an ongoing calibration and validation data stream directly to the international (NASA and CNES sponsored) Ocean Surface Topography Science Team (OSTST). This contribution, importantly, spans multiple altimeter missions, thus enabling the assimilation of multi-mission satellite data to determine ongoing changes in sea level with flow-on effects to other uses of this data. The OSTST collectively oversees the operation and calibration of the precision altimetry missions, and ensures each of these missions is performing as accurately as possible in order to meet mission objectives. The IMOS supported calibration site in Australia is one of four primary in situ calibration/validation sites that contribute to the OSTST. The remaining sites include Harvest (USA), Corsica (France), and Gavdos (Greece). The Australian calibration site includes two comparison points where in situ data is compared against the altimeter \u2013 Bass Strait and Storm Bay. These two locations both lie on descending (N -> S) pass 088 of the satellite altimeter, and thus share similar satellite orbit characteristics. The use of these two sites allows detailed investigation into the accuracy of the altimeter over two different wave climates. The average significant wave height at Storm Bay is approximately double that observed at the comparatively sheltered Bass Strait location. One of the ongoing issues with satellite altimeter missions is the \u201cSea-state bias\u201d correction which is related to wave shape and height. We plan to use the different wave climates at the two locations, coupled with the fact that some other things (such as orbit errors) are the same at both locations to improve the quality of this correction.",
     "Documentation": "https://catalogue-imos.aodn.org.au/geonetwork/srv/eng/catalog.search#/metadata/78d588ed-79dd-47e2-b806-d39025194e7e",
     "Contact": "info@aodn.org.au",
     "ManagedBy": "AODN",
     "UpdateFrequency": "As Needed",
     "Tags": [
-      "FILL UP MANUALLY - CHECK DOCUMENTATION"
+      "oceans",
+      "ocean currents",
+      "chemistry"
     ],
     "License": "http://creativecommons.org/licenses/by/4.0/",
     "Resources": [
@@ -338,6 +340,6 @@
         }
       ]
     },
-    "Citation": "IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access]"
+    "Citation": "The citation in a list of references is: \"IMOS [year-of-data-download], [Title], [data-access-URL], accessed [date-of-access].\""
   }
 }