Merge branch 'develop' into fix_short_name_not_updated

Author: simon-20

db-migrations/20250605_01_Rsfu1.rollback.sql

@@ -0,0 +1,5 @@
+--
+-- depends: 20250319_04_RmMUy
+
+alter table iati_datasets
+    rename column licence_id to license_id;

db-migrations/20250605_01_Rsfu1.sql

@@ -0,0 +1,5 @@
+--
+-- depends: 20250319_04_RmMUy
+
+alter table iati_datasets
+    rename column license_id to licence_id;

src/bulk_data_service/dataset_indexing.py

@@ -107,7 +107,7 @@ def get_dataset_index_entry(context: dict, dataset: dict, index_type: str) -> di
         "reporting_org_id": None,
         "reporting_org_short_name": None,
         "source_url": None,
-        "license_id": None,
+        "licence_id": None,
         "last_update_check": None,
         "last_known_good_dataset": {
             "downloaded": None,
@@ -185,7 +185,7 @@ def create_cached_dataset_url(dataset, db_field, index_prefix, index_field):
         ("reporting_org_id", None, "reporting_org_id", None),
         ("reporting_org_short_name", None, "reporting_org_short_name", None),
         ("source_url", None, "source_url", None),
-        ("license_id", None, "license_id", None),
+        ("licence_id", None, "licence_id", None),
         ("last_update_check", None, "last_update_check", None),
         ("most_recent_head_attempt_datetime", "most_recent_head_attempt", "datetime", None),
         ("most_recent_head_attempt_http_status", "most_recent_head_attempt", "http_status", None),
@@ -234,7 +234,7 @@ def get_minimal_index_dataset_fields(context: dict) -> list[str]:
         "reporting_org_id",
         "reporting_org_short_name",
         "source_url",
-        "license_id",
+        "licence_id",
         "last_known_good_dataset_hash",
         "last_known_good_dataset_hash_excluding_generated_timestamp",
         "last_known_good_dataset_downloaded",

src/bulk_data_service/dataset_updater.py

@@ -371,7 +371,7 @@ def create_bds_dataset(registered_dataset: dict) -> dict:
         "reporting_org_id": registered_dataset["reporting_org_id"],
         "reporting_org_short_name": registered_dataset["reporting_org_short_name"],
         "source_url": registered_dataset["source_url"],
-        "license_id": registered_dataset["license_id"],
+        "licence_id": registered_dataset["licence_id"],
         "registration_service_dataset_metadata": registered_dataset["registration_service_dataset_metadata"],
         "registration_service_name": registered_dataset["registration_service_name"],
         "last_update_check": None,
@@ -401,7 +401,7 @@ def update_bds_dataset_registration_info(bds_dataset: dict, registered_dataset:
         "reporting_org_id",
         "reporting_org_short_name",
         "source_url",
-        "license_id",
+        "licence_id",
         "registration_service_dataset_metadata",
         "registration_service_name",
     ]:

src/bulk_data_service/zippers.py

@@ -13,7 +13,7 @@
     filter_dict_by_structure,
     get_number_xml_files_in_dir,
     get_timestamp_as_str_z,
-    lookup_license_title_from_id,
+    lookup_licence_title_from_id,
 )
 
 
@@ -127,8 +127,8 @@ def get_dataset_metadata_in_ckan_format(self, dataset: dict) -> str:
         return json.dumps(
             {
                 "id": str(dataset["id"]),
-                "license_id": dataset["license_id"],
-                "license_title": lookup_license_title_from_id(dataset["license_id"]),
+                "license_id": dataset["licence_id"],
+                "license_title": lookup_licence_title_from_id(dataset["licence_id"]),
                 "name": dataset["short_name"],
                 "organization": {"id": str(dataset["reporting_org_id"]), "name": dataset["reporting_org_short_name"]},
                 "resources": [{"url": dataset["source_url"]}],

src/dataset_registration/iati_registry_ckan.py

@@ -201,7 +201,7 @@ def convert_datasets_metadata(datasets_from_registry: list[dict]) -> dict[uuid.U
             "reporting_org_id": uuid.UUID(dataset["organization"]["id"]),
             "reporting_org_short_name": dataset["organization"]["name"],
             "source_url": get_source_url(dataset),
-            "license_id": dataset["license_id"],
+            "licence_id": dataset["license_id"],
             "registration_service_dataset_metadata": json.dumps(
                 {k: dataset[k] for k in dataset if k != "registration_service_publisher_metadata"}
             ),

src/utilities/db.py

@@ -71,7 +71,7 @@ def insert_or_update_dataset(connection: psycopg.Connection, data):
                         reporting_org_id = %(reporting_org_id)s,
                         reporting_org_short_name = %(reporting_org_short_name)s,
                         source_url = %(source_url)s,
-                        license_id = %(license_id)s,
+                        licence_id = %(licence_id)s,
                         registration_service_dataset_metadata = %(registration_service_dataset_metadata)s,
                         registration_service_name = %(registration_service_name)s,
 

src/utilities/misc.py

@@ -134,9 +134,9 @@ def find_object_by_key(objects: list, key: str, value_to_find: Any):
     return next(filter(lambda x: x[key] == value_to_find, objects), None)
 
 
-def lookup_license_title_from_id(license_id: str) -> str:
-    LICENSE_TITLE_LOOKUP = {
-        "notspecified": "License Not Specified",
+def lookup_licence_title_from_id(licence_id: str) -> str:
+    LICENCE_TITLE_LOOKUP = {
+        "notspecified": "Licence Not Specified",
         "odc-pddl": "Open Data Commons Public Domain Dedication and Licence (PDDL)",
         "odc-odbl": "Open Data Commons Open Database License (ODbL)",
         "odc-by": "Open Data Commons Attribution Licence",
@@ -216,4 +216,4 @@ def lookup_license_title_from_id(license_id: str) -> str:
         "zpl": "Zope Public License",
         "zlib-license": "zlib/libpng license",
     }
-    return LICENSE_TITLE_LOOKUP[license_id] if license_id in LICENSE_TITLE_LOOKUP else "Unknown License"
+    return LICENCE_TITLE_LOOKUP[licence_id] if licence_id in LICENCE_TITLE_LOOKUP else "Unknown License"

tests/helpers/data_helpers.py

@@ -49,7 +49,7 @@ def check_index_registration_fields(dataset: dict, dataset_index_item: dict):
     assert uuid.UUID(dataset_index_item["reporting_org_id"]) == dataset["reporting_org_id"]
     assert dataset_index_item["reporting_org_short_name"] == dataset["reporting_org_short_name"]
     assert dataset_index_item["source_url"] == dataset["source_url"]
-    assert dataset_index_item["license_id"] == dataset["license_id"]
+    assert dataset_index_item["licence_id"] == dataset["licence_id"]
 
 
 def check_index_most_recent_fields(context: dict, field_grouping: str, dataset: dict, dataset_index_item: dict):
@@ -90,7 +90,7 @@ def expected_values_for_dataset_registration_fields(source_url: str) -> list:
             ("reporting_org_id", uuid.UUID("ea055d99-f7e9-456f-9f99-963e95493c1b")),
             ("reporting_org_short_name", "test_foundation_a"),
             ("source_url", source_url),
-            ("license_id", "other-at"),
+            ("licence_id", "other-at"),
             ("registration_service_name", "ckan-registry"),
             ("registration_service_dataset_metadata", json.dumps(
                     {

tests/integration/test_dataset_update.py

@@ -17,7 +17,7 @@
     ("source_url", "http://localhost:3000/data/test_foundation_a-dataset-001.xml",
         "http://localhost:3000/not_found"),
     ("short_name", "test_foundation_a-dataset-001", "test_foundation_a-dataset-001-newname"),
-    ("license_id", "other-at", "uk-ogl"),
+    ("licence_id", "other-at", "uk-ogl"),
     ("registration_service_dataset_metadata", json.dumps(
             {
                 "author": None,
@@ -234,7 +234,7 @@ def test_update_dataset_registration_details(get_and_clear_up_context,  # noqa:
     assert len(datasets_in_bds) == 1
     assert datasets_in_bds[dataset_id][field] == original
 
-    # this is same dataset as above, with a different url and license
+    # this is same dataset as above, with a different url and licence
     context["DATA_REGISTRY_BASE_URL"] = "http://localhost:3000/ckan-registration/datasets-05-1-dataset-updated"
     checker_run(context, datasets_in_bds)