From 7be95b9f16f9f18e2df2c88c5474c3c973f6d68c Mon Sep 17 00:00:00 2001 From: DanielElisenberg Date: Tue, 3 Jun 2025 15:02:41 +0200 Subject: [PATCH 1/3] cache-metadata-all: avoid caching draft --- metadata_service/adapter/datastore.py | 27 +++++++++++++++++++-------- 1 file changed, 19 insertions(+), 8 deletions(-) diff --git a/metadata_service/adapter/datastore.py b/metadata_service/adapter/datastore.py index 59d0f85..84c5681 100644 --- a/metadata_service/adapter/datastore.py +++ b/metadata_service/adapter/datastore.py @@ -22,19 +22,30 @@ def get_datastore_versions() -> dict: return json.load(f) -@lru_cache(maxsize=32) -def get_metadata_all(version: Version) -> str: - if version.is_draft(): - file_version = "DRAFT" - else: - file_version = version.to_3_underscored() +def _get_draft_metadata_all(): + metadata_all_file_path = ( + f"{DATASTORE_ROOT_DIR}/datastore/metadata_all__DRAFT.json" + ) + with open(metadata_all_file_path, "r", encoding="utf-8") as f: + return json.load(f) + +@lru_cache(maxsize=32) +def _get_versioned_metadata_all(version: Version): + file_version = version.to_3_underscored() metadata_all_file_path = ( f"{DATASTORE_ROOT_DIR}/datastore/metadata_all__{file_version}.json" ) + with open(metadata_all_file_path, "r", encoding="utf-8") as f: + return json.load(f) + + +def get_metadata_all(version: Version) -> str: try: - with open(metadata_all_file_path, "r", encoding="utf-8") as f: - return json.load(f) + if version.is_draft(): + return _get_draft_metadata_all() + else: + return _get_versioned_metadata_all(version) except FileNotFoundError as e: raise DataNotFoundException( f"metadata_all for version {version} not found" From 85342017adcdd4451b936168d516eea5044fca5e Mon Sep 17 00:00:00 2001 From: DanielElisenberg Date: Tue, 3 Jun 2025 15:32:48 +0200 Subject: [PATCH 2/3] cache-metadata-all: make version hashable --- metadata_service/api/metadata_api.py | 7 ++++--- metadata_service/domain/version.py | 14 ++++++-------- tests/unit/api/test_metadata_api.py | 14 +++++++------- tests/unit/domain/test_metadata.py | 22 +++++++++++++--------- 4 files changed, 30 insertions(+), 27 deletions(-) diff --git a/metadata_service/api/metadata_api.py b/metadata_service/api/metadata_api.py index b44df94..6f081a0 100644 --- a/metadata_service/api/metadata_api.py +++ b/metadata_service/api/metadata_api.py @@ -4,7 +4,7 @@ from metadata_service.api.request_models import NameParam, MetadataQuery from metadata_service.domain import metadata -from metadata_service.domain.version import Version +from metadata_service.domain.version import get_version_from_string logger = logging.getLogger() metadata_api = Blueprint("metadata_api", __name__) @@ -59,7 +59,7 @@ def get_data_structures(): response = jsonify( metadata.find_data_structures( validated_query.names, - Version(validated_query.version), + get_version_from_string(validated_query.version), validated_query.include_attributes, validated_query.skip_code_lists, ) @@ -84,7 +84,8 @@ def get_all_metadata(): response = jsonify( metadata.find_all_metadata( - Version(validated_query.version), validated_query.skip_code_lists + get_version_from_string(validated_query.version), + validated_query.skip_code_lists, ) ) response.headers.set("content-language", "no") diff --git a/metadata_service/domain/version.py b/metadata_service/domain/version.py index 73f02f9..c58fdbf 100644 --- a/metadata_service/domain/version.py +++ b/metadata_service/domain/version.py @@ -1,20 +1,13 @@ from dataclasses import dataclass -@dataclass +@dataclass(frozen=True) class Version: major: str minor: str patch: str draft: str - def __init__(self, version: str): - split = version.split(".") - self.major = split[0] - self.minor = split[1] - self.patch = split[2] - self.draft = split[3] - def to_3_underscored(self): return "_".join([self.major, self.minor, self.patch]) @@ -26,3 +19,8 @@ def is_draft(self): def __str__(self): return ".".join([self.major, self.minor, self.patch, self.draft]) + + +def get_version_from_string(version: str): + split = version.split(".") + return Version(split[0], split[1], split[2], split[3]) diff --git a/tests/unit/api/test_metadata_api.py b/tests/unit/api/test_metadata_api.py index 231144d..687092f 100644 --- a/tests/unit/api/test_metadata_api.py +++ b/tests/unit/api/test_metadata_api.py @@ -4,7 +4,7 @@ from flask import url_for, Response from metadata_service.domain import metadata -from metadata_service.domain.version import Version +from metadata_service.domain.version import Version, get_version_from_string MOCKED_DATASTORE_VERSIONS = { "name": "SSB-RAIRD", @@ -168,7 +168,7 @@ def test_get_data_structures(flask_app, mocker): ) spy.assert_called_with( - ["FNR", "AKT_ARBAP"], Version("3.2.1.0"), True, False + ["FNR", "AKT_ARBAP"], get_version_from_string("3.2.1.0"), True, False ) assert response.headers["Content-Type"] == "application/json" assert response.json == mocked_data_structures @@ -194,7 +194,7 @@ def test_get_data_structures_with_messagepack(flask_app, mocker): }, ) spy.assert_called_with( - ["FNR", "AKT_ARBAP"], Version("3.2.1.0"), True, False + ["FNR", "AKT_ARBAP"], get_version_from_string("3.2.1.0"), True, False ) assert response.headers["Content-Type"] == "application/x-msgpack" assert msgpack.loads(response.data) == mocked_data_structures @@ -246,7 +246,7 @@ def test_get_all_metadata(flask_app, mocker): "Accept": "application/json", }, ) - spy.assert_called_with(Version("3.2.1.0"), False) + spy.assert_called_with(get_version_from_string("3.2.1.0"), False) assert response.headers["Content-Type"] == "application/json" assert response.json == mocked_metadata_all @@ -276,7 +276,7 @@ def test_get_all_metadata_long_version_numbers(flask_app, mocker): "Accept": "application/json", }, ) - spy.assert_called_with(Version("1234.5678.9012.0"), False) + spy.assert_called_with(get_version_from_string("1234.5678.9012.0"), False) assert response.headers["Content-Type"] == "application/json" assert response.json == mocked_metadata_all @@ -313,7 +313,7 @@ def test_get_all_metadata_skip_code_lists(flask_app, mocker): "Accept": "application/json", }, ) - spy.assert_called_with(Version("3.2.1.0"), True) + spy.assert_called_with(get_version_from_string("3.2.1.0"), True) assert response.headers["Content-Type"] == "application/json" assert response.json == mocked_metadata_all @@ -339,7 +339,7 @@ def test_get_data_structures_skip_code_lists(flask_app, mocker): }, ) spy.assert_called_with( - ["FNR", "AKT_ARBAP"], Version("3.2.1.0"), True, True + ["FNR", "AKT_ARBAP"], get_version_from_string("3.2.1.0"), True, True ) assert response.headers["Content-Type"] == "application/json" assert response.json == mocked_data_structures diff --git a/tests/unit/domain/test_metadata.py b/tests/unit/domain/test_metadata.py index e23906e..aaed32d 100644 --- a/tests/unit/domain/test_metadata.py +++ b/tests/unit/domain/test_metadata.py @@ -7,7 +7,7 @@ from metadata_service.adapter import datastore from metadata_service.config import environment from metadata_service.domain import metadata -from metadata_service.domain.version import Version +from metadata_service.domain.version import Version, get_version_from_string from metadata_service.exceptions.exceptions import ( InvalidStorageFormatException, InvalidDraftVersionException, @@ -37,7 +37,7 @@ def test_find_two_data_structures_with_attrs(mocker): ) actual = metadata.find_data_structures( ["TEST_PERSON_INCOME", "TEST_PERSON_PETS"], - Version("1.0.0.0"), + get_version_from_string("1.0.0.0"), True, skip_code_lists=False, ) @@ -64,7 +64,7 @@ def test_find_two_data_structures_without_attrs(mocker): ) actual = metadata.find_data_structures( ["TEST_PERSON_INCOME", "TEST_PERSON_PETS"], - Version("1.0.0.0"), + get_version_from_string("1.0.0.0"), False, skip_code_lists=False, ) @@ -90,7 +90,7 @@ def test_find_data_structures_no_name_filter(mocker): datastore, "get_metadata_all", return_value=mocked_metadata_all ) actual = metadata.find_data_structures( - [], Version("1.0.0.0"), True, skip_code_lists=False + [], get_version_from_string("1.0.0.0"), True, skip_code_lists=False ) assert len(actual) == 2 @@ -237,7 +237,7 @@ def test_get_metadata_all_skip_code_list_and_missing_values(mocker): ) filtered_metadata = ( metadata.find_all_metadata_skip_code_list_and_missing_values( - Version("1.0.0.0") + get_version_from_string("1.0.0.0") ) ) _assert_code_list_and_missing_values(filtered_metadata["dataStructures"]) @@ -257,7 +257,7 @@ def test_find_all_metadata_skip_code_list_and_missing_values_invalid_model( ) with pytest.raises(InvalidStorageFormatException) as e: metadata.find_all_metadata_skip_code_list_and_missing_values( - Version("1.0.0.0") + get_version_from_string("1.0.0.0") ) assert "Invalid metadata format" == e.value.to_dict()["message"] @@ -269,7 +269,9 @@ def test_get_draft_metadata_all(mocker): mocker.patch.object( datastore, "get_metadata_all", return_value=mocked_metadata_all ) - filtered_metadata = metadata.find_all_metadata(Version("0.0.0.1608000000")) + filtered_metadata = metadata.find_all_metadata( + get_version_from_string("0.0.0.1608000000") + ) assert "dataStructures" in filtered_metadata @@ -281,7 +283,9 @@ def test_get_draft_metadata_all_0_0_0_0(mocker): mocker.patch.object( datastore, "get_metadata_all", return_value=mocked_metadata_all ) - filtered_metadata = metadata.find_all_metadata(Version("0.0.0.0")) + filtered_metadata = metadata.find_all_metadata( + get_version_from_string("0.0.0.0") + ) assert "dataStructures" in filtered_metadata @@ -295,7 +299,7 @@ def test_get_draft_metadata_all_invalid_draft_version(mocker): ) with pytest.raises(InvalidDraftVersionException) as e: - metadata.find_all_metadata(Version("0.0.0.2")) + metadata.find_all_metadata(get_version_from_string("0.0.0.2")) assert "Requested draft version" in str(e) From 6d6db3d268b4dae50bfdb30f6d36c8b1bb59d77c Mon Sep 17 00:00:00 2001 From: DanielElisenberg Date: Tue, 3 Jun 2025 15:37:29 +0200 Subject: [PATCH 3/3] cache-metadata-all: ruff fix --- tests/unit/api/test_metadata_api.py | 2 +- tests/unit/domain/test_metadata.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/unit/api/test_metadata_api.py b/tests/unit/api/test_metadata_api.py index 687092f..751d958 100644 --- a/tests/unit/api/test_metadata_api.py +++ b/tests/unit/api/test_metadata_api.py @@ -4,7 +4,7 @@ from flask import url_for, Response from metadata_service.domain import metadata -from metadata_service.domain.version import Version, get_version_from_string +from metadata_service.domain.version import get_version_from_string MOCKED_DATASTORE_VERSIONS = { "name": "SSB-RAIRD", diff --git a/tests/unit/domain/test_metadata.py b/tests/unit/domain/test_metadata.py index aaed32d..eb15ae5 100644 --- a/tests/unit/domain/test_metadata.py +++ b/tests/unit/domain/test_metadata.py @@ -7,7 +7,7 @@ from metadata_service.adapter import datastore from metadata_service.config import environment from metadata_service.domain import metadata -from metadata_service.domain.version import Version, get_version_from_string +from metadata_service.domain.version import get_version_from_string from metadata_service.exceptions.exceptions import ( InvalidStorageFormatException, InvalidDraftVersionException,