Merge pull request #65 from statisticsnorway/cache-metadata-all

cache: add thread-safe cache with 32 versions max

Author: pawbu

metadata_service/adapter/datastore.py

@@ -1,4 +1,5 @@
 import json
+from functools import lru_cache
 
 from metadata_service.config import environment
 from metadata_service.domain.version import Version
@@ -21,18 +22,30 @@ def get_datastore_versions() -> dict:
         return json.load(f)
 
 
-def get_metadata_all(version: Version) -> str:
-    if version.is_draft():
-        file_version = "DRAFT"
-    else:
-        file_version = version.to_3_underscored()
+def _get_draft_metadata_all():
+    metadata_all_file_path = (
+        f"{DATASTORE_ROOT_DIR}/datastore/metadata_all__DRAFT.json"
+    )
+    with open(metadata_all_file_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
 
+@lru_cache(maxsize=32)
+def _get_versioned_metadata_all(version: Version):
+    file_version = version.to_3_underscored()
     metadata_all_file_path = (
         f"{DATASTORE_ROOT_DIR}/datastore/metadata_all__{file_version}.json"
     )
+    with open(metadata_all_file_path, "r", encoding="utf-8") as f:
+        return json.load(f)
+
+
+def get_metadata_all(version: Version) -> str:
     try:
-        with open(metadata_all_file_path, "r", encoding="utf-8") as f:
-            return json.load(f)
+        if version.is_draft():
+            return _get_draft_metadata_all()
+        else:
+            return _get_versioned_metadata_all(version)
     except FileNotFoundError as e:
         raise DataNotFoundException(
             f"metadata_all for version {version} not found"

metadata_service/api/metadata_api.py

@@ -4,7 +4,7 @@
 
 from metadata_service.api.request_models import NameParam, MetadataQuery
 from metadata_service.domain import metadata
-from metadata_service.domain.version import Version
+from metadata_service.domain.version import get_version_from_string
 
 logger = logging.getLogger()
 metadata_api = Blueprint("metadata_api", __name__)
@@ -59,7 +59,7 @@ def get_data_structures():
     response = jsonify(
         metadata.find_data_structures(
             validated_query.names,
-            Version(validated_query.version),
+            get_version_from_string(validated_query.version),
             validated_query.include_attributes,
             validated_query.skip_code_lists,
         )
@@ -84,7 +84,8 @@ def get_all_metadata():
 
     response = jsonify(
         metadata.find_all_metadata(
-            Version(validated_query.version), validated_query.skip_code_lists
+            get_version_from_string(validated_query.version),
+            validated_query.skip_code_lists,
         )
     )
     response.headers.set("content-language", "no")

metadata_service/domain/version.py

@@ -1,20 +1,13 @@
 from dataclasses import dataclass
 
 
-@dataclass
+@dataclass(frozen=True)
 class Version:
     major: str
     minor: str
     patch: str
     draft: str
 
-    def __init__(self, version: str):
-        split = version.split(".")
-        self.major = split[0]
-        self.minor = split[1]
-        self.patch = split[2]
-        self.draft = split[3]
-
     def to_3_underscored(self):
         return "_".join([self.major, self.minor, self.patch])
 
@@ -26,3 +19,8 @@ def is_draft(self):
 
     def __str__(self):
         return ".".join([self.major, self.minor, self.patch, self.draft])
+
+
+def get_version_from_string(version: str):
+    split = version.split(".")
+    return Version(split[0], split[1], split[2], split[3])

tests/unit/api/test_metadata_api.py

@@ -4,7 +4,7 @@
 from flask import url_for, Response
 
 from metadata_service.domain import metadata
-from metadata_service.domain.version import Version
+from metadata_service.domain.version import get_version_from_string
 
 MOCKED_DATASTORE_VERSIONS = {
     "name": "SSB-RAIRD",
@@ -168,7 +168,7 @@ def test_get_data_structures(flask_app, mocker):
     )
 
     spy.assert_called_with(
-        ["FNR", "AKT_ARBAP"], Version("3.2.1.0"), True, False
+        ["FNR", "AKT_ARBAP"], get_version_from_string("3.2.1.0"), True, False
     )
     assert response.headers["Content-Type"] == "application/json"
     assert response.json == mocked_data_structures
@@ -194,7 +194,7 @@ def test_get_data_structures_with_messagepack(flask_app, mocker):
         },
     )
     spy.assert_called_with(
-        ["FNR", "AKT_ARBAP"], Version("3.2.1.0"), True, False
+        ["FNR", "AKT_ARBAP"], get_version_from_string("3.2.1.0"), True, False
     )
     assert response.headers["Content-Type"] == "application/x-msgpack"
     assert msgpack.loads(response.data) == mocked_data_structures
@@ -246,7 +246,7 @@ def test_get_all_metadata(flask_app, mocker):
             "Accept": "application/json",
         },
     )
-    spy.assert_called_with(Version("3.2.1.0"), False)
+    spy.assert_called_with(get_version_from_string("3.2.1.0"), False)
     assert response.headers["Content-Type"] == "application/json"
     assert response.json == mocked_metadata_all
 
@@ -276,7 +276,7 @@ def test_get_all_metadata_long_version_numbers(flask_app, mocker):
             "Accept": "application/json",
         },
     )
-    spy.assert_called_with(Version("1234.5678.9012.0"), False)
+    spy.assert_called_with(get_version_from_string("1234.5678.9012.0"), False)
     assert response.headers["Content-Type"] == "application/json"
     assert response.json == mocked_metadata_all
 
@@ -313,7 +313,7 @@ def test_get_all_metadata_skip_code_lists(flask_app, mocker):
             "Accept": "application/json",
         },
     )
-    spy.assert_called_with(Version("3.2.1.0"), True)
+    spy.assert_called_with(get_version_from_string("3.2.1.0"), True)
     assert response.headers["Content-Type"] == "application/json"
     assert response.json == mocked_metadata_all
 
@@ -339,7 +339,7 @@ def test_get_data_structures_skip_code_lists(flask_app, mocker):
         },
     )
     spy.assert_called_with(
-        ["FNR", "AKT_ARBAP"], Version("3.2.1.0"), True, True
+        ["FNR", "AKT_ARBAP"], get_version_from_string("3.2.1.0"), True, True
     )
     assert response.headers["Content-Type"] == "application/json"
     assert response.json == mocked_data_structures

tests/unit/domain/test_metadata.py

@@ -7,7 +7,7 @@
 from metadata_service.adapter import datastore
 from metadata_service.config import environment
 from metadata_service.domain import metadata
-from metadata_service.domain.version import Version
+from metadata_service.domain.version import get_version_from_string
 from metadata_service.exceptions.exceptions import (
     InvalidStorageFormatException,
     InvalidDraftVersionException,
@@ -37,7 +37,7 @@ def test_find_two_data_structures_with_attrs(mocker):
     )
     actual = metadata.find_data_structures(
         ["TEST_PERSON_INCOME", "TEST_PERSON_PETS"],
-        Version("1.0.0.0"),
+        get_version_from_string("1.0.0.0"),
         True,
         skip_code_lists=False,
     )
@@ -64,7 +64,7 @@ def test_find_two_data_structures_without_attrs(mocker):
     )
     actual = metadata.find_data_structures(
         ["TEST_PERSON_INCOME", "TEST_PERSON_PETS"],
-        Version("1.0.0.0"),
+        get_version_from_string("1.0.0.0"),
         False,
         skip_code_lists=False,
     )
@@ -90,7 +90,7 @@ def test_find_data_structures_no_name_filter(mocker):
         datastore, "get_metadata_all", return_value=mocked_metadata_all
     )
     actual = metadata.find_data_structures(
-        [], Version("1.0.0.0"), True, skip_code_lists=False
+        [], get_version_from_string("1.0.0.0"), True, skip_code_lists=False
     )
     assert len(actual) == 2
 
@@ -237,7 +237,7 @@ def test_get_metadata_all_skip_code_list_and_missing_values(mocker):
     )
     filtered_metadata = (
         metadata.find_all_metadata_skip_code_list_and_missing_values(
-            Version("1.0.0.0")
+            get_version_from_string("1.0.0.0")
         )
     )
     _assert_code_list_and_missing_values(filtered_metadata["dataStructures"])
@@ -257,7 +257,7 @@ def test_find_all_metadata_skip_code_list_and_missing_values_invalid_model(
     )
     with pytest.raises(InvalidStorageFormatException) as e:
         metadata.find_all_metadata_skip_code_list_and_missing_values(
-            Version("1.0.0.0")
+            get_version_from_string("1.0.0.0")
         )
     assert "Invalid metadata format" == e.value.to_dict()["message"]
 
@@ -269,7 +269,9 @@ def test_get_draft_metadata_all(mocker):
     mocker.patch.object(
         datastore, "get_metadata_all", return_value=mocked_metadata_all
     )
-    filtered_metadata = metadata.find_all_metadata(Version("0.0.0.1608000000"))
+    filtered_metadata = metadata.find_all_metadata(
+        get_version_from_string("0.0.0.1608000000")
+    )
 
     assert "dataStructures" in filtered_metadata
 
@@ -281,7 +283,9 @@ def test_get_draft_metadata_all_0_0_0_0(mocker):
     mocker.patch.object(
         datastore, "get_metadata_all", return_value=mocked_metadata_all
     )
-    filtered_metadata = metadata.find_all_metadata(Version("0.0.0.0"))
+    filtered_metadata = metadata.find_all_metadata(
+        get_version_from_string("0.0.0.0")
+    )
 
     assert "dataStructures" in filtered_metadata
 
@@ -295,7 +299,7 @@ def test_get_draft_metadata_all_invalid_draft_version(mocker):
     )
 
     with pytest.raises(InvalidDraftVersionException) as e:
-        metadata.find_all_metadata(Version("0.0.0.2"))
+        metadata.find_all_metadata(get_version_from_string("0.0.0.2"))
 
     assert "Requested draft version" in str(e)