Add cache bound to version of sdk defs

.github/workflows/test_all.yml

@@ -13,6 +13,23 @@ on:
   pull_request:
 
 jobs:
+  windows_test:
+    runs-on: windows-latest
+    strategy:
+      matrix:
+        python-version: [ 3.7, 3.8, 3.9 ]
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Install requirements
+        run: |
+          python -m pip install ./
+      - name: Test
+        run: |
+          python3 -m unittest
+          python3 scripts/validate_my_definition.py -d alsdkdefs/apis/assets_query/
   test:
     runs-on: ubuntu-latest
     strategy:

alsdkdefs/__init__.py

@@ -12,21 +12,30 @@
 import requests
 import json
 import re
+from .persistent_storage import PersistentStorage
+
+try:
+    from .version import version as alsdkdefs_version
+except ImportError:
+    alsdkdefs_version = '0.0.1'
 
 try:
     import alsdkdefs_dev
+    from alsdkdefs_dev.version import version as alsdkdefs_dev_version
     DEV_SDK_DEFS = True
 except ImportError:
+    alsdkdefs_dev_version = '0.0.1dev'
     DEV_SDK_DEFS = False
 
 OPENAPI_SCHEMA_URL = 'https://raw.githubusercontent.com/OAI/OpenAPI-Specification/master/schemas/v3.0/schema.json'
 URI_SCHEMES = ['file', 'http', 'https']
 
 
 class ServiceDefinition:
-    def __init__(self, name, filespath):
+    def __init__(self, name, filespath, type='public'):
         self.service_name = name
         self.filespath = filespath
+        self.type = type
 
     def __str__(self):
         return self.service_name
@@ -43,6 +52,9 @@ def __eq__(self, other):
     def __lt__(self, other):
         return self.get_service_name() < other.get_service_name()
 
+    def get_service_type(self):
+        return self.type
+
     def get_service_name(self):
         return self.service_name
 
@@ -143,12 +155,35 @@ class _YamlOrderedLoader(yaml.SafeLoader):
 )
 
 
+class SpecCache(PersistentStorage):
+    def __init__(self, service_def, lib_version, load_fun, fun_args, **kwargs):
+        self.cache_name = f'normalized_schemas_{service_def.service_name}'
+        self.cache_key = self.make_cache_key(service_def, lib_version)
+        self.load_fun = load_fun
+        self.fun_args = fun_args
+        super().__init__(self.cache_name, **kwargs)
+
+    def get_spec(self):
+        normalized_spec = self.get(self.cache_key)
+        if normalized_spec:
+            return normalized_spec
+        else:
+            self.clear()
+            spec = self.load_fun(*self.fun_args)
+            self.set(self.cache_key, spec)
+            return spec
+
+    @staticmethod
+    def make_cache_key(service_def, lib_version):
+        return f"{service_def.get_service_name()}_{service_def.get_service_type()}_{lib_version}"
+
+
 def get_apis_dir():
     """Get absolute apis directory path on the fs"""
     return f"{pjoin(os.path.dirname(__file__), 'apis')}"
 
 
-def load_service_spec(service_name, apis_dir=None, version=None):
+def load_service_spec(service_name, apis_dir=None, version=None, use_persistent_cache=False):
     """Loads a version of service from library apis directory, if version is not specified, latest is loaded"""
     services = list_services(apis_dir)
     servicedef = services.get(service_name)
@@ -166,7 +201,12 @@ def load_service_spec(service_name, apis_dir=None, version=None):
     else:
         version = version[:1] != "v" and version or version[1:]
     service_spec_file_path = f"{pjoin(service_api_dir, service_name)}.v{version}.yaml"
-    return load_spec(service_spec_file_path)
+    if use_persistent_cache:
+        ver = {'dev': alsdkdefs_dev_version}.get(servicedef.type, alsdkdefs_version)
+        cache = SpecCache(servicedef, ver, load_spec, [service_spec_file_path])
+        return cache.get_spec()
+    else:
+        return load_spec(service_spec_file_path)
 
 
 def load_spec(uri_or_path):
@@ -197,7 +237,7 @@ def list_services(apis_dir=None):
     dev_services = []
     if DEV_SDK_DEFS:
         dev_dirs = alsdkdefs_dev.get_apis_dir()
-        dev_services = [ServiceDefinition(s, pjoin(dev_dirs, s)) for s in next(os.walk(dev_dirs))[1]]
+        dev_services = [ServiceDefinition(s, pjoin(dev_dirs, s), 'dev') for s in next(os.walk(dev_dirs))[1]]
     pub_services = [ServiceDefinition(s, pjoin(base_dir, s)) for s in next(os.walk(base_dir))[1]]
     services_search = OrderedDict(sorted([(str(servicedef), servicedef) for servicedef in pub_services + dev_services]))
     return services_search

alsdkdefs/persistent_storage.py

@@ -0,0 +1,111 @@
+import pickle
+from pathlib import Path
+import os
+from datetime import datetime
+import shutil
+
+DEFAULT_PATH = os.path.join(Path.home(), ".alertlogic", "cache")
+
+
+class PersistentStorage:
+    def __init__(self, namespace, storage_class_id='key_object_file_storage', **kwargs):
+        storage_class_selector = {'key_object_file_storage': KeyObjectFileStorage}
+        storage_class = storage_class_selector[storage_class_id]
+        self.storage = storage_class(namespace, **kwargs)
+
+    def set(self, key, value):
+        now = PersistentStorage.__now()
+        record = PersistentStorage.__storage_record(key, value, now)
+        return self.storage.set(key, record)
+
+    def get(self, key, default=None):
+        record = self.storage.get(key)
+        if record:
+            return PersistentStorage.__get_record_value(record)
+        else:
+            return default
+
+    def created_at(self, key):
+        record = self.storage.get(key)
+        if record:
+            return PersistentStorage.__get_record_created_at(record)
+
+    def clear(self):
+        return self.storage.clear()
+
+    def delete(self, key):
+        return self.storage.delete(key)
+
+    def list(self):
+        return self.storage.list()
+
+    @staticmethod
+    def __storage_record(key, value, created_at):
+        return {'key': key, 'value': value, 'created_at': created_at}
+
+    @staticmethod
+    def __get_record_value(record):
+        return record.get('value')
+
+    @staticmethod
+    def __get_record_created_at(record):
+        return record.get('created_at')
+
+    @staticmethod
+    def __now():
+        now = datetime.now()
+        timestamp = datetime.timestamp(now)
+        return int(timestamp)
+
+
+class KeyObjectStorage:
+    def __init__(self, namespace, **kwargs):
+        self.namespace = namespace
+
+    def set(self, key, value):
+        pass
+
+    def get(self, key, default=None):
+        pass
+
+    def delete(self, key):
+        pass
+
+    def list(self):
+        pass
+
+
+class KeyObjectFileStorage(KeyObjectStorage):
+    def __init__(self, namespace, **kwargs):
+        super().__init__(namespace, **kwargs)
+        storage_path = kwargs.get('storage_path', DEFAULT_PATH)
+        self.file_storage_path = os.path.join(storage_path, namespace)
+        self.__init()
+
+    def __init(self):
+        if not os.path.isdir(self.file_storage_path):
+            os.makedirs(self.file_storage_path)
+
+    def set(self, key, value):
+        with open(self.__key_file(key), 'wb') as f:
+            pickle.dump(value, f)
+
+    def get(self, key, default=None):
+        try:
+            with open(self.__key_file(key), 'rb') as f:
+                return pickle.load(f)
+        except FileNotFoundError:
+            return default
+
+    def delete(self, key):
+        os.remove(self.__key_file(key))
+
+    def clear(self):
+        shutil.rmtree(self.file_storage_path)
+        self.__init()
+
+    def list(self):
+        return os.listdir(self.file_storage_path)
+
+    def __key_file(self, key):
+        return os.path.join(self.file_storage_path, key)

tests/test_persistent_storage.py

@@ -0,0 +1,46 @@
+import unittest
+from alsdkdefs.persistent_storage import PersistentStorage
+from alsdkdefs import SpecCache
+from alsdkdefs import ServiceDefinition
+
+
+class MyTestCase(unittest.TestCase):
+    def test_ps(self):
+        stor = PersistentStorage('test_persistent_storage')
+        stor.clear()
+        assert stor.get('a_key') is None
+        assert stor.get('a_key', 123) == 123
+        stor.set('a_key', 'stored')
+        assert stor.get('a_key') == 'stored'
+        stor.set('a_key', 'stored2')
+        assert stor.get('a_key') == 'stored2'
+        assert isinstance(stor.created_at('a_key'), int)
+        stor.set('tuple', (1, 2, 3))
+        assert stor.get('tuple') == (1, 2, 3)
+        dictionary = {'a': 'b', 'b': 1, 'c': (1, 2, 3)}
+        stor.set('dict', dictionary)
+        assert stor.get('dict') == dictionary
+        assert sorted(stor.list()) == sorted(['a_key', 'dict', 'tuple'])
+        stor.delete('tuple')
+        assert sorted(stor.list()) == sorted(['a_key', 'dict'])
+        stor.clear()
+        assert stor.list() == []
+
+    def test_spec_stor(self):
+        sample_spec = {'key': 'value', 'key2': 'val2'}
+        service_def = ServiceDefinition('test_service', '/tmp/fake', 'public')
+        cache = SpecCache(service_def, 'v0.0.1', lambda x: {'key': 'value', 'key2': x}, ['val2'])
+        assert cache.get_spec() == sample_spec
+        assert cache.list() == ['test_service_public_v0.0.1']
+        assert cache.get('test_service_public_v0.0.1') == sample_spec
+        del cache
+        new_spec = {'key1': 'value', 'key3': 'val2'}
+        cache = SpecCache(service_def, 'v0.0.1', lambda x: {'key1': 'value', 'key3': x}, ['val2'])
+        assert cache.get_spec() == sample_spec
+        del cache
+        cache = SpecCache(service_def, 'v0.0.2', lambda x: {'key1': 'value', 'key3': x}, ['val2'])
+        assert cache.get_spec() == new_spec
+        assert cache.list() == ['test_service_public_v0.0.2']
+
+if __name__ == '__main__':
+    unittest.main()