diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index de6dde4..cb5bf04 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -44,7 +44,7 @@ jobs: uses: docker/setup-buildx-action@v3 - name: Replace run integration test command run: | - sed -i "s+mundialis/actinia:latest+mundialis/actinia:2.9.2_G83+g" docker/actinia-module-plugin-test/Dockerfile + sed -i "s+mundialis/actinia:latest+mundialis/actinia:grass8.3+g" docker/actinia-module-plugin-test/Dockerfile - name: Tests of actinia-module-plugin id: docker_build uses: docker/build-push-action@v6 diff --git a/README.md b/README.md index c3deca6..78d5fa7 100644 --- a/README.md +++ b/README.md @@ -29,7 +29,7 @@ in any module, import `from actinia_module_plugin.resources.logging import log` ### Running tests -You can run the tests in the actinia-modules-plugin-test docker. For that you can comment the execution of the test in the docker/actinia-modules-plugin-test/Dockerfile `RUN ./tests_with_redis.sh` and run the following commands: +You can run the tests in the actinia-modules-plugin-test docker. For that you can comment the execution of the test in the docker/actinia-modules-plugin-test/Dockerfile `RUN ./tests_with_kvdb.sh` and run the following commands: ```bash docker build -f docker/actinia-module-plugin-test/Dockerfile -t actinia-module-plugin-test . @@ -39,5 +39,5 @@ docker run -it actinia-module-plugin-test -i cd /src/actinia-module-plugin/ # run all tests -./tests_with_redis.sh +./tests_with_kvdb.sh ``` diff --git a/docker/actinia-module-plugin-test/Dockerfile b/docker/actinia-module-plugin-test/Dockerfile index 0c0a643..d10ea4b 100644 --- a/docker/actinia-module-plugin-test/Dockerfile +++ b/docker/actinia-module-plugin-test/Dockerfile @@ -12,7 +12,7 @@ ENV TEMPLATE_VALUE_ENV_RASTER=elevation ENV TEMPLATE_VALUE_ENV_TYPE=raster # install things only for tests -RUN apk add --no-cache redis +RUN apk add --no-cache valkey valkey-cli RUN pip install --upgrade setuptools && pip install pytest pytest-cov pwgen ENTRYPOINT ["/bin/sh"] @@ -35,7 +35,7 @@ COPY docker/actinia-module-plugin-test/actinia-module-plugin-test.cfg /etc/defau COPY docker/actinia-module-plugin-test/actinia-module-plugin-test.cfg /etc/default/actinia-module-plugin-test COPY . /src/actinia-module-plugin/ -RUN chmod a+x tests_with_redis.sh +RUN chmod a+x tests_with_kvdb.sh RUN pip install . -RUN ./tests_with_redis.sh +RUN ./tests_with_kvdb.sh diff --git a/docker/actinia-module-plugin-test/actinia-module-plugin-test.cfg b/docker/actinia-module-plugin-test/actinia-module-plugin-test.cfg index 1c0f02d..e2f8415 100644 --- a/docker/actinia-module-plugin-test/actinia-module-plugin-test.cfg +++ b/docker/actinia-module-plugin-test/actinia-module-plugin-test.cfg @@ -14,9 +14,9 @@ force_https_urls = True [QUEUE] queue_type = local -[REDIS] -redis_server_url = localhost -redis_server_port = 6379 +[KVDB] +kvdb_server_url = localhost +kvdb_server_port = 6379 worker_queue_name = actinia_job worker_logfile = /actinia_core/workspace/tmp/actinia_worker_test.log diff --git a/src/actinia_module_plugin/api/modules/actinia.py b/src/actinia_module_plugin/api/modules/actinia.py index 1a39593..f9dc078 100644 --- a/src/actinia_module_plugin/api/modules/actinia.py +++ b/src/actinia_module_plugin/api/modules/actinia.py @@ -17,7 +17,7 @@ actinia-module viewer -Templates can be stored file based and in redis +Templates can be stored file based and in kvdb * List all actinia-modules * Describe single actinia-module @@ -40,7 +40,7 @@ createProcessChainTemplateListFromFileSystem, ) from actinia_module_plugin.core.modules.actinia_user_templates import ( - createProcessChainTemplateListFromRedis, + createProcessChainTemplateListFromKvdb, ) from actinia_module_plugin.core.modules.actinia_common import ( createActiniaModule, @@ -59,8 +59,8 @@ def get(self): """Get a list of all actinia modules (process chain templates).""" pc_list_fs = createProcessChainTemplateListFromFileSystem() - pc_list_redis = createProcessChainTemplateListFromRedis() - pc_list = pc_list_fs + pc_list_redis + pc_list_kvdb = createProcessChainTemplateListFromKvdb() + pc_list = pc_list_fs + pc_list_kvdb pc_list = filter(pc_list) diff --git a/src/actinia_module_plugin/api/modules/combined.py b/src/actinia_module_plugin/api/modules/combined.py index c89a6c4..467edd1 100644 --- a/src/actinia_module_plugin/api/modules/combined.py +++ b/src/actinia_module_plugin/api/modules/combined.py @@ -17,7 +17,7 @@ actinia-module + grass-module viewer -Templates can be stored file based and in redis +Templates can be stored file based and in kvdb * List all GRASS GIS modules and actinia-modules * Describe single module @@ -39,7 +39,7 @@ createProcessChainTemplateListFromFileSystem, ) from actinia_module_plugin.core.modules.actinia_user_templates import ( - createProcessChainTemplateListFromRedis, + createProcessChainTemplateListFromKvdb, ) from actinia_module_plugin.core.modules.actinia_common import ( createActiniaModule, @@ -75,8 +75,8 @@ def get(self): final_grass_list = createFullModuleList(self, final_grass_list) pc_list_fs = createProcessChainTemplateListFromFileSystem() - pc_list_redis = createProcessChainTemplateListFromRedis() - module_list = final_grass_list + pc_list_fs + pc_list_redis + pc_list_kvdb = createProcessChainTemplateListFromKvdb() + module_list = final_grass_list + pc_list_fs + pc_list_kvdb module_list = filter(module_list) diff --git a/src/actinia_module_plugin/api/modules/grass.py b/src/actinia_module_plugin/api/modules/grass.py index 3b3aec7..cd85d66 100644 --- a/src/actinia_module_plugin/api/modules/grass.py +++ b/src/actinia_module_plugin/api/modules/grass.py @@ -39,8 +39,8 @@ from actinia_module_plugin.core.modules.grass import createGrassModule from actinia_module_plugin.core.modules.grass import createFullModuleList from actinia_module_plugin.core.modules.grass import installGrassAddon -from actinia_module_plugin.core.modules.accessible_modules_redis_interface import ( - addGrassAddonToModuleListRedis, +from actinia_module_plugin.core.modules.accessible_modules_kvdb_interface import ( + addGrassAddonToModuleListKvdb, ) from actinia_module_plugin.model.modules import ModuleList from actinia_module_plugin.model.responseModels import ( @@ -104,7 +104,7 @@ def post(self, grassmodule): response = installGrassAddon(self, grassmodule) if response["status"] == "finished": - addGrassAddonToModuleListRedis(self, grassmodule) + addGrassAddonToModuleListKvdb(self, grassmodule) msg = "Successfully installed GRASS addon " + grassmodule + "." status_code = 201 else: diff --git a/src/actinia_module_plugin/api/processing.py b/src/actinia_module_plugin/api/processing.py index 521583e..e0b4344 100644 --- a/src/actinia_module_plugin/api/processing.py +++ b/src/actinia_module_plugin/api/processing.py @@ -53,7 +53,7 @@ from flask import jsonify, make_response from flask_restful_swagger_2 import swagger -from actinia_core.core.common.redis_interface import enqueue_job +from actinia_core.core.common.kvdb_interface import enqueue_job from actinia_core.models.response_models import create_response_from_model from actinia_core.processing.common.ephemeral_processing_with_export import ( start_job as start_job_ephemeral_processing_with_export, @@ -73,7 +73,7 @@ createProcessChainTemplateListFromFileSystem, ) from actinia_module_plugin.core.modules.actinia_user_templates import ( - createProcessChainTemplateListFromRedis, + createProcessChainTemplateListFromKvdb, ) from actinia_module_plugin.core.modules.grass import createModuleList from actinia_module_plugin.core.processing import fillTemplateFromProcessChain @@ -105,7 +105,7 @@ def log_error_to_resource_logger(self, msg, rdc): resource_id=self.resource_id, iteration=1, document=data, - expiration=rdc.config.REDIS_RESOURCE_EXPIRE_TIME, + expiration=rdc.config.KVDB_RESOURCE_EXPIRE_TIME, ) @@ -167,7 +167,7 @@ def preprocess_build_pc_and_enqueue(self, preprocess_kwargs, start_job): # get grass and actinia module lists module_list = createModuleList(self) pc_global_list = createProcessChainTemplateListFromFileSystem() - pc_user_list = createProcessChainTemplateListFromRedis() + pc_user_list = createProcessChainTemplateListFromKvdb() grass_module_list = [] actinia_module_list = [] diff --git a/src/actinia_module_plugin/core/modules/accessible_modules_redis_interface.py b/src/actinia_module_plugin/core/modules/accessible_modules_kvdb_interface.py similarity index 68% rename from src/actinia_module_plugin/core/modules/accessible_modules_redis_interface.py rename to src/actinia_module_plugin/core/modules/accessible_modules_kvdb_interface.py index 3578a5e..1a11937 100644 --- a/src/actinia_module_plugin/core/modules/accessible_modules_redis_interface.py +++ b/src/actinia_module_plugin/core/modules/accessible_modules_kvdb_interface.py @@ -16,7 +16,7 @@ limitations under the License. -Accessible Modules Redis Interface +Accessible Modules Kvdb Interface """ __license__ = "Apache-2.0" @@ -24,37 +24,37 @@ __copyright__ = "Copyright 2021 - 2022, mundialis GmbH & Co. KG" __maintainer__ = "mundialis GmbH & Co. KG" -from actinia_core.core.redis_user import RedisUserInterface +from actinia_core.core.kvdb_user import KvdbUserInterface from actinia_core.core.common.config import Configuration -def getAccessibleModuleListRedis(self): - redis_interface = RedisUserInterface() +def getAccessibleModuleListKvdb(self): + kvdb_interface = KvdbUserInterface() conf = Configuration() try: conf.read() except Exception: pass - server = conf.REDIS_SERVER_URL - port = conf.REDIS_SERVER_PORT - if conf.REDIS_SERVER_PW: - redis_password = conf.REDIS_SERVER_PW + server = conf.KVDB_SERVER_URL + port = conf.KVDB_SERVER_PORT + if conf.KVDB_SERVER_PW: + kvdb_password = conf.KVDB_SERVER_PW else: - redis_password = None - redis_interface.connect(host=server, port=port, password=redis_password) + kvdb_password = None + kvdb_interface.connect(host=server, port=port, password=kvdb_password) user = self.user.get_id() - access_modules = redis_interface.get_credentials(user)["permissions"][ + access_modules = kvdb_interface.get_credentials(user)["permissions"][ "accessible_modules" ] - redis_interface.disconnect() + kvdb_interface.disconnect() return access_modules -def addGrassAddonToModuleListRedis(self, grassmodule): +def addGrassAddonToModuleListKvdb(self, grassmodule): """ This function adds installed GRASS addon to the user's module list - in redis. + in kvdb. """ self.user.add_accessible_modules( [ diff --git a/src/actinia_module_plugin/core/modules/actinia_common.py b/src/actinia_module_plugin/core/modules/actinia_common.py index 6936a28..2d8f935 100644 --- a/src/actinia_module_plugin/core/modules/actinia_common.py +++ b/src/actinia_module_plugin/core/modules/actinia_common.py @@ -17,7 +17,7 @@ actinia-module viewer -Common module for file based and redis templates +Common module for file based and kvdb templates """ __license__ = "Apache-2.0" @@ -207,7 +207,7 @@ def execute_interface_descr(self, vp): pc_item = { item_key: {"module": vp.module, "interface-description": True} } - # TODO make use of module in redis cache if exists + # TODO make use of module in kvdb cache if exists response = run_process_chain(self.resourceBaseSelf, pc_item) xml_strings = response["process_log"] self.count = self.count + 1 diff --git a/src/actinia_module_plugin/core/modules/actinia_user_templates.py b/src/actinia_module_plugin/core/modules/actinia_user_templates.py index 11530f4..e38a53e 100644 --- a/src/actinia_module_plugin/core/modules/actinia_user_templates.py +++ b/src/actinia_module_plugin/core/modules/actinia_user_templates.py @@ -17,7 +17,7 @@ actinia-module viewer -Module for templates stored in redis which are user defined +Module for templates stored in kvdb which are user defined """ __license__ = "Apache-2.0" @@ -31,7 +31,7 @@ from actinia_module_plugin.model.modules import Module -def createProcessChainTemplateListFromRedis(): +def createProcessChainTemplateListFromKvdb(): """ list all stored templates and return as actinia-module list """ diff --git a/src/actinia_module_plugin/core/modules/grass.py b/src/actinia_module_plugin/core/modules/grass.py index 375ba91..8cc9932 100644 --- a/src/actinia_module_plugin/core/modules/grass.py +++ b/src/actinia_module_plugin/core/modules/grass.py @@ -35,11 +35,11 @@ from actinia_module_plugin.core.modules.parser import ParseInterfaceDescription from actinia_module_plugin.model.modules import Module from actinia_module_plugin.resources.logging import log -from actinia_module_plugin.core.modules.grass_modules_redis_interface import ( - redis_grass_module_interface, +from actinia_module_plugin.core.modules.grass_modules_kvdb_interface import ( + kvdb_grass_module_interface, ) -from actinia_module_plugin.core.modules.accessible_modules_redis_interface import ( - getAccessibleModuleListRedis, +from actinia_module_plugin.core.modules.accessible_modules_kvdb_interface import ( + getAccessibleModuleListKvdb, ) @@ -114,7 +114,7 @@ def createModuleList(self): def createModuleUserList(self): - return getAccessibleModuleListRedis(self) + return getAccessibleModuleListKvdb(self) def build_and_run_iface_description_pc(self, module_list): @@ -144,33 +144,33 @@ def build_and_run_iface_description_pc(self, module_list): def connect(): - """This method initializes the connection with redis.""" + """This method initializes the connection with kvdb.""" conf = Configuration() try: conf.read() except Exception: pass - server = conf.REDIS_SERVER_URL - port = conf.REDIS_SERVER_PORT - if conf.REDIS_SERVER_PW: - redis_password = conf.REDIS_SERVER_PW + server = conf.KVDB_SERVER_URL + port = conf.KVDB_SERVER_PORT + if conf.KVDB_SERVER_PW: + kvdb_password = conf.KVDB_SERVER_PW else: - redis_password = None + kvdb_password = None - redis_grass_module_interface.connect( - host=server, port=port, password=redis_password + kvdb_grass_module_interface.connect( + host=server, port=port, password=kvdb_password ) - return redis_grass_module_interface + return kvdb_grass_module_interface def cacheGrassModule(grass_module): """ Insert grass_module into database """ - redis_grass_module_interface = connect() - cached_module = redis_grass_module_interface.create(grass_module) + kvdb_grass_module_interface = connect() + cached_module = kvdb_grass_module_interface.create(grass_module) return cached_module @@ -186,9 +186,9 @@ def createGrassModule(self, module): # This is currently the only mechanism to update a cached # module via API call. # If not, then cache it - redis_grass_module_interface = connect() - if redis_grass_module_interface.exists(module): - redis_grass_module_interface.update(module, grass_module) + kvdb_grass_module_interface = connect() + if kvdb_grass_module_interface.exists(module): + kvdb_grass_module_interface.update(module, grass_module) else: cacheGrassModule(grass_module) @@ -201,8 +201,8 @@ def createFullModuleList(self, module_list): # Check if modules are already cached, then use them for module in module_list: - redis_grass_module_interface = connect() - grass_module = redis_grass_module_interface.read(module["id"]) + kvdb_grass_module_interface = connect() + grass_module = kvdb_grass_module_interface.read(module["id"]) if grass_module: detailed_module_list.append(grass_module) else: diff --git a/src/actinia_module_plugin/core/modules/grass_modules_redis_interface.py b/src/actinia_module_plugin/core/modules/grass_modules_kvdb_interface.py similarity index 84% rename from src/actinia_module_plugin/core/modules/grass_modules_redis_interface.py rename to src/actinia_module_plugin/core/modules/grass_modules_kvdb_interface.py index e238d58..cea07b1 100644 --- a/src/actinia_module_plugin/core/modules/grass_modules_redis_interface.py +++ b/src/actinia_module_plugin/core/modules/grass_modules_kvdb_interface.py @@ -16,8 +16,8 @@ limitations under the License. -GRASS GIS module cache in redis -Redis interface +GRASS GIS module cache in kvdb +Kvdb interface """ __license__ = "Apache-2.0" @@ -28,12 +28,12 @@ import pickle -from actinia_core.core.common.redis_base import RedisBaseInterface +from actinia_core.core.common.kvdb_base import KvdbBaseInterface -class RedisActiniaGrassModuleInterface(RedisBaseInterface): +class KvdbActiniaGrassModuleInterface(KvdbBaseInterface): """ - The Redis GRASS GIS module database interface + The Kvdb GRASS GIS module database interface A single GRASS GIS module is stored as Hash with: - GRASS GIS module name that must be unique @@ -47,7 +47,7 @@ class RedisActiniaGrassModuleInterface(RedisBaseInterface): grass_module_id_db = "GRASS-MODULE-ID-DATABASE" def __init__(self): - RedisBaseInterface.__init__(self) + KvdbBaseInterface.__init__(self) def create(self, grass_module): """ @@ -64,7 +64,7 @@ def create(self, grass_module): grass_module_id = grass_module["id"] keyname = self.grass_module_id_hash_prefix + grass_module_id - exists = self.redis_server.exists(keyname) + exists = self.kvdb_server.exists(keyname) if exists == 1 or exists is True: return False @@ -74,14 +74,14 @@ def create(self, grass_module): "grass_module": grass_module_bytes, } - lock = self.redis_server.lock(name="add_grass_module_lock", timeout=1) + lock = self.kvdb_server.lock(name="add_grass_module_lock", timeout=1) lock.acquire() # First add the grass_module-id to the grass_module id database - self.redis_server.hset( + self.kvdb_server.hset( self.grass_module_id_db, grass_module_id, grass_module_id ) - self.redis_server.hset( + self.kvdb_server.hset( self.grass_module_id_hash_prefix + grass_module_id, mapping=mapping ) lock.release() @@ -104,7 +104,7 @@ def read(self, grass_module_id): try: grass_module = pickle.loads( - self.redis_server.hget( + self.kvdb_server.hget( self.grass_module_id_hash_prefix + grass_module_id, "grass_module", ) @@ -131,7 +131,7 @@ def update(self, grass_module_id, grass_module): """ keyname = self.grass_module_id_hash_prefix + grass_module_id - exists = self.redis_server.exists(keyname) + exists = self.kvdb_server.exists(keyname) if exists == 0 or exists is False: return False @@ -141,12 +141,12 @@ def update(self, grass_module_id, grass_module): "grass_module": grass_module_bytes, } - lock = self.redis_server.lock( + lock = self.kvdb_server.lock( name="update_grass_module_lock", timeout=1 ) lock.acquire() - self.redis_server.hset( + self.kvdb_server.hset( self.grass_module_id_hash_prefix + grass_module_id, mapping=mapping ) @@ -169,14 +169,14 @@ def update(self, grass_module_id, grass_module): # if exists == 0 or exists is False: # return False - # lock = self.redis_server.lock( + # lock = self.kvdb_server.lock( # name="delete_grass_module_lock", timeout=1) # lock.acquire() # # Delete the entry from the grass_module id database - # self.redis_server.hdel(self.grass_module_id_db, + # self.kvdb_server.hdel(self.grass_module_id_db, # grass_module_id) # # Delete the actual grass_module entry - # self.redis_server.delete( + # self.kvdb_server.delete( # self.grass_module_id_hash_prefix + grass_module_id) # lock.release() @@ -193,7 +193,7 @@ def update(self, grass_module_id, grass_module): # A list of all grass_module ids in the database # """ # values = [] - # list = self.redis_server.hkeys(self.grass_module_id_db) + # list = self.kvdb_server.hkeys(self.grass_module_id_db) # for entry in list: # entry = entry.decode() # values.append(entry) @@ -211,10 +211,10 @@ def exists(self, grass_module_id): bool: True is grass_module exists, False otherwise """ - return self.redis_server.exists( + return self.kvdb_server.exists( self.grass_module_id_hash_prefix + grass_module_id ) -# Create the Redis interface instance -redis_grass_module_interface = RedisActiniaGrassModuleInterface() +# Create the Kvdb interface instance +kvdb_grass_module_interface = KvdbActiniaGrassModuleInterface() diff --git a/src/actinia_module_plugin/core/processing.py b/src/actinia_module_plugin/core/processing.py index 81d650a..84f9345 100644 --- a/src/actinia_module_plugin/core/processing.py +++ b/src/actinia_module_plugin/core/processing.py @@ -101,7 +101,7 @@ def fill_env_values(filled_params, undef): def fillTemplateFromProcessChain(module): """ This method receives a process chain for an actinia module and loads - the according process chain template from redis or filesystem. The + the according process chain template from kvdb or filesystem. The received values will be replaced to be passed to actinia. In case the template has more placeholder values than it receives, the missing attribute is returned as string. diff --git a/src/actinia_module_plugin/core/templates/user_templates.py b/src/actinia_module_plugin/core/templates/user_templates.py index f1cfa5b..89737e8 100644 --- a/src/actinia_module_plugin/core/templates/user_templates.py +++ b/src/actinia_module_plugin/core/templates/user_templates.py @@ -17,7 +17,7 @@ Process Chain Template Management -Redis CRUD for user templates +Kvdb CRUD for user templates """ __license__ = "Apache-2.0" @@ -28,39 +28,39 @@ from actinia_core.core.common.config import Configuration -from actinia_module_plugin.core.templates.user_templates_redis_interface import ( - redis_actinia_template_interface, +from actinia_module_plugin.core.templates.user_templates_kvdb_interface import ( + kvdb_actinia_template_interface, ) def connect(): - """This method initializes the connection with redis.""" + """This method initializes the connection with kvdb.""" conf = Configuration() try: conf.read() except Exception: pass - server = conf.REDIS_SERVER_URL - port = conf.REDIS_SERVER_PORT - if conf.REDIS_SERVER_PW: - redis_password = conf.REDIS_SERVER_PW + server = conf.KVDB_SERVER_URL + port = conf.KVDB_SERVER_PORT + if conf.KVDB_SERVER_PW: + kvdb_password = conf.KVDB_SERVER_PW else: - redis_password = None + kvdb_password = None - redis_actinia_template_interface.connect( - host=server, port=port, password=redis_password + kvdb_actinia_template_interface.connect( + host=server, port=port, password=kvdb_password ) - return redis_actinia_template_interface + return kvdb_actinia_template_interface def readAll(): """ - Get all actinia templates from redis database + Get all actinia templates from kvdb database """ - redis_actinia_template_interface = connect() - actinia_template = redis_actinia_template_interface.list_all_ids() + kvdb_actinia_template_interface = connect() + actinia_template = kvdb_actinia_template_interface.list_all_ids() return actinia_template @@ -69,8 +69,8 @@ def createTemplate(pc_tpl): """ Insert actinia template into database """ - redis_actinia_template_interface = connect() - actinia_template = redis_actinia_template_interface.create(pc_tpl) + kvdb_actinia_template_interface = connect() + actinia_template = kvdb_actinia_template_interface.create(pc_tpl) return actinia_template @@ -79,8 +79,8 @@ def readTemplate(template_id): """ Get actinia template by id """ - redis_actinia_template_interface = connect() - actinia_template = redis_actinia_template_interface.read(template_id) + kvdb_actinia_template_interface = connect() + actinia_template = kvdb_actinia_template_interface.read(template_id) return actinia_template @@ -89,8 +89,8 @@ def updateTemplate(template_id, pc_tpl): """ Update actinia template by id """ - redis_actinia_template_interface = connect() - actinia_template = redis_actinia_template_interface.update( + kvdb_actinia_template_interface = connect() + actinia_template = kvdb_actinia_template_interface.update( template_id, pc_tpl ) @@ -101,7 +101,7 @@ def deleteTemplate(template_id): """ Delete actinia template by id """ - redis_actinia_template_interface = connect() - actinia_template = redis_actinia_template_interface.delete(template_id) + kvdb_actinia_template_interface = connect() + actinia_template = kvdb_actinia_template_interface.delete(template_id) return actinia_template diff --git a/src/actinia_module_plugin/core/templates/user_templates_redis_interface.py b/src/actinia_module_plugin/core/templates/user_templates_kvdb_interface.py similarity index 86% rename from src/actinia_module_plugin/core/templates/user_templates_redis_interface.py rename to src/actinia_module_plugin/core/templates/user_templates_kvdb_interface.py index 626a582..4b7546a 100644 --- a/src/actinia_module_plugin/core/templates/user_templates_redis_interface.py +++ b/src/actinia_module_plugin/core/templates/user_templates_kvdb_interface.py @@ -17,7 +17,7 @@ Process Chain Template Management -Redis interface +Kvdb interface """ __license__ = "Apache-2.0" @@ -28,12 +28,12 @@ import pickle -from actinia_core.core.common.redis_base import RedisBaseInterface +from actinia_core.core.common.kvdb_base import KvdbBaseInterface -class RedisActiniaTemplateInterface(RedisBaseInterface): +class KvdbActiniaTemplateInterface(KvdbBaseInterface): """ - The Redis actinia_template database interface + The Kvdb actinia_template database interface A single actinia_template is stored as Hash with: - actinia_template id aka actinia_template name that must be unique @@ -50,7 +50,7 @@ class RedisActiniaTemplateInterface(RedisBaseInterface): actinia_template_id_db = "ACTINIA-TEMPLATE-ID-DATABASE" def __init__(self): - RedisBaseInterface.__init__(self) + KvdbBaseInterface.__init__(self) def create(self, actinia_template): """ @@ -66,7 +66,7 @@ def create(self, actinia_template): actinia_template_id = actinia_template["id"] keyname = self.actinia_template_id_hash_prefix + actinia_template_id - exists = self.redis_server.exists(keyname) + exists = self.kvdb_server.exists(keyname) if exists == 1 or exists is True: return False @@ -76,18 +76,18 @@ def create(self, actinia_template): "actinia_template": actinia_template_bytes, } - lock = self.redis_server.lock( + lock = self.kvdb_server.lock( name="add_actinia_template_lock", timeout=1 ) lock.acquire() # First add the actinia_template-id to the actinia_template id database - self.redis_server.hset( + self.kvdb_server.hset( self.actinia_template_id_db, actinia_template_id, actinia_template_id, ) - self.redis_server.hset( + self.kvdb_server.hset( self.actinia_template_id_hash_prefix + actinia_template_id, mapping=mapping, ) @@ -111,7 +111,7 @@ def read(self, actinia_template_id): try: actinia_template = pickle.loads( - self.redis_server.hget( + self.kvdb_server.hget( self.actinia_template_id_hash_prefix + actinia_template_id, "actinia_template", ) @@ -138,7 +138,7 @@ def update(self, actinia_template_id, actinia_template): """ keyname = self.actinia_template_id_hash_prefix + actinia_template_id - exists = self.redis_server.exists(keyname) + exists = self.kvdb_server.exists(keyname) if exists == 0 or exists is False: return False @@ -148,12 +148,12 @@ def update(self, actinia_template_id, actinia_template): "actinia_template": actinia_template_bytes, } - lock = self.redis_server.lock( + lock = self.kvdb_server.lock( name="update_actinia_template_lock", timeout=1 ) lock.acquire() - self.redis_server.hset( + self.kvdb_server.hset( self.actinia_template_id_hash_prefix + actinia_template_id, mapping=mapping, ) @@ -177,16 +177,14 @@ def delete(self, actinia_template_id): if exists == 0 or exists is False: return False - lock = self.redis_server.lock( + lock = self.kvdb_server.lock( name="delete_actinia_template_lock", timeout=1 ) lock.acquire() # Delete the entry from the actinia_template id database - self.redis_server.hdel( - self.actinia_template_id_db, actinia_template_id - ) + self.kvdb_server.hdel(self.actinia_template_id_db, actinia_template_id) # Delete the actual actinia_template entry - self.redis_server.delete( + self.kvdb_server.delete( self.actinia_template_id_hash_prefix + actinia_template_id ) lock.release() @@ -204,7 +202,7 @@ def list_all_ids(self): A list of all actinia_template ids in the database """ values = [] - list = self.redis_server.hkeys(self.actinia_template_id_db) + list = self.kvdb_server.hkeys(self.actinia_template_id_db) for entry in list: entry = entry.decode() values.append(entry) @@ -222,10 +220,10 @@ def exists(self, actinia_template_id): bool: True is actinia_template exists, False otherwise """ - return self.redis_server.exists( + return self.kvdb_server.exists( self.actinia_template_id_hash_prefix + actinia_template_id ) -# Create the Redis interface instance -redis_actinia_template_interface = RedisActiniaTemplateInterface() +# Create the Kvdb interface instance +kvdb_actinia_template_interface = KvdbActiniaTemplateInterface() diff --git a/tests/test_modules.py b/tests/test_modules.py index 4537db3..03461ab 100644 --- a/tests/test_modules.py +++ b/tests/test_modules.py @@ -146,7 +146,7 @@ def test_filter_list_modules_get_user_2(self): assert resp.status_code == respStatusCode assert hasattr(resp, "json") # WARNING: this depends on existing GRASS GIS modules and possible - # installed GRASS GIS Addons. If redis db is empty, it should include + # installed GRASS GIS Addons. If kvdb db is empty, it should include # all pc_templates from templates/pc_templates including subdirs and # importer and exporter. num_of_actinia_modules = len( diff --git a/tests/test_modules_actinia_user.py b/tests/test_modules_actinia_user.py index 5099fec..9a91dd0 100644 --- a/tests/test_modules_actinia_user.py +++ b/tests/test_modules_actinia_user.py @@ -47,7 +47,7 @@ class ActiniaModulesTest(ActiniaTestCase): def test_read_user_module_get(self): - """Test HTTP GET /actinia_modules/ for redis based templates""" + """Test HTTP GET /actinia_modules/ for kvdb based templates""" import_user_template(self, "user_point_in_polygon") respStatusCode = 200 diff --git a/tests/test_modules_grass.py b/tests/test_modules_grass.py index a9d2521..25cc411 100644 --- a/tests/test_modules_grass.py +++ b/tests/test_modules_grass.py @@ -30,6 +30,8 @@ from actinia_api import URL_PREFIX +from actinia_core.version import init_versions, G_VERSION + from testsuite import ActiniaTestCase, compare_module_to_file @@ -37,6 +39,12 @@ class GmodulesTest(ActiniaTestCase): + + # For expected test results, which are dependent on GRASS GIS version + init_versions() + grass_version_s = G_VERSION["version"] + grass_version = [int(item) for item in grass_version_s.split(".")[:2]] + # @unittest.skip("demonstrating skipping") def test_list_modules_get_user(self): """Test HTTP GET /grass_modules for user""" @@ -139,7 +147,10 @@ def test_filter_list_modules_get_admin_3(self): assert hasattr(resp, "json") # WARNING: this depends on existing GRASS GIS modules and possible # installed GRASS GIS Addons - assert len(resp.json["processes"]) == 2 + if self.grass_version < [8, 5]: + assert len(resp.json["processes"]) == 2 + else: + assert len(resp.json["processes"]) == 3 assert resp.json["processes"][0]["categories"] != 0 assert hasattr(resp.json["processes"][0], "parameters") is False diff --git a/tests/testsuite.py b/tests/testsuite.py index dd6d034..6dec9bb 100644 --- a/tests/testsuite.py +++ b/tests/testsuite.py @@ -56,7 +56,7 @@ from werkzeug.datastructures import Headers from actinia_core.endpoints import create_endpoints -from actinia_core.core.common.redis_interface import connect, disconnect +from actinia_core.core.common.kvdb_interface import connect, disconnect from actinia_core.core.common.app import flask_app, URL_PREFIX from actinia_core.core.common.config import global_config from actinia_core.core.common.process_queue import create_process_queue @@ -100,17 +100,17 @@ def setUp(self): flask_app.testing = True self.app = flask_app.test_client() - # Start and connect the redis interface - redis_args = ( - global_config.REDIS_SERVER_URL, - global_config.REDIS_SERVER_PORT, + # Start and connect the kvdb interface + kvdb_args = ( + global_config.KVDB_SERVER_URL, + global_config.KVDB_SERVER_PORT, ) if ( - global_config.REDIS_SERVER_PW - and global_config.REDIS_SERVER_PW is not None + global_config.KVDB_SERVER_PW + and global_config.KVDB_SERVER_PW is not None ): - redis_args = (*redis_args, global_config.REDIS_SERVER_PW) - connect(*redis_args) + kvdb_args = (*kvdb_args, global_config.KVDB_SERVER_PW) + connect(*kvdb_args) # create test user for roles user (more to come) accessible_datasets = { @@ -167,7 +167,7 @@ def tearDown(self): self.app_context.pop() - # remove test user; disconnect redis + # remove test user; disconnect kvdb for user in self.users_list: user.delete() disconnect() @@ -236,7 +236,7 @@ def compare_module_to_file(self, uri_path="modules", module=None): def import_user_template(testCase, name): - """Imports user template to redis database (Create)""" + """Imports user template to kvdb database (Create)""" json_path = "tests/resources/actinia_templates/" + name + ".json" with open(json_path) as file: pc_template = json.load(file) @@ -250,7 +250,7 @@ def import_user_template(testCase, name): def delete_user_template(testCase, name): - """Deletes user template from redis database (Delete) if exists""" + """Deletes user template from kvdb database (Delete) if exists""" resp = testCase.app.get( URL_PREFIX + "/actinia_templates/" + name, headers=testCase.user_auth_header, diff --git a/tests_with_redis.sh b/tests_with_kvdb.sh similarity index 77% rename from tests_with_redis.sh rename to tests_with_kvdb.sh index f8f4be0..9d09c81 100755 --- a/tests_with_redis.sh +++ b/tests_with_kvdb.sh @@ -1,9 +1,9 @@ #!/usr/bin/env sh -# start redis server -redis-server & +# start kvdb server +valkey-server & sleep 1 -redis-cli ping +valkey-cli ping # # start webhook server # webhook-server --host "0.0.0.0" --port "5005" & @@ -23,7 +23,7 @@ fi TEST_RES=$? -# stop redis server -redis-cli shutdown +# stop kvdb server +valkey-cli shutdown return $TEST_RES